2022-12-12 15:06:24 +01:00
|
|
|
use std::fmt::{self, Debug, Display};
|
|
|
|
use std::fs::File;
|
2024-02-22 18:42:12 +01:00
|
|
|
use std::io::{self, BufWriter, Write};
|
2022-12-12 15:06:24 +01:00
|
|
|
use std::marker::PhantomData;
|
|
|
|
|
2022-12-03 11:52:20 +01:00
|
|
|
use memmap2::MmapOptions;
|
2022-10-11 18:03:10 +02:00
|
|
|
use milli::documents::{DocumentsBatchBuilder, Error};
|
2022-12-13 16:29:49 +01:00
|
|
|
use milli::Object;
|
2022-11-30 10:59:06 +01:00
|
|
|
use serde::de::{SeqAccess, Visitor};
|
2022-11-29 17:03:22 +01:00
|
|
|
use serde::{Deserialize, Deserializer};
|
2022-10-20 17:35:22 +02:00
|
|
|
use serde_json::error::Category;
|
2022-10-20 18:00:07 +02:00
|
|
|
|
2022-12-12 15:02:56 +01:00
|
|
|
use crate::error::{Code, ErrorCode};
|
|
|
|
|
2022-10-11 18:03:10 +02:00
|
|
|
type Result<T> = std::result::Result<T, DocumentFormatError>;
|
|
|
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
pub enum PayloadType {
|
|
|
|
Ndjson,
|
|
|
|
Json,
|
2023-02-20 14:53:37 +01:00
|
|
|
Csv { delimiter: u8 },
|
2022-10-11 18:03:10 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
impl fmt::Display for PayloadType {
|
|
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
|
|
match self {
|
|
|
|
PayloadType::Ndjson => f.write_str("ndjson"),
|
|
|
|
PayloadType::Json => f.write_str("json"),
|
2023-02-20 14:53:37 +01:00
|
|
|
PayloadType::Csv { .. } => f.write_str("csv"),
|
2022-10-11 18:03:10 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Debug)]
|
|
|
|
pub enum DocumentFormatError {
|
2022-12-19 20:50:40 +01:00
|
|
|
Io(io::Error),
|
2022-10-11 18:03:10 +02:00
|
|
|
MalformedPayload(Error, PayloadType),
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Display for DocumentFormatError {
|
|
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
|
|
match self {
|
2022-12-19 20:50:40 +01:00
|
|
|
Self::Io(e) => write!(f, "{e}"),
|
2023-08-10 08:41:25 +02:00
|
|
|
Self::MalformedPayload(me, b) => match me {
|
2022-10-11 18:03:10 +02:00
|
|
|
Error::Json(se) => {
|
2022-10-20 17:35:22 +02:00
|
|
|
let mut message = match se.classify() {
|
|
|
|
Category::Data => {
|
|
|
|
"data are neither an object nor a list of objects".to_string()
|
|
|
|
}
|
|
|
|
_ => se.to_string(),
|
|
|
|
};
|
|
|
|
|
2022-10-11 18:03:10 +02:00
|
|
|
// https://github.com/meilisearch/meilisearch/issues/2107
|
|
|
|
// The user input maybe insanely long. We need to truncate it.
|
|
|
|
let ellipsis = "...";
|
2022-10-20 17:24:15 +02:00
|
|
|
let trim_input_prefix_len = 50;
|
|
|
|
let trim_input_suffix_len = 85;
|
|
|
|
|
2022-10-20 17:35:22 +02:00
|
|
|
if message.len()
|
2022-10-20 17:24:15 +02:00
|
|
|
> trim_input_prefix_len + trim_input_suffix_len + ellipsis.len()
|
|
|
|
{
|
2022-10-20 17:35:22 +02:00
|
|
|
message.replace_range(
|
|
|
|
trim_input_prefix_len..message.len() - trim_input_suffix_len,
|
2022-10-20 17:24:15 +02:00
|
|
|
ellipsis,
|
|
|
|
);
|
2022-10-11 18:03:10 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
write!(
|
|
|
|
f,
|
|
|
|
"The `{}` payload provided is malformed. `Couldn't serialize document value: {}`.",
|
2022-10-20 17:35:22 +02:00
|
|
|
b, message
|
2022-10-20 17:24:15 +02:00
|
|
|
)
|
2022-10-11 18:03:10 +02:00
|
|
|
}
|
|
|
|
_ => write!(f, "The `{}` payload provided is malformed: `{}`.", b, me),
|
|
|
|
},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl std::error::Error for DocumentFormatError {}
|
|
|
|
|
|
|
|
impl From<(PayloadType, Error)> for DocumentFormatError {
|
|
|
|
fn from((ty, error): (PayloadType, Error)) -> Self {
|
|
|
|
match error {
|
2022-12-20 17:49:38 +01:00
|
|
|
Error::Io(e) => Self::Io(e),
|
2022-10-11 18:03:10 +02:00
|
|
|
e => Self::MalformedPayload(e, ty),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-12-19 20:50:40 +01:00
|
|
|
impl From<io::Error> for DocumentFormatError {
|
|
|
|
fn from(error: io::Error) -> Self {
|
|
|
|
Self::Io(error)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-10-11 18:03:10 +02:00
|
|
|
impl ErrorCode for DocumentFormatError {
|
|
|
|
fn error_code(&self) -> Code {
|
|
|
|
match self {
|
2022-12-19 20:50:40 +01:00
|
|
|
DocumentFormatError::Io(e) => e.error_code(),
|
2022-10-11 18:03:10 +02:00
|
|
|
DocumentFormatError::MalformedPayload(_, _) => Code::MalformedPayload,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Reads CSV from input and write an obkv batch to writer.
|
2024-02-22 18:42:12 +01:00
|
|
|
pub fn read_csv(file: &File, writer: impl Write, delimiter: u8) -> Result<u64> {
|
|
|
|
let mut builder = DocumentsBatchBuilder::new(BufWriter::new(writer));
|
2022-12-03 11:52:20 +01:00
|
|
|
let mmap = unsafe { MmapOptions::new().map(file)? };
|
2023-02-16 13:12:18 +01:00
|
|
|
let csv = csv::ReaderBuilder::new().delimiter(delimiter).from_reader(mmap.as_ref());
|
2023-02-20 14:53:37 +01:00
|
|
|
builder.append_csv(csv).map_err(|e| (PayloadType::Csv { delimiter }, e))?;
|
2022-10-11 18:03:10 +02:00
|
|
|
|
|
|
|
let count = builder.documents_count();
|
2022-12-20 17:49:38 +01:00
|
|
|
let _ = builder.into_inner().map_err(DocumentFormatError::Io)?;
|
2022-10-11 18:03:10 +02:00
|
|
|
|
2022-12-13 15:10:51 +01:00
|
|
|
Ok(count as u64)
|
2022-10-11 18:03:10 +02:00
|
|
|
}
|
|
|
|
|
2024-02-22 18:42:12 +01:00
|
|
|
/// Reads JSON from temporary file and write an obkv batch to writer.
|
|
|
|
pub fn read_json(file: &File, writer: impl Write) -> Result<u64> {
|
|
|
|
let mut builder = DocumentsBatchBuilder::new(BufWriter::new(writer));
|
2022-12-03 11:52:20 +01:00
|
|
|
let mmap = unsafe { MmapOptions::new().map(file)? };
|
2022-11-29 17:03:22 +01:00
|
|
|
let mut deserializer = serde_json::Deserializer::from_slice(&mmap);
|
2022-10-11 18:03:10 +02:00
|
|
|
|
2022-12-13 14:52:10 +01:00
|
|
|
match array_each(&mut deserializer, |obj| builder.append_json_object(&obj)) {
|
|
|
|
// The json data has been deserialized and does not need to be processed again.
|
|
|
|
// The data has been transferred to the writer during the deserialization process.
|
2022-12-13 14:58:32 +01:00
|
|
|
Ok(Ok(_)) => (),
|
2022-12-20 17:49:38 +01:00
|
|
|
Ok(Err(e)) => return Err(DocumentFormatError::Io(e)),
|
2022-12-21 04:27:15 +01:00
|
|
|
Err(e) => {
|
|
|
|
// Attempt to deserialize a single json string when the cause of the exception is not Category.data
|
|
|
|
// Other types of deserialisation exceptions are returned directly to the front-end
|
|
|
|
if e.classify() != serde_json::error::Category::Data {
|
|
|
|
return Err(DocumentFormatError::MalformedPayload(
|
|
|
|
Error::Json(e),
|
|
|
|
PayloadType::Json,
|
|
|
|
));
|
2022-12-13 16:29:49 +01:00
|
|
|
}
|
|
|
|
|
2022-12-21 04:27:15 +01:00
|
|
|
let content: Object = serde_json::from_slice(&mmap)
|
2022-11-30 10:59:06 +01:00
|
|
|
.map_err(Error::Json)
|
2022-12-21 04:27:15 +01:00
|
|
|
.map_err(|e| (PayloadType::Json, e))?;
|
|
|
|
builder.append_json_object(&content).map_err(DocumentFormatError::Io)?;
|
2022-11-30 10:59:06 +01:00
|
|
|
}
|
2022-10-11 18:03:10 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
let count = builder.documents_count();
|
2022-12-20 17:49:38 +01:00
|
|
|
let _ = builder.into_inner().map_err(DocumentFormatError::Io)?;
|
2022-10-11 18:03:10 +02:00
|
|
|
|
2022-12-13 15:10:51 +01:00
|
|
|
Ok(count as u64)
|
2022-10-11 18:03:10 +02:00
|
|
|
}
|
2022-11-29 17:03:22 +01:00
|
|
|
|
2022-12-21 04:27:15 +01:00
|
|
|
/// Reads JSON from temporary file and write an obkv batch to writer.
|
2024-02-22 18:42:12 +01:00
|
|
|
pub fn read_ndjson(file: &File, writer: impl Write) -> Result<u64> {
|
|
|
|
let mut builder = DocumentsBatchBuilder::new(BufWriter::new(writer));
|
2022-12-21 04:27:15 +01:00
|
|
|
let mmap = unsafe { MmapOptions::new().map(file)? };
|
|
|
|
|
2022-12-21 07:31:45 +01:00
|
|
|
for result in serde_json::Deserializer::from_slice(&mmap).into_iter() {
|
2022-12-21 04:27:15 +01:00
|
|
|
let object = result.map_err(Error::Json).map_err(|e| (PayloadType::Ndjson, e))?;
|
|
|
|
builder.append_json_object(&object).map_err(Into::into).map_err(DocumentFormatError::Io)?;
|
|
|
|
}
|
|
|
|
|
|
|
|
let count = builder.documents_count();
|
|
|
|
let _ = builder.into_inner().map_err(Into::into).map_err(DocumentFormatError::Io)?;
|
|
|
|
|
|
|
|
Ok(count as u64)
|
|
|
|
}
|
|
|
|
|
2022-12-13 14:52:10 +01:00
|
|
|
/// The actual handling of the deserialization process in serde
|
|
|
|
/// avoids storing the deserialized object in memory.
|
|
|
|
///
|
|
|
|
/// ## References
|
|
|
|
/// <https://serde.rs/stream-array.html>
|
|
|
|
/// <https://github.com/serde-rs/json/issues/160>
|
2022-11-29 17:03:22 +01:00
|
|
|
fn array_each<'de, D, T, F>(deserializer: D, f: F) -> std::result::Result<io::Result<u64>, D::Error>
|
|
|
|
where
|
|
|
|
D: Deserializer<'de>,
|
|
|
|
T: Deserialize<'de>,
|
|
|
|
F: FnMut(T) -> io::Result<()>,
|
|
|
|
{
|
|
|
|
struct SeqVisitor<T, F>(F, PhantomData<T>);
|
|
|
|
|
|
|
|
impl<'de, T, F> Visitor<'de> for SeqVisitor<T, F>
|
|
|
|
where
|
|
|
|
T: Deserialize<'de>,
|
|
|
|
F: FnMut(T) -> io::Result<()>,
|
|
|
|
{
|
|
|
|
type Value = io::Result<u64>;
|
|
|
|
|
|
|
|
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
|
|
|
|
formatter.write_str("a nonempty sequence")
|
|
|
|
}
|
|
|
|
|
2022-11-30 10:59:06 +01:00
|
|
|
fn visit_seq<A>(
|
|
|
|
mut self,
|
|
|
|
mut seq: A,
|
|
|
|
) -> std::result::Result<io::Result<u64>, <A as SeqAccess<'de>>::Error>
|
2022-11-29 17:03:22 +01:00
|
|
|
where
|
|
|
|
A: SeqAccess<'de>,
|
|
|
|
{
|
|
|
|
let mut max: u64 = 0;
|
|
|
|
while let Some(value) = seq.next_element::<T>()? {
|
|
|
|
match self.0(value) {
|
2022-11-30 10:59:06 +01:00
|
|
|
Ok(()) => max += 1,
|
2022-11-29 17:03:22 +01:00
|
|
|
Err(e) => return Ok(Err(e)),
|
|
|
|
};
|
|
|
|
}
|
|
|
|
Ok(Ok(max))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
let visitor = SeqVisitor(f, PhantomData);
|
|
|
|
deserializer.deserialize_seq(visitor)
|
2022-11-30 10:59:06 +01:00
|
|
|
}
|