1539: Use serdeval for validating json format. r=curquiza a=MarinPostma

uses [serdeval](https://github.com/MarinPostma/serdeval) to validate that the json payload is valid json, and in the correct format.

fix #1535


Co-authored-by: mpostma <postma.marin@protonmail.com>
This commit is contained in:
bors[bot] 2021-07-29 17:05:13 +00:00 committed by GitHub
commit 09c74c04a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 16 additions and 17 deletions

17
Cargo.lock generated
View File

@ -1666,7 +1666,6 @@ dependencies = [
"num_cpus", "num_cpus",
"obkv", "obkv",
"once_cell", "once_cell",
"oxidized-json-checker",
"parking_lot", "parking_lot",
"paste", "paste",
"pin-project", "pin-project",
@ -1679,6 +1678,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"serde_url_params", "serde_url_params",
"serdeval",
"sha-1 0.9.6", "sha-1 0.9.6",
"sha2", "sha2",
"siphasher", "siphasher",
@ -1969,12 +1969,6 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "oxidized-json-checker"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "938464aebf563f48ab86d1cfc0e2df952985c0b814d3108f41d1b85e7f5b0dac"
[[package]] [[package]]
name = "page_size" name = "page_size"
version = "0.4.2" version = "0.4.2"
@ -2784,6 +2778,15 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "serdeval"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94023adfd3d548a8bd9a1f09c09f44eaab7080c7a9ab20314bb65154bee62bd0"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "sha-1" name = "sha-1"
version = "0.8.2" version = "0.8.2"

View File

@ -55,7 +55,6 @@ milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.8.0" }
mime = "0.3.16" mime = "0.3.16"
num_cpus = "1.13.0" num_cpus = "1.13.0"
once_cell = "1.5.2" once_cell = "1.5.2"
oxidized-json-checker = "0.3.2"
parking_lot = "0.11.1" parking_lot = "0.11.1"
rand = "0.7.3" rand = "0.7.3"
rayon = "1.5.0" rayon = "1.5.0"
@ -77,6 +76,7 @@ obkv = "0.2.0"
pin-project = "1.0.7" pin-project = "1.0.7"
whoami = { version = "1.1.2", optional = true } whoami = { version = "1.1.2", optional = true }
reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true } reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true }
serdeval = "0.1.0"
[dependencies.sentry] [dependencies.sentry]
default-features = false default-features = false

View File

@ -7,7 +7,7 @@ use std::sync::Arc;
use async_stream::stream; use async_stream::stream;
use futures::StreamExt; use futures::StreamExt;
use log::trace; use log::trace;
use oxidized_json_checker::JsonChecker; use serdeval::*;
use tokio::fs; use tokio::fs;
use tokio::io::AsyncWriteExt; use tokio::io::AsyncWriteExt;
use tokio::sync::mpsc; use tokio::sync::mpsc;
@ -180,7 +180,7 @@ where
let update_store = self.store.clone(); let update_store = self.store.clone();
tokio::task::spawn_blocking(move || { tokio::task::spawn_blocking(move || {
use std::io::{copy, sink, BufReader, Seek}; use std::io::{BufReader, Seek};
// If the payload is empty, ignore the check. // If the payload is empty, ignore the check.
let update_uuid = if let Some((mut file, uuid)) = file_path { let update_uuid = if let Some((mut file, uuid)) = file_path {
@ -188,14 +188,10 @@ where
file.seek(SeekFrom::Start(0))?; file.seek(SeekFrom::Start(0))?;
// Check that the json payload is valid: // Check that the json payload is valid:
let reader = BufReader::new(&mut file); let reader = BufReader::new(&mut file);
let mut checker = JsonChecker::new(reader); // Validate that the payload is in the correct format.
let _: Seq<Map<Str, Any>> = serde_json::from_reader(reader)
.map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?;
if copy(&mut checker, &mut sink()).is_err() || checker.finish().is_err() {
// The json file is invalid, we use Serde to get a nice error message:
file.seek(SeekFrom::Start(0))?;
let _: serde_json::Value = serde_json::from_reader(file)
.map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?;
}
Some(uuid) Some(uuid)
} else { } else {
None None