From 67ecd7c1478af8e820651a0c2aeb2d98fc81fc66 Mon Sep 17 00:00:00 2001 From: Morgane Dubus <30866152+mdubus@users.noreply.github.com> Date: Wed, 16 Feb 2022 14:10:47 +0100 Subject: [PATCH 01/16] Update dashboard with v0.1.8 --- meilisearch-http/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 653584fd3..11c2476d4 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -105,5 +105,5 @@ default = ["analytics", "mini-dashboard"] tikv-jemallocator = "0.4.1" [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.7/build.zip" -sha1 = "e2feedf271917c4b7b88998eff5aaaea1d3925b9" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.8/build.zip" +sha1 = "093557c14c132e122d209526b4037221a1c42718" From 7bcaa2fd13f5ef41c31139fb57adae372a388fc1 Mon Sep 17 00:00:00 2001 From: Morgane Dubus <30866152+mdubus@users.noreply.github.com> Date: Wed, 16 Feb 2022 15:53:15 +0100 Subject: [PATCH 02/16] Update dashboard to v0.1.9 --- meilisearch-http/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 11c2476d4..5442b26bb 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -105,5 +105,5 @@ default = ["analytics", "mini-dashboard"] tikv-jemallocator = "0.4.1" [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.8/build.zip" -sha1 = "093557c14c132e122d209526b4037221a1c42718" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.9/build.zip" +sha1 = "b1833c3e5dc6b5d9d519ae4834935ae6c8a47024" From 05c8d81e6559397e827ad8c880ae4d8965525bfc Mon Sep 17 00:00:00 2001 From: Irevoire Date: Mon, 14 Feb 2022 15:32:41 +0100 Subject: [PATCH 03/16] chore: get rid of chrono in favor of time Chrono has been unmaintened for a few month now and there is a CVE on it. make clippy happy bump milli --- Cargo.lock | 72 ++++++++++--- meilisearch-auth/Cargo.toml | 2 +- meilisearch-auth/src/error.rs | 8 +- meilisearch-auth/src/key.rs | 57 ++++++---- meilisearch-auth/src/lib.rs | 4 +- meilisearch-auth/src/store.rs | 8 +- meilisearch-http/Cargo.toml | 5 +- .../src/analytics/segment_analytics.rs | 10 +- .../src/extractors/authentication/mod.rs | 4 +- meilisearch-http/src/routes/api_key.rs | 19 ++-- meilisearch-http/src/routes/indexes/mod.rs | 11 +- meilisearch-http/src/routes/indexes/tasks.rs | 11 +- meilisearch-http/src/routes/mod.rs | 24 +++-- meilisearch-http/src/task.rs | 59 ++++++++-- meilisearch-http/tests/auth/api_keys.rs | 26 ++--- meilisearch-http/tests/auth/authorization.rs | 25 ++--- meilisearch-http/tests/auth/tenant_token.rs | 101 +++++++++--------- .../tests/documents/add_documents.rs | 6 +- meilisearch-http/tests/index/update_index.rs | 8 +- meilisearch-http/tests/tasks/mod.rs | 6 +- meilisearch-lib/Cargo.toml | 4 +- meilisearch-lib/src/index/index.rs | 8 +- .../src/index_controller/dump_actor/actor.rs | 7 +- .../index_controller/dump_actor/compat/v2.rs | 12 +-- .../index_controller/dump_actor/compat/v3.rs | 10 +- .../src/index_controller/dump_actor/mod.rs | 24 +++-- meilisearch-lib/src/index_controller/mod.rs | 6 +- meilisearch-lib/src/index_resolver/mod.rs | 23 ++-- meilisearch-lib/src/tasks/batch.rs | 4 +- meilisearch-lib/src/tasks/scheduler.rs | 6 +- meilisearch-lib/src/tasks/task.rs | 16 +-- meilisearch-lib/src/tasks/task_store/mod.rs | 4 +- meilisearch-lib/src/tasks/update_loop.rs | 5 +- 33 files changed, 369 insertions(+), 226 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5059bd013..804fe8c70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -300,6 +300,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + [[package]] name = "as-slice" version = "0.1.5" @@ -647,7 +653,6 @@ dependencies = [ "libc", "num-integer", "num-traits", - "serde", "time 0.1.44", "winapi", ] @@ -989,9 +994,9 @@ dependencies = [ [[package]] name = "filter-parser" version = "0.1.0" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.22.1#ea15ad6c34492b32eb7ac06e69de02b6dc70a707" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.22.2#f2984f66e64838d51f5cce412693fa411ee3f2d4" dependencies = [ - "nom", + "nom 7.1.0", "nom_locate", ] @@ -1479,6 +1484,15 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" +[[package]] +name = "iso8601-duration" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b51dd97fa24074214b9eb14da518957573f4dec3189112610ae1ccec9ac464" +dependencies = [ + "nom 5.1.2", +] + [[package]] name = "itertools" version = "0.10.3" @@ -1568,6 +1582,19 @@ dependencies = [ "fst", ] +[[package]] +name = "lexical-core" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" +dependencies = [ + "arrayvec", + "bitflags", + "cfg-if 1.0.0", + "ryu", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.114" @@ -1688,7 +1715,6 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" name = "meilisearch-auth" version = "0.25.0" dependencies = [ - "chrono", "enum-iterator", "heed", "meilisearch-error", @@ -1697,6 +1723,7 @@ dependencies = [ "serde_json", "sha2", "thiserror", + "time 0.3.7", ] [[package]] @@ -1727,7 +1754,6 @@ dependencies = [ "byte-unit", "bytes", "cargo_toml", - "chrono", "clap", "crossbeam-channel", "either", @@ -1740,6 +1766,7 @@ dependencies = [ "hex", "http", "indexmap", + "iso8601-duration", "itertools", "jsonwebtoken", "log", @@ -1775,6 +1802,7 @@ dependencies = [ "tempfile", "thiserror", "tikv-jemallocator", + "time 0.3.7", "tokio", "tokio-stream", "urlencoding", @@ -1796,7 +1824,6 @@ dependencies = [ "atomic_refcell", "byte-unit", "bytes", - "chrono", "clap", "crossbeam-channel", "csv", @@ -1840,6 +1867,7 @@ dependencies = [ "tar", "tempfile", "thiserror", + "time 0.3.7", "tokio", "uuid", "walkdir", @@ -1888,14 +1916,13 @@ dependencies = [ [[package]] name = "milli" -version = "0.22.1" -source = "git+https://github.com/meilisearch/milli.git?tag=v0.22.1#ea15ad6c34492b32eb7ac06e69de02b6dc70a707" +version = "0.23.0" +source = "git+https://github.com/meilisearch/milli.git?tag=v0.22.2#f2984f66e64838d51f5cce412693fa411ee3f2d4" dependencies = [ "bimap", "bincode", "bstr", "byteorder", - "chrono", "concat-arrays", "crossbeam-channel", "csv", @@ -1927,6 +1954,7 @@ dependencies = [ "smallstr", "smallvec", "tempfile", + "time 0.3.7", "uuid", ] @@ -2029,6 +2057,17 @@ name = "nelson" version = "0.1.0" source = "git+https://github.com/MarinPostma/nelson.git?rev=675f13885548fb415ead8fbb447e9e6d9314000a#675f13885548fb415ead8fbb447e9e6d9314000a" +[[package]] +name = "nom" +version = "5.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" +dependencies = [ + "lexical-core", + "memchr", + "version_check", +] + [[package]] name = "nom" version = "7.1.0" @@ -2048,7 +2087,7 @@ checksum = "37794436ca3029a3089e0b95d42da1f0b565ad271e4d3bb4bad0c7bb70b10605" dependencies = [ "bytecount", "memchr", - "nom", + "nom 7.1.0", ] [[package]] @@ -2780,16 +2819,16 @@ dependencies = [ [[package]] name = "segment" -version = "0.1.2" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bdcc286fff0e7c5ccd46c06a301c7a8a848b06acedc6983707bd311eb358002" +checksum = "5c14967a911a216177366bac6dfa1209b597e311a32360431c63526e27b814fb" dependencies = [ "async-trait", - "chrono", "reqwest", "serde", "serde_json", "thiserror", + "time 0.3.7", ] [[package]] @@ -2976,6 +3015,12 @@ dependencies = [ "path-slash", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.10.0" @@ -3147,6 +3192,7 @@ dependencies = [ "itoa 1.0.1", "libc", "num_threads", + "serde", "time-macros", ] diff --git a/meilisearch-auth/Cargo.toml b/meilisearch-auth/Cargo.toml index 3b5379d8f..257396099 100644 --- a/meilisearch-auth/Cargo.toml +++ b/meilisearch-auth/Cargo.toml @@ -7,9 +7,9 @@ edition = "2021" enum-iterator = "0.7.0" heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" } sha2 = "0.9.6" -chrono = { version = "0.4.19", features = ["serde"] } meilisearch-error = { path = "../meilisearch-error" } serde_json = { version = "1.0.67", features = ["preserve_order"] } +time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } rand = "0.8.4" serde = { version = "1.0.130", features = ["derive"] } thiserror = "1.0.28" diff --git a/meilisearch-auth/src/error.rs b/meilisearch-auth/src/error.rs index 8fa6b8430..70cfd3905 100644 --- a/meilisearch-auth/src/error.rs +++ b/meilisearch-auth/src/error.rs @@ -10,13 +10,13 @@ pub type Result = std::result::Result; pub enum AuthControllerError { #[error("`{0}` field is mandatory.")] MissingParameter(&'static str), - #[error("actions field value `{0}` is invalid. It should be an array of string representing action names.")] + #[error("`actions` field value `{0}` is invalid. It should be an array of string representing action names.")] InvalidApiKeyActions(Value), - #[error("indexes field value `{0}` is invalid. It should be an array of string representing index names.")] + #[error("`indexes` field value `{0}` is invalid. It should be an array of string representing index names.")] InvalidApiKeyIndexes(Value), - #[error("expiresAt field value `{0}` is invalid. It should be in ISO-8601 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DDTHH:MM:SS'.")] + #[error("`expiresAt` field value `{0}` is invalid. It should follow the RFC 3339 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DD HH:MM:SS'.")] InvalidApiKeyExpiresAt(Value), - #[error("description field value `{0}` is invalid. It should be a string or specified as a null value.")] + #[error("`description` field value `{0}` is invalid. It should be a string or specified as a null value.")] InvalidApiKeyDescription(Value), #[error("API key `{0}` not found.")] ApiKeyNotFound(String), diff --git a/meilisearch-auth/src/key.rs b/meilisearch-auth/src/key.rs index ffa56fa88..1b06f34be 100644 --- a/meilisearch-auth/src/key.rs +++ b/meilisearch-auth/src/key.rs @@ -1,10 +1,12 @@ use crate::action::Action; use crate::error::{AuthControllerError, Result}; use crate::store::{KeyId, KEY_ID_LENGTH}; -use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc}; use rand::Rng; use serde::{Deserialize, Serialize}; use serde_json::{from_value, Value}; +use time::format_description::well_known::Rfc3339; +use time::macros::{format_description, time}; +use time::{Date, OffsetDateTime, PrimitiveDateTime}; #[derive(Debug, Deserialize, Serialize)] pub struct Key { @@ -13,9 +15,12 @@ pub struct Key { pub id: KeyId, pub actions: Vec, pub indexes: Vec, - pub expires_at: Option>, - pub created_at: DateTime, - pub updated_at: DateTime, + #[serde(with = "time::serde::rfc3339::option")] + pub expires_at: Option, + #[serde(with = "time::serde::rfc3339")] + pub created_at: OffsetDateTime, + #[serde(with = "time::serde::rfc3339")] + pub updated_at: OffsetDateTime, } impl Key { @@ -52,8 +57,8 @@ impl Key { .map(parse_expiration_date) .ok_or(AuthControllerError::MissingParameter("expiresAt"))??; - let created_at = Utc::now(); - let updated_at = Utc::now(); + let created_at = OffsetDateTime::now_utc(); + let updated_at = created_at; Ok(Self { description, @@ -89,24 +94,26 @@ impl Key { self.expires_at = parse_expiration_date(exp)?; } - self.updated_at = Utc::now(); + self.updated_at = OffsetDateTime::now_utc(); Ok(()) } pub(crate) fn default_admin() -> Self { + let now = OffsetDateTime::now_utc(); Self { description: Some("Default Admin API Key (Use it for all other operations. Caution! Do not use it on a public frontend)".to_string()), id: generate_id(), actions: vec![Action::All], indexes: vec!["*".to_string()], expires_at: None, - created_at: Utc::now(), - updated_at: Utc::now(), + created_at: now, + updated_at: now, } } pub(crate) fn default_search() -> Self { + let now = OffsetDateTime::now_utc(); Self { description: Some( "Default Search API Key (Use it to search from the frontend)".to_string(), @@ -115,8 +122,8 @@ impl Key { actions: vec![Action::Search], indexes: vec!["*".to_string()], expires_at: None, - created_at: Utc::now(), - updated_at: Utc::now(), + created_at: now, + updated_at: now, } } } @@ -134,22 +141,34 @@ fn generate_id() -> [u8; KEY_ID_LENGTH] { bytes } -fn parse_expiration_date(value: &Value) -> Result>> { +fn parse_expiration_date(value: &Value) -> Result> { match value { - Value::String(string) => DateTime::parse_from_rfc3339(string) - .map(|d| d.into()) + Value::String(string) => OffsetDateTime::parse(string, &Rfc3339) .or_else(|_| { - NaiveDateTime::parse_from_str(string, "%Y-%m-%dT%H:%M:%S") - .map(|naive| DateTime::from_utc(naive, Utc)) + PrimitiveDateTime::parse( + string, + format_description!( + "[year repr:full base:calendar]-[month repr:numerical]-[day]T[hour]:[minute]:[second]" + ), + ).map(|datetime| datetime.assume_utc()) }) .or_else(|_| { - NaiveDate::parse_from_str(string, "%Y-%m-%d") - .map(|naive| DateTime::from_utc(naive.and_hms(0, 0, 0), Utc)) + PrimitiveDateTime::parse( + string, + format_description!( + "[year repr:full base:calendar]-[month repr:numerical]-[day] [hour]:[minute]:[second]" + ), + ).map(|datetime| datetime.assume_utc()) + }) + .or_else(|_| { + Date::parse(string, format_description!( + "[year repr:full base:calendar]-[month repr:numerical]-[day]" + )).map(|date| PrimitiveDateTime::new(date, time!(00:00)).assume_utc()) }) .map_err(|_| AuthControllerError::InvalidApiKeyExpiresAt(value.clone())) // check if the key is already expired. .and_then(|d| { - if d > Utc::now() { + if d > OffsetDateTime::now_utc() { Ok(d) } else { Err(AuthControllerError::InvalidApiKeyExpiresAt(value.clone())) diff --git a/meilisearch-auth/src/lib.rs b/meilisearch-auth/src/lib.rs index cfaa605a3..70ef69c82 100644 --- a/meilisearch-auth/src/lib.rs +++ b/meilisearch-auth/src/lib.rs @@ -9,10 +9,10 @@ use std::path::Path; use std::str::from_utf8; use std::sync::Arc; -use chrono::Utc; use serde::{Deserialize, Serialize}; use serde_json::Value; use sha2::{Digest, Sha256}; +use time::OffsetDateTime; pub use action::{actions, Action}; use error::{AuthControllerError, Result}; @@ -148,7 +148,7 @@ impl AuthController { None => self.store.prefix_first_expiration_date(key, action)?, }) { // check expiration date. - Some(Some(exp)) => Ok(Utc::now() < exp), + Some(Some(exp)) => Ok(OffsetDateTime::now_utc() < exp), // no expiration date. Some(None) => Ok(true), // action or index forbidden. diff --git a/meilisearch-auth/src/store.rs b/meilisearch-auth/src/store.rs index 757a8338b..332f698bf 100644 --- a/meilisearch-auth/src/store.rs +++ b/meilisearch-auth/src/store.rs @@ -8,9 +8,9 @@ use std::path::Path; use std::str; use std::sync::Arc; -use chrono::{DateTime, Utc}; use heed::types::{ByteSlice, DecodeIgnore, SerdeJson}; use heed::{Database, Env, EnvOpenOptions, RwTxn}; +use time::OffsetDateTime; use super::error::Result; use super::{Action, Key}; @@ -27,7 +27,7 @@ pub type KeyId = [u8; KEY_ID_LENGTH]; pub struct HeedAuthStore { env: Arc, keys: Database>, - action_keyid_index_expiration: Database>>>, + action_keyid_index_expiration: Database>>, should_close_on_drop: bool, } @@ -146,7 +146,7 @@ impl HeedAuthStore { key: &[u8], action: Action, index: Option<&[u8]>, - ) -> Result>>> { + ) -> Result>> { let rtxn = self.env.read_txn()?; match self.get_key_id(key) { Some(id) => { @@ -161,7 +161,7 @@ impl HeedAuthStore { &self, key: &[u8], action: Action, - ) -> Result>>> { + ) -> Result>> { let rtxn = self.env.read_txn()?; match self.get_key_id(key) { Some(id) => { diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 6bda0dc31..743245b6d 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -32,7 +32,6 @@ async-trait = "0.1.51" bstr = "0.2.17" byte-unit = { version = "4.0.12", default-features = false, features = ["std", "serde"] } bytes = "1.1.0" -chrono = { version = "0.4.19", features = ["serde"] } crossbeam-channel = "0.5.1" either = "1.6.1" env_logger = "0.9.0" @@ -43,6 +42,7 @@ futures-util = "0.3.17" heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" } http = "0.2.4" indexmap = { version = "1.7.0", features = ["serde-1"] } +iso8601-duration = "0.1.0" itertools = "0.10.1" jsonwebtoken = "7" log = "0.4.14" @@ -61,7 +61,7 @@ rayon = "1.5.1" regex = "1.5.4" rustls = "0.20.2" rustls-pemfile = "0.2" -segment = { version = "0.1.2", optional = true } +segment = { version = "0.2.0", optional = true } serde = { version = "1.0.130", features = ["derive"] } serde_json = { version = "1.0.67", features = ["preserve_order"] } sha2 = "0.9.6" @@ -73,6 +73,7 @@ sysinfo = "0.20.2" tar = "0.4.37" tempfile = "3.2.0" thiserror = "1.0.28" +time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } tokio = { version = "1.11.0", features = ["full"] } tokio-stream = "0.1.7" uuid = { version = "0.8.2", features = ["serde"] } diff --git a/meilisearch-http/src/analytics/segment_analytics.rs b/meilisearch-http/src/analytics/segment_analytics.rs index 2936c4d5d..905d55281 100644 --- a/meilisearch-http/src/analytics/segment_analytics.rs +++ b/meilisearch-http/src/analytics/segment_analytics.rs @@ -6,7 +6,6 @@ use std::time::{Duration, Instant}; use actix_web::http::header::USER_AGENT; use actix_web::HttpRequest; -use chrono::{DateTime, Utc}; use http::header::CONTENT_TYPE; use meilisearch_auth::SearchRules; use meilisearch_lib::index::{SearchQuery, SearchResult}; @@ -18,6 +17,7 @@ use segment::message::{Identify, Track, User}; use segment::{AutoBatcher, Batcher, HttpClient}; use serde_json::{json, Value}; use sysinfo::{DiskExt, System, SystemExt}; +use time::OffsetDateTime; use tokio::select; use tokio::sync::mpsc::{self, Receiver, Sender}; use uuid::Uuid; @@ -323,7 +323,7 @@ impl Segment { #[derive(Default)] pub struct SearchAggregator { - timestamp: Option>, + timestamp: Option, // context user_agents: HashSet, @@ -360,7 +360,7 @@ pub struct SearchAggregator { impl SearchAggregator { pub fn from_query(query: &SearchQuery, request: &HttpRequest) -> Self { let mut ret = Self::default(); - ret.timestamp = Some(chrono::offset::Utc::now()); + ret.timestamp = Some(OffsetDateTime::now_utc()); ret.total_received = 1; ret.user_agents = extract_user_agents(request).into_iter().collect(); @@ -504,7 +504,7 @@ impl SearchAggregator { #[derive(Default)] pub struct DocumentsAggregator { - timestamp: Option>, + timestamp: Option, // set to true when at least one request was received updated: bool, @@ -524,7 +524,7 @@ impl DocumentsAggregator { request: &HttpRequest, ) -> Self { let mut ret = Self::default(); - ret.timestamp = Some(chrono::offset::Utc::now()); + ret.timestamp = Some(OffsetDateTime::now_utc()); ret.updated = true; ret.user_agents = extract_user_agents(request).into_iter().collect(); diff --git a/meilisearch-http/src/extractors/authentication/mod.rs b/meilisearch-http/src/extractors/authentication/mod.rs index d4c8d5534..bf35e1729 100644 --- a/meilisearch-http/src/extractors/authentication/mod.rs +++ b/meilisearch-http/src/extractors/authentication/mod.rs @@ -94,10 +94,10 @@ pub trait Policy { } pub mod policies { - use chrono::Utc; use jsonwebtoken::{dangerous_insecure_decode, decode, Algorithm, DecodingKey, Validation}; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; + use time::OffsetDateTime; use crate::extractors::authentication::Policy; use meilisearch_auth::{Action, AuthController, AuthFilter, SearchRules}; @@ -183,7 +183,7 @@ pub mod policies { // Check if token is expired. if let Some(exp) = exp { - if Utc::now().timestamp() > exp { + if OffsetDateTime::now_utc().unix_timestamp() > exp { return None; } } diff --git a/meilisearch-http/src/routes/api_key.rs b/meilisearch-http/src/routes/api_key.rs index 9e67c3195..54152ea2c 100644 --- a/meilisearch-http/src/routes/api_key.rs +++ b/meilisearch-http/src/routes/api_key.rs @@ -1,11 +1,11 @@ use std::str; use actix_web::{web, HttpRequest, HttpResponse}; -use chrono::SecondsFormat; use meilisearch_auth::{Action, AuthController, Key}; use serde::{Deserialize, Serialize}; use serde_json::Value; +use time::OffsetDateTime; use crate::extractors::authentication::{policies::*, GuardedData}; use meilisearch_error::ResponseError; @@ -92,9 +92,12 @@ struct KeyView { key: String, actions: Vec, indexes: Vec, - expires_at: Option, - created_at: String, - updated_at: String, + #[serde(serialize_with = "time::serde::rfc3339::option::serialize")] + expires_at: Option, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + created_at: OffsetDateTime, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + updated_at: OffsetDateTime, } impl KeyView { @@ -107,11 +110,9 @@ impl KeyView { key: generated_key, actions: key.actions, indexes: key.indexes, - expires_at: key - .expires_at - .map(|dt| dt.to_rfc3339_opts(SecondsFormat::Secs, true)), - created_at: key.created_at.to_rfc3339_opts(SecondsFormat::Secs, true), - updated_at: key.updated_at.to_rfc3339_opts(SecondsFormat::Secs, true), + expires_at: key.expires_at, + created_at: key.created_at, + updated_at: key.updated_at, } } } diff --git a/meilisearch-http/src/routes/indexes/mod.rs b/meilisearch-http/src/routes/indexes/mod.rs index fe7ba0b11..50e54e6b4 100644 --- a/meilisearch-http/src/routes/indexes/mod.rs +++ b/meilisearch-http/src/routes/indexes/mod.rs @@ -1,11 +1,11 @@ use actix_web::{web, HttpRequest, HttpResponse}; -use chrono::{DateTime, Utc}; use log::debug; use meilisearch_error::ResponseError; use meilisearch_lib::index_controller::Update; use meilisearch_lib::MeiliSearch; use serde::{Deserialize, Serialize}; use serde_json::json; +use time::OffsetDateTime; use crate::analytics::Analytics; use crate::extractors::authentication::{policies::*, GuardedData}; @@ -95,9 +95,12 @@ pub struct UpdateIndexRequest { pub struct UpdateIndexResponse { name: String, uid: String, - created_at: DateTime, - updated_at: DateTime, - primary_key: Option, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + created_at: OffsetDateTime, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + updated_at: OffsetDateTime, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + primary_key: OffsetDateTime, } pub async fn get_index( diff --git a/meilisearch-http/src/routes/indexes/tasks.rs b/meilisearch-http/src/routes/indexes/tasks.rs index f20a39d4a..8545831a0 100644 --- a/meilisearch-http/src/routes/indexes/tasks.rs +++ b/meilisearch-http/src/routes/indexes/tasks.rs @@ -1,10 +1,10 @@ use actix_web::{web, HttpRequest, HttpResponse}; -use chrono::{DateTime, Utc}; use log::debug; use meilisearch_error::ResponseError; use meilisearch_lib::MeiliSearch; use serde::{Deserialize, Serialize}; use serde_json::json; +use time::OffsetDateTime; use crate::analytics::Analytics; use crate::extractors::authentication::{policies::*, GuardedData}; @@ -20,9 +20,12 @@ pub fn configure(cfg: &mut web::ServiceConfig) { pub struct UpdateIndexResponse { name: String, uid: String, - created_at: DateTime, - updated_at: DateTime, - primary_key: Option, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + created_at: OffsetDateTime, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + updated_at: OffsetDateTime, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + primary_key: OffsetDateTime, } #[derive(Deserialize)] diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 63d537967..f94028a7d 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -1,7 +1,7 @@ use actix_web::{web, HttpResponse}; -use chrono::{DateTime, Utc}; use log::debug; use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; use meilisearch_error::ResponseError; use meilisearch_lib::index::{Settings, Unchecked}; @@ -54,8 +54,10 @@ pub struct ProcessedUpdateResult { #[serde(rename = "type")] pub update_type: UpdateType, pub duration: f64, // in seconds - pub enqueued_at: DateTime, - pub processed_at: DateTime, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + pub enqueued_at: OffsetDateTime, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + pub processed_at: OffsetDateTime, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -66,8 +68,10 @@ pub struct FailedUpdateResult { pub update_type: UpdateType, pub error: ResponseError, pub duration: f64, // in seconds - pub enqueued_at: DateTime, - pub processed_at: DateTime, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + pub enqueued_at: OffsetDateTime, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + pub processed_at: OffsetDateTime, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -76,9 +80,13 @@ pub struct EnqueuedUpdateResult { pub update_id: u64, #[serde(rename = "type")] pub update_type: UpdateType, - pub enqueued_at: DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub started_processing_at: Option>, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + pub enqueued_at: OffsetDateTime, + #[serde( + skip_serializing_if = "Option::is_none", + serialize_with = "time::serde::rfc3339::option::serialize" + )] + pub started_processing_at: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/meilisearch-http/src/task.rs b/meilisearch-http/src/task.rs index 9881854e3..7179b10db 100644 --- a/meilisearch-http/src/task.rs +++ b/meilisearch-http/src/task.rs @@ -1,4 +1,6 @@ -use chrono::{DateTime, Duration, Utc}; +use std::fmt::Write; +use std::write; + use meilisearch_error::ResponseError; use meilisearch_lib::index::{Settings, Unchecked}; use meilisearch_lib::milli::update::IndexDocumentsMethod; @@ -7,6 +9,7 @@ use meilisearch_lib::tasks::task::{ DocumentDeletion, Task, TaskContent, TaskEvent, TaskId, TaskResult, }; use serde::{Serialize, Serializer}; +use time::{Duration, OffsetDateTime}; use crate::AUTOBATCHING_ENABLED; @@ -79,14 +82,52 @@ enum TaskDetails { ClearAll { deleted_documents: Option }, } +/// Serialize a `time::Duration` as a best effort ISO 8601 while waiting for +/// https://github.com/time-rs/time/issues/378. +/// This code is a port of the old code of time that was removed in 0.2. fn serialize_duration( duration: &Option, serializer: S, ) -> Result { match duration { Some(duration) => { - let duration_str = duration.to_string(); - serializer.serialize_str(&duration_str) + // technically speaking, negative duration is not valid ISO 8601 + if duration.is_negative() { + return serializer.serialize_none(); + } + + const SECS_PER_DAY: i64 = Duration::DAY.whole_seconds(); + let secs = duration.whole_seconds(); + let days = secs / SECS_PER_DAY; + let secs = secs - days * SECS_PER_DAY; + let hasdate = days != 0; + let nanos = duration.subsec_nanoseconds(); + let hastime = (secs != 0 || nanos != 0) || !hasdate; + + // all the following unwrap can't fail + let mut res = String::new(); + write!(&mut res, "P").unwrap(); + + if hasdate { + write!(&mut res, "{}D", days).unwrap(); + } + + const NANOS_PER_MILLI: i32 = Duration::MILLISECOND.subsec_nanoseconds(); + const NANOS_PER_MICRO: i32 = Duration::MICROSECOND.subsec_nanoseconds(); + + if hastime { + if nanos == 0 { + write!(&mut res, "T{}S", secs).unwrap(); + } else if nanos % NANOS_PER_MILLI == 0 { + write!(&mut res, "T{}.{:03}S", secs, nanos / NANOS_PER_MILLI).unwrap(); + } else if nanos % NANOS_PER_MICRO == 0 { + write!(&mut res, "T{}.{:06}S", secs, nanos / NANOS_PER_MICRO).unwrap(); + } else { + write!(&mut res, "T{}.{:09}S", secs, nanos).unwrap(); + } + } + + serializer.serialize_str(&res) } None => serializer.serialize_none(), } @@ -106,9 +147,12 @@ pub struct TaskView { error: Option, #[serde(serialize_with = "serialize_duration")] duration: Option, - enqueued_at: DateTime, - started_at: Option>, - finished_at: Option>, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + enqueued_at: OffsetDateTime, + #[serde(serialize_with = "time::serde::rfc3339::option::serialize")] + started_at: Option, + #[serde(serialize_with = "time::serde::rfc3339::option::serialize")] + finished_at: Option, #[serde(skip_serializing_if = "Option::is_none")] batch_uid: Option>, } @@ -302,7 +346,8 @@ pub struct SummarizedTaskView { status: TaskStatus, #[serde(rename = "type")] task_type: TaskType, - enqueued_at: DateTime, + #[serde(serialize_with = "time::serde::rfc3339::serialize")] + enqueued_at: OffsetDateTime, } impl From for SummarizedTaskView { diff --git a/meilisearch-http/tests/auth/api_keys.rs b/meilisearch-http/tests/auth/api_keys.rs index 52915b96a..e9fb3d127 100644 --- a/meilisearch-http/tests/auth/api_keys.rs +++ b/meilisearch-http/tests/auth/api_keys.rs @@ -257,7 +257,7 @@ async fn error_add_api_key_missing_parameter() { "message": "`indexes` field is mandatory.", "code": "missing_parameter", "type": "invalid_request", - "link":"https://docs.meilisearch.com/errors#missing_parameter" + "link": "https://docs.meilisearch.com/errors#missing_parameter" }); assert_eq!(response, expected_response); @@ -275,7 +275,7 @@ async fn error_add_api_key_missing_parameter() { "message": "`actions` field is mandatory.", "code": "missing_parameter", "type": "invalid_request", - "link":"https://docs.meilisearch.com/errors#missing_parameter" + "link": "https://docs.meilisearch.com/errors#missing_parameter" }); assert_eq!(response, expected_response); @@ -293,7 +293,7 @@ async fn error_add_api_key_missing_parameter() { "message": "`expiresAt` field is mandatory.", "code": "missing_parameter", "type": "invalid_request", - "link":"https://docs.meilisearch.com/errors#missing_parameter" + "link": "https://docs.meilisearch.com/errors#missing_parameter" }); assert_eq!(response, expected_response); @@ -316,7 +316,7 @@ async fn error_add_api_key_invalid_parameters_description() { let (response, code) = server.add_api_key(content).await; let expected_response = json!({ - "message": r#"description field value `{"name":"products"}` is invalid. It should be a string or specified as a null value."#, + "message": r#"`description` field value `{"name":"products"}` is invalid. It should be a string or specified as a null value."#, "code": "invalid_api_key_description", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_description" @@ -342,7 +342,7 @@ async fn error_add_api_key_invalid_parameters_indexes() { let (response, code) = server.add_api_key(content).await; let expected_response = json!({ - "message": r#"indexes field value `{"name":"products"}` is invalid. It should be an array of string representing index names."#, + "message": r#"`indexes` field value `{"name":"products"}` is invalid. It should be an array of string representing index names."#, "code": "invalid_api_key_indexes", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_indexes" @@ -366,7 +366,7 @@ async fn error_add_api_key_invalid_parameters_actions() { let (response, code) = server.add_api_key(content).await; let expected_response = json!({ - "message": r#"actions field value `{"name":"products"}` is invalid. It should be an array of string representing action names."#, + "message": r#"`actions` field value `{"name":"products"}` is invalid. It should be an array of string representing action names."#, "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" @@ -386,7 +386,7 @@ async fn error_add_api_key_invalid_parameters_actions() { let (response, code) = server.add_api_key(content).await; let expected_response = json!({ - "message": r#"actions field value `["doc.add"]` is invalid. It should be an array of string representing action names."#, + "message": r#"`actions` field value `["doc.add"]` is invalid. It should be an array of string representing action names."#, "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" @@ -412,7 +412,7 @@ async fn error_add_api_key_invalid_parameters_expires_at() { let (response, code) = server.add_api_key(content).await; let expected_response = json!({ - "message": r#"expiresAt field value `{"name":"products"}` is invalid. It should be in ISO-8601 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DDTHH:MM:SS'."#, + "message": r#"`expiresAt` field value `{"name":"products"}` is invalid. It should follow the RFC 3339 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DD HH:MM:SS'."#, "code": "invalid_api_key_expires_at", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_expires_at" @@ -438,7 +438,7 @@ async fn error_add_api_key_invalid_parameters_expires_at_in_the_past() { let (response, code) = server.add_api_key(content).await; let expected_response = json!({ - "message": r#"expiresAt field value `"2010-11-13T00:00:00Z"` is invalid. It should be in ISO-8601 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DDTHH:MM:SS'."#, + "message": r#"`expiresAt` field value `"2010-11-13T00:00:00Z"` is invalid. It should follow the RFC 3339 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DD HH:MM:SS'."#, "code": "invalid_api_key_expires_at", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_expires_at" @@ -1213,7 +1213,7 @@ async fn error_patch_api_key_indexes_invalid_parameters() { let (response, code) = server.patch_api_key(&key, content).await; let expected_response = json!({ - "message": "description field value `13` is invalid. It should be a string or specified as a null value.", + "message": "`description` field value `13` is invalid. It should be a string or specified as a null value.", "code": "invalid_api_key_description", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_description" @@ -1230,7 +1230,7 @@ async fn error_patch_api_key_indexes_invalid_parameters() { let (response, code) = server.patch_api_key(&key, content).await; let expected_response = json!({ - "message": "indexes field value `13` is invalid. It should be an array of string representing index names.", + "message": "`indexes` field value `13` is invalid. It should be an array of string representing index names.", "code": "invalid_api_key_indexes", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_indexes" @@ -1246,7 +1246,7 @@ async fn error_patch_api_key_indexes_invalid_parameters() { let (response, code) = server.patch_api_key(&key, content).await; let expected_response = json!({ - "message": "actions field value `13` is invalid. It should be an array of string representing action names.", + "message": "`actions` field value `13` is invalid. It should be an array of string representing action names.", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" @@ -1262,7 +1262,7 @@ async fn error_patch_api_key_indexes_invalid_parameters() { let (response, code) = server.patch_api_key(&key, content).await; let expected_response = json!({ - "message": "expiresAt field value `13` is invalid. It should be in ISO-8601 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DDTHH:MM:SS'.", + "message": "`expiresAt` field value `13` is invalid. It should follow the RFC 3339 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DD HH:MM:SS'.", "code": "invalid_api_key_expires_at", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_expires_at" diff --git a/meilisearch-http/tests/auth/authorization.rs b/meilisearch-http/tests/auth/authorization.rs index de13c6194..30df2dd2d 100644 --- a/meilisearch-http/tests/auth/authorization.rs +++ b/meilisearch-http/tests/auth/authorization.rs @@ -1,9 +1,10 @@ use crate::common::Server; -use chrono::{Duration, Utc}; +use ::time::format_description::well_known::Rfc3339; use maplit::{hashmap, hashset}; use once_cell::sync::Lazy; use serde_json::{json, Value}; use std::collections::{HashMap, HashSet}; +use time::{Duration, OffsetDateTime}; pub static AUTHORIZATIONS: Lazy>> = Lazy::new(|| { @@ -76,7 +77,7 @@ async fn error_access_expired_key() { let content = json!({ "indexes": ["products"], "actions": ALL_ACTIONS.clone(), - "expiresAt": (Utc::now() + Duration::seconds(1)), + "expiresAt": (OffsetDateTime::now_utc() + Duration::seconds(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; @@ -106,7 +107,7 @@ async fn error_access_unauthorized_index() { let content = json!({ "indexes": ["sales"], "actions": ALL_ACTIONS.clone(), - "expiresAt": Utc::now() + Duration::hours(1), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; @@ -137,7 +138,7 @@ async fn error_access_unauthorized_action() { let content = json!({ "indexes": ["products"], "actions": [], - "expiresAt": Utc::now() + Duration::hours(1), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; @@ -174,7 +175,7 @@ async fn access_authorized_restricted_index() { let content = json!({ "indexes": ["products"], "actions": [], - "expiresAt": Utc::now() + Duration::hours(1), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; @@ -213,7 +214,7 @@ async fn access_authorized_no_index_restriction() { let content = json!({ "indexes": ["*"], "actions": [], - "expiresAt": Utc::now() + Duration::hours(1), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; @@ -263,7 +264,7 @@ async fn access_authorized_stats_restricted_index() { let content = json!({ "indexes": ["products"], "actions": ["stats.get"], - "expiresAt": Utc::now() + Duration::hours(1), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; assert_eq!(code, 201); @@ -303,7 +304,7 @@ async fn access_authorized_stats_no_index_restriction() { let content = json!({ "indexes": ["*"], "actions": ["stats.get"], - "expiresAt": Utc::now() + Duration::hours(1), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; assert_eq!(code, 201); @@ -343,7 +344,7 @@ async fn list_authorized_indexes_restricted_index() { let content = json!({ "indexes": ["products"], "actions": ["indexes.get"], - "expiresAt": Utc::now() + Duration::hours(1), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; assert_eq!(code, 201); @@ -384,7 +385,7 @@ async fn list_authorized_indexes_no_index_restriction() { let content = json!({ "indexes": ["*"], "actions": ["indexes.get"], - "expiresAt": Utc::now() + Duration::hours(1), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; assert_eq!(code, 201); @@ -424,7 +425,7 @@ async fn list_authorized_tasks_restricted_index() { let content = json!({ "indexes": ["products"], "actions": ["tasks.get"], - "expiresAt": Utc::now() + Duration::hours(1), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; assert_eq!(code, 201); @@ -464,7 +465,7 @@ async fn list_authorized_tasks_no_index_restriction() { let content = json!({ "indexes": ["*"], "actions": ["tasks.get"], - "expiresAt": Utc::now() + Duration::hours(1), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; assert_eq!(code, 201); diff --git a/meilisearch-http/tests/auth/tenant_token.rs b/meilisearch-http/tests/auth/tenant_token.rs index bad6de5e6..bb9224590 100644 --- a/meilisearch-http/tests/auth/tenant_token.rs +++ b/meilisearch-http/tests/auth/tenant_token.rs @@ -1,9 +1,10 @@ use crate::common::Server; -use chrono::{Duration, Utc}; +use ::time::format_description::well_known::Rfc3339; use maplit::hashmap; use once_cell::sync::Lazy; use serde_json::{json, Value}; use std::collections::HashMap; +use time::{Duration, OffsetDateTime}; use super::authorization::{ALL_ACTIONS, AUTHORIZATIONS}; @@ -63,22 +64,22 @@ static ACCEPTED_KEYS: Lazy> = Lazy::new(|| { json!({ "indexes": ["*"], "actions": ["*"], - "expiresAt": Utc::now() + Duration::days(1) + "expiresAt": (OffsetDateTime::now_utc() + Duration::days(1)).format(&Rfc3339).unwrap() }), json!({ "indexes": ["*"], "actions": ["search"], - "expiresAt": Utc::now() + Duration::days(1) + "expiresAt": (OffsetDateTime::now_utc() + Duration::days(1)).format(&Rfc3339).unwrap() }), json!({ "indexes": ["sales"], "actions": ["*"], - "expiresAt": Utc::now() + Duration::days(1) + "expiresAt": (OffsetDateTime::now_utc() + Duration::days(1)).format(&Rfc3339).unwrap() }), json!({ "indexes": ["sales"], "actions": ["search"], - "expiresAt": Utc::now() + Duration::days(1) + "expiresAt": (OffsetDateTime::now_utc() + Duration::days(1)).format(&Rfc3339).unwrap() }), ] }); @@ -89,23 +90,23 @@ static REFUSED_KEYS: Lazy> = Lazy::new(|| { json!({ "indexes": ["*"], "actions": ALL_ACTIONS.iter().cloned().filter(|a| *a != "search" && *a != "*").collect::>(), - "expiresAt": Utc::now() + Duration::days(1) + "expiresAt": (OffsetDateTime::now_utc() + Duration::days(1)).format(&Rfc3339).unwrap() }), json!({ "indexes": ["sales"], "actions": ALL_ACTIONS.iter().cloned().filter(|a| *a != "search" && *a != "*").collect::>(), - "expiresAt": Utc::now() + Duration::days(1) + "expiresAt": (OffsetDateTime::now_utc() + Duration::days(1)).format(&Rfc3339).unwrap() }), // bad index json!({ "indexes": ["products"], "actions": ["*"], - "expiresAt": Utc::now() + Duration::days(1) + "expiresAt": (OffsetDateTime::now_utc() + Duration::days(1)).format(&Rfc3339).unwrap() }), json!({ "indexes": ["products"], "actions": ["search"], - "expiresAt": Utc::now() + Duration::days(1) + "expiresAt": (OffsetDateTime::now_utc() + Duration::days(1)).format(&Rfc3339).unwrap() }), ] }); @@ -204,19 +205,19 @@ async fn search_authorized_simple_token() { let tenant_tokens = vec![ hashmap! { "searchRules" => json!({"*": {}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!(["*"]), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"sales": {}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!(["sales"]), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"*": {}}), @@ -253,19 +254,19 @@ async fn search_authorized_filter_token() { let tenant_tokens = vec![ hashmap! { "searchRules" => json!({"*": {"filter": "color = blue"}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"sales": {"filter": "color = blue"}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"*": {"filter": ["color = blue"]}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"sales": {"filter": ["color = blue"]}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, // filter on sales should override filters on * hashmap! { @@ -273,28 +274,28 @@ async fn search_authorized_filter_token() { "*": {"filter": "color = green"}, "sales": {"filter": "color = blue"} }), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({ "*": {}, "sales": {"filter": "color = blue"} }), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({ "*": {"filter": "color = green"}, "sales": {"filter": ["color = blue"]} }), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({ "*": {}, "sales": {"filter": ["color = blue"]} }), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, ]; @@ -307,19 +308,19 @@ async fn filter_search_authorized_filter_token() { let tenant_tokens = vec![ hashmap! { "searchRules" => json!({"*": {"filter": "color = blue"}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"sales": {"filter": "color = blue"}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"*": {"filter": ["color = blue"]}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"sales": {"filter": ["color = blue"]}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, // filter on sales should override filters on * hashmap! { @@ -327,28 +328,28 @@ async fn filter_search_authorized_filter_token() { "*": {"filter": "color = green"}, "sales": {"filter": "color = blue"} }), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({ "*": {}, "sales": {"filter": "color = blue"} }), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({ "*": {"filter": "color = green"}, "sales": {"filter": ["color = blue"]} }), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({ "*": {}, "sales": {"filter": ["color = blue"]} }), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, ]; @@ -361,27 +362,27 @@ async fn error_search_token_forbidden_parent_key() { let tenant_tokens = vec![ hashmap! { "searchRules" => json!({"*": {}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"*": Value::Null}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!(["*"]), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"sales": {}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"sales": Value::Null}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!(["sales"]), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, ]; @@ -395,11 +396,11 @@ async fn error_search_forbidden_token() { // bad index hashmap! { "searchRules" => json!({"products": {}}), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!(["products"]), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"products": {}}), @@ -416,27 +417,27 @@ async fn error_search_forbidden_token() { // expired token hashmap! { "searchRules" => json!({"*": {}}), - "exp" => json!((Utc::now() - Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"*": Value::Null}), - "exp" => json!((Utc::now() - Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!(["*"]), - "exp" => json!((Utc::now() - Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"sales": {}}), - "exp" => json!((Utc::now() - Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!({"sales": Value::Null}), - "exp" => json!((Utc::now() - Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp()) }, hashmap! { "searchRules" => json!(["sales"]), - "exp" => json!((Utc::now() - Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp()) }, ]; @@ -452,7 +453,7 @@ async fn error_access_forbidden_routes() { let content = json!({ "indexes": ["*"], "actions": ["*"], - "expiresAt": (Utc::now() + Duration::hours(1)), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; @@ -463,7 +464,7 @@ async fn error_access_forbidden_routes() { let tenant_token = hashmap! { "searchRules" => json!(["*"]), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }; let web_token = generate_tenant_token(&key, tenant_token); server.use_api_key(&web_token); @@ -487,7 +488,7 @@ async fn error_access_expired_parent_key() { let content = json!({ "indexes": ["*"], "actions": ["*"], - "expiresAt": (Utc::now() + Duration::seconds(1)), + "expiresAt": (OffsetDateTime::now_utc() + Duration::seconds(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; @@ -498,7 +499,7 @@ async fn error_access_expired_parent_key() { let tenant_token = hashmap! { "searchRules" => json!(["*"]), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }; let web_token = generate_tenant_token(&key, tenant_token); server.use_api_key(&web_token); @@ -529,7 +530,7 @@ async fn error_access_modified_token() { let content = json!({ "indexes": ["*"], "actions": ["*"], - "expiresAt": (Utc::now() + Duration::hours(1)), + "expiresAt": (OffsetDateTime::now_utc() + Duration::hours(1)).format(&Rfc3339).unwrap(), }); let (response, code) = server.add_api_key(content).await; @@ -540,7 +541,7 @@ async fn error_access_modified_token() { let tenant_token = hashmap! { "searchRules" => json!(["products"]), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }; let web_token = generate_tenant_token(&key, tenant_token); server.use_api_key(&web_token); @@ -554,7 +555,7 @@ async fn error_access_modified_token() { let tenant_token = hashmap! { "searchRules" => json!(["*"]), - "exp" => json!((Utc::now() + Duration::hours(1)).timestamp()) + "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp()) }; let alt = generate_tenant_token(&key, tenant_token); diff --git a/meilisearch-http/tests/documents/add_documents.rs b/meilisearch-http/tests/documents/add_documents.rs index a8340096c..c684458c5 100644 --- a/meilisearch-http/tests/documents/add_documents.rs +++ b/meilisearch-http/tests/documents/add_documents.rs @@ -1,8 +1,8 @@ use crate::common::{GetAllDocumentsOptions, Server}; use actix_web::test; -use chrono::DateTime; use meilisearch_http::{analytics, create_app}; use serde_json::{json, Value}; +use time::{format_description::well_known::Rfc3339, OffsetDateTime}; /// This is the basic usage of our API and every other tests uses the content-type application/json #[actix_rt::test] @@ -568,9 +568,9 @@ async fn add_documents_no_index_creation() { assert_eq!(response["details"]["indexedDocuments"], 1); let processed_at = - DateTime::parse_from_rfc3339(response["finishedAt"].as_str().unwrap()).unwrap(); + OffsetDateTime::parse(response["finishedAt"].as_str().unwrap(), &Rfc3339).unwrap(); let enqueued_at = - DateTime::parse_from_rfc3339(response["enqueuedAt"].as_str().unwrap()).unwrap(); + OffsetDateTime::parse(response["enqueuedAt"].as_str().unwrap(), &Rfc3339).unwrap(); assert!(processed_at > enqueued_at); // index was created, and primary key was infered. diff --git a/meilisearch-http/tests/index/update_index.rs b/meilisearch-http/tests/index/update_index.rs index 0246b55ef..1896f731f 100644 --- a/meilisearch-http/tests/index/update_index.rs +++ b/meilisearch-http/tests/index/update_index.rs @@ -1,6 +1,6 @@ use crate::common::Server; -use chrono::DateTime; use serde_json::json; +use time::{format_description::well_known::Rfc3339, OffsetDateTime}; #[actix_rt::test] async fn update_primary_key() { @@ -25,8 +25,10 @@ async fn update_primary_key() { assert!(response.get("createdAt").is_some()); assert!(response.get("updatedAt").is_some()); - let created_at = DateTime::parse_from_rfc3339(response["createdAt"].as_str().unwrap()).unwrap(); - let updated_at = DateTime::parse_from_rfc3339(response["updatedAt"].as_str().unwrap()).unwrap(); + let created_at = + OffsetDateTime::parse(response["createdAt"].as_str().unwrap(), &Rfc3339).unwrap(); + let updated_at = + OffsetDateTime::parse(response["updatedAt"].as_str().unwrap(), &Rfc3339).unwrap(); assert!(created_at < updated_at); assert_eq!(response["primaryKey"], "primary"); diff --git a/meilisearch-http/tests/tasks/mod.rs b/meilisearch-http/tests/tasks/mod.rs index 3edb89376..167b7b05f 100644 --- a/meilisearch-http/tests/tasks/mod.rs +++ b/meilisearch-http/tests/tasks/mod.rs @@ -1,6 +1,7 @@ use crate::common::Server; -use chrono::{DateTime, Utc}; use serde_json::json; +use time::format_description::well_known::Rfc3339; +use time::OffsetDateTime; #[actix_rt::test] async fn error_get_task_unexisting_index() { @@ -98,7 +99,8 @@ macro_rules! assert_valid_summarized_task { assert_eq!($response["status"], "enqueued"); assert_eq!($response["type"], $task_type); let date = $response["enqueuedAt"].as_str().expect("missing date"); - date.parse::>().unwrap(); + + OffsetDateTime::parse(date, &Rfc3339).unwrap(); }}; } diff --git a/meilisearch-lib/Cargo.toml b/meilisearch-lib/Cargo.toml index 367ab84b2..03456999f 100644 --- a/meilisearch-lib/Cargo.toml +++ b/meilisearch-lib/Cargo.toml @@ -12,7 +12,6 @@ async-stream = "0.3.2" async-trait = "0.1.51" byte-unit = { version = "4.0.12", default-features = false, features = ["std"] } bytes = "1.1.0" -chrono = { version = "0.4.19", features = ["serde"] } csv = "1.1.6" crossbeam-channel = "0.5.1" either = "1.6.1" @@ -28,7 +27,7 @@ lazy_static = "1.4.0" log = "0.4.14" meilisearch-error = { path = "../meilisearch-error" } meilisearch-auth = { path = "../meilisearch-auth" } -milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.22.1" } +milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.22.2" } mime = "0.3.16" num_cpus = "1.13.0" once_cell = "1.8.0" @@ -45,6 +44,7 @@ clap = { version = "3.0", features = ["derive", "env"] } tar = "0.4.37" tempfile = "3.2.0" thiserror = "1.0.28" +time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] } tokio = { version = "1.11.0", features = ["full"] } uuid = { version = "0.8.2", features = ["serde"] } walkdir = "2.3.2" diff --git a/meilisearch-lib/src/index/index.rs b/meilisearch-lib/src/index/index.rs index 41803b6d5..8eadc8c4a 100644 --- a/meilisearch-lib/src/index/index.rs +++ b/meilisearch-lib/src/index/index.rs @@ -5,12 +5,12 @@ use std::ops::Deref; use std::path::Path; use std::sync::Arc; -use chrono::{DateTime, Utc}; use heed::{EnvOpenOptions, RoTxn}; use milli::update::{IndexerConfig, Setting}; use milli::{obkv_to_json, FieldDistribution, FieldId}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +use time::OffsetDateTime; use uuid::Uuid; use crate::EnvSizer; @@ -24,8 +24,10 @@ pub type Document = Map; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMeta { - pub created_at: DateTime, - pub updated_at: DateTime, + #[serde(with = "time::serde::rfc3339")] + pub created_at: OffsetDateTime, + #[serde(with = "time::serde::rfc3339")] + pub updated_at: OffsetDateTime, pub primary_key: Option, } diff --git a/meilisearch-lib/src/index_controller/dump_actor/actor.rs b/meilisearch-lib/src/index_controller/dump_actor/actor.rs index c9b871c0e..a94eac6ce 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/actor.rs @@ -3,9 +3,10 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use async_stream::stream; -use chrono::Utc; use futures::{lock::Mutex, stream::StreamExt}; use log::{error, trace}; +use time::macros::format_description; +use time::OffsetDateTime; use tokio::sync::{mpsc, oneshot, RwLock}; use super::error::{DumpActorError, Result}; @@ -29,7 +30,9 @@ pub struct DumpActor { /// Generate uid from creation date fn generate_uid() -> String { - Utc::now().format("%Y%m%d-%H%M%S%3f").to_string() + OffsetDateTime::now_utc() + .format(format_description!("%Y%m%d-%H%M%S%3f")) + .unwrap() } impl DumpActor { diff --git a/meilisearch-lib/src/index_controller/dump_actor/compat/v2.rs b/meilisearch-lib/src/index_controller/dump_actor/compat/v2.rs index 9af0f11b5..d7ba41479 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/compat/v2.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/compat/v2.rs @@ -1,8 +1,8 @@ use anyhow::bail; -use chrono::{DateTime, Utc}; use meilisearch_error::Code; use milli::update::IndexDocumentsMethod; use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; use uuid::Uuid; use crate::index::{Settings, Unchecked}; @@ -51,7 +51,7 @@ pub enum UpdateMeta { pub struct Enqueued { pub update_id: u64, pub meta: UpdateMeta, - pub enqueued_at: DateTime, + pub enqueued_at: OffsetDateTime, pub content: Option, } @@ -59,7 +59,7 @@ pub struct Enqueued { #[serde(rename_all = "camelCase")] pub struct Processed { pub success: UpdateResult, - pub processed_at: DateTime, + pub processed_at: OffsetDateTime, #[serde(flatten)] pub from: Processing, } @@ -69,7 +69,7 @@ pub struct Processed { pub struct Processing { #[serde(flatten)] pub from: Enqueued, - pub started_processing_at: DateTime, + pub started_processing_at: OffsetDateTime, } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -77,7 +77,7 @@ pub struct Processing { pub struct Aborted { #[serde(flatten)] pub from: Enqueued, - pub aborted_at: DateTime, + pub aborted_at: OffsetDateTime, } #[derive(Debug, Serialize, Deserialize)] @@ -86,7 +86,7 @@ pub struct Failed { #[serde(flatten)] pub from: Processing, pub error: ResponseError, - pub failed_at: DateTime, + pub failed_at: OffsetDateTime, } #[derive(Debug, Serialize, Deserialize)] diff --git a/meilisearch-lib/src/index_controller/dump_actor/compat/v3.rs b/meilisearch-lib/src/index_controller/dump_actor/compat/v3.rs index 597c11fe0..1e378805d 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/compat/v3.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/compat/v3.rs @@ -1,7 +1,7 @@ -use chrono::{DateTime, Utc}; use meilisearch_error::{Code, ResponseError}; use milli::update::IndexDocumentsMethod; use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; use uuid::Uuid; use crate::index::{Settings, Unchecked}; @@ -107,7 +107,7 @@ pub enum UpdateMeta { pub struct Enqueued { pub update_id: u64, pub meta: Update, - pub enqueued_at: DateTime, + pub enqueued_at: OffsetDateTime, } impl Enqueued { @@ -122,7 +122,7 @@ impl Enqueued { #[serde(rename_all = "camelCase")] pub struct Processed { pub success: v2::UpdateResult, - pub processed_at: DateTime, + pub processed_at: OffsetDateTime, #[serde(flatten)] pub from: Processing, } @@ -144,7 +144,7 @@ impl Processed { pub struct Processing { #[serde(flatten)] pub from: Enqueued, - pub started_processing_at: DateTime, + pub started_processing_at: OffsetDateTime, } impl Processing { @@ -163,7 +163,7 @@ pub struct Failed { pub from: Processing, pub msg: String, pub code: Code, - pub failed_at: DateTime, + pub failed_at: OffsetDateTime, } impl Failed { diff --git a/meilisearch-lib/src/index_controller/dump_actor/mod.rs b/meilisearch-lib/src/index_controller/dump_actor/mod.rs index 2c0f464d2..d9ff63f2c 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/mod.rs @@ -3,9 +3,9 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::bail; -use chrono::{DateTime, Utc}; use log::{info, trace}; use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; pub use actor::DumpActor; pub use handle_impl::*; @@ -40,7 +40,7 @@ pub struct Metadata { db_version: String, index_db_size: usize, update_db_size: usize, - dump_date: DateTime, + dump_date: OffsetDateTime, } impl Metadata { @@ -49,7 +49,7 @@ impl Metadata { db_version: env!("CARGO_PKG_VERSION").to_string(), index_db_size, update_db_size, - dump_date: Utc::now(), + dump_date: OffsetDateTime::now_utc(), } } } @@ -144,7 +144,7 @@ impl MetadataVersion { } } - pub fn dump_date(&self) -> Option<&DateTime> { + pub fn dump_date(&self) -> Option<&OffsetDateTime> { match self { MetadataVersion::V1(_) => None, MetadataVersion::V2(meta) | MetadataVersion::V3(meta) | MetadataVersion::V4(meta) => { @@ -169,9 +169,13 @@ pub struct DumpInfo { pub status: DumpStatus, #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, - started_at: DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - finished_at: Option>, + #[serde(with = "time::serde::rfc3339")] + started_at: OffsetDateTime, + #[serde( + skip_serializing_if = "Option::is_none", + with = "time::serde::rfc3339::option" + )] + finished_at: Option, } impl DumpInfo { @@ -180,19 +184,19 @@ impl DumpInfo { uid, status, error: None, - started_at: Utc::now(), + started_at: OffsetDateTime::now_utc(), finished_at: None, } } pub fn with_error(&mut self, error: String) { self.status = DumpStatus::Failed; - self.finished_at = Some(Utc::now()); + self.finished_at = Some(OffsetDateTime::now_utc()); self.error = Some(error); } pub fn done(&mut self) { - self.finished_at = Some(Utc::now()); + self.finished_at = Some(OffsetDateTime::now_utc()); self.status = DumpStatus::Done; } diff --git a/meilisearch-lib/src/index_controller/mod.rs b/meilisearch-lib/src/index_controller/mod.rs index 5dfaae848..8995df39c 100644 --- a/meilisearch-lib/src/index_controller/mod.rs +++ b/meilisearch-lib/src/index_controller/mod.rs @@ -8,11 +8,11 @@ use std::time::Duration; use actix_web::error::PayloadError; use bytes::Bytes; -use chrono::{DateTime, Utc}; use futures::Stream; use futures::StreamExt; use milli::update::IndexDocumentsMethod; use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; use tokio::sync::{mpsc, RwLock}; use tokio::task::spawn_blocking; use tokio::time::sleep; @@ -107,7 +107,7 @@ impl fmt::Display for DocumentAdditionFormat { #[serde(rename_all = "camelCase")] pub struct Stats { pub database_size: u64, - pub last_update: Option>, + pub last_update: Option, pub indexes: BTreeMap, } @@ -579,7 +579,7 @@ where } pub async fn get_all_stats(&self, search_rules: &SearchRules) -> Result { - let mut last_task: Option> = None; + let mut last_task: Option = None; let mut indexes = BTreeMap::new(); let mut database_size = 0; let processing_tasks = self.scheduler.read().await.get_processing_tasks().await?; diff --git a/meilisearch-lib/src/index_resolver/mod.rs b/meilisearch-lib/src/index_resolver/mod.rs index 48201d39a..9428d4a78 100644 --- a/meilisearch-lib/src/index_resolver/mod.rs +++ b/meilisearch-lib/src/index_resolver/mod.rs @@ -6,7 +6,6 @@ use std::convert::{TryFrom, TryInto}; use std::path::Path; use std::sync::Arc; -use chrono::Utc; use error::{IndexResolverError, Result}; use heed::Env; use index_store::{IndexStore, MapIndexStore}; @@ -14,6 +13,7 @@ use meilisearch_error::ResponseError; use meta_store::{HeedMetaStore, IndexMetaStore}; use milli::update::{DocumentDeletionResult, IndexerConfig}; use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; use tokio::sync::oneshot; use tokio::task::spawn_blocking; use uuid::Uuid; @@ -115,18 +115,19 @@ where self.process_document_addition_batch(batch).await } else { if let Some(task) = batch.tasks.first_mut() { - task.events.push(TaskEvent::Processing(Utc::now())); + task.events + .push(TaskEvent::Processing(OffsetDateTime::now_utc())); match self.process_task(task).await { Ok(success) => { task.events.push(TaskEvent::Succeded { result: success, - timestamp: Utc::now(), + timestamp: OffsetDateTime::now_utc(), }); } Err(err) => task.events.push(TaskEvent::Failed { error: err.into(), - timestamp: Utc::now(), + timestamp: OffsetDateTime::now_utc(), }), } } @@ -225,7 +226,7 @@ where // If the index doesn't exist and we are not allowed to create it with the first // task, we must fails the whole batch. - let now = Utc::now(); + let now = OffsetDateTime::now_utc(); let index = match index { Ok(index) => index, Err(e) => { @@ -253,17 +254,17 @@ where let event = match result { Ok(Ok(result)) => TaskEvent::Succeded { - timestamp: Utc::now(), + timestamp: OffsetDateTime::now_utc(), result: TaskResult::DocumentAddition { indexed_documents: result.indexed_documents, }, }, Ok(Err(e)) => TaskEvent::Failed { - timestamp: Utc::now(), + timestamp: OffsetDateTime::now_utc(), error: e.into(), }, Err(e) => TaskEvent::Failed { - timestamp: Utc::now(), + timestamp: OffsetDateTime::now_utc(), error: IndexResolverError::from(e).into(), }, }; @@ -524,7 +525,7 @@ mod test { }; if primary_key.is_some() { mocker.when::>("update_primary_key") - .then(move |_| Ok(IndexMeta{ created_at: Utc::now(), updated_at: Utc::now(), primary_key: None })); + .then(move |_| Ok(IndexMeta{ created_at: OffsetDateTime::now_utc(), updated_at: OffsetDateTime::now_utc(), primary_key: None })); } mocker.when::<(IndexDocumentsMethod, Option, UpdateFileStore, IntoIter), IndexResult>("update_documents") .then(move |(_, _, _, _)| result()); @@ -569,7 +570,7 @@ mod test { | TaskContent::IndexCreation { primary_key } => { if primary_key.is_some() { let result = move || if !index_op_fails { - Ok(IndexMeta{ created_at: Utc::now(), updated_at: Utc::now(), primary_key: None }) + Ok(IndexMeta{ created_at: OffsetDateTime::now_utc(), updated_at: OffsetDateTime::now_utc(), primary_key: None }) } else { // return this error because it's easy to generate... Err(IndexError::DocumentNotFound("a doc".into())) @@ -640,7 +641,7 @@ mod test { let update_file_store = UpdateFileStore::mock(mocker); let index_resolver = IndexResolver::new(uuid_store, index_store, update_file_store); - let batch = Batch { id: 1, created_at: Utc::now(), tasks: vec![task.clone()] }; + let batch = Batch { id: 1, created_at: OffsetDateTime::now_utc(), tasks: vec![task.clone()] }; let result = index_resolver.process_batch(batch).await; // Test for some expected output scenarios: diff --git a/meilisearch-lib/src/tasks/batch.rs b/meilisearch-lib/src/tasks/batch.rs index eff81acc5..4a8cf7907 100644 --- a/meilisearch-lib/src/tasks/batch.rs +++ b/meilisearch-lib/src/tasks/batch.rs @@ -1,4 +1,4 @@ -use chrono::{DateTime, Utc}; +use time::OffsetDateTime; use super::task::Task; @@ -7,7 +7,7 @@ pub type BatchId = u64; #[derive(Debug)] pub struct Batch { pub id: BatchId, - pub created_at: DateTime, + pub created_at: OffsetDateTime, pub tasks: Vec, } diff --git a/meilisearch-lib/src/tasks/scheduler.rs b/meilisearch-lib/src/tasks/scheduler.rs index bbb9cb2e2..0aa72fcf4 100644 --- a/meilisearch-lib/src/tasks/scheduler.rs +++ b/meilisearch-lib/src/tasks/scheduler.rs @@ -6,8 +6,8 @@ use std::sync::Arc; use std::time::Duration; use atomic_refcell::AtomicRefCell; -use chrono::Utc; use milli::update::IndexDocumentsMethod; +use time::OffsetDateTime; use tokio::sync::{watch, RwLock}; use crate::options::SchedulerConfig; @@ -357,7 +357,7 @@ impl Scheduler { tasks.iter_mut().for_each(|t| { t.events.push(TaskEvent::Batched { batch_id: id, - timestamp: Utc::now(), + timestamp: OffsetDateTime::now_utc(), }) }); @@ -365,7 +365,7 @@ impl Scheduler { let batch = Batch { id, - created_at: Utc::now(), + created_at: OffsetDateTime::now_utc(), tasks, }; diff --git a/meilisearch-lib/src/tasks/task.rs b/meilisearch-lib/src/tasks/task.rs index f5d6687cd..1c67fafa8 100644 --- a/meilisearch-lib/src/tasks/task.rs +++ b/meilisearch-lib/src/tasks/task.rs @@ -1,9 +1,9 @@ use std::path::PathBuf; -use chrono::{DateTime, Utc}; use meilisearch_error::ResponseError; use milli::update::{DocumentAdditionResult, IndexDocumentsMethod}; use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; use tokio::sync::oneshot; use uuid::Uuid; @@ -36,22 +36,22 @@ impl From for TaskResult { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr(test, derive(proptest_derive::Arbitrary))] pub enum TaskEvent { - Created(#[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] DateTime), + Created(#[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] OffsetDateTime), Batched { #[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] - timestamp: DateTime, + timestamp: OffsetDateTime, batch_id: BatchId, }, - Processing(#[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] DateTime), + Processing(#[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] OffsetDateTime), Succeded { result: TaskResult, #[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] - timestamp: DateTime, + timestamp: OffsetDateTime, }, Failed { error: ResponseError, #[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] - timestamp: DateTime, + timestamp: OffsetDateTime, }, } @@ -165,7 +165,7 @@ mod test { ] } - pub(super) fn datetime_strategy() -> impl Strategy> { - Just(Utc::now()) + pub(super) fn datetime_strategy() -> impl Strategy { + Just(OffsetDateTime::now_utc()) } } diff --git a/meilisearch-lib/src/tasks/task_store/mod.rs b/meilisearch-lib/src/tasks/task_store/mod.rs index 88f12ddd1..695981d25 100644 --- a/meilisearch-lib/src/tasks/task_store/mod.rs +++ b/meilisearch-lib/src/tasks/task_store/mod.rs @@ -5,9 +5,9 @@ use std::io::{BufWriter, Write}; use std::path::Path; use std::sync::Arc; -use chrono::Utc; use heed::{Env, RwTxn}; use log::debug; +use time::OffsetDateTime; use super::error::TaskError; use super::task::{Task, TaskContent, TaskId}; @@ -72,7 +72,7 @@ impl TaskStore { let task = tokio::task::spawn_blocking(move || -> Result { let mut txn = store.wtxn()?; let next_task_id = store.next_task_id(&mut txn)?; - let created_at = TaskEvent::Created(Utc::now()); + let created_at = TaskEvent::Created(OffsetDateTime::now_utc()); let task = Task { id: next_task_id, index_uid, diff --git a/meilisearch-lib/src/tasks/update_loop.rs b/meilisearch-lib/src/tasks/update_loop.rs index 5cdbf1b46..b09811721 100644 --- a/meilisearch-lib/src/tasks/update_loop.rs +++ b/meilisearch-lib/src/tasks/update_loop.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::time::Duration; -use chrono::Utc; +use time::OffsetDateTime; use tokio::sync::{watch, RwLock}; use tokio::time::interval_at; @@ -63,7 +63,8 @@ where match pending { Pending::Batch(mut batch) => { for task in &mut batch.tasks { - task.events.push(TaskEvent::Processing(Utc::now())); + task.events + .push(TaskEvent::Processing(OffsetDateTime::now_utc())); } batch.tasks = { From 21d277a0ef935fe2481abdd0c99ad67e458bf814 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 22 Feb 2022 01:57:46 +0100 Subject: [PATCH 04/16] fix(all): fix two dates that were wrongly formatted --- meilisearch-http/tests/stats/mod.rs | 7 +++- .../src/index_controller/dump_actor/actor.rs | 34 ++++++++++++++++++- meilisearch-lib/src/index_controller/mod.rs | 1 + 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/meilisearch-http/tests/stats/mod.rs b/meilisearch-http/tests/stats/mod.rs index e89d145e1..b9d185ca3 100644 --- a/meilisearch-http/tests/stats/mod.rs +++ b/meilisearch-http/tests/stats/mod.rs @@ -1,4 +1,5 @@ use serde_json::json; +use time::{format_description::well_known::Rfc3339, OffsetDateTime}; use crate::common::Server; @@ -57,11 +58,15 @@ async fn stats() { index.wait_task(1).await; + let timestamp = OffsetDateTime::now_utc(); let (response, code) = server.stats().await; assert_eq!(code, 200); assert!(response["databaseSize"].as_u64().unwrap() > 0); - assert!(response.get("lastUpdate").is_some()); + let last_update = + OffsetDateTime::parse(response["lastUpdate"].as_str().unwrap(), &Rfc3339).unwrap(); + assert!(last_update - timestamp < time::Duration::SECOND); + assert_eq!(response["indexes"]["test"]["numberOfDocuments"], 2); assert!(response["indexes"]["test"]["isIndexing"] == false); assert_eq!(response["indexes"]["test"]["fieldDistribution"]["id"], 2); diff --git a/meilisearch-lib/src/index_controller/dump_actor/actor.rs b/meilisearch-lib/src/index_controller/dump_actor/actor.rs index a94eac6ce..48fc077ca 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/actor.rs @@ -31,7 +31,9 @@ pub struct DumpActor { /// Generate uid from creation date fn generate_uid() -> String { OffsetDateTime::now_utc() - .format(format_description!("%Y%m%d-%H%M%S%3f")) + .format(format_description!( + "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]" + )) .unwrap() } @@ -157,3 +159,33 @@ impl DumpActor { } } } + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_generate_uid() { + let current = OffsetDateTime::now_utc(); + + let uid = generate_uid(); + let (date, time) = uid.split_once('-').unwrap(); + + let date = time::Date::parse( + date, + &format_description!("[year repr:full][month repr:numerical][day padding:zero]"), + ) + .unwrap(); + let time = time::Time::parse( + time, + &format_description!( + "[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]" + ), + ) + .unwrap(); + let datetime = time::PrimitiveDateTime::new(date, time); + let datetime = datetime.assume_utc(); + + assert!(current - datetime < time::Duration::SECOND); + } +} diff --git a/meilisearch-lib/src/index_controller/mod.rs b/meilisearch-lib/src/index_controller/mod.rs index 8995df39c..06f64f1c1 100644 --- a/meilisearch-lib/src/index_controller/mod.rs +++ b/meilisearch-lib/src/index_controller/mod.rs @@ -107,6 +107,7 @@ impl fmt::Display for DocumentAdditionFormat { #[serde(rename_all = "camelCase")] pub struct Stats { pub database_size: u64, + #[serde(serialize_with = "time::serde::rfc3339::option::serialize")] pub last_update: Option, pub indexes: BTreeMap, } From 6312e7f1f33c79f4326f72c6c97900b53045c8e5 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 22 Feb 2022 15:45:36 +0100 Subject: [PATCH 05/16] fix(analytics): flatten the scheduler options --- meilisearch-http/src/option.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index b6cc3db22..0f1ff5970 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -42,6 +42,7 @@ pub struct Opt { /// Do not send analytics to Meili. #[cfg(all(not(debug_assertions), feature = "analytics"))] + #[serde(skip)] // we can't send true #[clap(long, env = "MEILI_NO_ANALYTICS")] pub no_analytics: bool, @@ -148,6 +149,7 @@ pub struct Opt { #[clap(skip)] pub indexer_options: IndexerOpts, + #[serde(flatten)] #[clap(flatten)] pub scheduler_options: SchedulerConfig, } From 4fbb83a34dfda62ffadb269815815e502d193d49 Mon Sep 17 00:00:00 2001 From: ad hoc Date: Tue, 22 Feb 2022 18:16:02 +0100 Subject: [PATCH 06/16] bug(snapshot): Correctly open environments in snapshots --- meilisearch-auth/src/lib.rs | 1 + meilisearch-auth/src/store.rs | 12 ++++++++---- meilisearch-http/src/routes/mod.rs | 1 - meilisearch-lib/src/index_controller/mod.rs | 13 ++++++++----- meilisearch-lib/src/snapshot.rs | 18 ++++++++---------- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/meilisearch-auth/src/lib.rs b/meilisearch-auth/src/lib.rs index 70ef69c82..d141aa0ff 100644 --- a/meilisearch-auth/src/lib.rs +++ b/meilisearch-auth/src/lib.rs @@ -17,6 +17,7 @@ use time::OffsetDateTime; pub use action::{actions, Action}; use error::{AuthControllerError, Result}; pub use key::Key; +pub use store::open_auth_store_env; use store::HeedAuthStore; #[derive(Clone)] diff --git a/meilisearch-auth/src/store.rs b/meilisearch-auth/src/store.rs index 332f698bf..dab75cc5f 100644 --- a/meilisearch-auth/src/store.rs +++ b/meilisearch-auth/src/store.rs @@ -39,14 +39,18 @@ impl Drop for HeedAuthStore { } } +pub fn open_auth_store_env(path: &Path) -> heed::Result { + let mut options = EnvOpenOptions::new(); + options.map_size(AUTH_STORE_SIZE); // 1GB + options.max_dbs(2); + options.open(path) +} + impl HeedAuthStore { pub fn new(path: impl AsRef) -> Result { let path = path.as_ref().join(AUTH_DB_PATH); create_dir_all(&path)?; - let mut options = EnvOpenOptions::new(); - options.map_size(AUTH_STORE_SIZE); // 1GB - options.max_dbs(2); - let env = Arc::new(options.open(path)?); + let env = Arc::new(open_auth_store_env(path.as_ref())?); let keys = env.create_database(Some(KEY_DB_NAME))?; let action_keyid_index_expiration = env.create_database(Some(KEY_ID_ACTION_INDEX_EXPIRATION_DB_NAME))?; diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index f94028a7d..4b9fab978 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -136,7 +136,6 @@ async fn get_stats( meilisearch: GuardedData, MeiliSearch>, ) -> Result { let search_rules = &meilisearch.filters().search_rules; - let response = meilisearch.get_all_stats(search_rules).await?; debug!("returns: {:?}", response); diff --git a/meilisearch-lib/src/index_controller/mod.rs b/meilisearch-lib/src/index_controller/mod.rs index 06f64f1c1..108084c57 100644 --- a/meilisearch-lib/src/index_controller/mod.rs +++ b/meilisearch-lib/src/index_controller/mod.rs @@ -48,6 +48,13 @@ pub type Payload = Box< dyn Stream> + Send + Sync + 'static + Unpin, >; +pub fn open_meta_env(path: &Path, size: usize) -> heed::Result { + let mut options = heed::EnvOpenOptions::new(); + options.map_size(size); + options.max_dbs(20); + options.open(path) +} + #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct IndexMetadata { @@ -202,11 +209,7 @@ impl IndexControllerBuilder { std::fs::create_dir_all(db_path.as_ref())?; - let mut options = heed::EnvOpenOptions::new(); - options.map_size(task_store_size); - options.max_dbs(20); - - let meta_env = Arc::new(options.open(&db_path)?); + let meta_env = Arc::new(open_meta_env(db_path.as_ref(), task_store_size)?); let update_file_store = UpdateFileStore::new(&db_path)?; // Create or overwrite the version file for this DB diff --git a/meilisearch-lib/src/snapshot.rs b/meilisearch-lib/src/snapshot.rs index 528c0f64a..3667d1021 100644 --- a/meilisearch-lib/src/snapshot.rs +++ b/meilisearch-lib/src/snapshot.rs @@ -6,11 +6,13 @@ use std::time::Duration; use anyhow::bail; use fs_extra::dir::{self, CopyOptions}; use log::{info, trace}; +use meilisearch_auth::open_auth_store_env; use tokio::sync::RwLock; use tokio::time::sleep; use walkdir::WalkDir; use crate::compression::from_tar_gz; +use crate::index_controller::open_meta_env; use crate::index_controller::versioning::VERSION_FILE_NAME; use crate::tasks::task::Job; use crate::tasks::Scheduler; @@ -39,7 +41,6 @@ impl SnapshotService { }; let job = Job::Snapshot(snapshot_job); self.scheduler.write().await.schedule_job(job).await; - sleep(self.snapshot_period).await; } } @@ -145,9 +146,7 @@ impl SnapshotJob { } fn snapshot_meta_env(&self, path: &Path) -> anyhow::Result<()> { - let mut options = heed::EnvOpenOptions::new(); - options.map_size(self.meta_env_size); - let env = options.open(&self.src_path)?; + let env = open_meta_env(&self.src_path, self.meta_env_size)?; let dst = path.join("data.mdb"); env.copy_to_path(dst, heed::CompactionOption::Enabled)?; @@ -183,9 +182,10 @@ impl SnapshotJob { let mut options = heed::EnvOpenOptions::new(); options.map_size(self.index_size); - let env = options.open(entry.path())?; - - env.copy_to_path(dst, heed::CompactionOption::Enabled)?; + let index = milli::Index::new(options, entry.path())?; + index + .env + .copy_to_path(dst, heed::CompactionOption::Enabled)?; } Ok(()) @@ -197,9 +197,7 @@ impl SnapshotJob { std::fs::create_dir_all(&dst)?; let dst = dst.join("data.mdb"); - let mut options = heed::EnvOpenOptions::new(); - options.map_size(1_073_741_824); - let env = options.open(auth_path)?; + let env = open_auth_store_env(&auth_path)?; env.copy_to_path(dst, heed::CompactionOption::Enabled)?; Ok(()) From 3b2e467ca622929be655f61523f0d99deca813f8 Mon Sep 17 00:00:00 2001 From: Rob Ede Date: Fri, 25 Feb 2022 23:28:55 +0000 Subject: [PATCH 07/16] update actix-web dependency to 4.0 --- Cargo.lock | 358 ++++++++++++++++++++--------------- meilisearch-error/Cargo.toml | 2 +- meilisearch-http/Cargo.toml | 6 +- meilisearch-http/src/lib.rs | 12 +- meilisearch-lib/Cargo.toml | 2 +- 5 files changed, 219 insertions(+), 161 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65c8ee2eb..fdc14ea2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "actix-codec" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a36c014a3e811624313b51a227b775ecba55d36ef9462bbaac7d4f13e54c9271" +checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" dependencies = [ "bitflags", "bytes", @@ -16,14 +16,14 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", - "tokio-util", + "tokio-util 0.7.0", ] [[package]] name = "actix-cors" -version = "0.6.0-beta.8" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4f1bd0e31c745df129f0e94efd374d21f2a455bcc386c15d78ed9a9e7d4dd50" +checksum = "30dbd116ef7532f56e2f6d7c511736ea0b124d914ee8820a5271247bf89f06aa" dependencies = [ "actix-service", "actix-utils", @@ -37,9 +37,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.0.0-beta.19" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae58d21721388ea9b2cd0d4c11756b0f34424cdcd6e5cc74c3ce37b4641c8af0" +checksum = "0f3fdd63b9cfeaf92eeeece719dabbddddb420a57d3fd171ce1490ecfb7086b1" dependencies = [ "actix-codec", "actix-rt", @@ -85,9 +85,9 @@ dependencies = [ [[package]] name = "actix-router" -version = "0.5.0-rc.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e0b59ad08167ffbb686ddb495846707231e96908b829b1fc218198ec581e2ad" +checksum = "eb60846b52c118f2f04a56cc90880a274271c489b2498623d58176f8ca21fa80" dependencies = [ "bytestring", "firestorm", @@ -120,7 +120,7 @@ dependencies = [ "futures-core", "futures-util", "log", - "mio 0.8.0", + "mio", "num_cpus", "socket2", "tokio", @@ -139,20 +139,19 @@ dependencies = [ [[package]] name = "actix-tls" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b161450ff646361005a300716e85856780ddc2fde569fd81335cc3c15b2e0933" +checksum = "9fde0cf292f7cdc7f070803cb9a0d45c018441321a78b1042ffbbb81ec333297" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-utils", - "derive_more", "futures-core", "log", "pin-project-lite", "tokio-rustls", - "tokio-util", + "tokio-util 0.7.0", "webpki-roots", ] @@ -168,9 +167,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.0.0-beta.21" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "606fc29a9bde2907243086ceb93ce56df7584276c2c46abc64a524f645c63c5e" +checksum = "f4e5ebffd51d50df56a3ae0de0e59487340ca456f05dd0b90c0a7a6dd6a74d31" dependencies = [ "actix-codec", "actix-http", @@ -184,6 +183,7 @@ dependencies = [ "actix-web-codegen", "ahash 0.7.6", "bytes", + "bytestring", "cfg-if 1.0.0", "cookie", "derive_more", @@ -208,9 +208,9 @@ dependencies = [ [[package]] name = "actix-web-codegen" -version = "0.5.0-rc.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98a793e4a7bd059e06e1bc1bd9943b57a47f806de3599d2437441682292c333e" +checksum = "7525bedf54704abb1d469e88d7e7e9226df73778798a69cea5022d53b2ae91bc" dependencies = [ "actix-router", "proc-macro2 1.0.36", @@ -221,7 +221,7 @@ dependencies = [ [[package]] name = "actix-web-static-files" version = "3.0.5" -source = "git+https://github.com/robjtede/actix-web-static-files.git?rev=ed74153#ed741539d0c72507f4ebe419306c45fdcaff4a09" +source = "git+https://github.com/robjtede/actix-web-static-files.git?rev=baebe8e3#baebe8e36d4a019113b43fbed908192cf34fa6c4" dependencies = [ "actix-web", "derive_more", @@ -287,9 +287,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.53" +version = "1.0.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a45b455c14666b85fc40a019e8ab9eb75e3a124e05494f5397122bc9eb06e0" +checksum = "159bb86af3a200e19a068f4224eae4c8bb2d0fa054c7e5d1cacd5cef95e684cd" dependencies = [ "backtrace", ] @@ -379,15 +379,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.63" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321629d8ba6513061f26707241fa9bc89524ff1cd7a915a97ef0c62c666ce1b6" +checksum = "5e121dee8023ce33ab248d9ce1493df03c3b38a659b240096fcbd7048ff9c31f" dependencies = [ "addr2line", "cc", @@ -460,9 +460,9 @@ dependencies = [ [[package]] name = "block-buffer" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d36a02058e76b040de25a4464ba1c80935655595b661505c8b39b664828b95" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" dependencies = [ "generic-array 0.14.5", ] @@ -597,9 +597,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.72" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" dependencies = [ "jobserver", ] @@ -659,9 +659,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.0.13" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08799f92c961c7a1cf0cc398a9073da99e21ce388b46372c37f3191f2f3eed3e" +checksum = "5177fac1ab67102d8989464efd043c6ff44191b1557ec1ddd489b4f7e1447e77" dependencies = [ "atty", "bitflags", @@ -676,9 +676,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.0.12" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd2078197a22f338bd4fbf7d6387eb6f0d6a3c69e6cbc09f5c93e97321fd92a" +checksum = "01d42c94ce7c2252681b5fed4d3627cc807b13dfc033246bd05d5b252399000e" dependencies = [ "heck", "proc-macro-error", @@ -738,9 +738,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2209c310e29876f7f0b2721e7e26b84aff178aa3da5d091f9bfbf47669e60e3" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ "cfg-if 1.0.0", ] @@ -752,7 +752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.6", + "crossbeam-utils 0.8.7", ] [[package]] @@ -763,17 +763,17 @@ checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.8.6", + "crossbeam-utils 0.8.7", ] [[package]] name = "crossbeam-epoch" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97242a70df9b89a65d0b6df3c4bf5b9ce03c5b7309019777fbde37e7537f8762" +checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.6", + "crossbeam-utils 0.8.7", "lazy_static", "memoffset", "scopeguard", @@ -800,9 +800,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcae03edb34f947e64acdb1c33ec169824e20657e9ecb61cef6c8c74dcb8120" +checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" dependencies = [ "cfg-if 1.0.0", "lazy_static", @@ -810,11 +810,12 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d6b536309245c849479fba3da410962a43ed8e51c26b729208ec0ac2798d0" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" dependencies = [ "generic-array 0.14.5", + "typenum", ] [[package]] @@ -886,13 +887,12 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b697d66081d42af4fba142d56918a3cb21dc8eb63372c6b85d14f44fb9c5979b" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ - "block-buffer 0.10.0", + "block-buffer 0.10.2", "crypto-common", - "generic-array 0.14.5", ] [[package]] @@ -1063,9 +1063,9 @@ checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" [[package]] name = "futures" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28560757fe2bb34e79f907794bb6b22ae8b0e5c669b638a1132f2592b19035b4" +checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" dependencies = [ "futures-channel", "futures-core", @@ -1078,9 +1078,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3dda0b6588335f360afc675d0564c17a77a2bda81ca178a4b6081bd86c7f0b" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" dependencies = [ "futures-core", "futures-sink", @@ -1088,15 +1088,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c8ff0461b82559810cdccfde3215c3f373807f5e5232b71479bff7bb2583d7" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" [[package]] name = "futures-executor" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29d6d2ff5bb10fb95c85b8ce46538a2e5f5e7fdc755623a7d4529ab8a4ed9d2a" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" dependencies = [ "futures-core", "futures-task", @@ -1105,15 +1105,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9d34af5a1aac6fb380f735fe510746c38067c5bf16c7fd250280503c971b2" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" [[package]] name = "futures-macro" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbd947adfffb0efc70599b3ddcf7b5597bb5fa9e245eb99f62b3a5f7bb8bd3c" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" dependencies = [ "proc-macro2 1.0.36", "quote 1.0.15", @@ -1122,21 +1122,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3055baccb68d74ff6480350f8d6eb8fcfa3aa11bdc1a1ae3afdd0514617d508" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" [[package]] name = "futures-task" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee7c6485c30167ce4dfb83ac568a849fe53274c831081476ee13e0dce1aad72" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" [[package]] name = "futures-util" -version = "0.3.19" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b5cf40b47a271f77a8b1bec03ca09044d99d2372c0de244e66430761127164" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" dependencies = [ "futures-channel", "futures-core", @@ -1195,9 +1195,9 @@ checksum = "9e006f616a407d396ace1d2ebb3f43ed73189db8b098079bd129928d7645dd1e" [[package]] name = "getrandom" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c" +checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1224,9 +1224,9 @@ checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" [[package]] name = "git2" -version = "0.13.25" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" +checksum = "94781080dd1a6b55dea7c46540d5bac87742a22f6dc2d84e54a5071ad6f0e387" dependencies = [ "bitflags", "libc", @@ -1267,7 +1267,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util", + "tokio-util 0.6.9", "tracing", ] @@ -1388,9 +1388,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" +checksum = "9100414882e15fb7feccb4897e5f0ff0ff1ca7d1a86a23208ada4d7a18e6c6c4" [[package]] name = "httpdate" @@ -1412,9 +1412,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.16" +version = "0.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7ec3e62bdc98a2f0393a5048e4c30ef659440ea6e0e572965103e72bd836f55" +checksum = "043f0e083e9901b6cc658a77d1eb86f4fc650bbb977a4337dd63192826aa85dd" dependencies = [ "bytes", "futures-channel", @@ -1425,7 +1425,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 0.4.8", + "itoa 1.0.1", "pin-project-lite", "socket2", "tokio", @@ -1442,7 +1442,7 @@ checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" dependencies = [ "http", "hyper", - "rustls 0.20.2", + "rustls 0.20.4", "tokio", "tokio-rustls", ] @@ -1597,15 +1597,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.114" +version = "0.2.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0005d08a8f7b65fb8073cb697aa0b12b631ed251ce73d862ce50eeb52ce3b50" +checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" [[package]] name = "libgit2-sys" -version = "0.12.26+1.3.0" +version = "0.13.0+1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" +checksum = "864e22fc06cae62860398cd854c93d5867f11c02ec916aa1417b440f170df23a" dependencies = [ "cc", "libc", @@ -1661,9 +1661,9 @@ checksum = "902eb695eb0591864543cbfbf6d742510642a605a61fc5e97fe6ceb5a30ac4fb" [[package]] name = "lock_api" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" +checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" dependencies = [ "scopeguard", ] @@ -1778,7 +1778,7 @@ dependencies = [ "num_cpus", "obkv", "once_cell", - "parking_lot", + "parking_lot 0.11.2", "paste", "pin-project", "platform-dirs", @@ -1786,7 +1786,7 @@ dependencies = [ "rayon", "regex", "reqwest", - "rustls 0.20.2", + "rustls 0.20.4", "rustls-pemfile", "segment", "serde", @@ -1849,7 +1849,7 @@ dependencies = [ "num_cpus", "obkv", "once_cell", - "parking_lot", + "parking_lot 0.11.2", "paste", "pin-project", "proptest", @@ -1898,9 +1898,9 @@ checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "memmap2" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe3179b85e1fd8b14447cbebadb75e45a1002f541b925f0bfec366d56a81c56d" +checksum = "057a3db23999c867821a7a59feb06a578fcb03685e983dff90daf9e7d24ac08f" dependencies = [ "libc", ] @@ -1966,9 +1966,9 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" [[package]] name = "mime_guess" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" dependencies = [ "mime", "unicase", @@ -1990,19 +1990,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "mio" -version = "0.7.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" -dependencies = [ - "libc", - "log", - "miow", - "ntapi", - "winapi", -] - [[package]] name = "mio" version = "0.8.0" @@ -2098,9 +2085,9 @@ checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" [[package]] name = "ntapi" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" dependencies = [ "winapi", ] @@ -2217,7 +2204,17 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.5", +] + +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.1", ] [[package]] @@ -2234,6 +2231,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + [[package]] name = "paste" version = "1.0.6" @@ -2503,14 +2513,13 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", "rand_core", - "rand_hc", ] [[package]] @@ -2532,15 +2541,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_hc" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" -dependencies = [ - "rand_core", -] - [[package]] name = "rand_xorshift" version = "0.3.0" @@ -2570,7 +2570,7 @@ checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" dependencies = [ "crossbeam-channel", "crossbeam-deque", - "crossbeam-utils 0.8.6", + "crossbeam-utils 0.8.7", "lazy_static", "num_cpus", ] @@ -2649,7 +2649,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rustls 0.20.2", + "rustls 0.20.4", "rustls-pemfile", "serde", "serde_json", @@ -2666,9 +2666,9 @@ dependencies = [ [[package]] name = "retain_mut" -version = "0.1.5" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11000e6ba5020e53e7cc26f73b91ae7d5496b4977851479edb66b694c0675c21" +checksum = "8c31b5c4033f8fdde8700e4657be2c497e7288f01515be52168c631e2e4d4086" [[package]] name = "ring" @@ -2739,9 +2739,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.2" +version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d37e5e2290f3e040b594b1a9e04377c2c671f1a1cfd9bfdef82106ac1c113f84" +checksum = "4fbfeb8d0ddb84706bc597a5574ab8912817c52a397f819e5b614e2265206921" dependencies = [ "log", "ring", @@ -2833,9 +2833,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012" +checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" [[package]] name = "serde" @@ -2859,9 +2859,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d23c1ba4cf0efd44be32017709280b32d1cea5c3f1275c3b6d9e8bc54f758085" +checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" dependencies = [ "indexmap", "itoa 1.0.1", @@ -2912,7 +2912,7 @@ checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" dependencies = [ "cfg-if 1.0.0", "cpufeatures", - "digest 0.10.1", + "digest 0.10.3", ] [[package]] @@ -3153,9 +3153,9 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" -version = "0.4.2+5.2.1-patched.2" +version = "0.4.3+5.2.1-patched.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5844e429d797c62945a566f8da4e24c7fe3fbd5d6617fd8bf7a0b7dc1ee0f22e" +checksum = "a1792ccb507d955b46af42c123ea8863668fae24d03721e40cad6a41773dbb49" dependencies = [ "cc", "fs_extra", @@ -3164,9 +3164,9 @@ dependencies = [ [[package]] name = "tikv-jemallocator" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c14a5a604eb8715bc5785018a37d00739b180bcf609916ddf4393d33d49ccdf" +checksum = "a5b7bcecfafe4998587d636f9ae9d55eb9d0499877b88757767c346875067098" dependencies = [ "libc", "tikv-jemalloc-sys", @@ -3219,19 +3219,20 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" +checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" dependencies = [ "bytes", "libc", "memchr", - "mio 0.7.14", + "mio", "num_cpus", "once_cell", - "parking_lot", + "parking_lot 0.12.0", "pin-project-lite", "signal-hook-registry", + "socket2", "tokio-macros", "winapi", ] @@ -3253,7 +3254,7 @@ version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b" dependencies = [ - "rustls 0.20.2", + "rustls 0.20.4", "tokio", "webpki 0.22.0", ] @@ -3283,6 +3284,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64910e1b9c1901aaf5375561e35b9c057d95ff41a44ede043a03e09279eabaf1" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.5.8" @@ -3300,9 +3315,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.29" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" +checksum = "f6c650a8ef0cd2dd93736f033d21cbd1224c5a967aa0c258d00fcf7dafef9b9f" dependencies = [ "cfg-if 1.0.0", "pin-project-lite", @@ -3311,9 +3326,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" +checksum = "03cfcb51380632a72d3111cb8d3447a8d908e577d31beeac006f836383d29a23" dependencies = [ "lazy_static", ] @@ -3356,9 +3371,9 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" +checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" [[package]] name = "unicode-xid" @@ -3631,6 +3646,49 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" + +[[package]] +name = "windows_i686_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" + +[[package]] +name = "windows_i686_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" + [[package]] name = "winreg" version = "0.7.0" @@ -3686,18 +3744,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.9.2+zstd.1.5.1" +version = "0.10.0+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2390ea1bf6c038c39674f22d95f0564725fc06034a47129179810b2fc58caa54" +checksum = "3b1365becbe415f3f0fcd024e2f7b45bacfb5bdd055f0dc113571394114e7bdd" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "4.1.3+zstd.1.5.1" +version = "4.1.4+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e99d81b99fb3c2c2c794e3fe56c305c63d5173a16a46b5850b07c935ffc7db79" +checksum = "2f7cd17c9af1a4d6c24beb1cc54b17e2ef7b593dc92f19e9d9acad8b182bbaee" dependencies = [ "libc", "zstd-sys", @@ -3705,9 +3763,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "1.6.2+zstd.1.5.1" +version = "1.6.3+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2daf2f248d9ea44454bfcb2516534e8b8ad2fc91bf818a1885495fc42bc8ac9f" +checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8" dependencies = [ "cc", "libc", diff --git a/meilisearch-error/Cargo.toml b/meilisearch-error/Cargo.toml index 5e970e345..0b75e4151 100644 --- a/meilisearch-error/Cargo.toml +++ b/meilisearch-error/Cargo.toml @@ -5,7 +5,7 @@ authors = ["marin "] edition = "2021" [dependencies] -actix-web = { version = "4.0.0-beta.21", default-features = false } +actix-web = { version = "4", default-features = false } proptest = { version = "1.0.0", optional = true } proptest-derive = { version = "0.3.0", optional = true } serde = { version = "1.0.130", features = ["derive"] } diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 7f47b5968..acf38a0a6 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -22,9 +22,9 @@ vergen = { version = "5.1.15", default-features = false, features = ["git"] } zip = { version = "0.5.13", optional = true } [dependencies] -actix-cors = "0.6.0-beta.8" -actix-web = { version = "4.0.0-beta.21", features = ["rustls"] } -actix-web-static-files = { git = "https://github.com/robjtede/actix-web-static-files.git", rev = "ed74153", optional = true } +actix-cors = "0.6" +actix-web = { version = "4", features = ["rustls"] } +actix-web-static-files = { git = "https://github.com/robjtede/actix-web-static-files.git", rev = "baebe8e3", optional = true } anyhow = { version = "1.0.43", features = ["backtrace"] } arc-swap = "1.3.2" async-stream = "0.3.2" diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 907a4423e..71932dd9b 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -122,13 +122,13 @@ pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) { } = resource; // Redirect index.html to / if path == "index.html" { - config.service(web::resource("/").route( - web::get().to(move || HttpResponse::Ok().content_type(mime_type).body(data)), - )); + config.service(web::resource("/").route(web::get().to(move || async move { + HttpResponse::Ok().content_type(mime_type).body(data) + }))); } else { - config.service(web::resource(path).route( - web::get().to(move || HttpResponse::Ok().content_type(mime_type).body(data)), - )); + config.service(web::resource(path).route(web::get().to(move || async move { + HttpResponse::Ok().content_type(mime_type).body(data) + }))); } } } else { diff --git a/meilisearch-lib/Cargo.toml b/meilisearch-lib/Cargo.toml index 9c37edba3..b64104219 100644 --- a/meilisearch-lib/Cargo.toml +++ b/meilisearch-lib/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -actix-web = { version = "4.0.0-beta.21", default-features = false } +actix-web = { version = "4", default-features = false } anyhow = { version = "1.0.43", features = ["backtrace"] } async-stream = "0.3.2" async-trait = "0.1.51" From 15150db957bc5cf0928a54f02e0d41c74df170ff Mon Sep 17 00:00:00 2001 From: Rob Ede Date: Mon, 28 Feb 2022 13:43:22 +0000 Subject: [PATCH 08/16] clippy --- Cargo.lock | 2 +- meilisearch-http/Cargo.toml | 2 +- meilisearch-http/src/analytics/mod.rs | 2 +- meilisearch-http/tests/snapshot/mod.rs | 5 +---- meilisearch-lib/src/index/index.rs | 2 +- .../src/index_controller/dump_actor/compat/mod.rs | 4 ++-- meilisearch-lib/src/tasks/scheduler.rs | 2 +- meilisearch-lib/src/tasks/task_store/store.rs | 10 +++++----- 8 files changed, 13 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdc14ea2f..1ba427a1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -221,7 +221,7 @@ dependencies = [ [[package]] name = "actix-web-static-files" version = "3.0.5" -source = "git+https://github.com/robjtede/actix-web-static-files.git?rev=baebe8e3#baebe8e36d4a019113b43fbed908192cf34fa6c4" +source = "git+https://github.com/kilork/actix-web-static-files.git?rev=2d3b6160#2d3b6160f0de4ba061c5d76b5704f34fb677f6df" dependencies = [ "actix-web", "derive_more", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index acf38a0a6..c3dd1026f 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -24,7 +24,7 @@ zip = { version = "0.5.13", optional = true } [dependencies] actix-cors = "0.6" actix-web = { version = "4", features = ["rustls"] } -actix-web-static-files = { git = "https://github.com/robjtede/actix-web-static-files.git", rev = "baebe8e3", optional = true } +actix-web-static-files = { git = "https://github.com/kilork/actix-web-static-files.git", rev = "2d3b6160", optional = true } anyhow = { version = "1.0.43", features = ["backtrace"] } arc-swap = "1.3.2" async-stream = "0.3.2" diff --git a/meilisearch-http/src/analytics/mod.rs b/meilisearch-http/src/analytics/mod.rs index caeb7b7c0..1d37a053d 100644 --- a/meilisearch-http/src/analytics/mod.rs +++ b/meilisearch-http/src/analytics/mod.rs @@ -44,7 +44,7 @@ fn config_user_id_path(db_path: &Path) -> Option { path.join("instance-uid") .display() .to_string() - .replace("/", "-") + .replace('/', "-") }) .zip(MEILISEARCH_CONFIG_PATH.as_ref()) .map(|(filename, config_path)| config_path.join(filename.trim_start_matches('-'))) diff --git a/meilisearch-http/tests/snapshot/mod.rs b/meilisearch-http/tests/snapshot/mod.rs index 3a5c99b2d..5c626a888 100644 --- a/meilisearch-http/tests/snapshot/mod.rs +++ b/meilisearch-http/tests/snapshot/mod.rs @@ -60,10 +60,7 @@ async fn perform_snapshot() { let temp = tempfile::tempdir().unwrap(); - let snapshot_path = snapshot_dir - .path() - .to_owned() - .join("db.snapshot".to_string()); + let snapshot_path = snapshot_dir.path().to_owned().join("db.snapshot"); let options = Opt { import_snapshot: Some(snapshot_path), diff --git a/meilisearch-lib/src/index/index.rs b/meilisearch-lib/src/index/index.rs index 8eadc8c4a..a17ed8504 100644 --- a/meilisearch-lib/src/index/index.rs +++ b/meilisearch-lib/src/index/index.rs @@ -152,7 +152,7 @@ impl Index { Ok(stop_words.stream().into_strs()?.into_iter().collect()) }) .transpose()? - .unwrap_or_else(BTreeSet::new); + .unwrap_or_default(); let distinct_field = self.distinct_field(txn)?.map(String::from); // in milli each word in the synonyms map were split on their separator. Since we lost diff --git a/meilisearch-lib/src/index_controller/dump_actor/compat/mod.rs b/meilisearch-lib/src/index_controller/dump_actor/compat/mod.rs index ad069f61c..93f3f9dd7 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/compat/mod.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/compat/mod.rs @@ -4,13 +4,13 @@ pub mod v3; /// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name. pub fn asc_ranking_rule(text: &str) -> Option<&str> { text.split_once("asc(") - .and_then(|(_, tail)| tail.rsplit_once(")")) + .and_then(|(_, tail)| tail.rsplit_once(')')) .map(|(field, _)| field) } /// Parses the v1 version of the Desc ranking rules `desc(price)`and returns the field name. pub fn desc_ranking_rule(text: &str) -> Option<&str> { text.split_once("desc(") - .and_then(|(_, tail)| tail.rsplit_once(")")) + .and_then(|(_, tail)| tail.rsplit_once(')')) .map(|(field, _)| field) } diff --git a/meilisearch-lib/src/tasks/scheduler.rs b/meilisearch-lib/src/tasks/scheduler.rs index 0aa72fcf4..50f40a8e2 100644 --- a/meilisearch-lib/src/tasks/scheduler.rs +++ b/meilisearch-lib/src/tasks/scheduler.rs @@ -443,7 +443,7 @@ mod test { fn gen_task(id: TaskId, index_uid: &str, content: TaskContent) -> Task { Task { id, - index_uid: IndexUid::new_unchecked(index_uid.to_owned()), + index_uid: IndexUid::new_unchecked(index_uid), content, events: vec![], } diff --git a/meilisearch-lib/src/tasks/task_store/store.rs b/meilisearch-lib/src/tasks/task_store/store.rs index 6032eec2c..582cfe27c 100644 --- a/meilisearch-lib/src/tasks/task_store/store.rs +++ b/meilisearch-lib/src/tasks/task_store/store.rs @@ -325,7 +325,7 @@ pub mod test { let tasks = (0..100) .map(|_| Task { id: rand::random(), - index_uid: IndexUid::new_unchecked("test".to_string()), + index_uid: IndexUid::new_unchecked("test"), content: TaskContent::IndexDeletion, events: vec![], }) @@ -356,14 +356,14 @@ pub mod test { let task_1 = Task { id: 1, - index_uid: IndexUid::new_unchecked("test".to_string()), + index_uid: IndexUid::new_unchecked("test"), content: TaskContent::IndexDeletion, events: vec![], }; let task_2 = Task { id: 0, - index_uid: IndexUid::new_unchecked("test1".to_string()), + index_uid: IndexUid::new_unchecked("test1"), content: TaskContent::IndexDeletion, events: vec![], }; @@ -384,13 +384,13 @@ pub mod test { // same thing but invert the ids let task_1 = Task { id: 0, - index_uid: IndexUid::new_unchecked("test".to_string()), + index_uid: IndexUid::new_unchecked("test"), content: TaskContent::IndexDeletion, events: vec![], }; let task_2 = Task { id: 1, - index_uid: IndexUid::new_unchecked("test1".to_string()), + index_uid: IndexUid::new_unchecked("test1"), content: TaskContent::IndexDeletion, events: vec![], }; From c2b58720d1086cdb848f268cad023cc83fc6db03 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Tue, 1 Mar 2022 19:02:32 +0100 Subject: [PATCH 09/16] Fix(dumps): Explicitly define serde for time --- meilisearch-http/src/routes/mod.rs | 12 ++++++------ .../src/index_controller/dump_actor/compat/v2.rs | 5 +++++ .../src/index_controller/dump_actor/compat/v3.rs | 4 ++++ .../src/index_controller/dump_actor/mod.rs | 1 + meilisearch-lib/src/tasks/task.rs | 15 +++++++++++++-- 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index 4b9fab978..49397444f 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -54,9 +54,9 @@ pub struct ProcessedUpdateResult { #[serde(rename = "type")] pub update_type: UpdateType, pub duration: f64, // in seconds - #[serde(serialize_with = "time::serde::rfc3339::serialize")] + #[serde(with = "time::serde::rfc3339")] pub enqueued_at: OffsetDateTime, - #[serde(serialize_with = "time::serde::rfc3339::serialize")] + #[serde(with = "time::serde::rfc3339")] pub processed_at: OffsetDateTime, } @@ -68,9 +68,9 @@ pub struct FailedUpdateResult { pub update_type: UpdateType, pub error: ResponseError, pub duration: f64, // in seconds - #[serde(serialize_with = "time::serde::rfc3339::serialize")] + #[serde(with = "time::serde::rfc3339")] pub enqueued_at: OffsetDateTime, - #[serde(serialize_with = "time::serde::rfc3339::serialize")] + #[serde(with = "time::serde::rfc3339")] pub processed_at: OffsetDateTime, } @@ -80,11 +80,11 @@ pub struct EnqueuedUpdateResult { pub update_id: u64, #[serde(rename = "type")] pub update_type: UpdateType, - #[serde(serialize_with = "time::serde::rfc3339::serialize")] + #[serde(with = "time::serde::rfc3339")] pub enqueued_at: OffsetDateTime, #[serde( skip_serializing_if = "Option::is_none", - serialize_with = "time::serde::rfc3339::option::serialize" + with = "time::serde::rfc3339::option" )] pub started_processing_at: Option, } diff --git a/meilisearch-lib/src/index_controller/dump_actor/compat/v2.rs b/meilisearch-lib/src/index_controller/dump_actor/compat/v2.rs index d7ba41479..a30e24794 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/compat/v2.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/compat/v2.rs @@ -51,6 +51,7 @@ pub enum UpdateMeta { pub struct Enqueued { pub update_id: u64, pub meta: UpdateMeta, + #[serde(with = "time::serde::rfc3339")] pub enqueued_at: OffsetDateTime, pub content: Option, } @@ -59,6 +60,7 @@ pub struct Enqueued { #[serde(rename_all = "camelCase")] pub struct Processed { pub success: UpdateResult, + #[serde(with = "time::serde::rfc3339")] pub processed_at: OffsetDateTime, #[serde(flatten)] pub from: Processing, @@ -69,6 +71,7 @@ pub struct Processed { pub struct Processing { #[serde(flatten)] pub from: Enqueued, + #[serde(with = "time::serde::rfc3339")] pub started_processing_at: OffsetDateTime, } @@ -77,6 +80,7 @@ pub struct Processing { pub struct Aborted { #[serde(flatten)] pub from: Enqueued, + #[serde(with = "time::serde::rfc3339")] pub aborted_at: OffsetDateTime, } @@ -86,6 +90,7 @@ pub struct Failed { #[serde(flatten)] pub from: Processing, pub error: ResponseError, + #[serde(with = "time::serde::rfc3339")] pub failed_at: OffsetDateTime, } diff --git a/meilisearch-lib/src/index_controller/dump_actor/compat/v3.rs b/meilisearch-lib/src/index_controller/dump_actor/compat/v3.rs index 1e378805d..7cd670bad 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/compat/v3.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/compat/v3.rs @@ -107,6 +107,7 @@ pub enum UpdateMeta { pub struct Enqueued { pub update_id: u64, pub meta: Update, + #[serde(with = "time::serde::rfc3339")] pub enqueued_at: OffsetDateTime, } @@ -122,6 +123,7 @@ impl Enqueued { #[serde(rename_all = "camelCase")] pub struct Processed { pub success: v2::UpdateResult, + #[serde(with = "time::serde::rfc3339")] pub processed_at: OffsetDateTime, #[serde(flatten)] pub from: Processing, @@ -144,6 +146,7 @@ impl Processed { pub struct Processing { #[serde(flatten)] pub from: Enqueued, + #[serde(with = "time::serde::rfc3339")] pub started_processing_at: OffsetDateTime, } @@ -163,6 +166,7 @@ pub struct Failed { pub from: Processing, pub msg: String, pub code: Code, + #[serde(with = "time::serde::rfc3339")] pub failed_at: OffsetDateTime, } diff --git a/meilisearch-lib/src/index_controller/dump_actor/mod.rs b/meilisearch-lib/src/index_controller/dump_actor/mod.rs index d9ff63f2c..16e328e3b 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/mod.rs @@ -40,6 +40,7 @@ pub struct Metadata { db_version: String, index_db_size: usize, update_db_size: usize, + #[serde(with = "time::serde::rfc3339")] dump_date: OffsetDateTime, } diff --git a/meilisearch-lib/src/tasks/task.rs b/meilisearch-lib/src/tasks/task.rs index 1c67fafa8..ecbd4ca62 100644 --- a/meilisearch-lib/src/tasks/task.rs +++ b/meilisearch-lib/src/tasks/task.rs @@ -36,21 +36,32 @@ impl From for TaskResult { #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr(test, derive(proptest_derive::Arbitrary))] pub enum TaskEvent { - Created(#[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] OffsetDateTime), + Created( + #[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] + #[serde(with = "time::serde::rfc3339")] + OffsetDateTime, + ), Batched { #[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] + #[serde(with = "time::serde::rfc3339")] timestamp: OffsetDateTime, batch_id: BatchId, }, - Processing(#[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] OffsetDateTime), + Processing( + #[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] + #[serde(with = "time::serde::rfc3339")] + OffsetDateTime, + ), Succeded { result: TaskResult, #[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] + #[serde(with = "time::serde::rfc3339")] timestamp: OffsetDateTime, }, Failed { error: ResponseError, #[cfg_attr(test, proptest(strategy = "test::datetime_strategy()"))] + #[serde(with = "time::serde::rfc3339")] timestamp: OffsetDateTime, }, } From d6400aef274cc60ac12f6193074511aac04fb3b7 Mon Sep 17 00:00:00 2001 From: ad hoc Date: Wed, 2 Mar 2022 18:22:34 +0100 Subject: [PATCH 10/16] remove async from meilsearch-authentication --- meilisearch-auth/src/lib.rs | 12 +++--- meilisearch-http/src/routes/api_key.rs | 54 ++++++++++++++++++-------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/meilisearch-auth/src/lib.rs b/meilisearch-auth/src/lib.rs index d141aa0ff..22263735e 100644 --- a/meilisearch-auth/src/lib.rs +++ b/meilisearch-auth/src/lib.rs @@ -40,18 +40,18 @@ impl AuthController { }) } - pub async fn create_key(&self, value: Value) -> Result { + pub fn create_key(&self, value: Value) -> Result { let key = Key::create_from_value(value)?; self.store.put_api_key(key) } - pub async fn update_key(&self, key: impl AsRef, value: Value) -> Result { - let mut key = self.get_key(key).await?; + pub fn update_key(&self, key: impl AsRef, value: Value) -> Result { + let mut key = self.get_key(key)?; key.update_from_value(value)?; self.store.put_api_key(key) } - pub async fn get_key(&self, key: impl AsRef) -> Result { + pub fn get_key(&self, key: impl AsRef) -> Result { self.store .get_api_key(&key)? .ok_or_else(|| AuthControllerError::ApiKeyNotFound(key.as_ref().to_string())) @@ -101,11 +101,11 @@ impl AuthController { Ok(filters) } - pub async fn list_keys(&self) -> Result> { + pub fn list_keys(&self) -> Result> { self.store.list_api_keys() } - pub async fn delete_key(&self, key: impl AsRef) -> Result<()> { + pub fn delete_key(&self, key: impl AsRef) -> Result<()> { if self.store.delete_api_key(&key)? { Ok(()) } else { diff --git a/meilisearch-http/src/routes/api_key.rs b/meilisearch-http/src/routes/api_key.rs index 54152ea2c..aea35e68e 100644 --- a/meilisearch-http/src/routes/api_key.rs +++ b/meilisearch-http/src/routes/api_key.rs @@ -2,13 +2,13 @@ use std::str; use actix_web::{web, HttpRequest, HttpResponse}; -use meilisearch_auth::{Action, AuthController, Key}; +use meilisearch_auth::{error::AuthControllerError, Action, AuthController, Key}; use serde::{Deserialize, Serialize}; use serde_json::Value; use time::OffsetDateTime; use crate::extractors::authentication::{policies::*, GuardedData}; -use meilisearch_error::ResponseError; +use meilisearch_error::{Code, ResponseError}; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( @@ -29,8 +29,13 @@ pub async fn create_api_key( body: web::Json, _req: HttpRequest, ) -> Result { - let key = auth_controller.create_key(body.into_inner()).await?; - let res = KeyView::from_key(key, &auth_controller); + let v = body.into_inner(); + let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> { + let key = auth_controller.create_key(v)?; + Ok(KeyView::from_key(key, &auth_controller)) + }) + .await + .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; Ok(HttpResponse::Created().json(res)) } @@ -39,11 +44,16 @@ pub async fn list_api_keys( auth_controller: GuardedData, _req: HttpRequest, ) -> Result { - let keys = auth_controller.list_keys().await?; - let res: Vec<_> = keys - .into_iter() - .map(|k| KeyView::from_key(k, &auth_controller)) - .collect(); + let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> { + let keys = auth_controller.list_keys()?; + let res: Vec<_> = keys + .into_iter() + .map(|k| KeyView::from_key(k, &auth_controller)) + .collect(); + Ok(res) + }) + .await + .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; Ok(HttpResponse::Ok().json(KeyListView::from(res))) } @@ -52,8 +62,13 @@ pub async fn get_api_key( auth_controller: GuardedData, path: web::Path, ) -> Result { - let key = auth_controller.get_key(&path.api_key).await?; - let res = KeyView::from_key(key, &auth_controller); + let api_key = path.into_inner().api_key; + let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> { + let key = auth_controller.get_key(&api_key)?; + Ok(KeyView::from_key(key, &auth_controller)) + }) + .await + .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; Ok(HttpResponse::Ok().json(res)) } @@ -63,10 +78,14 @@ pub async fn patch_api_key( body: web::Json, path: web::Path, ) -> Result { - let key = auth_controller - .update_key(&path.api_key, body.into_inner()) - .await?; - let res = KeyView::from_key(key, &auth_controller); + let api_key = path.into_inner().api_key; + let body = body.into_inner(); + let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> { + let key = auth_controller.update_key(&api_key, body)?; + Ok(KeyView::from_key(key, &auth_controller)) + }) + .await + .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; Ok(HttpResponse::Ok().json(res)) } @@ -75,7 +94,10 @@ pub async fn delete_api_key( auth_controller: GuardedData, path: web::Path, ) -> Result { - auth_controller.delete_key(&path.api_key).await?; + let api_key = path.into_inner().api_key; + tokio::task::spawn_blocking(move || auth_controller.delete_key(&api_key)) + .await + .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??; Ok(HttpResponse::NoContent().finish()) } From af8a5f2c219b20ba84780a4425fc83ed3758c938 Mon Sep 17 00:00:00 2001 From: ad hoc Date: Wed, 2 Mar 2022 19:20:57 +0100 Subject: [PATCH 11/16] async auth --- .../src/extractors/authentication/mod.rs | 119 ++++++++++++------ 1 file changed, 82 insertions(+), 37 deletions(-) diff --git a/meilisearch-http/src/extractors/authentication/mod.rs b/meilisearch-http/src/extractors/authentication/mod.rs index bf35e1729..ab0030fc1 100644 --- a/meilisearch-http/src/extractors/authentication/mod.rs +++ b/meilisearch-http/src/extractors/authentication/mod.rs @@ -2,28 +2,88 @@ mod error; use std::marker::PhantomData; use std::ops::Deref; +use std::pin::Pin; use actix_web::FromRequest; use futures::future::err; -use futures::future::{ok, Ready}; -use meilisearch_error::ResponseError; +use futures::Future; +use meilisearch_error::{Code, ResponseError}; use error::AuthenticationError; use meilisearch_auth::{AuthController, AuthFilter}; -pub struct GuardedData { +pub struct GuardedData { data: D, filters: AuthFilter, - _marker: PhantomData, + _marker: PhantomData

, } -impl GuardedData { +impl GuardedData { pub fn filters(&self) -> &AuthFilter { &self.filters } + + async fn auth_bearer( + auth: AuthController, + token: String, + index: Option, + data: Option, + ) -> Result + where + P: Policy + 'static, + { + match Self::authenticate(auth, token, index).await? { + (_, Some(filters)) => match data { + Some(data) => Ok(Self { + data, + filters, + _marker: PhantomData, + }), + None => Err(AuthenticationError::IrretrievableState.into()), + }, + (token, None) => { + let token = token.to_string(); + Err(AuthenticationError::InvalidToken(token).into()) + } + } + } + + async fn auth_token(auth: AuthController, data: Option) -> Result + where + P: Policy + 'static, + { + match Self::authenticate(auth, "", None).await?.1 { + Some(filters) => match data { + Some(data) => Ok(Self { + data, + filters, + _marker: PhantomData, + }), + None => Err(AuthenticationError::IrretrievableState.into()), + }, + None => Err(AuthenticationError::MissingAuthorizationHeader.into()), + } + } + + async fn authenticate( + auth: AuthController, + token: S, + index: Option, + ) -> Result<(S, Option), ResponseError> + where + P: Policy + 'static, + S: AsRef + 'static + Send, + { + Ok(tokio::task::spawn_blocking(move || { + let res = P::authenticate(auth, token.as_ref(), index.as_deref()); + (token, res) + }) + .await + .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))?) + } } -impl Deref for GuardedData { +impl Deref for GuardedData { type Target = D; fn deref(&self) -> &Self::Target { @@ -34,7 +94,7 @@ impl Deref for GuardedData { impl FromRequest for GuardedData { type Error = ResponseError; - type Future = Ready>; + type Future = Pin>>>; fn from_request( req: &actix_web::HttpRequest, @@ -51,40 +111,25 @@ impl FromRequest for GuardedData // TODO: find a less hardcoded way? let index = req.match_info().get("index_uid"); match type_token.next() { - Some(token) => match P::authenticate(auth, token, index) { - Some(filters) => match req.app_data::().cloned() { - Some(data) => ok(Self { - data, - filters, - _marker: PhantomData, - }), - None => err(AuthenticationError::IrretrievableState.into()), - }, - None => { - let token = token.to_string(); - err(AuthenticationError::InvalidToken(token).into()) - } - }, - None => { - err(AuthenticationError::InvalidToken("unknown".to_string()).into()) - } + Some(token) => Box::pin(Self::auth_bearer( + auth, + token.to_string(), + index.map(String::from), + req.app_data::().cloned(), + )), + None => Box::pin(err(AuthenticationError::InvalidToken( + "unknown".to_string(), + ) + .into())), } } - _otherwise => err(AuthenticationError::MissingAuthorizationHeader.into()), - }, - None => match P::authenticate(auth, "", None) { - Some(filters) => match req.app_data::().cloned() { - Some(data) => ok(Self { - data, - filters, - _marker: PhantomData, - }), - None => err(AuthenticationError::IrretrievableState.into()), - }, - None => err(AuthenticationError::MissingAuthorizationHeader.into()), + _otherwise => { + Box::pin(err(AuthenticationError::MissingAuthorizationHeader.into())) + } }, + None => Box::pin(Self::auth_token(auth, req.app_data::().cloned())), }, - None => err(AuthenticationError::IrretrievableState.into()), + None => Box::pin(err(AuthenticationError::IrretrievableState.into())), } } } From b57c59baa4f0683d521d91d7d9e23dd1f98dd08b Mon Sep 17 00:00:00 2001 From: ad hoc Date: Fri, 4 Mar 2022 20:12:44 +0100 Subject: [PATCH 12/16] sequential extractor --- Cargo.lock | 2 +- meilisearch-http/Cargo.toml | 2 +- .../src/extractors/authentication/mod.rs | 5 +- meilisearch-http/src/extractors/mod.rs | 1 + .../src/extractors/sequential_extractor.rs | 148 ++++++++++++++++++ meilisearch-http/src/routes/api_key.rs | 15 +- meilisearch-http/src/routes/dump.rs | 7 +- .../src/routes/indexes/documents.rs | 15 +- meilisearch-http/src/routes/indexes/mod.rs | 11 +- meilisearch-http/src/routes/indexes/search.rs | 5 +- .../src/routes/indexes/settings.rs | 14 +- meilisearch-http/src/routes/indexes/tasks.rs | 5 +- meilisearch-http/src/routes/tasks.rs | 5 +- meilisearch-http/tests/auth/authorization.rs | 1 + 14 files changed, 198 insertions(+), 38 deletions(-) create mode 100644 meilisearch-http/src/extractors/sequential_extractor.rs diff --git a/Cargo.lock b/Cargo.lock index 1ba427a1e..815fdf3b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1780,7 +1780,7 @@ dependencies = [ "once_cell", "parking_lot 0.11.2", "paste", - "pin-project", + "pin-project-lite", "platform-dirs", "rand", "rayon", diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index c3dd1026f..da7d9e61a 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -54,7 +54,6 @@ num_cpus = "1.13.0" obkv = "0.2.0" once_cell = "1.8.0" parking_lot = "0.11.2" -pin-project = "1.0.8" platform-dirs = "0.3.0" rand = "0.8.4" rayon = "1.5.1" @@ -78,6 +77,7 @@ tokio = { version = "1.11.0", features = ["full"] } tokio-stream = "0.1.7" uuid = { version = "0.8.2", features = ["serde"] } walkdir = "2.3.2" +pin-project-lite = "0.2.8" [dev-dependencies] actix-rt = "2.2.0" diff --git a/meilisearch-http/src/extractors/authentication/mod.rs b/meilisearch-http/src/extractors/authentication/mod.rs index ab0030fc1..873f7cbcd 100644 --- a/meilisearch-http/src/extractors/authentication/mod.rs +++ b/meilisearch-http/src/extractors/authentication/mod.rs @@ -41,10 +41,7 @@ impl GuardedData { }), None => Err(AuthenticationError::IrretrievableState.into()), }, - (token, None) => { - let token = token.to_string(); - Err(AuthenticationError::InvalidToken(token).into()) - } + (token, None) => Err(AuthenticationError::InvalidToken(token).into()), } } diff --git a/meilisearch-http/src/extractors/mod.rs b/meilisearch-http/src/extractors/mod.rs index 09a56e4a0..98a22f8c9 100644 --- a/meilisearch-http/src/extractors/mod.rs +++ b/meilisearch-http/src/extractors/mod.rs @@ -1,3 +1,4 @@ pub mod payload; #[macro_use] pub mod authentication; +pub mod sequential_extractor; diff --git a/meilisearch-http/src/extractors/sequential_extractor.rs b/meilisearch-http/src/extractors/sequential_extractor.rs new file mode 100644 index 000000000..1176334ad --- /dev/null +++ b/meilisearch-http/src/extractors/sequential_extractor.rs @@ -0,0 +1,148 @@ +#![allow(non_snake_case)] +use std::{future::Future, pin::Pin, task::Poll}; + +use actix_web::{dev::Payload, FromRequest, Handler, HttpRequest}; +use pin_project_lite::pin_project; + +/// `SeqHandler` is an actix `Handler` that enforces that extractors errors are returned in the +/// same order as they are defined in the wrapped handler. This is needed because, by default, actix +/// to resolves the extractors concurrently, whereas we always need the authentication extractor to +/// throw first. +#[derive(Clone)] +pub struct SeqHandler(pub H); + +pub struct SeqFromRequest(T); + +/// This macro implements `FromRequest` for arbitrary arity handler, except for one, which is +/// useless anyway. +macro_rules! gen_seq { + ($ty:ident; $($T:ident)+) => { + pin_project! { + pub struct $ty<$($T: FromRequest), +> { + $( + #[pin] + $T: ExtractFuture<$T::Future, $T, $T::Error>, + )+ + } + } + + impl<$($T: FromRequest), +> Future for $ty<$($T),+> { + type Output = Result, actix_web::Error>; + + fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { + let mut this = self.project(); + + let mut count_fut = 0; + let mut count_finished = 0; + + $( + count_fut += 1; + match this.$T.as_mut().project() { + ExtractProj::Future { fut } => match fut.poll(cx) { + Poll::Ready(Ok(output)) => { + count_finished += 1; + let _ = this + .$T + .as_mut() + .project_replace(ExtractFuture::Done { output }); + } + Poll::Ready(Err(error)) => { + count_finished += 1; + let _ = this + .$T + .as_mut() + .project_replace(ExtractFuture::Error { error }); + } + Poll::Pending => (), + }, + ExtractProj::Done { .. } => count_finished += 1, + ExtractProj::Error { .. } => { + // short circuit if all previous are finished and we had an error. + if count_finished == count_fut { + match this.$T.project_replace(ExtractFuture::Empty) { + ExtractReplaceProj::Error { error } => { + return Poll::Ready(Err(error.into())) + } + _ => unreachable!("Invalid future state"), + } + } else { + count_finished += 1; + } + } + ExtractProj::Empty => unreachable!("From request polled after being finished. {}", stringify!($T)), + } + )+ + + if count_fut == count_finished { + let result = ( + $( + match this.$T.project_replace(ExtractFuture::Empty) { + ExtractReplaceProj::Done { output } => output, + ExtractReplaceProj::Error { error } => return Poll::Ready(Err(error.into())), + _ => unreachable!("Invalid future state"), + }, + )+ + ); + + Poll::Ready(Ok(SeqFromRequest(result))) + } else { + Poll::Pending + } + } + } + + impl<$($T: FromRequest,)+> FromRequest for SeqFromRequest<($($T,)+)> { + type Error = actix_web::Error; + + type Future = $ty<$($T),+>; + + fn from_request(req: &HttpRequest, payload: &mut Payload) -> Self::Future { + $ty { + $( + $T: ExtractFuture::Future { + fut: $T::from_request(req, payload), + }, + )+ + } + } + } + + impl Handler> for SeqHandler + where + Han: Handler<($($T),+)>, + { + type Output = Han::Output; + type Future = Han::Future; + + fn call(&self, args: SeqFromRequest<($($T),+)>) -> Self::Future { + self.0.call(args.0) + } + } + }; +} + +// Not working for a single argument, but then, it is not really necessary. +// gen_seq! { SeqFromRequestFut1; A } +gen_seq! { SeqFromRequestFut2; A B } +gen_seq! { SeqFromRequestFut3; A B C } +gen_seq! { SeqFromRequestFut4; A B C D } +gen_seq! { SeqFromRequestFut5; A B C D E } +gen_seq! { SeqFromRequestFut6; A B C D E F } + +pin_project! { + #[project = ExtractProj] + #[project_replace = ExtractReplaceProj] + enum ExtractFuture { + Future { + #[pin] + fut: Fut, + }, + Done { + output: Res, + }, + Error { + error: Err, + }, + Empty, + } +} diff --git a/meilisearch-http/src/routes/api_key.rs b/meilisearch-http/src/routes/api_key.rs index aea35e68e..310b09c4d 100644 --- a/meilisearch-http/src/routes/api_key.rs +++ b/meilisearch-http/src/routes/api_key.rs @@ -7,20 +7,23 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use time::OffsetDateTime; -use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::extractors::{ + authentication::{policies::*, GuardedData}, + sequential_extractor::SeqHandler, +}; use meilisearch_error::{Code, ResponseError}; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") - .route(web::post().to(create_api_key)) - .route(web::get().to(list_api_keys)), + .route(web::post().to(SeqHandler(create_api_key))) + .route(web::get().to(SeqHandler(list_api_keys))), ) .service( web::resource("/{api_key}") - .route(web::get().to(get_api_key)) - .route(web::patch().to(patch_api_key)) - .route(web::delete().to(delete_api_key)), + .route(web::get().to(SeqHandler(get_api_key))) + .route(web::patch().to(SeqHandler(patch_api_key))) + .route(web::delete().to(SeqHandler(delete_api_key))), ); } diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index 0627ea378..65cd7521f 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -7,10 +7,13 @@ use serde_json::json; use crate::analytics::Analytics; use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::extractors::sequential_extractor::SeqHandler; pub fn configure(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("").route(web::post().to(create_dump))) - .service(web::resource("/{dump_uid}/status").route(web::get().to(get_dump_status))); + cfg.service(web::resource("").route(web::post().to(SeqHandler(create_dump)))) + .service( + web::resource("/{dump_uid}/status").route(web::get().to(SeqHandler(get_dump_status))), + ); } pub async fn create_dump( diff --git a/meilisearch-http/src/routes/indexes/documents.rs b/meilisearch-http/src/routes/indexes/documents.rs index d18c600af..66551ec77 100644 --- a/meilisearch-http/src/routes/indexes/documents.rs +++ b/meilisearch-http/src/routes/indexes/documents.rs @@ -20,6 +20,7 @@ use crate::analytics::Analytics; use crate::error::MeilisearchHttpError; use crate::extractors::authentication::{policies::*, GuardedData}; use crate::extractors::payload::Payload; +use crate::extractors::sequential_extractor::SeqHandler; use crate::task::SummarizedTaskView; const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0; @@ -71,17 +72,17 @@ pub struct DocumentParam { pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") - .route(web::get().to(get_all_documents)) - .route(web::post().to(add_documents)) - .route(web::put().to(update_documents)) - .route(web::delete().to(clear_all_documents)), + .route(web::get().to(SeqHandler(get_all_documents))) + .route(web::post().to(SeqHandler(add_documents))) + .route(web::put().to(SeqHandler(update_documents))) + .route(web::delete().to(SeqHandler(clear_all_documents))), ) // this route needs to be before the /documents/{document_id} to match properly - .service(web::resource("/delete-batch").route(web::post().to(delete_documents))) + .service(web::resource("/delete-batch").route(web::post().to(SeqHandler(delete_documents)))) .service( web::resource("/{document_id}") - .route(web::get().to(get_document)) - .route(web::delete().to(delete_document)), + .route(web::get().to(SeqHandler(get_document))) + .route(web::delete().to(SeqHandler(delete_document))), ); } diff --git a/meilisearch-http/src/routes/indexes/mod.rs b/meilisearch-http/src/routes/indexes/mod.rs index 50e54e6b4..bd74fd724 100644 --- a/meilisearch-http/src/routes/indexes/mod.rs +++ b/meilisearch-http/src/routes/indexes/mod.rs @@ -9,6 +9,7 @@ use time::OffsetDateTime; use crate::analytics::Analytics; use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::extractors::sequential_extractor::SeqHandler; use crate::task::SummarizedTaskView; pub mod documents; @@ -20,17 +21,17 @@ pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") .route(web::get().to(list_indexes)) - .route(web::post().to(create_index)), + .route(web::post().to(SeqHandler(create_index))), ) .service( web::scope("/{index_uid}") .service( web::resource("") - .route(web::get().to(get_index)) - .route(web::put().to(update_index)) - .route(web::delete().to(delete_index)), + .route(web::get().to(SeqHandler(get_index))) + .route(web::put().to(SeqHandler(update_index))) + .route(web::delete().to(SeqHandler(delete_index))), ) - .service(web::resource("/stats").route(web::get().to(get_index_stats))) + .service(web::resource("/stats").route(web::get().to(SeqHandler(get_index_stats)))) .service(web::scope("/documents").configure(documents::configure)) .service(web::scope("/search").configure(search::configure)) .service(web::scope("/tasks").configure(tasks::configure)) diff --git a/meilisearch-http/src/routes/indexes/search.rs b/meilisearch-http/src/routes/indexes/search.rs index a1695633e..14b3f74f5 100644 --- a/meilisearch-http/src/routes/indexes/search.rs +++ b/meilisearch-http/src/routes/indexes/search.rs @@ -9,12 +9,13 @@ use serde_json::Value; use crate::analytics::{Analytics, SearchAggregator}; use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::extractors::sequential_extractor::SeqHandler; pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") - .route(web::get().to(search_with_url_query)) - .route(web::post().to(search_with_post)), + .route(web::get().to(SeqHandler(search_with_url_query))) + .route(web::post().to(SeqHandler(search_with_post))), ); } diff --git a/meilisearch-http/src/routes/indexes/settings.rs b/meilisearch-http/src/routes/indexes/settings.rs index 8b38072c4..eeb3e71b3 100644 --- a/meilisearch-http/src/routes/indexes/settings.rs +++ b/meilisearch-http/src/routes/indexes/settings.rs @@ -23,6 +23,7 @@ macro_rules! make_setting_route { use crate::analytics::Analytics; use crate::extractors::authentication::{policies::*, GuardedData}; + use crate::extractors::sequential_extractor::SeqHandler; use crate::task::SummarizedTaskView; use meilisearch_error::ResponseError; @@ -98,9 +99,9 @@ macro_rules! make_setting_route { pub fn resources() -> Resource { Resource::new($route) - .route(web::get().to(get)) - .route(web::post().to(update)) - .route(web::delete().to(delete)) + .route(web::get().to(SeqHandler(get))) + .route(web::post().to(SeqHandler(update))) + .route(web::delete().to(SeqHandler(delete))) } } }; @@ -226,11 +227,12 @@ make_setting_route!( macro_rules! generate_configure { ($($mod:ident),*) => { pub fn configure(cfg: &mut web::ServiceConfig) { + use crate::extractors::sequential_extractor::SeqHandler; cfg.service( web::resource("") - .route(web::post().to(update_all)) - .route(web::get().to(get_all)) - .route(web::delete().to(delete_all))) + .route(web::post().to(SeqHandler(update_all))) + .route(web::get().to(SeqHandler(get_all))) + .route(web::delete().to(SeqHandler(delete_all)))) $(.service($mod::resources()))*; } }; diff --git a/meilisearch-http/src/routes/indexes/tasks.rs b/meilisearch-http/src/routes/indexes/tasks.rs index 8545831a0..01ed85db8 100644 --- a/meilisearch-http/src/routes/indexes/tasks.rs +++ b/meilisearch-http/src/routes/indexes/tasks.rs @@ -8,11 +8,12 @@ use time::OffsetDateTime; use crate::analytics::Analytics; use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::extractors::sequential_extractor::SeqHandler; use crate::task::{TaskListView, TaskView}; pub fn configure(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("").route(web::get().to(get_all_tasks_status))) - .service(web::resource("{task_id}").route(web::get().to(get_task_status))); + cfg.service(web::resource("").route(web::get().to(SeqHandler(get_all_tasks_status)))) + .service(web::resource("{task_id}").route(web::get().to(SeqHandler(get_task_status)))); } #[derive(Debug, Serialize)] diff --git a/meilisearch-http/src/routes/tasks.rs b/meilisearch-http/src/routes/tasks.rs index 350cef3dc..ae932253a 100644 --- a/meilisearch-http/src/routes/tasks.rs +++ b/meilisearch-http/src/routes/tasks.rs @@ -7,11 +7,12 @@ use serde_json::json; use crate::analytics::Analytics; use crate::extractors::authentication::{policies::*, GuardedData}; +use crate::extractors::sequential_extractor::SeqHandler; use crate::task::{TaskListView, TaskView}; pub fn configure(cfg: &mut web::ServiceConfig) { - cfg.service(web::resource("").route(web::get().to(get_tasks))) - .service(web::resource("/{task_id}").route(web::get().to(get_task))); + cfg.service(web::resource("").route(web::get().to(SeqHandler(get_tasks)))) + .service(web::resource("/{task_id}").route(web::get().to(SeqHandler(get_task)))); } async fn get_tasks( diff --git a/meilisearch-http/tests/auth/authorization.rs b/meilisearch-http/tests/auth/authorization.rs index 30df2dd2d..dcb4504c6 100644 --- a/meilisearch-http/tests/auth/authorization.rs +++ b/meilisearch-http/tests/auth/authorization.rs @@ -91,6 +91,7 @@ async fn error_access_expired_key() { thread::sleep(time::Duration::new(1, 0)); for (method, route) in AUTHORIZATIONS.keys() { + dbg!(route); let (response, code) = server.dummy_request(method, route).await; assert_eq!(response, INVALID_RESPONSE.clone()); From 0026410c611d795492fe9a906ee8bbb53220777f Mon Sep 17 00:00:00 2001 From: ad hoc Date: Mon, 7 Mar 2022 14:21:44 +0100 Subject: [PATCH 13/16] review edits --- meilisearch-http/tests/auth/authorization.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/meilisearch-http/tests/auth/authorization.rs b/meilisearch-http/tests/auth/authorization.rs index dcb4504c6..30df2dd2d 100644 --- a/meilisearch-http/tests/auth/authorization.rs +++ b/meilisearch-http/tests/auth/authorization.rs @@ -91,7 +91,6 @@ async fn error_access_expired_key() { thread::sleep(time::Duration::new(1, 0)); for (method, route) in AUTHORIZATIONS.keys() { - dbg!(route); let (response, code) = server.dummy_request(method, route).await; assert_eq!(response, INVALID_RESPONSE.clone()); From 19da45c53bbecef0f4da0d4b62bc41f6a723b083 Mon Sep 17 00:00:00 2001 From: ad hoc Date: Mon, 7 Mar 2022 15:02:07 +0100 Subject: [PATCH 14/16] Update meilisearch-http/src/extractors/sequential_extractor.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Renault --- meilisearch-http/src/extractors/sequential_extractor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meilisearch-http/src/extractors/sequential_extractor.rs b/meilisearch-http/src/extractors/sequential_extractor.rs index 1176334ad..d6cee6083 100644 --- a/meilisearch-http/src/extractors/sequential_extractor.rs +++ b/meilisearch-http/src/extractors/sequential_extractor.rs @@ -6,7 +6,7 @@ use pin_project_lite::pin_project; /// `SeqHandler` is an actix `Handler` that enforces that extractors errors are returned in the /// same order as they are defined in the wrapped handler. This is needed because, by default, actix -/// to resolves the extractors concurrently, whereas we always need the authentication extractor to +/// resolves the extractors concurrently, whereas we always need the authentication extractor to /// throw first. #[derive(Clone)] pub struct SeqHandler(pub H); From ddd25bfe019ff138b6966e4ff3ec0da46465cd87 Mon Sep 17 00:00:00 2001 From: ad hoc Date: Mon, 7 Mar 2022 15:15:51 +0100 Subject: [PATCH 15/16] remove token from InvalidToken error --- .../src/extractors/authentication/error.rs | 4 ++-- .../src/extractors/authentication/mod.rs | 21 +++++++------------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/meilisearch-http/src/extractors/authentication/error.rs b/meilisearch-http/src/extractors/authentication/error.rs index c1af9a3ce..6d362dcbf 100644 --- a/meilisearch-http/src/extractors/authentication/error.rs +++ b/meilisearch-http/src/extractors/authentication/error.rs @@ -5,7 +5,7 @@ pub enum AuthenticationError { #[error("The Authorization header is missing. It must use the bearer authorization method.")] MissingAuthorizationHeader, #[error("The provided API key is invalid.")] - InvalidToken(String), + InvalidToken, // Triggered on configuration error. #[error("An internal error has occurred. `Irretrievable state`.")] IrretrievableState, @@ -15,7 +15,7 @@ impl ErrorCode for AuthenticationError { fn error_code(&self) -> Code { match self { AuthenticationError::MissingAuthorizationHeader => Code::MissingAuthorizationHeader, - AuthenticationError::InvalidToken(_) => Code::InvalidToken, + AuthenticationError::InvalidToken => Code::InvalidToken, AuthenticationError::IrretrievableState => Code::Internal, } } diff --git a/meilisearch-http/src/extractors/authentication/mod.rs b/meilisearch-http/src/extractors/authentication/mod.rs index 873f7cbcd..0a0d9ecfe 100644 --- a/meilisearch-http/src/extractors/authentication/mod.rs +++ b/meilisearch-http/src/extractors/authentication/mod.rs @@ -33,7 +33,7 @@ impl GuardedData { P: Policy + 'static, { match Self::authenticate(auth, token, index).await? { - (_, Some(filters)) => match data { + Some(filters) => match data { Some(data) => Ok(Self { data, filters, @@ -41,7 +41,7 @@ impl GuardedData { }), None => Err(AuthenticationError::IrretrievableState.into()), }, - (token, None) => Err(AuthenticationError::InvalidToken(token).into()), + None => Err(AuthenticationError::InvalidToken.into()), } } @@ -49,7 +49,7 @@ impl GuardedData { where P: Policy + 'static, { - match Self::authenticate(auth, "", None).await?.1 { + match Self::authenticate(auth, String::new(), None).await? { Some(filters) => match data { Some(data) => Ok(Self { data, @@ -62,18 +62,16 @@ impl GuardedData { } } - async fn authenticate( + async fn authenticate( auth: AuthController, - token: S, + token: String, index: Option, - ) -> Result<(S, Option), ResponseError> + ) -> Result, ResponseError> where P: Policy + 'static, - S: AsRef + 'static + Send, { Ok(tokio::task::spawn_blocking(move || { - let res = P::authenticate(auth, token.as_ref(), index.as_deref()); - (token, res) + P::authenticate(auth, token.as_ref(), index.as_deref()) }) .await .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))?) @@ -114,10 +112,7 @@ impl FromRequest for GuardedData index.map(String::from), req.app_data::().cloned(), )), - None => Box::pin(err(AuthenticationError::InvalidToken( - "unknown".to_string(), - ) - .into())), + None => Box::pin(err(AuthenticationError::InvalidToken.into())), } } _otherwise => { From 62ce8e0bdaec7eece58283c84c06430d190de2fa Mon Sep 17 00:00:00 2001 From: ad hoc Date: Mon, 7 Mar 2022 14:53:09 +0100 Subject: [PATCH 16/16] chore(http): rename auto batching cli option --- meilisearch-http/src/lib.rs | 2 +- meilisearch-lib/src/options.rs | 2 +- meilisearch-lib/src/tasks/scheduler.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 71932dd9b..d1f5d9da1 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -32,7 +32,7 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result { // enable autobatching? let _ = AUTOBATCHING_ENABLED.store( - opt.scheduler_options.enable_autobatching, + opt.scheduler_options.enable_auto_batching, std::sync::atomic::Ordering::Relaxed, ); diff --git a/meilisearch-lib/src/options.rs b/meilisearch-lib/src/options.rs index d6657cae6..54a411250 100644 --- a/meilisearch-lib/src/options.rs +++ b/meilisearch-lib/src/options.rs @@ -48,7 +48,7 @@ pub struct IndexerOpts { pub struct SchedulerConfig { /// enable the autobatching experimental feature #[clap(long, hide = true)] - pub enable_autobatching: bool, + pub enable_auto_batching: bool, // The maximum number of updates of the same type that can be batched together. // If unspecified, this is unlimited. A value of 0 is interpreted as 1. diff --git a/meilisearch-lib/src/tasks/scheduler.rs b/meilisearch-lib/src/tasks/scheduler.rs index 50f40a8e2..0e540a646 100644 --- a/meilisearch-lib/src/tasks/scheduler.rs +++ b/meilisearch-lib/src/tasks/scheduler.rs @@ -218,7 +218,7 @@ impl Scheduler { let debounce_time = config.debounce_duration_sec; // Disable autobatching - if !config.enable_autobatching { + if !config.enable_auto_batching { config.max_batch_size = Some(1); }