From ddd778971351d8eb5b937813ea5b375b1af9b68c Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 13 Jan 2021 17:50:36 +0100 Subject: [PATCH 01/16] WIP: IndexController --- Cargo.lock | 28 +++--- Cargo.toml | 4 +- src/data/mod.rs | 26 +++-- src/data/search.rs | 36 +++---- src/index_controller/mod.rs | 187 ++++++++++++++++++++++++++++++++++++ src/lib.rs | 4 +- 6 files changed, 230 insertions(+), 55 deletions(-) create mode 100644 src/index_controller/mod.rs diff --git a/Cargo.lock b/Cargo.lock index f717be694..4415db555 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -777,6 +777,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if 1.0.0", + "num_cpus", +] + [[package]] name = "debugid" version = "0.7.2" @@ -1201,7 +1211,6 @@ dependencies = [ "lmdb-rkv-sys", "once_cell", "page_size", - "serde", "synchronoise", "url", "zerocopy", @@ -1588,7 +1597,7 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "meilisearch-error" -version = "0.15.0" +version = "0.18.0" dependencies = [ "actix-http", ] @@ -1609,6 +1618,7 @@ dependencies = [ "bytes 0.6.0", "chrono", "crossbeam-channel", + "dashmap", "env_logger 0.8.2", "flate2", "fst", @@ -1627,6 +1637,7 @@ dependencies = [ "milli", "mime", "once_cell", + "page_size", "rand 0.7.3", "rayon", "regex", @@ -1699,7 +1710,6 @@ dependencies = [ "bstr", "byte-unit", "byteorder", - "chrono", "crossbeam-channel", "csv", "either", @@ -1714,13 +1724,13 @@ dependencies = [ "levenshtein_automata", "linked-hash-map", "log", - "meilisearch-tokenizer", "memmap", "near-proximity", "num-traits", "obkv", "once_cell", "ordered-float", + "page_size", "pest 2.1.3 (git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67)", "pest_derive", "rayon", @@ -1729,7 +1739,6 @@ dependencies = [ "roaring", "serde", "serde_json", - "serde_millis", "slice-group-by", "smallstr", "smallvec", @@ -2596,15 +2605,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_millis" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e2dc780ca5ee2c369d1d01d100270203c4ff923d2a4264812d723766434d00" -dependencies = [ - "serde", -] - [[package]] name = "serde_qs" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index fb8531a96..47bbc09df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ fst = "0.4.5" futures = "0.3.7" futures-util = "0.3.8" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = "0.10.6" +heed = { version = "0.10.6", default-features = false, features = ["lmdb", "sync-read-txn"] } http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } log = "0.4.8" @@ -58,6 +58,8 @@ tokio = { version = "0.2", features = ["full"] } ureq = { version = "1.5.1", default-features = false, features = ["tls"] } walkdir = "2.3.1" whoami = "1.0.0" +dashmap = "4.0.2" +page_size = "0.4.2" [dependencies.sentry] default-features = false diff --git a/src/data/mod.rs b/src/data/mod.rs index 9d64052af..944690e0c 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -3,7 +3,6 @@ mod updates; pub use search::{SearchQuery, SearchResult}; -use std::collections::HashMap; use std::fs::create_dir_all; use std::ops::Deref; use std::sync::Arc; @@ -13,6 +12,7 @@ use sha2::Digest; use crate::{option::Opt, updates::Settings}; use crate::updates::UpdateQueue; +use crate::index_controller::IndexController; #[derive(Clone)] pub struct Data { @@ -29,7 +29,7 @@ impl Deref for Data { #[derive(Clone)] pub struct DataInner { - pub indexes: Arc, + pub indexes: Arc, pub update_queue: Arc, api_keys: ApiKeys, options: Opt, @@ -62,9 +62,7 @@ impl ApiKeys { impl Data { pub fn new(options: Opt) -> anyhow::Result { let db_size = options.max_mdb_size.get_bytes() as usize; - let path = options.db_path.join("main"); - create_dir_all(&path)?; - let indexes = Index::new(&path, Some(db_size))?; + let indexes = IndexController::new(&options.db_path)?; let indexes = Arc::new(indexes); let update_queue = Arc::new(UpdateQueue::new(&options, indexes.clone())?); @@ -90,28 +88,26 @@ impl Data { let displayed_attributes = self.indexes .displayed_fields(&txn)? - .map(|fields| {println!("{:?}", fields); fields.iter().filter_map(|f| fields_map.name(*f).map(String::from)).collect()}) + .map(|fields| fields.into_iter().map(String::from).collect()) .unwrap_or_else(|| vec!["*".to_string()]); let searchable_attributes = self.indexes .searchable_fields(&txn)? .map(|fields| fields - .iter() - .filter_map(|f| fields_map.name(*f).map(String::from)) + .into_iter() + .map(String::from) .collect()) .unwrap_or_else(|| vec!["*".to_string()]); - let faceted_attributes = self.indexes - .faceted_fields(&txn)? - .iter() - .filter_map(|(f, t)| Some((fields_map.name(*f)?.to_string(), t.to_string()))) - .collect::>() - .into(); + let faceted_attributes = self.indexes.faceted_fields(&txn)? + .into_iter() + .map(|(k, v)| (k, v.to_string())) + .collect(); Ok(Settings { displayed_attributes: Some(Some(displayed_attributes)), searchable_attributes: Some(Some(searchable_attributes)), - faceted_attributes: Some(faceted_attributes), + faceted_attributes: Some(Some(faceted_attributes)), criteria: None, }) } diff --git a/src/data/search.rs b/src/data/search.rs index bd22a959b..44e0a54e6 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -1,4 +1,3 @@ -use std::borrow::Cow; use std::collections::HashSet; use std::mem; use std::time::Instant; @@ -8,17 +7,22 @@ use serde::{Deserialize, Serialize}; use milli::{SearchResult as Results, obkv_to_json}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; +use crate::error::Error; + use super::Data; const DEFAULT_SEARCH_LIMIT: usize = 20; +const fn default_search_limit() -> usize { DEFAULT_SEARCH_LIMIT } + #[derive(Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] #[allow(dead_code)] pub struct SearchQuery { q: Option, offset: Option, - limit: Option, + #[serde(default = "default_search_limit")] + limit: usize, attributes_to_retrieve: Option>, attributes_to_crop: Option>, crop_length: Option, @@ -100,30 +104,18 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { } impl Data { - pub fn search>(&self, _index: S, search_query: SearchQuery) -> anyhow::Result { + pub fn search>(&self, index: S, search_query: SearchQuery) -> anyhow::Result { let start = Instant::now(); - let index = &self.indexes; - let rtxn = index.read_txn()?; - - let mut search = index.search(&rtxn); - if let Some(query) = &search_query.q { - search.query(query); - } - - if let Some(offset) = search_query.offset { - search.offset(offset); - } - - let limit = search_query.limit.unwrap_or(DEFAULT_SEARCH_LIMIT); - search.limit(limit); - - let Results { found_words, documents_ids, nb_hits, .. } = search.execute().unwrap(); + let index = self.indexes + .get(index)? + .ok_or_else(|| Error::OpenIndex(format!("Index {} doesn't exists.", index.as_ref())))?; + let Results { found_words, documents_ids, nb_hits, .. } = index.search(search_query)?; let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); - let displayed_fields = match index.displayed_fields(&rtxn).unwrap() { - Some(fields) => Cow::Borrowed(fields), - None => Cow::Owned(fields_ids_map.iter().map(|(id, _)| id).collect()), + let displayed_fields = match index.displayed_fields_ids(&rtxn).unwrap() { + Some(fields) => fields, + None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; let attributes_to_highlight = match search_query.attributes_to_highlight { diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs new file mode 100644 index 000000000..9dfa23ce5 --- /dev/null +++ b/src/index_controller/mod.rs @@ -0,0 +1,187 @@ +use std::fs::File; +use std::io::{Read, Write}; +use std::path::Path; +use std::ops::Deref; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use dashmap::DashMap; +use heed::types::{Str, SerdeBincode}; +use heed::{EnvOpenOptions, Env, Database}; +use milli::Index; +use serde::{Serialize, Deserialize}; + +use crate::data::{SearchQuery, SearchResult}; + +const CONTROLLER_META_FILENAME: &str = "index_controller_meta"; +const INDEXES_CONTROLLER_FILENAME: &str = "indexes_db"; +const INDEXES_DB_NAME: &str = "indexes_db"; + +trait UpdateStore {} + +pub struct IndexController { + update_store: U, + env: Env, + indexes_db: Database>, + indexes: DashMap, +} + +#[derive(Debug, Serialize, Deserialize)] +struct IndexControllerMeta { + open_options: EnvOpenOptions, + created_at: DateTime, +} + +impl IndexControllerMeta { + fn from_path(path: impl AsRef) -> Result> { + let path = path.as_ref().to_path_buf().push(CONTROLLER_META_FILENAME); + if path.exists() { + let mut file = File::open(path)?; + let mut buffer = Vec::new(); + let n = file.read_to_end(&mut buffer)?; + let meta: IndexControllerMeta = serde_json::from_slice(&buffer[..n])?; + Ok(Some(meta)) + } else { + Ok(None) + } + } + + fn to_path(self, path: impl AsRef) -> Result<()> { + let path = path.as_ref().to_path_buf().push(CONTROLLER_META_FILENAME); + if path.exists() { + Err(anyhow::anyhow!("Index controller metadata already exists")) + } else { + let mut file = File::create(path)?; + let json = serde_json::to_vec(&self)?; + file.write_all(&json)?; + Ok(()) + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct IndexMetadata { + created_at: DateTime, + open_options: EnvOpenOptions, + id: String, +} + +impl IndexMetadata { + fn open_index(&self) -> Result { + todo!() + } +} + +struct IndexView<'a, U> { + txn: heed::RoTxn<'a>, + index: &'a Index, + update_store: &'a U, +} + +struct IndexViewMut<'a, U> { + txn: heed::RwTxn<'a>, + index: &'a Index, + update_store: &'a U, +} + +impl<'a, U> Deref for IndexViewMut<'a, U> { + type Target = IndexView<'a, U>; + + fn deref(&self) -> &Self::Target { + IndexView { + txn: *self.txn, + index: self.index, + update_store: self.update_store, + } + } +} + +impl<'a, U: UpdateStore> IndexView<'a, U> { + fn search(&self, search_query: SearchQuery) -> Result { + let mut search = self.index.search(self.txn); + if let Some(query) = &search_query.q { + search.query(query); + } + + if let Some(offset) = search_query.offset { + search.offset(offset); + } + + let limit = search_query.limit; + search.limit(limit); + + Ok(search.execute()?) + } +} + +impl IndexController { + /// Open the index controller from meta found at path, and create a new one if no meta is + /// found. + pub fn new(path: impl AsRef, update_store: U) -> Result { + // If index controller metadata is present, we return the env, otherwise, we create a new + // metadata from scratch before returning a new env. + let env = match IndexControllerMeta::from_path(path)? { + Some(meta) => meta.open_options.open(INDEXES_CONTROLLER_FILENAME)?, + None => { + let open_options = EnvOpenOptions::new() + .map_size(page_size::get() * 1000); + let env = open_options.open(INDEXES_CONTROLLER_FILENAME)?; + let created_at = Utc::now(); + let meta = IndexControllerMeta { open_options, created_at }; + meta.to_path(path)?; + env + } + }; + let indexes = DashMap::new(); + let indexes_db = match env.open_database(INDEXES_DB_NAME)? { + Some(indexes_db) => indexes_db, + None => env.create_database(INDEXES_DB_NAME)?, + }; + + Ok(Self { env, indexes, indexes_db, update_store }) + } + + pub fn get_or_create>(&mut self, name: S) -> Result> { + todo!() + } + + /// Get an index with read access to the db. The index are lazily loaded, meaning that we first + /// check for its exixtence in the indexes map, and if it doesn't exist, the index db is check + /// for metadata to launch the index. + pub fn get>(&self, name: S) -> Result>> { + match self.indexes.get(name.as_ref()) { + Some(index) => { + let txn = index.read_txn()?; + let update_store = &self.update_store; + Ok(Some(IndexView { index, update_store, txn })) + } + None => { + let txn = self.env.read_txn()?; + match self.indexes_db.get(&txn, name.as_ref())? { + Some(meta) => { + let index = meta.open_index()?; + self.indexes.insert(name.as_ref().to_owned(), index); + Ok(self.indexes.get(name.as_ref())) + } + None => Ok(None) + } + } + } + } + + pub fn get_mut>(&self, name: S) -> Result>> { + todo!() + } + + pub async fn delete_index>(&self, name:S) -> Result<()> { + todo!() + } + + pub async fn list_indices(&self) -> Result> { + todo!() + } + + pub async fn rename_index(&self, old: &str, new: &str) -> Result<()> { + todo!() + } +} diff --git a/src/lib.rs b/src/lib.rs index e0ae9aedb..f5dd79b0b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,9 +6,7 @@ pub mod helpers; pub mod option; pub mod routes; mod updates; -//pub mod analytics; -//pub mod snapshot; -//pub mod dump; +mod index_controller; use actix_http::Error; use actix_service::ServiceFactory; From d22fab5baec0fdc6029056e9c8af87acb1fddeae Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 13 Jan 2021 18:18:52 +0100 Subject: [PATCH 02/16] implement open index --- Cargo.lock | 13 ++++++++++++- src/index_controller/mod.rs | 9 +++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4415db555..40528367e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1710,6 +1710,7 @@ dependencies = [ "bstr", "byte-unit", "byteorder", + "chrono", "crossbeam-channel", "csv", "either", @@ -1724,13 +1725,13 @@ dependencies = [ "levenshtein_automata", "linked-hash-map", "log", + "meilisearch-tokenizer", "memmap", "near-proximity", "num-traits", "obkv", "once_cell", "ordered-float", - "page_size", "pest 2.1.3 (git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67)", "pest_derive", "rayon", @@ -1739,6 +1740,7 @@ dependencies = [ "roaring", "serde", "serde_json", + "serde_millis", "slice-group-by", "smallstr", "smallvec", @@ -2605,6 +2607,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_millis" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e2dc780ca5ee2c369d1d01d100270203c4ff923d2a4264812d723766434d00" +dependencies = [ + "serde", +] + [[package]] name = "serde_qs" version = "0.8.2" diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 9dfa23ce5..0c6a95c1f 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -67,8 +67,9 @@ struct IndexMetadata { } impl IndexMetadata { - fn open_index(&self) -> Result { - todo!() + fn open_index(&self, path: impl AsRef) -> Result { + let path = path.as_ref().to_path_buf().push("indexes").push(&self.id); + Ok(Index::new(self.options, path)?) } } @@ -79,7 +80,7 @@ struct IndexView<'a, U> { } struct IndexViewMut<'a, U> { - txn: heed::RwTxn<'a>, + txn: heed::RwTxn<'a, 'a>, index: &'a Index, update_store: &'a U, } @@ -97,7 +98,7 @@ impl<'a, U> Deref for IndexViewMut<'a, U> { } impl<'a, U: UpdateStore> IndexView<'a, U> { - fn search(&self, search_query: SearchQuery) -> Result { + pub fn search(&self, search_query: SearchQuery) -> Result { let mut search = self.index.search(self.txn); if let Some(query) = &search_query.q { search.query(query); From 334933b874d811599f2e893ca561c81e32729b0a Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 13 Jan 2021 18:29:17 +0100 Subject: [PATCH 03/16] fix search --- src/data/search.rs | 9 +++++---- src/index_controller/mod.rs | 14 +++++++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/data/search.rs b/src/data/search.rs index 44e0a54e6..8c8e0da69 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -109,11 +109,12 @@ impl Data { let index = self.indexes .get(index)? .ok_or_else(|| Error::OpenIndex(format!("Index {} doesn't exists.", index.as_ref())))?; - let Results { found_words, documents_ids, nb_hits, .. } = index.search(search_query)?; - let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); + let Results { found_words, documents_ids, nb_hits, limit, .. } = index.search(search_query)?; - let displayed_fields = match index.displayed_fields_ids(&rtxn).unwrap() { + let fields_ids_map = index.fields_ids_map()?; + + let displayed_fields = match index.displayed_fields_ids()? { Some(fields) => fields, None => fields_ids_map.iter().map(|(id, _)| id).collect(), }; @@ -126,7 +127,7 @@ impl Data { let stop_words = fst::Set::default(); let highlighter = Highlighter::new(&stop_words); let mut documents = Vec::new(); - for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { + for (_id, obkv) in index.documents(&documents_ids)? { let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv).unwrap(); highlighter.highlight_record(&mut object, &found_words, &attributes_to_highlight); documents.push(object); diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 0c6a95c1f..25e5d4787 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -8,7 +8,7 @@ use chrono::{DateTime, Utc}; use dashmap::DashMap; use heed::types::{Str, SerdeBincode}; use heed::{EnvOpenOptions, Env, Database}; -use milli::Index; +use milli::{Index, FieldsIdsMap}; use serde::{Serialize, Deserialize}; use crate::data::{SearchQuery, SearchResult}; @@ -113,6 +113,18 @@ impl<'a, U: UpdateStore> IndexView<'a, U> { Ok(search.execute()?) } + + pub fn fields_ids_map(&self) -> Result { + self.index.fields_ids_map(self.txn) + } + + pub fn fields_displayed_fields_ids(&self) -> Result { + self.index.fields_displayed_fields_ids(self.txn) + } + + pub fn documents(&self, ids: &[u32]) -> Result> { + self.index.documents(self.txn, ids) + } } impl IndexController { From 686f9871806505151d9c624723a76c0570209187 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 14 Jan 2021 11:27:07 +0100 Subject: [PATCH 04/16] fix compile errors --- Cargo.lock | 2 + Cargo.toml | 3 +- src/data/mod.rs | 5 +-- src/data/search.rs | 22 +++++----- src/index_controller/mod.rs | 86 ++++++++++++++++++------------------- src/updates/mod.rs | 2 + 6 files changed, 59 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40528367e..e0457b4d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1211,6 +1211,7 @@ dependencies = [ "lmdb-rkv-sys", "once_cell", "page_size", + "serde", "synchronoise", "url", "zerocopy", @@ -1636,6 +1637,7 @@ dependencies = [ "memmap", "milli", "mime", + "obkv", "once_cell", "page_size", "rand 0.7.3", diff --git a/Cargo.toml b/Cargo.toml index 47bbc09df..c06847253 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ fst = "0.4.5" futures = "0.3.7" futures-util = "0.3.8" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = { version = "0.10.6", default-features = false, features = ["lmdb", "sync-read-txn"] } +heed = { version = "0.10.6", default-features = false, features = ["serde", "lmdb", "sync-read-txn"] } http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } log = "0.4.8" @@ -60,6 +60,7 @@ walkdir = "2.3.1" whoami = "1.0.0" dashmap = "4.0.2" page_size = "0.4.2" +obkv = "0.1.1" [dependencies.sentry] default-features = false diff --git a/src/data/mod.rs b/src/data/mod.rs index 944690e0c..4600531bf 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -3,11 +3,9 @@ mod updates; pub use search::{SearchQuery, SearchResult}; -use std::fs::create_dir_all; use std::ops::Deref; use std::sync::Arc; -use milli::Index; use sha2::Digest; use crate::{option::Opt, updates::Settings}; @@ -29,8 +27,7 @@ impl Deref for Data { #[derive(Clone)] pub struct DataInner { - pub indexes: Arc, - pub update_queue: Arc, + pub indexes: Arc>, api_keys: ApiKeys, options: Opt, } diff --git a/src/data/search.rs b/src/data/search.rs index 8c8e0da69..69029d8a9 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -19,18 +19,18 @@ const fn default_search_limit() -> usize { DEFAULT_SEARCH_LIMIT } #[serde(rename_all = "camelCase", deny_unknown_fields)] #[allow(dead_code)] pub struct SearchQuery { - q: Option, - offset: Option, + pub q: Option, + pub offset: Option, #[serde(default = "default_search_limit")] - limit: usize, - attributes_to_retrieve: Option>, - attributes_to_crop: Option>, - crop_length: Option, - attributes_to_highlight: Option>, - filters: Option, - matches: Option, - facet_filters: Option, - facets_distribution: Option>, + pub limit: usize, + pub attributes_to_retrieve: Option>, + pub attributes_to_crop: Option>, + pub crop_length: Option, + pub attributes_to_highlight: Option>, + pub filters: Option, + pub matches: Option, + pub facet_filters: Option, + pub facets_distribution: Option>, } #[derive(Serialize)] diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 25e5d4787..a3b29879a 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,25 +1,26 @@ use std::fs::File; use std::io::{Read, Write}; -use std::path::Path; -use std::ops::Deref; +use std::path::{Path, PathBuf}; use anyhow::Result; use chrono::{DateTime, Utc}; use dashmap::DashMap; +use dashmap::mapref::one::Ref; use heed::types::{Str, SerdeBincode}; use heed::{EnvOpenOptions, Env, Database}; -use milli::{Index, FieldsIdsMap}; +use milli::{Index, FieldsIdsMap, SearchResult, FieldId}; use serde::{Serialize, Deserialize}; -use crate::data::{SearchQuery, SearchResult}; +use crate::data::SearchQuery; const CONTROLLER_META_FILENAME: &str = "index_controller_meta"; const INDEXES_CONTROLLER_FILENAME: &str = "indexes_db"; const INDEXES_DB_NAME: &str = "indexes_db"; -trait UpdateStore {} +pub trait UpdateStore {} pub struct IndexController { + path: PathBuf, update_store: U, env: Env, indexes_db: Database>, @@ -34,7 +35,8 @@ struct IndexControllerMeta { impl IndexControllerMeta { fn from_path(path: impl AsRef) -> Result> { - let path = path.as_ref().to_path_buf().push(CONTROLLER_META_FILENAME); + let mut path = path.as_ref().to_path_buf(); + path.push(CONTROLLER_META_FILENAME); if path.exists() { let mut file = File::open(path)?; let mut buffer = Vec::new(); @@ -47,7 +49,8 @@ impl IndexControllerMeta { } fn to_path(self, path: impl AsRef) -> Result<()> { - let path = path.as_ref().to_path_buf().push(CONTROLLER_META_FILENAME); + let mut path = path.as_ref().to_path_buf(); + path.push(CONTROLLER_META_FILENAME); if path.exists() { Err(anyhow::anyhow!("Index controller metadata already exists")) } else { @@ -67,39 +70,24 @@ struct IndexMetadata { } impl IndexMetadata { - fn open_index(&self, path: impl AsRef) -> Result { - let path = path.as_ref().to_path_buf().push("indexes").push(&self.id); - Ok(Index::new(self.options, path)?) + fn open_index(&self, path: impl AsRef) -> Result { + // create a path in the form "db_path/indexes/index_id" + let mut path = path.as_ref().to_path_buf(); + path.push("indexes"); + path.push(&self.id); + Ok(Index::new(self.open_options, path)?) } } struct IndexView<'a, U> { txn: heed::RoTxn<'a>, - index: &'a Index, + index: Ref<'a, String, Index>, update_store: &'a U, } -struct IndexViewMut<'a, U> { - txn: heed::RwTxn<'a, 'a>, - index: &'a Index, - update_store: &'a U, -} - -impl<'a, U> Deref for IndexViewMut<'a, U> { - type Target = IndexView<'a, U>; - - fn deref(&self) -> &Self::Target { - IndexView { - txn: *self.txn, - index: self.index, - update_store: self.update_store, - } - } -} - impl<'a, U: UpdateStore> IndexView<'a, U> { pub fn search(&self, search_query: SearchQuery) -> Result { - let mut search = self.index.search(self.txn); + let mut search = self.index.search(&self.txn); if let Some(query) = &search_query.q { search.query(query); } @@ -115,15 +103,15 @@ impl<'a, U: UpdateStore> IndexView<'a, U> { } pub fn fields_ids_map(&self) -> Result { - self.index.fields_ids_map(self.txn) + Ok(self.index.fields_ids_map(&self.txn)?) } - pub fn fields_displayed_fields_ids(&self) -> Result { - self.index.fields_displayed_fields_ids(self.txn) + pub fn fields_displayed_fields_ids(&self) -> Result>> { + Ok(self.index.displayed_fields_ids(&self.txn)?) } - pub fn documents(&self, ids: &[u32]) -> Result> { - self.index.documents(self.txn, ids) + pub fn documents(&self, ids: Vec) -> Result)>> { + Ok(self.index.documents(&self.txn, ids)?) } } @@ -133,28 +121,29 @@ impl IndexController { pub fn new(path: impl AsRef, update_store: U) -> Result { // If index controller metadata is present, we return the env, otherwise, we create a new // metadata from scratch before returning a new env. - let env = match IndexControllerMeta::from_path(path)? { + let path = path.as_ref().to_path_buf(); + let env = match IndexControllerMeta::from_path(&path)? { Some(meta) => meta.open_options.open(INDEXES_CONTROLLER_FILENAME)?, None => { let open_options = EnvOpenOptions::new() .map_size(page_size::get() * 1000); let env = open_options.open(INDEXES_CONTROLLER_FILENAME)?; let created_at = Utc::now(); - let meta = IndexControllerMeta { open_options, created_at }; + let meta = IndexControllerMeta { open_options: open_options.clone(), created_at }; meta.to_path(path)?; env } }; let indexes = DashMap::new(); - let indexes_db = match env.open_database(INDEXES_DB_NAME)? { + let indexes_db = match env.open_database(Some(INDEXES_DB_NAME))? { Some(indexes_db) => indexes_db, - None => env.create_database(INDEXES_DB_NAME)?, + None => env.create_database(Some(INDEXES_DB_NAME))?, }; - Ok(Self { env, indexes, indexes_db, update_store }) + Ok(Self { env, indexes, indexes_db, update_store, path }) } - pub fn get_or_create>(&mut self, name: S) -> Result> { + pub fn get_or_create>(&mut self, name: S) -> Result> { todo!() } @@ -162,19 +151,26 @@ impl IndexController { /// check for its exixtence in the indexes map, and if it doesn't exist, the index db is check /// for metadata to launch the index. pub fn get>(&self, name: S) -> Result>> { + let update_store = &self.update_store; match self.indexes.get(name.as_ref()) { Some(index) => { let txn = index.read_txn()?; - let update_store = &self.update_store; Ok(Some(IndexView { index, update_store, txn })) } None => { let txn = self.env.read_txn()?; match self.indexes_db.get(&txn, name.as_ref())? { Some(meta) => { - let index = meta.open_index()?; + let index = meta.open_index(self.path)?; self.indexes.insert(name.as_ref().to_owned(), index); - Ok(self.indexes.get(name.as_ref())) + // TODO: create index view + match self.indexes.get(name.as_ref()) { + Some(index) => { + let txn = index.read_txn()?; + Ok(Some(IndexView { index, txn, update_store })) + } + None => Ok(None) + } } None => Ok(None) } @@ -182,7 +178,7 @@ impl IndexController { } } - pub fn get_mut>(&self, name: S) -> Result>> { + pub fn get_mut>(&self, name: S) -> Result>> { todo!() } diff --git a/src/updates/mod.rs b/src/updates/mod.rs index d92a3b16f..faefe7804 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -55,6 +55,8 @@ pub struct UpdateQueue { inner: Arc>, } +impl crate::index_controller::UpdateStore for UpdateQueue {} + impl Deref for UpdateQueue { type Target = Arc>; From 6a3f625e11bc9ab41a2bd01a306b3c8a63ea65cd Mon Sep 17 00:00:00 2001 From: mpostma Date: Sat, 16 Jan 2021 15:09:48 +0100 Subject: [PATCH 05/16] WIP: refactor IndexController change the architecture of the index controller to allow it to own an index store. --- Cargo.lock | 40 ++ Cargo.toml | 1 + src/data/mod.rs | 47 +- src/data/search.rs | 4 +- src/data/updates.rs | 66 ++- src/index_controller/index_store.rs | 255 ++++++++++ src/index_controller/mod.rs | 293 +++++------- src/index_controller/update_store/mod.rs | 49 ++ src/lib.rs | 2 +- src/option.rs | 52 +- src/routes/document.rs | 2 +- src/routes/index.rs | 66 +-- src/routes/settings/mod.rs | 14 +- src/updates/mod.rs | 12 +- src/updates/update_store.rs | 581 +++++++++++++++++++++++ 15 files changed, 1197 insertions(+), 287 deletions(-) create mode 100644 src/index_controller/index_store.rs create mode 100644 src/index_controller/update_store/mod.rs create mode 100644 src/updates/update_store.rs diff --git a/Cargo.lock b/Cargo.lock index e0457b4d5..0282bfe92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,15 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "actix-codec" version = "0.3.0" @@ -1639,6 +1649,7 @@ dependencies = [ "mime", "obkv", "once_cell", + "ouroboros", "page_size", "rand 0.7.3", "rayon", @@ -1941,6 +1952,29 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ouroboros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069fb33e127cabdc8ad6a287eed9719b85c612d36199777f6dc41ad91f7be41a" +dependencies = [ + "ouroboros_macro", + "stable_deref_trait", +] + +[[package]] +name = "ouroboros_macro" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad938cc920f299d6dce91e43d3ce316e785f4aa4bc4243555634dc2967098fc6" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "page_size" version = "0.4.2" @@ -2771,6 +2805,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "standback" version = "0.2.13" diff --git a/Cargo.toml b/Cargo.toml index c06847253..26a3dc1b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ whoami = "1.0.0" dashmap = "4.0.2" page_size = "0.4.2" obkv = "0.1.1" +ouroboros = "0.8.0" [dependencies.sentry] default-features = false diff --git a/src/data/mod.rs b/src/data/mod.rs index 4600531bf..4258278d2 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -8,17 +8,16 @@ use std::sync::Arc; use sha2::Digest; -use crate::{option::Opt, updates::Settings}; -use crate::updates::UpdateQueue; -use crate::index_controller::IndexController; +use crate::{option::Opt, index_controller::Settings}; +use crate::index_controller::{IndexStore, UpdateStore}; #[derive(Clone)] pub struct Data { - inner: Arc, + inner: Arc>, } impl Deref for Data { - type Target = DataInner; + type Target = DataInner; fn deref(&self) -> &Self::Target { &self.inner @@ -26,8 +25,8 @@ impl Deref for Data { } #[derive(Clone)] -pub struct DataInner { - pub indexes: Arc>, +pub struct DataInner { + pub indexes: Arc, api_keys: ApiKeys, options: Opt, } @@ -58,11 +57,10 @@ impl ApiKeys { impl Data { pub fn new(options: Opt) -> anyhow::Result { - let db_size = options.max_mdb_size.get_bytes() as usize; - let indexes = IndexController::new(&options.db_path)?; - let indexes = Arc::new(indexes); - - let update_queue = Arc::new(UpdateQueue::new(&options, indexes.clone())?); + let path = options.db_path.clone(); + let index_store = IndexStore::new(&path)?; + let index_controller = UpdateStore::new(index_store); + let indexes = Arc::new(index_controller); let mut api_keys = ApiKeys { master: options.clone().master_key, @@ -72,31 +70,28 @@ impl Data { api_keys.generate_missing_api_keys(); - let inner = DataInner { indexes, options, update_queue, api_keys }; + let inner = DataInner { indexes, options, api_keys }; let inner = Arc::new(inner); Ok(Data { inner }) } - pub fn settings>(&self, _index: S) -> anyhow::Result { - let txn = self.indexes.env.read_txn()?; - let fields_map = self.indexes.fields_ids_map(&txn)?; - println!("fields_map: {:?}", fields_map); + pub fn settings>(&self, index_uid: S) -> anyhow::Result { + let index = self.indexes + .get(&index_uid)? + .ok_or_else(|| anyhow::anyhow!("Index {} does not exist.", index_uid.as_ref()))?; - let displayed_attributes = self.indexes - .displayed_fields(&txn)? + let displayed_attributes = index + .displayed_fields()? .map(|fields| fields.into_iter().map(String::from).collect()) .unwrap_or_else(|| vec!["*".to_string()]); - let searchable_attributes = self.indexes - .searchable_fields(&txn)? - .map(|fields| fields - .into_iter() - .map(String::from) - .collect()) + let searchable_attributes = index + .searchable_fields()? + .map(|fields| fields.into_iter().map(String::from).collect()) .unwrap_or_else(|| vec!["*".to_string()]); - let faceted_attributes = self.indexes.faceted_fields(&txn)? + let faceted_attributes = index.faceted_fields()? .into_iter() .map(|(k, v)| (k, v.to_string())) .collect(); diff --git a/src/data/search.rs b/src/data/search.rs index 69029d8a9..c30345073 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -107,10 +107,10 @@ impl Data { pub fn search>(&self, index: S, search_query: SearchQuery) -> anyhow::Result { let start = Instant::now(); let index = self.indexes - .get(index)? + .get(&index)? .ok_or_else(|| Error::OpenIndex(format!("Index {} doesn't exists.", index.as_ref())))?; - let Results { found_words, documents_ids, nb_hits, limit, .. } = index.search(search_query)?; + let Results { found_words, documents_ids, nb_hits, limit, .. } = index.search(&search_query)?; let fields_ids_map = index.fields_ids_map()?; diff --git a/src/data/updates.rs b/src/data/updates.rs index 0654f6fd3..507223d41 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -1,18 +1,20 @@ use std::ops::Deref; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; +//use milli::update_store::UpdateStatus; use async_compression::tokio_02::write::GzipEncoder; use futures_util::stream::StreamExt; use tokio::io::AsyncWriteExt; -use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use milli::update_store::UpdateStatus; use super::Data; -use crate::updates::{UpdateMeta, UpdateResult, UpdateStatusResponse, Settings}; +use crate::index_controller::IndexController; +use crate::index_controller::{UpdateStatusResponse, Settings}; + impl Data { pub async fn add_documents( &self, - _index: S, + index: S, method: IndexDocumentsMethod, format: UpdateFormat, mut stream: impl futures::Stream> + Unpin, @@ -20,7 +22,7 @@ impl Data { where B: Deref, E: std::error::Error + Send + Sync + 'static, - S: AsRef, + S: AsRef + Send + Sync + 'static, { let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; let file = tokio::fs::File::from_std(file?); @@ -37,43 +39,39 @@ impl Data { let file = file.into_std().await; let mmap = unsafe { memmap::Mmap::map(&file)? }; - let meta = UpdateMeta::DocumentsAddition { method, format }; - - let queue = self.update_queue.clone(); - let update = tokio::task::spawn_blocking(move || queue.register_update(meta, &mmap[..])).await??; - + let indexes = self.indexes.clone(); + let update = tokio::task::spawn_blocking(move ||indexes.add_documents(index, method, format, &mmap[..])).await??; Ok(update.into()) } - pub async fn update_settings>( + pub async fn update_settings + Send + Sync + 'static>( &self, - _index: S, + index: S, settings: Settings ) -> anyhow::Result { - let meta = UpdateMeta::Settings(settings); - let queue = self.update_queue.clone(); - let update = tokio::task::spawn_blocking(move || queue.register_update(meta, &[])).await??; + let indexes = self.indexes.clone(); + let update = tokio::task::spawn_blocking(move || indexes.update_settings(index, settings)).await??; Ok(update.into()) } - #[inline] - pub fn get_update_status(&self, _index: &str, uid: u64) -> anyhow::Result>> { - self.update_queue.get_update_status(uid) - } + //#[inline] + //pub fn get_update_status>(&self, _index: S, uid: u64) -> anyhow::Result>> { + //self.indexes.get_update_status(uid) + //} - pub fn get_updates_status(&self, _index: &str) -> anyhow::Result>> { - let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| { - let mut metas = processing - .map(UpdateStatus::from) - .into_iter() - .chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - .chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - .chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - .chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - .collect::>(); - metas.sort_by(|a, b| a.id().cmp(&b.id())); - Ok(metas) - })?; - Ok(result) - } + //pub fn get_updates_status(&self, _index: &str) -> anyhow::Result>> { + //let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| { + //let mut metas = processing + //.map(UpdateStatus::from) + //.into_iter() + //.chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.collect::>(); + //metas.sort_by(|a, b| a.id().cmp(&b.id())); + //Ok(metas) + //})?; + //Ok(result) + //} } diff --git a/src/index_controller/index_store.rs b/src/index_controller/index_store.rs new file mode 100644 index 000000000..6ba6621c3 --- /dev/null +++ b/src/index_controller/index_store.rs @@ -0,0 +1,255 @@ +use std::fs::File; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::collections::HashMap; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use dashmap::DashMap; +use heed::types::{Str, SerdeBincode}; +use heed::{EnvOpenOptions, Env, Database}; +use milli::{Index, FieldsIdsMap, SearchResult, FieldId, facet::FacetType}; +use serde::{Serialize, Deserialize}; +use ouroboros::self_referencing; + +use crate::data::SearchQuery; + +const CONTROLLER_META_FILENAME: &str = "index_controller_meta"; +const INDEXES_CONTROLLER_FILENAME: &str = "indexes_db"; +const INDEXES_DB_NAME: &str = "indexes_db"; + + +#[derive(Debug, Serialize, Deserialize)] +struct IndexStoreMeta { + open_options: EnvOpenOptions, + created_at: DateTime, +} + +impl IndexStoreMeta { + fn from_path(path: impl AsRef) -> Result> { + let mut path = path.as_ref().to_path_buf(); + path.push(CONTROLLER_META_FILENAME); + if path.exists() { + let mut file = File::open(path)?; + let mut buffer = Vec::new(); + let n = file.read_to_end(&mut buffer)?; + let meta: IndexStoreMeta = serde_json::from_slice(&buffer[..n])?; + Ok(Some(meta)) + } else { + Ok(None) + } + } + + fn to_path(self, path: impl AsRef) -> Result<()> { + let mut path = path.as_ref().to_path_buf(); + path.push(CONTROLLER_META_FILENAME); + if path.exists() { + Err(anyhow::anyhow!("Index controller metadata already exists")) + } else { + let mut file = File::create(path)?; + let json = serde_json::to_vec(&self)?; + file.write_all(&json)?; + Ok(()) + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct IndexMetadata { + created_at: DateTime, + open_options: EnvOpenOptions, + uuid: String, +} + +impl IndexMetadata { + fn open_index(self, path: impl AsRef) -> Result { + // create a path in the form "db_path/indexes/index_id" + let mut path = path.as_ref().to_path_buf(); + path.push("indexes"); + path.push(&self.uuid); + Ok(Index::new(self.open_options, path)?) + } +} + + +#[self_referencing] +pub struct IndexView { + pub index: Arc, + #[borrows(index)] + #[covariant] + pub txn: heed::RoTxn<'this>, + uuid: String, +} + +impl IndexView { + pub fn search(&self, search_query: &SearchQuery) -> Result { + self.with(|this| { + let mut search = this.index.search(&this.txn); + if let Some(query) = &search_query.q { + search.query(query); + } + + if let Some(offset) = search_query.offset { + search.offset(offset); + } + + let limit = search_query.limit; + search.limit(limit); + + Ok(search.execute()?) + }) + } + + #[inline] + pub fn fields_ids_map(&self) -> Result { + self.with(|this| Ok(this.index.fields_ids_map(&this.txn)?)) + + } + + #[inline] + pub fn displayed_fields_ids(&self) -> Result>> { + self.with(|this| Ok(this.index.displayed_fields_ids(&this.txn)?)) + } + + #[inline] + pub fn displayed_fields(&self) -> Result>> { + self.with(|this| Ok(this.index + .displayed_fields(&this.txn)? + .map(|fields| fields.into_iter().map(String::from).collect()))) + } + + #[inline] + pub fn searchable_fields(&self) -> Result>> { + self.with(|this| Ok(this.index + .searchable_fields(&this.txn)? + .map(|fields| fields.into_iter().map(String::from).collect()))) + } + + #[inline] + pub fn faceted_fields(&self) -> Result> { + self.with(|this| Ok(this.index.faceted_fields(&this.txn)?)) + } + + pub fn documents(&self, ids: &[u32]) -> Result)>> { + let txn = self.borrow_txn(); + let index = self.borrow_index(); + Ok(index.documents(txn, ids.into_iter().copied())?) + } + + //pub async fn add_documents( + //&self, + //method: IndexDocumentsMethod, + //format: UpdateFormat, + //mut stream: impl futures::Stream> + Unpin, + //) -> anyhow::Result + //where + //B: Deref, + //E: std::error::Error + Send + Sync + 'static, + //{ + //let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; + //let file = tokio::fs::File::from_std(file?); + //let mut encoder = GzipEncoder::new(file); + + //while let Some(result) = stream.next().await { + //let bytes = &*result?; + //encoder.write_all(&bytes[..]).await?; + //} + + //encoder.shutdown().await?; + //let mut file = encoder.into_inner(); + //file.sync_all().await?; + //let file = file.into_std().await; + //let mmap = unsafe { memmap::Mmap::map(&file)? }; + + //let meta = UpdateMeta::DocumentsAddition { method, format }; + + //let index = self.index.clone(); + //let queue = self.update_store.clone(); + //let update = tokio::task::spawn_blocking(move || queue.register_update(index, meta, &mmap[..])).await??; + //Ok(update.into()) + //} +} + +pub struct IndexStore { + path: PathBuf, + env: Env, + indexes_db: Database>, + indexes: DashMap)>, +} + +impl IndexStore { + /// Open the index controller from meta found at path, and create a new one if no meta is + /// found. + pub fn new(path: impl AsRef) -> Result { + // If index controller metadata is present, we return the env, otherwise, we create a new + // metadata from scratch before returning a new env. + let path = path.as_ref().to_path_buf(); + let env = match IndexStoreMeta::from_path(&path)? { + Some(meta) => meta.open_options.open(INDEXES_CONTROLLER_FILENAME)?, + None => { + let mut open_options = EnvOpenOptions::new(); + open_options.map_size(page_size::get() * 1000); + let env = open_options.open(INDEXES_CONTROLLER_FILENAME)?; + let created_at = Utc::now(); + let meta = IndexStoreMeta { open_options: open_options.clone(), created_at }; + meta.to_path(&path)?; + env + } + }; + let indexes = DashMap::new(); + let indexes_db = match env.open_database(Some(INDEXES_DB_NAME))? { + Some(indexes_db) => indexes_db, + None => env.create_database(Some(INDEXES_DB_NAME))?, + }; + + Ok(Self { env, indexes, indexes_db, path }) + } + + pub fn get_or_create>(&self, _name: S) -> Result { + todo!() + } + + /// Get an index with read access to the db. The index are lazily loaded, meaning that we first + /// check for its exixtence in the indexes map, and if it doesn't exist, the index db is check + /// for metadata to launch the index. + pub fn get>(&self, name: S) -> Result> { + match self.indexes.get(name.as_ref()) { + Some(entry) => { + let index = entry.1.clone(); + let uuid = entry.0.clone(); + let view = IndexView::try_new(index, |index| index.read_txn(), uuid)?; + Ok(Some(view)) + } + None => { + let txn = self.env.read_txn()?; + match self.indexes_db.get(&txn, name.as_ref())? { + Some(meta) => { + let uuid = meta.uuid.clone(); + let index = Arc::new(meta.open_index(&self.path)?); + self.indexes.insert(name.as_ref().to_owned(), (uuid.clone(), index.clone())); + let view = IndexView::try_new(index, |index| index.read_txn(), uuid)?; + Ok(Some(view)) + } + None => Ok(None) + } + } + } + } + + pub fn get_mut>(&self, _name: S) -> Result> { + todo!() + } + + pub async fn delete_index>(&self, _name:S) -> Result<()> { + todo!() + } + + pub async fn list_indices(&self) -> Result> { + todo!() + } + + pub async fn rename_index(&self, _old: &str, _new: &str) -> Result<()> { + todo!() + } +} diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index a3b29879a..c927a6c5b 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,196 +1,145 @@ -use std::fs::File; -use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; +mod index_store; +mod update_store; + +pub use index_store::IndexStore; +pub use update_store::UpdateStore; + +use std::num::NonZeroUsize; +use std::ops::Deref; +use std::collections::HashMap; use anyhow::Result; -use chrono::{DateTime, Utc}; -use dashmap::DashMap; -use dashmap::mapref::one::Ref; -use heed::types::{Str, SerdeBincode}; -use heed::{EnvOpenOptions, Env, Database}; -use milli::{Index, FieldsIdsMap, SearchResult, FieldId}; -use serde::{Serialize, Deserialize}; +use milli::update::{IndexDocumentsMethod, UpdateFormat}; +use milli::update_store::{Processed, Processing, Failed, Pending, Aborted}; +use serde::{Serialize, Deserialize, de::Deserializer}; -use crate::data::SearchQuery; +pub type UpdateStatusResponse = UpdateStatus; -const CONTROLLER_META_FILENAME: &str = "index_controller_meta"; -const INDEXES_CONTROLLER_FILENAME: &str = "indexes_db"; -const INDEXES_DB_NAME: &str = "indexes_db"; - -pub trait UpdateStore {} - -pub struct IndexController { - path: PathBuf, - update_store: U, - env: Env, - indexes_db: Database>, - indexes: DashMap, +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum UpdateMeta { + DocumentsAddition { method: IndexDocumentsMethod, format: UpdateFormat }, + ClearDocuments, + Settings(Settings), + Facets(Facets), } -#[derive(Debug, Serialize, Deserialize)] -struct IndexControllerMeta { - open_options: EnvOpenOptions, - created_at: DateTime, +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct Facets { + pub level_group_size: Option, + pub min_level_size: Option, } -impl IndexControllerMeta { - fn from_path(path: impl AsRef) -> Result> { - let mut path = path.as_ref().to_path_buf(); - path.push(CONTROLLER_META_FILENAME); - if path.exists() { - let mut file = File::open(path)?; - let mut buffer = Vec::new(); - let n = file.read_to_end(&mut buffer)?; - let meta: IndexControllerMeta = serde_json::from_slice(&buffer[..n])?; - Ok(Some(meta)) - } else { - Ok(None) - } - } +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "type")] +pub enum UpdateStatus { + Pending { update_id: u64, meta: Pending }, + Progressing { update_id: u64, meta: P }, + Processed { update_id: u64, meta: Processed }, + Aborted { update_id: u64, meta: Aborted }, +} - fn to_path(self, path: impl AsRef) -> Result<()> { - let mut path = path.as_ref().to_path_buf(); - path.push(CONTROLLER_META_FILENAME); - if path.exists() { - Err(anyhow::anyhow!("Index controller metadata already exists")) - } else { - let mut file = File::create(path)?; - let json = serde_json::to_vec(&self)?; - file.write_all(&json)?; - Ok(()) +fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> +where T: Deserialize<'de>, + D: Deserializer<'de> +{ + Deserialize::deserialize(deserializer).map(Some) +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +pub struct Settings { + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + pub displayed_attributes: Option>>, + + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + pub searchable_attributes: Option>>, + + #[serde(default)] + pub faceted_attributes: Option>>, + + #[serde( + default, + deserialize_with = "deserialize_some", + skip_serializing_if = "Option::is_none", + )] + pub criteria: Option>>, +} + +impl Settings { + pub fn cleared() -> Self { + Self { + displayed_attributes: Some(None), + searchable_attributes: Some(None), + faceted_attributes: Some(None), + criteria: Some(None), } } } - -#[derive(Debug, Serialize, Deserialize)] -struct IndexMetadata { - created_at: DateTime, - open_options: EnvOpenOptions, - id: String, +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum UpdateResult { + //DocumentsAddition(DocumentAdditionResult), + Other, } -impl IndexMetadata { - fn open_index(&self, path: impl AsRef) -> Result { - // create a path in the form "db_path/indexes/index_id" - let mut path = path.as_ref().to_path_buf(); - path.push("indexes"); - path.push(&self.id); - Ok(Index::new(self.open_options, path)?) - } -} +/// The `IndexController` is in charge of the access to the underlying indices. It splits the logic +/// for read access which is provided, and write access which must be provided. This allows the +/// implementer to define the behaviour of write accesses to the indices, and abstract the +/// scheduling of the updates. The implementer must be able to provide an instance of `IndexStore` +pub trait IndexController: Deref { -struct IndexView<'a, U> { - txn: heed::RoTxn<'a>, - index: Ref<'a, String, Index>, - update_store: &'a U, -} + /* + * Write operations + * + * Logic for the write operation need to be provided by the implementer, since they can be made + * asynchronous thanks to an update_store for example. + * + * */ -impl<'a, U: UpdateStore> IndexView<'a, U> { - pub fn search(&self, search_query: SearchQuery) -> Result { - let mut search = self.index.search(&self.txn); - if let Some(query) = &search_query.q { - search.query(query); - } + /// Perform document addition on the database. If the provided index does not exist, it will be + /// created when the addition is applied to the index. + fn add_documents>( + &self, + index: S, + method: IndexDocumentsMethod, + format: UpdateFormat, + data: &[u8], + ) -> anyhow::Result; - if let Some(offset) = search_query.offset { - search.offset(offset); - } + /// Updates an index settings. If the index does not exist, it will be created when the update + /// is applied to the index. + fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; - let limit = search_query.limit; - search.limit(limit); + /// Create an index with the given `index_uid`. + fn create_index>(&self, index_uid: S) -> Result<()>; - Ok(search.execute()?) - } + /// Delete index with the given `index_uid`, attempting to close it beforehand. + fn delete_index>(&self, index_uid: S) -> Result<()>; - pub fn fields_ids_map(&self) -> Result { - Ok(self.index.fields_ids_map(&self.txn)?) - } + /// Swap two indexes, concretely, it simply swaps the index the names point to. + fn swap_indices, S2: AsRef>(&self, index1_uid: S1, index2_uid: S2) -> Result<()>; - pub fn fields_displayed_fields_ids(&self) -> Result>> { - Ok(self.index.displayed_fields_ids(&self.txn)?) - } - - pub fn documents(&self, ids: Vec) -> Result)>> { - Ok(self.index.documents(&self.txn, ids)?) - } -} - -impl IndexController { - /// Open the index controller from meta found at path, and create a new one if no meta is - /// found. - pub fn new(path: impl AsRef, update_store: U) -> Result { - // If index controller metadata is present, we return the env, otherwise, we create a new - // metadata from scratch before returning a new env. - let path = path.as_ref().to_path_buf(); - let env = match IndexControllerMeta::from_path(&path)? { - Some(meta) => meta.open_options.open(INDEXES_CONTROLLER_FILENAME)?, - None => { - let open_options = EnvOpenOptions::new() - .map_size(page_size::get() * 1000); - let env = open_options.open(INDEXES_CONTROLLER_FILENAME)?; - let created_at = Utc::now(); - let meta = IndexControllerMeta { open_options: open_options.clone(), created_at }; - meta.to_path(path)?; - env - } - }; - let indexes = DashMap::new(); - let indexes_db = match env.open_database(Some(INDEXES_DB_NAME))? { - Some(indexes_db) => indexes_db, - None => env.create_database(Some(INDEXES_DB_NAME))?, - }; - - Ok(Self { env, indexes, indexes_db, update_store, path }) - } - - pub fn get_or_create>(&mut self, name: S) -> Result> { - todo!() - } - - /// Get an index with read access to the db. The index are lazily loaded, meaning that we first - /// check for its exixtence in the indexes map, and if it doesn't exist, the index db is check - /// for metadata to launch the index. - pub fn get>(&self, name: S) -> Result>> { - let update_store = &self.update_store; - match self.indexes.get(name.as_ref()) { - Some(index) => { - let txn = index.read_txn()?; - Ok(Some(IndexView { index, update_store, txn })) - } - None => { - let txn = self.env.read_txn()?; - match self.indexes_db.get(&txn, name.as_ref())? { - Some(meta) => { - let index = meta.open_index(self.path)?; - self.indexes.insert(name.as_ref().to_owned(), index); - // TODO: create index view - match self.indexes.get(name.as_ref()) { - Some(index) => { - let txn = index.read_txn()?; - Ok(Some(IndexView { index, txn, update_store })) - } - None => Ok(None) - } - } - None => Ok(None) - } - } - } - } - - pub fn get_mut>(&self, name: S) -> Result>> { - todo!() - } - - pub async fn delete_index>(&self, name:S) -> Result<()> { - todo!() - } - - pub async fn list_indices(&self) -> Result> { - todo!() - } - - pub async fn rename_index(&self, old: &str, new: &str) -> Result<()> { + /// Apply an update to the given index. This method can be called when an update is ready to be + /// processed + fn handle_update>( + &self, + _index: S, + _update_id: u64, + _meta: Processing, + _content: &[u8] + ) -> Result, Failed> { todo!() } } + diff --git a/src/index_controller/update_store/mod.rs b/src/index_controller/update_store/mod.rs new file mode 100644 index 000000000..84db2f63d --- /dev/null +++ b/src/index_controller/update_store/mod.rs @@ -0,0 +1,49 @@ +use std::ops::Deref; + +use super::{IndexStore, IndexController}; + +pub struct UpdateStore { + index_store: IndexStore, +} + +impl Deref for UpdateStore { + type Target = IndexStore; + + fn deref(&self) -> &Self::Target { + &self.index_store + } +} + +impl UpdateStore { + pub fn new(index_store: IndexStore) -> Self { + Self { index_store } + } +} + +impl IndexController for UpdateStore { + fn add_documents>( + &self, + _index: S, + _method: milli::update::IndexDocumentsMethod, + _format: milli::update::UpdateFormat, + _data: &[u8], + ) -> anyhow::Result { + todo!() + } + + fn update_settings>(&self, _index_uid: S, _settings: crate::index_controller::Settings) -> anyhow::Result { + todo!() + } + + fn create_index>(&self, _index_uid: S) -> anyhow::Result<()> { + todo!() + } + + fn delete_index>(&self, _index_uid: S) -> anyhow::Result<()> { + todo!() + } + + fn swap_indices, S2: AsRef>(&self, _index1_uid: S1, _index2_uid: S2) -> anyhow::Result<()> { + todo!() + } +} diff --git a/src/lib.rs b/src/lib.rs index f5dd79b0b..df9381914 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,7 @@ pub mod error; pub mod helpers; pub mod option; pub mod routes; -mod updates; +//mod updates; mod index_controller; use actix_http::Error; diff --git a/src/option.rs b/src/option.rs index f9e98f4fa..f280553e1 100644 --- a/src/option.rs +++ b/src/option.rs @@ -9,10 +9,60 @@ use rustls::{ AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth, RootCertStore, }; +use grenad::CompressionType; use structopt::StructOpt; -use crate::updates::IndexerOpts; +#[derive(Debug, Clone, StructOpt)] +pub struct IndexerOpts { + /// The amount of documents to skip before printing + /// a log regarding the indexing advancement. + #[structopt(long, default_value = "100000")] // 100k + pub log_every_n: usize, + /// MTBL max number of chunks in bytes. + #[structopt(long)] + pub max_nb_chunks: Option, + + /// The maximum amount of memory to use for the MTBL buffer. It is recommended + /// to use something like 80%-90% of the available memory. + /// + /// It is automatically split by the number of jobs e.g. if you use 7 jobs + /// and 7 GB of max memory, each thread will use a maximum of 1 GB. + #[structopt(long, default_value = "7 GiB")] + pub max_memory: Byte, + + /// Size of the linked hash map cache when indexing. + /// The bigger it is, the faster the indexing is but the more memory it takes. + #[structopt(long, default_value = "500")] + pub linked_hash_map_size: usize, + + /// The name of the compression algorithm to use when compressing intermediate + /// chunks during indexing documents. + /// + /// Choosing a fast algorithm will make the indexing faster but may consume more memory. + #[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])] + pub chunk_compression_type: CompressionType, + + /// The level of compression of the chosen algorithm. + #[structopt(long, requires = "chunk-compression-type")] + pub chunk_compression_level: Option, + + /// The number of bytes to remove from the begining of the chunks while reading/sorting + /// or merging them. + /// + /// File fusing must only be enable on file systems that support the `FALLOC_FL_COLLAPSE_RANGE`, + /// (i.e. ext4 and XFS). File fusing will only work if the `enable-chunk-fusing` is set. + #[structopt(long, default_value = "4 GiB")] + pub chunk_fusing_shrink_size: Byte, + + /// Enable the chunk fusing or not, this reduces the amount of disk used by a factor of 2. + #[structopt(long)] + pub enable_chunk_fusing: bool, + + /// Number of parallel jobs for indexing, defaults to # of CPUs. + #[structopt(long)] + pub indexing_jobs: Option, +} const POSSIBLE_ENV: [&str; 2] = ["development", "production"]; #[derive(Debug, Clone, StructOpt)] diff --git a/src/routes/document.rs b/src/routes/document.rs index 6c5f93991..aeec0e5df 100644 --- a/src/routes/document.rs +++ b/src/routes/document.rs @@ -122,7 +122,7 @@ async fn add_documents_json( ) -> Result { let addition_result = data .add_documents( - &path.index_uid, + path.into_inner().index_uid, IndexDocumentsMethod::UpdateDocuments, UpdateFormat::Json, body diff --git a/src/routes/index.rs b/src/routes/index.rs index 515e771e1..fa4ae0679 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -1,7 +1,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; -use log::error; +//use log::error; use serde::{Deserialize, Serialize}; use crate::Data; @@ -94,8 +94,8 @@ async fn delete_index( #[derive(Deserialize)] struct UpdateParam { - index_uid: String, - update_id: u64, + _index_uid: String, + _update_id: u64, } #[get( @@ -103,39 +103,41 @@ struct UpdateParam { wrap = "Authentication::Private" )] async fn get_update_status( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { - let result = data.get_update_status(&path.index_uid, path.update_id); - match result { - Ok(Some(meta)) => { - let json = serde_json::to_string(&meta).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } - Ok(None) => { - todo!() - } - Err(e) => { - error!("{}", e); - todo!() - } - } + todo!() + //let result = data.get_update_status(&path.index_uid, path.update_id); + //match result { + //Ok(Some(meta)) => { + //let json = serde_json::to_string(&meta).unwrap(); + //Ok(HttpResponse::Ok().body(json)) + //} + //Ok(None) => { + //todo!() + //} + //Err(e) => { + //error!("{}", e); + //todo!() + //} + //} } #[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] async fn get_all_updates_status( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { - let result = data.get_updates_status(&path.index_uid); - match result { - Ok(metas) => { - let json = serde_json::to_string(&metas).unwrap(); - Ok(HttpResponse::Ok().body(json)) - } - Err(e) => { - error!("{}", e); - todo!() - } - } + todo!() + //let result = data.get_updates_status(&path.index_uid); + //match result { + //Ok(metas) => { + //let json = serde_json::to_string(&metas).unwrap(); + //Ok(HttpResponse::Ok().body(json)) + //} + //Err(e) => { + //error!("{}", e); + //todo!() + //} + //} } diff --git a/src/routes/settings/mod.rs b/src/routes/settings/mod.rs index d54ad469c..56c1d34a0 100644 --- a/src/routes/settings/mod.rs +++ b/src/routes/settings/mod.rs @@ -3,7 +3,7 @@ use log::error; use crate::Data; use crate::error::ResponseError; -use crate::updates::Settings; +use crate::index_controller::Settings; use crate::helpers::Authentication; #[macro_export] @@ -15,19 +15,19 @@ macro_rules! make_setting_route { use crate::data; use crate::error::ResponseError; use crate::helpers::Authentication; - use crate::updates::Settings; + use crate::index_controller::Settings; #[actix_web::delete($route, wrap = "Authentication::Private")] pub async fn delete( data: web::Data, index_uid: web::Path, ) -> Result { - use crate::updates::Settings; + use crate::index_controller::Settings; let settings = Settings { $attr: Some(None), ..Default::default() }; - match data.update_settings(index_uid.as_ref(), settings).await { + match data.update_settings(index_uid.into_inner(), settings).await { Ok(update_status) => { let json = serde_json::to_string(&update_status).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -50,7 +50,7 @@ macro_rules! make_setting_route { ..Default::default() }; - match data.update_settings(index_uid.as_ref(), settings).await { + match data.update_settings(index_uid.into_inner(), settings).await { Ok(update_status) => { let json = serde_json::to_string(&update_status).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -141,7 +141,7 @@ async fn update_all( index_uid: web::Path, body: web::Json, ) -> Result { - match data.update_settings(index_uid.as_ref(), body.into_inner()).await { + match data.update_settings(index_uid.into_inner(), body.into_inner()).await { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) @@ -176,7 +176,7 @@ async fn delete_all( index_uid: web::Path, ) -> Result { let settings = Settings::cleared(); - match data.update_settings(index_uid.as_ref(), settings).await { + match data.update_settings(index_uid.into_inner(), settings).await { Ok(update_result) => { let json = serde_json::to_string(&update_result).unwrap(); Ok(HttpResponse::Ok().body(json)) diff --git a/src/updates/mod.rs b/src/updates/mod.rs index faefe7804..7e0802595 100644 --- a/src/updates/mod.rs +++ b/src/updates/mod.rs @@ -1,10 +1,10 @@ mod settings; +mod update_store; pub use settings::{Settings, Facets}; use std::io; use std::sync::Arc; -use std::ops::Deref; use std::fs::create_dir_all; use std::collections::HashMap; @@ -55,16 +55,6 @@ pub struct UpdateQueue { inner: Arc>, } -impl crate::index_controller::UpdateStore for UpdateQueue {} - -impl Deref for UpdateQueue { - type Target = Arc>; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - #[derive(Debug, Clone, StructOpt)] pub struct IndexerOpts { /// The amount of documents to skip before printing diff --git a/src/updates/update_store.rs b/src/updates/update_store.rs new file mode 100644 index 000000000..f750fc38b --- /dev/null +++ b/src/updates/update_store.rs @@ -0,0 +1,581 @@ +use std::path::Path; +use std::sync::{Arc, RwLock}; + +use crossbeam_channel::Sender; +use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; +use heed::{EnvOpenOptions, Env, Database}; +use serde::{Serialize, Deserialize}; +use chrono::{DateTime, Utc}; + +type BEU64 = heed::zerocopy::U64; + +#[derive(Clone)] +pub struct UpdateStore { + env: Env, + pending_meta: Database, SerdeJson>>, + pending: Database, ByteSlice>, + processed_meta: Database, SerdeJson>>, + failed_meta: Database, SerdeJson>>, + aborted_meta: Database, SerdeJson>>, + processing: Arc>>>, + notification_sender: Sender<()>, +} + +pub trait UpdateHandler { + fn handle_update(&mut self, update_id: u64, meta: Processing, content: &[u8]) -> Result, Failed>; +} + +impl UpdateHandler for F +where F: FnMut(u64, Processing, &[u8]) -> Result, Failed> + Send + 'static { + fn handle_update(&mut self, update_id: u64, meta: Processing, content: &[u8]) -> Result, Failed> { + self(update_id, meta, content) + } +} + +impl UpdateStore { + pub fn open( + size: Option, + path: P, + mut update_handler: U, + ) -> heed::Result>> + where + P: AsRef, + U: UpdateHandler + Send + 'static, + M: for<'a> Deserialize<'a> + Serialize + Send + Sync + Clone, + N: Serialize, + E: Serialize, + { + let mut options = EnvOpenOptions::new(); + if let Some(size) = size { + options.map_size(size); + } + options.max_dbs(5); + + let env = options.open(path)?; + let pending_meta = env.create_database(Some("pending-meta"))?; + let pending = env.create_database(Some("pending"))?; + let processed_meta = env.create_database(Some("processed-meta"))?; + let aborted_meta = env.create_database(Some("aborted-meta"))?; + let failed_meta = env.create_database(Some("failed-meta"))?; + let processing = Arc::new(RwLock::new(None)); + + let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); + // Send a first notification to trigger the process. + let _ = notification_sender.send(()); + + let update_store = Arc::new(UpdateStore { + env, + pending, + pending_meta, + processed_meta, + aborted_meta, + notification_sender, + failed_meta, + processing, + }); + + let update_store_cloned = update_store.clone(); + std::thread::spawn(move || { + // Block and wait for something to process. + for () in notification_receiver { + loop { + match update_store_cloned.process_pending_update(&mut update_handler) { + Ok(Some(_)) => (), + Ok(None) => break, + Err(e) => eprintln!("error while processing update: {}", e), + } + } + } + }); + + Ok(update_store) + } + + /// Returns the new biggest id to use to store the new update. + fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { + let last_pending = self.pending_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_processed = self.processed_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_aborted = self.aborted_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_update_id = [last_pending, last_processed, last_aborted] + .iter() + .copied() + .flatten() + .max(); + + match last_update_id { + Some(last_id) => Ok(last_id + 1), + None => Ok(0), + } + } + + /// Registers the update content in the pending store and the meta + /// into the pending-meta store. Returns the new unique update id. + pub fn register_update(&self, meta: M, content: &[u8]) -> heed::Result> + where M: Serialize, + { + let mut wtxn = self.env.write_txn()?; + + // We ask the update store to give us a new update id, this is safe, + // no other update can have the same id because we use a write txn before + // asking for the id and registering it so other update registering + // will be forced to wait for a new write txn. + let update_id = self.new_update_id(&wtxn)?; + let update_key = BEU64::new(update_id); + + let meta = Pending::new(meta, update_id); + self.pending_meta.put(&mut wtxn, &update_key, &meta)?; + self.pending.put(&mut wtxn, &update_key, content)?; + + wtxn.commit()?; + + if let Err(e) = self.notification_sender.try_send(()) { + assert!(!e.is_disconnected(), "update notification channel is disconnected"); + } + Ok(meta) + } + /// Executes the user provided function on the next pending update (the one with the lowest id). + /// This is asynchronous as it let the user process the update with a read-only txn and + /// only writing the result meta to the processed-meta store *after* it has been processed. + fn process_pending_update(&self, handler: &mut U) -> heed::Result> + where + U: UpdateHandler, + M: for<'a> Deserialize<'a> + Serialize + Clone, + N: Serialize, + E: Serialize, + { + // Create a read transaction to be able to retrieve the pending update in order. + let rtxn = self.env.read_txn()?; + let first_meta = self.pending_meta.first(&rtxn)?; + + // If there is a pending update we process and only keep + // a reader while processing it, not a writer. + match first_meta { + Some((first_id, pending)) => { + let first_content = self.pending + .get(&rtxn, &first_id)? + .expect("associated update content"); + + // we cahnge the state of the update from pending to processing before we pass it + // to the update handler. Processing store is non persistent to be able recover + // from a failure + let processing = pending.processing(); + self.processing + .write() + .unwrap() + .replace(processing.clone()); + // Process the pending update using the provided user function. + let result = handler.handle_update(first_id.get(), processing, first_content); + drop(rtxn); + + // Once the pending update have been successfully processed + // we must remove the content from the pending and processing stores and + // write the *new* meta to the processed-meta store and commit. + let mut wtxn = self.env.write_txn()?; + self.processing + .write() + .unwrap() + .take(); + self.pending_meta.delete(&mut wtxn, &first_id)?; + self.pending.delete(&mut wtxn, &first_id)?; + match result { + Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, + Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, + } + wtxn.commit()?; + + Ok(Some(())) + }, + None => Ok(None) + } + } + + /// The id and metadata of the update that is currently being processed, + /// `None` if no update is being processed. + pub fn processing_update(&self) -> heed::Result)>> + where M: for<'a> Deserialize<'a>, + { + let rtxn = self.env.read_txn()?; + match self.pending_meta.first(&rtxn)? { + Some((key, meta)) => Ok(Some((key.get(), meta))), + None => Ok(None), + } + } + + /// Execute the user defined function with the meta-store iterators, the first + /// iterator is the *processed* meta one, the second the *aborted* meta one + /// and, the last is the *pending* meta one. + pub fn iter_metas(&self, mut f: F) -> heed::Result + where + M: for<'a> Deserialize<'a> + Clone, + N: for<'a> Deserialize<'a>, + F: for<'a> FnMut( + Option>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + ) -> heed::Result, + { + let rtxn = self.env.read_txn()?; + + // We get the pending, processed and aborted meta iterators. + let processed_iter = self.processed_meta.iter(&rtxn)?; + let aborted_iter = self.aborted_meta.iter(&rtxn)?; + let pending_iter = self.pending_meta.iter(&rtxn)?; + let processing = self.processing.read().unwrap().clone(); + let failed_iter = self.failed_meta.iter(&rtxn)?; + + // We execute the user defined function with both iterators. + (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) + } + + /// Returns the update associated meta or `None` if the update doesn't exist. + pub fn meta(&self, update_id: u64) -> heed::Result>> + where + M: for<'a> Deserialize<'a> + Clone, + N: for<'a> Deserialize<'a>, + E: for<'a> Deserialize<'a>, + { + let rtxn = self.env.read_txn()?; + let key = BEU64::new(update_id); + + if let Some(ref meta) = *self.processing.read().unwrap() { + if meta.id() == update_id { + return Ok(Some(UpdateStatus::Processing(meta.clone()))); + } + } + + println!("pending"); + if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Pending(meta))); + } + + println!("processed"); + if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Processed(meta))); + } + + if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Aborted(meta))); + } + + if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Failed(meta))); + } + + Ok(None) + } + + /// Aborts an update, an aborted update content is deleted and + /// the meta of it is moved into the aborted updates database. + /// + /// Trying to abort an update that is currently being processed, an update + /// that as already been processed or which doesn't actually exist, will + /// return `None`. + pub fn abort_update(&self, update_id: u64) -> heed::Result>> + where M: Serialize + for<'a> Deserialize<'a>, + { + let mut wtxn = self.env.write_txn()?; + let key = BEU64::new(update_id); + + // We cannot abort an update that is currently being processed. + if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { + return Ok(None); + } + + let pending = match self.pending_meta.get(&wtxn, &key)? { + Some(meta) => meta, + None => return Ok(None), + }; + + let aborted = pending.abort(); + + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + + wtxn.commit()?; + + Ok(Some(aborted)) + } + + /// Aborts all the pending updates, and not the one being currently processed. + /// Returns the update metas and ids that were successfully aborted. + pub fn abort_pendings(&self) -> heed::Result)>> + where M: Serialize + for<'a> Deserialize<'a>, + { + let mut wtxn = self.env.write_txn()?; + let mut aborted_updates = Vec::new(); + + // We skip the first pending update as it is currently being processed. + for result in self.pending_meta.iter(&wtxn)?.skip(1) { + let (key, pending) = result?; + let id = key.get(); + aborted_updates.push((id, pending.abort())); + } + + for (id, aborted) in &aborted_updates { + let key = BEU64::new(*id); + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + } + + wtxn.commit()?; + + Ok(aborted_updates) + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Pending { + update_id: u64, + meta: M, + enqueued_at: DateTime, +} + +impl Pending { + fn new(meta: M, update_id: u64) -> Self { + Self { + enqueued_at: Utc::now(), + meta, + update_id, + } + } + + pub fn processing(self) -> Processing { + Processing { + from: self, + started_processing_at: Utc::now(), + } + } + + pub fn abort(self) -> Aborted { + Aborted { + from: self, + aborted_at: Utc::now(), + } + } + + pub fn meta(&self) -> &M { + &self.meta + } + + pub fn id(&self) -> u64 { + self.update_id + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Processed { + success: N, + processed_at: DateTime, + #[serde(flatten)] + from: Processing, +} + +impl Processed { + fn id(&self) -> u64 { + self.from.id() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Processing { + #[serde(flatten)] + from: Pending, + started_processing_at: DateTime, +} + +impl Processing { + pub fn id(&self) -> u64 { + self.from.id() + } + + pub fn meta(&self) -> &M { + self.from.meta() + } + + pub fn process(self, meta: N) -> Processed { + Processed { + success: meta, + from: self, + processed_at: Utc::now(), + } + } + + pub fn fail(self, error: E) -> Failed { + Failed { + from: self, + error, + failed_at: Utc::now(), + } + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Aborted { + #[serde(flatten)] + from: Pending, + aborted_at: DateTime, +} + +impl Aborted { + fn id(&self) -> u64 { + self.from.id() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Failed { + #[serde(flatten)] + from: Processing, + error: E, + failed_at: DateTime, +} + +impl Failed { + fn id(&self) -> u64 { + self.from.id() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize)] +#[serde(tag = "status")] +pub enum UpdateStatus { + Processing(Processing), + Pending(Pending), + Processed(Processed), + Aborted(Aborted), + Failed(Failed), +} + +impl UpdateStatus { + pub fn id(&self) -> u64 { + match self { + UpdateStatus::Processing(u) => u.id(), + UpdateStatus::Pending(u) => u.id(), + UpdateStatus::Processed(u) => u.id(), + UpdateStatus::Aborted(u) => u.id(), + UpdateStatus::Failed(u) => u.id(), + } + } +} + +impl From> for UpdateStatus { + fn from(other: Pending) -> Self { + Self::Pending(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Aborted) -> Self { + Self::Aborted(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Processed) -> Self { + Self::Processed(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Processing) -> Self { + Self::Processing(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Failed) -> Self { + Self::Failed(other) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + use std::time::{Duration, Instant}; + + #[test] + fn simple() { + let dir = tempfile::tempdir().unwrap(); + let update_store = UpdateStore::open(None, dir, |_id, meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + let new_meta = meta.meta().to_string() + " processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let meta = String::from("kiki"); + let update = update_store.register_update(meta, &[]).unwrap(); + thread::sleep(Duration::from_millis(100)); + let meta = update_store.meta(update.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + } + + #[test] + #[ignore] + fn long_running_update() { + let dir = tempfile::tempdir().unwrap(); + let update_store = UpdateStore::open(None, dir, |_id, meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { + thread::sleep(Duration::from_millis(400)); + let new_meta = meta.meta().to_string() + "processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let before_register = Instant::now(); + + let meta = String::from("kiki"); + let update_kiki = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("coco"); + let update_coco = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("cucu"); + let update_cucu = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + thread::sleep(Duration::from_millis(400 * 3 + 100)); + + let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "coco processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "cucu processed"); + } else { + panic!() + } + } +} From 74410d8c6b0e20c79df179a26f228ad058b93e3e Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 14:12:34 +0100 Subject: [PATCH 06/16] architecture rework --- Cargo.lock | 24 +- Cargo.toml | 1 + src/data/mod.rs | 31 +- src/data/search.rs | 107 +++--- src/data/updates.rs | 11 +- src/index_controller/_update_store/mod.rs | 17 + src/index_controller/index_store.rs | 181 ---------- .../local_index_controller/index_store.rs | 188 +++++++++++ .../local_index_controller/mod.rs | 57 ++++ .../local_index_controller/update_handler.rs | 206 ++++++++++++ .../local_index_controller/update_store.rs | 311 ++++++++++++++++++ src/index_controller/mod.rs | 25 +- src/index_controller/update_store/mod.rs | 49 --- src/index_controller/updates.rs | 167 ++++++++++ 14 files changed, 1065 insertions(+), 310 deletions(-) create mode 100644 src/index_controller/_update_store/mod.rs create mode 100644 src/index_controller/local_index_controller/index_store.rs create mode 100644 src/index_controller/local_index_controller/mod.rs create mode 100644 src/index_controller/local_index_controller/update_handler.rs create mode 100644 src/index_controller/local_index_controller/update_store.rs delete mode 100644 src/index_controller/update_store/mod.rs create mode 100644 src/index_controller/updates.rs diff --git a/Cargo.lock b/Cargo.lock index 0282bfe92..5ceaef691 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1137,6 +1137,17 @@ dependencies = [ "wasi 0.9.0+wasi-snapshot-preview1", ] +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", +] + [[package]] name = "gimli" version = "0.23.0" @@ -1608,7 +1619,7 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "meilisearch-error" -version = "0.18.0" +version = "0.18.1" dependencies = [ "actix-http", ] @@ -1669,6 +1680,7 @@ dependencies = [ "tempfile", "tokio", "ureq", + "uuid", "vergen", "walkdir", "whoami", @@ -2263,7 +2275,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom", + "getrandom 0.1.15", "libc", "rand_chacha", "rand_core 0.5.1", @@ -2302,7 +2314,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom", + "getrandom 0.1.15", ] [[package]] @@ -3365,11 +3377,11 @@ checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa" [[package]] name = "uuid" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "rand 0.7.3", + "getrandom 0.2.2", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index 26a3dc1b8..eb2fca43d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ dashmap = "4.0.2" page_size = "0.4.2" obkv = "0.1.1" ouroboros = "0.8.0" +uuid = "0.8.2" [dependencies.sentry] default-features = false diff --git a/src/data/mod.rs b/src/data/mod.rs index 4258278d2..7494792bc 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -5,19 +5,20 @@ pub use search::{SearchQuery, SearchResult}; use std::ops::Deref; use std::sync::Arc; +use std::fs::create_dir_all; use sha2::Digest; use crate::{option::Opt, index_controller::Settings}; -use crate::index_controller::{IndexStore, UpdateStore}; +use crate::index_controller::{IndexController, LocalIndexController}; #[derive(Clone)] pub struct Data { - inner: Arc>, + inner: Arc, } impl Deref for Data { - type Target = DataInner; + type Target = DataInner; fn deref(&self) -> &Self::Target { &self.inner @@ -25,8 +26,8 @@ impl Deref for Data { } #[derive(Clone)] -pub struct DataInner { - pub indexes: Arc, +pub struct DataInner { + pub index_controller: Arc, api_keys: ApiKeys, options: Opt, } @@ -58,8 +59,9 @@ impl ApiKeys { impl Data { pub fn new(options: Opt) -> anyhow::Result { let path = options.db_path.clone(); - let index_store = IndexStore::new(&path)?; - let index_controller = UpdateStore::new(index_store); + let indexer_opts = options.indexer_options.clone(); + create_dir_all(&path)?; + let index_controller = LocalIndexController::new(&path, indexer_opts)?; let indexes = Arc::new(index_controller); let mut api_keys = ApiKeys { @@ -70,28 +72,31 @@ impl Data { api_keys.generate_missing_api_keys(); - let inner = DataInner { indexes, options, api_keys }; + let inner = DataInner { index_controller: indexes, options, api_keys }; let inner = Arc::new(inner); Ok(Data { inner }) } pub fn settings>(&self, index_uid: S) -> anyhow::Result { - let index = self.indexes - .get(&index_uid)? + let index = self.index_controller + .index(&index_uid)? .ok_or_else(|| anyhow::anyhow!("Index {} does not exist.", index_uid.as_ref()))?; + let txn = index.read_txn()?; + let displayed_attributes = index - .displayed_fields()? + .displayed_fields(&txn)? .map(|fields| fields.into_iter().map(String::from).collect()) .unwrap_or_else(|| vec!["*".to_string()]); let searchable_attributes = index - .searchable_fields()? + .searchable_fields(&txn)? .map(|fields| fields.into_iter().map(String::from).collect()) .unwrap_or_else(|| vec!["*".to_string()]); - let faceted_attributes = index.faceted_fields()? + let faceted_attributes = index + .faceted_fields(&txn)? .into_iter() .map(|(k, v)| (k, v.to_string())) .collect(); diff --git a/src/data/search.rs b/src/data/search.rs index c30345073..246e3bdac 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -4,11 +4,11 @@ use std::time::Instant; use serde_json::{Value, Map}; use serde::{Deserialize, Serialize}; -use milli::{SearchResult as Results, obkv_to_json}; +use milli::{Index, obkv_to_json, FacetCondition}; use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; +use anyhow::bail; -use crate::error::Error; - +use crate::index_controller::IndexController; use super::Data; const DEFAULT_SEARCH_LIMIT: usize = 20; @@ -26,11 +26,68 @@ pub struct SearchQuery { pub attributes_to_retrieve: Option>, pub attributes_to_crop: Option>, pub crop_length: Option, - pub attributes_to_highlight: Option>, + pub attributes_to_highlight: Option>, pub filters: Option, pub matches: Option, pub facet_filters: Option, pub facets_distribution: Option>, + pub facet_condition: Option, +} + +impl SearchQuery { + pub fn perform(&self, index: impl AsRef) -> anyhow::Result{ + let index = index.as_ref(); + + let before_search = Instant::now(); + let rtxn = index.read_txn().unwrap(); + + let mut search = index.search(&rtxn); + + if let Some(ref query) = self.q { + search.query(query); + } + + if let Some(ref condition) = self.facet_condition { + if !condition.trim().is_empty() { + let condition = FacetCondition::from_str(&rtxn, &index, &condition).unwrap(); + search.facet_condition(condition); + } + } + + if let Some(offset) = self.offset { + search.offset(offset); + } + + let milli::SearchResult { documents_ids, found_words, nb_hits, limit, } = search.execute()?; + + let mut documents = Vec::new(); + let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); + + let displayed_fields = match index.displayed_fields_ids(&rtxn).unwrap() { + Some(fields) => fields, + None => fields_ids_map.iter().map(|(id, _)| id).collect(), + }; + + let stop_words = fst::Set::default(); + let highlighter = Highlighter::new(&stop_words); + + for (_id, obkv) in index.documents(&rtxn, documents_ids).unwrap() { + let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv).unwrap(); + if let Some(ref attributes_to_highlight) = self.attributes_to_highlight { + highlighter.highlight_record(&mut object, &found_words, attributes_to_highlight); + } + documents.push(object); + } + + Ok(SearchResult { + hits: documents, + nb_hits, + query: self.q.clone().unwrap_or_default(), + limit, + offset: self.offset.unwrap_or_default(), + processing_time_ms: before_search.elapsed().as_millis(), + }) + } } #[derive(Serialize)] @@ -105,45 +162,9 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> { impl Data { pub fn search>(&self, index: S, search_query: SearchQuery) -> anyhow::Result { - let start = Instant::now(); - let index = self.indexes - .get(&index)? - .ok_or_else(|| Error::OpenIndex(format!("Index {} doesn't exists.", index.as_ref())))?; - - let Results { found_words, documents_ids, nb_hits, limit, .. } = index.search(&search_query)?; - - let fields_ids_map = index.fields_ids_map()?; - - let displayed_fields = match index.displayed_fields_ids()? { - Some(fields) => fields, - None => fields_ids_map.iter().map(|(id, _)| id).collect(), - }; - - let attributes_to_highlight = match search_query.attributes_to_highlight { - Some(fields) => fields.iter().map(ToOwned::to_owned).collect(), - None => HashSet::new(), - }; - - let stop_words = fst::Set::default(); - let highlighter = Highlighter::new(&stop_words); - let mut documents = Vec::new(); - for (_id, obkv) in index.documents(&documents_ids)? { - let mut object = obkv_to_json(&displayed_fields, &fields_ids_map, obkv).unwrap(); - highlighter.highlight_record(&mut object, &found_words, &attributes_to_highlight); - documents.push(object); + match self.index_controller.index(&index)? { + Some(index) => Ok(search_query.perform(index)?), + None => bail!("index {:?} doesn't exists", index.as_ref()), } - - let processing_time_ms = start.elapsed().as_millis(); - - let result = SearchResult { - hits: documents, - nb_hits, - query: search_query.q.unwrap_or_default(), - offset: search_query.offset.unwrap_or(0), - limit, - processing_time_ms, - }; - - Ok(result) } } diff --git a/src/data/updates.rs b/src/data/updates.rs index 507223d41..d05617361 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -1,15 +1,12 @@ use std::ops::Deref; use milli::update::{IndexDocumentsMethod, UpdateFormat}; -//use milli::update_store::UpdateStatus; use async_compression::tokio_02::write::GzipEncoder; use futures_util::stream::StreamExt; use tokio::io::AsyncWriteExt; use super::Data; -use crate::index_controller::IndexController; -use crate::index_controller::{UpdateStatusResponse, Settings}; - +use crate::index_controller::{IndexController, UpdateStatusResponse, Settings}; impl Data { pub async fn add_documents( @@ -39,8 +36,8 @@ impl Data { let file = file.into_std().await; let mmap = unsafe { memmap::Mmap::map(&file)? }; - let indexes = self.indexes.clone(); - let update = tokio::task::spawn_blocking(move ||indexes.add_documents(index, method, format, &mmap[..])).await??; + let index_controller = self.index_controller.clone(); + let update = tokio::task::spawn_blocking(move ||index_controller.add_documents(index, method, format, &mmap[..])).await??; Ok(update.into()) } @@ -49,7 +46,7 @@ impl Data { index: S, settings: Settings ) -> anyhow::Result { - let indexes = self.indexes.clone(); + let indexes = self.index_controller.clone(); let update = tokio::task::spawn_blocking(move || indexes.update_settings(index, settings)).await??; Ok(update.into()) } diff --git a/src/index_controller/_update_store/mod.rs b/src/index_controller/_update_store/mod.rs new file mode 100644 index 000000000..ef8711ded --- /dev/null +++ b/src/index_controller/_update_store/mod.rs @@ -0,0 +1,17 @@ +use std::sync::Arc; + +use heed::Env; + +use super::IndexStore; + +pub struct UpdateStore { + env: Env, + index_store: Arc, +} + +impl UpdateStore { + pub fn new(env: Env, index_store: Arc) -> anyhow::Result { + Ok(Self { env, index_store }) + } +} + diff --git a/src/index_controller/index_store.rs b/src/index_controller/index_store.rs index 6ba6621c3..6652086f9 100644 --- a/src/index_controller/index_store.rs +++ b/src/index_controller/index_store.rs @@ -1,78 +1,12 @@ -use std::fs::File; -use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; use std::sync::Arc; use std::collections::HashMap; use anyhow::Result; -use chrono::{DateTime, Utc}; -use dashmap::DashMap; -use heed::types::{Str, SerdeBincode}; -use heed::{EnvOpenOptions, Env, Database}; use milli::{Index, FieldsIdsMap, SearchResult, FieldId, facet::FacetType}; -use serde::{Serialize, Deserialize}; use ouroboros::self_referencing; use crate::data::SearchQuery; -const CONTROLLER_META_FILENAME: &str = "index_controller_meta"; -const INDEXES_CONTROLLER_FILENAME: &str = "indexes_db"; -const INDEXES_DB_NAME: &str = "indexes_db"; - - -#[derive(Debug, Serialize, Deserialize)] -struct IndexStoreMeta { - open_options: EnvOpenOptions, - created_at: DateTime, -} - -impl IndexStoreMeta { - fn from_path(path: impl AsRef) -> Result> { - let mut path = path.as_ref().to_path_buf(); - path.push(CONTROLLER_META_FILENAME); - if path.exists() { - let mut file = File::open(path)?; - let mut buffer = Vec::new(); - let n = file.read_to_end(&mut buffer)?; - let meta: IndexStoreMeta = serde_json::from_slice(&buffer[..n])?; - Ok(Some(meta)) - } else { - Ok(None) - } - } - - fn to_path(self, path: impl AsRef) -> Result<()> { - let mut path = path.as_ref().to_path_buf(); - path.push(CONTROLLER_META_FILENAME); - if path.exists() { - Err(anyhow::anyhow!("Index controller metadata already exists")) - } else { - let mut file = File::create(path)?; - let json = serde_json::to_vec(&self)?; - file.write_all(&json)?; - Ok(()) - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct IndexMetadata { - created_at: DateTime, - open_options: EnvOpenOptions, - uuid: String, -} - -impl IndexMetadata { - fn open_index(self, path: impl AsRef) -> Result { - // create a path in the form "db_path/indexes/index_id" - let mut path = path.as_ref().to_path_buf(); - path.push("indexes"); - path.push(&self.uuid); - Ok(Index::new(self.open_options, path)?) - } -} - - #[self_referencing] pub struct IndexView { pub index: Arc, @@ -136,120 +70,5 @@ impl IndexView { let index = self.borrow_index(); Ok(index.documents(txn, ids.into_iter().copied())?) } - - //pub async fn add_documents( - //&self, - //method: IndexDocumentsMethod, - //format: UpdateFormat, - //mut stream: impl futures::Stream> + Unpin, - //) -> anyhow::Result - //where - //B: Deref, - //E: std::error::Error + Send + Sync + 'static, - //{ - //let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; - //let file = tokio::fs::File::from_std(file?); - //let mut encoder = GzipEncoder::new(file); - - //while let Some(result) = stream.next().await { - //let bytes = &*result?; - //encoder.write_all(&bytes[..]).await?; - //} - - //encoder.shutdown().await?; - //let mut file = encoder.into_inner(); - //file.sync_all().await?; - //let file = file.into_std().await; - //let mmap = unsafe { memmap::Mmap::map(&file)? }; - - //let meta = UpdateMeta::DocumentsAddition { method, format }; - - //let index = self.index.clone(); - //let queue = self.update_store.clone(); - //let update = tokio::task::spawn_blocking(move || queue.register_update(index, meta, &mmap[..])).await??; - //Ok(update.into()) - //} } -pub struct IndexStore { - path: PathBuf, - env: Env, - indexes_db: Database>, - indexes: DashMap)>, -} - -impl IndexStore { - /// Open the index controller from meta found at path, and create a new one if no meta is - /// found. - pub fn new(path: impl AsRef) -> Result { - // If index controller metadata is present, we return the env, otherwise, we create a new - // metadata from scratch before returning a new env. - let path = path.as_ref().to_path_buf(); - let env = match IndexStoreMeta::from_path(&path)? { - Some(meta) => meta.open_options.open(INDEXES_CONTROLLER_FILENAME)?, - None => { - let mut open_options = EnvOpenOptions::new(); - open_options.map_size(page_size::get() * 1000); - let env = open_options.open(INDEXES_CONTROLLER_FILENAME)?; - let created_at = Utc::now(); - let meta = IndexStoreMeta { open_options: open_options.clone(), created_at }; - meta.to_path(&path)?; - env - } - }; - let indexes = DashMap::new(); - let indexes_db = match env.open_database(Some(INDEXES_DB_NAME))? { - Some(indexes_db) => indexes_db, - None => env.create_database(Some(INDEXES_DB_NAME))?, - }; - - Ok(Self { env, indexes, indexes_db, path }) - } - - pub fn get_or_create>(&self, _name: S) -> Result { - todo!() - } - - /// Get an index with read access to the db. The index are lazily loaded, meaning that we first - /// check for its exixtence in the indexes map, and if it doesn't exist, the index db is check - /// for metadata to launch the index. - pub fn get>(&self, name: S) -> Result> { - match self.indexes.get(name.as_ref()) { - Some(entry) => { - let index = entry.1.clone(); - let uuid = entry.0.clone(); - let view = IndexView::try_new(index, |index| index.read_txn(), uuid)?; - Ok(Some(view)) - } - None => { - let txn = self.env.read_txn()?; - match self.indexes_db.get(&txn, name.as_ref())? { - Some(meta) => { - let uuid = meta.uuid.clone(); - let index = Arc::new(meta.open_index(&self.path)?); - self.indexes.insert(name.as_ref().to_owned(), (uuid.clone(), index.clone())); - let view = IndexView::try_new(index, |index| index.read_txn(), uuid)?; - Ok(Some(view)) - } - None => Ok(None) - } - } - } - } - - pub fn get_mut>(&self, _name: S) -> Result> { - todo!() - } - - pub async fn delete_index>(&self, _name:S) -> Result<()> { - todo!() - } - - pub async fn list_indices(&self) -> Result> { - todo!() - } - - pub async fn rename_index(&self, _old: &str, _new: &str) -> Result<()> { - todo!() - } -} diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs new file mode 100644 index 000000000..bfe63459a --- /dev/null +++ b/src/index_controller/local_index_controller/index_store.rs @@ -0,0 +1,188 @@ +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use dashmap::DashMap; +use dashmap::mapref::entry::Entry; +use heed::{Env, EnvOpenOptions, Database, types::{Str, SerdeJson, ByteSlice}, RoTxn, RwTxn}; +use milli::Index; +use rayon::ThreadPool; +use uuid::Uuid; +use serde::{Serialize, Deserialize}; + +use super::update_store::UpdateStore; +use super::update_handler::UpdateHandler; +use crate::option::IndexerOpts; + +#[derive(Serialize, Deserialize, Debug)] +struct IndexMeta { + update_size: usize, + index_size: usize, + uid: Uuid, +} + +impl IndexMeta { + fn open( + &self, + path: impl AsRef, + thread_pool: Arc, + opt: &IndexerOpts, + ) -> anyhow::Result<(Arc, Arc)> { + let update_path = make_update_db_path(&path, &self.uid); + let index_path = make_index_db_path(&path, &self.uid); + + let mut options = EnvOpenOptions::new(); + options.map_size(self.index_size); + let index = Arc::new(Index::new(options, index_path)?); + + let mut options = EnvOpenOptions::new(); + options.map_size(self.update_size); + let handler = UpdateHandler::new(opt, index.clone(), thread_pool)?; + let update_store = UpdateStore::open(options, update_path, handler)?; + Ok((index, update_store)) + } +} + +pub struct IndexStore { + env: Env, + name_to_uid: DashMap, + name_to_uid_db: Database, + uid_to_index: DashMap, Arc)>, + uid_to_index_db: Database>, + + thread_pool: Arc, + opt: IndexerOpts, +} + +impl IndexStore { + pub fn new(path: impl AsRef, opt: IndexerOpts) -> anyhow::Result { + let env = EnvOpenOptions::new() + .map_size(4096 * 100) + .max_dbs(2) + .open(path)?; + + let name_to_uid = DashMap::new(); + let uid_to_index = DashMap::new(); + let name_to_uid_db = open_or_create_database(&env, Some("name_to_uid"))?; + let uid_to_index_db = open_or_create_database(&env, Some("uid_to_index_db"))?; + + let thread_pool = rayon::ThreadPoolBuilder::new() + .num_threads(opt.indexing_jobs.unwrap_or(0)) + .build()?; + let thread_pool = Arc::new(thread_pool); + + Ok(Self { + env, + name_to_uid, + name_to_uid_db, + uid_to_index, + uid_to_index_db, + + thread_pool, + opt, + }) + } + + fn index_uid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { + match self.name_to_uid.entry(name.as_ref().to_string()) { + Entry::Vacant(entry) => { + match self.name_to_uid_db.get(txn, name.as_ref())? { + Some(bytes) => { + let uuid = Uuid::from_slice(bytes)?; + entry.insert(uuid); + Ok(Some(uuid)) + } + None => Ok(None) + } + } + Entry::Occupied(entry) => Ok(Some(entry.get().clone())), + } + } + + fn retrieve_index(&self, txn: &RoTxn, uid: Uuid) -> anyhow::Result, Arc)>> { + match self.uid_to_index.entry(uid.clone()) { + Entry::Vacant(entry) => { + match self.uid_to_index_db.get(txn, uid.as_bytes())? { + Some(meta) => { + let path = self.env.path(); + let (index, updates) = meta.open(path, self.thread_pool.clone(), &self.opt)?; + entry.insert((index.clone(), updates.clone())); + Ok(Some((index, updates))) + }, + None => Ok(None) + } + } + Entry::Occupied(entry) => { + let (index, updates) = entry.get(); + Ok(Some((index.clone(), updates.clone()))) + } + } + } + + fn _get_index(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result, Arc)>> { + match self.index_uid(&txn, name)? { + Some(uid) => self.retrieve_index(&txn, uid), + None => Ok(None), + } + } + + pub fn index(&self, name: impl AsRef) -> anyhow::Result, Arc)>> { + let txn = self.env.read_txn()?; + self._get_index(&txn, name) + } + + pub fn get_or_create_index( + &self, name: impl AsRef, + update_size: usize, + index_size: usize, + ) -> anyhow::Result<(Arc, Arc)> { + let mut txn = self.env.write_txn()?; + match self._get_index(&txn, name.as_ref())? { + Some(res) => Ok(res), + None => { + let uid = Uuid::new_v4(); + // TODO: clean in case of error + Ok(self.create_index(&mut txn, uid, name, update_size, index_size)?) + }, + } + } + + fn create_index( &self, + txn: &mut RwTxn, + uid: Uuid, + name: impl AsRef, + update_size: usize, + index_size: usize, + ) -> anyhow::Result<(Arc, Arc)> { + let meta = IndexMeta { update_size, index_size, uid: uid.clone() }; + + self.name_to_uid_db.put(txn, name.as_ref(), uid.as_bytes())?; + self.uid_to_index_db.put(txn, uid.as_bytes(), &meta)?; + + let path = self.env.path(); + let (index, update_store) = meta.open(path, self.thread_pool.clone(), &self.opt)?; + + self.name_to_uid.insert(name.as_ref().to_string(), uid); + self.uid_to_index.insert(uid, (index.clone(), update_store.clone())); + + Ok((index, update_store)) + } +} + +fn open_or_create_database(env: &Env, name: Option<&str>) -> anyhow::Result> { + match env.open_database::(name)? { + Some(db) => Ok(db), + None => Ok(env.create_database::(name)?), + } +} + +fn make_update_db_path(path: impl AsRef, uid: &Uuid) -> PathBuf { + let mut path = path.as_ref().to_path_buf(); + path.push(format!("update{}", uid)); + path +} + +fn make_index_db_path(path: impl AsRef, uid: &Uuid) -> PathBuf { + let mut path = path.as_ref().to_path_buf(); + path.push(format!("index{}", uid)); + path +} diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs new file mode 100644 index 000000000..dc26d0da8 --- /dev/null +++ b/src/index_controller/local_index_controller/mod.rs @@ -0,0 +1,57 @@ +mod update_store; +mod index_store; +mod update_handler; + +use index_store::IndexStore; + +use std::path::Path; +use std::sync::Arc; + +use milli::Index; + +use crate::option::IndexerOpts; +use super::IndexController; + +pub struct LocalIndexController { + indexes: IndexStore, +} + +impl LocalIndexController { + pub fn new(path: impl AsRef, opt: IndexerOpts) -> anyhow::Result { + let indexes = IndexStore::new(path, opt)?; + Ok(Self { indexes }) + } +} + +impl IndexController for LocalIndexController { + fn add_documents>( + &self, + _index: S, + _method: milli::update::IndexDocumentsMethod, + _format: milli::update::UpdateFormat, + _data: &[u8], + ) -> anyhow::Result { + todo!() + } + + fn update_settings>(&self, _index_uid: S, _settings: super::Settings) -> anyhow::Result { + todo!() + } + + fn create_index>(&self, _index_uid: S) -> anyhow::Result<()> { + todo!() + } + + fn delete_index>(&self, _index_uid: S) -> anyhow::Result<()> { + todo!() + } + + fn swap_indices, S2: AsRef>(&self, _index1_uid: S1, _index2_uid: S2) -> anyhow::Result<()> { + todo!() + } + + fn index(&self, name: impl AsRef) -> anyhow::Result>> { + let index = self.indexes.index(name)?.map(|(i, _)| i); + Ok(index) + } +} diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs new file mode 100644 index 000000000..fae3ad0ae --- /dev/null +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -0,0 +1,206 @@ +use std::io; +use std::sync::Arc; +use std::collections::HashMap; + +use anyhow::Result; +use flate2::read::GzDecoder; +use grenad::CompressionType; +use log::info; +use milli::Index; +use milli::update::{UpdateBuilder, UpdateFormat, IndexDocumentsMethod}; +use rayon::ThreadPool; + +use crate::index_controller::updates::{Processing, Processed, Failed}; +use crate::index_controller::{UpdateResult, UpdateMeta, Settings, Facets}; +use crate::option::IndexerOpts; +use super::update_store::HandleUpdate; + +pub struct UpdateHandler { + index: Arc, + max_nb_chunks: Option, + chunk_compression_level: Option, + thread_pool: Arc, + log_frequency: usize, + max_memory: usize, + linked_hash_map_size: usize, + chunk_compression_type: CompressionType, + chunk_fusing_shrink_size: u64, +} + +impl UpdateHandler { + pub fn new( + opt: &IndexerOpts, + index: Arc, + thread_pool: Arc, + ) -> anyhow::Result { + Ok(Self { + index, + max_nb_chunks: opt.max_nb_chunks, + chunk_compression_level: opt.chunk_compression_level, + thread_pool, + log_frequency: opt.log_every_n, + max_memory: opt.max_memory.get_bytes() as usize, + linked_hash_map_size: opt.linked_hash_map_size, + chunk_compression_type: opt.chunk_compression_type, + chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), + }) + } + + fn update_buidler(&self, update_id: u64) -> UpdateBuilder { + // We prepare the update by using the update builder. + let mut update_builder = UpdateBuilder::new(update_id); + if let Some(max_nb_chunks) = self.max_nb_chunks { + update_builder.max_nb_chunks(max_nb_chunks); + } + if let Some(chunk_compression_level) = self.chunk_compression_level { + update_builder.chunk_compression_level(chunk_compression_level); + } + update_builder.thread_pool(&self.thread_pool); + update_builder.log_every_n(self.log_frequency); + update_builder.max_memory(self.max_memory); + update_builder.linked_hash_map_size(self.linked_hash_map_size); + update_builder.chunk_compression_type(self.chunk_compression_type); + update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size); + update_builder + } + + fn update_documents( + &self, + format: UpdateFormat, + method: IndexDocumentsMethod, + content: &[u8], + update_builder: UpdateBuilder, + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.index.write_txn()?; + let mut builder = update_builder.index_documents(&mut wtxn, &self.index); + builder.update_format(format); + builder.index_documents_method(method); + + let gzipped = true; + let reader = if gzipped { + Box::new(GzDecoder::new(content)) + } else { + Box::new(content) as Box + }; + + let result = builder.execute(reader, |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); + + match result { + Ok(addition_result) => wtxn + .commit() + .and(Ok(UpdateResult::DocumentsAddition(addition_result))) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } + + fn clear_documents(&self, update_builder: UpdateBuilder) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.index.write_txn()?; + let builder = update_builder.clear_documents(&mut wtxn, &self.index); + + match builder.execute() { + Ok(_count) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } + + fn update_settings(&self, settings: &Settings, update_builder: UpdateBuilder) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.index.write_txn()?; + let mut builder = update_builder.settings(&mut wtxn, &self.index); + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.searchable_attributes { + match names { + Some(names) => builder.set_searchable_fields(names.clone()), + None => builder.reset_searchable_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref names) = settings.displayed_attributes { + match names { + Some(names) => builder.set_displayed_fields(names.clone()), + None => builder.reset_displayed_fields(), + } + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref facet_types) = settings.faceted_attributes { + let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); + builder.set_faceted_fields(facet_types); + } + + // We transpose the settings JSON struct into a real setting update. + if let Some(ref criteria) = settings.criteria { + match criteria { + Some(criteria) => builder.set_criteria(criteria.clone()), + None => builder.reset_criteria(), + } + } + + let result = builder.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); + + match result { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } + + fn update_facets( + &self, + levels: &Facets, + update_builder: UpdateBuilder + ) -> anyhow::Result { + // We must use the write transaction of the update here. + let mut wtxn = self.index.write_txn()?; + let mut builder = update_builder.facets(&mut wtxn, &self.index); + if let Some(value) = levels.level_group_size { + builder.level_group_size(value); + } + if let Some(value) = levels.min_level_size { + builder.min_level_size(value); + } + match builder.execute() { + Ok(()) => wtxn + .commit() + .and(Ok(UpdateResult::Other)) + .map_err(Into::into), + Err(e) => Err(e.into()) + } + } +} + +impl HandleUpdate for UpdateHandler { + fn handle_update( + &mut self, + update_id: u64, + meta: Processing, + content: &[u8] + ) -> Result, Failed> { + use UpdateMeta::*; + + let update_builder = self.update_buidler(update_id); + + let result = match meta.meta() { + DocumentsAddition { method, format } => self.update_documents(*format, *method, content, update_builder), + ClearDocuments => self.clear_documents(update_builder), + Settings(settings) => self.update_settings(settings, update_builder), + Facets(levels) => self.update_facets(levels, update_builder), + }; + + match result { + Ok(result) => Ok(meta.process(result)), + Err(e) => Err(meta.fail(e.to_string())), + } + } +} + diff --git a/src/index_controller/local_index_controller/update_store.rs b/src/index_controller/local_index_controller/update_store.rs new file mode 100644 index 000000000..94543b0a1 --- /dev/null +++ b/src/index_controller/local_index_controller/update_store.rs @@ -0,0 +1,311 @@ +use std::path::Path; +use std::sync::{Arc, RwLock}; + +use crossbeam_channel::Sender; +use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; +use heed::{EnvOpenOptions, Env, Database}; + +use crate::index_controller::updates::*; +use crate::index_controller::{UpdateMeta, UpdateResult}; + +type BEU64 = heed::zerocopy::U64; + +#[derive(Clone)] +pub struct UpdateStore { + env: Env, + pending_meta: Database, SerdeJson>>, + pending: Database, ByteSlice>, + processed_meta: Database, SerdeJson>>, + failed_meta: Database, SerdeJson>>, + aborted_meta: Database, SerdeJson>>, + processing: Arc>>>, + notification_sender: Sender<()>, +} + +pub trait HandleUpdate { + fn handle_update(&mut self, update_id: u64, meta: Processing, content: &[u8]) -> Result, Failed>; +} + +impl UpdateStore { + pub fn open( + mut options: EnvOpenOptions, + path: P, + mut update_handler: U, + ) -> heed::Result> + where + P: AsRef, + U: HandleUpdate + Send + 'static, + { + options.max_dbs(5); + + let env = options.open(path)?; + let pending_meta = env.create_database(Some("pending-meta"))?; + let pending = env.create_database(Some("pending"))?; + let processed_meta = env.create_database(Some("processed-meta"))?; + let aborted_meta = env.create_database(Some("aborted-meta"))?; + let failed_meta = env.create_database(Some("failed-meta"))?; + let processing = Arc::new(RwLock::new(None)); + + let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); + // Send a first notification to trigger the process. + let _ = notification_sender.send(()); + + let update_store = Arc::new(UpdateStore { + env, + pending, + pending_meta, + processed_meta, + aborted_meta, + notification_sender, + failed_meta, + processing, + }); + + let update_store_cloned = update_store.clone(); + std::thread::spawn(move || { + // Block and wait for something to process. + for () in notification_receiver { + loop { + match update_store_cloned.process_pending_update(&mut update_handler) { + Ok(Some(_)) => (), + Ok(None) => break, + Err(e) => eprintln!("error while processing update: {}", e), + } + } + } + }); + + Ok(update_store) + } + + /// Returns the new biggest id to use to store the new update. + fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { + let last_pending = self.pending_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_processed = self.processed_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_aborted = self.aborted_meta + .remap_data_type::() + .last(txn)? + .map(|(k, _)| k.get()); + + let last_update_id = [last_pending, last_processed, last_aborted] + .iter() + .copied() + .flatten() + .max(); + + match last_update_id { + Some(last_id) => Ok(last_id + 1), + None => Ok(0), + } + } + + /// Registers the update content in the pending store and the meta + /// into the pending-meta store. Returns the new unique update id. + pub fn register_update( + &self, + meta: UpdateMeta, + content: &[u8] + ) -> heed::Result> { + let mut wtxn = self.env.write_txn()?; + + // We ask the update store to give us a new update id, this is safe, + // no other update can have the same id because we use a write txn before + // asking for the id and registering it so other update registering + // will be forced to wait for a new write txn. + let update_id = self.new_update_id(&wtxn)?; + let update_key = BEU64::new(update_id); + + let meta = Pending::new(meta, update_id); + self.pending_meta.put(&mut wtxn, &update_key, &meta)?; + self.pending.put(&mut wtxn, &update_key, content)?; + + wtxn.commit()?; + + if let Err(e) = self.notification_sender.try_send(()) { + assert!(!e.is_disconnected(), "update notification channel is disconnected"); + } + Ok(meta) + } + /// Executes the user provided function on the next pending update (the one with the lowest id). + /// This is asynchronous as it let the user process the update with a read-only txn and + /// only writing the result meta to the processed-meta store *after* it has been processed. + fn process_pending_update(&self, handler: &mut U) -> heed::Result> + where + U: HandleUpdate + Send + 'static, + { + // Create a read transaction to be able to retrieve the pending update in order. + let rtxn = self.env.read_txn()?; + let first_meta = self.pending_meta.first(&rtxn)?; + + // If there is a pending update we process and only keep + // a reader while processing it, not a writer. + match first_meta { + Some((first_id, pending)) => { + let first_content = self.pending + .get(&rtxn, &first_id)? + .expect("associated update content"); + + // we cahnge the state of the update from pending to processing before we pass it + // to the update handler. Processing store is non persistent to be able recover + // from a failure + let processing = pending.processing(); + self.processing + .write() + .unwrap() + .replace(processing.clone()); + // Process the pending update using the provided user function. + let result = handler.handle_update(first_id.get(), processing, first_content); + drop(rtxn); + + // Once the pending update have been successfully processed + // we must remove the content from the pending and processing stores and + // write the *new* meta to the processed-meta store and commit. + let mut wtxn = self.env.write_txn()?; + self.processing + .write() + .unwrap() + .take(); + self.pending_meta.delete(&mut wtxn, &first_id)?; + self.pending.delete(&mut wtxn, &first_id)?; + match result { + Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, + Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, + } + wtxn.commit()?; + + Ok(Some(())) + }, + None => Ok(None) + } + } + + /// The id and metadata of the update that is currently being processed, + /// `None` if no update is being processed. + pub fn processing_update(&self) -> heed::Result)>> { + let rtxn = self.env.read_txn()?; + match self.pending_meta.first(&rtxn)? { + Some((key, meta)) => Ok(Some((key.get(), meta))), + None => Ok(None), + } + } + + /// Execute the user defined function with the meta-store iterators, the first + /// iterator is the *processed* meta one, the second the *aborted* meta one + /// and, the last is the *pending* meta one. + pub fn iter_metas(&self, mut f: F) -> heed::Result + where + F: for<'a> FnMut( + Option>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + ) -> heed::Result, + { + let rtxn = self.env.read_txn()?; + + // We get the pending, processed and aborted meta iterators. + let processed_iter = self.processed_meta.iter(&rtxn)?; + let aborted_iter = self.aborted_meta.iter(&rtxn)?; + let pending_iter = self.pending_meta.iter(&rtxn)?; + let processing = self.processing.read().unwrap().clone(); + let failed_iter = self.failed_meta.iter(&rtxn)?; + + // We execute the user defined function with both iterators. + (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) + } + + /// Returns the update associated meta or `None` if the update doesn't exist. + pub fn meta(&self, update_id: u64) -> heed::Result>> { + let rtxn = self.env.read_txn()?; + let key = BEU64::new(update_id); + + if let Some(ref meta) = *self.processing.read().unwrap() { + if meta.id() == update_id { + return Ok(Some(UpdateStatus::Processing(meta.clone()))); + } + } + + if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Pending(meta))); + } + + if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Processed(meta))); + } + + if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Aborted(meta))); + } + + if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { + return Ok(Some(UpdateStatus::Failed(meta))); + } + + Ok(None) + } + + /// Aborts an update, an aborted update content is deleted and + /// the meta of it is moved into the aborted updates database. + /// + /// Trying to abort an update that is currently being processed, an update + /// that as already been processed or which doesn't actually exist, will + /// return `None`. + pub fn abort_update(&self, update_id: u64) -> heed::Result>> { + let mut wtxn = self.env.write_txn()?; + let key = BEU64::new(update_id); + + // We cannot abort an update that is currently being processed. + if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { + return Ok(None); + } + + let pending = match self.pending_meta.get(&wtxn, &key)? { + Some(meta) => meta, + None => return Ok(None), + }; + + let aborted = pending.abort(); + + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + + wtxn.commit()?; + + Ok(Some(aborted)) + } + + /// Aborts all the pending updates, and not the one being currently processed. + /// Returns the update metas and ids that were successfully aborted. + pub fn abort_pendings(&self) -> heed::Result)>> { + let mut wtxn = self.env.write_txn()?; + let mut aborted_updates = Vec::new(); + + // We skip the first pending update as it is currently being processed. + for result in self.pending_meta.iter(&wtxn)?.skip(1) { + let (key, pending) = result?; + let id = key.get(); + aborted_updates.push((id, pending.abort())); + } + + for (id, aborted) in &aborted_updates { + let key = BEU64::new(*id); + self.aborted_meta.put(&mut wtxn, &key, &aborted)?; + self.pending_meta.delete(&mut wtxn, &key)?; + self.pending.delete(&mut wtxn, &key)?; + } + + wtxn.commit()?; + + Ok(aborted_updates) + } +} diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index c927a6c5b..d59575f7d 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,18 +1,19 @@ -mod index_store; -mod update_store; +mod local_index_controller; +mod updates; -pub use index_store::IndexStore; -pub use update_store::UpdateStore; +pub use local_index_controller::LocalIndexController; -use std::num::NonZeroUsize; -use std::ops::Deref; use std::collections::HashMap; +use std::num::NonZeroUsize; +use std::sync::Arc; use anyhow::Result; -use milli::update::{IndexDocumentsMethod, UpdateFormat}; -use milli::update_store::{Processed, Processing, Failed, Pending, Aborted}; +use milli::Index; +use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; +use updates::{Processed, Processing, Failed, Pending, Aborted}; + pub type UpdateStatusResponse = UpdateStatus; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -89,7 +90,7 @@ impl Settings { } #[derive(Debug, Clone, Serialize, Deserialize)] pub enum UpdateResult { - //DocumentsAddition(DocumentAdditionResult), + DocumentsAddition(DocumentAdditionResult), Other, } @@ -97,7 +98,7 @@ pub enum UpdateResult { /// for read access which is provided, and write access which must be provided. This allows the /// implementer to define the behaviour of write accesses to the indices, and abstract the /// scheduling of the updates. The implementer must be able to provide an instance of `IndexStore` -pub trait IndexController: Deref { +pub trait IndexController { /* * Write operations @@ -141,5 +142,7 @@ pub trait IndexController: Deref { ) -> Result, Failed> { todo!() } -} + /// Returns, if it exists, an `IndexView` to the requested index. + fn index(&self, uid: impl AsRef) -> anyhow::Result>>; +} diff --git a/src/index_controller/update_store/mod.rs b/src/index_controller/update_store/mod.rs deleted file mode 100644 index 84db2f63d..000000000 --- a/src/index_controller/update_store/mod.rs +++ /dev/null @@ -1,49 +0,0 @@ -use std::ops::Deref; - -use super::{IndexStore, IndexController}; - -pub struct UpdateStore { - index_store: IndexStore, -} - -impl Deref for UpdateStore { - type Target = IndexStore; - - fn deref(&self) -> &Self::Target { - &self.index_store - } -} - -impl UpdateStore { - pub fn new(index_store: IndexStore) -> Self { - Self { index_store } - } -} - -impl IndexController for UpdateStore { - fn add_documents>( - &self, - _index: S, - _method: milli::update::IndexDocumentsMethod, - _format: milli::update::UpdateFormat, - _data: &[u8], - ) -> anyhow::Result { - todo!() - } - - fn update_settings>(&self, _index_uid: S, _settings: crate::index_controller::Settings) -> anyhow::Result { - todo!() - } - - fn create_index>(&self, _index_uid: S) -> anyhow::Result<()> { - todo!() - } - - fn delete_index>(&self, _index_uid: S) -> anyhow::Result<()> { - todo!() - } - - fn swap_indices, S2: AsRef>(&self, _index1_uid: S1, _index2_uid: S2) -> anyhow::Result<()> { - todo!() - } -} diff --git a/src/index_controller/updates.rs b/src/index_controller/updates.rs new file mode 100644 index 000000000..7c67ea8c2 --- /dev/null +++ b/src/index_controller/updates.rs @@ -0,0 +1,167 @@ +use chrono::{Utc, DateTime}; +use serde::{Serialize, Deserialize}; + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Pending { + update_id: u64, + meta: M, + enqueued_at: DateTime, +} + +impl Pending { + pub fn new(meta: M, update_id: u64) -> Self { + Self { + enqueued_at: Utc::now(), + meta, + update_id, + } + } + + pub fn processing(self) -> Processing { + Processing { + from: self, + started_processing_at: Utc::now(), + } + } + + pub fn abort(self) -> Aborted { + Aborted { + from: self, + aborted_at: Utc::now(), + } + } + + pub fn meta(&self) -> &M { + &self.meta + } + + pub fn id(&self) -> u64 { + self.update_id + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Processed { + success: N, + processed_at: DateTime, + #[serde(flatten)] + from: Processing, +} + +impl Processed { + pub fn id(&self) -> u64 { + self.from.id() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Processing { + #[serde(flatten)] + from: Pending, + started_processing_at: DateTime, +} + +impl Processing { + pub fn id(&self) -> u64 { + self.from.id() + } + + pub fn meta(&self) -> &M { + self.from.meta() + } + + pub fn process(self, meta: N) -> Processed { + Processed { + success: meta, + from: self, + processed_at: Utc::now(), + } + } + + pub fn fail(self, error: E) -> Failed { + Failed { + from: self, + error, + failed_at: Utc::now(), + } + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Aborted { + #[serde(flatten)] + from: Pending, + aborted_at: DateTime, +} + +impl Aborted { + pub fn id(&self) -> u64 { + self.from.id() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] +pub struct Failed { + #[serde(flatten)] + from: Processing, + error: E, + failed_at: DateTime, +} + +impl Failed { + pub fn id(&self) -> u64 { + self.from.id() + } +} + +#[derive(Debug, PartialEq, Eq, Hash, Serialize)] +#[serde(tag = "status")] +pub enum UpdateStatus { + Processing(Processing), + Pending(Pending), + Processed(Processed), + Aborted(Aborted), + Failed(Failed), +} + +impl UpdateStatus { + pub fn id(&self) -> u64 { + match self { + UpdateStatus::Processing(u) => u.id(), + UpdateStatus::Pending(u) => u.id(), + UpdateStatus::Processed(u) => u.id(), + UpdateStatus::Aborted(u) => u.id(), + UpdateStatus::Failed(u) => u.id(), + } + } +} + +impl From> for UpdateStatus { + fn from(other: Pending) -> Self { + Self::Pending(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Aborted) -> Self { + Self::Aborted(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Processed) -> Self { + Self::Processed(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Processing) -> Self { + Self::Processing(other) + } +} + +impl From> for UpdateStatus { + fn from(other: Failed) -> Self { + Self::Failed(other) + } +} From 81832028689de6c298c38acffa03fbab8e45afd4 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 15:14:48 +0100 Subject: [PATCH 07/16] documetn addition and search --- src/data/mod.rs | 7 ++- src/data/updates.rs | 7 +-- .../local_index_controller/index_store.rs | 45 +++++++++++++++---- .../local_index_controller/mod.rs | 34 ++++++++++---- .../local_index_controller/update_handler.rs | 4 ++ src/index_controller/mod.rs | 18 ++------ 6 files changed, 79 insertions(+), 36 deletions(-) diff --git a/src/data/mod.rs b/src/data/mod.rs index 7494792bc..175aedba5 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -61,7 +61,12 @@ impl Data { let path = options.db_path.clone(); let indexer_opts = options.indexer_options.clone(); create_dir_all(&path)?; - let index_controller = LocalIndexController::new(&path, indexer_opts)?; + let index_controller = LocalIndexController::new( + &path, + indexer_opts, + options.max_mdb_size.get_bytes(), + options.max_udb_size.get_bytes(), + )?; let indexes = Arc::new(index_controller); let mut api_keys = ApiKeys { diff --git a/src/data/updates.rs b/src/data/updates.rs index d05617361..880579bf6 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -6,7 +6,8 @@ use futures_util::stream::StreamExt; use tokio::io::AsyncWriteExt; use super::Data; -use crate::index_controller::{IndexController, UpdateStatusResponse, Settings}; +use crate::index_controller::{IndexController, Settings, UpdateResult, UpdateMeta}; +use crate::index_controller::updates::UpdateStatus; impl Data { pub async fn add_documents( @@ -15,7 +16,7 @@ impl Data { method: IndexDocumentsMethod, format: UpdateFormat, mut stream: impl futures::Stream> + Unpin, - ) -> anyhow::Result + ) -> anyhow::Result> where B: Deref, E: std::error::Error + Send + Sync + 'static, @@ -45,7 +46,7 @@ impl Data { &self, index: S, settings: Settings - ) -> anyhow::Result { + ) -> anyhow::Result> { let indexes = self.index_controller.clone(); let update = tokio::task::spawn_blocking(move || indexes.update_settings(index, settings)).await??; Ok(update.into()) diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index bfe63459a..4c9a98472 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -1,4 +1,5 @@ use std::path::{Path, PathBuf}; +use std::fs::create_dir_all; use std::sync::Arc; use dashmap::DashMap; @@ -8,6 +9,7 @@ use milli::Index; use rayon::ThreadPool; use uuid::Uuid; use serde::{Serialize, Deserialize}; +use log::warn; use super::update_store::UpdateStore; use super::update_handler::UpdateHandler; @@ -15,8 +17,8 @@ use crate::option::IndexerOpts; #[derive(Serialize, Deserialize, Debug)] struct IndexMeta { - update_size: usize, - index_size: usize, + update_size: u64, + index_size: u64, uid: Uuid, } @@ -30,12 +32,15 @@ impl IndexMeta { let update_path = make_update_db_path(&path, &self.uid); let index_path = make_index_db_path(&path, &self.uid); + create_dir_all(&update_path)?; + create_dir_all(&index_path)?; + let mut options = EnvOpenOptions::new(); - options.map_size(self.index_size); + options.map_size(self.index_size as usize); let index = Arc::new(Index::new(options, index_path)?); let mut options = EnvOpenOptions::new(); - options.map_size(self.update_size); + options.map_size(self.update_size as usize); let handler = UpdateHandler::new(opt, index.clone(), thread_pool)?; let update_store = UpdateStore::open(options, update_path, handler)?; Ok((index, update_store)) @@ -132,8 +137,8 @@ impl IndexStore { pub fn get_or_create_index( &self, name: impl AsRef, - update_size: usize, - index_size: usize, + update_size: u64, + index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { let mut txn = self.env.write_txn()?; match self._get_index(&txn, name.as_ref())? { @@ -141,17 +146,39 @@ impl IndexStore { None => { let uid = Uuid::new_v4(); // TODO: clean in case of error - Ok(self.create_index(&mut txn, uid, name, update_size, index_size)?) + let result = self.create_index(&mut txn, uid, name, update_size, index_size); + match result { + Ok((index, update_store)) => { + match txn.commit() { + Ok(_) => Ok((index, update_store)), + Err(e) => { + self.clean_uid(&uid); + Err(anyhow::anyhow!("error creating index: {}", e)) + } + } + } + Err(e) => { + self.clean_uid(&uid); + Err(e) + } + } }, } } + /// removes all data acociated with an index Uuid. This is called when index creation failed + /// and outstanding files and data need to be cleaned. + fn clean_uid(&self, _uid: &Uuid) { + // TODO! + warn!("creating cleanup is not yet implemented"); + } + fn create_index( &self, txn: &mut RwTxn, uid: Uuid, name: impl AsRef, - update_size: usize, - index_size: usize, + update_size: u64, + index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { let meta = IndexMeta { update_size, index_size, uid: uid.clone() }; diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index dc26d0da8..00a6cc363 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -11,30 +11,46 @@ use milli::Index; use crate::option::IndexerOpts; use super::IndexController; +use super::updates::UpdateStatus; +use super::{UpdateMeta, UpdateResult}; pub struct LocalIndexController { indexes: IndexStore, + update_db_size: u64, + index_db_size: u64, } impl LocalIndexController { - pub fn new(path: impl AsRef, opt: IndexerOpts) -> anyhow::Result { + pub fn new( + path: impl AsRef, + opt: IndexerOpts, + index_db_size: u64, + update_db_size: u64, + ) -> anyhow::Result { let indexes = IndexStore::new(path, opt)?; - Ok(Self { indexes }) + Ok(Self { indexes, index_db_size, update_db_size }) } } impl IndexController for LocalIndexController { fn add_documents>( &self, - _index: S, - _method: milli::update::IndexDocumentsMethod, - _format: milli::update::UpdateFormat, - _data: &[u8], - ) -> anyhow::Result { - todo!() + index: S, + method: milli::update::IndexDocumentsMethod, + format: milli::update::UpdateFormat, + data: &[u8], + ) -> anyhow::Result> { + let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; + let meta = UpdateMeta::DocumentsAddition { method, format }; + let pending = update_store.register_update(meta, data).unwrap(); + Ok(pending.into()) } - fn update_settings>(&self, _index_uid: S, _settings: super::Settings) -> anyhow::Result { + fn update_settings>( + &self, + _index_uid: S, + _settings: super::Settings + ) -> anyhow::Result> { todo!() } diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs index fae3ad0ae..24b9ab405 100644 --- a/src/index_controller/local_index_controller/update_handler.rs +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -188,6 +188,8 @@ impl HandleUpdate for UpdateHandler { ) -> Result, Failed> { use UpdateMeta::*; + println!("handling update {}", update_id); + let update_builder = self.update_buidler(update_id); let result = match meta.meta() { @@ -197,6 +199,8 @@ impl HandleUpdate for UpdateHandler { Facets(levels) => self.update_facets(levels, update_builder), }; + println!("{:?}", result); + match result { Ok(result) => Ok(meta.process(result)), Err(e) => Err(meta.fail(e.to_string())), diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index d59575f7d..0907ba0c8 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,5 +1,5 @@ mod local_index_controller; -mod updates; +pub mod updates; pub use local_index_controller::LocalIndexController; @@ -12,9 +12,8 @@ use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; -use updates::{Processed, Processing, Failed, Pending, Aborted}; +use updates::{Processed, Processing, Failed, UpdateStatus}; -pub type UpdateStatusResponse = UpdateStatus; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] @@ -33,15 +32,6 @@ pub struct Facets { pub min_level_size: Option, } -#[derive(Debug, Clone, Serialize)] -#[serde(tag = "type")] -pub enum UpdateStatus { - Pending { update_id: u64, meta: Pending }, - Progressing { update_id: u64, meta: P }, - Processed { update_id: u64, meta: Processed }, - Aborted { update_id: u64, meta: Aborted }, -} - fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> where T: Deserialize<'de>, D: Deserializer<'de> @@ -116,11 +106,11 @@ pub trait IndexController { method: IndexDocumentsMethod, format: UpdateFormat, data: &[u8], - ) -> anyhow::Result; + ) -> anyhow::Result>; /// Updates an index settings. If the index does not exist, it will be created when the update /// is applied to the index. - fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; + fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result>; /// Create an index with the given `index_uid`. fn create_index>(&self, index_uid: S) -> Result<()>; From 4119ae865528dfad19f935fea300c5f9866c18a6 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 16:57:53 +0100 Subject: [PATCH 08/16] setttings update --- src/index_controller/local_index_controller/mod.rs | 9 ++++++--- .../local_index_controller/update_handler.rs | 4 ---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index 00a6cc363..f70050512 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -48,10 +48,13 @@ impl IndexController for LocalIndexController { fn update_settings>( &self, - _index_uid: S, - _settings: super::Settings + index: S, + settings: super::Settings ) -> anyhow::Result> { - todo!() + let (_, update_store) = self.indexes.get_or_create_index(&index, self.update_db_size, self.index_db_size)?; + let meta = UpdateMeta::Settings(settings); + let pending = update_store.register_update(meta, &[]).unwrap(); + Ok(pending.into()) } fn create_index>(&self, _index_uid: S) -> anyhow::Result<()> { diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs index 24b9ab405..fae3ad0ae 100644 --- a/src/index_controller/local_index_controller/update_handler.rs +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -188,8 +188,6 @@ impl HandleUpdate for UpdateHandler { ) -> Result, Failed> { use UpdateMeta::*; - println!("handling update {}", update_id); - let update_builder = self.update_buidler(update_id); let result = match meta.meta() { @@ -199,8 +197,6 @@ impl HandleUpdate for UpdateHandler { Facets(levels) => self.update_facets(levels, update_builder), }; - println!("{:?}", result); - match result { Ok(result) => Ok(meta.process(result)), Err(e) => Err(meta.fail(e.to_string())), From 60371b9dcf77bcbbb3aea3a27d71f6386b50c2f7 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 17:20:51 +0100 Subject: [PATCH 09/16] get update id --- src/data/updates.rs | 36 ++++++++--------- .../local_index_controller/mod.rs | 8 ++++ src/index_controller/mod.rs | 2 + src/routes/index.rs | 39 +++++++++---------- 4 files changed, 47 insertions(+), 38 deletions(-) diff --git a/src/data/updates.rs b/src/data/updates.rs index 880579bf6..6f44ec57d 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -10,7 +10,7 @@ use crate::index_controller::{IndexController, Settings, UpdateResult, UpdateMet use crate::index_controller::updates::UpdateStatus; impl Data { - pub async fn add_documents( + pub async fn add_documents( &self, index: S, method: IndexDocumentsMethod, @@ -52,24 +52,24 @@ impl Data { Ok(update.into()) } - //#[inline] - //pub fn get_update_status>(&self, _index: S, uid: u64) -> anyhow::Result>> { - //self.indexes.get_update_status(uid) - //} + #[inline] + pub fn get_update_status>(&self, index: S, uid: u64) -> anyhow::Result>> { + self.index_controller.update_status(index, uid) + } //pub fn get_updates_status(&self, _index: &str) -> anyhow::Result>> { - //let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| { - //let mut metas = processing - //.map(UpdateStatus::from) - //.into_iter() - //.chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.collect::>(); - //metas.sort_by(|a, b| a.id().cmp(&b.id())); - //Ok(metas) - //})?; - //Ok(result) + //let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| { + //let mut metas = processing + //.map(UpdateStatus::from) + //.into_iter() + //.chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) + //.collect::>(); + //metas.sort_by(|a, b| a.id().cmp(&b.id())); + //Ok(metas) + //})?; + //Ok(result) //} } diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index f70050512..b36009ce5 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -8,6 +8,7 @@ use std::path::Path; use std::sync::Arc; use milli::Index; +use anyhow::bail; use crate::option::IndexerOpts; use super::IndexController; @@ -73,4 +74,11 @@ impl IndexController for LocalIndexController { let index = self.indexes.index(name)?.map(|(i, _)| i); Ok(index) } + + fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>> { + match self.indexes.index(&index)? { + Some((_, update_store)) => Ok(update_store.meta(id)?), + None => bail!("index {:?} doesn't exist", index.as_ref()), + } + } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 0907ba0c8..b0d75a266 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -135,4 +135,6 @@ pub trait IndexController { /// Returns, if it exists, an `IndexView` to the requested index. fn index(&self, uid: impl AsRef) -> anyhow::Result>>; + + fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>>; } diff --git a/src/routes/index.rs b/src/routes/index.rs index fa4ae0679..cccf28a2d 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -1,7 +1,7 @@ use actix_web::{delete, get, post, put}; use actix_web::{web, HttpResponse}; use chrono::{DateTime, Utc}; -//use log::error; +use log::error; use serde::{Deserialize, Serialize}; use crate::Data; @@ -94,8 +94,8 @@ async fn delete_index( #[derive(Deserialize)] struct UpdateParam { - _index_uid: String, - _update_id: u64, + index_uid: String, + update_id: u64, } #[get( @@ -103,24 +103,23 @@ struct UpdateParam { wrap = "Authentication::Private" )] async fn get_update_status( - _data: web::Data, - _path: web::Path, + data: web::Data, + path: web::Path, ) -> Result { - todo!() - //let result = data.get_update_status(&path.index_uid, path.update_id); - //match result { - //Ok(Some(meta)) => { - //let json = serde_json::to_string(&meta).unwrap(); - //Ok(HttpResponse::Ok().body(json)) - //} - //Ok(None) => { - //todo!() - //} - //Err(e) => { - //error!("{}", e); - //todo!() - //} - //} + let result = data.get_update_status(&path.index_uid, path.update_id); + match result { + Ok(Some(meta)) => { + let json = serde_json::to_string(&meta).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Ok(None) => { + todo!() + } + Err(e) => { + error!("{}", e); + todo!() + } + } } #[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] From 6c63ee6798db69cd19bf82021afd5984ab5c7b0d Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 18:32:24 +0100 Subject: [PATCH 10/16] implement list all indexes --- Cargo.lock | 14 ++++++++-- Cargo.toml | 1 + src/data/updates.rs | 18 +++---------- .../local_index_controller/mod.rs | 22 +++++++++++++++ src/index_controller/mod.rs | 1 + src/routes/index.rs | 27 +++++++++---------- 6 files changed, 52 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ceaef691..fc102336a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1452,6 +1452,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.6" @@ -1650,6 +1659,7 @@ dependencies = [ "heed", "http", "indexmap", + "itertools 0.10.0", "jemallocator", "log", "main_error", @@ -1745,7 +1755,7 @@ dependencies = [ "grenad", "heed", "human_format", - "itertools", + "itertools 0.9.0", "jemallocator", "levenshtein_automata", "linked-hash-map", @@ -3699,6 +3709,6 @@ checksum = "a1e6e8778706838f43f771d80d37787cb2fe06dafe89dd3aebaf6721b9eaec81" dependencies = [ "cc", "glob", - "itertools", + "itertools 0.9.0", "libc", ] diff --git a/Cargo.toml b/Cargo.toml index eb2fca43d..f68414beb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,6 +63,7 @@ page_size = "0.4.2" obkv = "0.1.1" ouroboros = "0.8.0" uuid = "0.8.2" +itertools = "0.10.0" [dependencies.sentry] default-features = false diff --git a/src/data/updates.rs b/src/data/updates.rs index 6f44ec57d..194ec346b 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -57,19 +57,7 @@ impl Data { self.index_controller.update_status(index, uid) } - //pub fn get_updates_status(&self, _index: &str) -> anyhow::Result>> { - //let result = self.update_queue.iter_metas(|processing, processed, pending, aborted, failed| { - //let mut metas = processing - //.map(UpdateStatus::from) - //.into_iter() - //.chain(processed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.chain(pending.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.chain(aborted.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.chain(failed.filter_map(|i| Some(i.ok()?.1)).map(UpdateStatus::from)) - //.collect::>(); - //metas.sort_by(|a, b| a.id().cmp(&b.id())); - //Ok(metas) - //})?; - //Ok(result) - //} + pub fn get_updates_status(&self, index: &str) -> anyhow::Result>> { + self.index_controller.all_update_status(index) + } } diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index b36009ce5..debc15a1a 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use milli::Index; use anyhow::bail; +use itertools::Itertools; use crate::option::IndexerOpts; use super::IndexController; @@ -81,4 +82,25 @@ impl IndexController for LocalIndexController { None => bail!("index {:?} doesn't exist", index.as_ref()), } } + + fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>> { + match self.indexes.index(index)? { + Some((_, update_store)) => { + let updates = update_store.iter_metas(|processing, processed, pending, aborted, failed| { + Ok(processing + .map(UpdateStatus::from) + .into_iter() + .chain(pending.filter_map(|p| p.ok()).map(|(_, u)| UpdateStatus::from(u))) + .chain(aborted.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) + .chain(processed.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) + .chain(failed.filter_map(Result::ok).map(|(_, u)| UpdateStatus::from(u))) + .sorted_by(|a, b| a.id().cmp(&b.id())) + .collect()) + })?; + Ok(updates) + } + None => Ok(Vec::new()) + } + + } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index b0d75a266..f1ba8f7ce 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -137,4 +137,5 @@ pub trait IndexController { fn index(&self, uid: impl AsRef) -> anyhow::Result>>; fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>>; + fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>>; } diff --git a/src/routes/index.rs b/src/routes/index.rs index cccf28a2d..774039f2b 100644 --- a/src/routes/index.rs +++ b/src/routes/index.rs @@ -124,19 +124,18 @@ async fn get_update_status( #[get("/indexes/{index_uid}/updates", wrap = "Authentication::Private")] async fn get_all_updates_status( - _data: web::Data, - _path: web::Path, + data: web::Data, + path: web::Path, ) -> Result { - todo!() - //let result = data.get_updates_status(&path.index_uid); - //match result { - //Ok(metas) => { - //let json = serde_json::to_string(&metas).unwrap(); - //Ok(HttpResponse::Ok().body(json)) - //} - //Err(e) => { - //error!("{}", e); - //todo!() - //} - //} + let result = data.get_updates_status(&path.index_uid); + match result { + Ok(metas) => { + let json = serde_json::to_string(&metas).unwrap(); + Ok(HttpResponse::Ok().body(json)) + } + Err(e) => { + error!("{}", e); + todo!() + } + } } From e9c95f66231d5059f65d8bd13be977611bd593e6 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 19:43:54 +0100 Subject: [PATCH 11/16] remove useless files --- Cargo.lock | 2 +- src/index_controller/_update_store/mod.rs | 17 - src/index_controller/index_store.rs | 74 --- src/updates/mod.rs | 318 ------------ src/updates/settings.rs | 61 --- src/updates/update_store.rs | 581 ---------------------- 6 files changed, 1 insertion(+), 1052 deletions(-) delete mode 100644 src/index_controller/_update_store/mod.rs delete mode 100644 src/index_controller/index_store.rs delete mode 100644 src/updates/mod.rs delete mode 100644 src/updates/settings.rs delete mode 100644 src/updates/update_store.rs diff --git a/Cargo.lock b/Cargo.lock index fc102336a..4471f5127 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1628,7 +1628,7 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "meilisearch-error" -version = "0.18.1" +version = "0.18.0" dependencies = [ "actix-http", ] diff --git a/src/index_controller/_update_store/mod.rs b/src/index_controller/_update_store/mod.rs deleted file mode 100644 index ef8711ded..000000000 --- a/src/index_controller/_update_store/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -use std::sync::Arc; - -use heed::Env; - -use super::IndexStore; - -pub struct UpdateStore { - env: Env, - index_store: Arc, -} - -impl UpdateStore { - pub fn new(env: Env, index_store: Arc) -> anyhow::Result { - Ok(Self { env, index_store }) - } -} - diff --git a/src/index_controller/index_store.rs b/src/index_controller/index_store.rs deleted file mode 100644 index 6652086f9..000000000 --- a/src/index_controller/index_store.rs +++ /dev/null @@ -1,74 +0,0 @@ -use std::sync::Arc; -use std::collections::HashMap; - -use anyhow::Result; -use milli::{Index, FieldsIdsMap, SearchResult, FieldId, facet::FacetType}; -use ouroboros::self_referencing; - -use crate::data::SearchQuery; - -#[self_referencing] -pub struct IndexView { - pub index: Arc, - #[borrows(index)] - #[covariant] - pub txn: heed::RoTxn<'this>, - uuid: String, -} - -impl IndexView { - pub fn search(&self, search_query: &SearchQuery) -> Result { - self.with(|this| { - let mut search = this.index.search(&this.txn); - if let Some(query) = &search_query.q { - search.query(query); - } - - if let Some(offset) = search_query.offset { - search.offset(offset); - } - - let limit = search_query.limit; - search.limit(limit); - - Ok(search.execute()?) - }) - } - - #[inline] - pub fn fields_ids_map(&self) -> Result { - self.with(|this| Ok(this.index.fields_ids_map(&this.txn)?)) - - } - - #[inline] - pub fn displayed_fields_ids(&self) -> Result>> { - self.with(|this| Ok(this.index.displayed_fields_ids(&this.txn)?)) - } - - #[inline] - pub fn displayed_fields(&self) -> Result>> { - self.with(|this| Ok(this.index - .displayed_fields(&this.txn)? - .map(|fields| fields.into_iter().map(String::from).collect()))) - } - - #[inline] - pub fn searchable_fields(&self) -> Result>> { - self.with(|this| Ok(this.index - .searchable_fields(&this.txn)? - .map(|fields| fields.into_iter().map(String::from).collect()))) - } - - #[inline] - pub fn faceted_fields(&self) -> Result> { - self.with(|this| Ok(this.index.faceted_fields(&this.txn)?)) - } - - pub fn documents(&self, ids: &[u32]) -> Result)>> { - let txn = self.borrow_txn(); - let index = self.borrow_index(); - Ok(index.documents(txn, ids.into_iter().copied())?) - } -} - diff --git a/src/updates/mod.rs b/src/updates/mod.rs deleted file mode 100644 index 7e0802595..000000000 --- a/src/updates/mod.rs +++ /dev/null @@ -1,318 +0,0 @@ -mod settings; -mod update_store; - -pub use settings::{Settings, Facets}; - -use std::io; -use std::sync::Arc; -use std::fs::create_dir_all; -use std::collections::HashMap; - -use anyhow::Result; -use byte_unit::Byte; -use flate2::read::GzDecoder; -use grenad::CompressionType; -use log::info; -use milli::Index; -use milli::update::{UpdateBuilder, UpdateFormat, IndexDocumentsMethod, DocumentAdditionResult }; -use milli::update_store::{UpdateStore, UpdateHandler as Handler, UpdateStatus, Processing, Processed, Failed}; -use rayon::ThreadPool; -use serde::{Serialize, Deserialize}; -use structopt::StructOpt; - -use crate::option::Opt; - -pub type UpdateStatusResponse = UpdateStatus; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type")] -pub enum UpdateMeta { - DocumentsAddition { method: IndexDocumentsMethod, format: UpdateFormat }, - ClearDocuments, - Settings(Settings), - Facets(Facets), -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type")] -pub enum UpdateMetaProgress { - DocumentsAddition { - step: usize, - total_steps: usize, - current: usize, - total: Option, - }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum UpdateResult { - DocumentsAddition(DocumentAdditionResult), - Other, -} - -#[derive(Clone)] -pub struct UpdateQueue { - inner: Arc>, -} - -#[derive(Debug, Clone, StructOpt)] -pub struct IndexerOpts { - /// The amount of documents to skip before printing - /// a log regarding the indexing advancement. - #[structopt(long, default_value = "100000")] // 100k - pub log_every_n: usize, - - /// MTBL max number of chunks in bytes. - #[structopt(long)] - pub max_nb_chunks: Option, - - /// The maximum amount of memory to use for the MTBL buffer. It is recommended - /// to use something like 80%-90% of the available memory. - /// - /// It is automatically split by the number of jobs e.g. if you use 7 jobs - /// and 7 GB of max memory, each thread will use a maximum of 1 GB. - #[structopt(long, default_value = "7 GiB")] - pub max_memory: Byte, - - /// Size of the linked hash map cache when indexing. - /// The bigger it is, the faster the indexing is but the more memory it takes. - #[structopt(long, default_value = "500")] - pub linked_hash_map_size: usize, - - /// The name of the compression algorithm to use when compressing intermediate - /// chunks during indexing documents. - /// - /// Choosing a fast algorithm will make the indexing faster but may consume more memory. - #[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])] - pub chunk_compression_type: CompressionType, - - /// The level of compression of the chosen algorithm. - #[structopt(long, requires = "chunk-compression-type")] - pub chunk_compression_level: Option, - - /// The number of bytes to remove from the begining of the chunks while reading/sorting - /// or merging them. - /// - /// File fusing must only be enable on file systems that support the `FALLOC_FL_COLLAPSE_RANGE`, - /// (i.e. ext4 and XFS). File fusing will only work if the `enable-chunk-fusing` is set. - #[structopt(long, default_value = "4 GiB")] - pub chunk_fusing_shrink_size: Byte, - - /// Enable the chunk fusing or not, this reduces the amount of disk used by a factor of 2. - #[structopt(long)] - pub enable_chunk_fusing: bool, - - /// Number of parallel jobs for indexing, defaults to # of CPUs. - #[structopt(long)] - pub indexing_jobs: Option, -} - -struct UpdateHandler { - indexes: Arc, - max_nb_chunks: Option, - chunk_compression_level: Option, - thread_pool: ThreadPool, - log_frequency: usize, - max_memory: usize, - linked_hash_map_size: usize, - chunk_compression_type: CompressionType, - chunk_fusing_shrink_size: u64, -} - -impl UpdateHandler { - fn new( - opt: &IndexerOpts, - indexes: Arc, - ) -> Result { - let thread_pool = rayon::ThreadPoolBuilder::new() - .num_threads(opt.indexing_jobs.unwrap_or(0)) - .build()?; - Ok(Self { - indexes, - max_nb_chunks: opt.max_nb_chunks, - chunk_compression_level: opt.chunk_compression_level, - thread_pool, - log_frequency: opt.log_every_n, - max_memory: opt.max_memory.get_bytes() as usize, - linked_hash_map_size: opt.linked_hash_map_size, - chunk_compression_type: opt.chunk_compression_type, - chunk_fusing_shrink_size: opt.chunk_fusing_shrink_size.get_bytes(), - }) - } - - fn update_buidler(&self, update_id: u64) -> UpdateBuilder { - // We prepare the update by using the update builder. - let mut update_builder = UpdateBuilder::new(update_id); - if let Some(max_nb_chunks) = self.max_nb_chunks { - update_builder.max_nb_chunks(max_nb_chunks); - } - if let Some(chunk_compression_level) = self.chunk_compression_level { - update_builder.chunk_compression_level(chunk_compression_level); - } - update_builder.thread_pool(&self.thread_pool); - update_builder.log_every_n(self.log_frequency); - update_builder.max_memory(self.max_memory); - update_builder.linked_hash_map_size(self.linked_hash_map_size); - update_builder.chunk_compression_type(self.chunk_compression_type); - update_builder.chunk_fusing_shrink_size(self.chunk_fusing_shrink_size); - update_builder - } - - fn update_documents( - &self, - format: UpdateFormat, - method: IndexDocumentsMethod, - content: &[u8], - update_builder: UpdateBuilder, - ) -> Result { - // We must use the write transaction of the update here. - let mut wtxn = self.indexes.write_txn()?; - let mut builder = update_builder.index_documents(&mut wtxn, &self.indexes); - builder.update_format(format); - builder.index_documents_method(method); - - let gzipped = true; - let reader = if gzipped { - Box::new(GzDecoder::new(content)) - } else { - Box::new(content) as Box - }; - - let result = builder.execute(reader, |indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); - - match result { - Ok(addition_result) => wtxn - .commit() - .and(Ok(UpdateResult::DocumentsAddition(addition_result))) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } - - fn clear_documents(&self, update_builder: UpdateBuilder) -> Result { - // We must use the write transaction of the update here. - let mut wtxn = self.indexes.write_txn()?; - let builder = update_builder.clear_documents(&mut wtxn, &self.indexes); - - match builder.execute() { - Ok(_count) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } - - fn update_settings(&self, settings: &Settings, update_builder: UpdateBuilder) -> Result { - // We must use the write transaction of the update here. - let mut wtxn = self.indexes.write_txn()?; - let mut builder = update_builder.settings(&mut wtxn, &self.indexes); - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.searchable_attributes { - match names { - Some(names) => builder.set_searchable_fields(names.clone()), - None => builder.reset_searchable_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref names) = settings.displayed_attributes { - match names { - Some(names) => builder.set_displayed_fields(names.clone()), - None => builder.reset_displayed_fields(), - } - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref facet_types) = settings.faceted_attributes { - let facet_types = facet_types.clone().unwrap_or_else(|| HashMap::new()); - builder.set_faceted_fields(facet_types); - } - - // We transpose the settings JSON struct into a real setting update. - if let Some(ref criteria) = settings.criteria { - match criteria { - Some(criteria) => builder.set_criteria(criteria.clone()), - None => builder.reset_criteria(), - } - } - - let result = builder.execute(|indexing_step, update_id| info!("update {}: {:?}", update_id, indexing_step)); - - match result { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } - - fn update_facets(&self, levels: &Facets, update_builder: UpdateBuilder) -> Result { - // We must use the write transaction of the update here. - let mut wtxn = self.indexes.write_txn()?; - let mut builder = update_builder.facets(&mut wtxn, &self.indexes); - if let Some(value) = levels.level_group_size { - builder.level_group_size(value); - } - if let Some(value) = levels.min_level_size { - builder.min_level_size(value); - } - match builder.execute() { - Ok(()) => wtxn - .commit() - .and(Ok(UpdateResult::Other)) - .map_err(Into::into), - Err(e) => Err(e.into()) - } - } -} - -impl Handler for UpdateHandler { - fn handle_update( - &mut self, - update_id: u64, - meta: Processing, - content: &[u8] - ) -> Result, Failed> { - use UpdateMeta::*; - - let update_builder = self.update_buidler(update_id); - - let result = match meta.meta() { - DocumentsAddition { method, format } => self.update_documents(*format, *method, content, update_builder), - ClearDocuments => self.clear_documents(update_builder), - Settings(settings) => self.update_settings(settings, update_builder), - Facets(levels) => self.update_facets(levels, update_builder), - }; - - match result { - Ok(result) => Ok(meta.process(result)), - Err(e) => Err(meta.fail(e.to_string())), - } - } -} - -impl UpdateQueue { - pub fn new( - opt: &Opt, - indexes: Arc, - ) -> Result { - let handler = UpdateHandler::new(&opt.indexer_options, indexes)?; - let size = opt.max_udb_size.get_bytes() as usize; - let path = opt.db_path.join("updates.mdb"); - create_dir_all(&path)?; - let inner = UpdateStore::open( - Some(size), - path, - handler - )?; - Ok(Self { inner }) - } - - #[inline] - pub fn get_update_status(&self, update_id: u64) -> Result> { - Ok(self.inner.meta(update_id)?) - } -} diff --git a/src/updates/settings.rs b/src/updates/settings.rs deleted file mode 100644 index 91381edc5..000000000 --- a/src/updates/settings.rs +++ /dev/null @@ -1,61 +0,0 @@ -use std::num::NonZeroUsize; -use std::collections::HashMap; - -use serde::{Serialize, Deserialize, de::Deserializer}; - -// Any value that is present is considered Some value, including null. -fn deserialize_some<'de, T, D>(deserializer: D) -> Result, D::Error> -where T: Deserialize<'de>, - D: Deserializer<'de> -{ - Deserialize::deserialize(deserializer).map(Some) -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -pub struct Settings { - #[serde( - default, - deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", - )] - pub displayed_attributes: Option>>, - - #[serde( - default, - deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", - )] - pub searchable_attributes: Option>>, - - #[serde(default)] - pub faceted_attributes: Option>>, - - #[serde( - default, - deserialize_with = "deserialize_some", - skip_serializing_if = "Option::is_none", - )] - pub criteria: Option>>, -} - -impl Settings { - pub fn cleared() -> Self { - Self { - displayed_attributes: Some(None), - searchable_attributes: Some(None), - faceted_attributes: Some(None), - criteria: Some(None), - } - } -} - - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -pub struct Facets { - pub level_group_size: Option, - pub min_level_size: Option, -} diff --git a/src/updates/update_store.rs b/src/updates/update_store.rs deleted file mode 100644 index f750fc38b..000000000 --- a/src/updates/update_store.rs +++ /dev/null @@ -1,581 +0,0 @@ -use std::path::Path; -use std::sync::{Arc, RwLock}; - -use crossbeam_channel::Sender; -use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; -use heed::{EnvOpenOptions, Env, Database}; -use serde::{Serialize, Deserialize}; -use chrono::{DateTime, Utc}; - -type BEU64 = heed::zerocopy::U64; - -#[derive(Clone)] -pub struct UpdateStore { - env: Env, - pending_meta: Database, SerdeJson>>, - pending: Database, ByteSlice>, - processed_meta: Database, SerdeJson>>, - failed_meta: Database, SerdeJson>>, - aborted_meta: Database, SerdeJson>>, - processing: Arc>>>, - notification_sender: Sender<()>, -} - -pub trait UpdateHandler { - fn handle_update(&mut self, update_id: u64, meta: Processing, content: &[u8]) -> Result, Failed>; -} - -impl UpdateHandler for F -where F: FnMut(u64, Processing, &[u8]) -> Result, Failed> + Send + 'static { - fn handle_update(&mut self, update_id: u64, meta: Processing, content: &[u8]) -> Result, Failed> { - self(update_id, meta, content) - } -} - -impl UpdateStore { - pub fn open( - size: Option, - path: P, - mut update_handler: U, - ) -> heed::Result>> - where - P: AsRef, - U: UpdateHandler + Send + 'static, - M: for<'a> Deserialize<'a> + Serialize + Send + Sync + Clone, - N: Serialize, - E: Serialize, - { - let mut options = EnvOpenOptions::new(); - if let Some(size) = size { - options.map_size(size); - } - options.max_dbs(5); - - let env = options.open(path)?; - let pending_meta = env.create_database(Some("pending-meta"))?; - let pending = env.create_database(Some("pending"))?; - let processed_meta = env.create_database(Some("processed-meta"))?; - let aborted_meta = env.create_database(Some("aborted-meta"))?; - let failed_meta = env.create_database(Some("failed-meta"))?; - let processing = Arc::new(RwLock::new(None)); - - let (notification_sender, notification_receiver) = crossbeam_channel::bounded(1); - // Send a first notification to trigger the process. - let _ = notification_sender.send(()); - - let update_store = Arc::new(UpdateStore { - env, - pending, - pending_meta, - processed_meta, - aborted_meta, - notification_sender, - failed_meta, - processing, - }); - - let update_store_cloned = update_store.clone(); - std::thread::spawn(move || { - // Block and wait for something to process. - for () in notification_receiver { - loop { - match update_store_cloned.process_pending_update(&mut update_handler) { - Ok(Some(_)) => (), - Ok(None) => break, - Err(e) => eprintln!("error while processing update: {}", e), - } - } - } - }); - - Ok(update_store) - } - - /// Returns the new biggest id to use to store the new update. - fn new_update_id(&self, txn: &heed::RoTxn) -> heed::Result { - let last_pending = self.pending_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_processed = self.processed_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_aborted = self.aborted_meta - .remap_data_type::() - .last(txn)? - .map(|(k, _)| k.get()); - - let last_update_id = [last_pending, last_processed, last_aborted] - .iter() - .copied() - .flatten() - .max(); - - match last_update_id { - Some(last_id) => Ok(last_id + 1), - None => Ok(0), - } - } - - /// Registers the update content in the pending store and the meta - /// into the pending-meta store. Returns the new unique update id. - pub fn register_update(&self, meta: M, content: &[u8]) -> heed::Result> - where M: Serialize, - { - let mut wtxn = self.env.write_txn()?; - - // We ask the update store to give us a new update id, this is safe, - // no other update can have the same id because we use a write txn before - // asking for the id and registering it so other update registering - // will be forced to wait for a new write txn. - let update_id = self.new_update_id(&wtxn)?; - let update_key = BEU64::new(update_id); - - let meta = Pending::new(meta, update_id); - self.pending_meta.put(&mut wtxn, &update_key, &meta)?; - self.pending.put(&mut wtxn, &update_key, content)?; - - wtxn.commit()?; - - if let Err(e) = self.notification_sender.try_send(()) { - assert!(!e.is_disconnected(), "update notification channel is disconnected"); - } - Ok(meta) - } - /// Executes the user provided function on the next pending update (the one with the lowest id). - /// This is asynchronous as it let the user process the update with a read-only txn and - /// only writing the result meta to the processed-meta store *after* it has been processed. - fn process_pending_update(&self, handler: &mut U) -> heed::Result> - where - U: UpdateHandler, - M: for<'a> Deserialize<'a> + Serialize + Clone, - N: Serialize, - E: Serialize, - { - // Create a read transaction to be able to retrieve the pending update in order. - let rtxn = self.env.read_txn()?; - let first_meta = self.pending_meta.first(&rtxn)?; - - // If there is a pending update we process and only keep - // a reader while processing it, not a writer. - match first_meta { - Some((first_id, pending)) => { - let first_content = self.pending - .get(&rtxn, &first_id)? - .expect("associated update content"); - - // we cahnge the state of the update from pending to processing before we pass it - // to the update handler. Processing store is non persistent to be able recover - // from a failure - let processing = pending.processing(); - self.processing - .write() - .unwrap() - .replace(processing.clone()); - // Process the pending update using the provided user function. - let result = handler.handle_update(first_id.get(), processing, first_content); - drop(rtxn); - - // Once the pending update have been successfully processed - // we must remove the content from the pending and processing stores and - // write the *new* meta to the processed-meta store and commit. - let mut wtxn = self.env.write_txn()?; - self.processing - .write() - .unwrap() - .take(); - self.pending_meta.delete(&mut wtxn, &first_id)?; - self.pending.delete(&mut wtxn, &first_id)?; - match result { - Ok(processed) => self.processed_meta.put(&mut wtxn, &first_id, &processed)?, - Err(failed) => self.failed_meta.put(&mut wtxn, &first_id, &failed)?, - } - wtxn.commit()?; - - Ok(Some(())) - }, - None => Ok(None) - } - } - - /// The id and metadata of the update that is currently being processed, - /// `None` if no update is being processed. - pub fn processing_update(&self) -> heed::Result)>> - where M: for<'a> Deserialize<'a>, - { - let rtxn = self.env.read_txn()?; - match self.pending_meta.first(&rtxn)? { - Some((key, meta)) => Ok(Some((key.get(), meta))), - None => Ok(None), - } - } - - /// Execute the user defined function with the meta-store iterators, the first - /// iterator is the *processed* meta one, the second the *aborted* meta one - /// and, the last is the *pending* meta one. - pub fn iter_metas(&self, mut f: F) -> heed::Result - where - M: for<'a> Deserialize<'a> + Clone, - N: for<'a> Deserialize<'a>, - F: for<'a> FnMut( - Option>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - ) -> heed::Result, - { - let rtxn = self.env.read_txn()?; - - // We get the pending, processed and aborted meta iterators. - let processed_iter = self.processed_meta.iter(&rtxn)?; - let aborted_iter = self.aborted_meta.iter(&rtxn)?; - let pending_iter = self.pending_meta.iter(&rtxn)?; - let processing = self.processing.read().unwrap().clone(); - let failed_iter = self.failed_meta.iter(&rtxn)?; - - // We execute the user defined function with both iterators. - (f)(processing, processed_iter, aborted_iter, pending_iter, failed_iter) - } - - /// Returns the update associated meta or `None` if the update doesn't exist. - pub fn meta(&self, update_id: u64) -> heed::Result>> - where - M: for<'a> Deserialize<'a> + Clone, - N: for<'a> Deserialize<'a>, - E: for<'a> Deserialize<'a>, - { - let rtxn = self.env.read_txn()?; - let key = BEU64::new(update_id); - - if let Some(ref meta) = *self.processing.read().unwrap() { - if meta.id() == update_id { - return Ok(Some(UpdateStatus::Processing(meta.clone()))); - } - } - - println!("pending"); - if let Some(meta) = self.pending_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Pending(meta))); - } - - println!("processed"); - if let Some(meta) = self.processed_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Processed(meta))); - } - - if let Some(meta) = self.aborted_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Aborted(meta))); - } - - if let Some(meta) = self.failed_meta.get(&rtxn, &key)? { - return Ok(Some(UpdateStatus::Failed(meta))); - } - - Ok(None) - } - - /// Aborts an update, an aborted update content is deleted and - /// the meta of it is moved into the aborted updates database. - /// - /// Trying to abort an update that is currently being processed, an update - /// that as already been processed or which doesn't actually exist, will - /// return `None`. - pub fn abort_update(&self, update_id: u64) -> heed::Result>> - where M: Serialize + for<'a> Deserialize<'a>, - { - let mut wtxn = self.env.write_txn()?; - let key = BEU64::new(update_id); - - // We cannot abort an update that is currently being processed. - if self.pending_meta.first(&wtxn)?.map(|(key, _)| key.get()) == Some(update_id) { - return Ok(None); - } - - let pending = match self.pending_meta.get(&wtxn, &key)? { - Some(meta) => meta, - None => return Ok(None), - }; - - let aborted = pending.abort(); - - self.aborted_meta.put(&mut wtxn, &key, &aborted)?; - self.pending_meta.delete(&mut wtxn, &key)?; - self.pending.delete(&mut wtxn, &key)?; - - wtxn.commit()?; - - Ok(Some(aborted)) - } - - /// Aborts all the pending updates, and not the one being currently processed. - /// Returns the update metas and ids that were successfully aborted. - pub fn abort_pendings(&self) -> heed::Result)>> - where M: Serialize + for<'a> Deserialize<'a>, - { - let mut wtxn = self.env.write_txn()?; - let mut aborted_updates = Vec::new(); - - // We skip the first pending update as it is currently being processed. - for result in self.pending_meta.iter(&wtxn)?.skip(1) { - let (key, pending) = result?; - let id = key.get(); - aborted_updates.push((id, pending.abort())); - } - - for (id, aborted) in &aborted_updates { - let key = BEU64::new(*id); - self.aborted_meta.put(&mut wtxn, &key, &aborted)?; - self.pending_meta.delete(&mut wtxn, &key)?; - self.pending.delete(&mut wtxn, &key)?; - } - - wtxn.commit()?; - - Ok(aborted_updates) - } -} - -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] -pub struct Pending { - update_id: u64, - meta: M, - enqueued_at: DateTime, -} - -impl Pending { - fn new(meta: M, update_id: u64) -> Self { - Self { - enqueued_at: Utc::now(), - meta, - update_id, - } - } - - pub fn processing(self) -> Processing { - Processing { - from: self, - started_processing_at: Utc::now(), - } - } - - pub fn abort(self) -> Aborted { - Aborted { - from: self, - aborted_at: Utc::now(), - } - } - - pub fn meta(&self) -> &M { - &self.meta - } - - pub fn id(&self) -> u64 { - self.update_id - } -} - -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] -pub struct Processed { - success: N, - processed_at: DateTime, - #[serde(flatten)] - from: Processing, -} - -impl Processed { - fn id(&self) -> u64 { - self.from.id() - } -} - -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] -pub struct Processing { - #[serde(flatten)] - from: Pending, - started_processing_at: DateTime, -} - -impl Processing { - pub fn id(&self) -> u64 { - self.from.id() - } - - pub fn meta(&self) -> &M { - self.from.meta() - } - - pub fn process(self, meta: N) -> Processed { - Processed { - success: meta, - from: self, - processed_at: Utc::now(), - } - } - - pub fn fail(self, error: E) -> Failed { - Failed { - from: self, - error, - failed_at: Utc::now(), - } - } -} - -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] -pub struct Aborted { - #[serde(flatten)] - from: Pending, - aborted_at: DateTime, -} - -impl Aborted { - fn id(&self) -> u64 { - self.from.id() - } -} - -#[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] -pub struct Failed { - #[serde(flatten)] - from: Processing, - error: E, - failed_at: DateTime, -} - -impl Failed { - fn id(&self) -> u64 { - self.from.id() - } -} - -#[derive(Debug, PartialEq, Eq, Hash, Serialize)] -#[serde(tag = "status")] -pub enum UpdateStatus { - Processing(Processing), - Pending(Pending), - Processed(Processed), - Aborted(Aborted), - Failed(Failed), -} - -impl UpdateStatus { - pub fn id(&self) -> u64 { - match self { - UpdateStatus::Processing(u) => u.id(), - UpdateStatus::Pending(u) => u.id(), - UpdateStatus::Processed(u) => u.id(), - UpdateStatus::Aborted(u) => u.id(), - UpdateStatus::Failed(u) => u.id(), - } - } -} - -impl From> for UpdateStatus { - fn from(other: Pending) -> Self { - Self::Pending(other) - } -} - -impl From> for UpdateStatus { - fn from(other: Aborted) -> Self { - Self::Aborted(other) - } -} - -impl From> for UpdateStatus { - fn from(other: Processed) -> Self { - Self::Processed(other) - } -} - -impl From> for UpdateStatus { - fn from(other: Processing) -> Self { - Self::Processing(other) - } -} - -impl From> for UpdateStatus { - fn from(other: Failed) -> Self { - Self::Failed(other) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::thread; - use std::time::{Duration, Instant}; - - #[test] - fn simple() { - let dir = tempfile::tempdir().unwrap(); - let update_store = UpdateStore::open(None, dir, |_id, meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { - let new_meta = meta.meta().to_string() + " processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); - - let meta = String::from("kiki"); - let update = update_store.register_update(meta, &[]).unwrap(); - thread::sleep(Duration::from_millis(100)); - let meta = update_store.meta(update.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } - } - - #[test] - #[ignore] - fn long_running_update() { - let dir = tempfile::tempdir().unwrap(); - let update_store = UpdateStore::open(None, dir, |_id, meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { - thread::sleep(Duration::from_millis(400)); - let new_meta = meta.meta().to_string() + "processed"; - let processed = meta.process(new_meta); - Ok(processed) - }).unwrap(); - - let before_register = Instant::now(); - - let meta = String::from("kiki"); - let update_kiki = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - let meta = String::from("coco"); - let update_coco = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - let meta = String::from("cucu"); - let update_cucu = update_store.register_update(meta, &[]).unwrap(); - assert!(before_register.elapsed() < Duration::from_millis(200)); - - thread::sleep(Duration::from_millis(400 * 3 + 100)); - - let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "kiki processed"); - } else { - panic!() - } - - let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "coco processed"); - } else { - panic!() - } - - let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); - if let UpdateStatus::Processed(Processed { success, .. }) = meta { - assert_eq!(success, "cucu processed"); - } else { - panic!() - } - } -} From da056a687794caf5f9a3725c14fe14b45184e9b4 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 28 Jan 2021 20:55:29 +0100 Subject: [PATCH 12/16] comment tests out --- .../local_index_controller/index_store.rs | 6 +- .../local_index_controller/update_handler.rs | 4 +- .../local_index_controller/update_store.rs | 144 +- src/index_controller/updates.rs | 16 +- tests/assets/dumps/v1/metadata.json | 12 - tests/assets/dumps/v1/test/documents.jsonl | 77 - tests/assets/dumps/v1/test/settings.json | 59 - tests/assets/dumps/v1/test/updates.jsonl | 2 - tests/assets/test_set.json | 1613 -------------- tests/common.rs | 569 ----- tests/dashboard.rs | 12 - tests/documents_add.rs | 222 -- tests/documents_delete.rs | 67 - tests/documents_get.rs | 23 - tests/dump.rs | 395 ---- tests/errors.rs | 200 -- tests/health.rs | 11 - tests/index.rs | 809 ------- tests/index_update.rs | 200 -- tests/lazy_index_creation.rs | 446 ---- tests/placeholder_search.rs | 629 ------ tests/search.rs | 1879 ----------------- tests/search_settings.rs | 538 ----- tests/settings.rs | 523 ----- tests/settings_ranking_rules.rs | 182 -- tests/settings_stop_words.rs | 61 - tests/url_normalizer.rs | 18 - 27 files changed, 132 insertions(+), 8585 deletions(-) delete mode 100644 tests/assets/dumps/v1/metadata.json delete mode 100644 tests/assets/dumps/v1/test/documents.jsonl delete mode 100644 tests/assets/dumps/v1/test/settings.json delete mode 100644 tests/assets/dumps/v1/test/updates.jsonl delete mode 100644 tests/assets/test_set.json delete mode 100644 tests/common.rs delete mode 100644 tests/dashboard.rs delete mode 100644 tests/documents_add.rs delete mode 100644 tests/documents_delete.rs delete mode 100644 tests/documents_get.rs delete mode 100644 tests/dump.rs delete mode 100644 tests/errors.rs delete mode 100644 tests/health.rs delete mode 100644 tests/index.rs delete mode 100644 tests/index_update.rs delete mode 100644 tests/lazy_index_creation.rs delete mode 100644 tests/placeholder_search.rs delete mode 100644 tests/search.rs delete mode 100644 tests/search_settings.rs delete mode 100644 tests/settings.rs delete mode 100644 tests/settings_ranking_rules.rs delete mode 100644 tests/settings_stop_words.rs delete mode 100644 tests/url_normalizer.rs diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 4c9a98472..b94d9e492 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -11,9 +11,11 @@ use uuid::Uuid; use serde::{Serialize, Deserialize}; use log::warn; -use super::update_store::UpdateStore; -use super::update_handler::UpdateHandler; use crate::option::IndexerOpts; +use super::update_handler::UpdateHandler; +use super::{UpdateMeta, UpdateResult}; + +type UpdateStore = super::update_store::UpdateStore; #[derive(Serialize, Deserialize, Debug)] struct IndexMeta { diff --git a/src/index_controller/local_index_controller/update_handler.rs b/src/index_controller/local_index_controller/update_handler.rs index fae3ad0ae..c89b9f686 100644 --- a/src/index_controller/local_index_controller/update_handler.rs +++ b/src/index_controller/local_index_controller/update_handler.rs @@ -182,12 +182,13 @@ impl UpdateHandler { impl HandleUpdate for UpdateHandler { fn handle_update( &mut self, - update_id: u64, meta: Processing, content: &[u8] ) -> Result, Failed> { use UpdateMeta::*; + let update_id = meta.id(); + let update_builder = self.update_buidler(update_id); let result = match meta.meta() { @@ -203,4 +204,3 @@ impl HandleUpdate for UpdateHandler { } } } - diff --git a/src/index_controller/local_index_controller/update_store.rs b/src/index_controller/local_index_controller/update_store.rs index 94543b0a1..9a17ec00f 100644 --- a/src/index_controller/local_index_controller/update_store.rs +++ b/src/index_controller/local_index_controller/update_store.rs @@ -4,37 +4,42 @@ use std::sync::{Arc, RwLock}; use crossbeam_channel::Sender; use heed::types::{OwnedType, DecodeIgnore, SerdeJson, ByteSlice}; use heed::{EnvOpenOptions, Env, Database}; +use serde::{Serialize, Deserialize}; use crate::index_controller::updates::*; -use crate::index_controller::{UpdateMeta, UpdateResult}; type BEU64 = heed::zerocopy::U64; #[derive(Clone)] -pub struct UpdateStore { +pub struct UpdateStore { env: Env, - pending_meta: Database, SerdeJson>>, + pending_meta: Database, SerdeJson>>, pending: Database, ByteSlice>, - processed_meta: Database, SerdeJson>>, - failed_meta: Database, SerdeJson>>, - aborted_meta: Database, SerdeJson>>, - processing: Arc>>>, + processed_meta: Database, SerdeJson>>, + failed_meta: Database, SerdeJson>>, + aborted_meta: Database, SerdeJson>>, + processing: Arc>>>, notification_sender: Sender<()>, } pub trait HandleUpdate { - fn handle_update(&mut self, update_id: u64, meta: Processing, content: &[u8]) -> Result, Failed>; + fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed>; } -impl UpdateStore { +impl UpdateStore +where + M: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync + Clone, + N: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, + E: for<'a> Deserialize<'a> + Serialize + 'static + Send + Sync, +{ pub fn open( mut options: EnvOpenOptions, path: P, mut update_handler: U, - ) -> heed::Result> + ) -> heed::Result> where P: AsRef, - U: HandleUpdate + Send + 'static, + U: HandleUpdate + Send + 'static, { options.max_dbs(5); @@ -111,9 +116,9 @@ impl UpdateStore { /// into the pending-meta store. Returns the new unique update id. pub fn register_update( &self, - meta: UpdateMeta, + meta: M, content: &[u8] - ) -> heed::Result> { + ) -> heed::Result> { let mut wtxn = self.env.write_txn()?; // We ask the update store to give us a new update id, this is safe, @@ -139,7 +144,7 @@ impl UpdateStore { /// only writing the result meta to the processed-meta store *after* it has been processed. fn process_pending_update(&self, handler: &mut U) -> heed::Result> where - U: HandleUpdate + Send + 'static, + U: HandleUpdate + Send + 'static, { // Create a read transaction to be able to retrieve the pending update in order. let rtxn = self.env.read_txn()?; @@ -153,7 +158,7 @@ impl UpdateStore { .get(&rtxn, &first_id)? .expect("associated update content"); - // we cahnge the state of the update from pending to processing before we pass it + // we change the state of the update from pending to processing before we pass it // to the update handler. Processing store is non persistent to be able recover // from a failure let processing = pending.processing(); @@ -162,7 +167,7 @@ impl UpdateStore { .unwrap() .replace(processing.clone()); // Process the pending update using the provided user function. - let result = handler.handle_update(first_id.get(), processing, first_content); + let result = handler.handle_update(processing, first_content); drop(rtxn); // Once the pending update have been successfully processed @@ -189,10 +194,10 @@ impl UpdateStore { /// The id and metadata of the update that is currently being processed, /// `None` if no update is being processed. - pub fn processing_update(&self) -> heed::Result)>> { + pub fn processing_update(&self) -> heed::Result>> { let rtxn = self.env.read_txn()?; match self.pending_meta.first(&rtxn)? { - Some((key, meta)) => Ok(Some((key.get(), meta))), + Some((_, meta)) => Ok(Some(meta)), None => Ok(None), } } @@ -203,11 +208,11 @@ impl UpdateStore { pub fn iter_metas(&self, mut f: F) -> heed::Result where F: for<'a> FnMut( - Option>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, - heed::RoIter<'a, OwnedType, SerdeJson>>, + Option>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, + heed::RoIter<'a, OwnedType, SerdeJson>>, ) -> heed::Result, { let rtxn = self.env.read_txn()?; @@ -224,7 +229,7 @@ impl UpdateStore { } /// Returns the update associated meta or `None` if the update doesn't exist. - pub fn meta(&self, update_id: u64) -> heed::Result>> { + pub fn meta(&self, update_id: u64) -> heed::Result>> { let rtxn = self.env.read_txn()?; let key = BEU64::new(update_id); @@ -259,7 +264,7 @@ impl UpdateStore { /// Trying to abort an update that is currently being processed, an update /// that as already been processed or which doesn't actually exist, will /// return `None`. - pub fn abort_update(&self, update_id: u64) -> heed::Result>> { + pub fn abort_update(&self, update_id: u64) -> heed::Result>> { let mut wtxn = self.env.write_txn()?; let key = BEU64::new(update_id); @@ -286,7 +291,7 @@ impl UpdateStore { /// Aborts all the pending updates, and not the one being currently processed. /// Returns the update metas and ids that were successfully aborted. - pub fn abort_pendings(&self) -> heed::Result)>> { + pub fn abort_pendings(&self) -> heed::Result)>> { let mut wtxn = self.env.write_txn()?; let mut aborted_updates = Vec::new(); @@ -309,3 +314,90 @@ impl UpdateStore { Ok(aborted_updates) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + use std::time::{Duration, Instant}; + + impl HandleUpdate for F + where F: FnMut(Processing, &[u8]) -> Result, Failed> + Send + 'static { + fn handle_update(&mut self, meta: Processing, content: &[u8]) -> Result, Failed> { + self(meta, content) + } + } + + #[test] + fn simple() { + let dir = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100); + let update_store = UpdateStore::open(options, dir, |meta: Processing, _content: &_| -> Result<_, Failed<_, ()>> { + let new_meta = meta.meta().to_string() + " processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let meta = String::from("kiki"); + let update = update_store.register_update(meta, &[]).unwrap(); + thread::sleep(Duration::from_millis(100)); + let meta = update_store.meta(update.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + } + + #[test] + #[ignore] + fn long_running_update() { + let dir = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(4096 * 100); + let update_store = UpdateStore::open(options, dir, |meta: Processing, _content:&_| -> Result<_, Failed<_, ()>> { + thread::sleep(Duration::from_millis(400)); + let new_meta = meta.meta().to_string() + "processed"; + let processed = meta.process(new_meta); + Ok(processed) + }).unwrap(); + + let before_register = Instant::now(); + + let meta = String::from("kiki"); + let update_kiki = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("coco"); + let update_coco = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + let meta = String::from("cucu"); + let update_cucu = update_store.register_update(meta, &[]).unwrap(); + assert!(before_register.elapsed() < Duration::from_millis(200)); + + thread::sleep(Duration::from_millis(400 * 3 + 100)); + + let meta = update_store.meta(update_kiki.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "kiki processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_coco.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "coco processed"); + } else { + panic!() + } + + let meta = update_store.meta(update_cucu.id()).unwrap().unwrap(); + if let UpdateStatus::Processed(Processed { success, .. }) = meta { + assert_eq!(success, "cucu processed"); + } else { + panic!() + } + } +} diff --git a/src/index_controller/updates.rs b/src/index_controller/updates.rs index 7c67ea8c2..4eb94dc4a 100644 --- a/src/index_controller/updates.rs +++ b/src/index_controller/updates.rs @@ -3,9 +3,9 @@ use serde::{Serialize, Deserialize}; #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] pub struct Pending { - update_id: u64, - meta: M, - enqueued_at: DateTime, + pub update_id: u64, + pub meta: M, + pub enqueued_at: DateTime, } impl Pending { @@ -42,10 +42,10 @@ impl Pending { #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] pub struct Processed { - success: N, - processed_at: DateTime, + pub success: N, + pub processed_at: DateTime, #[serde(flatten)] - from: Processing, + pub from: Processing, } impl Processed { @@ -57,8 +57,8 @@ impl Processed { #[derive(Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone)] pub struct Processing { #[serde(flatten)] - from: Pending, - started_processing_at: DateTime, + pub from: Pending, + pub started_processing_at: DateTime, } impl Processing { diff --git a/tests/assets/dumps/v1/metadata.json b/tests/assets/dumps/v1/metadata.json deleted file mode 100644 index 6fe302324..000000000 --- a/tests/assets/dumps/v1/metadata.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "indices": [{ - "uid": "test", - "primaryKey": "id" - }, { - "uid": "test2", - "primaryKey": "test2_id" - } - ], - "dbVersion": "0.13.0", - "dumpVersion": "1" -} diff --git a/tests/assets/dumps/v1/test/documents.jsonl b/tests/assets/dumps/v1/test/documents.jsonl deleted file mode 100644 index 7af80f342..000000000 --- a/tests/assets/dumps/v1/test/documents.jsonl +++ /dev/null @@ -1,77 +0,0 @@ -{"id":0,"isActive":false,"balance":"$2,668.55","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Lucas Hess","gender":"male","email":"lucashess@chorizon.com","phone":"+1 (998) 478-2597","address":"412 Losee Terrace, Blairstown, Georgia, 2825","about":"Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n","registered":"2016-06-21T09:30:25 -02:00","latitude":-44.174957,"longitude":-145.725388,"tags":["bug","bug"]} -{"id":1,"isActive":true,"balance":"$1,706.13","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Cherry Orr","gender":"female","email":"cherryorr@chorizon.com","phone":"+1 (995) 479-3174","address":"442 Beverly Road, Ventress, New Mexico, 3361","about":"Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n","registered":"2020-03-18T11:12:21 -01:00","latitude":-24.356932,"longitude":27.184808,"tags":["new issue","bug"]} -{"id":2,"isActive":true,"balance":"$2,467.47","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Patricia Goff","gender":"female","email":"patriciagoff@chorizon.com","phone":"+1 (864) 463-2277","address":"866 Hornell Loop, Cresaptown, Ohio, 1700","about":"Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n","registered":"2014-10-28T12:59:30 -01:00","latitude":-64.008555,"longitude":11.867098,"tags":["good first issue"]} -{"id":3,"isActive":true,"balance":"$3,344.40","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Adeline Flynn","gender":"female","email":"adelineflynn@chorizon.com","phone":"+1 (994) 600-2840","address":"428 Paerdegat Avenue, Hollymead, Pennsylvania, 948","about":"Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n","registered":"2014-03-27T06:24:45 -01:00","latitude":-74.485173,"longitude":-11.059859,"tags":["bug","good first issue","wontfix","new issue"]} -{"id":4,"isActive":false,"balance":"$2,575.78","picture":"http://placehold.it/32x32","age":39,"color":"Green","name":"Mariana Pacheco","gender":"female","email":"marianapacheco@chorizon.com","phone":"+1 (820) 414-2223","address":"664 Rapelye Street, Faywood, California, 7320","about":"Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n","registered":"2015-09-02T03:23:35 -02:00","latitude":75.763501,"longitude":-78.777124,"tags":["new issue"]} -{"id":5,"isActive":true,"balance":"$3,793.09","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Warren Watson","gender":"male","email":"warrenwatson@chorizon.com","phone":"+1 (807) 583-2427","address":"671 Prince Street, Faxon, Connecticut, 4275","about":"Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n","registered":"2017-06-04T06:02:17 -02:00","latitude":29.979223,"longitude":25.358943,"tags":["wontfix","wontfix","wontfix"]} -{"id":6,"isActive":true,"balance":"$2,919.70","picture":"http://placehold.it/32x32","age":20,"color":"blue","name":"Shelia Berry","gender":"female","email":"sheliaberry@chorizon.com","phone":"+1 (853) 511-2651","address":"437 Forrest Street, Coventry, Illinois, 2056","about":"Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n","registered":"2018-07-11T02:45:01 -02:00","latitude":54.815991,"longitude":-118.690609,"tags":["good first issue","bug","wontfix","new issue"]} -{"id":7,"isActive":true,"balance":"$1,349.50","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Chrystal Boyd","gender":"female","email":"chrystalboyd@chorizon.com","phone":"+1 (936) 563-2802","address":"670 Croton Loop, Sussex, Florida, 4692","about":"Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n","registered":"2016-11-01T07:36:04 -01:00","latitude":-24.711933,"longitude":147.246705,"tags":[]} -{"id":8,"isActive":false,"balance":"$3,999.56","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Martin Porter","gender":"male","email":"martinporter@chorizon.com","phone":"+1 (895) 580-2304","address":"577 Regent Place, Aguila, Guam, 6554","about":"Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n","registered":"2014-09-20T02:08:30 -02:00","latitude":-88.344273,"longitude":37.964466,"tags":[]} -{"id":9,"isActive":true,"balance":"$3,729.71","picture":"http://placehold.it/32x32","age":26,"color":"blue","name":"Kelli Mendez","gender":"female","email":"kellimendez@chorizon.com","phone":"+1 (936) 401-2236","address":"242 Caton Place, Grazierville, Alabama, 3968","about":"Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n","registered":"2018-05-04T10:35:30 -02:00","latitude":49.37551,"longitude":41.872323,"tags":["new issue","new issue"]} -{"id":10,"isActive":false,"balance":"$1,127.47","picture":"http://placehold.it/32x32","age":27,"color":"blue","name":"Maddox Johns","gender":"male","email":"maddoxjohns@chorizon.com","phone":"+1 (892) 470-2357","address":"756 Beard Street, Avalon, Louisiana, 114","about":"Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n","registered":"2016-04-22T06:41:25 -02:00","latitude":66.640229,"longitude":-17.222666,"tags":["new issue","good first issue","good first issue","new issue"]} -{"id":11,"isActive":true,"balance":"$1,351.43","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Evans Wagner","gender":"male","email":"evanswagner@chorizon.com","phone":"+1 (889) 496-2332","address":"118 Monaco Place, Lutsen, Delaware, 6209","about":"Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n","registered":"2016-10-27T01:26:31 -02:00","latitude":-77.673222,"longitude":-142.657214,"tags":["good first issue","good first issue"]} -{"id":12,"isActive":false,"balance":"$3,394.96","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Aida Kirby","gender":"female","email":"aidakirby@chorizon.com","phone":"+1 (942) 532-2325","address":"797 Engert Avenue, Wilsonia, Idaho, 6532","about":"Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n","registered":"2018-06-18T04:39:57 -02:00","latitude":-58.062041,"longitude":34.999254,"tags":["new issue","wontfix","bug","new issue"]} -{"id":13,"isActive":true,"balance":"$2,812.62","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Nelda Burris","gender":"female","email":"neldaburris@chorizon.com","phone":"+1 (813) 600-2576","address":"160 Opal Court, Fowlerville, Tennessee, 2170","about":"Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n","registered":"2015-08-15T12:39:53 -02:00","latitude":66.6871,"longitude":179.549488,"tags":["wontfix"]} -{"id":14,"isActive":true,"balance":"$1,718.33","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Jennifer Hart","gender":"female","email":"jenniferhart@chorizon.com","phone":"+1 (850) 537-2513","address":"124 Veranda Place, Nash, Utah, 985","about":"Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n","registered":"2016-09-04T11:46:59 -02:00","latitude":-66.827751,"longitude":99.220079,"tags":["wontfix","bug","new issue","new issue"]} -{"id":15,"isActive":false,"balance":"$2,698.16","picture":"http://placehold.it/32x32","age":28,"color":"blue","name":"Aurelia Contreras","gender":"female","email":"aureliacontreras@chorizon.com","phone":"+1 (932) 442-3103","address":"655 Dwight Street, Grapeview, Palau, 8356","about":"Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n","registered":"2014-09-11T10:43:15 -02:00","latitude":-71.328973,"longitude":133.404895,"tags":["wontfix","bug","good first issue"]} -{"id":16,"isActive":true,"balance":"$3,303.25","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Estella Bass","gender":"female","email":"estellabass@chorizon.com","phone":"+1 (825) 436-2909","address":"435 Rockwell Place, Garberville, Wisconsin, 2230","about":"Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n","registered":"2017-11-23T09:32:09 -01:00","latitude":81.17014,"longitude":-145.262693,"tags":["new issue"]} -{"id":17,"isActive":false,"balance":"$3,579.20","picture":"http://placehold.it/32x32","age":25,"color":"brown","name":"Ortega Brennan","gender":"male","email":"ortegabrennan@chorizon.com","phone":"+1 (906) 526-2287","address":"440 Berry Street, Rivera, Maine, 1849","about":"Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n","registered":"2016-03-31T02:17:13 -02:00","latitude":-68.407524,"longitude":-113.642067,"tags":["new issue","wontfix"]} -{"id":18,"isActive":false,"balance":"$1,484.92","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Leonard Tillman","gender":"male","email":"leonardtillman@chorizon.com","phone":"+1 (864) 541-3456","address":"985 Provost Street, Charco, New Hampshire, 8632","about":"Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n","registered":"2018-05-06T08:21:27 -02:00","latitude":-8.581801,"longitude":-61.910062,"tags":["wontfix","new issue","bug","bug"]} -{"id":19,"isActive":true,"balance":"$3,572.55","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Dale Payne","gender":"male","email":"dalepayne@chorizon.com","phone":"+1 (814) 469-3499","address":"536 Dare Court, Ironton, Arkansas, 8605","about":"Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n","registered":"2019-10-11T01:01:33 -02:00","latitude":-18.280968,"longitude":-126.091797,"tags":["bug","wontfix","wontfix","wontfix"]} -{"id":20,"isActive":true,"balance":"$1,986.48","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Florence Long","gender":"female","email":"florencelong@chorizon.com","phone":"+1 (972) 557-3858","address":"519 Hendrickson Street, Templeton, Hawaii, 2389","about":"Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n","registered":"2016-05-02T09:18:59 -02:00","latitude":-27.110866,"longitude":-45.09445,"tags":[]} -{"id":21,"isActive":true,"balance":"$1,440.09","picture":"http://placehold.it/32x32","age":40,"color":"blue","name":"Levy Whitley","gender":"male","email":"levywhitley@chorizon.com","phone":"+1 (911) 458-2411","address":"187 Thomas Street, Hachita, North Carolina, 2989","about":"Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n","registered":"2014-04-30T07:31:38 -02:00","latitude":-6.537315,"longitude":171.813536,"tags":["bug"]} -{"id":22,"isActive":false,"balance":"$2,938.57","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Bernard Mcfarland","gender":"male","email":"bernardmcfarland@chorizon.com","phone":"+1 (979) 442-3386","address":"409 Hall Street, Keyport, Federated States Of Micronesia, 7011","about":"Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n","registered":"2017-08-10T10:07:59 -02:00","latitude":63.766795,"longitude":68.177069,"tags":[]} -{"id":23,"isActive":true,"balance":"$1,678.49","picture":"http://placehold.it/32x32","age":31,"color":"brown","name":"Blanca Mcclain","gender":"female","email":"blancamcclain@chorizon.com","phone":"+1 (976) 439-2772","address":"176 Crooke Avenue, Valle, Virginia, 5373","about":"Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n","registered":"2015-10-12T11:57:28 -02:00","latitude":-8.944564,"longitude":-150.711709,"tags":["bug","wontfix","good first issue"]} -{"id":24,"isActive":true,"balance":"$2,276.87","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Espinoza Ford","gender":"male","email":"espinozaford@chorizon.com","phone":"+1 (945) 429-3975","address":"137 Bowery Street, Itmann, District Of Columbia, 1864","about":"Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n","registered":"2014-03-26T02:16:08 -01:00","latitude":-37.137666,"longitude":-51.811757,"tags":["wontfix","bug"]} -{"id":25,"isActive":true,"balance":"$3,973.43","picture":"http://placehold.it/32x32","age":29,"color":"Green","name":"Sykes Conley","gender":"male","email":"sykesconley@chorizon.com","phone":"+1 (851) 401-3916","address":"345 Grand Street, Woodlands, Missouri, 4461","about":"Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n","registered":"2015-09-12T06:03:56 -02:00","latitude":67.282955,"longitude":-64.341323,"tags":["wontfix"]} -{"id":26,"isActive":false,"balance":"$1,431.50","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Barlow Duran","gender":"male","email":"barlowduran@chorizon.com","phone":"+1 (995) 436-2562","address":"481 Everett Avenue, Allison, Nebraska, 3065","about":"Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n","registered":"2017-06-29T04:28:43 -02:00","latitude":-38.70606,"longitude":55.02816,"tags":["new issue"]} -{"id":27,"isActive":true,"balance":"$3,478.27","picture":"http://placehold.it/32x32","age":31,"color":"blue","name":"Schwartz Morgan","gender":"male","email":"schwartzmorgan@chorizon.com","phone":"+1 (861) 507-2067","address":"451 Lincoln Road, Fairlee, Washington, 2717","about":"Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n","registered":"2016-05-10T08:34:54 -02:00","latitude":-75.886403,"longitude":93.044471,"tags":["bug","bug","wontfix","wontfix"]} -{"id":28,"isActive":true,"balance":"$2,825.59","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Kristy Leon","gender":"female","email":"kristyleon@chorizon.com","phone":"+1 (948) 465-2563","address":"594 Macon Street, Floris, South Dakota, 3565","about":"Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n","registered":"2014-12-14T04:10:29 -01:00","latitude":-50.01615,"longitude":-68.908804,"tags":["wontfix","good first issue"]} -{"id":29,"isActive":false,"balance":"$3,028.03","picture":"http://placehold.it/32x32","age":39,"color":"blue","name":"Ashley Pittman","gender":"male","email":"ashleypittman@chorizon.com","phone":"+1 (928) 507-3523","address":"646 Adelphi Street, Clara, Colorado, 6056","about":"Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n","registered":"2016-01-07T10:40:48 -01:00","latitude":-58.766037,"longitude":-124.828485,"tags":["wontfix"]} -{"id":30,"isActive":true,"balance":"$2,021.11","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Stacy Espinoza","gender":"female","email":"stacyespinoza@chorizon.com","phone":"+1 (999) 487-3253","address":"931 Alabama Avenue, Bangor, Alaska, 8215","about":"Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n","registered":"2014-07-16T06:15:53 -02:00","latitude":41.560197,"longitude":177.697,"tags":["new issue","new issue","bug"]} -{"id":31,"isActive":false,"balance":"$3,609.82","picture":"http://placehold.it/32x32","age":32,"color":"blue","name":"Vilma Garza","gender":"female","email":"vilmagarza@chorizon.com","phone":"+1 (944) 585-2021","address":"565 Tech Place, Sedley, Puerto Rico, 858","about":"Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n","registered":"2017-06-30T07:43:52 -02:00","latitude":-12.574889,"longitude":-54.771186,"tags":["new issue","wontfix","wontfix"]} -{"id":32,"isActive":false,"balance":"$2,882.34","picture":"http://placehold.it/32x32","age":38,"color":"brown","name":"June Dunlap","gender":"female","email":"junedunlap@chorizon.com","phone":"+1 (997) 504-2937","address":"353 Cozine Avenue, Goodville, Indiana, 1438","about":"Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n","registered":"2016-08-23T08:54:11 -02:00","latitude":-27.883363,"longitude":-163.919683,"tags":["new issue","new issue","bug","wontfix"]} -{"id":33,"isActive":true,"balance":"$3,556.54","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Cecilia Greer","gender":"female","email":"ceciliagreer@chorizon.com","phone":"+1 (977) 573-3498","address":"696 Withers Street, Lydia, Oklahoma, 3220","about":"Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n","registered":"2017-01-13T11:30:12 -01:00","latitude":60.467215,"longitude":84.684575,"tags":["wontfix","good first issue","good first issue","wontfix"]} -{"id":34,"isActive":true,"balance":"$1,413.35","picture":"http://placehold.it/32x32","age":33,"color":"brown","name":"Mckay Schroeder","gender":"male","email":"mckayschroeder@chorizon.com","phone":"+1 (816) 480-3657","address":"958 Miami Court, Rehrersburg, Northern Mariana Islands, 567","about":"Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n","registered":"2016-02-08T04:50:15 -01:00","latitude":-72.413287,"longitude":-159.254371,"tags":["good first issue"]} -{"id":35,"isActive":true,"balance":"$2,306.53","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Sawyer Mccormick","gender":"male","email":"sawyermccormick@chorizon.com","phone":"+1 (829) 569-3012","address":"749 Apollo Street, Eastvale, Texas, 7373","about":"Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n","registered":"2019-11-30T11:53:23 -01:00","latitude":-48.978194,"longitude":110.950191,"tags":["good first issue","new issue","new issue","bug"]} -{"id":36,"isActive":false,"balance":"$1,844.54","picture":"http://placehold.it/32x32","age":37,"color":"brown","name":"Barbra Valenzuela","gender":"female","email":"barbravalenzuela@chorizon.com","phone":"+1 (992) 512-2649","address":"617 Schenck Court, Reinerton, Michigan, 2908","about":"Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n","registered":"2019-03-29T01:59:31 -01:00","latitude":45.193723,"longitude":-12.486778,"tags":["new issue","new issue","wontfix","wontfix"]} -{"id":37,"isActive":false,"balance":"$3,469.82","picture":"http://placehold.it/32x32","age":39,"color":"brown","name":"Opal Weiss","gender":"female","email":"opalweiss@chorizon.com","phone":"+1 (809) 400-3079","address":"535 Bogart Street, Frizzleburg, Arizona, 5222","about":"Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n","registered":"2019-09-04T07:22:28 -02:00","latitude":72.50376,"longitude":61.656435,"tags":["bug","bug","good first issue","good first issue"]} -{"id":38,"isActive":true,"balance":"$1,992.38","picture":"http://placehold.it/32x32","age":40,"color":"Green","name":"Christina Short","gender":"female","email":"christinashort@chorizon.com","phone":"+1 (884) 589-2705","address":"594 Willmohr Street, Dexter, Montana, 660","about":"Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n","registered":"2014-01-21T09:31:56 -01:00","latitude":-42.762739,"longitude":77.052349,"tags":["bug","new issue"]} -{"id":39,"isActive":false,"balance":"$1,722.85","picture":"http://placehold.it/32x32","age":29,"color":"brown","name":"Golden Horton","gender":"male","email":"goldenhorton@chorizon.com","phone":"+1 (903) 426-2489","address":"191 Schenck Avenue, Mayfair, North Dakota, 5000","about":"Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n","registered":"2015-08-19T02:56:41 -02:00","latitude":69.922534,"longitude":9.881433,"tags":["bug"]} -{"id":40,"isActive":false,"balance":"$1,656.54","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Stafford Emerson","gender":"male","email":"staffordemerson@chorizon.com","phone":"+1 (992) 455-2573","address":"523 Thornton Street, Conway, Vermont, 6331","about":"Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n","registered":"2019-02-16T04:07:08 -01:00","latitude":-29.143111,"longitude":-57.207703,"tags":["wontfix","good first issue","good first issue"]} -{"id":41,"isActive":false,"balance":"$1,861.56","picture":"http://placehold.it/32x32","age":21,"color":"brown","name":"Salinas Gamble","gender":"male","email":"salinasgamble@chorizon.com","phone":"+1 (901) 525-2373","address":"991 Nostrand Avenue, Kansas, Mississippi, 6756","about":"Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n","registered":"2017-08-21T05:47:53 -02:00","latitude":-22.593819,"longitude":-63.613004,"tags":["good first issue","bug","bug","wontfix"]} -{"id":42,"isActive":true,"balance":"$3,179.74","picture":"http://placehold.it/32x32","age":34,"color":"brown","name":"Graciela Russell","gender":"female","email":"gracielarussell@chorizon.com","phone":"+1 (893) 464-3951","address":"361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713","about":"Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n","registered":"2015-05-18T09:52:56 -02:00","latitude":-14.634444,"longitude":12.931783,"tags":["wontfix","bug","wontfix"]} -{"id":43,"isActive":true,"balance":"$1,777.38","picture":"http://placehold.it/32x32","age":25,"color":"blue","name":"Arnold Bender","gender":"male","email":"arnoldbender@chorizon.com","phone":"+1 (945) 581-3808","address":"781 Lorraine Street, Gallina, American Samoa, 1832","about":"Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n","registered":"2018-12-23T02:26:30 -01:00","latitude":41.208579,"longitude":51.948925,"tags":["bug","good first issue","good first issue","wontfix"]} -{"id":44,"isActive":true,"balance":"$2,893.45","picture":"http://placehold.it/32x32","age":22,"color":"Green","name":"Joni Spears","gender":"female","email":"jonispears@chorizon.com","phone":"+1 (916) 565-2124","address":"307 Harwood Place, Canterwood, Maryland, 2047","about":"Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n","registered":"2015-03-01T12:38:28 -01:00","latitude":8.19071,"longitude":146.323808,"tags":["wontfix","new issue","good first issue","good first issue"]} -{"id":45,"isActive":true,"balance":"$2,830.36","picture":"http://placehold.it/32x32","age":20,"color":"brown","name":"Irene Bennett","gender":"female","email":"irenebennett@chorizon.com","phone":"+1 (904) 431-2211","address":"353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686","about":"Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n","registered":"2018-04-17T05:18:51 -02:00","latitude":-36.435177,"longitude":-127.552573,"tags":["bug","wontfix"]} -{"id":46,"isActive":true,"balance":"$1,348.04","picture":"http://placehold.it/32x32","age":34,"color":"Green","name":"Lawson Curtis","gender":"male","email":"lawsoncurtis@chorizon.com","phone":"+1 (896) 532-2172","address":"942 Gerritsen Avenue, Southmont, Kansas, 8915","about":"Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n","registered":"2016-08-23T01:41:09 -02:00","latitude":-48.783539,"longitude":20.492944,"tags":[]} -{"id":47,"isActive":true,"balance":"$1,132.41","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goff May","gender":"male","email":"goffmay@chorizon.com","phone":"+1 (859) 453-3415","address":"225 Rutledge Street, Boonville, Massachusetts, 4081","about":"Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n","registered":"2014-10-25T07:32:30 -02:00","latitude":13.079225,"longitude":76.215086,"tags":["bug"]} -{"id":48,"isActive":true,"balance":"$1,201.87","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Goodman Becker","gender":"male","email":"goodmanbecker@chorizon.com","phone":"+1 (825) 470-3437","address":"388 Seigel Street, Sisquoc, Kentucky, 8231","about":"Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n","registered":"2019-09-05T04:49:03 -02:00","latitude":-23.792094,"longitude":-13.621221,"tags":["bug","bug","wontfix","new issue"]} -{"id":49,"isActive":true,"balance":"$1,476.39","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Maureen Dale","gender":"female","email":"maureendale@chorizon.com","phone":"+1 (984) 538-3684","address":"817 Newton Street, Bannock, Wyoming, 1468","about":"Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n","registered":"2018-04-26T06:04:40 -02:00","latitude":-64.196802,"longitude":-117.396238,"tags":["wontfix"]} -{"id":50,"isActive":true,"balance":"$1,947.08","picture":"http://placehold.it/32x32","age":21,"color":"Green","name":"Guerra Mcintyre","gender":"male","email":"guerramcintyre@chorizon.com","phone":"+1 (951) 536-2043","address":"423 Lombardy Street, Stewart, West Virginia, 908","about":"Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n","registered":"2015-07-16T05:11:42 -02:00","latitude":79.733743,"longitude":-20.602356,"tags":["bug","good first issue","good first issue"]} -{"id":51,"isActive":true,"balance":"$2,960.90","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Key Cervantes","gender":"male","email":"keycervantes@chorizon.com","phone":"+1 (931) 474-3865","address":"410 Barbey Street, Vernon, Oregon, 2328","about":"Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n","registered":"2019-12-15T12:13:35 -01:00","latitude":47.627647,"longitude":117.049918,"tags":["new issue"]} -{"id":52,"isActive":false,"balance":"$1,884.02","picture":"http://placehold.it/32x32","age":35,"color":"blue","name":"Karen Nelson","gender":"female","email":"karennelson@chorizon.com","phone":"+1 (993) 528-3607","address":"930 Frank Court, Dunbar, New York, 8810","about":"Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n","registered":"2014-06-23T09:21:44 -02:00","latitude":-59.059033,"longitude":76.565373,"tags":["new issue","bug"]} -{"id":53,"isActive":true,"balance":"$3,559.55","picture":"http://placehold.it/32x32","age":32,"color":"brown","name":"Caitlin Burnett","gender":"female","email":"caitlinburnett@chorizon.com","phone":"+1 (945) 480-2796","address":"516 Senator Street, Emory, Iowa, 4145","about":"In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n","registered":"2019-01-09T02:26:31 -01:00","latitude":-82.774237,"longitude":42.316194,"tags":["bug","good first issue"]} -{"id":54,"isActive":true,"balance":"$2,113.29","picture":"http://placehold.it/32x32","age":28,"color":"Green","name":"Richards Walls","gender":"male","email":"richardswalls@chorizon.com","phone":"+1 (865) 517-2982","address":"959 Brightwater Avenue, Stevens, Nevada, 2968","about":"Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n","registered":"2014-09-25T06:51:22 -02:00","latitude":80.09202,"longitude":87.49759,"tags":["wontfix","wontfix","bug"]} -{"id":55,"isActive":true,"balance":"$1,977.66","picture":"http://placehold.it/32x32","age":36,"color":"brown","name":"Combs Stanley","gender":"male","email":"combsstanley@chorizon.com","phone":"+1 (827) 419-2053","address":"153 Beverley Road, Siglerville, South Carolina, 3666","about":"Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n","registered":"2019-08-22T07:53:15 -02:00","latitude":78.386181,"longitude":143.661058,"tags":[]} -{"id":56,"isActive":false,"balance":"$3,886.12","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Tucker Barry","gender":"male","email":"tuckerbarry@chorizon.com","phone":"+1 (808) 544-3433","address":"805 Jamaica Avenue, Cornfields, Minnesota, 3689","about":"Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n","registered":"2016-08-29T07:28:00 -02:00","latitude":71.701551,"longitude":9.903068,"tags":[]} -{"id":57,"isActive":false,"balance":"$1,844.56","picture":"http://placehold.it/32x32","age":20,"color":"Green","name":"Kaitlin Conner","gender":"female","email":"kaitlinconner@chorizon.com","phone":"+1 (862) 467-2666","address":"501 Knight Court, Joppa, Rhode Island, 274","about":"Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n","registered":"2019-05-30T06:38:24 -02:00","latitude":15.613464,"longitude":171.965629,"tags":[]} -{"id":58,"isActive":true,"balance":"$2,876.10","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Mamie Fischer","gender":"female","email":"mamiefischer@chorizon.com","phone":"+1 (948) 545-3901","address":"599 Hunterfly Place, Haena, Georgia, 6005","about":"Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n","registered":"2019-05-27T05:07:10 -02:00","latitude":70.915079,"longitude":-48.813584,"tags":["bug","wontfix","wontfix","good first issue"]} -{"id":59,"isActive":true,"balance":"$1,921.58","picture":"http://placehold.it/32x32","age":31,"color":"Green","name":"Harper Carson","gender":"male","email":"harpercarson@chorizon.com","phone":"+1 (912) 430-3243","address":"883 Dennett Place, Knowlton, New Mexico, 9219","about":"Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n","registered":"2019-12-07T07:33:15 -01:00","latitude":-60.812605,"longitude":-27.129016,"tags":["bug","new issue"]} -{"id":60,"isActive":true,"balance":"$1,770.93","picture":"http://placehold.it/32x32","age":23,"color":"brown","name":"Jody Herrera","gender":"female","email":"jodyherrera@chorizon.com","phone":"+1 (890) 583-3222","address":"261 Jay Street, Strykersville, Ohio, 9248","about":"Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n","registered":"2016-05-21T01:00:02 -02:00","latitude":-36.846586,"longitude":131.156223,"tags":[]} -{"id":61,"isActive":false,"balance":"$2,813.41","picture":"http://placehold.it/32x32","age":37,"color":"Green","name":"Charles Castillo","gender":"male","email":"charlescastillo@chorizon.com","phone":"+1 (934) 467-2108","address":"675 Morton Street, Rew, Pennsylvania, 137","about":"Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n","registered":"2019-06-10T02:54:22 -02:00","latitude":-16.423202,"longitude":-146.293752,"tags":["new issue","new issue"]} -{"id":62,"isActive":true,"balance":"$3,341.35","picture":"http://placehold.it/32x32","age":33,"color":"blue","name":"Estelle Ramirez","gender":"female","email":"estelleramirez@chorizon.com","phone":"+1 (816) 459-2073","address":"636 Nolans Lane, Camptown, California, 7794","about":"Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n","registered":"2015-02-14T01:05:50 -01:00","latitude":-46.591249,"longitude":-83.385587,"tags":["good first issue","bug"]} -{"id":63,"isActive":true,"balance":"$2,478.30","picture":"http://placehold.it/32x32","age":21,"color":"blue","name":"Knowles Hebert","gender":"male","email":"knowleshebert@chorizon.com","phone":"+1 (819) 409-2308","address":"361 Kathleen Court, Gratton, Connecticut, 7254","about":"Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n","registered":"2016-03-08T08:34:52 -01:00","latitude":71.042482,"longitude":152.460406,"tags":["good first issue","wontfix"]} -{"id":64,"isActive":false,"balance":"$2,559.09","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Thelma Mckenzie","gender":"female","email":"thelmamckenzie@chorizon.com","phone":"+1 (941) 596-2777","address":"202 Leonard Street, Riverton, Illinois, 8577","about":"Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n","registered":"2020-04-14T12:43:06 -02:00","latitude":16.026129,"longitude":105.464476,"tags":[]} -{"id":65,"isActive":true,"balance":"$1,025.08","picture":"http://placehold.it/32x32","age":34,"color":"blue","name":"Carole Rowland","gender":"female","email":"carolerowland@chorizon.com","phone":"+1 (862) 558-3448","address":"941 Melba Court, Bluetown, Florida, 9555","about":"Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n","registered":"2014-12-01T05:55:35 -01:00","latitude":-0.191998,"longitude":43.389652,"tags":["wontfix"]} -{"id":66,"isActive":true,"balance":"$1,061.49","picture":"http://placehold.it/32x32","age":35,"color":"brown","name":"Higgins Aguilar","gender":"male","email":"higginsaguilar@chorizon.com","phone":"+1 (911) 540-3791","address":"132 Sackman Street, Layhill, Guam, 8729","about":"Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n","registered":"2015-04-05T02:10:07 -02:00","latitude":74.702813,"longitude":151.314972,"tags":["bug"]} -{"id":67,"isActive":true,"balance":"$3,510.14","picture":"http://placehold.it/32x32","age":28,"color":"brown","name":"Ilene Gillespie","gender":"female","email":"ilenegillespie@chorizon.com","phone":"+1 (937) 575-2676","address":"835 Lake Street, Naomi, Alabama, 4131","about":"Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n","registered":"2015-06-28T09:41:45 -02:00","latitude":71.573342,"longitude":-95.295989,"tags":["wontfix","wontfix"]} -{"id":68,"isActive":false,"balance":"$1,539.98","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Angelina Dyer","gender":"female","email":"angelinadyer@chorizon.com","phone":"+1 (948) 574-3949","address":"575 Division Place, Gorham, Louisiana, 3458","about":"Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n","registered":"2014-07-08T06:34:36 -02:00","latitude":-85.649593,"longitude":66.126018,"tags":["good first issue"]} -{"id":69,"isActive":true,"balance":"$3,367.69","picture":"http://placehold.it/32x32","age":30,"color":"brown","name":"Marks Burt","gender":"male","email":"marksburt@chorizon.com","phone":"+1 (895) 497-3138","address":"819 Village Road, Wadsworth, Delaware, 6099","about":"Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n","registered":"2014-08-31T06:12:18 -02:00","latitude":26.854112,"longitude":-143.313948,"tags":["good first issue"]} -{"id":70,"isActive":false,"balance":"$3,755.72","picture":"http://placehold.it/32x32","age":23,"color":"blue","name":"Glass Perkins","gender":"male","email":"glassperkins@chorizon.com","phone":"+1 (923) 486-3725","address":"899 Roosevelt Court, Belleview, Idaho, 1737","about":"Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n","registered":"2015-05-22T05:44:33 -02:00","latitude":54.27147,"longitude":-65.065604,"tags":["wontfix"]} -{"id":71,"isActive":true,"balance":"$3,381.63","picture":"http://placehold.it/32x32","age":38,"color":"Green","name":"Candace Sawyer","gender":"female","email":"candacesawyer@chorizon.com","phone":"+1 (830) 404-2636","address":"334 Arkansas Drive, Bordelonville, Tennessee, 8449","about":"Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n","registered":"2014-04-04T08:45:00 -02:00","latitude":6.484262,"longitude":-37.054928,"tags":["new issue","new issue"]} -{"id":72,"isActive":true,"balance":"$1,640.98","picture":"http://placehold.it/32x32","age":27,"color":"Green","name":"Hendricks Martinez","gender":"male","email":"hendricksmartinez@chorizon.com","phone":"+1 (857) 566-3245","address":"636 Agate Court, Newry, Utah, 3304","about":"Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n","registered":"2018-06-15T10:36:11 -02:00","latitude":86.746034,"longitude":10.347893,"tags":["new issue"]} -{"id":73,"isActive":false,"balance":"$1,239.74","picture":"http://placehold.it/32x32","age":38,"color":"blue","name":"Eleanor Shepherd","gender":"female","email":"eleanorshepherd@chorizon.com","phone":"+1 (894) 567-2617","address":"670 Lafayette Walk, Darlington, Palau, 8803","about":"Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n","registered":"2020-02-29T12:15:28 -01:00","latitude":35.749621,"longitude":-94.40842,"tags":["good first issue","new issue","new issue","bug"]} -{"id":74,"isActive":true,"balance":"$1,180.90","picture":"http://placehold.it/32x32","age":36,"color":"Green","name":"Stark Wong","gender":"male","email":"starkwong@chorizon.com","phone":"+1 (805) 575-3055","address":"522 Bond Street, Bawcomville, Wisconsin, 324","about":"Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n","registered":"2020-01-25T10:47:48 -01:00","latitude":-80.452139,"longitude":160.72546,"tags":["wontfix"]} -{"id":75,"isActive":false,"balance":"$1,913.42","picture":"http://placehold.it/32x32","age":24,"color":"Green","name":"Emma Jacobs","gender":"female","email":"emmajacobs@chorizon.com","phone":"+1 (899) 554-3847","address":"173 Tapscott Street, Esmont, Maine, 7450","about":"Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n","registered":"2019-03-29T06:24:13 -01:00","latitude":-35.53722,"longitude":155.703874,"tags":[]} -{"id":76,"isActive":false,"balance":"$1,274.29","picture":"http://placehold.it/32x32","age":25,"color":"Green","name":"Clarice Gardner","gender":"female","email":"claricegardner@chorizon.com","phone":"+1 (810) 407-3258","address":"894 Brooklyn Road, Utting, New Hampshire, 6404","about":"Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n","registered":"2014-10-20T10:13:32 -02:00","latitude":17.11935,"longitude":65.38197,"tags":["new issue","wontfix"]} \ No newline at end of file diff --git a/tests/assets/dumps/v1/test/settings.json b/tests/assets/dumps/v1/test/settings.json deleted file mode 100644 index 918cfab53..000000000 --- a/tests/assets/dumps/v1/test/settings.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": "email", - "searchableAttributes": [ - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags" - ], - "displayedAttributes": [ - "id", - "isActive", - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags" - ], - "stopWords": [ - "in", - "ad" - ], - "synonyms": { - "wolverine": ["xmen", "logan"], - "logan": ["wolverine", "xmen"] - }, - "attributesForFaceting": [ - "gender", - "color", - "tags", - "isActive" - ] -} diff --git a/tests/assets/dumps/v1/test/updates.jsonl b/tests/assets/dumps/v1/test/updates.jsonl deleted file mode 100644 index 0dcffdce0..000000000 --- a/tests/assets/dumps/v1/test/updates.jsonl +++ /dev/null @@ -1,2 +0,0 @@ -{"status": "processed","updateId": 0,"type": {"name":"Settings","settings":{"ranking_rules":{"Update":["Typo","Words","Proximity","Attribute","WordsPosition","Exactness"]},"distinct_attribute":"Nothing","primary_key":"Nothing","searchable_attributes":{"Update":["balance","picture","age","color","name","gender","email","phone","address","about","registered","latitude","longitude","tags"]},"displayed_attributes":{"Update":["about","address","age","balance","color","email","gender","id","isActive","latitude","longitude","name","phone","picture","registered","tags"]},"stop_words":"Nothing","synonyms":"Nothing","attributes_for_faceting":"Nothing"}}} -{"status": "processed", "updateId": 1, "type": { "name": "DocumentsAddition"}} \ No newline at end of file diff --git a/tests/assets/test_set.json b/tests/assets/test_set.json deleted file mode 100644 index 63534c896..000000000 --- a/tests/assets/test_set.json +++ /dev/null @@ -1,1613 +0,0 @@ -[ - { - "id": 0, - "isActive": false, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ] - }, - { - "id": 1, - "isActive": true, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ] - }, - { - "id": 2, - "isActive": true, - "balance": "$2,467.47", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", - "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", - "registered": "2014-10-28T12:59:30 -01:00", - "latitude": -64.008555, - "longitude": 11.867098, - "tags": [ - "good first issue" - ] - }, - { - "id": 3, - "isActive": true, - "balance": "$3,344.40", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "blue", - "name": "Adeline Flynn", - "gender": "female", - "email": "adelineflynn@chorizon.com", - "phone": "+1 (994) 600-2840", - "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948", - "about": "Ex velit magna minim labore dolor id laborum incididunt. Proident dolor fugiat exercitation ad adipisicing amet dolore. Veniam nisi pariatur aute eu amet sint elit duis exercitation. Eu fugiat Lorem nostrud consequat aute sunt. Minim excepteur cillum laboris enim tempor adipisicing nulla reprehenderit ea velit Lorem qui in incididunt. Esse ipsum mollit deserunt ea exercitation ex aliqua anim magna cupidatat culpa.\r\n", - "registered": "2014-03-27T06:24:45 -01:00", - "latitude": -74.485173, - "longitude": -11.059859, - "tags": [ - "bug", - "good first issue", - "wontfix", - "new issue" - ] - }, - { - "id": 4, - "isActive": false, - "balance": "$2,575.78", - "picture": "http://placehold.it/32x32", - "age": 39, - "color": "Green", - "name": "Mariana Pacheco", - "gender": "female", - "email": "marianapacheco@chorizon.com", - "phone": "+1 (820) 414-2223", - "address": "664 Rapelye Street, Faywood, California, 7320", - "about": "Sint cillum enim eu Lorem dolore. Est excepteur cillum consequat incididunt. Ut consectetur et do culpa eiusmod ex ut id proident aliqua. Sunt dolor anim minim labore incididunt deserunt enim velit sunt ut in velit. Nulla ipsum cillum qui est minim officia in occaecat exercitation Lorem sunt. Aliqua minim excepteur tempor incididunt dolore. Quis amet ullamco et proident aliqua magna consequat.\r\n", - "registered": "2015-09-02T03:23:35 -02:00", - "latitude": 75.763501, - "longitude": -78.777124, - "tags": [ - "new issue" - ] - }, - { - "id": 5, - "isActive": true, - "balance": "$3,793.09", - "picture": "http://placehold.it/32x32", - "age": 20, - "color": "Green", - "name": "Warren Watson", - "gender": "male", - "email": "warrenwatson@chorizon.com", - "phone": "+1 (807) 583-2427", - "address": "671 Prince Street, Faxon, Connecticut, 4275", - "about": "Cillum incididunt mollit labore ipsum elit ea. Lorem labore consectetur nulla ea fugiat sint esse cillum ea commodo id qui. Sint cillum mollit dolore enim quis esse. Nisi labore duis dolor tempor laborum laboris ad minim pariatur in excepteur sit. Aliqua anim amet sunt ullamco labore amet culpa irure esse eiusmod deserunt consequat Lorem nostrud.\r\n", - "registered": "2017-06-04T06:02:17 -02:00", - "latitude": 29.979223, - "longitude": 25.358943, - "tags": [ - "wontfix", - "wontfix", - "wontfix" - ] - }, - { - "id": 6, - "isActive": true, - "balance": "$2,919.70", - "picture": "http://placehold.it/32x32", - "age": 20, - "color": "blue", - "name": "Shelia Berry", - "gender": "female", - "email": "sheliaberry@chorizon.com", - "phone": "+1 (853) 511-2651", - "address": "437 Forrest Street, Coventry, Illinois, 2056", - "about": "Id occaecat qui voluptate proident culpa cillum nisi reprehenderit. Pariatur nostrud proident adipisicing reprehenderit eiusmod qui minim proident aliqua id cupidatat laboris deserunt. Proident sint laboris sit mollit dolor qui incididunt quis veniam cillum cupidatat ad nostrud ut. Aliquip consequat eiusmod eiusmod irure tempor do incididunt id culpa laboris eiusmod.\r\n", - "registered": "2018-07-11T02:45:01 -02:00", - "latitude": 54.815991, - "longitude": -118.690609, - "tags": [ - "good first issue", - "bug", - "wontfix", - "new issue" - ] - }, - { - "id": 7, - "isActive": true, - "balance": "$1,349.50", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "Green", - "name": "Chrystal Boyd", - "gender": "female", - "email": "chrystalboyd@chorizon.com", - "phone": "+1 (936) 563-2802", - "address": "670 Croton Loop, Sussex, Florida, 4692", - "about": "Consequat ex voluptate consectetur laborum nulla. Qui voluptate Lorem amet labore est esse sunt. Nulla cupidatat consequat quis incididunt exercitation aliquip reprehenderit ea ea adipisicing reprehenderit id consectetur quis. Exercitation est incididunt ullamco non proident consequat. Nisi veniam aliquip fugiat voluptate ex id aute duis ullamco magna ipsum ad laborum ipsum. Cupidatat velit dolore esse nisi.\r\n", - "registered": "2016-11-01T07:36:04 -01:00", - "latitude": -24.711933, - "longitude": 147.246705, - "tags": [] - }, - { - "id": 8, - "isActive": false, - "balance": "$3,999.56", - "picture": "http://placehold.it/32x32", - "age": 30, - "color": "brown", - "name": "Martin Porter", - "gender": "male", - "email": "martinporter@chorizon.com", - "phone": "+1 (895) 580-2304", - "address": "577 Regent Place, Aguila, Guam, 6554", - "about": "Nostrud nulla labore ex excepteur labore enim cillum pariatur in do Lorem eiusmod ullamco est. Labore aliquip id ut nisi commodo pariatur ea esse laboris. Incididunt eu dolor esse excepteur nulla minim proident non cillum nisi dolore incididunt ipsum tempor.\r\n", - "registered": "2014-09-20T02:08:30 -02:00", - "latitude": -88.344273, - "longitude": 37.964466, - "tags": [] - }, - { - "id": 9, - "isActive": true, - "balance": "$3,729.71", - "picture": "http://placehold.it/32x32", - "age": 26, - "color": "blue", - "name": "Kelli Mendez", - "gender": "female", - "email": "kellimendez@chorizon.com", - "phone": "+1 (936) 401-2236", - "address": "242 Caton Place, Grazierville, Alabama, 3968", - "about": "Consectetur occaecat dolore esse eiusmod enim ea aliqua eiusmod amet velit laborum. Velit quis consequat consectetur velit fugiat labore commodo amet do. Magna minim est ad commodo consequat fugiat. Laboris duis Lorem ipsum irure sit ipsum consequat tempor sit. Est ad nulla duis quis velit anim id nulla. Cupidatat ea esse laboris eu veniam cupidatat proident veniam quis.\r\n", - "registered": "2018-05-04T10:35:30 -02:00", - "latitude": 49.37551, - "longitude": 41.872323, - "tags": [ - "new issue", - "new issue" - ] - }, - { - "id": 10, - "isActive": false, - "balance": "$1,127.47", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "blue", - "name": "Maddox Johns", - "gender": "male", - "email": "maddoxjohns@chorizon.com", - "phone": "+1 (892) 470-2357", - "address": "756 Beard Street, Avalon, Louisiana, 114", - "about": "Voluptate et dolor magna do do. Id do enim ut nulla esse culpa fugiat excepteur quis. Nostrud ad aliquip aliqua qui esse ut consequat proident deserunt esse cupidatat do elit fugiat. Sint cillum aliquip cillum laboris laborum laboris ad aliquip enim reprehenderit cillum eu sint. Sint ut ad duis do culpa non eiusmod amet non ipsum commodo. Pariatur aliquip sit deserunt non. Ut consequat pariatur deserunt veniam est sit eiusmod officia aliquip commodo sunt in eu duis.\r\n", - "registered": "2016-04-22T06:41:25 -02:00", - "latitude": 66.640229, - "longitude": -17.222666, - "tags": [ - "new issue", - "good first issue", - "good first issue", - "new issue" - ] - }, - { - "id": 11, - "isActive": true, - "balance": "$1,351.43", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "Green", - "name": "Evans Wagner", - "gender": "male", - "email": "evanswagner@chorizon.com", - "phone": "+1 (889) 496-2332", - "address": "118 Monaco Place, Lutsen, Delaware, 6209", - "about": "Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n", - "registered": "2016-10-27T01:26:31 -02:00", - "latitude": -77.673222, - "longitude": -142.657214, - "tags": [ - "good first issue", - "good first issue" - ] - }, - { - "id": 12, - "isActive": false, - "balance": "$3,394.96", - "picture": "http://placehold.it/32x32", - "age": 25, - "color": "blue", - "name": "Aida Kirby", - "gender": "female", - "email": "aidakirby@chorizon.com", - "phone": "+1 (942) 532-2325", - "address": "797 Engert Avenue, Wilsonia, Idaho, 6532", - "about": "Mollit aute esse Lorem do laboris anim reprehenderit excepteur. Ipsum culpa esse voluptate officia cupidatat minim. Velit officia proident nostrud sunt irure labore. Culpa ex commodo amet dolor amet voluptate Lorem ex esse commodo fugiat quis non. Ex est adipisicing veniam sunt dolore ut aliqua nisi ex sit. Esse voluptate esse anim id adipisicing enim aute ea exercitation tempor cillum.\r\n", - "registered": "2018-06-18T04:39:57 -02:00", - "latitude": -58.062041, - "longitude": 34.999254, - "tags": [ - "new issue", - "wontfix", - "bug", - "new issue" - ] - }, - { - "id": 13, - "isActive": true, - "balance": "$2,812.62", - "picture": "http://placehold.it/32x32", - "age": 40, - "color": "blue", - "name": "Nelda Burris", - "gender": "female", - "email": "neldaburris@chorizon.com", - "phone": "+1 (813) 600-2576", - "address": "160 Opal Court, Fowlerville, Tennessee, 2170", - "about": "Ipsum aliquip adipisicing elit magna. Veniam irure quis laborum laborum sint velit amet. Irure non eiusmod laborum fugiat qui quis Lorem culpa veniam commodo. Fugiat cupidatat dolore et consequat pariatur enim ex velit consequat deserunt quis. Deserunt et quis laborum cupidatat cillum minim cupidatat nisi do commodo commodo labore cupidatat ea. In excepteur sit nostrud nulla nostrud dolor sint. Et anim culpa aliquip laborum Lorem elit.\r\n", - "registered": "2015-08-15T12:39:53 -02:00", - "latitude": 66.6871, - "longitude": 179.549488, - "tags": [ - "wontfix" - ] - }, - { - "id": 14, - "isActive": true, - "balance": "$1,718.33", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "blue", - "name": "Jennifer Hart", - "gender": "female", - "email": "jenniferhart@chorizon.com", - "phone": "+1 (850) 537-2513", - "address": "124 Veranda Place, Nash, Utah, 985", - "about": "Amet amet voluptate in occaecat pariatur. Nulla ipsum esse quis qui in quis qui. Non est non nisi qui tempor commodo consequat fugiat. Sint eu ipsum aute anim anim. Ea nostrud excepteur exercitation consectetur Lorem.\r\n", - "registered": "2016-09-04T11:46:59 -02:00", - "latitude": -66.827751, - "longitude": 99.220079, - "tags": [ - "wontfix", - "bug", - "new issue", - "new issue" - ] - }, - { - "id": 15, - "isActive": false, - "balance": "$2,698.16", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "blue", - "name": "Aurelia Contreras", - "gender": "female", - "email": "aureliacontreras@chorizon.com", - "phone": "+1 (932) 442-3103", - "address": "655 Dwight Street, Grapeview, Palau, 8356", - "about": "Qui adipisicing consectetur aute veniam culpa ipsum. Occaecat occaecat ut mollit enim enim elit Lorem nostrud Lorem. Consequat laborum mollit nulla aute cillum sunt mollit commodo velit culpa. Pariatur pariatur velit nostrud tempor. In minim enim cillum exercitation in laboris labore ea sunt in incididunt fugiat.\r\n", - "registered": "2014-09-11T10:43:15 -02:00", - "latitude": -71.328973, - "longitude": 133.404895, - "tags": [ - "wontfix", - "bug", - "good first issue" - ] - }, - { - "id": 16, - "isActive": true, - "balance": "$3,303.25", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Estella Bass", - "gender": "female", - "email": "estellabass@chorizon.com", - "phone": "+1 (825) 436-2909", - "address": "435 Rockwell Place, Garberville, Wisconsin, 2230", - "about": "Sit eiusmod mollit velit non. Qui ea in exercitation elit reprehenderit occaecat tempor minim officia. Culpa amet voluptate sit eiusmod pariatur.\r\n", - "registered": "2017-11-23T09:32:09 -01:00", - "latitude": 81.17014, - "longitude": -145.262693, - "tags": [ - "new issue" - ] - }, - { - "id": 17, - "isActive": false, - "balance": "$3,579.20", - "picture": "http://placehold.it/32x32", - "age": 25, - "color": "brown", - "name": "Ortega Brennan", - "gender": "male", - "email": "ortegabrennan@chorizon.com", - "phone": "+1 (906) 526-2287", - "address": "440 Berry Street, Rivera, Maine, 1849", - "about": "Veniam velit non laboris consectetur sit aliquip enim proident velit in ipsum reprehenderit reprehenderit. Dolor qui nulla adipisicing ad magna dolore do ut duis et aute est. Qui est elit cupidatat nostrud. Laboris voluptate reprehenderit minim sint exercitation cupidatat ipsum sint consectetur velit sunt et officia incididunt. Ut amet Lorem minim deserunt officia officia irure qui et Lorem deserunt culpa sit.\r\n", - "registered": "2016-03-31T02:17:13 -02:00", - "latitude": -68.407524, - "longitude": -113.642067, - "tags": [ - "new issue", - "wontfix" - ] - }, - { - "id": 18, - "isActive": false, - "balance": "$1,484.92", - "picture": "http://placehold.it/32x32", - "age": 39, - "color": "blue", - "name": "Leonard Tillman", - "gender": "male", - "email": "leonardtillman@chorizon.com", - "phone": "+1 (864) 541-3456", - "address": "985 Provost Street, Charco, New Hampshire, 8632", - "about": "Consectetur ut magna sit id officia nostrud ipsum. Lorem cupidatat laborum nostrud aliquip magna qui est cupidatat exercitation et. Officia qui magna commodo id cillum magna ut ad veniam sunt sint ex. Id minim do in do exercitation aliquip incididunt ex esse. Nisi aliqua quis excepteur qui aute excepteur dolore eu pariatur irure id eu cupidatat eiusmod. Aliqua amet et dolore enim et eiusmod qui irure pariatur qui officia adipisicing nulla duis.\r\n", - "registered": "2018-05-06T08:21:27 -02:00", - "latitude": -8.581801, - "longitude": -61.910062, - "tags": [ - "wontfix", - "new issue", - "bug", - "bug" - ] - }, - { - "id": 19, - "isActive": true, - "balance": "$3,572.55", - "picture": "http://placehold.it/32x32", - "age": 33, - "color": "brown", - "name": "Dale Payne", - "gender": "male", - "email": "dalepayne@chorizon.com", - "phone": "+1 (814) 469-3499", - "address": "536 Dare Court, Ironton, Arkansas, 8605", - "about": "Et velit cupidatat velit incididunt mollit. Occaecat do labore aliqua dolore excepteur occaecat ut veniam ad ullamco tempor. Ut anim laboris deserunt culpa esse. Pariatur Lorem nulla cillum cupidatat nostrud Lorem commodo reprehenderit ut est. In dolor cillum reprehenderit laboris incididunt ad reprehenderit aute ipsum officia id in consequat. Culpa exercitation voluptate fugiat est Lorem ipsum in dolore dolor consequat Lorem et.\r\n", - "registered": "2019-10-11T01:01:33 -02:00", - "latitude": -18.280968, - "longitude": -126.091797, - "tags": [ - "bug", - "wontfix", - "wontfix", - "wontfix" - ] - }, - { - "id": 20, - "isActive": true, - "balance": "$1,986.48", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "Green", - "name": "Florence Long", - "gender": "female", - "email": "florencelong@chorizon.com", - "phone": "+1 (972) 557-3858", - "address": "519 Hendrickson Street, Templeton, Hawaii, 2389", - "about": "Quis officia occaecat veniam veniam. Ex minim enim labore cupidatat qui. Proident esse deserunt laborum laboris sunt nostrud.\r\n", - "registered": "2016-05-02T09:18:59 -02:00", - "latitude": -27.110866, - "longitude": -45.09445, - "tags": [] - }, - { - "id": 21, - "isActive": true, - "balance": "$1,440.09", - "picture": "http://placehold.it/32x32", - "age": 40, - "color": "blue", - "name": "Levy Whitley", - "gender": "male", - "email": "levywhitley@chorizon.com", - "phone": "+1 (911) 458-2411", - "address": "187 Thomas Street, Hachita, North Carolina, 2989", - "about": "Velit laboris non minim elit sint deserunt fugiat. Aute minim ex commodo aute cillum aliquip fugiat pariatur nulla eiusmod pariatur consectetur. Qui ex ea qui laborum veniam adipisicing magna minim ut. In irure anim voluptate mollit et. Adipisicing labore ea mollit magna aliqua culpa velit est. Excepteur nisi veniam enim velit in ad officia irure laboris.\r\n", - "registered": "2014-04-30T07:31:38 -02:00", - "latitude": -6.537315, - "longitude": 171.813536, - "tags": [ - "bug" - ] - }, - { - "id": 22, - "isActive": false, - "balance": "$2,938.57", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "blue", - "name": "Bernard Mcfarland", - "gender": "male", - "email": "bernardmcfarland@chorizon.com", - "phone": "+1 (979) 442-3386", - "address": "409 Hall Street, Keyport, Federated States Of Micronesia, 7011", - "about": "Reprehenderit irure aute et anim ullamco enim est tempor id ipsum mollit veniam aute ullamco. Consectetur dolor velit tempor est reprehenderit ut id non est ullamco voluptate. Commodo aute ullamco culpa non voluptate incididunt non culpa culpa nisi id proident cupidatat.\r\n", - "registered": "2017-08-10T10:07:59 -02:00", - "latitude": 63.766795, - "longitude": 68.177069, - "tags": [] - }, - { - "id": 23, - "isActive": true, - "balance": "$1,678.49", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "brown", - "name": "Blanca Mcclain", - "gender": "female", - "email": "blancamcclain@chorizon.com", - "phone": "+1 (976) 439-2772", - "address": "176 Crooke Avenue, Valle, Virginia, 5373", - "about": "Aliquip sunt irure ut consectetur elit. Cillum amet incididunt et anim elit in incididunt adipisicing fugiat veniam esse veniam. Nisi qui sit occaecat tempor nostrud est aute cillum anim excepteur laboris magna in. Fugiat fugiat veniam cillum laborum ut pariatur amet nulla nulla. Nostrud mollit in laborum minim exercitation aute. Lorem aute ipsum laboris est adipisicing qui ullamco tempor adipisicing cupidatat mollit.\r\n", - "registered": "2015-10-12T11:57:28 -02:00", - "latitude": -8.944564, - "longitude": -150.711709, - "tags": [ - "bug", - "wontfix", - "good first issue" - ] - }, - { - "id": 24, - "isActive": true, - "balance": "$2,276.87", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Espinoza Ford", - "gender": "male", - "email": "espinozaford@chorizon.com", - "phone": "+1 (945) 429-3975", - "address": "137 Bowery Street, Itmann, District Of Columbia, 1864", - "about": "Deserunt nisi aliquip esse occaecat laborum qui aliqua excepteur ea cupidatat dolore magna consequat. Culpa aliquip cillum incididunt proident est officia consequat duis. Elit tempor ut cupidatat nisi ea sint non labore aliquip amet. Deserunt labore cupidatat laboris dolor duis occaecat velit aliquip reprehenderit esse. Sit ad qui consectetur id anim nisi amet eiusmod.\r\n", - "registered": "2014-03-26T02:16:08 -01:00", - "latitude": -37.137666, - "longitude": -51.811757, - "tags": [ - "wontfix", - "bug" - ] - }, - { - "id": 25, - "isActive": true, - "balance": "$3,973.43", - "picture": "http://placehold.it/32x32", - "age": 29, - "color": "Green", - "name": "Sykes Conley", - "gender": "male", - "email": "sykesconley@chorizon.com", - "phone": "+1 (851) 401-3916", - "address": "345 Grand Street, Woodlands, Missouri, 4461", - "about": "Pariatur ullamco duis reprehenderit ad sit dolore. Dolore ex fugiat labore incididunt nostrud. Minim deserunt officia sunt enim magna elit veniam reprehenderit nisi cupidatat dolor eiusmod. Veniam laboris sint cillum et laboris nostrud culpa laboris anim. Incididunt velit pariatur cupidatat sit dolore in. Voluptate consectetur officia id nostrud velit mollit dolor. Id laboris consectetur culpa sunt pariatur minim sunt laboris sit.\r\n", - "registered": "2015-09-12T06:03:56 -02:00", - "latitude": 67.282955, - "longitude": -64.341323, - "tags": [ - "wontfix" - ] - }, - { - "id": 26, - "isActive": false, - "balance": "$1,431.50", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "blue", - "name": "Barlow Duran", - "gender": "male", - "email": "barlowduran@chorizon.com", - "phone": "+1 (995) 436-2562", - "address": "481 Everett Avenue, Allison, Nebraska, 3065", - "about": "Proident quis eu officia adipisicing aliquip. Lorem laborum magna dolor et incididunt cillum excepteur et amet. Veniam consectetur officia fugiat magna consequat dolore elit aute exercitation fugiat excepteur ullamco. Sit qui proident reprehenderit ea ad qui culpa exercitation reprehenderit anim cupidatat. Nulla et duis Lorem cillum duis pariatur amet voluptate labore ut aliqua mollit anim ea. Nostrud incididunt et proident adipisicing non consequat tempor ullamco adipisicing incididunt. Incididunt cupidatat tempor fugiat officia qui eiusmod reprehenderit.\r\n", - "registered": "2017-06-29T04:28:43 -02:00", - "latitude": -38.70606, - "longitude": 55.02816, - "tags": [ - "new issue" - ] - }, - { - "id": 27, - "isActive": true, - "balance": "$3,478.27", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "blue", - "name": "Schwartz Morgan", - "gender": "male", - "email": "schwartzmorgan@chorizon.com", - "phone": "+1 (861) 507-2067", - "address": "451 Lincoln Road, Fairlee, Washington, 2717", - "about": "Labore eiusmod sint dolore sunt eiusmod esse et in id aliquip. Aliqua consequat occaecat laborum labore ipsum enim non nostrud adipisicing adipisicing cillum occaecat. Duis minim est culpa sunt nulla ullamco adipisicing magna irure. Occaecat quis irure eiusmod fugiat quis commodo reprehenderit labore cillum commodo id et.\r\n", - "registered": "2016-05-10T08:34:54 -02:00", - "latitude": -75.886403, - "longitude": 93.044471, - "tags": [ - "bug", - "bug", - "wontfix", - "wontfix" - ] - }, - { - "id": 28, - "isActive": true, - "balance": "$2,825.59", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Kristy Leon", - "gender": "female", - "email": "kristyleon@chorizon.com", - "phone": "+1 (948) 465-2563", - "address": "594 Macon Street, Floris, South Dakota, 3565", - "about": "Proident veniam voluptate magna id do. Laboris enim dolor culpa quis. Esse voluptate elit commodo duis incididunt velit aliqua. Qui aute commodo incididunt elit eu Lorem dolore. Non esse duis do reprehenderit culpa minim. Ullamco consequat id do exercitation exercitation mollit ipsum velit eiusmod quis.\r\n", - "registered": "2014-12-14T04:10:29 -01:00", - "latitude": -50.01615, - "longitude": -68.908804, - "tags": [ - "wontfix", - "good first issue" - ] - }, - { - "id": 29, - "isActive": false, - "balance": "$3,028.03", - "picture": "http://placehold.it/32x32", - "age": 39, - "color": "blue", - "name": "Ashley Pittman", - "gender": "male", - "email": "ashleypittman@chorizon.com", - "phone": "+1 (928) 507-3523", - "address": "646 Adelphi Street, Clara, Colorado, 6056", - "about": "Incididunt cillum consectetur nulla sit sit labore nulla sit. Ullamco nisi mollit reprehenderit tempor irure in Lorem duis. Sunt eu aute laboris dolore commodo ipsum sint cupidatat veniam amet culpa incididunt aute ad. Quis dolore aliquip id aute mollit eiusmod nisi ipsum ut labore adipisicing do culpa.\r\n", - "registered": "2016-01-07T10:40:48 -01:00", - "latitude": -58.766037, - "longitude": -124.828485, - "tags": [ - "wontfix" - ] - }, - { - "id": 30, - "isActive": true, - "balance": "$2,021.11", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Stacy Espinoza", - "gender": "female", - "email": "stacyespinoza@chorizon.com", - "phone": "+1 (999) 487-3253", - "address": "931 Alabama Avenue, Bangor, Alaska, 8215", - "about": "Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n", - "registered": "2014-07-16T06:15:53 -02:00", - "latitude": 41.560197, - "longitude": 177.697, - "tags": [ - "new issue", - "new issue", - "bug" - ] - }, - { - "id": 31, - "isActive": false, - "balance": "$3,609.82", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Vilma Garza", - "gender": "female", - "email": "vilmagarza@chorizon.com", - "phone": "+1 (944) 585-2021", - "address": "565 Tech Place, Sedley, Puerto Rico, 858", - "about": "Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n", - "registered": "2017-06-30T07:43:52 -02:00", - "latitude": -12.574889, - "longitude": -54.771186, - "tags": [ - "new issue", - "wontfix", - "wontfix" - ] - }, - { - "id": 32, - "isActive": false, - "balance": "$2,882.34", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "brown", - "name": "June Dunlap", - "gender": "female", - "email": "junedunlap@chorizon.com", - "phone": "+1 (997) 504-2937", - "address": "353 Cozine Avenue, Goodville, Indiana, 1438", - "about": "Non dolore ut Lorem dolore amet veniam fugiat reprehenderit ut amet ea ut. Non aliquip cillum ad occaecat non et sint quis proident velit laborum ullamco et. Quis qui tempor eu voluptate et proident duis est commodo laboris ex enim. Nisi aliquip laboris nostrud veniam aliqua ullamco. Et officia proident dolor aliqua incididunt veniam proident.\r\n", - "registered": "2016-08-23T08:54:11 -02:00", - "latitude": -27.883363, - "longitude": -163.919683, - "tags": [ - "new issue", - "new issue", - "bug", - "wontfix" - ] - }, - { - "id": 33, - "isActive": true, - "balance": "$3,556.54", - "picture": "http://placehold.it/32x32", - "age": 33, - "color": "brown", - "name": "Cecilia Greer", - "gender": "female", - "email": "ceciliagreer@chorizon.com", - "phone": "+1 (977) 573-3498", - "address": "696 Withers Street, Lydia, Oklahoma, 3220", - "about": "Dolor pariatur veniam ad enim eiusmod fugiat ullamco nulla veniam. Dolore dolor sit excepteur veniam adipisicing adipisicing excepteur commodo qui reprehenderit magna exercitation enim reprehenderit. Cupidatat eu ullamco excepteur sint do. Et cupidatat ex adipisicing veniam eu tempor reprehenderit ut eiusmod amet proident veniam nostrud. Tempor ex enim mollit laboris magna tempor. Et aliqua nostrud esse pariatur quis. Ut pariatur ea ipsum pariatur.\r\n", - "registered": "2017-01-13T11:30:12 -01:00", - "latitude": 60.467215, - "longitude": 84.684575, - "tags": [ - "wontfix", - "good first issue", - "good first issue", - "wontfix" - ] - }, - { - "id": 34, - "isActive": true, - "balance": "$1,413.35", - "picture": "http://placehold.it/32x32", - "age": 33, - "color": "brown", - "name": "Mckay Schroeder", - "gender": "male", - "email": "mckayschroeder@chorizon.com", - "phone": "+1 (816) 480-3657", - "address": "958 Miami Court, Rehrersburg, Northern Mariana Islands, 567", - "about": "Amet do velit excepteur tempor sit eu voluptate. Excepteur amet culpa ipsum in pariatur mollit amet nisi veniam. Laboris elit consectetur id anim qui laboris. Reprehenderit mollit laboris occaecat esse sunt Lorem Lorem sunt occaecat.\r\n", - "registered": "2016-02-08T04:50:15 -01:00", - "latitude": -72.413287, - "longitude": -159.254371, - "tags": [ - "good first issue" - ] - }, - { - "id": 35, - "isActive": true, - "balance": "$2,306.53", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Sawyer Mccormick", - "gender": "male", - "email": "sawyermccormick@chorizon.com", - "phone": "+1 (829) 569-3012", - "address": "749 Apollo Street, Eastvale, Texas, 7373", - "about": "Est irure ex occaecat aute. Lorem ad ullamco esse cillum deserunt qui proident anim officia dolore. Incididunt tempor cupidatat nulla cupidatat ullamco reprehenderit Lorem. Laboris tempor do pariatur sint non officia id qui deserunt amet Lorem pariatur consectetur exercitation. Adipisicing reprehenderit pariatur duis ex cupidatat cillum ad laboris ex. Sunt voluptate pariatur esse amet dolore minim aliquip reprehenderit nisi velit mollit.\r\n", - "registered": "2019-11-30T11:53:23 -01:00", - "latitude": -48.978194, - "longitude": 110.950191, - "tags": [ - "good first issue", - "new issue", - "new issue", - "bug" - ] - }, - { - "id": 36, - "isActive": false, - "balance": "$1,844.54", - "picture": "http://placehold.it/32x32", - "age": 37, - "color": "brown", - "name": "Barbra Valenzuela", - "gender": "female", - "email": "barbravalenzuela@chorizon.com", - "phone": "+1 (992) 512-2649", - "address": "617 Schenck Court, Reinerton, Michigan, 2908", - "about": "Deserunt adipisicing nisi et amet aliqua amet. Veniam occaecat et elit excepteur veniam. Aute irure culpa nostrud occaecat. Excepteur sit aute mollit commodo. Do ex pariatur consequat sint Lorem veniam laborum excepteur. Non voluptate ex laborum enim irure. Adipisicing excepteur anim elit esse.\r\n", - "registered": "2019-03-29T01:59:31 -01:00", - "latitude": 45.193723, - "longitude": -12.486778, - "tags": [ - "new issue", - "new issue", - "wontfix", - "wontfix" - ] - }, - { - "id": 37, - "isActive": false, - "balance": "$3,469.82", - "picture": "http://placehold.it/32x32", - "age": 39, - "color": "brown", - "name": "Opal Weiss", - "gender": "female", - "email": "opalweiss@chorizon.com", - "phone": "+1 (809) 400-3079", - "address": "535 Bogart Street, Frizzleburg, Arizona, 5222", - "about": "Reprehenderit nostrud minim adipisicing voluptate nisi consequat id sint. Proident tempor est esse cupidatat minim irure esse do do sint dolor. In officia duis et voluptate Lorem minim cupidatat ipsum enim qui dolor quis in Lorem. Aliquip commodo ex quis exercitation reprehenderit. Lorem id reprehenderit cillum adipisicing sunt ipsum incididunt incididunt.\r\n", - "registered": "2019-09-04T07:22:28 -02:00", - "latitude": 72.50376, - "longitude": 61.656435, - "tags": [ - "bug", - "bug", - "good first issue", - "good first issue" - ] - }, - { - "id": 38, - "isActive": true, - "balance": "$1,992.38", - "picture": "http://placehold.it/32x32", - "age": 40, - "color": "Green", - "name": "Christina Short", - "gender": "female", - "email": "christinashort@chorizon.com", - "phone": "+1 (884) 589-2705", - "address": "594 Willmohr Street, Dexter, Montana, 660", - "about": "Quis commodo eu dolor incididunt. Nisi magna mollit nostrud do consequat irure exercitation mollit aute deserunt. Magna aute quis occaecat incididunt deserunt tempor nostrud sint ullamco ipsum. Anim in occaecat exercitation laborum nostrud eiusmod reprehenderit ea culpa et sit. Culpa voluptate consectetur nostrud do eu fugiat excepteur officia pariatur enim duis amet.\r\n", - "registered": "2014-01-21T09:31:56 -01:00", - "latitude": -42.762739, - "longitude": 77.052349, - "tags": [ - "bug", - "new issue" - ] - }, - { - "id": 39, - "isActive": false, - "balance": "$1,722.85", - "picture": "http://placehold.it/32x32", - "age": 29, - "color": "brown", - "name": "Golden Horton", - "gender": "male", - "email": "goldenhorton@chorizon.com", - "phone": "+1 (903) 426-2489", - "address": "191 Schenck Avenue, Mayfair, North Dakota, 5000", - "about": "Cillum velit aliqua velit in quis do mollit in et veniam. Nostrud proident non irure commodo. Ea culpa duis enim adipisicing do sint et est culpa reprehenderit officia laborum. Non et nostrud tempor nostrud nostrud ea duis esse laboris occaecat laborum. In eu ipsum sit tempor esse eiusmod enim aliquip aute. Officia ea anim ea ea. Consequat aute deserunt tempor nulla nisi tempor velit.\r\n", - "registered": "2015-08-19T02:56:41 -02:00", - "latitude": 69.922534, - "longitude": 9.881433, - "tags": [ - "bug" - ] - }, - { - "id": 40, - "isActive": false, - "balance": "$1,656.54", - "picture": "http://placehold.it/32x32", - "age": 21, - "color": "blue", - "name": "Stafford Emerson", - "gender": "male", - "email": "staffordemerson@chorizon.com", - "phone": "+1 (992) 455-2573", - "address": "523 Thornton Street, Conway, Vermont, 6331", - "about": "Adipisicing cupidatat elit minim elit nostrud elit non eiusmod sunt ut. Enim minim irure officia irure occaecat mollit eu nostrud eiusmod adipisicing sunt. Elit deserunt commodo minim dolor qui. Nostrud officia ex proident mollit et dolor tempor pariatur. Ex consequat tempor eiusmod irure mollit cillum laboris est veniam ea mollit deserunt. Tempor sit voluptate excepteur elit ullamco.\r\n", - "registered": "2019-02-16T04:07:08 -01:00", - "latitude": -29.143111, - "longitude": -57.207703, - "tags": [ - "wontfix", - "good first issue", - "good first issue" - ] - }, - { - "id": 41, - "isActive": false, - "balance": "$1,861.56", - "picture": "http://placehold.it/32x32", - "age": 21, - "color": "brown", - "name": "Salinas Gamble", - "gender": "male", - "email": "salinasgamble@chorizon.com", - "phone": "+1 (901) 525-2373", - "address": "991 Nostrand Avenue, Kansas, Mississippi, 6756", - "about": "Consequat tempor adipisicing cupidatat aliquip. Mollit proident incididunt ad ipsum laborum. Dolor in elit minim aliquip aliquip voluptate reprehenderit mollit eiusmod excepteur aliquip minim nulla cupidatat.\r\n", - "registered": "2017-08-21T05:47:53 -02:00", - "latitude": -22.593819, - "longitude": -63.613004, - "tags": [ - "good first issue", - "bug", - "bug", - "wontfix" - ] - }, - { - "id": 42, - "isActive": true, - "balance": "$3,179.74", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "brown", - "name": "Graciela Russell", - "gender": "female", - "email": "gracielarussell@chorizon.com", - "phone": "+1 (893) 464-3951", - "address": "361 Greenpoint Avenue, Shrewsbury, New Jersey, 4713", - "about": "Ex amet duis incididunt consequat minim dolore deserunt reprehenderit adipisicing in mollit aliqua adipisicing sunt. In ullamco eu qui est eiusmod qui. Fugiat esse est Lorem dolore nisi mollit exercitation. Aliquip occaecat esse exercitation ex non aute velit excepteur duis aliquip id. Velit id non aliquip fugiat minim qui exercitation culpa tempor consectetur. Minim dolor labore ea aute aute eu.\r\n", - "registered": "2015-05-18T09:52:56 -02:00", - "latitude": -14.634444, - "longitude": 12.931783, - "tags": [ - "wontfix", - "bug", - "wontfix" - ] - }, - { - "id": 43, - "isActive": true, - "balance": "$1,777.38", - "picture": "http://placehold.it/32x32", - "age": 25, - "color": "blue", - "name": "Arnold Bender", - "gender": "male", - "email": "arnoldbender@chorizon.com", - "phone": "+1 (945) 581-3808", - "address": "781 Lorraine Street, Gallina, American Samoa, 1832", - "about": "Et mollit laboris duis ut duis eiusmod aute laborum duis irure labore deserunt. Ut occaecat ullamco quis excepteur. Et commodo non sint laboris tempor laboris aliqua consequat magna ea aute minim tempor pariatur. Dolore occaecat qui irure Lorem nulla consequat non.\r\n", - "registered": "2018-12-23T02:26:30 -01:00", - "latitude": 41.208579, - "longitude": 51.948925, - "tags": [ - "bug", - "good first issue", - "good first issue", - "wontfix" - ] - }, - { - "id": 44, - "isActive": true, - "balance": "$2,893.45", - "picture": "http://placehold.it/32x32", - "age": 22, - "color": "Green", - "name": "Joni Spears", - "gender": "female", - "email": "jonispears@chorizon.com", - "phone": "+1 (916) 565-2124", - "address": "307 Harwood Place, Canterwood, Maryland, 2047", - "about": "Dolore consequat deserunt aliquip duis consequat minim occaecat enim est. Nulla aute reprehenderit est enim duis cillum ullamco aliquip eiusmod sunt. Labore eiusmod aliqua Lorem velit aliqua quis ex mollit mollit duis culpa et qui in. Cupidatat est id ullamco irure dolor nulla.\r\n", - "registered": "2015-03-01T12:38:28 -01:00", - "latitude": 8.19071, - "longitude": 146.323808, - "tags": [ - "wontfix", - "new issue", - "good first issue", - "good first issue" - ] - }, - { - "id": 45, - "isActive": true, - "balance": "$2,830.36", - "picture": "http://placehold.it/32x32", - "age": 20, - "color": "brown", - "name": "Irene Bennett", - "gender": "female", - "email": "irenebennett@chorizon.com", - "phone": "+1 (904) 431-2211", - "address": "353 Ridgecrest Terrace, Springdale, Marshall Islands, 2686", - "about": "Consectetur Lorem dolor reprehenderit sunt duis. Pariatur non velit velit veniam elit reprehenderit in. Aute quis Lorem quis pariatur Lorem incididunt nulla magna adipisicing. Et id occaecat labore officia occaecat occaecat adipisicing.\r\n", - "registered": "2018-04-17T05:18:51 -02:00", - "latitude": -36.435177, - "longitude": -127.552573, - "tags": [ - "bug", - "wontfix" - ] - }, - { - "id": 46, - "isActive": true, - "balance": "$1,348.04", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "Green", - "name": "Lawson Curtis", - "gender": "male", - "email": "lawsoncurtis@chorizon.com", - "phone": "+1 (896) 532-2172", - "address": "942 Gerritsen Avenue, Southmont, Kansas, 8915", - "about": "Amet consectetur minim aute nostrud excepteur sint labore in culpa. Mollit qui quis ea amet sint ex incididunt nulla. Elit id esse ea consectetur laborum consequat occaecat aute consectetur ex. Commodo duis aute elit occaecat cupidatat non consequat ad officia qui dolore nostrud reprehenderit. Occaecat velit velit adipisicing exercitation consectetur. Incididunt et amet nostrud tempor do esse ullamco est Lorem irure. Eu aliqua eu exercitation sint.\r\n", - "registered": "2016-08-23T01:41:09 -02:00", - "latitude": -48.783539, - "longitude": 20.492944, - "tags": [] - }, - { - "id": 47, - "isActive": true, - "balance": "$1,132.41", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "Green", - "name": "Goff May", - "gender": "male", - "email": "goffmay@chorizon.com", - "phone": "+1 (859) 453-3415", - "address": "225 Rutledge Street, Boonville, Massachusetts, 4081", - "about": "Sint occaecat velit anim sint reprehenderit est. Adipisicing ea pariatur amet id non ex. Aute id laborum tempor aliquip magna ex eu incididunt aliquip eiusmod elit quis dolor. Anim est minim deserunt amet exercitation nulla elit nulla nulla culpa ullamco. Velit consectetur ipsum amet proident labore excepteur ut id excepteur voluptate commodo. Exercitation et laboris labore esse est laboris consectetur et sint.\r\n", - "registered": "2014-10-25T07:32:30 -02:00", - "latitude": 13.079225, - "longitude": 76.215086, - "tags": [ - "bug" - ] - }, - { - "id": 48, - "isActive": true, - "balance": "$1,201.87", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "Green", - "name": "Goodman Becker", - "gender": "male", - "email": "goodmanbecker@chorizon.com", - "phone": "+1 (825) 470-3437", - "address": "388 Seigel Street, Sisquoc, Kentucky, 8231", - "about": "Velit excepteur aute esse fugiat laboris aliqua magna. Est ex sit do labore ullamco aliquip. Duis ea commodo nostrud in fugiat. Aliqua consequat mollit dolore excepteur nisi ullamco commodo ea nostrud ea minim. Minim occaecat ut laboris ea consectetur veniam ipsum qui sit tempor incididunt anim amet eu. Velit sint incididunt eu adipisicing ipsum qui labore. Anim commodo labore reprehenderit aliquip labore elit minim deserunt amet exercitation officia non ea consectetur.\r\n", - "registered": "2019-09-05T04:49:03 -02:00", - "latitude": -23.792094, - "longitude": -13.621221, - "tags": [ - "bug", - "bug", - "wontfix", - "new issue" - ] - }, - { - "id": 49, - "isActive": true, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ] - }, - { - "id": 50, - "isActive": true, - "balance": "$1,947.08", - "picture": "http://placehold.it/32x32", - "age": 21, - "color": "Green", - "name": "Guerra Mcintyre", - "gender": "male", - "email": "guerramcintyre@chorizon.com", - "phone": "+1 (951) 536-2043", - "address": "423 Lombardy Street, Stewart, West Virginia, 908", - "about": "Sunt proident proident deserunt exercitation consectetur deserunt labore non commodo amet. Duis aute aliqua amet deserunt consectetur velit. Quis Lorem dolore occaecat deserunt reprehenderit non esse ullamco nostrud enim sunt ea fugiat. Elit amet veniam eu magna tempor. Mollit cupidatat laboris ex deserunt et labore sit tempor nostrud anim. Tempor aliqua occaecat voluptate reprehenderit eiusmod aliqua incididunt officia.\r\n", - "registered": "2015-07-16T05:11:42 -02:00", - "latitude": 79.733743, - "longitude": -20.602356, - "tags": [ - "bug", - "good first issue", - "good first issue" - ] - }, - { - "id": 51, - "isActive": true, - "balance": "$2,960.90", - "picture": "http://placehold.it/32x32", - "age": 23, - "color": "blue", - "name": "Key Cervantes", - "gender": "male", - "email": "keycervantes@chorizon.com", - "phone": "+1 (931) 474-3865", - "address": "410 Barbey Street, Vernon, Oregon, 2328", - "about": "Duis amet minim eu consectetur laborum ad exercitation eiusmod nulla velit cillum consectetur. Nostrud aliqua cillum minim veniam quis do cupidatat mollit laborum. Culpa fugiat consectetur cillum non occaecat tempor non fugiat esse pariatur in ullamco. Occaecat amet officia et culpa officia deserunt in qui magna aute consequat eiusmod.\r\n", - "registered": "2019-12-15T12:13:35 -01:00", - "latitude": 47.627647, - "longitude": 117.049918, - "tags": [ - "new issue" - ] - }, - { - "id": 52, - "isActive": false, - "balance": "$1,884.02", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "blue", - "name": "Karen Nelson", - "gender": "female", - "email": "karennelson@chorizon.com", - "phone": "+1 (993) 528-3607", - "address": "930 Frank Court, Dunbar, New York, 8810", - "about": "Occaecat officia veniam consectetur aliqua laboris dolor irure nulla. Lorem ipsum sit nisi veniam mollit ea sint nisi irure. Eiusmod officia do laboris nostrud enim ullamco nulla officia in Lorem qui. Sint sunt incididunt quis reprehenderit incididunt. Sit dolore nulla consequat ea magna.\r\n", - "registered": "2014-06-23T09:21:44 -02:00", - "latitude": -59.059033, - "longitude": 76.565373, - "tags": [ - "new issue", - "bug" - ] - }, - { - "id": 53, - "isActive": true, - "balance": "$3,559.55", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "brown", - "name": "Caitlin Burnett", - "gender": "female", - "email": "caitlinburnett@chorizon.com", - "phone": "+1 (945) 480-2796", - "address": "516 Senator Street, Emory, Iowa, 4145", - "about": "In aliqua ea esse in. Magna aute cupidatat culpa enim proident ad adipisicing laborum consequat exercitation nisi. Qui esse aliqua duis anim nulla esse enim nostrud ipsum tempor. Lorem deserunt ullamco do mollit culpa ipsum duis Lorem velit duis occaecat.\r\n", - "registered": "2019-01-09T02:26:31 -01:00", - "latitude": -82.774237, - "longitude": 42.316194, - "tags": [ - "bug", - "good first issue" - ] - }, - { - "id": 54, - "isActive": true, - "balance": "$2,113.29", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "Green", - "name": "Richards Walls", - "gender": "male", - "email": "richardswalls@chorizon.com", - "phone": "+1 (865) 517-2982", - "address": "959 Brightwater Avenue, Stevens, Nevada, 2968", - "about": "Ad aute Lorem non pariatur anim ullamco ad amet eiusmod tempor velit. Mollit et tempor nisi aute adipisicing exercitation mollit do amet amet est fugiat enim. Ex voluptate nulla id tempor officia ullamco cillum dolor irure irure mollit et magna nisi. Pariatur voluptate qui laboris dolor id. Eu ipsum nulla dolore aute voluptate deserunt anim aliqua. Ut enim enim velit officia est nisi. Duis amet ut veniam aliquip minim tempor Lorem amet Lorem dolor duis.\r\n", - "registered": "2014-09-25T06:51:22 -02:00", - "latitude": 80.09202, - "longitude": 87.49759, - "tags": [ - "wontfix", - "wontfix", - "bug" - ] - }, - { - "id": 55, - "isActive": true, - "balance": "$1,977.66", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "brown", - "name": "Combs Stanley", - "gender": "male", - "email": "combsstanley@chorizon.com", - "phone": "+1 (827) 419-2053", - "address": "153 Beverley Road, Siglerville, South Carolina, 3666", - "about": "Commodo ullamco consequat eu ipsum eiusmod aute voluptate in. Ea laboris id deserunt nostrud pariatur et laboris minim tempor quis qui consequat non esse. Magna elit commodo mollit veniam Lorem enim nisi pariatur. Nisi non nisi adipisicing ea ipsum laborum dolore cillum. Amet do nisi esse laboris ipsum proident non veniam ullamco ea cupidatat sunt. Aliquip aute cillum quis laboris consectetur enim eiusmod nisi non id ullamco cupidatat sunt.\r\n", - "registered": "2019-08-22T07:53:15 -02:00", - "latitude": 78.386181, - "longitude": 143.661058, - "tags": [] - }, - { - "id": 56, - "isActive": false, - "balance": "$3,886.12", - "picture": "http://placehold.it/32x32", - "age": 23, - "color": "brown", - "name": "Tucker Barry", - "gender": "male", - "email": "tuckerbarry@chorizon.com", - "phone": "+1 (808) 544-3433", - "address": "805 Jamaica Avenue, Cornfields, Minnesota, 3689", - "about": "Enim est sunt ullamco nulla aliqua commodo. Enim minim veniam non fugiat id tempor ad velit quis velit ad sunt consectetur laborum. Cillum deserunt tempor est adipisicing Lorem esse qui. Magna quis sunt cillum ea officia adipisicing eiusmod eu et nisi consectetur.\r\n", - "registered": "2016-08-29T07:28:00 -02:00", - "latitude": 71.701551, - "longitude": 9.903068, - "tags": [] - }, - { - "id": 57, - "isActive": false, - "balance": "$1,844.56", - "picture": "http://placehold.it/32x32", - "age": 20, - "color": "Green", - "name": "Kaitlin Conner", - "gender": "female", - "email": "kaitlinconner@chorizon.com", - "phone": "+1 (862) 467-2666", - "address": "501 Knight Court, Joppa, Rhode Island, 274", - "about": "Occaecat id reprehenderit pariatur ea. Incididunt laborum reprehenderit ipsum velit labore excepteur nostrud voluptate officia ut culpa. Sint sunt in qui duis cillum aliqua do ullamco. Non do aute excepteur non labore sint consectetur tempor ad ea fugiat commodo labore. Dolor tempor culpa Lorem voluptate esse nostrud anim tempor irure reprehenderit. Deserunt ipsum cillum fugiat ut labore labore anim. In aliqua sunt dolore irure reprehenderit voluptate commodo consequat mollit amet laboris sit anim.\r\n", - "registered": "2019-05-30T06:38:24 -02:00", - "latitude": 15.613464, - "longitude": 171.965629, - "tags": [] - }, - { - "id": 58, - "isActive": true, - "balance": "$2,876.10", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "Green", - "name": "Mamie Fischer", - "gender": "female", - "email": "mamiefischer@chorizon.com", - "phone": "+1 (948) 545-3901", - "address": "599 Hunterfly Place, Haena, Georgia, 6005", - "about": "Cillum eu aliquip ipsum anim in dolore labore ea. Laboris velit esse ea ea aute do adipisicing ullamco elit laborum aute tempor. Esse consectetur quis irure occaecat nisi cillum et consectetur cillum cillum quis quis commodo.\r\n", - "registered": "2019-05-27T05:07:10 -02:00", - "latitude": 70.915079, - "longitude": -48.813584, - "tags": [ - "bug", - "wontfix", - "wontfix", - "good first issue" - ] - }, - { - "id": 59, - "isActive": true, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ] - }, - { - "id": 60, - "isActive": true, - "balance": "$1,770.93", - "picture": "http://placehold.it/32x32", - "age": 23, - "color": "brown", - "name": "Jody Herrera", - "gender": "female", - "email": "jodyherrera@chorizon.com", - "phone": "+1 (890) 583-3222", - "address": "261 Jay Street, Strykersville, Ohio, 9248", - "about": "Sit adipisicing pariatur irure non sint cupidatat ex ipsum pariatur exercitation ea. Enim consequat enim eu eu sint eu elit ex esse aliquip. Pariatur ipsum dolore veniam nisi id tempor elit exercitation dolore ad fugiat labore velit.\r\n", - "registered": "2016-05-21T01:00:02 -02:00", - "latitude": -36.846586, - "longitude": 131.156223, - "tags": [] - }, - { - "id": 61, - "isActive": false, - "balance": "$2,813.41", - "picture": "http://placehold.it/32x32", - "age": 37, - "color": "Green", - "name": "Charles Castillo", - "gender": "male", - "email": "charlescastillo@chorizon.com", - "phone": "+1 (934) 467-2108", - "address": "675 Morton Street, Rew, Pennsylvania, 137", - "about": "Velit amet laborum amet sunt sint sit cupidatat deserunt dolor laborum consectetur veniam. Minim cupidatat amet exercitation nostrud ex deserunt ad Lorem amet aute consectetur labore reprehenderit. Minim mollit aliqua et deserunt ex nisi. Id irure dolor labore consequat ipsum consectetur.\r\n", - "registered": "2019-06-10T02:54:22 -02:00", - "latitude": -16.423202, - "longitude": -146.293752, - "tags": [ - "new issue", - "new issue" - ] - }, - { - "id": 62, - "isActive": true, - "balance": "$3,341.35", - "picture": "http://placehold.it/32x32", - "age": 33, - "color": "blue", - "name": "Estelle Ramirez", - "gender": "female", - "email": "estelleramirez@chorizon.com", - "phone": "+1 (816) 459-2073", - "address": "636 Nolans Lane, Camptown, California, 7794", - "about": "Dolor proident incididunt ex labore quis ullamco duis. Sit esse laboris nisi eu voluptate nulla cupidatat nulla fugiat veniam. Culpa cillum est esse dolor consequat. Pariatur ex sit irure qui do fugiat. Fugiat culpa veniam est nisi excepteur quis cupidatat et minim in esse minim dolor et. Anim aliquip labore dolor occaecat nisi sunt dolore pariatur veniam nostrud est ut.\r\n", - "registered": "2015-02-14T01:05:50 -01:00", - "latitude": -46.591249, - "longitude": -83.385587, - "tags": [ - "good first issue", - "bug" - ] - }, - { - "id": 63, - "isActive": true, - "balance": "$2,478.30", - "picture": "http://placehold.it/32x32", - "age": 21, - "color": "blue", - "name": "Knowles Hebert", - "gender": "male", - "email": "knowleshebert@chorizon.com", - "phone": "+1 (819) 409-2308", - "address": "361 Kathleen Court, Gratton, Connecticut, 7254", - "about": "Esse mollit nulla eiusmod esse duis non proident excepteur labore. Nisi ex culpa do mollit dolor ea deserunt elit anim ipsum nostrud. Cupidatat nostrud duis ipsum dolore amet et. Veniam in cillum ea cillum deserunt excepteur officia laboris nulla. Commodo incididunt aliquip qui sunt dolore occaecat labore do laborum irure. Labore culpa duis pariatur reprehenderit ad laboris occaecat anim cillum et fugiat ea.\r\n", - "registered": "2016-03-08T08:34:52 -01:00", - "latitude": 71.042482, - "longitude": 152.460406, - "tags": [ - "good first issue", - "wontfix" - ] - }, - { - "id": 64, - "isActive": false, - "balance": "$2,559.09", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Thelma Mckenzie", - "gender": "female", - "email": "thelmamckenzie@chorizon.com", - "phone": "+1 (941) 596-2777", - "address": "202 Leonard Street, Riverton, Illinois, 8577", - "about": "Non ad ipsum elit commodo fugiat Lorem ipsum reprehenderit. Commodo incididunt officia cillum eiusmod officia proident ea incididunt ullamco magna commodo consectetur dolor. Nostrud esse nisi ea laboris. Veniam et dolore nulla excepteur pariatur laborum non. Eiusmod reprehenderit do tempor esse eu eu aliquip. Magna quis consectetur ipsum adipisicing mollit elit ad elit.\r\n", - "registered": "2020-04-14T12:43:06 -02:00", - "latitude": 16.026129, - "longitude": 105.464476, - "tags": [] - }, - { - "id": 65, - "isActive": true, - "balance": "$1,025.08", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Carole Rowland", - "gender": "female", - "email": "carolerowland@chorizon.com", - "phone": "+1 (862) 558-3448", - "address": "941 Melba Court, Bluetown, Florida, 9555", - "about": "Ullamco occaecat ipsum aliqua sit proident eu. Occaecat ut consectetur proident culpa aliqua excepteur quis qui anim irure sit proident mollit irure. Proident cupidatat deserunt dolor adipisicing.\r\n", - "registered": "2014-12-01T05:55:35 -01:00", - "latitude": -0.191998, - "longitude": 43.389652, - "tags": [ - "wontfix" - ] - }, - { - "id": 66, - "isActive": true, - "balance": "$1,061.49", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "brown", - "name": "Higgins Aguilar", - "gender": "male", - "email": "higginsaguilar@chorizon.com", - "phone": "+1 (911) 540-3791", - "address": "132 Sackman Street, Layhill, Guam, 8729", - "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", - "registered": "2015-04-05T02:10:07 -02:00", - "latitude": 74.702813, - "longitude": 151.314972, - "tags": [ - "bug" - ] - }, - { - "id": 67, - "isActive": true, - "balance": "$3,510.14", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Ilene Gillespie", - "gender": "female", - "email": "ilenegillespie@chorizon.com", - "phone": "+1 (937) 575-2676", - "address": "835 Lake Street, Naomi, Alabama, 4131", - "about": "Quis laborum consequat id cupidatat exercitation aute ad ex nulla dolore velit qui proident minim. Et do consequat nisi eiusmod exercitation exercitation enim voluptate elit ullamco. Cupidatat ut adipisicing consequat aute est voluptate sit ipsum culpa ullamco. Ex pariatur ex qui quis qui.\r\n", - "registered": "2015-06-28T09:41:45 -02:00", - "latitude": 71.573342, - "longitude": -95.295989, - "tags": [ - "wontfix", - "wontfix" - ] - }, - { - "id": 68, - "isActive": false, - "balance": "$1,539.98", - "picture": "http://placehold.it/32x32", - "age": 24, - "color": "Green", - "name": "Angelina Dyer", - "gender": "female", - "email": "angelinadyer@chorizon.com", - "phone": "+1 (948) 574-3949", - "address": "575 Division Place, Gorham, Louisiana, 3458", - "about": "Cillum magna eu est veniam incididunt laboris laborum elit mollit incididunt proident non mollit. Dolor mollit culpa ullamco dolore aliqua adipisicing culpa officia. Reprehenderit minim nisi fugiat consectetur dolore.\r\n", - "registered": "2014-07-08T06:34:36 -02:00", - "latitude": -85.649593, - "longitude": 66.126018, - "tags": [ - "good first issue" - ] - }, - { - "id": 69, - "isActive": true, - "balance": "$3,367.69", - "picture": "http://placehold.it/32x32", - "age": 30, - "color": "brown", - "name": "Marks Burt", - "gender": "male", - "email": "marksburt@chorizon.com", - "phone": "+1 (895) 497-3138", - "address": "819 Village Road, Wadsworth, Delaware, 6099", - "about": "Fugiat tempor aute voluptate proident exercitation tempor esse dolor id. Duis aliquip exercitation Lorem elit magna sint sit. Culpa adipisicing occaecat aliqua officia reprehenderit laboris sint aliquip. Magna do sunt consequat excepteur nisi do commodo non. Cillum officia nostrud consequat excepteur elit proident in. Tempor ipsum in ut qui cupidatat exercitation est nulla exercitation voluptate.\r\n", - "registered": "2014-08-31T06:12:18 -02:00", - "latitude": 26.854112, - "longitude": -143.313948, - "tags": [ - "good first issue" - ] - }, - { - "id": 70, - "isActive": false, - "balance": "$3,755.72", - "picture": "http://placehold.it/32x32", - "age": 23, - "color": "blue", - "name": "Glass Perkins", - "gender": "male", - "email": "glassperkins@chorizon.com", - "phone": "+1 (923) 486-3725", - "address": "899 Roosevelt Court, Belleview, Idaho, 1737", - "about": "Esse magna id labore sunt qui eu enim esse cillum consequat enim eu culpa enim. Duis veniam cupidatat deserunt sunt irure ad Lorem proident aliqua mollit. Laborum mollit aute nulla est. Sunt id proident incididunt ipsum et dolor consectetur laborum enim dolor officia dolore laborum. Est commodo duis et ea consequat labore id id eu aliqua. Qui veniam sit eu aliquip ad sit dolor ullamco et laborum voluptate quis fugiat ex. Exercitation dolore cillum amet ad nisi consectetur occaecat sit aliqua laborum qui proident aliqua exercitation.\r\n", - "registered": "2015-05-22T05:44:33 -02:00", - "latitude": 54.27147, - "longitude": -65.065604, - "tags": [ - "wontfix" - ] - }, - { - "id": 71, - "isActive": true, - "balance": "$3,381.63", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "Green", - "name": "Candace Sawyer", - "gender": "female", - "email": "candacesawyer@chorizon.com", - "phone": "+1 (830) 404-2636", - "address": "334 Arkansas Drive, Bordelonville, Tennessee, 8449", - "about": "Et aliqua elit incididunt et aliqua. Deserunt ut elit proident ullamco ut. Ex exercitation amet non eu reprehenderit ea voluptate qui sit reprehenderit ad sint excepteur.\r\n", - "registered": "2014-04-04T08:45:00 -02:00", - "latitude": 6.484262, - "longitude": -37.054928, - "tags": [ - "new issue", - "new issue" - ] - }, - { - "id": 72, - "isActive": true, - "balance": "$1,640.98", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Hendricks Martinez", - "gender": "male", - "email": "hendricksmartinez@chorizon.com", - "phone": "+1 (857) 566-3245", - "address": "636 Agate Court, Newry, Utah, 3304", - "about": "Do sit culpa amet incididunt officia enim occaecat incididunt excepteur enim tempor deserunt qui. Excepteur adipisicing anim consectetur adipisicing proident anim laborum qui. Aliquip nostrud cupidatat sit ullamco.\r\n", - "registered": "2018-06-15T10:36:11 -02:00", - "latitude": 86.746034, - "longitude": 10.347893, - "tags": [ - "new issue" - ] - }, - { - "id": 73, - "isActive": false, - "balance": "$1,239.74", - "picture": "http://placehold.it/32x32", - "age": 38, - "color": "blue", - "name": "Eleanor Shepherd", - "gender": "female", - "email": "eleanorshepherd@chorizon.com", - "phone": "+1 (894) 567-2617", - "address": "670 Lafayette Walk, Darlington, Palau, 8803", - "about": "Adipisicing ad incididunt id veniam magna cupidatat et labore eu deserunt mollit. Lorem voluptate exercitation elit eu aliquip cupidatat occaecat anim excepteur reprehenderit est est. Ipsum excepteur ea mollit qui nisi laboris ex qui. Cillum velit culpa culpa commodo laboris nisi Lorem non elit deserunt incididunt. Officia quis velit nulla sint incididunt duis mollit tempor adipisicing qui officia eu nisi Lorem. Do proident pariatur ex enim nostrud eu aute esse deserunt eu velit quis culpa exercitation. Occaecat ad cupidatat ullamco consequat duis anim deserunt occaecat aliqua sunt consectetur ipsum magna.\r\n", - "registered": "2020-02-29T12:15:28 -01:00", - "latitude": 35.749621, - "longitude": -94.40842, - "tags": [ - "good first issue", - "new issue", - "new issue", - "bug" - ] - }, - { - "id": 74, - "isActive": true, - "balance": "$1,180.90", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Stark Wong", - "gender": "male", - "email": "starkwong@chorizon.com", - "phone": "+1 (805) 575-3055", - "address": "522 Bond Street, Bawcomville, Wisconsin, 324", - "about": "Aute qui sit incididunt eu adipisicing exercitation sunt nostrud. Id laborum incididunt proident ipsum est cillum esse. Officia ullamco eu ut Lorem do minim ea dolor consequat sit eu est voluptate. Id commodo cillum enim culpa aliquip ullamco nisi Lorem cillum ipsum cupidatat anim officia eu. Dolore sint elit labore pariatur. Officia duis nulla voluptate et nulla ut voluptate laboris eu commodo veniam qui veniam.\r\n", - "registered": "2020-01-25T10:47:48 -01:00", - "latitude": -80.452139, - "longitude": 160.72546, - "tags": [ - "wontfix" - ] - }, - { - "id": 75, - "isActive": false, - "balance": "$1,913.42", - "picture": "http://placehold.it/32x32", - "age": 24, - "color": "Green", - "name": "Emma Jacobs", - "gender": "female", - "email": "emmajacobs@chorizon.com", - "phone": "+1 (899) 554-3847", - "address": "173 Tapscott Street, Esmont, Maine, 7450", - "about": "Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n", - "registered": "2019-03-29T06:24:13 -01:00", - "latitude": -35.53722, - "longitude": 155.703874, - "tags": [] - }, - { - "id": 76, - "isActive": false, - "balance": "$1,274.29", - "picture": "http://placehold.it/32x32", - "age": 25, - "color": "Green", - "name": "Clarice Gardner", - "gender": "female", - "email": "claricegardner@chorizon.com", - "phone": "+1 (810) 407-3258", - "address": "894 Brooklyn Road, Utting, New Hampshire, 6404", - "about": "Elit occaecat aute ea adipisicing mollit cupidatat aliquip excepteur veniam minim. Sunt quis dolore in commodo aute esse quis. Lorem in cillum commodo eu anim commodo mollit. Adipisicing enim sunt adipisicing cupidatat adipisicing eiusmod eu do sit nisi.\r\n", - "registered": "2014-10-20T10:13:32 -02:00", - "latitude": 17.11935, - "longitude": 65.38197, - "tags": [ - "new issue", - "wontfix" - ] - } -] diff --git a/tests/common.rs b/tests/common.rs deleted file mode 100644 index 43cea1447..000000000 --- a/tests/common.rs +++ /dev/null @@ -1,569 +0,0 @@ -#![allow(dead_code)] - -use actix_web::{http::StatusCode, test}; -use serde_json::{json, Value}; -use std::time::Duration; -use tempdir::TempDir; -use tokio::time::delay_for; - -use meilisearch_core::DatabaseOptions; -use meilisearch_http::data::Data; -use meilisearch_http::helpers::NormalizePath; -use meilisearch_http::option::Opt; - -/// Performs a search test on both post and get routes -#[macro_export] -macro_rules! test_post_get_search { - ($server:expr, $query:expr, |$response:ident, $status_code:ident | $block:expr) => { - let post_query: meilisearch_http::routes::search::SearchQueryPost = - serde_json::from_str(&$query.clone().to_string()).unwrap(); - let get_query: meilisearch_http::routes::search::SearchQuery = post_query.into(); - let get_query = ::serde_url_params::to_string(&get_query).unwrap(); - let ($response, $status_code) = $server.search_get(&get_query).await; - let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { - panic!( - "panic in get route: {:?}", - e.downcast_ref::<&str>().unwrap() - ) - }); - let ($response, $status_code) = $server.search_post($query).await; - let _ = ::std::panic::catch_unwind(|| $block).map_err(|e| { - panic!( - "panic in post route: {:?}", - e.downcast_ref::<&str>().unwrap() - ) - }); - }; -} - -pub struct Server { - pub uid: String, - pub data: Data, -} - -impl Server { - pub fn with_uid(uid: &str) -> Server { - let tmp_dir = TempDir::new("meilisearch").unwrap(); - - let default_db_options = DatabaseOptions::default(); - - let opt = Opt { - db_path: tmp_dir.path().join("db").to_str().unwrap().to_string(), - dumps_dir: tmp_dir.path().join("dump"), - dump_batch_size: 16, - http_addr: "127.0.0.1:7700".to_owned(), - master_key: None, - env: "development".to_owned(), - no_analytics: true, - max_mdb_size: default_db_options.main_map_size, - max_udb_size: default_db_options.update_map_size, - http_payload_size_limit: 10000000, - ..Opt::default() - }; - - let data = Data::new(opt).unwrap(); - - Server { - uid: uid.to_string(), - data, - } - } - - pub async fn test_server() -> Self { - let mut server = Self::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - ], - "searchableAttributes": [ - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags", - ], - "displayedAttributes": [ - "id", - "isActive", - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags", - ], - }); - - server.update_all_settings(body).await; - - let dataset = include_bytes!("assets/test_set.json"); - - let body: Value = serde_json::from_slice(dataset).unwrap(); - - server.add_or_replace_multiple_documents(body).await; - server - } - - pub fn data(&self) -> &Data { - &self.data - } - - pub async fn wait_update_id(&mut self, update_id: u64) { - // try 10 times to get status, or panic to not wait forever - for _ in 0..10 { - let (response, status_code) = self.get_update_status(update_id).await; - assert_eq!(status_code, 200); - - if response["status"] == "processed" || response["status"] == "failed" { - // eprintln!("{:#?}", response); - return; - } - - delay_for(Duration::from_secs(1)).await; - } - panic!("Timeout waiting for update id"); - } - - // Global Http request GET/POST/DELETE async or sync - - pub async fn get_request(&mut self, url: &str) -> (Value, StatusCode) { - eprintln!("get_request: {}", url); - - let mut app = - test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - let req = test::TestRequest::get().uri(url).to_request(); - let res = test::call_service(&mut app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) - } - - pub async fn post_request(&self, url: &str, body: Value) -> (Value, StatusCode) { - eprintln!("post_request: {}", url); - - let mut app = - test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - let req = test::TestRequest::post() - .uri(url) - .set_json(&body) - .to_request(); - let res = test::call_service(&mut app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) - } - - pub async fn post_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { - eprintln!("post_request_async: {}", url); - - let (response, status_code) = self.post_request(url, body).await; - eprintln!("response: {}", response); - assert!(response["updateId"].as_u64().is_some()); - self.wait_update_id(response["updateId"].as_u64().unwrap()) - .await; - (response, status_code) - } - - pub async fn put_request(&mut self, url: &str, body: Value) -> (Value, StatusCode) { - eprintln!("put_request: {}", url); - - let mut app = - test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - let req = test::TestRequest::put() - .uri(url) - .set_json(&body) - .to_request(); - let res = test::call_service(&mut app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) - } - - pub async fn put_request_async(&mut self, url: &str, body: Value) -> (Value, StatusCode) { - eprintln!("put_request_async: {}", url); - - let (response, status_code) = self.put_request(url, body).await; - assert!(response["updateId"].as_u64().is_some()); - assert_eq!(status_code, 202); - self.wait_update_id(response["updateId"].as_u64().unwrap()) - .await; - (response, status_code) - } - - pub async fn delete_request(&mut self, url: &str) -> (Value, StatusCode) { - eprintln!("delete_request: {}", url); - - let mut app = - test::init_service(meilisearch_http::create_app(&self.data, true).wrap(NormalizePath)).await; - - let req = test::TestRequest::delete().uri(url).to_request(); - let res = test::call_service(&mut app, req).await; - let status_code = res.status(); - - let body = test::read_body(res).await; - let response = serde_json::from_slice(&body).unwrap_or_default(); - (response, status_code) - } - - pub async fn delete_request_async(&mut self, url: &str) -> (Value, StatusCode) { - eprintln!("delete_request_async: {}", url); - - let (response, status_code) = self.delete_request(url).await; - assert!(response["updateId"].as_u64().is_some()); - assert_eq!(status_code, 202); - self.wait_update_id(response["updateId"].as_u64().unwrap()) - .await; - (response, status_code) - } - - // All Routes - - pub async fn list_indexes(&mut self) -> (Value, StatusCode) { - self.get_request("/indexes").await - } - - pub async fn create_index(&mut self, body: Value) -> (Value, StatusCode) { - self.post_request("/indexes", body).await - } - - pub async fn search_multi_index(&mut self, query: &str) -> (Value, StatusCode) { - let url = format!("/indexes/search?{}", query); - self.get_request(&url).await - } - - pub async fn get_index(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}", self.uid); - self.get_request(&url).await - } - - pub async fn update_index(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}", self.uid); - self.put_request(&url, body).await - } - - pub async fn delete_index(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}", self.uid); - self.delete_request(&url).await - } - - pub async fn search_get(&mut self, query: &str) -> (Value, StatusCode) { - let url = format!("/indexes/{}/search?{}", self.uid, query); - self.get_request(&url).await - } - - pub async fn search_post(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/search", self.uid); - self.post_request(&url, body).await - } - - pub async fn get_all_updates_status(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/updates", self.uid); - self.get_request(&url).await - } - - pub async fn get_update_status(&mut self, update_id: u64) -> (Value, StatusCode) { - let url = format!("/indexes/{}/updates/{}", self.uid, update_id); - self.get_request(&url).await - } - - pub async fn get_all_documents(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents", self.uid); - self.get_request(&url).await - } - - pub async fn add_or_replace_multiple_documents(&mut self, body: Value) { - let url = format!("/indexes/{}/documents", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn add_or_replace_multiple_documents_sync( - &mut self, - body: Value, - ) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents", self.uid); - self.post_request(&url, body).await - } - - pub async fn add_or_update_multiple_documents(&mut self, body: Value) { - let url = format!("/indexes/{}/documents", self.uid); - self.put_request_async(&url, body).await; - } - - pub async fn clear_all_documents(&mut self) { - let url = format!("/indexes/{}/documents", self.uid); - self.delete_request_async(&url).await; - } - - pub async fn get_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) { - let url = format!( - "/indexes/{}/documents/{}", - self.uid, - document_id.to_string() - ); - self.get_request(&url).await - } - - pub async fn delete_document(&mut self, document_id: impl ToString) -> (Value, StatusCode) { - let url = format!( - "/indexes/{}/documents/{}", - self.uid, - document_id.to_string() - ); - self.delete_request_async(&url).await - } - - pub async fn delete_multiple_documents(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/documents/delete-batch", self.uid); - self.post_request_async(&url, body).await - } - - pub async fn get_all_settings(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", self.uid); - self.get_request(&url).await - } - - pub async fn update_all_settings(&mut self, body: Value) { - let url = format!("/indexes/{}/settings", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_all_settings_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_all_settings(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_ranking_rules(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - self.get_request(&url).await - } - - pub async fn update_ranking_rules(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_ranking_rules_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_ranking_rules(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/ranking-rules", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_distinct_attribute(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - self.get_request(&url).await - } - - pub async fn update_distinct_attribute(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_distinct_attribute_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_distinct_attribute(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/distinct-attribute", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_primary_key(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/primary_key", self.uid); - self.get_request(&url).await - } - - pub async fn get_searchable_attributes(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - self.get_request(&url).await - } - - pub async fn update_searchable_attributes(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_searchable_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_searchable_attributes(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/searchable-attributes", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_displayed_attributes(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - self.get_request(&url).await - } - - pub async fn update_displayed_attributes(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_displayed_attributes_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_displayed_attributes(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/displayed-attributes", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_attributes_for_faceting(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - self.get_request(&url).await - } - - pub async fn update_attributes_for_faceting(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_attributes_for_faceting_sync( - &mut self, - body: Value, - ) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_attributes_for_faceting(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/attributes-for-faceting", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_synonyms(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/synonyms", self.uid); - self.get_request(&url).await - } - - pub async fn update_synonyms(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/synonyms", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_synonyms_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/synonyms", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_synonyms(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/synonyms", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_stop_words(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/stop-words", self.uid); - self.get_request(&url).await - } - - pub async fn update_stop_words(&mut self, body: Value) { - let url = format!("/indexes/{}/settings/stop-words", self.uid); - self.post_request_async(&url, body).await; - } - - pub async fn update_stop_words_sync(&mut self, body: Value) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/stop-words", self.uid); - self.post_request(&url, body).await - } - - pub async fn delete_stop_words(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/settings/stop-words", self.uid); - self.delete_request_async(&url).await - } - - pub async fn get_index_stats(&mut self) -> (Value, StatusCode) { - let url = format!("/indexes/{}/stats", self.uid); - self.get_request(&url).await - } - - pub async fn list_keys(&mut self) -> (Value, StatusCode) { - self.get_request("/keys").await - } - - pub async fn get_health(&mut self) -> (Value, StatusCode) { - self.get_request("/health").await - } - - pub async fn update_health(&mut self, body: Value) -> (Value, StatusCode) { - self.put_request("/health", body).await - } - - pub async fn get_version(&mut self) -> (Value, StatusCode) { - self.get_request("/version").await - } - - pub async fn get_sys_info(&mut self) -> (Value, StatusCode) { - self.get_request("/sys-info").await - } - - pub async fn get_sys_info_pretty(&mut self) -> (Value, StatusCode) { - self.get_request("/sys-info/pretty").await - } - - pub async fn trigger_dump(&self) -> (Value, StatusCode) { - self.post_request("/dumps", Value::Null).await - } - - pub async fn get_dump_status(&mut self, dump_uid: &str) -> (Value, StatusCode) { - let url = format!("/dumps/{}/status", dump_uid); - self.get_request(&url).await - } - - pub async fn trigger_dump_importation(&mut self, dump_uid: &str) -> (Value, StatusCode) { - let url = format!("/dumps/{}/import", dump_uid); - self.get_request(&url).await - } -} diff --git a/tests/dashboard.rs b/tests/dashboard.rs deleted file mode 100644 index 2dbaf8f7d..000000000 --- a/tests/dashboard.rs +++ /dev/null @@ -1,12 +0,0 @@ -mod common; - -#[actix_rt::test] -async fn dashboard() { - let mut server = common::Server::with_uid("movies"); - - let (_response, status_code) = server.get_request("/").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("/bulma.min.css").await; - assert_eq!(status_code, 200); -} diff --git a/tests/documents_add.rs b/tests/documents_add.rs deleted file mode 100644 index 382a1ed43..000000000 --- a/tests/documents_add.rs +++ /dev/null @@ -1,222 +0,0 @@ -use serde_json::json; - -mod common; - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/519 -#[actix_rt::test] -async fn check_add_documents_with_primary_key_param() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add documents - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/movies/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/568 -#[actix_rt::test] -async fn check_add_documents_with_nested_boolean() { - let mut server = common::Server::with_uid("tasks"); - - // 1 - Create the index with no primary_key - - let body = json!({ "uid": "tasks" }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document that contains a boolean in a nested object - - let body = json!([{ - "id": 12161, - "created_at": "2019-04-10T14:57:57.522Z", - "foo": { - "bar": { - "id": 121, - "crash": false - }, - "id": 45912 - } - }]); - - let url = "/indexes/tasks/documents"; - let (response, status_code) = server.post_request(&url, body).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/571 -#[actix_rt::test] -async fn check_add_documents_with_nested_null() { - let mut server = common::Server::with_uid("tasks"); - - // 1 - Create the index with no primary_key - - let body = json!({ "uid": "tasks" }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document that contains a null in a nested object - - let body = json!([{ - "id": 0, - "foo": { - "bar": null - } - }]); - - let url = "/indexes/tasks/documents"; - let (response, status_code) = server.post_request(&url, body).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/574 -#[actix_rt::test] -async fn check_add_documents_with_nested_sequence() { - let mut server = common::Server::with_uid("tasks"); - - // 1 - Create the index with no primary_key - - let body = json!({ "uid": "tasks" }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document that contains a seq in a nested object - - let body = json!([{ - "id": 0, - "foo": { - "bar": [123,456], - "fez": [{ - "id": 255, - "baz": "leesz", - "fuzz": { - "fax": [234] - }, - "sas": [] - }], - "foz": [{ - "id": 255, - "baz": "leesz", - "fuzz": { - "fax": [234] - }, - "sas": [] - }, - { - "id": 256, - "baz": "loss", - "fuzz": { - "fax": [235] - }, - "sas": [321, 321] - }] - } - }]); - - let url = "/indexes/tasks/documents"; - let (response, status_code) = server.post_request(&url, body.clone()).await; - eprintln!("{:#?}", response); - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); - - let url = "/indexes/tasks/search?q=leesz"; - let (response, status_code) = server.get_request(&url).await; - assert_eq!(status_code, 200); - assert_eq!(response["hits"], body); -} - -#[actix_rt::test] -// test sample from #807 -async fn add_document_with_long_field() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - let body = json!([{ - "documentId":"de1c2adbb897effdfe0deae32a01035e46f932ce", - "rank":1, - "relurl":"/configuration/app/web.html#locations", - "section":"Web", - "site":"docs", - "text":" The locations block is the most powerful, and potentially most involved, section of the .platform.app.yaml file. It allows you to control how the application container responds to incoming requests at a very fine-grained level. Common patterns also vary between language containers due to the way PHP-FPM handles incoming requests.\nEach entry of the locations block is an absolute URI path (with leading /) and its value includes the configuration directives for how the web server should handle matching requests. That is, if your domain is example.com then '/' means “requests for example.com/”, while '/admin' means “requests for example.com/admin”. If multiple blocks could match an incoming request then the most-specific will apply.\nweb:locations:'/':# Rules for all requests that don't otherwise match....'/sites/default/files':# Rules for any requests that begin with /sites/default/files....The simplest possible locations configuration is one that simply passes all requests on to your application unconditionally:\nweb:locations:'/':passthru:trueThat is, all requests to /* should be forwarded to the process started by web.commands.start above. Note that for PHP containers the passthru key must specify what PHP file the request should be forwarded to, and must also specify a docroot under which the file lives. For example:\nweb:locations:'/':root:'web'passthru:'/app.php'This block will serve requests to / from the web directory in the application, and if a file doesn’t exist on disk then the request will be forwarded to the /app.php script.\nA full list of the possible subkeys for locations is below.\n root: The folder from which to serve static assets for this location relative to the application root. The application root is the directory in which the .platform.app.yaml file is located. Typical values for this property include public or web. Setting it to '' is not recommended, and its behavior may vary depending on the type of application. Absolute paths are not supported.\n passthru: Whether to forward disallowed and missing resources from this location to the application and can be true, false or an absolute URI path (with leading /). The default value is false. For non-PHP applications it will generally be just true or false. In a PHP application this will typically be the front controller such as /index.php or /app.php. This entry works similar to mod_rewrite under Apache. Note: If the value of passthru does not begin with the same value as the location key it is under, the passthru may evaluate to another entry. That may be useful when you want different cache settings for different paths, for instance, but want missing files in all of them to map back to the same front controller. See the example block below.\n index: The files to consider when serving a request for a directory: an array of file names or null. (typically ['index.html']). Note that in order for this to work, access to the static files named must be allowed by the allow or rules keys for this location.\n expires: How long to allow static assets from this location to be cached (this enables the Cache-Control and Expires headers) and can be a time or -1 for no caching (default). Times can be suffixed with “ms” (milliseconds), “s” (seconds), “m” (minutes), “h” (hours), “d” (days), “w” (weeks), “M” (months, 30d) or “y” (years, 365d).\n scripts: Whether to allow loading scripts in that location (true or false). This directive is only meaningful on PHP.\n allow: Whether to allow serving files which don’t match a rule (true or false, default: true).\n headers: Any additional headers to apply to static assets. This section is a mapping of header names to header values. Responses from the application aren’t affected, to avoid overlap with the application’s own ability to include custom headers in the response.\n rules: Specific overrides for a specific location. The key is a PCRE (regular expression) that is matched against the full request path.\n request_buffering: Most application servers do not support chunked requests (e.g. fpm, uwsgi), so Platform.sh enables request_buffering by default to handle them. That default configuration would look like this if it was present in .platform.app.yaml:\nweb:locations:'/':passthru:truerequest_buffering:enabled:truemax_request_size:250mIf the application server can already efficiently handle chunked requests, the request_buffering subkey can be modified to disable it entirely (enabled: false). Additionally, applications that frequently deal with uploads greater than 250MB in size can update the max_request_size key to the application’s needs. Note that modifications to request_buffering will need to be specified at each location where it is desired.\n ", - "title":"Locations", - "url":"/configuration/app/web.html#locations" - }]); - server.add_or_replace_multiple_documents(body).await; - let (response, _status) = server - .search_post(json!({ "q": "request_buffering" })) - .await; - assert!(!response["hits"].as_array().unwrap().is_empty()); -} - -#[actix_rt::test] -async fn documents_with_same_id_are_overwritten() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test"})).await; - let documents = json!([ - { - "id": 1, - "content": "test1" - }, - { - "id": 1, - "content": "test2" - }, - ]); - server.add_or_replace_multiple_documents(documents).await; - let (response, _status) = server.get_all_documents().await; - assert_eq!(response.as_array().unwrap().len(), 1); - assert_eq!( - response.as_array().unwrap()[0].as_object().unwrap()["content"], - "test2" - ); -} diff --git a/tests/documents_delete.rs b/tests/documents_delete.rs deleted file mode 100644 index 4353a5355..000000000 --- a/tests/documents_delete.rs +++ /dev/null @@ -1,67 +0,0 @@ -mod common; - -use serde_json::json; - -#[actix_rt::test] -async fn delete() { - let mut server = common::Server::test_server().await; - - let (_response, status_code) = server.get_document(50).await; - assert_eq!(status_code, 200); - - server.delete_document(50).await; - - let (_response, status_code) = server.get_document(50).await; - assert_eq!(status_code, 404); -} - -// Resolve the issue https://github.com/meilisearch/MeiliSearch/issues/493 -#[actix_rt::test] -async fn delete_batch() { - let mut server = common::Server::test_server().await; - - let doc_ids = vec!(50, 55, 60); - for doc_id in &doc_ids { - let (_response, status_code) = server.get_document(doc_id).await; - assert_eq!(status_code, 200); - } - - let body = serde_json::json!(&doc_ids); - server.delete_multiple_documents(body).await; - - for doc_id in &doc_ids { - let (_response, status_code) = server.get_document(doc_id).await; - assert_eq!(status_code, 404); - } -} - -#[actix_rt::test] -async fn text_clear_all_placeholder_search() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - }); - - server.create_index(body).await; - let settings = json!({ - "attributesForFaceting": ["genre"], - }); - - server.update_all_settings(settings).await; - - let documents = json!([ - { "id": 2, "title": "Pride and Prejudice", "author": "Jane Austin", "genre": "romance" }, - { "id": 456, "title": "Le Petit Prince", "author": "Antoine de Saint-Exupéry", "genre": "adventure" }, - { "id": 1, "title": "Alice In Wonderland", "author": "Lewis Carroll", "genre": "fantasy" }, - { "id": 1344, "title": "The Hobbit", "author": "J. R. R. Tolkien", "genre": "fantasy" }, - { "id": 4, "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling", "genre": "fantasy" }, - { "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams" } - ]); - - server.add_or_update_multiple_documents(documents).await; - server.clear_all_documents().await; - let (response, _) = server.search_post(json!({ "q": "", "facetsDistribution": ["genre"] })).await; - assert_eq!(response["nbHits"], 0); - let (response, _) = server.search_post(json!({ "q": "" })).await; - assert_eq!(response["nbHits"], 0); -} diff --git a/tests/documents_get.rs b/tests/documents_get.rs deleted file mode 100644 index 35e04f494..000000000 --- a/tests/documents_get.rs +++ /dev/null @@ -1,23 +0,0 @@ -use serde_json::json; -use actix_web::http::StatusCode; - -mod common; - -#[actix_rt::test] -async fn get_documents_from_unexisting_index_is_error() { - let mut server = common::Server::with_uid("test"); - let (response, status) = server.get_all_documents().await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(response["errorCode"], "index_not_found"); - assert_eq!(response["errorType"], "invalid_request_error"); - assert_eq!(response["errorLink"], "https://docs.meilisearch.com/errors#index_not_found"); -} - -#[actix_rt::test] -async fn get_empty_documents_list() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - let (response, status) = server.get_all_documents().await; - assert_eq!(status, StatusCode::OK); - assert!(response.as_array().unwrap().is_empty()); -} diff --git a/tests/dump.rs b/tests/dump.rs deleted file mode 100644 index 701b754aa..000000000 --- a/tests/dump.rs +++ /dev/null @@ -1,395 +0,0 @@ -use assert_json_diff::{assert_json_eq, assert_json_include}; -use meilisearch_http::helpers::compression; -use serde_json::{json, Value}; -use std::fs::File; -use std::path::Path; -use std::thread; -use std::time::Duration; -use tempfile::TempDir; - -#[macro_use] mod common; - -async fn trigger_and_wait_dump(server: &mut common::Server) -> String { - let (value, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - for _ in 0..20 as u8 { - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - assert_eq!(status_code, 200); - assert_ne!(value["status"].as_str(), Some("dump_process_failed")); - - if value["status"].as_str() == Some("done") { return dump_uid } - thread::sleep(Duration::from_millis(100)); - } - - unreachable!("dump creation runned out of time") -} - -fn current_db_version() -> (String, String, String) { - let current_version_major = env!("CARGO_PKG_VERSION_MAJOR").to_string(); - let current_version_minor = env!("CARGO_PKG_VERSION_MINOR").to_string(); - let current_version_patch = env!("CARGO_PKG_VERSION_PATCH").to_string(); - - (current_version_major, current_version_minor, current_version_patch) -} - -fn current_dump_version() -> String { - "V1".into() -} - -fn read_all_jsonline(r: R) -> Value { - let deserializer = serde_json::Deserializer::from_reader(r); - let iterator = deserializer.into_iter::(); - - json!(iterator.map(|v| v.unwrap()).collect::>()) -} - -#[actix_rt::test] -#[ignore] -async fn trigger_dump_should_return_ok() { - let server = common::Server::test_server().await; - - let (_, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); -} - -#[actix_rt::test] -#[ignore] -async fn trigger_dump_twice_should_return_conflict() { - let server = common::Server::test_server().await; - - let expected = json!({ - "message": "Another dump is already in progress", - "errorCode": "dump_already_in_progress", - "errorType": "invalid_request_error", - "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" - }); - - let (_, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let (value, status_code) = server.trigger_dump().await; - - - assert_json_eq!(expected, value, ordered: false); - assert_eq!(status_code, 409); -} - -#[actix_rt::test] -#[ignore] -async fn trigger_dump_concurently_should_return_conflict() { - let server = common::Server::test_server().await; - - let expected = json!({ - "message": "Another dump is already in progress", - "errorCode": "dump_already_in_progress", - "errorType": "invalid_request_error", - "errorLink": "https://docs.meilisearch.com/errors#dump_already_in_progress" - }); - - let ((_value_1, _status_code_1), (value_2, status_code_2)) = futures::join!(server.trigger_dump(), server.trigger_dump()); - - assert_json_eq!(expected, value_2, ordered: false); - assert_eq!(status_code_2, 409); -} - -#[actix_rt::test] -#[ignore] -async fn get_dump_status_early_should_return_in_progress() { - let mut server = common::Server::test_server().await; - - - - let (value, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - let expected = json!({ - "uid": dump_uid, - "status": "in_progress" - }); - - assert_eq!(status_code, 200); - - assert_json_eq!(expected, value, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn get_dump_status_should_return_done() { - let mut server = common::Server::test_server().await; - - - let (value, status_code) = server.trigger_dump().await; - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - let expected = json!({ - "uid": dump_uid.clone(), - "status": "done" - }); - - thread::sleep(Duration::from_secs(1)); // wait dump until process end - - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - assert_eq!(status_code, 200); - - assert_json_eq!(expected, value, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn get_dump_status_should_return_error_provoking_it() { - let mut server = common::Server::test_server().await; - - - let (value, status_code) = server.trigger_dump().await; - - // removing destination directory provoking `No such file or directory` error - std::fs::remove_dir(server.data().dumps_dir.clone()).unwrap(); - - assert_eq!(status_code, 202); - - let dump_uid = value["uid"].as_str().unwrap().to_string(); - - let expected = json!({ - "uid": dump_uid.clone(), - "status": "failed", - "message": "Dump process failed: compressing dump; No such file or directory (os error 2)", - "errorCode": "dump_process_failed", - "errorType": "internal_error", - "errorLink": "https://docs.meilisearch.com/errors#dump_process_failed" - }); - - thread::sleep(Duration::from_secs(1)); // wait dump until process end - - let (value, status_code) = server.get_dump_status(&dump_uid).await; - - assert_eq!(status_code, 200); - - assert_json_eq!(expected, value, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn dump_metadata_should_be_valid() { - let mut server = common::Server::test_server().await; - - let body = json!({ - "uid": "test2", - "primaryKey": "test2_id", - }); - - server.create_index(body).await; - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("metadata.json")).unwrap(); - let mut metadata: serde_json::Value = serde_json::from_reader(file).unwrap(); - - // fields are randomly ordered - metadata.get_mut("indexes").unwrap() - .as_array_mut().unwrap() - .sort_by(|a, b| - a.get("uid").unwrap().as_str().cmp(&b.get("uid").unwrap().as_str()) - ); - - let (major, minor, patch) = current_db_version(); - - let expected = json!({ - "indexes": [{ - "uid": "test", - "primaryKey": "id", - }, { - "uid": "test2", - "primaryKey": "test2_id", - } - ], - "dbVersion": format!("{}.{}.{}", major, minor, patch), - "dumpVersion": current_dump_version() - }); - - assert_json_include!(expected: expected, actual: metadata); -} - -#[actix_rt::test] -#[ignore] -async fn dump_gzip_should_have_been_created() { - let mut server = common::Server::test_server().await; - - - let dump_uid = trigger_and_wait_dump(&mut server).await; - let dumps_dir = Path::new(&server.data().dumps_dir); - - let compressed_path = dumps_dir.join(format!("{}.dump", dump_uid)); - assert!(File::open(compressed_path).is_ok()); -} - -#[actix_rt::test] -#[ignore] -async fn dump_index_settings_should_be_valid() { - let mut server = common::Server::test_server().await; - - let expected = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": "email", - "searchableAttributes": [ - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags" - ], - "displayedAttributes": [ - "id", - "isActive", - "balance", - "picture", - "age", - "color", - "name", - "gender", - "email", - "phone", - "address", - "about", - "registered", - "latitude", - "longitude", - "tags" - ], - "stopWords": [ - "in", - "ad" - ], - "synonyms": { - "wolverine": ["xmen", "logan"], - "logan": ["wolverine", "xmen"] - }, - "attributesForFaceting": [ - "gender", - "color", - "tags" - ] - }); - - server.update_all_settings(expected.clone()).await; - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("test").join("settings.json")).unwrap(); - let settings: serde_json::Value = serde_json::from_reader(file).unwrap(); - - assert_json_eq!(expected, settings, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn dump_index_documents_should_be_valid() { - let mut server = common::Server::test_server().await; - - let dataset = include_bytes!("assets/dumps/v1/test/documents.jsonl"); - let mut slice: &[u8] = dataset; - - let expected: Value = read_all_jsonline(&mut slice); - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("test").join("documents.jsonl")).unwrap(); - let documents = read_all_jsonline(file); - - assert_json_eq!(expected, documents, ordered: false); -} - -#[actix_rt::test] -#[ignore] -async fn dump_index_updates_should_be_valid() { - let mut server = common::Server::test_server().await; - - let dataset = include_bytes!("assets/dumps/v1/test/updates.jsonl"); - let mut slice: &[u8] = dataset; - - let expected: Value = read_all_jsonline(&mut slice); - - let uid = trigger_and_wait_dump(&mut server).await; - - let dumps_dir = Path::new(&server.data().dumps_dir); - let tmp_dir = TempDir::new().unwrap(); - let tmp_dir_path = tmp_dir.path(); - - compression::from_tar_gz(&dumps_dir.join(&format!("{}.dump", uid)), tmp_dir_path).unwrap(); - - let file = File::open(tmp_dir_path.join("test").join("updates.jsonl")).unwrap(); - let mut updates = read_all_jsonline(file); - - - // hotfix until #943 is fixed (https://github.com/meilisearch/MeiliSearch/issues/943) - updates.as_array_mut().unwrap() - .get_mut(0).unwrap() - .get_mut("type").unwrap() - .get_mut("settings").unwrap() - .get_mut("displayed_attributes").unwrap() - .get_mut("Update").unwrap() - .as_array_mut().unwrap().sort_by(|a, b| a.as_str().cmp(&b.as_str())); - - eprintln!("{}\n", updates.to_string()); - eprintln!("{}", expected.to_string()); - assert_json_include!(expected: expected, actual: updates); -} - -#[actix_rt::test] -#[ignore] -async fn get_unexisting_dump_status_should_return_not_found() { - let mut server = common::Server::test_server().await; - - let (_, status_code) = server.get_dump_status("4242").await; - - assert_eq!(status_code, 404); -} diff --git a/tests/errors.rs b/tests/errors.rs deleted file mode 100644 index e11483356..000000000 --- a/tests/errors.rs +++ /dev/null @@ -1,200 +0,0 @@ -mod common; - -use std::thread; -use std::time::Duration; - -use actix_http::http::StatusCode; -use serde_json::{json, Map, Value}; - -macro_rules! assert_error { - ($code:literal, $type:literal, $status:path, $req:expr) => { - let (response, status_code) = $req; - assert_eq!(status_code, $status); - assert_eq!(response["errorCode"].as_str().unwrap(), $code); - assert_eq!(response["errorType"].as_str().unwrap(), $type); - }; -} - -macro_rules! assert_error_async { - ($code:literal, $type:literal, $server:expr, $req:expr) => { - let (response, _) = $req; - let update_id = response["updateId"].as_u64().unwrap(); - for _ in 1..10 { - let (response, status_code) = $server.get_update_status(update_id).await; - assert_eq!(status_code, StatusCode::OK); - if response["status"] == "processed" || response["status"] == "failed" { - println!("response: {}", response); - assert_eq!(response["status"], "failed"); - assert_eq!(response["errorCode"], $code); - assert_eq!(response["errorType"], $type); - return - } - thread::sleep(Duration::from_secs(1)); - } - }; -} - -#[actix_rt::test] -async fn index_already_exists_error() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test" - }); - let (response, status_code) = server.create_index(body.clone()).await; - println!("{}", response); - assert_eq!(status_code, StatusCode::CREATED); - - let (response, status_code) = server.create_index(body.clone()).await; - println!("{}", response); - - assert_error!( - "index_already_exists", - "invalid_request_error", - StatusCode::BAD_REQUEST, - (response, status_code)); -} - -#[actix_rt::test] -async fn index_not_found_error() { - let mut server = common::Server::with_uid("test"); - assert_error!( - "index_not_found", - "invalid_request_error", - StatusCode::NOT_FOUND, - server.get_index().await); -} - -#[actix_rt::test] -async fn primary_key_already_present_error() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "test" - }); - server.create_index(body.clone()).await; - let body = json!({ - "primaryKey": "t" - }); - assert_error!( - "primary_key_already_present", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.update_index(body).await); -} - -#[actix_rt::test] -async fn max_field_limit_exceeded_error() { - let mut server = common::Server::test_server().await; - let body = json!({ - "uid": "test", - }); - server.create_index(body).await; - let mut doc = Map::with_capacity(70_000); - doc.insert("id".into(), Value::String("foo".into())); - for i in 0..69_999 { - doc.insert(format!("field{}", i), Value::String("foo".into())); - } - let docs = json!([doc]); - assert_error_async!( - "max_fields_limit_exceeded", - "invalid_request_error", - server, - server.add_or_replace_multiple_documents_sync(docs).await); -} - -#[actix_rt::test] -async fn missing_document_id() { - let mut server = common::Server::test_server().await; - let body = json!({ - "uid": "test", - "primaryKey": "test" - }); - server.create_index(body).await; - let docs = json!([ - { - "foo": "bar", - } - ]); - assert_error_async!( - "missing_document_id", - "invalid_request_error", - server, - server.add_or_replace_multiple_documents_sync(docs).await); -} - -#[actix_rt::test] -async fn facet_error() { - let mut server = common::Server::test_server().await; - let search = json!({ - "q": "foo", - "facetFilters": ["test:hello"] - }); - assert_error!( - "invalid_facet", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.search_post(search).await); -} - -#[actix_rt::test] -async fn filters_error() { - let mut server = common::Server::test_server().await; - let search = json!({ - "q": "foo", - "filters": "fo:12" - }); - assert_error!( - "invalid_filter", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.search_post(search).await); -} - -#[actix_rt::test] -async fn bad_request_error() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "foo": "bar", - }); - assert_error!( - "bad_request", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.search_post(body).await); -} - -#[actix_rt::test] -async fn document_not_found_error() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({"uid": "test"})).await; - assert_error!( - "document_not_found", - "invalid_request_error", - StatusCode::NOT_FOUND, - server.get_document(100).await); -} - -#[actix_rt::test] -async fn payload_too_large_error() { - let mut server = common::Server::with_uid("test"); - let bigvec = vec![0u64; 10_000_000]; // 80mb - assert_error!( - "payload_too_large", - "invalid_request_error", - StatusCode::PAYLOAD_TOO_LARGE, - server.create_index(json!(bigvec)).await); -} - -#[actix_rt::test] -async fn missing_primary_key_error() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({"uid": "test"})).await; - let document = json!([{ - "content": "test" - }]); - assert_error!( - "missing_primary_key", - "invalid_request_error", - StatusCode::BAD_REQUEST, - server.add_or_replace_multiple_documents_sync(document).await); -} diff --git a/tests/health.rs b/tests/health.rs deleted file mode 100644 index f72127431..000000000 --- a/tests/health.rs +++ /dev/null @@ -1,11 +0,0 @@ -mod common; - -#[actix_rt::test] -async fn test_healthyness() { - let mut server = common::Server::with_uid("movies"); - - // Check that the server is healthy - - let (_response, status_code) = server.get_health().await; - assert_eq!(status_code, 204); -} diff --git a/tests/index.rs b/tests/index.rs deleted file mode 100644 index 271507e03..000000000 --- a/tests/index.rs +++ /dev/null @@ -1,809 +0,0 @@ -use actix_web::http::StatusCode; -use assert_json_diff::assert_json_eq; -use serde_json::{json, Value}; - -mod common; - -#[actix_rt::test] -async fn create_index_with_name() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 8); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid.len(), r1_uid.len()); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn create_index_with_uid() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "uid": "movies", - }); - - let (res1_value, status_code) = server.create_index(body.clone()).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid, "movies"); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 1.5 verify that error is thrown when trying to create the same index - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - assert_eq!( - response["errorCode"].as_str().unwrap(), - "index_already_exists" - ); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid, r1_uid); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn create_index_with_name_and_uid() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "Films", - "uid": "fr_movies", - }); - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "Films"); - assert_eq!(r1_uid, "fr_movies"); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid, r1_uid); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn rename_index() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - "uid": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 6); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Update an index name - - let body = json!({ - "name": "TV Shows", - }); - - let (res2_value, status_code) = server.update_index(body).await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_object().unwrap().len(), 5); - let r2_name = res2_value["name"].as_str().unwrap(); - let r2_uid = res2_value["uid"].as_str().unwrap(); - let r2_created_at = res2_value["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, "TV Shows"); - assert_eq!(r2_uid, r1_uid); - assert_eq!(r2_created_at, r1_created_at); - assert!(r2_updated_at.len() > 1); - - // 3 - Check the list of indexes - - let (res3_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res3_value.as_array().unwrap().len(), 1); - assert_eq!(res3_value[0].as_object().unwrap().len(), 5); - let r3_name = res3_value[0]["name"].as_str().unwrap(); - let r3_uid = res3_value[0]["uid"].as_str().unwrap(); - let r3_created_at = res3_value[0]["createdAt"].as_str().unwrap(); - let r3_updated_at = res3_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r3_name, r2_name); - assert_eq!(r3_uid.len(), r1_uid.len()); - assert_eq!(r3_created_at.len(), r1_created_at.len()); - assert_eq!(r3_updated_at.len(), r2_updated_at.len()); -} - -#[actix_rt::test] -async fn delete_index_and_recreate_it() { - let mut server = common::Server::with_uid("movies"); - - // 0 - delete unexisting index is error - - let (response, status_code) = server.delete_request("/indexes/test").await; - assert_eq!(status_code, 404); - assert_eq!(&response["errorCode"], "index_not_found"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - "uid": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 6); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid.len(), r1_uid.len()); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); - - // 3- Delete an index - - let (_res2_value, status_code) = server.delete_index().await; - - assert_eq!(status_code, 204); - - // 4 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 0); - - // 5 - Create a new index - - let body = json!({ - "name": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 8); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 6 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_name = res2_value[0]["name"].as_str().unwrap(); - let r2_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_name, r1_name); - assert_eq!(r2_uid.len(), r1_uid.len()); - assert_eq!(r2_created_at.len(), r1_created_at.len()); - assert_eq!(r2_updated_at.len(), r1_updated_at.len()); -} - -#[actix_rt::test] -async fn check_multiples_indexes() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create a new index - - let body = json!({ - "name": "movies", - }); - - let (res1_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res1_value.as_object().unwrap().len(), 5); - let r1_name = res1_value["name"].as_str().unwrap(); - let r1_uid = res1_value["uid"].as_str().unwrap(); - let r1_created_at = res1_value["createdAt"].as_str().unwrap(); - let r1_updated_at = res1_value["updatedAt"].as_str().unwrap(); - assert_eq!(r1_name, "movies"); - assert_eq!(r1_uid.len(), 8); - assert!(r1_created_at.len() > 1); - assert!(r1_updated_at.len() > 1); - - // 2 - Check the list of indexes - - let (res2_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res2_value.as_array().unwrap().len(), 1); - assert_eq!(res2_value[0].as_object().unwrap().len(), 5); - let r2_0_name = res2_value[0]["name"].as_str().unwrap(); - let r2_0_uid = res2_value[0]["uid"].as_str().unwrap(); - let r2_0_created_at = res2_value[0]["createdAt"].as_str().unwrap(); - let r2_0_updated_at = res2_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(r2_0_name, r1_name); - assert_eq!(r2_0_uid.len(), r1_uid.len()); - assert_eq!(r2_0_created_at.len(), r1_created_at.len()); - assert_eq!(r2_0_updated_at.len(), r1_updated_at.len()); - - // 3 - Create a new index - - let body = json!({ - "name": "films", - }); - - let (res3_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 201); - assert_eq!(res3_value.as_object().unwrap().len(), 5); - let r3_name = res3_value["name"].as_str().unwrap(); - let r3_uid = res3_value["uid"].as_str().unwrap(); - let r3_created_at = res3_value["createdAt"].as_str().unwrap(); - let r3_updated_at = res3_value["updatedAt"].as_str().unwrap(); - assert_eq!(r3_name, "films"); - assert_eq!(r3_uid.len(), 8); - assert!(r3_created_at.len() > 1); - assert!(r3_updated_at.len() > 1); - - // 4 - Check the list of indexes - - let (res4_value, status_code) = server.list_indexes().await; - - assert_eq!(status_code, 200); - assert_eq!(res4_value.as_array().unwrap().len(), 2); - assert_eq!(res4_value[0].as_object().unwrap().len(), 5); - let r4_0_name = res4_value[0]["name"].as_str().unwrap(); - let r4_0_uid = res4_value[0]["uid"].as_str().unwrap(); - let r4_0_created_at = res4_value[0]["createdAt"].as_str().unwrap(); - let r4_0_updated_at = res4_value[0]["updatedAt"].as_str().unwrap(); - assert_eq!(res4_value[1].as_object().unwrap().len(), 5); - let r4_1_name = res4_value[1]["name"].as_str().unwrap(); - let r4_1_uid = res4_value[1]["uid"].as_str().unwrap(); - let r4_1_created_at = res4_value[1]["createdAt"].as_str().unwrap(); - let r4_1_updated_at = res4_value[1]["updatedAt"].as_str().unwrap(); - if r4_0_name == r1_name { - assert_eq!(r4_0_name, r1_name); - assert_eq!(r4_0_uid.len(), r1_uid.len()); - assert_eq!(r4_0_created_at.len(), r1_created_at.len()); - assert_eq!(r4_0_updated_at.len(), r1_updated_at.len()); - } else { - assert_eq!(r4_0_name, r3_name); - assert_eq!(r4_0_uid.len(), r3_uid.len()); - assert_eq!(r4_0_created_at.len(), r3_created_at.len()); - assert_eq!(r4_0_updated_at.len(), r3_updated_at.len()); - } - if r4_1_name == r1_name { - assert_eq!(r4_1_name, r1_name); - assert_eq!(r4_1_uid.len(), r1_uid.len()); - assert_eq!(r4_1_created_at.len(), r1_created_at.len()); - assert_eq!(r4_1_updated_at.len(), r1_updated_at.len()); - } else { - assert_eq!(r4_1_name, r3_name); - assert_eq!(r4_1_uid.len(), r3_uid.len()); - assert_eq!(r4_1_created_at.len(), r3_created_at.len()); - assert_eq!(r4_1_updated_at.len(), r3_updated_at.len()); - } -} - -#[actix_rt::test] -async fn create_index_failed() { - let mut server = common::Server::with_uid("movies"); - - // 2 - Push index creation with empty json body - - let body = json!({}); - - let (res_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = res_value["message"].as_str().unwrap(); - assert_eq!(res_value.as_object().unwrap().len(), 4); - assert_eq!(message, "Index creation must have an uid"); - - // 3 - Create a index with extra data - - let body = json!({ - "name": "movies", - "active": true - }); - - let (_res_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - - // 3 - Create a index with wrong data type - - let body = json!({ - "name": "movies", - "uid": 0 - }); - - let (_res_value, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); -} - -// Resolve issue https://github.com/meilisearch/MeiliSearch/issues/492 -#[actix_rt::test] -async fn create_index_with_primary_key_and_index() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index - - let body = json!({ - "uid": "movies", - "primaryKey": "id", - }); - - let (_response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - - // 2 - Add content - - let body = json!([{ - "id": 123, - "text": "The mask" - }]); - - server.add_or_replace_multiple_documents(body.clone()).await; - - // 3 - Retreive document - - let (response, _status_code) = server.get_document(123).await; - - let expect = json!({ - "id": 123, - "text": "The mask" - }); - - assert_json_eq!(response, expect, ordered: false); -} - -// Resolve issue https://github.com/meilisearch/MeiliSearch/issues/497 -// Test when the given index uid is not valid -// Should have a 400 status code -// Should have the right error message -#[actix_rt::test] -async fn create_index_with_invalid_uid() { - let mut server = common::Server::with_uid(""); - - // 1 - Create the index with invalid uid - - let body = json!({ - "uid": "the movies" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); - - // 2 - Create the index with invalid uid - - let body = json!({ - "uid": "%$#" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); - - // 3 - Create the index with invalid uid - - let body = json!({ - "uid": "the~movies" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); - - // 4 - Create the index with invalid uid - - let body = json!({ - "uid": "🎉" - }); - - let (response, status_code) = server.create_index(body).await; - - assert_eq!(status_code, 400); - let message = response["message"].as_str().unwrap(); - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(message, "Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_)."); -} - -// Test that it's possible to add primary_key if it's not already set on index creation -#[actix_rt::test] -async fn create_index_and_add_indentifier_after() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Update the index and add an primary_key. - - let body = json!({ - "primaryKey": "id", - }); - - let (response, status_code) = server.update_index(body).await; - assert_eq!(status_code, 200); - eprintln!("response: {:#?}", response); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); - - // 3 - Get index to verify if the primary_key is good - - let (response, status_code) = server.get_index().await; - assert_eq!(status_code, 200); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); -} - -// Test that it's impossible to change the primary_key -#[actix_rt::test] -async fn create_index_and_update_indentifier_after() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - "primaryKey": "id", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); - - // 2 - Update the index and add an primary_key. - - let body = json!({ - "primaryKey": "skuid", - }); - - let (_response, status_code) = server.update_index(body).await; - assert_eq!(status_code, 400); - - // 3 - Get index to verify if the primary_key still the first one - - let (response, status_code) = server.get_index().await; - assert_eq!(status_code, 200); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); -} - -// Test that schema inference work well -#[actix_rt::test] -async fn create_index_without_primary_key_and_add_document() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Add a document - - let body = json!([{ - "id": 123, - "title": "I'm a legend", - }]); - - server.add_or_update_multiple_documents(body).await; - - // 3 - Get index to verify if the primary_key is good - - let (response, status_code) = server.get_index().await; - assert_eq!(status_code, 200); - assert_eq!(response["primaryKey"].as_str().unwrap(), "id"); -} - -// Test search with no primary_key -#[actix_rt::test] -async fn create_index_without_primary_key_and_search() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2 - Search - - let query = "q=captain&limit=3"; - - let (response, status_code) = server.search_get(&query).await; - assert_eq!(status_code, 200); - assert_eq!(response["hits"].as_array().unwrap().len(), 0); -} - -// Test the error message when we push an document update and impossibility to find primary key -// Test issue https://github.com/meilisearch/MeiliSearch/issues/517 -#[actix_rt::test] -async fn check_add_documents_without_primary_key() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Create the index with no primary_key - - let body = json!({ - "uid": "movies", - }); - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2- Add document - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let (response, status_code) = server.add_or_replace_multiple_documents_sync(body).await; - - assert_eq!(response.as_object().unwrap().len(), 4); - assert_eq!(response["errorCode"], "missing_primary_key"); - assert_eq!(status_code, 400); -} - -#[actix_rt::test] -async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() { - let mut server = common::Server::with_uid("movies"); - - let body = json!({ - "uid": "movies", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let dataset = include_bytes!("./assets/test_set.json"); - - let body: Value = serde_json::from_slice(dataset).unwrap(); - - // 2. Index the documents from movies.json, present inside of assets directory - server.add_or_replace_multiple_documents(body).await; - - // 3. Fetch the status of the indexing done above. - let (response, status_code) = server.get_all_updates_status().await; - - // 4. Verify the fetch is successful and indexing status is 'processed' - assert_eq!(status_code, 200); - assert_eq!(response[0]["status"], "processed"); -} - -#[actix_rt::test] -async fn get_empty_index() { - let mut server = common::Server::with_uid("test"); - let (response, _status) = server.list_indexes().await; - assert!(response.as_array().unwrap().is_empty()); -} - -#[actix_rt::test] -async fn create_and_list_multiple_indices() { - let mut server = common::Server::with_uid("test"); - for i in 0..10 { - server - .create_index(json!({ "uid": format!("test{}", i) })) - .await; - } - let (response, _status) = server.list_indexes().await; - assert_eq!(response.as_array().unwrap().len(), 10); -} - -#[actix_rt::test] -async fn get_unexisting_index_is_error() { - let mut server = common::Server::with_uid("test"); - let (response, status) = server.get_index().await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(response["errorCode"], "index_not_found"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn create_index_twice_is_error() { - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - let (response, status) = server.create_index(json!({ "uid": "test" })).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(response["errorCode"], "index_already_exists"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn badly_formatted_index_name_is_error() { - let mut server = common::Server::with_uid("$__test"); - let (response, status) = server.create_index(json!({ "uid": "$__test" })).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(response["errorCode"], "invalid_index_uid"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn correct_response_no_primary_key_index() { - let mut server = common::Server::with_uid("test"); - let (response, _status) = server.create_index(json!({ "uid": "test" })).await; - assert_eq!(response["primaryKey"], Value::Null); -} - -#[actix_rt::test] -async fn correct_response_with_primary_key_index() { - let mut server = common::Server::with_uid("test"); - let (response, _status) = server - .create_index(json!({ "uid": "test", "primaryKey": "test" })) - .await; - assert_eq!(response["primaryKey"], "test"); -} - -#[actix_rt::test] -async fn udpate_unexisting_index_is_error() { - let mut server = common::Server::with_uid("test"); - let (response, status) = server.update_index(json!({ "primaryKey": "foobar" })).await; - assert_eq!(status, StatusCode::NOT_FOUND); - assert_eq!(response["errorCode"], "index_not_found"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn update_existing_primary_key_is_error() { - let mut server = common::Server::with_uid("test"); - server - .create_index(json!({ "uid": "test", "primaryKey": "key" })) - .await; - let (response, status) = server.update_index(json!({ "primaryKey": "test2" })).await; - assert_eq!(status, StatusCode::BAD_REQUEST); - assert_eq!(response["errorCode"], "primary_key_already_present"); - assert_eq!(response["errorType"], "invalid_request_error"); -} - -#[actix_rt::test] -async fn test_facets_distribution_attribute() { - let mut server = common::Server::test_server().await; - - let (response, _status_code) = server.get_index_stats().await; - - let expected = json!({ - "isIndexing": false, - "numberOfDocuments":77, - "fieldsDistribution":{ - "age":77, - "gender":77, - "phone":77, - "name":77, - "registered":77, - "latitude":77, - "email":77, - "tags":77, - "longitude":77, - "color":77, - "address":77, - "balance":77, - "about":77, - "picture":77, - }, - }); - - assert_json_eq!(expected, response, ordered: true); -} diff --git a/tests/index_update.rs b/tests/index_update.rs deleted file mode 100644 index df4639252..000000000 --- a/tests/index_update.rs +++ /dev/null @@ -1,200 +0,0 @@ -use serde_json::json; -use serde_json::Value; -use assert_json_diff::assert_json_include; - -mod common; - -#[actix_rt::test] -async fn check_first_update_should_bring_up_processed_status_after_first_docs_addition() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let dataset = include_bytes!("assets/test_set.json"); - - let body: Value = serde_json::from_slice(dataset).unwrap(); - - // 2. Index the documents from movies.json, present inside of assets directory - server.add_or_replace_multiple_documents(body).await; - - // 3. Fetch the status of the indexing done above. - let (response, status_code) = server.get_all_updates_status().await; - - // 4. Verify the fetch is successful and indexing status is 'processed' - assert_eq!(status_code, 200); - assert_eq!(response[0]["status"], "processed"); -} - -#[actix_rt::test] -async fn return_error_when_get_update_status_of_unexisting_index() { - let mut server = common::Server::with_uid("test"); - - // 1. Fetch the status of unexisting index. - let (_, status_code) = server.get_all_updates_status().await; - - // 2. Verify the fetch returned 404 - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn return_empty_when_get_update_status_of_empty_index() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - // 2. Fetch the status of empty index. - let (response, status_code) = server.get_all_updates_status().await; - - // 3. Verify the fetch is successful, and no document are returned - assert_eq!(status_code, 200); - assert_eq!(response, json!([])); -} - -#[actix_rt::test] -async fn return_update_status_of_pushed_documents() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - - let bodies = vec![ - json!([{ - "title": "Test", - "comment": "comment test" - }]), - json!([{ - "title": "Test1", - "comment": "comment test1" - }]), - json!([{ - "title": "Test2", - "comment": "comment test2" - }]), - ]; - - let mut update_ids = Vec::new(); - - let url = "/indexes/test/documents?primaryKey=title"; - for body in bodies { - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - update_ids.push(update_id); - } - - // 2. Fetch the status of index. - let (response, status_code) = server.get_all_updates_status().await; - - // 3. Verify the fetch is successful, and updates are returned - - let expected = json!([{ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_ids[0] - },{ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_ids[1] - },{ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_ids[2] - },]); - - assert_eq!(status_code, 200); - assert_json_include!(actual: json!(response), expected: expected); -} - -#[actix_rt::test] -async fn return_error_if_index_does_not_exist() { - let mut server = common::Server::with_uid("test"); - - let (response, status_code) = server.get_update_status(42).await; - - assert_eq!(status_code, 404); - assert_eq!(response["errorCode"], "index_not_found"); -} - -#[actix_rt::test] -async fn return_error_if_update_does_not_exist() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let (response, status_code) = server.get_update_status(42).await; - - assert_eq!(status_code, 404); - assert_eq!(response["errorCode"], "not_found"); -} - -#[actix_rt::test] -async fn should_return_existing_update() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - }); - - // 1. Create Index - let (response, status_code) = server.create_index(body).await; - assert_eq!(status_code, 201); - assert_eq!(response["primaryKey"], json!(null)); - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/test/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - - let update_id = response["updateId"].as_u64().unwrap(); - - let expected = json!({ - "type": { - "name": "DocumentsAddition", - "number": 1, - }, - "updateId": update_id - }); - - let (response, status_code) = server.get_update_status(update_id).await; - - assert_eq!(status_code, 200); - assert_json_include!(actual: json!(response), expected: expected); -} diff --git a/tests/lazy_index_creation.rs b/tests/lazy_index_creation.rs deleted file mode 100644 index 6730db82e..000000000 --- a/tests/lazy_index_creation.rs +++ /dev/null @@ -1,446 +0,0 @@ -use serde_json::json; - -mod common; - -#[actix_rt::test] -async fn create_index_lazy_by_pushing_documents() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Add documents - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/movies/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -#[actix_rt::test] -async fn create_index_lazy_by_pushing_documents_and_discover_pk() { - let mut server = common::Server::with_uid("movies"); - - // 1 - Add documents - - let body = json!([{ - "id": 1, - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/movies/documents"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 202); - let update_id = response["updateId"].as_u64().unwrap(); - server.wait_update_id(update_id).await; - - // 3 - Check update success - - let (response, status_code) = server.get_update_status(update_id).await; - assert_eq!(status_code, 200); - assert_eq!(response["status"], "processed"); -} - -#[actix_rt::test] -async fn create_index_lazy_by_pushing_documents_with_wrong_name() { - let server = common::Server::with_uid("wrong&name"); - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/wrong&name/documents?primaryKey=title"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_index_uid"); -} - -#[actix_rt::test] -async fn create_index_lazy_add_documents_failed() { - let mut server = common::Server::with_uid("wrong&name"); - - let body = json!([{ - "title": "Test", - "comment": "comment test" - }]); - - let url = "/indexes/wrong&name/documents"; - let (response, status_code) = server.post_request(&url, body).await; - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_index_uid"); - - let (_, status_code) = server.get_index().await; - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_settings() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_settings_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "other", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "anotherSettings": ["name"], - }); - - let (_, status_code) = server.update_all_settings_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_ranking_rules() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ]); - - server.update_ranking_rules(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_ranking_rules_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "rankingRules": 123, - }); - - let (_, status_code) = server.update_ranking_rules_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_distinct_attribute() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!("type"); - - server.update_distinct_attribute(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_distinct_attribute_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (resp, status_code) = server.update_distinct_attribute_sync(body.clone()).await; - eprintln!("resp: {:?}", resp); - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (resp, status_code) = server.get_all_settings().await; - eprintln!("resp: {:?}", resp); - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_searchable_attributes() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["title", "description"]); - - server.update_searchable_attributes(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_searchable_attributes_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_searchable_attributes_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_displayed_attributes() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["title", "description"]); - - server.update_displayed_attributes(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_displayed_attributes_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_displayed_attributes_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_attributes_for_faceting() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["title", "description"]); - - server.update_attributes_for_faceting(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_attributes_for_faceting_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server - .update_attributes_for_faceting_sync(body.clone()) - .await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_synonyms() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!({ - "road": ["street", "avenue"], - "street": ["avenue"], - }); - - server.update_synonyms(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_synonyms_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_synonyms_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_stop_words() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(["le", "la", "les"]); - - server.update_stop_words(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 200); -} - -#[actix_rt::test] -async fn create_index_lazy_by_sending_stop_words_with_error() { - let mut server = common::Server::with_uid("movies"); - // 2 - Send the settings - - let body = json!(123); - - let (_, status_code) = server.update_stop_words_sync(body.clone()).await; - assert_eq!(status_code, 400); - - // 3 - Get all settings and compare to the previous one - - let (_, status_code) = server.get_all_settings().await; - - assert_eq!(status_code, 404); -} diff --git a/tests/placeholder_search.rs b/tests/placeholder_search.rs deleted file mode 100644 index 048ab7f8b..000000000 --- a/tests/placeholder_search.rs +++ /dev/null @@ -1,629 +0,0 @@ -use std::convert::Into; - -use serde_json::json; -use serde_json::Value; -use std::cell::RefCell; -use std::sync::Mutex; - -#[macro_use] -mod common; - -#[actix_rt::test] -async fn placeholder_search_with_limit() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "limit": 3 - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - assert_eq!(response["hits"].as_array().unwrap().len(), 3); - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_offset() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "limit": 6, - }); - - // hack to take a value out of macro (must implement UnwindSafe) - let expected = Mutex::new(RefCell::new(Vec::new())); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - // take results at offset 3 as reference - let lock = expected.lock().unwrap(); - lock.replace(response["hits"].as_array().unwrap()[3..6].to_vec()); - }); - let expected = expected.into_inner().unwrap().into_inner(); - - let query = json!({ - "limit": 3, - "offset": 3, - }); - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - let response = response["hits"].as_array().unwrap(); - assert_eq!(&expected, response); - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_attribute_to_highlight_wildcard() { - // there should be no highlight in placeholder search - let mut server = common::Server::test_server().await; - - let query = json!({ - "limit": 1, - "attributesToHighlight": ["*"] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - let result = response["hits"].as_array().unwrap()[0].as_object().unwrap(); - for value in result.values() { - assert!(value.to_string().find("").is_none()); - } - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_matches() { - // matches is always empty - let mut server = common::Server::test_server().await; - - let query = json!({ - "matches": true - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - let result = response["hits"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_object().unwrap()["_matchesInfo"].clone()) - .all(|m| m.as_object().unwrap().is_empty()); - assert!(result); - }); -} - -#[actix_rt::test] -async fn placeholder_search_witch_crop() { - // placeholder search crop always crop from beggining - let mut server = common::Server::test_server().await; - - let query = json!({ - "attributesToCrop": ["about"], - "cropLength": 20 - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 200); - - let hits = response["hits"].as_array().unwrap(); - - for hit in hits { - let hit = hit.as_object().unwrap(); - let formatted = hit["_formatted"].as_object().unwrap(); - - let about = hit["about"].as_str().unwrap(); - let about_formatted = formatted["about"].as_str().unwrap(); - // the formatted about length should be about 20 characters long - assert!(about_formatted.len() < 20 + 10); - // the formatted part should be located at the beginning of the original one - assert_eq!(about.find(&about_formatted).unwrap(), 0); - } - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_attributes_to_retrieve() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "limit": 1, - "attributesToRetrieve": ["gender", "about"], - }); - - test_post_get_search!(server, query, |response, _status_code| { - let hit = response["hits"].as_array().unwrap()[0].as_object().unwrap(); - assert_eq!(hit.values().count(), 2); - let _ = hit["gender"]; - let _ = hit["about"]; - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_filter() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "filters": "color='green'" - }); - - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "Green")); - }); - - let query = json!({ - "filters": "tags=bug" - }); - - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - let value = Value::String(String::from("bug")); - assert!(hits - .iter() - .all(|v| v["tags"].as_array().unwrap().contains(&value))); - }); - - let query = json!({ - "filters": "color='green' AND (tags='bug' OR tags='wontfix')" - }); - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - let bug = Value::String(String::from("bug")); - let wontfix = Value::String(String::from("wontfix")); - assert!(hits.iter().all(|v| v["color"].as_str().unwrap() == "Green" - && v["tags"].as_array().unwrap().contains(&bug) - || v["tags"].as_array().unwrap().contains(&wontfix))); - }); -} - -#[actix_rt::test] -async fn placeholder_test_faceted_search_valid() { - let mut server = common::Server::test_server().await; - - // simple tests on attributes with string value - let body = json!({ - "attributesForFaceting": ["color"] - }); - - server.update_all_settings(body).await; - - let query = json!({ - "facetFilters": ["color:green"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "Green")); - }); - - let query = json!({ - "facetFilters": [["color:blue"]] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - let query = json!({ - "facetFilters": ["color:Blue"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - // test on arrays: ["tags:bug"] - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - - server.update_all_settings(body).await; - - let query = json!({ - "facetFilters": ["tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())))); - }); - - // test and: ["color:blue", "tags:bug"] - let query = json!({ - "facetFilters": ["color:blue", "tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue" - && value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())))); - }); - - // test or: [["color:blue", "color:green"]] - let query = json!({ - "facetFilters": [["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue" - || value.get("color").unwrap() == "Green")); - }); - // test and-or: ["tags:bug", ["color:blue", "color:green"]] - let query = json!({ - "facetFilters": ["tags:bug", ["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())) - && (value.get("color").unwrap() == "blue" - || value.get("color").unwrap() == "Green"))); - }); -} - -#[actix_rt::test] -async fn placeholder_test_faceted_search_invalid() { - let mut server = common::Server::test_server().await; - - //no faceted attributes set - let query = json!({ - "facetFilters": ["color:blue"] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // empty arrays are error - // [] - let query = json!({ - "facetFilters": [] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // [[]] - let query = json!({ - "facetFilters": [[]] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // ["color:green", []] - let query = json!({ - "facetFilters": ["color:green", []] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - - // too much depth - // [[[]]] - let query = json!({ - "facetFilters": [[[]]] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // [["color:green", ["color:blue"]]] - let query = json!({ - "facetFilters": [["color:green", ["color:blue"]]] - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); - // "color:green" - let query = json!({ - "facetFilters": "color:green" - }); - test_post_get_search!(server, query, |_response, status_code| assert_ne!( - status_code, - 202 - )); -} - -#[actix_rt::test] -async fn placeholder_test_facet_count() { - let mut server = common::Server::test_server().await; - - // test without facet distribution - let query = json!({}); - test_post_get_search!(server, query, |response, _status_code| { - assert!(response.get("exhaustiveFacetsCount").is_none()); - assert!(response.get("facetsDistribution").is_none()); - }); - - // test no facets set, search on color - let query = json!({ - "facetsDistribution": ["color"] - }); - test_post_get_search!(server, query.clone(), |_response, status_code| { - assert_eq!(status_code, 400); - }); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // same as before, but now facets are set: - test_post_get_search!(server, query, |response, _status_code| { - println!("{}", response); - assert!(response.get("exhaustiveFacetsCount").is_some()); - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 1 - ); - }); - // searching on color and tags - let query = json!({ - "facetsDistribution": ["color", "tags"] - }); - test_post_get_search!(server, query, |response, _status_code| { - let facets = response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap(); - assert_eq!(facets.values().count(), 2); - assert_ne!( - !facets - .get("color") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 0 - ); - assert_ne!( - !facets - .get("tags") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 0 - ); - }); - // wildcard - let query = json!({ - "facetsDistribution": ["*"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 2 - ); - }); - // wildcard with other attributes: - let query = json!({ - "facetsDistribution": ["color", "*"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 2 - ); - }); - - // empty facet list - let query = json!({ - "facetsDistribution": [] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!( - response - .get("facetsDistribution") - .unwrap() - .as_object() - .unwrap() - .values() - .count(), - 0 - ); - }); - - // attr not set as facet passed: - let query = json!({ - "facetsDistribution": ["gender"] - }); - test_post_get_search!(server, query, |_response, status_code| { - assert_eq!(status_code, 400); - }); -} - -#[actix_rt::test] -#[should_panic] -async fn placeholder_test_bad_facet_distribution() { - let mut server = common::Server::test_server().await; - // string instead of array: - let query = json!({ - "facetsDistribution": "color" - }); - test_post_get_search!(server, query, |_response, _status_code| {}); - - // invalid value in array: - let query = json!({ - "facetsDistribution": ["color", true] - }); - test_post_get_search!(server, query, |_response, _status_code| {}); -} - -#[actix_rt::test] -async fn placeholder_test_sort() { - let mut server = common::Server::test_server().await; - - let body = json!({ - "rankingRules": ["asc(age)"], - "attributesForFaceting": ["color"] - }); - server.update_all_settings(body).await; - let query = json!({}); - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - hits.iter() - .map(|v| v["age"].as_u64().unwrap()) - .fold(0, |prev, cur| { - assert!(cur >= prev); - cur - }); - }); - - let query = json!({ - "facetFilters": ["color:green"] - }); - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - hits.iter() - .map(|v| v["age"].as_u64().unwrap()) - .fold(0, |prev, cur| { - assert!(cur >= prev); - cur - }); - }); -} - -#[actix_rt::test] -async fn placeholder_search_with_empty_query() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "q": "", - "limit": 3 - }); - - test_post_get_search!(server, query, |response, status_code| { - eprintln!("{}", response); - assert_eq!(status_code, 200); - assert_eq!(response["hits"].as_array().unwrap().len(), 3); - }); -} - -#[actix_rt::test] -async fn test_filter_nb_hits_search_placeholder() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - let documents = json!([ - { - "id": 1, - "content": "a", - "color": "green", - "size": 1, - }, - { - "id": 2, - "content": "a", - "color": "green", - "size": 2, - }, - { - "id": 3, - "content": "a", - "color": "blue", - "size": 3, - }, - ]); - - server.add_or_update_multiple_documents(documents).await; - let (response, _) = server.search_post(json!({})).await; - assert_eq!(response["nbHits"], 3); - - server.update_distinct_attribute(json!("color")).await; - - let (response, _) = server.search_post(json!({})).await; - assert_eq!(response["nbHits"], 2); - - let (response, _) = server.search_post(json!({"filters": "size < 3"})).await; - println!("result: {}", response); - assert_eq!(response["nbHits"], 1); -} diff --git a/tests/search.rs b/tests/search.rs deleted file mode 100644 index 267c98265..000000000 --- a/tests/search.rs +++ /dev/null @@ -1,1879 +0,0 @@ -use std::convert::Into; - -use assert_json_diff::assert_json_eq; -use serde_json::json; -use serde_json::Value; - -#[macro_use] mod common; - -#[actix_rt::test] -async fn search() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "q": "exercitation" - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 49, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ], - "isActive": true - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - let hits = response["hits"].as_array().unwrap(); - let hits: Vec = hits.iter().cloned().take(3).collect(); - assert_json_eq!(expected.clone(), serde_json::to_value(hits).unwrap(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_no_params() { - let mut server = common::Server::test_server().await; - - let query = json! ({}); - - // an empty search should return the 20 first indexed document - let dataset: Vec = serde_json::from_slice(include_bytes!("assets/test_set.json")).unwrap(); - let expected: Vec = dataset.into_iter().take(20).collect(); - let expected: Value = serde_json::to_value(expected).unwrap(); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_in_unexisting_index() { - let mut server = common::Server::with_uid("test"); - - let query = json! ({ - "q": "exercitation" - }); - - let expected = json! ({ - "message": "Index test not found", - "errorCode": "index_not_found", - "errorType": "invalid_request_error", - "errorLink": "https://docs.meilisearch.com/errors#index_not_found" - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(404, status_code); - assert_json_eq!(expected.clone(), response.clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_unexpected_params() { - - let query = json! ({"lol": "unexpected"}); - - let expected = "unknown field `lol`, expected one of `q`, `offset`, `limit`, `attributesToRetrieve`, `attributesToCrop`, `cropLength`, `attributesToHighlight`, `filters`, `matches`, `facetFilters`, `facetsDistribution` at line 1 column 6"; - - let post_query = serde_json::from_str::(&query.to_string()); - assert!(post_query.is_err()); - assert_eq!(expected, post_query.err().unwrap().to_string()); - - let get_query: Result = serde_json::from_str(&query.to_string()); - assert!(get_query.is_err()); - assert_eq!(expected, get_query.err().unwrap().to_string()); -} - -#[actix_rt::test] -async fn search_with_limit() { - let mut server = common::Server::test_server().await; - - let query = json! ({ - "q": "exercitation", - "limit": 3 - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 49, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ], - "isActive": true - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_offset() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 3, - "offset": 1 - }); - - let expected = json!([ - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 49, - "balance": "$1,476.39", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468", - "about": "Tempor mollit exercitation excepteur cupidatat reprehenderit ad ex. Nulla laborum proident incididunt quis. Esse laborum deserunt qui anim. Sunt incididunt pariatur cillum anim proident eu ullamco dolor excepteur. Ullamco amet culpa nostrud adipisicing duis aliqua consequat duis non eu id mollit velit. Deserunt ullamco amet in occaecat.\r\n", - "registered": "2018-04-26T06:04:40 -02:00", - "latitude": -64.196802, - "longitude": -117.396238, - "tags": [ - "wontfix" - ], - "isActive": true - }, - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attribute_to_highlight_wildcard() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToHighlight": ["*"] - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attribute_to_highlight_1() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToHighlight": ["name"] - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_matches() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "matches": true - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_matchesInfo": { - "name": [ - { - "start": 0, - "length": 6 - } - ], - "email": [ - { - "start": 0, - "length": 6 - } - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_crop() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 1, - "attributesToCrop": ["about"], - "cropLength": 20 - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_retrieve() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","color","gender"], - }); - - let expected = json!([ - { - "name": "Cherry Orr", - "age": 27, - "color": "Green", - "gender": "female" - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_retrieve_wildcard() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["*"], - }); - - let expected = json!([ - { - "id": 1, - "isActive": true, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ] - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_filter() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 3, - "filters": "gender='male'" - }); - - let expected = json!([ - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - }, - { - "id": 66, - "balance": "$1,061.49", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "brown", - "name": "Higgins Aguilar", - "gender": "male", - "email": "higginsaguilar@chorizon.com", - "phone": "+1 (911) 540-3791", - "address": "132 Sackman Street, Layhill, Guam, 8729", - "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", - "registered": "2015-04-05T02:10:07 -02:00", - "latitude": 74.702813, - "longitude": 151.314972, - "tags": [ - "bug" - ], - "isActive": true - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - } - ]); - - let query = json!({ - "q": "exercitation", - "limit": 3, - "filters": "name='Lucas Hess'" - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 2, - "balance": "$2,467.47", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", - "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", - "registered": "2014-10-28T12:59:30 -01:00", - "latitude": -64.008555, - "longitude": 11.867098, - "tags": [ - "good first issue" - ], - "isActive": true - }, - { - "id": 75, - "balance": "$1,913.42", - "picture": "http://placehold.it/32x32", - "age": 24, - "color": "Green", - "name": "Emma Jacobs", - "gender": "female", - "email": "emmajacobs@chorizon.com", - "phone": "+1 (899) 554-3847", - "address": "173 Tapscott Street, Esmont, Maine, 7450", - "about": "Laboris consequat consectetur tempor labore ullamco ullamco voluptate quis quis duis ut ad. In est irure quis amet sunt nulla ad ut sit labore ut eu quis duis. Nostrud cupidatat aliqua sunt occaecat minim id consequat officia deserunt laborum. Ea dolor reprehenderit laborum veniam exercitation est nostrud excepteur laborum minim id qui et.\r\n", - "registered": "2019-03-29T06:24:13 -01:00", - "latitude": -35.53722, - "longitude": 155.703874, - "tags": [], - "isActive": false - } - ]); - let query = json!({ - "q": "exercitation", - "limit": 3, - "filters": "gender='female' AND (name='Patricia Goff' OR name='Emma Jacobs')" - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 30, - "balance": "$2,021.11", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Stacy Espinoza", - "gender": "female", - "email": "stacyespinoza@chorizon.com", - "phone": "+1 (999) 487-3253", - "address": "931 Alabama Avenue, Bangor, Alaska, 8215", - "about": "Id reprehenderit cupidatat exercitation anim ad nisi irure. Minim est proident mollit laborum. Duis ad duis eiusmod quis.\r\n", - "registered": "2014-07-16T06:15:53 -02:00", - "latitude": 41.560197, - "longitude": 177.697, - "tags": [ - "new issue", - "new issue", - "bug" - ], - "isActive": true - }, - { - "id": 31, - "balance": "$3,609.82", - "picture": "http://placehold.it/32x32", - "age": 32, - "color": "blue", - "name": "Vilma Garza", - "gender": "female", - "email": "vilmagarza@chorizon.com", - "phone": "+1 (944) 585-2021", - "address": "565 Tech Place, Sedley, Puerto Rico, 858", - "about": "Excepteur et fugiat mollit incididunt cupidatat. Mollit nisi veniam sint eu exercitation amet labore. Voluptate est magna est amet qui minim excepteur cupidatat dolor quis id excepteur aliqua reprehenderit. Proident nostrud ex veniam officia nisi enim occaecat ex magna officia id consectetur ad eu. In et est reprehenderit cupidatat ad minim veniam proident nulla elit nisi veniam proident ex. Eu in irure sit veniam amet incididunt fugiat proident quis ullamco laboris.\r\n", - "registered": "2017-06-30T07:43:52 -02:00", - "latitude": -12.574889, - "longitude": -54.771186, - "tags": [ - "new issue", - "wontfix", - "wontfix" - ], - "isActive": false - }, - { - "id": 2, - "balance": "$2,467.47", - "picture": "http://placehold.it/32x32", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700", - "about": "Non culpa duis dolore Lorem aliqua. Labore veniam laborum cupidatat nostrud ea exercitation. Esse nostrud sit veniam laborum minim ullamco nulla aliqua est cillum magna. Duis non esse excepteur veniam voluptate sunt cupidatat nostrud consequat sint adipisicing ut excepteur. Incididunt sit aliquip non id magna amet deserunt esse quis dolor.\r\n", - "registered": "2014-10-28T12:59:30 -01:00", - "latitude": -64.008555, - "longitude": 11.867098, - "tags": [ - "good first issue" - ], - "isActive": true - } - ]); - let query = json!({ - "q": "exerciatation", - "limit": 3, - "filters": "gender='female' AND (name='Patricia Goff' OR age > 30)" - }); - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 59, - "balance": "$1,921.58", - "picture": "http://placehold.it/32x32", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219", - "about": "Exercitation minim esse proident cillum velit et deserunt incididunt adipisicing minim. Cillum Lorem consectetur laborum id consequat exercitation velit. Magna dolor excepteur sunt deserunt dolor ullamco non sint proident ipsum. Reprehenderit voluptate sit veniam consectetur ea sunt duis labore deserunt ipsum aute. Eiusmod aliqua anim voluptate id duis tempor aliqua commodo sunt. Do officia ea consectetur nostrud eiusmod laborum.\r\n", - "registered": "2019-12-07T07:33:15 -01:00", - "latitude": -60.812605, - "longitude": -27.129016, - "tags": [ - "bug", - "new issue" - ], - "isActive": true - }, - { - "id": 0, - "balance": "$2,668.55", - "picture": "http://placehold.it/32x32", - "age": 36, - "color": "Green", - "name": "Lucas Hess", - "gender": "male", - "email": "lucashess@chorizon.com", - "phone": "+1 (998) 478-2597", - "address": "412 Losee Terrace, Blairstown, Georgia, 2825", - "about": "Mollit ad in exercitation quis. Anim est ut consequat fugiat duis magna aliquip velit nisi. Commodo eiusmod est consequat proident consectetur aliqua enim fugiat. Aliqua adipisicing laboris elit proident enim veniam laboris mollit. Incididunt fugiat minim ad nostrud deserunt tempor in. Id irure officia labore qui est labore nulla nisi. Magna sit quis tempor esse consectetur amet labore duis aliqua consequat.\r\n", - "registered": "2016-06-21T09:30:25 -02:00", - "latitude": -44.174957, - "longitude": -145.725388, - "tags": [ - "bug", - "bug" - ], - "isActive": false - }, - { - "id": 66, - "balance": "$1,061.49", - "picture": "http://placehold.it/32x32", - "age": 35, - "color": "brown", - "name": "Higgins Aguilar", - "gender": "male", - "email": "higginsaguilar@chorizon.com", - "phone": "+1 (911) 540-3791", - "address": "132 Sackman Street, Layhill, Guam, 8729", - "about": "Anim ea dolore exercitation minim. Proident cillum non deserunt cupidatat veniam non occaecat aute ullamco irure velit laboris ex aliquip. Voluptate incididunt non ex nulla est ipsum. Amet anim do velit sunt irure sint minim nisi occaecat proident tempor elit exercitation nostrud.\r\n", - "registered": "2015-04-05T02:10:07 -02:00", - "latitude": 74.702813, - "longitude": 151.314972, - "tags": [ - "bug" - ], - "isActive": true - } - ]); - let query = json!({ - "q": "exerciatation", - "limit": 3, - "filters": "NOT gender = 'female' AND age > 30" - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); - - let expected = json!([ - { - "id": 11, - "balance": "$1,351.43", - "picture": "http://placehold.it/32x32", - "age": 28, - "color": "Green", - "name": "Evans Wagner", - "gender": "male", - "email": "evanswagner@chorizon.com", - "phone": "+1 (889) 496-2332", - "address": "118 Monaco Place, Lutsen, Delaware, 6209", - "about": "Sunt consectetur enim ipsum consectetur occaecat reprehenderit nulla pariatur. Cupidatat do exercitation tempor voluptate duis nostrud dolor consectetur. Excepteur aliquip Lorem voluptate cillum est. Nisi velit nulla nostrud ea id officia laboris et.\r\n", - "registered": "2016-10-27T01:26:31 -02:00", - "latitude": -77.673222, - "longitude": -142.657214, - "tags": [ - "good first issue", - "good first issue" - ], - "isActive": true - } - ]); - let query = json!({ - "q": "exerciatation", - "filters": "NOT gender = 'female' AND name='Evans Wagner'" - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_highlight_and_matches() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToHighlight": ["name","email"], - "matches": true, - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - "_matchesInfo": { - "email": [ - { - "start": 0, - "length": 6 - } - ], - "name": [ - { - "start": 0, - "length": 6 - } - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_attributes_to_highlight_and_matches_and_crop() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exerciatation", - "limit": 1, - "attributesToCrop": ["about"], - "cropLength": 20, - "attributesToHighlight": ["about"], - "matches": true, - }); - - let expected = json!([ - { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia mollit proident nostrud ea. Pariatur voluptate labore nostrud magna duis non elit et incididunt Lorem velit duis amet commodo. Irure in velit laboris pariatur. Do tempor ex deserunt duis minim amet.\r\n", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true, - "_formatted": { - "id": 1, - "balance": "$1,706.13", - "picture": "http://placehold.it/32x32", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "about": "Exercitation officia", - "registered": "2020-03-18T11:12:21 -01:00", - "latitude": -24.356932, - "longitude": 27.184808, - "tags": [ - "new issue", - "bug" - ], - "isActive": true - }, - "_matchesInfo": { - "about": [ - { - "start": 0, - "length": 12 - } - ] - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","gender","email"], - "attributesToHighlight": ["name"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "name": "Cherry Orr" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_2() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 1, - "attributesToRetrieve": ["name","age","gender"], - "attributesToCrop": ["about"], - "cropLength": 20, - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "_formatted": { - "about": "Exercitation officia" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_3() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "exercitation", - "limit": 1, - "attributesToRetrieve": ["name","age","gender"], - "attributesToCrop": ["about:20"], - }); - - let expected = json!( [ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "_formatted": { - "about": "Exercitation officia" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_4() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender"], - "attributesToCrop": ["name:0","email:6"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "name": "Cherry", - "email": "cherryorr" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_5() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender"], - "attributesToCrop": ["*","email:6"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "name": "Cherry Orr", - "email": "cherryorr", - "age": 27, - "gender": "female" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_6() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender"], - "attributesToCrop": ["*","email:10"], - "attributesToHighlight": ["name"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_7() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","gender","email"], - "attributesToCrop": ["*","email:6"], - "attributesToHighlight": ["*"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "_formatted": { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn search_with_differents_attributes_8() { - let mut server = common::Server::test_server().await; - - let query = json!({ - "q": "cherry", - "limit": 1, - "attributesToRetrieve": ["name","age","email","gender","address"], - "attributesToCrop": ["*","email:6"], - "attributesToHighlight": ["*","address"], - }); - - let expected = json!([ - { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "address": "442 Beverly Road, Ventress, New Mexico, 3361", - "_formatted": { - "age": 27, - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - } - } - ]); - - test_post_get_search!(server, query, |response, _status_code| { - assert_json_eq!(expected.clone(), response["hits"].clone(), ordered: false); - }); -} - -#[actix_rt::test] -async fn test_faceted_search_valid() { - // set facetting attributes before adding documents - let mut server = common::Server::with_uid("test"); - server.create_index(json!({ "uid": "test" })).await; - - let body = json!({ - "attributesForFaceting": ["color"] - }); - server.update_all_settings(body).await; - - let dataset = include_bytes!("assets/test_set.json"); - let body: Value = serde_json::from_slice(dataset).unwrap(); - server.add_or_update_multiple_documents(body).await; - - // simple tests on attributes with string value - - let query = json!({ - "q": "a", - "facetFilters": ["color:green"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "Green")); - }); - - let query = json!({ - "q": "a", - "facetFilters": [["color:blue"]] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - let query = json!({ - "q": "a", - "facetFilters": ["color:Blue"] - }); - - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("color").unwrap() == "blue")); - }); - - // test on arrays: ["tags:bug"] - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - - server.update_all_settings(body).await; - - let query = json!({ - "q": "a", - "facetFilters": ["tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned())))); - }); - - // test and: ["color:blue", "tags:bug"] - let query = json!({ - "q": "a", - "facetFilters": ["color:blue", "tags:bug"] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| value - .get("color") - .unwrap() == "blue" - && value.get("tags").unwrap().as_array().unwrap().contains(&Value::String("bug".to_owned())))); - }); - - // test or: [["color:blue", "color:green"]] - let query = json!({ - "q": "a", - "facetFilters": [["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| - value - .get("color") - .unwrap() == "blue" - || value - .get("color") - .unwrap() == "Green")); - }); - // test and-or: ["tags:bug", ["color:blue", "color:green"]] - let query = json!({ - "q": "a", - "facetFilters": ["tags:bug", ["color:blue", "color:green"]] - }); - test_post_get_search!(server, query, |response, _status_code| { - assert!(!response.get("hits").unwrap().as_array().unwrap().is_empty()); - assert!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .iter() - .all(|value| - value - .get("tags") - .unwrap() - .as_array() - .unwrap() - .contains(&Value::String("bug".to_owned())) - && (value - .get("color") - .unwrap() == "blue" - || value - .get("color") - .unwrap() == "Green"))); - - }); -} - -#[actix_rt::test] -async fn test_faceted_search_invalid() { - let mut server = common::Server::test_server().await; - - //no faceted attributes set - let query = json!({ - "q": "a", - "facetFilters": ["color:blue"] - }); - - test_post_get_search!(server, query, |response, status_code| { - - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // empty arrays are error - // [] - let query = json!({ - "q": "a", - "facetFilters": [] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - // [[]] - let query = json!({ - "q": "a", - "facetFilters": [[]] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // ["color:green", []] - let query = json!({ - "q": "a", - "facetFilters": ["color:green", []] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // too much depth - // [[[]]] - let query = json!({ - "q": "a", - "facetFilters": [[[]]] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // [["color:green", ["color:blue"]]] - let query = json!({ - "q": "a", - "facetFilters": [["color:green", ["color:blue"]]] - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); - - // "color:green" - let query = json!({ - "q": "a", - "facetFilters": "color:green" - }); - - test_post_get_search!(server, query, |response, status_code| { - assert_eq!(status_code, 400); - assert_eq!(response["errorCode"], "invalid_facet"); - }); -} - -#[actix_rt::test] -async fn test_facet_count() { - let mut server = common::Server::test_server().await; - - // test without facet distribution - let query = json!({ - "q": "a", - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert!(response.get("exhaustiveFacetsCount").is_none()); - assert!(response.get("facetsDistribution").is_none()); - }); - - // test no facets set, search on color - let query = json!({ - "q": "a", - "facetsDistribution": ["color"] - }); - test_post_get_search!(server, query.clone(), |_response, status_code|{ - assert_eq!(status_code, 400); - }); - - let body = json!({ - "attributesForFaceting": ["color", "tags"] - }); - server.update_all_settings(body).await; - // same as before, but now facets are set: - test_post_get_search!(server, query, |response, _status_code|{ - println!("{}", response); - assert!(response.get("exhaustiveFacetsCount").is_some()); - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 1); - // assert that case is preserved - assert!(response["facetsDistribution"] - .as_object() - .unwrap()["color"] - .as_object() - .unwrap() - .get("Green") - .is_some()); - }); - // searching on color and tags - let query = json!({ - "q": "a", - "facetsDistribution": ["color", "tags"] - }); - test_post_get_search!(server, query, |response, _status_code|{ - let facets = response.get("facetsDistribution").unwrap().as_object().unwrap(); - assert_eq!(facets.values().count(), 2); - assert_ne!(!facets.get("color").unwrap().as_object().unwrap().values().count(), 0); - assert_ne!(!facets.get("tags").unwrap().as_object().unwrap().values().count(), 0); - }); - // wildcard - let query = json!({ - "q": "a", - "facetsDistribution": ["*"] - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2); - }); - // wildcard with other attributes: - let query = json!({ - "q": "a", - "facetsDistribution": ["color", "*"] - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 2); - }); - - // empty facet list - let query = json!({ - "q": "a", - "facetsDistribution": [] - }); - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response.get("facetsDistribution").unwrap().as_object().unwrap().values().count(), 0); - }); - - // attr not set as facet passed: - let query = json!({ - "q": "a", - "facetsDistribution": ["gender"] - }); - test_post_get_search!(server, query, |_response, status_code|{ - assert_eq!(status_code, 400); - }); - -} - -#[actix_rt::test] -#[should_panic] -async fn test_bad_facet_distribution() { - let mut server = common::Server::test_server().await; - // string instead of array: - let query = json!({ - "q": "a", - "facetsDistribution": "color" - }); - test_post_get_search!(server, query, |_response, _status_code| {}); - - // invalid value in array: - let query = json!({ - "q": "a", - "facetsDistribution": ["color", true] - }); - test_post_get_search!(server, query, |_response, _status_code| {}); -} - -#[actix_rt::test] -async fn highlight_cropped_text() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - let doc = json!([ - { - "id": 1, - "body": r##"well, it may not work like that, try the following: -1. insert your trip -2. google your `searchQuery` -3. find a solution -> say hello"## - } - ]); - server.add_or_replace_multiple_documents(doc).await; - - // tests from #680 - //let query = "q=insert&attributesToHighlight=*&attributesToCrop=body&cropLength=30"; - let query = json!({ - "q": "insert", - "attributesToHighlight": ["*"], - "attributesToCrop": ["body"], - "cropLength": 30, - }); - let expected_response = "that, try the following: \n1. insert your trip\n2. google your"; - test_post_get_search!(server, query, |response, _status_code|{ - assert_eq!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .get(0) - .unwrap() - .as_object() - .unwrap() - .get("_formatted") - .unwrap() - .as_object() - .unwrap() - .get("body") - .unwrap() - , &Value::String(expected_response.to_owned())); - }); - - //let query = "q=insert&attributesToHighlight=*&attributesToCrop=body&cropLength=80"; - let query = json!({ - "q": "insert", - "attributesToHighlight": ["*"], - "attributesToCrop": ["body"], - "cropLength": 80, - }); - let expected_response = "well, it may not work like that, try the following: \n1. insert your trip\n2. google your `searchQuery`\n3. find a solution \n> say hello"; - test_post_get_search!(server, query, |response, _status_code| { - assert_eq!(response - .get("hits") - .unwrap() - .as_array() - .unwrap() - .get(0) - .unwrap() - .as_object() - .unwrap() - .get("_formatted") - .unwrap() - .as_object() - .unwrap() - .get("body") - .unwrap() - , &Value::String(expected_response.to_owned())); - }); -} - -#[actix_rt::test] -async fn well_formated_error_with_bad_request_params() { - let mut server = common::Server::with_uid("test"); - let query = "foo=bar"; - let (response, _status_code) = server.search_get(query).await; - assert!(response.get("message").is_some()); - assert!(response.get("errorCode").is_some()); - assert!(response.get("errorType").is_some()); - assert!(response.get("errorLink").is_some()); -} - - -#[actix_rt::test] -async fn update_documents_with_facet_distribution() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - let settings = json!({ - "attributesForFaceting": ["genre"], - "displayedAttributes": ["genre"], - "searchableAttributes": ["genre"] - }); - server.update_all_settings(settings).await; - let update1 = json!([ - { - "id": "1", - "type": "album", - "title": "Nevermind", - "genre": ["grunge", "alternative"] - }, - { - "id": "2", - "type": "album", - "title": "Mellon Collie and the Infinite Sadness", - "genre": ["alternative", "rock"] - }, - { - "id": "3", - "type": "album", - "title": "The Queen Is Dead", - "genre": ["indie", "rock"] - } - ]); - server.add_or_update_multiple_documents(update1).await; - let search = json!({ - "q": "album", - "facetsDistribution": ["genre"] - }); - let (response1, _) = server.search_post(search.clone()).await; - let expected_facet_distribution = json!({ - "genre": { - "grunge": 1, - "alternative": 2, - "rock": 2, - "indie": 1 - } - }); - assert_json_eq!(expected_facet_distribution.clone(), response1["facetsDistribution"].clone()); - - let update2 = json!([ - { - "id": "3", - "title": "The Queen Is Very Dead" - } - ]); - server.add_or_update_multiple_documents(update2).await; - let (response2, _) = server.search_post(search).await; - assert_json_eq!(expected_facet_distribution, response2["facetsDistribution"].clone()); -} - -#[actix_rt::test] -async fn test_filter_nb_hits_search_normal() { - let mut server = common::Server::with_uid("test"); - - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - - server.create_index(body).await; - let documents = json!([ - { - "id": 1, - "content": "a", - "color": "green", - "size": 1, - }, - { - "id": 2, - "content": "a", - "color": "green", - "size": 2, - }, - { - "id": 3, - "content": "a", - "color": "blue", - "size": 3, - }, - ]); - - server.add_or_update_multiple_documents(documents).await; - let (response, _) = server.search_post(json!({"q": "a"})).await; - assert_eq!(response["nbHits"], 3); - - let (response, _) = server.search_post(json!({"q": "a", "filters": "size = 1"})).await; - assert_eq!(response["nbHits"], 1); - - server.update_distinct_attribute(json!("color")).await; - - let (response, _) = server.search_post(json!({"q": "a"})).await; - assert_eq!(response["nbHits"], 2); - - let (response, _) = server.search_post(json!({"q": "a", "filters": "size < 3"})).await; - println!("result: {}", response); - assert_eq!(response["nbHits"], 1); -} diff --git a/tests/search_settings.rs b/tests/search_settings.rs deleted file mode 100644 index 46417498d..000000000 --- a/tests/search_settings.rs +++ /dev/null @@ -1,538 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; -use std::convert::Into; - -mod common; - -#[actix_rt::test] -async fn search_with_settings_basic() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=ea%20exercitation&limit=3"; - - let expect = json!([ - { - "balance": "$2,467.47", - "age": 34, - "color": "blue", - "name": "Patricia Goff", - "gender": "female", - "email": "patriciagoff@chorizon.com", - "phone": "+1 (864) 463-2277", - "address": "866 Hornell Loop, Cresaptown, Ohio, 1700" - }, - { - "balance": "$3,344.40", - "age": 35, - "color": "blue", - "name": "Adeline Flynn", - "gender": "female", - "email": "adelineflynn@chorizon.com", - "phone": "+1 (994) 600-2840", - "address": "428 Paerdegat Avenue, Hollymead, Pennsylvania, 948" - }, - { - "balance": "$3,394.96", - "age": 25, - "color": "blue", - "name": "Aida Kirby", - "gender": "female", - "email": "aidakirby@chorizon.com", - "phone": "+1 (942) 532-2325", - "address": "797 Engert Avenue, Wilsonia, Idaho, 6532" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_stop_words() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": ["ea"], - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=ea%20exercitation&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_synonyms() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": { - "application": [ - "exercitation" - ] - }, - }); - - server.update_all_settings(config).await; - - let query = "q=application&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_ranking_rules() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "age", - "color", - "gender", - "email", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=exarcitation&limit=3"; - let expect = json!([ - { - "balance": "$1,921.58", - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243", - "address": "883 Dennett Place, Knowlton, New Mexico, 9219" - }, - { - "balance": "$1,706.13", - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174", - "address": "442 Beverly Road, Ventress, New Mexico, 3361" - }, - { - "balance": "$1,476.39", - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684", - "address": "817 Newton Street, Bannock, Wyoming, 1468" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - println!("{}", response["hits"].clone()); - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_searchable_attributes() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "age", - "color", - "gender", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone", - "address", - "balance" - ], - "stopWords": null, - "synonyms": { - "exarcitation": [ - "exercitation" - ] - }, - }); - - server.update_all_settings(config).await; - - let query = "q=Carol&limit=3"; - let expect = json!([ - { - "balance": "$1,440.09", - "age": 40, - "color": "blue", - "name": "Levy Whitley", - "gender": "male", - "email": "levywhitley@chorizon.com", - "phone": "+1 (911) 458-2411", - "address": "187 Thomas Street, Hachita, North Carolina, 2989" - }, - { - "balance": "$1,977.66", - "age": 36, - "color": "brown", - "name": "Combs Stanley", - "gender": "male", - "email": "combsstanley@chorizon.com", - "phone": "+1 (827) 419-2053", - "address": "153 Beverley Road, Siglerville, South Carolina, 3666" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_displayed_attributes() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "age", - "color", - "gender", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender", - "color", - "email", - "phone" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=exercitation&limit=3"; - let expect = json!([ - { - "age": 31, - "color": "Green", - "name": "Harper Carson", - "gender": "male", - "email": "harpercarson@chorizon.com", - "phone": "+1 (912) 430-3243" - }, - { - "age": 27, - "color": "Green", - "name": "Cherry Orr", - "gender": "female", - "email": "cherryorr@chorizon.com", - "phone": "+1 (995) 479-3174" - }, - { - "age": 28, - "color": "brown", - "name": "Maureen Dale", - "gender": "female", - "email": "maureendale@chorizon.com", - "phone": "+1 (984) 538-3684" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -#[actix_rt::test] -async fn search_with_settings_searchable_attributes_2() { - let mut server = common::Server::test_server().await; - - let config = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "desc(age)", - "exactness", - "desc(balance)" - ], - "distinctAttribute": null, - "searchableAttributes": [ - "age", - "color", - "gender", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "age", - "gender" - ], - "stopWords": null, - "synonyms": null, - }); - - server.update_all_settings(config).await; - - let query = "q=exercitation&limit=3"; - let expect = json!([ - { - "age": 31, - "name": "Harper Carson", - "gender": "male" - }, - { - "age": 27, - "name": "Cherry Orr", - "gender": "female" - }, - { - "age": 28, - "name": "Maureen Dale", - "gender": "female" - } - ]); - - let (response, _status_code) = server.search_get(query).await; - assert_json_eq!(expect, response["hits"].clone(), ordered: false); -} - -// issue #798 -#[actix_rt::test] -async fn distinct_attributes_returns_name_not_id() { - let mut server = common::Server::test_server().await; - let settings = json!({ - "distinctAttribute": "color", - }); - server.update_all_settings(settings).await; - let (response, _) = server.get_all_settings().await; - assert_eq!(response["distinctAttribute"], "color"); - let (response, _) = server.get_distinct_attribute().await; - assert_eq!(response, "color"); -} diff --git a/tests/settings.rs b/tests/settings.rs deleted file mode 100644 index 6b125c13a..000000000 --- a/tests/settings.rs +++ /dev/null @@ -1,523 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; -use std::convert::Into; -mod common; - -#[actix_rt::test] -async fn write_all_and_delete() { - let mut server = common::Server::test_server().await; - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Delete all settings - - server.delete_all_settings().await; - - // 5 - Get all settings and check if they are set to default values - - let (response, _status_code) = server.get_all_settings().await; - - let expect = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": null, - "searchableAttributes": ["*"], - "displayedAttributes": ["*"], - "stopWords": [], - "synonyms": {}, - "attributesForFaceting": [], - }); - - assert_json_eq!(expect, response, ordered: false); -} - -#[actix_rt::test] -async fn write_all_and_update() { - let mut server = common::Server::test_server().await; - - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Update all settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "color", - "age", - ], - "displayedAttributes": [ - "name", - "color", - "age", - "registered", - "picture", - ], - "stopWords": [], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["title"], - }); - - server.update_all_settings(body).await; - - // 5 - Get all settings and check if the content is the same of (4) - - let (response, _status_code) = server.get_all_settings().await; - - let expected = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - ], - "distinctAttribute": null, - "searchableAttributes": [ - "name", - "color", - "age", - ], - "displayedAttributes": [ - "name", - "color", - "age", - "registered", - "picture", - ], - "stopWords": [], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": ["title"], - }); - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn test_default_settings() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - }); - server.create_index(body).await; - - // 1 - Get all settings and compare to the previous one - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": null, - "searchableAttributes": ["*"], - "displayedAttributes": ["*"], - "stopWords": [], - "synonyms": {}, - "attributesForFaceting": [], - }); - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); -} - -#[actix_rt::test] -async fn test_default_settings_2() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - // 1 - Get all settings and compare to the previous one - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ], - "distinctAttribute": null, - "searchableAttributes": ["*"], - "displayedAttributes": ["*"], - "stopWords": [], - "synonyms": {}, - "attributesForFaceting": [], - }); - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: false); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/516 -#[actix_rt::test] -async fn write_setting_and_update_partial() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - }); - server.create_index(body).await; - - // 2 - Send the settings - - let body = json!({ - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ] - }); - - server.update_all_settings(body.clone()).await; - - // 2 - Send the settings - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - "desc(registered)", - ], - "distinctAttribute": "id", - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - }); - - server.update_all_settings(body.clone()).await; - - // 2 - Send the settings - - let expected = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(age)", - "desc(registered)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "about" - ], - "displayedAttributes": [ - "name", - "gender", - "email", - "registered", - "age", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["street", "avenue"], - "street": ["avenue"], - }, - "attributesForFaceting": [], - }); - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn attributes_for_faceting_settings() { - let mut server = common::Server::test_server().await; - // initial attributes array should be empty - let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; - assert_eq!(response, json!([])); - // add an attribute and test for its presence - let (_response, _status_code) = server.post_request_async( - "/indexes/test/settings/attributes-for-faceting", - json!(["foobar"])).await; - let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; - assert_eq!(response, json!(["foobar"])); - // remove all attributes and test for emptiness - let (_response, _status_code) = server.delete_request_async( - "/indexes/test/settings/attributes-for-faceting").await; - let (response, _status_code) = server.get_request("/indexes/test/settings/attributes-for-faceting").await; - assert_eq!(response, json!([])); -} - -#[actix_rt::test] -async fn setting_ranking_rules_dont_mess_with_other_settings() { - let mut server = common::Server::test_server().await; - let body = json!({ - "rankingRules": ["asc(foobar)"] - }); - server.update_all_settings(body).await; - let (response, _) = server.get_all_settings().await; - assert_eq!(response["rankingRules"].as_array().unwrap().len(), 1); - assert_eq!(response["rankingRules"].as_array().unwrap().first().unwrap().as_str().unwrap(), "asc(foobar)"); - assert!(!response["searchableAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar")); - assert!(!response["displayedAttributes"].as_array().unwrap().iter().any(|e| e.as_str().unwrap() == "foobar")); -} - -#[actix_rt::test] -async fn displayed_and_searchable_attributes_reset_to_wildcard() { - let mut server = common::Server::test_server().await; - server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await; - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color"); - - server.delete_searchable_attributes().await; - server.delete_displayed_attributes().await; - - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); - - let mut server = common::Server::test_server().await; - server.update_all_settings(json!({ "searchableAttributes": ["color"], "displayedAttributes": ["color"] })).await; - let (response, _) = server.get_all_settings().await; - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "color"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "color"); - - server.update_all_settings(json!({ "searchableAttributes": [], "displayedAttributes": [] })).await; - - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); -} - -#[actix_rt::test] -async fn settings_that_contains_wildcard_is_wildcard() { - let mut server = common::Server::test_server().await; - server.update_all_settings(json!({ "searchableAttributes": ["color", "*"], "displayedAttributes": ["color", "*"] })).await; - - let (response, _) = server.get_all_settings().await; - - assert_eq!(response["searchableAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["displayedAttributes"].as_array().unwrap().len(), 1); - assert_eq!(response["searchableAttributes"].as_array().unwrap()[0], "*"); - assert_eq!(response["displayedAttributes"].as_array().unwrap()[0], "*"); -} - -#[actix_rt::test] -async fn test_displayed_attributes_field() { - let mut server = common::Server::test_server().await; - - let body = json!({ - "rankingRules": [ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ], - "distinctAttribute": "id", - "searchableAttributes": [ - "id", - "name", - "color", - "gender", - "email", - "phone", - "address", - "registered", - "about" - ], - "displayedAttributes": [ - "age", - "email", - "gender", - "name", - "registered", - ], - "stopWords": [ - "ad", - "in", - "ut", - ], - "synonyms": { - "road": ["avenue", "street"], - "street": ["avenue"], - }, - "attributesForFaceting": ["name"], - }); - - server.update_all_settings(body.clone()).await; - - let (response, _status_code) = server.get_all_settings().await; - - assert_json_eq!(body, response, ordered: true); -} \ No newline at end of file diff --git a/tests/settings_ranking_rules.rs b/tests/settings_ranking_rules.rs deleted file mode 100644 index ac9a1e00c..000000000 --- a/tests/settings_ranking_rules.rs +++ /dev/null @@ -1,182 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; - -mod common; - -#[actix_rt::test] -async fn write_all_and_delete() { - let mut server = common::Server::test_server().await; - - // 2 - Send the settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ]); - - server.update_ranking_rules(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_ranking_rules().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Delete all settings - - server.delete_ranking_rules().await; - - // 5 - Get all settings and check if they are empty - - let (response, _status_code) = server.get_ranking_rules().await; - - let expected = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ]); - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn write_all_and_update() { - let mut server = common::Server::test_server().await; - - // 2 - Send the settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - "desc(age)", - ]); - - server.update_ranking_rules(body.clone()).await; - - // 3 - Get all settings and compare to the previous one - - let (response, _status_code) = server.get_ranking_rules().await; - - assert_json_eq!(body, response, ordered: false); - - // 4 - Update all settings - - let body = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - ]); - - server.update_ranking_rules(body).await; - - // 5 - Get all settings and check if the content is the same of (4) - - let (response, _status_code) = server.get_ranking_rules().await; - - let expected = json!([ - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness", - "desc(registered)", - ]); - - assert_json_eq!(expected, response, ordered: false); -} - -#[actix_rt::test] -async fn send_undefined_rule() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - let body = json!(["typos",]); - - let (_response, status_code) = server.update_ranking_rules_sync(body).await; - assert_eq!(status_code, 400); -} - -#[actix_rt::test] -async fn send_malformed_custom_rule() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - let body = json!(["dsc(truc)",]); - - let (_response, status_code) = server.update_ranking_rules_sync(body).await; - assert_eq!(status_code, 400); -} - -// Test issue https://github.com/meilisearch/MeiliSearch/issues/521 -#[actix_rt::test] -async fn write_custom_ranking_and_index_documents() { - let mut server = common::Server::with_uid("test"); - let body = json!({ - "uid": "test", - "primaryKey": "id", - }); - server.create_index(body).await; - - // 1 - Add ranking rules with one custom ranking on a string - - let body = json!(["asc(name)", "typo"]); - - server.update_ranking_rules(body).await; - - // 2 - Add documents - - let body = json!([ - { - "id": 1, - "name": "Cherry Orr", - "color": "green" - }, - { - "id": 2, - "name": "Lucas Hess", - "color": "yellow" - } - ]); - - server.add_or_replace_multiple_documents(body).await; - - // 3 - Get the first document and compare - - let expected = json!({ - "id": 1, - "name": "Cherry Orr", - "color": "green" - }); - - let (response, status_code) = server.get_document(1).await; - assert_eq!(status_code, 200); - - assert_json_eq!(response, expected, ordered: false); -} diff --git a/tests/settings_stop_words.rs b/tests/settings_stop_words.rs deleted file mode 100644 index 3ff2e8bb7..000000000 --- a/tests/settings_stop_words.rs +++ /dev/null @@ -1,61 +0,0 @@ -use assert_json_diff::assert_json_eq; -use serde_json::json; - -mod common; - -#[actix_rt::test] -async fn update_stop_words() { - let mut server = common::Server::test_server().await; - - // 1 - Get stop words - - let (response, _status_code) = server.get_stop_words().await; - assert_eq!(response.as_array().unwrap().is_empty(), true); - - // 2 - Update stop words - - let body = json!(["ut", "ea"]); - server.update_stop_words(body.clone()).await; - - // 3 - Get all stop words and compare to the previous one - - let (response, _status_code) = server.get_stop_words().await; - assert_json_eq!(body, response, ordered: false); - - // 4 - Delete all stop words - - server.delete_stop_words().await; - - // 5 - Get all stop words and check if they are empty - - let (response, _status_code) = server.get_stop_words().await; - assert_eq!(response.as_array().unwrap().is_empty(), true); -} - -#[actix_rt::test] -async fn add_documents_and_stop_words() { - let mut server = common::Server::test_server().await; - - // 2 - Update stop words - - let body = json!(["ad", "in"]); - server.update_stop_words(body.clone()).await; - - // 3 - Search for a document with stop words - - let (response, _status_code) = server.search_get("q=in%20exercitation").await; - assert!(!response["hits"].as_array().unwrap().is_empty()); - - // 4 - Search for documents with *only* stop words - - let (response, _status_code) = server.search_get("q=ad%20in").await; - assert!(response["hits"].as_array().unwrap().is_empty()); - - // 5 - Delete all stop words - - // server.delete_stop_words(); - - // // 6 - Search for a document with one stop word - - // assert!(!response["hits"].as_array().unwrap().is_empty()); -} diff --git a/tests/url_normalizer.rs b/tests/url_normalizer.rs deleted file mode 100644 index c2c9187ee..000000000 --- a/tests/url_normalizer.rs +++ /dev/null @@ -1,18 +0,0 @@ -mod common; - -#[actix_rt::test] -async fn url_normalizer() { - let mut server = common::Server::with_uid("movies"); - - let (_response, status_code) = server.get_request("/version").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("//version").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("/version/").await; - assert_eq!(status_code, 200); - - let (_response, status_code) = server.get_request("//version/").await; - assert_eq!(status_code, 200); -} From ccb7104dee63b631d28ad5f4356157221d4325f9 Mon Sep 17 00:00:00 2001 From: mpostma Date: Fri, 29 Jan 2021 19:14:23 +0100 Subject: [PATCH 13/16] add tests for IndexStore --- .../local_index_controller/index_store.rs | 155 +++++++++++++++--- src/option.rs | 18 ++ 2 files changed, 153 insertions(+), 20 deletions(-) diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index b94d9e492..c96884155 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -17,11 +17,11 @@ use super::{UpdateMeta, UpdateResult}; type UpdateStore = super::update_store::UpdateStore; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, PartialEq)] struct IndexMeta { update_size: u64, index_size: u64, - uid: Uuid, + uuid: Uuid, } impl IndexMeta { @@ -31,8 +31,8 @@ impl IndexMeta { thread_pool: Arc, opt: &IndexerOpts, ) -> anyhow::Result<(Arc, Arc)> { - let update_path = make_update_db_path(&path, &self.uid); - let index_path = make_index_db_path(&path, &self.uid); + let update_path = make_update_db_path(&path, &self.uuid); + let index_path = make_index_db_path(&path, &self.uuid); create_dir_all(&update_path)?; create_dir_all(&index_path)?; @@ -51,7 +51,6 @@ impl IndexMeta { pub struct IndexStore { env: Env, - name_to_uid: DashMap, name_to_uid_db: Database, uid_to_index: DashMap, Arc)>, uid_to_index_db: Database>, @@ -67,7 +66,6 @@ impl IndexStore { .max_dbs(2) .open(path)?; - let name_to_uid = DashMap::new(); let uid_to_index = DashMap::new(); let name_to_uid_db = open_or_create_database(&env, Some("name_to_uid"))?; let uid_to_index_db = open_or_create_database(&env, Some("uid_to_index_db"))?; @@ -79,7 +77,6 @@ impl IndexStore { Ok(Self { env, - name_to_uid, name_to_uid_db, uid_to_index, uid_to_index_db, @@ -90,18 +87,12 @@ impl IndexStore { } fn index_uid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { - match self.name_to_uid.entry(name.as_ref().to_string()) { - Entry::Vacant(entry) => { - match self.name_to_uid_db.get(txn, name.as_ref())? { - Some(bytes) => { - let uuid = Uuid::from_slice(bytes)?; - entry.insert(uuid); - Ok(Some(uuid)) - } - None => Ok(None) - } + match self.name_to_uid_db.get(txn, name.as_ref())? { + Some(bytes) => { + let uuid = Uuid::from_slice(bytes)?; + Ok(Some(uuid)) } - Entry::Occupied(entry) => Ok(Some(entry.get().clone())), + None => Ok(None) } } @@ -182,7 +173,7 @@ impl IndexStore { update_size: u64, index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { - let meta = IndexMeta { update_size, index_size, uid: uid.clone() }; + let meta = IndexMeta { update_size, index_size, uuid: uid.clone() }; self.name_to_uid_db.put(txn, name.as_ref(), uid.as_bytes())?; self.uid_to_index_db.put(txn, uid.as_bytes(), &meta)?; @@ -190,7 +181,6 @@ impl IndexStore { let path = self.env.path(); let (index, update_store) = meta.open(path, self.thread_pool.clone(), &self.opt)?; - self.name_to_uid.insert(name.as_ref().to_string(), uid); self.uid_to_index.insert(uid, (index.clone(), update_store.clone())); Ok((index, update_store)) @@ -215,3 +205,128 @@ fn make_index_db_path(path: impl AsRef, uid: &Uuid) -> PathBuf { path.push(format!("index{}", uid)); path } + +#[cfg(test)] +mod test { + use super::*; + use std::path::PathBuf; + + #[test] + fn test_make_update_db_path() { + let uuid = Uuid::new_v4(); + assert_eq!( + make_update_db_path("/home", &uuid), + PathBuf::from(format!("/home/update{}", uuid)) + ); + } + + #[test] + fn test_make_index_db_path() { + let uuid = Uuid::new_v4(); + assert_eq!( + make_index_db_path("/home", &uuid), + PathBuf::from(format!("/home/index{}", uuid)) + ); + } + + mod index_store { + use super::*; + + #[test] + fn test_index_uuid() { + let temp = tempfile::tempdir().unwrap(); + let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); + + let name = "foobar"; + let txn = store.env.read_txn().unwrap(); + // name is not found if the uuid in not present in the db + assert!(store.index_uid(&txn, &name).unwrap().is_none()); + drop(txn); + + // insert an uuid in the the name_to_uuid_db: + let uuid = Uuid::new_v4(); + let mut txn = store.env.write_txn().unwrap(); + store.name_to_uid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + txn.commit().unwrap(); + + // check that the uuid is there + let txn = store.env.read_txn().unwrap(); + assert_eq!(store.index_uid(&txn, &name).unwrap(), Some(uuid)); + } + + #[test] + fn test_retrieve_index() { + let temp = tempfile::tempdir().unwrap(); + let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); + let uuid = Uuid::new_v4(); + + let txn = store.env.read_txn().unwrap(); + assert!(store.retrieve_index(&txn, uuid).unwrap().is_none()); + + let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; + let mut txn = store.env.write_txn().unwrap(); + store.uid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); + txn.commit().unwrap(); + + // the index cache should be empty + assert!(store.uid_to_index.is_empty()); + + let txn = store.env.read_txn().unwrap(); + assert!(store.retrieve_index(&txn, uuid).unwrap().is_some()); + assert_eq!(store.uid_to_index.len(), 1); + } + + #[test] + fn test_index() { + let temp = tempfile::tempdir().unwrap(); + let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); + let name = "foobar"; + + assert!(store.index(&name).unwrap().is_none()); + + let uuid = Uuid::new_v4(); + let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; + let mut txn = store.env.write_txn().unwrap(); + store.name_to_uid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store.uid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); + txn.commit().unwrap(); + + assert!(store.index(&name).unwrap().is_some()); + } + + #[test] + fn test_get_or_create_index() { + let temp = tempfile::tempdir().unwrap(); + let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); + let name = "foobar"; + + store.get_or_create_index(&name, 4096 * 100, 4096 * 100).unwrap(); + let txn = store.env.read_txn().unwrap(); + let uuid = store.name_to_uid_db.get(&txn, &name).unwrap(); + assert_eq!(store.uid_to_index.len(), 1); + assert!(uuid.is_some()); + let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); + let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; + assert_eq!(store.uid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); + } + + #[test] + fn test_create_index() { + let temp = tempfile::tempdir().unwrap(); + let store = IndexStore::new(temp, IndexerOpts::default()).unwrap(); + let name = "foobar"; + + let update_size = 4096 * 100; + let index_size = 4096 * 100; + let uuid = Uuid::new_v4(); + let mut txn = store.env.write_txn().unwrap(); + store.create_index(&mut txn, uuid, name, update_size, index_size).unwrap(); + let uuid = store.name_to_uid_db.get(&txn, &name).unwrap(); + assert_eq!(store.uid_to_index.len(), 1); + assert!(uuid.is_some()); + let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); + let meta = IndexMeta { update_size , index_size, uuid: uuid.clone() }; + assert_eq!(store.uid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); + } + } +} diff --git a/src/option.rs b/src/option.rs index f280553e1..dffa7d483 100644 --- a/src/option.rs +++ b/src/option.rs @@ -63,6 +63,24 @@ pub struct IndexerOpts { #[structopt(long)] pub indexing_jobs: Option, } + +#[cfg(test)] +impl Default for IndexerOpts { + fn default() -> Self { + Self { + log_every_n: 0, + max_nb_chunks: None, + max_memory: Byte::from_str("0Kb").unwrap(), + linked_hash_map_size: 0, + chunk_compression_type: CompressionType::None, + chunk_compression_level: None, + chunk_fusing_shrink_size: Byte::from_str("0Kb").unwrap(), + enable_chunk_fusing: false, + indexing_jobs: None, + } + } +} + const POSSIBLE_ENV: [&str; 2] = ["development", "production"]; #[derive(Debug, Clone, StructOpt)] From 17c463ca61eaaccc9bbf106e871506a030bb878d Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 1 Feb 2021 13:32:21 +0100 Subject: [PATCH 14/16] remove unused deps --- Cargo.lock | 161 ++++------------------------------------------------- Cargo.toml | 5 -- 2 files changed, 11 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4471f5127..9240548d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,15 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "Inflector" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" -dependencies = [ - "lazy_static", - "regex", -] - [[package]] name = "actix-codec" version = "0.3.0" @@ -41,7 +31,7 @@ dependencies = [ "futures-util", "http", "log", - "rustls 0.18.1", + "rustls", "tokio-rustls", "trust-dns-proto", "trust-dns-resolver", @@ -217,10 +207,10 @@ dependencies = [ "actix-service", "actix-utils", "futures-util", - "rustls 0.18.1", + "rustls", "tokio-rustls", "webpki", - "webpki-roots 0.20.0", + "webpki-roots", ] [[package]] @@ -273,7 +263,7 @@ dependencies = [ "mime", "pin-project 1.0.2", "regex", - "rustls 0.18.1", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -342,7 +332,7 @@ checksum = "68803225a7b13e47191bab76f2687382b60d259e8cf37f6e1893658b84bb9479" [[package]] name = "assert-json-diff" version = "1.0.1" -source = "git+https://github.com/qdequele/assert-json-diff?branch=master#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" +source = "git+https://github.com/qdequele/assert-json-diff#9012a0c8866d0f2db0ef9a6242e4a19d1e8c67e4" dependencies = [ "serde", "serde_json", @@ -408,7 +398,7 @@ dependencies = [ "mime", "percent-encoding", "rand 0.7.3", - "rustls 0.18.1", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -634,12 +624,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "chunked_transfer" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7477065d45a8fe57167bf3cf8bcd3729b54cfcb81cca49bda2d038ea89ae82ca" - [[package]] name = "clap" version = "2.33.3" @@ -1365,7 +1349,7 @@ dependencies = [ "futures-util", "hyper", "log", - "rustls 0.18.1", + "rustls", "tokio", "tokio-rustls", "webpki", @@ -1670,16 +1654,14 @@ dependencies = [ "mime", "obkv", "once_cell", - "ouroboros", "page_size", "rand 0.7.3", "rayon", "regex", - "rustls 0.18.1", + "rustls", "sentry", "serde", "serde_json", - "serde_qs", "serde_url_params", "sha2", "siphasher", @@ -1689,11 +1671,8 @@ dependencies = [ "tempdir", "tempfile", "tokio", - "ureq", "uuid", "vergen", - "walkdir", - "whoami", ] [[package]] @@ -1974,29 +1953,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "ouroboros" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069fb33e127cabdc8ad6a287eed9719b85c612d36199777f6dc41ad91f7be41a" -dependencies = [ - "ouroboros_macro", - "stable_deref_trait", -] - -[[package]] -name = "ouroboros_macro" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad938cc920f299d6dce91e43d3ce316e785f4aa4bc4243555634dc2967098fc6" -dependencies = [ - "Inflector", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "page_size" version = "0.4.2" @@ -2242,15 +2198,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "qstring" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d464fae65fff2680baf48019211ce37aaec0c78e9264c84a3e484717f965104e" -dependencies = [ - "percent-encoding", -] - [[package]] name = "quick-error" version = "1.2.3" @@ -2453,7 +2400,7 @@ dependencies = [ "mime_guess", "percent-encoding", "pin-project-lite 0.2.0", - "rustls 0.18.1", + "rustls", "serde", "serde_json", "serde_urlencoded", @@ -2463,7 +2410,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.20.0", + "webpki-roots", "winreg 0.7.0", ] @@ -2535,34 +2482,12 @@ dependencies = [ "webpki", ] -[[package]] -name = "rustls" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" -dependencies = [ - "base64 0.13.0", - "log", - "ring", - "sct", - "webpki", -] - [[package]] name = "ryu" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "scopeguard" version = "1.1.0" @@ -2674,17 +2599,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_qs" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5cb0f0564a84554436c4ceff5c896308d4e09d0eb4bd0215b8f698f88084601" -dependencies = [ - "percent-encoding", - "serde", - "thiserror", -] - [[package]] name = "serde_url_params" version = "0.2.0" @@ -2827,12 +2741,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - [[package]] name = "standback" version = "0.2.13" @@ -3175,7 +3083,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a" dependencies = [ "futures-core", - "rustls 0.18.1", + "rustls", "tokio", "webpki", ] @@ -3349,23 +3257,6 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" -[[package]] -name = "ureq" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "294b85ef5dbc3670a72e82a89971608a1fcc4ed5c7c5a2895230d31a95f0569b" -dependencies = [ - "base64 0.13.0", - "chunked_transfer", - "log", - "once_cell", - "qstring", - "rustls 0.19.0", - "url", - "webpki", - "webpki-roots 0.21.0", -] - [[package]] name = "url" version = "2.2.0" @@ -3417,17 +3308,6 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" -[[package]] -name = "walkdir" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" -dependencies = [ - "same-file", - "winapi 0.3.9", - "winapi-util", -] - [[package]] name = "want" version = "0.3.0" @@ -3547,15 +3427,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "webpki-roots" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82015b7e0b8bad8185994674a13a93306bea76cf5a16c5a181382fd3a5ec2376" -dependencies = [ - "webpki", -] - [[package]] name = "whatlang" version = "0.9.0" @@ -3565,16 +3436,6 @@ dependencies = [ "hashbrown 0.7.2", ] -[[package]] -name = "whoami" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e35495e7faf4c657051a8e9725d9c37ac57879e915be3ed55bb401af84382035" -dependencies = [ - "wasm-bindgen", - "web-sys", -] - [[package]] name = "widestring" version = "0.4.3" diff --git a/Cargo.toml b/Cargo.toml index f68414beb..1d7590d47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,6 @@ regex = "1.4.2" rustls = "0.18" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0.59", features = ["preserve_order"] } -serde_qs = "0.8.1" sha2 = "0.9.1" siphasher = "0.3.2" slice-group-by = "0.2.6" @@ -55,13 +54,9 @@ structopt = "0.3.20" tar = "0.4.29" tempfile = "3.1.0" tokio = { version = "0.2", features = ["full"] } -ureq = { version = "1.5.1", default-features = false, features = ["tls"] } -walkdir = "2.3.1" -whoami = "1.0.0" dashmap = "4.0.2" page_size = "0.4.2" obkv = "0.1.1" -ouroboros = "0.8.0" uuid = "0.8.2" itertools = "0.10.0" From 9af0a08122e3558d90a0ac32fff63dcc7ba1a912 Mon Sep 17 00:00:00 2001 From: mpostma Date: Mon, 1 Feb 2021 19:51:47 +0100 Subject: [PATCH 15/16] post review fixes --- Cargo.lock | 113 ++++++++++-- Cargo.toml | 1 + src/data/mod.rs | 8 +- src/data/search.rs | 24 ++- src/data/updates.rs | 27 ++- .../local_index_controller/index_store.rs | 173 +++++++++--------- .../local_index_controller/mod.rs | 9 +- .../local_index_controller/update_store.rs | 10 - src/index_controller/mod.rs | 25 +-- src/lib.rs | 1 - 10 files changed, 233 insertions(+), 158 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9240548d9..f036664f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -808,6 +808,12 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80115a2dfde04491e181c2440a39e4be26e52d9ca4e92bed213f65b94e0b8db1" +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" + [[package]] name = "digest" version = "0.8.1" @@ -832,6 +838,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "downcast" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d" + [[package]] name = "either" version = "1.6.1" @@ -937,6 +949,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -953,6 +974,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69a039c3498dc930fe810151a34ba0c1c70b02b8625035592e74432f678591f2" + [[package]] name = "fs_extra" version = "1.2.0" @@ -1612,7 +1639,7 @@ checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] name = "meilisearch-error" -version = "0.18.0" +version = "0.19.0" dependencies = [ "actix-http", ] @@ -1652,6 +1679,7 @@ dependencies = [ "memmap", "milli", "mime", + "mockall", "obkv", "once_cell", "page_size", @@ -1724,7 +1752,6 @@ dependencies = [ "bstr", "byte-unit", "byteorder", - "chrono", "crossbeam-channel", "csv", "either", @@ -1754,7 +1781,6 @@ dependencies = [ "roaring", "serde", "serde_json", - "serde_millis", "slice-group-by", "smallstr", "smallvec", @@ -1854,6 +1880,33 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "mockall" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "619634fd9149c4a06e66d8fd9256e85326d8eeee75abee4565ff76c92e4edfe0" +dependencies = [ + "cfg-if 1.0.0", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83714c95dbf4c24202f0f1b208f0f248e6bd65abfa8989303611a71c0f781548" +dependencies = [ + "cfg-if 1.0.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "near-proximity" version = "0.1.0" @@ -1885,6 +1938,12 @@ dependencies = [ "libc", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "num-integer" version = "0.1.44" @@ -2153,6 +2212,35 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "predicates" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb433456c1a57cc93554dea3ce40b4c19c4057e41c55d4a0f3d84ea71c325aa" +dependencies = [ + "difference", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451" + +[[package]] +name = "predicates-tree" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2" +dependencies = [ + "predicates-core", + "treeline", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2447,9 +2535,9 @@ checksum = "21215c1b9d8f7832b433255bd9eea3e2779aa55b21b2f8e13aad62c74749b237" [[package]] name = "roaring" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb550891a98438463978260676feef06c12bfb0eb0b05e191f888fb785cc9374" +checksum = "4d60b41c8f25d07cecab125cb46ebbf234fc055effc61ca2392a3ef4f9422304" dependencies = [ "byteorder", ] @@ -2590,15 +2678,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_millis" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e2dc780ca5ee2c369d1d01d100270203c4ff923d2a4264812d723766434d00" -dependencies = [ - "serde", -] - [[package]] name = "serde_url_params" version = "0.2.0" @@ -3139,6 +3218,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "treeline" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" + [[package]] name = "trust-dns-proto" version = "0.19.6" diff --git a/Cargo.toml b/Cargo.toml index 1d7590d47..ad2d034ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ serde_url_params = "0.2.0" tempdir = "0.3.7" assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } tokio = { version = "0.2", features = ["macros", "time"] } +mockall = "0.9.0" [features] default = ["sentry"] diff --git a/src/data/mod.rs b/src/data/mod.rs index 175aedba5..de24d0a06 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -3,14 +3,14 @@ mod updates; pub use search::{SearchQuery, SearchResult}; +use std::fs::create_dir_all; use std::ops::Deref; use std::sync::Arc; -use std::fs::create_dir_all; use sha2::Digest; -use crate::{option::Opt, index_controller::Settings}; use crate::index_controller::{IndexController, LocalIndexController}; +use crate::{option::Opt, index_controller::Settings}; #[derive(Clone)] pub struct Data { @@ -67,7 +67,7 @@ impl Data { options.max_mdb_size.get_bytes(), options.max_udb_size.get_bytes(), )?; - let indexes = Arc::new(index_controller); + let index_controller = Arc::new(index_controller); let mut api_keys = ApiKeys { master: options.clone().master_key, @@ -77,7 +77,7 @@ impl Data { api_keys.generate_missing_api_keys(); - let inner = DataInner { index_controller: indexes, options, api_keys }; + let inner = DataInner { index_controller, options, api_keys }; let inner = Arc::new(inner); Ok(Data { inner }) diff --git a/src/data/search.rs b/src/data/search.rs index 246e3bdac..2e05988aa 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -2,11 +2,11 @@ use std::collections::HashSet; use std::mem; use std::time::Instant; -use serde_json::{Value, Map}; -use serde::{Deserialize, Serialize}; -use milli::{Index, obkv_to_json, FacetCondition}; -use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; use anyhow::bail; +use meilisearch_tokenizer::{Analyzer, AnalyzerConfig}; +use milli::{Index, obkv_to_json, FacetCondition}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, Map}; use crate::index_controller::IndexController; use super::Data; @@ -37,7 +37,6 @@ pub struct SearchQuery { impl SearchQuery { pub fn perform(&self, index: impl AsRef) -> anyhow::Result{ let index = index.as_ref(); - let before_search = Instant::now(); let rtxn = index.read_txn().unwrap(); @@ -47,6 +46,9 @@ impl SearchQuery { search.query(query); } + search.limit(self.limit); + search.offset(self.offset.unwrap_or_default()); + if let Some(ref condition) = self.facet_condition { if !condition.trim().is_empty() { let condition = FacetCondition::from_str(&rtxn, &index, &condition).unwrap(); @@ -54,11 +56,7 @@ impl SearchQuery { } } - if let Some(offset) = self.offset { - search.offset(offset); - } - - let milli::SearchResult { documents_ids, found_words, nb_hits, limit, } = search.execute()?; + let milli::SearchResult { documents_ids, found_words, candidates } = search.execute()?; let mut documents = Vec::new(); let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); @@ -81,9 +79,9 @@ impl SearchQuery { Ok(SearchResult { hits: documents, - nb_hits, + nb_hits: candidates.len(), query: self.q.clone().unwrap_or_default(), - limit, + limit: self.limit, offset: self.offset.unwrap_or_default(), processing_time_ms: before_search.elapsed().as_millis(), }) @@ -94,7 +92,7 @@ impl SearchQuery { #[serde(rename_all = "camelCase")] pub struct SearchResult { hits: Vec>, - nb_hits: usize, + nb_hits: u64, query: String, limit: usize, offset: usize, diff --git a/src/data/updates.rs b/src/data/updates.rs index 194ec346b..27fc6537e 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -6,21 +6,20 @@ use futures_util::stream::StreamExt; use tokio::io::AsyncWriteExt; use super::Data; -use crate::index_controller::{IndexController, Settings, UpdateResult, UpdateMeta}; -use crate::index_controller::updates::UpdateStatus; +use crate::index_controller::{IndexController, Settings}; +use crate::index_controller::UpdateStatus; impl Data { - pub async fn add_documents( + pub async fn add_documents( &self, - index: S, + index: impl AsRef + Send + Sync + 'static, method: IndexDocumentsMethod, format: UpdateFormat, mut stream: impl futures::Stream> + Unpin, - ) -> anyhow::Result> + ) -> anyhow::Result where B: Deref, E: std::error::Error + Send + Sync + 'static, - S: AsRef + Send + Sync + 'static, { let file = tokio::task::spawn_blocking(tempfile::tempfile).await?; let file = tokio::fs::File::from_std(file?); @@ -38,26 +37,26 @@ impl Data { let mmap = unsafe { memmap::Mmap::map(&file)? }; let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move ||index_controller.add_documents(index, method, format, &mmap[..])).await??; + let update = tokio::task::spawn_blocking(move || index_controller.add_documents(index, method, format, &mmap[..])).await??; Ok(update.into()) } - pub async fn update_settings + Send + Sync + 'static>( + pub async fn update_settings( &self, - index: S, + index: impl AsRef + Send + Sync + 'static, settings: Settings - ) -> anyhow::Result> { - let indexes = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move || indexes.update_settings(index, settings)).await??; + ) -> anyhow::Result { + let index_controller = self.index_controller.clone(); + let update = tokio::task::spawn_blocking(move || index_controller.update_settings(index, settings)).await??; Ok(update.into()) } #[inline] - pub fn get_update_status>(&self, index: S, uid: u64) -> anyhow::Result>> { + pub fn get_update_status(&self, index: impl AsRef, uid: u64) -> anyhow::Result> { self.index_controller.update_status(index, uid) } - pub fn get_updates_status(&self, index: &str) -> anyhow::Result>> { + pub fn get_updates_status(&self, index: impl AsRef) -> anyhow::Result> { self.index_controller.all_update_status(index) } } diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index c96884155..16df83d2c 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -1,15 +1,13 @@ +use std::fs::{create_dir_all, remove_dir_all}; use std::path::{Path, PathBuf}; -use std::fs::create_dir_all; use std::sync::Arc; -use dashmap::DashMap; -use dashmap::mapref::entry::Entry; +use dashmap::{DashMap, mapref::entry::Entry}; use heed::{Env, EnvOpenOptions, Database, types::{Str, SerdeJson, ByteSlice}, RoTxn, RwTxn}; use milli::Index; use rayon::ThreadPool; -use uuid::Uuid; use serde::{Serialize, Deserialize}; -use log::warn; +use uuid::Uuid; use crate::option::IndexerOpts; use super::update_handler::UpdateHandler; @@ -29,7 +27,7 @@ impl IndexMeta { &self, path: impl AsRef, thread_pool: Arc, - opt: &IndexerOpts, + indexer_options: &IndexerOpts, ) -> anyhow::Result<(Arc, Arc)> { let update_path = make_update_db_path(&path, &self.uuid); let index_path = make_index_db_path(&path, &self.uuid); @@ -43,24 +41,25 @@ impl IndexMeta { let mut options = EnvOpenOptions::new(); options.map_size(self.update_size as usize); - let handler = UpdateHandler::new(opt, index.clone(), thread_pool)?; + let handler = UpdateHandler::new(indexer_options, index.clone(), thread_pool)?; let update_store = UpdateStore::open(options, update_path, handler)?; + Ok((index, update_store)) } } pub struct IndexStore { env: Env, - name_to_uid_db: Database, - uid_to_index: DashMap, Arc)>, - uid_to_index_db: Database>, + name_to_uuid_db: Database, + uuid_to_index: DashMap, Arc)>, + uuid_to_index_db: Database>, thread_pool: Arc, - opt: IndexerOpts, + indexer_options: IndexerOpts, } impl IndexStore { - pub fn new(path: impl AsRef, opt: IndexerOpts) -> anyhow::Result { + pub fn new(path: impl AsRef, indexer_options: IndexerOpts) -> anyhow::Result { let env = EnvOpenOptions::new() .map_size(4096 * 100) .max_dbs(2) @@ -71,23 +70,23 @@ impl IndexStore { let uid_to_index_db = open_or_create_database(&env, Some("uid_to_index_db"))?; let thread_pool = rayon::ThreadPoolBuilder::new() - .num_threads(opt.indexing_jobs.unwrap_or(0)) + .num_threads(indexer_options.indexing_jobs.unwrap_or(0)) .build()?; let thread_pool = Arc::new(thread_pool); Ok(Self { env, - name_to_uid_db, - uid_to_index, - uid_to_index_db, + name_to_uuid_db: name_to_uid_db, + uuid_to_index: uid_to_index, + uuid_to_index_db: uid_to_index_db, thread_pool, - opt, + indexer_options, }) } - fn index_uid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { - match self.name_to_uid_db.get(txn, name.as_ref())? { + fn index_uuid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { + match self.name_to_uuid_db.get(txn, name.as_ref())? { Some(bytes) => { let uuid = Uuid::from_slice(bytes)?; Ok(Some(uuid)) @@ -97,12 +96,12 @@ impl IndexStore { } fn retrieve_index(&self, txn: &RoTxn, uid: Uuid) -> anyhow::Result, Arc)>> { - match self.uid_to_index.entry(uid.clone()) { + match self.uuid_to_index.entry(uid.clone()) { Entry::Vacant(entry) => { - match self.uid_to_index_db.get(txn, uid.as_bytes())? { + match self.uuid_to_index_db.get(txn, uid.as_bytes())? { Some(meta) => { let path = self.env.path(); - let (index, updates) = meta.open(path, self.thread_pool.clone(), &self.opt)?; + let (index, updates) = meta.open(path, self.thread_pool.clone(), &self.indexer_options)?; entry.insert((index.clone(), updates.clone())); Ok(Some((index, updates))) }, @@ -117,7 +116,7 @@ impl IndexStore { } fn _get_index(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result, Arc)>> { - match self.index_uid(&txn, name)? { + match self.index_uuid(&txn, name)? { Some(uid) => self.retrieve_index(&txn, uid), None => Ok(None), } @@ -129,59 +128,61 @@ impl IndexStore { } pub fn get_or_create_index( - &self, name: impl AsRef, - update_size: u64, - index_size: u64, - ) -> anyhow::Result<(Arc, Arc)> { - let mut txn = self.env.write_txn()?; - match self._get_index(&txn, name.as_ref())? { - Some(res) => Ok(res), - None => { - let uid = Uuid::new_v4(); - // TODO: clean in case of error - let result = self.create_index(&mut txn, uid, name, update_size, index_size); - match result { - Ok((index, update_store)) => { - match txn.commit() { - Ok(_) => Ok((index, update_store)), - Err(e) => { - self.clean_uid(&uid); - Err(anyhow::anyhow!("error creating index: {}", e)) - } - } - } - Err(e) => { - self.clean_uid(&uid); - Err(e) - } - } - }, - } - } - - /// removes all data acociated with an index Uuid. This is called when index creation failed - /// and outstanding files and data need to be cleaned. - fn clean_uid(&self, _uid: &Uuid) { - // TODO! - warn!("creating cleanup is not yet implemented"); - } - - fn create_index( &self, - txn: &mut RwTxn, - uid: Uuid, + &self, name: impl AsRef, update_size: u64, index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { - let meta = IndexMeta { update_size, index_size, uuid: uid.clone() }; + let mut txn = self.env.write_txn()?; + match self._get_index(&txn, name.as_ref())? { + Some(res) => Ok(res), + None => { + let uuid = Uuid::new_v4(); + let result = self.create_index(&mut txn, uuid, name, update_size, index_size)?; + // If we fail to commit the transaction, we must delete the database from the + // file-system. + if let Err(e) = txn.commit() { + self.clean_db(uuid); + return Err(e)?; + } + Ok(result) + }, + } + } - self.name_to_uid_db.put(txn, name.as_ref(), uid.as_bytes())?; - self.uid_to_index_db.put(txn, uid.as_bytes(), &meta)?; + // Remove all the files and data associated with a db uuid. + fn clean_db(&self, uuid: Uuid) { + let update_db_path = make_update_db_path(self.env.path(), &uuid); + let index_db_path = make_index_db_path(self.env.path(), &uuid); + + remove_dir_all(update_db_path).expect("Failed to clean database"); + remove_dir_all(index_db_path).expect("Failed to clean database"); + + self.uuid_to_index.remove(&uuid); + } + + fn create_index( &self, + txn: &mut RwTxn, + uuid: Uuid, + name: impl AsRef, + update_size: u64, + index_size: u64, + ) -> anyhow::Result<(Arc, Arc)> { + let meta = IndexMeta { update_size, index_size, uuid: uuid.clone() }; + + self.name_to_uuid_db.put(txn, name.as_ref(), uuid.as_bytes())?; + self.uuid_to_index_db.put(txn, uuid.as_bytes(), &meta)?; let path = self.env.path(); - let (index, update_store) = meta.open(path, self.thread_pool.clone(), &self.opt)?; + let (index, update_store) = match meta.open(path, self.thread_pool.clone(), &self.indexer_options) { + Ok(res) => res, + Err(e) => { + self.clean_db(uuid); + return Err(e) + } + }; - self.uid_to_index.insert(uid, (index.clone(), update_store.clone())); + self.uuid_to_index.insert(uuid, (index.clone(), update_store.clone())); Ok((index, update_store)) } @@ -194,15 +195,15 @@ fn open_or_create_database(env: &Env, name: Option<&str> } } -fn make_update_db_path(path: impl AsRef, uid: &Uuid) -> PathBuf { +fn make_update_db_path(path: impl AsRef, uuid: &Uuid) -> PathBuf { let mut path = path.as_ref().to_path_buf(); - path.push(format!("update{}", uid)); + path.push(format!("update{}", uuid)); path } -fn make_index_db_path(path: impl AsRef, uid: &Uuid) -> PathBuf { +fn make_index_db_path(path: impl AsRef, uuid: &Uuid) -> PathBuf { let mut path = path.as_ref().to_path_buf(); - path.push(format!("index{}", uid)); + path.push(format!("index{}", uuid)); path } @@ -240,18 +241,18 @@ mod test { let name = "foobar"; let txn = store.env.read_txn().unwrap(); // name is not found if the uuid in not present in the db - assert!(store.index_uid(&txn, &name).unwrap().is_none()); + assert!(store.index_uuid(&txn, &name).unwrap().is_none()); drop(txn); // insert an uuid in the the name_to_uuid_db: let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); - store.name_to_uid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store.name_to_uuid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); txn.commit().unwrap(); // check that the uuid is there let txn = store.env.read_txn().unwrap(); - assert_eq!(store.index_uid(&txn, &name).unwrap(), Some(uuid)); + assert_eq!(store.index_uuid(&txn, &name).unwrap(), Some(uuid)); } #[test] @@ -265,15 +266,15 @@ mod test { let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; let mut txn = store.env.write_txn().unwrap(); - store.uid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); + store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); txn.commit().unwrap(); // the index cache should be empty - assert!(store.uid_to_index.is_empty()); + assert!(store.uuid_to_index.is_empty()); let txn = store.env.read_txn().unwrap(); assert!(store.retrieve_index(&txn, uuid).unwrap().is_some()); - assert_eq!(store.uid_to_index.len(), 1); + assert_eq!(store.uuid_to_index.len(), 1); } #[test] @@ -287,8 +288,8 @@ mod test { let uuid = Uuid::new_v4(); let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; let mut txn = store.env.write_txn().unwrap(); - store.name_to_uid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); - store.uid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); + store.name_to_uuid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); txn.commit().unwrap(); assert!(store.index(&name).unwrap().is_some()); @@ -302,12 +303,12 @@ mod test { store.get_or_create_index(&name, 4096 * 100, 4096 * 100).unwrap(); let txn = store.env.read_txn().unwrap(); - let uuid = store.name_to_uid_db.get(&txn, &name).unwrap(); - assert_eq!(store.uid_to_index.len(), 1); + let uuid = store.name_to_uuid_db.get(&txn, &name).unwrap(); + assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; - assert_eq!(store.uid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); + assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); } #[test] @@ -321,12 +322,12 @@ mod test { let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); store.create_index(&mut txn, uuid, name, update_size, index_size).unwrap(); - let uuid = store.name_to_uid_db.get(&txn, &name).unwrap(); - assert_eq!(store.uid_to_index.len(), 1); + let uuid = store.name_to_uuid_db.get(&txn, &name).unwrap(); + assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); let meta = IndexMeta { update_size , index_size, uuid: uuid.clone() }; - assert_eq!(store.uid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); + assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); } } } diff --git a/src/index_controller/local_index_controller/mod.rs b/src/index_controller/local_index_controller/mod.rs index debc15a1a..b59eb2a99 100644 --- a/src/index_controller/local_index_controller/mod.rs +++ b/src/index_controller/local_index_controller/mod.rs @@ -2,16 +2,15 @@ mod update_store; mod index_store; mod update_handler; -use index_store::IndexStore; - use std::path::Path; use std::sync::Arc; -use milli::Index; use anyhow::bail; use itertools::Itertools; +use milli::Index; use crate::option::IndexerOpts; +use index_store::IndexStore; use super::IndexController; use super::updates::UpdateStatus; use super::{UpdateMeta, UpdateResult}; @@ -84,7 +83,7 @@ impl IndexController for LocalIndexController { } fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>> { - match self.indexes.index(index)? { + match self.indexes.index(&index)? { Some((_, update_store)) => { let updates = update_store.iter_metas(|processing, processed, pending, aborted, failed| { Ok(processing @@ -99,7 +98,7 @@ impl IndexController for LocalIndexController { })?; Ok(updates) } - None => Ok(Vec::new()) + None => bail!("index {} doesn't exist.", index.as_ref()), } } diff --git a/src/index_controller/local_index_controller/update_store.rs b/src/index_controller/local_index_controller/update_store.rs index 9a17ec00f..d4b796993 100644 --- a/src/index_controller/local_index_controller/update_store.rs +++ b/src/index_controller/local_index_controller/update_store.rs @@ -192,16 +192,6 @@ where } } - /// The id and metadata of the update that is currently being processed, - /// `None` if no update is being processed. - pub fn processing_update(&self) -> heed::Result>> { - let rtxn = self.env.read_txn()?; - match self.pending_meta.first(&rtxn)? { - Some((_, meta)) => Ok(Some(meta)), - None => Ok(None), - } - } - /// Execute the user defined function with the meta-store iterators, the first /// iterator is the *processed* meta one, the second the *aborted* meta one /// and, the last is the *pending* meta one. diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index f1ba8f7ce..0ea654dfb 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -1,5 +1,5 @@ mod local_index_controller; -pub mod updates; +mod updates; pub use local_index_controller::LocalIndexController; @@ -12,7 +12,9 @@ use milli::Index; use milli::update::{IndexDocumentsMethod, UpdateFormat, DocumentAdditionResult}; use serde::{Serialize, Deserialize, de::Deserializer}; -use updates::{Processed, Processing, Failed, UpdateStatus}; +pub use updates::{Processed, Processing, Failed}; + +pub type UpdateStatus = updates::UpdateStatus; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -85,9 +87,10 @@ pub enum UpdateResult { } /// The `IndexController` is in charge of the access to the underlying indices. It splits the logic -/// for read access which is provided, and write access which must be provided. This allows the -/// implementer to define the behaviour of write accesses to the indices, and abstract the -/// scheduling of the updates. The implementer must be able to provide an instance of `IndexStore` +/// for read access which is provided thanks to an handle to the index, and write access which must +/// be provided. This allows the implementer to define the behaviour of write accesses to the +/// indices, and abstract the scheduling of the updates. The implementer must be able to provide an +/// instance of `IndexStore` pub trait IndexController { /* @@ -106,11 +109,11 @@ pub trait IndexController { method: IndexDocumentsMethod, format: UpdateFormat, data: &[u8], - ) -> anyhow::Result>; + ) -> anyhow::Result; /// Updates an index settings. If the index does not exist, it will be created when the update /// is applied to the index. - fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result>; + fn update_settings>(&self, index_uid: S, settings: Settings) -> anyhow::Result; /// Create an index with the given `index_uid`. fn create_index>(&self, index_uid: S) -> Result<()>; @@ -133,9 +136,9 @@ pub trait IndexController { todo!() } - /// Returns, if it exists, an `IndexView` to the requested index. - fn index(&self, uid: impl AsRef) -> anyhow::Result>>; + /// Returns, if it exists, the `Index` with the povided name. + fn index(&self, name: impl AsRef) -> anyhow::Result>>; - fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>>; - fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>>; + fn update_status(&self, index: impl AsRef, id: u64) -> anyhow::Result>; + fn all_update_status(&self, index: impl AsRef) -> anyhow::Result>; } diff --git a/src/lib.rs b/src/lib.rs index df9381914..d542fd6d7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,7 +5,6 @@ pub mod error; pub mod helpers; pub mod option; pub mod routes; -//mod updates; mod index_controller; use actix_http::Error; From f8f02af23e372ae0b07a44ba1eaf8503e62f0f67 Mon Sep 17 00:00:00 2001 From: mpostma Date: Thu, 4 Feb 2021 13:21:15 +0100 Subject: [PATCH 16/16] incorporate review changes --- Cargo.lock | 98 ------------------- Cargo.toml | 5 +- src/data/search.rs | 2 +- src/data/updates.rs | 15 ++- .../local_index_controller/index_store.rs | 40 ++++---- src/index_controller/mod.rs | 1 - src/option.rs | 16 +-- 7 files changed, 43 insertions(+), 134 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f036664f4..0b8128118 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -808,12 +808,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80115a2dfde04491e181c2440a39e4be26e52d9ca4e92bed213f65b94e0b8db1" -[[package]] -name = "difference" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" - [[package]] name = "digest" version = "0.8.1" @@ -838,12 +832,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" -[[package]] -name = "downcast" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d" - [[package]] name = "either" version = "1.6.1" @@ -949,15 +937,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "float-cmp" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1267f4ac4f343772758f7b1bdcbe767c218bbab93bb432acbf5162bbf85a6c4" -dependencies = [ - "num-traits", -] - [[package]] name = "fnv" version = "1.0.7" @@ -974,12 +953,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fragile" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a039c3498dc930fe810151a34ba0c1c70b02b8625035592e74432f678591f2" - [[package]] name = "fs_extra" version = "1.2.0" @@ -1679,10 +1652,7 @@ dependencies = [ "memmap", "milli", "mime", - "mockall", - "obkv", "once_cell", - "page_size", "rand 0.7.3", "rayon", "regex", @@ -1880,33 +1850,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "mockall" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "619634fd9149c4a06e66d8fd9256e85326d8eeee75abee4565ff76c92e4edfe0" -dependencies = [ - "cfg-if 1.0.0", - "downcast", - "fragile", - "lazy_static", - "mockall_derive", - "predicates", - "predicates-tree", -] - -[[package]] -name = "mockall_derive" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83714c95dbf4c24202f0f1b208f0f248e6bd65abfa8989303611a71c0f781548" -dependencies = [ - "cfg-if 1.0.0", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "near-proximity" version = "0.1.0" @@ -1938,12 +1881,6 @@ dependencies = [ "libc", ] -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - [[package]] name = "num-integer" version = "0.1.44" @@ -2212,35 +2149,6 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" -[[package]] -name = "predicates" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb433456c1a57cc93554dea3ce40b4c19c4057e41c55d4a0f3d84ea71c325aa" -dependencies = [ - "difference", - "float-cmp", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates-core" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451" - -[[package]] -name = "predicates-tree" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2" -dependencies = [ - "predicates-core", - "treeline", -] - [[package]] name = "proc-macro-error" version = "1.0.4" @@ -3218,12 +3126,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "treeline" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" - [[package]] name = "trust-dns-proto" version = "0.19.6" diff --git a/Cargo.toml b/Cargo.toml index ad2d034ad..4668a6897 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ fst = "0.4.5" futures = "0.3.7" futures-util = "0.3.8" grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } -heed = { version = "0.10.6", default-features = false, features = ["serde", "lmdb", "sync-read-txn"] } +heed = "0.10.6" http = "0.2.1" indexmap = { version = "1.3.2", features = ["serde-1"] } log = "0.4.8" @@ -55,8 +55,6 @@ tar = "0.4.29" tempfile = "3.1.0" tokio = { version = "0.2", features = ["full"] } dashmap = "4.0.2" -page_size = "0.4.2" -obkv = "0.1.1" uuid = "0.8.2" itertools = "0.10.0" @@ -72,7 +70,6 @@ serde_url_params = "0.2.0" tempdir = "0.3.7" assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" } tokio = { version = "0.2", features = ["macros", "time"] } -mockall = "0.9.0" [features] default = ["sentry"] diff --git a/src/data/search.rs b/src/data/search.rs index 2e05988aa..d0858d704 100644 --- a/src/data/search.rs +++ b/src/data/search.rs @@ -51,7 +51,7 @@ impl SearchQuery { if let Some(ref condition) = self.facet_condition { if !condition.trim().is_empty() { - let condition = FacetCondition::from_str(&rtxn, &index, &condition).unwrap(); + let condition = FacetCondition::from_str(&rtxn, &index, &condition)?; search.facet_condition(condition); } } diff --git a/src/data/updates.rs b/src/data/updates.rs index 27fc6537e..06aed8faa 100644 --- a/src/data/updates.rs +++ b/src/data/updates.rs @@ -25,7 +25,9 @@ impl Data { let file = tokio::fs::File::from_std(file?); let mut encoder = GzipEncoder::new(file); + let mut empty_update = true; while let Some(result) = stream.next().await { + empty_update = false; let bytes = &*result?; encoder.write_all(&bytes[..]).await?; } @@ -34,10 +36,19 @@ impl Data { let mut file = encoder.into_inner(); file.sync_all().await?; let file = file.into_std().await; - let mmap = unsafe { memmap::Mmap::map(&file)? }; + let index_controller = self.index_controller.clone(); - let update = tokio::task::spawn_blocking(move || index_controller.add_documents(index, method, format, &mmap[..])).await??; + let update = tokio::task::spawn_blocking(move ||{ + let mmap; + let bytes = if empty_update { + &[][..] + } else { + mmap = unsafe { memmap::Mmap::map(&file)? }; + &mmap + }; + index_controller.add_documents(index, method, format, &bytes) + }).await??; Ok(update.into()) } diff --git a/src/index_controller/local_index_controller/index_store.rs b/src/index_controller/local_index_controller/index_store.rs index 16df83d2c..483b6f5d6 100644 --- a/src/index_controller/local_index_controller/index_store.rs +++ b/src/index_controller/local_index_controller/index_store.rs @@ -17,8 +17,8 @@ type UpdateStore = super::update_store::UpdateStore, + name_to_uuid_meta: Database, uuid_to_index: DashMap, Arc)>, uuid_to_index_db: Database>, @@ -76,7 +76,7 @@ impl IndexStore { Ok(Self { env, - name_to_uuid_db: name_to_uid_db, + name_to_uuid_meta: name_to_uid_db, uuid_to_index: uid_to_index, uuid_to_index_db: uid_to_index_db, @@ -86,7 +86,7 @@ impl IndexStore { } fn index_uuid(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result> { - match self.name_to_uuid_db.get(txn, name.as_ref())? { + match self.name_to_uuid_meta.get(txn, name.as_ref())? { Some(bytes) => { let uuid = Uuid::from_slice(bytes)?; Ok(Some(uuid)) @@ -115,7 +115,7 @@ impl IndexStore { } } - fn _get_index(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result, Arc)>> { + fn get_index_txn(&self, txn: &RoTxn, name: impl AsRef) -> anyhow::Result, Arc)>> { match self.index_uuid(&txn, name)? { Some(uid) => self.retrieve_index(&txn, uid), None => Ok(None), @@ -124,7 +124,7 @@ impl IndexStore { pub fn index(&self, name: impl AsRef) -> anyhow::Result, Arc)>> { let txn = self.env.read_txn()?; - self._get_index(&txn, name) + self.get_index_txn(&txn, name) } pub fn get_or_create_index( @@ -134,7 +134,7 @@ impl IndexStore { index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { let mut txn = self.env.write_txn()?; - match self._get_index(&txn, name.as_ref())? { + match self.get_index_txn(&txn, name.as_ref())? { Some(res) => Ok(res), None => { let uuid = Uuid::new_v4(); @@ -168,9 +168,9 @@ impl IndexStore { update_size: u64, index_size: u64, ) -> anyhow::Result<(Arc, Arc)> { - let meta = IndexMeta { update_size, index_size, uuid: uuid.clone() }; + let meta = IndexMeta { update_store_size: update_size, index_store_size: index_size, uuid: uuid.clone() }; - self.name_to_uuid_db.put(txn, name.as_ref(), uuid.as_bytes())?; + self.name_to_uuid_meta.put(txn, name.as_ref(), uuid.as_bytes())?; self.uuid_to_index_db.put(txn, uuid.as_bytes(), &meta)?; let path = self.env.path(); @@ -247,7 +247,7 @@ mod test { // insert an uuid in the the name_to_uuid_db: let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); - store.name_to_uuid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store.name_to_uuid_meta.put(&mut txn, &name, uuid.as_bytes()).unwrap(); txn.commit().unwrap(); // check that the uuid is there @@ -264,7 +264,7 @@ mod test { let txn = store.env.read_txn().unwrap(); assert!(store.retrieve_index(&txn, uuid).unwrap().is_none()); - let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; + let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() }; let mut txn = store.env.write_txn().unwrap(); store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); txn.commit().unwrap(); @@ -286,9 +286,9 @@ mod test { assert!(store.index(&name).unwrap().is_none()); let uuid = Uuid::new_v4(); - let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; + let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() }; let mut txn = store.env.write_txn().unwrap(); - store.name_to_uuid_db.put(&mut txn, &name, uuid.as_bytes()).unwrap(); + store.name_to_uuid_meta.put(&mut txn, &name, uuid.as_bytes()).unwrap(); store.uuid_to_index_db.put(&mut txn, uuid.as_bytes(), &meta).unwrap(); txn.commit().unwrap(); @@ -303,11 +303,11 @@ mod test { store.get_or_create_index(&name, 4096 * 100, 4096 * 100).unwrap(); let txn = store.env.read_txn().unwrap(); - let uuid = store.name_to_uuid_db.get(&txn, &name).unwrap(); + let uuid = store.name_to_uuid_meta.get(&txn, &name).unwrap(); assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = IndexMeta { update_size: 4096 * 100, index_size: 4096 * 100, uuid: uuid.clone() }; + let meta = IndexMeta { update_store_size: 4096 * 100, index_store_size: 4096 * 100, uuid: uuid.clone() }; assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); } @@ -322,11 +322,11 @@ mod test { let uuid = Uuid::new_v4(); let mut txn = store.env.write_txn().unwrap(); store.create_index(&mut txn, uuid, name, update_size, index_size).unwrap(); - let uuid = store.name_to_uuid_db.get(&txn, &name).unwrap(); + let uuid = store.name_to_uuid_meta.get(&txn, &name).unwrap(); assert_eq!(store.uuid_to_index.len(), 1); assert!(uuid.is_some()); let uuid = Uuid::from_slice(uuid.unwrap()).unwrap(); - let meta = IndexMeta { update_size , index_size, uuid: uuid.clone() }; + let meta = IndexMeta { update_store_size: update_size , index_store_size: index_size, uuid: uuid.clone() }; assert_eq!(store.uuid_to_index_db.get(&txn, uuid.as_bytes()).unwrap(), Some(meta)); } } diff --git a/src/index_controller/mod.rs b/src/index_controller/mod.rs index 0ea654dfb..d348ee876 100644 --- a/src/index_controller/mod.rs +++ b/src/index_controller/mod.rs @@ -16,7 +16,6 @@ pub use updates::{Processed, Processing, Failed}; pub type UpdateStatus = updates::UpdateStatus; - #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type")] pub enum UpdateMeta { diff --git a/src/option.rs b/src/option.rs index dffa7d483..38f99880e 100644 --- a/src/option.rs +++ b/src/option.rs @@ -19,11 +19,11 @@ pub struct IndexerOpts { #[structopt(long, default_value = "100000")] // 100k pub log_every_n: usize, - /// MTBL max number of chunks in bytes. + /// Grenad max number of chunks in bytes. #[structopt(long)] pub max_nb_chunks: Option, - /// The maximum amount of memory to use for the MTBL buffer. It is recommended + /// The maximum amount of memory to use for the Grenad buffer. It is recommended /// to use something like 80%-90% of the available memory. /// /// It is automatically split by the number of jobs e.g. if you use 7 jobs @@ -37,7 +37,7 @@ pub struct IndexerOpts { pub linked_hash_map_size: usize, /// The name of the compression algorithm to use when compressing intermediate - /// chunks during indexing documents. + /// Grenad chunks while indexing documents. /// /// Choosing a fast algorithm will make the indexing faster but may consume more memory. #[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])] @@ -55,7 +55,7 @@ pub struct IndexerOpts { #[structopt(long, default_value = "4 GiB")] pub chunk_fusing_shrink_size: Byte, - /// Enable the chunk fusing or not, this reduces the amount of disk used by a factor of 2. + /// Enable the chunk fusing or not, this reduces the amount of disk space used. #[structopt(long)] pub enable_chunk_fusing: bool, @@ -68,13 +68,13 @@ pub struct IndexerOpts { impl Default for IndexerOpts { fn default() -> Self { Self { - log_every_n: 0, + log_every_n: 100_000, max_nb_chunks: None, - max_memory: Byte::from_str("0Kb").unwrap(), - linked_hash_map_size: 0, + max_memory: Byte::from_str("1GiB").unwrap(), + linked_hash_map_size: 500, chunk_compression_type: CompressionType::None, chunk_compression_level: None, - chunk_fusing_shrink_size: Byte::from_str("0Kb").unwrap(), + chunk_fusing_shrink_size: Byte::from_str("4GiB").unwrap(), enable_chunk_fusing: false, indexing_jobs: None, }