searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 20:37:15 +02:00
Code Issues Releases Wiki Activity

Introduce a rustfmt file

Browse source
This commit is contained in:
Clément Renault 2022-10-20 18:00:07 +02:00
parent 52e858a588
commit 80b2e70ee7
No known key found for this signature in database
GPG key ID: 92ADA4E935E71FA4
92 changed files with 1250 additions and 2855 deletions
Download patch file Download diff file Expand all files Collapse all files

87
dump/src/lib.rs
View file

@ -1,11 +1,9 @@
use meilisearch_types::{
error::ResponseError,
keys::Key,
milli::update::IndexDocumentsMethod,
settings::Unchecked,
tasks::{Details, KindWithContent, Status, Task, TaskId},
InstanceUid,
};
use meilisearch_types::error::ResponseError;
use meilisearch_types::keys::Key;
use meilisearch_types::milli::update::IndexDocumentsMethod;
use meilisearch_types::settings::Unchecked;
use meilisearch_types::tasks::{Details, KindWithContent, Status, Task, TaskId};
use meilisearch_types::InstanceUid;
use roaring::RoaringBitmap;
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
@ -168,15 +166,8 @@ impl From<KindWithContent> for KindDump {
}
KindWithContent::DocumentClear { .. } => KindDump::DocumentClear,
KindWithContent::Settings {
new_settings,
is_deletion,
allow_index_creation,
..
} => KindDump::Settings {
settings: new_settings,
is_deletion,
allow_index_creation,
},
new_settings, is_deletion, allow_index_creation, ..
} => KindDump::Settings { settings: new_settings, is_deletion, allow_index_creation },
KindWithContent::IndexDeletion { .. } => KindDump::IndexDeletion,
KindWithContent::IndexCreation { primary_key, .. } => {
KindDump::IndexCreation { primary_key }
@ -191,15 +182,9 @@ impl From<KindWithContent> for KindDump {
KindWithContent::TaskDeletion { query, tasks } => {
KindDump::TasksDeletion { query, tasks }
}
KindWithContent::DumpExport {
dump_uid,
keys,
instance_uid,
} => KindDump::DumpExport {
dump_uid,
keys,
instance_uid,
},
KindWithContent::DumpExport { dump_uid, keys, instance_uid } => {
KindDump::DumpExport { dump_uid, keys, instance_uid }
}
KindWithContent::Snapshot => KindDump::Snapshot,
}
}
@ -207,29 +192,25 @@ impl From<KindWithContent> for KindDump {
#[cfg(test)]
pub(crate) mod test {
use std::{
fs::File,
io::{Seek, SeekFrom},
str::FromStr,
};
use std::fs::File;
use std::io::{Seek, SeekFrom};
use std::str::FromStr;
use big_s::S;
use maplit::btreeset;
use meilisearch_types::index_uid::IndexUid;
use meilisearch_types::keys::{Action, Key};
use meilisearch_types::milli::{self, update::Setting};
use meilisearch_types::tasks::Status;
use meilisearch_types::{index_uid::IndexUid, star_or::StarOr};
use meilisearch_types::{
settings::{Checked, Settings},
tasks::Details,
};
use meilisearch_types::milli::update::Setting;
use meilisearch_types::milli::{self};
use meilisearch_types::settings::{Checked, Settings};
use meilisearch_types::star_or::StarOr;
use meilisearch_types::tasks::{Details, Status};
use serde_json::{json, Map, Value};
use time::macros::datetime;
use uuid::Uuid;
use crate::{
reader::Document, DumpReader, DumpWriter, IndexMetadata, KindDump, TaskDump, Version,
};
use crate::reader::Document;
use crate::{DumpReader, DumpWriter, IndexMetadata, KindDump, TaskDump, Version};
pub fn create_test_instance_uid() -> Uuid {
Uuid::parse_str("9e15e977-f2ae-4761-943f-1eaf75fd736d").unwrap()
@ -326,14 +307,8 @@ pub(crate) mod test {
finished_at: None,
},
Some(vec![
json!({ "id": 4, "race": "leonberg" })
.as_object()
.unwrap()
.clone(),
json!({ "id": 5, "race": "patou" })
.as_object()
.unwrap()
.clone(),
json!({ "id": 4, "race": "leonberg" }).as_object().unwrap().clone(),
json!({ "id": 5, "race": "patou" }).as_object().unwrap().clone(),
]),
),
(
@ -397,9 +372,7 @@ pub(crate) mod test {
let documents = create_test_documents();
let settings = create_test_settings();
let mut index = dump
.create_index("doggos", &create_test_index_metadata())
.unwrap();
let mut index = dump.create_index("doggos", &create_test_index_metadata()).unwrap();
for document in &documents {
index.push_document(document).unwrap();
}
@ -445,10 +418,7 @@ pub(crate) mod test {
// ==== checking the top level infos
assert_eq!(dump.version(), Version::V6);
assert!(dump.date().is_some());
assert_eq!(
dump.instance_uid().unwrap().unwrap(),
create_test_instance_uid()
);
assert_eq!(dump.instance_uid().unwrap().unwrap(), create_test_instance_uid());
// ==== checking the index
let mut indexes = dump.indexes().unwrap();
@ -475,10 +445,7 @@ pub(crate) mod test {
"A content file was expected for the task {}.",
expected.0.uid
);
let updates = content_file
.unwrap()
.collect::<Result<Vec<_>, _>>()
.unwrap();
let updates = content_file.unwrap().collect::<Result<Vec<_>, _>>().unwrap();
assert_eq!(updates, expected_update);
}
}

115
dump/src/reader/compat/v2_to_v3.rs
View file

@ -4,11 +4,10 @@ use std::str::FromStr;
use time::OffsetDateTime;
use uuid::Uuid;
use super::v3_to_v4::CompatV3ToV4;
use crate::reader::{v2, v3, Document};
use crate::Result;
use super::v3_to_v4::CompatV3ToV4;
pub struct CompatV2ToV3 {
pub from: v2::V2Reader,
}
@ -22,10 +21,7 @@ impl CompatV2ToV3 {
self.from
.index_uuid()
.into_iter()
.map(|index| v3::meta::IndexUuid {
uid: index.uid,
uuid: index.uuid,
})
.map(|index| v3::meta::IndexUuid { uid: index.uid, uuid: index.uuid })
.collect()
}
@ -65,10 +61,7 @@ impl CompatV2ToV3 {
.tasks()
.map(move |task| {
task.map(|(task, content_file)| {
let task = v3::Task {
uuid: task.uuid,
update: task.update.into(),
};
let task = v3::Task { uuid: task.uuid, update: task.update.into() };
Some((
task,
@ -216,22 +209,22 @@ impl TryFrom<(v2::updates::UpdateMeta, Option<Uuid>)> for v3::updates::Update {
fn try_from((update, uuid): (v2::updates::UpdateMeta, Option<Uuid>)) -> Result<Self> {
Ok(match update {
v2::updates::UpdateMeta::DocumentsAddition {
method,
format: _,
primary_key,
} if uuid.is_some() => v3::updates::Update::DocumentAddition {
primary_key,
method: match method {
v2::updates::IndexDocumentsMethod::ReplaceDocuments => {
v3::updates::IndexDocumentsMethod::ReplaceDocuments
}
v2::updates::IndexDocumentsMethod::UpdateDocuments => {
v3::updates::IndexDocumentsMethod::UpdateDocuments
}
},
content_uuid: uuid.unwrap(),
},
v2::updates::UpdateMeta::DocumentsAddition { method, format: _, primary_key }
if uuid.is_some() =>
{
v3::updates::Update::DocumentAddition {
primary_key,
method: match method {
v2::updates::IndexDocumentsMethod::ReplaceDocuments => {
v3::updates::IndexDocumentsMethod::ReplaceDocuments
}
v2::updates::IndexDocumentsMethod::UpdateDocuments => {
v3::updates::IndexDocumentsMethod::UpdateDocuments
}
},
content_uuid: uuid.unwrap(),
}
}
v2::updates::UpdateMeta::DocumentsAddition { .. } => {
return Err(crate::Error::MalformedTask)
}
@ -248,23 +241,21 @@ impl TryFrom<(v2::updates::UpdateMeta, Option<Uuid>)> for v3::updates::Update {
pub fn update_from_unchecked_update_meta(update: v2::updates::UpdateMeta) -> v3::updates::Update {
match update {
v2::updates::UpdateMeta::DocumentsAddition {
method,
format: _,
primary_key,
} => v3::updates::Update::DocumentAddition {
primary_key,
method: match method {
v2::updates::IndexDocumentsMethod::ReplaceDocuments => {
v3::updates::IndexDocumentsMethod::ReplaceDocuments
}
v2::updates::IndexDocumentsMethod::UpdateDocuments => {
v3::updates::IndexDocumentsMethod::UpdateDocuments
}
},
// we use this special uuid so we can recognize it if one day there is a bug related to this field.
content_uuid: Uuid::from_str("00112233-4455-6677-8899-aabbccddeeff").unwrap(),
},
v2::updates::UpdateMeta::DocumentsAddition { method, format: _, primary_key } => {
v3::updates::Update::DocumentAddition {
primary_key,
method: match method {
v2::updates::IndexDocumentsMethod::ReplaceDocuments => {
v3::updates::IndexDocumentsMethod::ReplaceDocuments
}
v2::updates::IndexDocumentsMethod::UpdateDocuments => {
v3::updates::IndexDocumentsMethod::UpdateDocuments
}
},
// we use this special uuid so we can recognize it if one day there is a bug related to this field.
content_uuid: Uuid::from_str("00112233-4455-6677-8899-aabbccddeeff").unwrap(),
}
}
v2::updates::UpdateMeta::ClearDocuments => v3::updates::Update::ClearDocuments,
v2::updates::UpdateMeta::DeleteDocuments { ids } => {
v3::updates::Update::DeleteDocuments(ids)
@ -354,10 +345,7 @@ impl<T> From<v2::Settings<T>> for v3::Settings<v3::Unchecked> {
.map(|f| f.into_iter().collect()),
sortable_attributes: v3::Setting::NotSet,
ranking_rules: option_to_setting(settings.ranking_rules).map(|criteria| {
criteria
.into_iter()
.map(|criterion| patch_ranking_rules(&criterion))
.collect()
criteria.into_iter().map(|criterion| patch_ranking_rules(&criterion)).collect()
}),
stop_words: option_to_setting(settings.stop_words),
synonyms: option_to_setting(settings.synonyms),
@ -383,7 +371,8 @@ fn patch_ranking_rules(ranking_rule: &str) -> String {
#[cfg(test)]
pub(crate) mod test {
use std::{fs::File, io::BufReader};
use std::fs::File;
use std::io::BufReader;
use flate2::bufread::GzDecoder;
use tempfile::TempDir;
@ -412,11 +401,7 @@ pub(crate) mod test {
assert!(update_files[0].is_some()); // the enqueued document addition
assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
let update_file = update_files
.remove(0)
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let update_file = update_files.remove(0).unwrap().collect::<Result<Vec<_>>>().unwrap();
meili_snap::snapshot_hash!(meili_snap::json_string!(update_file), @"7b8889539b669c7b9ddba448bafa385d");
// indexes
@ -441,11 +426,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"f43338ecceeddd1ce13ffd55438b2347");
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
@ -460,11 +441,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"0d76c745cb334e8c20d6d6a14df733e1");
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 110);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
@ -479,11 +456,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @"09a2f7c571729f70f4cd93e24e8e3f28");
let documents = movies2
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 0);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
@ -498,11 +471,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"09a2f7c571729f70f4cd93e24e8e3f28");
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
}

57
dump/src/reader/compat/v3_to_v4.rs
View file

@ -1,8 +1,7 @@
use crate::reader::{v3, v4, UpdateFile};
use crate::Result;
use super::v2_to_v3::{CompatIndexV2ToV3, CompatV2ToV3};
use super::v4_to_v5::CompatV4ToV5;
use crate::reader::{v3, v4, UpdateFile};
use crate::Result;
pub enum CompatV3ToV4 {
V3(v3::V3Reader),
@ -38,18 +37,15 @@ impl CompatV3ToV4 {
pub fn indexes(&self) -> Result<impl Iterator<Item = Result<CompatIndexV3ToV4>> + '_> {
Ok(match self {
CompatV3ToV4::V3(v3) => Box::new(
v3.indexes()?
.map(|index| index.map(CompatIndexV3ToV4::from)),
)
as Box<dyn Iterator<Item = Result<CompatIndexV3ToV4>> + '_>,
CompatV3ToV4::V3(v3) => {
Box::new(v3.indexes()?.map(|index| index.map(CompatIndexV3ToV4::from)))
as Box<dyn Iterator<Item = Result<CompatIndexV3ToV4>> + '_>
}
CompatV3ToV4::Compat(compat) => Box::new(
compat
.indexes()?
.map(|index| index.map(CompatIndexV3ToV4::from)),
)
as Box<dyn Iterator<Item = Result<CompatIndexV3ToV4>> + '_>,
CompatV3ToV4::Compat(compat) => {
Box::new(compat.indexes()?.map(|index| index.map(CompatIndexV3ToV4::from)))
as Box<dyn Iterator<Item = Result<CompatIndexV3ToV4>> + '_>
}
})
}
@ -341,7 +337,8 @@ impl<T> From<v3::Settings<T>> for v4::Settings<v4::Unchecked> {
#[cfg(test)]
pub(crate) mod test {
use std::{fs::File, io::BufReader};
use std::fs::File;
use std::io::BufReader;
use flate2::bufread::GzDecoder;
use tempfile::TempDir;
@ -370,11 +367,7 @@ pub(crate) mod test {
assert!(update_files[0].is_some()); // the enqueued document addition
assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
let update_file = update_files
.remove(0)
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let update_file = update_files.remove(0).unwrap().collect::<Result<Vec<_>>>().unwrap();
meili_snap::snapshot_hash!(meili_snap::json_string!(update_file), @"7b8889539b669c7b9ddba448bafa385d");
// keys
@ -403,11 +396,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"ea46dd6b58c5e1d65c1c8159a32695ea");
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
@ -422,11 +411,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"4df4074ef6bfb71e8dc66d08ff8c9dfd");
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 110);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
@ -441,11 +426,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @"24eaf4046d9718dabff36f35103352d4");
let documents = movies2
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 0);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
@ -460,11 +441,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"24eaf4046d9718dabff36f35103352d4");
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
}

61
dump/src/reader/compat/v4_to_v5.rs
View file

@ -1,8 +1,7 @@
use crate::reader::{v4, v5, Document};
use crate::Result;
use super::v3_to_v4::{CompatIndexV3ToV4, CompatV3ToV4};
use super::v5_to_v6::CompatV5ToV6;
use crate::reader::{v4, v5, Document};
use crate::Result;
pub enum CompatV4ToV5 {
V4(v4::V4Reader),
@ -41,18 +40,15 @@ impl CompatV4ToV5 {
pub fn indexes(&self) -> Result<Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>> {
Ok(match self {
CompatV4ToV5::V4(v4) => Box::new(
v4.indexes()?
.map(|index| index.map(CompatIndexV4ToV5::from)),
)
as Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>,
CompatV4ToV5::V4(v4) => {
Box::new(v4.indexes()?.map(|index| index.map(CompatIndexV4ToV5::from)))
as Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>
}
CompatV4ToV5::Compat(compat) => Box::new(
compat
.indexes()?
.map(|index| index.map(CompatIndexV4ToV5::from)),
)
as Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>,
CompatV4ToV5::Compat(compat) => {
Box::new(compat.indexes()?.map(|index| index.map(CompatIndexV4ToV5::from)))
as Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>
}
})
}
@ -138,13 +134,9 @@ impl CompatV4ToV5 {
v4::tasks::TaskEvent::Created(date) => {
v5::tasks::TaskEvent::Created(date)
}
v4::tasks::TaskEvent::Batched {
timestamp,
batch_id,
} => v5::tasks::TaskEvent::Batched {
timestamp,
batch_id,
},
v4::tasks::TaskEvent::Batched { timestamp, batch_id } => {
v5::tasks::TaskEvent::Batched { timestamp, batch_id }
}
v4::tasks::TaskEvent::Processing(date) => {
v5::tasks::TaskEvent::Processing(date)
}
@ -196,11 +188,7 @@ impl CompatV4ToV5 {
description: key.description,
name: None,
uid: v5::keys::KeyId::new_v4(),
actions: key
.actions
.into_iter()
.filter_map(|action| action.into())
.collect(),
actions: key.actions.into_iter().filter_map(|action| action.into()).collect(),
indexes: key
.indexes
.into_iter()
@ -385,7 +373,8 @@ impl From<v4::Action> for Option<v5::Action> {
#[cfg(test)]
pub(crate) mod test {
use std::{fs::File, io::BufReader};
use std::fs::File;
use std::io::BufReader;
use flate2::bufread::GzDecoder;
use tempfile::TempDir;
@ -440,11 +429,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"ed1a6977a832b1ab49cd5068b77ce498");
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
@ -459,11 +444,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"70681af1d52411218036fbd5a9b94ab5");
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 110);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab");
@ -478,11 +459,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"7019bb8f146004dcdd91fc3c3254b742");
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
}

68
dump/src/reader/compat/v5_to_v6.rs
View file

@ -1,8 +1,7 @@
use super::v4_to_v5::{CompatIndexV4ToV5, CompatV4ToV5};
use crate::reader::{v5, v6, Document, UpdateFile};
use crate::Result;
use super::v4_to_v5::{CompatIndexV4ToV5, CompatV4ToV5};
pub enum CompatV5ToV6 {
V5(v5::V5Reader),
Compat(CompatV4ToV5),
@ -36,18 +35,15 @@ impl CompatV5ToV6 {
pub fn indexes(&self) -> Result<Box<dyn Iterator<Item = Result<CompatIndexV5ToV6>> + '_>> {
let indexes = match self {
CompatV5ToV6::V5(v5) => Box::new(
v5.indexes()?
.map(|index| index.map(CompatIndexV5ToV6::from)),
)
as Box<dyn Iterator<Item = Result<CompatIndexV5ToV6>> + '_>,
CompatV5ToV6::V5(v5) => {
Box::new(v5.indexes()?.map(|index| index.map(CompatIndexV5ToV6::from)))
as Box<dyn Iterator<Item = Result<CompatIndexV5ToV6>> + '_>
}
CompatV5ToV6::Compat(compat) => Box::new(
compat
.indexes()?
.map(|index| index.map(CompatIndexV5ToV6::from)),
)
as Box<dyn Iterator<Item = Result<CompatIndexV5ToV6>> + '_>,
CompatV5ToV6::Compat(compat) => {
Box::new(compat.indexes()?.map(|index| index.map(CompatIndexV5ToV6::from)))
as Box<dyn Iterator<Item = Result<CompatIndexV5ToV6>> + '_>
}
};
Ok(indexes)
}
@ -127,16 +123,15 @@ impl CompatV5ToV6 {
},
canceled_by: None,
details: task_view.details.map(|details| match details {
v5::Details::DocumentAddition {
received_documents,
indexed_documents,
} => v6::Details::DocumentAddition {
received_documents: received_documents as u64,
indexed_documents: indexed_documents.map(|i| i as u64),
},
v5::Details::Settings { settings } => v6::Details::Settings {
settings: settings.into(),
},
v5::Details::DocumentAddition { received_documents, indexed_documents } => {
v6::Details::DocumentAddition {
received_documents: received_documents as u64,
indexed_documents: indexed_documents.map(|i| i as u64),
}
}
v5::Details::Settings { settings } => {
v6::Details::Settings { settings: settings.into() }
}
v5::Details::IndexInfo { primary_key } => {
v6::Details::IndexInfo { primary_key }
}
@ -174,11 +169,7 @@ impl CompatV5ToV6 {
description: key.description,
name: key.name,
uid: key.uid,
actions: key
.actions
.into_iter()
.map(|action| action.into())
.collect(),
actions: key.actions.into_iter().map(|action| action.into()).collect(),
indexes: key
.indexes
.into_iter()
@ -396,7 +387,8 @@ impl From<v5::Action> for v6::Action {
#[cfg(test)]
pub(crate) mod test {
use std::{fs::File, io::BufReader};
use std::fs::File;
use std::io::BufReader;
use flate2::bufread::GzDecoder;
use tempfile::TempDir;
@ -452,11 +444,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"9896a66a399c24a0f4f6a3c8563cd14a");
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
@ -471,11 +459,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"d0dc7efd1360f95fce57d7931a70b7c9");
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 200);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"e962baafd2fbae4cdd14e876053b0c5a");
@ -490,11 +474,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"59c8e30c2022897987ea7b4394167b06");
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
}

103
dump/src/reader/mod.rs
View file

@ -1,5 +1,9 @@
use std::io::Read;
use std::{fs::File, io::BufReader};
use std::fs::File;
use std::io::{BufReader, Read};
use flate2::bufread::GzDecoder;
use serde::Deserialize;
use tempfile::TempDir;
use self::compat::v4_to_v5::CompatV4ToV5;
use self::compat::v5_to_v6::{CompatIndexV5ToV6, CompatV5ToV6};
@ -7,10 +11,6 @@ use self::v5::V5Reader;
use self::v6::{V6IndexReader, V6Reader};
use crate::{Error, Result, Version};
use flate2::bufread::GzDecoder;
use serde::Deserialize;
use tempfile::TempDir;
mod compat;
// pub(self) mod v1;
@ -47,12 +47,7 @@ impl DumpReader {
match dump_version {
// Version::V1 => Ok(Box::new(v1::Reader::open(path)?)),
Version::V1 => Err(Error::DumpV1Unsupported),
Version::V2 => Ok(v2::V2Reader::open(path)?
.to_v3()
.to_v4()
.to_v5()
.to_v6()
.into()),
Version::V2 => Ok(v2::V2Reader::open(path)?.to_v3().to_v4().to_v5().to_v6().into()),
Version::V3 => Ok(v3::V3Reader::open(path)?.to_v4().to_v5().to_v6().into()),
Version::V4 => Ok(v4::V4Reader::open(path)?.to_v5().to_v6().into()),
Version::V5 => Ok(v5::V5Reader::open(path)?.to_v6().into()),
@ -234,11 +229,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"9896a66a399c24a0f4f6a3c8563cd14a");
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
@ -253,11 +244,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"d0dc7efd1360f95fce57d7931a70b7c9");
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 200);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"e962baafd2fbae4cdd14e876053b0c5a");
@ -272,11 +259,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"59c8e30c2022897987ea7b4394167b06");
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
}
@ -323,11 +306,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"ed1a6977a832b1ab49cd5068b77ce498");
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
@ -342,11 +321,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"70681af1d52411218036fbd5a9b94ab5");
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 110);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab");
@ -361,11 +336,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"7019bb8f146004dcdd91fc3c3254b742");
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
}
@ -413,11 +384,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"1a5ed16d00e6163662d9d7ffe400c5d0");
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
@ -432,11 +399,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"9a6b511669b8f53d193d2f0bd1671baa");
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 110);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
@ -451,11 +414,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @"4fdf905496d9a511800ff523728728ac");
let documents = movies2
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 0);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
@ -470,11 +429,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"4fdf905496d9a511800ff523728728ac");
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
}
@ -522,11 +477,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"a7d4fed93bfc91d0f1126d3371abf48e");
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
@ -541,11 +492,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"e79c3cc4eef44bd22acfb60957b459d9");
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 110);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
@ -560,11 +507,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @"7917f954b6f345336073bb155540ad6d");
let documents = movies2
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 0);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
@ -579,11 +522,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"7917f954b6f345336073bb155540ad6d");
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
}

59
dump/src/reader/v2/mod.rs
View file

@ -22,11 +22,9 @@
//! └── update_202573df-718b-4d80-9a65-2ee397c23dc3
//! ```
use std::{
fs::{self, File},
io::{BufRead, BufReader},
path::Path,
};
use std::fs::{self, File};
use std::io::{BufRead, BufReader};
use std::path::Path;
use serde::{Deserialize, Serialize};
use tempfile::TempDir;
@ -37,11 +35,10 @@ pub mod meta;
pub mod settings;
pub mod updates;
use crate::{IndexMetadata, Result, Version};
use self::meta::{DumpMeta, IndexUuid};
use super::{compat::v2_to_v3::CompatV2ToV3, Document};
use super::compat::v2_to_v3::CompatV2ToV3;
use super::Document;
use crate::{IndexMetadata, Result, Version};
pub type Settings<T> = settings::Settings<T>;
pub type Checked = settings::Checked;
@ -110,11 +107,7 @@ impl V2Reader {
Ok(self.index_uuid.iter().map(|index| -> Result<_> {
Ok(V2IndexReader::new(
index.uid.clone(),
&self
.dump
.path()
.join("indexes")
.join(format!("index-{}", index.uuid.to_string())),
&self.dump.path().join("indexes").join(format!("index-{}", index.uuid.to_string())),
)?)
}))
}
@ -193,10 +186,7 @@ pub struct UpdateFile {
impl UpdateFile {
fn new(path: &Path) -> Result<Self> {
let reader = BufReader::new(File::open(path)?);
Ok(UpdateFile {
documents: serde_json::from_reader(reader)?,
index: 0,
})
Ok(UpdateFile { documents: serde_json::from_reader(reader)?, index: 0 })
}
}
@ -211,7 +201,8 @@ impl Iterator for UpdateFile {
#[cfg(test)]
pub(crate) mod test {
use std::{fs::File, io::BufReader};
use std::fs::File;
use std::io::BufReader;
use flate2::bufread::GzDecoder;
use tempfile::TempDir;
@ -240,11 +231,7 @@ pub(crate) mod test {
assert!(update_files[0].is_some()); // the enqueued document addition
assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
let update_file = update_files
.remove(0)
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let update_file = update_files.remove(0).unwrap().collect::<Result<Vec<_>>>().unwrap();
meili_snap::snapshot_hash!(meili_snap::json_string!(update_file), @"7b8889539b669c7b9ddba448bafa385d");
// indexes
@ -269,11 +256,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"b4814eab5e73e2dcfc90aad50aa583d1");
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
@ -288,11 +271,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"59dd69f590635a58f3d99edc9e1fa21f");
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 110);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
@ -307,11 +286,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @"ac041085004c43373fe90dc48f5c23ab");
let documents = movies2
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 0);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
@ -326,11 +301,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"ac041085004c43373fe90dc48f5c23ab");
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
}

13
dump/src/reader/v2/settings.rs
View file

@ -1,8 +1,6 @@
use std::{
collections::{BTreeMap, BTreeSet, HashSet},
marker::PhantomData,
str::FromStr,
};
use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::marker::PhantomData;
use std::str::FromStr;
use once_cell::sync::Lazy;
use regex::Regex;
@ -39,10 +37,7 @@ pub struct Unchecked;
#[cfg_attr(test, derive(serde::Serialize))]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
#[serde(bound(
serialize = "T: serde::Serialize",
deserialize = "T: Deserialize<'static>"
))]
#[serde(bound(serialize = "T: serde::Serialize", deserialize = "T: Deserialize<'static>"))]
pub struct Settings<T> {
#[serde(
default,

58
dump/src/reader/v3/mod.rs
View file

@ -22,11 +22,9 @@
//! └── 66d3f12d-fcf3-4b53-88cb-407017373de7
//! ```
use std::{
fs::{self, File},
io::{BufRead, BufReader},
path::Path,
};
use std::fs::{self, File};
use std::io::{BufRead, BufReader};
use std::path::Path;
use serde::{Deserialize, Serialize};
use tempfile::TempDir;
@ -37,11 +35,10 @@ pub mod meta;
pub mod settings;
pub mod updates;
use crate::{Error, IndexMetadata, Result, Version};
use self::meta::{DumpMeta, IndexUuid};
use super::{compat::v3_to_v4::CompatV3ToV4, Document};
use super::compat::v3_to_v4::CompatV3ToV4;
use super::Document;
use crate::{Error, IndexMetadata, Result, Version};
pub type Settings<T> = settings::Settings<T>;
pub type Checked = settings::Checked;
@ -116,11 +113,7 @@ impl V3Reader {
Ok(self.index_uuid.iter().map(|index| -> Result<_> {
Ok(V3IndexReader::new(
index.uid.clone(),
&self
.dump
.path()
.join("indexes")
.join(index.uuid.to_string()),
&self.dump.path().join("indexes").join(index.uuid.to_string()),
)?)
}))
}
@ -204,9 +197,7 @@ pub struct UpdateFile {
impl UpdateFile {
fn new(path: &Path) -> Result<Self> {
Ok(UpdateFile {
reader: BufReader::new(File::open(path)?),
})
Ok(UpdateFile { reader: BufReader::new(File::open(path)?) })
}
}
@ -226,7 +217,8 @@ impl Iterator for UpdateFile {
#[cfg(test)]
pub(crate) mod test {
use std::{fs::File, io::BufReader};
use std::fs::File;
use std::io::BufReader;
use flate2::bufread::GzDecoder;
use tempfile::TempDir;
@ -255,11 +247,7 @@ pub(crate) mod test {
assert!(update_files[0].is_some()); // the enqueued document addition
assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
let update_file = update_files
.remove(0)
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let update_file = update_files.remove(0).unwrap().collect::<Result<Vec<_>>>().unwrap();
meili_snap::snapshot_hash!(meili_snap::json_string!(update_file), @"7b8889539b669c7b9ddba448bafa385d");
// indexes
@ -284,11 +272,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"7460d4b242b5c8b1bda223f63bbbf349");
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
@ -303,11 +287,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"d83ab8e79bb44595667d6ce3e6629a4f");
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 110);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
@ -322,11 +302,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @"44d3b5a3b3aa6cd950373ff751d05bb7");
let documents = movies2
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 0);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
@ -341,11 +317,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"44d3b5a3b3aa6cd950373ff751d05bb7");
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
}

13
dump/src/reader/v3/settings.rs
View file

@ -1,8 +1,6 @@
use std::{
collections::{BTreeMap, BTreeSet},
marker::PhantomData,
num::NonZeroUsize,
};
use std::collections::{BTreeMap, BTreeSet};
use std::marker::PhantomData;
use std::num::NonZeroUsize;
use serde::{Deserialize, Deserializer};
@ -40,10 +38,7 @@ pub struct Unchecked;
#[cfg_attr(test, derive(serde::Serialize))]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
#[serde(bound(
serialize = "T: serde::Serialize",
deserialize = "T: Deserialize<'static>"
))]
#[serde(bound(serialize = "T: serde::Serialize", deserialize = "T: Deserialize<'static>"))]
pub struct Settings<T> {
#[serde(
default,

30
dump/src/reader/v4/errors.rs
View file

@ -8,10 +8,7 @@ use serde::{Deserialize, Serialize};
#[cfg_attr(feature = "test-traits", derive(proptest_derive::Arbitrary))]
pub struct ResponseError {
#[serde(skip)]
#[cfg_attr(
feature = "test-traits",
proptest(strategy = "strategy::status_code_strategy()")
)]
#[cfg_attr(feature = "test-traits", proptest(strategy = "strategy::status_code_strategy()"))]
pub code: StatusCode,
pub message: String,
#[serde(rename = "code")]
@ -206,10 +203,9 @@ impl Code {
BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST),
BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST),
DatabaseSizeLimitReached => ErrCode::internal(
"database_size_limit_reached",
StatusCode::INTERNAL_SERVER_ERROR,
),
DatabaseSizeLimitReached => {
ErrCode::internal("database_size_limit_reached", StatusCode::INTERNAL_SERVER_ERROR)
}
DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND),
Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR),
InvalidGeoField => ErrCode::invalid("invalid_geo_field", StatusCode::BAD_REQUEST),
@ -302,26 +298,14 @@ struct ErrCode {
impl ErrCode {
fn authentication(error_name: &'static str, status_code: StatusCode) -> ErrCode {
ErrCode {
status_code,
error_name,
error_type: ErrorType::AuthenticationError,
}
ErrCode { status_code, error_name, error_type: ErrorType::AuthenticationError }
}
fn internal(error_name: &'static str, status_code: StatusCode) -> ErrCode {
ErrCode {
status_code,
error_name,
error_type: ErrorType::InternalError,
}
ErrCode { status_code, error_name, error_type: ErrorType::InternalError }
}
fn invalid(error_name: &'static str, status_code: StatusCode) -> ErrCode {
ErrCode {
status_code,
error_name,
error_type: ErrorType::InvalidRequestError,
}
ErrCode { status_code, error_name, error_type: ErrorType::InvalidRequestError }
}
}

15
dump/src/reader/v4/meta.rs
View file

@ -1,10 +1,9 @@
use std::{
fmt::{self, Display, Formatter},
marker::PhantomData,
str::FromStr,
};
use std::fmt::{self, Display, Formatter};
use std::marker::PhantomData;
use std::str::FromStr;
use serde::{de::Visitor, Deserialize, Deserializer};
use serde::de::Visitor;
use serde::{Deserialize, Deserializer};
use uuid::Uuid;
use super::settings::{Settings, Unchecked};
@ -39,9 +38,7 @@ impl TryFrom<String> for IndexUid {
type Error = IndexUidFormatError;
fn try_from(uid: String) -> Result<Self, Self::Error> {
if !uid
.chars()
.all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
if !uid.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
|| uid.is_empty()
|| uid.len() > 400
{

53
dump/src/reader/v4/mod.rs
View file

@ -1,8 +1,6 @@
use std::{
fs::{self, File},
io::{BufRead, BufReader},
path::Path,
};
use std::fs::{self, File};
use std::io::{BufRead, BufReader};
use std::path::Path;
use serde::{Deserialize, Serialize};
use tempfile::TempDir;
@ -15,11 +13,9 @@ pub mod meta;
pub mod settings;
pub mod tasks;
use crate::{Error, IndexMetadata, Result, Version};
use self::meta::{DumpMeta, IndexUuid};
use super::compat::v4_to_v5::CompatV4ToV5;
use crate::{Error, IndexMetadata, Result, Version};
pub type Document = serde_json::Map<String, serde_json::Value>;
pub type Settings<T> = settings::Settings<T>;
@ -100,11 +96,7 @@ impl V4Reader {
Ok(self.index_uuid.iter().map(|index| -> Result<_> {
Ok(V4IndexReader::new(
index.uid.clone(),
&self
.dump
.path()
.join("indexes")
.join(index.index_meta.uuid.to_string()),
&self.dump.path().join("indexes").join(index.index_meta.uuid.to_string()),
)?)
}))
}
@ -139,9 +131,7 @@ impl V4Reader {
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<Key>> + '_> {
Box::new(
(&mut self.keys)
.lines()
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
(&mut self.keys).lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
)
}
}
@ -196,9 +186,7 @@ pub struct UpdateFile {
impl UpdateFile {
fn new(path: &Path) -> Result<Self> {
Ok(UpdateFile {
reader: BufReader::new(File::open(path)?),
})
Ok(UpdateFile { reader: BufReader::new(File::open(path)?) })
}
}
@ -218,7 +206,8 @@ impl Iterator for UpdateFile {
#[cfg(test)]
pub(crate) mod test {
use std::{fs::File, io::BufReader};
use std::fs::File;
use std::io::BufReader;
use flate2::bufread::GzDecoder;
use tempfile::TempDir;
@ -248,11 +237,7 @@ pub(crate) mod test {
assert!(update_files[0].is_some()); // the enqueued document addition
assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
let update_file = update_files
.remove(0)
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let update_file = update_files.remove(0).unwrap().collect::<Result<Vec<_>>>().unwrap();
meili_snap::snapshot_hash!(meili_snap::json_string!(update_file), @"7b8889539b669c7b9ddba448bafa385d");
// keys
@ -280,11 +265,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"ace6546a6eb856ecb770b2409975c01d");
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
@ -299,11 +280,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"4dfa34fa34f2c03259482e1e4555faa8");
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 110);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab");
@ -318,11 +295,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"1aa241a5e3afd8c85a4e7b9db42362d7");
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
}

13
dump/src/reader/v4/settings.rs
View file

@ -1,8 +1,6 @@
use std::{
collections::{BTreeMap, BTreeSet},
marker::PhantomData,
num::NonZeroUsize,
};
use std::collections::{BTreeMap, BTreeSet};
use std::marker::PhantomData;
use std::num::NonZeroUsize;
use serde::{Deserialize, Deserializer};
@ -65,10 +63,7 @@ pub struct TypoSettings {
#[cfg_attr(test, derive(serde::Serialize))]
#[serde(deny_unknown_fields)]
#[serde(rename_all = "camelCase")]
#[serde(bound(
serialize = "T: serde::Serialize",
deserialize = "T: Deserialize<'static>"
))]
#[serde(bound(serialize = "T: serde::Serialize", deserialize = "T: Deserialize<'static>"))]
pub struct Settings<T> {
#[serde(
default,

15
dump/src/reader/v4/tasks.rs
View file

@ -2,11 +2,9 @@ use serde::Deserialize;
use time::OffsetDateTime;
use uuid::Uuid;
use super::{
errors::ResponseError,
meta::IndexUid,
settings::{Settings, Unchecked},
};
use super::errors::ResponseError;
use super::meta::IndexUid;
use super::settings::{Settings, Unchecked};
pub type TaskId = u32;
pub type BatchId = u32;
@ -109,10 +107,9 @@ impl Task {
/// Return the content_uuid of the `Task` if there is one.
pub fn get_content_uuid(&self) -> Option<Uuid> {
match self {
Task {
content: TaskContent::DocumentAddition { content_uuid, .. },
..
} => Some(*content_uuid),
Task { content: TaskContent::DocumentAddition { content_uuid, .. }, .. } => {
Some(*content_uuid)
}
_ => None,
}
}

25
dump/src/reader/v5/errors.rs
View file

@ -142,10 +142,9 @@ impl Code {
BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST),
BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST),
DatabaseSizeLimitReached => ErrCode::internal(
"database_size_limit_reached",
StatusCode::INTERNAL_SERVER_ERROR,
),
DatabaseSizeLimitReached => {
ErrCode::internal("database_size_limit_reached", StatusCode::INTERNAL_SERVER_ERROR)
}
DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND),
Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR),
InvalidGeoField => ErrCode::invalid("invalid_geo_field", StatusCode::BAD_REQUEST),
@ -241,27 +240,15 @@ struct ErrCode {
impl ErrCode {
fn authentication(error_name: &'static str, status_code: StatusCode) -> ErrCode {
ErrCode {
status_code,
error_name,
error_type: ErrorType::AuthenticationError,
}
ErrCode { status_code, error_name, error_type: ErrorType::AuthenticationError }
}
fn internal(error_name: &'static str, status_code: StatusCode) -> ErrCode {
ErrCode {
status_code,
error_name,
error_type: ErrorType::InternalError,
}
ErrCode { status_code, error_name, error_type: ErrorType::InternalError }
}
fn invalid(error_name: &'static str, status_code: StatusCode) -> ErrCode {
ErrCode {
status_code,
error_name,
error_type: ErrorType::InvalidRequestError,
}
ErrCode { status_code, error_name, error_type: ErrorType::InvalidRequestError }
}
}

15
dump/src/reader/v5/meta.rs
View file

@ -1,10 +1,9 @@
use std::{
fmt::{self, Display, Formatter},
marker::PhantomData,
str::FromStr,
};
use std::fmt::{self, Display, Formatter};
use std::marker::PhantomData;
use std::str::FromStr;
use serde::{de::Visitor, Deserialize, Deserializer};
use serde::de::Visitor;
use serde::{Deserialize, Deserializer};
use uuid::Uuid;
use super::settings::{Settings, Unchecked};
@ -39,9 +38,7 @@ impl TryFrom<String> for IndexUid {
type Error = IndexUidFormatError;
fn try_from(uid: String) -> Result<Self, Self::Error> {
if !uid
.chars()
.all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
if !uid.chars().all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
|| uid.is_empty()
|| uid.len() > 400
{

55
dump/src/reader/v5/mod.rs
View file

@ -32,21 +32,19 @@
//! ```
//!
use std::{
fs::{self, File},
io::{BufRead, BufReader, Seek, SeekFrom},
path::Path,
};
use std::fs::{self, File};
use std::io::{BufRead, BufReader, Seek, SeekFrom};
use std::path::Path;
use serde::{Deserialize, Serialize};
use tempfile::TempDir;
use time::OffsetDateTime;
use uuid::Uuid;
use super::compat::v5_to_v6::CompatV5ToV6;
use super::Document;
use crate::{Error, IndexMetadata, Result, Version};
use super::{compat::v5_to_v6::CompatV5ToV6, Document};
pub mod errors;
pub mod keys;
pub mod meta;
@ -139,11 +137,7 @@ impl V5Reader {
Ok(self.index_uuid.iter().map(|index| -> Result<_> {
Ok(V5IndexReader::new(
index.uid.clone(),
&self
.dump
.path()
.join("indexes")
.join(index.index_meta.uuid.to_string()),
&self.dump.path().join("indexes").join(index.index_meta.uuid.to_string()),
)?)
}))
}
@ -178,9 +172,9 @@ impl V5Reader {
pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<Key>> + '_>> {
self.keys.seek(SeekFrom::Start(0))?;
Ok(Box::new((&mut self.keys).lines().map(
|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) },
)))
Ok(Box::new(
(&mut self.keys).lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
))
}
}
@ -234,9 +228,7 @@ pub struct UpdateFile {
impl UpdateFile {
fn new(path: &Path) -> Result<Self> {
Ok(UpdateFile {
reader: BufReader::new(File::open(path)?),
})
Ok(UpdateFile { reader: BufReader::new(File::open(path)?) })
}
}
@ -256,7 +248,8 @@ impl Iterator for UpdateFile {
#[cfg(test)]
pub(crate) mod test {
use std::{fs::File, io::BufReader};
use std::fs::File;
use std::io::BufReader;
use flate2::bufread::GzDecoder;
use tempfile::TempDir;
@ -287,11 +280,7 @@ pub(crate) mod test {
assert!(update_files[1].is_some()); // the enqueued document addition
assert!(update_files[2..].iter().all(|u| u.is_none())); // everything already processed
let update_file = update_files
.remove(1)
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let update_file = update_files.remove(1).unwrap().collect::<Result<Vec<_>>>().unwrap();
meili_snap::snapshot_hash!(meili_snap::json_string!(update_file), @"7b8889539b669c7b9ddba448bafa385d");
// keys
@ -319,11 +308,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"9896a66a399c24a0f4f6a3c8563cd14a");
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
@ -338,11 +323,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"d0dc7efd1360f95fce57d7931a70b7c9");
let documents = movies
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 200);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"e962baafd2fbae4cdd14e876053b0c5a");
@ -357,11 +338,7 @@ pub(crate) mod test {
"###);
meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"59c8e30c2022897987ea7b4394167b06");
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
assert_eq!(documents.len(), 10);
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
}

6
dump/src/reader/v5/settings.rs
View file

@ -1,7 +1,5 @@
use std::{
collections::{BTreeMap, BTreeSet},
marker::PhantomData,
};
use std::collections::{BTreeMap, BTreeSet};
use std::marker::PhantomData;
use serde::{Deserialize, Deserializer, Serialize};

144
dump/src/reader/v5/tasks.rs
View file

@ -2,11 +2,9 @@ use serde::Deserialize;
use time::{Duration, OffsetDateTime};
use uuid::Uuid;
use super::{
errors::ResponseError,
meta::IndexUid,
settings::{Settings, Unchecked},
};
use super::errors::ResponseError;
use super::meta::IndexUid;
use super::settings::{Settings, Unchecked};
pub type TaskId = u32;
pub type BatchId = u32;
@ -117,20 +115,16 @@ impl Task {
/// A task is finished when its last state is either `Succeeded` or `Failed`.
pub fn is_finished(&self) -> bool {
self.events.last().map_or(false, |event| {
matches!(
event,
TaskEvent::Succeeded { .. } | TaskEvent::Failed { .. }
)
matches!(event, TaskEvent::Succeeded { .. } | TaskEvent::Failed { .. })
})
}
/// Return the content_uuid of the `Task` if there is one.
pub fn get_content_uuid(&self) -> Option<Uuid> {
match self {
Task {
content: TaskContent::DocumentAddition { content_uuid, .. },
..
} => Some(*content_uuid),
Task { content: TaskContent::DocumentAddition { content_uuid, .. }, .. } => {
Some(*content_uuid)
}
_ => None,
}
}
@ -184,31 +178,19 @@ pub struct TaskView {
pub duration: Option<Duration>,
#[cfg_attr(test, serde(serialize_with = "time::serde::rfc3339::serialize"))]
pub enqueued_at: OffsetDateTime,
#[cfg_attr(
test,
serde(serialize_with = "time::serde::rfc3339::option::serialize")
)]
#[cfg_attr(test, serde(serialize_with = "time::serde::rfc3339::option::serialize"))]
pub started_at: Option<OffsetDateTime>,
#[cfg_attr(
test,
serde(serialize_with = "time::serde::rfc3339::option::serialize")
)]
#[cfg_attr(test, serde(serialize_with = "time::serde::rfc3339::option::serialize"))]
pub finished_at: Option<OffsetDateTime>,
}
impl From<Task> for TaskView {
fn from(task: Task) -> Self {
let index_uid = task.index_uid().map(String::from);
let Task {
id,
content,
events,
} = task;
let Task { id, content, events } = task;
let (task_type, mut details) = match content {
TaskContent::DocumentAddition {
documents_count, ..
} => {
TaskContent::DocumentAddition { documents_count, .. } => {
let details = TaskDetails::DocumentAddition {
received_documents: documents_count,
indexed_documents: None,
@ -216,47 +198,32 @@ impl From<Task> for TaskView {
(TaskType::DocumentAdditionOrUpdate, Some(details))
}
TaskContent::DocumentDeletion {
deletion: DocumentDeletion::Ids(ids),
..
} => (
TaskContent::DocumentDeletion { deletion: DocumentDeletion::Ids(ids), .. } => (
TaskType::DocumentDeletion,
Some(TaskDetails::DocumentDeletion {
received_document_ids: ids.len(),
deleted_documents: None,
}),
),
TaskContent::DocumentDeletion {
deletion: DocumentDeletion::Clear,
..
} => (
TaskContent::DocumentDeletion { deletion: DocumentDeletion::Clear, .. } => (
TaskType::DocumentDeletion,
Some(TaskDetails::ClearAll {
deleted_documents: None,
}),
),
TaskContent::IndexDeletion { .. } => (
TaskType::IndexDeletion,
Some(TaskDetails::ClearAll {
deleted_documents: None,
}),
),
TaskContent::SettingsUpdate { settings, .. } => (
TaskType::SettingsUpdate,
Some(TaskDetails::Settings { settings }),
),
TaskContent::IndexCreation { primary_key, .. } => (
TaskType::IndexCreation,
Some(TaskDetails::IndexInfo { primary_key }),
),
TaskContent::IndexUpdate { primary_key, .. } => (
TaskType::IndexUpdate,
Some(TaskDetails::IndexInfo { primary_key }),
),
TaskContent::Dump { uid } => (
TaskType::DumpCreation,
Some(TaskDetails::Dump { dump_uid: uid }),
Some(TaskDetails::ClearAll { deleted_documents: None }),
),
TaskContent::IndexDeletion { .. } => {
(TaskType::IndexDeletion, Some(TaskDetails::ClearAll { deleted_documents: None }))
}
TaskContent::SettingsUpdate { settings, .. } => {
(TaskType::SettingsUpdate, Some(TaskDetails::Settings { settings }))
}
TaskContent::IndexCreation { primary_key, .. } => {
(TaskType::IndexCreation, Some(TaskDetails::IndexInfo { primary_key }))
}
TaskContent::IndexUpdate { primary_key, .. } => {
(TaskType::IndexUpdate, Some(TaskDetails::IndexInfo { primary_key }))
}
TaskContent::Dump { uid } => {
(TaskType::DumpCreation, Some(TaskDetails::Dump { dump_uid: uid }))
}
};
// An event always has at least one event: "Created"
@ -267,36 +234,20 @@ impl From<Task> for TaskView {
TaskEvent::Succeeded { timestamp, result } => {
match (result, &mut details) {
(
TaskResult::DocumentAddition {
indexed_documents: num,
..
},
Some(TaskDetails::DocumentAddition {
ref mut indexed_documents,
..
}),
TaskResult::DocumentAddition { indexed_documents: num, .. },
Some(TaskDetails::DocumentAddition { ref mut indexed_documents, .. }),
) => {
indexed_documents.replace(*num);
}
(
TaskResult::DocumentDeletion {
deleted_documents: docs,
..
},
Some(TaskDetails::DocumentDeletion {
ref mut deleted_documents,
..
}),
TaskResult::DocumentDeletion { deleted_documents: docs, .. },
Some(TaskDetails::DocumentDeletion { ref mut deleted_documents, .. }),
) => {
deleted_documents.replace(*docs);
}
(
TaskResult::ClearAll {
deleted_documents: docs,
},
Some(TaskDetails::ClearAll {
ref mut deleted_documents,
}),
TaskResult::ClearAll { deleted_documents: docs },
Some(TaskDetails::ClearAll { ref mut deleted_documents }),
) => {
deleted_documents.replace(*docs);
}
@ -306,22 +257,13 @@ impl From<Task> for TaskView {
}
TaskEvent::Failed { timestamp, error } => {
match details {
Some(TaskDetails::DocumentDeletion {
ref mut deleted_documents,
..
}) => {
Some(TaskDetails::DocumentDeletion { ref mut deleted_documents, .. }) => {
deleted_documents.replace(0);
}
Some(TaskDetails::ClearAll {
ref mut deleted_documents,
..
}) => {
Some(TaskDetails::ClearAll { ref mut deleted_documents, .. }) => {
deleted_documents.replace(0);
}
Some(TaskDetails::DocumentAddition {
ref mut indexed_documents,
..
}) => {
Some(TaskDetails::DocumentAddition { ref mut indexed_documents, .. }) => {
indexed_documents.replace(0);
}
_ => (),
@ -400,10 +342,7 @@ pub enum TaskStatus {
#[allow(clippy::large_enum_variant)]
pub enum TaskDetails {
#[cfg_attr(test, serde(rename_all = "camelCase"))]
DocumentAddition {
received_documents: usize,
indexed_documents: Option<u64>,
},
DocumentAddition { received_documents: usize, indexed_documents: Option<u64> },
#[cfg_attr(test, serde(rename_all = "camelCase"))]
Settings {
#[cfg_attr(test, serde(flatten))]
@ -412,10 +351,7 @@ pub enum TaskDetails {
#[cfg_attr(test, serde(rename_all = "camelCase"))]
IndexInfo { primary_key: Option<String> },
#[cfg_attr(test, serde(rename_all = "camelCase"))]
DocumentDeletion {
received_document_ids: usize,
deleted_documents: Option<u64>,
},
DocumentDeletion { received_document_ids: usize, deleted_documents: Option<u64> },
#[cfg_attr(test, serde(rename_all = "camelCase"))]
ClearAll { deleted_documents: Option<u64> },
#[cfg_attr(test, serde(rename_all = "camelCase"))]

30
dump/src/reader/v6/mod.rs
View file

@ -1,19 +1,15 @@
use std::{
fs::{self, File},
io::{BufRead, BufReader},
path::Path,
str::FromStr,
};
use std::fs::{self, File};
use std::io::{BufRead, BufReader};
use std::path::Path;
use std::str::FromStr;
pub use meilisearch_types::milli;
use tempfile::TempDir;
use time::OffsetDateTime;
use uuid::Uuid;
use crate::{Error, IndexMetadata, Result, Version};
pub use meilisearch_types::milli;
use super::Document;
use crate::{Error, IndexMetadata, Result, Version};
pub type Metadata = crate::Metadata;
@ -89,11 +85,7 @@ impl V6Reader {
let entry = entry?;
if entry.file_type()?.is_dir() {
let index = V6IndexReader::new(
entry
.file_name()
.to_str()
.ok_or(Error::BadIndexName)?
.to_string(),
entry.file_name().to_str().ok_or(Error::BadIndexName)?.to_string(),
&entry.path(),
)?;
Ok(Some(index))
@ -132,9 +124,7 @@ impl V6Reader {
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<Key>> + '_> {
Box::new(
(&mut self.keys)
.lines()
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
(&mut self.keys).lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
)
}
}
@ -145,9 +135,7 @@ pub struct UpdateFile {
impl UpdateFile {
fn new(path: &Path) -> Result<Self> {
Ok(UpdateFile {
reader: BufReader::new(File::open(path)?),
})
Ok(UpdateFile { reader: BufReader::new(File::open(path)?) })
}
}

83
dump/src/writer.rs
View file

@ -1,20 +1,18 @@
use std::{
fs::{self, File},
io::{BufWriter, Write},
path::PathBuf,
};
use std::fs::{self, File};
use std::io::{BufWriter, Write};
use std::path::PathBuf;
use flate2::{write::GzEncoder, Compression};
use meilisearch_types::{
keys::Key,
settings::{Checked, Settings},
};
use flate2::write::GzEncoder;
use flate2::Compression;
use meilisearch_types::keys::Key;
use meilisearch_types::settings::{Checked, Settings};
use serde_json::{Map, Value};
use tempfile::TempDir;
use time::OffsetDateTime;
use uuid::Uuid;
use crate::{reader::Document, IndexMetadata, Metadata, Result, TaskDump, CURRENT_DUMP_VERSION};
use crate::reader::Document;
use crate::{IndexMetadata, Metadata, Result, TaskDump, CURRENT_DUMP_VERSION};
pub struct DumpWriter {
dir: TempDir,
@ -36,10 +34,7 @@ impl DumpWriter {
db_version: env!("CARGO_PKG_VERSION").to_string(),
dump_date: OffsetDateTime::now_utc(),
};
fs::write(
dir.path().join("metadata.json"),
serde_json::to_string(&metadata)?,
)?;
fs::write(dir.path().join("metadata.json"), serde_json::to_string(&metadata)?)?;
std::fs::create_dir(&dir.path().join("indexes"))?;
@ -77,9 +72,7 @@ pub struct KeyWriter {
impl KeyWriter {
pub(crate) fn new(path: PathBuf) -> Result<Self> {
let keys = File::create(path.join("keys.jsonl"))?;
Ok(KeyWriter {
keys: BufWriter::new(keys),
})
Ok(KeyWriter { keys: BufWriter::new(keys) })
}
pub fn push_key(&mut self, key: &Key) -> Result<()> {
@ -107,10 +100,7 @@ impl TaskWriter {
let update_files = path.join("update_files");
std::fs::create_dir(&update_files)?;
Ok(TaskWriter {
queue: BufWriter::new(queue),
update_files,
})
Ok(TaskWriter { queue: BufWriter::new(queue), update_files })
}
/// Pushes tasks in the dump.
@ -119,9 +109,7 @@ impl TaskWriter {
self.queue.write_all(&serde_json::to_vec(task)?)?;
self.queue.write_all(b"\n")?;
Ok(UpdateFile::new(
self.update_files.join(format!("{}.jsonl", task.uid)),
))
Ok(UpdateFile::new(self.update_files.join(format!("{}.jsonl", task.uid))))
}
pub fn flush(mut self) -> Result<()> {
@ -175,10 +163,7 @@ impl IndexWriter {
let documents = File::create(path.join("documents.jsonl"))?;
let settings = File::create(path.join("settings.json"))?;
Ok(IndexWriter {
documents: BufWriter::new(documents),
settings,
})
Ok(IndexWriter { documents: BufWriter::new(documents), settings })
}
pub fn push_document(&mut self, document: &Map<String, Value>) -> Result<()> {
@ -200,20 +185,20 @@ impl IndexWriter {
#[cfg(test)]
pub(crate) mod test {
use std::{fmt::Write, io::BufReader, path::Path, str::FromStr};
use std::fmt::Write;
use std::io::BufReader;
use std::path::Path;
use std::str::FromStr;
use flate2::bufread::GzDecoder;
use meilisearch_types::settings::Unchecked;
use crate::{
reader::Document,
test::{
create_test_api_keys, create_test_documents, create_test_dump,
create_test_instance_uid, create_test_settings, create_test_tasks,
},
};
use super::*;
use crate::reader::Document;
use crate::test::{
create_test_api_keys, create_test_documents, create_test_dump, create_test_instance_uid,
create_test_settings, create_test_tasks,
};
fn create_directory_hierarchy(dir: &Path) -> String {
let mut ret = String::new();
@ -226,10 +211,8 @@ pub(crate) mod test {
let mut ret = String::new();
// the entries are not guarenteed to be returned in the same order thus we need to sort them.
let mut entries = fs::read_dir(dir)
.unwrap()
.collect::<std::result::Result<Vec<_>, _>>()
.unwrap();
let mut entries =
fs::read_dir(dir).unwrap().collect::<std::result::Result<Vec<_>, _>>().unwrap();
// I want the directories first and then sort by name.
entries.sort_by(|a, b| {
@ -317,18 +300,12 @@ pub(crate) mod test {
"###);
let instance_uid = fs::read_to_string(dump_path.join("instance_uid.uuid")).unwrap();
assert_eq!(
Uuid::from_str(&instance_uid).unwrap(),
create_test_instance_uid()
);
assert_eq!(Uuid::from_str(&instance_uid).unwrap(), create_test_instance_uid());
// ==== checking the index
let docs = fs::read_to_string(dump_path.join("indexes/doggos/documents.jsonl")).unwrap();
for (document, expected) in docs.lines().zip(create_test_documents()) {
assert_eq!(
serde_json::from_str::<Map<String, Value>>(document).unwrap(),
expected
);
assert_eq!(serde_json::from_str::<Map<String, Value>>(document).unwrap(), expected);
}
let test_settings =
fs::read_to_string(dump_path.join("indexes/doggos/settings.json")).unwrap();
@ -356,10 +333,8 @@ pub(crate) mod test {
let path = dump_path.join(format!("tasks/update_files/{}.jsonl", expected.0.uid));
println!("trying to open {}", path.display());
let update = fs::read_to_string(path).unwrap();
let documents: Vec<Document> = update
.lines()
.map(|line| serde_json::from_str(line).unwrap())
.collect();
let documents: Vec<Document> =
update.lines().map(|line| serde_json::from_str(line).unwrap()).collect();
assert_eq!(documents, expected_update);
}
}