searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-29 16:24:26 +01:00
Code Issues Releases Wiki Activity

Use finished_at for v3 dumps instead

Browse Source
This commit is contained in:
Francis Murillo 2023-01-17 23:11:49 +08:00
parent a97281af08
commit 6993924f32
No known key found for this signature in database
GPG Key ID: 2D816FFCF0F44A70
2 changed files with 28 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
dump/src/reader/v3/mod.rs
View File

@ -112,7 +112,6 @@ impl V3Reader {
pub fn indexes(&self) -> Result<impl Iterator<Item = Result<V3IndexReader>> + '_> { pub fn indexes(&self) -> Result<impl Iterator<Item = Result<V3IndexReader>> + '_> {
Ok(self.index_uuid.iter().map(|index| -> Result<_> { Ok(self.index_uuid.iter().map(|index| -> Result<_> {
V3IndexReader::new( V3IndexReader::new(
index.uid.clone(),
&self.dump.path().join("indexes").join(index.uuid.to_string()), &self.dump.path().join("indexes").join(index.uuid.to_string()),
&index, &index,
BufReader::new( BufReader::new(
@ -160,43 +159,42 @@ pub struct V3IndexReader {
} }
impl V3IndexReader { impl V3IndexReader {
pub fn new( pub fn new(path: &Path, index_uuid: &IndexUuid, tasks: BufReader<File>) -> Result<Self> {
name: String,
path: &Path,
index_uuid: &IndexUuid,
tasks: BufReader<File>,
) -> Result<Self> {
let meta = File::open(path.join("meta.json"))?; let meta = File::open(path.join("meta.json"))?;
let meta: DumpMeta = serde_json::from_reader(meta)?; let meta: DumpMeta = serde_json::from_reader(meta)?;
let mut created_entry = (u64::MAX, None); let mut created_at = None;
let mut updated_entry = (u64::MIN, None); let mut updated_at = None;
for line in tasks.lines() { for line in tasks.lines() {
let task: Task = serde_json::from_str(&line?)?; let task: Task = serde_json::from_str(&line?)?;
if &task.uuid == &index_uuid.uuid { if task.uuid != index_uuid.uuid || !task.is_finished() {
let update_id = task.update.id(); continue;
}
if update_id <= created_entry.0 { let new_created_at = match task.update.meta() {
created_entry.0 = update_id; Kind::DocumentAddition { .. } | Kind::Settings(_) => task.update.finished_at(),
created_entry.1 = task.update.created_at(); _ => None,
} };
let new_updated_at = task.update.finished_at();
if update_id >= updated_entry.0 { if created_at.is_none() || created_at > new_created_at {
updated_entry.0 = update_id; created_at = new_created_at;
updated_entry.1 = task.update.processed_at(); }
}
if updated_at.is_none() || updated_at < new_updated_at {
updated_at = new_updated_at;
} }
} }
let current_time = OffsetDateTime::now_utc(); let current_time = OffsetDateTime::now_utc();
let metadata = IndexMetadata { let metadata = IndexMetadata {
uid: name, uid: index_uuid.uid.clone(),
primary_key: meta.primary_key, primary_key: meta.primary_key,
created_at: created_entry.1.unwrap_or(current_time), created_at: created_at.unwrap_or(current_time),
updated_at: updated_entry.1.unwrap_or(current_time), updated_at: updated_at.unwrap_or(current_time),
}; };
let ret = V3IndexReader { let ret = V3IndexReader {
@ -299,7 +297,7 @@ pub(crate) mod test {
{ {
"uid": "products", "uid": "products",
"primaryKey": "sku", "primaryKey": "sku",
"createdAt": "2022-10-07T11:38:54.734594617Z", "createdAt": "2022-10-07T11:38:54.74389899Z",
"updatedAt": "2022-10-07T11:38:55.963185778Z" "updatedAt": "2022-10-07T11:38:55.963185778Z"
} }
"###); "###);
@ -314,7 +312,7 @@ pub(crate) mod test {
{ {
"uid": "movies", "uid": "movies",
"primaryKey": "id", "primaryKey": "id",
"createdAt": "2022-10-07T11:38:54.004402239Z", "createdAt": "2022-10-07T11:38:54.026649575Z",
"updatedAt": "2022-10-07T11:39:04.188852537Z" "updatedAt": "2022-10-07T11:39:04.188852537Z"
} }
"###); "###);
@ -325,11 +323,11 @@ pub(crate) mod test {
meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022"); meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
// movies2 // movies2
insta::assert_json_snapshot!(movies2.metadata(), { ".updatedAt" => "[now]" }, @r###" insta::assert_json_snapshot!(movies2.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
{ {
"uid": "movies_2", "uid": "movies_2",
"primaryKey": null, "primaryKey": null,
"createdAt": "2022-10-07T11:39:03.703667164Z", "createdAt": "[now]",
"updatedAt": "[now]" "updatedAt": "[now]"
} }
"###); "###);
@ -344,7 +342,7 @@ pub(crate) mod test {
{ {
"uid": "dnd_spells", "uid": "dnd_spells",
"primaryKey": "index", "primaryKey": "index",
"createdAt": "2022-10-07T11:38:56.263041061Z", "createdAt": "2022-10-07T11:38:56.265951133Z",
"updatedAt": "2022-10-07T11:38:56.521004328Z" "updatedAt": "2022-10-07T11:38:56.521004328Z"
} }
"###); "###);

6
dump/src/reader/v3/updates.rs
View File

@ -75,7 +75,7 @@ impl UpdateStatus {
} }
} }
pub fn created_at(&self) -> Option<OffsetDateTime> { pub fn enqueued_at(&self) -> Option<OffsetDateTime> {
match self { match self {
UpdateStatus::Processing(u) => Some(u.from.enqueued_at.clone()), UpdateStatus::Processing(u) => Some(u.from.enqueued_at.clone()),
UpdateStatus::Enqueued(u) => Some(u.enqueued_at.clone()), UpdateStatus::Enqueued(u) => Some(u.enqueued_at.clone()),
@ -85,9 +85,9 @@ impl UpdateStatus {
} }
} }
pub fn processed_at(&self) -> Option<OffsetDateTime> { pub fn finished_at(&self) -> Option<OffsetDateTime> {
match self { match self {
UpdateStatus::Processing(u) => Some(u.started_processing_at.clone()), UpdateStatus::Processing(_) => None,
UpdateStatus::Enqueued(_) => None, UpdateStatus::Enqueued(_) => None,
UpdateStatus::Processed(u) => Some(u.processed_at.clone()), UpdateStatus::Processed(u) => Some(u.processed_at.clone()),
UpdateStatus::Aborted(_) => None, UpdateStatus::Aborted(_) => None,