3013: Extract the dates out of the dumpv5. r=loiclec a=funilrys

Hi there, 

please review this PR that tries to fix #2986. I'm still learning Rust and I found that #2986 is an excellent way for me to read and learn what others do with Rust. So please excuse my semantics ...

Stay safe and healthy.

---

# Pull Request

This patch possibly fixes #2986.

This patch introduces a way to fill the IndexMetadata.created_at and IndexMetadata.updated_at keys from the tasks events. This is done by reading the creation date of the first event (created_at) and the creation date of the last event (updated_at).


## Related issue
Fixes #2986

## What does this PR do?
- Extract the dates out of the dumpv5.

## PR checklist
Please check if your PR fulfills the following requirements:
- [x] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)?
- [x] Have you read the contributing guidelines?
- [x] Have you made sure that the title is accurate and descriptive of the changes?

Thank you so much for contributing to Meilisearch!


Co-authored-by: funilrys <contact@funilrys.com>
This commit is contained in:
bors[bot] 2023-01-05 08:23:52 +00:00 committed by GitHub
commit a402fc4486
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 70 additions and 4 deletions

View File

@ -141,6 +141,10 @@ impl V5Reader {
V5IndexReader::new( V5IndexReader::new(
index.uid.clone(), index.uid.clone(),
&self.dump.path().join("indexes").join(index.index_meta.uuid.to_string()), &self.dump.path().join("indexes").join(index.index_meta.uuid.to_string()),
&index.index_meta,
BufReader::new(
File::open(self.dump.path().join("updates").join("data.jsonl")).unwrap(),
),
) )
})) }))
} }
@ -189,16 +193,39 @@ pub struct V5IndexReader {
} }
impl V5IndexReader { impl V5IndexReader {
pub fn new(name: String, path: &Path) -> Result<Self> { pub fn new(
name: String,
path: &Path,
index_metadata: &meta::IndexMeta,
tasks: BufReader<File>,
) -> Result<Self> {
let meta = File::open(path.join("meta.json"))?; let meta = File::open(path.join("meta.json"))?;
let meta: meta::DumpMeta = serde_json::from_reader(meta)?; let meta: meta::DumpMeta = serde_json::from_reader(meta)?;
let mut created_at = None;
let mut updated_at = None;
for line in tasks.lines() {
let task: Task = serde_json::from_str(&line?)?;
if *task.index_uid().unwrap_or_default().to_string() == name {
if updated_at.is_none() {
updated_at = task.processed_at()
}
if task.id as usize == index_metadata.creation_task_id {
created_at = task.created_at();
break;
}
}
}
let metadata = IndexMetadata { let metadata = IndexMetadata {
uid: name, uid: name,
primary_key: meta.primary_key, primary_key: meta.primary_key,
// FIXME: Iterate over the whole task queue to find the creation and last update date. created_at: created_at.unwrap_or_else(OffsetDateTime::now_utc),
created_at: OffsetDateTime::now_utc(), updated_at: updated_at.unwrap_or_else(OffsetDateTime::now_utc),
updated_at: OffsetDateTime::now_utc(),
}; };
let ret = V5IndexReader { let ret = V5IndexReader {

View File

@ -140,6 +140,45 @@ impl Task {
TaskContent::Dump { .. } => None, TaskContent::Dump { .. } => None,
} }
} }
pub fn processed_at(&self) -> Option<OffsetDateTime> {
match self.events.last() {
Some(TaskEvent::Succeeded { result: _, timestamp }) => Some(*timestamp),
_ => None,
}
}
pub fn created_at(&self) -> Option<OffsetDateTime> {
match &self.content {
TaskContent::IndexCreation { index_uid: _, primary_key: _ } => {
match self.events.first() {
Some(TaskEvent::Created(ts)) => Some(*ts),
_ => None,
}
}
TaskContent::DocumentAddition {
index_uid: _,
content_uuid: _,
merge_strategy: _,
primary_key: _,
documents_count: _,
allow_index_creation: _,
} => match self.events.first() {
Some(TaskEvent::Created(ts)) => Some(*ts),
_ => None,
},
TaskContent::SettingsUpdate {
index_uid: _,
settings: _,
is_deletion: _,
allow_index_creation: _,
} => match self.events.first() {
Some(TaskEvent::Created(ts)) => Some(*ts),
_ => None,
},
_ => None,
}
}
} }
impl IndexUid { impl IndexUid {