mirror of
https://github.com/meilisearch/MeiliSearch
synced 2025-07-04 20:37:15 +02:00
refactor the Task a little bit
This commit is contained in:
Tamo
Clément Renault
parent
510ce9fc51
commit
2f1eb78b1d
17 changed files with 1358 additions and 833 deletions
|
|
@ -72,15 +72,23 @@ impl CompatV5ToV6 {
|
|||
v5::Status::Succeeded => v6::Status::Succeeded,
|
||||
v5::Status::Failed => v6::Status::Failed,
|
||||
},
|
||||
kind: match &task.content {
|
||||
v5::tasks::TaskContent::IndexCreation { .. } => v6::Kind::IndexCreation,
|
||||
v5::tasks::TaskContent::IndexUpdate { .. } => v6::Kind::IndexUpdate,
|
||||
kind: match task.content.clone() {
|
||||
v5::tasks::TaskContent::IndexCreation { primary_key, .. } => {
|
||||
v6::Kind::IndexCreation { primary_key }
|
||||
}
|
||||
v5::tasks::TaskContent::IndexUpdate { primary_key, .. } => {
|
||||
v6::Kind::IndexUpdate { primary_key }
|
||||
}
|
||||
v5::tasks::TaskContent::IndexDeletion { .. } => v6::Kind::IndexDeletion,
|
||||
v5::tasks::TaskContent::DocumentAddition {
|
||||
merge_strategy,
|
||||
allow_index_creation,
|
||||
primary_key,
|
||||
documents_count,
|
||||
..
|
||||
} => v6::Kind::DocumentImport {
|
||||
primary_key,
|
||||
documents_count: documents_count as u64,
|
||||
method: match merge_strategy {
|
||||
v5::tasks::IndexDocumentsMethod::ReplaceDocuments => {
|
||||
v6::milli::update::IndexDocumentsMethod::ReplaceDocuments
|
||||
|
|
@ -91,14 +99,22 @@ impl CompatV5ToV6 {
|
|||
},
|
||||
allow_index_creation: allow_index_creation.clone(),
|
||||
},
|
||||
v5::tasks::TaskContent::DocumentDeletion { .. } => {
|
||||
v6::Kind::DocumentDeletion
|
||||
}
|
||||
v5::tasks::TaskContent::DocumentDeletion { deletion, .. } => match deletion
|
||||
{
|
||||
v5::tasks::DocumentDeletion::Clear => v6::Kind::DocumentClear,
|
||||
v5::tasks::DocumentDeletion::Ids(documents_ids) => {
|
||||
v6::Kind::DocumentDeletion { documents_ids }
|
||||
}
|
||||
},
|
||||
v5::tasks::TaskContent::SettingsUpdate {
|
||||
allow_index_creation,
|
||||
is_deletion,
|
||||
settings,
|
||||
..
|
||||
} => v6::Kind::Settings {
|
||||
allow_index_creation: allow_index_creation.clone(),
|
||||
is_deletion,
|
||||
allow_index_creation,
|
||||
settings: settings.into(),
|
||||
},
|
||||
v5::tasks::TaskContent::Dump { .. } => v6::Kind::DumpExport,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1,200 +0,0 @@
|
|||
use std::{
|
||||
fs::{self, File},
|
||||
io::{BufRead, BufReader},
|
||||
path::Path,
|
||||
str::FromStr,
|
||||
};
|
||||
|
||||
use tempfile::TempDir;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{Error, IndexMetadata, Result, Version};
|
||||
|
||||
pub use meilisearch_types::milli;
|
||||
|
||||
use super::Document;
|
||||
|
||||
pub type Metadata = crate::Metadata;
|
||||
|
||||
pub type Settings<T> = meilisearch_types::settings::Settings<T>;
|
||||
pub type Checked = meilisearch_types::settings::Checked;
|
||||
pub type Unchecked = meilisearch_types::settings::Unchecked;
|
||||
|
||||
pub type Task = meilisearch_types::tasks::TaskDump;
|
||||
pub type Key = meilisearch_auth::Key;
|
||||
|
||||
// ===== Other types to clarify the code of the compat module
|
||||
// everything related to the tasks
|
||||
pub type Status = meilisearch_types::tasks::Status;
|
||||
pub type Kind = meilisearch_types::tasks::Kind;
|
||||
pub type Details = meilisearch_types::tasks::Details;
|
||||
|
||||
// everything related to the settings
|
||||
pub type Setting<T> = meilisearch_types::milli::update::Setting<T>;
|
||||
pub type TypoTolerance = meilisearch_types::settings::TypoSettings;
|
||||
pub type MinWordSizeForTypos = meilisearch_types::settings::MinWordSizeTyposSetting;
|
||||
pub type FacetingSettings = meilisearch_types::settings::FacetingSettings;
|
||||
pub type PaginationSettings = meilisearch_types::settings::PaginationSettings;
|
||||
|
||||
// everything related to the api keys
|
||||
pub type Action = meilisearch_auth::Action;
|
||||
pub type StarOr<T> = meilisearch_types::star_or::StarOr<T>;
|
||||
pub type IndexUid = meilisearch_types::index_uid::IndexUid;
|
||||
|
||||
// everything related to the errors
|
||||
pub type ResponseError = meilisearch_types::error::ResponseError;
|
||||
pub type Code = meilisearch_types::error::Code;
|
||||
|
||||
pub struct V6Reader {
|
||||
dump: TempDir,
|
||||
instance_uid: Uuid,
|
||||
metadata: Metadata,
|
||||
tasks: BufReader<File>,
|
||||
keys: BufReader<File>,
|
||||
}
|
||||
|
||||
impl V6Reader {
|
||||
pub fn open(dump: TempDir) -> Result<Self> {
|
||||
let meta_file = fs::read(dump.path().join("metadata.json"))?;
|
||||
let instance_uid = fs::read_to_string(dump.path().join("instance_uid.uuid"))?;
|
||||
let instance_uid = Uuid::from_str(&instance_uid)?;
|
||||
|
||||
Ok(V6Reader {
|
||||
metadata: serde_json::from_reader(&*meta_file)?,
|
||||
instance_uid,
|
||||
tasks: BufReader::new(File::open(dump.path().join("tasks").join("queue.jsonl"))?),
|
||||
keys: BufReader::new(File::open(dump.path().join("keys.jsonl"))?),
|
||||
dump,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn version(&self) -> Version {
|
||||
Version::V6
|
||||
}
|
||||
|
||||
pub fn date(&self) -> Option<OffsetDateTime> {
|
||||
Some(self.metadata.dump_date)
|
||||
}
|
||||
|
||||
pub fn instance_uid(&self) -> Result<Option<Uuid>> {
|
||||
Ok(Some(self.instance_uid))
|
||||
}
|
||||
|
||||
pub fn indexes(&self) -> Result<Box<dyn Iterator<Item = Result<V6IndexReader>> + '_>> {
|
||||
let entries = fs::read_dir(self.dump.path().join("indexes"))?;
|
||||
Ok(Box::new(
|
||||
entries
|
||||
.map(|entry| -> Result<Option<_>> {
|
||||
let entry = entry?;
|
||||
if entry.file_type()?.is_dir() {
|
||||
let index = V6IndexReader::new(
|
||||
entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.ok_or(Error::BadIndexName)?
|
||||
.to_string(),
|
||||
&entry.path(),
|
||||
)?;
|
||||
Ok(Some(index))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
})
|
||||
.filter_map(|entry| entry.transpose()),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn tasks(
|
||||
&mut self,
|
||||
) -> Box<dyn Iterator<Item = Result<(Task, Option<Box<super::UpdateFile>>)>> + '_> {
|
||||
Box::new((&mut self.tasks).lines().map(|line| -> Result<_> {
|
||||
let task: Task = serde_json::from_str(&line?)?;
|
||||
|
||||
let update_file_path = self
|
||||
.dump
|
||||
.path()
|
||||
.join("tasks")
|
||||
.join("update_files")
|
||||
.join(format!("{}.jsonl", task.uid.to_string()));
|
||||
|
||||
if update_file_path.exists() {
|
||||
Ok((
|
||||
task,
|
||||
Some(Box::new(UpdateFile::new(&update_file_path)?) as Box<super::UpdateFile>),
|
||||
))
|
||||
} else {
|
||||
Ok((task, None))
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<Key>> + '_> {
|
||||
Box::new(
|
||||
(&mut self.keys)
|
||||
.lines()
|
||||
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct UpdateFile {
|
||||
reader: BufReader<File>,
|
||||
}
|
||||
|
||||
impl UpdateFile {
|
||||
fn new(path: &Path) -> Result<Self> {
|
||||
Ok(UpdateFile {
|
||||
reader: BufReader::new(File::open(path)?),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for UpdateFile {
|
||||
type Item = Result<Document>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
(&mut self.reader)
|
||||
.lines()
|
||||
.map(|line| {
|
||||
line.map_err(Error::from)
|
||||
.and_then(|line| serde_json::from_str(&line).map_err(Error::from))
|
||||
})
|
||||
.next()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct V6IndexReader {
|
||||
metadata: IndexMetadata,
|
||||
documents: BufReader<File>,
|
||||
settings: BufReader<File>,
|
||||
}
|
||||
|
||||
impl V6IndexReader {
|
||||
pub fn new(_name: String, path: &Path) -> Result<Self> {
|
||||
let metadata = File::open(path.join("metadata.json"))?;
|
||||
|
||||
let ret = V6IndexReader {
|
||||
metadata: serde_json::from_reader(metadata)?,
|
||||
documents: BufReader::new(File::open(path.join("documents.jsonl"))?),
|
||||
settings: BufReader::new(File::open(path.join("settings.json"))?),
|
||||
};
|
||||
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
pub fn metadata(&self) -> &IndexMetadata {
|
||||
&self.metadata
|
||||
}
|
||||
|
||||
pub fn documents(&mut self) -> Result<impl Iterator<Item = Result<Document>> + '_> {
|
||||
Ok((&mut self.documents)
|
||||
.lines()
|
||||
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }))
|
||||
}
|
||||
|
||||
pub fn settings(&mut self) -> Result<Settings<Checked>> {
|
||||
let settings: Settings<Unchecked> = serde_json::from_reader(&mut self.settings)?;
|
||||
Ok(settings.check())
|
||||
}
|
||||
}
|
||||
|
|
@ -6,8 +6,10 @@ use std::{
|
|||
|
||||
use flate2::{write::GzEncoder, Compression};
|
||||
use meilisearch_auth::Key;
|
||||
use meilisearch_types::settings::{Checked, Settings};
|
||||
use meilisearch_types::tasks::TaskDump;
|
||||
use meilisearch_types::{
|
||||
settings::{Checked, Settings},
|
||||
tasks::Task,
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
use tempfile::TempDir;
|
||||
use time::OffsetDateTime;
|
||||
|
|
@ -105,7 +107,7 @@ impl TaskWriter {
|
|||
|
||||
/// Pushes tasks in the dump.
|
||||
/// If the tasks has an associated `update_file` it'll use the `task_id` as its name.
|
||||
pub fn push_task(&mut self, task: &TaskDump) -> Result<UpdateFile> {
|
||||
pub fn push_task(&mut self, task: &Task) -> Result<UpdateFile> {
|
||||
self.queue.write_all(&serde_json::to_vec(task)?)?;
|
||||
self.queue.write_all(b"\n")?;
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue