searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-25 22:34:28 +01:00
Code Issues Releases Wiki Activity

get rids of the trait in most places

Browse Source
This commit is contained in:
Tamo 2022-10-06 14:24:28 +02:00 committed by Clément Renault
parent 0972587cfc
commit 6dcc5851b5
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
6 changed files with 131 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
dump/src/reader/mod.rs
View File

@ -10,32 +10,20 @@ use tempfile::TempDir;
use time::OffsetDateTime;
use uuid::Uuid;
use crate::reader::compat::Compat;
// use crate::reader::compat::Compat;
use crate::{IndexMetadata, Result, Version};
// use self::loaders::{v2, v3, v4, v5};
// pub mod error;
mod compat;
// mod compat;
// mod loaders;
// mod v1;
pub(self) mod v4;
pub(self) mod v5;
pub(self) mod v6;
pub fn open(
dump: impl Read,
) -> Result<
Box<
dyn DumpReader<
Document = serde_json::Map<String, serde_json::Value>,
Settings = v6::Settings<v6::Checked>,
Task = TaskView,
UpdateFile = File,
Key = Key,
>,
>,
> {
pub fn open(dump: impl Read) -> Result<Box<dyn DumpReader>> {
let path = TempDir::new()?;
let mut dump = BufReader::new(dump);
let gz = GzDecoder::new(&mut dump);
@ -57,6 +45,7 @@ pub fn open(
Version::V3 => todo!(),
Version::V4 => todo!(),
Version::V5 => {
/*
let dump_reader = Box::new(v5::V5Reader::open(path)?);
let dump_reader = Box::new(Compat::<
dyn DumpReader<
@ -77,33 +66,14 @@ pub fn open(
>,
>;
Ok(dump_reader)
*/
todo!()
}
Version::V6 => {
let dump_reader = Box::new(v6::V6Reader::open(path)?)
as Box<
dyn DumpReader<
Document = v6::Document,
Settings = v6::Settings<v6::Checked>,
Task = v6::Task,
UpdateFile = v6::UpdateFile,
Key = v6::Key,
>,
>;
Ok(dump_reader)
}
Version::V6 => Ok(Box::new(v6::V6Reader::open(path)?)),
}
}
pub trait DumpReader {
type Document;
type Settings;
type Task;
type UpdateFile;
type Key;
/// Return the version of the dump.
fn version(&self) -> Version;
@ -114,35 +84,19 @@ pub trait DumpReader {
fn instance_uid(&self) -> Result<Option<Uuid>>;
/// Return an iterator over each indexes.
fn indexes(
&self,
) -> Result<
Box<
dyn Iterator<
Item = Result<
Box<
dyn IndexReader<Document = Self::Document, Settings = Self::Settings>
+ '_,
>,
>,
> + '_,
>,
>;
fn indexes(&self) -> Result<Box<dyn Iterator<Item = Result<Box<dyn IndexReader + '_>>> + '_>>;
/// Return all the tasks in the dump with a possible update file.
fn tasks(
&mut self,
) -> Box<dyn Iterator<Item = Result<(Self::Task, Option<Self::UpdateFile>)>> + '_>;
) -> Box<dyn Iterator<Item = Result<(v6::Task, Option<v6::UpdateFile>)>> + '_>;
/// Return all the keys.
fn keys(&mut self) -> Box<dyn Iterator<Item = Result<Self::Key>> + '_>;
fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v6::Key>> + '_>;
}
pub trait IndexReader {
type Document;
type Settings;
fn metadata(&self) -> &IndexMetadata;
fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Self::Document>> + '_>>;
fn settings(&mut self) -> Result<Self::Settings>;
fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Document>> + '_>>;
fn settings(&mut self) -> Result<v6::Settings<v6::Checked>>;
}

71
dump/src/reader/v5/mod.rs
View File

@ -99,16 +99,6 @@ impl V5Reader {
dump,
})
}
}
impl DumpReader for V5Reader {
type Document = serde_json::Map<String, serde_json::Value>;
type Settings = Settings<Checked>;
type Task = Task;
type UpdateFile = File;
type Key = Key;
fn version(&self) -> Version {
Version::V5
@ -123,24 +113,9 @@ impl DumpReader for V5Reader {
Ok(Some(Uuid::parse_str(&uuid)?))
}
fn indexes(
&self,
) -> Result<
Box<
dyn Iterator<
Item = Result<
Box<
dyn super::IndexReader<
Document = Self::Document,
Settings = Self::Settings,
> + '_,
>,
>,
> + '_,
>,
> {
Ok(Box::new(self.index_uuid.iter().map(|index| -> Result<_> {
Ok(Box::new(V5IndexReader::new(
fn indexes(&self) -> Result<impl Iterator<Item = Result<V5IndexReader>> + '_> {
Ok(self.index_uuid.iter().map(|index| -> Result<_> {
Ok(V5IndexReader::new(
index.uid.clone(),
&self
.dump
@ -148,17 +123,12 @@ impl DumpReader for V5Reader {
.join("indexes")
.join(index.index_meta.uuid.to_string()),
)?)
as Box<
dyn IndexReader<Document = Self::Document, Settings = Self::Settings>,
>)
})))
}))
}
fn tasks(
&mut self,
) -> Box<dyn Iterator<Item = Result<(Self::Task, Option<Self::UpdateFile>)>> + '_> {
Box::new((&mut self.tasks).lines().map(|line| -> Result<_> {
let task: Self::Task = serde_json::from_str(&line?)?;
fn tasks(&mut self) -> impl Iterator<Item = Result<(Task, Option<UpdateFile>)>> + '_ {
(&mut self.tasks).lines().map(|line| -> Result<_> {
let task: Task = serde_json::from_str(&line?)?;
if !task.is_finished() {
if let Some(uuid) = task.get_content_uuid() {
let update_file_path = self
@ -175,15 +145,13 @@ impl DumpReader for V5Reader {
} else {
Ok((task, None))
}
}))
})
}
fn keys(&mut self) -> Box<dyn Iterator<Item = Result<Self::Key>> + '_> {
Box::new(
(&mut self.keys)
.lines()
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
)
fn keys(&mut self) -> impl Iterator<Item = Result<Key>> + '_ {
(&mut self.keys)
.lines()
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })
}
}
@ -215,23 +183,18 @@ impl V5IndexReader {
Ok(ret)
}
}
impl IndexReader for V5IndexReader {
type Document = serde_json::Map<String, serde_json::Value>;
type Settings = Settings<Checked>;
fn metadata(&self) -> &IndexMetadata {
&self.metadata
}
fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Self::Document>> + '_>> {
Ok(Box::new((&mut self.documents).lines().map(
|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) },
)))
fn documents(&mut self) -> Result<impl Iterator<Item = Result<Document>> + '_> {
Ok((&mut self.documents)
.lines()
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }))
}
fn settings(&mut self) -> Result<Self::Settings> {
fn settings(&mut self) -> Result<Settings<Checked>> {
Ok(self.settings.clone())
}
}

2
dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-12.snap
View File

@ -66,6 +66,6 @@ Ok(
),
},
),
_kind: PhantomData,
_kind: PhantomData<dump::reader::v5::settings::Checked>,
},
)

2
dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-6.snap
View File

@ -80,6 +80,6 @@ Ok(
),
},
),
_kind: PhantomData,
_kind: PhantomData<dump::reader::v5::settings::Checked>,
},
)

2
dump/src/reader/v5/snapshots/dump__reader__v5__test__read_dump_v5-9.snap
View File

@ -72,6 +72,6 @@ Ok(
),
},
),
_kind: PhantomData,
_kind: PhantomData<dump::reader::v5::settings::Checked>,
},
)

192
dump/src/reader/v6.rs
View File

@ -54,12 +54,6 @@ pub struct V6Reader {
keys: BufReader<File>,
}
struct V6IndexReader {
metadata: IndexMetadata,
documents: BufReader<File>,
settings: BufReader<File>,
}
impl V6Reader {
pub fn open(dump: TempDir) -> Result<Self> {
let meta_file = fs::read(dump.path().join("metadata.json"))?;
@ -74,31 +68,6 @@ impl V6Reader {
dump,
})
}
}
impl V6IndexReader {
pub fn new(name: String, path: &Path) -> Result<Self> {
let metadata = File::open(path.join("metadata.json"))?;
let ret = V6IndexReader {
metadata: serde_json::from_reader(metadata)?,
documents: BufReader::new(File::open(path.join("documents.jsonl"))?),
settings: BufReader::new(File::open(path.join("settings.json"))?),
};
Ok(ret)
}
}
impl DumpReader for V6Reader {
type Document = serde_json::Map<String, serde_json::Value>;
type Settings = Settings<Checked>;
type Task = Task;
type UpdateFile = File;
type Key = Key;
fn version(&self) -> Version {
Version::V6
}
@ -111,55 +80,30 @@ impl DumpReader for V6Reader {
Ok(Some(self.instance_uid))
}
fn indexes(
&self,
) -> Result<
Box<
dyn Iterator<
Item = Result<
Box<
dyn super::IndexReader<
Document = Self::Document,
Settings = Self::Settings,
> + '_,
>,
>,
> + '_,
>,
> {
fn indexes(&self) -> Result<impl Iterator<Item = Result<V6IndexReader>> + '_> {
let entries = fs::read_dir(self.dump.path().join("indexes"))?;
Ok(Box::new(
entries
.map(|entry| -> Result<Option<_>> {
let entry = entry?;
if entry.file_type()?.is_dir() {
let index = Box::new(V6IndexReader::new(
entry
.file_name()
.to_str()
.ok_or(Error::BadIndexName)?
.to_string(),
&entry.path(),
)?)
as Box<
dyn IndexReader<
Document = Self::Document,
Settings = Self::Settings,
>,
>;
Ok(Some(index))
} else {
Ok(None)
}
})
.filter_map(|entry| entry.transpose()),
))
Ok(entries
.map(|entry| -> Result<Option<_>> {
let entry = entry?;
if entry.file_type()?.is_dir() {
let index = V6IndexReader::new(
entry
.file_name()
.to_str()
.ok_or(Error::BadIndexName)?
.to_string(),
&entry.path(),
)?;
Ok(Some(index))
} else {
Ok(None)
}
})
.filter_map(|entry| entry.transpose()))
}
fn tasks(
&mut self,
) -> Box<dyn Iterator<Item = Result<(Self::Task, Option<Self::UpdateFile>)>> + '_> {
Box::new((&mut self.tasks).lines().map(|line| -> Result<_> {
fn tasks(&mut self) -> impl Iterator<Item = Result<(Task, Option<UpdateFile>)>> + '_ {
(&mut self.tasks).lines().map(|line| -> Result<_> {
let mut task: index_scheduler::TaskView = serde_json::from_str(&line?)?;
// TODO: this can be removed once we can `Deserialize` the duration from the `TaskView`.
if let Some((started_at, finished_at)) = task.started_at.zip(task.finished_at) {
@ -177,34 +121,96 @@ impl DumpReader for V6Reader {
} else {
Ok((task, None))
}
}))
})
}
fn keys(&mut self) -> Box<dyn Iterator<Item = Result<Self::Key>> + '_> {
Box::new(
(&mut self.keys)
.lines()
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
)
fn keys(&mut self) -> impl Iterator<Item = Result<Key>> + '_ {
(&mut self.keys)
.lines()
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })
}
}
impl IndexReader for V6IndexReader {
type Document = serde_json::Map<String, serde_json::Value>;
type Settings = Settings<Checked>;
impl DumpReader for V6Reader {
fn version(&self) -> Version {
self.version()
}
fn date(&self) -> Option<OffsetDateTime> {
self.date()
}
fn instance_uid(&self) -> Result<Option<Uuid>> {
self.instance_uid()
}
fn indexes(
&self,
) -> Result<Box<dyn Iterator<Item = Result<Box<dyn super::IndexReader + '_>>> + '_>> {
self.indexes().map(|iter| {
Box::new(iter.map(|result| {
result.map(|index| Box::new(index) as Box<dyn super::IndexReader + '_>)
})) as Box<dyn Iterator<Item = _>>
})
}
fn tasks(
&mut self,
) -> Box<dyn Iterator<Item = Result<(self::Task, Option<self::UpdateFile>)>> + '_> {
Box::new(self.tasks())
}
fn keys(&mut self) -> Box<dyn Iterator<Item = Result<self::Key>> + '_> {
Box::new(self.keys())
}
}
struct V6IndexReader {
metadata: IndexMetadata,
documents: BufReader<File>,
settings: BufReader<File>,
}
impl V6IndexReader {
pub fn new(name: String, path: &Path) -> Result<Self> {
let metadata = File::open(path.join("metadata.json"))?;
let ret = V6IndexReader {
metadata: serde_json::from_reader(metadata)?,
documents: BufReader::new(File::open(path.join("documents.jsonl"))?),
settings: BufReader::new(File::open(path.join("settings.json"))?),
};
Ok(ret)
}
fn metadata(&self) -> &IndexMetadata {
&self.metadata
}
fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Self::Document>> + '_>> {
Ok(Box::new((&mut self.documents).lines().map(
|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) },
)))
fn documents(&mut self) -> Result<impl Iterator<Item = Result<Document>> + '_> {
Ok((&mut self.documents)
.lines()
.map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }))
}
fn settings(&mut self) -> Result<Self::Settings> {
let settings: index::Settings<Unchecked> = serde_json::from_reader(&mut self.settings)?;
fn settings(&mut self) -> Result<Settings<Checked>> {
let settings: Settings<Unchecked> = serde_json::from_reader(&mut self.settings)?;
Ok(settings.check())
}
}
impl IndexReader for V6IndexReader {
fn metadata(&self) -> &IndexMetadata {
self.metadata()
}
fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Document>> + '_>> {
self.documents()
.map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>)
}
fn settings(&mut self) -> Result<Settings<Checked>> {
self.settings()
}
}