searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-05 02:55:46 +01:00
Code Issues Releases Wiki Activity

wip: starts working on importing the snapshots

Browse Source
This commit is contained in:
Tamo 2023-08-16 18:41:05 +02:00
parent 777eebb759
commit 854745c670
2 changed files with 38 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
index-scheduler/src/index_mapper/mod.rs
View File

@ -61,7 +61,7 @@ pub struct IndexMapper {
pub(crate) index_stats: Database<UuidCodec, SerdeJson<IndexStats>>, pub(crate) index_stats: Database<UuidCodec, SerdeJson<IndexStats>>,
/// Path to the folder where the LMDB environments of each index are. /// Path to the folder where the LMDB environments of each index are.
base_path: PathBuf, pub(crate) base_path: PathBuf,
/// The map size an index is opened with on the first time. /// The map size an index is opened with on the first time.
index_base_map_size: usize, index_base_map_size: usize,
/// The quantity by which the map size of an index is incremented upon reopening, in bytes. /// The quantity by which the map size of an index is incremented upon reopening, in bytes.

51
index-scheduler/src/lib.rs
View File

@ -683,24 +683,46 @@ impl IndexScheduler {
let snapshot_id = path.strip_prefix("/snapshots/snapshot-").unwrap(); let snapshot_id = path.strip_prefix("/snapshots/snapshot-").unwrap();
let snapshot_dir = let snapshot_dir =
PathBuf::from(format!("~/zk-snapshots/{}", snapshot_id)); PathBuf::from(format!("{}/zk-snapshots/{}", env!("HOME"), snapshot_id));
// TODO: everything
// 1. TODO: Ensure the snapshot version file is the same as our version. // 1. TODO: Ensure the snapshot version file is the same as our version.
// 2. Download and import the index-scheduler database // 2. Download all the databases
log::info!("Importing the index scheduler."); let tasks_file = tempfile::NamedTempFile::new_in(run.env.path()).unwrap();
let tasks =
log::info!("Downloading the index scheduler database.");
let tasks_snapshot =
snapshot_dir.join("tasks.mdb"); snapshot_dir.join("tasks.mdb");
std::fs::copy(tasks_snapshot, tasks_file).unwrap();
log::info!("Downloading the indexes databases");
let indexes_files = tempfile::TempDir::new_in(&run.index_mapper.base_path).unwrap();
let mut indexes = Vec::new();
let dst = snapshot_dir.join("indexes");
let mut indexes_snapshot = tokio::fs::read_dir(&dst).await.unwrap();
while let Some(file) = indexes_snapshot.next_entry().await.unwrap() {
let uuid = file.file_name().as_os_str().to_str().unwrap().to_string();
log::info!("\tDownloading the index {}", uuid.to_string());
std::fs::copy(dst.join(&uuid), indexes_files.path().join(&uuid)).unwrap();
indexes.push(uuid);
}
// 3. Lock the index-mapper and close all the env
// TODO: continue here
// run.env.close();
// 4. Move all the databases
// 5. Unlock the index-mapper
// 2. Download and import the index-scheduler database
// 3. Snapshot every indexes // 3. Snapshot every indexes
log::info!("Importing the indexes");
let dst = snapshot_dir.join("indexes");
let mut indexes = tokio::fs::read_dir(dst).await.unwrap();
while let Some(uuid) = indexes.next_entry().await.unwrap() {
// TODO: Import the index
}
} }
_ => (), _ => (),
}, },
@ -733,9 +755,10 @@ impl IndexScheduler {
.await .await
.unwrap(); .unwrap();
tokio::fs::create_dir_all("~/zk-snapshots").await.unwrap(); let zk_snapshots = format!("{}/zk-snapshots", env!("HOME"));
tokio::fs::create_dir_all(&zk_snapshots).await.unwrap();
let snapshot_dir = let snapshot_dir =
PathBuf::from(format!("~/zk-snapshots/{snapshot_id}")); PathBuf::from(format!("{zk_snapshots}/{snapshot_id}"));
tokio::fs::create_dir(&snapshot_dir).await.unwrap(); tokio::fs::create_dir(&snapshot_dir).await.unwrap();
// 1. Snapshot the version file. // 1. Snapshot the version file.