searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-30 00:34:26 +01:00
Code Issues Releases Wiki Activity

Fix some bugs with files

Browse Source
This commit is contained in:
Kerollmops 2022-10-25 18:44:07 +02:00 committed by Clément Renault
parent 4736e00253
commit e35fe33712
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
2 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
index-scheduler/src/batch.rs
View File

@ -508,7 +508,7 @@ impl IndexScheduler {
_ => unreachable!(), _ => unreachable!(),
} }
// We must only remove the content files if the transaction is successfuly committed // We must only remove the content files if the transaction is successfully committed
// and if errors occurs when we are deleting files we must do our best to delete // and if errors occurs when we are deleting files we must do our best to delete
// everything. We do not return the encountered errors when deleting the content // everything. We do not return the encountered errors when deleting the content
// files as it is not a breaking operation and we can safely continue our job. // files as it is not a breaking operation and we can safely continue our job.
@ -577,9 +577,10 @@ impl IndexScheduler {
// Note that there cannot be any update files deleted between those // Note that there cannot be any update files deleted between those
// two read operations as the task processing is synchronous. // two read operations as the task processing is synchronous.
// 2.1 First copy the LMDB env and reorganize pages to reduce its size. // 2.1 First copy the LMDB env of the index-scheduler
let dst = temp_snapshot_dir.path().join("data.mdb"); let dst = temp_snapshot_dir.path().join("tasks");
self.env.copy_to_path(dst, CompactionOption::Enabled)?; fs::create_dir_all(&dst)?;
self.env.copy_to_path(dst.join("data.mdb"), CompactionOption::Enabled)?;
// 2.2 Create a read transaction on the index-scheduler // 2.2 Create a read transaction on the index-scheduler
let rtxn = self.env.read_txn()?; let rtxn = self.env.read_txn()?;
@ -605,22 +606,22 @@ impl IndexScheduler {
for result in self.index_mapper.index_mapping.iter(&rtxn)? { for result in self.index_mapper.index_mapping.iter(&rtxn)? {
let (name, uuid) = result?; let (name, uuid) = result?;
let index = self.index_mapper.index(&rtxn, name)?; let index = self.index_mapper.index(&rtxn, name)?;
let dst = temp_snapshot_dir let dst = temp_snapshot_dir.path().join("indexes").join(uuid.to_string());
.path() fs::create_dir_all(&dst)?;
.join("indexes") index.copy_to_path(dst.join("data.mdb"), CompactionOption::Enabled)?;
.join(uuid.to_string())
.join("data.mdb");
index.copy_to_path(dst, CompactionOption::Enabled)?;
} }
drop(rtxn); drop(rtxn);
// 4. Snapshot the auth LMDB env // 4. Snapshot the auth LMDB env
let dst = temp_snapshot_dir.path().join("auth").join("data.mdb"); let dst = temp_snapshot_dir.path().join("auth");
fs::create_dir_all(&dst)?; fs::create_dir_all(&dst)?;
let src = self.auth_path.join("data.mdb"); // TODO We can't use the open_auth_store_env function here but we should
let auth = milli::heed::EnvOpenOptions::new().open(src)?; let auth = milli::heed::EnvOpenOptions::new()
auth.copy_to_path(dst, CompactionOption::Enabled)?; .map_size(1 * 1024 * 1024 * 1024) // 1 GiB
.max_dbs(2)
.open(&self.auth_path)?;
auth.copy_to_path(dst.join("data.mdb"), CompactionOption::Enabled)?;
// 5. Copy and tarball the flat snapshot // 5. Copy and tarball the flat snapshot
// 5.1 Find the original name of the database // 5.1 Find the original name of the database
@ -630,7 +631,7 @@ impl IndexScheduler {
let db_name = base_path.file_name().and_then(OsStr::to_str).unwrap_or("data.ms"); let db_name = base_path.file_name().and_then(OsStr::to_str).unwrap_or("data.ms");
// 5.2 Tarball the content of the snapshot in a tempfile with a .snapshot extension // 5.2 Tarball the content of the snapshot in a tempfile with a .snapshot extension
let snapshot_path = self.snapshots_path.join(db_name).with_extension("snapshot"); let snapshot_path = self.snapshots_path.join(format!("{}.snapshot", db_name));
let temp_snapshot_file = tempfile::NamedTempFile::new_in(&self.snapshots_path)?; let temp_snapshot_file = tempfile::NamedTempFile::new_in(&self.snapshots_path)?;
compression::to_tar_gz(temp_snapshot_dir.path(), temp_snapshot_file.path())?; compression::to_tar_gz(temp_snapshot_dir.path(), temp_snapshot_file.path())?;
let file = temp_snapshot_file.persist(&snapshot_path)?; let file = temp_snapshot_file.persist(&snapshot_path)?;

2
index-scheduler/src/lib.rs
View File

@ -336,8 +336,8 @@ impl IndexScheduler {
tasks_path: PathBuf, tasks_path: PathBuf,
update_file_path: PathBuf, update_file_path: PathBuf,
indexes_path: PathBuf, indexes_path: PathBuf,
dumps_path: PathBuf,
snapshots_path: PathBuf, snapshots_path: PathBuf,
dumps_path: PathBuf,
task_db_size: usize, task_db_size: usize,
index_size: usize, index_size: usize,
indexer_config: IndexerConfig, indexer_config: IndexerConfig,