Clamp the databases size to the page size

This commit is contained in:
Kerollmops 2022-11-29 10:38:27 +01:00 committed by curquiza
parent 24c79b79f9
commit e4e4370a3c
5 changed files with 22 additions and 5 deletions

13
Cargo.lock generated
View File

@ -1625,7 +1625,7 @@ dependencies = [
"libc",
"lmdb-rkv-sys",
"once_cell",
"page_size",
"page_size 0.4.2",
"synchronoise",
"url",
"zerocopy",
@ -1783,6 +1783,7 @@ dependencies = [
"meili-snap",
"meilisearch-types",
"nelson",
"page_size 0.5.0",
"roaring",
"serde",
"serde_json",
@ -2663,6 +2664,16 @@ dependencies = [
"winapi",
]
[[package]]
name = "page_size"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b7663cbd190cfd818d08efa8497f6cd383076688c49a391ef7c0d03cd12b561"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "parking_lot"
version = "0.12.1"

View File

@ -13,6 +13,7 @@ enum-iterator = "1.1.3"
file-store = { path = "../file-store" }
log = "0.4.14"
meilisearch-types = { path = "../meilisearch-types" }
page_size = "0.5.0"
roaring = { version = "0.10.0", features = ["serde"] }
serde = { version = "1.0.136", features = ["derive"] }
serde_json = { version = "1.0.85", features = ["preserve_order"] }

View File

@ -13,7 +13,7 @@ use uuid::Uuid;
use self::IndexStatus::{Available, BeingDeleted};
use crate::uuid_codec::UuidCodec;
use crate::{Error, Result};
use crate::{clamp_to_page_size, Error, Result};
const INDEX_MAPPING: &str = "index-mapping";
@ -68,7 +68,7 @@ impl IndexMapper {
/// The path *must* exists or an error will be thrown.
fn create_or_open_index(&self, path: &Path) -> Result<Index> {
let mut options = EnvOpenOptions::new();
options.map_size(self.index_size);
options.map_size(clamp_to_page_size(self.index_size));
options.max_readers(1024);
Ok(Index::new(options, path)?)
}

View File

@ -55,7 +55,7 @@ use utils::{filter_out_references_to_newer_tasks, keep_tasks_within_datetimes, m
use uuid::Uuid;
use crate::index_mapper::IndexMapper;
use crate::utils::check_index_swap_validity;
use crate::utils::{check_index_swap_validity, clamp_to_page_size};
pub(crate) type BEI128 =
meilisearch_types::heed::zerocopy::I128<meilisearch_types::heed::byteorder::BE>;
@ -362,7 +362,7 @@ impl IndexScheduler {
let env = heed::EnvOpenOptions::new()
.max_dbs(10)
.map_size(options.task_db_size)
.map_size(clamp_to_page_size(options.task_db_size))
.open(options.tasks_path)?;
let file_store = FileStore::new(&options.update_file_path)?;

View File

@ -324,6 +324,11 @@ pub(crate) fn check_index_swap_validity(task: &Task) -> Result<()> {
Ok(())
}
/// Clamp the provided value to be a multiple of system page size.
pub fn clamp_to_page_size(size: usize) -> usize {
size / page_size::get() * page_size::get()
}
#[cfg(test)]
impl IndexScheduler {
/// Asserts that the index scheduler's content is internally consistent.