fix the bad index version on opening

This commit is contained in:
Tamo 2025-01-23 12:31:10 +01:00 committed by Louis Dureuil
parent 4f21ee6c66
commit 7197ced673
No known key found for this signature in database
16 changed files with 31 additions and 21 deletions

View File

@ -38,7 +38,7 @@ fn setup_index() -> Index {
let mut options = EnvOpenOptions::new();
options.map_size(100 * 1024 * 1024 * 1024); // 100 GB
options.max_readers(100);
Index::new(options, path).unwrap()
Index::new(options, path, true).unwrap()
}
fn setup_settings<'t>(

View File

@ -68,7 +68,7 @@ pub fn base_setup(conf: &Conf) -> Index {
let mut options = EnvOpenOptions::new();
options.map_size(100 * 1024 * 1024 * 1024); // 100 GB
options.max_readers(100);
let index = Index::new(options, conf.database_name).unwrap();
let index = Index::new(options, conf.database_name, true).unwrap();
let config = IndexerConfig::default();
let mut wtxn = index.write_txn().unwrap();

View File

@ -63,7 +63,7 @@ fn main() {
Some(path) => TempDir::new_in(path).unwrap(),
None => TempDir::new().unwrap(),
};
let index = Index::new(options, tempdir.path()).unwrap();
let index = Index::new(options, tempdir.path(), true).unwrap();
let indexer_config = IndexerConfig::default();
std::thread::scope(|s| {

View File

@ -102,7 +102,7 @@ impl ReopenableIndex {
return Ok(());
}
map.unavailable.remove(&self.uuid);
map.create(&self.uuid, path, None, self.enable_mdb_writemap, self.map_size)?;
map.create(&self.uuid, path, None, self.enable_mdb_writemap, self.map_size, false)?;
}
Ok(())
}
@ -171,11 +171,12 @@ impl IndexMap {
date: Option<(OffsetDateTime, OffsetDateTime)>,
enable_mdb_writemap: bool,
map_size: usize,
creation: bool,
) -> Result<Index> {
if !matches!(self.get_unavailable(uuid), Missing) {
panic!("Attempt to open an index that was unavailable");
}
let index = create_or_open_index(path, date, enable_mdb_writemap, map_size)?;
let index = create_or_open_index(path, date, enable_mdb_writemap, map_size, creation)?;
match self.available.insert(*uuid, index.clone()) {
InsertionOutcome::InsertedNew => (),
InsertionOutcome::Evicted(evicted_uuid, evicted_index) => {
@ -299,6 +300,7 @@ fn create_or_open_index(
date: Option<(OffsetDateTime, OffsetDateTime)>,
enable_mdb_writemap: bool,
map_size: usize,
creation: bool,
) -> Result<Index> {
let mut options = EnvOpenOptions::new();
options.map_size(clamp_to_page_size(map_size));
@ -308,9 +310,9 @@ fn create_or_open_index(
}
if let Some((created, updated)) = date {
Ok(Index::new_with_creation_dates(options, path, created, updated)?)
Ok(Index::new_with_creation_dates(options, path, created, updated, creation)?)
} else {
Ok(Index::new(options, path)?)
Ok(Index::new(options, path, creation)?)
}
}

View File

@ -198,6 +198,7 @@ impl IndexMapper {
date,
self.enable_mdb_writemap,
self.index_base_map_size,
true,
)
.map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?;
let index_rtxn = index.read_txn()?;
@ -396,6 +397,7 @@ impl IndexMapper {
None,
self.enable_mdb_writemap,
self.index_base_map_size,
false,
)
.map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?;
}

Binary file not shown.

View File

@ -303,7 +303,7 @@ fn export_a_dump(
for result in index_mapping.iter(&rtxn)? {
let (uid, uuid) = result?;
let index_path = db_path.join("indexes").join(uuid.to_string());
let index = Index::new(EnvOpenOptions::new(), &index_path).with_context(|| {
let index = Index::new(EnvOpenOptions::new(), &index_path, false).with_context(|| {
format!("While trying to open the index at path {:?}", index_path.display())
})?;

View File

@ -173,10 +173,11 @@ fn rebuild_field_distribution(db_path: &Path) -> anyhow::Result<()> {
println!("\t- Rebuilding field distribution");
let index = meilisearch_types::milli::Index::new(EnvOpenOptions::new(), &index_path)
.with_context(|| {
format!("while opening index {uid} at '{}'", index_path.display())
})?;
let index =
meilisearch_types::milli::Index::new(EnvOpenOptions::new(), &index_path, false)
.with_context(|| {
format!("while opening index {uid} at '{}'", index_path.display())
})?;
let mut index_txn = index.write_txn()?;

View File

@ -178,6 +178,7 @@ impl Index {
path: P,
created_at: time::OffsetDateTime,
updated_at: time::OffsetDateTime,
creation: bool,
) -> Result<Index> {
use db_name::*;
@ -253,7 +254,7 @@ impl Index {
embedder_category_id,
documents,
};
if this.get_version(&wtxn)?.is_none() {
if this.get_version(&wtxn)?.is_none() && creation {
this.put_version(
&mut wtxn,
(
@ -270,9 +271,13 @@ impl Index {
Ok(this)
}
pub fn new<P: AsRef<Path>>(options: heed::EnvOpenOptions, path: P) -> Result<Index> {
pub fn new<P: AsRef<Path>>(
options: heed::EnvOpenOptions,
path: P,
creation: bool,
) -> Result<Index> {
let now = time::OffsetDateTime::now_utc();
Self::new_with_creation_dates(options, path, now, now)
Self::new_with_creation_dates(options, path, now, now, creation)
}
fn set_creation_dates(
@ -1802,7 +1807,7 @@ pub(crate) mod tests {
let mut options = EnvOpenOptions::new();
options.map_size(size);
let _tempdir = TempDir::new_in(".").unwrap();
let inner = Index::new(options, _tempdir.path()).unwrap();
let inner = Index::new(options, _tempdir.path(), true).unwrap();
let indexer_config = IndexerConfig::default();
let index_documents_config = IndexDocumentsConfig::default();
Self { inner, indexer_config, index_documents_config, _tempdir }

View File

@ -17,7 +17,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = Index::new(options, &path, true).unwrap();
let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();

View File

@ -15,7 +15,7 @@ fn test_facet_distribution_with_no_facet_values() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = Index::new(options, &path, true).unwrap();
let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();

View File

@ -34,7 +34,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = Index::new(options, &path, true).unwrap();
let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();

View File

@ -264,7 +264,7 @@ fn criteria_ascdesc() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(12 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = Index::new(options, &path, true).unwrap();
let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();

View File

@ -110,7 +110,7 @@ fn test_typo_disabled_on_word() {
let tmp = tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(4096 * 100);
let index = Index::new(options, tmp.path()).unwrap();
let index = Index::new(options, tmp.path(), true).unwrap();
let doc1: Object = from_value(json!({ "id": 1usize, "data": "zealand" })).unwrap();
let doc2: Object = from_value(json!({ "id": 2usize, "data": "zearand" })).unwrap();