diff --git a/crates/benchmarks/benches/indexing.rs b/crates/benchmarks/benches/indexing.rs index 4acd7b22a..7c1783a1a 100644 --- a/crates/benchmarks/benches/indexing.rs +++ b/crates/benchmarks/benches/indexing.rs @@ -38,7 +38,7 @@ fn setup_index() -> Index { let mut options = EnvOpenOptions::new(); options.map_size(100 * 1024 * 1024 * 1024); // 100 GB options.max_readers(100); - Index::new(options, path).unwrap() + Index::new(options, path, true).unwrap() } fn setup_settings<'t>( diff --git a/crates/benchmarks/benches/utils.rs b/crates/benchmarks/benches/utils.rs index 889478d40..b472b4f6b 100644 --- a/crates/benchmarks/benches/utils.rs +++ b/crates/benchmarks/benches/utils.rs @@ -68,7 +68,7 @@ pub fn base_setup(conf: &Conf) -> Index { let mut options = EnvOpenOptions::new(); options.map_size(100 * 1024 * 1024 * 1024); // 100 GB options.max_readers(100); - let index = Index::new(options, conf.database_name).unwrap(); + let index = Index::new(options, conf.database_name, true).unwrap(); let config = IndexerConfig::default(); let mut wtxn = index.write_txn().unwrap(); diff --git a/crates/fuzzers/src/bin/fuzz-indexing.rs b/crates/fuzzers/src/bin/fuzz-indexing.rs index 08711e5e3..1216083ca 100644 --- a/crates/fuzzers/src/bin/fuzz-indexing.rs +++ b/crates/fuzzers/src/bin/fuzz-indexing.rs @@ -63,7 +63,7 @@ fn main() { Some(path) => TempDir::new_in(path).unwrap(), None => TempDir::new().unwrap(), }; - let index = Index::new(options, tempdir.path()).unwrap(); + let index = Index::new(options, tempdir.path(), true).unwrap(); let indexer_config = IndexerConfig::default(); std::thread::scope(|s| { diff --git a/crates/index-scheduler/src/index_mapper/index_map.rs b/crates/index-scheduler/src/index_mapper/index_map.rs index 947f558aa..3031043a9 100644 --- a/crates/index-scheduler/src/index_mapper/index_map.rs +++ b/crates/index-scheduler/src/index_mapper/index_map.rs @@ -102,7 +102,7 @@ impl ReopenableIndex { return Ok(()); } map.unavailable.remove(&self.uuid); - map.create(&self.uuid, path, None, self.enable_mdb_writemap, self.map_size)?; + map.create(&self.uuid, path, None, self.enable_mdb_writemap, self.map_size, false)?; } Ok(()) } @@ -171,11 +171,12 @@ impl IndexMap { date: Option<(OffsetDateTime, OffsetDateTime)>, enable_mdb_writemap: bool, map_size: usize, + creation: bool, ) -> Result { if !matches!(self.get_unavailable(uuid), Missing) { panic!("Attempt to open an index that was unavailable"); } - let index = create_or_open_index(path, date, enable_mdb_writemap, map_size)?; + let index = create_or_open_index(path, date, enable_mdb_writemap, map_size, creation)?; match self.available.insert(*uuid, index.clone()) { InsertionOutcome::InsertedNew => (), InsertionOutcome::Evicted(evicted_uuid, evicted_index) => { @@ -299,6 +300,7 @@ fn create_or_open_index( date: Option<(OffsetDateTime, OffsetDateTime)>, enable_mdb_writemap: bool, map_size: usize, + creation: bool, ) -> Result { let mut options = EnvOpenOptions::new(); options.map_size(clamp_to_page_size(map_size)); @@ -308,9 +310,9 @@ fn create_or_open_index( } if let Some((created, updated)) = date { - Ok(Index::new_with_creation_dates(options, path, created, updated)?) + Ok(Index::new_with_creation_dates(options, path, created, updated, creation)?) } else { - Ok(Index::new(options, path)?) + Ok(Index::new(options, path, creation)?) } } diff --git a/crates/index-scheduler/src/index_mapper/mod.rs b/crates/index-scheduler/src/index_mapper/mod.rs index cc5e616ed..dad73d4c6 100644 --- a/crates/index-scheduler/src/index_mapper/mod.rs +++ b/crates/index-scheduler/src/index_mapper/mod.rs @@ -198,6 +198,7 @@ impl IndexMapper { date, self.enable_mdb_writemap, self.index_base_map_size, + true, ) .map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?; let index_rtxn = index.read_txn()?; @@ -396,6 +397,7 @@ impl IndexMapper { None, self.enable_mdb_writemap, self.index_base_map_size, + false, ) .map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?; } diff --git a/crates/meilisearch/db.snapshot b/crates/meilisearch/db.snapshot index 73a3f0f8e..3cec193fb 100644 Binary files a/crates/meilisearch/db.snapshot and b/crates/meilisearch/db.snapshot differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb index 9154fcee2..2f097330d 100644 Binary files a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb differ diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb index 2fcbf7530..b27d4eb0b 100644 Binary files a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb and b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb differ diff --git a/crates/meilitool/src/main.rs b/crates/meilitool/src/main.rs index 599bc3274..0d59333e3 100644 --- a/crates/meilitool/src/main.rs +++ b/crates/meilitool/src/main.rs @@ -303,7 +303,7 @@ fn export_a_dump( for result in index_mapping.iter(&rtxn)? { let (uid, uuid) = result?; let index_path = db_path.join("indexes").join(uuid.to_string()); - let index = Index::new(EnvOpenOptions::new(), &index_path).with_context(|| { + let index = Index::new(EnvOpenOptions::new(), &index_path, false).with_context(|| { format!("While trying to open the index at path {:?}", index_path.display()) })?; diff --git a/crates/meilitool/src/upgrade/v1_12.rs b/crates/meilitool/src/upgrade/v1_12.rs index 67a19c370..3ad171c31 100644 --- a/crates/meilitool/src/upgrade/v1_12.rs +++ b/crates/meilitool/src/upgrade/v1_12.rs @@ -173,10 +173,11 @@ fn rebuild_field_distribution(db_path: &Path) -> anyhow::Result<()> { println!("\t- Rebuilding field distribution"); - let index = meilisearch_types::milli::Index::new(EnvOpenOptions::new(), &index_path) - .with_context(|| { - format!("while opening index {uid} at '{}'", index_path.display()) - })?; + let index = + meilisearch_types::milli::Index::new(EnvOpenOptions::new(), &index_path, false) + .with_context(|| { + format!("while opening index {uid} at '{}'", index_path.display()) + })?; let mut index_txn = index.write_txn()?; diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index bda57b531..944fb6cd4 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -178,6 +178,7 @@ impl Index { path: P, created_at: time::OffsetDateTime, updated_at: time::OffsetDateTime, + creation: bool, ) -> Result { use db_name::*; @@ -253,7 +254,7 @@ impl Index { embedder_category_id, documents, }; - if this.get_version(&wtxn)?.is_none() { + if this.get_version(&wtxn)?.is_none() && creation { this.put_version( &mut wtxn, ( @@ -270,9 +271,13 @@ impl Index { Ok(this) } - pub fn new>(options: heed::EnvOpenOptions, path: P) -> Result { + pub fn new>( + options: heed::EnvOpenOptions, + path: P, + creation: bool, + ) -> Result { let now = time::OffsetDateTime::now_utc(); - Self::new_with_creation_dates(options, path, now, now) + Self::new_with_creation_dates(options, path, now, now, creation) } fn set_creation_dates( @@ -1802,7 +1807,7 @@ pub(crate) mod tests { let mut options = EnvOpenOptions::new(); options.map_size(size); let _tempdir = TempDir::new_in(".").unwrap(); - let inner = Index::new(options, _tempdir.path()).unwrap(); + let inner = Index::new(options, _tempdir.path(), true).unwrap(); let indexer_config = IndexerConfig::default(); let index_documents_config = IndexDocumentsConfig::default(); Self { inner, indexer_config, index_documents_config, _tempdir } diff --git a/crates/milli/src/search/new/tests/integration.rs b/crates/milli/src/search/new/tests/integration.rs index fc15b5f12..99d5dc033 100644 --- a/crates/milli/src/search/new/tests/integration.rs +++ b/crates/milli/src/search/new/tests/integration.rs @@ -17,7 +17,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { let path = tempfile::tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = Index::new(options, &path, true).unwrap(); let mut wtxn = index.write_txn().unwrap(); let config = IndexerConfig::default(); diff --git a/crates/milli/tests/search/facet_distribution.rs b/crates/milli/tests/search/facet_distribution.rs index ced81409d..db9f86357 100644 --- a/crates/milli/tests/search/facet_distribution.rs +++ b/crates/milli/tests/search/facet_distribution.rs @@ -15,7 +15,7 @@ fn test_facet_distribution_with_no_facet_values() { let path = tempfile::tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = Index::new(options, &path, true).unwrap(); let mut wtxn = index.write_txn().unwrap(); let config = IndexerConfig::default(); diff --git a/crates/milli/tests/search/mod.rs b/crates/milli/tests/search/mod.rs index 30690969b..662715638 100644 --- a/crates/milli/tests/search/mod.rs +++ b/crates/milli/tests/search/mod.rs @@ -34,7 +34,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { let path = tempfile::tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(10 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = Index::new(options, &path, true).unwrap(); let mut wtxn = index.write_txn().unwrap(); let config = IndexerConfig::default(); diff --git a/crates/milli/tests/search/query_criteria.rs b/crates/milli/tests/search/query_criteria.rs index 304059915..d47c9539d 100644 --- a/crates/milli/tests/search/query_criteria.rs +++ b/crates/milli/tests/search/query_criteria.rs @@ -264,7 +264,7 @@ fn criteria_ascdesc() { let path = tempfile::tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(12 * 1024 * 1024); // 10 MB - let index = Index::new(options, &path).unwrap(); + let index = Index::new(options, &path, true).unwrap(); let mut wtxn = index.write_txn().unwrap(); let config = IndexerConfig::default(); diff --git a/crates/milli/tests/search/typo_tolerance.rs b/crates/milli/tests/search/typo_tolerance.rs index d33d79e54..b640fa910 100644 --- a/crates/milli/tests/search/typo_tolerance.rs +++ b/crates/milli/tests/search/typo_tolerance.rs @@ -110,7 +110,7 @@ fn test_typo_disabled_on_word() { let tmp = tempdir().unwrap(); let mut options = EnvOpenOptions::new(); options.map_size(4096 * 100); - let index = Index::new(options, tmp.path()).unwrap(); + let index = Index::new(options, tmp.path(), true).unwrap(); let doc1: Object = from_value(json!({ "id": 1usize, "data": "zealand" })).unwrap(); let doc2: Object = from_value(json!({ "id": 2usize, "data": "zearand" })).unwrap();