mirror of
https://github.com/meilisearch/MeiliSearch
synced 2025-02-03 09:03:47 +01:00
fix the bad index version on opening
This commit is contained in:
parent
4f21ee6c66
commit
7197ced673
@ -38,7 +38,7 @@ fn setup_index() -> Index {
|
|||||||
let mut options = EnvOpenOptions::new();
|
let mut options = EnvOpenOptions::new();
|
||||||
options.map_size(100 * 1024 * 1024 * 1024); // 100 GB
|
options.map_size(100 * 1024 * 1024 * 1024); // 100 GB
|
||||||
options.max_readers(100);
|
options.max_readers(100);
|
||||||
Index::new(options, path).unwrap()
|
Index::new(options, path, true).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn setup_settings<'t>(
|
fn setup_settings<'t>(
|
||||||
|
@ -68,7 +68,7 @@ pub fn base_setup(conf: &Conf) -> Index {
|
|||||||
let mut options = EnvOpenOptions::new();
|
let mut options = EnvOpenOptions::new();
|
||||||
options.map_size(100 * 1024 * 1024 * 1024); // 100 GB
|
options.map_size(100 * 1024 * 1024 * 1024); // 100 GB
|
||||||
options.max_readers(100);
|
options.max_readers(100);
|
||||||
let index = Index::new(options, conf.database_name).unwrap();
|
let index = Index::new(options, conf.database_name, true).unwrap();
|
||||||
|
|
||||||
let config = IndexerConfig::default();
|
let config = IndexerConfig::default();
|
||||||
let mut wtxn = index.write_txn().unwrap();
|
let mut wtxn = index.write_txn().unwrap();
|
||||||
|
@ -63,7 +63,7 @@ fn main() {
|
|||||||
Some(path) => TempDir::new_in(path).unwrap(),
|
Some(path) => TempDir::new_in(path).unwrap(),
|
||||||
None => TempDir::new().unwrap(),
|
None => TempDir::new().unwrap(),
|
||||||
};
|
};
|
||||||
let index = Index::new(options, tempdir.path()).unwrap();
|
let index = Index::new(options, tempdir.path(), true).unwrap();
|
||||||
let indexer_config = IndexerConfig::default();
|
let indexer_config = IndexerConfig::default();
|
||||||
|
|
||||||
std::thread::scope(|s| {
|
std::thread::scope(|s| {
|
||||||
|
@ -102,7 +102,7 @@ impl ReopenableIndex {
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
map.unavailable.remove(&self.uuid);
|
map.unavailable.remove(&self.uuid);
|
||||||
map.create(&self.uuid, path, None, self.enable_mdb_writemap, self.map_size)?;
|
map.create(&self.uuid, path, None, self.enable_mdb_writemap, self.map_size, false)?;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@ -171,11 +171,12 @@ impl IndexMap {
|
|||||||
date: Option<(OffsetDateTime, OffsetDateTime)>,
|
date: Option<(OffsetDateTime, OffsetDateTime)>,
|
||||||
enable_mdb_writemap: bool,
|
enable_mdb_writemap: bool,
|
||||||
map_size: usize,
|
map_size: usize,
|
||||||
|
creation: bool,
|
||||||
) -> Result<Index> {
|
) -> Result<Index> {
|
||||||
if !matches!(self.get_unavailable(uuid), Missing) {
|
if !matches!(self.get_unavailable(uuid), Missing) {
|
||||||
panic!("Attempt to open an index that was unavailable");
|
panic!("Attempt to open an index that was unavailable");
|
||||||
}
|
}
|
||||||
let index = create_or_open_index(path, date, enable_mdb_writemap, map_size)?;
|
let index = create_or_open_index(path, date, enable_mdb_writemap, map_size, creation)?;
|
||||||
match self.available.insert(*uuid, index.clone()) {
|
match self.available.insert(*uuid, index.clone()) {
|
||||||
InsertionOutcome::InsertedNew => (),
|
InsertionOutcome::InsertedNew => (),
|
||||||
InsertionOutcome::Evicted(evicted_uuid, evicted_index) => {
|
InsertionOutcome::Evicted(evicted_uuid, evicted_index) => {
|
||||||
@ -299,6 +300,7 @@ fn create_or_open_index(
|
|||||||
date: Option<(OffsetDateTime, OffsetDateTime)>,
|
date: Option<(OffsetDateTime, OffsetDateTime)>,
|
||||||
enable_mdb_writemap: bool,
|
enable_mdb_writemap: bool,
|
||||||
map_size: usize,
|
map_size: usize,
|
||||||
|
creation: bool,
|
||||||
) -> Result<Index> {
|
) -> Result<Index> {
|
||||||
let mut options = EnvOpenOptions::new();
|
let mut options = EnvOpenOptions::new();
|
||||||
options.map_size(clamp_to_page_size(map_size));
|
options.map_size(clamp_to_page_size(map_size));
|
||||||
@ -308,9 +310,9 @@ fn create_or_open_index(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some((created, updated)) = date {
|
if let Some((created, updated)) = date {
|
||||||
Ok(Index::new_with_creation_dates(options, path, created, updated)?)
|
Ok(Index::new_with_creation_dates(options, path, created, updated, creation)?)
|
||||||
} else {
|
} else {
|
||||||
Ok(Index::new(options, path)?)
|
Ok(Index::new(options, path, creation)?)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -198,6 +198,7 @@ impl IndexMapper {
|
|||||||
date,
|
date,
|
||||||
self.enable_mdb_writemap,
|
self.enable_mdb_writemap,
|
||||||
self.index_base_map_size,
|
self.index_base_map_size,
|
||||||
|
true,
|
||||||
)
|
)
|
||||||
.map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?;
|
.map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?;
|
||||||
let index_rtxn = index.read_txn()?;
|
let index_rtxn = index.read_txn()?;
|
||||||
@ -396,6 +397,7 @@ impl IndexMapper {
|
|||||||
None,
|
None,
|
||||||
self.enable_mdb_writemap,
|
self.enable_mdb_writemap,
|
||||||
self.index_base_map_size,
|
self.index_base_map_size,
|
||||||
|
false,
|
||||||
)
|
)
|
||||||
.map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?;
|
.map_err(|e| Error::from_milli(e, Some(uuid.to_string())))?;
|
||||||
}
|
}
|
||||||
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -303,7 +303,7 @@ fn export_a_dump(
|
|||||||
for result in index_mapping.iter(&rtxn)? {
|
for result in index_mapping.iter(&rtxn)? {
|
||||||
let (uid, uuid) = result?;
|
let (uid, uuid) = result?;
|
||||||
let index_path = db_path.join("indexes").join(uuid.to_string());
|
let index_path = db_path.join("indexes").join(uuid.to_string());
|
||||||
let index = Index::new(EnvOpenOptions::new(), &index_path).with_context(|| {
|
let index = Index::new(EnvOpenOptions::new(), &index_path, false).with_context(|| {
|
||||||
format!("While trying to open the index at path {:?}", index_path.display())
|
format!("While trying to open the index at path {:?}", index_path.display())
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
|
@ -173,10 +173,11 @@ fn rebuild_field_distribution(db_path: &Path) -> anyhow::Result<()> {
|
|||||||
|
|
||||||
println!("\t- Rebuilding field distribution");
|
println!("\t- Rebuilding field distribution");
|
||||||
|
|
||||||
let index = meilisearch_types::milli::Index::new(EnvOpenOptions::new(), &index_path)
|
let index =
|
||||||
.with_context(|| {
|
meilisearch_types::milli::Index::new(EnvOpenOptions::new(), &index_path, false)
|
||||||
format!("while opening index {uid} at '{}'", index_path.display())
|
.with_context(|| {
|
||||||
})?;
|
format!("while opening index {uid} at '{}'", index_path.display())
|
||||||
|
})?;
|
||||||
|
|
||||||
let mut index_txn = index.write_txn()?;
|
let mut index_txn = index.write_txn()?;
|
||||||
|
|
||||||
|
@ -178,6 +178,7 @@ impl Index {
|
|||||||
path: P,
|
path: P,
|
||||||
created_at: time::OffsetDateTime,
|
created_at: time::OffsetDateTime,
|
||||||
updated_at: time::OffsetDateTime,
|
updated_at: time::OffsetDateTime,
|
||||||
|
creation: bool,
|
||||||
) -> Result<Index> {
|
) -> Result<Index> {
|
||||||
use db_name::*;
|
use db_name::*;
|
||||||
|
|
||||||
@ -253,7 +254,7 @@ impl Index {
|
|||||||
embedder_category_id,
|
embedder_category_id,
|
||||||
documents,
|
documents,
|
||||||
};
|
};
|
||||||
if this.get_version(&wtxn)?.is_none() {
|
if this.get_version(&wtxn)?.is_none() && creation {
|
||||||
this.put_version(
|
this.put_version(
|
||||||
&mut wtxn,
|
&mut wtxn,
|
||||||
(
|
(
|
||||||
@ -270,9 +271,13 @@ impl Index {
|
|||||||
Ok(this)
|
Ok(this)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn new<P: AsRef<Path>>(options: heed::EnvOpenOptions, path: P) -> Result<Index> {
|
pub fn new<P: AsRef<Path>>(
|
||||||
|
options: heed::EnvOpenOptions,
|
||||||
|
path: P,
|
||||||
|
creation: bool,
|
||||||
|
) -> Result<Index> {
|
||||||
let now = time::OffsetDateTime::now_utc();
|
let now = time::OffsetDateTime::now_utc();
|
||||||
Self::new_with_creation_dates(options, path, now, now)
|
Self::new_with_creation_dates(options, path, now, now, creation)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn set_creation_dates(
|
fn set_creation_dates(
|
||||||
@ -1802,7 +1807,7 @@ pub(crate) mod tests {
|
|||||||
let mut options = EnvOpenOptions::new();
|
let mut options = EnvOpenOptions::new();
|
||||||
options.map_size(size);
|
options.map_size(size);
|
||||||
let _tempdir = TempDir::new_in(".").unwrap();
|
let _tempdir = TempDir::new_in(".").unwrap();
|
||||||
let inner = Index::new(options, _tempdir.path()).unwrap();
|
let inner = Index::new(options, _tempdir.path(), true).unwrap();
|
||||||
let indexer_config = IndexerConfig::default();
|
let indexer_config = IndexerConfig::default();
|
||||||
let index_documents_config = IndexDocumentsConfig::default();
|
let index_documents_config = IndexDocumentsConfig::default();
|
||||||
Self { inner, indexer_config, index_documents_config, _tempdir }
|
Self { inner, indexer_config, index_documents_config, _tempdir }
|
||||||
|
@ -17,7 +17,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
|
|||||||
let path = tempfile::tempdir().unwrap();
|
let path = tempfile::tempdir().unwrap();
|
||||||
let mut options = EnvOpenOptions::new();
|
let mut options = EnvOpenOptions::new();
|
||||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||||
let index = Index::new(options, &path).unwrap();
|
let index = Index::new(options, &path, true).unwrap();
|
||||||
|
|
||||||
let mut wtxn = index.write_txn().unwrap();
|
let mut wtxn = index.write_txn().unwrap();
|
||||||
let config = IndexerConfig::default();
|
let config = IndexerConfig::default();
|
||||||
|
@ -15,7 +15,7 @@ fn test_facet_distribution_with_no_facet_values() {
|
|||||||
let path = tempfile::tempdir().unwrap();
|
let path = tempfile::tempdir().unwrap();
|
||||||
let mut options = EnvOpenOptions::new();
|
let mut options = EnvOpenOptions::new();
|
||||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||||
let index = Index::new(options, &path).unwrap();
|
let index = Index::new(options, &path, true).unwrap();
|
||||||
|
|
||||||
let mut wtxn = index.write_txn().unwrap();
|
let mut wtxn = index.write_txn().unwrap();
|
||||||
let config = IndexerConfig::default();
|
let config = IndexerConfig::default();
|
||||||
|
@ -34,7 +34,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
|
|||||||
let path = tempfile::tempdir().unwrap();
|
let path = tempfile::tempdir().unwrap();
|
||||||
let mut options = EnvOpenOptions::new();
|
let mut options = EnvOpenOptions::new();
|
||||||
options.map_size(10 * 1024 * 1024); // 10 MB
|
options.map_size(10 * 1024 * 1024); // 10 MB
|
||||||
let index = Index::new(options, &path).unwrap();
|
let index = Index::new(options, &path, true).unwrap();
|
||||||
|
|
||||||
let mut wtxn = index.write_txn().unwrap();
|
let mut wtxn = index.write_txn().unwrap();
|
||||||
let config = IndexerConfig::default();
|
let config = IndexerConfig::default();
|
||||||
|
@ -264,7 +264,7 @@ fn criteria_ascdesc() {
|
|||||||
let path = tempfile::tempdir().unwrap();
|
let path = tempfile::tempdir().unwrap();
|
||||||
let mut options = EnvOpenOptions::new();
|
let mut options = EnvOpenOptions::new();
|
||||||
options.map_size(12 * 1024 * 1024); // 10 MB
|
options.map_size(12 * 1024 * 1024); // 10 MB
|
||||||
let index = Index::new(options, &path).unwrap();
|
let index = Index::new(options, &path, true).unwrap();
|
||||||
|
|
||||||
let mut wtxn = index.write_txn().unwrap();
|
let mut wtxn = index.write_txn().unwrap();
|
||||||
let config = IndexerConfig::default();
|
let config = IndexerConfig::default();
|
||||||
|
@ -110,7 +110,7 @@ fn test_typo_disabled_on_word() {
|
|||||||
let tmp = tempdir().unwrap();
|
let tmp = tempdir().unwrap();
|
||||||
let mut options = EnvOpenOptions::new();
|
let mut options = EnvOpenOptions::new();
|
||||||
options.map_size(4096 * 100);
|
options.map_size(4096 * 100);
|
||||||
let index = Index::new(options, tmp.path()).unwrap();
|
let index = Index::new(options, tmp.path(), true).unwrap();
|
||||||
|
|
||||||
let doc1: Object = from_value(json!({ "id": 1usize, "data": "zealand" })).unwrap();
|
let doc1: Object = from_value(json!({ "id": 1usize, "data": "zealand" })).unwrap();
|
||||||
let doc2: Object = from_value(json!({ "id": 2usize, "data": "zearand" })).unwrap();
|
let doc2: Object = from_value(json!({ "id": 2usize, "data": "zearand" })).unwrap();
|
||||||
|
Loading…
x
Reference in New Issue
Block a user