searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 23:04:26 +01:00
Code Issues Releases Wiki Activity

Merge #600

Browse Source 
600: Simplify some unit tests r=ManyTheFish a=loiclec

# Pull Request

## What does this PR do?
Simplify the code that is used in unit tests to create and modify an index. Basically, the following code:
```rust
  let path = tempfile::tempdir().unwrap();
  let mut options = EnvOpenOptions::new();
  options.map_size(10 * 1024 * 1024); // 10 MB
  let index = Index::new(options, &path).unwrap();

  let mut wtxn = index.write_txn().unwrap();
  let content = documents!([
      { "id": 0, "name": "kevin" },
  ]);
  let config = IndexerConfig::default();
  let indexing_config = IndexDocumentsConfig::default();
  let builder =
      IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
  let (builder, user_error) = builder.add_documents(content).unwrap();
  user_error.unwrap();
  builder.execute().unwrap();
  wtxn.commit.unwrap();

  let mut wtxn = index.write_txn().unwrap();
  let config = IndexerConfig::default();
  let mut builder = Settings::new(&mut wtxn, &index, &config);
  builder.set_primary_key(S("docid"));
  builder.set_filterable_fields(hashset! { S("label") });
  builder.execute(|_| ()).unwrap();
  wtxn.commit().unwrap();
```
becomes:
```rust
let index = TempIndex::new():
index.add_documents(documents!(
      { "id": 0, "name": "kevin" },
)).unwrap();
index.update_settings(|settings| {
    settings.set_primary_key(S("docid"));
    settings.set_filterable_fields(hashset! { S("label") });
}).unwrap();
```

Then there is a bunch of options to modify the indexing configs, the map size, to reuse a transaction, etc. For example:
```rust
let mut index = TempIndex::new_with_map_size(1000 * 4096 * 10);
index.index_documents_config.autogenerate_docids = true;
let mut wtxn = index.write_txn().unwrap();
index.update_settings_using_wtxn(&mut wtxn, |settings| {
    settings.set_primary_key(S("docids"));
}).unwrap();
wtxn.commit().unwrap();
```

Co-authored-by: Loïc Lecrenier <loic@meilisearch.com>
Co-authored-by: Loïc Lecrenier <loic.lecrenier@me.com>
Co-authored-by: bors[bot] <26634292+bors[bot]@users.noreply.github.com>
This commit is contained in:
bors[bot] 2022-08-04 10:19:42 +00:00 committed by GitHub
commit 950d8e4c44
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 1123 additions and 1816 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
milli/src/documents/mod.rs
View File

@ -179,6 +179,17 @@ macro_rules! documents {
}}; }};
} }
#[cfg(test)]
pub fn documents_batch_reader_from_objects(
objects: impl IntoIterator<Item = Object>,
) -> DocumentsBatchReader<std::io::Cursor<Vec<u8>>> {
let mut builder = DocumentsBatchBuilder::new(Vec::new());
for object in objects {
builder.append_json_object(&object).unwrap();
}
DocumentsBatchReader::from_reader(std::io::Cursor::new(builder.into_inner().unwrap())).unwrap()
}
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use std::io::Cursor; use std::io::Cursor;

229
milli/src/index.rs
View File

@ -1182,16 +1182,19 @@ pub(crate) mod tests {
use std::ops::Deref; use std::ops::Deref;
use big_s::S; use big_s::S;
use heed::EnvOpenOptions; use heed::{EnvOpenOptions, RwTxn};
use maplit::btreemap; use maplit::btreemap;
use tempfile::TempDir; use tempfile::TempDir;
use crate::documents::DocumentsBatchReader;
use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS}; use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS};
use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig}; use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
use crate::Index; use crate::Index;
pub(crate) struct TempIndex { pub(crate) struct TempIndex {
inner: Index, pub inner: Index,
pub indexer_config: IndexerConfig,
pub index_documents_config: IndexDocumentsConfig,
_tempdir: TempDir, _tempdir: TempDir,
} }
@ -1204,43 +1207,88 @@ pub(crate) mod tests {
} }
impl TempIndex { impl TempIndex {
/// Creates a temporary index, with a default `4096 * 100` size. This should be enough for /// Creates a temporary index
/// most tests. pub fn new_with_map_size(size: usize) -> Self {
pub fn new() -> Self {
let mut options = EnvOpenOptions::new(); let mut options = EnvOpenOptions::new();
options.map_size(100 * 4096); options.map_size(size);
let _tempdir = TempDir::new_in(".").unwrap(); let _tempdir = TempDir::new_in(".").unwrap();
let inner = Index::new(options, _tempdir.path()).unwrap(); let inner = Index::new(options, _tempdir.path()).unwrap();
Self { inner, _tempdir } let indexer_config = IndexerConfig::default();
let index_documents_config = IndexDocumentsConfig::default();
Self { inner, indexer_config, index_documents_config, _tempdir }
}
/// Creates a temporary index, with a default `4096 * 1000` size. This should be enough for
/// most tests.
pub fn new() -> Self {
Self::new_with_map_size(4096 * 1000)
}
pub fn add_documents_using_wtxn<'t, R>(
&'t self,
wtxn: &mut RwTxn<'t, '_>,
documents: DocumentsBatchReader<R>,
) -> Result<(), crate::error::Error>
where
R: std::io::Read + std::io::Seek,
{
let builder = IndexDocuments::new(
wtxn,
&self,
&self.indexer_config,
self.index_documents_config.clone(),
|_| (),
)
.unwrap();
let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error?;
builder.execute()?;
Ok(())
}
pub fn add_documents<R>(
&self,
documents: DocumentsBatchReader<R>,
) -> Result<(), crate::error::Error>
where
R: std::io::Read + std::io::Seek,
{
let mut wtxn = self.write_txn().unwrap();
self.add_documents_using_wtxn(&mut wtxn, documents)?;
wtxn.commit().unwrap();
Ok(())
}
pub fn update_settings(
&self,
update: impl Fn(&mut Settings),
) -> Result<(), crate::error::Error> {
let mut wtxn = self.write_txn().unwrap();
self.update_settings_using_wtxn(&mut wtxn, update)?;
wtxn.commit().unwrap();
Ok(())
}
pub fn update_settings_using_wtxn<'t>(
&'t self,
wtxn: &mut RwTxn<'t, '_>,
update: impl Fn(&mut Settings),
) -> Result<(), crate::error::Error> {
let mut builder = update::Settings::new(wtxn, &self.inner, &self.indexer_config);
update(&mut builder);
builder.execute(drop)?;
Ok(())
} }
} }
#[test] #[test]
fn initial_field_distribution() { fn initial_field_distribution() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new(); index
options.map_size(10 * 1024 * 1024); // 10 MB .add_documents(documents!([
let index = Index::new(options, &path).unwrap(); { "id": 1, "name": "kevin" },
{ "id": 2, "name": "bob", "age": 20 },
let mut wtxn = index.write_txn().unwrap(); { "id": 2, "name": "bob", "age": 20 },
let content = documents!([ ]))
{ "id": 1, "name": "kevin" }, .unwrap();
{ "id": 2, "name": "bob", "age": 20 },
{ "id": 2, "name": "bob", "age": 20 },
]);
let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
let field_distribution = index.field_distribution(&rtxn).unwrap(); let field_distribution = index.field_distribution(&rtxn).unwrap();
assert_eq!( assert_eq!(
field_distribution, field_distribution,
@ -1253,19 +1301,13 @@ pub(crate) mod tests {
// we add all the documents a second time. we are supposed to get the same // we add all the documents a second time. we are supposed to get the same
// field_distribution in the end // field_distribution in the end
let mut wtxn = index.write_txn().unwrap(); index
let builder = .add_documents(documents!([
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) { "id": 1, "name": "kevin" },
.unwrap(); { "id": 2, "name": "bob", "age": 20 },
let content = documents!([ { "id": 2, "name": "bob", "age": 20 },
{ "id": 1, "name": "kevin" }, ]))
{ "id": 2, "name": "bob", "age": 20 }, .unwrap();
{ "id": 2, "name": "bob", "age": 20 },
]);
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
@ -1280,19 +1322,12 @@ pub(crate) mod tests {
); );
// then we update a document by removing one field and another by adding one field // then we update a document by removing one field and another by adding one field
let content = documents!([ index
{ "id": 1, "name": "kevin", "has_dog": true }, .add_documents(documents!([
{ "id": 2, "name": "bob" } { "id": 1, "name": "kevin", "has_dog": true },
]); { "id": 2, "name": "bob" }
]))
let mut wtxn = index.write_txn().unwrap(); .unwrap();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
@ -1341,35 +1376,19 @@ pub(crate) mod tests {
#[test] #[test]
fn add_documents_and_set_searchable_fields() { fn add_documents_and_set_searchable_fields() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new(); index
options.map_size(10 * 1024 * 1024); // 10 MB .add_documents(documents!([
let index = Index::new(options, &path).unwrap(); { "id": 1, "doggo": "kevin" },
{ "id": 2, "doggo": { "name": "bob", "age": 20 } },
let mut wtxn = index.write_txn().unwrap(); { "id": 3, "name": "jean", "age": 25 },
let content = documents!([ ]))
{ "id": 1, "doggo": "kevin" }, .unwrap();
{ "id": 2, "doggo": { "name": "bob", "age": 20 } }, index
{ "id": 3, "name": "jean", "age": 25 }, .update_settings(|settings| {
]); settings.set_searchable_fields(vec![S("doggo"), S("name")]);
})
let config = IndexerConfig::default(); .unwrap();
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// set searchable fields
let mut wtxn = index.write_txn().unwrap();
let mut builder = update::Settings::new(&mut wtxn, &index, &config);
builder.set_searchable_fields(vec![S("doggo"), S("name")]);
builder.execute(drop).unwrap();
wtxn.commit().unwrap();
// ensure we get the right real searchable fields + user defined searchable fields // ensure we get the right real searchable fields + user defined searchable fields
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
@ -1383,19 +1402,13 @@ pub(crate) mod tests {
#[test] #[test]
fn set_searchable_fields_and_add_documents() { fn set_searchable_fields_and_add_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let config = IndexerConfig::default();
// set searchable fields index
let mut wtxn = index.write_txn().unwrap(); .update_settings(|settings| {
let mut builder = update::Settings::new(&mut wtxn, &index, &config); settings.set_searchable_fields(vec![S("doggo"), S("name")]);
builder.set_searchable_fields(vec![S("doggo"), S("name")]); })
.unwrap();
builder.execute(drop).unwrap();
wtxn.commit().unwrap();
// ensure we get the right real searchable fields + user defined searchable fields // ensure we get the right real searchable fields + user defined searchable fields
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
@ -1405,21 +1418,13 @@ pub(crate) mod tests {
let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap(); let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap();
assert_eq!(user_defined, &["doggo", "name"]); assert_eq!(user_defined, &["doggo", "name"]);
let mut wtxn = index.write_txn().unwrap(); index
let content = documents!([ .add_documents(documents!([
{ "id": 1, "doggo": "kevin" }, { "id": 1, "doggo": "kevin" },
{ "id": 2, "doggo": { "name": "bob", "age": 20 } }, { "id": 2, "doggo": { "name": "bob", "age": 20 } },
{ "id": 3, "name": "jean", "age": 25 }, { "id": 3, "name": "jean", "age": 25 },
]); ]))
.unwrap();
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// ensure we get the right real searchable fields + user defined searchable fields // ensure we get the right real searchable fields + user defined searchable fields
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();

138
milli/src/search/facet/filter.rs
View File

@ -494,28 +494,21 @@ mod tests {
use big_s::S; use big_s::S;
use either::Either; use either::Either;
use heed::EnvOpenOptions;
use maplit::hashset; use maplit::hashset;
use super::*; use crate::index::tests::TempIndex;
use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings}; use crate::Filter;
use crate::Index;
#[test] #[test]
fn empty_db() { fn empty_db() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
// Set the filterable fields to be the channel. // Set the filterable fields to be the channel.
let config = IndexerConfig::default(); index
let mut wtxn = index.write_txn().unwrap(); .update_settings(|settings| {
let mut builder = Settings::new(&mut wtxn, &index, &config); settings.set_searchable_fields(vec![S("PrIcE")]); // to keep the fields order
builder.set_searchable_fields(vec![S("PrIcE")]); // to keep the fields order settings.set_filterable_fields(hashset! { S("PrIcE") });
builder.set_filterable_fields(hashset! { S("PrIcE") }); })
builder.execute(|_| ()).unwrap(); .unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
@ -592,10 +585,7 @@ mod tests {
#[test] #[test]
fn not_filterable() { fn not_filterable() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
let filter = Filter::from_str("_geoRadius(42, 150, 10)").unwrap().unwrap(); let filter = Filter::from_str("_geoRadius(42, 150, 10)").unwrap().unwrap();
@ -611,14 +601,12 @@ mod tests {
)); ));
drop(rtxn); drop(rtxn);
let config = IndexerConfig::default(); index
// Set the filterable fields to be the channel. .update_settings(|settings| {
let mut wtxn = index.write_txn().unwrap(); settings.set_searchable_fields(vec![S("title")]);
let mut builder = Settings::new(&mut wtxn, &index, &config); settings.set_filterable_fields(hashset! { S("title") });
builder.set_searchable_fields(vec![S("title")]); })
builder.set_filterable_fields(hashset! { S("title") }); .unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
@ -637,92 +625,64 @@ mod tests {
#[test] #[test]
fn escaped_quote_in_filter_value_2380() { fn escaped_quote_in_filter_value_2380() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); index
let content = documents!([ .add_documents(documents!([
{ {
"id": "test_1", "id": "test_1",
"monitor_diagonal": "27' to 30'" "monitor_diagonal": "27' to 30'"
}, },
{ {
"id": "test_2", "id": "test_2",
"monitor_diagonal": "27\" to 30\"" "monitor_diagonal": "27\" to 30\""
}, },
{ {
"id": "test_3", "id": "test_3",
"monitor_diagonal": "27\" to 30'" "monitor_diagonal": "27\" to 30'"
}, },
]); ]))
.unwrap();
let config = IndexerConfig::default(); index
let indexing_config = IndexDocumentsConfig::default(); .update_settings(|settings| {
let builder = settings.set_filterable_fields(hashset!(S("monitor_diagonal")));
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) })
.unwrap(); .unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
let mut wtxn = index.write_txn().unwrap();
let mut builder = update::Settings::new(&mut wtxn, &index, &config);
builder.set_filterable_fields(hashset!(S("monitor_diagonal")));
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
let mut search = crate::Search::new(&rtxn, &index); let mut search = crate::Search::new(&rtxn, &index);
// this filter is copy pasted from #2380 with the exact same espace sequence // this filter is copy pasted from #2380 with the exact same espace sequence
search.filter( search.filter(Filter::from_str("monitor_diagonal = '27\" to 30\\''").unwrap().unwrap());
crate::Filter::from_str("monitor_diagonal = '27\" to 30\\''").unwrap().unwrap(),
);
let crate::SearchResult { documents_ids, .. } = search.execute().unwrap(); let crate::SearchResult { documents_ids, .. } = search.execute().unwrap();
assert_eq!(documents_ids, vec![2]); assert_eq!(documents_ids, vec![2]);
search.filter( search.filter(Filter::from_str(r#"monitor_diagonal = "27' to 30'" "#).unwrap().unwrap());
crate::Filter::from_str(r#"monitor_diagonal = "27' to 30'" "#).unwrap().unwrap(),
);
let crate::SearchResult { documents_ids, .. } = search.execute().unwrap(); let crate::SearchResult { documents_ids, .. } = search.execute().unwrap();
assert_eq!(documents_ids, vec![0]); assert_eq!(documents_ids, vec![0]);
search.filter( search.filter(Filter::from_str(r#"monitor_diagonal = "27\" to 30\"" "#).unwrap().unwrap());
crate::Filter::from_str(r#"monitor_diagonal = "27\" to 30\"" "#).unwrap().unwrap(),
);
let crate::SearchResult { documents_ids, .. } = search.execute().unwrap(); let crate::SearchResult { documents_ids, .. } = search.execute().unwrap();
assert_eq!(documents_ids, vec![1]); assert_eq!(documents_ids, vec![1]);
search.filter( search.filter(Filter::from_str(r#"monitor_diagonal = "27\" to 30'" "#).unwrap().unwrap());
crate::Filter::from_str(r#"monitor_diagonal = "27\" to 30'" "#).unwrap().unwrap(),
);
let crate::SearchResult { documents_ids, .. } = search.execute().unwrap(); let crate::SearchResult { documents_ids, .. } = search.execute().unwrap();
assert_eq!(documents_ids, vec![2]); assert_eq!(documents_ids, vec![2]);
} }
#[test] #[test]
fn geo_radius_error() { fn geo_radius_error() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let config = IndexerConfig::default(); index
// Set the filterable fields to be the channel. .update_settings(|settings| {
let mut wtxn = index.write_txn().unwrap(); settings.set_searchable_fields(vec![S("_geo"), S("price")]); // to keep the fields order
let mut builder = Settings::new(&mut wtxn, &index, &config); settings.set_filterable_fields(hashset! { S("_geo"), S("price") });
builder.set_searchable_fields(vec![S("_geo"), S("price")]); // to keep the fields order })
builder.set_filterable_fields(hashset! { S("_geo"), S("price") }); .unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();
// georadius have a bad latitude // georadius have a bad latitude
let filter = Filter::from_str("_geoRadius(-100, 150, 10)").unwrap().unwrap(); let filter = Filter::from_str("_geoRadius(-100, 150, 10)").unwrap().unwrap();
let error = filter.evaluate(&rtxn, &index).unwrap_err(); let error = filter.evaluate(&rtxn, &index).unwrap_err();

29
milli/src/update/clear_documents.rs
View File

@ -82,36 +82,25 @@ impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use heed::EnvOpenOptions;
use super::*; use super::*;
use crate::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig}; use crate::index::tests::TempIndex;
#[test] #[test]
fn clear_documents() { fn clear_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let content = documents!([ index
{ "id": 0, "name": "kevin", "age": 20 }, .add_documents_using_wtxn(&mut wtxn, documents!([
{ "id": 1, "name": "kevina" }, { "id": 0, "name": "kevin", "age": 20 },
{ "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } } { "id": 1, "name": "kevina" },
]); { "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } }
let indexing_config = IndexDocumentsConfig::default(); ]))
let config = IndexerConfig::default(); .unwrap();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
// Clear all documents from the database. // Clear all documents from the database.
let builder = ClearDocuments::new(&mut wtxn, &index); let builder = ClearDocuments::new(&mut wtxn, &index);
assert_eq!(builder.execute().unwrap(), 3); assert_eq!(builder.execute().unwrap(), 3);
wtxn.commit().unwrap(); wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap(); let rtxn = index.read_txn().unwrap();

376
milli/src/update/delete_documents.rs
View File

@ -654,26 +654,13 @@ where
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use big_s::S; use big_s::S;
use heed::{EnvOpenOptions, RwTxn}; use heed::RwTxn;
use maplit::hashset; use maplit::hashset;
use super::*; use super::*;
use crate::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings}; use crate::index::tests::TempIndex;
use crate::Filter; use crate::Filter;
fn insert_documents<'t, R: std::io::Read + std::io::Seek>(
wtxn: &mut RwTxn<'t, '_>,
index: &'t Index,
documents: crate::documents::DocumentsBatchReader<R>,
) {
let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default();
let builder = IndexDocuments::new(wtxn, &index, &config, indexing_config, |_| ()).unwrap();
let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap();
}
fn delete_documents<'t>( fn delete_documents<'t>(
wtxn: &mut RwTxn<'t, '_>, wtxn: &mut RwTxn<'t, '_>,
index: &'t Index, index: &'t Index,
@ -695,24 +682,19 @@ mod tests {
#[test] #[test]
fn delete_documents_with_numbers_as_primary_key() { fn delete_documents_with_numbers_as_primary_key() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let content = documents!([ index
{ "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } }, .add_documents_using_wtxn(
{ "id": 1, "name": "kevina", "array": ["I", "am", "fine"] }, &mut wtxn,
{ "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] } documents!([
]); { "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } },
let config = IndexerConfig::default(); { "id": 1, "name": "kevina", "array": ["I", "am", "fine"] },
let indexing_config = IndexDocumentsConfig::default(); { "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] }
let builder = ]),
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); )
let (builder, user_error) = builder.add_documents(content).unwrap(); .unwrap();
user_error.unwrap();
builder.execute().unwrap();
// delete those documents, ids are synchronous therefore 0, 1, and 2. // delete those documents, ids are synchronous therefore 0, 1, and 2.
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
@ -730,25 +712,19 @@ mod tests {
#[test] #[test]
fn delete_documents_with_strange_primary_key() { fn delete_documents_with_strange_primary_key() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let content = documents!([ index
{ "mysuperid": 0, "name": "kevin" }, .add_documents_using_wtxn(
{ "mysuperid": 1, "name": "kevina" }, &mut wtxn,
{ "mysuperid": 2, "name": "benoit" } documents!([
]); { "mysuperid": 0, "name": "kevin" },
{ "mysuperid": 1, "name": "kevina" },
let config = IndexerConfig::default(); { "mysuperid": 2, "name": "benoit" }
let indexing_config = IndexDocumentsConfig::default(); ]),
let builder = )
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap(); .unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
// Delete not all of the documents but some of them. // Delete not all of the documents but some of them.
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap(); let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
@ -761,42 +737,45 @@ mod tests {
#[test] #[test]
fn filtered_placeholder_search_should_not_return_deleted_documents() { fn filtered_placeholder_search_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_primary_key(S("docid"));
builder.set_filterable_fields(hashset! { S("label") });
builder.execute(|_| ()).unwrap();
let content = documents!([ index
{ "docid": "1_4", "label": "sign" }, .update_settings_using_wtxn(&mut wtxn, |settings| {
{ "docid": "1_5", "label": "letter" }, settings.set_primary_key(S("docid"));
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" }, settings.set_filterable_fields(hashset! { S("label") });
{ "docid": "1_36", "label": "drawing,painting,pattern" }, })
{ "docid": "1_37", "label": "art,drawing,outdoor" }, .unwrap();
{ "docid": "1_38", "label": "aquarium,art,drawing" },
{ "docid": "1_39", "label": "abstract" }, index
{ "docid": "1_40", "label": "cartoon" }, .add_documents_using_wtxn(
{ "docid": "1_41", "label": "art,drawing" }, &mut wtxn,
{ "docid": "1_42", "label": "art,pattern" }, documents!([
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" }, { "docid": "1_4", "label": "sign" },
{ "docid": "1_44", "label": "drawing" }, { "docid": "1_5", "label": "letter" },
{ "docid": "1_45", "label": "art" }, { "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" }, { "docid": "1_36", "label": "drawing,painting,pattern" },
{ "docid": "1_47", "label": "abstract,pattern" }, { "docid": "1_37", "label": "art,drawing,outdoor" },
{ "docid": "1_52", "label": "abstract,cartoon" }, { "docid": "1_38", "label": "aquarium,art,drawing" },
{ "docid": "1_57", "label": "abstract,drawing,pattern" }, { "docid": "1_39", "label": "abstract" },
{ "docid": "1_58", "label": "abstract,art,cartoon" }, { "docid": "1_40", "label": "cartoon" },
{ "docid": "1_68", "label": "design" }, { "docid": "1_41", "label": "art,drawing" },
{ "docid": "1_69", "label": "geometry" } { "docid": "1_42", "label": "art,pattern" },
]); { "docid": "1_43", "label": "abstract,art,drawing,pattern" },
{ "docid": "1_44", "label": "drawing" },
{ "docid": "1_45", "label": "art" },
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" },
{ "docid": "1_47", "label": "abstract,pattern" },
{ "docid": "1_52", "label": "abstract,cartoon" },
{ "docid": "1_57", "label": "abstract,drawing,pattern" },
{ "docid": "1_58", "label": "abstract,art,cartoon" },
{ "docid": "1_68", "label": "design" },
{ "docid": "1_69", "label": "geometry" }
]),
)
.unwrap();
insert_documents(&mut wtxn, &index, content);
delete_documents(&mut wtxn, &index, &["1_4"]); delete_documents(&mut wtxn, &index, &["1_4"]);
// Placeholder search with filter // Placeholder search with filter
@ -809,41 +788,43 @@ mod tests {
#[test] #[test]
fn placeholder_search_should_not_return_deleted_documents() { fn placeholder_search_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); index
let mut builder = Settings::new(&mut wtxn, &index, &config); .update_settings_using_wtxn(&mut wtxn, |settings| {
builder.set_primary_key(S("docid")); settings.set_primary_key(S("docid"));
builder.execute(|_| ()).unwrap(); })
.unwrap();
let content = documents!([ index
{ "docid": "1_4", "label": "sign" }, .add_documents_using_wtxn(
{ "docid": "1_5", "label": "letter" }, &mut wtxn,
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" }, documents!([
{ "docid": "1_36", "label": "drawing,painting,pattern" }, { "docid": "1_4", "label": "sign" },
{ "docid": "1_37", "label": "art,drawing,outdoor" }, { "docid": "1_5", "label": "letter" },
{ "docid": "1_38", "label": "aquarium,art,drawing" }, { "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
{ "docid": "1_39", "label": "abstract" }, { "docid": "1_36", "label": "drawing,painting,pattern" },
{ "docid": "1_40", "label": "cartoon" }, { "docid": "1_37", "label": "art,drawing,outdoor" },
{ "docid": "1_41", "label": "art,drawing" }, { "docid": "1_38", "label": "aquarium,art,drawing" },
{ "docid": "1_42", "label": "art,pattern" }, { "docid": "1_39", "label": "abstract" },
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" }, { "docid": "1_40", "label": "cartoon" },
{ "docid": "1_44", "label": "drawing" }, { "docid": "1_41", "label": "art,drawing" },
{ "docid": "1_45", "label": "art" }, { "docid": "1_42", "label": "art,pattern" },
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" }, { "docid": "1_43", "label": "abstract,art,drawing,pattern" },
{ "docid": "1_47", "label": "abstract,pattern" }, { "docid": "1_44", "label": "drawing" },
{ "docid": "1_52", "label": "abstract,cartoon" }, { "docid": "1_45", "label": "art" },
{ "docid": "1_57", "label": "abstract,drawing,pattern" }, { "docid": "1_46", "label": "abstract,colorfulness,pattern" },
{ "docid": "1_58", "label": "abstract,art,cartoon" }, { "docid": "1_47", "label": "abstract,pattern" },
{ "docid": "1_68", "label": "design" }, { "docid": "1_52", "label": "abstract,cartoon" },
{ "docid": "1_69", "label": "geometry" } { "docid": "1_57", "label": "abstract,drawing,pattern" },
]); { "docid": "1_58", "label": "abstract,art,cartoon" },
{ "docid": "1_68", "label": "design" },
{ "docid": "1_69", "label": "geometry" }
]),
)
.unwrap();
insert_documents(&mut wtxn, &index, content);
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_4"]); let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_4"]);
// Placeholder search // Placeholder search
@ -862,41 +843,43 @@ mod tests {
#[test] #[test]
fn search_should_not_return_deleted_documents() { fn search_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); index
let mut builder = Settings::new(&mut wtxn, &index, &config); .update_settings_using_wtxn(&mut wtxn, |settings| {
builder.set_primary_key(S("docid")); settings.set_primary_key(S("docid"));
builder.execute(|_| ()).unwrap(); })
.unwrap();
let content = documents!([ index
{"docid": "1_4", "label": "sign"}, .add_documents_using_wtxn(
{"docid": "1_5", "label": "letter"}, &mut wtxn,
{"docid": "1_7", "label": "abstract,cartoon,design,pattern"}, documents!([
{"docid": "1_36","label": "drawing,painting,pattern"}, {"docid": "1_4", "label": "sign"},
{"docid": "1_37","label": "art,drawing,outdoor"}, {"docid": "1_5", "label": "letter"},
{"docid": "1_38","label": "aquarium,art,drawing"}, {"docid": "1_7", "label": "abstract,cartoon,design,pattern"},
{"docid": "1_39","label": "abstract"}, {"docid": "1_36","label": "drawing,painting,pattern"},
{"docid": "1_40","label": "cartoon"}, {"docid": "1_37","label": "art,drawing,outdoor"},
{"docid": "1_41","label": "art,drawing"}, {"docid": "1_38","label": "aquarium,art,drawing"},
{"docid": "1_42","label": "art,pattern"}, {"docid": "1_39","label": "abstract"},
{"docid": "1_43","label": "abstract,art,drawing,pattern"}, {"docid": "1_40","label": "cartoon"},
{"docid": "1_44","label": "drawing"}, {"docid": "1_41","label": "art,drawing"},
{"docid": "1_45","label": "art"}, {"docid": "1_42","label": "art,pattern"},
{"docid": "1_46","label": "abstract,colorfulness,pattern"}, {"docid": "1_43","label": "abstract,art,drawing,pattern"},
{"docid": "1_47","label": "abstract,pattern"}, {"docid": "1_44","label": "drawing"},
{"docid": "1_52","label": "abstract,cartoon"}, {"docid": "1_45","label": "art"},
{"docid": "1_57","label": "abstract,drawing,pattern"}, {"docid": "1_46","label": "abstract,colorfulness,pattern"},
{"docid": "1_58","label": "abstract,art,cartoon"}, {"docid": "1_47","label": "abstract,pattern"},
{"docid": "1_68","label": "design"}, {"docid": "1_52","label": "abstract,cartoon"},
{"docid": "1_69","label": "geometry"} {"docid": "1_57","label": "abstract,drawing,pattern"},
]); {"docid": "1_58","label": "abstract,art,cartoon"},
{"docid": "1_68","label": "design"},
{"docid": "1_69","label": "geometry"}
]),
)
.unwrap();
insert_documents(&mut wtxn, &index, content);
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_7", "1_52"]); let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_7", "1_52"]);
// search for abstract // search for abstract
@ -915,20 +898,18 @@ mod tests {
#[test] #[test]
fn geo_filtered_placeholder_search_should_not_return_deleted_documents() { fn geo_filtered_placeholder_search_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); index
let mut builder = Settings::new(&mut wtxn, &index, &config); .update_settings_using_wtxn(&mut wtxn, |settings| {
builder.set_primary_key(S("id")); settings.set_primary_key(S("id"));
builder.set_filterable_fields(hashset!(S("_geo"))); settings.set_filterable_fields(hashset!(S("_geo")));
builder.set_sortable_fields(hashset!(S("_geo"))); settings.set_sortable_fields(hashset!(S("_geo")));
builder.execute(|_| ()).unwrap(); })
.unwrap();
let content = documents!([ index.add_documents_using_wtxn(&mut wtxn, documents!([
{ "id": "1", "city": "Lille", "_geo": { "lat": 50.6299, "lng": 3.0569 } }, { "id": "1", "city": "Lille", "_geo": { "lat": 50.6299, "lng": 3.0569 } },
{ "id": "2", "city": "Mons-en-Barœul", "_geo": { "lat": 50.6415, "lng": 3.1106 } }, { "id": "2", "city": "Mons-en-Barœul", "_geo": { "lat": 50.6415, "lng": 3.1106 } },
{ "id": "3", "city": "Hellemmes", "_geo": { "lat": 50.6312, "lng": 3.1106 } }, { "id": "3", "city": "Hellemmes", "_geo": { "lat": 50.6312, "lng": 3.1106 } },
@ -949,10 +930,9 @@ mod tests {
{ "id": "18", "city": "Amiens", "_geo": { "lat": 49.9314, "lng": 2.2710 } }, { "id": "18", "city": "Amiens", "_geo": { "lat": 49.9314, "lng": 2.2710 } },
{ "id": "19", "city": "Compiègne", "_geo": { "lat": 49.4449, "lng": 2.7913 } }, { "id": "19", "city": "Compiègne", "_geo": { "lat": 49.4449, "lng": 2.7913 } },
{ "id": "20", "city": "Paris", "_geo": { "lat": 48.9021, "lng": 2.3708 } } { "id": "20", "city": "Paris", "_geo": { "lat": 48.9021, "lng": 2.3708 } }
]); ])).unwrap();
let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"];
insert_documents(&mut wtxn, &index, content); let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"];
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &external_ids_to_delete); let deleted_internal_ids = delete_documents(&mut wtxn, &index, &external_ids_to_delete);
// Placeholder search with geo filter // Placeholder search with geo filter
@ -972,41 +952,43 @@ mod tests {
#[test] #[test]
fn get_documents_should_not_return_deleted_documents() { fn get_documents_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default(); index
let mut builder = Settings::new(&mut wtxn, &index, &config); .update_settings_using_wtxn(&mut wtxn, |settings| {
builder.set_primary_key(S("docid")); settings.set_primary_key(S("docid"));
builder.execute(|_| ()).unwrap(); })
.unwrap();
let content = documents!([ index
{ "docid": "1_4", "label": "sign" }, .add_documents_using_wtxn(
{ "docid": "1_5", "label": "letter" }, &mut wtxn,
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" }, documents!([
{ "docid": "1_36", "label": "drawing,painting,pattern" }, { "docid": "1_4", "label": "sign" },
{ "docid": "1_37", "label": "art,drawing,outdoor" }, { "docid": "1_5", "label": "letter" },
{ "docid": "1_38", "label": "aquarium,art,drawing" }, { "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
{ "docid": "1_39", "label": "abstract" }, { "docid": "1_36", "label": "drawing,painting,pattern" },
{ "docid": "1_40", "label": "cartoon" }, { "docid": "1_37", "label": "art,drawing,outdoor" },
{ "docid": "1_41", "label": "art,drawing" }, { "docid": "1_38", "label": "aquarium,art,drawing" },
{ "docid": "1_42", "label": "art,pattern" }, { "docid": "1_39", "label": "abstract" },
{ "docid": "1_43", "label": "abstract,art,drawing,pattern" }, { "docid": "1_40", "label": "cartoon" },
{ "docid": "1_44", "label": "drawing" }, { "docid": "1_41", "label": "art,drawing" },
{ "docid": "1_45", "label": "art" }, { "docid": "1_42", "label": "art,pattern" },
{ "docid": "1_46", "label": "abstract,colorfulness,pattern" }, { "docid": "1_43", "label": "abstract,art,drawing,pattern" },
{ "docid": "1_47", "label": "abstract,pattern" }, { "docid": "1_44", "label": "drawing" },
{ "docid": "1_52", "label": "abstract,cartoon" }, { "docid": "1_45", "label": "art" },
{ "docid": "1_57", "label": "abstract,drawing,pattern" }, { "docid": "1_46", "label": "abstract,colorfulness,pattern" },
{ "docid": "1_58", "label": "abstract,art,cartoon" }, { "docid": "1_47", "label": "abstract,pattern" },
{ "docid": "1_68", "label": "design" }, { "docid": "1_52", "label": "abstract,cartoon" },
{ "docid": "1_69", "label": "geometry" } { "docid": "1_57", "label": "abstract,drawing,pattern" },
]); { "docid": "1_58", "label": "abstract,art,cartoon" },
{ "docid": "1_68", "label": "design" },
{ "docid": "1_69", "label": "geometry" }
]),
)
.unwrap();
insert_documents(&mut wtxn, &index, content);
let deleted_external_ids = ["1_7", "1_52"]; let deleted_external_ids = ["1_7", "1_52"];
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &deleted_external_ids); let deleted_internal_ids = delete_documents(&mut wtxn, &index, &deleted_external_ids);
@ -1042,18 +1024,17 @@ mod tests {
#[test] #[test]
fn stats_should_not_return_deleted_documents() { fn stats_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap(); let index = TempIndex::new();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap(); let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_primary_key(S("docid"));
builder.execute(|_| ()).unwrap();
let content = documents!([ index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_primary_key(S("docid"));
})
.unwrap();
index.add_documents_using_wtxn(&mut wtxn, documents!([
{ "docid": "1_4", "label": "sign"}, { "docid": "1_4", "label": "sign"},
{ "docid": "1_5", "label": "letter"}, { "docid": "1_5", "label": "letter"},
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern", "title": "Mickey Mouse"}, { "docid": "1_7", "label": "abstract,cartoon,design,pattern", "title": "Mickey Mouse"},
@ -1074,9 +1055,8 @@ mod tests {
{ "docid": "1_58", "label": "abstract,art,cartoon"}, { "docid": "1_58", "label": "abstract,art,cartoon"},
{ "docid": "1_68", "label": "design"}, { "docid": "1_68", "label": "design"},
{ "docid": "1_69", "label": "geometry"} { "docid": "1_69", "label": "geometry"}
]); ])).unwrap();
insert_documents(&mut wtxn, &index, content);
delete_documents(&mut wtxn, &index, &["1_7", "1_52"]); delete_documents(&mut wtxn, &index, &["1_7", "1_52"]);
// count internal documents // count internal documents

1333
milli/src/update/index_documents/mod.rs
View File

File diff suppressed because it is too large Load Diff

823
milli/src/update/settings.rs
View File

File diff suppressed because it is too large Load Diff