searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-22 21:04:27 +01:00
Code Issues Releases Wiki Activity

Merge #600

Browse Source 
600: Simplify some unit tests r=ManyTheFish a=loiclec

# Pull Request

## What does this PR do?
Simplify the code that is used in unit tests to create and modify an index. Basically, the following code:
```rust
  let path = tempfile::tempdir().unwrap();
  let mut options = EnvOpenOptions::new();
  options.map_size(10 * 1024 * 1024); // 10 MB
  let index = Index::new(options, &path).unwrap();

  let mut wtxn = index.write_txn().unwrap();
  let content = documents!([
      { "id": 0, "name": "kevin" },
  ]);
  let config = IndexerConfig::default();
  let indexing_config = IndexDocumentsConfig::default();
  let builder =
      IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
  let (builder, user_error) = builder.add_documents(content).unwrap();
  user_error.unwrap();
  builder.execute().unwrap();
  wtxn.commit.unwrap();

  let mut wtxn = index.write_txn().unwrap();
  let config = IndexerConfig::default();
  let mut builder = Settings::new(&mut wtxn, &index, &config);
  builder.set_primary_key(S("docid"));
  builder.set_filterable_fields(hashset! { S("label") });
  builder.execute(|_| ()).unwrap();
  wtxn.commit().unwrap();
```
becomes:
```rust
let index = TempIndex::new():
index.add_documents(documents!(
      { "id": 0, "name": "kevin" },
)).unwrap();
index.update_settings(|settings| {
    settings.set_primary_key(S("docid"));
    settings.set_filterable_fields(hashset! { S("label") });
}).unwrap();
```

Then there is a bunch of options to modify the indexing configs, the map size, to reuse a transaction, etc. For example:
```rust
let mut index = TempIndex::new_with_map_size(1000 * 4096 * 10);
index.index_documents_config.autogenerate_docids = true;
let mut wtxn = index.write_txn().unwrap();
index.update_settings_using_wtxn(&mut wtxn, |settings| {
    settings.set_primary_key(S("docids"));
}).unwrap();
wtxn.commit().unwrap();
```

Co-authored-by: Loïc Lecrenier <loic@meilisearch.com>
Co-authored-by: Loïc Lecrenier <loic.lecrenier@me.com>
Co-authored-by: bors[bot] <26634292+bors[bot]@users.noreply.github.com>
This commit is contained in:
bors[bot] 2022-08-04 10:19:42 +00:00 committed by GitHub
commit 950d8e4c44
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 1123 additions and 1816 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
milli/src/documents/mod.rs
View File

@ -179,6 +179,17 @@ macro_rules! documents {
}};
}
#[cfg(test)]
pub fn documents_batch_reader_from_objects(
objects: impl IntoIterator<Item = Object>,
) -> DocumentsBatchReader<std::io::Cursor<Vec<u8>>> {
let mut builder = DocumentsBatchBuilder::new(Vec::new());
for object in objects {
builder.append_json_object(&object).unwrap();
}
DocumentsBatchReader::from_reader(std::io::Cursor::new(builder.into_inner().unwrap())).unwrap()
}
#[cfg(test)]
mod test {
use std::io::Cursor;

193
milli/src/index.rs
View File

@ -1182,16 +1182,19 @@ pub(crate) mod tests {
use std::ops::Deref;
use big_s::S;
use heed::EnvOpenOptions;
use heed::{EnvOpenOptions, RwTxn};
use maplit::btreemap;
use tempfile::TempDir;
use crate::documents::DocumentsBatchReader;
use crate::index::{DEFAULT_MIN_WORD_LEN_ONE_TYPO, DEFAULT_MIN_WORD_LEN_TWO_TYPOS};
use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig};
use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
use crate::Index;
pub(crate) struct TempIndex {
inner: Index,
pub inner: Index,
pub indexer_config: IndexerConfig,
pub index_documents_config: IndexDocumentsConfig,
_tempdir: TempDir,
}
@ -1204,43 +1207,88 @@ pub(crate) mod tests {
}
impl TempIndex {
/// Creates a temporary index, with a default `4096 * 100` size. This should be enough for
/// most tests.
pub fn new() -> Self {
/// Creates a temporary index
pub fn new_with_map_size(size: usize) -> Self {
let mut options = EnvOpenOptions::new();
options.map_size(100 * 4096);
options.map_size(size);
let _tempdir = TempDir::new_in(".").unwrap();
let inner = Index::new(options, _tempdir.path()).unwrap();
Self { inner, _tempdir }
let indexer_config = IndexerConfig::default();
let index_documents_config = IndexDocumentsConfig::default();
Self { inner, indexer_config, index_documents_config, _tempdir }
}
/// Creates a temporary index, with a default `4096 * 1000` size. This should be enough for
/// most tests.
pub fn new() -> Self {
Self::new_with_map_size(4096 * 1000)
}
pub fn add_documents_using_wtxn<'t, R>(
&'t self,
wtxn: &mut RwTxn<'t, '_>,
documents: DocumentsBatchReader<R>,
) -> Result<(), crate::error::Error>
where
R: std::io::Read + std::io::Seek,
{
let builder = IndexDocuments::new(
wtxn,
&self,
&self.indexer_config,
self.index_documents_config.clone(),
|_| (),
)
.unwrap();
let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error?;
builder.execute()?;
Ok(())
}
pub fn add_documents<R>(
&self,
documents: DocumentsBatchReader<R>,
) -> Result<(), crate::error::Error>
where
R: std::io::Read + std::io::Seek,
{
let mut wtxn = self.write_txn().unwrap();
self.add_documents_using_wtxn(&mut wtxn, documents)?;
wtxn.commit().unwrap();
Ok(())
}
pub fn update_settings(
&self,
update: impl Fn(&mut Settings),
) -> Result<(), crate::error::Error> {
let mut wtxn = self.write_txn().unwrap();
self.update_settings_using_wtxn(&mut wtxn, update)?;
wtxn.commit().unwrap();
Ok(())
}
pub fn update_settings_using_wtxn<'t>(
&'t self,
wtxn: &mut RwTxn<'t, '_>,
update: impl Fn(&mut Settings),
) -> Result<(), crate::error::Error> {
let mut builder = update::Settings::new(wtxn, &self.inner, &self.indexer_config);
update(&mut builder);
builder.execute(drop)?;
Ok(())
}
}
#[test]
fn initial_field_distribution() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
let index = TempIndex::new();
index
.add_documents(documents!([
{ "id": 1, "name": "kevin" },
{ "id": 2, "name": "bob", "age": 20 },
{ "id": 2, "name": "bob", "age": 20 },
]);
let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]))
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();
let field_distribution = index.field_distribution(&rtxn).unwrap();
assert_eq!(
field_distribution,
@ -1253,19 +1301,13 @@ pub(crate) mod tests {
// we add all the documents a second time. we are supposed to get the same
// field_distribution in the end
let mut wtxn = index.write_txn().unwrap();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap();
let content = documents!([
index
.add_documents(documents!([
{ "id": 1, "name": "kevin" },
{ "id": 2, "name": "bob", "age": 20 },
{ "id": 2, "name": "bob", "age": 20 },
]);
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
]))
.unwrap();
let rtxn = index.read_txn().unwrap();
@ -1280,19 +1322,12 @@ pub(crate) mod tests {
);
// then we update a document by removing one field and another by adding one field
let content = documents!([
index
.add_documents(documents!([
{ "id": 1, "name": "kevin", "has_dog": true },
{ "id": 2, "name": "bob" }
]);
let mut wtxn = index.write_txn().unwrap();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]))
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();
@ -1341,35 +1376,19 @@ pub(crate) mod tests {
#[test]
fn add_documents_and_set_searchable_fields() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
let index = TempIndex::new();
index
.add_documents(documents!([
{ "id": 1, "doggo": "kevin" },
{ "id": 2, "doggo": { "name": "bob", "age": 20 } },
{ "id": 3, "name": "jean", "age": 25 },
]);
let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]))
.unwrap();
index
.update_settings(|settings| {
settings.set_searchable_fields(vec![S("doggo"), S("name")]);
})
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// set searchable fields
let mut wtxn = index.write_txn().unwrap();
let mut builder = update::Settings::new(&mut wtxn, &index, &config);
builder.set_searchable_fields(vec![S("doggo"), S("name")]);
builder.execute(drop).unwrap();
wtxn.commit().unwrap();
// ensure we get the right real searchable fields + user defined searchable fields
let rtxn = index.read_txn().unwrap();
@ -1383,19 +1402,13 @@ pub(crate) mod tests {
#[test]
fn set_searchable_fields_and_add_documents() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let config = IndexerConfig::default();
let index = TempIndex::new();
// set searchable fields
let mut wtxn = index.write_txn().unwrap();
let mut builder = update::Settings::new(&mut wtxn, &index, &config);
builder.set_searchable_fields(vec![S("doggo"), S("name")]);
builder.execute(drop).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.set_searchable_fields(vec![S("doggo"), S("name")]);
})
.unwrap();
// ensure we get the right real searchable fields + user defined searchable fields
let rtxn = index.read_txn().unwrap();
@ -1405,21 +1418,13 @@ pub(crate) mod tests {
let user_defined = index.user_defined_searchable_fields(&rtxn).unwrap().unwrap();
assert_eq!(user_defined, &["doggo", "name"]);
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
index
.add_documents(documents!([
{ "id": 1, "doggo": "kevin" },
{ "id": 2, "doggo": { "name": "bob", "age": 20 } },
{ "id": 3, "name": "jean", "age": 25 },
]);
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]))
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// ensure we get the right real searchable fields + user defined searchable fields
let rtxn = index.read_txn().unwrap();

112
milli/src/search/facet/filter.rs
View File

@ -494,28 +494,21 @@ mod tests {
use big_s::S;
use either::Either;
use heed::EnvOpenOptions;
use maplit::hashset;
use super::*;
use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
use crate::Index;
use crate::index::tests::TempIndex;
use crate::Filter;
#[test]
fn empty_db() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
// Set the filterable fields to be the channel.
let config = IndexerConfig::default();
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_searchable_fields(vec![S("PrIcE")]); // to keep the fields order
builder.set_filterable_fields(hashset! { S("PrIcE") });
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.set_searchable_fields(vec![S("PrIcE")]); // to keep the fields order
settings.set_filterable_fields(hashset! { S("PrIcE") });
})
.unwrap();
let rtxn = index.read_txn().unwrap();
@ -592,10 +585,7 @@ mod tests {
#[test]
fn not_filterable() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
let rtxn = index.read_txn().unwrap();
let filter = Filter::from_str("_geoRadius(42, 150, 10)").unwrap().unwrap();
@ -611,14 +601,12 @@ mod tests {
));
drop(rtxn);
let config = IndexerConfig::default();
// Set the filterable fields to be the channel.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_searchable_fields(vec![S("title")]);
builder.set_filterable_fields(hashset! { S("title") });
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.set_searchable_fields(vec![S("title")]);
settings.set_filterable_fields(hashset! { S("title") });
})
.unwrap();
let rtxn = index.read_txn().unwrap();
@ -637,13 +625,10 @@ mod tests {
#[test]
fn escaped_quote_in_filter_value_2380() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
index
.add_documents(documents!([
{
"id": "test_1",
"monitor_diagonal": "27' to 30'"
@ -656,73 +641,48 @@ mod tests {
"id": "test_3",
"monitor_diagonal": "27\" to 30'"
},
]);
let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]))
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
let mut wtxn = index.write_txn().unwrap();
let mut builder = update::Settings::new(&mut wtxn, &index, &config);
builder.set_filterable_fields(hashset!(S("monitor_diagonal")));
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.set_filterable_fields(hashset!(S("monitor_diagonal")));
})
.unwrap();
let rtxn = index.read_txn().unwrap();
let mut search = crate::Search::new(&rtxn, &index);
// this filter is copy pasted from #2380 with the exact same espace sequence
search.filter(
crate::Filter::from_str("monitor_diagonal = '27\" to 30\\''").unwrap().unwrap(),
);
search.filter(Filter::from_str("monitor_diagonal = '27\" to 30\\''").unwrap().unwrap());
let crate::SearchResult { documents_ids, .. } = search.execute().unwrap();
assert_eq!(documents_ids, vec![2]);
search.filter(
crate::Filter::from_str(r#"monitor_diagonal = "27' to 30'" "#).unwrap().unwrap(),
);
search.filter(Filter::from_str(r#"monitor_diagonal = "27' to 30'" "#).unwrap().unwrap());
let crate::SearchResult { documents_ids, .. } = search.execute().unwrap();
assert_eq!(documents_ids, vec![0]);
search.filter(
crate::Filter::from_str(r#"monitor_diagonal = "27\" to 30\"" "#).unwrap().unwrap(),
);
search.filter(Filter::from_str(r#"monitor_diagonal = "27\" to 30\"" "#).unwrap().unwrap());
let crate::SearchResult { documents_ids, .. } = search.execute().unwrap();
assert_eq!(documents_ids, vec![1]);
search.filter(
crate::Filter::from_str(r#"monitor_diagonal = "27\" to 30'" "#).unwrap().unwrap(),
);
search.filter(Filter::from_str(r#"monitor_diagonal = "27\" to 30'" "#).unwrap().unwrap());
let crate::SearchResult { documents_ids, .. } = search.execute().unwrap();
assert_eq!(documents_ids, vec![2]);
}
#[test]
fn geo_radius_error() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
let config = IndexerConfig::default();
// Set the filterable fields to be the channel.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_searchable_fields(vec![S("_geo"), S("price")]); // to keep the fields order
builder.set_filterable_fields(hashset! { S("_geo"), S("price") });
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.set_searchable_fields(vec![S("_geo"), S("price")]); // to keep the fields order
settings.set_filterable_fields(hashset! { S("_geo"), S("price") });
})
.unwrap();
let rtxn = index.read_txn().unwrap();
// georadius have a bad latitude
let filter = Filter::from_str("_geoRadius(-100, 150, 10)").unwrap().unwrap();
let error = filter.evaluate(&rtxn, &index).unwrap_err();

23
milli/src/update/clear_documents.rs
View File

@ -82,36 +82,25 @@ impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> {
#[cfg(test)]
mod tests {
use heed::EnvOpenOptions;
use super::*;
use crate::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig};
use crate::index::tests::TempIndex;
#[test]
fn clear_documents() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
index
.add_documents_using_wtxn(&mut wtxn, documents!([
{ "id": 0, "name": "kevin", "age": 20 },
{ "id": 1, "name": "kevina" },
{ "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } }
]);
let indexing_config = IndexDocumentsConfig::default();
let config = IndexerConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
]))
.unwrap();
// Clear all documents from the database.
let builder = ClearDocuments::new(&mut wtxn, &index);
assert_eq!(builder.execute().unwrap(), 3);
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();

204
milli/src/update/delete_documents.rs
View File

@ -654,26 +654,13 @@ where
#[cfg(test)]
mod tests {
use big_s::S;
use heed::{EnvOpenOptions, RwTxn};
use heed::RwTxn;
use maplit::hashset;
use super::*;
use crate::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
use crate::index::tests::TempIndex;
use crate::Filter;
fn insert_documents<'t, R: std::io::Read + std::io::Seek>(
wtxn: &mut RwTxn<'t, '_>,
index: &'t Index,
documents: crate::documents::DocumentsBatchReader<R>,
) {
let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default();
let builder = IndexDocuments::new(wtxn, &index, &config, indexing_config, |_| ()).unwrap();
let (builder, user_error) = builder.add_documents(documents).unwrap();
user_error.unwrap();
builder.execute().unwrap();
}
fn delete_documents<'t>(
wtxn: &mut RwTxn<'t, '_>,
index: &'t Index,
@ -695,24 +682,19 @@ mod tests {
#[test]
fn delete_documents_with_numbers_as_primary_key() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
index
.add_documents_using_wtxn(
&mut wtxn,
documents!([
{ "id": 0, "name": "kevin", "object": { "key1": "value1", "key2": "value2" } },
{ "id": 1, "name": "kevina", "array": ["I", "am", "fine"] },
{ "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] }
]);
let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
]),
)
.unwrap();
// delete those documents, ids are synchronous therefore 0, 1, and 2.
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
@ -730,25 +712,19 @@ mod tests {
#[test]
fn delete_documents_with_strange_primary_key() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
index
.add_documents_using_wtxn(
&mut wtxn,
documents!([
{ "mysuperid": 0, "name": "kevin" },
{ "mysuperid": 1, "name": "kevina" },
{ "mysuperid": 2, "name": "benoit" }
]);
let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| ()).unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
]),
)
.unwrap();
// Delete not all of the documents but some of them.
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
@ -761,19 +737,21 @@ mod tests {
#[test]
fn filtered_placeholder_search_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_primary_key(S("docid"));
builder.set_filterable_fields(hashset! { S("label") });
builder.execute(|_| ()).unwrap();
let content = documents!([
index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_primary_key(S("docid"));
settings.set_filterable_fields(hashset! { S("label") });
})
.unwrap();
index
.add_documents_using_wtxn(
&mut wtxn,
documents!([
{ "docid": "1_4", "label": "sign" },
{ "docid": "1_5", "label": "letter" },
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
@ -794,9 +772,10 @@ mod tests {
{ "docid": "1_58", "label": "abstract,art,cartoon" },
{ "docid": "1_68", "label": "design" },
{ "docid": "1_69", "label": "geometry" }
]);
]),
)
.unwrap();
insert_documents(&mut wtxn, &index, content);
delete_documents(&mut wtxn, &index, &["1_4"]);
// Placeholder search with filter
@ -809,18 +788,19 @@ mod tests {
#[test]
fn placeholder_search_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_primary_key(S("docid"));
builder.execute(|_| ()).unwrap();
index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_primary_key(S("docid"));
})
.unwrap();
let content = documents!([
index
.add_documents_using_wtxn(
&mut wtxn,
documents!([
{ "docid": "1_4", "label": "sign" },
{ "docid": "1_5", "label": "letter" },
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
@ -841,9 +821,10 @@ mod tests {
{ "docid": "1_58", "label": "abstract,art,cartoon" },
{ "docid": "1_68", "label": "design" },
{ "docid": "1_69", "label": "geometry" }
]);
]),
)
.unwrap();
insert_documents(&mut wtxn, &index, content);
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_4"]);
// Placeholder search
@ -862,18 +843,19 @@ mod tests {
#[test]
fn search_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_primary_key(S("docid"));
builder.execute(|_| ()).unwrap();
index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_primary_key(S("docid"));
})
.unwrap();
let content = documents!([
index
.add_documents_using_wtxn(
&mut wtxn,
documents!([
{"docid": "1_4", "label": "sign"},
{"docid": "1_5", "label": "letter"},
{"docid": "1_7", "label": "abstract,cartoon,design,pattern"},
@ -894,9 +876,10 @@ mod tests {
{"docid": "1_58","label": "abstract,art,cartoon"},
{"docid": "1_68","label": "design"},
{"docid": "1_69","label": "geometry"}
]);
]),
)
.unwrap();
insert_documents(&mut wtxn, &index, content);
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &["1_7", "1_52"]);
// search for abstract
@ -915,20 +898,18 @@ mod tests {
#[test]
fn geo_filtered_placeholder_search_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_primary_key(S("id"));
builder.set_filterable_fields(hashset!(S("_geo")));
builder.set_sortable_fields(hashset!(S("_geo")));
builder.execute(|_| ()).unwrap();
index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_primary_key(S("id"));
settings.set_filterable_fields(hashset!(S("_geo")));
settings.set_sortable_fields(hashset!(S("_geo")));
})
.unwrap();
let content = documents!([
index.add_documents_using_wtxn(&mut wtxn, documents!([
{ "id": "1", "city": "Lille", "_geo": { "lat": 50.6299, "lng": 3.0569 } },
{ "id": "2", "city": "Mons-en-Barœul", "_geo": { "lat": 50.6415, "lng": 3.1106 } },
{ "id": "3", "city": "Hellemmes", "_geo": { "lat": 50.6312, "lng": 3.1106 } },
@ -949,10 +930,9 @@ mod tests {
{ "id": "18", "city": "Amiens", "_geo": { "lat": 49.9314, "lng": 2.2710 } },
{ "id": "19", "city": "Compiègne", "_geo": { "lat": 49.4449, "lng": 2.7913 } },
{ "id": "20", "city": "Paris", "_geo": { "lat": 48.9021, "lng": 2.3708 } }
]);
let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"];
])).unwrap();
insert_documents(&mut wtxn, &index, content);
let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"];
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &external_ids_to_delete);
// Placeholder search with geo filter
@ -972,18 +952,19 @@ mod tests {
#[test]
fn get_documents_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_primary_key(S("docid"));
builder.execute(|_| ()).unwrap();
index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_primary_key(S("docid"));
})
.unwrap();
let content = documents!([
index
.add_documents_using_wtxn(
&mut wtxn,
documents!([
{ "docid": "1_4", "label": "sign" },
{ "docid": "1_5", "label": "letter" },
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern" },
@ -1004,9 +985,10 @@ mod tests {
{ "docid": "1_58", "label": "abstract,art,cartoon" },
{ "docid": "1_68", "label": "design" },
{ "docid": "1_69", "label": "geometry" }
]);
]),
)
.unwrap();
insert_documents(&mut wtxn, &index, content);
let deleted_external_ids = ["1_7", "1_52"];
let deleted_internal_ids = delete_documents(&mut wtxn, &index, &deleted_external_ids);
@ -1042,18 +1024,17 @@ mod tests {
#[test]
fn stats_should_not_return_deleted_documents() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
let mut wtxn = index.write_txn().unwrap();
let config = IndexerConfig::default();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_primary_key(S("docid"));
builder.execute(|_| ()).unwrap();
let content = documents!([
index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_primary_key(S("docid"));
})
.unwrap();
index.add_documents_using_wtxn(&mut wtxn, documents!([
{ "docid": "1_4", "label": "sign"},
{ "docid": "1_5", "label": "letter"},
{ "docid": "1_7", "label": "abstract,cartoon,design,pattern", "title": "Mickey Mouse"},
@ -1074,9 +1055,8 @@ mod tests {
{ "docid": "1_58", "label": "abstract,art,cartoon"},
{ "docid": "1_68", "label": "design"},
{ "docid": "1_69", "label": "geometry"}
]);
])).unwrap();
insert_documents(&mut wtxn, &index, content);
delete_documents(&mut wtxn, &index, &["1_7", "1_52"]);
// count internal documents

1003
milli/src/update/index_documents/mod.rs
View File

File diff suppressed because it is too large Load Diff

609
milli/src/update/settings.rs
View File

@ -709,45 +709,38 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
mod tests {
use big_s::S;
use heed::types::ByteSlice;
use heed::EnvOpenOptions;
use maplit::{btreeset, hashmap, hashset};
use super::*;
use crate::error::Error;
use crate::index::tests::TempIndex;
use crate::update::IndexDocuments;
use crate::{Criterion, Filter, SearchResult};
#[test]
fn set_and_reset_searchable_fields() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let index = TempIndex::new();
// First we send 3 documents with ids from 1 to 3.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
index
.add_documents_using_wtxn(
&mut wtxn,
documents!([
{ "id": 1, "name": "kevin", "age": 23 },
{ "id": 2, "name": "kevina", "age": 21},
{ "id": 3, "name": "benoit", "age": 34 }
]);
let config = IndexerConfig::default();
let indexing_config = IndexDocumentsConfig::default();
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]),
)
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// We change the searchable fields to be the "name" field only.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_searchable_fields(vec!["name".into()]);
builder.execute(|_| ()).unwrap();
index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_searchable_fields(vec!["name".into()]);
})
.unwrap();
wtxn.commit().unwrap();
// Check that the searchable field is correctly set to "name" only.
@ -766,11 +759,11 @@ mod tests {
drop(rtxn);
// We change the searchable fields to be the "name" field only.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.reset_searchable_fields();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.reset_searchable_fields();
})
.unwrap();
// Check that the searchable field have been reset and documents are found now.
let rtxn = index.read_txn().unwrap();
@ -784,36 +777,30 @@ mod tests {
#[test]
fn mixup_searchable_with_displayed_fields() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut index = TempIndex::new();
index.index_documents_config.autogenerate_docids = true;
// First we send 3 documents with ids from 1 to 3.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
// First we send 3 documents with ids from 1 to 3.
index
.add_documents_using_wtxn(
&mut wtxn,
documents!([
{ "name": "kevin", "age": 23},
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 }
]);
let config = IndexerConfig::default();
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]),
)
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// In the same transaction we change the displayed fields to be only the "age".
// We also change the searchable fields to be the "name" field only.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_displayed_fields(vec!["age".into()]);
builder.set_searchable_fields(vec!["name".into()]);
builder.execute(|_| ()).unwrap();
index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_displayed_fields(vec!["age".into()]);
settings.set_searchable_fields(vec!["name".into()]);
})
.unwrap();
wtxn.commit().unwrap();
// Check that the displayed fields are correctly set to `None` (default value).
@ -823,11 +810,11 @@ mod tests {
drop(rtxn);
// We change the searchable fields to be the "name" field only.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.reset_searchable_fields();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.reset_searchable_fields();
})
.unwrap();
// Check that the displayed fields always contains only the "age" field.
let rtxn = index.read_txn().unwrap();
@ -837,28 +824,17 @@ mod tests {
#[test]
fn default_displayed_fields() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut index = TempIndex::new();
index.index_documents_config.autogenerate_docids = true;
// First we send 3 documents with ids from 1 to 3.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
index
.add_documents(documents!([
{ "name": "kevin", "age": 23},
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 }
]);
let config = IndexerConfig::default();
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]))
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// Check that the displayed fields are correctly set to `None` (default value).
let rtxn = index.read_txn().unwrap();
@ -868,32 +844,25 @@ mod tests {
#[test]
fn set_and_reset_displayed_field() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut index = TempIndex::new();
index.index_documents_config.autogenerate_docids = true;
// First we send 3 documents with ids from 1 to 3.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
index
.add_documents_using_wtxn(
&mut wtxn,
documents!([
{ "name": "kevin", "age": 23},
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 }
]);
let config = IndexerConfig::default();
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]),
)
.unwrap();
index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_displayed_fields(vec!["age".into()]);
})
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
// In the same transaction we change the displayed fields to be only the age.
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_displayed_fields(vec!["age".into()]);
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
// Check that the displayed fields are correctly set to only the "age" field.
@ -903,11 +872,11 @@ mod tests {
drop(rtxn);
// We reset the fields ids to become `None`, the default value.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.reset_displayed_fields();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.reset_displayed_fields();
})
.unwrap();
// Check that the displayed fields are correctly set to `None` (default value).
let rtxn = index.read_txn().unwrap();
@ -917,34 +886,24 @@ mod tests {
#[test]
fn set_filterable_fields() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let config = IndexerConfig::default();
let mut index = TempIndex::new();
index.index_documents_config.autogenerate_docids = true;
// Set the filterable fields to be the age.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_filterable_fields(hashset! { S("age") });
builder.execute(|_| ()).unwrap();
index
.update_settings(|settings| {
settings.set_filterable_fields(hashset! { S("age") });
})
.unwrap();
// Then index some documents.
let content = documents!([
index
.add_documents(documents!([
{ "name": "kevin", "age": 23},
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 }
]);
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]))
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// Check that the displayed fields are correctly set.
let rtxn = index.read_txn().unwrap();
@ -970,22 +929,13 @@ mod tests {
drop(rtxn);
// Index a little more documents with new and current facets values.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
index
.add_documents(documents!([
{ "name": "kevin2", "age": 23},
{ "name": "kevina2", "age": 21 },
{ "name": "benoit", "age": 35 }
]);
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]))
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();
// Only count the field_id 0 and level 0 facet values.
@ -1000,35 +950,25 @@ mod tests {
#[test]
fn set_asc_desc_field() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let config = IndexerConfig::default();
let mut index = TempIndex::new();
index.index_documents_config.autogenerate_docids = true;
// Set the filterable fields to be the age.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
// Don't display the generated `id` field.
builder.set_displayed_fields(vec![S("name")]);
builder.set_criteria(vec![S("age:asc")]);
builder.execute(|_| ()).unwrap();
index
.update_settings(|settings| {
settings.set_displayed_fields(vec![S("name")]);
settings.set_criteria(vec![S("age:asc")]);
})
.unwrap();
// Then index some documents.
let content = documents!([
index
.add_documents(documents!([
{ "name": "kevin", "age": 23},
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 }
]);
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]))
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// Run an empty query just to ensure that the search results are ordered.
let rtxn = index.read_txn().unwrap();
@ -1048,22 +988,21 @@ mod tests {
#[test]
fn set_distinct_field() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let config = IndexerConfig::default();
let mut index = TempIndex::new();
index.index_documents_config.autogenerate_docids = true;
// Set the filterable fields to be the age.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
index
.update_settings(|settings| {
// Don't display the generated `id` field.
builder.set_displayed_fields(vec![S("name"), S("age")]);
builder.set_distinct_field(S("age"));
builder.execute(|_| ()).unwrap();
settings.set_displayed_fields(vec![S("name"), S("age")]);
settings.set_distinct_field(S("age"));
})
.unwrap();
// Then index some documents.
let content = documents!([
index
.add_documents(documents!([
{ "name": "kevin", "age": 23 },
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 },
@ -1071,16 +1010,8 @@ mod tests {
{ "name": "bertrand", "age": 34 },
{ "name": "bernie", "age": 34 },
{ "name": "ben", "age": 34 }
]);
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]))
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// Run an empty query just to ensure that the search results are ordered.
let rtxn = index.read_txn().unwrap();
@ -1092,22 +1023,21 @@ mod tests {
#[test]
fn set_nested_distinct_field() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let config = IndexerConfig::default();
let mut index = TempIndex::new();
index.index_documents_config.autogenerate_docids = true;
// Set the filterable fields to be the age.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
index
.update_settings(|settings| {
// Don't display the generated `id` field.
builder.set_displayed_fields(vec![S("person")]);
builder.set_distinct_field(S("person.age"));
builder.execute(|_| ()).unwrap();
settings.set_displayed_fields(vec![S("person")]);
settings.set_distinct_field(S("person.age"));
})
.unwrap();
// Then index some documents.
let content = documents!([
index
.add_documents(documents!([
{ "person": { "name": "kevin", "age": 23 }},
{ "person": { "name": "kevina", "age": 21 }},
{ "person": { "name": "benoit", "age": 34 }},
@ -1115,16 +1045,8 @@ mod tests {
{ "person": { "name": "bertrand", "age": 34 }},
{ "person": { "name": "bernie", "age": 34 }},
{ "person": { "name": "ben", "age": 34 }}
]);
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]))
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// Run an empty query just to ensure that the search results are ordered.
let rtxn = index.read_txn().unwrap();
@ -1136,28 +1058,17 @@ mod tests {
#[test]
fn default_stop_words() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut index = TempIndex::new();
index.index_documents_config.autogenerate_docids = true;
// First we send 3 documents with ids from 1 to 3.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
index
.add_documents(documents!([
{ "name": "kevin", "age": 23},
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 }
]);
let config = IndexerConfig::default();
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]))
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// Ensure there is no stop_words by default
let rtxn = index.read_txn().unwrap();
@ -1167,33 +1078,30 @@ mod tests {
#[test]
fn set_and_reset_stop_words() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut index = TempIndex::new();
index.index_documents_config.autogenerate_docids = true;
// First we send 3 documents with ids from 1 to 3.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
// First we send 3 documents with ids from 1 to 3.
index
.add_documents_using_wtxn(
&mut wtxn,
documents!([
{ "name": "kevin", "age": 23, "maxim": "I love dogs" },
{ "name": "kevina", "age": 21, "maxim": "Doggos are the best" },
{ "name": "benoit", "age": 34, "maxim": "The crepes are really good" },
]);
let config = IndexerConfig::default();
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]),
)
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
// In the same transaction we provide some stop_words
let mut builder = Settings::new(&mut wtxn, &index, &config);
let set = btreeset! { "i".to_string(), "the".to_string(), "are".to_string() };
builder.set_stop_words(set.clone());
builder.execute(|_| ()).unwrap();
index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_stop_words(set.clone());
})
.unwrap();
wtxn.commit().unwrap();
// Ensure stop_words are effectively stored
@ -1220,11 +1128,11 @@ mod tests {
assert_eq!(result.documents_ids.len(), 1); // there is one benoit in our data
// now we'll reset the stop_words and ensure it's None
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.reset_stop_words();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.reset_stop_words();
})
.unwrap();
let rtxn = index.read_txn().unwrap();
let stop_words = index.stop_words(&rtxn).unwrap();
@ -1247,36 +1155,32 @@ mod tests {
#[test]
fn set_and_reset_synonyms() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let mut index = TempIndex::new();
index.index_documents_config.autogenerate_docids = true;
// Send 3 documents with ids from 1 to 3.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([
// Send 3 documents with ids from 1 to 3.
index
.add_documents_using_wtxn(
&mut wtxn,
documents!([
{ "name": "kevin", "age": 23, "maxim": "I love dogs"},
{ "name": "kevina", "age": 21, "maxim": "Doggos are the best"},
{ "name": "benoit", "age": 34, "maxim": "The crepes are really good"},
]);
let config = IndexerConfig::default();
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]),
)
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
// In the same transaction provide some synonyms
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_synonyms(hashmap! {
index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_synonyms(hashmap! {
"blini".to_string() => vec!["crepes".to_string()],
"super like".to_string() => vec!["love".to_string()],
"puppies".to_string() => vec!["dogs".to_string(), "doggos".to_string()]
});
builder.execute(|_| ()).unwrap();
})
.unwrap();
wtxn.commit().unwrap();
// Ensure synonyms are effectively stored
@ -1293,11 +1197,11 @@ mod tests {
assert_eq!(result.documents_ids.len(), 2);
// Reset the synonyms
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.reset_synonyms();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.reset_synonyms();
})
.unwrap();
// Ensure synonyms are reset
let rtxn = index.read_txn().unwrap();
@ -1315,20 +1219,16 @@ mod tests {
#[test]
fn setting_searchable_recomputes_other_settings() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let config = IndexerConfig::default();
let index = TempIndex::new();
// Set all the settings except searchable
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_displayed_fields(vec!["hello".to_string()]);
builder.set_filterable_fields(hashset! { S("age"), S("toto") });
builder.set_criteria(vec!["toto:asc".to_string()]);
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.set_displayed_fields(vec!["hello".to_string()]);
settings.set_filterable_fields(hashset! { S("age"), S("toto") });
settings.set_criteria(vec!["toto:asc".to_string()]);
})
.unwrap();
// check the output
let rtxn = index.read_txn().unwrap();
@ -1339,11 +1239,11 @@ mod tests {
drop(rtxn);
// We set toto and age as searchable to force reordering of the fields
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_searchable_fields(vec!["toto".to_string(), "age".to_string()]);
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.set_searchable_fields(vec!["toto".to_string(), "age".to_string()]);
})
.unwrap();
let rtxn = index.read_txn().unwrap();
assert_eq!(&["hello"][..], index.displayed_fields(&rtxn).unwrap().unwrap());
@ -1353,20 +1253,16 @@ mod tests {
#[test]
fn setting_not_filterable_cant_filter() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let config = IndexerConfig::default();
let index = TempIndex::new();
// Set all the settings except searchable
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_displayed_fields(vec!["hello".to_string()]);
index
.update_settings(|settings| {
settings.set_displayed_fields(vec!["hello".to_string()]);
// It is only Asc(toto), there is a facet database but it is denied to filter with toto.
builder.set_criteria(vec!["toto:asc".to_string()]);
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
settings.set_criteria(vec!["toto:asc".to_string()]);
})
.unwrap();
let rtxn = index.read_txn().unwrap();
let filter = Filter::from_str("toto = 32").unwrap().unwrap();
@ -1375,22 +1271,23 @@ mod tests {
#[test]
fn setting_primary_key() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let config = IndexerConfig::default();
let mut index = TempIndex::new();
index.index_documents_config.autogenerate_docids = true;
// Set the primary key settings
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_primary_key(S("mykey"));
builder.execute(|_| ()).unwrap();
// Set the primary key settings
index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.set_primary_key(S("mykey"));
})
.unwrap();
assert_eq!(index.primary_key(&wtxn).unwrap(), Some("mykey"));
// Then index some documents with the "mykey" primary key.
let content = documents!([
index
.add_documents_using_wtxn(
&mut wtxn,
documents!([
{ "mykey": 1, "name": "kevin", "age": 23 },
{ "mykey": 2, "name": "kevina", "age": 21 },
{ "mykey": 3, "name": "benoit", "age": 34 },
@ -1398,53 +1295,47 @@ mod tests {
{ "mykey": 5, "name": "bertrand", "age": 34 },
{ "mykey": 6, "name": "bernie", "age": 34 },
{ "mykey": 7, "name": "ben", "age": 34 }
]);
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
]),
)
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
// We now try to reset the primary key
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.reset_primary_key();
let err = builder.execute(|_| ()).unwrap_err();
assert!(matches!(err, Error::UserError(UserError::PrimaryKeyCannotBeChanged(_))));
let error = index
.update_settings_using_wtxn(&mut wtxn, |settings| {
settings.reset_primary_key();
})
.unwrap_err();
assert!(matches!(error, Error::UserError(UserError::PrimaryKeyCannotBeChanged(_))));
wtxn.abort().unwrap();
// But if we clear the database...
let mut wtxn = index.write_txn().unwrap();
let builder = ClearDocuments::new(&mut wtxn, &index);
builder.execute().unwrap();
wtxn.commit().unwrap();
// ...we can change the primary key
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_primary_key(S("myid"));
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
.update_settings(|settings| {
settings.set_primary_key(S("myid"));
})
.unwrap();
}
#[test]
fn setting_impact_relevancy() {
let path = tempfile::tempdir().unwrap();
let mut options = EnvOpenOptions::new();
options.map_size(10 * 1024 * 1024); // 10 MB
let index = Index::new(options, &path).unwrap();
let config = IndexerConfig::default();
let mut index = TempIndex::new();
index.index_documents_config.autogenerate_docids = true;
// Set the genres setting
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, &config);
builder.set_filterable_fields(hashset! { S("genres") });
builder.execute(|_| ()).unwrap();
index
.update_settings(|settings| {
settings.set_filterable_fields(hashset! { S("genres") });
})
.unwrap();
let content = documents!([
index.add_documents(documents!([
{
"id": 11,
"title": "Star Wars",
@ -1462,18 +1353,8 @@ mod tests {
"poster": "https://image.tmdb.org/t/p/w500/gSuHDeWemA1menrwfMRChnSmMVN.jpg",
"release_date": 819676800
}
]);
let indexing_config =
IndexDocumentsConfig { autogenerate_docids: true, ..Default::default() };
let builder =
IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ())
.unwrap();
let (builder, user_error) = builder.add_documents(content).unwrap();
user_error.unwrap();
builder.execute().unwrap();
wtxn.commit().unwrap();
])).unwrap();
// We now try to reset the primary key
let rtxn = index.read_txn().unwrap();
let SearchResult { documents_ids, .. } = index.search(&rtxn).query("S").execute().unwrap();
let first_id = documents_ids[0];
@ -1490,45 +1371,41 @@ mod tests {
let index = TempIndex::new();
let mut txn = index.write_txn().unwrap();
let config = IndexerConfig::default();
assert!(index.authorize_typos(&txn).unwrap());
let mut builder = Settings::new(&mut txn, &index, &config);
builder.set_autorize_typos(false);
builder.execute(|_| ()).unwrap();
index
.update_settings_using_wtxn(&mut txn, |settings| {
settings.set_autorize_typos(false);
})
.unwrap();
assert!(!index.authorize_typos(&txn).unwrap());
}
#[test]
fn update_min_word_len_for_typo() {
let index = TempIndex::new();
let config = IndexerConfig::default();
// Set the genres setting
let mut txn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut txn, &index, &config);
builder.set_min_word_len_one_typo(8);
builder.set_min_word_len_two_typos(8);
builder.execute(|_| ()).unwrap();
txn.commit().unwrap();
index
.update_settings(|settings| {
settings.set_min_word_len_one_typo(8);
settings.set_min_word_len_two_typos(8);
})
.unwrap();
let txn = index.read_txn().unwrap();
assert_eq!(index.min_word_len_one_typo(&txn).unwrap(), 8);
assert_eq!(index.min_word_len_two_typos(&txn).unwrap(), 8);
let mut txn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut txn, &index, &config);
builder.reset_min_word_len_one_typo();
builder.reset_min_word_len_two_typos();
builder.execute(|_| ()).unwrap();
txn.commit().unwrap();
index
.update_settings(|settings| {
settings.reset_min_word_len_one_typo();
settings.reset_min_word_len_two_typos();
})
.unwrap();
let txn = index.read_txn().unwrap();
assert_eq!(index.min_word_len_one_typo(&txn).unwrap(), DEFAULT_MIN_WORD_LEN_ONE_TYPO);
assert_eq!(index.min_word_len_two_typos(&txn).unwrap(), DEFAULT_MIN_WORD_LEN_TWO_TYPOS);
}
@ -1536,28 +1413,29 @@ mod tests {
#[test]
fn update_invalid_min_word_len_for_typo() {
let index = TempIndex::new();
let config = IndexerConfig::default();
// Set the genres setting
let mut txn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut txn, &index, &config);
builder.set_min_word_len_one_typo(10);
builder.set_min_word_len_two_typos(7);
assert!(builder.execute(|_| ()).is_err());
index
.update_settings(|settings| {
settings.set_min_word_len_one_typo(10);
settings.set_min_word_len_two_typos(7);
})
.unwrap_err();
}
#[test]
fn update_exact_words_normalization() {
let index = TempIndex::new();
let config = IndexerConfig::default();
// Set the genres setting
let mut txn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut txn, &index, &config);
// Set the genres setting
index
.update_settings_using_wtxn(&mut txn, |settings| {
let words = btreeset! { S("Ab"), S("ac") };
builder.set_exact_words(words);
assert!(builder.execute(|_| ()).is_ok());
settings.set_exact_words(words);
})
.unwrap();
let exact_words = index.exact_words(&txn).unwrap().unwrap();
for word in exact_words.into_fst().stream().into_str_vec().unwrap() {
assert!(word.0 == "ac" || word.0 == "ab");
@ -1567,10 +1445,10 @@ mod tests {
#[test]
fn test_correct_settings_init() {
let index = TempIndex::new();
let config = IndexerConfig::default();
let mut txn = index.write_txn().unwrap();
let builder = Settings::new(&mut txn, &index, &config);
index
.update_settings(|settings| {
// we don't actually update the settings, just check their content
let Settings {
wtxn: _,
index: _,
@ -1591,8 +1469,7 @@ mod tests {
exact_attributes,
max_values_per_facet,
pagination_max_total_hits,
} = builder;
} = settings;
assert!(matches!(searchable_fields, Setting::NotSet));
assert!(matches!(displayed_fields, Setting::NotSet));
assert!(matches!(filterable_fields, Setting::NotSet));
@ -1609,5 +1486,7 @@ mod tests {
assert!(matches!(exact_attributes, Setting::NotSet));
assert!(matches!(max_values_per_facet, Setting::NotSet));
assert!(matches!(pagination_max_total_hits, Setting::NotSet));
})
.unwrap();
}
}