searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-22 21:04:27 +01:00
Code Issues Releases Wiki Activity

remove update_id in UpdateBuilder

Browse Source
This commit is contained in:
Marin Postma 2021-11-03 13:12:01 +01:00
parent 21b78f3926
commit 6eb47ab792
17 changed files with 217 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
benchmarks/benches/indexing.rs
View File

@ -39,7 +39,7 @@ fn indexing_songs_default(c: &mut Criterion) {
move || {
let index = setup_index();
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let mut builder = update_builder.settings(&mut wtxn, &index);
@ -61,17 +61,17 @@ fn indexing_songs_default(c: &mut Criterion) {
.map(|s| s.to_string())
.collect();
builder.set_filterable_fields(faceted_fields);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
},
move |index| {
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let builder = update_builder.index_documents(&mut wtxn, &index);
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
builder.execute(documents, |_, _| ()).unwrap();
builder.execute(documents, |_| ()).unwrap();
wtxn.commit().unwrap();
index.prepare_for_closing().wait();
@ -88,7 +88,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
move || {
let index = setup_index();
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let mut builder = update_builder.settings(&mut wtxn, &index);
@ -107,17 +107,17 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
let faceted_fields =
["genre", "country", "artist"].iter().map(|s| s.to_string()).collect();
builder.set_filterable_fields(faceted_fields);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
},
move |index| {
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let builder = update_builder.index_documents(&mut wtxn, &index);
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
builder.execute(documents, |_, _| ()).unwrap();
builder.execute(documents, |_| ()).unwrap();
wtxn.commit().unwrap();
index.prepare_for_closing().wait();
@ -134,7 +134,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
move || {
let index = setup_index();
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let mut builder = update_builder.settings(&mut wtxn, &index);
@ -149,17 +149,17 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
let searchable_fields =
["title", "album", "artist"].iter().map(|s| s.to_string()).collect();
builder.set_searchable_fields(searchable_fields);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
},
move |index| {
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let builder = update_builder.index_documents(&mut wtxn, &index);
let documents = utils::documents_from(datasets_paths::SMOL_SONGS, "csv");
builder.execute(documents, |_, _| ()).unwrap();
builder.execute(documents, |_| ()).unwrap();
wtxn.commit().unwrap();
index.prepare_for_closing().wait();
@ -176,7 +176,7 @@ fn indexing_wiki(c: &mut Criterion) {
move || {
let index = setup_index();
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let mut builder = update_builder.settings(&mut wtxn, &index);
@ -190,18 +190,18 @@ fn indexing_wiki(c: &mut Criterion) {
// there is NO faceted fields at all
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
},
move |index| {
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let mut builder = update_builder.index_documents(&mut wtxn, &index);
builder.enable_autogenerate_docids();
let documents = utils::documents_from(datasets_paths::SMOL_WIKI_ARTICLES, "csv");
builder.execute(documents, |_, _| ()).unwrap();
builder.execute(documents, |_| ()).unwrap();
wtxn.commit().unwrap();
index.prepare_for_closing().wait();
@ -218,7 +218,7 @@ fn indexing_movies_default(c: &mut Criterion) {
move || {
let index = setup_index();
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let mut builder = update_builder.settings(&mut wtxn, &index);
@ -237,17 +237,17 @@ fn indexing_movies_default(c: &mut Criterion) {
["released_date", "genres"].iter().map(|s| s.to_string()).collect();
builder.set_filterable_fields(faceted_fields);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
},
move |index| {
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let builder = update_builder.index_documents(&mut wtxn, &index);
let documents = utils::documents_from(datasets_paths::MOVIES, "json");
builder.execute(documents, |_, _| ()).unwrap();
builder.execute(documents, |_| ()).unwrap();
wtxn.commit().unwrap();
index.prepare_for_closing().wait();
@ -264,7 +264,7 @@ fn indexing_geo(c: &mut Criterion) {
move || {
let index = setup_index();
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let mut builder = update_builder.settings(&mut wtxn, &index);
@ -288,17 +288,17 @@ fn indexing_geo(c: &mut Criterion) {
["_geo", "population", "elevation"].iter().map(|s| s.to_string()).collect();
builder.set_sortable_fields(sortable_fields);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
index
},
move |index| {
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let builder = update_builder.index_documents(&mut wtxn, &index);
let documents = utils::documents_from(datasets_paths::SMOL_ALL_COUNTRIES, "jsonl");
builder.execute(documents, |_, _| ()).unwrap();
builder.execute(documents, |_| ()).unwrap();
wtxn.commit().unwrap();

8
benchmarks/benches/utils.rs
View File

@ -65,7 +65,7 @@ pub fn base_setup(conf: &Conf) -> Index {
options.max_readers(10);
let index = Index::new(options, conf.database_name).unwrap();
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let mut builder = update_builder.settings(&mut wtxn, &index);
@ -84,10 +84,10 @@ pub fn base_setup(conf: &Conf) -> Index {
(conf.configure)(&mut builder);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
let update_builder = UpdateBuilder::new(0);
let update_builder = UpdateBuilder::new();
let mut wtxn = index.write_txn().unwrap();
let mut builder = update_builder.index_documents(&mut wtxn, &index);
if let None = conf.primary_key {
@ -96,7 +96,7 @@ pub fn base_setup(conf: &Conf) -> Index {
let documents = documents_from(conf.dataset, conf.dataset_format);
builder.index_documents_method(IndexDocumentsMethod::ReplaceDocuments);
builder.execute(documents, |_, _| ()).unwrap();
builder.execute(documents, |_| ()).unwrap();
wtxn.commit().unwrap();
index

8
cli/src/main.rs
View File

@ -122,7 +122,7 @@ impl DocumentAddition {
println!("Adding {} documents to the index.", reader.len());
let mut txn = index.env.write_txn()?;
let mut addition = milli::update::IndexDocuments::new(&mut txn, &index, 0);
let mut addition = milli::update::IndexDocuments::new(&mut txn, &index);
if self.update_documents {
addition.index_documents_method(milli::update::IndexDocumentsMethod::UpdateDocuments);
@ -146,7 +146,7 @@ impl DocumentAddition {
progesses.join().unwrap();
});
let result = addition.execute(reader, |step, _| indexing_callback(step, &bars))?;
let result = addition.execute(reader, |step| indexing_callback(step, &bars))?;
txn.commit()?;
@ -292,7 +292,7 @@ impl SettingsUpdate {
fn perform(&self, index: milli::Index) -> Result<()> {
let mut txn = index.env.write_txn()?;
let mut update = milli::update::Settings::new(&mut txn, &index, 0);
let mut update = milli::update::Settings::new(&mut txn, &index);
update.log_every_n(100);
if let Some(ref filterable_attributes) = self.filterable_attributes {
@ -315,7 +315,7 @@ impl SettingsUpdate {
progesses.join().unwrap();
});
update.execute(|step, _| indexing_callback(step, &bars))?;
update.execute(|step| indexing_callback(step, &bars))?;
txn.commit()?;
Ok(())

6
http-ui/src/main.rs
View File

@ -343,7 +343,7 @@ async fn main() -> anyhow::Result<()> {
// the type hint is necessary: https://github.com/rust-lang/rust/issues/32600
move |update_id, meta, content: &_| {
// We prepare the update by using the update builder.
let mut update_builder = UpdateBuilder::new(update_id);
let mut update_builder = UpdateBuilder::new();
if let Some(max_nb_chunks) = indexer_opt_cloned.max_nb_chunks {
update_builder.max_nb_chunks(max_nb_chunks);
}
@ -393,7 +393,7 @@ async fn main() -> anyhow::Result<()> {
let documents = DocumentBatchReader::from_reader(Cursor::new(documents))?;
let result = builder.execute(documents, |indexing_step, update_id| {
let result = builder.execute(documents, |indexing_step| {
let (current, total) = match indexing_step {
RemapDocumentAddition { documents_seen } => (documents_seen, None),
ComputeIdsAndMergeDocuments { documents_seen, total_documents } => {
@ -494,7 +494,7 @@ async fn main() -> anyhow::Result<()> {
Setting::NotSet => (),
}
let result = builder.execute(|indexing_step, update_id| {
let result = builder.execute(|indexing_step| {
let (current, total) = match indexing_step {
RemapDocumentAddition { documents_seen } => (documents_seen, None),
ComputeIdsAndMergeDocuments { documents_seen, total_documents } => {

12
milli/src/index.rs
View File

@ -908,8 +908,8 @@ pub(crate) mod tests {
{ "id": 2, "name": "bob", "age": 20 },
{ "id": 2, "name": "bob", "age": 20 }
]);
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();
@ -927,13 +927,13 @@ pub(crate) mod tests {
// we add all the documents a second time. we are supposed to get the same
// field_distribution in the end
let mut wtxn = index.write_txn().unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
let builder = IndexDocuments::new(&mut wtxn, &index);
let content = documents!([
{ "id": 1, "name": "kevin" },
{ "id": 2, "name": "bob", "age": 20 },
{ "id": 2, "name": "bob", "age": 20 }
]);
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();
@ -955,8 +955,8 @@ pub(crate) mod tests {
]);
let mut wtxn = index.write_txn().unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();

8
milli/src/search/distinct/mod.rs
View File

@ -84,19 +84,19 @@ mod test {
let mut txn = index.write_txn().unwrap();
// set distinct and faceted attributes for the index.
let builder = UpdateBuilder::new(0);
let builder = UpdateBuilder::new();
let mut update = builder.settings(&mut txn, &index);
update.set_distinct_field(distinct.to_string());
update.execute(|_, _| ()).unwrap();
update.execute(|_| ()).unwrap();
// add documents to the index
let builder = UpdateBuilder::new(1);
let builder = UpdateBuilder::new();
let mut addition = builder.index_documents(&mut txn, &index);
addition.index_documents_method(IndexDocumentsMethod::ReplaceDocuments);
let reader =
crate::documents::DocumentBatchReader::from_reader(Cursor::new(&*JSON)).unwrap();
addition.execute(reader, |_, _| ()).unwrap();
addition.execute(reader, |_| ()).unwrap();
let fields_map = index.fields_ids_map(&txn).unwrap();
let fid = fields_map.id(&distinct).unwrap();

8
milli/src/search/facet/filter.rs
View File

@ -512,10 +512,10 @@ mod tests {
// Set the filterable fields to be the channel.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_searchable_fields(vec![S("title")]);
builder.set_filterable_fields(hashset! { S("title") });
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();
@ -542,10 +542,10 @@ mod tests {
// Set the filterable fields to be the channel.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_searchable_fields(vec![S("_geo"), S("price")]); // to keep the fields order
builder.set_filterable_fields(hashset! { S("_geo"), S("price") });
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();

13
milli/src/update/clear_documents.rs
View File

@ -6,16 +6,11 @@ use crate::{ExternalDocumentsIds, FieldDistribution, Index, Result};
pub struct ClearDocuments<'t, 'u, 'i> {
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
_update_id: u64,
}
impl<'t, 'u, 'i> ClearDocuments<'t, 'u, 'i> {
pub fn new(
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
update_id: u64,
) -> ClearDocuments<'t, 'u, 'i> {
ClearDocuments { wtxn, index, _update_id: update_id }
pub fn new(wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index) -> ClearDocuments<'t, 'u, 'i> {
ClearDocuments { wtxn, index }
}
pub fn execute(self) -> Result<u64> {
@ -97,10 +92,10 @@ mod tests {
{ "id": 1, "name": "kevina" },
{ "id": 2, "name": "benoit", "country": "France", "_geo": { "lng": 42, "lat": 35 } }
]);
IndexDocuments::new(&mut wtxn, &index, 0).execute(content, |_, _| ()).unwrap();
IndexDocuments::new(&mut wtxn, &index).execute(content, |_| ()).unwrap();
// Clear all documents from the database.
let builder = ClearDocuments::new(&mut wtxn, &index, 1);
let builder = ClearDocuments::new(&mut wtxn, &index);
assert_eq!(builder.execute().unwrap(), 3);
wtxn.commit().unwrap();

36
milli/src/update/delete_documents.rs
View File

@ -23,7 +23,6 @@ pub struct DeleteDocuments<'t, 'u, 'i> {
index: &'i Index,
external_documents_ids: ExternalDocumentsIds<'static>,
documents_ids: RoaringBitmap,
update_id: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@ -36,7 +35,6 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
pub fn new(
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
update_id: u64,
) -> Result<DeleteDocuments<'t, 'u, 'i>> {
let external_documents_ids = index.external_documents_ids(wtxn)?.into_static();
@ -45,7 +43,6 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
index,
external_documents_ids,
documents_ids: RoaringBitmap::new(),
update_id,
})
}
@ -85,8 +82,7 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
// We can execute a ClearDocuments operation when the number of documents
// to delete is exactly the number of documents in the database.
if current_documents_ids_len == self.documents_ids.len() {
let remaining_documents =
ClearDocuments::new(self.wtxn, self.index, self.update_id).execute()?;
let remaining_documents = ClearDocuments::new(self.wtxn, self.index).execute()?;
return Ok(DocumentDeletionResult {
deleted_documents: current_documents_ids_len,
remaining_documents,
@ -600,11 +596,11 @@ mod tests {
{ "id": 1, "name": "kevina", "array": ["I", "am", "fine"] },
{ "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] }
]);
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
// delete those documents, ids are synchronous therefore 0, 1, and 2.
let mut builder = DeleteDocuments::new(&mut wtxn, &index, 1).unwrap();
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.delete_document(0);
builder.delete_document(1);
builder.delete_document(2);
@ -630,11 +626,11 @@ mod tests {
{ "mysuperid": 1, "name": "kevina" },
{ "mysuperid": 2, "name": "benoit" }
]);
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
// Delete not all of the documents but some of them.
let mut builder = DeleteDocuments::new(&mut wtxn, &index, 1).unwrap();
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.delete_external_id("0");
builder.delete_external_id("1");
builder.execute().unwrap();
@ -650,10 +646,10 @@ mod tests {
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_primary_key(S("docid"));
builder.set_filterable_fields(hashset! { S("label") });
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
let content = documents!([
{"docid":"1_4","label":"sign"},
@ -677,11 +673,11 @@ mod tests {
{"docid":"1_68","label":"design"},
{"docid":"1_69","label":"geometry"}
]);
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
// Delete not all of the documents but some of them.
let mut builder = DeleteDocuments::new(&mut wtxn, &index, 1).unwrap();
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.delete_external_id("1_4");
builder.execute().unwrap();
@ -700,11 +696,11 @@ mod tests {
let index = Index::new(options, &path).unwrap();
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_primary_key(S("id"));
builder.set_filterable_fields(hashset!(S("_geo")));
builder.set_sortable_fields(hashset!(S("_geo")));
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
let content = documents!([
{"id":"1","city":"Lille", "_geo": { "lat": 50.629973371633746, "lng": 3.0569447399419570 } },
@ -730,7 +726,7 @@ mod tests {
]);
let external_ids_to_delete = ["5", "6", "7", "12", "17", "19"];
IndexDocuments::new(&mut wtxn, &index, 0).execute(content, |_, _| ()).unwrap();
IndexDocuments::new(&mut wtxn, &index).execute(content, |_| ()).unwrap();
let external_document_ids = index.external_documents_ids(&wtxn).unwrap();
let ids_to_delete: Vec<u32> = external_ids_to_delete
@ -739,7 +735,7 @@ mod tests {
.collect();
// Delete some documents.
let mut builder = DeleteDocuments::new(&mut wtxn, &index, 1).unwrap();
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
external_ids_to_delete.iter().for_each(|id| drop(builder.delete_external_id(id)));
builder.execute().unwrap();

8
milli/src/update/facets.rs
View File

@ -27,15 +27,10 @@ pub struct Facets<'t, 'u, 'i> {
pub(crate) chunk_compression_level: Option<u32>,
level_group_size: NonZeroUsize,
min_level_size: NonZeroUsize,
_update_id: u64,
}
impl<'t, 'u, 'i> Facets<'t, 'u, 'i> {
pub fn new(
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
update_id: u64,
) -> Facets<'t, 'u, 'i> {
pub fn new(wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index) -> Facets<'t, 'u, 'i> {
Facets {
wtxn,
index,
@ -43,7 +38,6 @@ impl<'t, 'u, 'i> Facets<'t, 'u, 'i> {
chunk_compression_level: None,
level_group_size: NonZeroUsize::new(4).unwrap(),
min_level_size: NonZeroUsize::new(5).unwrap(),
_update_id: update_id,
}
}

114
milli/src/update/index_documents/mod.rs
View File

@ -80,14 +80,12 @@ pub struct IndexDocuments<'t, 'u, 'i, 'a> {
words_positions_min_level_size: Option<NonZeroU32>,
update_method: IndexDocumentsMethod,
autogenerate_docids: bool,
update_id: u64,
}
impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
pub fn new(
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
update_id: u64,
) -> IndexDocuments<'t, 'u, 'i, 'a> {
IndexDocuments {
wtxn,
@ -107,7 +105,6 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
words_positions_min_level_size: None,
update_method: IndexDocumentsMethod::ReplaceDocuments,
autogenerate_docids: false,
update_id,
max_positions_per_attributes: None,
}
}
@ -136,7 +133,7 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
) -> Result<DocumentAdditionResult>
where
R: Read + Seek,
F: Fn(UpdateIndexingStep, u64) + Sync,
F: Fn(UpdateIndexingStep) + Sync,
{
// Early return when there is no document to add
if reader.is_empty() {
@ -148,8 +145,6 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
self.index.set_updated_at(self.wtxn, &Utc::now())?;
let before_transform = Instant::now();
let update_id = self.update_id;
let progress_callback = |step| progress_callback(step, update_id);
let transform = Transform {
rtxn: &self.wtxn,
index: self.index,
@ -162,7 +157,7 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
autogenerate_docids: self.autogenerate_docids,
};
let output = transform.read_documents(reader, progress_callback)?;
let output = transform.read_documents(reader, &progress_callback)?;
let indexed_documents = output.documents_count as u64;
info!("Update transformed in {:.02?}", before_transform.elapsed());
@ -295,7 +290,6 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
chunk_compression_level: self.chunk_compression_level,
thread_pool: self.thread_pool,
max_positions_per_attributes: self.max_positions_per_attributes,
update_id: self.update_id,
};
let mut deletion_builder = update_builder.delete_documents(self.wtxn, self.index)?;
debug!("documents to delete {:?}", replaced_documents_ids);
@ -364,7 +358,7 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
let mut databases_seen = MERGED_DATABASE_COUNT;
// Run the facets update operation.
let mut builder = Facets::new(self.wtxn, self.index, self.update_id);
let mut builder = Facets::new(self.wtxn, self.index);
builder.chunk_compression_type = self.chunk_compression_type;
builder.chunk_compression_level = self.chunk_compression_level;
if let Some(value) = self.facet_level_group_size {
@ -382,7 +376,7 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
});
// Run the words prefixes update operation.
let mut builder = WordsPrefixesFst::new(self.wtxn, self.index, self.update_id);
let mut builder = WordsPrefixesFst::new(self.wtxn, self.index);
if let Some(value) = self.words_prefix_threshold {
builder.threshold(value);
}
@ -475,8 +469,8 @@ mod tests {
{ "id": 2, "name": "kevina" },
{ "id": 3, "name": "benoit" }
]);
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is 3 documents now.
@ -488,8 +482,8 @@ mod tests {
// Second we send 1 document with id 1, to erase the previous ones.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([ { "id": 1, "name": "updated kevin" } ]);
let builder = IndexDocuments::new(&mut wtxn, &index, 1);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is **always** 3 documents.
@ -505,8 +499,8 @@ mod tests {
{ "id": 2, "name": "updated kevina" },
{ "id": 3, "name": "updated benoit" }
]);
let builder = IndexDocuments::new(&mut wtxn, &index, 2);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is **always** 3 documents.
@ -531,9 +525,9 @@ mod tests {
{ "id": 1, "name": "kevina" },
{ "id": 1, "name": "benoit" }
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.index_documents_method(IndexDocumentsMethod::UpdateDocuments);
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is only 1 document now.
@ -557,9 +551,9 @@ mod tests {
// Second we send 1 document with id 1, to force it to be merged with the previous one.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([ { "id": 1, "age": 25 } ]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 1);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.index_documents_method(IndexDocumentsMethod::UpdateDocuments);
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is **always** 1 document.
@ -596,8 +590,8 @@ mod tests {
{ "name": "kevina" },
{ "name": "benoit" }
]);
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
assert!(builder.execute(content, |_, _| ()).is_err());
let builder = IndexDocuments::new(&mut wtxn, &index);
assert!(builder.execute(content, |_| ()).is_err());
wtxn.commit().unwrap();
// Check that there is no document.
@ -621,9 +615,9 @@ mod tests {
{ "name": "kevina" },
{ "name": "benoit" }
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.enable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is 3 documents now.
@ -639,8 +633,8 @@ mod tests {
// Second we send 1 document with the generated uuid, to erase the previous ones.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([ { "name": "updated kevin", "id": kevin_uuid } ]);
let builder = IndexDocuments::new(&mut wtxn, &index, 1);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is **always** 3 documents.
@ -676,8 +670,8 @@ mod tests {
{ "id": 2, "name": "kevina" },
{ "id": 3, "name": "benoit" }
]);
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is 3 documents now.
@ -689,9 +683,9 @@ mod tests {
// Second we send 1 document without specifying the id.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([ { "name": "new kevin" } ]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 1);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.enable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is 4 documents now.
@ -711,8 +705,8 @@ mod tests {
// First we send 0 documents and only headers.
let mut wtxn = index.write_txn().unwrap();
let content = documents!([]);
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is no documents.
@ -733,16 +727,16 @@ mod tests {
let mut wtxn = index.write_txn().unwrap();
// There is a space in the document id.
let content = documents!([ { "id": "brume bleue", "name": "kevin" } ]);
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
assert!(builder.execute(content, |_, _| ()).is_err());
let builder = IndexDocuments::new(&mut wtxn, &index);
assert!(builder.execute(content, |_| ()).is_err());
wtxn.commit().unwrap();
// First we send 1 document with a valid id.
let mut wtxn = index.write_txn().unwrap();
// There is a space in the document id.
let content = documents!([ { "id": 32, "name": "kevin" } ]);
let builder = IndexDocuments::new(&mut wtxn, &index, 1);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is 1 document now.
@ -766,8 +760,8 @@ mod tests {
{ "id": 1, "name": "kevina", "array": ["I", "am", "fine"] },
{ "id": 2, "name": "benoit", "array_of_object": [{ "wow": "amazing" }] }
]);
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is 1 documents now.
@ -805,13 +799,13 @@ mod tests {
{ "id": 4, "title": "Harry Potter and the Half-Blood Prince", "author": "J. K. Rowling", "genre": "fantasy" },
{ "id": 42, "title": "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "_geo": { "lat": 35, "lng": 23 } }
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.index_documents_method(IndexDocumentsMethod::ReplaceDocuments);
builder.execute(documents, |_, _| ()).unwrap();
builder.execute(documents, |_| ()).unwrap();
wtxn.commit().unwrap();
let mut wtxn = index.write_txn().unwrap();
let mut builder = IndexDocuments::new(&mut wtxn, &index, 1);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.index_documents_method(IndexDocumentsMethod::UpdateDocuments);
let documents = documents!([
{
@ -821,7 +815,7 @@ mod tests {
}
]);
builder.execute(documents, |_, _| ()).unwrap();
builder.execute(documents, |_| ()).unwrap();
wtxn.commit().unwrap();
}
@ -839,12 +833,12 @@ mod tests {
{ "objectId": 1, "title": "Alice In Wonderland", "comment": "A weird book" },
{ "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } }
]);
IndexDocuments::new(&mut wtxn, &index, 0).execute(content, |_, _| ()).unwrap();
IndexDocuments::new(&mut wtxn, &index).execute(content, |_| ()).unwrap();
assert_eq!(index.primary_key(&wtxn).unwrap(), Some("objectId"));
// Delete not all of the documents but some of them.
let mut builder = DeleteDocuments::new(&mut wtxn, &index, 1).unwrap();
let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
builder.delete_external_id("30");
builder.execute().unwrap();
@ -854,7 +848,7 @@ mod tests {
let content = documents!([
{ "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } }
]);
IndexDocuments::new(&mut wtxn, &index, 0).execute(content, |_, _| ()).unwrap();
IndexDocuments::new(&mut wtxn, &index).execute(content, |_| ()).unwrap();
let external_documents_ids = index.external_documents_ids(&wtxn).unwrap();
assert!(external_documents_ids.get("30").is_some());
@ -862,7 +856,7 @@ mod tests {
let content = documents!([
{ "objectId": 30, "title": "Hamlet", "_geo": { "lat": 12, "lng": 89 } }
]);
IndexDocuments::new(&mut wtxn, &index, 0).execute(content, |_, _| ()).unwrap();
IndexDocuments::new(&mut wtxn, &index).execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
}
@ -892,8 +886,8 @@ mod tests {
cursor.set_position(0);
let content = DocumentBatchReader::from_reader(cursor).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
}
@ -922,8 +916,8 @@ mod tests {
cursor.set_position(0);
let content = DocumentBatchReader::from_reader(cursor).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
@ -975,8 +969,8 @@ mod tests {
},
]);
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
}
@ -996,8 +990,8 @@ mod tests {
]);
let mut wtxn = index.write_txn().unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is 4 document now.
@ -1008,8 +1002,8 @@ mod tests {
let content = documents!([]);
let mut wtxn = index.write_txn().unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index, 1);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is 4 document now.
@ -1025,8 +1019,8 @@ mod tests {
]);
let mut wtxn = index.write_txn().unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index, 2);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that there is 4 document now.
@ -1048,8 +1042,8 @@ mod tests {
]);
let mut wtxn = index.write_txn().unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();

152
milli/src/update/settings.rs
View File

@ -85,7 +85,6 @@ pub struct Settings<'a, 't, 'u, 'i> {
pub(crate) chunk_compression_level: Option<u32>,
pub(crate) thread_pool: Option<&'a ThreadPool>,
pub(crate) max_positions_per_attributes: Option<u32>,
update_id: u64,
searchable_fields: Setting<Vec<String>>,
displayed_fields: Setting<Vec<String>>,
@ -99,11 +98,7 @@ pub struct Settings<'a, 't, 'u, 'i> {
}
impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
pub fn new(
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
update_id: u64,
) -> Settings<'a, 't, 'u, 'i> {
pub fn new(wtxn: &'t mut heed::RwTxn<'i, 'u>, index: &'i Index) -> Settings<'a, 't, 'u, 'i> {
Settings {
wtxn,
index,
@ -123,7 +118,6 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
distinct_field: Setting::NotSet,
synonyms: Setting::NotSet,
primary_key: Setting::NotSet,
update_id,
max_positions_per_attributes: None,
}
}
@ -207,11 +201,9 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
fn reindex<F>(&mut self, cb: &F, old_fields_ids_map: FieldsIdsMap) -> Result<()>
where
F: Fn(UpdateIndexingStep, u64) + Sync,
F: Fn(UpdateIndexingStep) + Sync,
{
let fields_ids_map = self.index.fields_ids_map(self.wtxn)?;
let update_id = self.update_id;
let cb = |step| cb(step, update_id);
// if the settings are set before any document update, we don't need to do anything, and
// will set the primary key during the first document addition.
if self.index.number_of_documents(&self.wtxn)? == 0 {
@ -242,11 +234,11 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
)?;
// We clear the full database (words-fst, documents ids and documents content).
ClearDocuments::new(self.wtxn, self.index, self.update_id).execute()?;
ClearDocuments::new(self.wtxn, self.index).execute()?;
// We index the generated `TransformOutput` which must contain
// all the documents with fields in the newly defined searchable order.
let mut indexing_builder = IndexDocuments::new(self.wtxn, self.index, self.update_id);
let mut indexing_builder = IndexDocuments::new(self.wtxn, self.index);
indexing_builder.log_every_n = self.log_every_n;
indexing_builder.max_nb_chunks = self.max_nb_chunks;
indexing_builder.max_memory = self.max_memory;
@ -484,7 +476,7 @@ impl<'a, 't, 'u, 'i> Settings<'a, 't, 'u, 'i> {
pub fn execute<F>(mut self, progress_callback: F) -> Result<()>
where
F: Fn(UpdateIndexingStep, u64) + Sync,
F: Fn(UpdateIndexingStep) + Sync,
{
self.index.set_updated_at(self.wtxn, &Utc::now())?;
@ -543,15 +535,15 @@ mod tests {
{ "id": 2, "name": "kevina", "age": 21},
{ "id": 3, "name": "benoit", "age": 34 }
]);
let builder = IndexDocuments::new(&mut wtxn, &index, 0);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// We change the searchable fields to be the "name" field only.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 1);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_searchable_fields(vec!["name".into()]);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
// Check that the searchable field is correctly set to "name" only.
@ -571,9 +563,9 @@ mod tests {
// We change the searchable fields to be the "name" field only.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 2);
let mut builder = Settings::new(&mut wtxn, &index);
builder.reset_searchable_fields();
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
// Check that the searchable field have been reset and documents are found now.
@ -600,18 +592,18 @@ mod tests {
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 }
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.enable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// In the same transaction we change the displayed fields to be only the "age".
// We also change the searchable fields to be the "name" field only.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 1);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_displayed_fields(vec!["age".into()]);
builder.set_searchable_fields(vec!["name".into()]);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
// Check that the displayed fields are correctly set to `None` (default value).
@ -622,9 +614,9 @@ mod tests {
// We change the searchable fields to be the "name" field only.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 2);
let mut builder = Settings::new(&mut wtxn, &index);
builder.reset_searchable_fields();
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
// Check that the displayed fields always contains only the "age" field.
@ -647,9 +639,9 @@ mod tests {
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 }
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.enable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that the displayed fields are correctly set to `None` (default value).
@ -672,14 +664,14 @@ mod tests {
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 }
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.enable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
// In the same transaction we change the displayed fields to be only the age.
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_displayed_fields(vec!["age".into()]);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
// Check that the displayed fields are correctly set to only the "age" field.
@ -690,9 +682,9 @@ mod tests {
// We reset the fields ids to become `None`, the default value.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.reset_displayed_fields();
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
// Check that the displayed fields are correctly set to `None` (default value).
@ -710,9 +702,9 @@ mod tests {
// Set the filterable fields to be the age.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_filterable_fields(hashset! { S("age") });
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
// Then index some documents.
let content = documents!([
@ -720,9 +712,9 @@ mod tests {
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 }
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 1);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.enable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Check that the displayed fields are correctly set.
@ -757,9 +749,9 @@ mod tests {
{ "name": "benoit", "age": 35 }
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 2);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.enable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();
@ -782,11 +774,11 @@ mod tests {
// Set the filterable fields to be the age.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
// Don't display the generated `id` field.
builder.set_displayed_fields(vec![S("name")]);
builder.set_criteria(vec![S("age:asc")]);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
// Then index some documents.
let content = documents!([
@ -794,9 +786,9 @@ mod tests {
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 }
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 1);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.enable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Run an empty query just to ensure that the search results are ordered.
@ -824,11 +816,11 @@ mod tests {
// Set the filterable fields to be the age.
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
// Don't display the generated `id` field.
builder.set_displayed_fields(vec![S("name"), S("age")]);
builder.set_distinct_field(S("age"));
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
// Then index some documents.
let content = documents!([
@ -840,9 +832,9 @@ mod tests {
{ "name": "bernie", "age": 34 },
{ "name": "ben", "age": 34 }
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 1);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.enable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Run an empty query just to ensure that the search results are ordered.
@ -867,9 +859,9 @@ mod tests {
{ "name": "kevina", "age": 21 },
{ "name": "benoit", "age": 34 }
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.enable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// Ensure there is no stop_words by default
@ -892,15 +884,15 @@ mod tests {
{ "name": "kevina", "age": 21, "maxim": "Doggos are the best" },
{ "name": "benoit", "age": 34, "maxim": "The crepes are really good" },
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.enable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
// In the same transaction we provide some stop_words
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
let set = btreeset! { "i".to_string(), "the".to_string(), "are".to_string() };
builder.set_stop_words(set.clone());
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
// Ensure stop_words are effectively stored
@ -928,9 +920,9 @@ mod tests {
// now we'll reset the stop_words and ensure it's None
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.reset_stop_words();
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();
@ -966,18 +958,18 @@ mod tests {
{ "name": "kevina", "age": 21, "maxim": "Doggos are the best"},
{ "name": "benoit", "age": 34, "maxim": "The crepes are really good"},
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 0);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.enable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
// In the same transaction provide some synonyms
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_synonyms(hashmap! {
"blini".to_string() => vec!["crepes".to_string()],
"super like".to_string() => vec!["love".to_string()],
"puppies".to_string() => vec!["dogs".to_string(), "doggos".to_string()]
});
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
// Ensure synonyms are effectively stored
@ -995,9 +987,9 @@ mod tests {
// Reset the synonyms
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.reset_synonyms();
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
// Ensure synonyms are reset
@ -1023,11 +1015,11 @@ mod tests {
// Set all the settings except searchable
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_displayed_fields(vec!["hello".to_string()]);
builder.set_filterable_fields(hashset! { S("age"), S("toto") });
builder.set_criteria(vec!["toto:asc".to_string()]);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
// check the output
@ -1040,9 +1032,9 @@ mod tests {
// We set toto and age as searchable to force reordering of the fields
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 1);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_searchable_fields(vec!["toto".to_string(), "age".to_string()]);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();
@ -1060,11 +1052,11 @@ mod tests {
// Set all the settings except searchable
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_displayed_fields(vec!["hello".to_string()]);
// It is only Asc(toto), there is a facet database but it is denied to filter with toto.
builder.set_criteria(vec!["toto:asc".to_string()]);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();
@ -1081,10 +1073,10 @@ mod tests {
// Set the primary key settings
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_primary_key(S("mykey"));
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
assert_eq!(index.primary_key(&wtxn).unwrap(), Some("mykey"));
// Then index some documents with the "mykey" primary key.
@ -1097,29 +1089,29 @@ mod tests {
{ "mykey": 6, "name": "bernie", "age": 34 },
{ "mykey": 7, "name": "ben", "age": 34 }
]);
let mut builder = IndexDocuments::new(&mut wtxn, &index, 1);
let mut builder = IndexDocuments::new(&mut wtxn, &index);
builder.disable_autogenerate_docids();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// We now try to reset the primary key
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.reset_primary_key();
let err = builder.execute(|_, _| ()).unwrap_err();
let err = builder.execute(|_| ()).unwrap_err();
assert!(matches!(err, Error::UserError(UserError::PrimaryKeyCannotBeChanged(_))));
wtxn.abort().unwrap();
// But if we clear the database...
let mut wtxn = index.write_txn().unwrap();
let builder = ClearDocuments::new(&mut wtxn, &index, 0);
let builder = ClearDocuments::new(&mut wtxn, &index);
builder.execute().unwrap();
// ...we can change the primary key
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_primary_key(S("myid"));
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
}
@ -1132,9 +1124,9 @@ mod tests {
// Set the genres setting
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_filterable_fields(hashset! { S("genres") });
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
let content = documents!([
{
@ -1155,8 +1147,8 @@ mod tests {
"release_date": 819676800
}
]);
let builder = IndexDocuments::new(&mut wtxn, &index, 1);
builder.execute(content, |_, _| ()).unwrap();
let builder = IndexDocuments::new(&mut wtxn, &index);
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();
// We now try to reset the primary key

14
milli/src/update/update_builder.rs
View File

@ -13,11 +13,10 @@ pub struct UpdateBuilder<'a> {
pub(crate) chunk_compression_level: Option<u32>,
pub(crate) thread_pool: Option<&'a ThreadPool>,
pub(crate) max_positions_per_attributes: Option<u32>,
pub(crate) update_id: u64,
}
impl<'a> UpdateBuilder<'a> {
pub fn new(update_id: u64) -> UpdateBuilder<'a> {
pub fn new() -> UpdateBuilder<'a> {
UpdateBuilder {
log_every_n: None,
max_nb_chunks: None,
@ -27,7 +26,6 @@ impl<'a> UpdateBuilder<'a> {
chunk_compression_level: None,
thread_pool: None,
max_positions_per_attributes: None,
update_id,
}
}
@ -68,7 +66,7 @@ impl<'a> UpdateBuilder<'a> {
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
) -> ClearDocuments<'t, 'u, 'i> {
ClearDocuments::new(wtxn, index, self.update_id)
ClearDocuments::new(wtxn, index)
}
pub fn delete_documents<'t, 'u, 'i>(
@ -76,7 +74,7 @@ impl<'a> UpdateBuilder<'a> {
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
) -> Result<DeleteDocuments<'t, 'u, 'i>> {
DeleteDocuments::new(wtxn, index, self.update_id)
DeleteDocuments::new(wtxn, index)
}
pub fn index_documents<'t, 'u, 'i>(
@ -84,7 +82,7 @@ impl<'a> UpdateBuilder<'a> {
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
) -> IndexDocuments<'t, 'u, 'i, 'a> {
let mut builder = IndexDocuments::new(wtxn, index, self.update_id);
let mut builder = IndexDocuments::new(wtxn, index);
builder.log_every_n = self.log_every_n;
builder.max_nb_chunks = self.max_nb_chunks;
@ -103,7 +101,7 @@ impl<'a> UpdateBuilder<'a> {
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
) -> Settings<'a, 't, 'u, 'i> {
let mut builder = Settings::new(wtxn, index, self.update_id);
let mut builder = Settings::new(wtxn, index);
builder.log_every_n = self.log_every_n;
builder.max_nb_chunks = self.max_nb_chunks;
@ -122,7 +120,7 @@ impl<'a> UpdateBuilder<'a> {
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
) -> Facets<'t, 'u, 'i> {
let mut builder = Facets::new(wtxn, index, self.update_id);
let mut builder = Facets::new(wtxn, index);
builder.chunk_compression_type = self.chunk_compression_type;
builder.chunk_compression_level = self.chunk_compression_level;

10
milli/src/update/words_prefixes_fst.rs
View File

@ -10,22 +10,14 @@ pub struct WordsPrefixesFst<'t, 'u, 'i> {
index: &'i Index,
threshold: u32,
max_prefix_length: usize,
_update_id: u64,
}
impl<'t, 'u, 'i> WordsPrefixesFst<'t, 'u, 'i> {
pub fn new(
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
update_id: u64,
) -> WordsPrefixesFst<'t, 'u, 'i> {
WordsPrefixesFst {
wtxn,
index,
threshold: 100,
max_prefix_length: 4,
_update_id: update_id,
}
WordsPrefixesFst { wtxn, index, threshold: 100, max_prefix_length: 4 }
}
/// Set the number of words required to make a prefix be part of the words prefixes

4
milli/tests/search/distinct.rs
View File

@ -16,9 +16,9 @@ macro_rules! test_distinct {
// update distinct attribute
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_distinct_field(S(stringify!($distinct)));
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
let rtxn = index.read_txn().unwrap();

8
milli/tests/search/mod.rs
View File

@ -32,7 +32,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
let criteria = criteria.iter().map(|c| c.to_string()).collect();
builder.set_criteria(criteria);
@ -51,10 +51,10 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
S("america") => vec![S("the united states")],
});
builder.set_searchable_fields(vec![S("title"), S("description")]);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
// index documents
let mut builder = UpdateBuilder::new(0);
let mut builder = UpdateBuilder::new();
builder.max_memory(10 * 1024 * 1024); // 10MiB
let mut builder = builder.index_documents(&mut wtxn, &index);
builder.enable_autogenerate_docids();
@ -73,7 +73,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index {
// index documents
let content = DocumentBatchReader::from_reader(cursor).unwrap();
builder.execute(content, |_, _| ()).unwrap();
builder.execute(content, |_| ()).unwrap();
wtxn.commit().unwrap();

16
milli/tests/search/query_criteria.rs
View File

@ -341,9 +341,9 @@ fn criteria_mixup() {
eprintln!("Testing with criteria order: {:?}", &criteria);
//update criteria
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_criteria(criteria.iter().map(ToString::to_string).collect());
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
let mut rtxn = index.read_txn().unwrap();
@ -376,16 +376,16 @@ fn criteria_ascdesc() {
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_sortable_fields(hashset! {
S("name"),
S("age"),
});
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
// index documents
let mut builder = UpdateBuilder::new(0);
let mut builder = UpdateBuilder::new();
builder.max_memory(10 * 1024 * 1024); // 10MiB
let mut builder = builder.index_documents(&mut wtxn, &index);
builder.enable_autogenerate_docids();
@ -419,7 +419,7 @@ fn criteria_ascdesc() {
let reader = DocumentBatchReader::from_reader(cursor).unwrap();
builder.execute(reader, |_, _| ()).unwrap();
builder.execute(reader, |_| ()).unwrap();
wtxn.commit().unwrap();
@ -430,9 +430,9 @@ fn criteria_ascdesc() {
eprintln!("Testing with criterion: {:?}", &criterion);
let mut wtxn = index.write_txn().unwrap();
let mut builder = Settings::new(&mut wtxn, &index, 0);
let mut builder = Settings::new(&mut wtxn, &index);
builder.set_criteria(vec![criterion.to_string()]);
builder.execute(|_, _| ()).unwrap();
builder.execute(|_| ()).unwrap();
wtxn.commit().unwrap();
let mut rtxn = index.read_txn().unwrap();