mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-11 15:38:55 +01:00
Merge #543
543: Fix wrong internal ids assignments r=irevoire a=irevoire Fix https://github.com/meilisearch/meilisearch/issues/2470 Co-authored-by: ad hoc <postma.marin@protonmail.com> Co-authored-by: Tamo <tamo@meilisearch.com>
This commit is contained in:
commit
306d2f37ff
@ -1881,4 +1881,52 @@ mod tests {
|
|||||||
|
|
||||||
wtxn.commit().unwrap();
|
wtxn.commit().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn index_documents_in_multiple_transforms() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let mut options = EnvOpenOptions::new();
|
||||||
|
options.map_size(4096 * 100);
|
||||||
|
let index = Index::new(options, tmp).unwrap();
|
||||||
|
let mut wtxn = index.write_txn().unwrap();
|
||||||
|
let indexer_config = IndexerConfig::default();
|
||||||
|
let mut builder = IndexDocuments::new(
|
||||||
|
&mut wtxn,
|
||||||
|
&index,
|
||||||
|
&indexer_config,
|
||||||
|
IndexDocumentsConfig::default(),
|
||||||
|
|_| (),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let doc1 = documents! {[{
|
||||||
|
"id": 228142,
|
||||||
|
"title": "asdsad",
|
||||||
|
"state": "automated",
|
||||||
|
"priority": "normal",
|
||||||
|
"public_uid": "37ccf021",
|
||||||
|
"project_id": 78207,
|
||||||
|
"branch_id_number": 0
|
||||||
|
}]};
|
||||||
|
|
||||||
|
let doc2 = documents! {[{
|
||||||
|
"id": 228143,
|
||||||
|
"title": "something",
|
||||||
|
"state": "automated",
|
||||||
|
"priority": "normal",
|
||||||
|
"public_uid": "39c6499b",
|
||||||
|
"project_id": 78207,
|
||||||
|
"branch_id_number": 0
|
||||||
|
}]};
|
||||||
|
|
||||||
|
builder.add_documents(doc1).unwrap();
|
||||||
|
builder.add_documents(doc2).unwrap();
|
||||||
|
|
||||||
|
builder.execute().unwrap();
|
||||||
|
|
||||||
|
let map = index.external_documents_ids(&wtxn).unwrap().to_hash_map();
|
||||||
|
let ids = map.values().collect::<HashSet<_>>();
|
||||||
|
|
||||||
|
assert_eq!(ids.len(), map.len());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -51,6 +51,7 @@ pub struct Transform<'a, 'i> {
|
|||||||
indexer_settings: &'a IndexerConfig,
|
indexer_settings: &'a IndexerConfig,
|
||||||
pub autogenerate_docids: bool,
|
pub autogenerate_docids: bool,
|
||||||
pub index_documents_method: IndexDocumentsMethod,
|
pub index_documents_method: IndexDocumentsMethod,
|
||||||
|
available_documents_ids: AvailableDocumentsIds,
|
||||||
|
|
||||||
original_sorter: grenad::Sorter<MergeFn>,
|
original_sorter: grenad::Sorter<MergeFn>,
|
||||||
flattened_sorter: grenad::Sorter<MergeFn>,
|
flattened_sorter: grenad::Sorter<MergeFn>,
|
||||||
@ -128,12 +129,14 @@ impl<'a, 'i> Transform<'a, 'i> {
|
|||||||
indexer_settings.max_nb_chunks,
|
indexer_settings.max_nb_chunks,
|
||||||
indexer_settings.max_memory.map(|mem| mem / 2),
|
indexer_settings.max_memory.map(|mem| mem / 2),
|
||||||
);
|
);
|
||||||
|
let documents_ids = index.documents_ids(wtxn)?;
|
||||||
|
|
||||||
Ok(Transform {
|
Ok(Transform {
|
||||||
index,
|
index,
|
||||||
fields_ids_map: index.fields_ids_map(wtxn)?,
|
fields_ids_map: index.fields_ids_map(wtxn)?,
|
||||||
indexer_settings,
|
indexer_settings,
|
||||||
autogenerate_docids,
|
autogenerate_docids,
|
||||||
|
available_documents_ids: AvailableDocumentsIds::from_documents_ids(&documents_ids),
|
||||||
original_sorter,
|
original_sorter,
|
||||||
flattened_sorter,
|
flattened_sorter,
|
||||||
index_documents_method,
|
index_documents_method,
|
||||||
@ -156,8 +159,6 @@ impl<'a, 'i> Transform<'a, 'i> {
|
|||||||
{
|
{
|
||||||
let fields_index = reader.index();
|
let fields_index = reader.index();
|
||||||
let external_documents_ids = self.index.external_documents_ids(wtxn)?;
|
let external_documents_ids = self.index.external_documents_ids(wtxn)?;
|
||||||
let documents_ids = self.index.documents_ids(wtxn)?;
|
|
||||||
let mut available_documents_ids = AvailableDocumentsIds::from_documents_ids(&documents_ids);
|
|
||||||
|
|
||||||
let mapping = create_fields_mapping(&mut self.fields_ids_map, fields_index)?;
|
let mapping = create_fields_mapping(&mut self.fields_ids_map, fields_index)?;
|
||||||
|
|
||||||
@ -261,7 +262,8 @@ impl<'a, 'i> Transform<'a, 'i> {
|
|||||||
// if the document has never been encountered we give it a new docid
|
// if the document has never been encountered we give it a new docid
|
||||||
// and push this new docid to the external documents ids builder
|
// and push this new docid to the external documents ids builder
|
||||||
Entry::Vacant(entry) => {
|
Entry::Vacant(entry) => {
|
||||||
let new_docid = available_documents_ids
|
let new_docid = self
|
||||||
|
.available_documents_ids
|
||||||
.next()
|
.next()
|
||||||
.ok_or(UserError::DocumentLimitReached)?;
|
.ok_or(UserError::DocumentLimitReached)?;
|
||||||
entry.insert(new_docid as u64);
|
entry.insert(new_docid as u64);
|
||||||
|
Loading…
Reference in New Issue
Block a user