searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-15 13:58:36 +02:00
Code Issues Releases Wiki Activity

First compiling version with compressed documents iterators

Browse source
This commit is contained in:
Clément Renault 2024-07-02 11:08:10 +02:00
parent 2f0567fad1
commit e9d6b4222b
No known key found for this signature in database
GPG key ID: F250A4C4E3AE5F5F
16 changed files with 197 additions and 99 deletions
Download patch file Download diff file Expand all files Collapse all files

11
index-scheduler/src/batch.rs
View file

@ -908,16 +908,23 @@ impl IndexScheduler {
let mut index_dumper = dump.create_index(uid, &metadata)?;
let fields_ids_map = index.fields_ids_map(&rtxn)?;
let dictionary = index.document_compression_dictionary(&rtxn)?;
let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
let embedding_configs = index.embedding_configs(&rtxn)?;
let mut buffer = Vec::new();
// 3.1. Dump the documents
for ret in index.all_documents(&rtxn)? {
for ret in index.all_compressed_documents(&rtxn)? {
if self.must_stop_processing.get() {
return Err(Error::AbortedTask);
}
let (id, doc) = ret?;
let (id, compressed) = ret?;
let doc = match dictionary {
// TODO manage this unwrap correctly
Some(dict) => compressed.decompress_with(&mut buffer, dict).unwrap(),
None => compressed.as_non_compressed(),
};
let mut document = milli::obkv_to_json(&all_fields, &fields_ids_map, doc)?;