searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 20:37:15 +02:00
Code Issues Releases Wiki Activity

Add the payload size to the parameters

Browse source
This commit is contained in:
Kerollmops 2025-06-25 18:24:50 +02:00 • committed by Clément Renault
parent a743da3061
commit 63031219c5
No known key found for this signature in database
GPG key ID: F250A4C4E3AE5F5F
13 changed files with 130 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

11
crates/index-scheduler/src/scheduler/process_batch.rs
View file

@ -362,12 +362,19 @@ impl IndexScheduler {
Ok((vec![task], ProcessBatchInfo::default()))
}
Batch::Export { mut task } => {
let KindWithContent::Export { url, indexes, api_key } = &task.kind else {
let KindWithContent::Export { url, api_key, payload_size, indexes } = &task.kind
else {
unreachable!()
};
let ret = catch_unwind(AssertUnwindSafe(|| {
self.process_export(url, indexes, api_key.as_deref(), progress)
self.process_export(
url,
api_key.as_deref(),
payload_size.as_ref(),
indexes,
progress,
)
}));
match ret {

6
crates/index-scheduler/src/scheduler/process_export.rs
View file

@ -4,6 +4,7 @@ use std::sync::atomic;
use std::time::Duration;
use backoff::ExponentialBackoff;
use byte_unit::Byte;
use flate2::write::GzEncoder;
use flate2::Compression;
use meilisearch_types::index_uid_pattern::IndexUidPattern;
@ -25,8 +26,9 @@ impl IndexScheduler {
pub(super) fn process_export(
&self,
base_url: &str,
indexes: &BTreeMap<IndexUidPattern, ExportIndexSettings>,
api_key: Option<&str>,
payload_size: Option<&Byte>,
indexes: &BTreeMap<IndexUidPattern, ExportIndexSettings>,
progress: Progress,
) -> Result<()> {
#[cfg(test)]
@ -122,7 +124,7 @@ impl IndexScheduler {
let (step, progress_step) = AtomicDocumentStep::new(total_documents);
progress.update_progress(progress_step);
let limit = 50 * 1024 * 1024; // 50 MiB
let limit = payload_size.map(|ps| ps.as_u64() as usize).unwrap_or(50 * 1024 * 1024); // defaults to 50 MiB
let documents_url = format!("{base_url}/indexes/{uid}/documents");
request_threads()