searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-03 11:57:07 +02:00
Code Issues Releases Wiki Activity

Display an accurate number of uploaded documents

Browse source
This commit is contained in:
Clément Renault 2025-06-30 18:46:47 +02:00
parent 85037352b9
commit ad03c86c44
No known key found for this signature in database
GPG key ID: F250A4C4E3AE5F5F
3 changed files with 24 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

10
crates/index-scheduler/src/scheduler/process_batch.rs
View file

@ -377,9 +377,8 @@ impl IndexScheduler {
) )
})); }));
match ret { let stats = match ret {
// TODO return the matched and exported documents Ok(Ok(stats)) => stats,
Ok(Ok(())) => (),
Ok(Err(Error::AbortedTask)) => return Err(Error::AbortedTask), Ok(Err(Error::AbortedTask)) => return Err(Error::AbortedTask),
Ok(Err(e)) => return Err(Error::Export(Box::new(e))), Ok(Err(e)) => return Err(Error::Export(Box::new(e))),
Err(e) => { Err(e) => {
@ -394,9 +393,12 @@ impl IndexScheduler {
msg.to_string(), msg.to_string(),
)))); ))));
} }
} };
task.status = Status::Succeeded; task.status = Status::Succeeded;
if let Some(Details::Export { indexes, .. }) = task.details.as_mut() {
*indexes = stats;
}
Ok((vec![task], ProcessBatchInfo::default())) Ok((vec![task], ProcessBatchInfo::default()))
} }
Batch::UpgradeDatabase { mut tasks } => { Batch::UpgradeDatabase { mut tasks } => {

21
crates/index-scheduler/src/scheduler/process_export.rs
View file

@ -14,7 +14,7 @@ use meilisearch_types::milli::update::{request_threads, Setting};
use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors}; use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors};
use meilisearch_types::milli::{self, obkv_to_json, Filter, InternalError}; use meilisearch_types::milli::{self, obkv_to_json, Filter, InternalError};
use meilisearch_types::settings::{self, SecretPolicy}; use meilisearch_types::settings::{self, SecretPolicy};
use meilisearch_types::tasks::ExportIndexSettings; use meilisearch_types::tasks::{DetailsExportIndexSettings, ExportIndexSettings};
use serde::Deserialize; use serde::Deserialize;
use ureq::{json, Response}; use ureq::{json, Response};
@ -30,7 +30,7 @@ impl IndexScheduler {
payload_size: Option<&Byte>, payload_size: Option<&Byte>,
indexes: &BTreeMap<IndexUidPattern, ExportIndexSettings>, indexes: &BTreeMap<IndexUidPattern, ExportIndexSettings>,
progress: Progress, progress: Progress,
) -> Result<()> { ) -> Result<BTreeMap<IndexUidPattern, DetailsExportIndexSettings>> {
#[cfg(test)] #[cfg(test)]
self.maybe_fail(crate::test_utils::FailureLocation::ProcessExport)?; self.maybe_fail(crate::test_utils::FailureLocation::ProcessExport)?;
@ -41,13 +41,14 @@ impl IndexScheduler {
indexes indexes
.iter() .iter()
.find(|(pattern, _)| pattern.matches_str(&uid)) .find(|(pattern, _)| pattern.matches_str(&uid))
.map(|(_pattern, settings)| (uid, settings)) .map(|(pattern, settings)| (pattern, uid, settings))
}) })
.collect(); .collect();
let mut output = BTreeMap::new();
let agent = ureq::AgentBuilder::new().timeout(Duration::from_secs(5)).build(); let agent = ureq::AgentBuilder::new().timeout(Duration::from_secs(5)).build();
let must_stop_processing = self.scheduler.must_stop_processing.clone(); let must_stop_processing = self.scheduler.must_stop_processing.clone();
for (i, (uid, settings)) in indexes.iter().enumerate() { for (i, (pattern, uid, export_settings)) in indexes.iter().enumerate() {
if must_stop_processing.get() { if must_stop_processing.get() {
return Err(Error::AbortedTask); return Err(Error::AbortedTask);
} }
@ -58,7 +59,7 @@ impl IndexScheduler {
indexes.len() as u32, indexes.len() as u32,
)); ));
let ExportIndexSettings { filter } = settings; let ExportIndexSettings { filter } = export_settings;
let index = self.index(uid)?; let index = self.index(uid)?;
let index_rtxn = index.read_txn()?; let index_rtxn = index.read_txn()?;
@ -125,6 +126,14 @@ impl IndexScheduler {
let (step, progress_step) = AtomicDocumentStep::new(total_documents); let (step, progress_step) = AtomicDocumentStep::new(total_documents);
progress.update_progress(progress_step); progress.update_progress(progress_step);
output.insert(
(*pattern).clone(),
DetailsExportIndexSettings {
settings: (*export_settings).clone(),
matched_documents: Some(total_documents as u64),
},
);
let limit = payload_size.map(|ps| ps.as_u64() as usize).unwrap_or(50 * 1024 * 1024); // defaults to 50 MiB let limit = payload_size.map(|ps| ps.as_u64() as usize).unwrap_or(50 * 1024 * 1024); // defaults to 50 MiB
let documents_url = format!("{base_url}/indexes/{uid}/documents"); let documents_url = format!("{base_url}/indexes/{uid}/documents");
@ -265,7 +274,7 @@ impl IndexScheduler {
step.store(total_documents, atomic::Ordering::Relaxed); step.store(total_documents, atomic::Ordering::Relaxed);
} }
Ok(()) Ok(output)
} }
} }

8
crates/meilisearch-types/src/tasks.rs
View file

@ -707,16 +707,14 @@ pub enum Details {
#[schema(rename_all = "camelCase")] #[schema(rename_all = "camelCase")]
pub struct DetailsExportIndexSettings { pub struct DetailsExportIndexSettings {
#[serde(flatten)] #[serde(flatten)]
settings: ExportIndexSettings, pub settings: ExportIndexSettings,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
matched_documents: Option<u64>, pub matched_documents: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
exported_documents: Option<u64>,
} }
impl From<ExportIndexSettings> for DetailsExportIndexSettings { impl From<ExportIndexSettings> for DetailsExportIndexSettings {
fn from(settings: ExportIndexSettings) -> Self { fn from(settings: ExportIndexSettings) -> Self {
DetailsExportIndexSettings { settings, matched_documents: None, exported_documents: None } DetailsExportIndexSettings { settings, matched_documents: None }
} }
} }