searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-01-09 04:54:31 +01:00
Code Issues Releases Wiki Activity

add the batches

Browse Source
This commit is contained in:
Tamo 2024-12-26 17:16:52 +01:00
parent 8a2a1e4d27
commit 1dd33af8a3
No known key found for this signature in database
GPG Key ID: 20CD8020AFA88D69
7 changed files with 156 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
crates/meilisearch-types/src/batch_view.rs
View File

@ -1,13 +1,15 @@
use milli::progress::ProgressView;
use serde::Serialize;
use time::{Duration, OffsetDateTime};
use utoipa::ToSchema;
use crate::batches::{Batch, BatchId, BatchStats};
use crate::task_view::DetailsView;
use crate::tasks::serialize_duration;
#[derive(Debug, Clone, Serialize)]
#[derive(Debug, Clone, Serialize, ToSchema)]
#[serde(rename_all = "camelCase")]
#[schema(rename_all = "camelCase")]
pub struct BatchView {
pub uid: BatchId,
pub progress: Option<ProgressView>,

4
crates/meilisearch-types/src/batches.rs
View File

@ -3,6 +3,7 @@ use std::collections::BTreeMap;
use milli::progress::ProgressView;
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
use utoipa::ToSchema;
use crate::task_view::DetailsView;
use crate::tasks::{Kind, Status};
@ -25,8 +26,9 @@ pub struct Batch {
pub finished_at: Option<OffsetDateTime>,
}
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
#[derive(Default, Debug, Clone, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
#[schema(rename_all = "camelCase")]
pub struct BatchStats {
pub total_nb_tasks: BatchId,
pub status: BTreeMap<Status, u32>,

1
crates/meilisearch-types/src/task_view.rs
View File

@ -69,6 +69,7 @@ impl TaskView {
#[derive(Default, Debug, PartialEq, Eq, Clone, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
#[schema(rename_all = "camelCase")]
pub struct DetailsView {
/// Number of documents received for documentAdditionOrUpdate task.
#[serde(skip_serializing_if = "Option::is_none")]

123
crates/meilisearch/src/routes/batches.rs
View File

@ -8,17 +8,77 @@ use meilisearch_types::deserr::DeserrQueryParamError;
use meilisearch_types::error::ResponseError;
use meilisearch_types::keys::actions;
use serde::Serialize;
use utoipa::{OpenApi, ToSchema};
use super::tasks::TasksFilterQuery;
use super::ActionPolicy;
use crate::extractors::authentication::GuardedData;
use crate::extractors::sequential_extractor::SeqHandler;
#[derive(OpenApi)]
#[openapi(
paths(get_batch, get_batches),
tags((
name = "Batches",
description = "The /batches route gives information about the progress of batches of asynchronous operations.",
external_docs(url = "https://www.meilisearch.com/docs/reference/api/batches"),
)),
)]
pub struct BatchesApi;
pub fn configure(cfg: &mut web::ServiceConfig) {
cfg.service(web::resource("").route(web::get().to(SeqHandler(get_batches))))
.service(web::resource("/{batch_id}").route(web::get().to(SeqHandler(get_batch))));
}
/// Get one batch
///
/// Get a single batch.
#[utoipa::path(
get,
path = "/{batchUid}",
tag = "Batches",
security(("Bearer" = ["tasks.get", "tasks.*", "*"])),
params(
("batchUid" = String, Path, example = "8685", description = "The unique batch id", nullable = false),
),
responses(
(status = OK, description = "Return the batch", body = BatchView, content_type = "application/json", example = json!(
{
"uid": 1,
"details": {
"receivedDocuments": 1,
"indexedDocuments": 1
},
"progress": null,
"stats": {
"totalNbTasks": 1,
"status": {
"succeeded": 1
},
"types": {
"documentAdditionOrUpdate": 1
},
"indexUids": {
"INDEX_NAME": 1
}
},
"duration": "PT0.364788S",
"startedAt": "2024-12-10T15:48:49.672141Z",
"finishedAt": "2024-12-10T15:48:50.036929Z"
}
)),
(status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!(
{
"message": "The Authorization header is missing. It must use the bearer authorization method.",
"code": "missing_authorization_header",
"type": "auth",
"link": "https://docs.meilisearch.com/errors#missing_authorization_header"
}
)),
)
)]
async fn get_batch(
index_scheduler: GuardedData<ActionPolicy<{ actions::TASKS_GET }>, Data<IndexScheduler>>,
batch_uid: web::Path<String>,
@ -39,14 +99,14 @@ async fn get_batch(
let (batches, _) = index_scheduler.get_batches_from_authorized_indexes(&query, filters)?;
if let Some(batch) = batches.first() {
let task_view = BatchView::from_batch(batch);
Ok(HttpResponse::Ok().json(task_view))
let batch_view = BatchView::from_batch(batch);
Ok(HttpResponse::Ok().json(batch_view))
} else {
Err(index_scheduler::Error::BatchNotFound(batch_uid).into())
}
}
#[derive(Debug, Serialize)]
#[derive(Debug, Serialize, ToSchema)]
pub struct AllBatches {
results: Vec<BatchView>,
total: u64,
@ -55,6 +115,63 @@ pub struct AllBatches {
next: Option<u32>,
}
/// Get batches
///
/// List all batches, regardless of index. The batch objects are contained in the results array.
/// Batches are always returned in descending order of uid. This means that by default, the most recently created batch objects appear first.
/// Batch results are paginated and can be filtered with query parameters.
#[utoipa::path(
get,
path = "/",
tag = "Batches",
security(("Bearer" = ["tasks.get", "tasks.*", "*"])),
params(TasksFilterQuery),
responses(
(status = OK, description = "Return the batches", body = AllBatches, content_type = "application/json", example = json!(
{
"results": [
{
"uid": 2,
"details": {
"stopWords": [
"of",
"the"
]
},
"progress": null,
"stats": {
"totalNbTasks": 1,
"status": {
"succeeded": 1
},
"types": {
"settingsUpdate": 1
},
"indexUids": {
"INDEX_NAME": 1
}
},
"duration": "PT0.110083S",
"startedAt": "2024-12-10T15:49:04.995321Z",
"finishedAt": "2024-12-10T15:49:05.105404Z"
}
],
"total": 3,
"limit": 1,
"from": 2,
"next": 1
}
)),
(status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!(
{
"message": "The Authorization header is missing. It must use the bearer authorization method.",
"code": "missing_authorization_header",
"type": "auth",
"link": "https://docs.meilisearch.com/errors#missing_authorization_header"
}
)),
)
)]
async fn get_batches(
index_scheduler: GuardedData<ActionPolicy<{ actions::TASKS_GET }>, Data<IndexScheduler>>,
params: AwebQueryParameter<TasksFilterQuery, DeserrQueryParamError>,

2
crates/meilisearch/src/routes/features.rs
View File

@ -17,7 +17,7 @@ use crate::extractors::sequential_extractor::SeqHandler;
#[derive(OpenApi)]
#[openapi(
paths(get_features),
paths(get_features, patch_features),
tags((
name = "Experimental features",
description = "The `/experimental-features` route allows you to activate or deactivate some of Meilisearch's experimental features.

40
crates/meilisearch/src/routes/mod.rs
View File

@ -2,6 +2,9 @@ use std::collections::BTreeMap;
use crate::extractors::authentication::policies::*;
use crate::extractors::authentication::GuardedData;
use crate::milli::progress::ProgressStepView;
use crate::milli::progress::ProgressView;
use crate::routes::batches::AllBatches;
use crate::routes::features::RuntimeTogglableFeatures;
use crate::routes::indexes::documents::DocumentEditionByFunction;
use crate::routes::multi_search::SearchResults;
@ -16,6 +19,8 @@ use actix_web::web::Data;
use actix_web::{web, HttpRequest, HttpResponse};
use index_scheduler::IndexScheduler;
use meilisearch_auth::AuthController;
use meilisearch_types::batch_view::BatchView;
use meilisearch_types::batches::BatchStats;
use meilisearch_types::error::{Code, ErrorType, ResponseError};
use meilisearch_types::index_uid::IndexUid;
use meilisearch_types::keys::CreateApiKey;
@ -63,6 +68,7 @@ pub mod tasks;
#[openapi(
nest(
(path = "/tasks", api = tasks::TaskApi),
(path = "/batches", api = batches::BatchesApi),
(path = "/indexes", api = indexes::IndexesApi),
// We must stop the search path here because the rest must be configured by each route individually
(path = "/indexes", api = indexes::search::SearchApi),
@ -80,29 +86,29 @@ pub mod tasks;
(name = "Stats", description = "Stats gives extended information and metrics about indexes and the Meilisearch database."),
),
modifiers(&OpenApiAuth),
components(schemas(RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView<serde_json::Value>, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings<Unchecked>, Settings<Checked>, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind))
components(schemas(AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView<serde_json::Value>, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings<Unchecked>, Settings<Checked>, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind))
)]
pub struct MeilisearchApi;
pub fn configure(cfg: &mut web::ServiceConfig) {
let openapi = MeilisearchApi::openapi();
cfg.service(web::scope("/tasks").configure(tasks::configure)) // done
.service(web::scope("/batches").configure(batches::configure)) // TODO
.service(Scalar::with_url("/scalar", openapi.clone())) // done
.service(RapiDoc::with_openapi("/api-docs/openapi.json", openapi.clone()).path("/rapidoc")) // done
.service(Redoc::with_url("/redoc", openapi)) // done
.service(web::resource("/health").route(web::get().to(get_health))) // done
.service(web::scope("/logs").configure(logs::configure)) // done
.service(web::scope("/keys").configure(api_key::configure)) // done
.service(web::scope("/dumps").configure(dump::configure)) // done
.service(web::scope("/snapshots").configure(snapshot::configure)) // done
.service(web::resource("/stats").route(web::get().to(get_stats))) // done
.service(web::resource("/version").route(web::get().to(get_version))) // done
.service(web::scope("/indexes").configure(indexes::configure)) // done
.service(web::scope("/multi-search").configure(multi_search::configure)) // done
.service(web::scope("/swap-indexes").configure(swap_indexes::configure)) // done
.service(web::scope("/metrics").configure(metrics::configure)) // done
cfg.service(web::scope("/tasks").configure(tasks::configure))
.service(web::scope("/batches").configure(batches::configure))
.service(Scalar::with_url("/scalar", openapi.clone()))
.service(RapiDoc::with_openapi("/api-docs/openapi.json", openapi.clone()).path("/rapidoc"))
.service(Redoc::with_url("/redoc", openapi))
.service(web::resource("/health").route(web::get().to(get_health)))
.service(web::scope("/logs").configure(logs::configure))
.service(web::scope("/keys").configure(api_key::configure))
.service(web::scope("/dumps").configure(dump::configure))
.service(web::scope("/snapshots").configure(snapshot::configure))
.service(web::resource("/stats").route(web::get().to(get_stats)))
.service(web::resource("/version").route(web::get().to(get_version)))
.service(web::scope("/indexes").configure(indexes::configure))
.service(web::scope("/multi-search").configure(multi_search::configure))
.service(web::scope("/swap-indexes").configure(swap_indexes::configure))
.service(web::scope("/metrics").configure(metrics::configure))
.service(web::scope("/experimental-features").configure(features::configure));
}

7
crates/milli/src/progress.rs
View File

@ -4,6 +4,7 @@ use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::{Arc, RwLock};
use serde::Serialize;
use utoipa::ToSchema;
pub trait Step: 'static + Send + Sync {
fn name(&self) -> Cow<'static, str>;
@ -136,15 +137,17 @@ macro_rules! make_atomic_progress {
make_atomic_progress!(Document alias AtomicDocumentStep => "document" );
make_atomic_progress!(Payload alias AtomicPayloadStep => "payload" );
#[derive(Debug, Serialize, Clone)]
#[derive(Debug, Serialize, Clone, ToSchema)]
#[serde(rename_all = "camelCase")]
#[schema(rename_all = "camelCase")]
pub struct ProgressView {
pub steps: Vec<ProgressStepView>,
pub percentage: f32,
}
#[derive(Debug, Serialize, Clone)]
#[derive(Debug, Serialize, Clone, ToSchema)]
#[serde(rename_all = "camelCase")]
#[schema(rename_all = "camelCase")]
pub struct ProgressStepView {
pub current_step: Cow<'static, str>,
pub finished: u32,