Merge pull request #5670 from meilisearch/export-and-transfer-route

Introduce a new route to export indexes

crates/meilisearch/src/routes/export.rs

@@ -0,0 +1,183 @@
+use std::collections::BTreeMap;
+use std::convert::Infallible;
+use std::str::FromStr as _;
+
+use actix_web::web::{self, Data};
+use actix_web::{HttpRequest, HttpResponse};
+use byte_unit::Byte;
+use deserr::actix_web::AwebJson;
+use deserr::Deserr;
+use index_scheduler::IndexScheduler;
+use meilisearch_types::deserr::DeserrJsonError;
+use meilisearch_types::error::deserr_codes::*;
+use meilisearch_types::error::ResponseError;
+use meilisearch_types::index_uid_pattern::IndexUidPattern;
+use meilisearch_types::keys::actions;
+use meilisearch_types::tasks::{ExportIndexSettings as DbExportIndexSettings, KindWithContent};
+use serde::Serialize;
+use serde_json::Value;
+use tracing::debug;
+use utoipa::{OpenApi, ToSchema};
+
+use crate::analytics::Analytics;
+use crate::extractors::authentication::policies::ActionPolicy;
+use crate::extractors::authentication::GuardedData;
+use crate::routes::export_analytics::ExportAnalytics;
+use crate::routes::{get_task_id, is_dry_run, SummarizedTaskView};
+use crate::Opt;
+
+#[derive(OpenApi)]
+#[openapi(
+    paths(export),
+    tags((
+        name = "Export",
+        description = "The `/export` route allows you to trigger an export process to a remote Meilisearch instance.",
+        external_docs(url = "https://www.meilisearch.com/docs/reference/api/export"),
+    )),
+)]
+pub struct ExportApi;
+
+pub fn configure(cfg: &mut web::ServiceConfig) {
+    cfg.service(web::resource("").route(web::post().to(export)));
+}
+
+#[utoipa::path(
+    post,
+    path = "",
+    tag = "Export",
+    security(("Bearer" = ["export", "*"])),
+    responses(
+        (status = 202, description = "Export successfully enqueued", body = SummarizedTaskView, content_type = "application/json", example = json!(
+            {
+                "taskUid": 1,
+                "status": "enqueued",
+                "type": "export",
+                "enqueuedAt": "2021-08-11T09:25:53.000000Z"
+            })),
+        (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!(
+            {
+                "message": "The Authorization header is missing. It must use the bearer authorization method.",
+                "code": "missing_authorization_header",
+                "type": "auth",
+                "link": "https://docs.meilisearch.com/errors#missing_authorization_header"
+            }
+        )),
+    )
+)]
+async fn export(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::EXPORT }>, Data<IndexScheduler>>,
+    export: AwebJson<Export, DeserrJsonError>,
+    req: HttpRequest,
+    opt: web::Data<Opt>,
+    analytics: Data<Analytics>,
+) -> Result<HttpResponse, ResponseError> {
+    let export = export.into_inner();
+    debug!(returns = ?export, "Trigger export");
+
+    let analytics_aggregate = ExportAnalytics::from_export(&export);
+
+    let Export { url, api_key, payload_size, indexes } = export;
+
+    let indexes = match indexes {
+        Some(indexes) => indexes
+            .into_iter()
+            .map(|(pattern, ExportIndexSettings { filter, override_settings })| {
+                (pattern, DbExportIndexSettings { filter, override_settings })
+            })
+            .collect(),
+        None => BTreeMap::from([(
+            IndexUidPattern::new_unchecked("*"),
+            DbExportIndexSettings::default(),
+        )]),
+    };
+
+    let task = KindWithContent::Export {
+        url,
+        api_key,
+        payload_size: payload_size.map(|ByteWithDeserr(bytes)| bytes),
+        indexes,
+    };
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
+    let task: SummarizedTaskView =
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
+
+    analytics.publish(analytics_aggregate, &req);
+
+    Ok(HttpResponse::Ok().json(task))
+}
+
+#[derive(Debug, Deserr, ToSchema, Serialize)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
+#[serde(rename_all = "camelCase")]
+#[schema(rename_all = "camelCase")]
+pub struct Export {
+    #[schema(value_type = Option<String>, example = json!("https://ms-1234.heaven.meilisearch.com"))]
+    #[serde(default)]
+    #[deserr(default, error = DeserrJsonError<InvalidExportUrl>)]
+    pub url: String,
+    #[schema(value_type = Option<String>, example = json!("1234abcd"))]
+    #[serde(default)]
+    #[deserr(default, error = DeserrJsonError<InvalidExportApiKey>)]
+    pub api_key: Option<String>,
+    #[schema(value_type = Option<String>, example = json!("24MiB"))]
+    #[serde(default)]
+    #[deserr(default, error = DeserrJsonError<InvalidExportPayloadSize>)]
+    pub payload_size: Option<ByteWithDeserr>,
+    #[schema(value_type = Option<BTreeMap<String, ExportIndexSettings>>, example = json!({ "*": { "filter": null } }))]
+    #[deserr(default)]
+    #[serde(default)]
+    pub indexes: Option<BTreeMap<IndexUidPattern, ExportIndexSettings>>,
+}
+
+/// A wrapper around the `Byte` type that implements `Deserr`.
+#[derive(Debug, Serialize)]
+#[serde(transparent)]
+pub struct ByteWithDeserr(pub Byte);
+
+impl<E> deserr::Deserr<E> for ByteWithDeserr
+where
+    E: deserr::DeserializeError,
+{
+    fn deserialize_from_value<V: deserr::IntoValue>(
+        value: deserr::Value<V>,
+        location: deserr::ValuePointerRef,
+    ) -> Result<Self, E> {
+        use deserr::{ErrorKind, Value, ValueKind};
+        match value {
+            Value::Integer(integer) => Ok(ByteWithDeserr(Byte::from_u64(integer))),
+            Value::String(string) => Byte::from_str(&string).map(ByteWithDeserr).map_err(|e| {
+                deserr::take_cf_content(E::error::<Infallible>(
+                    None,
+                    ErrorKind::Unexpected { msg: e.to_string() },
+                    location,
+                ))
+            }),
+            actual => Err(deserr::take_cf_content(E::error(
+                None,
+                ErrorKind::IncorrectValueKind {
+                    actual,
+                    accepted: &[ValueKind::Integer, ValueKind::String],
+                },
+                location,
+            ))),
+        }
+    }
+}
+
+#[derive(Debug, Deserr, ToSchema, Serialize)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
+#[serde(rename_all = "camelCase")]
+#[schema(rename_all = "camelCase")]
+pub struct ExportIndexSettings {
+    #[schema(value_type = Option<String>, example = json!("genres = action"))]
+    #[serde(default)]
+    #[deserr(default, error = DeserrJsonError<InvalidExportIndexFilter>)]
+    pub filter: Option<Value>,
+    #[schema(value_type = Option<bool>, example = json!(true))]
+    #[serde(default)]
+    #[deserr(default, error = DeserrJsonError<InvalidExportIndexOverrideSettings>)]
+    pub override_settings: bool,
+}

crates/meilisearch/src/routes/export_analytics.rs

@@ -0,0 +1,93 @@
+use crate::analytics::Aggregate;
+use crate::routes::export::Export;
+
+#[derive(Default)]
+pub struct ExportAnalytics {
+    total_received: usize,
+    has_api_key: bool,
+    sum_index_patterns: usize,
+    sum_patterns_with_filter: usize,
+    sum_patterns_with_override_settings: usize,
+    payload_sizes: Vec<u64>,
+}
+
+impl ExportAnalytics {
+    pub fn from_export(export: &Export) -> Self {
+        let Export { url: _, api_key, payload_size, indexes } = export;
+
+        let has_api_key = api_key.is_some();
+        let index_patterns_count = indexes.as_ref().map_or(0, |indexes| indexes.len());
+        let patterns_with_filter_count = indexes.as_ref().map_or(0, |indexes| {
+            indexes.values().filter(|settings| settings.filter.is_some()).count()
+        });
+        let patterns_with_override_settings_count = indexes.as_ref().map_or(0, |indexes| {
+            indexes.values().filter(|settings| settings.override_settings).count()
+        });
+        let payload_sizes =
+            if let Some(crate::routes::export::ByteWithDeserr(byte_size)) = payload_size {
+                vec![byte_size.as_u64()]
+            } else {
+                vec![]
+            };
+
+        Self {
+            total_received: 1,
+            has_api_key,
+            sum_index_patterns: index_patterns_count,
+            sum_patterns_with_filter: patterns_with_filter_count,
+            sum_patterns_with_override_settings: patterns_with_override_settings_count,
+            payload_sizes,
+        }
+    }
+}
+
+impl Aggregate for ExportAnalytics {
+    fn event_name(&self) -> &'static str {
+        "Export Triggered"
+    }
+
+    fn aggregate(mut self: Box<Self>, other: Box<Self>) -> Box<Self> {
+        self.total_received += other.total_received;
+        self.has_api_key |= other.has_api_key;
+        self.sum_index_patterns += other.sum_index_patterns;
+        self.sum_patterns_with_filter += other.sum_patterns_with_filter;
+        self.sum_patterns_with_override_settings += other.sum_patterns_with_override_settings;
+        self.payload_sizes.extend(other.payload_sizes);
+        self
+    }
+
+    fn into_event(self: Box<Self>) -> serde_json::Value {
+        let avg_payload_size = if self.payload_sizes.is_empty() {
+            None
+        } else {
+            Some(self.payload_sizes.iter().sum::<u64>() / self.payload_sizes.len() as u64)
+        };
+
+        let avg_index_patterns = if self.total_received == 0 {
+            None
+        } else {
+            Some(self.sum_index_patterns as f64 / self.total_received as f64)
+        };
+
+        let avg_patterns_with_filter = if self.total_received == 0 {
+            None
+        } else {
+            Some(self.sum_patterns_with_filter as f64 / self.total_received as f64)
+        };
+
+        let avg_patterns_with_override_settings = if self.total_received == 0 {
+            None
+        } else {
+            Some(self.sum_patterns_with_override_settings as f64 / self.total_received as f64)
+        };
+
+        serde_json::json!({
+            "total_received": self.total_received,
+            "has_api_key": self.has_api_key,
+            "avg_index_patterns": avg_index_patterns,
+            "avg_patterns_with_filter": avg_patterns_with_filter,
+            "avg_patterns_with_override_settings": avg_patterns_with_override_settings,
+            "avg_payload_size": avg_payload_size,
+        })
+    }
+}

crates/meilisearch/src/routes/mod.rs

@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
 
 use actix_web::web::Data;
 use actix_web::{web, HttpRequest, HttpResponse};
+use export::Export;
 use index_scheduler::IndexScheduler;
 use meilisearch_auth::AuthController;
 use meilisearch_types::batch_view::BatchView;
@@ -54,6 +55,8 @@ mod api_key;
 pub mod batches;
 pub mod chats;
 mod dump;
+mod export;
+mod export_analytics;
 pub mod features;
 pub mod indexes;
 mod logs;
@@ -84,6 +87,7 @@ mod tasks_test;
         (path = "/multi-search", api = multi_search::MultiSearchApi),
         (path = "/swap-indexes", api = swap_indexes::SwapIndexesApi),
         (path = "/experimental-features", api = features::ExperimentalFeaturesApi),
+        (path = "/export", api = export::ExportApi),
         (path = "/network", api = network::NetworkApi),
     ),
     paths(get_health, get_version, get_stats),
@@ -95,7 +99,7 @@ mod tasks_test;
         url = "/",
         description = "Local server",
     )),
-    components(schemas(PaginationView<KeyView>, PaginationView<IndexView>, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView<serde_json::Value>, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings<Unchecked>, Settings<Checked>, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures))
+    components(schemas(PaginationView<KeyView>, PaginationView<IndexView>, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView<serde_json::Value>, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings<Unchecked>, Settings<Checked>, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export))
 )]
 pub struct MeilisearchApi;
 
@@ -115,6 +119,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         .service(web::scope("/metrics").configure(metrics::configure))
         .service(web::scope("/experimental-features").configure(features::configure))
         .service(web::scope("/network").configure(network::configure))
+        .service(web::scope("/export").configure(export::configure))
         .service(web::scope("/chats").configure(chats::configure));
 
     #[cfg(feature = "swagger")]

crates/meilisearch/src/routes/tasks_test.rs

@@ -228,7 +228,7 @@ mod tests {
             let err = deserr_query_params::<TaskDeletionOrCancelationQuery>(params).unwrap_err();
             snapshot!(meili_snap::json_string!(err), @r#"
             {
-              "message": "Invalid value in parameter `types`: `createIndex` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`, `upgradeDatabase`.",
+              "message": "Invalid value in parameter `types`: `createIndex` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`, `export`, `upgradeDatabase`.",
               "code": "invalid_task_types",
               "type": "invalid_request",
               "link": "https://docs.meilisearch.com/errors#invalid_task_types"