From 70ce0095ea3853d72d3817f4ef4cf8e2d61902cc Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 16 Apr 2024 18:48:03 +0200 Subject: [PATCH 1/4] remove the Version Seen analytic --- meilisearch/src/routes/mod.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/meilisearch/src/routes/mod.rs b/meilisearch/src/routes/mod.rs index 7cf886017..f2356fb7f 100644 --- a/meilisearch/src/routes/mod.rs +++ b/meilisearch/src/routes/mod.rs @@ -355,11 +355,7 @@ struct VersionResponse { async fn get_version( _index_scheduler: GuardedData, Data>, - req: HttpRequest, - analytics: web::Data, ) -> HttpResponse { - analytics.publish("Version Seen".to_string(), json!(null), Some(&req)); - let build_info = build_info::BuildInfo::from_build(); HttpResponse::Ok().json(VersionResponse { From abae31aee06874b62b215ff5fbef36337d94cbcb Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 16 Apr 2024 18:48:10 +0200 Subject: [PATCH 2/4] remove the Task Seen analytic --- meilisearch/src/analytics/mock_analytics.rs | 2 - meilisearch/src/analytics/mod.rs | 4 - .../src/analytics/segment_analytics.rs | 132 ------------------ meilisearch/src/routes/tasks.rs | 8 -- 4 files changed, 146 deletions(-) diff --git a/meilisearch/src/analytics/mock_analytics.rs b/meilisearch/src/analytics/mock_analytics.rs index 8fa1916a5..aa8989c3e 100644 --- a/meilisearch/src/analytics/mock_analytics.rs +++ b/meilisearch/src/analytics/mock_analytics.rs @@ -7,7 +7,6 @@ use serde_json::Value; use super::{find_user_id, Analytics, DocumentDeletionKind, DocumentFetchKind}; use crate::routes::indexes::documents::UpdateDocumentsQuery; -use crate::routes::tasks::TasksFilterQuery; use crate::Opt; pub struct MockAnalytics { @@ -86,6 +85,5 @@ impl Analytics for MockAnalytics { } fn get_fetch_documents(&self, _documents_query: &DocumentFetchKind, _request: &HttpRequest) {} fn post_fetch_documents(&self, _documents_query: &DocumentFetchKind, _request: &HttpRequest) {} - fn get_tasks(&self, _query: &TasksFilterQuery, _request: &HttpRequest) {} fn health_seen(&self, _request: &HttpRequest) {} } diff --git a/meilisearch/src/analytics/mod.rs b/meilisearch/src/analytics/mod.rs index d21838dbb..d6e8f1449 100644 --- a/meilisearch/src/analytics/mod.rs +++ b/meilisearch/src/analytics/mod.rs @@ -14,7 +14,6 @@ use platform_dirs::AppDirs; use serde_json::Value; use crate::routes::indexes::documents::UpdateDocumentsQuery; -use crate::routes::tasks::TasksFilterQuery; // if the analytics feature is disabled // the `SegmentAnalytics` point to the mock instead of the real analytics @@ -118,9 +117,6 @@ pub trait Analytics: Sync + Send { request: &HttpRequest, ); - // this method should be called to aggregate the get tasks requests. - fn get_tasks(&self, query: &TasksFilterQuery, request: &HttpRequest); - // this method should be called to aggregate a add documents request fn health_seen(&self, request: &HttpRequest); } diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index c49a04576..0dc471e78 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -33,7 +33,6 @@ use crate::option::{ }; use crate::routes::indexes::documents::UpdateDocumentsQuery; use crate::routes::indexes::facet_search::FacetSearchQuery; -use crate::routes::tasks::TasksFilterQuery; use crate::routes::{create_all_stats, Stats}; use crate::search::{ FacetSearchResult, MatchingStrategy, SearchQuery, SearchQueryWithIndex, SearchResult, @@ -81,7 +80,6 @@ pub enum AnalyticsMsg { AggregateUpdateDocuments(DocumentsAggregator), AggregateGetFetchDocuments(DocumentsFetchAggregator), AggregatePostFetchDocuments(DocumentsFetchAggregator), - AggregateTasks(TasksAggregator), AggregateHealth(HealthAggregator), } @@ -152,7 +150,6 @@ impl SegmentAnalytics { update_documents_aggregator: DocumentsAggregator::default(), get_fetch_documents_aggregator: DocumentsFetchAggregator::default(), post_fetch_documents_aggregator: DocumentsFetchAggregator::default(), - get_tasks_aggregator: TasksAggregator::default(), health_aggregator: HealthAggregator::default(), }); tokio::spawn(segment.run(index_scheduler.clone(), auth_controller.clone())); @@ -232,11 +229,6 @@ impl super::Analytics for SegmentAnalytics { let _ = self.sender.try_send(AnalyticsMsg::AggregatePostFetchDocuments(aggregate)); } - fn get_tasks(&self, query: &TasksFilterQuery, request: &HttpRequest) { - let aggregate = TasksAggregator::from_query(query, request); - let _ = self.sender.try_send(AnalyticsMsg::AggregateTasks(aggregate)); - } - fn health_seen(&self, request: &HttpRequest) { let aggregate = HealthAggregator::from_query(request); let _ = self.sender.try_send(AnalyticsMsg::AggregateHealth(aggregate)); @@ -394,7 +386,6 @@ pub struct Segment { update_documents_aggregator: DocumentsAggregator, get_fetch_documents_aggregator: DocumentsFetchAggregator, post_fetch_documents_aggregator: DocumentsFetchAggregator, - get_tasks_aggregator: TasksAggregator, health_aggregator: HealthAggregator, } @@ -458,7 +449,6 @@ impl Segment { Some(AnalyticsMsg::AggregateUpdateDocuments(agreg)) => self.update_documents_aggregator.aggregate(agreg), Some(AnalyticsMsg::AggregateGetFetchDocuments(agreg)) => self.get_fetch_documents_aggregator.aggregate(agreg), Some(AnalyticsMsg::AggregatePostFetchDocuments(agreg)) => self.post_fetch_documents_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregateTasks(agreg)) => self.get_tasks_aggregator.aggregate(agreg), Some(AnalyticsMsg::AggregateHealth(agreg)) => self.health_aggregator.aggregate(agreg), None => (), } @@ -513,7 +503,6 @@ impl Segment { update_documents_aggregator, get_fetch_documents_aggregator, post_fetch_documents_aggregator, - get_tasks_aggregator, health_aggregator, } = self; @@ -562,9 +551,6 @@ impl Segment { { let _ = self.batcher.push(post_fetch_documents).await; } - if let Some(get_tasks) = take(get_tasks_aggregator).into_event(user, "Tasks Seen") { - let _ = self.batcher.push(get_tasks).await; - } if let Some(health) = take(health_aggregator).into_event(user, "Health Seen") { let _ = self.batcher.push(health).await; } @@ -1503,124 +1489,6 @@ impl DocumentsDeletionAggregator { } } -#[derive(Default, Serialize)] -pub struct TasksAggregator { - #[serde(skip)] - timestamp: Option, - - // context - #[serde(rename = "user-agent")] - user_agents: HashSet, - - filtered_by_uid: bool, - filtered_by_index_uid: bool, - filtered_by_type: bool, - filtered_by_status: bool, - filtered_by_canceled_by: bool, - filtered_by_before_enqueued_at: bool, - filtered_by_after_enqueued_at: bool, - filtered_by_before_started_at: bool, - filtered_by_after_started_at: bool, - filtered_by_before_finished_at: bool, - filtered_by_after_finished_at: bool, - total_received: usize, -} - -impl TasksAggregator { - pub fn from_query(query: &TasksFilterQuery, request: &HttpRequest) -> Self { - let TasksFilterQuery { - limit: _, - from: _, - uids, - index_uids, - types, - statuses, - canceled_by, - before_enqueued_at, - after_enqueued_at, - before_started_at, - after_started_at, - before_finished_at, - after_finished_at, - } = query; - - Self { - timestamp: Some(OffsetDateTime::now_utc()), - user_agents: extract_user_agents(request).into_iter().collect(), - filtered_by_uid: uids.is_some(), - filtered_by_index_uid: index_uids.is_some(), - filtered_by_type: types.is_some(), - filtered_by_status: statuses.is_some(), - filtered_by_canceled_by: canceled_by.is_some(), - filtered_by_before_enqueued_at: before_enqueued_at.is_some(), - filtered_by_after_enqueued_at: after_enqueued_at.is_some(), - filtered_by_before_started_at: before_started_at.is_some(), - filtered_by_after_started_at: after_started_at.is_some(), - filtered_by_before_finished_at: before_finished_at.is_some(), - filtered_by_after_finished_at: after_finished_at.is_some(), - total_received: 1, - } - } - - /// Aggregate one [TasksAggregator] into another. - pub fn aggregate(&mut self, other: Self) { - let Self { - timestamp, - user_agents, - total_received, - filtered_by_uid, - filtered_by_index_uid, - filtered_by_type, - filtered_by_status, - filtered_by_canceled_by, - filtered_by_before_enqueued_at, - filtered_by_after_enqueued_at, - filtered_by_before_started_at, - filtered_by_after_started_at, - filtered_by_before_finished_at, - filtered_by_after_finished_at, - } = other; - - if self.timestamp.is_none() { - self.timestamp = timestamp; - } - - // we can't create a union because there is no `into_union` method - for user_agent in user_agents { - self.user_agents.insert(user_agent); - } - - self.filtered_by_uid |= filtered_by_uid; - self.filtered_by_index_uid |= filtered_by_index_uid; - self.filtered_by_type |= filtered_by_type; - self.filtered_by_status |= filtered_by_status; - self.filtered_by_canceled_by |= filtered_by_canceled_by; - self.filtered_by_before_enqueued_at |= filtered_by_before_enqueued_at; - self.filtered_by_after_enqueued_at |= filtered_by_after_enqueued_at; - self.filtered_by_before_started_at |= filtered_by_before_started_at; - self.filtered_by_after_started_at |= filtered_by_after_started_at; - self.filtered_by_before_finished_at |= filtered_by_before_finished_at; - self.filtered_by_after_finished_at |= filtered_by_after_finished_at; - self.filtered_by_after_finished_at |= filtered_by_after_finished_at; - - self.total_received = self.total_received.saturating_add(total_received); - } - - pub fn into_event(self, user: &User, event_name: &str) -> Option { - // if we had no timestamp it means we never encountered any events and - // thus we don't need to send this event. - let timestamp = self.timestamp?; - - Some(Track { - timestamp: Some(timestamp), - user: user.clone(), - event: event_name.to_string(), - properties: serde_json::to_value(self).ok()?, - ..Default::default() - }) - } -} - #[derive(Default, Serialize)] pub struct HealthAggregator { #[serde(skip)] diff --git a/meilisearch/src/routes/tasks.rs b/meilisearch/src/routes/tasks.rs index f35d97fe6..a4cf68d57 100644 --- a/meilisearch/src/routes/tasks.rs +++ b/meilisearch/src/routes/tasks.rs @@ -270,12 +270,8 @@ pub struct AllTasks { async fn get_tasks( index_scheduler: GuardedData, Data>, params: AwebQueryParameter, - req: HttpRequest, - analytics: web::Data, ) -> Result { let mut params = params.into_inner(); - analytics.get_tasks(¶ms, &req); - // We +1 just to know if there is more after this "page" or not. params.limit.0 = params.limit.0.saturating_add(1); let limit = params.limit.0; @@ -298,8 +294,6 @@ async fn get_tasks( async fn get_task( index_scheduler: GuardedData, Data>, task_uid: web::Path, - req: HttpRequest, - analytics: web::Data, ) -> Result { let task_uid_string = task_uid.into_inner(); @@ -310,8 +304,6 @@ async fn get_task( } }; - analytics.publish("Tasks Seen".to_string(), json!({ "per_task_uid": true }), Some(&req)); - let query = index_scheduler::Query { uids: Some(vec![task_uid]), ..Query::default() }; let filters = index_scheduler.filters(); let (tasks, _) = index_scheduler.get_tasks_from_authorized_indexes(query, filters)?; From e1f27de51aca1a5c1187de196405d02810216e70 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 16 Apr 2024 18:49:41 +0200 Subject: [PATCH 3/4] remove the Stats Seen analytic --- meilisearch/src/routes/indexes/mod.rs | 4 ---- meilisearch/src/routes/mod.rs | 3 --- 2 files changed, 7 deletions(-) diff --git a/meilisearch/src/routes/indexes/mod.rs b/meilisearch/src/routes/indexes/mod.rs index 59fa02dff..651977723 100644 --- a/meilisearch/src/routes/indexes/mod.rs +++ b/meilisearch/src/routes/indexes/mod.rs @@ -269,12 +269,8 @@ impl From for IndexStats { pub async fn get_index_stats( index_scheduler: GuardedData, Data>, index_uid: web::Path, - req: HttpRequest, - analytics: web::Data, ) -> Result { let index_uid = IndexUid::try_from(index_uid.into_inner())?; - analytics.publish("Stats Seen".to_string(), json!({ "per_index_uid": true }), Some(&req)); - let stats = IndexStats::from(index_scheduler.index_stats(&index_uid)?); debug!(returns = ?stats, "Get index stats"); diff --git a/meilisearch/src/routes/mod.rs b/meilisearch/src/routes/mod.rs index f2356fb7f..92d4af8e2 100644 --- a/meilisearch/src/routes/mod.rs +++ b/meilisearch/src/routes/mod.rs @@ -296,10 +296,7 @@ pub struct Stats { async fn get_stats( index_scheduler: GuardedData, Data>, auth_controller: GuardedData, Data>, - req: HttpRequest, - analytics: web::Data, ) -> Result { - analytics.publish("Stats Seen".to_string(), json!({ "per_index_uid": false }), Some(&req)); let filters = index_scheduler.filters(); let stats = create_all_stats((*index_scheduler).clone(), (*auth_controller).clone(), filters)?; From 2dd9dd6d0ae09f4970737e3a9eff83205f1f177f Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 17 Apr 2024 11:43:40 +0200 Subject: [PATCH 4/4] remove the Health Seen analytic --- meilisearch/src/analytics/mock_analytics.rs | 1 - meilisearch/src/analytics/mod.rs | 3 - .../src/analytics/segment_analytics.rs | 65 ------------------- meilisearch/src/routes/mod.rs | 6 -- 4 files changed, 75 deletions(-) diff --git a/meilisearch/src/analytics/mock_analytics.rs b/meilisearch/src/analytics/mock_analytics.rs index aa8989c3e..1687e9e19 100644 --- a/meilisearch/src/analytics/mock_analytics.rs +++ b/meilisearch/src/analytics/mock_analytics.rs @@ -85,5 +85,4 @@ impl Analytics for MockAnalytics { } fn get_fetch_documents(&self, _documents_query: &DocumentFetchKind, _request: &HttpRequest) {} fn post_fetch_documents(&self, _documents_query: &DocumentFetchKind, _request: &HttpRequest) {} - fn health_seen(&self, _request: &HttpRequest) {} } diff --git a/meilisearch/src/analytics/mod.rs b/meilisearch/src/analytics/mod.rs index d6e8f1449..09c0a05df 100644 --- a/meilisearch/src/analytics/mod.rs +++ b/meilisearch/src/analytics/mod.rs @@ -116,7 +116,4 @@ pub trait Analytics: Sync + Send { index_creation: bool, request: &HttpRequest, ); - - // this method should be called to aggregate a add documents request - fn health_seen(&self, request: &HttpRequest); } diff --git a/meilisearch/src/analytics/segment_analytics.rs b/meilisearch/src/analytics/segment_analytics.rs index 0dc471e78..8c20c82c2 100644 --- a/meilisearch/src/analytics/segment_analytics.rs +++ b/meilisearch/src/analytics/segment_analytics.rs @@ -80,7 +80,6 @@ pub enum AnalyticsMsg { AggregateUpdateDocuments(DocumentsAggregator), AggregateGetFetchDocuments(DocumentsFetchAggregator), AggregatePostFetchDocuments(DocumentsFetchAggregator), - AggregateHealth(HealthAggregator), } pub struct SegmentAnalytics { @@ -150,7 +149,6 @@ impl SegmentAnalytics { update_documents_aggregator: DocumentsAggregator::default(), get_fetch_documents_aggregator: DocumentsFetchAggregator::default(), post_fetch_documents_aggregator: DocumentsFetchAggregator::default(), - health_aggregator: HealthAggregator::default(), }); tokio::spawn(segment.run(index_scheduler.clone(), auth_controller.clone())); @@ -228,11 +226,6 @@ impl super::Analytics for SegmentAnalytics { let aggregate = DocumentsFetchAggregator::from_query(documents_query, request); let _ = self.sender.try_send(AnalyticsMsg::AggregatePostFetchDocuments(aggregate)); } - - fn health_seen(&self, request: &HttpRequest) { - let aggregate = HealthAggregator::from_query(request); - let _ = self.sender.try_send(AnalyticsMsg::AggregateHealth(aggregate)); - } } /// This structure represent the `infos` field we send in the analytics. @@ -386,7 +379,6 @@ pub struct Segment { update_documents_aggregator: DocumentsAggregator, get_fetch_documents_aggregator: DocumentsFetchAggregator, post_fetch_documents_aggregator: DocumentsFetchAggregator, - health_aggregator: HealthAggregator, } impl Segment { @@ -449,7 +441,6 @@ impl Segment { Some(AnalyticsMsg::AggregateUpdateDocuments(agreg)) => self.update_documents_aggregator.aggregate(agreg), Some(AnalyticsMsg::AggregateGetFetchDocuments(agreg)) => self.get_fetch_documents_aggregator.aggregate(agreg), Some(AnalyticsMsg::AggregatePostFetchDocuments(agreg)) => self.post_fetch_documents_aggregator.aggregate(agreg), - Some(AnalyticsMsg::AggregateHealth(agreg)) => self.health_aggregator.aggregate(agreg), None => (), } } @@ -503,7 +494,6 @@ impl Segment { update_documents_aggregator, get_fetch_documents_aggregator, post_fetch_documents_aggregator, - health_aggregator, } = self; if let Some(get_search) = @@ -551,9 +541,6 @@ impl Segment { { let _ = self.batcher.push(post_fetch_documents).await; } - if let Some(health) = take(health_aggregator).into_event(user, "Health Seen") { - let _ = self.batcher.push(health).await; - } let _ = self.batcher.flush().await; } } @@ -1489,58 +1476,6 @@ impl DocumentsDeletionAggregator { } } -#[derive(Default, Serialize)] -pub struct HealthAggregator { - #[serde(skip)] - timestamp: Option, - - // context - #[serde(rename = "user-agent")] - user_agents: HashSet, - - #[serde(rename = "requests.total_received")] - total_received: usize, -} - -impl HealthAggregator { - pub fn from_query(request: &HttpRequest) -> Self { - Self { - timestamp: Some(OffsetDateTime::now_utc()), - user_agents: extract_user_agents(request).into_iter().collect(), - total_received: 1, - } - } - - /// Aggregate one [HealthAggregator] into another. - pub fn aggregate(&mut self, other: Self) { - let Self { timestamp, user_agents, total_received } = other; - - if self.timestamp.is_none() { - self.timestamp = timestamp; - } - - // we can't create a union because there is no `into_union` method - for user_agent in user_agents { - self.user_agents.insert(user_agent); - } - self.total_received = self.total_received.saturating_add(total_received); - } - - pub fn into_event(self, user: &User, event_name: &str) -> Option { - // if we had no timestamp it means we never encountered any events and - // thus we don't need to send this event. - let timestamp = self.timestamp?; - - Some(Track { - timestamp: Some(timestamp), - user: user.clone(), - event: event_name.to_string(), - properties: serde_json::to_value(self).ok()?, - ..Default::default() - }) - } -} - #[derive(Default, Serialize)] pub struct DocumentsFetchAggregator { #[serde(skip)] diff --git a/meilisearch/src/routes/mod.rs b/meilisearch/src/routes/mod.rs index 92d4af8e2..0e6ab7765 100644 --- a/meilisearch/src/routes/mod.rs +++ b/meilisearch/src/routes/mod.rs @@ -8,11 +8,9 @@ use meilisearch_types::error::{Code, ResponseError}; use meilisearch_types::settings::{Settings, Unchecked}; use meilisearch_types::tasks::{Kind, Status, Task, TaskId}; use serde::{Deserialize, Serialize}; -use serde_json::json; use time::OffsetDateTime; use tracing::debug; -use crate::analytics::Analytics; use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::search_queue::SearchQueue; @@ -376,14 +374,10 @@ struct KeysResponse { } pub async fn get_health( - req: HttpRequest, index_scheduler: Data, auth_controller: Data, search_queue: Data, - analytics: web::Data, ) -> Result { - analytics.health_seen(&req); - search_queue.health().unwrap(); index_scheduler.health().unwrap(); auth_controller.health().unwrap();