From e51ea55ae31f81cadcf7bb99e1583c38cd7b0813 Mon Sep 17 00:00:00 2001 From: qdequele Date: Tue, 26 May 2020 19:03:13 +0200 Subject: [PATCH 1/2] add more analytics --- meilisearch-http/src/analytics.rs | 76 +++++++++++++++++++++++++++++-- meilisearch-http/src/lib.rs | 2 + meilisearch-http/src/main.rs | 16 ++++--- 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/meilisearch-http/src/analytics.rs b/meilisearch-http/src/analytics.rs index da5b600ce..379a25030 100644 --- a/meilisearch-http/src/analytics.rs +++ b/meilisearch-http/src/analytics.rs @@ -1,20 +1,72 @@ use std::hash::{Hash, Hasher}; -use std::thread; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::{error, thread}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use log::error; use serde::Serialize; use serde_qs as qs; use siphasher::sip::SipHasher; +use walkdir::WalkDir; + +use crate::Data; +use crate::Opt; const AMPLITUDE_API_KEY: &str = "f7fba398780e06d8fe6666a9be7e3d47"; +#[derive(Debug, Serialize)] +struct EventProperties { + database_size: u64, + last_update_timestamp: Option, //timestamp + number_of_documents: Vec, +} + +impl EventProperties { + fn from(data: Data) -> Result> { + let mut index_list = Vec::new(); + + let reader = data.db.main_read_txn()?; + + for index_uid in data.db.indexes_uids() { + if let Some(index) = data.db.open_index(&index_uid) { + let number_of_documents = index.main.number_of_documents(&reader)?; + index_list.push(number_of_documents); + } + } + + let database_size = WalkDir::new(&data.db_path) + .into_iter() + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.metadata().ok()) + .filter(|metadata| metadata.is_file()) + .fold(0, |acc, m| acc + m.len()); + + let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp()); + + Ok(EventProperties { + database_size, + last_update_timestamp, + number_of_documents: index_list, + }) + } +} + +#[derive(Debug, Serialize)] +struct UserProperties<'a> { + env: &'a str, + start_since_days: u64, + user_email: Option, + server_provider: Option, +} + #[derive(Debug, Serialize)] struct Event<'a> { user_id: &'a str, event_type: &'a str, device_id: &'a str, time: u64, + app_version: &'a str, + user_properties: UserProperties<'a>, + event_properties: Option, } #[derive(Debug, Serialize)] @@ -23,7 +75,7 @@ struct AmplitudeRequest<'a> { event: &'a str, } -pub fn analytics_sender() { +pub fn analytics_sender(data: Data, opt: Opt) { let username = whoami::username(); let hostname = whoami::hostname(); let platform = whoami::platform(); @@ -36,6 +88,7 @@ pub fn analytics_sender() { let uid = format!("{:X}", hash); let platform = platform.to_string(); + let first_start = Instant::now(); loop { let n = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); @@ -43,12 +96,27 @@ pub fn analytics_sender() { let device_id = &platform; let time = n.as_secs(); let event_type = "runtime_tick"; + let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day + let event_properties = EventProperties::from(data.clone()).ok(); + let app_version = env!("CARGO_PKG_VERSION").to_string(); + let app_version = app_version.as_str(); + let user_email = std::env::var("MEILI_USER_EMAIL").ok(); + let server_provider = std::env::var("MEILI_SERVER_PROVIDER").ok(); + let user_properties = UserProperties { + env: &opt.env, + start_since_days: elapsed_since_start, + user_email, + server_provider, + }; let event = Event { user_id, event_type, device_id, time, + app_version, + user_properties, + event_properties }; let event = serde_json::to_string(&event).unwrap(); @@ -64,6 +132,6 @@ pub fn analytics_sender() { error!("Unsuccessful call to Amplitude: {}", body); } - thread::sleep(Duration::from_secs(86_400)) // one day + thread::sleep(Duration::from_secs(3600)) // one hour } } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 8e81d9221..4b68c5120 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -6,6 +6,7 @@ pub mod helpers; pub mod models; pub mod option; pub mod routes; +pub mod analytics; use actix_http::Error; use actix_service::ServiceFactory; @@ -15,6 +16,7 @@ use log::error; use meilisearch_core::ProcessedUpdateResult; +pub use option::Opt; pub use self::data::Data; use self::error::{json_error_handler, ResponseError}; diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index f430c9f55..c60cb2f1a 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -3,10 +3,8 @@ use std::{env, thread}; use actix_cors::Cors; use actix_web::{middleware, HttpServer}; use main_error::MainError; -use meilisearch_http::data::Data; use meilisearch_http::helpers::NormalizePath; -use meilisearch_http::option::Opt; -use meilisearch_http::{create_app, index_update_callback}; +use meilisearch_http::{Data, Opt, create_app, index_update_callback}; use structopt::StructOpt; mod analytics; @@ -49,12 +47,16 @@ async fn main() -> Result<(), MainError> { _ => unreachable!(), } - if !opt.no_analytics { - thread::spawn(analytics::analytics_sender); - } - let data = Data::new(opt.clone()); + if !opt.no_analytics { + let analytics_data = data.clone(); + let analytics_opt = opt.clone(); + thread::spawn(move|| { + analytics::analytics_sender(analytics_data, analytics_opt) + }); + } + let data_cloned = data.clone(); data.db.set_update_callback(Box::new(move |name, status| { index_update_callback(name, &data_cloned, status); From a2321d156265e20ccba7d1cf79ad9ae219a4bccc Mon Sep 17 00:00:00 2001 From: qdequele Date: Wed, 27 May 2020 12:03:23 +0200 Subject: [PATCH 2/2] update changelog and readme --- CHANGELOG.md | 1 + README.md | 18 +++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index caa6df4b3..805de9e47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Rename fieldsFrequency into fieldsDistribution in stats (#719) - Add support for error code reporting (#703) - Allow the dashboard to query private servers (#732) + - Add telemetry (#720) ## v0.10.1 diff --git a/README.md b/README.md index dff726780..c785efcf4 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@

MeiliSearch

- Website | - Blog | - LinkedIn | - Twitter | - Documentation | + Website | + Blog | + LinkedIn | + Twitter | + Documentation | FAQ

@@ -122,7 +122,7 @@ curl -i -X POST 'http://127.0.0.1:7700/indexes/movies/documents' \ #### In command line -The search engine is now aware of your documents and can serve those via an HTTP server. +The search engine is now aware of your documents and can serve those via an HTTP server. The [`jq` command-line tool](https://stedolan.github.io/jq/) can greatly help you read the server responses. @@ -167,7 +167,7 @@ You can access the web interface in your web browser at the root of the server. ### Documentation -Now that your MeiliSearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com). +Now that your MeiliSearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com). ## Contributing @@ -176,8 +176,8 @@ Hey! We're glad you're thinking about contributing to MeiliSearch! If you think ### Analytic Events -Once a day, events are being sent to our Amplitude instance so we can know how many people are using MeiliSearch.
-Only information about the platform on which the server runs is stored. No other information is being sent.
+Every hour, events are being sent to our Amplitude instance so we can know how many people are using MeiliSearch.
+To see what information we're retrieving, please see the complete list [on the dedicated issue](https://github.com/meilisearch/MeiliSearch/issues/720).
We also use Sentry to make us crash and error reports. If you want to know more about what Sentry collects, please visit their [privacy policy website](https://sentry.io/privacy/).
If this doesn't suit you, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.