Merge #209

209: Integrate amplitude r=MarinPostma a=irevoire

And merge the sentry and amplitude usage under one “Enable analytics” flag

closes #180


Co-authored-by: Tamo <tamo@meilisearch.com>
Co-authored-by: Irevoire <tamo@meilisearch.com>

Cargo.lock

@@ -1,5 +1,7 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
 [[package]]
 name = "actix-codec"
 version = "0.4.0"
@@ -641,6 +643,22 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "core-foundation"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b"
+
 [[package]]
 name = "cow-utils"
 version = "0.1.2"
@@ -887,6 +905,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.0.1"
@@ -1296,6 +1329,19 @@ dependencies = [
  "webpki",
 ]
 
+[[package]]
+name = "hyper-tls"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+dependencies = [
+ "bytes 1.0.1",
+ "hyper",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+]
+
 [[package]]
 name = "idna"
 version = "0.2.3"
@@ -1614,6 +1660,7 @@ dependencies = [
  "uuid",
  "vergen",
  "walkdir",
+ "whoami",
  "zip",
 ]
 
@@ -1775,6 +1822,24 @@ dependencies = [
  "syn 1.0.73",
 ]
 
+[[package]]
+name = "native-tls"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4"
+dependencies = [
+ "lazy_static",
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
 [[package]]
 name = "nix"
 version = "0.19.1"
@@ -1864,6 +1929,39 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
+[[package]]
+name = "openssl"
+version = "0.10.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8"
+dependencies = [
+ "bitflags",
+ "cfg-if 1.0.0",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.63"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98"
+dependencies = [
+ "autocfg",
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "ordered-float"
 version = "2.5.1"
@@ -2384,11 +2482,13 @@ dependencies = [
  "http-body",
  "hyper",
  "hyper-rustls",
+ "hyper-tls",
  "ipnet",
  "js-sys",
  "lazy_static",
  "log",
  "mime",
+ "native-tls",
  "percent-encoding",
  "pin-project-lite",
  "rustls",
@@ -2396,6 +2496,7 @@ dependencies = [
  "serde_json",
  "serde_urlencoded",
  "tokio",
+ "tokio-native-tls",
  "tokio-rustls",
  "url",
  "wasm-bindgen",
@@ -2498,6 +2599,16 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "schannel"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
+dependencies = [
+ "lazy_static",
+ "winapi",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -2514,6 +2625,29 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "security-framework"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467"
+dependencies = [
+ "bitflags",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "semver"
 version = "0.9.0"
@@ -3115,6 +3249,16 @@ dependencies = [
  "syn 1.0.73",
 ]
 
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.22.0"
@@ -3300,6 +3444,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "vcpkg"
+version = "0.2.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "025ce40a007e1907e58d5bc1a594def78e5573bb0b1160bc389634e8f12e4faa"
+
 [[package]]
 name = "vec_map"
 version = "0.8.2"
@@ -3462,6 +3612,16 @@ dependencies = [
  "hashbrown 0.7.2",
 ]
 
+[[package]]
+name = "whoami"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4abacf325c958dfeaf1046931d37f2a901b6dfe0968ee965a29e94c6766b2af6"
+dependencies = [
+ "wasm-bindgen",
+ "web-sys",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"

meilisearch-http/Cargo.toml

@@ -74,6 +74,8 @@ uuid = { version =  "0.8.2", features = ["serde"] }
 walkdir = "2.3.2"
 obkv = "0.1.1"
 pin-project = "1.0.7"
+whoami = { version = "1.1.2", optional = true }
+reqwest = { version = "0.11.3", features = ["json"], optional = true }
 
 [dependencies.sentry]
 default-features = false
@@ -109,8 +111,8 @@ mini-dashboard = [
     "tempfile",
     "zip",
 ]
-telemetry = ["sentry"]
+analytics = ["sentry", "whoami", "reqwest"]
-default = ["telemetry", "mini-dashboard"]
+default = ["analytics", "mini-dashboard"]
 
 [target.'cfg(target_os = "linux")'.dependencies]
 jemallocator = "0.3.2"

meilisearch-http/src/analytics.rs

@@ -1,14 +1,10 @@
 use std::hash::{Hash, Hasher};
-use std::{error, thread};
 use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
 
-use log::error;
+use log::debug;
 use serde::Serialize;
-use serde_qs as qs;
 use siphasher::sip::SipHasher;
-use walkdir::WalkDir;
 
-use crate::helpers::EnvSizer;
 use crate::Data;
 use crate::Opt;
 
@@ -22,26 +18,21 @@ struct EventProperties {
 }
 
 impl EventProperties {
-    fn from(data: Data) -> Result<EventProperties, Box<dyn error::Error>> {
+    async fn from(data: Data) -> anyhow::Result<EventProperties> {
-        let mut index_list = Vec::new();
+        let stats = data.index_controller.get_all_stats().await?;
 
-        let reader = data.db.main_read_txn()?;
+        let database_size = stats.database_size;
-
+        let last_update_timestamp = stats.last_update.map(|u| u.timestamp());
-        for index_uid in data.db.indexes_uids() {
+        let number_of_documents = stats
-            if let Some(index) = data.db.open_index(&index_uid) {
+            .indexes
-                let number_of_documents = index.main.number_of_documents(&reader)?;
+            .values()
-                index_list.push(number_of_documents);
+            .map(|index| index.number_of_documents)
-            }
+            .collect();
-        }
-
-        let database_size = data.env.size();
-
-        let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp());
 
         Ok(EventProperties {
             database_size,
             last_update_timestamp,
-            number_of_documents: index_list,
+            number_of_documents,
         })
     }
 }
@@ -68,10 +59,10 @@ struct Event<'a> {
 #[derive(Debug, Serialize)]
 struct AmplitudeRequest<'a> {
     api_key: &'a str,
-    event: &'a str,
+    events: Vec<Event<'a>>,
 }
 
-pub fn analytics_sender(data: Data, opt: Opt) {
+pub async fn analytics_sender(data: Data, opt: Opt) {
     let username = whoami::username();
     let hostname = whoami::hostname();
     let platform = whoami::platform();
@@ -93,7 +84,7 @@ pub fn analytics_sender(data: Data, opt: Opt) {
         let time = n.as_secs();
         let event_type = "runtime_tick";
         let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day
-        let event_properties = EventProperties::from(data.clone()).ok();
+        let event_properties = EventProperties::from(data.clone()).await.ok();
         let app_version = env!("CARGO_PKG_VERSION").to_string();
         let app_version = app_version.as_str();
         let user_email = std::env::var("MEILI_USER_EMAIL").ok();
@@ -114,20 +105,22 @@ pub fn analytics_sender(data: Data, opt: Opt) {
             user_properties,
             event_properties,
         };
-        let event = serde_json::to_string(&event).unwrap();
 
         let request = AmplitudeRequest {
             api_key: AMPLITUDE_API_KEY,
-            event: &event,
+            events: vec![event],
         };
 
-        let body = qs::to_string(&request).unwrap();
+        let response = reqwest::Client::new()
-        let response = ureq::post("https://api.amplitude.com/httpapi").send_string(&body);
+            .post("https://api2.amplitude.com/2/httpapi")
-        if !response.ok() {
+            .timeout(Duration::from_secs(60)) // 1 minute max
-            let body = response.into_string().unwrap();
+            .json(&request)
-            error!("Unsuccessful call to Amplitude: {}", body);
+            .send()
+            .await;
+        if let Err(e) = response {
+            debug!("Unsuccessful call to Amplitude: {}", e);
         }
 
-        thread::sleep(Duration::from_secs(3600)) // one hour
+        tokio::time::sleep(Duration::from_secs(3600)).await;
     }
 }

meilisearch-http/src/lib.rs

@@ -6,6 +6,9 @@ mod index_controller;
 pub mod option;
 pub mod routes;
 
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+pub mod analytics;
+
 pub use self::data::Data;
 pub use option::Opt;
 

meilisearch-http/src/main.rs

@@ -5,12 +5,16 @@ use main_error::MainError;
 use meilisearch_http::{create_app, Data, Opt};
 use structopt::StructOpt;
 
-//mod analytics;
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+use meilisearch_http::analytics;
 
 #[cfg(target_os = "linux")]
 #[global_allocator]
 static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+const SENTRY_DSN: &str = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337";
+
 #[actix_web::main]
 async fn main() -> Result<(), MainError> {
     let opt = Opt::from_args();
@@ -26,16 +30,17 @@ async fn main() -> Result<(), MainError> {
                         .into(),
                 );
             }
-            #[cfg(all(not(debug_assertions), feature = "sentry"))]
+            #[cfg(all(not(debug_assertions), feature = "analytics"))]
-            if !opt.no_sentry {
+            if !opt.no_analytics {
-                let logger = sentry::integrations::log::SentryLogger::with_dest(log_builder.build());
+                let logger =
+                    sentry::integrations::log::SentryLogger::with_dest(log_builder.build());
                 log::set_boxed_logger(Box::new(logger))
                     .map(|()| log::set_max_level(log::LevelFilter::Info))
                     .unwrap();
 
                 let sentry = sentry::init(sentry::ClientOptions {
                     release: sentry::release_name!(),
-                    dsn: Some(opt.sentry_dsn.parse()?),
+                    dsn: Some(SENTRY_DSN.parse()?),
                     ..Default::default()
                 });
                 // sentry must stay alive as long as possible
@@ -50,25 +55,14 @@ async fn main() -> Result<(), MainError> {
         _ => unreachable!(),
     }
 
-    //if let Some(path) = &opt.import_snapshot {
-    //snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?;
-    //}
-
     let data = Data::new(opt.clone())?;
 
-    //if !opt.no_analytics {
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
-    //let analytics_data = data.clone();
+    if !opt.no_analytics {
-    //let analytics_opt = opt.clone();
+        let analytics_data = data.clone();
-    //thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt));
+        let analytics_opt = opt.clone();
-    //}
+        tokio::task::spawn(analytics::analytics_sender(analytics_data, analytics_opt));
-
+    }
-    //if let Some(path) = &opt.import_dump {
-    //dump::import_dump(&data, path, opt.dump_batch_size)?;
-    //}
-
-    //if opt.schedule_snapshot {
-    //snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?;
-    //}
 
     print_launch_resume(&opt, &data);
 
@@ -127,24 +121,21 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) {
         env!("CARGO_PKG_VERSION").to_string()
     );
 
-    #[cfg(all(not(debug_assertions), feature = "sentry"))]
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
-    eprintln!(
+    {
-        "Sentry DSN:\t\t{:?}",
+        if opt.no_analytics {
-        if !opt.no_sentry {
+            eprintln!("Anonymous telemetry:\t\"Disabled\"");
-            &opt.sentry_dsn
         } else {
-            "Disabled"
+            eprintln!(
-        }
+                "
-    );
+Thank you for using MeiliSearch!
 
-    eprintln!(
+We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/reference/features/configuration.html#analytics
-        "Anonymous telemetry:\t{:?}",
+
-        if !opt.no_analytics {
+Anonymous telemetry:   \"Enabled\""
-            "Enabled"
+            );
-        } else {
-            "Disabled"
         }
-    );
+    }
 
     eprintln!();
 

meilisearch-http/src/option.rs

@@ -96,21 +96,6 @@ pub struct Opt {
     #[structopt(long, env = "MEILI_MASTER_KEY")]
     pub master_key: Option<String>,
 
-    /// The Sentry DSN to use for error reporting. This defaults to the MeiliSearch Sentry project.
-    /// You can disable sentry all together using the `--no-sentry` flag or `MEILI_NO_SENTRY` environment variable.
-    #[cfg(all(not(debug_assertions), feature = "sentry"))]
-    #[structopt(
-        long,
-        env = "SENTRY_DSN",
-        default_value = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337"
-    )]
-    pub sentry_dsn: String,
-
-    /// Disable Sentry error reporting.
-    #[structopt(long, env = "MEILI_NO_SENTRY")]
-    #[cfg(all(not(debug_assertions), feature = "sentry"))]
-    pub no_sentry: bool,
-
     /// This environment variable must be set to `production` if you are running in production.
     /// If the server is running in development mode more logs will be displayed,
     /// and the master key can be avoided which implies that there is no security on the updates routes.
@@ -119,6 +104,7 @@ pub struct Opt {
     pub env: String,
 
     /// Do not send analytics to Meili.
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
     #[structopt(long, env = "MEILI_NO_ANALYTICS")]
     pub no_analytics: bool,
 

meilisearch-http/tests/common/server.rs

@@ -71,6 +71,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
         http_addr: "127.0.0.1:7700".to_owned(),
         master_key: None,
         env: "development".to_owned(),
+        #[cfg(all(not(debug_assertions), feature = "analytics"))]
         no_analytics: true,
         max_mdb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
         max_udb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
@@ -90,9 +91,5 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
         snapshot_interval_sec: 0,
         import_dump: None,
         indexer_options: IndexerOpts::default(),
-        #[cfg(all(not(debug_assertions), feature = "sentry"))]
-        sentry_dsn: String::from(""),
-        #[cfg(all(not(debug_assertions), feature = "sentry"))]
-        no_sentry: true,
     }
 }