diff --git a/Cargo.lock b/Cargo.lock index 4bc1f6ec9..080a67a59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "actix-codec" version = "0.4.0" @@ -641,6 +643,22 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + [[package]] name = "cow-utils" version = "0.1.2" @@ -887,6 +905,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -1296,6 +1329,19 @@ dependencies = [ "webpki", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes 1.0.1", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.2.3" @@ -1614,6 +1660,7 @@ dependencies = [ "uuid", "vergen", "walkdir", + "whoami", "zip", ] @@ -1775,6 +1822,24 @@ dependencies = [ "syn 1.0.73", ] +[[package]] +name = "native-tls" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.19.1" @@ -1864,6 +1929,39 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-float" version = "2.5.1" @@ -2384,11 +2482,13 @@ dependencies = [ "http-body", "hyper", "hyper-rustls", + "hyper-tls", "ipnet", "js-sys", "lazy_static", "log", "mime", + "native-tls", "percent-encoding", "pin-project-lite", "rustls", @@ -2396,6 +2496,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", + "tokio-native-tls", "tokio-rustls", "url", "wasm-bindgen", @@ -2498,6 +2599,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -2514,6 +2625,29 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "0.9.0" @@ -3115,6 +3249,16 @@ dependencies = [ "syn 1.0.73", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.22.0" @@ -3300,6 +3444,12 @@ dependencies = [ "serde", ] +[[package]] +name = "vcpkg" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "025ce40a007e1907e58d5bc1a594def78e5573bb0b1160bc389634e8f12e4faa" + [[package]] name = "vec_map" version = "0.8.2" @@ -3462,6 +3612,16 @@ dependencies = [ "hashbrown 0.7.2", ] +[[package]] +name = "whoami" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4abacf325c958dfeaf1046931d37f2a901b6dfe0968ee965a29e94c6766b2af6" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 64efe4c62..5bb9cc6a4 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -74,6 +74,8 @@ uuid = { version = "0.8.2", features = ["serde"] } walkdir = "2.3.2" obkv = "0.1.1" pin-project = "1.0.7" +whoami = { version = "1.1.2", optional = true } +reqwest = { version = "0.11.3", features = ["json"], optional = true } [dependencies.sentry] default-features = false @@ -109,8 +111,8 @@ mini-dashboard = [ "tempfile", "zip", ] -telemetry = ["sentry"] -default = ["telemetry", "mini-dashboard"] +analytics = ["sentry", "whoami", "reqwest"] +default = ["analytics", "mini-dashboard"] [target.'cfg(target_os = "linux")'.dependencies] jemallocator = "0.3.2" diff --git a/meilisearch-http/src/analytics.rs b/meilisearch-http/src/analytics.rs index a01cfabe0..11347175b 100644 --- a/meilisearch-http/src/analytics.rs +++ b/meilisearch-http/src/analytics.rs @@ -1,14 +1,10 @@ use std::hash::{Hash, Hasher}; -use std::{error, thread}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use log::error; +use log::debug; use serde::Serialize; -use serde_qs as qs; use siphasher::sip::SipHasher; -use walkdir::WalkDir; -use crate::helpers::EnvSizer; use crate::Data; use crate::Opt; @@ -22,26 +18,21 @@ struct EventProperties { } impl EventProperties { - fn from(data: Data) -> Result> { - let mut index_list = Vec::new(); + async fn from(data: Data) -> anyhow::Result { + let stats = data.index_controller.get_all_stats().await?; - let reader = data.db.main_read_txn()?; - - for index_uid in data.db.indexes_uids() { - if let Some(index) = data.db.open_index(&index_uid) { - let number_of_documents = index.main.number_of_documents(&reader)?; - index_list.push(number_of_documents); - } - } - - let database_size = data.env.size(); - - let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp()); + let database_size = stats.database_size; + let last_update_timestamp = stats.last_update.map(|u| u.timestamp()); + let number_of_documents = stats + .indexes + .values() + .map(|index| index.number_of_documents) + .collect(); Ok(EventProperties { database_size, last_update_timestamp, - number_of_documents: index_list, + number_of_documents, }) } } @@ -68,10 +59,10 @@ struct Event<'a> { #[derive(Debug, Serialize)] struct AmplitudeRequest<'a> { api_key: &'a str, - event: &'a str, + events: Vec>, } -pub fn analytics_sender(data: Data, opt: Opt) { +pub async fn analytics_sender(data: Data, opt: Opt) { let username = whoami::username(); let hostname = whoami::hostname(); let platform = whoami::platform(); @@ -93,7 +84,7 @@ pub fn analytics_sender(data: Data, opt: Opt) { let time = n.as_secs(); let event_type = "runtime_tick"; let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day - let event_properties = EventProperties::from(data.clone()).ok(); + let event_properties = EventProperties::from(data.clone()).await.ok(); let app_version = env!("CARGO_PKG_VERSION").to_string(); let app_version = app_version.as_str(); let user_email = std::env::var("MEILI_USER_EMAIL").ok(); @@ -114,20 +105,22 @@ pub fn analytics_sender(data: Data, opt: Opt) { user_properties, event_properties, }; - let event = serde_json::to_string(&event).unwrap(); let request = AmplitudeRequest { api_key: AMPLITUDE_API_KEY, - event: &event, + events: vec![event], }; - let body = qs::to_string(&request).unwrap(); - let response = ureq::post("https://api.amplitude.com/httpapi").send_string(&body); - if !response.ok() { - let body = response.into_string().unwrap(); - error!("Unsuccessful call to Amplitude: {}", body); + let response = reqwest::Client::new() + .post("https://api2.amplitude.com/2/httpapi") + .timeout(Duration::from_secs(60)) // 1 minute max + .json(&request) + .send() + .await; + if let Err(e) = response { + debug!("Unsuccessful call to Amplitude: {}", e); } - thread::sleep(Duration::from_secs(3600)) // one hour + tokio::time::sleep(Duration::from_secs(3600)).await; } } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 26b6a784c..8788d45d1 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -6,6 +6,9 @@ mod index_controller; pub mod option; pub mod routes; +#[cfg(feature = "analytics")] +pub mod analytics; + pub use self::data::Data; pub use option::Opt; diff --git a/meilisearch-http/src/main.rs b/meilisearch-http/src/main.rs index a29299673..120684dfc 100644 --- a/meilisearch-http/src/main.rs +++ b/meilisearch-http/src/main.rs @@ -5,12 +5,16 @@ use main_error::MainError; use meilisearch_http::{create_app, Data, Opt}; use structopt::StructOpt; -//mod analytics; +#[cfg(feature = "analytics")] +use meilisearch_http::analytics; #[cfg(target_os = "linux")] #[global_allocator] static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +#[cfg(all(not(debug_assertions), feature = "analytics"))] +const SENTRY_DSN: &str = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337"; + #[actix_web::main] async fn main() -> Result<(), MainError> { let opt = Opt::from_args(); @@ -26,16 +30,17 @@ async fn main() -> Result<(), MainError> { .into(), ); } - #[cfg(all(not(debug_assertions), feature = "sentry"))] - if !opt.no_sentry { - let logger = sentry::integrations::log::SentryLogger::with_dest(log_builder.build()); + #[cfg(all(not(debug_assertions), feature = "analytics"))] + if !opt.no_analytics { + let logger = + sentry::integrations::log::SentryLogger::with_dest(log_builder.build()); log::set_boxed_logger(Box::new(logger)) .map(|()| log::set_max_level(log::LevelFilter::Info)) .unwrap(); let sentry = sentry::init(sentry::ClientOptions { release: sentry::release_name!(), - dsn: Some(opt.sentry_dsn.parse()?), + dsn: Some(SENTRY_DSN.parse()?), ..Default::default() }); // sentry must stay alive as long as possible @@ -50,25 +55,14 @@ async fn main() -> Result<(), MainError> { _ => unreachable!(), } - //if let Some(path) = &opt.import_snapshot { - //snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?; - //} - let data = Data::new(opt.clone())?; - //if !opt.no_analytics { - //let analytics_data = data.clone(); - //let analytics_opt = opt.clone(); - //thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt)); - //} - - //if let Some(path) = &opt.import_dump { - //dump::import_dump(&data, path, opt.dump_batch_size)?; - //} - - //if opt.schedule_snapshot { - //snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?; - //} + #[cfg(feature = "analytics")] + if !opt.no_analytics { + let analytics_data = data.clone(); + let analytics_opt = opt.clone(); + tokio::task::spawn(analytics::analytics_sender(analytics_data, analytics_opt)); + } print_launch_resume(&opt, &data); @@ -127,24 +121,21 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) { env!("CARGO_PKG_VERSION").to_string() ); - #[cfg(all(not(debug_assertions), feature = "sentry"))] - eprintln!( - "Sentry DSN:\t\t{:?}", - if !opt.no_sentry { - &opt.sentry_dsn + #[cfg(feature = "analytics")] + { + if opt.no_analytics { + eprintln!("Anonymous telemetry:\t\"Disabled\""); } else { - "Disabled" - } - ); + eprintln!( + " +Thank you for using MeiliSearch! +We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/reference/features/configuration.html#analytics - eprintln!( - "Anonymous telemetry:\t{:?}", - if !opt.no_analytics { - "Enabled" - } else { - "Disabled" +Anonymous telemetry: \"Enabled\" +" + ); } - ); + } eprintln!(); diff --git a/meilisearch-http/src/option.rs b/meilisearch-http/src/option.rs index eb81ab9fd..54a50742a 100644 --- a/meilisearch-http/src/option.rs +++ b/meilisearch-http/src/option.rs @@ -96,21 +96,6 @@ pub struct Opt { #[structopt(long, env = "MEILI_MASTER_KEY")] pub master_key: Option, - /// The Sentry DSN to use for error reporting. This defaults to the MeiliSearch Sentry project. - /// You can disable sentry all together using the `--no-sentry` flag or `MEILI_NO_SENTRY` environment variable. - #[cfg(all(not(debug_assertions), feature = "sentry"))] - #[structopt( - long, - env = "SENTRY_DSN", - default_value = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337" - )] - pub sentry_dsn: String, - - /// Disable Sentry error reporting. - #[structopt(long, env = "MEILI_NO_SENTRY")] - #[cfg(all(not(debug_assertions), feature = "sentry"))] - pub no_sentry: bool, - /// This environment variable must be set to `production` if you are running in production. /// If the server is running in development mode more logs will be displayed, /// and the master key can be avoided which implies that there is no security on the updates routes. @@ -119,6 +104,7 @@ pub struct Opt { pub env: String, /// Do not send analytics to Meili. + #[cfg(feature = "analytics")] #[structopt(long, env = "MEILI_NO_ANALYTICS")] pub no_analytics: bool, diff --git a/meilisearch-http/tests/common/server.rs b/meilisearch-http/tests/common/server.rs index 0fb801d7f..2c47a8ca8 100644 --- a/meilisearch-http/tests/common/server.rs +++ b/meilisearch-http/tests/common/server.rs @@ -71,6 +71,7 @@ pub fn default_settings(dir: impl AsRef) -> Opt { http_addr: "127.0.0.1:7700".to_owned(), master_key: None, env: "development".to_owned(), + #[cfg(feature = "analytics")] no_analytics: true, max_mdb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), max_udb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), @@ -90,9 +91,5 @@ pub fn default_settings(dir: impl AsRef) -> Opt { snapshot_interval_sec: 0, import_dump: None, indexer_options: IndexerOpts::default(), - #[cfg(all(not(debug_assertions), feature = "sentry"))] - sentry_dsn: String::from(""), - #[cfg(all(not(debug_assertions), feature = "sentry"))] - no_sentry: true, } }