Integrate amplitude

And merge the sentry and amplitude usage under one “Enable analytics”
flag
This commit is contained in:
Tamo 2021-06-15 15:36:30 +02:00
parent b119bb4ab0
commit def1596eaf
No known key found for this signature in database
GPG Key ID: 20CD8020AFA88D69
7 changed files with 221 additions and 89 deletions

160
Cargo.lock generated
View File

@ -1,5 +1,7 @@
# This file is automatically @generated by Cargo. # This file is automatically @generated by Cargo.
# It is not intended for manual editing. # It is not intended for manual editing.
version = 3
[[package]] [[package]]
name = "actix-codec" name = "actix-codec"
version = "0.4.0" version = "0.4.0"
@ -641,6 +643,22 @@ dependencies = [
"version_check", "version_check",
] ]
[[package]]
name = "core-foundation"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "core-foundation-sys"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b"
[[package]] [[package]]
name = "cow-utils" name = "cow-utils"
version = "0.1.2" version = "0.1.2"
@ -887,6 +905,21 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]] [[package]]
name = "form_urlencoded" name = "form_urlencoded"
version = "1.0.1" version = "1.0.1"
@ -1296,6 +1329,19 @@ dependencies = [
"webpki", "webpki",
] ]
[[package]]
name = "hyper-tls"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
dependencies = [
"bytes 1.0.1",
"hyper",
"native-tls",
"tokio",
"tokio-native-tls",
]
[[package]] [[package]]
name = "idna" name = "idna"
version = "0.2.3" version = "0.2.3"
@ -1614,6 +1660,7 @@ dependencies = [
"uuid", "uuid",
"vergen", "vergen",
"walkdir", "walkdir",
"whoami",
"zip", "zip",
] ]
@ -1775,6 +1822,24 @@ dependencies = [
"syn 1.0.73", "syn 1.0.73",
] ]
[[package]]
name = "native-tls"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4"
dependencies = [
"lazy_static",
"libc",
"log",
"openssl",
"openssl-probe",
"openssl-sys",
"schannel",
"security-framework",
"security-framework-sys",
"tempfile",
]
[[package]] [[package]]
name = "nix" name = "nix"
version = "0.19.1" version = "0.19.1"
@ -1864,6 +1929,39 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "openssl"
version = "0.10.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8"
dependencies = [
"bitflags",
"cfg-if 1.0.0",
"foreign-types",
"libc",
"once_cell",
"openssl-sys",
]
[[package]]
name = "openssl-probe"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a"
[[package]]
name = "openssl-sys"
version = "0.9.63"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98"
dependencies = [
"autocfg",
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]] [[package]]
name = "ordered-float" name = "ordered-float"
version = "2.5.1" version = "2.5.1"
@ -2384,11 +2482,13 @@ dependencies = [
"http-body", "http-body",
"hyper", "hyper",
"hyper-rustls", "hyper-rustls",
"hyper-tls",
"ipnet", "ipnet",
"js-sys", "js-sys",
"lazy_static", "lazy_static",
"log", "log",
"mime", "mime",
"native-tls",
"percent-encoding", "percent-encoding",
"pin-project-lite", "pin-project-lite",
"rustls", "rustls",
@ -2396,6 +2496,7 @@ dependencies = [
"serde_json", "serde_json",
"serde_urlencoded", "serde_urlencoded",
"tokio", "tokio",
"tokio-native-tls",
"tokio-rustls", "tokio-rustls",
"url", "url",
"wasm-bindgen", "wasm-bindgen",
@ -2498,6 +2599,16 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "schannel"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
dependencies = [
"lazy_static",
"winapi",
]
[[package]] [[package]]
name = "scopeguard" name = "scopeguard"
version = "1.1.0" version = "1.1.0"
@ -2514,6 +2625,29 @@ dependencies = [
"untrusted", "untrusted",
] ]
[[package]]
name = "security-framework"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467"
dependencies = [
"bitflags",
"core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
]
[[package]]
name = "security-framework-sys"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]] [[package]]
name = "semver" name = "semver"
version = "0.9.0" version = "0.9.0"
@ -3115,6 +3249,16 @@ dependencies = [
"syn 1.0.73", "syn 1.0.73",
] ]
[[package]]
name = "tokio-native-tls"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
dependencies = [
"native-tls",
"tokio",
]
[[package]] [[package]]
name = "tokio-rustls" name = "tokio-rustls"
version = "0.22.0" version = "0.22.0"
@ -3300,6 +3444,12 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "vcpkg"
version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "025ce40a007e1907e58d5bc1a594def78e5573bb0b1160bc389634e8f12e4faa"
[[package]] [[package]]
name = "vec_map" name = "vec_map"
version = "0.8.2" version = "0.8.2"
@ -3462,6 +3612,16 @@ dependencies = [
"hashbrown 0.7.2", "hashbrown 0.7.2",
] ]
[[package]]
name = "whoami"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4abacf325c958dfeaf1046931d37f2a901b6dfe0968ee965a29e94c6766b2af6"
dependencies = [
"wasm-bindgen",
"web-sys",
]
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.3.9" version = "0.3.9"

View File

@ -74,6 +74,8 @@ uuid = { version = "0.8.2", features = ["serde"] }
walkdir = "2.3.2" walkdir = "2.3.2"
obkv = "0.1.1" obkv = "0.1.1"
pin-project = "1.0.7" pin-project = "1.0.7"
whoami = { version = "1.1.2", optional = true }
reqwest = { version = "0.11.3", features = ["json"], optional = true }
[dependencies.sentry] [dependencies.sentry]
default-features = false default-features = false
@ -109,8 +111,8 @@ mini-dashboard = [
"tempfile", "tempfile",
"zip", "zip",
] ]
telemetry = ["sentry"] analytics = ["sentry", "whoami", "reqwest"]
default = ["telemetry", "mini-dashboard"] default = ["analytics", "mini-dashboard"]
[target.'cfg(target_os = "linux")'.dependencies] [target.'cfg(target_os = "linux")'.dependencies]
jemallocator = "0.3.2" jemallocator = "0.3.2"

View File

@ -1,14 +1,10 @@
use std::hash::{Hash, Hasher}; use std::hash::{Hash, Hasher};
use std::{error, thread};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use log::error; use log::debug;
use serde::Serialize; use serde::Serialize;
use serde_qs as qs;
use siphasher::sip::SipHasher; use siphasher::sip::SipHasher;
use walkdir::WalkDir;
use crate::helpers::EnvSizer;
use crate::Data; use crate::Data;
use crate::Opt; use crate::Opt;
@ -22,26 +18,21 @@ struct EventProperties {
} }
impl EventProperties { impl EventProperties {
fn from(data: Data) -> Result<EventProperties, Box<dyn error::Error>> { async fn from(data: Data) -> anyhow::Result<EventProperties> {
let mut index_list = Vec::new(); let stats = data.index_controller.get_all_stats().await?;
let reader = data.db.main_read_txn()?; let database_size = stats.database_size;
let last_update_timestamp = stats.last_update.map(|u| u.timestamp());
for index_uid in data.db.indexes_uids() { let number_of_documents = stats
if let Some(index) = data.db.open_index(&index_uid) { .indexes
let number_of_documents = index.main.number_of_documents(&reader)?; .values()
index_list.push(number_of_documents); .map(|index| index.number_of_documents)
} .collect();
}
let database_size = data.env.size();
let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp());
Ok(EventProperties { Ok(EventProperties {
database_size, database_size,
last_update_timestamp, last_update_timestamp,
number_of_documents: index_list, number_of_documents,
}) })
} }
} }
@ -68,10 +59,10 @@ struct Event<'a> {
#[derive(Debug, Serialize)] #[derive(Debug, Serialize)]
struct AmplitudeRequest<'a> { struct AmplitudeRequest<'a> {
api_key: &'a str, api_key: &'a str,
event: &'a str, events: Vec<Event<'a>>,
} }
pub fn analytics_sender(data: Data, opt: Opt) { pub async fn analytics_sender(data: Data, opt: Opt) {
let username = whoami::username(); let username = whoami::username();
let hostname = whoami::hostname(); let hostname = whoami::hostname();
let platform = whoami::platform(); let platform = whoami::platform();
@ -93,7 +84,7 @@ pub fn analytics_sender(data: Data, opt: Opt) {
let time = n.as_secs(); let time = n.as_secs();
let event_type = "runtime_tick"; let event_type = "runtime_tick";
let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day
let event_properties = EventProperties::from(data.clone()).ok(); let event_properties = EventProperties::from(data.clone()).await.ok();
let app_version = env!("CARGO_PKG_VERSION").to_string(); let app_version = env!("CARGO_PKG_VERSION").to_string();
let app_version = app_version.as_str(); let app_version = app_version.as_str();
let user_email = std::env::var("MEILI_USER_EMAIL").ok(); let user_email = std::env::var("MEILI_USER_EMAIL").ok();
@ -114,20 +105,22 @@ pub fn analytics_sender(data: Data, opt: Opt) {
user_properties, user_properties,
event_properties, event_properties,
}; };
let event = serde_json::to_string(&event).unwrap();
let request = AmplitudeRequest { let request = AmplitudeRequest {
api_key: AMPLITUDE_API_KEY, api_key: AMPLITUDE_API_KEY,
event: &event, events: vec![event],
}; };
let body = qs::to_string(&request).unwrap(); let response = reqwest::Client::new()
let response = ureq::post("https://api.amplitude.com/httpapi").send_string(&body); .post("https://api2.amplitude.com/2/httpapi")
if !response.ok() { .timeout(Duration::from_secs(60)) // 1 minute max
let body = response.into_string().unwrap(); .json(&request)
error!("Unsuccessful call to Amplitude: {}", body); .send()
.await;
if let Err(e) = response {
debug!("Unsuccessful call to Amplitude: {}", e);
} }
thread::sleep(Duration::from_secs(3600)) // one hour tokio::time::sleep(Duration::from_secs(3600)).await;
} }
} }

View File

@ -6,6 +6,9 @@ mod index_controller;
pub mod option; pub mod option;
pub mod routes; pub mod routes;
#[cfg(feature = "analytics")]
pub mod analytics;
pub use self::data::Data; pub use self::data::Data;
pub use option::Opt; pub use option::Opt;

View File

@ -5,12 +5,16 @@ use main_error::MainError;
use meilisearch_http::{create_app, Data, Opt}; use meilisearch_http::{create_app, Data, Opt};
use structopt::StructOpt; use structopt::StructOpt;
//mod analytics; #[cfg(feature = "analytics")]
use meilisearch_http::analytics;
#[cfg(target_os = "linux")] #[cfg(target_os = "linux")]
#[global_allocator] #[global_allocator]
static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
#[cfg(all(not(debug_assertions), feature = "analytics"))]
const SENTRY_DSN: &str = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337";
#[actix_web::main] #[actix_web::main]
async fn main() -> Result<(), MainError> { async fn main() -> Result<(), MainError> {
let opt = Opt::from_args(); let opt = Opt::from_args();
@ -26,16 +30,17 @@ async fn main() -> Result<(), MainError> {
.into(), .into(),
); );
} }
#[cfg(all(not(debug_assertions), feature = "sentry"))] #[cfg(all(not(debug_assertions), feature = "analytics"))]
if !opt.no_sentry { if !opt.no_analytics {
let logger = sentry::integrations::log::SentryLogger::with_dest(log_builder.build()); let logger =
sentry::integrations::log::SentryLogger::with_dest(log_builder.build());
log::set_boxed_logger(Box::new(logger)) log::set_boxed_logger(Box::new(logger))
.map(|()| log::set_max_level(log::LevelFilter::Info)) .map(|()| log::set_max_level(log::LevelFilter::Info))
.unwrap(); .unwrap();
let sentry = sentry::init(sentry::ClientOptions { let sentry = sentry::init(sentry::ClientOptions {
release: sentry::release_name!(), release: sentry::release_name!(),
dsn: Some(opt.sentry_dsn.parse()?), dsn: Some(SENTRY_DSN.parse()?),
..Default::default() ..Default::default()
}); });
// sentry must stay alive as long as possible // sentry must stay alive as long as possible
@ -50,25 +55,14 @@ async fn main() -> Result<(), MainError> {
_ => unreachable!(), _ => unreachable!(),
} }
//if let Some(path) = &opt.import_snapshot {
//snapshot::load_snapshot(&opt.db_path, path, opt.ignore_snapshot_if_db_exists, opt.ignore_missing_snapshot)?;
//}
let data = Data::new(opt.clone())?; let data = Data::new(opt.clone())?;
//if !opt.no_analytics { #[cfg(feature = "analytics")]
//let analytics_data = data.clone(); if !opt.no_analytics {
//let analytics_opt = opt.clone(); let analytics_data = data.clone();
//thread::spawn(move || analytics::analytics_sender(analytics_data, analytics_opt)); let analytics_opt = opt.clone();
//} tokio::task::spawn(analytics::analytics_sender(analytics_data, analytics_opt));
}
//if let Some(path) = &opt.import_dump {
//dump::import_dump(&data, path, opt.dump_batch_size)?;
//}
//if opt.schedule_snapshot {
//snapshot::schedule_snapshot(data.clone(), &opt.snapshot_dir, opt.snapshot_interval_sec.unwrap_or(86400))?;
//}
print_launch_resume(&opt, &data); print_launch_resume(&opt, &data);
@ -127,24 +121,21 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) {
env!("CARGO_PKG_VERSION").to_string() env!("CARGO_PKG_VERSION").to_string()
); );
#[cfg(all(not(debug_assertions), feature = "sentry"))] #[cfg(feature = "analytics")]
eprintln!( {
"Sentry DSN:\t\t{:?}", if opt.no_analytics {
if !opt.no_sentry { eprintln!("Anonymous telemetry:\t\"Disabled\"");
&opt.sentry_dsn
} else { } else {
"Disabled" eprintln!(
} "
); Thank you for using MeiliSearch!
We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/reference/features/configuration.html#analytics
eprintln!( Anonymous telemetry: \"Enabled\"
"Anonymous telemetry:\t{:?}", "
if !opt.no_analytics { );
"Enabled"
} else {
"Disabled"
} }
); }
eprintln!(); eprintln!();

View File

@ -96,21 +96,6 @@ pub struct Opt {
#[structopt(long, env = "MEILI_MASTER_KEY")] #[structopt(long, env = "MEILI_MASTER_KEY")]
pub master_key: Option<String>, pub master_key: Option<String>,
/// The Sentry DSN to use for error reporting. This defaults to the MeiliSearch Sentry project.
/// You can disable sentry all together using the `--no-sentry` flag or `MEILI_NO_SENTRY` environment variable.
#[cfg(all(not(debug_assertions), feature = "sentry"))]
#[structopt(
long,
env = "SENTRY_DSN",
default_value = "https://5ddfa22b95f241198be2271aaf028653@sentry.io/3060337"
)]
pub sentry_dsn: String,
/// Disable Sentry error reporting.
#[structopt(long, env = "MEILI_NO_SENTRY")]
#[cfg(all(not(debug_assertions), feature = "sentry"))]
pub no_sentry: bool,
/// This environment variable must be set to `production` if you are running in production. /// This environment variable must be set to `production` if you are running in production.
/// If the server is running in development mode more logs will be displayed, /// If the server is running in development mode more logs will be displayed,
/// and the master key can be avoided which implies that there is no security on the updates routes. /// and the master key can be avoided which implies that there is no security on the updates routes.
@ -119,6 +104,7 @@ pub struct Opt {
pub env: String, pub env: String,
/// Do not send analytics to Meili. /// Do not send analytics to Meili.
#[cfg(feature = "analytics")]
#[structopt(long, env = "MEILI_NO_ANALYTICS")] #[structopt(long, env = "MEILI_NO_ANALYTICS")]
pub no_analytics: bool, pub no_analytics: bool,

View File

@ -71,6 +71,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
http_addr: "127.0.0.1:7700".to_owned(), http_addr: "127.0.0.1:7700".to_owned(),
master_key: None, master_key: None,
env: "development".to_owned(), env: "development".to_owned(),
#[cfg(feature = "analytics")]
no_analytics: true, no_analytics: true,
max_mdb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), max_mdb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
max_udb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(), max_udb_size: Byte::from_unit(4.0, ByteUnit::GiB).unwrap(),
@ -90,9 +91,5 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
snapshot_interval_sec: 0, snapshot_interval_sec: 0,
import_dump: None, import_dump: None,
indexer_options: IndexerOpts::default(), indexer_options: IndexerOpts::default(),
#[cfg(all(not(debug_assertions), feature = "sentry"))]
sentry_dsn: String::from(""),
#[cfg(all(not(debug_assertions), feature = "sentry"))]
no_sentry: true,
} }
} }