Merge pull request #718 from meilisearch/add-more-analytics-reporting

Add more analytics
This commit is contained in:
Clément Renault 2020-06-02 17:05:09 +02:00 committed by GitHub
commit adaf74bc87
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 93 additions and 20 deletions

View File

@ -20,6 +20,7 @@
- Rename fieldsFrequency into fieldsDistribution in stats (#719)
- Add support for error code reporting (#703)
- Allow the dashboard to query private servers (#732)
- Add telemetry (#720)
## v0.10.1

View File

@ -176,8 +176,8 @@ Hey! We're glad you're thinking about contributing to MeiliSearch! If you think
### Analytic Events
Once a day, events are being sent to our Amplitude instance so we can know how many people are using MeiliSearch.<br/>
Only information about the platform on which the server runs is stored. No other information is being sent.<br/>
Every hour, events are being sent to our Amplitude instance so we can know how many people are using MeiliSearch.<br/>
To see what information we're retrieving, please see the complete list [on the dedicated issue](https://github.com/meilisearch/MeiliSearch/issues/720).<br/>
We also use Sentry to make us crash and error reports. If you want to know more about what Sentry collects, please visit their [privacy policy website](https://sentry.io/privacy/).<br/>
If this doesn't suit you, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.

View File

@ -1,20 +1,72 @@
use std::hash::{Hash, Hasher};
use std::thread;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use std::{error, thread};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use log::error;
use serde::Serialize;
use serde_qs as qs;
use siphasher::sip::SipHasher;
use walkdir::WalkDir;
use crate::Data;
use crate::Opt;
const AMPLITUDE_API_KEY: &str = "f7fba398780e06d8fe6666a9be7e3d47";
#[derive(Debug, Serialize)]
struct EventProperties {
database_size: u64,
last_update_timestamp: Option<i64>, //timestamp
number_of_documents: Vec<u64>,
}
impl EventProperties {
fn from(data: Data) -> Result<EventProperties, Box<dyn error::Error>> {
let mut index_list = Vec::new();
let reader = data.db.main_read_txn()?;
for index_uid in data.db.indexes_uids() {
if let Some(index) = data.db.open_index(&index_uid) {
let number_of_documents = index.main.number_of_documents(&reader)?;
index_list.push(number_of_documents);
}
}
let database_size = WalkDir::new(&data.db_path)
.into_iter()
.filter_map(|entry| entry.ok())
.filter_map(|entry| entry.metadata().ok())
.filter(|metadata| metadata.is_file())
.fold(0, |acc, m| acc + m.len());
let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp());
Ok(EventProperties {
database_size,
last_update_timestamp,
number_of_documents: index_list,
})
}
}
#[derive(Debug, Serialize)]
struct UserProperties<'a> {
env: &'a str,
start_since_days: u64,
user_email: Option<String>,
server_provider: Option<String>,
}
#[derive(Debug, Serialize)]
struct Event<'a> {
user_id: &'a str,
event_type: &'a str,
device_id: &'a str,
time: u64,
app_version: &'a str,
user_properties: UserProperties<'a>,
event_properties: Option<EventProperties>,
}
#[derive(Debug, Serialize)]
@ -23,7 +75,7 @@ struct AmplitudeRequest<'a> {
event: &'a str,
}
pub fn analytics_sender() {
pub fn analytics_sender(data: Data, opt: Opt) {
let username = whoami::username();
let hostname = whoami::hostname();
let platform = whoami::platform();
@ -36,6 +88,7 @@ pub fn analytics_sender() {
let uid = format!("{:X}", hash);
let platform = platform.to_string();
let first_start = Instant::now();
loop {
let n = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
@ -43,12 +96,27 @@ pub fn analytics_sender() {
let device_id = &platform;
let time = n.as_secs();
let event_type = "runtime_tick";
let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day
let event_properties = EventProperties::from(data.clone()).ok();
let app_version = env!("CARGO_PKG_VERSION").to_string();
let app_version = app_version.as_str();
let user_email = std::env::var("MEILI_USER_EMAIL").ok();
let server_provider = std::env::var("MEILI_SERVER_PROVIDER").ok();
let user_properties = UserProperties {
env: &opt.env,
start_since_days: elapsed_since_start,
user_email,
server_provider,
};
let event = Event {
user_id,
event_type,
device_id,
time,
app_version,
user_properties,
event_properties
};
let event = serde_json::to_string(&event).unwrap();
@ -64,6 +132,6 @@ pub fn analytics_sender() {
error!("Unsuccessful call to Amplitude: {}", body);
}
thread::sleep(Duration::from_secs(86_400)) // one day
thread::sleep(Duration::from_secs(3600)) // one hour
}
}

View File

@ -6,6 +6,7 @@ pub mod helpers;
pub mod models;
pub mod option;
pub mod routes;
pub mod analytics;
use actix_http::Error;
use actix_service::ServiceFactory;
@ -15,6 +16,7 @@ use log::error;
use meilisearch_core::ProcessedUpdateResult;
pub use option::Opt;
pub use self::data::Data;
use self::error::{json_error_handler, ResponseError};

View File

@ -3,10 +3,8 @@ use std::{env, thread};
use actix_cors::Cors;
use actix_web::{middleware, HttpServer};
use main_error::MainError;
use meilisearch_http::data::Data;
use meilisearch_http::helpers::NormalizePath;
use meilisearch_http::option::Opt;
use meilisearch_http::{create_app, index_update_callback};
use meilisearch_http::{Data, Opt, create_app, index_update_callback};
use structopt::StructOpt;
mod analytics;
@ -49,12 +47,16 @@ async fn main() -> Result<(), MainError> {
_ => unreachable!(),
}
if !opt.no_analytics {
thread::spawn(analytics::analytics_sender);
}
let data = Data::new(opt.clone());
if !opt.no_analytics {
let analytics_data = data.clone();
let analytics_opt = opt.clone();
thread::spawn(move|| {
analytics::analytics_sender(analytics_data, analytics_opt)
});
}
let data_cloned = data.clone();
data.db.set_update_callback(Box::new(move |name, status| {
index_update_callback(name, &data_cloned, status);