Merge pull request #718 from meilisearch/add-more-analytics-reporting

Add more analytics
This commit is contained in:
Clément Renault 2020-06-02 17:05:09 +02:00 committed by GitHub
commit adaf74bc87
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 93 additions and 20 deletions

View File

@ -20,6 +20,7 @@
- Rename fieldsFrequency into fieldsDistribution in stats (#719)
- Add support for error code reporting (#703)
- Allow the dashboard to query private servers (#732)
- Add telemetry (#720)
## v0.10.1

View File

@ -6,11 +6,11 @@
<h1 align="center">MeiliSearch</h1>
<h4 align="center">
<a href="https://www.meilisearch.com">Website</a> |
<a href="https://blog.meilisearch.com">Blog</a> |
<a href="https://fr.linkedin.com/company/meilisearch">LinkedIn</a> |
<a href="https://twitter.com/meilisearch">Twitter</a> |
<a href="https://docs.meilisearch.com">Documentation</a> |
<a href="https://www.meilisearch.com">Website</a> |
<a href="https://blog.meilisearch.com">Blog</a> |
<a href="https://fr.linkedin.com/company/meilisearch">LinkedIn</a> |
<a href="https://twitter.com/meilisearch">Twitter</a> |
<a href="https://docs.meilisearch.com">Documentation</a> |
<a href="https://docs.meilisearch.com/faq/">FAQ</a>
</h4>
@ -122,7 +122,7 @@ curl -i -X POST 'http://127.0.0.1:7700/indexes/movies/documents' \
#### In command line
The search engine is now aware of your documents and can serve those via an HTTP server.
The search engine is now aware of your documents and can serve those via an HTTP server.
The [`jq` command-line tool](https://stedolan.github.io/jq/) can greatly help you read the server responses.
@ -167,7 +167,7 @@ You can access the web interface in your web browser at the root of the server.
### Documentation
Now that your MeiliSearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com).
Now that your MeiliSearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com).
## Contributing
@ -176,8 +176,8 @@ Hey! We're glad you're thinking about contributing to MeiliSearch! If you think
### Analytic Events
Once a day, events are being sent to our Amplitude instance so we can know how many people are using MeiliSearch.<br/>
Only information about the platform on which the server runs is stored. No other information is being sent.<br/>
Every hour, events are being sent to our Amplitude instance so we can know how many people are using MeiliSearch.<br/>
To see what information we're retrieving, please see the complete list [on the dedicated issue](https://github.com/meilisearch/MeiliSearch/issues/720).<br/>
We also use Sentry to make us crash and error reports. If you want to know more about what Sentry collects, please visit their [privacy policy website](https://sentry.io/privacy/).<br/>
If this doesn't suit you, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.

View File

@ -1,20 +1,72 @@
use std::hash::{Hash, Hasher};
use std::thread;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use std::{error, thread};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use log::error;
use serde::Serialize;
use serde_qs as qs;
use siphasher::sip::SipHasher;
use walkdir::WalkDir;
use crate::Data;
use crate::Opt;
const AMPLITUDE_API_KEY: &str = "f7fba398780e06d8fe6666a9be7e3d47";
#[derive(Debug, Serialize)]
struct EventProperties {
database_size: u64,
last_update_timestamp: Option<i64>, //timestamp
number_of_documents: Vec<u64>,
}
impl EventProperties {
fn from(data: Data) -> Result<EventProperties, Box<dyn error::Error>> {
let mut index_list = Vec::new();
let reader = data.db.main_read_txn()?;
for index_uid in data.db.indexes_uids() {
if let Some(index) = data.db.open_index(&index_uid) {
let number_of_documents = index.main.number_of_documents(&reader)?;
index_list.push(number_of_documents);
}
}
let database_size = WalkDir::new(&data.db_path)
.into_iter()
.filter_map(|entry| entry.ok())
.filter_map(|entry| entry.metadata().ok())
.filter(|metadata| metadata.is_file())
.fold(0, |acc, m| acc + m.len());
let last_update_timestamp = data.db.last_update(&reader)?.map(|u| u.timestamp());
Ok(EventProperties {
database_size,
last_update_timestamp,
number_of_documents: index_list,
})
}
}
#[derive(Debug, Serialize)]
struct UserProperties<'a> {
env: &'a str,
start_since_days: u64,
user_email: Option<String>,
server_provider: Option<String>,
}
#[derive(Debug, Serialize)]
struct Event<'a> {
user_id: &'a str,
event_type: &'a str,
device_id: &'a str,
time: u64,
app_version: &'a str,
user_properties: UserProperties<'a>,
event_properties: Option<EventProperties>,
}
#[derive(Debug, Serialize)]
@ -23,7 +75,7 @@ struct AmplitudeRequest<'a> {
event: &'a str,
}
pub fn analytics_sender() {
pub fn analytics_sender(data: Data, opt: Opt) {
let username = whoami::username();
let hostname = whoami::hostname();
let platform = whoami::platform();
@ -36,6 +88,7 @@ pub fn analytics_sender() {
let uid = format!("{:X}", hash);
let platform = platform.to_string();
let first_start = Instant::now();
loop {
let n = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
@ -43,12 +96,27 @@ pub fn analytics_sender() {
let device_id = &platform;
let time = n.as_secs();
let event_type = "runtime_tick";
let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day
let event_properties = EventProperties::from(data.clone()).ok();
let app_version = env!("CARGO_PKG_VERSION").to_string();
let app_version = app_version.as_str();
let user_email = std::env::var("MEILI_USER_EMAIL").ok();
let server_provider = std::env::var("MEILI_SERVER_PROVIDER").ok();
let user_properties = UserProperties {
env: &opt.env,
start_since_days: elapsed_since_start,
user_email,
server_provider,
};
let event = Event {
user_id,
event_type,
device_id,
time,
app_version,
user_properties,
event_properties
};
let event = serde_json::to_string(&event).unwrap();
@ -64,6 +132,6 @@ pub fn analytics_sender() {
error!("Unsuccessful call to Amplitude: {}", body);
}
thread::sleep(Duration::from_secs(86_400)) // one day
thread::sleep(Duration::from_secs(3600)) // one hour
}
}

View File

@ -6,6 +6,7 @@ pub mod helpers;
pub mod models;
pub mod option;
pub mod routes;
pub mod analytics;
use actix_http::Error;
use actix_service::ServiceFactory;
@ -15,6 +16,7 @@ use log::error;
use meilisearch_core::ProcessedUpdateResult;
pub use option::Opt;
pub use self::data::Data;
use self::error::{json_error_handler, ResponseError};

View File

@ -3,10 +3,8 @@ use std::{env, thread};
use actix_cors::Cors;
use actix_web::{middleware, HttpServer};
use main_error::MainError;
use meilisearch_http::data::Data;
use meilisearch_http::helpers::NormalizePath;
use meilisearch_http::option::Opt;
use meilisearch_http::{create_app, index_update_callback};
use meilisearch_http::{Data, Opt, create_app, index_update_callback};
use structopt::StructOpt;
mod analytics;
@ -49,12 +47,16 @@ async fn main() -> Result<(), MainError> {
_ => unreachable!(),
}
if !opt.no_analytics {
thread::spawn(analytics::analytics_sender);
}
let data = Data::new(opt.clone());
if !opt.no_analytics {
let analytics_data = data.clone();
let analytics_opt = opt.clone();
thread::spawn(move|| {
analytics::analytics_sender(analytics_data, analytics_opt)
});
}
let data_cloned = data.clone();
data.db.set_update_callback(Box::new(move |name, status| {
index_update_callback(name, &data_cloned, status);