searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 20:37:15 +02:00
Code Issues Releases Wiki Activity

Logging the memory usage over time

Browse source
This commit is contained in:
Clément Renault 2024-01-18 18:14:47 +01:00 • committed by Louis Dureuil
parent 7e47cea0c4
commit 771861599b
No known key found for this signature in database
8 changed files with 222 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

99
tracing-trace/src/processor/firefox_profiler.rs
View file

@ -5,10 +5,11 @@ use fxprof_processed_profile::{
MarkerFieldFormat, MarkerLocation, MarkerSchema, MarkerSchemaField, Profile, ProfilerMarker,
ReferenceTimestamp, SamplingInterval, StringHandle, Timestamp,
};
use once_cell::unsync::Lazy;
use serde_json::json;
use crate::entry::{
Entry, NewCallsite, NewSpan, ResourceId, SpanClose, SpanEnter, SpanExit, SpanId,
Entry, MemoryStats, NewCallsite, NewSpan, ResourceId, SpanClose, SpanEnter, SpanExit, SpanId,
};
use crate::{Error, TraceReader};
@ -33,6 +34,16 @@ pub fn to_firefox_profile<R: std::io::Read>(
let subcategory = profile.add_subcategory(category, "subcategory");
// TODO kero: add counters profile.add_counters + last_memory_value
let mut current_memory = MemoryStats::default();
let mut allocations_counter = Lazy::new(|| {
profile.add_counter(main, "mimmalloc", "Memory", "Amount of allocation calls")
});
let mut deallocations_counter = Lazy::new(|| {
profile.add_counter(main, "mimmalloc", "Memory", "Amount of deallocation calls")
});
let mut reallocations_counter = Lazy::new(|| {
profile.add_counter(main, "mimmalloc", "Memory", "Amount of reallocation calls")
});
for entry in trace {
let entry = entry?;
@ -56,7 +67,7 @@ pub fn to_firefox_profile<R: std::io::Read>(
Entry::NewSpan(span) => {
spans.insert(span.id, (span, SpanStatus::Outside));
}
Entry::SpanEnter(SpanEnter { id, time }) => {
Entry::SpanEnter(SpanEnter { id, time, memory }) => {
let (_span, status) = spans.get_mut(&id).unwrap();
let SpanStatus::Outside = status else {
@ -67,16 +78,41 @@ pub fn to_firefox_profile<R: std::io::Read>(
last_timestamp = Timestamp::from_nanos_since_reference(time.as_nanos() as u64);
/* TODO kero: compute delta and update them
profile.add_counter_sample(
counter,
timestamp,
value_delta,
number_of_operations_delta,
)
*/
if let Some(stats) = memory {
let MemoryStats {
allocations,
deallocations,
reallocations,
bytes_allocated,
bytes_deallocated,
bytes_reallocated,
} = current_memory - stats;
profile.add_counter_sample(
*allocations_counter,
last_timestamp,
bytes_allocated as f64,
allocations.try_into().unwrap(),
);
profile.add_counter_sample(
*deallocations_counter,
last_timestamp,
bytes_deallocated as f64,
deallocations.try_into().unwrap(),
);
profile.add_counter_sample(
*reallocations_counter,
last_timestamp,
bytes_reallocated as f64,
reallocations.try_into().unwrap(),
);
current_memory = stats;
}
}
Entry::SpanExit(SpanExit { id, time }) => {
Entry::SpanExit(SpanExit { id, time, memory }) => {
let (span, status) = spans.get_mut(&id).unwrap();
let SpanStatus::Inside(begin) = status else {
@ -108,14 +144,39 @@ pub fn to_firefox_profile<R: std::io::Read>(
1,
);
/* TODO kero: compute delta and update them
profile.add_counter_sample(
counter,
timestamp,
value_delta,
number_of_operations_delta,
)
*/
if let Some(stats) = memory {
let MemoryStats {
allocations,
deallocations,
reallocations,
bytes_allocated,
bytes_deallocated,
bytes_reallocated,
} = current_memory - stats;
profile.add_counter_sample(
*allocations_counter,
last_timestamp,
bytes_allocated as f64,
allocations.try_into().unwrap(),
);
profile.add_counter_sample(
*deallocations_counter,
last_timestamp,
bytes_deallocated as f64,
deallocations.try_into().unwrap(),
);
profile.add_counter_sample(
*reallocations_counter,
last_timestamp,
bytes_reallocated as f64,
reallocations.try_into().unwrap(),
);
current_memory = stats;
}
let (callsite, _) = calls.get(&span.call_id).unwrap();

56
tracing-trace/src/processor/fmt.rs
View file

@ -2,7 +2,8 @@ use std::collections::HashMap;
use std::io::Read;
use crate::entry::{
Entry, NewCallsite, NewSpan, NewThread, ResourceId, SpanClose, SpanEnter, SpanExit, SpanId,
Entry, MemoryStats, NewCallsite, NewSpan, NewThread, ResourceId, SpanClose, SpanEnter,
SpanExit, SpanId,
};
use crate::{Error, TraceReader};
@ -28,7 +29,7 @@ pub fn print_trace<R: Read>(trace: TraceReader<R>) -> Result<(), Error> {
Entry::NewSpan(span) => {
spans.insert(span.id, (span, SpanStatus::Outside));
}
Entry::SpanEnter(SpanEnter { id, time }) => {
Entry::SpanEnter(SpanEnter { id, time, memory }) => {
let (span, status) = spans.get_mut(&id).unwrap();
let SpanStatus::Outside = status else {
@ -39,14 +40,23 @@ pub fn print_trace<R: Read>(trace: TraceReader<R>) -> Result<(), Error> {
let span = *span;
println!(
"[{}]{}::{} <-",
print_thread(&threads, span.thread_id),
print_backtrace(&spans, &calls, &span),
print_span(&calls, &span)
);
match memory {
Some(stats) => println!(
"[{}]{}::{} ({}) <-",
print_thread(&threads, span.thread_id),
print_backtrace(&spans, &calls, &span),
print_span(&calls, &span),
print_memory(stats),
),
None => println!(
"[{}]{}::{} <-",
print_thread(&threads, span.thread_id),
print_backtrace(&spans, &calls, &span),
print_span(&calls, &span),
),
}
}
Entry::SpanExit(SpanExit { id, time }) => {
Entry::SpanExit(SpanExit { id, time, memory }) => {
let (span, status) = spans.get_mut(&id).unwrap();
let SpanStatus::Inside(begin) = status else {
@ -58,13 +68,23 @@ pub fn print_trace<R: Read>(trace: TraceReader<R>) -> Result<(), Error> {
let span = *span;
println!(
"[{}]{}::{} -> {}",
print_thread(&threads, span.thread_id),
print_backtrace(&spans, &calls, &span),
print_span(&calls, &span),
print_duration(time - begin),
)
match memory {
Some(stats) => println!(
"[{}]{}::{} ({}) -> {}",
print_thread(&threads, span.thread_id),
print_backtrace(&spans, &calls, &span),
print_span(&calls, &span),
print_memory(stats),
print_duration(time - begin),
),
None => println!(
"[{}]{}::{} -> {}",
print_thread(&threads, span.thread_id),
print_backtrace(&spans, &calls, &span),
print_span(&calls, &span),
print_duration(time - begin),
),
}
}
Entry::SpanClose(SpanClose { id, time: _ }) => {
spans.remove(&id);
@ -126,3 +146,7 @@ fn print_duration(duration: std::time::Duration) -> String {
format!("{}d", duration.as_secs_f64() / 3600.0 / 24.0)
}
}
fn print_memory(memory: MemoryStats) -> String {
// Format only the total allocations in GiB, MiB, KiB, Bytes
}