searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 04:17:10 +02:00
Code Issues Releases Wiki Activity

Replace stats_alloc with procfs

Browse source
This commit is contained in:
Clément Renault 2024-02-06 14:41:14 +01:00 • committed by Louis Dureuil
parent e773dfa9ba
commit b393823f36
No known key found for this signature in database
9 changed files with 99 additions and 192 deletions
Download patch file Download diff file Expand all files Collapse all files

74
tracing-trace/src/entry.rs
View file

@ -101,58 +101,46 @@ pub struct SpanClose {
}
/// A struct with a lot of memory allocation stats akin
/// to the `stats_alloc::Stats` one but implements the
/// `Serialize/Deserialize` serde traits.
/// to the `procfs::Process::StatsM` one plus the OOM score.
///
/// Note that all the values are in bytes not in pages.
#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)]
pub struct MemoryStats {
pub allocations: usize,
pub deallocations: usize,
pub reallocations: usize,
pub bytes_allocated: usize,
pub bytes_deallocated: usize,
pub bytes_reallocated: isize,
}
impl From<stats_alloc::Stats> for MemoryStats {
fn from(stats: stats_alloc::Stats) -> Self {
let stats_alloc::Stats {
allocations,
deallocations,
reallocations,
bytes_allocated,
bytes_deallocated,
bytes_reallocated,
} = stats;
MemoryStats {
allocations,
deallocations,
reallocations,
bytes_allocated,
bytes_deallocated,
bytes_reallocated,
}
}
/// Resident set size, measured in bytes.
/// (same as VmRSS in /proc/<pid>/status).
pub resident: u64,
/// Number of resident shared bytes (i.e., backed by a file).
/// (same as RssFile+RssShmem in /proc/<pid>/status).
pub shared: u64,
/// The current score that the kernel gives to this process
/// for the purpose of selecting a process for the OOM-killer
///
/// A higher score means that the process is more likely to be selected
/// by the OOM-killer. The basis for this score is the amount of memory used
/// by the process, plus other factors.
///
/// (Since linux 2.6.11)
pub oom_score: u32,
}
impl MemoryStats {
#[cfg(target_os = "linux")]
pub fn fetch() -> procfs::ProcResult<Self> {
let process = procfs::process::Process::myself().unwrap();
let procfs::process::StatM { resident, shared, .. } = process.statm()?;
let oom_score = process.oom_score()?;
let page_size = procfs::page_size();
Ok(MemoryStats { resident: resident * page_size, shared: shared * page_size, oom_score })
}
pub fn checked_sub(self, other: Self) -> Option<Self> {
Some(Self {
allocations: self.allocations.checked_sub(other.allocations)?,
deallocations: self.deallocations.checked_sub(other.deallocations)?,
reallocations: self.reallocations.checked_sub(other.reallocations)?,
bytes_allocated: self.bytes_allocated.checked_sub(other.bytes_allocated)?,
bytes_deallocated: self.bytes_deallocated.checked_sub(other.bytes_deallocated)?,
bytes_reallocated: self.bytes_reallocated.checked_sub(other.bytes_reallocated)?,
resident: self.resident.checked_sub(other.resident)?,
shared: self.shared.checked_sub(other.shared)?,
oom_score: self.oom_score.checked_sub(other.oom_score)?,
})
}
pub fn usage(&self) -> isize {
(self.bytes_allocated - self.bytes_deallocated) as isize + self.bytes_reallocated
}
pub fn operations(&self) -> usize {
self.allocations + self.deallocations + self.reallocations
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]

45
tracing-trace/src/layer.rs
View file

@ -1,11 +1,9 @@
use std::alloc::{GlobalAlloc, System};
use std::borrow::Cow;
use std::collections::HashMap;
use std::io::Write;
use std::ops::ControlFlow;
use std::sync::RwLock;
use stats_alloc::StatsAlloc;
use tracing::span::{Attributes, Id as TracingId};
use tracing::{Metadata, Subscriber};
use tracing_subscriber::layer::Context;
@ -18,55 +16,31 @@ use crate::entry::{
use crate::{Error, Trace, TraceWriter};
/// Layer that measures the time spent in spans.
pub struct TraceLayer<A: GlobalAlloc + 'static = System> {
pub struct TraceLayer {
sender: tokio::sync::mpsc::UnboundedSender<Entry>,
callsites: RwLock<HashMap<OpaqueIdentifier, ResourceId>>,
start_time: std::time::Instant,
memory_allocator: Option<&'static StatsAlloc<A>>,
}
impl Trace {
pub fn new() -> (Self, TraceLayer<System>) {
pub fn new() -> (Self, TraceLayer) {
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
let trace = Trace { receiver };
let layer = TraceLayer {
sender,
callsites: Default::default(),
start_time: std::time::Instant::now(),
memory_allocator: None,
};
(trace, layer)
}
pub fn with_stats_alloc<A: GlobalAlloc>(
stats_alloc: &'static StatsAlloc<A>,
) -> (Self, TraceLayer<A>) {
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
let trace = Trace { receiver };
let layer = TraceLayer {
sender,
callsites: Default::default(),
start_time: std::time::Instant::now(),
memory_allocator: Some(stats_alloc),
};
(trace, layer)
}
}
impl<W: Write> TraceWriter<W> {
pub fn new(writer: W) -> (Self, TraceLayer<System>) {
pub fn new(writer: W) -> (Self, TraceLayer) {
let (trace, layer) = Trace::new();
(trace.into_writer(writer), layer)
}
pub fn with_stats_alloc<A: GlobalAlloc>(
writer: W,
stats_alloc: &'static StatsAlloc<A>,
) -> (Self, TraceLayer<A>) {
let (trace, layer) = Trace::with_stats_alloc(stats_alloc);
(trace.into_writer(writer), layer)
}
pub async fn receive(&mut self) -> Result<ControlFlow<(), ()>, Error> {
let Some(entry) = self.receiver.recv().await else {
return Ok(ControlFlow::Break(()));
@ -107,7 +81,7 @@ enum OpaqueIdentifier {
Call(tracing::callsite::Identifier),
}
impl<A: GlobalAlloc> TraceLayer<A> {
impl TraceLayer {
fn resource_id(&self, opaque: OpaqueIdentifier) -> Option<ResourceId> {
self.callsites.read().unwrap().get(&opaque).copied()
}
@ -122,8 +96,14 @@ impl<A: GlobalAlloc> TraceLayer<A> {
self.start_time.elapsed()
}
#[cfg(target_os = "linux")]
fn memory_stats(&self) -> Option<MemoryStats> {
self.memory_allocator.map(|ma| ma.stats().into())
Some(MemoryStats::fetch().unwrap())
}
#[cfg(not(target_os = "linux"))]
fn memory_stats(&self) -> Option<MemoryStats> {
None
}
fn send(&self, entry: Entry) {
@ -160,10 +140,9 @@ impl<A: GlobalAlloc> TraceLayer<A> {
}
}
impl<S, A> Layer<S> for TraceLayer<A>
impl<S> Layer<S> for TraceLayer
where
S: Subscriber,
A: GlobalAlloc,
{
fn on_new_span(&self, attrs: &Attributes<'_>, id: &TracingId, _ctx: Context<'_, S>) {
let call_id = self

106
tracing-trace/src/processor/firefox_profiler.rs
View file

@ -227,8 +227,8 @@ fn add_memory_samples(
profile.add_counter_sample(
memory_counters.usage,
last_timestamp,
stats.usage() as f64 - last_memory.usage() as f64,
stats.operations().checked_sub(last_memory.operations()).unwrap_or_default() as u32,
stats.resident as f64 - last_memory.resident as f64,
0,
);
let delta = stats.checked_sub(*last_memory);
@ -317,39 +317,21 @@ impl<'a> ProfilerMarker for SpanMarker<'a> {
searchable: true,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "allocations",
label: "Number of allocation operations while this function was executing",
format: MarkerFieldFormat::Integer,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "deallocations",
label: "Number of deallocation operations while this function was executing",
format: MarkerFieldFormat::Integer,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "reallocations",
label: "Number of reallocation operations while this function was executing",
format: MarkerFieldFormat::Integer,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "allocated_bytes",
label: "Number of allocated bytes while this function was executing",
key: "resident",
label: "Resident set size, measured in bytes while this function was executing",
format: MarkerFieldFormat::Bytes,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "deallocated_bytes",
label: "Number of deallocated bytes while this function was executing",
key: "shared",
label: "Number of resident shared pages (i.e., backed by a file) while this function was executing",
format: MarkerFieldFormat::Bytes,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "reallocated_bytes",
label: "Number of reallocated bytes while this function was executing",
format: MarkerFieldFormat::Bytes,
key: "oom_score",
label: "The current score that the kernel gives to this process for the purpose of selecting a process for the OOM-killer while this function was executing",
format: MarkerFieldFormat::Integer,
searchable: false,
}),
];
@ -384,21 +366,10 @@ impl<'a> ProfilerMarker for SpanMarker<'a> {
"thread_id": thread_id,
});
if let Some(MemoryStats {
allocations,
deallocations,
reallocations,
bytes_allocated,
bytes_deallocated,
bytes_reallocated,
}) = self.memory_delta
{
value["allocations"] = json!(allocations);
value["deallocations"] = json!(deallocations);
value["reallocations"] = json!(reallocations);
value["allocated_bytes"] = json!(bytes_allocated);
value["deallocated_bytes"] = json!(bytes_deallocated);
value["reallocated_bytes"] = json!(bytes_reallocated);
if let Some(MemoryStats { resident, shared, oom_score }) = self.memory_delta {
value["resident"] = json!(resident);
value["shared"] = json!(shared);
value["oom_score"] = json!(oom_score);
}
value
@ -447,39 +418,21 @@ impl<'a> ProfilerMarker for EventMarker<'a> {
searchable: true,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "allocations",
label: "Number of allocation operations since last measure",
format: MarkerFieldFormat::Integer,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "deallocations",
label: "Number of deallocation operations since last measure",
format: MarkerFieldFormat::Integer,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "reallocations",
label: "Number of reallocation operations since last measure",
format: MarkerFieldFormat::Integer,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "allocated_bytes",
label: "Number of allocated bytes since last measure",
key: "resident",
label: "Resident set size, measured in bytes while this function was executing",
format: MarkerFieldFormat::Bytes,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "deallocated_bytes",
label: "Number of deallocated bytes since last measure",
key: "shared",
label: "Number of resident shared pages (i.e., backed by a file) while this function was executing",
format: MarkerFieldFormat::Bytes,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "reallocated_bytes",
label: "Number of reallocated bytes since last measure",
format: MarkerFieldFormat::Bytes,
key: "oom_score",
label: "The current score that the kernel gives to this process for the purpose of selecting a process for the OOM-killer while this function was executing",
format: MarkerFieldFormat::Integer,
searchable: false,
}),
];
@ -514,21 +467,10 @@ impl<'a> ProfilerMarker for EventMarker<'a> {
"thread_id": thread_id,
});
if let Some(MemoryStats {
allocations,
deallocations,
reallocations,
bytes_allocated,
bytes_deallocated,
bytes_reallocated,
}) = self.memory_delta
{
value["allocations"] = json!(allocations);
value["deallocations"] = json!(deallocations);
value["reallocations"] = json!(reallocations);
value["allocated_bytes"] = json!(bytes_allocated);
value["deallocated_bytes"] = json!(bytes_deallocated);
value["reallocated_bytes"] = json!(bytes_reallocated);
if let Some(MemoryStats { resident, shared, oom_score }) = self.memory_delta {
value["resident"] = json!(resident);
value["shared"] = json!(shared);
value["oom_score"] = json!(oom_score);
}
value

22
tracing-trace/src/processor/fmt.rs
View file

@ -188,23 +188,9 @@ fn print_duration(duration: std::time::Duration) -> String {
}
/// Format only the allocated bytes, deallocated bytes and reallocated bytes in GiB, MiB, KiB, Bytes.
fn print_memory(memory: MemoryStats) -> String {
fn print_memory(MemoryStats { resident, shared, oom_score }: MemoryStats) -> String {
use byte_unit::Byte;
let allocated_bytes = Byte::from_bytes(memory.bytes_allocated.try_into().unwrap());
let deallocated_bytes = Byte::from_bytes(memory.bytes_deallocated.try_into().unwrap());
let reallocated_sign = if memory.bytes_reallocated < 0 { "-" } else { "" };
let reallocated_bytes =
Byte::from_bytes(memory.bytes_reallocated.abs_diff(0).try_into().unwrap());
let adjusted_allocated_bytes = allocated_bytes.get_appropriate_unit(true);
let adjusted_deallocated_bytes = deallocated_bytes.get_appropriate_unit(true);
let adjusted_reallocated_bytes = reallocated_bytes.get_appropriate_unit(true);
format!(
"Allocated {adjusted_allocated_bytes:.2}, \
Deallocated {adjusted_deallocated_bytes:.2}, \
Reallocated {reallocated_sign}{adjusted_reallocated_bytes:.2}"
)
let rss_bytes = Byte::from_bytes(resident).get_appropriate_unit(true);
let shared_bytes = Byte::from_bytes(shared).get_appropriate_unit(true);
format!("RSS {rss_bytes:.2}, Shared {shared_bytes:.2}, OOM score {oom_score}")
}