searchengines-web/MeiliSearch
searchengines-web
/
MeiliSearch
mirror of https://github.com/meilisearch/MeiliSearch
1
0
Fork 0
Code Issues Releases Wiki Activity

Replace the procfs by libproc

This commit is contained in:
Clément Renault 2024-02-06 18:12:04 +01:00 committed by Louis Dureuil
parent d78ada07b5
commit 02dcaf07db
No known key found for this signature in database
6 changed files with 100 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
Cargo.lock generated
View File

@ -535,6 +535,26 @@ dependencies = [
"serde",
]
[[package]]
name = "bindgen"
version = "0.68.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078"
dependencies = [
"bitflags 2.4.1",
"cexpr",
"clang-sys",
"lazy_static",
"lazycell",
"peeking_take_while",
"proc-macro2",
"quote",
"regex",
"rustc-hash",
"shlex",
"syn 2.0.48",
]
[[package]]
name = "bit-set"
version = "0.5.3"
@ -831,6 +851,15 @@ dependencies = [
"smallvec",
]
[[package]]
name = "cexpr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -915,6 +944,17 @@ dependencies = [
"inout",
]
[[package]]
name = "clang-sys"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1"
dependencies = [
"glob",
"libc",
"libloading",
]
[[package]]
name = "clap"
version = "4.4.17"
@ -3119,6 +3159,12 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "levenshtein_automata"
version = "0.2.1"
@ -3146,6 +3192,16 @@ dependencies = [
"pkg-config",
]
[[package]]
name = "libloading"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161"
dependencies = [
"cfg-if",
"windows-sys 0.48.0",
]
[[package]]
name = "libm"
version = "0.2.7"
@ -3162,6 +3218,17 @@ dependencies = [
"libc",
]
[[package]]
name = "libproc"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "229004ebba9d1d5caf41623f1523b6d52abb47d9f6ab87f7e6fc992e3b854aef"
dependencies = [
"bindgen",
"errno",
"libc",
]
[[package]]
name = "libz-sys"
version = "1.1.12"
@ -4177,6 +4244,12 @@ dependencies = [
"sha2",
]
[[package]]
name = "peeking_take_while"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
[[package]]
name = "pem"
version = "1.1.1"
@ -4432,29 +4505,6 @@ dependencies = [
"rustix 0.36.16",
]
[[package]]
name = "procfs"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4"
dependencies = [
"bitflags 2.4.1",
"hex",
"lazy_static",
"procfs-core",
"rustix 0.38.26",
]
[[package]]
name = "procfs-core"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29"
dependencies = [
"bitflags 2.4.1",
"hex",
]
[[package]]
name = "prometheus"
version = "0.13.3"
@ -4467,7 +4517,7 @@ dependencies = [
"libc",
"memchr",
"parking_lot",
"procfs 0.14.2",
"procfs",
"protobuf",
"thiserror",
]
@ -5079,6 +5129,12 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "signal-hook-registry"
version = "1.4.1"
@ -5722,7 +5778,7 @@ dependencies = [
"byte-unit",
"color-spantrace",
"fxprof-processed-profile",
"procfs 0.16.0",
"libproc",
"serde",
"serde_json",
"tokio",

4
tracing-trace/Cargo.toml
View File

@ -19,5 +19,5 @@ byte-unit = { version = "4.0.19", default-features = false, features = [
] }
tokio = { version = "1.35.1", features = ["sync"] }
[target.'cfg(target_os = "linux")'.dependencies]
procfs = { version = "0.16.0", default-features = false }
[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
libproc = "0.14.2"

39
tracing-trace/src/entry.rs
View File

@ -100,46 +100,29 @@ pub struct SpanClose {
pub time: std::time::Duration,
}
/// A struct with a lot of memory allocation stats akin
/// to the `procfs::Process::StatsM` one plus the OOM score.
///
/// Note that all the values are in bytes not in pages.
/// A struct with a memory allocation stat.
#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)]
pub struct MemoryStats {
/// Resident set size, measured in bytes.
/// (same as VmRSS in /proc/<pid>/status).
pub resident: u64,
/// Number of resident shared bytes (i.e., backed by a file).
/// (same as RssFile+RssShmem in /proc/<pid>/status).
pub shared: u64,
/// The current score that the kernel gives to this process
/// for the purpose of selecting a process for the OOM-killer
///
/// A higher score means that the process is more likely to be selected
/// by the OOM-killer. The basis for this score is the amount of memory used
/// by the process, plus other factors.
///
/// (Since linux 2.6.11)
pub oom_score: u32,
}
impl MemoryStats {
#[cfg(target_os = "linux")]
pub fn fetch() -> procfs::ProcResult<Self> {
let process = procfs::process::Process::myself().unwrap();
let procfs::process::StatM { resident, shared, .. } = process.statm()?;
let oom_score = process.oom_score()?;
let page_size = procfs::page_size();
#[cfg(any(target_os = "linux", target_os = "macos"))]
pub fn fetch() -> Option<Self> {
use libproc::libproc::pid_rusage::{pidrusage, RUsageInfoV0};
Ok(MemoryStats { resident: resident * page_size, shared: shared * page_size, oom_score })
match pidrusage(std::process::id() as i32) {
Ok(RUsageInfoV0 { ri_resident_size, .. }) => {
Some(MemoryStats { resident: ri_resident_size })
}
Err(_) => None, /* ignoring error to avoid spamming */
}
}
pub fn checked_sub(self, other: Self) -> Option<Self> {
Some(Self {
resident: self.resident.checked_sub(other.resident)?,
shared: self.shared.checked_sub(other.shared)?,
oom_score: self.oom_score.checked_sub(other.oom_score)?,
})
Some(Self { resident: self.resident.checked_sub(other.resident)? })
}
}

4
tracing-trace/src/layer.rs
View File

@ -96,12 +96,12 @@ impl TraceLayer {
self.start_time.elapsed()
}
#[cfg(target_os = "linux")]
#[cfg(any(target_os = "linux", target_os = "macos"))]
fn memory_stats(&self) -> Option<MemoryStats> {
Some(MemoryStats::fetch().unwrap())
}
#[cfg(not(target_os = "linux"))]
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
fn memory_stats(&self) -> Option<MemoryStats> {
None
}

32
tracing-trace/src/processor/firefox_profiler.rs
View File

@ -322,18 +322,6 @@ impl<'a> ProfilerMarker for SpanMarker<'a> {
format: MarkerFieldFormat::Bytes,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "shared",
label: "Number of resident shared pages (i.e., backed by a file) while this function was executing",
format: MarkerFieldFormat::Bytes,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "oom_score",
label: "The current score that the kernel gives to this process for the purpose of selecting a process for the OOM-killer while this function was executing",
format: MarkerFieldFormat::Integer,
searchable: false,
}),
];
MarkerSchema {
@ -366,10 +354,8 @@ impl<'a> ProfilerMarker for SpanMarker<'a> {
"thread_id": thread_id,
});
if let Some(MemoryStats { resident, shared, oom_score }) = self.memory_delta {
if let Some(MemoryStats { resident }) = self.memory_delta {
value["resident"] = json!(resident);
value["shared"] = json!(shared);
value["oom_score"] = json!(oom_score);
}
value
@ -423,18 +409,6 @@ impl<'a> ProfilerMarker for EventMarker<'a> {
format: MarkerFieldFormat::Bytes,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "shared",
label: "Number of resident shared pages (i.e., backed by a file) while this function was executing",
format: MarkerFieldFormat::Bytes,
searchable: false,
}),
MarkerSchemaField::Dynamic(MarkerDynamicField {
key: "oom_score",
label: "The current score that the kernel gives to this process for the purpose of selecting a process for the OOM-killer while this function was executing",
format: MarkerFieldFormat::Integer,
searchable: false,
}),
];
MarkerSchema {
@ -467,10 +441,8 @@ impl<'a> ProfilerMarker for EventMarker<'a> {
"thread_id": thread_id,
});
if let Some(MemoryStats { resident, shared, oom_score }) = self.memory_delta {
if let Some(MemoryStats { resident }) = self.memory_delta {
value["resident"] = json!(resident);
value["shared"] = json!(shared);
value["oom_score"] = json!(oom_score);
}
value

5
tracing-trace/src/processor/fmt.rs
View File

@ -188,9 +188,8 @@ fn print_duration(duration: std::time::Duration) -> String {
}
/// Format only the allocated bytes, deallocated bytes and reallocated bytes in GiB, MiB, KiB, Bytes.
fn print_memory(MemoryStats { resident, shared, oom_score }: MemoryStats) -> String {
fn print_memory(MemoryStats { resident }: MemoryStats) -> String {
use byte_unit::Byte;
let rss_bytes = Byte::from_bytes(resident).get_appropriate_unit(true);
let shared_bytes = Byte::from_bytes(shared).get_appropriate_unit(true);
format!("RSS {rss_bytes:.2}, Shared {shared_bytes:.2}, OOM score {oom_score}")
format!("RSS {rss_bytes:.2}")
}