mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-11 15:38:55 +01:00
Replace the procfs by libproc
This commit is contained in:
parent
d78ada07b5
commit
02dcaf07db
106
Cargo.lock
generated
106
Cargo.lock
generated
@ -535,6 +535,26 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "bindgen"
|
||||||
|
version = "0.68.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.4.1",
|
||||||
|
"cexpr",
|
||||||
|
"clang-sys",
|
||||||
|
"lazy_static",
|
||||||
|
"lazycell",
|
||||||
|
"peeking_take_while",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"regex",
|
||||||
|
"rustc-hash",
|
||||||
|
"shlex",
|
||||||
|
"syn 2.0.48",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bit-set"
|
name = "bit-set"
|
||||||
version = "0.5.3"
|
version = "0.5.3"
|
||||||
@ -831,6 +851,15 @@ dependencies = [
|
|||||||
"smallvec",
|
"smallvec",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "cexpr"
|
||||||
|
version = "0.6.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
||||||
|
dependencies = [
|
||||||
|
"nom",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cfg-if"
|
name = "cfg-if"
|
||||||
version = "1.0.0"
|
version = "1.0.0"
|
||||||
@ -915,6 +944,17 @@ dependencies = [
|
|||||||
"inout",
|
"inout",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "clang-sys"
|
||||||
|
version = "1.7.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1"
|
||||||
|
dependencies = [
|
||||||
|
"glob",
|
||||||
|
"libc",
|
||||||
|
"libloading",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clap"
|
name = "clap"
|
||||||
version = "4.4.17"
|
version = "4.4.17"
|
||||||
@ -3119,6 +3159,12 @@ version = "1.4.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lazycell"
|
||||||
|
version = "1.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "levenshtein_automata"
|
name = "levenshtein_automata"
|
||||||
version = "0.2.1"
|
version = "0.2.1"
|
||||||
@ -3146,6 +3192,16 @@ dependencies = [
|
|||||||
"pkg-config",
|
"pkg-config",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libloading"
|
||||||
|
version = "0.8.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"windows-sys 0.48.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libm"
|
name = "libm"
|
||||||
version = "0.2.7"
|
version = "0.2.7"
|
||||||
@ -3162,6 +3218,17 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libproc"
|
||||||
|
version = "0.14.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "229004ebba9d1d5caf41623f1523b6d52abb47d9f6ab87f7e6fc992e3b854aef"
|
||||||
|
dependencies = [
|
||||||
|
"bindgen",
|
||||||
|
"errno",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libz-sys"
|
name = "libz-sys"
|
||||||
version = "1.1.12"
|
version = "1.1.12"
|
||||||
@ -4177,6 +4244,12 @@ dependencies = [
|
|||||||
"sha2",
|
"sha2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "peeking_take_while"
|
||||||
|
version = "0.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pem"
|
name = "pem"
|
||||||
version = "1.1.1"
|
version = "1.1.1"
|
||||||
@ -4432,29 +4505,6 @@ dependencies = [
|
|||||||
"rustix 0.36.16",
|
"rustix 0.36.16",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "procfs"
|
|
||||||
version = "0.16.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 2.4.1",
|
|
||||||
"hex",
|
|
||||||
"lazy_static",
|
|
||||||
"procfs-core",
|
|
||||||
"rustix 0.38.26",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "procfs-core"
|
|
||||||
version = "0.16.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags 2.4.1",
|
|
||||||
"hex",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "prometheus"
|
name = "prometheus"
|
||||||
version = "0.13.3"
|
version = "0.13.3"
|
||||||
@ -4467,7 +4517,7 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
"memchr",
|
"memchr",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
"procfs 0.14.2",
|
"procfs",
|
||||||
"protobuf",
|
"protobuf",
|
||||||
"thiserror",
|
"thiserror",
|
||||||
]
|
]
|
||||||
@ -5079,6 +5129,12 @@ dependencies = [
|
|||||||
"lazy_static",
|
"lazy_static",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "shlex"
|
||||||
|
version = "1.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "signal-hook-registry"
|
name = "signal-hook-registry"
|
||||||
version = "1.4.1"
|
version = "1.4.1"
|
||||||
@ -5722,7 +5778,7 @@ dependencies = [
|
|||||||
"byte-unit",
|
"byte-unit",
|
||||||
"color-spantrace",
|
"color-spantrace",
|
||||||
"fxprof-processed-profile",
|
"fxprof-processed-profile",
|
||||||
"procfs 0.16.0",
|
"libproc",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
@ -19,5 +19,5 @@ byte-unit = { version = "4.0.19", default-features = false, features = [
|
|||||||
] }
|
] }
|
||||||
tokio = { version = "1.35.1", features = ["sync"] }
|
tokio = { version = "1.35.1", features = ["sync"] }
|
||||||
|
|
||||||
[target.'cfg(target_os = "linux")'.dependencies]
|
[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
|
||||||
procfs = { version = "0.16.0", default-features = false }
|
libproc = "0.14.2"
|
||||||
|
@ -100,46 +100,29 @@ pub struct SpanClose {
|
|||||||
pub time: std::time::Duration,
|
pub time: std::time::Duration,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A struct with a lot of memory allocation stats akin
|
/// A struct with a memory allocation stat.
|
||||||
/// to the `procfs::Process::StatsM` one plus the OOM score.
|
|
||||||
///
|
|
||||||
/// Note that all the values are in bytes not in pages.
|
|
||||||
#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)]
|
#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)]
|
||||||
pub struct MemoryStats {
|
pub struct MemoryStats {
|
||||||
/// Resident set size, measured in bytes.
|
/// Resident set size, measured in bytes.
|
||||||
/// (same as VmRSS in /proc/<pid>/status).
|
/// (same as VmRSS in /proc/<pid>/status).
|
||||||
pub resident: u64,
|
pub resident: u64,
|
||||||
/// Number of resident shared bytes (i.e., backed by a file).
|
|
||||||
/// (same as RssFile+RssShmem in /proc/<pid>/status).
|
|
||||||
pub shared: u64,
|
|
||||||
/// The current score that the kernel gives to this process
|
|
||||||
/// for the purpose of selecting a process for the OOM-killer
|
|
||||||
///
|
|
||||||
/// A higher score means that the process is more likely to be selected
|
|
||||||
/// by the OOM-killer. The basis for this score is the amount of memory used
|
|
||||||
/// by the process, plus other factors.
|
|
||||||
///
|
|
||||||
/// (Since linux 2.6.11)
|
|
||||||
pub oom_score: u32,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MemoryStats {
|
impl MemoryStats {
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||||
pub fn fetch() -> procfs::ProcResult<Self> {
|
pub fn fetch() -> Option<Self> {
|
||||||
let process = procfs::process::Process::myself().unwrap();
|
use libproc::libproc::pid_rusage::{pidrusage, RUsageInfoV0};
|
||||||
let procfs::process::StatM { resident, shared, .. } = process.statm()?;
|
|
||||||
let oom_score = process.oom_score()?;
|
|
||||||
let page_size = procfs::page_size();
|
|
||||||
|
|
||||||
Ok(MemoryStats { resident: resident * page_size, shared: shared * page_size, oom_score })
|
match pidrusage(std::process::id() as i32) {
|
||||||
|
Ok(RUsageInfoV0 { ri_resident_size, .. }) => {
|
||||||
|
Some(MemoryStats { resident: ri_resident_size })
|
||||||
|
}
|
||||||
|
Err(_) => None, /* ignoring error to avoid spamming */
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn checked_sub(self, other: Self) -> Option<Self> {
|
pub fn checked_sub(self, other: Self) -> Option<Self> {
|
||||||
Some(Self {
|
Some(Self { resident: self.resident.checked_sub(other.resident)? })
|
||||||
resident: self.resident.checked_sub(other.resident)?,
|
|
||||||
shared: self.shared.checked_sub(other.shared)?,
|
|
||||||
oom_score: self.oom_score.checked_sub(other.oom_score)?,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -96,12 +96,12 @@ impl TraceLayer {
|
|||||||
self.start_time.elapsed()
|
self.start_time.elapsed()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||||
fn memory_stats(&self) -> Option<MemoryStats> {
|
fn memory_stats(&self) -> Option<MemoryStats> {
|
||||||
Some(MemoryStats::fetch().unwrap())
|
Some(MemoryStats::fetch().unwrap())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(target_os = "linux"))]
|
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||||
fn memory_stats(&self) -> Option<MemoryStats> {
|
fn memory_stats(&self) -> Option<MemoryStats> {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
@ -322,18 +322,6 @@ impl<'a> ProfilerMarker for SpanMarker<'a> {
|
|||||||
format: MarkerFieldFormat::Bytes,
|
format: MarkerFieldFormat::Bytes,
|
||||||
searchable: false,
|
searchable: false,
|
||||||
}),
|
}),
|
||||||
MarkerSchemaField::Dynamic(MarkerDynamicField {
|
|
||||||
key: "shared",
|
|
||||||
label: "Number of resident shared pages (i.e., backed by a file) while this function was executing",
|
|
||||||
format: MarkerFieldFormat::Bytes,
|
|
||||||
searchable: false,
|
|
||||||
}),
|
|
||||||
MarkerSchemaField::Dynamic(MarkerDynamicField {
|
|
||||||
key: "oom_score",
|
|
||||||
label: "The current score that the kernel gives to this process for the purpose of selecting a process for the OOM-killer while this function was executing",
|
|
||||||
format: MarkerFieldFormat::Integer,
|
|
||||||
searchable: false,
|
|
||||||
}),
|
|
||||||
];
|
];
|
||||||
|
|
||||||
MarkerSchema {
|
MarkerSchema {
|
||||||
@ -366,10 +354,8 @@ impl<'a> ProfilerMarker for SpanMarker<'a> {
|
|||||||
"thread_id": thread_id,
|
"thread_id": thread_id,
|
||||||
});
|
});
|
||||||
|
|
||||||
if let Some(MemoryStats { resident, shared, oom_score }) = self.memory_delta {
|
if let Some(MemoryStats { resident }) = self.memory_delta {
|
||||||
value["resident"] = json!(resident);
|
value["resident"] = json!(resident);
|
||||||
value["shared"] = json!(shared);
|
|
||||||
value["oom_score"] = json!(oom_score);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
value
|
value
|
||||||
@ -423,18 +409,6 @@ impl<'a> ProfilerMarker for EventMarker<'a> {
|
|||||||
format: MarkerFieldFormat::Bytes,
|
format: MarkerFieldFormat::Bytes,
|
||||||
searchable: false,
|
searchable: false,
|
||||||
}),
|
}),
|
||||||
MarkerSchemaField::Dynamic(MarkerDynamicField {
|
|
||||||
key: "shared",
|
|
||||||
label: "Number of resident shared pages (i.e., backed by a file) while this function was executing",
|
|
||||||
format: MarkerFieldFormat::Bytes,
|
|
||||||
searchable: false,
|
|
||||||
}),
|
|
||||||
MarkerSchemaField::Dynamic(MarkerDynamicField {
|
|
||||||
key: "oom_score",
|
|
||||||
label: "The current score that the kernel gives to this process for the purpose of selecting a process for the OOM-killer while this function was executing",
|
|
||||||
format: MarkerFieldFormat::Integer,
|
|
||||||
searchable: false,
|
|
||||||
}),
|
|
||||||
];
|
];
|
||||||
|
|
||||||
MarkerSchema {
|
MarkerSchema {
|
||||||
@ -467,10 +441,8 @@ impl<'a> ProfilerMarker for EventMarker<'a> {
|
|||||||
"thread_id": thread_id,
|
"thread_id": thread_id,
|
||||||
});
|
});
|
||||||
|
|
||||||
if let Some(MemoryStats { resident, shared, oom_score }) = self.memory_delta {
|
if let Some(MemoryStats { resident }) = self.memory_delta {
|
||||||
value["resident"] = json!(resident);
|
value["resident"] = json!(resident);
|
||||||
value["shared"] = json!(shared);
|
|
||||||
value["oom_score"] = json!(oom_score);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
value
|
value
|
||||||
|
@ -188,9 +188,8 @@ fn print_duration(duration: std::time::Duration) -> String {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Format only the allocated bytes, deallocated bytes and reallocated bytes in GiB, MiB, KiB, Bytes.
|
/// Format only the allocated bytes, deallocated bytes and reallocated bytes in GiB, MiB, KiB, Bytes.
|
||||||
fn print_memory(MemoryStats { resident, shared, oom_score }: MemoryStats) -> String {
|
fn print_memory(MemoryStats { resident }: MemoryStats) -> String {
|
||||||
use byte_unit::Byte;
|
use byte_unit::Byte;
|
||||||
let rss_bytes = Byte::from_bytes(resident).get_appropriate_unit(true);
|
let rss_bytes = Byte::from_bytes(resident).get_appropriate_unit(true);
|
||||||
let shared_bytes = Byte::from_bytes(shared).get_appropriate_unit(true);
|
format!("RSS {rss_bytes:.2}")
|
||||||
format!("RSS {rss_bytes:.2}, Shared {shared_bytes:.2}, OOM score {oom_score}")
|
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user