feat(telemetry): report process CPU and FDs plus host memory and load
This commit is contained in:
7 files changed
+301
-8
No files matched your search
@@ -309,7 +309,7 @@ static METRICS: OnceLock<Instruments> = OnceLock::new();
|
||||
mod fleet;
|
||||
mod process;
|
||||
pub use fleet::{AgentDeviceCounts, FleetCounts, FleetMetrics};
|
||||
pub use process::ProcessMetricsSource;
|
||||
pub use process::{HostLoad, HostMemory, ProcessMetricsSource};
|
||||
struct Instruments {
|
||||
operations: opentelemetry::metrics::Counter<u64>,
|
||||
duration: opentelemetry::metrics::Histogram<f64>,
|
||||
|
||||
@@ -1,22 +1,55 @@
|
||||
//! OS-independent process/host instruments. Callers supply a platform reader.
|
||||
//! Failed reads omit a sample: zero must not masquerade as a restart or freed RAM.
|
||||
|
||||
use opentelemetry::metrics::{Meter, ObservableGauge};
|
||||
use opentelemetry::KeyValue;
|
||||
use opentelemetry::metrics::{Meter, ObservableCounter, ObservableGauge};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Values are seconds/bytes, or None when unsupported/unavailable. Readers run
|
||||
/// Host-visible memory in bytes; available is an estimate, not merely free RAM.
|
||||
pub struct HostMemory {
|
||||
pub total_bytes: u64,
|
||||
pub available_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
/// Host load averages, not CPU utilization percentages.
|
||||
pub struct HostLoad {
|
||||
pub one_minute: f64,
|
||||
pub five_minutes: f64,
|
||||
pub fifteen_minutes: f64,
|
||||
}
|
||||
|
||||
/// Values use the documented seconds/bytes/count units, or None when unavailable. Readers run
|
||||
/// on the metric collection thread: keep work small and local, never perform
|
||||
/// network requests or emit exporter-directed logs from these callbacks.
|
||||
pub trait ProcessMetricsSource: Send + Sync + 'static {
|
||||
fn process_uptime_seconds(&self) -> Option<f64>;
|
||||
fn resident_memory_bytes(&self) -> Option<u64>;
|
||||
fn host_uptime_seconds(&self) -> Option<f64>;
|
||||
/// Cumulative user + system CPU seconds for the process, excluding children.
|
||||
/// Optional additions default to unavailable for existing platform readers.
|
||||
fn cpu_time_seconds(&self) -> Option<f64> {
|
||||
None
|
||||
}
|
||||
fn open_file_descriptors(&self) -> Option<u64> {
|
||||
None
|
||||
}
|
||||
fn host_memory(&self) -> Option<HostMemory> {
|
||||
None
|
||||
}
|
||||
fn host_load(&self) -> Option<HostLoad> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct ProcessMetrics {
|
||||
_uptime: ObservableGauge<f64>,
|
||||
_resident: ObservableGauge<u64>,
|
||||
_host_uptime: ObservableGauge<f64>,
|
||||
_cpu_time: ObservableCounter<f64>,
|
||||
_open_fds: ObservableGauge<u64>,
|
||||
_host_total_memory: ObservableGauge<u64>,
|
||||
_host_available_memory: ObservableGauge<u64>,
|
||||
_host_load: ObservableGauge<f64>,
|
||||
}
|
||||
|
||||
impl ProcessMetrics {
|
||||
@@ -24,6 +57,11 @@ impl ProcessMetrics {
|
||||
let source = Arc::new(source);
|
||||
let uptime = source.clone();
|
||||
let resident = source.clone();
|
||||
let host = source.clone();
|
||||
let cpu = source.clone();
|
||||
let fds = source.clone();
|
||||
let total_memory = source.clone();
|
||||
let available_memory = source.clone();
|
||||
Self {
|
||||
_uptime: meter
|
||||
.f64_observable_gauge("wakey.process.uptime")
|
||||
@@ -53,7 +91,7 @@ impl ProcessMetrics {
|
||||
.with_unit("s")
|
||||
.with_description("Host uptime reported by the platform reader")
|
||||
.with_callback(move |observer| {
|
||||
if let Some(seconds) = source.host_uptime_seconds()
|
||||
if let Some(seconds) = host.host_uptime_seconds()
|
||||
&& seconds.is_finite()
|
||||
&& seconds >= 0.0
|
||||
{
|
||||
@@ -61,6 +99,69 @@ impl ProcessMetrics {
|
||||
}
|
||||
})
|
||||
.build(),
|
||||
_cpu_time: meter
|
||||
.f64_observable_counter("wakey.process.cpu.time")
|
||||
.with_unit("s")
|
||||
.with_description("Cumulative user and system process CPU time, excluding children")
|
||||
.with_callback(move |observer| {
|
||||
if let Some(seconds) = cpu.cpu_time_seconds()
|
||||
&& seconds.is_finite()
|
||||
&& seconds >= 0.0
|
||||
{
|
||||
observer.observe(seconds, &[]);
|
||||
}
|
||||
})
|
||||
.build(),
|
||||
_open_fds: meter
|
||||
.u64_observable_gauge("wakey.process.open_fds")
|
||||
.with_description(
|
||||
"Current open file-descriptor count reported by the platform reader",
|
||||
)
|
||||
.with_callback(move |observer| {
|
||||
if let Some(count) = fds.open_file_descriptors() {
|
||||
observer.observe(count, &[]);
|
||||
}
|
||||
})
|
||||
.build(),
|
||||
_host_total_memory: meter
|
||||
.u64_observable_gauge("wakey.host.memory.total")
|
||||
.with_unit("By")
|
||||
.with_description("Total usable RAM reported by the platform reader")
|
||||
.with_callback(move |observer| {
|
||||
if let Some(memory) = total_memory.host_memory() {
|
||||
observer.observe(memory.total_bytes, &[]);
|
||||
}
|
||||
})
|
||||
.build(),
|
||||
_host_available_memory: meter
|
||||
.u64_observable_gauge("wakey.host.memory.available")
|
||||
.with_unit("By")
|
||||
.with_description("Estimated RAM available for applications without swapping")
|
||||
.with_callback(move |observer| {
|
||||
if let Some(memory) = available_memory.host_memory()
|
||||
&& let Some(bytes) = memory.available_bytes
|
||||
{
|
||||
observer.observe(bytes, &[]);
|
||||
}
|
||||
})
|
||||
.build(),
|
||||
_host_load: meter
|
||||
.f64_observable_gauge("wakey.host.load")
|
||||
.with_description("Host load average over the labeled period, not CPU percent")
|
||||
.with_callback(move |observer| {
|
||||
if let Some(load) = source.host_load() {
|
||||
for (period, value) in [
|
||||
("1m", load.one_minute),
|
||||
("5m", load.five_minutes),
|
||||
("15m", load.fifteen_minutes),
|
||||
] {
|
||||
if value.is_finite() && value >= 0.0 {
|
||||
observer.observe(value, &[KeyValue::new("period", period)]);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.build(),
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user