fix(telemetry): use seconds-based buckets for operation durations

This commit is contained in:
lda committed 2026-10-20 01:47:48 +07:00 Verified
1 parent 25c6eb9eb4
commit 03c1f14dba
3 files changed
+46

No files matched your search

+9
View File
@@ -155,6 +155,15 @@ This needs classic histogram bucket series. Keep `le` in the grouping: it is
the bucket boundary, in seconds. p95 is an estimate from those boundaries, the bucket boundary, in seconds. p95 is an estimate from those boundaries,
not an exact sort of recorded durations. Few operations make percentiles noisy. not an exact sort of recorded durations. Few operations make percentiles noisy.
Operation-duration buckets span 100 microseconds to 60 seconds, with an overflow
bucket above that. Older binaries used SDK defaults whose first positive boundary
was 5 seconds: fast operations could therefore report an estimated p95 of 4.75
seconds. Updating both CC and agents fixes future measurements, not historical
data. Wait until the query's window contains only new-version measurements after
rollout; mixing old and new bucket layouts makes the estimate unreliable.
With roughly one snapshot per minute, a five-minute percentile is based on only
a handful of observations and can still look stepped. Use the mean above alongside it.
CC relay duration includes waiting for the agent; agent execution duration CC relay duration includes waiting for the agent; agent execution duration
measures its dispatch separately. Do not sum both and call that user latency. measures its dispatch separately. Do not sum both and call that user latency.
+6
View File
@@ -338,6 +338,12 @@ impl Instruments {
duration: meter duration: meter
.f64_histogram("wakey.operation.duration") .f64_histogram("wakey.operation.duration")
.with_unit("s") .with_unit("s")
// Measurements are seconds; SDK defaults start at 5, which
// collapses millisecond operations into one unusably wide bucket.
.with_boundaries(vec![
0.0001, 0.0005, 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0,
10.0, 30.0, 60.0,
])
.build(), .build(),
snapshot_devices: meter.u64_counter("wakey.snapshot.devices").build(), snapshot_devices: meter.u64_counter("wakey.snapshot.devices").build(),
terminal_active: meter.i64_up_down_counter("wakey.terminal.active").build(), terminal_active: meter.i64_up_down_counter("wakey.terminal.active").build(),
+31
View File
@@ -409,6 +409,37 @@ async fn exports_all_signals_and_preserves_remote_command_parent() {
assert!(names.contains(&name), "missing {name}"); assert!(names.contains(&name), "missing {name}");
} }
let latest = metrics.last().unwrap(); let latest = metrics.last().unwrap();
{
let duration = latest
.resource_metrics
.iter()
.flat_map(|r| &r.scope_metrics)
.flat_map(|s| &s.metrics)
.find(|m| m.name == "wakey.operation.duration")
.unwrap();
assert_eq!(duration.unit, "s");
let Some(metric::Data::Histogram(histogram)) = &duration.data else {
panic!("operation duration must export a histogram");
};
let point = histogram.data_points.first().unwrap();
assert_eq!(
point.explicit_bounds,
vec![
0.0001, 0.0005, 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0,
10.0, 30.0, 60.0,
]
);
// One inventory operation is recorded above. Derive its expected bucket
// from the exported sum, avoiding elapsed-time thresholds or sleeps.
assert_eq!(point.count, 1);
let sample = point.sum.unwrap();
let bucket = point
.explicit_bounds
.partition_point(|bound| *bound < sample);
let mut expected = vec![0; point.explicit_bounds.len() + 1];
expected[bucket] = 1;
assert_eq!(point.bucket_counts, expected);
}
{ {
for (name, unit, expected) in [ for (name, unit, expected) in [
( (