fix(telemetry): use seconds-based buckets for operation durations
This commit is contained in:
3 files changed
+46
No files matched your search
@@ -155,6 +155,15 @@ This needs classic histogram bucket series. Keep `le` in the grouping: it is
|
|||||||
the bucket boundary, in seconds. p95 is an estimate from those boundaries,
|
the bucket boundary, in seconds. p95 is an estimate from those boundaries,
|
||||||
not an exact sort of recorded durations. Few operations make percentiles noisy.
|
not an exact sort of recorded durations. Few operations make percentiles noisy.
|
||||||
|
|
||||||
|
Operation-duration buckets span 100 microseconds to 60 seconds, with an overflow
|
||||||
|
bucket above that. Older binaries used SDK defaults whose first positive boundary
|
||||||
|
was 5 seconds: fast operations could therefore report an estimated p95 of 4.75
|
||||||
|
seconds. Updating both CC and agents fixes future measurements, not historical
|
||||||
|
data. Wait until the query's window contains only new-version measurements after
|
||||||
|
rollout; mixing old and new bucket layouts makes the estimate unreliable.
|
||||||
|
With roughly one snapshot per minute, a five-minute percentile is based on only
|
||||||
|
a handful of observations and can still look stepped. Use the mean above alongside it.
|
||||||
|
|
||||||
CC relay duration includes waiting for the agent; agent execution duration
|
CC relay duration includes waiting for the agent; agent execution duration
|
||||||
measures its dispatch separately. Do not sum both and call that user latency.
|
measures its dispatch separately. Do not sum both and call that user latency.
|
||||||
|
|
||||||
|
|||||||
@@ -338,6 +338,12 @@ impl Instruments {
|
|||||||
duration: meter
|
duration: meter
|
||||||
.f64_histogram("wakey.operation.duration")
|
.f64_histogram("wakey.operation.duration")
|
||||||
.with_unit("s")
|
.with_unit("s")
|
||||||
|
// Measurements are seconds; SDK defaults start at 5, which
|
||||||
|
// collapses millisecond operations into one unusably wide bucket.
|
||||||
|
.with_boundaries(vec![
|
||||||
|
0.0001, 0.0005, 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0,
|
||||||
|
10.0, 30.0, 60.0,
|
||||||
|
])
|
||||||
.build(),
|
.build(),
|
||||||
snapshot_devices: meter.u64_counter("wakey.snapshot.devices").build(),
|
snapshot_devices: meter.u64_counter("wakey.snapshot.devices").build(),
|
||||||
terminal_active: meter.i64_up_down_counter("wakey.terminal.active").build(),
|
terminal_active: meter.i64_up_down_counter("wakey.terminal.active").build(),
|
||||||
|
|||||||
@@ -409,6 +409,37 @@ async fn exports_all_signals_and_preserves_remote_command_parent() {
|
|||||||
assert!(names.contains(&name), "missing {name}");
|
assert!(names.contains(&name), "missing {name}");
|
||||||
}
|
}
|
||||||
let latest = metrics.last().unwrap();
|
let latest = metrics.last().unwrap();
|
||||||
|
{
|
||||||
|
let duration = latest
|
||||||
|
.resource_metrics
|
||||||
|
.iter()
|
||||||
|
.flat_map(|r| &r.scope_metrics)
|
||||||
|
.flat_map(|s| &s.metrics)
|
||||||
|
.find(|m| m.name == "wakey.operation.duration")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(duration.unit, "s");
|
||||||
|
let Some(metric::Data::Histogram(histogram)) = &duration.data else {
|
||||||
|
panic!("operation duration must export a histogram");
|
||||||
|
};
|
||||||
|
let point = histogram.data_points.first().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
point.explicit_bounds,
|
||||||
|
vec![
|
||||||
|
0.0001, 0.0005, 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0,
|
||||||
|
10.0, 30.0, 60.0,
|
||||||
|
]
|
||||||
|
);
|
||||||
|
// One inventory operation is recorded above. Derive its expected bucket
|
||||||
|
// from the exported sum, avoiding elapsed-time thresholds or sleeps.
|
||||||
|
assert_eq!(point.count, 1);
|
||||||
|
let sample = point.sum.unwrap();
|
||||||
|
let bucket = point
|
||||||
|
.explicit_bounds
|
||||||
|
.partition_point(|bound| *bound < sample);
|
||||||
|
let mut expected = vec![0; point.explicit_bounds.len() + 1];
|
||||||
|
expected[bucket] = 1;
|
||||||
|
assert_eq!(point.bucket_counts, expected);
|
||||||
|
}
|
||||||
{
|
{
|
||||||
for (name, unit, expected) in [
|
for (name, unit, expected) in [
|
||||||
(
|
(
|
||||||
|
|||||||
Reference in new issue
Block a user