feat(telemetry): expose current fleet device and endpoint counts

wakey-core/src/telemetry.rs

@@ -291,6 +291,8 @@ fn local_timer() -> fmt::time::OffsetTime<time::format_description::well_known::
 // Construct instruments once, after the real provider exists. Disabled telemetry
 // therefore does no per-PTY-read allocations or instrument registration.
 static METRICS: OnceLock<Instruments> = OnceLock::new();
+mod fleet;
+pub use fleet::{AgentDeviceCounts, FleetCounts, FleetMetrics};
 struct Instruments {
     operations: opentelemetry::metrics::Counter<u64>,
     duration: opentelemetry::metrics::Histogram<f64>,

wakey-core/src/telemetry/fleet.rs

@@ -0,0 +1,91 @@
+use std::collections::BTreeMap;
+use std::sync::{Arc, RwLock};
+
+use opentelemetry::{KeyValue, metrics::ObservableGauge};
+
+/// Retained CC rows, not unique addresses or online-device counts.
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct AgentDeviceCounts {
+    pub devices: u64,
+    pub endpoints: u64,
+}
+
+/// One unfiltered, revision-checked CC sample. Agent IDs are the only identity labels.
+#[derive(Debug, Default, PartialEq, Eq)]
+pub struct FleetCounts {
+    pub agents: BTreeMap<String, AgentDeviceCounts>,
+    pub saved: u64,
+    pub unsaved: u64,
+}
+
+/// Callbacks only read memory; async DB work stays outside the export thread.
+/// Replacing the entire sample removes vanished labels instead of retaining their series.
+pub struct FleetMetrics {
+    sample: Arc<RwLock<Option<FleetCounts>>>,
+    _devices: ObservableGauge<u64>,
+    _endpoints: ObservableGauge<u64>,
+    _fleet: ObservableGauge<u64>,
+}
+
+impl FleetMetrics {
+    /// Avoid polling the DB at all when OTLP metrics are disabled.
+    pub fn new() -> Option<Self> {
+        super::METRICS.get()?;
+        let meter = opentelemetry::global::meter("wakey");
+        let sample = Arc::new(RwLock::new(None::<FleetCounts>));
+        let devices = sample.clone();
+        let endpoints = sample.clone();
+        let fleet = sample.clone();
+        Some(Self {
+            sample,
+            _devices: meter
+                .u64_observable_gauge("wakey.stored.devices")
+                .with_callback(move |observer| {
+                    if let Ok(sample) = devices.read()
+                        && let Some(sample) = sample.as_ref()
+                    {
+                        for (agent, counts) in &sample.agents {
+                            observer.observe(
+                                counts.devices,
+                                &[KeyValue::new("agent_id", agent.clone())],
+                            );
+                        }
+                    }
+                })
+                .build(),
+            _endpoints: meter
+                .u64_observable_gauge("wakey.stored.endpoints")
+                .with_callback(move |observer| {
+                    if let Ok(sample) = endpoints.read()
+                        && let Some(sample) = sample.as_ref()
+                    {
+                        for (agent, counts) in &sample.agents {
+                            observer.observe(
+                                counts.endpoints,
+                                &[KeyValue::new("agent_id", agent.clone())],
+                            );
+                        }
+                    }
+                })
+                .build(),
+            _fleet: meter
+                .u64_observable_gauge("wakey.fleet.devices")
+                .with_callback(move |observer| {
+                    if let Ok(sample) = fleet.read()
+                        && let Some(sample) = sample.as_ref()
+                    {
+                        observer.observe(sample.saved, &[KeyValue::new("identity", "saved")]);
+                        observer.observe(sample.unsaved, &[KeyValue::new("identity", "unsaved")]);
+                    }
+                })
+                .build(),
+        })
+    }
+
+    /// None suppresses observations after a failed refresh, rather than reporting stale counts.
+    pub fn replace(&self, sample: Option<FleetCounts>) {
+        if let Ok(mut current) = self.sample.write() {
+            *current = sample;
+        }
+    }
+}

wakey-core/tests/telemetry.rs

@@ -3,6 +3,7 @@
 use opentelemetry_proto::tonic::collector::{
     logs::v1 as logs, metrics::v1 as metrics, trace::v1 as traces,
 };
+use opentelemetry_proto::tonic::metrics::v1::{metric, number_data_point};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex};
 use tonic::{Request, Response, Status};
@@ -212,6 +213,36 @@ async fn exports_all_signals_and_preserves_remote_command_parent() {
     )
     .unwrap();
 
+    let fleet_metrics = telemetry::FleetMetrics::new().unwrap();
+    fleet_metrics.replace(Some(telemetry::FleetCounts {
+        agents: [(
+            "removed-agent".into(),
+            telemetry::AgentDeviceCounts {
+                devices: 9,
+                endpoints: 20,
+            },
+        )]
+        .into(),
+        saved: 9,
+        unsaved: 0,
+    }));
+    fleet_metrics.replace(None);
+    fleet_metrics.replace(Some(telemetry::FleetCounts {
+        agents: [
+            (
+                "router".into(),
+                telemetry::AgentDeviceCounts {
+                    devices: 2,
+                    endpoints: 7,
+                },
+            ),
+            ("empty-agent".into(), Default::default()),
+        ]
+        .into(),
+        saved: 1,
+        unsaved: 1,
+    }));
+
     let parent = tracing::info_span!("relay_command");
     let context = parent.in_scope(telemetry::current_trace_context);
     assert!(context.contains_key("traceparent"));
@@ -306,7 +337,48 @@ async fn exports_all_signals_and_preserves_remote_command_parent() {
         "wakey.operation.duration",
         "wakey.snapshot.devices",
         "wakey.terminal.active",
+        "wakey.stored.devices",
+        "wakey.stored.endpoints",
+        "wakey.fleet.devices",
     ] {
         assert!(names.contains(&name), "missing {name}");
     }
+    let latest = metrics.last().unwrap();
+    for (name, label, expected) in [
+        (
+            "wakey.stored.devices",
+            "agent_id",
+            vec![("router", 2), ("empty-agent", 0)],
+        ),
+        (
+            "wakey.stored.endpoints",
+            "agent_id",
+            vec![("router", 7), ("empty-agent", 0)],
+        ),
+        (
+            "wakey.fleet.devices",
+            "identity",
+            vec![("saved", 1), ("unsaved", 1)],
+        ),
+    ] {
+        let metric = latest
+            .resource_metrics
+            .iter()
+            .flat_map(|r| &r.scope_metrics)
+            .flat_map(|s| &s.metrics)
+            .find(|m| m.name == name)
+            .unwrap();
+        let Some(metric::Data::Gauge(gauge)) = &metric.data else {
+            panic!("{name} must be a gauge");
+        };
+        assert_eq!(gauge.data_points.len(), expected.len());
+        for (value, count) in expected {
+            let point = gauge.data_points.iter().find(|p| p.attributes.iter().any(|a| {
+                a.key == label && a.value.as_ref().and_then(|v| v.value.as_ref())
+                    == Some(&opentelemetry_proto::tonic::common::v1::any_value::Value::StringValue(value.into()))
+            })).unwrap();
+            assert_eq!(point.value, Some(number_data_point::Value::AsInt(count)));
+            assert_eq!(point.attributes.len(), 1, "no device/address labels");
+        }
+    }
 }