feat(telemetry): expose current fleet device and endpoint counts
This commit is contained in:
12 files changed
+355
-1
No files matched your search
@@ -13,5 +13,6 @@ pub use enroll::{
|
||||
set_agent_nickname,
|
||||
};
|
||||
pub(crate) use fleet::FleetCache;
|
||||
pub(crate) use fleet::sample_fleet_counts;
|
||||
pub use fleet::{list_fleet_devices, refresh_fleet_devices, wake_fleet_device};
|
||||
pub use stats::{StateStatsResponse, state_stats};
|
||||
@@ -49,6 +49,35 @@ async fn fleet_view(state: &AppState) -> anyhow::Result<std::sync::Arc<Vec<Fleet
|
||||
.await
|
||||
}
|
||||
|
||||
/// Count stored rows separately from grouped identities. Reject mixed revisions;
|
||||
/// filtered UI results must never become the source of current-state metrics.
|
||||
pub(crate) async fn sample_fleet_counts(
|
||||
state: &AppState,
|
||||
) -> anyhow::Result<wakey_core::telemetry::FleetCounts> {
|
||||
for _ in 0..3 {
|
||||
let revision = state.store.fleet_revision();
|
||||
let mut counts = wakey_core::telemetry::FleetCounts::default();
|
||||
for (agent, _) in state.store.list_agents_with_nicknames().await? {
|
||||
counts.agents.entry(agent).or_default();
|
||||
}
|
||||
for row in state.store.list_agent_device_rows().await? {
|
||||
let agent = counts.agents.entry(row.device.agent_id).or_default();
|
||||
agent.devices += 1;
|
||||
agent.endpoints += row.endpoints.len() as u64;
|
||||
}
|
||||
let fleet = fleet_view(state).await?;
|
||||
counts.saved = fleet
|
||||
.iter()
|
||||
.filter(|device| device.known_device.is_some())
|
||||
.count() as u64;
|
||||
counts.unsaved = fleet.len() as u64 - counts.saved;
|
||||
if state.store.fleet_revision() == revision {
|
||||
return Ok(counts);
|
||||
}
|
||||
}
|
||||
Err(crate::runtime::view_cache::RebuildLimit.into())
|
||||
}
|
||||
|
||||
const REFRESH_CONCURRENCY: usize = 8;
|
||||
|
||||
fn fleet_error(err: anyhow::Error, fallback_code: &str) -> ApiError {
|
||||
|
||||
@@ -53,11 +53,18 @@ async fn fleet_cache_tracks_store_edits_and_connections_without_caching_filters(
|
||||
fleet: Arc::new(super::FleetCache::default()),
|
||||
};
|
||||
let store = &state.store;
|
||||
assert_eq!(
|
||||
super::sample_fleet_counts(&state).await.unwrap(),
|
||||
Default::default()
|
||||
);
|
||||
let token = store
|
||||
.issue_enroll_token(Duration::from_secs(60))
|
||||
.await
|
||||
.unwrap();
|
||||
let agent = store.enroll(&token.enroll_token).await.unwrap();
|
||||
let empty = super::sample_fleet_counts(&state).await.unwrap();
|
||||
assert_eq!(empty.agents[&agent.agent_id].devices, 0);
|
||||
assert_eq!(empty.agents[&agent.agent_id].endpoints, 0);
|
||||
let observed = wakey_core::Device::from_parts(
|
||||
vec![wakey_core::NeighborEntry {
|
||||
ip: "192.168.1.10".parse().unwrap(),
|
||||
@@ -73,6 +80,10 @@ async fn fleet_cache_tracks_store_edits_and_connections_without_caching_filters(
|
||||
.unwrap();
|
||||
let initial = super::fleet_view(&state).await.unwrap();
|
||||
assert_eq!(initial.len(), 1);
|
||||
let counts = super::sample_fleet_counts(&state).await.unwrap();
|
||||
assert_eq!(counts.agents[&agent.agent_id].devices, 1);
|
||||
assert_eq!(counts.agents[&agent.agent_id].endpoints, 1);
|
||||
assert_eq!((counts.saved, counts.unsaved), (0, 1));
|
||||
assert!(Arc::ptr_eq(
|
||||
&initial,
|
||||
&super::fleet_view(&state).await.unwrap()
|
||||
@@ -148,6 +159,8 @@ async fn fleet_cache_tracks_store_edits_and_connections_without_caching_filters(
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
let counts = super::sample_fleet_counts(&state).await.unwrap();
|
||||
assert_eq!((counts.saved, counts.unsaved), (1, 0));
|
||||
let all = ListFleetDevicesQuery {
|
||||
visibility: Some("all".into()),
|
||||
..Default::default()
|
||||
@@ -162,6 +175,8 @@ async fn fleet_cache_tracks_store_edits_and_connections_without_caching_filters(
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(super::fleet_view(&state).await.unwrap().len(), 2);
|
||||
let counts = super::sample_fleet_counts(&state).await.unwrap();
|
||||
assert_eq!((counts.saved, counts.unsaved), (1, 1));
|
||||
store
|
||||
.attach_device_identifier(
|
||||
&saved.device_id,
|
||||
@@ -195,6 +210,8 @@ async fn fleet_cache_tracks_store_edits_and_connections_without_caching_filters(
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
let counts = super::sample_fleet_counts(&state).await.unwrap();
|
||||
assert_eq!((counts.saved, counts.unsaved), (2, 0));
|
||||
assert_eq!(
|
||||
super::load_fleet_devices(&state, &all).await.unwrap().len(),
|
||||
2
|
||||
@@ -212,6 +229,9 @@ async fn fleet_cache_tracks_store_edits_and_connections_without_caching_filters(
|
||||
);
|
||||
|
||||
state.sessions.write().await.clear();
|
||||
let counts = super::sample_fleet_counts(&state).await.unwrap();
|
||||
assert_eq!(counts.agents[&agent.agent_id].devices, 1);
|
||||
assert_eq!((counts.saved, counts.unsaved), (0, 1));
|
||||
assert!(
|
||||
super::fleet_view(&state).await.unwrap()[0]
|
||||
.recommended_route
|
||||
@@ -234,6 +254,41 @@ async fn fleet_cache_tracks_store_edits_and_connections_without_caching_filters(
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(super::fleet_view(&state).await.unwrap().is_empty());
|
||||
assert_eq!(
|
||||
super::sample_fleet_counts(&state).await.unwrap(),
|
||||
Default::default()
|
||||
);
|
||||
// Identical devices on two agents count twice as stored rows, once as a fleet identity.
|
||||
// Offline endpoints are retained too; these metrics are not reachability counters.
|
||||
let offline = wakey_core::Device::from_parts(
|
||||
vec![wakey_core::NeighborEntry {
|
||||
ip: "192.168.1.10".parse().unwrap(),
|
||||
mac: Some("aa:bb:cc:dd:ee:01".parse().unwrap()),
|
||||
dev: Some("br-lan".into()),
|
||||
state: wakey_core::NeighborState::Failed,
|
||||
}],
|
||||
vec![],
|
||||
);
|
||||
for id in ["agent-a", "agent-b"] {
|
||||
store
|
||||
.replace_agent_device_snapshot(id, std::slice::from_ref(&offline))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let counts = super::sample_fleet_counts(&state).await.unwrap();
|
||||
assert_eq!(counts.agents.len(), 2);
|
||||
for count in counts.agents.values() {
|
||||
assert_eq!(count.devices, 1);
|
||||
assert_eq!(count.endpoints, 1);
|
||||
}
|
||||
assert_eq!((counts.saved, counts.unsaved), (0, 1));
|
||||
store
|
||||
.replace_agent_device_snapshot("agent-a", &[])
|
||||
.await
|
||||
.unwrap();
|
||||
let counts = super::sample_fleet_counts(&state).await.unwrap();
|
||||
assert!(!counts.agents.contains_key("agent-a"));
|
||||
assert_eq!(counts.agents["agent-b"].devices, 1);
|
||||
drop(state);
|
||||
std::fs::remove_dir_all(dir).unwrap();
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ mod commands;
|
||||
mod control;
|
||||
mod terminals;
|
||||
pub(crate) use control::FleetCache;
|
||||
pub(crate) use control::sample_fleet_counts;
|
||||
|
||||
pub use alerts::{AlertsHub, active_alerts, alert_history, alerts_stream, run_alert_evaluator};
|
||||
pub use audit::list_audit_events;
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use wakey_core::telemetry::FleetMetrics;
|
||||
|
||||
use super::AppState;
|
||||
|
||||
/// Tie polling to the daemon lifetime, including early server errors.
|
||||
pub(super) struct FleetMetricsTask(tokio::task::JoinHandle<()>);
|
||||
|
||||
impl FleetMetricsTask {
|
||||
pub(super) fn start(state: AppState) -> Option<Self> {
|
||||
let metrics = FleetMetrics::new()?;
|
||||
Some(Self(tokio::spawn(async move {
|
||||
let mut tick = tokio::time::interval(Duration::from_secs(30));
|
||||
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
loop {
|
||||
tick.tick().await;
|
||||
match crate::api::sample_fleet_counts(&state).await {
|
||||
Ok(sample) => metrics.replace(Some(sample)),
|
||||
Err(error) => {
|
||||
metrics.replace(None);
|
||||
tracing::warn!(%error, "failed to refresh fleet metrics");
|
||||
}
|
||||
}
|
||||
}
|
||||
})))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FleetMetricsTask {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ use crate::state;
|
||||
use crate::ws;
|
||||
|
||||
mod admin;
|
||||
mod fleet_metrics;
|
||||
mod process;
|
||||
pub mod terminals;
|
||||
pub(crate) mod view_cache;
|
||||
@@ -196,6 +197,7 @@ pub async fn serve(daemon: config::DaemonConfig) -> Result<()> {
|
||||
};
|
||||
|
||||
tokio::spawn(api::run_alert_evaluator(app_state.clone()));
|
||||
let _fleet_metrics = fleet_metrics::FleetMetricsTask::start(app_state.clone());
|
||||
|
||||
// Keep route classes explicit so edge policy can map directly:
|
||||
// - public_api_routes: intended internet-facing agent endpoints
|
||||
|
||||
Reference in new issue
Block a user