alert history work

- dedup error handling
- alert now clears with observation_retenstion_seconds
- evaluate recent alerts once per server per 5s
- some iter stuff, some type stuff, and some store api stuff.

wakey-control-plane/src/api/alerts.rs

@@ -1,4 +1,5 @@
 use std::collections::{BTreeMap, HashSet};
+use std::sync::Arc;
 use std::time::{SystemTime, UNIX_EPOCH};
 
 use axum::Json;
@@ -7,12 +8,13 @@ use axum::extract::{Query, State};
 use axum::http::StatusCode;
 use axum::response::IntoResponse;
 use serde::Deserialize;
+use tokio::sync::{RwLock, broadcast};
 use tokio::time::Duration;
 use tracing::warn;
 
 use crate::api::ApiError;
 use crate::runtime::AppState;
-use crate::state::{AlertState, AuditEvent, AuditEventFilter};
+use crate::state::{AlertState, AlertTransition, AuditEvent, AuditEventFilter};
 
 struct AlertBuildContext<'a> {
     now_unix: u64,
@@ -26,21 +28,13 @@ struct AlertBuildContext<'a> {
     enroll_rejected_threshold: u64,
 }
 
-#[derive(Debug, Deserialize)]
-pub struct ActiveAlertsQuery {
-    pub lookback_seconds: Option<u64>,
-    pub timeout_threshold: Option<u64>,
-    pub auth_rejected_threshold: Option<u64>,
-    pub enroll_rejected_threshold: Option<u64>,
-}
-
 #[derive(Debug, Deserialize)]
 pub struct AlertHistoryQuery {
     pub since_unix: Option<u64>,
     pub limit: Option<usize>,
 }
 
-#[derive(Debug, Deserialize, Clone)]
+#[derive(Debug, Clone, Deserialize)]
 pub struct AlertRuleConfig {
     pub lookback_seconds: Option<u64>,
     pub timeout_threshold: Option<u64>,
@@ -48,25 +42,120 @@ pub struct AlertRuleConfig {
     pub enroll_rejected_threshold: Option<u64>,
 }
 
-pub async fn active_alerts(
-    State(state): State<AppState>,
-    Query(query): Query<ActiveAlertsQuery>,
-) -> Result<impl IntoResponse, ApiError> {
-    let alerts = evaluate_alerts(
-        &state,
-        AlertRuleConfig {
-            lookback_seconds: query.lookback_seconds,
-            timeout_threshold: query.timeout_threshold,
-            auth_rejected_threshold: query.auth_rejected_threshold,
-            enroll_rejected_threshold: query.enroll_rejected_threshold,
-        },
-    )
-    .await?;
+impl Default for AlertRuleConfig {
+    fn default() -> Self {
+        Self {
+            lookback_seconds: Some(900),
+            timeout_threshold: Some(3),
+            auth_rejected_threshold: Some(3),
+            enroll_rejected_threshold: Some(5),
+        }
+    }
+}
 
-    if let Err(err) = state.store.sync_alert_transitions(&alerts).await {
-        warn!(error = %err, "failed to sync alert transitions");
+#[derive(Clone, Default)]
+pub struct AlertsSnapshot {
+    pub ts_unix: u64,
+    pub alerts: Vec<AlertState>,
+    pub recent_transitions: Vec<AlertTransition>,
+}
+
+/// Holds the latest alert snapshot computed by the background evaluator and
+/// broadcasts each new snapshot to connected `alerts/ws` subscribers.
+#[derive(Clone)]
+pub struct AlertsHub {
+    tx: broadcast::Sender<String>,
+    latest_payload: Arc<RwLock<Option<String>>>,
+    latest_alerts: Arc<RwLock<Vec<AlertState>>>,
+}
+
+impl Default for AlertsHub {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl AlertsHub {
+    pub fn new() -> Self {
+        let (tx, _) = broadcast::channel(16);
+        Self {
+            tx,
+            latest_payload: Arc::new(RwLock::new(None)),
+            latest_alerts: Arc::new(RwLock::new(Vec::new())),
+        }
     }
 
+    pub fn subscribe(&self) -> broadcast::Receiver<String> {
+        self.tx.subscribe()
+    }
+
+    pub async fn latest_alerts(&self) -> Vec<AlertState> {
+        self.latest_alerts.read().await.clone()
+    }
+
+    pub async fn latest_payload(&self) -> Option<String> {
+        self.latest_payload.read().await.clone()
+    }
+
+    pub async fn publish(&self, snapshot: AlertsSnapshot) {
+        let payload = serde_json::json!({
+            "type": "alerts_snapshot",
+            "ts_unix": snapshot.ts_unix,
+            "alerts": snapshot.alerts,
+            "recent_transitions": snapshot.recent_transitions,
+        });
+        let Ok(encoded) = serde_json::to_string(&payload) else {
+            warn!("failed to encode alerts snapshot payload");
+            return;
+        };
+        *self.latest_alerts.write().await = snapshot.alerts;
+        *self.latest_payload.write().await = Some(encoded.clone());
+        let _ = self.tx.send(encoded);
+    }
+}
+
+/// Background task that owns alert evaluation: on a fixed ticker it recomputes
+/// the snapshot, persists transitions, and publishes to subscribers so neither
+/// the GET endpoint nor individual WS clients evaluate alerts themselves.
+pub async fn run_alert_evaluator(state: AppState) {
+    let config = AlertRuleConfig::default();
+    let mut tick = tokio::time::interval(Duration::from_secs(5));
+    tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+    loop {
+        tick.tick().await;
+        let alerts = match evaluate_alerts(&state, &config).await {
+            Ok(alerts) => alerts,
+            Err(err) => {
+                warn!(code = %err.code, "failed to evaluate alerts in background task");
+                continue;
+            }
+        };
+
+        if let Err(err) = state.store.sync_alert_transitions(&alerts).await {
+            warn!(error = %err, "failed to sync alert transitions in background task");
+        }
+
+        let recent_transitions = match state.store.list_alert_transitions(None, 20).await {
+            Ok(history) => history,
+            Err(err) => {
+                warn!(error = %err, "failed to load alert transition history for snapshot");
+                Vec::new()
+            }
+        };
+
+        state
+            .alerts
+            .publish(AlertsSnapshot {
+                ts_unix: now_unix(),
+                alerts,
+                recent_transitions,
+            })
+            .await;
+    }
+}
+
+pub async fn active_alerts(State(state): State<AppState>) -> Result<impl IntoResponse, ApiError> {
+    let alerts = state.alerts.latest_alerts().await;
     Ok((StatusCode::OK, Json(alerts)))
 }
 
@@ -94,64 +183,34 @@ pub async fn alert_history(
 pub async fn alerts_stream(
     ws: WebSocketUpgrade,
     State(state): State<AppState>,
-    Query(query): Query<ActiveAlertsQuery>,
 ) -> impl IntoResponse {
-    let config = AlertRuleConfig {
-        lookback_seconds: query.lookback_seconds,
-        timeout_threshold: query.timeout_threshold,
-        auth_rejected_threshold: query.auth_rejected_threshold,
-        enroll_rejected_threshold: query.enroll_rejected_threshold,
-    };
-    ws.on_upgrade(move |socket| alerts_stream_socket(state, socket, config))
+    ws.on_upgrade(move |socket| alerts_stream_socket(state, socket))
 }
 
-async fn alerts_stream_socket(state: AppState, mut socket: WebSocket, config: AlertRuleConfig) {
-    let mut tick = tokio::time::interval(Duration::from_secs(5));
-    loop {
-        tick.tick().await;
-        let alerts = match evaluate_alerts(&state, config.clone()).await {
-            Ok(alerts) => alerts,
-            Err(err) => {
-                warn!(code = %err.code, "failed to evaluate alerts for stream");
-                continue;
-            }
-        };
+async fn alerts_stream_socket(state: AppState, mut socket: WebSocket) {
+    let mut rx = state.alerts.subscribe();
 
-        if let Err(err) = state.store.sync_alert_transitions(&alerts).await {
-            warn!(error = %err, "failed to sync alert transitions in stream");
+    if let Some(encoded) = state.alerts.latest_payload().await
+        && socket.send(Message::Text(encoded.into())).await.is_err() {
+            return;
         }
 
-        let history = match state.store.list_alert_transitions(None, 20).await {
-            Ok(h) => h,
-            Err(err) => {
-                warn!(error = %err, "failed to load alert transition history for stream");
-                Vec::new()
+    loop {
+        match rx.recv().await {
+            Ok(encoded) => {
+                if socket.send(Message::Text(encoded.into())).await.is_err() {
+                    break;
+                }
             }
-        };
-
-        let payload = serde_json::json!({
-            "type": "alerts_snapshot",
-            "ts_unix": now_unix(),
-            "alerts": alerts,
-            "recent_transitions": history,
-        });
-        let encoded = match serde_json::to_string(&payload) {
-            Ok(s) => s,
-            Err(err) => {
-                warn!(error = %err, "failed to encode alerts stream payload");
-                continue;
-            }
-        };
-
-        if socket.send(Message::Text(encoded.into())).await.is_err() {
-            break;
+            Err(broadcast::error::RecvError::Lagged(_)) => continue,
+            Err(broadcast::error::RecvError::Closed) => break,
         }
     }
 }
 
 async fn evaluate_alerts(
     state: &AppState,
-    config: AlertRuleConfig,
+    config: &AlertRuleConfig,
 ) -> Result<Vec<AlertState>, ApiError> {
     let lookback_seconds = config.lookback_seconds.unwrap_or(900).clamp(60, 86_400);
     let timeout_threshold = config.timeout_threshold.unwrap_or(3).max(1);
@@ -161,7 +220,14 @@ async fn evaluate_alerts(
     let now = now_unix();
     let since_unix = now.saturating_sub(lookback_seconds);
 
-    let enrolled_agents = state.store.list_agents().await;
+    let enrolled_agents = state.store.list_agents().await.map_err(|err| {
+        warn!(error = %err, "failed listing agents for alert evaluation");
+        ApiError::new(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "alerts_query_failed",
+            err.to_string(),
+        )
+    })?;
     let connected_agents = state
         .sessions
         .read()

wakey-control-plane/src/api/commands.rs

@@ -40,7 +40,18 @@ pub struct RelayCommandResponse {
 }
 
 pub async fn list_agents(State(state): State<AppState>) -> Result<impl IntoResponse, ApiError> {
-    let enrolled = state.store.list_agents_with_nicknames().await;
+    let enrolled = state
+        .store
+        .list_agents_with_nicknames()
+        .await
+        .map_err(|err| {
+            warn!(error = %err, "failed listing agents");
+            ApiError::new(
+                StatusCode::INTERNAL_SERVER_ERROR,
+                "list_agents_failed",
+                err.to_string(),
+            )
+        })?;
     let sessions = state.sessions.read().await;
 
     let agents = enrolled
@@ -120,9 +131,9 @@ pub async fn relay_agent_command(
         .insert(request_id_string.clone(), pending_tx);
 
     info!("dispatching command to agent");
-    if let Err(err) = state
+    state
         .store
-        .append_audit_event(AuditEventInput {
+        .audit(AuditEventInput {
             actor_type: "admin_api".into(),
             actor_id: None,
             agent_id: Some(agent_id.to_string()),
@@ -133,10 +144,7 @@ pub async fn relay_agent_command(
             message: "dispatched command to connected agent".into(),
             metadata: serde_json::json!({ "command": command_kind }),
         })
-        .await
-    {
-        warn!(error = %err, "failed to append audit event for command dispatch");
-    }
+        .await;
 
     if let Err(err) = tx.send(SessionEvent::Message(ServerMessage::Command {
         request_id,
@@ -144,9 +152,9 @@ pub async fn relay_agent_command(
     })) {
         state.pending.lock().await.remove(&request_id_string);
         warn!(error = %err, "failed sending command to agent session");
-        if let Err(audit_err) = state
+        state
             .store
-            .append_audit_event(AuditEventInput {
+            .audit(AuditEventInput {
                 actor_type: "admin_api".into(),
                 actor_id: None,
                 agent_id: Some(agent_id.to_string()),
@@ -157,10 +165,7 @@ pub async fn relay_agent_command(
                 message: err.to_string(),
                 metadata: serde_json::json!({ "command": command_kind }),
             })
-            .await
-        {
-            warn!(error = %audit_err, "failed to append audit event for command send failure");
-        }
+            .await;
         return Err(ApiError::new(
             StatusCode::BAD_GATEWAY,
             "agent_send_failed",
@@ -177,9 +182,9 @@ pub async fn relay_agent_command(
     let response = match outcome {
         Ok(Ok(AgentReply::Result(result))) => {
             info!("agent command completed");
-            if let Err(err) = state
+            state
                 .store
-                .append_audit_event(AuditEventInput {
+                .audit(AuditEventInput {
                     actor_type: "admin_api".into(),
                     actor_id: None,
                     agent_id: Some(agent_id.to_string()),
@@ -190,10 +195,7 @@ pub async fn relay_agent_command(
                     message: "agent command completed".into(),
                     metadata: serde_json::json!({ "command": command_kind }),
                 })
-                .await
-            {
-                warn!(error = %err, "failed to append audit event for command success");
-            }
+                .await;
             RelayCommandResponse {
                 request_id: request_id_string,
                 status: "ok".into(),
@@ -203,9 +205,9 @@ pub async fn relay_agent_command(
         }
         Ok(Ok(AgentReply::Error(error))) => {
             warn!(code = %error.code, "agent command returned error");
-            if let Err(err) = state
+            state
                 .store
-                .append_audit_event(AuditEventInput {
+                .audit(AuditEventInput {
                     actor_type: "admin_api".into(),
                     actor_id: None,
                     agent_id: Some(agent_id.to_string()),
@@ -216,10 +218,7 @@ pub async fn relay_agent_command(
                     message: error.message.clone(),
                     metadata: serde_json::json!({ "command": command_kind, "code": error.code }),
                 })
-                .await
-            {
-                warn!(error = %err, "failed to append audit event for command error result");
-            }
+                .await;
             RelayCommandResponse {
                 request_id: request_id_string,
                 status: "error".into(),
@@ -229,9 +228,9 @@ pub async fn relay_agent_command(
         }
         Ok(Err(_)) => {
             warn!("agent response channel dropped");
-            if let Err(err) = state
+            state
                 .store
-                .append_audit_event(AuditEventInput {
+                .audit(AuditEventInput {
                     actor_type: "admin_api".into(),
                     actor_id: None,
                     agent_id: Some(agent_id.to_string()),
@@ -242,10 +241,7 @@ pub async fn relay_agent_command(
                     message: "agent response channel dropped".into(),
                     metadata: serde_json::json!({ "command": command_kind }),
                 })
-                .await
-            {
-                warn!(error = %err, "failed to append audit event for dropped response");
-            }
+                .await;
             return Err(ApiError::new(
                 StatusCode::BAD_GATEWAY,
                 "agent_response_dropped",
@@ -258,9 +254,9 @@ pub async fn relay_agent_command(
                 timeout_ms = timeout.as_millis() as u64,
                 "agent command timed out"
             );
-            if let Err(err) = state
+            state
                 .store
-                .append_audit_event(AuditEventInput {
+                .audit(AuditEventInput {
                     actor_type: "admin_api".into(),
                     actor_id: None,
                     agent_id: Some(agent_id.to_string()),
@@ -271,10 +267,7 @@ pub async fn relay_agent_command(
                     message: "agent command timed out".into(),
                     metadata: serde_json::json!({ "command": command_kind, "timeout_ms": timeout.as_millis() as u64 }),
                 })
-                .await
-            {
-                warn!(error = %err, "failed to append audit event for timeout");
-            }
+                .await;
             return Err(ApiError::new(
                 StatusCode::GATEWAY_TIMEOUT,
                 "agent_timeout",

wakey-control-plane/src/api/control/enroll.rs

@@ -74,9 +74,9 @@ pub async fn enroll(
     match state.store.enroll(&req.enroll_token).await {
         Ok(issued) => {
             info!(agent_id = %issued.agent_id, "agent enrollment accepted");
-            if let Err(err) = state
+            state
                 .store
-                .append_audit_event(AuditEventInput {
+                .audit(AuditEventInput {
                     actor_type: "agent".into(),
                     actor_id: Some(issued.agent_id.clone()),
                     agent_id: Some(issued.agent_id.clone()),
@@ -87,10 +87,7 @@ pub async fn enroll(
                     message: "agent enrollment accepted".into(),
                     metadata: serde_json::json!({}),
                 })
-                .await
-            {
-                warn!(error = %err, "failed to append audit event for enroll success");
-            }
+                .await;
             Ok((
                 StatusCode::OK,
                 Json(EnrollResponse {
@@ -102,9 +99,9 @@ pub async fn enroll(
         }
         Err(err) => {
             warn!(error = %err, "agent enrollment rejected");
-            if let Err(audit_err) = state
+            state
                 .store
-                .append_audit_event(AuditEventInput {
+                .audit(AuditEventInput {
                     actor_type: "agent".into(),
                     actor_id: None,
                     agent_id: None,
@@ -115,10 +112,7 @@ pub async fn enroll(
                     message: err.to_string(),
                     metadata: serde_json::json!({}),
                 })
-                .await
-            {
-                warn!(error = %audit_err, "failed to append audit event for enroll rejection");
-            }
+                .await;
             Err(ApiError::new(
                 StatusCode::UNAUTHORIZED,
                 "enrollment_rejected",
@@ -145,9 +139,9 @@ pub async fn issue_enroll_token(
                 expires_at_unix = issued.expires_at_unix,
                 "issued enroll token"
             );
-            if let Err(err) = state
+            state
                 .store
-                .append_audit_event(AuditEventInput {
+                .audit(AuditEventInput {
                     actor_type: "admin_api".into(),
                     actor_id: None,
                     agent_id: None,
@@ -161,10 +155,7 @@ pub async fn issue_enroll_token(
                         "expires_at_unix": issued.expires_at_unix,
                     }),
                 })
-                .await
-            {
-                warn!(error = %err, "failed to append audit event for token issuance");
-            }
+                .await;
             Ok((
                 StatusCode::OK,
                 Json(IssueEnrollTokenResponse {
@@ -189,9 +180,9 @@ pub async fn list_enroll_tokens(
 ) -> Result<impl IntoResponse, ApiError> {
     match state.store.list_enroll_tokens().await {
         Ok(tokens) => {
-            if let Err(err) = state
+            state
                 .store
-                .append_audit_event(AuditEventInput {
+                .audit(AuditEventInput {
                     actor_type: "admin_api".into(),
                     actor_id: None,
                     agent_id: None,
@@ -204,10 +195,7 @@ pub async fn list_enroll_tokens(
                         "count": tokens.len(),
                     }),
                 })
-                .await
-            {
-                warn!(error = %err, "failed to append audit event for token listing");
-            }
+                .await;
             let body = tokens
                 .into_iter()
                 .map(|t| EnrollTokenStatus {
@@ -235,9 +223,9 @@ pub async fn revoke_enroll_token(
 ) -> Result<impl IntoResponse, ApiError> {
     match state.store.revoke_enroll_token(&token).await {
         Ok(revoked) => {
-            if let Err(err) = state
+            state
                 .store
-                .append_audit_event(AuditEventInput {
+                .audit(AuditEventInput {
                     actor_type: "admin_api".into(),
                     actor_id: None,
                     agent_id: None,
@@ -256,10 +244,7 @@ pub async fn revoke_enroll_token(
                     },
                     metadata: serde_json::json!({ "token": token }),
                 })
-                .await
-            {
-                warn!(error = %err, "failed to append audit event for token revoke");
-            }
+                .await;
             Ok((
                 StatusCode::OK,
                 Json(RevokeEnrollTokenResponse { token, revoked }),
@@ -289,9 +274,9 @@ pub async fn revoke_agent(
                 }
             }
 
-            if let Err(err) = state
+            state
                 .store
-                .append_audit_event(AuditEventInput {
+                .audit(AuditEventInput {
                     actor_type: "admin_api".into(),
                     actor_id: None,
                     agent_id: Some(agent_id.clone()),
@@ -310,10 +295,7 @@ pub async fn revoke_agent(
                     },
                     metadata: serde_json::json!({ "agent_id": agent_id }),
                 })
-                .await
-            {
-                warn!(error = %err, "failed to append audit event for agent revoke");
-            }
+                .await;
 
             Ok((
                 StatusCode::OK,
@@ -349,9 +331,9 @@ pub async fn set_agent_nickname(
         .await
     {
         Ok(updated) => {
-            if let Err(err) = state
+            state
                 .store
-                .append_audit_event(AuditEventInput {
+                .audit(AuditEventInput {
                     actor_type: "admin_api".into(),
                     actor_id: None,
                     agent_id: Some(agent_id.clone()),
@@ -373,10 +355,7 @@ pub async fn set_agent_nickname(
                         "nickname": normalized,
                     }),
                 })
-                .await
-            {
-                warn!(error = %err, "failed to append audit event for nickname set");
-            }
+                .await;
 
             Ok((
                 StatusCode::OK,

wakey-control-plane/src/api/control/fleet/mod.rs

@@ -4,11 +4,12 @@ use axum::Json;
 use axum::extract::{Query, State};
 use axum::http::StatusCode;
 use axum::response::IntoResponse;
+use futures_util::StreamExt;
 use tracing::warn;
 use wakey_agent::protocol::{AgentCommand, InventoryRequest, WakeRequest};
 
 use crate::api::ApiError;
-use crate::api::commands::relay_agent_command;
+use crate::api::commands::{RelayCommandResponse, relay_agent_command};
 use crate::runtime::AppState;
 
 mod build;
@@ -26,6 +27,9 @@ use types::{
     RefreshFleetDevicesResponse, WakeFleetDeviceRequest, WakeFleetDeviceResponse,
 };
 
+const MAX_REFRESH_AGENTS: usize = 128;
+const REFRESH_CONCURRENCY: usize = 8;
+
 pub async fn list_fleet_devices(
     State(state): State<AppState>,
     Query(query): Query<ListFleetDevicesQuery>,
@@ -55,88 +59,43 @@ pub async fn refresh_fleet_devices(
     };
     agent_ids.sort();
     agent_ids.dedup();
-
-    let mut results = Vec::with_capacity(agent_ids.len());
-    let mut total_accepted = 0usize;
-    for agent_id in agent_ids {
-        let response = relay_agent_command(
-            &state,
-            &agent_id,
-            AgentCommand::Inventory(InventoryRequest {
-                query: None,
-                name: None,
-                ips: Vec::new(),
-                devs: Vec::new(),
-                nuds: Vec::new(),
-                macs: Vec::new(),
-            }),
-            req.timeout_ms,
-        )
-        .await;
-
-        match response {
-            Ok(response) if response.status == "ok" => {
-                let Some(result) = response.result else {
-                    results.push(RefreshFleetAgentResult {
-                        agent_id,
-                        status: "error".into(),
-                        accepted: 0,
-                        error: Some("inventory command returned no result".into()),
-                    });
-                    continue;
-                };
-                match serde_json::from_value::<wakey_core::DeviceInventory>(result)
-                    .map(|i| i.devices)
-                {
-                    Ok(devices) => {
-                        match state
-                            .store
-                            .replace_agent_device_snapshot(&agent_id, &devices)
-                            .await
-                        {
-                            Ok(accepted) => {
-                                total_accepted = total_accepted.saturating_add(accepted);
-                                results.push(RefreshFleetAgentResult {
-                                    agent_id,
-                                    status: "ok".into(),
-                                    accepted,
-                                    error: None,
-                                });
-                            }
-                            Err(err) => results.push(RefreshFleetAgentResult {
-                                agent_id,
-                                status: "error".into(),
-                                accepted: 0,
-                                error: Some(err.to_string()),
-                            }),
-                        }
-                    }
-                    Err(err) => results.push(RefreshFleetAgentResult {
-                        agent_id,
-                        status: "error".into(),
-                        accepted: 0,
-                        error: Some(err.to_string()),
-                    }),
-                }
-            }
-            Ok(response) => results.push(RefreshFleetAgentResult {
-                agent_id,
-                status: "error".into(),
-                accepted: 0,
-                error: response
-                    .error
-                    .map(|error| error.message)
-                    .or_else(|| Some("inventory command failed".into())),
-            }),
-            Err(err) => results.push(RefreshFleetAgentResult {
-                agent_id,
-                status: "error".into(),
-                accepted: 0,
-                error: Some(format!("{}: {}", err.status, err.message)),
-            }),
-        }
+    if agent_ids.len() > MAX_REFRESH_AGENTS {
+        warn!(
+            requested = agent_ids.len(),
+            capped = MAX_REFRESH_AGENTS,
+            "capping fleet refresh agent count"
+        );
+        agent_ids.truncate(MAX_REFRESH_AGENTS);
     }
 
+    let results = futures_util::stream::iter(agent_ids.into_iter().map(|agent_id| {
+        let state = state.clone();
+        async move {
+            let response = relay_agent_command(
+                &state,
+                &agent_id,
+                AgentCommand::Inventory(InventoryRequest {
+                    query: None,
+                    name: None,
+                    ips: Vec::new(),
+                    devs: Vec::new(),
+                    nuds: Vec::new(),
+                    macs: Vec::new(),
+                }),
+                req.timeout_ms,
+            )
+            .await;
+            refresh_fleet_agent_result(&state, &agent_id, response).await
+        }
+    }))
+    .buffer_unordered(REFRESH_CONCURRENCY)
+    .collect::<Vec<_>>()
+    .await;
+
+    let mut results = results;
+    results.sort_by(|a, b| a.agent_id.cmp(&b.agent_id));
+    let total_accepted = results.iter().map(|result| result.accepted).sum::<usize>();
+
     Ok((
         StatusCode::OK,
         Json(RefreshFleetDevicesResponse {
@@ -146,6 +105,70 @@ pub async fn refresh_fleet_devices(
     ))
 }
 
+async fn refresh_fleet_agent_result(
+    state: &AppState,
+    agent_id: &str,
+    response: Result<RelayCommandResponse, ApiError>,
+) -> RefreshFleetAgentResult {
+    match response {
+        Ok(response) if response.status == "ok" => {
+            let Some(result) = response.result else {
+                return RefreshFleetAgentResult {
+                    agent_id: agent_id.to_string(),
+                    status: "error".into(),
+                    accepted: 0,
+                    error: Some("inventory command returned no result".into()),
+                };
+            };
+            match serde_json::from_value::<wakey_core::DeviceInventory>(result)
+                .map(|inventory| inventory.devices)
+            {
+                Ok(devices) => {
+                    match state
+                        .store
+                        .replace_agent_device_snapshot(agent_id, &devices)
+                        .await
+                    {
+                        Ok(accepted) => RefreshFleetAgentResult {
+                            agent_id: agent_id.to_string(),
+                            status: "ok".into(),
+                            accepted,
+                            error: None,
+                        },
+                        Err(err) => RefreshFleetAgentResult {
+                            agent_id: agent_id.to_string(),
+                            status: "error".into(),
+                            accepted: 0,
+                            error: Some(err.to_string()),
+                        },
+                    }
+                }
+                Err(err) => RefreshFleetAgentResult {
+                    agent_id: agent_id.to_string(),
+                    status: "error".into(),
+                    accepted: 0,
+                    error: Some(err.to_string()),
+                },
+            }
+        }
+        Ok(response) => RefreshFleetAgentResult {
+            agent_id: agent_id.to_string(),
+            status: "error".into(),
+            accepted: 0,
+            error: response
+                .error
+                .map(|error| error.message)
+                .or_else(|| Some("inventory command failed".into())),
+        },
+        Err(err) => RefreshFleetAgentResult {
+            agent_id: agent_id.to_string(),
+            status: "error".into(),
+            accepted: 0,
+            error: Some(format!("{}: {}", err.status, err.message)),
+        },
+    }
+}
+
 pub async fn wake_fleet_device(
     State(state): State<AppState>,
     Json(req): Json<WakeFleetDeviceRequest>,
@@ -243,7 +266,7 @@ async fn load_fleet_devices(
         let sessions = state.sessions.read().await;
         sessions.keys().cloned().collect::<BTreeSet<_>>()
     };
-    let agents = state.store.list_agents_with_nicknames().await;
+    let agents = state.store.list_agents_with_nicknames().await?;
     let agent_status = agents
         .into_iter()
         .map(|(agent_id, nickname)| {

wakey-control-plane/src/api/mod.rs

@@ -7,7 +7,7 @@ mod commands;
 mod control;
 mod terminals;
 
-pub use alerts::{active_alerts, alert_history, alerts_stream};
+pub use alerts::{AlertsHub, active_alerts, alert_history, alerts_stream, run_alert_evaluator};
 pub use audit::list_audit_events;
 pub use commands::{list_agents, run_command};
 pub use control::{

wakey-control-plane/src/api/terminals.rs

@@ -649,9 +649,9 @@ async fn append_terminal_audit(
     terminal_id: &str,
     audit: TerminalAudit<'_>,
 ) {
-    if let Err(err) = state
+    state
         .store
-        .append_audit_event(AuditEventInput {
+        .audit(AuditEventInput {
             actor_type: audit.actor_type.into(),
             actor_id: None,
             agent_id: Some(agent_id.to_string()),
@@ -662,10 +662,7 @@ async fn append_terminal_audit(
             message: audit.message.into(),
             metadata: audit.metadata,
         })
-        .await
-    {
-        warn!(terminal_id, event_type = audit.event_type, error = %err, "failed to append terminal audit event");
-    }
+        .await;
 }
 
 fn validate_size(rows: u16, cols: u16) -> Result<(), ApiError> {