alert history work
- dedup error handling - alert now clears with observation_retenstion_seconds - evaluate recent alerts once per server per 5s - some iter stuff, some type stuff, and some store api stuff.
This commit is contained in:
29 files changed
+750
-294
No files matched your search
+12
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "INSERT INTO audit_events\n (event_key, event_id, ts_unix, actor_type, actor_id, agent_id, request_id,\n event_type, outcome, latency_ms, message, metadata_json)\n VALUES (?1, ?2, ?3, 'test', NULL, NULL, NULL, 'test', 'ok', NULL, 'old', '{}')",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 3
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "6ce45cc94cbfdd948aad5fb986f9b5976c79bb63759bc9ecff1a7873e2d70137"
|
||||||
|
}
|
||||||
+20
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "SELECT COUNT(*) FROM alert_transitions WHERE transition_id = ?1",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"name": "COUNT(*)",
|
||||||
|
"ordinal": 0,
|
||||||
|
"type_info": "Integer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 1
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "b324587b94ee2e98dc8eff5f6735303a2784b74f49068ce8e946cedcd8d12c4e"
|
||||||
|
}
|
||||||
+20
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "SELECT COUNT(*) FROM audit_events WHERE message = ?1",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"name": "COUNT(*)",
|
||||||
|
"ordinal": 0,
|
||||||
|
"type_info": "Integer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 1
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "bd1666cea8840df04ddcdc8a0c42b9b7771f9bf63c45613c4b6f8b86be479d70"
|
||||||
|
}
|
||||||
+12
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "DELETE FROM audit_events WHERE ts_unix < ?1",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 1
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "ca612fee8c80f815b33dbc088cc7e0a7552f1f4e38390959e14ea5cc765bf2c7"
|
||||||
|
}
|
||||||
+12
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "DELETE FROM alert_transitions WHERE ts_unix < ?1",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 1
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "ce850a32c8702595bbc3207bf86d53209c48be145562192b3a8f84903d39eac1"
|
||||||
|
}
|
||||||
+20
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "SELECT COUNT(*) FROM audit_events WHERE event_id = ?1",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"name": "COUNT(*)",
|
||||||
|
"ordinal": 0,
|
||||||
|
"type_info": "Integer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 1
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "dc7056b7421dcc83648cbd360273ebcce1156ce0f7da2d39f054025ab9b66b6b"
|
||||||
|
}
|
||||||
+12
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "INSERT INTO alert_transitions\n (transition_key, transition_id, ts_unix, alert_id, kind, agent_id,\n from_status, to_status, message, metadata_json)\n VALUES (?1, ?2, ?3, 'alert-old', 'test', NULL, NULL, 'active', 'old', '{}')",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 3
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "e83a43217a3bfaacd2f5e76df1fd05447558646fb2b47963751a7ab53b3a99b6"
|
||||||
|
}
|
||||||
+12
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "INSERT INTO audit_events\n (event_key, event_id, ts_unix, actor_type, actor_id, agent_id, request_id,\n event_type, outcome, latency_ms, message, metadata_json)\n VALUES (?1, ?2, ?3, 'test', NULL, NULL, NULL, 'test', 'ok', NULL, 'old', '{}')",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 3
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "6ce45cc94cbfdd948aad5fb986f9b5976c79bb63759bc9ecff1a7873e2d70137"
|
||||||
|
}
|
||||||
+20
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "SELECT COUNT(*) FROM alert_transitions WHERE transition_id = ?1",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"name": "COUNT(*)",
|
||||||
|
"ordinal": 0,
|
||||||
|
"type_info": "Integer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 1
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "b324587b94ee2e98dc8eff5f6735303a2784b74f49068ce8e946cedcd8d12c4e"
|
||||||
|
}
|
||||||
+20
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "SELECT COUNT(*) FROM audit_events WHERE message = ?1",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"name": "COUNT(*)",
|
||||||
|
"ordinal": 0,
|
||||||
|
"type_info": "Integer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 1
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "bd1666cea8840df04ddcdc8a0c42b9b7771f9bf63c45613c4b6f8b86be479d70"
|
||||||
|
}
|
||||||
+12
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "DELETE FROM audit_events WHERE ts_unix < ?1",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 1
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "ca612fee8c80f815b33dbc088cc7e0a7552f1f4e38390959e14ea5cc765bf2c7"
|
||||||
|
}
|
||||||
+12
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "DELETE FROM alert_transitions WHERE ts_unix < ?1",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 1
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "ce850a32c8702595bbc3207bf86d53209c48be145562192b3a8f84903d39eac1"
|
||||||
|
}
|
||||||
+20
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "SELECT COUNT(*) FROM audit_events WHERE event_id = ?1",
|
||||||
|
"describe": {
|
||||||
|
"columns": [
|
||||||
|
{
|
||||||
|
"name": "COUNT(*)",
|
||||||
|
"ordinal": 0,
|
||||||
|
"type_info": "Integer"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 1
|
||||||
|
},
|
||||||
|
"nullable": [
|
||||||
|
false
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"hash": "dc7056b7421dcc83648cbd360273ebcce1156ce0f7da2d39f054025ab9b66b6b"
|
||||||
|
}
|
||||||
+12
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"db_name": "SQLite",
|
||||||
|
"query": "INSERT INTO alert_transitions\n (transition_key, transition_id, ts_unix, alert_id, kind, agent_id,\n from_status, to_status, message, metadata_json)\n VALUES (?1, ?2, ?3, 'alert-old', 'test', NULL, NULL, 'active', 'old', '{}')",
|
||||||
|
"describe": {
|
||||||
|
"columns": [],
|
||||||
|
"parameters": {
|
||||||
|
"Right": 3
|
||||||
|
},
|
||||||
|
"nullable": []
|
||||||
|
},
|
||||||
|
"hash": "e83a43217a3bfaacd2f5e76df1fd05447558646fb2b47963751a7ab53b3a99b6"
|
||||||
|
}
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
use std::collections::{BTreeMap, HashSet};
|
use std::collections::{BTreeMap, HashSet};
|
||||||
|
use std::sync::Arc;
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use axum::Json;
|
use axum::Json;
|
||||||
@@ -7,12 +8,13 @@ use axum::extract::{Query, State};
|
|||||||
use axum::http::StatusCode;
|
use axum::http::StatusCode;
|
||||||
use axum::response::IntoResponse;
|
use axum::response::IntoResponse;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
|
use tokio::sync::{RwLock, broadcast};
|
||||||
use tokio::time::Duration;
|
use tokio::time::Duration;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
use crate::api::ApiError;
|
use crate::api::ApiError;
|
||||||
use crate::runtime::AppState;
|
use crate::runtime::AppState;
|
||||||
use crate::state::{AlertState, AuditEvent, AuditEventFilter};
|
use crate::state::{AlertState, AlertTransition, AuditEvent, AuditEventFilter};
|
||||||
|
|
||||||
struct AlertBuildContext<'a> {
|
struct AlertBuildContext<'a> {
|
||||||
now_unix: u64,
|
now_unix: u64,
|
||||||
@@ -26,21 +28,13 @@ struct AlertBuildContext<'a> {
|
|||||||
enroll_rejected_threshold: u64,
|
enroll_rejected_threshold: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
pub struct ActiveAlertsQuery {
|
|
||||||
pub lookback_seconds: Option<u64>,
|
|
||||||
pub timeout_threshold: Option<u64>,
|
|
||||||
pub auth_rejected_threshold: Option<u64>,
|
|
||||||
pub enroll_rejected_threshold: Option<u64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Debug, Deserialize)]
|
#[derive(Debug, Deserialize)]
|
||||||
pub struct AlertHistoryQuery {
|
pub struct AlertHistoryQuery {
|
||||||
pub since_unix: Option<u64>,
|
pub since_unix: Option<u64>,
|
||||||
pub limit: Option<usize>,
|
pub limit: Option<usize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, Clone)]
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
pub struct AlertRuleConfig {
|
pub struct AlertRuleConfig {
|
||||||
pub lookback_seconds: Option<u64>,
|
pub lookback_seconds: Option<u64>,
|
||||||
pub timeout_threshold: Option<u64>,
|
pub timeout_threshold: Option<u64>,
|
||||||
@@ -48,25 +42,120 @@ pub struct AlertRuleConfig {
|
|||||||
pub enroll_rejected_threshold: Option<u64>,
|
pub enroll_rejected_threshold: Option<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn active_alerts(
|
impl Default for AlertRuleConfig {
|
||||||
State(state): State<AppState>,
|
fn default() -> Self {
|
||||||
Query(query): Query<ActiveAlertsQuery>,
|
Self {
|
||||||
) -> Result<impl IntoResponse, ApiError> {
|
lookback_seconds: Some(900),
|
||||||
let alerts = evaluate_alerts(
|
timeout_threshold: Some(3),
|
||||||
&state,
|
auth_rejected_threshold: Some(3),
|
||||||
AlertRuleConfig {
|
enroll_rejected_threshold: Some(5),
|
||||||
lookback_seconds: query.lookback_seconds,
|
}
|
||||||
timeout_threshold: query.timeout_threshold,
|
}
|
||||||
auth_rejected_threshold: query.auth_rejected_threshold,
|
}
|
||||||
enroll_rejected_threshold: query.enroll_rejected_threshold,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
if let Err(err) = state.store.sync_alert_transitions(&alerts).await {
|
#[derive(Clone, Default)]
|
||||||
warn!(error = %err, "failed to sync alert transitions");
|
pub struct AlertsSnapshot {
|
||||||
|
pub ts_unix: u64,
|
||||||
|
pub alerts: Vec<AlertState>,
|
||||||
|
pub recent_transitions: Vec<AlertTransition>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Holds the latest alert snapshot computed by the background evaluator and
|
||||||
|
/// broadcasts each new snapshot to connected `alerts/ws` subscribers.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct AlertsHub {
|
||||||
|
tx: broadcast::Sender<String>,
|
||||||
|
latest_payload: Arc<RwLock<Option<String>>>,
|
||||||
|
latest_alerts: Arc<RwLock<Vec<AlertState>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for AlertsHub {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AlertsHub {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
let (tx, _) = broadcast::channel(16);
|
||||||
|
Self {
|
||||||
|
tx,
|
||||||
|
latest_payload: Arc::new(RwLock::new(None)),
|
||||||
|
latest_alerts: Arc::new(RwLock::new(Vec::new())),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn subscribe(&self) -> broadcast::Receiver<String> {
|
||||||
|
self.tx.subscribe()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn latest_alerts(&self) -> Vec<AlertState> {
|
||||||
|
self.latest_alerts.read().await.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn latest_payload(&self) -> Option<String> {
|
||||||
|
self.latest_payload.read().await.clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn publish(&self, snapshot: AlertsSnapshot) {
|
||||||
|
let payload = serde_json::json!({
|
||||||
|
"type": "alerts_snapshot",
|
||||||
|
"ts_unix": snapshot.ts_unix,
|
||||||
|
"alerts": snapshot.alerts,
|
||||||
|
"recent_transitions": snapshot.recent_transitions,
|
||||||
|
});
|
||||||
|
let Ok(encoded) = serde_json::to_string(&payload) else {
|
||||||
|
warn!("failed to encode alerts snapshot payload");
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
*self.latest_alerts.write().await = snapshot.alerts;
|
||||||
|
*self.latest_payload.write().await = Some(encoded.clone());
|
||||||
|
let _ = self.tx.send(encoded);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Background task that owns alert evaluation: on a fixed ticker it recomputes
|
||||||
|
/// the snapshot, persists transitions, and publishes to subscribers so neither
|
||||||
|
/// the GET endpoint nor individual WS clients evaluate alerts themselves.
|
||||||
|
pub async fn run_alert_evaluator(state: AppState) {
|
||||||
|
let config = AlertRuleConfig::default();
|
||||||
|
let mut tick = tokio::time::interval(Duration::from_secs(5));
|
||||||
|
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||||
|
loop {
|
||||||
|
tick.tick().await;
|
||||||
|
let alerts = match evaluate_alerts(&state, &config).await {
|
||||||
|
Ok(alerts) => alerts,
|
||||||
|
Err(err) => {
|
||||||
|
warn!(code = %err.code, "failed to evaluate alerts in background task");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Err(err) = state.store.sync_alert_transitions(&alerts).await {
|
||||||
|
warn!(error = %err, "failed to sync alert transitions in background task");
|
||||||
|
}
|
||||||
|
|
||||||
|
let recent_transitions = match state.store.list_alert_transitions(None, 20).await {
|
||||||
|
Ok(history) => history,
|
||||||
|
Err(err) => {
|
||||||
|
warn!(error = %err, "failed to load alert transition history for snapshot");
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
state
|
||||||
|
.alerts
|
||||||
|
.publish(AlertsSnapshot {
|
||||||
|
ts_unix: now_unix(),
|
||||||
|
alerts,
|
||||||
|
recent_transitions,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn active_alerts(State(state): State<AppState>) -> Result<impl IntoResponse, ApiError> {
|
||||||
|
let alerts = state.alerts.latest_alerts().await;
|
||||||
Ok((StatusCode::OK, Json(alerts)))
|
Ok((StatusCode::OK, Json(alerts)))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -94,64 +183,34 @@ pub async fn alert_history(
|
|||||||
pub async fn alerts_stream(
|
pub async fn alerts_stream(
|
||||||
ws: WebSocketUpgrade,
|
ws: WebSocketUpgrade,
|
||||||
State(state): State<AppState>,
|
State(state): State<AppState>,
|
||||||
Query(query): Query<ActiveAlertsQuery>,
|
|
||||||
) -> impl IntoResponse {
|
) -> impl IntoResponse {
|
||||||
let config = AlertRuleConfig {
|
ws.on_upgrade(move |socket| alerts_stream_socket(state, socket))
|
||||||
lookback_seconds: query.lookback_seconds,
|
|
||||||
timeout_threshold: query.timeout_threshold,
|
|
||||||
auth_rejected_threshold: query.auth_rejected_threshold,
|
|
||||||
enroll_rejected_threshold: query.enroll_rejected_threshold,
|
|
||||||
};
|
|
||||||
ws.on_upgrade(move |socket| alerts_stream_socket(state, socket, config))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn alerts_stream_socket(state: AppState, mut socket: WebSocket, config: AlertRuleConfig) {
|
async fn alerts_stream_socket(state: AppState, mut socket: WebSocket) {
|
||||||
let mut tick = tokio::time::interval(Duration::from_secs(5));
|
let mut rx = state.alerts.subscribe();
|
||||||
loop {
|
|
||||||
tick.tick().await;
|
|
||||||
let alerts = match evaluate_alerts(&state, config.clone()).await {
|
|
||||||
Ok(alerts) => alerts,
|
|
||||||
Err(err) => {
|
|
||||||
warn!(code = %err.code, "failed to evaluate alerts for stream");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if let Err(err) = state.store.sync_alert_transitions(&alerts).await {
|
if let Some(encoded) = state.alerts.latest_payload().await
|
||||||
warn!(error = %err, "failed to sync alert transitions in stream");
|
&& socket.send(Message::Text(encoded.into())).await.is_err() {
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let history = match state.store.list_alert_transitions(None, 20).await {
|
loop {
|
||||||
Ok(h) => h,
|
match rx.recv().await {
|
||||||
Err(err) => {
|
Ok(encoded) => {
|
||||||
warn!(error = %err, "failed to load alert transition history for stream");
|
if socket.send(Message::Text(encoded.into())).await.is_err() {
|
||||||
Vec::new()
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
Err(broadcast::error::RecvError::Lagged(_)) => continue,
|
||||||
|
Err(broadcast::error::RecvError::Closed) => break,
|
||||||
let payload = serde_json::json!({
|
|
||||||
"type": "alerts_snapshot",
|
|
||||||
"ts_unix": now_unix(),
|
|
||||||
"alerts": alerts,
|
|
||||||
"recent_transitions": history,
|
|
||||||
});
|
|
||||||
let encoded = match serde_json::to_string(&payload) {
|
|
||||||
Ok(s) => s,
|
|
||||||
Err(err) => {
|
|
||||||
warn!(error = %err, "failed to encode alerts stream payload");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if socket.send(Message::Text(encoded.into())).await.is_err() {
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn evaluate_alerts(
|
async fn evaluate_alerts(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
config: AlertRuleConfig,
|
config: &AlertRuleConfig,
|
||||||
) -> Result<Vec<AlertState>, ApiError> {
|
) -> Result<Vec<AlertState>, ApiError> {
|
||||||
let lookback_seconds = config.lookback_seconds.unwrap_or(900).clamp(60, 86_400);
|
let lookback_seconds = config.lookback_seconds.unwrap_or(900).clamp(60, 86_400);
|
||||||
let timeout_threshold = config.timeout_threshold.unwrap_or(3).max(1);
|
let timeout_threshold = config.timeout_threshold.unwrap_or(3).max(1);
|
||||||
@@ -161,7 +220,14 @@ async fn evaluate_alerts(
|
|||||||
let now = now_unix();
|
let now = now_unix();
|
||||||
let since_unix = now.saturating_sub(lookback_seconds);
|
let since_unix = now.saturating_sub(lookback_seconds);
|
||||||
|
|
||||||
let enrolled_agents = state.store.list_agents().await;
|
let enrolled_agents = state.store.list_agents().await.map_err(|err| {
|
||||||
|
warn!(error = %err, "failed listing agents for alert evaluation");
|
||||||
|
ApiError::new(
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
"alerts_query_failed",
|
||||||
|
err.to_string(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
let connected_agents = state
|
let connected_agents = state
|
||||||
.sessions
|
.sessions
|
||||||
.read()
|
.read()
|
||||||
|
|||||||
@@ -40,7 +40,18 @@ pub struct RelayCommandResponse {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn list_agents(State(state): State<AppState>) -> Result<impl IntoResponse, ApiError> {
|
pub async fn list_agents(State(state): State<AppState>) -> Result<impl IntoResponse, ApiError> {
|
||||||
let enrolled = state.store.list_agents_with_nicknames().await;
|
let enrolled = state
|
||||||
|
.store
|
||||||
|
.list_agents_with_nicknames()
|
||||||
|
.await
|
||||||
|
.map_err(|err| {
|
||||||
|
warn!(error = %err, "failed listing agents");
|
||||||
|
ApiError::new(
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
|
"list_agents_failed",
|
||||||
|
err.to_string(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
let sessions = state.sessions.read().await;
|
let sessions = state.sessions.read().await;
|
||||||
|
|
||||||
let agents = enrolled
|
let agents = enrolled
|
||||||
@@ -120,9 +131,9 @@ pub async fn relay_agent_command(
|
|||||||
.insert(request_id_string.clone(), pending_tx);
|
.insert(request_id_string.clone(), pending_tx);
|
||||||
|
|
||||||
info!("dispatching command to agent");
|
info!("dispatching command to agent");
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "admin_api".into(),
|
actor_type: "admin_api".into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: Some(agent_id.to_string()),
|
agent_id: Some(agent_id.to_string()),
|
||||||
@@ -133,10 +144,7 @@ pub async fn relay_agent_command(
|
|||||||
message: "dispatched command to connected agent".into(),
|
message: "dispatched command to connected agent".into(),
|
||||||
metadata: serde_json::json!({ "command": command_kind }),
|
metadata: serde_json::json!({ "command": command_kind }),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for command dispatch");
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Err(err) = tx.send(SessionEvent::Message(ServerMessage::Command {
|
if let Err(err) = tx.send(SessionEvent::Message(ServerMessage::Command {
|
||||||
request_id,
|
request_id,
|
||||||
@@ -144,9 +152,9 @@ pub async fn relay_agent_command(
|
|||||||
})) {
|
})) {
|
||||||
state.pending.lock().await.remove(&request_id_string);
|
state.pending.lock().await.remove(&request_id_string);
|
||||||
warn!(error = %err, "failed sending command to agent session");
|
warn!(error = %err, "failed sending command to agent session");
|
||||||
if let Err(audit_err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "admin_api".into(),
|
actor_type: "admin_api".into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: Some(agent_id.to_string()),
|
agent_id: Some(agent_id.to_string()),
|
||||||
@@ -157,10 +165,7 @@ pub async fn relay_agent_command(
|
|||||||
message: err.to_string(),
|
message: err.to_string(),
|
||||||
metadata: serde_json::json!({ "command": command_kind }),
|
metadata: serde_json::json!({ "command": command_kind }),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %audit_err, "failed to append audit event for command send failure");
|
|
||||||
}
|
|
||||||
return Err(ApiError::new(
|
return Err(ApiError::new(
|
||||||
StatusCode::BAD_GATEWAY,
|
StatusCode::BAD_GATEWAY,
|
||||||
"agent_send_failed",
|
"agent_send_failed",
|
||||||
@@ -177,9 +182,9 @@ pub async fn relay_agent_command(
|
|||||||
let response = match outcome {
|
let response = match outcome {
|
||||||
Ok(Ok(AgentReply::Result(result))) => {
|
Ok(Ok(AgentReply::Result(result))) => {
|
||||||
info!("agent command completed");
|
info!("agent command completed");
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "admin_api".into(),
|
actor_type: "admin_api".into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: Some(agent_id.to_string()),
|
agent_id: Some(agent_id.to_string()),
|
||||||
@@ -190,10 +195,7 @@ pub async fn relay_agent_command(
|
|||||||
message: "agent command completed".into(),
|
message: "agent command completed".into(),
|
||||||
metadata: serde_json::json!({ "command": command_kind }),
|
metadata: serde_json::json!({ "command": command_kind }),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for command success");
|
|
||||||
}
|
|
||||||
RelayCommandResponse {
|
RelayCommandResponse {
|
||||||
request_id: request_id_string,
|
request_id: request_id_string,
|
||||||
status: "ok".into(),
|
status: "ok".into(),
|
||||||
@@ -203,9 +205,9 @@ pub async fn relay_agent_command(
|
|||||||
}
|
}
|
||||||
Ok(Ok(AgentReply::Error(error))) => {
|
Ok(Ok(AgentReply::Error(error))) => {
|
||||||
warn!(code = %error.code, "agent command returned error");
|
warn!(code = %error.code, "agent command returned error");
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "admin_api".into(),
|
actor_type: "admin_api".into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: Some(agent_id.to_string()),
|
agent_id: Some(agent_id.to_string()),
|
||||||
@@ -216,10 +218,7 @@ pub async fn relay_agent_command(
|
|||||||
message: error.message.clone(),
|
message: error.message.clone(),
|
||||||
metadata: serde_json::json!({ "command": command_kind, "code": error.code }),
|
metadata: serde_json::json!({ "command": command_kind, "code": error.code }),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for command error result");
|
|
||||||
}
|
|
||||||
RelayCommandResponse {
|
RelayCommandResponse {
|
||||||
request_id: request_id_string,
|
request_id: request_id_string,
|
||||||
status: "error".into(),
|
status: "error".into(),
|
||||||
@@ -229,9 +228,9 @@ pub async fn relay_agent_command(
|
|||||||
}
|
}
|
||||||
Ok(Err(_)) => {
|
Ok(Err(_)) => {
|
||||||
warn!("agent response channel dropped");
|
warn!("agent response channel dropped");
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "admin_api".into(),
|
actor_type: "admin_api".into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: Some(agent_id.to_string()),
|
agent_id: Some(agent_id.to_string()),
|
||||||
@@ -242,10 +241,7 @@ pub async fn relay_agent_command(
|
|||||||
message: "agent response channel dropped".into(),
|
message: "agent response channel dropped".into(),
|
||||||
metadata: serde_json::json!({ "command": command_kind }),
|
metadata: serde_json::json!({ "command": command_kind }),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for dropped response");
|
|
||||||
}
|
|
||||||
return Err(ApiError::new(
|
return Err(ApiError::new(
|
||||||
StatusCode::BAD_GATEWAY,
|
StatusCode::BAD_GATEWAY,
|
||||||
"agent_response_dropped",
|
"agent_response_dropped",
|
||||||
@@ -258,9 +254,9 @@ pub async fn relay_agent_command(
|
|||||||
timeout_ms = timeout.as_millis() as u64,
|
timeout_ms = timeout.as_millis() as u64,
|
||||||
"agent command timed out"
|
"agent command timed out"
|
||||||
);
|
);
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "admin_api".into(),
|
actor_type: "admin_api".into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: Some(agent_id.to_string()),
|
agent_id: Some(agent_id.to_string()),
|
||||||
@@ -271,10 +267,7 @@ pub async fn relay_agent_command(
|
|||||||
message: "agent command timed out".into(),
|
message: "agent command timed out".into(),
|
||||||
metadata: serde_json::json!({ "command": command_kind, "timeout_ms": timeout.as_millis() as u64 }),
|
metadata: serde_json::json!({ "command": command_kind, "timeout_ms": timeout.as_millis() as u64 }),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for timeout");
|
|
||||||
}
|
|
||||||
return Err(ApiError::new(
|
return Err(ApiError::new(
|
||||||
StatusCode::GATEWAY_TIMEOUT,
|
StatusCode::GATEWAY_TIMEOUT,
|
||||||
"agent_timeout",
|
"agent_timeout",
|
||||||
|
|||||||
@@ -74,9 +74,9 @@ pub async fn enroll(
|
|||||||
match state.store.enroll(&req.enroll_token).await {
|
match state.store.enroll(&req.enroll_token).await {
|
||||||
Ok(issued) => {
|
Ok(issued) => {
|
||||||
info!(agent_id = %issued.agent_id, "agent enrollment accepted");
|
info!(agent_id = %issued.agent_id, "agent enrollment accepted");
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "agent".into(),
|
actor_type: "agent".into(),
|
||||||
actor_id: Some(issued.agent_id.clone()),
|
actor_id: Some(issued.agent_id.clone()),
|
||||||
agent_id: Some(issued.agent_id.clone()),
|
agent_id: Some(issued.agent_id.clone()),
|
||||||
@@ -87,10 +87,7 @@ pub async fn enroll(
|
|||||||
message: "agent enrollment accepted".into(),
|
message: "agent enrollment accepted".into(),
|
||||||
metadata: serde_json::json!({}),
|
metadata: serde_json::json!({}),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for enroll success");
|
|
||||||
}
|
|
||||||
Ok((
|
Ok((
|
||||||
StatusCode::OK,
|
StatusCode::OK,
|
||||||
Json(EnrollResponse {
|
Json(EnrollResponse {
|
||||||
@@ -102,9 +99,9 @@ pub async fn enroll(
|
|||||||
}
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
warn!(error = %err, "agent enrollment rejected");
|
warn!(error = %err, "agent enrollment rejected");
|
||||||
if let Err(audit_err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "agent".into(),
|
actor_type: "agent".into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: None,
|
agent_id: None,
|
||||||
@@ -115,10 +112,7 @@ pub async fn enroll(
|
|||||||
message: err.to_string(),
|
message: err.to_string(),
|
||||||
metadata: serde_json::json!({}),
|
metadata: serde_json::json!({}),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %audit_err, "failed to append audit event for enroll rejection");
|
|
||||||
}
|
|
||||||
Err(ApiError::new(
|
Err(ApiError::new(
|
||||||
StatusCode::UNAUTHORIZED,
|
StatusCode::UNAUTHORIZED,
|
||||||
"enrollment_rejected",
|
"enrollment_rejected",
|
||||||
@@ -145,9 +139,9 @@ pub async fn issue_enroll_token(
|
|||||||
expires_at_unix = issued.expires_at_unix,
|
expires_at_unix = issued.expires_at_unix,
|
||||||
"issued enroll token"
|
"issued enroll token"
|
||||||
);
|
);
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "admin_api".into(),
|
actor_type: "admin_api".into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: None,
|
agent_id: None,
|
||||||
@@ -161,10 +155,7 @@ pub async fn issue_enroll_token(
|
|||||||
"expires_at_unix": issued.expires_at_unix,
|
"expires_at_unix": issued.expires_at_unix,
|
||||||
}),
|
}),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for token issuance");
|
|
||||||
}
|
|
||||||
Ok((
|
Ok((
|
||||||
StatusCode::OK,
|
StatusCode::OK,
|
||||||
Json(IssueEnrollTokenResponse {
|
Json(IssueEnrollTokenResponse {
|
||||||
@@ -189,9 +180,9 @@ pub async fn list_enroll_tokens(
|
|||||||
) -> Result<impl IntoResponse, ApiError> {
|
) -> Result<impl IntoResponse, ApiError> {
|
||||||
match state.store.list_enroll_tokens().await {
|
match state.store.list_enroll_tokens().await {
|
||||||
Ok(tokens) => {
|
Ok(tokens) => {
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "admin_api".into(),
|
actor_type: "admin_api".into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: None,
|
agent_id: None,
|
||||||
@@ -204,10 +195,7 @@ pub async fn list_enroll_tokens(
|
|||||||
"count": tokens.len(),
|
"count": tokens.len(),
|
||||||
}),
|
}),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for token listing");
|
|
||||||
}
|
|
||||||
let body = tokens
|
let body = tokens
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|t| EnrollTokenStatus {
|
.map(|t| EnrollTokenStatus {
|
||||||
@@ -235,9 +223,9 @@ pub async fn revoke_enroll_token(
|
|||||||
) -> Result<impl IntoResponse, ApiError> {
|
) -> Result<impl IntoResponse, ApiError> {
|
||||||
match state.store.revoke_enroll_token(&token).await {
|
match state.store.revoke_enroll_token(&token).await {
|
||||||
Ok(revoked) => {
|
Ok(revoked) => {
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "admin_api".into(),
|
actor_type: "admin_api".into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: None,
|
agent_id: None,
|
||||||
@@ -256,10 +244,7 @@ pub async fn revoke_enroll_token(
|
|||||||
},
|
},
|
||||||
metadata: serde_json::json!({ "token": token }),
|
metadata: serde_json::json!({ "token": token }),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for token revoke");
|
|
||||||
}
|
|
||||||
Ok((
|
Ok((
|
||||||
StatusCode::OK,
|
StatusCode::OK,
|
||||||
Json(RevokeEnrollTokenResponse { token, revoked }),
|
Json(RevokeEnrollTokenResponse { token, revoked }),
|
||||||
@@ -289,9 +274,9 @@ pub async fn revoke_agent(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "admin_api".into(),
|
actor_type: "admin_api".into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: Some(agent_id.clone()),
|
agent_id: Some(agent_id.clone()),
|
||||||
@@ -310,10 +295,7 @@ pub async fn revoke_agent(
|
|||||||
},
|
},
|
||||||
metadata: serde_json::json!({ "agent_id": agent_id }),
|
metadata: serde_json::json!({ "agent_id": agent_id }),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for agent revoke");
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok((
|
Ok((
|
||||||
StatusCode::OK,
|
StatusCode::OK,
|
||||||
@@ -349,9 +331,9 @@ pub async fn set_agent_nickname(
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(updated) => {
|
Ok(updated) => {
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "admin_api".into(),
|
actor_type: "admin_api".into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: Some(agent_id.clone()),
|
agent_id: Some(agent_id.clone()),
|
||||||
@@ -373,10 +355,7 @@ pub async fn set_agent_nickname(
|
|||||||
"nickname": normalized,
|
"nickname": normalized,
|
||||||
}),
|
}),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for nickname set");
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok((
|
Ok((
|
||||||
StatusCode::OK,
|
StatusCode::OK,
|
||||||
|
|||||||
@@ -4,11 +4,12 @@ use axum::Json;
|
|||||||
use axum::extract::{Query, State};
|
use axum::extract::{Query, State};
|
||||||
use axum::http::StatusCode;
|
use axum::http::StatusCode;
|
||||||
use axum::response::IntoResponse;
|
use axum::response::IntoResponse;
|
||||||
|
use futures_util::StreamExt;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
use wakey_agent::protocol::{AgentCommand, InventoryRequest, WakeRequest};
|
use wakey_agent::protocol::{AgentCommand, InventoryRequest, WakeRequest};
|
||||||
|
|
||||||
use crate::api::ApiError;
|
use crate::api::ApiError;
|
||||||
use crate::api::commands::relay_agent_command;
|
use crate::api::commands::{RelayCommandResponse, relay_agent_command};
|
||||||
use crate::runtime::AppState;
|
use crate::runtime::AppState;
|
||||||
|
|
||||||
mod build;
|
mod build;
|
||||||
@@ -26,6 +27,9 @@ use types::{
|
|||||||
RefreshFleetDevicesResponse, WakeFleetDeviceRequest, WakeFleetDeviceResponse,
|
RefreshFleetDevicesResponse, WakeFleetDeviceRequest, WakeFleetDeviceResponse,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const MAX_REFRESH_AGENTS: usize = 128;
|
||||||
|
const REFRESH_CONCURRENCY: usize = 8;
|
||||||
|
|
||||||
pub async fn list_fleet_devices(
|
pub async fn list_fleet_devices(
|
||||||
State(state): State<AppState>,
|
State(state): State<AppState>,
|
||||||
Query(query): Query<ListFleetDevicesQuery>,
|
Query(query): Query<ListFleetDevicesQuery>,
|
||||||
@@ -55,88 +59,43 @@ pub async fn refresh_fleet_devices(
|
|||||||
};
|
};
|
||||||
agent_ids.sort();
|
agent_ids.sort();
|
||||||
agent_ids.dedup();
|
agent_ids.dedup();
|
||||||
|
if agent_ids.len() > MAX_REFRESH_AGENTS {
|
||||||
let mut results = Vec::with_capacity(agent_ids.len());
|
warn!(
|
||||||
let mut total_accepted = 0usize;
|
requested = agent_ids.len(),
|
||||||
for agent_id in agent_ids {
|
capped = MAX_REFRESH_AGENTS,
|
||||||
let response = relay_agent_command(
|
"capping fleet refresh agent count"
|
||||||
&state,
|
);
|
||||||
&agent_id,
|
agent_ids.truncate(MAX_REFRESH_AGENTS);
|
||||||
AgentCommand::Inventory(InventoryRequest {
|
|
||||||
query: None,
|
|
||||||
name: None,
|
|
||||||
ips: Vec::new(),
|
|
||||||
devs: Vec::new(),
|
|
||||||
nuds: Vec::new(),
|
|
||||||
macs: Vec::new(),
|
|
||||||
}),
|
|
||||||
req.timeout_ms,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
match response {
|
|
||||||
Ok(response) if response.status == "ok" => {
|
|
||||||
let Some(result) = response.result else {
|
|
||||||
results.push(RefreshFleetAgentResult {
|
|
||||||
agent_id,
|
|
||||||
status: "error".into(),
|
|
||||||
accepted: 0,
|
|
||||||
error: Some("inventory command returned no result".into()),
|
|
||||||
});
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
match serde_json::from_value::<wakey_core::DeviceInventory>(result)
|
|
||||||
.map(|i| i.devices)
|
|
||||||
{
|
|
||||||
Ok(devices) => {
|
|
||||||
match state
|
|
||||||
.store
|
|
||||||
.replace_agent_device_snapshot(&agent_id, &devices)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(accepted) => {
|
|
||||||
total_accepted = total_accepted.saturating_add(accepted);
|
|
||||||
results.push(RefreshFleetAgentResult {
|
|
||||||
agent_id,
|
|
||||||
status: "ok".into(),
|
|
||||||
accepted,
|
|
||||||
error: None,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Err(err) => results.push(RefreshFleetAgentResult {
|
|
||||||
agent_id,
|
|
||||||
status: "error".into(),
|
|
||||||
accepted: 0,
|
|
||||||
error: Some(err.to_string()),
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(err) => results.push(RefreshFleetAgentResult {
|
|
||||||
agent_id,
|
|
||||||
status: "error".into(),
|
|
||||||
accepted: 0,
|
|
||||||
error: Some(err.to_string()),
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(response) => results.push(RefreshFleetAgentResult {
|
|
||||||
agent_id,
|
|
||||||
status: "error".into(),
|
|
||||||
accepted: 0,
|
|
||||||
error: response
|
|
||||||
.error
|
|
||||||
.map(|error| error.message)
|
|
||||||
.or_else(|| Some("inventory command failed".into())),
|
|
||||||
}),
|
|
||||||
Err(err) => results.push(RefreshFleetAgentResult {
|
|
||||||
agent_id,
|
|
||||||
status: "error".into(),
|
|
||||||
accepted: 0,
|
|
||||||
error: Some(format!("{}: {}", err.status, err.message)),
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let results = futures_util::stream::iter(agent_ids.into_iter().map(|agent_id| {
|
||||||
|
let state = state.clone();
|
||||||
|
async move {
|
||||||
|
let response = relay_agent_command(
|
||||||
|
&state,
|
||||||
|
&agent_id,
|
||||||
|
AgentCommand::Inventory(InventoryRequest {
|
||||||
|
query: None,
|
||||||
|
name: None,
|
||||||
|
ips: Vec::new(),
|
||||||
|
devs: Vec::new(),
|
||||||
|
nuds: Vec::new(),
|
||||||
|
macs: Vec::new(),
|
||||||
|
}),
|
||||||
|
req.timeout_ms,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
refresh_fleet_agent_result(&state, &agent_id, response).await
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
.buffer_unordered(REFRESH_CONCURRENCY)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let mut results = results;
|
||||||
|
results.sort_by(|a, b| a.agent_id.cmp(&b.agent_id));
|
||||||
|
let total_accepted = results.iter().map(|result| result.accepted).sum::<usize>();
|
||||||
|
|
||||||
Ok((
|
Ok((
|
||||||
StatusCode::OK,
|
StatusCode::OK,
|
||||||
Json(RefreshFleetDevicesResponse {
|
Json(RefreshFleetDevicesResponse {
|
||||||
@@ -146,6 +105,70 @@ pub async fn refresh_fleet_devices(
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn refresh_fleet_agent_result(
|
||||||
|
state: &AppState,
|
||||||
|
agent_id: &str,
|
||||||
|
response: Result<RelayCommandResponse, ApiError>,
|
||||||
|
) -> RefreshFleetAgentResult {
|
||||||
|
match response {
|
||||||
|
Ok(response) if response.status == "ok" => {
|
||||||
|
let Some(result) = response.result else {
|
||||||
|
return RefreshFleetAgentResult {
|
||||||
|
agent_id: agent_id.to_string(),
|
||||||
|
status: "error".into(),
|
||||||
|
accepted: 0,
|
||||||
|
error: Some("inventory command returned no result".into()),
|
||||||
|
};
|
||||||
|
};
|
||||||
|
match serde_json::from_value::<wakey_core::DeviceInventory>(result)
|
||||||
|
.map(|inventory| inventory.devices)
|
||||||
|
{
|
||||||
|
Ok(devices) => {
|
||||||
|
match state
|
||||||
|
.store
|
||||||
|
.replace_agent_device_snapshot(agent_id, &devices)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(accepted) => RefreshFleetAgentResult {
|
||||||
|
agent_id: agent_id.to_string(),
|
||||||
|
status: "ok".into(),
|
||||||
|
accepted,
|
||||||
|
error: None,
|
||||||
|
},
|
||||||
|
Err(err) => RefreshFleetAgentResult {
|
||||||
|
agent_id: agent_id.to_string(),
|
||||||
|
status: "error".into(),
|
||||||
|
accepted: 0,
|
||||||
|
error: Some(err.to_string()),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err) => RefreshFleetAgentResult {
|
||||||
|
agent_id: agent_id.to_string(),
|
||||||
|
status: "error".into(),
|
||||||
|
accepted: 0,
|
||||||
|
error: Some(err.to_string()),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(response) => RefreshFleetAgentResult {
|
||||||
|
agent_id: agent_id.to_string(),
|
||||||
|
status: "error".into(),
|
||||||
|
accepted: 0,
|
||||||
|
error: response
|
||||||
|
.error
|
||||||
|
.map(|error| error.message)
|
||||||
|
.or_else(|| Some("inventory command failed".into())),
|
||||||
|
},
|
||||||
|
Err(err) => RefreshFleetAgentResult {
|
||||||
|
agent_id: agent_id.to_string(),
|
||||||
|
status: "error".into(),
|
||||||
|
accepted: 0,
|
||||||
|
error: Some(format!("{}: {}", err.status, err.message)),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn wake_fleet_device(
|
pub async fn wake_fleet_device(
|
||||||
State(state): State<AppState>,
|
State(state): State<AppState>,
|
||||||
Json(req): Json<WakeFleetDeviceRequest>,
|
Json(req): Json<WakeFleetDeviceRequest>,
|
||||||
@@ -243,7 +266,7 @@ async fn load_fleet_devices(
|
|||||||
let sessions = state.sessions.read().await;
|
let sessions = state.sessions.read().await;
|
||||||
sessions.keys().cloned().collect::<BTreeSet<_>>()
|
sessions.keys().cloned().collect::<BTreeSet<_>>()
|
||||||
};
|
};
|
||||||
let agents = state.store.list_agents_with_nicknames().await;
|
let agents = state.store.list_agents_with_nicknames().await?;
|
||||||
let agent_status = agents
|
let agent_status = agents
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|(agent_id, nickname)| {
|
.map(|(agent_id, nickname)| {
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ mod commands;
|
|||||||
mod control;
|
mod control;
|
||||||
mod terminals;
|
mod terminals;
|
||||||
|
|
||||||
pub use alerts::{active_alerts, alert_history, alerts_stream};
|
pub use alerts::{AlertsHub, active_alerts, alert_history, alerts_stream, run_alert_evaluator};
|
||||||
pub use audit::list_audit_events;
|
pub use audit::list_audit_events;
|
||||||
pub use commands::{list_agents, run_command};
|
pub use commands::{list_agents, run_command};
|
||||||
pub use control::{
|
pub use control::{
|
||||||
|
|||||||
@@ -649,9 +649,9 @@ async fn append_terminal_audit(
|
|||||||
terminal_id: &str,
|
terminal_id: &str,
|
||||||
audit: TerminalAudit<'_>,
|
audit: TerminalAudit<'_>,
|
||||||
) {
|
) {
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: audit.actor_type.into(),
|
actor_type: audit.actor_type.into(),
|
||||||
actor_id: None,
|
actor_id: None,
|
||||||
agent_id: Some(agent_id.to_string()),
|
agent_id: Some(agent_id.to_string()),
|
||||||
@@ -662,10 +662,7 @@ async fn append_terminal_audit(
|
|||||||
message: audit.message.into(),
|
message: audit.message.into(),
|
||||||
metadata: audit.metadata,
|
metadata: audit.metadata,
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(terminal_id, event_type = audit.event_type, error = %err, "failed to append terminal audit event");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn validate_size(rows: u16, cols: u16) -> Result<(), ApiError> {
|
fn validate_size(rows: u16, cols: u16) -> Result<(), ApiError> {
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ pub struct DaemonConfig {
|
|||||||
pub state_file: PathBuf,
|
pub state_file: PathBuf,
|
||||||
pub command_timeout: Duration,
|
pub command_timeout: Duration,
|
||||||
pub enroll_token_ttl: Duration,
|
pub enroll_token_ttl: Duration,
|
||||||
#[allow(dead_code)]
|
|
||||||
pub observation_retention: Duration,
|
pub observation_retention: Duration,
|
||||||
pub pid_file: PathBuf,
|
pub pid_file: PathBuf,
|
||||||
pub ui_dist_dir: PathBuf,
|
pub ui_dist_dir: PathBuf,
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ pub struct AppState {
|
|||||||
pub command_timeout: Duration,
|
pub command_timeout: Duration,
|
||||||
pub enroll_token_ttl: Duration,
|
pub enroll_token_ttl: Duration,
|
||||||
pub terminals: terminals::TerminalRegistry,
|
pub terminals: terminals::TerminalRegistry,
|
||||||
|
pub alerts: api::AlertsHub,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -182,8 +183,11 @@ pub async fn serve(daemon: config::DaemonConfig) -> Result<()> {
|
|||||||
command_timeout: daemon.command_timeout,
|
command_timeout: daemon.command_timeout,
|
||||||
enroll_token_ttl: daemon.enroll_token_ttl,
|
enroll_token_ttl: daemon.enroll_token_ttl,
|
||||||
terminals: terminals::TerminalRegistry::new(),
|
terminals: terminals::TerminalRegistry::new(),
|
||||||
|
alerts: api::AlertsHub::new(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
tokio::spawn(api::run_alert_evaluator(app_state.clone()));
|
||||||
|
|
||||||
// Keep route classes explicit so edge policy can map directly:
|
// Keep route classes explicit so edge policy can map directly:
|
||||||
// - public_api_routes: intended internet-facing agent endpoints
|
// - public_api_routes: intended internet-facing agent endpoints
|
||||||
// - control_api_routes: intended admin-only endpoints behind Access
|
// - control_api_routes: intended admin-only endpoints behind Access
|
||||||
@@ -231,6 +235,18 @@ pub async fn serve(daemon: config::DaemonConfig) -> Result<()> {
|
|||||||
}
|
}
|
||||||
Err(err) => warn!(error = %err, "periodic gc failed"),
|
Err(err) => warn!(error = %err, "periodic gc failed"),
|
||||||
}
|
}
|
||||||
|
match app_state.store.gc_retention(daemon.observation_retention).await {
|
||||||
|
Ok(stats) => {
|
||||||
|
if stats.audit_events_removed > 0 || stats.alert_transitions_removed > 0 {
|
||||||
|
info!(
|
||||||
|
audit_events_removed = stats.audit_events_removed,
|
||||||
|
alert_transitions_removed = stats.alert_transitions_removed,
|
||||||
|
"periodic gc pruned rows older than retention window"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err) => warn!(error = %err, "periodic retention gc failed"),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
join = &mut server => {
|
join = &mut server => {
|
||||||
let _ = remove_pid_file(&daemon.pid_file);
|
let _ = remove_pid_file(&daemon.pid_file);
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ pub use store::Store;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub use types::AgentDeviceRow;
|
pub use types::AgentDeviceRow;
|
||||||
pub use types::{
|
pub use types::{
|
||||||
AgentDeviceWithChildren, AlertState, AuditEvent, AuditEventFilter, AuditEventInput,
|
AgentDeviceWithChildren, AlertState, AlertTransition, AuditEvent, AuditEventFilter,
|
||||||
DeviceIdentifier, DeviceIdentifierInput, KnownDevice, KnownDeviceInput, KnownDeviceSummary,
|
AuditEventInput, DeviceIdentifier, DeviceIdentifierInput, KnownDevice, KnownDeviceInput,
|
||||||
|
KnownDeviceSummary,
|
||||||
};
|
};
|
||||||
@@ -10,7 +10,7 @@ use uuid::Uuid;
|
|||||||
use crate::state::types::{
|
use crate::state::types::{
|
||||||
AlertState, AlertTransition, AuditEvent, AuditEventFilter, AuditEventInput, DeviceIdentifier,
|
AlertState, AlertTransition, AuditEvent, AuditEventFilter, AuditEventInput, DeviceIdentifier,
|
||||||
DeviceIdentifierInput, EnrollTokenInfo, IssuedAgent, IssuedEnrollToken, KnownDevice,
|
DeviceIdentifierInput, EnrollTokenInfo, IssuedAgent, IssuedEnrollToken, KnownDevice,
|
||||||
KnownDeviceInput, StateStats,
|
KnownDeviceInput, RetentionStats, StateStats,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub struct Store {
|
pub struct Store {
|
||||||
@@ -103,6 +103,124 @@ mod tests {
|
|||||||
assert_eq!(exists, 0);
|
assert_eq!(exists, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn enroll_rejects_negative_expiry() {
|
||||||
|
let ts = TestStore::new().await;
|
||||||
|
sqlx::query!(
|
||||||
|
"INSERT INTO enroll_tokens (token, expires_at_unix) VALUES (?1, ?2)",
|
||||||
|
"enr-negative-expiry-test",
|
||||||
|
-1_i64
|
||||||
|
)
|
||||||
|
.execute(&ts.store().pool)
|
||||||
|
.await
|
||||||
|
.expect("insert should succeed");
|
||||||
|
|
||||||
|
let err = ts
|
||||||
|
.store()
|
||||||
|
.enroll("enr-negative-expiry-test")
|
||||||
|
.await
|
||||||
|
.expect_err("negative expiry should be treated as expired");
|
||||||
|
|
||||||
|
assert!(err.to_string().contains("expired"));
|
||||||
|
let exists = sqlx::query_scalar!(
|
||||||
|
"SELECT COUNT(*) FROM enroll_tokens WHERE token = ?1",
|
||||||
|
"enr-negative-expiry-test"
|
||||||
|
)
|
||||||
|
.fetch_one(&ts.store().pool)
|
||||||
|
.await
|
||||||
|
.expect("read should succeed");
|
||||||
|
assert_eq!(exists, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn gc_retention_prunes_old_audit_and_alert_history() {
|
||||||
|
let ts = TestStore::new().await;
|
||||||
|
|
||||||
|
let now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.expect("clock should be after epoch")
|
||||||
|
.as_secs() as i64;
|
||||||
|
let old_ts = now - 86_400;
|
||||||
|
let old_evt = format!("{:020}:old-evt", old_ts);
|
||||||
|
sqlx::query!(
|
||||||
|
"INSERT INTO audit_events
|
||||||
|
(event_key, event_id, ts_unix, actor_type, actor_id, agent_id, request_id,
|
||||||
|
event_type, outcome, latency_ms, message, metadata_json)
|
||||||
|
VALUES (?1, ?2, ?3, 'test', NULL, NULL, NULL, 'test', 'ok', NULL, 'old', '{}')",
|
||||||
|
old_evt,
|
||||||
|
"old-evt",
|
||||||
|
old_ts
|
||||||
|
)
|
||||||
|
.execute(&ts.store().pool)
|
||||||
|
.await
|
||||||
|
.expect("old audit event should insert");
|
||||||
|
|
||||||
|
let old_atr = format!("{:020}:old-atr", old_ts);
|
||||||
|
sqlx::query!(
|
||||||
|
"INSERT INTO alert_transitions
|
||||||
|
(transition_key, transition_id, ts_unix, alert_id, kind, agent_id,
|
||||||
|
from_status, to_status, message, metadata_json)
|
||||||
|
VALUES (?1, ?2, ?3, 'alert-old', 'test', NULL, NULL, 'active', 'old', '{}')",
|
||||||
|
old_atr,
|
||||||
|
"old-atr",
|
||||||
|
old_ts
|
||||||
|
)
|
||||||
|
.execute(&ts.store().pool)
|
||||||
|
.await
|
||||||
|
.expect("old alert transition should insert");
|
||||||
|
|
||||||
|
ts.store()
|
||||||
|
.append_audit_event(crate::state::AuditEventInput {
|
||||||
|
actor_type: "test".into(),
|
||||||
|
actor_id: None,
|
||||||
|
agent_id: None,
|
||||||
|
request_id: None,
|
||||||
|
event_type: "test".into(),
|
||||||
|
outcome: "ok".into(),
|
||||||
|
latency_ms: None,
|
||||||
|
message: "fresh".into(),
|
||||||
|
metadata: serde_json::json!({}),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("fresh audit event should insert");
|
||||||
|
|
||||||
|
let stats = ts
|
||||||
|
.store()
|
||||||
|
.gc_retention(Duration::from_secs(3600))
|
||||||
|
.await
|
||||||
|
.expect("retention gc should succeed");
|
||||||
|
|
||||||
|
assert_eq!(stats.audit_events_removed, 1);
|
||||||
|
assert_eq!(stats.alert_transitions_removed, 1);
|
||||||
|
|
||||||
|
let old_audit_gone = sqlx::query_scalar!(
|
||||||
|
"SELECT COUNT(*) FROM audit_events WHERE event_id = ?1",
|
||||||
|
"old-evt"
|
||||||
|
)
|
||||||
|
.fetch_one(&ts.store().pool)
|
||||||
|
.await
|
||||||
|
.expect("read should succeed");
|
||||||
|
assert_eq!(old_audit_gone, 0);
|
||||||
|
|
||||||
|
let fresh_audit_remains = sqlx::query_scalar!(
|
||||||
|
"SELECT COUNT(*) FROM audit_events WHERE message = ?1",
|
||||||
|
"fresh"
|
||||||
|
)
|
||||||
|
.fetch_one(&ts.store().pool)
|
||||||
|
.await
|
||||||
|
.expect("read should succeed");
|
||||||
|
assert_eq!(fresh_audit_remains, 1);
|
||||||
|
|
||||||
|
let old_transition_gone = sqlx::query_scalar!(
|
||||||
|
"SELECT COUNT(*) FROM alert_transitions WHERE transition_id = ?1",
|
||||||
|
"old-atr"
|
||||||
|
)
|
||||||
|
.fetch_one(&ts.store().pool)
|
||||||
|
.await
|
||||||
|
.expect("read should succeed");
|
||||||
|
assert_eq!(old_transition_gone, 0);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn enroll_rejects_expired_token() {
|
async fn enroll_rejects_expired_token() {
|
||||||
let ts = TestStore::new().await;
|
let ts = TestStore::new().await;
|
||||||
@@ -209,7 +327,11 @@ mod tests {
|
|||||||
.expect("nickname set should succeed");
|
.expect("nickname set should succeed");
|
||||||
assert!(updated);
|
assert!(updated);
|
||||||
|
|
||||||
let listed = ts.store().list_agents_with_nicknames().await;
|
let listed = ts
|
||||||
|
.store()
|
||||||
|
.list_agents_with_nicknames()
|
||||||
|
.await
|
||||||
|
.expect("list agents with nicknames should succeed");
|
||||||
assert!(listed.iter().any(|(id, name)| {
|
assert!(listed.iter().any(|(id, name)| {
|
||||||
id == &issued.agent_id && name.as_deref() == Some("kitchen-router")
|
id == &issued.agent_id && name.as_deref() == Some("kitchen-router")
|
||||||
}));
|
}));
|
||||||
@@ -221,7 +343,11 @@ mod tests {
|
|||||||
.expect("nickname clear should succeed");
|
.expect("nickname clear should succeed");
|
||||||
assert!(cleared);
|
assert!(cleared);
|
||||||
|
|
||||||
let listed = ts.store().list_agents_with_nicknames().await;
|
let listed = ts
|
||||||
|
.store()
|
||||||
|
.list_agents_with_nicknames()
|
||||||
|
.await
|
||||||
|
.expect("list agents with nicknames should succeed");
|
||||||
assert!(
|
assert!(
|
||||||
listed
|
listed
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
@@ -1,6 +1,25 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
impl Store {
|
impl Store {
|
||||||
|
/// Best-effort audit append: logs the failure with event context instead of
|
||||||
|
/// bubbling it up. Auditing must never fail the operation that triggered it.
|
||||||
|
pub async fn audit(&self, input: AuditEventInput) {
|
||||||
|
let event_type = input.event_type.clone();
|
||||||
|
let outcome = input.outcome.clone();
|
||||||
|
let agent_id = input.agent_id.clone();
|
||||||
|
let request_id = input.request_id.clone();
|
||||||
|
if let Err(err) = self.append_audit_event(input).await {
|
||||||
|
warn!(
|
||||||
|
error = %err,
|
||||||
|
event_type = %event_type,
|
||||||
|
outcome = %outcome,
|
||||||
|
agent_id = ?agent_id,
|
||||||
|
request_id = ?request_id,
|
||||||
|
"failed to append audit event"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn append_audit_event(&self, input: AuditEventInput) -> Result<AuditEvent> {
|
pub async fn append_audit_event(&self, input: AuditEventInput) -> Result<AuditEvent> {
|
||||||
let event = AuditEvent {
|
let event = AuditEvent {
|
||||||
event_id: format!("evt-{}", Uuid::new_v4()),
|
event_id: format!("evt-{}", Uuid::new_v4()),
|
||||||
|
|||||||
@@ -154,6 +154,37 @@ impl Store {
|
|||||||
Ok(removed)
|
Ok(removed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Prunes audit and alert history older than the retention window.
|
||||||
|
///
|
||||||
|
/// Runs from the daemon's periodic maintenance loop; both tables have a
|
||||||
|
/// `ts_unix` index so the deletes stay cheap as history grows.
|
||||||
|
pub async fn gc_retention(&self, older_than: Duration) -> Result<RetentionStats> {
|
||||||
|
let cutoff = i64::try_from(now_unix().saturating_sub(older_than.as_secs().max(1)))
|
||||||
|
.context("retention cutoff does not fit SQLite integer")?;
|
||||||
|
let audit_events_removed =
|
||||||
|
sqlx::query!("DELETE FROM audit_events WHERE ts_unix < ?1", cutoff)
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.context("failed pruning old audit events")?
|
||||||
|
.rows_affected();
|
||||||
|
let alert_transitions_removed =
|
||||||
|
sqlx::query!("DELETE FROM alert_transitions WHERE ts_unix < ?1", cutoff)
|
||||||
|
.execute(&self.pool)
|
||||||
|
.await
|
||||||
|
.context("failed pruning old alert transitions")?
|
||||||
|
.rows_affected();
|
||||||
|
if audit_events_removed > 0 || alert_transitions_removed > 0 {
|
||||||
|
info!(
|
||||||
|
audit_events_removed,
|
||||||
|
alert_transitions_removed, "pruned state rows older than retention window"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(RetentionStats {
|
||||||
|
audit_events_removed,
|
||||||
|
alert_transitions_removed,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn ensure_schema_version(&self) -> Result<()> {
|
async fn ensure_schema_version(&self) -> Result<()> {
|
||||||
match sqlx::query_scalar!("SELECT value FROM meta WHERE key = ?1", SCHEMA_VERSION_KEY)
|
match sqlx::query_scalar!("SELECT value FROM meta WHERE key = ?1", SCHEMA_VERSION_KEY)
|
||||||
.fetch_optional(&self.pool)
|
.fetch_optional(&self.pool)
|
||||||
|
|||||||
@@ -17,7 +17,8 @@ impl Store {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let now = now_unix();
|
let now = now_unix();
|
||||||
if expires_at_unix as u64 <= now {
|
let now_i64 = i64::try_from(now).context("current time does not fit SQLite integer")?;
|
||||||
|
if expires_at_unix <= now_i64 {
|
||||||
sqlx::query!("DELETE FROM enroll_tokens WHERE token = ?1", enroll_token)
|
sqlx::query!("DELETE FROM enroll_tokens WHERE token = ?1", enroll_token)
|
||||||
.execute(&mut *tx)
|
.execute(&mut *tx)
|
||||||
.await
|
.await
|
||||||
@@ -165,20 +166,16 @@ impl Store {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn list_agents(&self) -> Vec<String> {
|
pub async fn list_agents(&self) -> Result<Vec<String>> {
|
||||||
match sqlx::query_scalar!(r#"SELECT agent_id as "agent_id!" FROM agents ORDER BY agent_id"#)
|
let ids =
|
||||||
.fetch_all(&self.pool)
|
sqlx::query_scalar!(r#"SELECT agent_id as "agent_id!" FROM agents ORDER BY agent_id"#)
|
||||||
.await
|
.fetch_all(&self.pool)
|
||||||
{
|
.await
|
||||||
Ok(out) => out,
|
.context("failed listing agents from state db")?;
|
||||||
Err(err) => {
|
Ok(ids)
|
||||||
warn!(error = %err, "failed to list agents from state db");
|
|
||||||
Vec::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn list_agents_with_nicknames(&self) -> Vec<(String, Option<String>)> {
|
pub async fn list_agents_with_nicknames(&self) -> Result<Vec<(String, Option<String>)>> {
|
||||||
let rows = sqlx::query!(
|
let rows = sqlx::query!(
|
||||||
r#"SELECT agents.agent_id as "agent_id!", agent_meta.nickname
|
r#"SELECT agents.agent_id as "agent_id!", agent_meta.nickname
|
||||||
FROM agents
|
FROM agents
|
||||||
@@ -186,23 +183,19 @@ impl Store {
|
|||||||
ORDER BY agents.agent_id"#,
|
ORDER BY agents.agent_id"#,
|
||||||
)
|
)
|
||||||
.fetch_all(&self.pool)
|
.fetch_all(&self.pool)
|
||||||
.await;
|
.await
|
||||||
|
.context("failed listing agents from state db")?;
|
||||||
|
|
||||||
match rows {
|
let agents = rows
|
||||||
Ok(rows) => rows
|
.into_iter()
|
||||||
.into_iter()
|
.map(|row| {
|
||||||
.map(|row| {
|
let nickname = row
|
||||||
let nickname = row
|
.nickname
|
||||||
.nickname
|
.map(|v| v.trim().to_string())
|
||||||
.map(|v| v.trim().to_string())
|
.filter(|v| !v.is_empty());
|
||||||
.filter(|v| !v.is_empty());
|
(row.agent_id, nickname)
|
||||||
(row.agent_id, nickname)
|
})
|
||||||
})
|
.collect();
|
||||||
.collect(),
|
Ok(agents)
|
||||||
Err(err) => {
|
|
||||||
warn!(error = %err, "failed to list agents from state db");
|
|
||||||
Vec::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -32,6 +32,12 @@ pub struct StateStats {
|
|||||||
pub expired_enroll_token_count: usize,
|
pub expired_enroll_token_count: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct RetentionStats {
|
||||||
|
pub audit_events_removed: u64,
|
||||||
|
pub alert_transitions_removed: u64,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct KnownDevice {
|
pub struct KnownDevice {
|
||||||
pub device_id: String,
|
pub device_id: String,
|
||||||
|
|||||||
@@ -159,9 +159,9 @@ async fn handle_agent_socket(state: AppState, socket: WebSocket) {
|
|||||||
sessions.remove(&agent_id);
|
sessions.remove(&agent_id);
|
||||||
}
|
}
|
||||||
drop(sessions);
|
drop(sessions);
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "agent".into(),
|
actor_type: "agent".into(),
|
||||||
actor_id: Some(agent_id.clone()),
|
actor_id: Some(agent_id.clone()),
|
||||||
agent_id: Some(agent_id),
|
agent_id: Some(agent_id),
|
||||||
@@ -172,10 +172,7 @@ async fn handle_agent_socket(state: AppState, socket: WebSocket) {
|
|||||||
message: "agent websocket disconnected".into(),
|
message: "agent websocket disconnected".into(),
|
||||||
metadata: serde_json::json!({}),
|
metadata: serde_json::json!({}),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for ws disconnect");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
writer.abort();
|
writer.abort();
|
||||||
@@ -251,9 +248,9 @@ async fn process_agent_text(
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
warn!(agent_id = %agent_id, "agent auth rejected");
|
warn!(agent_id = %agent_id, "agent auth rejected");
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "agent".into(),
|
actor_type: "agent".into(),
|
||||||
actor_id: Some(agent_id.clone()),
|
actor_id: Some(agent_id.clone()),
|
||||||
agent_id: Some(agent_id),
|
agent_id: Some(agent_id),
|
||||||
@@ -267,10 +264,7 @@ async fn process_agent_text(
|
|||||||
"hello_to_auth_ms": hello_to_auth_ms,
|
"hello_to_auth_ms": hello_to_auth_ms,
|
||||||
}),
|
}),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for auth rejection");
|
|
||||||
}
|
|
||||||
anyhow::bail!("agent auth rejected");
|
anyhow::bail!("agent auth rejected");
|
||||||
}
|
}
|
||||||
state.sessions.write().await.insert(
|
state.sessions.write().await.insert(
|
||||||
@@ -284,9 +278,9 @@ async fn process_agent_text(
|
|||||||
);
|
);
|
||||||
connection.authed_agent_id = Some(agent_id.clone());
|
connection.authed_agent_id = Some(agent_id.clone());
|
||||||
info!(agent_id = %agent_id, connect_to_auth_ms, hello_to_auth_ms = hello_to_auth_ms.unwrap_or(0), "agent authenticated");
|
info!(agent_id = %agent_id, connect_to_auth_ms, hello_to_auth_ms = hello_to_auth_ms.unwrap_or(0), "agent authenticated");
|
||||||
if let Err(err) = state
|
state
|
||||||
.store
|
.store
|
||||||
.append_audit_event(AuditEventInput {
|
.audit(AuditEventInput {
|
||||||
actor_type: "agent".into(),
|
actor_type: "agent".into(),
|
||||||
actor_id: Some(agent_id.clone()),
|
actor_id: Some(agent_id.clone()),
|
||||||
agent_id: Some(agent_id),
|
agent_id: Some(agent_id),
|
||||||
@@ -300,10 +294,7 @@ async fn process_agent_text(
|
|||||||
"hello_to_auth_ms": hello_to_auth_ms,
|
"hello_to_auth_ms": hello_to_auth_ms,
|
||||||
}),
|
}),
|
||||||
})
|
})
|
||||||
.await
|
.await;
|
||||||
{
|
|
||||||
warn!(error = %err, "failed to append audit event for auth success");
|
|
||||||
}
|
|
||||||
let _ = tx.send(SessionEvent::Message(ServerMessage::SyncDeviceSnapshot));
|
let _ = tx.send(SessionEvent::Message(ServerMessage::SyncDeviceSnapshot));
|
||||||
}
|
}
|
||||||
IncomingClientMessage::Heartbeat { agent_id } => {
|
IncomingClientMessage::Heartbeat { agent_id } => {
|
||||||
|
|||||||
Reference in new issue
Block a user