alert history work
- dedup error handling - alert now clears with observation_retenstion_seconds - evaluate recent alerts once per server per 5s - some iter stuff, some type stuff, and some store api stuff.
This commit is contained in:
29 files changed
+750
-294
No files matched your search
@@ -5,6 +5,7 @@ pub use store::Store;
|
||||
#[cfg(test)]
|
||||
pub use types::AgentDeviceRow;
|
||||
pub use types::{
|
||||
AgentDeviceWithChildren, AlertState, AuditEvent, AuditEventFilter, AuditEventInput,
|
||||
DeviceIdentifier, DeviceIdentifierInput, KnownDevice, KnownDeviceInput, KnownDeviceSummary,
|
||||
AgentDeviceWithChildren, AlertState, AlertTransition, AuditEvent, AuditEventFilter,
|
||||
AuditEventInput, DeviceIdentifier, DeviceIdentifierInput, KnownDevice, KnownDeviceInput,
|
||||
KnownDeviceSummary,
|
||||
};
|
||||
@@ -10,7 +10,7 @@ use uuid::Uuid;
|
||||
use crate::state::types::{
|
||||
AlertState, AlertTransition, AuditEvent, AuditEventFilter, AuditEventInput, DeviceIdentifier,
|
||||
DeviceIdentifierInput, EnrollTokenInfo, IssuedAgent, IssuedEnrollToken, KnownDevice,
|
||||
KnownDeviceInput, StateStats,
|
||||
KnownDeviceInput, RetentionStats, StateStats,
|
||||
};
|
||||
|
||||
pub struct Store {
|
||||
@@ -103,6 +103,124 @@ mod tests {
|
||||
assert_eq!(exists, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enroll_rejects_negative_expiry() {
|
||||
let ts = TestStore::new().await;
|
||||
sqlx::query!(
|
||||
"INSERT INTO enroll_tokens (token, expires_at_unix) VALUES (?1, ?2)",
|
||||
"enr-negative-expiry-test",
|
||||
-1_i64
|
||||
)
|
||||
.execute(&ts.store().pool)
|
||||
.await
|
||||
.expect("insert should succeed");
|
||||
|
||||
let err = ts
|
||||
.store()
|
||||
.enroll("enr-negative-expiry-test")
|
||||
.await
|
||||
.expect_err("negative expiry should be treated as expired");
|
||||
|
||||
assert!(err.to_string().contains("expired"));
|
||||
let exists = sqlx::query_scalar!(
|
||||
"SELECT COUNT(*) FROM enroll_tokens WHERE token = ?1",
|
||||
"enr-negative-expiry-test"
|
||||
)
|
||||
.fetch_one(&ts.store().pool)
|
||||
.await
|
||||
.expect("read should succeed");
|
||||
assert_eq!(exists, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn gc_retention_prunes_old_audit_and_alert_history() {
|
||||
let ts = TestStore::new().await;
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("clock should be after epoch")
|
||||
.as_secs() as i64;
|
||||
let old_ts = now - 86_400;
|
||||
let old_evt = format!("{:020}:old-evt", old_ts);
|
||||
sqlx::query!(
|
||||
"INSERT INTO audit_events
|
||||
(event_key, event_id, ts_unix, actor_type, actor_id, agent_id, request_id,
|
||||
event_type, outcome, latency_ms, message, metadata_json)
|
||||
VALUES (?1, ?2, ?3, 'test', NULL, NULL, NULL, 'test', 'ok', NULL, 'old', '{}')",
|
||||
old_evt,
|
||||
"old-evt",
|
||||
old_ts
|
||||
)
|
||||
.execute(&ts.store().pool)
|
||||
.await
|
||||
.expect("old audit event should insert");
|
||||
|
||||
let old_atr = format!("{:020}:old-atr", old_ts);
|
||||
sqlx::query!(
|
||||
"INSERT INTO alert_transitions
|
||||
(transition_key, transition_id, ts_unix, alert_id, kind, agent_id,
|
||||
from_status, to_status, message, metadata_json)
|
||||
VALUES (?1, ?2, ?3, 'alert-old', 'test', NULL, NULL, 'active', 'old', '{}')",
|
||||
old_atr,
|
||||
"old-atr",
|
||||
old_ts
|
||||
)
|
||||
.execute(&ts.store().pool)
|
||||
.await
|
||||
.expect("old alert transition should insert");
|
||||
|
||||
ts.store()
|
||||
.append_audit_event(crate::state::AuditEventInput {
|
||||
actor_type: "test".into(),
|
||||
actor_id: None,
|
||||
agent_id: None,
|
||||
request_id: None,
|
||||
event_type: "test".into(),
|
||||
outcome: "ok".into(),
|
||||
latency_ms: None,
|
||||
message: "fresh".into(),
|
||||
metadata: serde_json::json!({}),
|
||||
})
|
||||
.await
|
||||
.expect("fresh audit event should insert");
|
||||
|
||||
let stats = ts
|
||||
.store()
|
||||
.gc_retention(Duration::from_secs(3600))
|
||||
.await
|
||||
.expect("retention gc should succeed");
|
||||
|
||||
assert_eq!(stats.audit_events_removed, 1);
|
||||
assert_eq!(stats.alert_transitions_removed, 1);
|
||||
|
||||
let old_audit_gone = sqlx::query_scalar!(
|
||||
"SELECT COUNT(*) FROM audit_events WHERE event_id = ?1",
|
||||
"old-evt"
|
||||
)
|
||||
.fetch_one(&ts.store().pool)
|
||||
.await
|
||||
.expect("read should succeed");
|
||||
assert_eq!(old_audit_gone, 0);
|
||||
|
||||
let fresh_audit_remains = sqlx::query_scalar!(
|
||||
"SELECT COUNT(*) FROM audit_events WHERE message = ?1",
|
||||
"fresh"
|
||||
)
|
||||
.fetch_one(&ts.store().pool)
|
||||
.await
|
||||
.expect("read should succeed");
|
||||
assert_eq!(fresh_audit_remains, 1);
|
||||
|
||||
let old_transition_gone = sqlx::query_scalar!(
|
||||
"SELECT COUNT(*) FROM alert_transitions WHERE transition_id = ?1",
|
||||
"old-atr"
|
||||
)
|
||||
.fetch_one(&ts.store().pool)
|
||||
.await
|
||||
.expect("read should succeed");
|
||||
assert_eq!(old_transition_gone, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enroll_rejects_expired_token() {
|
||||
let ts = TestStore::new().await;
|
||||
@@ -209,7 +327,11 @@ mod tests {
|
||||
.expect("nickname set should succeed");
|
||||
assert!(updated);
|
||||
|
||||
let listed = ts.store().list_agents_with_nicknames().await;
|
||||
let listed = ts
|
||||
.store()
|
||||
.list_agents_with_nicknames()
|
||||
.await
|
||||
.expect("list agents with nicknames should succeed");
|
||||
assert!(listed.iter().any(|(id, name)| {
|
||||
id == &issued.agent_id && name.as_deref() == Some("kitchen-router")
|
||||
}));
|
||||
@@ -221,7 +343,11 @@ mod tests {
|
||||
.expect("nickname clear should succeed");
|
||||
assert!(cleared);
|
||||
|
||||
let listed = ts.store().list_agents_with_nicknames().await;
|
||||
let listed = ts
|
||||
.store()
|
||||
.list_agents_with_nicknames()
|
||||
.await
|
||||
.expect("list agents with nicknames should succeed");
|
||||
assert!(
|
||||
listed
|
||||
.iter()
|
||||
|
||||
@@ -1,6 +1,25 @@
|
||||
use super::*;
|
||||
|
||||
impl Store {
|
||||
/// Best-effort audit append: logs the failure with event context instead of
|
||||
/// bubbling it up. Auditing must never fail the operation that triggered it.
|
||||
pub async fn audit(&self, input: AuditEventInput) {
|
||||
let event_type = input.event_type.clone();
|
||||
let outcome = input.outcome.clone();
|
||||
let agent_id = input.agent_id.clone();
|
||||
let request_id = input.request_id.clone();
|
||||
if let Err(err) = self.append_audit_event(input).await {
|
||||
warn!(
|
||||
error = %err,
|
||||
event_type = %event_type,
|
||||
outcome = %outcome,
|
||||
agent_id = ?agent_id,
|
||||
request_id = ?request_id,
|
||||
"failed to append audit event"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn append_audit_event(&self, input: AuditEventInput) -> Result<AuditEvent> {
|
||||
let event = AuditEvent {
|
||||
event_id: format!("evt-{}", Uuid::new_v4()),
|
||||
|
||||
@@ -154,6 +154,37 @@ impl Store {
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
/// Prunes audit and alert history older than the retention window.
|
||||
///
|
||||
/// Runs from the daemon's periodic maintenance loop; both tables have a
|
||||
/// `ts_unix` index so the deletes stay cheap as history grows.
|
||||
pub async fn gc_retention(&self, older_than: Duration) -> Result<RetentionStats> {
|
||||
let cutoff = i64::try_from(now_unix().saturating_sub(older_than.as_secs().max(1)))
|
||||
.context("retention cutoff does not fit SQLite integer")?;
|
||||
let audit_events_removed =
|
||||
sqlx::query!("DELETE FROM audit_events WHERE ts_unix < ?1", cutoff)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.context("failed pruning old audit events")?
|
||||
.rows_affected();
|
||||
let alert_transitions_removed =
|
||||
sqlx::query!("DELETE FROM alert_transitions WHERE ts_unix < ?1", cutoff)
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.context("failed pruning old alert transitions")?
|
||||
.rows_affected();
|
||||
if audit_events_removed > 0 || alert_transitions_removed > 0 {
|
||||
info!(
|
||||
audit_events_removed,
|
||||
alert_transitions_removed, "pruned state rows older than retention window"
|
||||
);
|
||||
}
|
||||
Ok(RetentionStats {
|
||||
audit_events_removed,
|
||||
alert_transitions_removed,
|
||||
})
|
||||
}
|
||||
|
||||
async fn ensure_schema_version(&self) -> Result<()> {
|
||||
match sqlx::query_scalar!("SELECT value FROM meta WHERE key = ?1", SCHEMA_VERSION_KEY)
|
||||
.fetch_optional(&self.pool)
|
||||
|
||||
@@ -17,7 +17,8 @@ impl Store {
|
||||
};
|
||||
|
||||
let now = now_unix();
|
||||
if expires_at_unix as u64 <= now {
|
||||
let now_i64 = i64::try_from(now).context("current time does not fit SQLite integer")?;
|
||||
if expires_at_unix <= now_i64 {
|
||||
sqlx::query!("DELETE FROM enroll_tokens WHERE token = ?1", enroll_token)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
@@ -165,20 +166,16 @@ impl Store {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn list_agents(&self) -> Vec<String> {
|
||||
match sqlx::query_scalar!(r#"SELECT agent_id as "agent_id!" FROM agents ORDER BY agent_id"#)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
{
|
||||
Ok(out) => out,
|
||||
Err(err) => {
|
||||
warn!(error = %err, "failed to list agents from state db");
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
pub async fn list_agents(&self) -> Result<Vec<String>> {
|
||||
let ids =
|
||||
sqlx::query_scalar!(r#"SELECT agent_id as "agent_id!" FROM agents ORDER BY agent_id"#)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.context("failed listing agents from state db")?;
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
pub async fn list_agents_with_nicknames(&self) -> Vec<(String, Option<String>)> {
|
||||
pub async fn list_agents_with_nicknames(&self) -> Result<Vec<(String, Option<String>)>> {
|
||||
let rows = sqlx::query!(
|
||||
r#"SELECT agents.agent_id as "agent_id!", agent_meta.nickname
|
||||
FROM agents
|
||||
@@ -186,23 +183,19 @@ impl Store {
|
||||
ORDER BY agents.agent_id"#,
|
||||
)
|
||||
.fetch_all(&self.pool)
|
||||
.await;
|
||||
.await
|
||||
.context("failed listing agents from state db")?;
|
||||
|
||||
match rows {
|
||||
Ok(rows) => rows
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
let nickname = row
|
||||
.nickname
|
||||
.map(|v| v.trim().to_string())
|
||||
.filter(|v| !v.is_empty());
|
||||
(row.agent_id, nickname)
|
||||
})
|
||||
.collect(),
|
||||
Err(err) => {
|
||||
warn!(error = %err, "failed to list agents from state db");
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
let agents = rows
|
||||
.into_iter()
|
||||
.map(|row| {
|
||||
let nickname = row
|
||||
.nickname
|
||||
.map(|v| v.trim().to_string())
|
||||
.filter(|v| !v.is_empty());
|
||||
(row.agent_id, nickname)
|
||||
})
|
||||
.collect();
|
||||
Ok(agents)
|
||||
}
|
||||
}
|
||||
@@ -32,6 +32,12 @@ pub struct StateStats {
|
||||
pub expired_enroll_token_count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RetentionStats {
|
||||
pub audit_events_removed: u64,
|
||||
pub alert_transitions_removed: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct KnownDevice {
|
||||
pub device_id: String,
|
||||
|
||||
Reference in new issue
Block a user