push terminal session changes and alert evaluator health

ui/src/App.tsx

@@ -1,4 +1,4 @@
-import { lazy, Suspense, useEffect, useState } from "react";
+import { lazy, Suspense, useEffect, useRef, useState } from "react";
 import { Navigate, Route, Routes } from "react-router-dom";
 import { toast } from "sonner";
 
@@ -40,22 +40,48 @@ export function App() {
   const [history, setHistory] = useState<AlertTransition[]>([]);
   const [audit, setAudit] = useState<AuditEvent[]>([]);
   const [selectedAgentId, setSelectedAgentId] = useState("");
+  const [alertsHealth, setAlertsHealth] = useState<{
+    available: boolean;
+    reason?: string;
+  } | null>(null);
+  const [alertsStreamConnected, setAlertsStreamConnected] = useState(false);
+  const streamHealthReceived = useRef(false);
+  const alertsRevision = useRef(0);
 
   const [state, setState] = useState<LoadState>("idle");
 
+  async function loadAlertsWithoutBlocking() {
+    try {
+      const nextAlerts = await fetchAlerts();
+      if (!streamHealthReceived.current) {
+        setAlertsHealth({ available: true });
+      }
+      return nextAlerts;
+    } catch (err) {
+      if (!streamHealthReceived.current) {
+        setAlertsHealth({ available: false, reason: String(err) });
+      }
+      return null;
+    }
+  }
+
   async function loadAll() {
+    const revision = alertsRevision.current;
     setState("loading");
     try {
       const [nextAgents, nextAlerts, nextHistory, nextAudit] =
         await Promise.all([
           fetchAgents(),
-          fetchAlerts(),
+          loadAlertsWithoutBlocking(),
           fetchAlertHistory(20),
           fetchAudit(30),
         ]);
       setAgents(nextAgents);
-      setAlerts(nextAlerts);
-      setHistory(nextHistory);
+      // Streamed state wins over HTTP requests started before its arrival.
+      if (revision === alertsRevision.current) {
+        if (nextAlerts) setAlerts(nextAlerts);
+        setHistory(nextHistory);
+      }
       setAudit(nextAudit);
       const firstConnectedAgentId =
         nextAgents.find((agent) => agent.connected)?.agent_id ?? "";
@@ -100,12 +126,15 @@ export function App() {
   }
 
   async function refreshAlertsAndHistory() {
+    const revision = alertsRevision.current;
     const [nextAlerts, nextHistory] = await Promise.all([
-      fetchAlerts(),
+      loadAlertsWithoutBlocking(),
       fetchAlertHistory(20),
     ]);
-    setAlerts(nextAlerts);
-    setHistory(nextHistory);
+    if (revision === alertsRevision.current) {
+      if (nextAlerts) setAlerts(nextAlerts);
+      setHistory(nextHistory);
+    }
   }
 
   useEffect(() => {
@@ -117,15 +146,14 @@ export function App() {
     let disposed = false;
     let attempt = 0;
     let openedAt = 0;
-    let wsGeneration = 0;
     let reconnectTimer: number | undefined;
 
     const resync = () => {
-      const generation = wsGeneration;
+      const generation = alertsRevision.current;
       void Promise.all([fetchAlerts(), fetchAlertHistory(20)])
         .then(([nextAlerts, nextHistory]) => {
           // A WS snapshot delivered while this request was in flight is newer.
-          if (generation !== wsGeneration) return;
+          if (disposed || generation !== alertsRevision.current) return;
           setAlerts(nextAlerts);
           setHistory(nextHistory);
         })
@@ -138,18 +166,32 @@ export function App() {
       socket = new WebSocket(wsUrl);
 
       socket.onopen = () => {
+        if (disposed) return;
         openedAt = Date.now();
+        setAlertsStreamConnected(true);
         // Pull anything missed while the stream was down.
         resync();
       };
 
       socket.onmessage = (evt) => {
-        wsGeneration += 1;
+        if (disposed) return;
+        alertsRevision.current += 1;
         try {
           const payload = JSON.parse(String(evt.data)) as {
+            type?: string;
+            available?: boolean;
+            reason?: string;
             alerts?: Alert[];
             recent_transitions?: AlertTransition[];
           };
+          if (payload.type === "alerts_health") {
+            streamHealthReceived.current = true;
+            setAlertsHealth({
+              available: payload.available === true,
+              reason: payload.reason,
+            });
+            return;
+          }
           if (payload.alerts) setAlerts(payload.alerts);
           if (payload.recent_transitions)
             setHistory(payload.recent_transitions);
@@ -161,6 +203,7 @@ export function App() {
       socket.onclose = () => {
         socket = null;
         if (disposed) return;
+        setAlertsStreamConnected(false);
         // Only a connection that stayed up proves the stream is healthy; an
         // accept-then-close server must not pin us to the first backoff step.
         if (openedAt && Date.now() - openedAt > 10_000) attempt = 0;
@@ -217,6 +260,8 @@ export function App() {
                 transitions={history}
                 audit={audit}
                 loading={state === "loading"}
+                alertsHealth={alertsHealth}
+                alertsStreamConnected={alertsStreamConnected}
                 onRefresh={loadAll}
               />
             }
@@ -259,6 +304,8 @@ export function App() {
               <AlertsPage
                 alerts={alerts}
                 transitions={history}
+                alertsHealth={alertsHealth}
+                alertsStreamConnected={alertsStreamConnected}
                 onRefresh={refreshAlertsAndHistory}
               />
             }

ui/src/pages/AlertsPage.tsx

@@ -16,10 +16,18 @@ import { Textarea } from "@/components/ui/textarea";
 type Props = {
   alerts: Alert[];
   transitions: AlertTransition[];
+  alertsHealth: { available: boolean; reason?: string } | null;
+  alertsStreamConnected: boolean;
   onRefresh: () => Promise<void>;
 };
 
-export function AlertsPage({ alerts, transitions, onRefresh }: Props) {
+export function AlertsPage({
+  alerts,
+  transitions,
+  alertsHealth,
+  alertsStreamConnected,
+  onRefresh,
+}: Props) {
   const [severity, setSeverity] = useState("all");
   const [status, setStatus] = useState("all");
   const [kind, setKind] = useState("all");
@@ -66,9 +74,27 @@ export function AlertsPage({ alerts, transitions, onRefresh }: Props) {
       <Card>
         <CardHeader className="flex items-center justify-between gap-2">
           <CardTitle>Active Alerts</CardTitle>
-          <Button size="sm" variant="outline" onClick={() => void onRefresh()}>
-            Refresh
-          </Button>
+          <div className="flex flex-wrap items-center justify-end gap-x-3 gap-y-1">
+            <div
+              aria-live="polite"
+              className="text-right text-xs text-muted-foreground"
+            >
+              {!alertsStreamConnected && <div>Alert stream disconnected</div>}
+              {alertsHealth?.available === false && (
+                <div>
+                  Alert results may be stale or unavailable
+                  {alertsHealth.reason ? `: ${alertsHealth.reason}` : ""}
+                </div>
+              )}
+            </div>
+            <Button
+              size="sm"
+              variant="outline"
+              onClick={() => void onRefresh()}
+            >
+              Refresh
+            </Button>
+          </div>
         </CardHeader>
         <CardContent className="space-y-3">
           <div className="grid gap-2 sm:grid-cols-3">

ui/src/pages/DashboardPage.tsx

@@ -27,6 +27,8 @@ type Props = {
   transitions: AlertTransition[];
   audit: AuditEvent[];
   loading: boolean;
+  alertsHealth: { available: boolean; reason?: string } | null;
+  alertsStreamConnected: boolean;
   onRefresh: () => void;
 };
 
@@ -66,6 +68,8 @@ export function DashboardPage({
   transitions,
   audit,
   loading,
+  alertsHealth,
+  alertsStreamConnected,
   onRefresh,
 }: Props) {
   const connected = agents.filter((a) => a.connected).length;
@@ -165,6 +169,20 @@ export function DashboardPage({
           </Link>
         </CardHeader>
         <CardContent>
+          {(!alertsStreamConnected || alertsHealth?.available === false) && (
+            <div
+              aria-live="polite"
+              className="mb-2 text-xs text-muted-foreground"
+            >
+              {!alertsStreamConnected && <div>Alert stream disconnected</div>}
+              {alertsHealth?.available === false && (
+                <div>
+                  Alert results may be stale or unavailable
+                  {alertsHealth.reason ? `: ${alertsHealth.reason}` : ""}
+                </div>
+              )}
+            </div>
+          )}
           <div className="text-2xl font-bold tabular-nums">{alerts.length}</div>
           {Object.keys(severityBuckets).length > 0 ? (
             <div className="mt-2 flex flex-wrap gap-1.5">

ui/src/pages/TerminalPage.tsx

@@ -513,9 +513,38 @@ export function TerminalPage({
     terminalRef.current = terminal;
     fitRef.current = fit;
 
+    let cancelled = false;
+    let refreshInFlight: Promise<TerminalSession[]> | null = null;
+    let refreshRequested = false;
+    const refreshSessions = (): Promise<TerminalSession[]> => {
+      if (refreshInFlight) {
+        refreshRequested = true;
+        return refreshInFlight;
+      }
+      refreshInFlight = (async () => {
+        let listed: TerminalSession[] = [];
+        do {
+          refreshRequested = false;
+          listed = await listTerminals();
+          if (!cancelled) {
+            setSessions((current) =>
+              reconcileTerminalSessions(current, listed),
+            );
+          }
+        } while (refreshRequested && !cancelled);
+        return listed;
+      })().finally(() => {
+        refreshInFlight = null;
+        if (refreshRequested && !cancelled) {
+          void refreshSessions().catch(() => undefined);
+        }
+      });
+      return refreshInFlight;
+    };
+
     async function restoreTerminalSession() {
       try {
-        const listed = await listTerminals();
+        const listed = await refreshSessions();
         if (cancelled) return;
         setSessions(orderTerminalSessions(listed));
         const rememberedId = window.sessionStorage.getItem(
@@ -557,7 +586,6 @@ export function TerminalPage({
       }
     }
 
-    let cancelled = false;
     const input = terminal.onData((data) => writeTerminalData(data, true));
     let resizeFrame = 0;
     let lastHostWidth = 0;
@@ -575,30 +603,61 @@ export function TerminalPage({
       });
     });
     resizeObserver.observe(hostRef.current);
-    void restoreTerminalSession();
-    let refreshInFlight = false;
-    const refreshSessions = () => {
-      if (refreshInFlight) return;
-      refreshInFlight = true;
-      void listTerminals()
-        .then((listed) => {
-          if (!cancelled) {
-            setSessions((current) =>
-              reconcileTerminalSessions(current, listed),
-            );
+    let eventsSocket: WebSocket | null = null;
+    let reconnectTimer = 0;
+    let healthyTimer = 0;
+    let reconnectAttempt = 0;
+    const connectEvents = () => {
+      if (cancelled) return;
+      const socket = new WebSocket(
+        websocketUrl("/api/v1/control/terminals/events/ws"),
+      );
+      eventsSocket = socket;
+      socket.onmessage = (event) => {
+        if (cancelled || eventsSocket !== socket) return;
+        if (typeof event.data !== "string") return;
+        try {
+          const message = JSON.parse(event.data) as { type?: string };
+          if (message.type === "sessions_changed") {
+            void refreshSessions().catch(() => undefined);
           }
-        })
-        .catch(() => {
-          // The attached terminal transport remains authoritative while a
-          // background list refresh is temporarily unavailable.
-        })
-        .finally(() => {
-          refreshInFlight = false;
-        });
+        } catch {
+          // Ignore malformed event frames; the fallback refresh remains active.
+        }
+      };
+      socket.onopen = () => {
+        window.clearTimeout(healthyTimer);
+        healthyTimer = window.setTimeout(() => {
+          if (eventsSocket === socket && socket.readyState === WebSocket.OPEN) {
+            reconnectAttempt = 0;
+          }
+        }, 10_000);
+      };
+      socket.onclose = () => {
+        if (cancelled || eventsSocket !== socket) return;
+        window.clearTimeout(healthyTimer);
+        const ceiling = Math.min(500 * 2 ** reconnectAttempt, 12_000);
+        const delay = Math.min(
+          Math.round(ceiling * (0.8 + Math.random() * 0.4)),
+          15_000,
+        );
+        reconnectAttempt = Math.min(reconnectAttempt + 1, 5);
+        reconnectTimer = window.setTimeout(connectEvents, delay);
+      };
+      socket.onerror = () => socket.close();
     };
-    refreshSessionsRef.current = refreshSessions;
-    const refreshTimer = window.setInterval(refreshSessions, 15_000);
-    window.addEventListener("focus", refreshSessions);
+    connectEvents();
+    void restoreTerminalSession();
+    refreshSessionsRef.current = () => {
+      void refreshSessions().catch(() => undefined);
+    };
+    const refreshTimer = window.setInterval(() => {
+      void refreshSessions().catch(() => undefined);
+    }, 15_000);
+    const refreshOnFocus = () => {
+      void refreshSessions().catch(() => undefined);
+    };
+    window.addEventListener("focus", refreshOnFocus);
 
     return () => {
       cancelled = true;
@@ -606,7 +665,11 @@ export function TerminalPage({
       titleChange.dispose();
       resizeObserver.disconnect();
       window.clearInterval(refreshTimer);
-      window.removeEventListener("focus", refreshSessions);
+      window.removeEventListener("focus", refreshOnFocus);
+      window.clearTimeout(reconnectTimer);
+      window.clearTimeout(healthyTimer);
+      eventsSocket?.close();
+      eventsSocket = null;
       refreshSessionsRef.current = () => {};
       window.cancelAnimationFrame(resizeFrame);
       detachTransport();