Trying to save some events

.sqlx/query-52a39dd83e65374a5c9e5f55c8492d9927273712ec75814d08674784771fbad2.json

@@ -0,0 +1,86 @@
+{
+  "db_name": "SQLite",
+  "query": "SELECT events.event_id as \"event_id!\",\n                    ('agent:' || events.agent_id || ':' || events.kind || ':' ||\n                        CASE\n                            WHEN events.kind = 'neigh' AND events.mac IS NOT NULL AND events.ip IS NOT NULL\n                                THEN 'mac:' || events.mac || ':ip:' || events.ip\n                            WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac\n                            WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip\n                            ELSE ''\n                        END) as \"observation_key!\",\n                    events.agent_id as \"agent_id!\",\n                    events.kind as \"kind!\",\n                    events.action as \"action!\",\n                    events.mac,\n                    events.ip,\n                    events.hostname,\n                    events.ts_unix,\n                    known_devices.device_id,\n                    known_devices.display_name,\n                    known_devices.pinned\n             FROM agent_device_observation_events events\n             LEFT JOIN device_identifiers identifiers\n               ON identifiers.identifier_key =\n                  CASE\n                    WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac\n                    WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip\n                  END\n             LEFT JOIN known_devices ON known_devices.device_id = identifiers.device_id\n             WHERE (?1 IS NULL OR events.agent_id = ?1)\n               AND (?2 IS NULL OR events.kind = ?2)\n               AND (?3 IS NULL OR events.mac = ?3)\n               AND (?4 IS NULL OR events.ip = ?4)\n               AND (?5 IS NULL OR ('agent:' || events.agent_id || ':' || events.kind || ':' ||\n                    CASE\n                        WHEN events.kind = 'neigh' AND events.mac IS NOT NULL AND events.ip IS NOT NULL\n                            THEN 'mac:' || events.mac || ':ip:' || events.ip\n                        WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac\n                        WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip\n                        ELSE ''\n                    END) = ?5)\n             ORDER BY events.ts_unix DESC\n             LIMIT ?6",
+  "describe": {
+    "columns": [
+      {
+        "name": "event_id!",
+        "ordinal": 0,
+        "type_info": "Text"
+      },
+      {
+        "name": "observation_key!",
+        "ordinal": 1,
+        "type_info": "Text"
+      },
+      {
+        "name": "agent_id!",
+        "ordinal": 2,
+        "type_info": "Text"
+      },
+      {
+        "name": "kind!",
+        "ordinal": 3,
+        "type_info": "Text"
+      },
+      {
+        "name": "action!",
+        "ordinal": 4,
+        "type_info": "Text"
+      },
+      {
+        "name": "mac",
+        "ordinal": 5,
+        "type_info": "Text"
+      },
+      {
+        "name": "ip",
+        "ordinal": 6,
+        "type_info": "Text"
+      },
+      {
+        "name": "hostname",
+        "ordinal": 7,
+        "type_info": "Text"
+      },
+      {
+        "name": "ts_unix",
+        "ordinal": 8,
+        "type_info": "Integer"
+      },
+      {
+        "name": "device_id",
+        "ordinal": 9,
+        "type_info": "Text"
+      },
+      {
+        "name": "display_name",
+        "ordinal": 10,
+        "type_info": "Text"
+      },
+      {
+        "name": "pinned",
+        "ordinal": 11,
+        "type_info": "Integer"
+      }
+    ],
+    "parameters": {
+      "Right": 6
+    },
+    "nullable": [
+      true,
+      true,
+      false,
+      false,
+      false,
+      true,
+      true,
+      true,
+      false,
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "52a39dd83e65374a5c9e5f55c8492d9927273712ec75814d08674784771fbad2"
+}

.sqlx/query-908d763909c76fb6170b818784e360372c02295523ef9970c91199db7d4f535a.json

@@ -1,86 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "SELECT events.event_id as \"event_id!\",\n                    ('agent:' || events.agent_id || ':' || events.kind || ':' ||\n                        CASE\n                            WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac\n                            WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip\n                            ELSE ''\n                        END) as \"observation_key!\",\n                    events.agent_id as \"agent_id!\",\n                    events.kind as \"kind!\",\n                    events.action as \"action!\",\n                    events.mac,\n                    events.ip,\n                    events.hostname,\n                    events.ts_unix,\n                    known_devices.device_id,\n                    known_devices.display_name,\n                    known_devices.pinned\n             FROM agent_device_observation_events events\n             LEFT JOIN device_identifiers identifiers\n               ON identifiers.identifier_key =\n                  CASE\n                    WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac\n                    WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip\n                  END\n             LEFT JOIN known_devices ON known_devices.device_id = identifiers.device_id\n             WHERE (?1 IS NULL OR events.agent_id = ?1)\n               AND (?2 IS NULL OR events.kind = ?2)\n               AND (?3 IS NULL OR events.mac = ?3)\n               AND (?4 IS NULL OR events.ip = ?4)\n               AND (?5 IS NULL OR ('agent:' || events.agent_id || ':' || events.kind || ':' ||\n                    CASE\n                        WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac\n                        WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip\n                        ELSE ''\n                    END) = ?5)\n             ORDER BY events.ts_unix DESC\n             LIMIT ?6",
-  "describe": {
-    "columns": [
-      {
-        "name": "event_id!",
-        "ordinal": 0,
-        "type_info": "Text"
-      },
-      {
-        "name": "observation_key!",
-        "ordinal": 1,
-        "type_info": "Text"
-      },
-      {
-        "name": "agent_id!",
-        "ordinal": 2,
-        "type_info": "Text"
-      },
-      {
-        "name": "kind!",
-        "ordinal": 3,
-        "type_info": "Text"
-      },
-      {
-        "name": "action!",
-        "ordinal": 4,
-        "type_info": "Text"
-      },
-      {
-        "name": "mac",
-        "ordinal": 5,
-        "type_info": "Text"
-      },
-      {
-        "name": "ip",
-        "ordinal": 6,
-        "type_info": "Text"
-      },
-      {
-        "name": "hostname",
-        "ordinal": 7,
-        "type_info": "Text"
-      },
-      {
-        "name": "ts_unix",
-        "ordinal": 8,
-        "type_info": "Integer"
-      },
-      {
-        "name": "device_id",
-        "ordinal": 9,
-        "type_info": "Text"
-      },
-      {
-        "name": "display_name",
-        "ordinal": 10,
-        "type_info": "Text"
-      },
-      {
-        "name": "pinned",
-        "ordinal": 11,
-        "type_info": "Integer"
-      }
-    ],
-    "parameters": {
-      "Right": 6
-    },
-    "nullable": [
-      true,
-      true,
-      false,
-      false,
-      false,
-      true,
-      true,
-      true,
-      false,
-      true,
-      true,
-      true
-    ]
-  },
-  "hash": "908d763909c76fb6170b818784e360372c02295523ef9970c91199db7d4f535a"
-}

remote/copy.ps1

@@ -25,3 +25,11 @@ $remote = $files.ForEach({ "/etc/init.d/$_" })
 $remote | ForEach-Object {
     pscp.exe -l root -scp -pw $PASSWD "192.168.100.1:$_" "$initDir\"
 }
+
+$files = "dhcp/95-wakey", "neigh/95-wakey"
+$remote =  $files.ForEach({ "/etc/hotplug.d/$_" })
+$hotplugDir = Join-Path $PSScriptRoot 'hotplug.d'
+
+$remote | ForEach-Object {
+    pscp.exe -l root -scp -pw $PASSWD "192.168.100.1:$_" "$hotplugDir\"
+}

wakey-control-plane/.sqlx/query-52a39dd83e65374a5c9e5f55c8492d9927273712ec75814d08674784771fbad2.json

@@ -0,0 +1,86 @@
+{
+  "db_name": "SQLite",
+  "query": "SELECT events.event_id as \"event_id!\",\n                    ('agent:' || events.agent_id || ':' || events.kind || ':' ||\n                        CASE\n                            WHEN events.kind = 'neigh' AND events.mac IS NOT NULL AND events.ip IS NOT NULL\n                                THEN 'mac:' || events.mac || ':ip:' || events.ip\n                            WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac\n                            WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip\n                            ELSE ''\n                        END) as \"observation_key!\",\n                    events.agent_id as \"agent_id!\",\n                    events.kind as \"kind!\",\n                    events.action as \"action!\",\n                    events.mac,\n                    events.ip,\n                    events.hostname,\n                    events.ts_unix,\n                    known_devices.device_id,\n                    known_devices.display_name,\n                    known_devices.pinned\n             FROM agent_device_observation_events events\n             LEFT JOIN device_identifiers identifiers\n               ON identifiers.identifier_key =\n                  CASE\n                    WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac\n                    WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip\n                  END\n             LEFT JOIN known_devices ON known_devices.device_id = identifiers.device_id\n             WHERE (?1 IS NULL OR events.agent_id = ?1)\n               AND (?2 IS NULL OR events.kind = ?2)\n               AND (?3 IS NULL OR events.mac = ?3)\n               AND (?4 IS NULL OR events.ip = ?4)\n               AND (?5 IS NULL OR ('agent:' || events.agent_id || ':' || events.kind || ':' ||\n                    CASE\n                        WHEN events.kind = 'neigh' AND events.mac IS NOT NULL AND events.ip IS NOT NULL\n                            THEN 'mac:' || events.mac || ':ip:' || events.ip\n                        WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac\n                        WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip\n                        ELSE ''\n                    END) = ?5)\n             ORDER BY events.ts_unix DESC\n             LIMIT ?6",
+  "describe": {
+    "columns": [
+      {
+        "name": "event_id!",
+        "ordinal": 0,
+        "type_info": "Text"
+      },
+      {
+        "name": "observation_key!",
+        "ordinal": 1,
+        "type_info": "Text"
+      },
+      {
+        "name": "agent_id!",
+        "ordinal": 2,
+        "type_info": "Text"
+      },
+      {
+        "name": "kind!",
+        "ordinal": 3,
+        "type_info": "Text"
+      },
+      {
+        "name": "action!",
+        "ordinal": 4,
+        "type_info": "Text"
+      },
+      {
+        "name": "mac",
+        "ordinal": 5,
+        "type_info": "Text"
+      },
+      {
+        "name": "ip",
+        "ordinal": 6,
+        "type_info": "Text"
+      },
+      {
+        "name": "hostname",
+        "ordinal": 7,
+        "type_info": "Text"
+      },
+      {
+        "name": "ts_unix",
+        "ordinal": 8,
+        "type_info": "Integer"
+      },
+      {
+        "name": "device_id",
+        "ordinal": 9,
+        "type_info": "Text"
+      },
+      {
+        "name": "display_name",
+        "ordinal": 10,
+        "type_info": "Text"
+      },
+      {
+        "name": "pinned",
+        "ordinal": 11,
+        "type_info": "Integer"
+      }
+    ],
+    "parameters": {
+      "Right": 6
+    },
+    "nullable": [
+      true,
+      true,
+      false,
+      false,
+      false,
+      true,
+      true,
+      true,
+      false,
+      true,
+      true,
+      true
+    ]
+  },
+  "hash": "52a39dd83e65374a5c9e5f55c8492d9927273712ec75814d08674784771fbad2"
+}

wakey-control-plane/.sqlx/query-908d763909c76fb6170b818784e360372c02295523ef9970c91199db7d4f535a.json

@@ -1,86 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "SELECT events.event_id as \"event_id!\",\n                    ('agent:' || events.agent_id || ':' || events.kind || ':' ||\n                        CASE\n                            WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac\n                            WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip\n                            ELSE ''\n                        END) as \"observation_key!\",\n                    events.agent_id as \"agent_id!\",\n                    events.kind as \"kind!\",\n                    events.action as \"action!\",\n                    events.mac,\n                    events.ip,\n                    events.hostname,\n                    events.ts_unix,\n                    known_devices.device_id,\n                    known_devices.display_name,\n                    known_devices.pinned\n             FROM agent_device_observation_events events\n             LEFT JOIN device_identifiers identifiers\n               ON identifiers.identifier_key =\n                  CASE\n                    WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac\n                    WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip\n                  END\n             LEFT JOIN known_devices ON known_devices.device_id = identifiers.device_id\n             WHERE (?1 IS NULL OR events.agent_id = ?1)\n               AND (?2 IS NULL OR events.kind = ?2)\n               AND (?3 IS NULL OR events.mac = ?3)\n               AND (?4 IS NULL OR events.ip = ?4)\n               AND (?5 IS NULL OR ('agent:' || events.agent_id || ':' || events.kind || ':' ||\n                    CASE\n                        WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac\n                        WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip\n                        ELSE ''\n                    END) = ?5)\n             ORDER BY events.ts_unix DESC\n             LIMIT ?6",
-  "describe": {
-    "columns": [
-      {
-        "name": "event_id!",
-        "ordinal": 0,
-        "type_info": "Text"
-      },
-      {
-        "name": "observation_key!",
-        "ordinal": 1,
-        "type_info": "Text"
-      },
-      {
-        "name": "agent_id!",
-        "ordinal": 2,
-        "type_info": "Text"
-      },
-      {
-        "name": "kind!",
-        "ordinal": 3,
-        "type_info": "Text"
-      },
-      {
-        "name": "action!",
-        "ordinal": 4,
-        "type_info": "Text"
-      },
-      {
-        "name": "mac",
-        "ordinal": 5,
-        "type_info": "Text"
-      },
-      {
-        "name": "ip",
-        "ordinal": 6,
-        "type_info": "Text"
-      },
-      {
-        "name": "hostname",
-        "ordinal": 7,
-        "type_info": "Text"
-      },
-      {
-        "name": "ts_unix",
-        "ordinal": 8,
-        "type_info": "Integer"
-      },
-      {
-        "name": "device_id",
-        "ordinal": 9,
-        "type_info": "Text"
-      },
-      {
-        "name": "display_name",
-        "ordinal": 10,
-        "type_info": "Text"
-      },
-      {
-        "name": "pinned",
-        "ordinal": 11,
-        "type_info": "Integer"
-      }
-    ],
-    "parameters": {
-      "Right": 6
-    },
-    "nullable": [
-      true,
-      true,
-      false,
-      false,
-      false,
-      true,
-      true,
-      true,
-      false,
-      true,
-      true,
-      true
-    ]
-  },
-  "hash": "908d763909c76fb6170b818784e360372c02295523ef9970c91199db7d4f535a"
-}

wakey-control-plane/src/api/control/fleet.rs

@@ -323,6 +323,13 @@ pub async fn wake_fleet_device(
             "selected route agent is not connected",
         ));
     }
+    if !route.wakeable {
+        return Err(json_error(
+            StatusCode::BAD_REQUEST,
+            "wake_route_unavailable",
+            "selected route is not wakeable",
+        ));
+    }
 
     let mac = mac.parse().map_err(|err| {
         json_error(
@@ -530,6 +537,7 @@ fn add_observation_to_entry(
     observation: AgentDeviceObservation,
     context: &FleetBuildContext,
 ) {
+    let observation_offline = observation_is_offline(&observation);
     if let Some(summary) = observation_known_device(&observation, context)
         && entry.known_device.is_none()
     {
@@ -540,7 +548,7 @@ fn add_observation_to_entry(
     if let Some(mac) = observation.mac.as_deref() {
         entry.macs.insert(mac.to_string());
     }
-    if let Some(ip) = observation.ip.as_deref() {
+    if !observation_offline && let Some(ip) = observation.ip.as_deref() {
         entry.ips.insert(ip.to_string());
     }
     if let Some(hostname) = observation.hostname.as_deref() {
@@ -595,7 +603,7 @@ fn add_observation_to_entry(
         observation.ip.as_deref(),
         &observation.kind,
     );
-    let wakeable = status.connected && observation.mac.is_some();
+    let wakeable = status.connected && observation.mac.is_some() && !observation_offline;
     entry.routes.insert(
         route_id.clone(),
         FleetWakeRoute {
@@ -612,7 +620,7 @@ fn add_observation_to_entry(
         },
     );
     if let Some(route) = entry.routes.get_mut(&route_id) {
-        route.wakeable = route.connected && route.mac.is_some();
+        route.wakeable = route.connected && route.mac.is_some() && !observation_offline;
     }
 }
 
@@ -744,12 +752,17 @@ fn inventory_result_to_observations(
 
 fn observation_presence_rank(observation: &AgentDeviceObservation) -> u8 {
     match observation.last_action.as_str() {
-        "remove" => 0,
+        "remove" | "failed" => 0,
-        "add" | "old" | "update" => 2,
+        "permanent" | "reachable" => 3,
+        "stale" | "add" | "old" | "update" => 2,
         _ => 1,
     }
 }
 
+fn observation_is_offline(observation: &AgentDeviceObservation) -> bool {
+    matches!(observation.last_action.as_str(), "remove" | "failed")
+}
+
 fn rank_presence(rank: u8) -> &'static str {
     match rank {
         3 => "online",
@@ -932,6 +945,26 @@ mod tests {
         assert!(!devices[0].route_candidates[0].wakeable);
     }
 
+    #[test]
+    fn offline_observation_does_not_advertise_current_ip_or_wake_route() {
+        let mut offline = observation(
+            "agent-a",
+            Some("aa:bb:cc:dd:ee:ff"),
+            Some("192.168.1.2"),
+            20,
+        );
+        offline.kind = "neigh".into();
+        offline.last_action = "remove".into();
+
+        let devices = build_fleet_devices(Vec::new(), vec![offline], &context(&["agent-a"]));
+
+        assert_eq!(devices.len(), 1);
+        assert!(devices[0].ips.is_empty());
+        assert_eq!(devices[0].presence, "offline");
+        assert!(devices[0].recommended_route.is_none());
+        assert!(!devices[0].route_candidates[0].wakeable);
+    }
+
     #[test]
     fn inventory_result_maps_to_stored_observations() {
         let observations = inventory_result_to_observations(serde_json::json!({

wakey-control-plane/src/state/store/helpers/core.rs

@@ -259,10 +259,7 @@ pub(in crate::state::store) fn normalize_agent_observation(
     let mac = normalize_optional_text(input.mac.as_deref()).map(|value| value.to_ascii_lowercase());
     let ip = normalize_optional_text(input.ip.as_deref());
     let hostname = normalize_optional_text(input.hostname.as_deref());
-    let identifier = mac
+    let identifier = observation_identifier(&kind, mac.as_deref(), ip.as_deref())
-        .as_ref()
-        .map(|value| format!("mac:{value}"))
-        .or_else(|| ip.as_ref().map(|value| format!("ip:{value}")))
         .ok_or_else(|| anyhow::anyhow!("observation requires mac or ip"))?;
     let observation_key = format!("agent:{agent_id}:{kind}:{identifier}");
     Ok(AgentDeviceObservation {
@@ -278,6 +275,15 @@ pub(in crate::state::store) fn normalize_agent_observation(
     })
 }
 
+fn observation_identifier(kind: &str, mac: Option<&str>, ip: Option<&str>) -> Option<String> {
+    match (kind, mac, ip) {
+        ("neigh", Some(mac), Some(ip)) => Some(format!("mac:{mac}:ip:{ip}")),
+        (_, Some(mac), _) => Some(format!("mac:{mac}")),
+        (_, None, Some(ip)) => Some(format!("ip:{ip}")),
+        (_, None, None) => None,
+    }
+}
+
 pub(in crate::state::store) async fn insert_device_identifier_tx(
     tx: &mut Transaction<'_, Sqlite>,
     device_id: &str,

wakey-control-plane/src/state/store/observations.rs

@@ -211,6 +211,8 @@ impl Store {
             r#"SELECT events.event_id as "event_id!",
                     ('agent:' || events.agent_id || ':' || events.kind || ':' ||
                         CASE
+                            WHEN events.kind = 'neigh' AND events.mac IS NOT NULL AND events.ip IS NOT NULL
+                                THEN 'mac:' || events.mac || ':ip:' || events.ip
                             WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac
                             WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip
                             ELSE ''
@@ -239,6 +241,8 @@ impl Store {
                AND (?4 IS NULL OR events.ip = ?4)
                AND (?5 IS NULL OR ('agent:' || events.agent_id || ':' || events.kind || ':' ||
                     CASE
+                        WHEN events.kind = 'neigh' AND events.mac IS NOT NULL AND events.ip IS NOT NULL
+                            THEN 'mac:' || events.mac || ':ip:' || events.ip
                         WHEN events.mac IS NOT NULL THEN 'mac:' || events.mac
                         WHEN events.ip IS NOT NULL THEN 'ip:' || events.ip
                         ELSE ''

wakey-core/src/model/device.rs

@@ -174,7 +174,9 @@ impl Device {
 
 fn observation_presence(observation: &DeviceObservationFact) -> Presence {
     match (observation.kind.as_str(), observation.action.as_str()) {
-        (_, "remove") => Presence::Offline,
+        (_, "remove" | "failed") => Presence::Offline,
+        (_, "permanent" | "reachable") => Presence::Online,
+        (_, "stale") => Presence::LikelyOnline,
         ("neigh", "add" | "update" | "old") => Presence::LikelyOnline,
         _ => Presence::Unknown,
     }

wakey-linux/src/dhcp.rs

@@ -136,4 +136,47 @@ mod tests {
         let _ = tokio::fs::remove_file(observation_path).await;
         let _ = tokio::fs::remove_file(cache_path).await;
     }
+
+    #[tokio::test]
+    #[serial]
+    async fn neighbor_observations_are_keyed_by_mac_ip_pair() {
+        let observation_path = temp_file("neighbor-observations");
+        let _observation_guard = EnvGuard::set(OBSERVATION_STORE_ENV, &observation_path);
+        let mac = "aa:bb:cc:dd:ee:ff".parse().expect("mac should parse");
+
+        observe_neighbor_event(
+            "stale",
+            Some(mac),
+            Some("192.168.1.2".parse().expect("ip should parse")),
+        )
+        .await
+        .expect("first observation should write");
+        observe_neighbor_event(
+            "reachable",
+            Some(mac),
+            Some("192.168.1.3".parse().expect("ip should parse")),
+        )
+        .await
+        .expect("second observation should write");
+
+        let store = load_observation_store()
+            .await
+            .expect("observation store should read");
+        assert!(
+            store
+                .neighbors
+                .contains_key("mac:aa:bb:cc:dd:ee:ff:ip:192.168.1.2")
+        );
+        assert!(
+            store
+                .neighbors
+                .contains_key("mac:aa:bb:cc:dd:ee:ff:ip:192.168.1.3")
+        );
+        assert_eq!(
+            store.neighbors["mac:aa:bb:cc:dd:ee:ff:ip:192.168.1.2"].last_action,
+            "remove"
+        );
+
+        let _ = tokio::fs::remove_file(observation_path).await;
+    }
 }

wakey-linux/src/dhcp/observations.rs

@@ -1,8 +1,10 @@
 use std::io::{self, ErrorKind};
 use std::net::IpAddr;
+use std::str::FromStr;
 
 use macaddr::MacAddr;
 use serde::{Deserialize, Serialize};
+use wakey_core::NeighborState;
 
 use super::{mac_name_cache_path, now_unix, observation_store_path};
 
@@ -218,14 +220,11 @@ pub async fn observe_neighbor_event(
     mac: Option<MacAddr>,
     ip: Option<IpAddr>,
 ) -> io::Result<bool> {
-    if !matches!(action, "add" | "update" | "old" | "remove") {
+    let action = normalize_neighbor_action(action, mac, ip).await;
-        // update and remove not emitted by hotplug
+    if !is_neighbor_observation_action(&action) {
         return Ok(false);
     }
-    let Some(key) = mac
+    let Some(key) = neighbor_observation_key(mac, ip) else {
-        .map(|value| format!("mac:{}", value.to_string().to_ascii_lowercase()))
-        .or_else(|| ip.map(|value| format!("ip:{}", value)))
-    else {
         return Ok(false);
     };
 
@@ -244,7 +243,7 @@ pub async fn observe_neighbor_event(
             {
                 row.mac = mac.clone();
                 row.ip = ip;
-                row.last_action = action.to_string();
+                row.last_action = action.clone();
                 row.last_seen_unix = now;
                 changed = true;
             }
@@ -253,15 +252,105 @@ pub async fn observe_neighbor_event(
             changed = true;
             ObservedNeighbor {
                 key,
-                mac,
+                mac: mac.clone(),
                 ip,
                 first_seen_unix: now,
                 last_seen_unix: now,
-                last_action: action.to_string(),
+                last_action: action.clone(),
             }
         });
+    if let (Some(mac), Some(ip)) = (mac.as_deref(), ip)
+        && !is_offline_neighbor_action(&action)
+    {
+        mark_replaced_neighbor_ips_removed(&mut store, mac, ip, now, &mut changed);
+    }
     if changed {
         save_observation_store(&store).await?;
     }
     Ok(changed)
 }
+
+async fn normalize_neighbor_action(
+    action: &str,
+    mac: Option<MacAddr>,
+    ip: Option<IpAddr>,
+) -> String {
+    let action = action.trim().to_ascii_lowercase();
+    if action == "remove" || action == "del" {
+        return "remove".into();
+    }
+    if NeighborState::from_str(&action).is_ok() {
+        return action;
+    }
+    if matches!(action.as_str(), "add" | "update" | "old") {
+        if let Some(state) = current_neighbor_state(mac, ip).await {
+            return state.as_ip_neigh_arg().into();
+        }
+        return action;
+    }
+    action
+}
+
+async fn current_neighbor_state(mac: Option<MacAddr>, ip: Option<IpAddr>) -> Option<NeighborState> {
+    let ips = ip.into_iter().collect::<Vec<_>>();
+    let macs = mac.into_iter().collect::<Vec<_>>();
+    let rows = crate::devices::get_neighbors(&[] as &[&str], &ips, &[] as &[&str], &[], &macs)
+        .await
+        .ok()?;
+    rows.into_iter()
+        .filter(|row| ip.is_none_or(|expected| row.ip == expected))
+        .filter(|row| mac.is_none_or(|expected| row.mac == Some(expected)))
+        .map(|row| row.state)
+        .max()
+}
+
+fn is_neighbor_observation_action(action: &str) -> bool {
+    matches!(action, "add" | "update" | "old" | "remove" | "del")
+        || NeighborState::from_str(action).is_ok()
+}
+
+fn is_offline_neighbor_action(action: &str) -> bool {
+    matches!(action, "remove" | "failed")
+}
+
+fn neighbor_observation_key(mac: Option<MacAddr>, ip: Option<IpAddr>) -> Option<String> {
+    match (mac, ip) {
+        (Some(mac), Some(ip)) => Some(format!(
+            "mac:{}:ip:{ip}",
+            mac.to_string().to_ascii_lowercase()
+        )),
+        (Some(mac), None) => Some(format!("mac:{}", mac.to_string().to_ascii_lowercase())),
+        (None, Some(ip)) => Some(format!("ip:{ip}")),
+        (None, None) => None,
+    }
+}
+
+fn mark_replaced_neighbor_ips_removed(
+    store: &mut LocalObservationStore,
+    mac: &str,
+    current_ip: IpAddr,
+    now: u64,
+    changed: &mut bool,
+) {
+    for row in store.neighbors.values_mut() {
+        let Some(row_ip) = row.ip else {
+            continue;
+        };
+        if row.mac.as_deref() == Some(mac)
+            && row_ip != current_ip
+            && same_ip_family(row_ip, current_ip)
+            && !is_offline_neighbor_action(&row.last_action)
+        {
+            row.last_action = "remove".into();
+            row.last_seen_unix = now;
+            *changed = true;
+        }
+    }
+}
+
+fn same_ip_family(a: IpAddr, b: IpAddr) -> bool {
+    matches!(
+        (a, b),
+        (IpAddr::V4(_), IpAddr::V4(_)) | (IpAddr::V6(_), IpAddr::V6(_))
+    )
+}