sched: remediate R4 ownership, identity-match, dispatch-mark, and pending atomicity

src/wf_scheduling/recovery.py

@@ -97,18 +97,31 @@ def recover(
         elif status == StoredRunStatus.COMPLETED.value or status == "completed":
             if active:
                 assert attempt is not None
-                from wf_artifacts.runs.models import ResumeAttempt
+                try:
+                    latest = run_store.get_latest_checkpoint(run.id)
+                    completed_attempt = latest.attempt_id
+                except KeyError:
+                    completed_attempt = None
+                if (
+                    completed_attempt is not None
+                    and completed_attempt == attempt.attempt_id
+                ):
+                    from wf_artifacts.runs.models import ResumeAttempt
 
-                run_store.save_resume_attempt(
-                    ResumeAttempt(
-                        run_id=run.id,
-                        attempt_id=attempt.attempt_id,
-                        state="DONE",
-                        created_at=attempt.created_at,
-                        updated_at=now,
+                    run_store.save_resume_attempt(
+                        ResumeAttempt(
+                            run_id=run.id,
+                            attempt_id=attempt.attempt_id,
+                            state="DONE",
+                            created_at=attempt.created_at,
+                            updated_at=now,
+                        )
                     )
-                )
-                diags.append(f"{run.id}:attempt-reconciled")
+                    diags.append(f"{run.id}:attempt-reconciled")
+                else:
+                    _fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
+                    diags.append(f"{run.id}:failed-closed")
+                    continue
             if record_history is not None and not _has_terminal(
                 schedule_store, run.id, "completed"
             ):
@@ -154,23 +167,20 @@ def _has_terminal(schedule_store: Any, run_id: str, kind: str) -> bool:
 
 
 def _mark_pending(run_store: Any, run_id: str) -> None:
-    path = run_store._run_directory(run_id) / "pending_dispatch"
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text("pending", encoding="utf-8")
+    run_store.mark_pending_dispatch(run_id)
 
 
 def _is_pending(run_store: Any, run_id: str) -> bool:
     try:
-        return (run_store._run_directory(run_id) / "pending_dispatch").exists()
-    except ValueError:
+        return bool(run_store.is_pending_dispatch(run_id))
+    except AttributeError:
+        # Legacy stores without the pending protocol: treat as not pending.
         return False
 
 
 def clear_pending(run_store: Any, run_id: str) -> None:
     """Clear the pending-dispatch marker after the poll sweep dispatches."""
     try:
-        path = run_store._run_directory(run_id) / "pending_dispatch"
-    except ValueError:
+        run_store.clear_pending_dispatch(run_id)
+    except AttributeError:
         return
-    if path.exists():
-        path.unlink()