sched: record resumed stopped results live and idempotently (B3)

tests/wf_server/test_scheduler_integration.py

@@ -1260,3 +1260,174 @@ async def test_service_stop_drains_inflight_scheduled_resume(tmp_path: Path) ->
     finally:
         _gate_open.set()
         await service.stop()
+
+
+async def _kinds(root: Path, schedule_id: str) -> list[str]:
+    page = FileScheduleStore(root).list_occurrences(schedule_id, limit=100)
+    rows = cast(list[dict[str, Any]], page["occurrences"])
+    return [row["kind"] for row in rows]
+
+
+async def test_scheduled_resume_records_completion_live(tmp_path: Path) -> None:
+    """A resumed completion lands in occurrence history live (B3).
+
+    No restart recovery is involved: after the API resume plus ordinary
+    polls, history reads admitted/interrupted/completed, and repeated
+    polling plus a restart never duplicate the entries.
+    """
+    root = tmp_path / "store"
+    server = build_local_static_workflow_server(root)
+    await _seed(server, "ask1", "ask1.default", _single_interrupt_plan("ask1"), ["submitted"])
+    store = FileScheduleStore(root)
+    store.create_schedule(_one_shot("asker", "ask1.default", datetime.now(UTC)))
+    service = _scheduler(server, capacity=2)
+    try:
+        await service.start()
+        await _wait_for(lambda: len(_run_ids(root)) == 1)
+        ask_id = _run_ids(root)[0]
+        await _wait_for(
+            lambda: FileRunStore(root).get_run(ask_id).status.value == "interrupted"
+        )
+        resumed = await server.api.resume_run(
+            run_id=ask_id, resume_payload={}, resume_outcome="submitted"
+        )
+        assert resumed["status"] == "completed"
+        assert await _kinds(root, "asker") == ["admitted", "interrupted", "completed"]
+        completed = _entries(root, "asker", "completed")
+        assert len(completed) == 1
+        assert completed[0]["run_id"] == ask_id
+        assert (
+            completed[0]["checkpoint_id"]
+            == FileRunStore(root).get_run(ask_id).latest_checkpoint_id
+        )
+        assert completed[0]["revision"] == 1
+        await service.poll_once(datetime.now(UTC))
+        await service.poll_once(datetime.now(UTC))
+        assert await _kinds(root, "asker") == ["admitted", "interrupted", "completed"]
+    finally:
+        await service.stop()
+
+    server_b = build_local_static_workflow_server(root)
+    revived = _scheduler(server_b)
+    try:
+        await revived.start()
+        await revived.poll_once(datetime.now(UTC))
+        assert await _kinds(root, "asker") == ["admitted", "interrupted", "completed"]
+    finally:
+        await revived.stop()
+
+
+async def test_scheduled_resume_records_failure_and_reinterruption_live(
+    tmp_path: Path,
+) -> None:
+    """Resumed failures and re-interruptions each record live (B3)."""
+    root = tmp_path / "store"
+    server = build_local_static_workflow_server(root)
+    await _seed(
+        server,
+        "ask_twice",
+        "ask_twice.default",
+        _double_interrupt_plan("ask_twice"),
+        ["submitted"],
+    )
+    await _seed(
+        server,
+        "ask_fail",
+        "ask_fail.default",
+        _interrupt_then_fail_plan("ask_fail"),
+        ["submitted"],
+    )
+    store = FileScheduleStore(root)
+    store.create_schedule(_one_shot("twicer", "ask_twice.default", datetime.now(UTC)))
+    store.create_schedule(_one_shot("failer", "ask_fail.default", datetime.now(UTC)))
+    service = _scheduler(server, capacity=2)
+    try:
+        await service.start()
+        await _wait_for(lambda: len(_run_ids(root)) == 2)
+        twice_id = fail_id = ""
+        for rid in _run_ids(root):
+            admission = FileRunStore(root).get_admission(rid)
+            if admission.schedule_id == "twicer":
+                twice_id = rid
+            else:
+                fail_id = rid
+        assert twice_id and fail_id
+        await _wait_for(
+            lambda: FileRunStore(root).get_run(twice_id).status.value == "interrupted"
+        )
+        await _wait_for(
+            lambda: FileRunStore(root).get_run(fail_id).status.value == "interrupted"
+        )
+        first = await server.api.resume_run(
+            run_id=twice_id, resume_payload={}, resume_outcome="submitted"
+        )
+        assert first["status"] == "interrupted"
+        second = await server.api.resume_run(
+            run_id=twice_id, resume_payload={}, resume_outcome="submitted"
+        )
+        assert second["status"] == "completed"
+        failed = await server.api.resume_run(
+            run_id=fail_id, resume_payload={}, resume_outcome="submitted"
+        )
+        assert failed["status"] == "failed"
+
+        assert await _kinds(root, "twicer") == [
+            "admitted",
+            "interrupted",
+            "interrupted",
+            "completed",
+        ]
+        twicer_page = FileScheduleStore(root).list_occurrences("twicer", limit=100)
+        twicer_ckpts = [
+            row["checkpoint_id"]
+            for row in cast(list[dict[str, Any]], twicer_page["occurrences"])
+            if row["kind"] in ("interrupted", "completed")
+        ]
+        assert len(set(twicer_ckpts)) == 3
+        assert await _kinds(root, "failer") == ["admitted", "interrupted", "failed"]
+        fail_rows = _entries(root, "failer", "failed")
+        assert len(fail_rows) == 1
+        assert fail_rows[0]["run_id"] == fail_id
+    finally:
+        await service.stop()
+
+
+async def test_paused_and_deleted_schedules_keep_resume_history(
+    tmp_path: Path,
+) -> None:
+    """Pausing/deleting never suppresses retained resume history (B3)."""
+    root = tmp_path / "store"
+    server = build_local_static_workflow_server(root)
+    await _seed(server, "ask1", "ask1.default", _single_interrupt_plan("ask1"), ["submitted"])
+    store = FileScheduleStore(root)
+    store.create_schedule(_one_shot("pausable", "ask1.default", datetime.now(UTC)))
+    store.create_schedule(_one_shot("doomed", "ask1.default", datetime.now(UTC)))
+    service = _scheduler(server, capacity=2)
+    try:
+        await service.start()
+        await _wait_for(lambda: len(_run_ids(root)) == 2)
+        ids = {
+            FileRunStore(root).get_admission(rid).schedule_id or "": rid
+            for rid in _run_ids(root)
+        }
+        await _wait_for(
+            lambda: all(
+                FileRunStore(root).get_run(rid).status.value == "interrupted"
+                for rid in ids.values()
+            )
+        )
+        paused = store.get_schedule("pausable")
+        paused.paused = True
+        store.save_schedule(paused)
+        gone = store.get_schedule("doomed")
+        gone.deleted = True
+        store.save_schedule(gone)
+        for rid in ids.values():
+            resumed = await server.api.resume_run(
+                run_id=rid, resume_payload={}, resume_outcome="submitted"
+            )
+            assert resumed["status"] == "completed"
+        assert await _kinds(root, "pausable") == ["admitted", "interrupted", "completed"]
+        assert await _kinds(root, "doomed") == ["admitted", "interrupted", "completed"]
+    finally:
+        await service.stop()