sched: scope live abandon repair to the broken run; torn settle reconciles genuine result (B1)

2026-09-09 17:13:17 +07:00
parent b3302eb197
commit fd2f85ef96
3 changed files with 253 additions and 18 deletions
@@ -31,6 +31,7 @@ from wf_scheduling.lifecycle import (
 )
 from wf_scheduling.models import Schedule
 from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
+from wf_scheduling.poll import Scheduler
 from wf_scheduling.store import FileScheduleStore


@@ -556,3 +557,183 @@ async def test_store_deployment_directory_contract(tmp_path: Path) -> None:
        directory.deployment_revision("missing.deployment")
    with pytest.raises(KeyError):
        directory.required_inputs("missing.deployment")
+
+
+async def _poll_two(
+    service: SchedulerService, intended: datetime
+) -> tuple[str, str]:
+    service.schedule_store.create_schedule(_sched_model("a", intended))
+    service.schedule_store.create_schedule(_sched_model("b", intended))
+    result = await service.poll_once(intended + timedelta(seconds=1))
+    hang_id = result["a"].removeprefix("admit:")
+    sib_id = result["b"].removeprefix("admit:")
+    assert hang_id.startswith("run-") and sib_id.startswith("run-")
+    return hang_id, sib_id
+
+
+def _assert_sibling_live(service: SchedulerService, hang_id: str) -> None:
+    """The healthy sibling keeps its markers, status, and clean history."""
+    sibling = service.run_store.get_run(hang_id)
+    assert sibling.status.value == "admitted"
+    assert service.run_store.is_executing(hang_id)
+    assert not service.run_store.is_pending_dispatch(hang_id)
+    assert len(_entries(service.schedule_store, "a", "failed")) == 0
+
+
+async def test_live_runtime_failure_spares_healthy_sibling(
+    tmp_path: Path,
+) -> None:
+    """A live execution failure repairs only the broken run (B1).
+
+    Whole-store recovery during live execution used to fail the still
+    hanging sibling too, whose late success was then rejected as
+    ``settle non-admitted run``.
+    """
+    intended = ts(2026, 9, 8, 12, 0)
+    runtime = ScriptedRuntime(lambda n: "hang" if n == 1 else "raise")
+    service = _service(
+        tmp_path,
+        runtime,
+        SchedulerServiceConfig(poll_interval_s=0.01, capacity=2, auto_tick=False),
+    )
+    try:
+        await service.start()
+        hang_id, sib_id = await _poll_two(service, intended)
+        await _wait_for(lambda: runtime.started.is_set())
+        await _wait_for(
+            lambda: service.run_store.get_run(sib_id).status.value == "failed"
+        )
+        failed = service.run_store.get_run(sib_id)
+        assert failed.status.value == "failed"
+        assert not service.run_store.is_executing(sib_id)
+        assert not service.run_store.is_pending_dispatch(sib_id)
+        assert len(_entries(service.schedule_store, "b", "failed")) == 1
+        _assert_sibling_live(service, hang_id)
+        runtime.release.set()
+        await _wait_for(lambda: service.live_executions == 0)
+        assert service.run_store.get_run(hang_id).status.value == "completed"
+        assert len(_entries(service.schedule_store, "a", "completed")) == 1
+        assert not any("settle non-admitted run" in m for m in service.errors)
+    finally:
+        runtime.release.set()
+        report = await service.stop()
+    assert report.settled == 1
+    assert report.abandoned == 1
+
+
+async def test_live_settlement_failure_spares_healthy_sibling(
+    tmp_path: Path, monkeypatch: Any
+) -> None:
+    """A torn settle write repairs only the broken run (B1).
+
+    The settlement seam raises after genuine execution for one run while
+    its sibling hangs: only that run fails closed, and the sibling still
+    settles exactly once with its capacity slot freed.
+    """
+    intended = ts(2026, 9, 8, 12, 0)
+    runtime = ScriptedRuntime(lambda n: "hang" if n == 1 else "complete")
+    service = _service(
+        tmp_path,
+        runtime,
+        SchedulerServiceConfig(poll_interval_s=0.01, capacity=2, auto_tick=False),
+    )
+    try:
+        await service.start()
+        original = Scheduler.record_stopped_execution
+        fired = {"done": False}
+
+        def _flaky_settle(self: Any, run_id: str, state: Any, now: datetime) -> None:
+            try:
+                owned_by_b = (
+                    self.run_store.get_admission(run_id).schedule_id == "b"
+                )
+            except KeyError:
+                owned_by_b = False
+            if owned_by_b and not fired["done"]:
+                fired["done"] = True
+                raise RuntimeError("injected settlement failure")
+            original(self, run_id, state, now)
+
+        monkeypatch.setattr(Scheduler, "record_stopped_execution", _flaky_settle)
+        hang_id, sib_id = await _poll_two(service, intended)
+        await _wait_for(lambda: runtime.started.is_set())
+        await _wait_for(
+            lambda: service.run_store.get_run(sib_id).status.value == "failed"
+        )
+        assert fired["done"]
+        assert not service.run_store.is_executing(sib_id)
+        assert len(_entries(service.schedule_store, "b", "failed")) == 1
+        _assert_sibling_live(service, hang_id)
+        runtime.release.set()
+        await _wait_for(lambda: service.live_executions == 0)
+        assert service.run_store.get_run(hang_id).status.value == "completed"
+        assert len(_entries(service.schedule_store, "a", "completed")) == 1
+        assert not any("settle non-admitted run" in m for m in service.errors)
+    finally:
+        runtime.release.set()
+        report = await service.stop()
+    assert report.settled == 1
+    assert report.abandoned == 1
+
+
+async def test_live_torn_settlement_reconciles_genuine_result(
+    tmp_path: Path, monkeypatch: Any
+) -> None:
+    """A torn history write keeps the genuine stopped result (B1).
+
+    The stopped result is durably persisted but the history append is
+    lost: scoped repair must reconcile the genuine completion (with
+    checkpoint authority), never fail the run, and never touch the
+    hanging sibling.
+    """
+    intended = ts(2026, 9, 8, 12, 0)
+    runtime = ScriptedRuntime(lambda n: "hang" if n == 1 else "complete")
+    service = _service(
+        tmp_path,
+        runtime,
+        SchedulerServiceConfig(poll_interval_s=0.01, capacity=2, auto_tick=False),
+    )
+    try:
+        await service.start()
+        original = Scheduler._record
+        fired = {"done": False}
+
+        def _flaky_record(self: Any, **kwargs: Any) -> None:
+            run_id = kwargs.get("run_id")
+            try:
+                owned_by_b = (
+                    run_id is not None
+                    and self.run_store.get_admission(run_id).schedule_id == "b"
+                )
+            except KeyError:
+                owned_by_b = False
+            if (
+                kwargs.get("kind") == "completed"
+                and owned_by_b
+                and not fired["done"]
+            ):
+                fired["done"] = True
+                raise RuntimeError("injected history failure")
+            original(self, **kwargs)
+
+        monkeypatch.setattr(Scheduler, "_record", _flaky_record)
+        hang_id, sib_id = await _poll_two(service, intended)
+        await _wait_for(lambda: runtime.started.is_set())
+        await _wait_for(
+            lambda: len(_entries(service.schedule_store, "b", "completed")) == 1
+        )
+        assert fired["done"]
+        repaired = service.run_store.get_run(sib_id)
+        assert repaired.status.value == "completed"
+        assert not service.run_store.is_executing(sib_id)
+        assert len(_entries(service.schedule_store, "b", "failed")) == 0
+        _assert_sibling_live(service, hang_id)
+        runtime.release.set()
+        await _wait_for(lambda: service.live_executions == 0)
+        assert service.run_store.get_run(hang_id).status.value == "completed"
+        assert len(_entries(service.schedule_store, "a", "completed")) == 1
+    finally:
+        runtime.release.set()
+        report = await service.stop()
+    assert report.settled == 2
+    assert report.abandoned == 0