sched: crash-safe admin ordering, create watermark, poll freshness (T13 fix)

tests/scheduling/test_poll.py

@@ -303,3 +303,53 @@ def test_capacity_wait_then_expire_for_skip(tmp_path: Path) -> None:
     sched.poll(t0 + timedelta(seconds=30))
     assert len([r for r in _history(store, "a") if r["kind"] == "admitted"]) == 1
     sched.ownership.release()
+
+
+def test_poll_one_rereads_pause_before_deciding(tmp_path: Path) -> None:
+    sched, store, runs, sources = _harness(tmp_path, script={"*": "hang"})
+    t0 = ts(2026, 9, 8, 12, 0)
+    _add(sched, store, sources, "a", OneShotSource(t0), t0 - timedelta(hours=1))
+    stale = store.get_schedule("a")
+    # Same-process administration lands after the tick listed schedules:
+    # the decision must observe the pause, not the stale snapshot.
+    live = store.get_schedule("a")
+    live.paused = True
+    store.save_schedule(live)
+    assert sched._poll_one(stale, t0 + timedelta(seconds=1)) == "paused"
+    assert runs.list_runs() == []
+    assert runs.list_admissions() == []
+    sched.ownership.release()
+
+
+def test_poll_one_uses_fresh_definition_after_edit(tmp_path: Path) -> None:
+    sched, store, runs, sources = _harness(tmp_path, script={"*": "hang"})
+    t0 = ts(2026, 9, 8, 12, 0)
+    _add(
+        sched,
+        store,
+        sources,
+        "a",
+        OneShotSource(t0),
+        t0 - timedelta(hours=1),
+        input_bindings=[
+            {"target": "team", "expression": {"kind": "literal", "value": "old"}}
+        ],
+    )
+    stale = store.get_schedule("a")
+    live = store.get_schedule("a")
+    live.revision = 2
+    from wf_core.models.input_bindings import ScheduleInputBinding
+
+    live.input_bindings = [
+        ScheduleInputBinding.model_validate(
+            {"target": "team", "expression": {"kind": "literal", "value": "new"}}
+        )
+    ]
+    store.save_schedule(live)
+    result = sched._poll_one(stale, t0 + timedelta(seconds=1))
+    assert result.startswith("admit:run-")
+    run_id = result.split(":", 1)[1]
+    admission = runs.get_admission(run_id)
+    assert admission.resolved_input["team"] == "new"
+    assert admission.schedule_revision == 2
+    sched.ownership.release()

tests/wf_api/test_schedules.py

@@ -260,6 +260,32 @@ def _history_rows(
     return rows
 
 
+class FailConsumedOnce(FileScheduleStore):
+    """Real schedule store failing exactly one watermark write."""
+
+    def __init__(self, root: Path) -> None:
+        super().__init__(root)
+        self.armed = False
+
+    def save_consumed(self, schedule_id: str, consumed_through: datetime) -> None:
+        if self.armed:
+            self.armed = False
+            raise OSError("injected consumed failure")
+        super().save_consumed(schedule_id, consumed_through)
+
+
+def _fault_harness(
+    root: Path,
+) -> tuple[WorkflowScheduleApi, FailConsumedOnce, FileRunStore]:
+    artifact_store = FileWorkflowArtifactStore(root)
+    artifact_store.save_artifact(_artifact())
+    artifact_store.save_deployment(_deployment())
+    run_store = FileRunStore(root)
+    sched_store = FailConsumedOnce(root)
+    context = _context(artifact_store, run_store, sched_store)
+    return WorkflowScheduleApi(context), sched_store, run_store
+
+
 async def test_create_get_list_round_trip_with_defaults(tmp_path: Path) -> None:
     api, sched_store, _, _, _ = _harness(tmp_path / "round_trip")
 
@@ -463,13 +489,81 @@ async def test_update_happy_path_only_future_work(tmp_path: Path) -> None:
 
 
 async def test_update_stale_revision_rejected(tmp_path: Path) -> None:
-    api, _, _, _, _ = _harness(tmp_path / "stale")
+    api, sched_store, _, _, _ = _harness(tmp_path / "stale")
+
+    created = await api.create_schedule(
+        schedule_id="s", deployment_id="dep.personal", trigger=_cron()
+    )
+    intended = ts(2026, 9, 8, 12, 0)
+    sched_store.save_candidate(
+        PendingCandidate(schedule_id="s", intended_at=intended, revision=1),
+        schedule_id="s",
+    )
+    consumed_before = sched_store.get_consumed("s")
+    with pytest.raises(StaleScheduleRevisionError):
+        await api.update_schedule(schedule_id="s", expected_revision=99)
+    # Stale edits write nothing: no revision bump, no candidate or
+    # watermark change, no history row.
+    assert sched_store.get_schedule("s").revision == 1
+    assert sched_store.get_schedule("s").updated_at == datetime.fromisoformat(
+        created["updated_at"]
+    )
+    candidate = sched_store.get_candidate("s")
+    assert candidate is not None and candidate.intended_at == intended
+    assert sched_store.get_consumed("s") == consumed_before
+    assert _history_rows(sched_store, "s") == []
+
+
+async def test_create_initializes_consumed_no_backfill(tmp_path: Path) -> None:
+    api, sched_store, _, _, _ = _harness(tmp_path / "create_consumed")
+
+    created = await api.create_schedule(
+        schedule_id="s",
+        deployment_id="dep.personal",
+        trigger=_cron(),
+        misfire="latest",
+    )
+    created_at = datetime.fromisoformat(created["created_at"])
+    consumed = sched_store.get_consumed("s")
+    assert consumed is not None and consumed >= created_at
+
+
+async def test_update_watermark_failure_leaves_revision(tmp_path: Path) -> None:
+    api, sched_store, _ = _fault_harness(tmp_path / "torn_update")
 
     await api.create_schedule(
         schedule_id="s", deployment_id="dep.personal", trigger=_cron()
     )
-    with pytest.raises(StaleScheduleRevisionError):
-        await api.update_schedule(schedule_id="s", expected_revision=99)
+    intended = ts(2026, 9, 8, 12, 0)
+    sched_store.save_candidate(
+        PendingCandidate(schedule_id="s", intended_at=intended, revision=1),
+        schedule_id="s",
+    )
+    sched_store.armed = True
+    with pytest.raises(OSError, match="injected consumed failure"):
+        await api.update_schedule(
+            schedule_id="s",
+            expected_revision=1,
+            overlap="parallel",
+            max_active_runs=2,
+        )
+    # Crash-safe ordering: the watermark advance precedes the revision
+    # bump, so a torn edit leaves the old revision (retryable) and can
+    # never backfill pre-edit instants under the new revision.
+    assert sched_store.get_schedule("s").revision == 1
+    assert sched_store.get_schedule("s").overlap == "skip"
+
+
+async def test_pause_watermark_failure_leaves_flag(tmp_path: Path) -> None:
+    api, sched_store, _ = _fault_harness(tmp_path / "torn_pause")
+
+    await api.create_schedule(
+        schedule_id="s", deployment_id="dep.personal", trigger=_cron()
+    )
+    sched_store.armed = True
+    with pytest.raises(OSError, match="injected consumed failure"):
+        await api.pause_schedule(schedule_id="s")
+    assert sched_store.get_schedule("s").paused is False
 
 
 async def test_unknown_schedule_keyerror(tmp_path: Path) -> None:
@@ -491,10 +585,14 @@ async def test_unknown_schedule_keyerror(tmp_path: Path) -> None:
 
 async def test_pause_resume_delete_transitions(tmp_path: Path) -> None:
     api, sched_store, _, _, _ = _harness(tmp_path / "transitions")
-    await api.create_schedule(
+    created = await api.create_schedule(
         schedule_id="s", deployment_id="dep.personal", trigger=_cron()
     )
-    assert sched_store.get_consumed("s") is None
+    # Creation never backfills time before the revision: the consumed
+    # watermark starts at creation.
+    created_consumed = sched_store.get_consumed("s")
+    assert created_consumed is not None
+    assert created_consumed >= datetime.fromisoformat(created["created_at"])
 
     intended = ts(2026, 9, 8, 12, 0)
     sched_store.save_candidate(

tests/wf_client/test_schedules.py

@@ -128,6 +128,8 @@ async def test_schedule_snapshot_exposes_minimal_surface() -> None:
     assert schedule.enabled is True
     assert schedule.paused is False
     assert schedule.deleted is False
+    assert schedule.exhausted is False
+    assert schedule.blocked_reason is None
     assert schedule.overlap == "skip"
     assert schedule.misfire == "skip"
     assert schedule.max_active_runs == 1