sched: crash-safe admin ordering, create watermark, poll freshness (T13 fix)

tests/wf_api/test_schedules.py

@@ -260,6 +260,32 @@ def _history_rows(
     return rows
 
 
+class FailConsumedOnce(FileScheduleStore):
+    """Real schedule store failing exactly one watermark write."""
+
+    def __init__(self, root: Path) -> None:
+        super().__init__(root)
+        self.armed = False
+
+    def save_consumed(self, schedule_id: str, consumed_through: datetime) -> None:
+        if self.armed:
+            self.armed = False
+            raise OSError("injected consumed failure")
+        super().save_consumed(schedule_id, consumed_through)
+
+
+def _fault_harness(
+    root: Path,
+) -> tuple[WorkflowScheduleApi, FailConsumedOnce, FileRunStore]:
+    artifact_store = FileWorkflowArtifactStore(root)
+    artifact_store.save_artifact(_artifact())
+    artifact_store.save_deployment(_deployment())
+    run_store = FileRunStore(root)
+    sched_store = FailConsumedOnce(root)
+    context = _context(artifact_store, run_store, sched_store)
+    return WorkflowScheduleApi(context), sched_store, run_store
+
+
 async def test_create_get_list_round_trip_with_defaults(tmp_path: Path) -> None:
     api, sched_store, _, _, _ = _harness(tmp_path / "round_trip")
 
@@ -463,13 +489,81 @@ async def test_update_happy_path_only_future_work(tmp_path: Path) -> None:
 
 
 async def test_update_stale_revision_rejected(tmp_path: Path) -> None:
-    api, _, _, _, _ = _harness(tmp_path / "stale")
+    api, sched_store, _, _, _ = _harness(tmp_path / "stale")
+
+    created = await api.create_schedule(
+        schedule_id="s", deployment_id="dep.personal", trigger=_cron()
+    )
+    intended = ts(2026, 9, 8, 12, 0)
+    sched_store.save_candidate(
+        PendingCandidate(schedule_id="s", intended_at=intended, revision=1),
+        schedule_id="s",
+    )
+    consumed_before = sched_store.get_consumed("s")
+    with pytest.raises(StaleScheduleRevisionError):
+        await api.update_schedule(schedule_id="s", expected_revision=99)
+    # Stale edits write nothing: no revision bump, no candidate or
+    # watermark change, no history row.
+    assert sched_store.get_schedule("s").revision == 1
+    assert sched_store.get_schedule("s").updated_at == datetime.fromisoformat(
+        created["updated_at"]
+    )
+    candidate = sched_store.get_candidate("s")
+    assert candidate is not None and candidate.intended_at == intended
+    assert sched_store.get_consumed("s") == consumed_before
+    assert _history_rows(sched_store, "s") == []
+
+
+async def test_create_initializes_consumed_no_backfill(tmp_path: Path) -> None:
+    api, sched_store, _, _, _ = _harness(tmp_path / "create_consumed")
+
+    created = await api.create_schedule(
+        schedule_id="s",
+        deployment_id="dep.personal",
+        trigger=_cron(),
+        misfire="latest",
+    )
+    created_at = datetime.fromisoformat(created["created_at"])
+    consumed = sched_store.get_consumed("s")
+    assert consumed is not None and consumed >= created_at
+
+
+async def test_update_watermark_failure_leaves_revision(tmp_path: Path) -> None:
+    api, sched_store, _ = _fault_harness(tmp_path / "torn_update")
 
     await api.create_schedule(
         schedule_id="s", deployment_id="dep.personal", trigger=_cron()
     )
-    with pytest.raises(StaleScheduleRevisionError):
-        await api.update_schedule(schedule_id="s", expected_revision=99)
+    intended = ts(2026, 9, 8, 12, 0)
+    sched_store.save_candidate(
+        PendingCandidate(schedule_id="s", intended_at=intended, revision=1),
+        schedule_id="s",
+    )
+    sched_store.armed = True
+    with pytest.raises(OSError, match="injected consumed failure"):
+        await api.update_schedule(
+            schedule_id="s",
+            expected_revision=1,
+            overlap="parallel",
+            max_active_runs=2,
+        )
+    # Crash-safe ordering: the watermark advance precedes the revision
+    # bump, so a torn edit leaves the old revision (retryable) and can
+    # never backfill pre-edit instants under the new revision.
+    assert sched_store.get_schedule("s").revision == 1
+    assert sched_store.get_schedule("s").overlap == "skip"
+
+
+async def test_pause_watermark_failure_leaves_flag(tmp_path: Path) -> None:
+    api, sched_store, _ = _fault_harness(tmp_path / "torn_pause")
+
+    await api.create_schedule(
+        schedule_id="s", deployment_id="dep.personal", trigger=_cron()
+    )
+    sched_store.armed = True
+    with pytest.raises(OSError, match="injected consumed failure"):
+        await api.pause_schedule(schedule_id="s")
+    assert sched_store.get_schedule("s").paused is False
 
 
 async def test_unknown_schedule_keyerror(tmp_path: Path) -> None:
@@ -491,10 +585,14 @@ async def test_unknown_schedule_keyerror(tmp_path: Path) -> None:
 
 async def test_pause_resume_delete_transitions(tmp_path: Path) -> None:
     api, sched_store, _, _, _ = _harness(tmp_path / "transitions")
-    await api.create_schedule(
+    created = await api.create_schedule(
         schedule_id="s", deployment_id="dep.personal", trigger=_cron()
     )
-    assert sched_store.get_consumed("s") is None
+    # Creation never backfills time before the revision: the consumed
+    # watermark starts at creation.
+    created_consumed = sched_store.get_consumed("s")
+    assert created_consumed is not None
+    assert created_consumed >= datetime.fromisoformat(created["created_at"])
 
     intended = ts(2026, 9, 8, 12, 0)
     sched_store.save_candidate(