chore: tighten scheduling review coverage

This commit is contained in:
lda
2026-09-09 21:50:24 +07:00 Verified
parent afb291d55c
commit 2ecdbdaa7f
17 changed files with 358 additions and 366 deletions
+1 -1
View File
@@ -93,7 +93,7 @@ if (-not (Test-Path $include_markdown_filter)) {
} }
$agent_results = Join-Path $PSScriptRoot "agent-challenge-results.md" $agent_results = Join-Path $PSScriptRoot "agent-challenge-results.md"
if ((Test-RenderNeedsAgentResults $RemainingArgs) -and -not (Test-Path $agent_results)) { if ((Test-RenderNeedsAgentResults (@($InputFile) + $RemainingArgs)) -and -not (Test-Path $agent_results)) {
Write-Error "agent-challenge-results.md is missing. Run generate_agent_challenge_evaluation.py first." Write-Error "agent-challenge-results.md is missing. Run generate_agent_challenge_evaluation.py first."
exit 1 exit 1
} }
+8 -6
View File
@@ -1433,7 +1433,9 @@ would still need to preserve the lifecycle's transaction and ownership
contracts; changing the storage engine alone would not prove those contracts; changing the storage engine alone would not prove those
properties. properties.
Scheduled deployment execution is not implemented. Nor does exposing a Scheduled deployment execution is implemented within the documented first
slice. It remains bounded by the filesystem-backed, single-process store and
the explicitly enabled local/static server composition. Nor does exposing a
provider's callable operations imply that its interactive widgets or entire provider's callable operations imply that its interactive widgets or entire
user experience are reproduced through the workflow API. user experience are reproduced through the workflow API.
@@ -1501,11 +1503,11 @@ and data-binding model are understandable without implementation knowledge.
## Scheduling and the Surrounding Application ## Scheduling and the Surrounding Application
Scheduled deployment execution is a required product direction, not an Scheduled deployment execution is implemented for the current slice. It
implemented capability. It introduces trigger identity, overlap policy, introduces trigger identity, overlap policy, and recovery decisions in
and recovery decisions in addition to time-expression parsing. It should addition to time-expression parsing, and builds on the same run lifecycle
build on the same run lifecycle rather than create a separate execution rather than creating a separate execution model. Extending it to distributed
model. Suspending an already-running workflow until a time or event is a workers or suspending an already-running workflow until a time or event is a
related but distinct design question; a wait node is not specified here. related but distinct design question; a wait node is not specified here.
The surrounding application is intended to combine assistant-backed chat The surrounding application is intended to combine assistant-backed chat
+2 -1
View File
@@ -117,7 +117,8 @@ wf-rpc-server --store-root .wf_store --enable-scheduler
``` ```
or set `server.scheduler.enabled` (plus optional `poll_interval_s`, or set `server.scheduler.enabled` (plus optional `poll_interval_s`,
`max_concurrent_runs`, `drain_grace_s`) in the neutral config. See `max_concurrent_runs`, `drain_grace_s`) in a local/static neutral config.
MCP-backed servers reject this setting. See
[`deployment scheduling operations`](deployment_scheduling.md). [`deployment scheduling operations`](deployment_scheduling.md).
`admin registry` shows desired persisted source entries. It is separate from `admin registry` shows desired persisted source entries. It is separate from
-1
View File
@@ -442,7 +442,6 @@ class WorkflowScheduleApi:
schedule = store.get_schedule(schedule_id) schedule = store.get_schedule(schedule_id)
schedule.deleted = True schedule.deleted = True
store.save_schedule(schedule) store.save_schedule(schedule)
store.save_candidate(None, schedule_id=schedule_id)
return _PROJECT_SCHEDULE(schedule.model_dump(mode="json")) return _PROJECT_SCHEDULE(schedule.model_dump(mode="json"))
async def list_schedule_occurrences( async def list_schedule_occurrences(
+13 -27
View File
@@ -151,46 +151,32 @@ class FileRunStore(RunStore):
for path in sorted(self.runs_dir.glob("*/admission.json")) for path in sorted(self.runs_dir.glob("*/admission.json"))
] ]
def allocate_run_id(self) -> str: def _next_sequence(self, filename: str, label: str) -> int:
"""Allocate a store-backed run identity that survives restart.""" """Atomically allocate the next value from one durable sequence file."""
with self._lock: with self._lock:
seq_path = self.runs_dir / "_run_id_seq.json" seq_path = self.runs_dir / filename
seq = 0 seq = 0
if seq_path.exists(): if seq_path.exists():
try: try:
raw = json.loads(seq_path.read_text(encoding="utf-8")) raw = json.loads(seq_path.read_text(encoding="utf-8"))
seq_value = raw.get("seq", 0) if isinstance(raw, dict) else None seq_value = raw.get("seq", 0) if isinstance(raw, dict) else None
if not isinstance(seq_value, int) or seq_value < 0: if not isinstance(seq_value, int) or seq_value < 0:
raise ValueError( raise ValueError(f"corrupt {label} at {seq_path}: {raw!r}")
f"corrupt run-id sequence at {seq_path}: {raw!r}"
)
seq = seq_value seq = seq_value
except (ValueError, AttributeError, TypeError) as exc: except (ValueError, AttributeError, TypeError) as exc:
raise ValueError(f"corrupt run-id sequence at {seq_path}") from exc raise ValueError(f"corrupt {label} at {seq_path}") from exc
seq += 1
self._write_json(seq_path, {"seq": seq})
return f"run-{seq:06d}"
def allocate_resume_attempt_id(self) -> int:
"""Allocate a store-backed resume-attempt identity (no reuse)."""
with self._lock:
seq_path = self.runs_dir / "_resume_attempt_seq.json"
seq = 0
if seq_path.exists():
try:
raw = json.loads(seq_path.read_text(encoding="utf-8"))
seq_value = raw.get("seq", 0) if isinstance(raw, dict) else None
if not isinstance(seq_value, int) or seq_value < 0:
raise ValueError(
f"corrupt attempt sequence at {seq_path}: {raw!r}"
)
seq = seq_value
except (ValueError, AttributeError, TypeError) as exc:
raise ValueError(f"corrupt attempt sequence at {seq_path}") from exc
seq += 1 seq += 1
self._write_json(seq_path, {"seq": seq}) self._write_json(seq_path, {"seq": seq})
return seq return seq
def allocate_run_id(self) -> str:
"""Allocate a store-backed run identity that survives restart."""
return f"run-{self._next_sequence('_run_id_seq.json', 'run-id sequence'):06d}"
def allocate_resume_attempt_id(self) -> int:
"""Allocate a store-backed resume-attempt identity (no reuse)."""
return self._next_sequence("_resume_attempt_seq.json", "attempt sequence")
def save_resume_attempt(self, attempt: ResumeAttempt) -> None: def save_resume_attempt(self, attempt: ResumeAttempt) -> None:
with self._lock: with self._lock:
self._write_json( self._write_json(
+22
View File
@@ -94,6 +94,28 @@ def test_concurrent_deployment_saves_advance_revision_without_lost_updates(
assert store.get_deployment("concurrent.personal").revision == 9 assert store.get_deployment("concurrent.personal").revision == 9
def test_deployment_revision_increments_on_save(tmp_path) -> None:
artifacts = FileWorkflowArtifactStore(tmp_path)
artifacts.save_deployment(
WorkflowDeployment(
id="dep-1",
artifact_id="wf-1",
artifact_version=1,
bindings=[],
)
)
assert artifacts.get_deployment("dep-1").revision == 1
artifacts.save_deployment(
WorkflowDeployment(
id="dep-1",
artifact_id="wf-1",
artifact_version=2,
bindings=[],
)
)
assert artifacts.get_deployment("dep-1").revision == 2
def test_file_store_loads_legacy_artifact_and_rewrites_canonical_shape( def test_file_store_loads_legacy_artifact_and_rewrites_canonical_shape(
tmp_path, tmp_path,
) -> None: ) -> None:
+1 -1
View File
@@ -70,7 +70,7 @@ def test_thesis_retires_campaign_without_removing_historical_evidence() -> None:
archive = ROOT / "docs/historical/thesis/2026-09-07-retired-agent-evaluation.md" archive = ROOT / "docs/historical/thesis/2026-09-07-retired-agent-evaluation.md"
historical = archive.read_text(encoding="utf-8") historical = archive.read_text(encoding="utf-8")
assert "# Agent Challenge Harness" in historical assert "# Agent Challenge Harness" in historical
assert "36" in historical assert "36 audited trials" in historical
def test_thesis_bundle_has_reproducible_agent_evaluation_assets() -> None: def test_thesis_bundle_has_reproducible_agent_evaluation_assets() -> None:
+10 -3
View File
@@ -17,6 +17,7 @@ from pathlib import Path
from typing import Any, cast from typing import Any, cast
import pytest import pytest
from pydantic import ValidationError
from tests.artifacts.test_run_store import artifact as _artifact from tests.artifacts.test_run_store import artifact as _artifact
from tests.artifacts.test_run_store import deployment as _deployment from tests.artifacts.test_run_store import deployment as _deployment
@@ -588,7 +589,7 @@ async def test_tick_error_recorded_and_loop_continues(tmp_path: Path) -> None:
broken_path = tmp_path / "schedules" / "broken" / "schedule.json" broken_path = tmp_path / "schedules" / "broken" / "schedule.json"
good_text = broken_path.read_text(encoding="utf-8") good_text = broken_path.read_text(encoding="utf-8")
broken_path.write_text('{"id": "broken", "trigger": {"kind": "bogus"}}') broken_path.write_text('{"id": "broken", "trigger": {"kind": "bogus"}}')
with pytest.raises(Exception): with pytest.raises(ValidationError):
await service.poll_once(intended + timedelta(seconds=1)) await service.poll_once(intended + timedelta(seconds=1))
assert service.last_tick_error is not None assert service.last_tick_error is not None
assert service.run_store.list_runs() == [] assert service.run_store.list_runs() == []
@@ -689,7 +690,10 @@ async def test_live_runtime_failure_spares_healthy_sibling(
hang_id, sib_id = await _poll_two(service, intended) hang_id, sib_id = await _poll_two(service, intended)
await _wait_for(lambda: runtime.started.is_set()) await _wait_for(lambda: runtime.started.is_set())
await _wait_for( await _wait_for(
lambda: service.run_store.get_run(sib_id).status.value == "failed" lambda: (
service.run_store.get_run(sib_id).status.value == "failed"
and not service.run_store.is_executing(sib_id)
)
) )
failed = service.run_store.get_run(sib_id) failed = service.run_store.get_run(sib_id)
assert failed.status.value == "failed" assert failed.status.value == "failed"
@@ -744,7 +748,10 @@ async def test_live_settlement_failure_spares_healthy_sibling(
hang_id, sib_id = await _poll_two(service, intended) hang_id, sib_id = await _poll_two(service, intended)
await _wait_for(lambda: runtime.started.is_set()) await _wait_for(lambda: runtime.started.is_set())
await _wait_for( await _wait_for(
lambda: service.run_store.get_run(sib_id).status.value == "failed" lambda: (
service.run_store.get_run(sib_id).status.value == "failed"
and not service.run_store.is_executing(sib_id)
)
) )
assert fired["done"] assert fired["done"]
assert not service.run_store.is_executing(sib_id) assert not service.run_store.is_executing(sib_id)
+8 -4
View File
@@ -22,7 +22,6 @@ from tests.scheduling.controlled import (
) )
from wf_artifacts.runs.store import FileRunStore from wf_artifacts.runs.store import FileRunStore
from wf_scheduling.calendar import OneShotSource from wf_scheduling.calendar import OneShotSource
from wf_scheduling.dispatch import StillRunning
from wf_scheduling.models import Schedule from wf_scheduling.models import Schedule
from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
from wf_scheduling.poll import Scheduler from wf_scheduling.poll import Scheduler
@@ -80,13 +79,13 @@ def test_unrelated_held_lock_rejected_before_writes(tmp_path: Path) -> None:
def spy(admission: Any, now: datetime) -> Any: def spy(admission: Any, now: datetime) -> Any:
calls.append(admission.id) calls.append(admission.id)
return StillRunning() raise AssertionError("dispatcher must not run without covering ownership")
sched_store = FileScheduleStore(tmp_path / "sched") sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs") run_store = FileRunStore(tmp_path / "runs")
ownership = SchedulerOwnership(tmp_path / "other", owner="unrelated").acquire() ownership = SchedulerOwnership(tmp_path / "other", owner="unrelated").acquire()
try: try:
sched = _scheduler(sched_store, run_store, ownership) sched = _scheduler(sched_store, run_store, ownership, script={"*": spy})
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
_due(sched, sched_store, intended) _due(sched, sched_store, intended)
with pytest.raises(SecondOwnerError): with pytest.raises(SecondOwnerError):
@@ -103,9 +102,12 @@ def test_two_lock_dirs_cannot_operate_same_stores(tmp_path: Path) -> None:
sched_store = FileScheduleStore(tmp_path / "sched") sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs") run_store = FileRunStore(tmp_path / "runs")
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
covering: SchedulerOwnership | None = None
other: SchedulerOwnership | None = None
try:
covering = SchedulerOwnership(tmp_path, owner="covering").acquire() covering = SchedulerOwnership(tmp_path, owner="covering").acquire()
other = SchedulerOwnership(tmp_path / "other", owner="other").acquire() other = SchedulerOwnership(tmp_path / "other", owner="other").acquire()
try: assert covering is not None and other is not None
first = _scheduler(sched_store, run_store, covering, script={"*": "hang"}) first = _scheduler(sched_store, run_store, covering, script={"*": "hang"})
_due(first, sched_store, intended) _due(first, sched_store, intended)
assert first.poll(intended)["a"].startswith("admit:run-") assert first.poll(intended)["a"].startswith("admit:run-")
@@ -117,7 +119,9 @@ def test_two_lock_dirs_cannot_operate_same_stores(tmp_path: Path) -> None:
second.poll(intended + timedelta(minutes=1)) second.poll(intended + timedelta(minutes=1))
assert len(run_store.list_runs()) == 1 assert len(run_store.list_runs()) == 1
finally: finally:
if covering is not None:
covering.release() covering.release()
if other is not None:
other.release() other.release()
+32 -28
View File
@@ -114,12 +114,9 @@ def _history(store: FileScheduleStore, sid: str) -> list[dict]:
return page["occurrences"] # type: ignore[return-value] return page["occurrences"] # type: ignore[return-value]
def test_overlap_skip_blocks_and_late_drops() -> None: def test_overlap_skip_blocks_and_late_drops(tmp_path: Path) -> None:
import tempfile sched, store, runs, sources = _harness(tmp_path, capacity=4, script={"*": "hang"})
try:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched, store, runs, sources = _harness(root, capacity=4, script={"*": "hang"})
t0 = ts(2026, 9, 8, 12, 0) t0 = ts(2026, 9, 8, 12, 0)
_add( _add(
sched, sched,
@@ -137,15 +134,13 @@ def test_overlap_skip_blocks_and_late_drops() -> None:
sched.poll(t0 + timedelta(minutes=30)) sched.poll(t0 + timedelta(minutes=30))
kinds = [(r["kind"], r["resolved_at"]) for r in _history(store, "a")] kinds = [(r["kind"], r["resolved_at"]) for r in _history(store, "a")]
assert any(k == "skipped-misfire" for k, _ in kinds) assert any(k == "skipped-misfire" for k, _ in kinds)
finally:
sched.ownership.release() sched.ownership.release()
def test_latest_coalesces_to_one_candidate_and_no_double_admit() -> None: def test_latest_coalesces_to_one_candidate_and_no_double_admit(tmp_path: Path) -> None:
import tempfile sched, store, runs, sources = _harness(tmp_path, capacity=0)
try:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched, store, runs, sources = _harness(root, capacity=0)
_add( _add(
sched, sched,
store, store,
@@ -170,6 +165,7 @@ def test_latest_coalesces_to_one_candidate_and_no_double_admit() -> None:
2026, 9, 8, 13, 0 2026, 9, 8, 13, 0
) )
assert store.get_candidate("h") is None assert store.get_candidate("h") is None
finally:
sched.ownership.release() sched.ownership.release()
@@ -218,7 +214,6 @@ def test_schedule_edit_between_poll_snapshot_and_admission_cannot_overwrite_term
) )
super().update_schedule(edited, expected_revision=current.revision) super().update_schedule(edited, expected_revision=current.revision)
return current return current
return current
sched_store = EditOnPollRead(tmp_path / "sched") sched_store = EditOnPollRead(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs") run_store = FileRunStore(tmp_path / "runs")
@@ -277,12 +272,9 @@ def test_trigger_edit_refreshes_managed_calendar_before_polling(tmp_path: Path)
sched.ownership.release() sched.ownership.release()
def test_parallel_limits_and_interrupted_slots() -> None: def test_parallel_limits_and_interrupted_slots(tmp_path: Path) -> None:
import tempfile sched, store, runs, sources = _harness(tmp_path, capacity=4, script={"*": "hang"})
try:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched, store, runs, sources = _harness(root, capacity=4, script={"*": "hang"})
t0 = ts(2026, 9, 8, 12, 0) t0 = ts(2026, 9, 8, 12, 0)
_add( _add(
sched, sched,
@@ -319,17 +311,15 @@ def test_parallel_limits_and_interrupted_slots() -> None:
r["kind"] == "skipped-overlap" and "12:15" in str(r["resolved_at"]) r["kind"] == "skipped-overlap" and "12:15" in str(r["resolved_at"])
for r in _history(store, "p") for r in _history(store, "p")
) )
finally:
sched.ownership.release() sched.ownership.release()
def test_pause_is_not_downtime() -> None: def test_pause_is_not_downtime(tmp_path: Path) -> None:
import tempfile
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched, store, runs, sources = _harness( sched, store, runs, sources = _harness(
root, capacity=4, script={"*": "complete"} tmp_path, capacity=4, script={"*": "complete"}
) )
try:
_add( _add(
sched, sched,
store, store,
@@ -352,6 +342,7 @@ def test_pause_is_not_downtime() -> None:
] ]
assert "2026-09-08T10:00:00+00:00" not in admitted assert "2026-09-08T10:00:00+00:00" not in admitted
assert "2026-09-08T11:00:00+00:00" not in admitted assert "2026-09-08T11:00:00+00:00" not in admitted
finally:
sched.ownership.release() sched.ownership.release()
@@ -359,21 +350,34 @@ def test_long_downtime_is_bounded(tmp_path: Path) -> None:
sched, store, runs, sources = _harness( sched, store, runs, sources = _harness(
tmp_path, capacity=4, script={"*": "complete"} tmp_path, capacity=4, script={"*": "complete"}
) )
try:
src = PeriodicSource(timedelta(minutes=1), ts(2023, 9, 8, 12, 0)) src = PeriodicSource(timedelta(minutes=1), ts(2023, 9, 8, 12, 0))
_add(sched, store, sources, "m", src, ts(2023, 9, 8, 12, 0), misfire="latest") _add(
sched,
store,
sources,
"m",
src,
ts(2023, 9, 8, 12, 0),
misfire="latest",
)
sched.poll(ts(2026, 9, 8, 12, 0, 0)) sched.poll(ts(2026, 9, 8, 12, 0, 0))
assert src.next_calls + src.prev_calls <= SCAN_CAP + 2 assert src.next_calls + src.prev_calls <= SCAN_CAP + 2
admitted = [r for r in _history(store, "m") if r["kind"] == "admitted"] admitted = [r for r in _history(store, "m") if r["kind"] == "admitted"]
assert len(admitted) == 1 assert len(admitted) == 1
# Latest-eligible <= now: a poll exactly at a due instant selects that # Latest-eligible <= now: a poll exactly at a due instant selects that
# instant (F3), not its exclusive predecessor. # instant (F3), not its exclusive predecessor.
assert datetime.fromisoformat(admitted[0]["resolved_at"]) == ts(2026, 9, 8, 12, 0) assert datetime.fromisoformat(admitted[0]["resolved_at"]) == ts(
2026, 9, 8, 12, 0
)
assert ( assert (
len([r for r in _history(store, "m") if r["kind"] == "interval-summary"]) == 1 len([r for r in _history(store, "m") if r["kind"] == "interval-summary"])
== 1
) )
# A second poll at the same instant admits nothing more. # A second poll at the same instant admits nothing more.
sched.poll(ts(2026, 9, 8, 12, 0, 0)) sched.poll(ts(2026, 9, 8, 12, 0, 0))
assert len([r for r in _history(store, "m") if r["kind"] == "admitted"]) == 1 assert len([r for r in _history(store, "m") if r["kind"] == "admitted"]) == 1
finally:
sched.ownership.release() sched.ownership.release()
+16 -4
View File
@@ -10,6 +10,7 @@ inventing a run.
from __future__ import annotations from __future__ import annotations
import inspect import inspect
import typing
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from pathlib import Path from pathlib import Path
from typing import Any, cast from typing import Any, cast
@@ -105,9 +106,9 @@ def test_production_scheduler_has_no_fixture_seams() -> None:
# Collaborators are protocols consumed here, implemented elsewhere. # Collaborators are protocols consumed here, implemented elsewhere.
assert "InvocationPreparer" in source assert "InvocationPreparer" in source
assert "RunDispatcher" in source assert "RunDispatcher" in source
assert isinstance( hints = typing.get_type_hints(poll_module.Scheduler.__init__)
poll_module.Scheduler.__init__.__annotations__["preparer"], object assert hints["preparer"] is InvocationPreparer
) assert hints["dispatcher"] is RunDispatcher
def test_preparer_contract_is_a_protocol() -> None: def test_preparer_contract_is_a_protocol() -> None:
@@ -117,6 +118,7 @@ def test_preparer_contract_is_a_protocol() -> None:
def test_occurrence_bindings_resolve_into_admission(tmp_path: Path) -> None: def test_occurrence_bindings_resolve_into_admission(tmp_path: Path) -> None:
sched, store, runs = _scheduler(tmp_path, script={"*": "hang"}) sched, store, runs = _scheduler(tmp_path, script={"*": "hang"})
try:
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
model = _sched_model( model = _sched_model(
"b", "b",
@@ -127,7 +129,10 @@ def test_occurrence_bindings_resolve_into_admission(tmp_path: Path) -> None:
}, },
{ {
"target": "report_time", "target": "report_time",
"expression": {"kind": "occurrence", "field": "scheduled_at"}, "expression": {
"kind": "occurrence",
"field": "scheduled_at",
},
}, },
{ {
"target": "which", "target": "which",
@@ -147,11 +152,13 @@ def test_occurrence_bindings_resolve_into_admission(tmp_path: Path) -> None:
assert admission.resolved_input["which"] == "b" assert admission.resolved_input["which"] == "b"
assert admission.deployment_revision == 3 assert admission.deployment_revision == 3
assert admission.schedule_revision == 1 assert admission.schedule_revision == 1
finally:
sched.ownership.release() sched.ownership.release()
def test_unknown_deployment_rejects_without_a_run(tmp_path: Path) -> None: def test_unknown_deployment_rejects_without_a_run(tmp_path: Path) -> None:
sched, store, runs = _scheduler(tmp_path) sched, store, runs = _scheduler(tmp_path)
try:
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
store.create_schedule(_sched_model("gone", deployment_id="dep-missing")) store.create_schedule(_sched_model("gone", deployment_id="dep-missing"))
store.save_consumed("gone", intended - timedelta(hours=1)) store.save_consumed("gone", intended - timedelta(hours=1))
@@ -162,6 +169,7 @@ def test_unknown_deployment_rejects_without_a_run(tmp_path: Path) -> None:
page = store.list_occurrences("gone", limit=100) page = store.list_occurrences("gone", limit=100)
kinds = [r["kind"] for r in cast(list[dict[str, Any]], page["occurrences"])] kinds = [r["kind"] for r in cast(list[dict[str, Any]], page["occurrences"])]
assert kinds == ["preflight-rejected"] assert kinds == ["preflight-rejected"]
finally:
sched.ownership.release() sched.ownership.release()
@@ -198,6 +206,7 @@ def test_missing_required_input_rejects_without_a_run(tmp_path: Path) -> None:
dispatcher=ScriptedDispatcher({"*": "hang"}), dispatcher=ScriptedDispatcher({"*": "hang"}),
ownership=SchedulerOwnership(tmp_path, owner="test").acquire(), ownership=SchedulerOwnership(tmp_path, owner="test").acquire(),
) )
try:
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
sched_store.create_schedule(_sched_model("need")) sched_store.create_schedule(_sched_model("need"))
sched_store.save_consumed("need", intended - timedelta(hours=1)) sched_store.save_consumed("need", intended - timedelta(hours=1))
@@ -208,11 +217,13 @@ def test_missing_required_input_rejects_without_a_run(tmp_path: Path) -> None:
entry = cast(list[dict[str, Any]], page["occurrences"])[0] entry = cast(list[dict[str, Any]], page["occurrences"])[0]
assert entry["kind"] == "preflight-rejected" assert entry["kind"] == "preflight-rejected"
assert "missing-input" in entry["reason"] assert "missing-input" in entry["reason"]
finally:
sched.ownership.release() sched.ownership.release()
def test_conflicting_schedule_targets_reject_without_a_run(tmp_path: Path) -> None: def test_conflicting_schedule_targets_reject_without_a_run(tmp_path: Path) -> None:
sched, store, runs = _scheduler(tmp_path) sched, store, runs = _scheduler(tmp_path)
try:
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
store.create_schedule( store.create_schedule(
_sched_model( _sched_model(
@@ -237,6 +248,7 @@ def test_conflicting_schedule_targets_reject_without_a_run(tmp_path: Path) -> No
entry = cast(list[dict[str, Any]], page["occurrences"])[0] entry = cast(list[dict[str, Any]], page["occurrences"])[0]
assert entry["kind"] == "preflight-rejected" assert entry["kind"] == "preflight-rejected"
assert entry["reason"].startswith("invalid-input:") assert entry["reason"].startswith("invalid-input:")
finally:
sched.ownership.release() sched.ownership.release()
+2 -2
View File
@@ -70,7 +70,7 @@ def test_recovery_materializes_missing_view_as_pending(tmp_path: Path) -> None:
ownership.release() ownership.release()
assert any("pending-dispatch" in d for d in diags) assert any("pending-dispatch" in d for d in diags)
assert run_store.get_run(admission.id).status.value == "admitted" assert run_store.get_run(admission.id).status.value == "admitted"
assert sched_recovery._is_pending(run_store, admission.id) assert run_store.is_pending_dispatch(admission.id)
def test_recovery_fails_abandoned_admitted_without_replay(tmp_path: Path) -> None: def test_recovery_fails_abandoned_admitted_without_replay(tmp_path: Path) -> None:
@@ -207,4 +207,4 @@ def test_recovery_never_executes_pending_until_poll(tmp_path: Path) -> None:
finally: finally:
ownership.release() ownership.release()
assert run_store.get_run(admission.id).status.value == "completed" assert run_store.get_run(admission.id).status.value == "completed"
assert not sched_recovery._is_pending(run_store, admission.id) assert not run_store.is_pending_dispatch(admission.id)
+32 -63
View File
@@ -13,6 +13,8 @@ from datetime import UTC, datetime, timedelta
from pathlib import Path from pathlib import Path
from typing import Any, cast from typing import Any, cast
import pytest
from tests.artifacts.test_run_store import artifact as _artifact from tests.artifacts.test_run_store import artifact as _artifact
from tests.artifacts.test_run_store import deployment as _deployment from tests.artifacts.test_run_store import deployment as _deployment
from wf_api.run_lifecycle import ( from wf_api.run_lifecycle import (
@@ -23,7 +25,7 @@ from wf_api.run_lifecycle import (
restore_interrupted_run, restore_interrupted_run,
) )
from wf_artifacts import PinnedRunEnvironment from wf_artifacts import PinnedRunEnvironment
from wf_artifacts.runs.models import ResumeAttempt from wf_artifacts.runs.models import ResumeAttempt, StoredRunStatus
from wf_artifacts.runs.store import FileRunStore from wf_artifacts.runs.store import FileRunStore
from wf_core import RunState, RunStatus from wf_core import RunState, RunStatus
from wf_scheduling import recovery as sched_recovery from wf_scheduling import recovery as sched_recovery
@@ -128,13 +130,9 @@ def _entries(store: FileScheduleStore, sid: str, kind: str) -> list[dict[str, An
] ]
def test_abandonment_decision_is_stable_across_recovery() -> None: def test_abandonment_decision_is_stable_across_recovery(tmp_path: Path) -> None:
import tempfile sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched_store = FileScheduleStore(root / "sched")
run_store = FileRunStore(root / "runs")
sched_store.create_schedule(_sched_model("a")) sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
run_id = _admit_interrupted(run_store, intended) run_id = _admit_interrupted(run_store, intended)
@@ -146,27 +144,21 @@ def test_abandonment_decision_is_stable_across_recovery() -> None:
assert record.resume_readiness.value == "not_applicable" assert record.resume_readiness.value == "not_applicable"
assert len(record.diagnostics) == 1 assert len(record.diagnostics) == 1
# Fresh store objects across the restart boundary: everything stable. # Fresh store objects across the restart boundary: everything stable.
sched_store2 = FileScheduleStore(root / "sched") sched_store2 = FileScheduleStore(tmp_path / "sched")
run_store2 = FileRunStore(root / "runs") run_store2 = FileRunStore(tmp_path / "runs")
second = _recover(sched_store2, run_store2, intended + timedelta(minutes=1)) second = _recover(sched_store2, run_store2, intended + timedelta(minutes=1))
assert not any(run_id in d for d in second) assert not any(run_id in d for d in second)
again = run_store2.get_run(run_id) again = run_store2.get_run(run_id)
assert again.status.value == "failed" assert again.status.value == "failed"
assert again.resume_readiness.value == "not_applicable" assert again.resume_readiness.value == "not_applicable"
assert len(again.diagnostics) == 1 assert len(again.diagnostics) == 1
assert _entries(sched_store2, "a", "failed") == _entries( assert _entries(sched_store2, "a", "failed") == _entries(sched_store, "a", "failed")
sched_store, "a", "failed"
)
assert len(_entries(sched_store2, "a", "failed")) == 1 assert len(_entries(sched_store2, "a", "failed")) == 1
def test_failed_readiness_and_inspection_agree() -> None: def test_failed_readiness_and_inspection_agree(tmp_path: Path) -> None:
import tempfile sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched_store = FileScheduleStore(root / "sched")
run_store = FileRunStore(root / "runs")
sched_store.create_schedule(_sched_model("a")) sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
run_id = _admit_interrupted(run_store, intended) run_id = _admit_interrupted(run_store, intended)
@@ -174,20 +166,13 @@ def test_failed_readiness_and_inspection_agree() -> None:
_recover(sched_store, run_store, intended) _recover(sched_store, run_store, intended)
record, _ = load_stored_run(run_store, run_id) record, _ = load_stored_run(run_store, run_id)
assert record.resume_readiness.value == "not_applicable" assert record.resume_readiness.value == "not_applicable"
try: with pytest.raises(ValueError):
restore_interrupted_run(run_store, run_id) restore_interrupted_run(run_store, run_id)
raise AssertionError("failed run must not restore as interrupted")
except ValueError:
pass
def test_genuinely_newer_result_repairs_after_decision() -> None: def test_genuinely_newer_result_repairs_after_decision(tmp_path: Path) -> None:
import tempfile sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched_store = FileScheduleStore(root / "sched")
run_store = FileRunStore(root / "runs")
sched_store.create_schedule(_sched_model("a")) sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
run_id = _admit_interrupted(run_store, intended) run_id = _admit_interrupted(run_store, intended)
@@ -208,13 +193,9 @@ def test_genuinely_newer_result_repairs_after_decision() -> None:
assert {e["checkpoint_id"] for e in interrupted} == {f"{run_id}.000002"} assert {e["checkpoint_id"] for e in interrupted} == {f"{run_id}.000002"}
def test_completed_mismatch_decision_is_stable() -> None: def test_completed_mismatch_decision_is_stable(tmp_path: Path) -> None:
import tempfile sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched_store = FileScheduleStore(root / "sched")
run_store = FileRunStore(root / "runs")
sched_store.create_schedule(_sched_model("a")) sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
run_id = run_store.allocate_run_id() run_id = run_store.allocate_run_id()
@@ -234,27 +215,21 @@ def test_completed_mismatch_decision_is_stable() -> None:
_recover(sched_store, run_store, intended) _recover(sched_store, run_store, intended)
assert run_store.get_run(run_id).status.value == "failed" assert run_store.get_run(run_id).status.value == "failed"
second = _recover( second = _recover(
FileScheduleStore(root / "sched"), FileScheduleStore(tmp_path / "sched"),
FileRunStore(root / "runs"), FileRunStore(tmp_path / "runs"),
intended + timedelta(minutes=1), intended + timedelta(minutes=1),
) )
assert not any(run_id in d for d in second) assert not any(run_id in d for d in second)
assert len(FileRunStore(root / "runs").get_run(run_id).diagnostics) == 1 assert len(FileRunStore(tmp_path / "runs").get_run(run_id).diagnostics) == 1
def test_legacy_failed_run_gains_history_without_refail() -> None: def test_legacy_failed_run_gains_history_without_refail(tmp_path: Path) -> None:
import tempfile sched_store = FileScheduleStore(tmp_path / "sched")
run_store = FileRunStore(tmp_path / "runs")
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
sched_store = FileScheduleStore(root / "sched")
run_store = FileRunStore(root / "runs")
sched_store.create_schedule(_sched_model("a")) sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
run_id = _admit_interrupted(run_store, intended) run_id = _admit_interrupted(run_store, intended)
record = run_store.get_run(run_id) record = run_store.get_run(run_id)
from wf_artifacts.runs.models import StoredRunStatus
run_store.save_run(record.model_copy(update={"status": StoredRunStatus.FAILED})) run_store.save_run(record.model_copy(update={"status": StoredRunStatus.FAILED}))
assert _entries(sched_store, "a", "failed") == [] assert _entries(sched_store, "a", "failed") == []
diags = _recover(sched_store, run_store, intended) diags = _recover(sched_store, run_store, intended)
@@ -264,12 +239,7 @@ def test_legacy_failed_run_gains_history_without_refail() -> None:
assert len(_entries(sched_store, "a", "failed")) == 1 assert len(_entries(sched_store, "a", "failed")) == 1
def test_crash_between_decision_writes_recovers_history_once() -> None: def test_crash_between_decision_writes_recovers_history_once(tmp_path: Path) -> None:
import tempfile
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
class FailFailedHistoryOnce(FileScheduleStore): class FailFailedHistoryOnce(FileScheduleStore):
def __init__(self, root: Path) -> None: def __init__(self, root: Path) -> None:
super().__init__(root) super().__init__(root)
@@ -281,14 +251,13 @@ def test_crash_between_decision_writes_recovers_history_once() -> None:
raise OSError("injected failed-history failure") raise OSError("injected failed-history failure")
super().append_history(record) super().append_history(record)
sched_store = FailFailedHistoryOnce(root / "sched") sched_store = FailFailedHistoryOnce(tmp_path / "sched")
run_store = FileRunStore(root / "runs") run_store = FileRunStore(tmp_path / "runs")
sched_store.create_schedule(_sched_model("a")) sched_store.create_schedule(_sched_model("a"))
intended = ts(2026, 9, 8, 12, 0) intended = ts(2026, 9, 8, 12, 0)
run_id = _admit_interrupted(run_store, intended) run_id = _admit_interrupted(run_store, intended)
_mark_active(run_store, run_id, 9, intended) _mark_active(run_store, run_id, 9, intended)
sched_store.armed = True sched_store.armed = True
import pytest
with pytest.raises(OSError, match="injected failed-history failure"): with pytest.raises(OSError, match="injected failed-history failure"):
_recover(sched_store, run_store, intended) _recover(sched_store, run_store, intended)
@@ -298,10 +267,10 @@ def test_crash_between_decision_writes_recovers_history_once() -> None:
assert _entries(sched_store, "a", "failed") == [] assert _entries(sched_store, "a", "failed") == []
# The decision (status + reason) survived; only history is missing. # The decision (status + reason) survived; only history is missing.
second = _recover( second = _recover(
FileScheduleStore(root / "sched"), FileScheduleStore(tmp_path / "sched"),
FileRunStore(root / "runs"), FileRunStore(tmp_path / "runs"),
intended + timedelta(minutes=1), intended + timedelta(minutes=1),
) )
assert not any("failed-closed" in d for d in second) assert not any("failed-closed" in d for d in second)
assert len(FileRunStore(root / "runs").get_run(run_id).diagnostics) == 1 assert len(FileRunStore(tmp_path / "runs").get_run(run_id).diagnostics) == 1
assert len(_entries(FileScheduleStore(root / "sched"), "a", "failed")) == 1 assert len(_entries(FileScheduleStore(tmp_path / "sched"), "a", "failed")) == 1
+2 -26
View File
@@ -141,7 +141,6 @@ def test_candidate_is_at_most_one(tmp_path: Path) -> None:
store.save_candidate(first, schedule_id="a") store.save_candidate(first, schedule_id="a")
store.save_candidate(second, schedule_id="a") store.save_candidate(second, schedule_id="a")
assert store.get_candidate("a") is not None assert store.get_candidate("a") is not None
assert store.get_candidate("a") is not None
assert store.get_candidate("a").intended_at == datetime( # type: ignore[union-attr] assert store.get_candidate("a").intended_at == datetime( # type: ignore[union-attr]
2026, 9, 8, 13, 0, tzinfo=UTC 2026, 9, 8, 13, 0, tzinfo=UTC
) )
@@ -202,30 +201,6 @@ def test_inspection_payload_carries_contract_fields(tmp_path: Path) -> None:
assert row["reason"] == "active=['run-1']" assert row["reason"] == "active=['run-1']"
def test_deployment_revision_increments_on_save(tmp_path: Path) -> None:
from wf_artifacts import FileWorkflowArtifactStore, WorkflowDeployment
artifacts = FileWorkflowArtifactStore(tmp_path)
artifacts.save_deployment(
WorkflowDeployment(
id="dep-1",
artifact_id="wf-1",
artifact_version=1,
bindings=[],
)
)
assert artifacts.get_deployment("dep-1").revision == 1
artifacts.save_deployment(
WorkflowDeployment(
id="dep-1",
artifact_id="wf-1",
artifact_version=2,
bindings=[],
)
)
assert artifacts.get_deployment("dep-1").revision == 2
def _tied_entry( def _tied_entry(
kind: str, kind: str,
checkpoint_id: str | None, checkpoint_id: str | None,
@@ -248,13 +223,14 @@ def _traverse(store: FileScheduleStore, limit: int) -> list[dict]:
"""Walk every page to the end, returning all rows in visit order.""" """Walk every page to the end, returning all rows in visit order."""
rows: list[dict] = [] rows: list[dict] = []
cursor: str | None = None cursor: str | None = None
while True: for _ in range(1000):
page = store.list_occurrences("a", cursor=cursor, limit=limit) page = store.list_occurrences("a", cursor=cursor, limit=limit)
rows.extend(page["occurrences"]) # type: ignore[arg-type] rows.extend(page["occurrences"]) # type: ignore[arg-type]
cursor = page["next_cursor"] # type: ignore[assignment] cursor = page["next_cursor"] # type: ignore[assignment]
if cursor is None: if cursor is None:
assert page["total"] == len(rows) assert page["total"] == len(rows)
return rows return rows
raise AssertionError(f"pagination did not terminate; visited {len(rows)} rows")
def test_history_pagination_visits_every_tied_entry_once(tmp_path: Path) -> None: def test_history_pagination_visits_every_tied_entry_once(tmp_path: Path) -> None:
+4 -2
View File
@@ -40,7 +40,9 @@ def _api(root: Path) -> tuple[WorkflowRunApi, FileRunStore]:
return WorkflowRunApi(context), context.run_store return WorkflowRunApi(context), context.run_store
def test_resume_marks_active_attempt_with_store_backed_id(tmp_path: Path) -> None: def test_resume_attempt_ids_are_monotonic_across_store_instances(
tmp_path: Path,
) -> None:
api, store = _api(tmp_path / "resume-marker") api, store = _api(tmp_path / "resume-marker")
started = asyncio.run( started = asyncio.run(
api.run_deployment(deployment_id="echo.personal", workflow_input={"text": "hi"}) api.run_deployment(deployment_id="echo.personal", workflow_input={"text": "hi"})
@@ -55,7 +57,7 @@ def test_resume_marks_active_attempt_with_store_backed_id(tmp_path: Path) -> Non
assert second == first + 1 assert second == first + 1
def test_active_attempt_blocks_second_resume(tmp_path: Path) -> None: def test_active_attempt_marker_is_durable(tmp_path: Path) -> None:
from datetime import UTC, datetime from datetime import UTC, datetime
from wf_artifacts.runs.models import ResumeAttempt from wf_artifacts.runs.models import ResumeAttempt
+9
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import json import json
import pytest
from typer.testing import CliRunner from typer.testing import CliRunner
from wf_server.cli import app from wf_server.cli import app
@@ -583,6 +584,10 @@ def test_rpc_server_cli_enable_scheduler_rejects_mcp_backed_server(
"wf_server.cli.build_workflow_server_from_legacy_mcp_config", "wf_server.cli.build_workflow_server_from_legacy_mcp_config",
fake_build_mcp_server, fake_build_mcp_server,
) )
monkeypatch.setattr(
"wf_server.cli.uvicorn.run",
lambda *args, **kwargs: pytest.fail("server must not start"),
)
result = CliRunner().invoke( result = CliRunner().invoke(
app, app,
@@ -638,6 +643,10 @@ def test_rpc_server_cli_config_mcp_sources_reject_scheduler(
fake_build_server, fake_build_server,
) )
monkeypatch.setattr("wf_server.cli.create_rpc_app", fake_create_rpc_app) monkeypatch.setattr("wf_server.cli.create_rpc_app", fake_create_rpc_app)
monkeypatch.setattr(
"wf_server.cli.uvicorn.run",
lambda *args, **kwargs: pytest.fail("server must not start"),
)
result = CliRunner().invoke(app, ["--config", str(config_path)]) result = CliRunner().invoke(app, ["--config", str(config_path)])
@@ -90,9 +90,8 @@ def _constant_plan() -> RawWorkflowPlan:
) )
async def _seed_server(tmp_path: Any, name: str = "sched-art") -> Any: async def _seed_server(tmp_path: Any) -> Any:
"""Build a schedule-enabled server with one artifact + deployment.""" """Build a schedule-enabled server with one artifact + deployment."""
_ = name
server = build_local_static_workflow_server(tmp_path / "store", schedules=True) server = build_local_static_workflow_server(tmp_path / "store", schedules=True)
await server.api.create_artifact_from_plan( await server.api.create_artifact_from_plan(
artifact_id="sched-art", artifact_id="sched-art",