fix: join cancelled scheduler startup
This commit is contained in:
@@ -531,6 +531,62 @@ async def test_failed_startup_releases_lock(tmp_path: Path) -> None:
|
||||
await service.stop()
|
||||
|
||||
|
||||
async def test_cancelled_startup_joins_gated_recovery_before_releasing_ownership(
|
||||
tmp_path: Path, monkeypatch: Any
|
||||
) -> None:
|
||||
"""Cancelled startup waits for recovery before releasing its store lock."""
|
||||
service = _service(tmp_path, ScriptedRuntime("complete"))
|
||||
recovery_started = threading.Event()
|
||||
recovery_release = threading.Event()
|
||||
recovery_ready = threading.Event()
|
||||
recovery_continue = threading.Event()
|
||||
recovery_finished = threading.Event()
|
||||
original = SchedulerService._recover
|
||||
|
||||
def gated_recover(self: SchedulerService) -> list[str]:
|
||||
recovery_started.set()
|
||||
if not recovery_release.wait(5):
|
||||
raise AssertionError("recovery release was not signalled")
|
||||
recovery_ready.set()
|
||||
if not recovery_continue.wait(5):
|
||||
raise AssertionError("recovery continuation was not signalled")
|
||||
result = original(self)
|
||||
recovery_finished.set()
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(SchedulerService, "_recover", gated_recover)
|
||||
start_task = asyncio.create_task(service.start())
|
||||
try:
|
||||
assert await asyncio.to_thread(recovery_started.wait, 5)
|
||||
start_task.cancel()
|
||||
recovery_release.set()
|
||||
assert await asyncio.to_thread(recovery_ready.wait, 5)
|
||||
# The cancelled start is still joining the active recovery worker.
|
||||
assert not start_task.done()
|
||||
|
||||
retry = _service(tmp_path, ScriptedRuntime("complete"))
|
||||
with pytest.raises(SecondOwnerError):
|
||||
retry.ownership.acquire()
|
||||
|
||||
recovery_continue.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await asyncio.wait_for(start_task, 5)
|
||||
assert recovery_finished.is_set()
|
||||
assert service.running is False
|
||||
assert service.live_executions == 0
|
||||
assert service._tick_task is None
|
||||
|
||||
await retry.start()
|
||||
await retry.stop()
|
||||
finally:
|
||||
recovery_release.set()
|
||||
recovery_continue.set()
|
||||
if not start_task.done():
|
||||
start_task.cancel()
|
||||
await asyncio.gather(start_task, return_exceptions=True)
|
||||
await service.stop()
|
||||
|
||||
|
||||
async def test_corrupt_store_startup_releases_lock(tmp_path: Path) -> None:
|
||||
class BrokenRuns(FileRunStore):
|
||||
def list_admissions(self) -> list[Any]:
|
||||
|
||||
Reference in New Issue
Block a user