fix: join offloaded scheduler settlement
This commit is contained in:
@@ -11,6 +11,7 @@ dedicated integration tests.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
@@ -238,6 +239,95 @@ async def test_scheduled_failure_records_failed_history(tmp_path: Path) -> None:
|
||||
await service.stop()
|
||||
|
||||
|
||||
async def test_settlement_does_not_block_the_event_loop(
|
||||
tmp_path: Path, monkeypatch: Any
|
||||
) -> None:
|
||||
"""Synchronous settlement persistence must not stall other coroutines."""
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
service = _service(tmp_path, ScriptedRuntime("complete"))
|
||||
settlement_started = threading.Event()
|
||||
settlement_finished = threading.Event()
|
||||
release = threading.Event()
|
||||
original = Scheduler.record_stopped_execution
|
||||
|
||||
def _blocking_settle(self: Any, run_id: str, state: Any, now: datetime) -> None:
|
||||
settlement_started.set()
|
||||
release.wait(timeout=2)
|
||||
original(self, run_id, state, now)
|
||||
settlement_finished.set()
|
||||
|
||||
monkeypatch.setattr(Scheduler, "record_stopped_execution", _blocking_settle)
|
||||
release_timer = threading.Timer(1.0, release.set)
|
||||
release_timer.start()
|
||||
try:
|
||||
await service.start()
|
||||
service.schedule_store.create_schedule(_sched_model("a", intended))
|
||||
await service.poll_once(intended + timedelta(seconds=1))
|
||||
await asyncio.sleep(0.05)
|
||||
assert settlement_started.is_set()
|
||||
assert not settlement_finished.is_set()
|
||||
assert service.live_executions == 1
|
||||
release.set()
|
||||
await _wait_for(lambda: service.live_executions == 0)
|
||||
finally:
|
||||
release.set()
|
||||
release_timer.cancel()
|
||||
await service.stop()
|
||||
|
||||
|
||||
async def test_shutdown_joins_cancelled_settlement_before_releasing_ownership(
|
||||
tmp_path: Path, monkeypatch: Any
|
||||
) -> None:
|
||||
"""Shutdown waits for an offloaded settlement after cancelling its task."""
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
service = _service(
|
||||
tmp_path,
|
||||
ScriptedRuntime("complete"),
|
||||
SchedulerServiceConfig(
|
||||
poll_interval_s=0.01,
|
||||
drain_grace_s=0,
|
||||
auto_tick=False,
|
||||
),
|
||||
)
|
||||
settlement_started = threading.Event()
|
||||
release = threading.Event()
|
||||
original = Scheduler.record_stopped_execution
|
||||
|
||||
def _blocking_settle(self: Any, run_id: str, state: Any, now: datetime) -> None:
|
||||
settlement_started.set()
|
||||
release.wait(timeout=2)
|
||||
original(self, run_id, state, now)
|
||||
|
||||
monkeypatch.setattr(Scheduler, "record_stopped_execution", _blocking_settle)
|
||||
release_timer = threading.Timer(1.0, release.set)
|
||||
release_timer.start()
|
||||
try:
|
||||
await service.start()
|
||||
service.schedule_store.create_schedule(_sched_model("a", intended))
|
||||
await service.poll_once(intended + timedelta(seconds=1))
|
||||
for _ in range(50):
|
||||
if settlement_started.is_set():
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
else:
|
||||
pytest.fail("settlement did not start")
|
||||
|
||||
stop_task = asyncio.create_task(service.stop())
|
||||
await asyncio.sleep(0.05)
|
||||
assert not stop_task.done()
|
||||
release.set()
|
||||
report = await stop_task
|
||||
assert report.settled == 1
|
||||
assert report.cancelled == 0
|
||||
assert service.run_store.get_run(
|
||||
_only_run_id(service.run_store)
|
||||
).status.value == ("completed")
|
||||
finally:
|
||||
release.set()
|
||||
release_timer.cancel()
|
||||
await service.stop()
|
||||
|
||||
|
||||
async def test_scheduled_interrupt_stays_resumable(tmp_path: Path) -> None:
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
service = _service(tmp_path, ScriptedRuntime("interrupt"))
|
||||
|
||||
Reference in New Issue
Block a user