fix: join offloaded scheduler settlement

This commit is contained in:
lda
2026-09-09 21:29:01 +07:00 Verified
parent 384f610301
commit c5087fe037
2 changed files with 137 additions and 10 deletions
+90
View File
@@ -11,6 +11,7 @@ dedicated integration tests.
from __future__ import annotations
import asyncio
import threading
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, cast
@@ -238,6 +239,95 @@ async def test_scheduled_failure_records_failed_history(tmp_path: Path) -> None:
await service.stop()
async def test_settlement_does_not_block_the_event_loop(
tmp_path: Path, monkeypatch: Any
) -> None:
"""Synchronous settlement persistence must not stall other coroutines."""
intended = ts(2026, 9, 8, 12, 0)
service = _service(tmp_path, ScriptedRuntime("complete"))
settlement_started = threading.Event()
settlement_finished = threading.Event()
release = threading.Event()
original = Scheduler.record_stopped_execution
def _blocking_settle(self: Any, run_id: str, state: Any, now: datetime) -> None:
settlement_started.set()
release.wait(timeout=2)
original(self, run_id, state, now)
settlement_finished.set()
monkeypatch.setattr(Scheduler, "record_stopped_execution", _blocking_settle)
release_timer = threading.Timer(1.0, release.set)
release_timer.start()
try:
await service.start()
service.schedule_store.create_schedule(_sched_model("a", intended))
await service.poll_once(intended + timedelta(seconds=1))
await asyncio.sleep(0.05)
assert settlement_started.is_set()
assert not settlement_finished.is_set()
assert service.live_executions == 1
release.set()
await _wait_for(lambda: service.live_executions == 0)
finally:
release.set()
release_timer.cancel()
await service.stop()
async def test_shutdown_joins_cancelled_settlement_before_releasing_ownership(
tmp_path: Path, monkeypatch: Any
) -> None:
"""Shutdown waits for an offloaded settlement after cancelling its task."""
intended = ts(2026, 9, 8, 12, 0)
service = _service(
tmp_path,
ScriptedRuntime("complete"),
SchedulerServiceConfig(
poll_interval_s=0.01,
drain_grace_s=0,
auto_tick=False,
),
)
settlement_started = threading.Event()
release = threading.Event()
original = Scheduler.record_stopped_execution
def _blocking_settle(self: Any, run_id: str, state: Any, now: datetime) -> None:
settlement_started.set()
release.wait(timeout=2)
original(self, run_id, state, now)
monkeypatch.setattr(Scheduler, "record_stopped_execution", _blocking_settle)
release_timer = threading.Timer(1.0, release.set)
release_timer.start()
try:
await service.start()
service.schedule_store.create_schedule(_sched_model("a", intended))
await service.poll_once(intended + timedelta(seconds=1))
for _ in range(50):
if settlement_started.is_set():
break
await asyncio.sleep(0.01)
else:
pytest.fail("settlement did not start")
stop_task = asyncio.create_task(service.stop())
await asyncio.sleep(0.05)
assert not stop_task.done()
release.set()
report = await stop_task
assert report.settled == 1
assert report.cancelled == 0
assert service.run_store.get_run(
_only_run_id(service.run_store)
).status.value == ("completed")
finally:
release.set()
release_timer.cancel()
await service.stop()
async def test_scheduled_interrupt_stays_resumable(tmp_path: Path) -> None:
intended = ts(2026, 9, 8, 12, 0)
service = _service(tmp_path, ScriptedRuntime("interrupt"))