fix: avoid scheduler registration deadlock

This commit is contained in:
lda
2026-09-09 23:20:56 +07:00 Verified
parent fc95326250
commit 6a87711f89
4 changed files with 113 additions and 20 deletions
+76
View File
@@ -11,6 +11,8 @@ dedicated integration tests.
from __future__ import annotations
import asyncio
import subprocess
import sys
import threading
from datetime import UTC, datetime, timedelta
from pathlib import Path
@@ -329,6 +331,80 @@ async def test_shutdown_joins_cancelled_settlement_before_releasing_ownership(
await service.stop()
def test_submit_and_stop_do_not_deadlock_on_registration(tmp_path: Path) -> None:
"""The poll worker and event loop must not wait on each other."""
child = """
import asyncio
import sys
import threading
import time
from pathlib import Path
from tests.scheduling.test_lifecycle import (
SchedulerServiceConfig,
ScriptedRuntime,
_sched_model,
_service,
ts,
)
import wf_scheduling.lifecycle as lifecycle
async def main() -> None:
root = Path(sys.argv[1])
service = _service(
root,
ScriptedRuntime("complete"),
config=SchedulerServiceConfig(poll_interval_s=0.01, auto_tick=False),
)
await service.start()
service.schedule_store.create_schedule(
_sched_model("a", ts(2026, 9, 8, 12, 0))
)
registration_entered = threading.Event()
release_registration = threading.Event()
original = asyncio.run_coroutine_threadsafe
def blocked_registration(coro, loop):
registration_entered.set()
if not release_registration.wait(5):
raise AssertionError("registration release was not signalled")
return original(coro, loop)
lifecycle.asyncio.run_coroutine_threadsafe = blocked_registration
poll_task = asyncio.create_task(
service.poll_once(ts(2026, 9, 8, 12, 1))
)
if not await asyncio.to_thread(registration_entered.wait, 5):
raise AssertionError("poll did not reach the registration gate")
stop_task = asyncio.create_task(service.stop())
threading.Thread(
target=lambda: (time.sleep(0.1), release_registration.set()),
daemon=True,
).start()
await asyncio.wait_for(asyncio.gather(poll_task, stop_task), 4)
print("registration-stop-complete", flush=True)
asyncio.run(main())
"""
process = subprocess.Popen(
[sys.executable, "-c", child, str(tmp_path)],
cwd=Path(__file__).resolve().parents[2],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
try:
output, _ = process.communicate(timeout=7)
except subprocess.TimeoutExpired:
process.kill()
output, _ = process.communicate(timeout=5)
pytest.fail(f"registration/stop deadlocked; child output: {output}")
assert process.returncode == 0, output
assert "registration-stop-complete" in output
async def test_scheduled_interrupt_stays_resumable(tmp_path: Path) -> None:
intended = ts(2026, 9, 8, 12, 0)
service = _service(tmp_path, ScriptedRuntime("interrupt"))