fix: avoid scheduler registration deadlock
This commit is contained in:
@@ -11,6 +11,8 @@ dedicated integration tests.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -329,6 +331,80 @@ async def test_shutdown_joins_cancelled_settlement_before_releasing_ownership(
|
||||
await service.stop()
|
||||
|
||||
|
||||
def test_submit_and_stop_do_not_deadlock_on_registration(tmp_path: Path) -> None:
|
||||
"""The poll worker and event loop must not wait on each other."""
|
||||
child = """
|
||||
import asyncio
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from tests.scheduling.test_lifecycle import (
|
||||
SchedulerServiceConfig,
|
||||
ScriptedRuntime,
|
||||
_sched_model,
|
||||
_service,
|
||||
ts,
|
||||
)
|
||||
import wf_scheduling.lifecycle as lifecycle
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
root = Path(sys.argv[1])
|
||||
service = _service(
|
||||
root,
|
||||
ScriptedRuntime("complete"),
|
||||
config=SchedulerServiceConfig(poll_interval_s=0.01, auto_tick=False),
|
||||
)
|
||||
await service.start()
|
||||
service.schedule_store.create_schedule(
|
||||
_sched_model("a", ts(2026, 9, 8, 12, 0))
|
||||
)
|
||||
registration_entered = threading.Event()
|
||||
release_registration = threading.Event()
|
||||
original = asyncio.run_coroutine_threadsafe
|
||||
|
||||
def blocked_registration(coro, loop):
|
||||
registration_entered.set()
|
||||
if not release_registration.wait(5):
|
||||
raise AssertionError("registration release was not signalled")
|
||||
return original(coro, loop)
|
||||
|
||||
lifecycle.asyncio.run_coroutine_threadsafe = blocked_registration
|
||||
poll_task = asyncio.create_task(
|
||||
service.poll_once(ts(2026, 9, 8, 12, 1))
|
||||
)
|
||||
if not await asyncio.to_thread(registration_entered.wait, 5):
|
||||
raise AssertionError("poll did not reach the registration gate")
|
||||
stop_task = asyncio.create_task(service.stop())
|
||||
threading.Thread(
|
||||
target=lambda: (time.sleep(0.1), release_registration.set()),
|
||||
daemon=True,
|
||||
).start()
|
||||
await asyncio.wait_for(asyncio.gather(poll_task, stop_task), 4)
|
||||
print("registration-stop-complete", flush=True)
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
"""
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-c", child, str(tmp_path)],
|
||||
cwd=Path(__file__).resolve().parents[2],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
try:
|
||||
output, _ = process.communicate(timeout=7)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
output, _ = process.communicate(timeout=5)
|
||||
pytest.fail(f"registration/stop deadlocked; child output: {output}")
|
||||
assert process.returncode == 0, output
|
||||
assert "registration-stop-complete" in output
|
||||
|
||||
|
||||
async def test_scheduled_interrupt_stays_resumable(tmp_path: Path) -> None:
|
||||
intended = ts(2026, 9, 8, 12, 0)
|
||||
service = _service(tmp_path, ScriptedRuntime("interrupt"))
|
||||
|
||||
Reference in New Issue
Block a user