703 lines
28 KiB
Python
703 lines
28 KiB
Python
"""Poll loop: overlap, misfire, candidates, fairness, capacity (T08).
|
|
|
|
Mirrors the reference state model (probes/deployment_scheduling_verify/
|
|
test_schedule_state_model.py) against real file stores. Calendar iteration
|
|
uses ``OccurrenceSource`` (``next_after``/``prev_before`` only, never
|
|
enumeration); latest-missed catch-up is one bounded ``prev_before`` query
|
|
(F1). Overlap decisions precede capacity checks; terminal skips never
|
|
reappear; ``latest`` retains at most one candidate.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Protocol
|
|
|
|
from wf_scheduling.calendar import CronSource, OneShotSource
|
|
from wf_scheduling.models import OccurrenceRecord, PendingCandidate
|
|
from wf_scheduling.occurrences import occurrence_id
|
|
|
|
UTC = timezone.utc
|
|
SCAN_CAP = 100
|
|
EPOCH = datetime(2020, 1, 1, tzinfo=UTC)
|
|
|
|
|
|
class BlockedSchedule(ValueError):
|
|
"""A schedule is blocked by a corrupt record and fails closed."""
|
|
|
|
|
|
class InvalidScheduleDefinitionError(ValueError):
|
|
"""A schedule definition or its trigger source is invalid."""
|
|
|
|
|
|
class OccurrenceSource(Protocol):
|
|
"""Due-instant source over aware datetimes, UTC at the boundary."""
|
|
|
|
def next_after(self, instant: datetime) -> datetime | None: ...
|
|
def prev_before(self, instant: datetime) -> datetime | None: ...
|
|
|
|
|
|
def source_for_trigger(trigger: Any) -> OccurrenceSource:
|
|
"""Build a calendar source from a schedule trigger model."""
|
|
kind = trigger.kind if hasattr(trigger, "kind") else trigger.get("kind")
|
|
if kind == "cron":
|
|
expression = (
|
|
trigger.expression
|
|
if hasattr(trigger, "expression")
|
|
else trigger["expression"]
|
|
)
|
|
zone = (
|
|
trigger.timezone
|
|
if hasattr(trigger, "timezone")
|
|
else trigger.get("timezone", "UTC")
|
|
)
|
|
return CronSource(expression, zone)
|
|
if kind == "oneshot":
|
|
at = trigger.at if hasattr(trigger, "at") else trigger["at"]
|
|
if isinstance(at, str):
|
|
at = datetime.fromisoformat(at)
|
|
return OneShotSource(at)
|
|
raise ValueError(f"unknown trigger kind {kind!r}")
|
|
|
|
|
|
def _is_oneshot(source: OccurrenceSource) -> bool:
|
|
# Calendar and test-double one-shots share the class name and an ``at``
|
|
# instant; periodic sources carry a ``period`` instead. isinstance covers
|
|
# the calendar type; the name check covers duck-typed test doubles.
|
|
if isinstance(source, OneShotSource):
|
|
return True
|
|
return type(source).__name__ == "OneShotSource" and hasattr(source, "at")
|
|
|
|
|
|
class Scheduler:
|
|
"""File-store scheduler core with injected clock and controlled dispatch.
|
|
|
|
``outcomes`` maps run ids (or ``"*"``) to ``complete`` | ``hang`` |
|
|
``interrupt`` for deterministic tests. Production dispatch (real runtime)
|
|
is wired in T12; here ``hang`` keeps an admitted run occupying its
|
|
schedule slot, ``interrupt`` marks it interrupted (schedule slot only),
|
|
and ``complete`` marks it completed (releases overlap).
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
schedule_store: Any,
|
|
run_store: Any,
|
|
sources: dict[str, OccurrenceSource],
|
|
capacity: int,
|
|
outcomes: dict[str, str] | None = None,
|
|
deployments: dict[str, dict] | None = None,
|
|
) -> None:
|
|
self.schedule_store = schedule_store
|
|
self.run_store = run_store
|
|
self.sources = sources
|
|
self.capacity = capacity
|
|
self.outcomes = outcomes or {}
|
|
self.deployments = deployments or {"dep-1": {"rev": 1, "required": []}}
|
|
self._poll_cursor = 0
|
|
|
|
# -- helpers ------------------------------------------------------
|
|
@staticmethod
|
|
def _status_value(run: Any) -> Any:
|
|
status = getattr(run, "status", None)
|
|
return getattr(status, "value", status)
|
|
|
|
def _active(self, sched_id: str) -> list[Any]:
|
|
active: list[Any] = []
|
|
for run in self.run_store.list_runs():
|
|
if self._status_value(run) not in ("admitted", "interrupted"):
|
|
continue
|
|
direct = getattr(run, "sched_id", None)
|
|
if direct is not None:
|
|
if direct == sched_id:
|
|
active.append(run)
|
|
continue
|
|
if self._run_sched(run) == sched_id:
|
|
active.append(run)
|
|
return active
|
|
|
|
def _run_sched(self, run: Any) -> str | None:
|
|
try:
|
|
admission = self.run_store.get_admission(run.id)
|
|
except KeyError:
|
|
return None
|
|
return admission.schedule_id
|
|
|
|
def _task_load(self) -> int:
|
|
from wf_scheduling.recovery import _is_pending
|
|
|
|
count = 0
|
|
for run in self.run_store.list_runs():
|
|
if self._status_value(run) != "admitted":
|
|
continue
|
|
if _is_pending(self.run_store, run.id):
|
|
continue
|
|
count += 1
|
|
return count
|
|
|
|
def _record(
|
|
self,
|
|
*,
|
|
kind: str,
|
|
sched_id: str,
|
|
intended: datetime | None = None,
|
|
run_id: str | None = None,
|
|
reason: str = "",
|
|
interval: tuple[datetime, datetime] | None = None,
|
|
count: int = 0,
|
|
revision: int | None = None,
|
|
now: datetime | None = None,
|
|
admitted_at: datetime | None = None,
|
|
started_at: datetime | None = None,
|
|
) -> None:
|
|
created = now if now is not None else datetime.now(UTC)
|
|
if intended is not None:
|
|
oid = occurrence_id(sched_id, intended)
|
|
elif interval is not None:
|
|
oid = f"{sched_id}|summary|{interval[0].isoformat()}|{interval[1].isoformat()}"
|
|
else:
|
|
oid = f"{sched_id}|summary|{created.isoformat()}"
|
|
self.schedule_store.append_history(
|
|
OccurrenceRecord(
|
|
schedule_id=sched_id,
|
|
occurrence_id=oid,
|
|
kind=kind, # type: ignore[arg-type]
|
|
resolved_at=intended,
|
|
run_id=run_id,
|
|
revision=revision,
|
|
reason=reason,
|
|
admitted_at=admitted_at,
|
|
started_at=started_at,
|
|
interval_start=interval[0] if interval else None,
|
|
interval_end=interval[1] if interval else None,
|
|
interval_count=count,
|
|
created_at=created,
|
|
)
|
|
)
|
|
|
|
def _admit(self, sched: Any, intended: datetime, now: datetime) -> str | None:
|
|
if getattr(sched, "blocked_reason", None):
|
|
raise BlockedSchedule(getattr(sched, "blocked_reason"))
|
|
if not sched.enabled or sched.deleted or sched.paused:
|
|
return None
|
|
# Fail closed before any new admission when a corrupt active view
|
|
# exists for this schedule.
|
|
for run in self._active(sched.id):
|
|
try:
|
|
self.run_store.get_admission(run.id)
|
|
except KeyError:
|
|
reason = f"corrupt run view without admission: {run.id}"
|
|
sched.blocked_reason = reason
|
|
self.schedule_store.save_schedule(sched)
|
|
raise BlockedSchedule(reason) from None
|
|
dep = self.deployments.get(sched.deployment_id)
|
|
if dep is None:
|
|
self._record(
|
|
kind="preflight-rejected",
|
|
sched_id=sched.id,
|
|
intended=intended,
|
|
reason="deployment-deleted",
|
|
revision=sched.revision,
|
|
now=now,
|
|
)
|
|
cand = self.schedule_store.get_candidate(sched.id)
|
|
if cand is not None and cand.intended_at == intended:
|
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
|
self.schedule_store.save_consumed(sched.id, intended)
|
|
return None
|
|
frozen = {
|
|
"team": "eng",
|
|
"report_time": intended.isoformat(),
|
|
"sched": sched.id,
|
|
"dep_rev": dep["rev"],
|
|
}
|
|
missing = [k for k in dep.get("required", []) if k not in frozen]
|
|
if missing:
|
|
self._record(
|
|
kind="preflight-rejected",
|
|
sched_id=sched.id,
|
|
intended=intended,
|
|
reason=f"missing-input:{missing}",
|
|
revision=sched.revision,
|
|
)
|
|
cand = self.schedule_store.get_candidate(sched.id)
|
|
if cand is not None and cand.intended_at == intended:
|
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
|
self.schedule_store.save_consumed(sched.id, intended)
|
|
return None
|
|
active = self._active(sched.id)
|
|
if sched.overlap == "skip" and active:
|
|
self._record(
|
|
kind="skipped-overlap",
|
|
sched_id=sched.id,
|
|
intended=intended,
|
|
reason=f"active={[getattr(r, 'id', None) for r in active]}",
|
|
revision=sched.revision,
|
|
)
|
|
cand = self.schedule_store.get_candidate(sched.id)
|
|
if cand is not None and cand.intended_at == intended:
|
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
|
self.schedule_store.save_consumed(sched.id, intended)
|
|
return None
|
|
if sched.overlap == "parallel" and len(active) >= sched.max_active_runs:
|
|
self._record(
|
|
kind="skipped-overlap",
|
|
sched_id=sched.id,
|
|
intended=intended,
|
|
reason=f"max_active={sched.max_active_runs}",
|
|
revision=sched.revision,
|
|
)
|
|
cand = self.schedule_store.get_candidate(sched.id)
|
|
if cand is not None and cand.intended_at == intended:
|
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
|
self.schedule_store.save_consumed(sched.id, intended)
|
|
return None
|
|
if self._task_load() >= self.capacity:
|
|
if sched.misfire == "latest":
|
|
self.schedule_store.save_candidate(
|
|
PendingCandidate(
|
|
schedule_id=sched.id,
|
|
intended_at=intended,
|
|
revision=sched.revision,
|
|
),
|
|
schedule_id=sched.id,
|
|
)
|
|
self.schedule_store.save_consumed(sched.id, intended)
|
|
return "held"
|
|
if (now - intended).total_seconds() > sched.lateness_allowance_s:
|
|
self._record(
|
|
kind="skipped-misfire",
|
|
sched_id=sched.id,
|
|
intended=intended,
|
|
reason="capacity-deadline",
|
|
revision=sched.revision,
|
|
)
|
|
self.schedule_store.save_consumed(sched.id, intended)
|
|
return None
|
|
return "held-undecided"
|
|
run_id = self.run_store.allocate_run_id()
|
|
from wf_artifacts.runs.models import RunAdmission
|
|
|
|
admission = RunAdmission(
|
|
id=run_id,
|
|
environment=_test_environment(sched),
|
|
resolved_input=dict(frozen),
|
|
max_steps=getattr(sched, "max_steps", None),
|
|
scheduled_at=intended,
|
|
schedule_id=sched.id,
|
|
schedule_revision=sched.revision,
|
|
deployment_revision=dep["rev"],
|
|
created_at=now,
|
|
)
|
|
self.run_store.save_admission(admission)
|
|
self._record(
|
|
kind="admitted",
|
|
sched_id=sched.id,
|
|
intended=intended,
|
|
run_id=run_id,
|
|
reason=f"rev={sched.revision}",
|
|
revision=sched.revision,
|
|
now=now,
|
|
admitted_at=now,
|
|
)
|
|
if _is_oneshot(self.sources.get(sched.id)): # type: ignore[arg-type]
|
|
sched.exhausted = True
|
|
self.schedule_store.save_schedule(sched)
|
|
cand = self.schedule_store.get_candidate(sched.id)
|
|
if cand is not None and cand.intended_at == intended:
|
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
|
self.schedule_store.save_consumed(sched.id, intended)
|
|
from wf_api.run_lifecycle import materialize_admitted_view
|
|
|
|
# Dispatch mark precedes the view so a crash after admission but
|
|
# before/during first dispatch stays pending (not abandoned). Cleared
|
|
# after dispatch returns regardless of outcome (hang still dispatched).
|
|
try:
|
|
self.run_store.mark_pending_dispatch(run_id)
|
|
except AttributeError:
|
|
pass
|
|
materialize_admitted_view(store=self.run_store, admission=admission)
|
|
try:
|
|
self._dispatch(run_id, now)
|
|
finally:
|
|
try:
|
|
self.run_store.clear_pending_dispatch(run_id)
|
|
except AttributeError:
|
|
pass
|
|
return run_id
|
|
|
|
def _dispatch(self, run_id: str, now: datetime) -> None:
|
|
from wf_artifacts.runs.models import StoredRunStatus
|
|
|
|
outcome = self.outcomes.get(run_id, self.outcomes.get("*", "complete"))
|
|
if outcome not in ("complete", "hang", "interrupt"):
|
|
raise ValueError(f"unknown controlled outcome {outcome!r} for {run_id!r}")
|
|
try:
|
|
record = self.run_store.get_run(run_id)
|
|
except KeyError as exc:
|
|
raise BlockedSchedule(f"dispatch missing run view: {run_id!r}") from exc
|
|
sched_id = self._run_sched(record)
|
|
if sched_id is None:
|
|
raise BlockedSchedule(f"dispatch missing admission: {run_id!r}")
|
|
if outcome == "hang":
|
|
return
|
|
if outcome == "interrupt":
|
|
updated = record.model_copy(
|
|
update={"status": StoredRunStatus.INTERRUPTED, "updated_at": now}
|
|
)
|
|
self.run_store.save_run(updated)
|
|
self._record(
|
|
kind="interrupted",
|
|
sched_id=sched_id,
|
|
intended=_admission_intended(self.run_store, run_id),
|
|
run_id=run_id,
|
|
now=now,
|
|
started_at=now,
|
|
)
|
|
return
|
|
updated = record.model_copy(
|
|
update={"status": StoredRunStatus.COMPLETED, "updated_at": now}
|
|
)
|
|
self.run_store.save_run(updated)
|
|
self._record(
|
|
kind="completed",
|
|
sched_id=sched_id,
|
|
intended=_admission_intended(self.run_store, run_id),
|
|
run_id=run_id,
|
|
now=now,
|
|
started_at=now,
|
|
)
|
|
|
|
# -- polling --------------------------------------------------------
|
|
def poll(self, now: datetime) -> dict[str, str]:
|
|
self._dispatch_pending(now)
|
|
schedules = self.schedule_store.list_schedules(include_deleted=True)
|
|
ids = sorted(item.id for item in schedules)
|
|
if not ids:
|
|
return {}
|
|
cursor = self.schedule_store.get_poll_cursor()
|
|
start = cursor % len(ids)
|
|
order = ids[start:] + ids[:start]
|
|
self.schedule_store.save_poll_cursor(cursor + 1)
|
|
results: dict[str, str] = {}
|
|
by_id = {item.id: item for item in schedules}
|
|
for sid in order:
|
|
sched = by_id[sid]
|
|
if getattr(sched, "blocked_reason", None):
|
|
raise BlockedSchedule(getattr(sched, "blocked_reason"))
|
|
results[sid] = self._poll_one(sched, now)
|
|
return results
|
|
|
|
def _dispatch_pending(self, now: datetime) -> None:
|
|
"""Dispatch recovery-materialized runs through capacity checks.
|
|
|
|
Recovery NEVER executes: it only completes missing views flagged
|
|
pending for this sweep. Pending runs of blocked schedules stay
|
|
pending. Hanging admitted runs without a pending marker are never
|
|
re-executed here.
|
|
"""
|
|
from wf_scheduling.recovery import _is_pending, clear_pending
|
|
|
|
for run in sorted(self.run_store.list_runs(), key=lambda r: r.id):
|
|
if not _is_pending(self.run_store, run.id):
|
|
continue
|
|
try:
|
|
admission = self.run_store.get_admission(run.id)
|
|
except KeyError:
|
|
continue
|
|
if admission.schedule_id is None:
|
|
continue
|
|
try:
|
|
sched = self.schedule_store.get_schedule(admission.schedule_id)
|
|
except KeyError:
|
|
continue
|
|
if getattr(sched, "blocked_reason", None):
|
|
continue
|
|
if self._task_load() >= self.capacity:
|
|
continue
|
|
self._dispatch(run.id, now)
|
|
clear_pending(self.run_store, run.id)
|
|
|
|
def _poll_one(self, sched: Any, now: datetime) -> str:
|
|
if sched.deleted:
|
|
if self.schedule_store.get_candidate(sched.id) is not None:
|
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
|
return "deleted"
|
|
if not sched.enabled:
|
|
if self.schedule_store.get_candidate(sched.id) is not None:
|
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
|
consumed = self.schedule_store.get_consumed(sched.id) or EPOCH
|
|
self.schedule_store.save_consumed(sched.id, max(consumed, now))
|
|
return "disabled"
|
|
if sched.exhausted:
|
|
return "exhausted"
|
|
if sched.paused:
|
|
if self.schedule_store.get_candidate(sched.id) is not None:
|
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
|
consumed = self.schedule_store.get_consumed(sched.id) or EPOCH
|
|
self.schedule_store.save_consumed(sched.id, max(consumed, now))
|
|
return "paused"
|
|
# Fail closed on corrupt active views before calendar work (F5).
|
|
for run in self.run_store.list_runs():
|
|
if self._status_value(run) not in ("admitted", "interrupted"):
|
|
continue
|
|
try:
|
|
self.run_store.get_admission(run.id)
|
|
except KeyError:
|
|
reason = f"corrupt run view without admission: {run.id}"
|
|
sched.blocked_reason = reason
|
|
self.schedule_store.save_schedule(sched)
|
|
raise BlockedSchedule(reason) from None
|
|
try:
|
|
src = self.sources[sched.id]
|
|
except KeyError as exc:
|
|
raise InvalidScheduleDefinitionError(
|
|
f"no occurrence source for schedule {sched.id!r}"
|
|
) from exc
|
|
consumed = self.schedule_store.get_consumed(sched.id) or EPOCH
|
|
if consumed > now:
|
|
return "clock-rollback-held"
|
|
due: list[datetime] = []
|
|
cursor = consumed
|
|
jumped = False
|
|
while True:
|
|
nxt = src.next_after(cursor)
|
|
if nxt is None or nxt > now:
|
|
if (
|
|
not due
|
|
and _is_oneshot(src)
|
|
and nxt is None
|
|
and not sched.exhausted
|
|
and sched.misfire == "skip"
|
|
and getattr(src, "at", None) is not None
|
|
and getattr(src, "at") <= now
|
|
):
|
|
at = getattr(src, "at")
|
|
self._record(
|
|
kind="exhausted",
|
|
sched_id=sched.id,
|
|
intended=at,
|
|
reason="oneshot-expired-skip",
|
|
revision=sched.revision,
|
|
)
|
|
sched.exhausted = True
|
|
self.schedule_store.save_schedule(sched)
|
|
self.schedule_store.save_consumed(sched.id, now)
|
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
|
return "exhausted"
|
|
break
|
|
due.append(nxt)
|
|
cursor = nxt
|
|
if len(due) >= SCAN_CAP:
|
|
jumped = True
|
|
break
|
|
if jumped:
|
|
if sched.misfire == "latest":
|
|
from wf_scheduling.calendar import ScheduleExhaustedError
|
|
|
|
latest = src.prev_before(now)
|
|
if latest is None:
|
|
raise ScheduleExhaustedError(
|
|
f"latest-missed lookup exhausted for {sched.id!r}"
|
|
)
|
|
old = self.schedule_store.get_candidate(sched.id)
|
|
if old is not None and old.intended_at != latest:
|
|
self._record(
|
|
kind="superseded",
|
|
sched_id=sched.id,
|
|
intended=old.intended_at,
|
|
reason=f"coalesced-into:{latest.isoformat()}",
|
|
revision=sched.revision,
|
|
now=now,
|
|
)
|
|
self.schedule_store.save_candidate(
|
|
PendingCandidate(
|
|
schedule_id=sched.id,
|
|
intended_at=latest,
|
|
revision=sched.revision,
|
|
),
|
|
schedule_id=sched.id,
|
|
)
|
|
self._record(
|
|
kind="interval-summary",
|
|
sched_id=sched.id,
|
|
reason="coalesced-missed-span",
|
|
revision=sched.revision,
|
|
interval=(consumed, now),
|
|
count=-1,
|
|
now=now,
|
|
)
|
|
self.schedule_store.save_consumed(sched.id, now)
|
|
held = self._admit_held_candidate(sched, now)
|
|
if held is None:
|
|
# Terminal skip inside admission (e.g. overlap) clears the
|
|
# candidate; do not misreport it as held.
|
|
return "skipped-overlap"
|
|
return held
|
|
self._record(
|
|
kind="interval-summary",
|
|
sched_id=sched.id,
|
|
reason="skipped-missed-span",
|
|
revision=sched.revision,
|
|
interval=(consumed, now),
|
|
count=-1,
|
|
now=now,
|
|
)
|
|
self.schedule_store.save_consumed(sched.id, now)
|
|
return "span-skipped"
|
|
last_result = "idle"
|
|
for instant in due:
|
|
stored_consumed = self.schedule_store.get_consumed(sched.id) or EPOCH
|
|
if instant <= stored_consumed:
|
|
continue
|
|
age = (now - instant).total_seconds()
|
|
if age <= sched.lateness_allowance_s:
|
|
old = self.schedule_store.get_candidate(sched.id)
|
|
if old is not None and old.intended_at != instant:
|
|
self._record(
|
|
kind="superseded",
|
|
sched_id=sched.id,
|
|
intended=old.intended_at,
|
|
reason=f"admitted-newer:{instant.isoformat()}",
|
|
revision=sched.revision,
|
|
)
|
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
|
result = self._admit(sched, instant, now)
|
|
last_result = f"admit:{result}"
|
|
elif sched.misfire == "latest":
|
|
old = self.schedule_store.get_candidate(sched.id)
|
|
if old is not None and old.intended_at != instant:
|
|
self._record(
|
|
kind="superseded",
|
|
sched_id=sched.id,
|
|
intended=old.intended_at,
|
|
reason=f"coalesced-into:{instant.isoformat()}",
|
|
revision=sched.revision,
|
|
)
|
|
self.schedule_store.save_candidate(
|
|
PendingCandidate(
|
|
schedule_id=sched.id,
|
|
intended_at=instant,
|
|
revision=sched.revision,
|
|
),
|
|
schedule_id=sched.id,
|
|
)
|
|
self.schedule_store.save_consumed(sched.id, instant)
|
|
held = self._admit_held_candidate(sched, now)
|
|
if held is None:
|
|
last_result = "skipped-overlap"
|
|
elif held == "held-undecided":
|
|
last_result = "candidate-held"
|
|
else:
|
|
last_result = held
|
|
else:
|
|
if _is_oneshot(src) and not sched.exhausted:
|
|
self._record(
|
|
kind="exhausted",
|
|
sched_id=sched.id,
|
|
intended=instant,
|
|
reason="oneshot-expired-skip",
|
|
revision=sched.revision,
|
|
)
|
|
sched.exhausted = True
|
|
self.schedule_store.save_schedule(sched)
|
|
self.schedule_store.save_consumed(sched.id, instant)
|
|
last_result = "exhausted"
|
|
continue
|
|
self._record(
|
|
kind="skipped-misfire",
|
|
sched_id=sched.id,
|
|
intended=instant,
|
|
reason=f"age={age:.0f}s",
|
|
revision=sched.revision,
|
|
)
|
|
self.schedule_store.save_consumed(sched.id, instant)
|
|
last_result = "skipped-misfire"
|
|
if last_result in ("idle",):
|
|
cand = self.schedule_store.get_candidate(sched.id)
|
|
if cand is not None:
|
|
held = self._admit_held_candidate(sched, now)
|
|
if held is None:
|
|
last_result = "skipped-overlap"
|
|
elif held == "held-undecided":
|
|
last_result = "candidate-held"
|
|
else:
|
|
last_result = held
|
|
return last_result
|
|
|
|
def _admit_held_candidate(self, sched: Any, now: datetime) -> str | None:
|
|
cand = self.schedule_store.get_candidate(sched.id)
|
|
if cand is None or cand.revision != sched.revision:
|
|
if cand is not None and cand.revision != sched.revision:
|
|
self._record(
|
|
kind="superseded",
|
|
sched_id=sched.id,
|
|
intended=cand.intended_at,
|
|
reason="schedule-edit",
|
|
revision=sched.revision,
|
|
now=now,
|
|
)
|
|
self.schedule_store.save_candidate(None, schedule_id=sched.id)
|
|
return None
|
|
return self._admit(sched, cand.intended_at, now)
|
|
|
|
# -- administration ---------------------------------------------------
|
|
def resume_schedule(self, sid: str, now: datetime) -> None:
|
|
"""Unpause: resume selects the next future occurrence."""
|
|
sched = self.schedule_store.get_schedule(sid)
|
|
sched.paused = False
|
|
self.schedule_store.save_schedule(sched)
|
|
consumed = self.schedule_store.get_consumed(sid) or EPOCH
|
|
self.schedule_store.save_consumed(sid, max(consumed, now))
|
|
self.schedule_store.save_candidate(None, schedule_id=sid)
|
|
|
|
def edit_schedule(self, sid: str, now: datetime) -> None:
|
|
"""Definition edit: new revision, discard old candidates, no backfill."""
|
|
sched = self.schedule_store.get_schedule(sid)
|
|
sched.revision += 1
|
|
sched.updated_at = now
|
|
self.schedule_store.save_schedule(sched)
|
|
old = self.schedule_store.get_candidate(sid)
|
|
if old is not None:
|
|
self._record(
|
|
kind="superseded",
|
|
sched_id=sid,
|
|
intended=old.intended_at,
|
|
reason="schedule-edit",
|
|
revision=sched.revision,
|
|
)
|
|
self.schedule_store.save_candidate(None, schedule_id=sid)
|
|
consumed = self.schedule_store.get_consumed(sid) or EPOCH
|
|
self.schedule_store.save_consumed(sid, max(consumed, now))
|
|
|
|
|
|
def _test_environment(sched: Any) -> Any:
|
|
from wf_artifacts import PinnedRunEnvironment, WorkflowArtifact, WorkflowDeployment
|
|
|
|
deployment = WorkflowDeployment(
|
|
id=getattr(sched, "deployment_id", "dep-1"),
|
|
artifact_id="wf-1",
|
|
artifact_version=1,
|
|
bindings=[],
|
|
)
|
|
artifact = WorkflowArtifact(
|
|
id="wf-1",
|
|
version=1,
|
|
title="Wf-1",
|
|
input_schema={"type": "object", "properties": {}},
|
|
output_schema={"type": "object", "properties": {}},
|
|
outcomes=("ok",),
|
|
plan={"name": "wf-1", "nodes": [], "edges": []},
|
|
)
|
|
return PinnedRunEnvironment(
|
|
deployment=deployment, root_artifact=artifact, child_artifacts=[]
|
|
)
|
|
|
|
|
|
def _admission_intended(run_store: Any, run_id: str) -> datetime | None:
|
|
try:
|
|
return run_store.get_admission(run_id).scheduled_at
|
|
except KeyError:
|
|
return None
|