483 lines
20 KiB
Python
483 lines
20 KiB
Python
"""Schedule administration operations (T13).
|
|
|
|
Same-process admin surface over the file schedule store: create, inspect,
|
|
list, revision-checked update, pause, resume, delete, and paginated
|
|
occurrence inspection with a live pending-candidate synthesis.
|
|
|
|
There is deliberately no ownership requirement on these admin ops: this is
|
|
the same-process administration surface, the service observes fresh reads
|
|
from the store on every call, and second-process schedulers remain
|
|
unsupported per the store boundary (see ``wf_scheduling.ownership``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Callable
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from wf_core import RunLimits
|
|
from wf_scheduling.history import FileScheduleHistoryRecorder, HistoryEntry
|
|
from wf_scheduling.models import OccurrenceRecord, Schedule
|
|
from wf_scheduling.occurrences import occurrence_id
|
|
from wf_scheduling.prepare import PreparationRejected, SchedulePreparer
|
|
from wf_scheduling.store import ScheduleExistsError, StaleScheduleRevisionError
|
|
|
|
from .models import (
|
|
JsonProjector,
|
|
ListSchedulesResult,
|
|
OccurrencePage,
|
|
ScheduleResult,
|
|
)
|
|
from .operation_context import WorkflowOperationContext
|
|
from .run_lifecycle import create_pinned_environment
|
|
from .saved_subgraphs import resolve_saved_subgraph_tree
|
|
|
|
_PROJECT_SCHEDULE = JsonProjector(ScheduleResult)
|
|
_PROJECT_LIST_SCHEDULES = JsonProjector(ListSchedulesResult)
|
|
_PROJECT_OCCURRENCE_PAGE = JsonProjector(OccurrencePage)
|
|
|
|
|
|
class _ContextDeploymentDirectory:
|
|
"""Deployment contract source over the API artifact store.
|
|
|
|
Mirrors ``StoreDeploymentDirectory`` (``wf_scheduling.lifecycle``)
|
|
without importing the server-lifecycle layer: the deployment revision
|
|
plus the required keys of the pinned root artifact's input schema.
|
|
Unknown ids raise ``KeyError``.
|
|
"""
|
|
|
|
def __init__(self, artifact_store: Any) -> None:
|
|
self._artifact_store = artifact_store
|
|
|
|
def deployment_revision(self, deployment_id: str) -> int:
|
|
return int(self._artifact_store.get_deployment(deployment_id).revision)
|
|
|
|
def required_inputs(self, deployment_id: str) -> list[str]:
|
|
deployment = self._artifact_store.get_deployment(deployment_id)
|
|
artifact = self._artifact_store.get_artifact(
|
|
deployment.artifact_id, deployment.artifact_version
|
|
)
|
|
schema = getattr(artifact, "input_schema", None)
|
|
if not isinstance(schema, dict):
|
|
return []
|
|
required = schema.get("required", [])
|
|
if not isinstance(required, list):
|
|
return []
|
|
return [key for key in required if isinstance(key, str)]
|
|
|
|
|
|
def _build_pinned_environment(artifact_store: Any) -> Callable[[Any], Any]:
|
|
"""Pin deployment + root artifact + subgraph tree for sample validation."""
|
|
|
|
def build(sched: Any) -> Any:
|
|
deployment = artifact_store.get_deployment(sched.deployment_id)
|
|
artifact = artifact_store.get_artifact(
|
|
deployment.artifact_id, deployment.artifact_version
|
|
)
|
|
tree = resolve_saved_subgraph_tree(
|
|
root_artifact=artifact, artifact_store=artifact_store
|
|
)
|
|
return create_pinned_environment(
|
|
deployment=deployment, artifact=artifact, tree=tree
|
|
)
|
|
|
|
return build
|
|
|
|
|
|
class WorkflowScheduleApi:
|
|
"""Schedule administration over the configured schedule store.
|
|
|
|
The store resolves from the operation context (or from an explicitly
|
|
passed store, which wins); every method raises ``KeyError`` when no
|
|
schedule store is configured, exactly like runs with a missing store.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
context: WorkflowOperationContext,
|
|
*,
|
|
schedule_store: Any | None = None,
|
|
) -> None:
|
|
self.context = context
|
|
self._explicit_schedule_store = schedule_store
|
|
|
|
def _schedule_store(self) -> Any:
|
|
store = self._explicit_schedule_store
|
|
if store is None:
|
|
store = getattr(self.context, "schedule_store", None)
|
|
if store is None:
|
|
raise KeyError("workflow schedule store is not configured")
|
|
return store
|
|
|
|
def bind_schedule_store(self, store: Any) -> None:
|
|
"""Bind the composition-owned schedule store exactly once.
|
|
|
|
Server composition uses this seam when the scheduler is enabled on a
|
|
server that was built without the optional schedule surface. Refuse
|
|
to replace an already configured store: API and scheduler writes must
|
|
remain on one same-process lock and one durable root.
|
|
"""
|
|
if self._explicit_schedule_store is not None:
|
|
if self._explicit_schedule_store is not store:
|
|
raise ValueError("workflow schedule store is already configured")
|
|
return
|
|
context_store = getattr(self.context, "schedule_store", None)
|
|
if context_store is not None and context_store is not store:
|
|
raise ValueError("workflow schedule store is already configured")
|
|
self._explicit_schedule_store = store
|
|
|
|
def _artifact_store(self) -> Any:
|
|
if self.context.artifact_store is None:
|
|
raise KeyError("workflow artifact store is not configured")
|
|
return self.context.artifact_store
|
|
|
|
@staticmethod
|
|
def _check_trigger(schedule: Schedule) -> None:
|
|
"""Reject triggers the poll loop could not build a source for."""
|
|
from wf_scheduling.poll import source_for_trigger
|
|
|
|
try:
|
|
source_for_trigger(schedule.trigger)
|
|
except Exception as exc:
|
|
raise ValueError(f"invalid schedule trigger: {exc}") from exc
|
|
|
|
@staticmethod
|
|
def _check_max_steps(max_steps: int | None) -> None:
|
|
"""Reuse the manual-run step-budget validator for schedules."""
|
|
if max_steps is None:
|
|
return
|
|
try:
|
|
RunLimits(max_steps=max_steps)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError(f"invalid max_steps {max_steps!r}: {exc}") from exc
|
|
|
|
def _validate_sample(self, schedule: Schedule) -> None:
|
|
"""Resolve the bindings against a synthetic occurrence and validate.
|
|
|
|
A real ``SchedulePreparer`` wired to the context artifact store
|
|
resolves the schedule's input bindings for the current instant and
|
|
validates the result against the pinned root artifact's input
|
|
schema, so a schedule cannot be saved when its very first
|
|
occurrence would fail preflight. Failures raise ``ValueError``.
|
|
"""
|
|
artifact_store = self._artifact_store()
|
|
preparer = SchedulePreparer(
|
|
_ContextDeploymentDirectory(artifact_store),
|
|
_build_pinned_environment(artifact_store),
|
|
)
|
|
now = datetime.now(UTC)
|
|
result = preparer.prepare(sched=schedule, intended=now, now=now)
|
|
if isinstance(result, PreparationRejected):
|
|
raise ValueError(f"schedule sample occurrence rejected: {result.reason}")
|
|
|
|
def _validate_definition(self, schedule: Schedule) -> None:
|
|
"""Run the trigger/budget/sample checks shared by create and update."""
|
|
self._check_max_steps(schedule.max_steps)
|
|
self._check_trigger(schedule)
|
|
self._validate_sample(schedule)
|
|
|
|
async def create_schedule(
|
|
self,
|
|
*,
|
|
schedule_id: str,
|
|
deployment_id: str,
|
|
trigger: dict[str, Any],
|
|
input_bindings: list[dict[str, Any]] | None = None,
|
|
overlap: str = "skip",
|
|
misfire: str = "skip",
|
|
max_active_runs: int = 1,
|
|
lateness_allowance_s: float = 60.0,
|
|
max_steps: int | None = None,
|
|
enabled: bool = True,
|
|
) -> ScheduleResult:
|
|
"""Create one schedule after deployment, trigger, and sample checks.
|
|
|
|
Unknown deployments raise ``KeyError``; bad triggers, binding
|
|
shapes, budgets, and sample-occurrence rejections raise
|
|
``ValueError``; duplicate ids (deleted ids are never reusable)
|
|
raise ``ScheduleExistsError``.
|
|
"""
|
|
store = self._schedule_store()
|
|
now = datetime.now(UTC)
|
|
# KeyError first: the deployment must exist before any validation.
|
|
self._artifact_store().get_deployment(deployment_id)
|
|
schedule = Schedule.model_validate(
|
|
{
|
|
"id": schedule_id,
|
|
"deployment_id": deployment_id,
|
|
"trigger": trigger,
|
|
"input_bindings": input_bindings if input_bindings is not None else [],
|
|
"max_steps": max_steps,
|
|
"overlap": overlap,
|
|
"misfire": misfire,
|
|
"max_active_runs": max_active_runs,
|
|
"lateness_allowance_s": lateness_allowance_s,
|
|
"revision": 1,
|
|
"enabled": enabled,
|
|
"paused": False,
|
|
"deleted": False,
|
|
"exhausted": False,
|
|
"blocked_reason": None,
|
|
"created_at": now,
|
|
"updated_at": now,
|
|
}
|
|
)
|
|
self._validate_definition(schedule)
|
|
# Creation never backfills time before the revision: the consumed
|
|
# watermark starts at creation so catch-up only ever covers downtime
|
|
# after this revision, never pre-creation instants. Check the id
|
|
# before this write so a duplicate create cannot overwrite the
|
|
# existing schedule's watermark; publishing the schedule last keeps
|
|
# a failed watermark write invisible to the poller.
|
|
try:
|
|
store.get_schedule(schedule_id)
|
|
except KeyError:
|
|
pass
|
|
else:
|
|
raise ScheduleExistsError(f"schedule id already exists: {schedule_id!r}")
|
|
store.save_consumed(schedule_id, now)
|
|
stored = store.create_schedule(schedule)
|
|
return _PROJECT_SCHEDULE(stored.model_dump(mode="json"))
|
|
|
|
async def get_schedule(self, *, schedule_id: str) -> ScheduleResult:
|
|
"""Return one schedule payload; unknown ids raise ``KeyError``."""
|
|
schedule = self._schedule_store().get_schedule(schedule_id)
|
|
return _PROJECT_SCHEDULE(schedule.model_dump(mode="json"))
|
|
|
|
async def list_schedules(
|
|
self, *, include_deleted: bool = False
|
|
) -> ListSchedulesResult:
|
|
"""Return schedule payloads ordered by id (deleted excluded by default)."""
|
|
schedules = self._schedule_store().list_schedules(
|
|
include_deleted=include_deleted
|
|
)
|
|
return _PROJECT_LIST_SCHEDULES(
|
|
{
|
|
"schedules": [
|
|
_PROJECT_SCHEDULE(item.model_dump(mode="json"))
|
|
for item in schedules
|
|
]
|
|
}
|
|
)
|
|
|
|
async def update_schedule(
|
|
self,
|
|
*,
|
|
schedule_id: str,
|
|
expected_revision: int,
|
|
deployment_id: str | None = None,
|
|
trigger: dict[str, Any] | None = None,
|
|
input_bindings: list[dict[str, Any]] | None = None,
|
|
overlap: str | None = None,
|
|
misfire: str | None = None,
|
|
max_active_runs: int | None = None,
|
|
lateness_allowance_s: float | None = None,
|
|
max_steps: int | None = None,
|
|
enabled: bool | None = None,
|
|
) -> ScheduleResult:
|
|
"""Apply a partial edit under an optimistic revision check.
|
|
|
|
Only provided fields change; ``revision`` becomes
|
|
``expected_revision + 1`` and ``updated_at`` becomes now, while
|
|
``created_at`` and the admitted-run snapshots stay untouched (later
|
|
edits only affect future admissions). Stale revisions raise
|
|
``StaleScheduleRevisionError`` before any write. Crash-safe
|
|
ordering: the candidate is cleared (with a ``superseded`` /
|
|
``schedule-edit`` history row) and the consumed watermark advances
|
|
BEFORE the revision bump is persisted, so a crash can only leave
|
|
the edit unapplied (safe over-skip under the old revision, freely
|
|
retryable) and never a bumped revision that backfills pre-edit
|
|
instants on restart. A concurrent edit that commits first still
|
|
wins via the store's authoritative revision check; our already
|
|
applied watermark advance is a safe over-skip in that case too.
|
|
"""
|
|
store = self._schedule_store()
|
|
now = datetime.now(UTC)
|
|
current = store.get_schedule(schedule_id)
|
|
if current.revision != expected_revision:
|
|
raise StaleScheduleRevisionError(
|
|
f"stale schedule revision for {schedule_id!r}: "
|
|
f"expected {expected_revision}, found {current.revision}"
|
|
)
|
|
data = current.model_dump(mode="python")
|
|
if deployment_id is not None:
|
|
data["deployment_id"] = deployment_id
|
|
if trigger is not None:
|
|
data["trigger"] = trigger
|
|
if input_bindings is not None:
|
|
data["input_bindings"] = input_bindings
|
|
if overlap is not None:
|
|
data["overlap"] = overlap
|
|
if misfire is not None:
|
|
data["misfire"] = misfire
|
|
if max_active_runs is not None:
|
|
data["max_active_runs"] = max_active_runs
|
|
if lateness_allowance_s is not None:
|
|
data["lateness_allowance_s"] = lateness_allowance_s
|
|
if max_steps is not None:
|
|
data["max_steps"] = max_steps
|
|
if enabled is not None:
|
|
data["enabled"] = enabled
|
|
data["revision"] = expected_revision + 1
|
|
data["updated_at"] = now
|
|
# KeyError first, like create: the (possibly repointed) deployment
|
|
# must exist before definition validation.
|
|
self._artifact_store().get_deployment(data["deployment_id"])
|
|
updated = Schedule.model_validate(data)
|
|
self._validate_definition(updated)
|
|
# Crash-safe ordering: discard the old revision's unadmitted work
|
|
# and advance the watermark BEFORE the revision bump below. A
|
|
# crash here leaves the edit unapplied under the old revision
|
|
# (retryable); the bumped revision can never observe pre-edit
|
|
# instants on restart.
|
|
old_candidate = store.get_candidate(schedule_id)
|
|
if old_candidate is not None:
|
|
FileScheduleHistoryRecorder(store).record(
|
|
HistoryEntry(
|
|
schedule_id=schedule_id,
|
|
kind="superseded",
|
|
resolved_at=old_candidate.intended_at,
|
|
revision=expected_revision + 1,
|
|
reason="schedule-edit",
|
|
created_at=now,
|
|
)
|
|
)
|
|
store.save_candidate(None, schedule_id=schedule_id)
|
|
consumed = store.get_consumed(schedule_id)
|
|
store.save_consumed(
|
|
schedule_id, max(consumed, now) if consumed is not None else now
|
|
)
|
|
stored = store.update_schedule(updated, expected_revision=expected_revision)
|
|
return _PROJECT_SCHEDULE(stored.model_dump(mode="json"))
|
|
|
|
async def pause_schedule(self, *, schedule_id: str) -> ScheduleResult:
|
|
"""Pause one schedule (mirror the poll-loop paused branch).
|
|
|
|
Sets ``paused``, clears any held candidate, and advances the
|
|
consumed watermark to at least now so the paused span is never
|
|
backfilled on resume. No history row is written. Crash-safe
|
|
ordering: the candidate is cleared and the watermark advances
|
|
BEFORE the flag flip is persisted, so a crash can only leave the
|
|
schedule unpaused (retryable) and never a paused flag whose span
|
|
backfills on resume.
|
|
"""
|
|
store = self._schedule_store()
|
|
now = datetime.now(UTC)
|
|
# Existence first: unknown ids raise KeyError before any write.
|
|
store.get_schedule(schedule_id)
|
|
store.save_candidate(None, schedule_id=schedule_id)
|
|
consumed = store.get_consumed(schedule_id)
|
|
store.save_consumed(
|
|
schedule_id, max(consumed, now) if consumed is not None else now
|
|
)
|
|
schedule = store.get_schedule(schedule_id)
|
|
schedule.paused = True
|
|
store.save_schedule(schedule)
|
|
return _PROJECT_SCHEDULE(schedule.model_dump(mode="json"))
|
|
|
|
async def resume_schedule(self, *, schedule_id: str) -> ScheduleResult:
|
|
"""Resume one schedule (mirror ``Scheduler.resume_schedule``).
|
|
|
|
Clears ``paused`` (resume selects the next future occurrence),
|
|
clears any held candidate, and advances the consumed watermark to
|
|
at least now. No history row is written. Crash-safe ordering like
|
|
pause: candidate and watermark first, flag flip last.
|
|
"""
|
|
store = self._schedule_store()
|
|
now = datetime.now(UTC)
|
|
# Existence first: unknown ids raise KeyError before any write.
|
|
store.get_schedule(schedule_id)
|
|
store.save_candidate(None, schedule_id=schedule_id)
|
|
consumed = store.get_consumed(schedule_id)
|
|
store.save_consumed(
|
|
schedule_id, max(consumed, now) if consumed is not None else now
|
|
)
|
|
schedule = store.get_schedule(schedule_id)
|
|
schedule.paused = False
|
|
store.save_schedule(schedule)
|
|
return _PROJECT_SCHEDULE(schedule.model_dump(mode="json"))
|
|
|
|
async def delete_schedule(self, *, schedule_id: str) -> ScheduleResult:
|
|
"""Soft-delete one schedule (mirror the poll-loop deleted branch).
|
|
|
|
Clears any held candidate, then sets ``deleted``. Runs and history
|
|
are untouched, the consumed watermark does not advance, and the id
|
|
stays reserved (re-creation is rejected by create). Either crash
|
|
half heals: an unclearable candidate on a deleted schedule is
|
|
dropped by the poll loop, and a cleared candidate on a live
|
|
schedule is rebuilt from the untouched watermark.
|
|
"""
|
|
store = self._schedule_store()
|
|
# Existence first: unknown ids raise KeyError before any write.
|
|
store.get_schedule(schedule_id)
|
|
store.save_candidate(None, schedule_id=schedule_id)
|
|
schedule = store.get_schedule(schedule_id)
|
|
schedule.deleted = True
|
|
store.save_schedule(schedule)
|
|
store.save_candidate(None, schedule_id=schedule_id)
|
|
return _PROJECT_SCHEDULE(schedule.model_dump(mode="json"))
|
|
|
|
async def list_schedule_occurrences(
|
|
self,
|
|
*,
|
|
schedule_id: str,
|
|
cursor: str | None = None,
|
|
limit: int = 50,
|
|
) -> OccurrencePage:
|
|
"""Return one occurrence-history page with a live pending synthesis.
|
|
|
|
The stored page keeps the schedule-store shape (``occurrences``,
|
|
``total``, ``cursor``, ``next_cursor``, ``limit``). When a
|
|
candidate is currently held AND this is the first page (``cursor``
|
|
is ``None`` or the legacy first-page offset ``"0"``), one
|
|
synthesized ``pending`` row for that candidate
|
|
is prepended: ``occurrence_id`` derives from
|
|
``(schedule_id, intended instant)``, ``resolved_at`` is the
|
|
intended instant, ``revision`` is the candidate revision, and
|
|
``reason`` is empty; ``total`` grows by one. Later pages never
|
|
carry the row (a repeat fetch after the candidate admits therefore
|
|
shows the durable ``admitted`` entry instead of a duplicate
|
|
pending projection).
|
|
"""
|
|
store = self._schedule_store()
|
|
# KeyError first: an unknown schedule must not leak pagination.
|
|
store.get_schedule(schedule_id)
|
|
if limit < 1 or limit > 100:
|
|
raise ValueError("limit must be between 1 and 100")
|
|
page = store.list_occurrences(schedule_id, cursor=cursor, limit=limit)
|
|
candidate = store.get_candidate(schedule_id)
|
|
if candidate is not None and cursor in (None, "0"):
|
|
now = datetime.now(UTC)
|
|
intended = candidate.intended_at
|
|
pending = OccurrenceRecord(
|
|
schedule_id=schedule_id,
|
|
occurrence_id=occurrence_id(schedule_id, intended),
|
|
kind="pending",
|
|
resolved_at=intended,
|
|
run_id=None,
|
|
revision=candidate.revision,
|
|
reason="",
|
|
admitted_at=None,
|
|
started_at=None,
|
|
checkpoint_id=None,
|
|
interval_start=None,
|
|
interval_end=None,
|
|
interval_count=0,
|
|
created_at=now,
|
|
)
|
|
page = {
|
|
"occurrences": [
|
|
pending.model_dump(mode="json"),
|
|
*page["occurrences"],
|
|
],
|
|
"total": page["total"] + 1,
|
|
"cursor": page["cursor"],
|
|
"next_cursor": page["next_cursor"],
|
|
"limit": page["limit"],
|
|
}
|
|
return _PROJECT_OCCURRENCE_PAGE(page)
|
|
|
|
|
|
__all__ = ["WorkflowScheduleApi"]
|