sched: typed history recorder with exactly-once terminal reconcile (F4)
This commit is contained in:
+184
-69
@@ -36,6 +36,11 @@ from __future__ import annotations
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from wf_scheduling.history import (
|
||||
FileScheduleHistoryRecorder,
|
||||
HistoryEntry,
|
||||
HistoryRecorder,
|
||||
)
|
||||
from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
|
||||
|
||||
UTC = timezone.utc
|
||||
@@ -60,7 +65,7 @@ def recover(
|
||||
run_store: Any,
|
||||
now: datetime,
|
||||
ownership: SchedulerOwnership,
|
||||
record_history: Any | None = None,
|
||||
history: HistoryRecorder | None = None,
|
||||
) -> list[str]:
|
||||
"""Reconcile durable state after a restart without executing work."""
|
||||
from wf_artifacts.runs.models import StoredRunStatus
|
||||
@@ -70,6 +75,8 @@ def recover(
|
||||
"scheduler ownership is required before recovery: an unowned "
|
||||
"recovery could abandon or redispatch another owner's work"
|
||||
)
|
||||
if history is None:
|
||||
history = FileScheduleHistoryRecorder(schedule_store)
|
||||
diags: list[str] = []
|
||||
# Admission record is the recovery authority: admitted but never
|
||||
# materialized views are completed here and flagged pending for the
|
||||
@@ -102,40 +109,51 @@ def recover(
|
||||
clear_pending(run_store, run.id)
|
||||
diags.append(f"{run.id}:completion-window-cleared")
|
||||
if status == StoredRunStatus.INTERRUPTED.value or status == "interrupted":
|
||||
checkpoint_id, result_attempt = _latest_result(run_store, run.id)
|
||||
sched_id, intended, revision = _attribution(run_store, run.id)
|
||||
if active:
|
||||
assert attempt is not None
|
||||
try:
|
||||
latest = run_store.get_latest_checkpoint(run.id)
|
||||
result_attempt = latest.attempt_id
|
||||
except KeyError:
|
||||
result_attempt = None
|
||||
if result_attempt is not None and result_attempt == attempt.attempt_id:
|
||||
from wf_artifacts.runs.models import ResumeAttempt
|
||||
|
||||
run_store.save_resume_attempt(
|
||||
ResumeAttempt(
|
||||
run_id=run.id,
|
||||
attempt_id=attempt.attempt_id,
|
||||
state="DONE",
|
||||
created_at=attempt.created_at,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
_complete_attempt(run_store, run.id, attempt, now)
|
||||
diags.append(f"{run.id}:fresh-result-resumable")
|
||||
if _reconcile_terminal(
|
||||
history,
|
||||
sched_id,
|
||||
run.id,
|
||||
"interrupted",
|
||||
checkpoint_id,
|
||||
intended,
|
||||
revision,
|
||||
"fresh-result",
|
||||
now,
|
||||
):
|
||||
diags.append(f"{run.id}:terminal-reconciled")
|
||||
else:
|
||||
_fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
|
||||
_fail_run(run_store, run, AMBIGUOUS_REASON, now, history)
|
||||
diags.append(f"{run.id}:failed-closed")
|
||||
else:
|
||||
diags.append(f"{run.id}:waiting-resumable")
|
||||
if _reconcile_terminal(
|
||||
history,
|
||||
sched_id,
|
||||
run.id,
|
||||
"interrupted",
|
||||
checkpoint_id,
|
||||
intended,
|
||||
revision,
|
||||
"reconciled-on-recovery",
|
||||
now,
|
||||
):
|
||||
diags.append(f"{run.id}:terminal-reconciled")
|
||||
elif status == StoredRunStatus.ADMITTED.value or status == "admitted":
|
||||
if is_executing(run_store, run.id):
|
||||
# The executor may already have produced external effects:
|
||||
# abandon, never retry. Both markers are cleared so a later
|
||||
# recovery does not re-fail the now-terminal run.
|
||||
if active:
|
||||
_fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
|
||||
_fail_run(run_store, run, AMBIGUOUS_REASON, now, history)
|
||||
else:
|
||||
_fail_run(run_store, run, ABANDONED_REASON, now, record_history)
|
||||
_fail_run(run_store, run, ABANDONED_REASON, now, history)
|
||||
clear_pending(run_store, run.id)
|
||||
clear_executing(run_store, run.id)
|
||||
diags.append(f"{run.id}:failed-closed")
|
||||
@@ -151,80 +169,177 @@ def recover(
|
||||
diags.append(f"{run.id}:corrupt-blocked")
|
||||
continue
|
||||
if active:
|
||||
_fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
|
||||
_fail_run(run_store, run, AMBIGUOUS_REASON, now, history)
|
||||
else:
|
||||
_fail_run(run_store, run, ABANDONED_REASON, now, record_history)
|
||||
_fail_run(run_store, run, ABANDONED_REASON, now, history)
|
||||
diags.append(f"{run.id}:failed-closed")
|
||||
elif status == StoredRunStatus.COMPLETED.value or status == "completed":
|
||||
checkpoint_id, completed_attempt = _latest_result(run_store, run.id)
|
||||
sched_id, intended, revision = _attribution(run_store, run.id)
|
||||
if active:
|
||||
assert attempt is not None
|
||||
try:
|
||||
latest = run_store.get_latest_checkpoint(run.id)
|
||||
completed_attempt = latest.attempt_id
|
||||
except KeyError:
|
||||
completed_attempt = None
|
||||
if (
|
||||
completed_attempt is not None
|
||||
and completed_attempt == attempt.attempt_id
|
||||
):
|
||||
from wf_artifacts.runs.models import ResumeAttempt
|
||||
|
||||
run_store.save_resume_attempt(
|
||||
ResumeAttempt(
|
||||
run_id=run.id,
|
||||
attempt_id=attempt.attempt_id,
|
||||
state="DONE",
|
||||
created_at=attempt.created_at,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
_complete_attempt(run_store, run.id, attempt, now)
|
||||
diags.append(f"{run.id}:attempt-reconciled")
|
||||
else:
|
||||
_fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
|
||||
_fail_run(run_store, run, AMBIGUOUS_REASON, now, history)
|
||||
diags.append(f"{run.id}:failed-closed")
|
||||
continue
|
||||
if record_history is not None and not _has_terminal(
|
||||
schedule_store, run.id, "completed"
|
||||
if _reconcile_terminal(
|
||||
history,
|
||||
sched_id,
|
||||
run.id,
|
||||
"completed",
|
||||
checkpoint_id,
|
||||
intended,
|
||||
revision,
|
||||
"reconciled-on-recovery",
|
||||
now,
|
||||
):
|
||||
diags.append(f"{run.id}:terminal-reconciled")
|
||||
elif status == StoredRunStatus.FAILED.value or status == "failed":
|
||||
sched_id, intended, revision = _attribution(run_store, run.id)
|
||||
if _reconcile_terminal(
|
||||
history,
|
||||
sched_id,
|
||||
run.id,
|
||||
"failed",
|
||||
run.latest_checkpoint_id,
|
||||
intended,
|
||||
revision,
|
||||
"reconciled-on-recovery",
|
||||
now,
|
||||
):
|
||||
record_history(
|
||||
kind="completed",
|
||||
run_id=run.id,
|
||||
reason="reconciled-on-recovery",
|
||||
)
|
||||
diags.append(f"{run.id}:terminal-reconciled")
|
||||
return diags
|
||||
|
||||
|
||||
def _attribution(
|
||||
run_store: Any, run_id: str
|
||||
) -> tuple[str | None, datetime | None, int | None]:
|
||||
"""Return ``(schedule_id, scheduled_at, schedule_revision)`` for a run.
|
||||
|
||||
``(None, None, None)`` when no admission owns the run: schedule history
|
||||
cannot attribute it, so callers persist the reason on the run itself.
|
||||
"""
|
||||
try:
|
||||
admission = run_store.get_admission(run_id)
|
||||
except KeyError:
|
||||
return None, None, None
|
||||
return (
|
||||
admission.schedule_id,
|
||||
admission.scheduled_at,
|
||||
admission.schedule_revision,
|
||||
)
|
||||
|
||||
|
||||
def _latest_result(run_store: Any, run_id: str) -> tuple[str | None, Any]:
|
||||
"""Return ``(checkpoint_id, attempt_id)`` of the latest stopped checkpoint."""
|
||||
try:
|
||||
latest = run_store.get_latest_checkpoint(run_id)
|
||||
except KeyError:
|
||||
return None, None
|
||||
return latest.id, latest.attempt_id
|
||||
|
||||
|
||||
def _complete_attempt(run_store: Any, run_id: str, attempt: Any, now: datetime) -> None:
|
||||
"""Mark the ACTIVE attempt DONE after its fresh result was recognized."""
|
||||
from wf_artifacts.runs.models import ResumeAttempt
|
||||
|
||||
run_store.save_resume_attempt(
|
||||
ResumeAttempt(
|
||||
run_id=run_id,
|
||||
attempt_id=attempt.attempt_id,
|
||||
state="DONE",
|
||||
created_at=attempt.created_at,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _reconcile_terminal(
|
||||
history: HistoryRecorder,
|
||||
sched_id: str | None,
|
||||
run_id: str,
|
||||
kind: str,
|
||||
checkpoint_id: str | None,
|
||||
intended: datetime | None,
|
||||
revision: int | None,
|
||||
reason: str,
|
||||
now: datetime,
|
||||
) -> bool:
|
||||
"""Record one stopped-result entry unless this exact result is known.
|
||||
|
||||
Returns whether an entry was appended. Idempotency identity is
|
||||
``(run_id, kind, checkpoint_id)``: repeating recovery never duplicates
|
||||
an entry, while a resumed run that stops again (new checkpoint id)
|
||||
records a new one.
|
||||
"""
|
||||
if sched_id is None:
|
||||
return False
|
||||
if history.has_terminal(sched_id, run_id, kind, checkpoint_id):
|
||||
return False
|
||||
history.record(
|
||||
HistoryEntry(
|
||||
schedule_id=sched_id,
|
||||
kind=kind, # type: ignore[arg-type]
|
||||
resolved_at=intended,
|
||||
run_id=run_id,
|
||||
revision=revision,
|
||||
reason=reason,
|
||||
checkpoint_id=checkpoint_id,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _fail_run(
|
||||
run_store: Any, run: Any, reason: str, now: datetime, record_history: Any | None
|
||||
run_store: Any,
|
||||
run: Any,
|
||||
reason: str,
|
||||
now: datetime,
|
||||
history: HistoryRecorder,
|
||||
) -> None:
|
||||
"""Fail a run closed with its truthful reason durably preserved.
|
||||
|
||||
The reason is always written into the run summary diagnostics; when an
|
||||
admission attributes the run to a schedule, a failed occurrence entry is
|
||||
reconciled exactly once through the history recorder.
|
||||
"""
|
||||
from wf_artifacts import DependencyDiagnostic, DiagnosticSeverity
|
||||
from wf_artifacts.runs.models import StoredRunStatus
|
||||
|
||||
diagnostic = DependencyDiagnostic(
|
||||
severity=DiagnosticSeverity.ERROR,
|
||||
code="schedule-recovery",
|
||||
logical_ref=run.id,
|
||||
message=reason,
|
||||
repair_hint=None,
|
||||
)
|
||||
updated = run.model_copy(
|
||||
update={"status": StoredRunStatus.FAILED, "updated_at": now}
|
||||
update={
|
||||
"status": StoredRunStatus.FAILED,
|
||||
"updated_at": now,
|
||||
"diagnostics": [*run.diagnostics, diagnostic],
|
||||
}
|
||||
)
|
||||
run_store.save_run(updated)
|
||||
if record_history is not None:
|
||||
try:
|
||||
admission = run_store.get_admission(run.id)
|
||||
sched_id = admission.schedule_id or ""
|
||||
intended = admission.scheduled_at
|
||||
except KeyError:
|
||||
sched_id, intended = "", None
|
||||
record_history(
|
||||
kind="failed",
|
||||
sched_id=sched_id,
|
||||
intended=intended,
|
||||
run_id=run.id,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
|
||||
def _has_terminal(schedule_store: Any, run_id: str, kind: str) -> bool:
|
||||
# Scheduler history lives per schedule; without a schedule index, skip
|
||||
# dedup here (poll reconciliation in T08 already guards re-admission via
|
||||
# consumed watermarks). Kept as a seam for T13 inspection.
|
||||
return False
|
||||
sched_id, intended, revision = _attribution(run_store, run.id)
|
||||
_reconcile_terminal(
|
||||
history,
|
||||
sched_id,
|
||||
run.id,
|
||||
"failed",
|
||||
run.latest_checkpoint_id,
|
||||
intended,
|
||||
revision,
|
||||
reason,
|
||||
now,
|
||||
)
|
||||
|
||||
|
||||
def _mark_pending(run_store: Any, run_id: str) -> None:
|
||||
|
||||
Reference in New Issue
Block a user