lda-wf/src/wf_scheduling/recovery.py

"""Startup recovery and reconciliation (T10).

Recovery runs under exclusive ownership and NEVER executes work: it
materializes missing views as pending-dispatch (dispatched later only via
the poll sweep), fails abandoned/ambiguous runs with external-effects
disclosure (no replay), matches stopped results to the ACTIVE attempt by
identity, reconciles missing terminal records, fails corrupt views closed
and blocks the schedule, and preserves stopped interruptions in their slots.

Restart behavior at each marker/admission/view write boundary (admission
persist → pending mark → view materialize → executing mark → pending
clear → executor → stopped persist → executing clear → history):

- admission only: view is materialized and flagged pending-dispatch; the
  occurrence was already consumed, so the later poll dispatches exactly
  once through capacity checks.
- admission + pending, no view: same as above (the view write was lost).
- admission + view + pending, no executing mark: provably undispatched
  (the executor is unreachable without the executing mark); kept pending.
- any admitted run with the executing mark: the executor may already have
  produced external effects; failed (ACTIVE attempt: ambiguous, else
  abandoned) with external-effects disclosure, never redispatched.
- admitted + view with neither mark and no ACTIVE attempt: a durable
  stopped checkpoint reconciles the summary first (checkpoint-first rule);
  only when no checkpoint exists is the outcome unprovable, and the run is
  failed closed without replay (current code never produces this shape on
  crash).
- stopped summary + executing/pending marks: completion-window leftovers
  (stopped result persisted, marker clearing lost); markers are cleared,
  the stopped status stands, nothing is re-executed. Checkpoint-vs-summary
  reconciliation across the torn ``save_checkpoint``/``save_run`` boundary
  is owned by the F2 reconcile step, which runs before marker handling.
- stopped summary, no marks: existing terminal/attempt reconciliation.
"""

from __future__ import annotations

from datetime import datetime, timezone
from typing import Any

from wf_scheduling.history import (
    FileScheduleHistoryRecorder,
    HistoryEntry,
    HistoryRecorder,
)
from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError

UTC = timezone.utc

ABANDONED_REASON = (
    "abandoned execution: outcome unknown; external effects may already "
    "have occurred; no replay"
)
AMBIGUOUS_REASON = (
    "ambiguous resume attempt: may have executed; external effects may "
    "already have occurred; no retry"
)
CORRUPT_PENDING_REASON = (
    "corrupt pending marker: no admission owns this run; the marker cannot "
    "be trusted for dispatch"
)


def recover(
    *,
    schedule_store: Any,
    run_store: Any,
    now: datetime,
    ownership: SchedulerOwnership,
    history: HistoryRecorder | None = None,
) -> list[str]:
    """Reconcile durable state after a restart without executing work."""
    from wf_artifacts.runs.models import StoredRunStatus

    if ownership is None or not ownership.held:
        raise SecondOwnerError(
            "scheduler ownership is required before recovery: an unowned "
            "recovery could abandon or redispatch another owner's work"
        )
    if history is None:
        history = FileScheduleHistoryRecorder(schedule_store)
    diags: list[str] = []
    # Admission record is the recovery authority: admitted but never
    # materialized views are completed here and flagged pending for the
    # capacity-checked poll sweep (exactly once, occurrence already consumed).
    for admission in run_store.list_admissions():
        try:
            run_store.get_run(admission.id)
        except KeyError:
            from wf_api.run_lifecycle import materialize_admitted_view

            materialize_admitted_view(store=run_store, admission=admission)
            _mark_pending(run_store, admission.id)
            diags.append(f"{admission.id}:view-completed-pending-dispatch")
    for run in run_store.list_runs():
        status = getattr(run.status, "value", run.status)
        attempt = run_store.get_resume_attempt(run.id)
        active = attempt is not None and attempt.state == "ACTIVE"
        # Checkpoint-first reconcile across the torn
        # save_checkpoint/save_run boundary: a durable stopped checkpoint
        # wins over a stale summary (status, checkpoint pointer, and
        # readiness are rewritten together from the checkpoint's own
        # persisted fields; nothing is fabricated). Attempt matching below
        # still distinguishes fresh results from stale ones.
        try:
            changed, status = _reconcile_summary_from_checkpoint(run_store, run, now)
        except (ValueError, OSError) as exc:
            _fail_run(run_store, run, f"corrupt checkpoint: {exc}", now, history)
            diags.append(f"{run.id}:failed-closed")
            continue
        if changed:
            run = run_store.get_run(run.id)
            diags.append(f"{run.id}:summary-reconciled")
        if status in (
            StoredRunStatus.INTERRUPTED.value,
            StoredRunStatus.COMPLETED.value,
            StoredRunStatus.FAILED.value,
            "interrupted",
            "completed",
            "failed",
        ) and (is_executing(run_store, run.id) or _is_pending(run_store, run.id)):
            # Completion-window leftovers: the stopped result was persisted
            # but marker clearing was lost. The stopped status stands;
            # nothing is re-executed.
            clear_executing(run_store, run.id)
            clear_pending(run_store, run.id)
            diags.append(f"{run.id}:completion-window-cleared")
        if status == StoredRunStatus.INTERRUPTED.value or status == "interrupted":
            checkpoint_id, result_attempt = _latest_result(run_store, run.id)
            sched_id, intended, revision = _attribution(run_store, run.id)
            if active:
                assert attempt is not None
                if result_attempt is not None and result_attempt == attempt.attempt_id:
                    _complete_attempt(run_store, run.id, attempt, now)
                    diags.append(f"{run.id}:fresh-result-resumable")
                    if _reconcile_terminal(
                        history,
                        sched_id,
                        run.id,
                        "interrupted",
                        checkpoint_id,
                        intended,
                        revision,
                        "fresh-result",
                        now,
                    ):
                        diags.append(f"{run.id}:terminal-reconciled")
                else:
                    _fail_run(run_store, run, AMBIGUOUS_REASON, now, history)
                    diags.append(f"{run.id}:failed-closed")
            else:
                diags.append(f"{run.id}:waiting-resumable")
                if _reconcile_terminal(
                    history,
                    sched_id,
                    run.id,
                    "interrupted",
                    checkpoint_id,
                    intended,
                    revision,
                    "reconciled-on-recovery",
                    now,
                ):
                    diags.append(f"{run.id}:terminal-reconciled")
        elif status == StoredRunStatus.ADMITTED.value or status == "admitted":
            if is_executing(run_store, run.id):
                # The executor may already have produced external effects:
                # abandon, never retry. Both markers are cleared so a later
                # recovery does not re-fail the now-terminal run.
                if active:
                    _fail_run(run_store, run, AMBIGUOUS_REASON, now, history)
                else:
                    _fail_run(run_store, run, ABANDONED_REASON, now, history)
                clear_pending(run_store, run.id)
                clear_executing(run_store, run.id)
                diags.append(f"{run.id}:failed-closed")
                continue
            if _is_pending(run_store, run.id):
                diags.append(f"{run.id}:pending-dispatch-kept")
                continue
            try:
                run_store.get_admission(run.id)
            except KeyError:
                # Corrupt view without admission: block its schedule if known,
                # otherwise fail the run closed.
                diags.append(f"{run.id}:corrupt-blocked")
                continue
            if active:
                _fail_run(run_store, run, AMBIGUOUS_REASON, now, history)
            else:
                _fail_run(run_store, run, ABANDONED_REASON, now, history)
            diags.append(f"{run.id}:failed-closed")
        elif status == StoredRunStatus.COMPLETED.value or status == "completed":
            checkpoint_id, completed_attempt = _latest_result(run_store, run.id)
            sched_id, intended, revision = _attribution(run_store, run.id)
            if active:
                assert attempt is not None
                if (
                    completed_attempt is not None
                    and completed_attempt == attempt.attempt_id
                ):
                    _complete_attempt(run_store, run.id, attempt, now)
                    diags.append(f"{run.id}:attempt-reconciled")
                else:
                    _fail_run(run_store, run, AMBIGUOUS_REASON, now, history)
                    diags.append(f"{run.id}:failed-closed")
                    continue
            if _reconcile_terminal(
                history,
                sched_id,
                run.id,
                "completed",
                checkpoint_id,
                intended,
                revision,
                "reconciled-on-recovery",
                now,
            ):
                diags.append(f"{run.id}:terminal-reconciled")
        elif status == StoredRunStatus.FAILED.value or status == "failed":
            sched_id, intended, revision = _attribution(run_store, run.id)
            if _reconcile_terminal(
                history,
                sched_id,
                run.id,
                "failed",
                run.latest_checkpoint_id,
                intended,
                revision,
                "reconciled-on-recovery",
                now,
            ):
                diags.append(f"{run.id}:terminal-reconciled")
    return diags


def _reconcile_summary_from_checkpoint(
    run_store: Any, run: Any, now: datetime
) -> tuple[bool, str]:
    """Rewrite a stale summary from its durable stopped checkpoint.

    Returns ``(changed, status)``. Only the checkpoint's own persisted
    fields (status, checkpoint id, readiness derived exactly as
    ``persist_stopped_run`` derives it) are copied; creation time,
    environment, and diagnostics are preserved. A run without checkpoints
    is untouched (an admitted summary with no checkpoint is genuinely
    undispatched-or-unknown, handled by the marker rules).
    """
    from wf_artifacts.runs.models import ResumeReadiness, StoredRunStatus

    try:
        latest = run_store.get_latest_checkpoint(run.id)
    except KeyError:
        return False, getattr(run.status, "value", run.status)
    expected = StoredRunStatus(latest.reason.value)
    readiness = (
        ResumeReadiness.READY
        if expected is StoredRunStatus.INTERRUPTED
        else ResumeReadiness.NOT_APPLICABLE
    )
    if (
        run.status == expected
        and run.latest_checkpoint_id == latest.id
        and run.resume_readiness == readiness
    ):
        return False, getattr(run.status, "value", run.status)
    run_store.save_run(
        run.model_copy(
            update={
                "status": expected,
                "latest_checkpoint_id": latest.id,
                "resume_readiness": readiness,
                "updated_at": now,
            }
        )
    )
    return True, expected.value


def _attribution(
    run_store: Any, run_id: str
) -> tuple[str | None, datetime | None, int | None]:
    """Return ``(schedule_id, scheduled_at, schedule_revision)`` for a run.

    ``(None, None, None)`` when no admission owns the run: schedule history
    cannot attribute it, so callers persist the reason on the run itself.
    """
    try:
        admission = run_store.get_admission(run_id)
    except KeyError:
        return None, None, None
    return (
        admission.schedule_id,
        admission.scheduled_at,
        admission.schedule_revision,
    )


def _latest_result(run_store: Any, run_id: str) -> tuple[str | None, Any]:
    """Return ``(checkpoint_id, attempt_id)`` of the latest stopped checkpoint."""
    try:
        latest = run_store.get_latest_checkpoint(run_id)
    except KeyError:
        return None, None
    return latest.id, latest.attempt_id


def _complete_attempt(run_store: Any, run_id: str, attempt: Any, now: datetime) -> None:
    """Mark the ACTIVE attempt DONE after its fresh result was recognized."""
    from wf_artifacts.runs.models import ResumeAttempt

    run_store.save_resume_attempt(
        ResumeAttempt(
            run_id=run_id,
            attempt_id=attempt.attempt_id,
            state="DONE",
            created_at=attempt.created_at,
            updated_at=now,
        )
    )


def _reconcile_terminal(
    history: HistoryRecorder,
    sched_id: str | None,
    run_id: str,
    kind: str,
    checkpoint_id: str | None,
    intended: datetime | None,
    revision: int | None,
    reason: str,
    now: datetime,
) -> bool:
    """Record one stopped-result entry unless this exact result is known.

    Returns whether an entry was appended. Idempotency identity is
    ``(run_id, kind, checkpoint_id)``: repeating recovery never duplicates
    an entry, while a resumed run that stops again (new checkpoint id)
    records a new one.
    """
    if sched_id is None:
        return False
    if history.has_terminal(sched_id, run_id, kind, checkpoint_id):
        return False
    history.record(
        HistoryEntry(
            schedule_id=sched_id,
            kind=kind,  # type: ignore[arg-type]
            resolved_at=intended,
            run_id=run_id,
            revision=revision,
            reason=reason,
            checkpoint_id=checkpoint_id,
            created_at=now,
        )
    )
    return True


def _fail_run(
    run_store: Any,
    run: Any,
    reason: str,
    now: datetime,
    history: HistoryRecorder,
) -> None:
    """Fail a run closed with its truthful reason durably preserved.

    The reason is always written into the run summary diagnostics; when an
    admission attributes the run to a schedule, a failed occurrence entry is
    reconciled exactly once through the history recorder.
    """
    from wf_artifacts import DependencyDiagnostic, DiagnosticSeverity
    from wf_artifacts.runs.models import StoredRunStatus

    diagnostic = DependencyDiagnostic(
        severity=DiagnosticSeverity.ERROR,
        code="schedule-recovery",
        logical_ref=run.id,
        message=reason,
        repair_hint=None,
    )
    updated = run.model_copy(
        update={
            "status": StoredRunStatus.FAILED,
            "updated_at": now,
            "diagnostics": [*run.diagnostics, diagnostic],
        }
    )
    run_store.save_run(updated)
    sched_id, intended, revision = _attribution(run_store, run.id)
    _reconcile_terminal(
        history,
        sched_id,
        run.id,
        "failed",
        run.latest_checkpoint_id,
        intended,
        revision,
        reason,
        now,
    )


def _mark_pending(run_store: Any, run_id: str) -> None:
    run_store.mark_pending_dispatch(run_id)


def is_executing(run_store: Any, run_id: str) -> bool:
    return bool(run_store.is_executing(run_id))


def clear_executing(run_store: Any, run_id: str) -> None:
    """Clear the executing mark after a stopped result is durably persisted."""
    run_store.clear_executing(run_id)


def _is_pending(run_store: Any, run_id: str) -> bool:
    return bool(run_store.is_pending_dispatch(run_id))


def clear_pending(run_store: Any, run_id: str) -> None:
    """Clear the pending-dispatch marker after the poll sweep dispatches."""
    run_store.clear_pending_dispatch(run_id)