lda-wf/src/wf_scheduling/recovery.py

"""Startup recovery and reconciliation (T10).

Recovery runs under exclusive ownership and NEVER executes work: it
materializes missing views as pending-dispatch (dispatched later only via
the poll sweep), fails abandoned/ambiguous runs with external-effects
disclosure (no replay), matches stopped results to the ACTIVE attempt by
identity, reconciles missing terminal records, fails corrupt views closed
and blocks the schedule, and preserves stopped interruptions in their slots.
"""

from __future__ import annotations

from datetime import datetime, timezone
from typing import Any

UTC = timezone.utc

ABANDONED_REASON = (
    "abandoned execution: outcome unknown; external effects may already "
    "have occurred; no replay"
)
AMBIGUOUS_REASON = (
    "ambiguous resume attempt: may have executed; external effects may "
    "already have occurred; no retry"
)


def recover(
    *,
    schedule_store: Any,
    run_store: Any,
    now: datetime,
    record_history: Any | None = None,
) -> list[str]:
    """Reconcile durable state after a restart without executing work."""
    from wf_artifacts.runs.models import StoredRunStatus

    diags: list[str] = []
    # Admission record is the recovery authority: admitted but never
    # materialized views are completed here and flagged pending for the
    # capacity-checked poll sweep (exactly once, occurrence already consumed).
    for admission in run_store.list_admissions():
        try:
            run_store.get_run(admission.id)
        except KeyError:
            from wf_api.run_lifecycle import materialize_admitted_view

            materialize_admitted_view(store=run_store, admission=admission)
            _mark_pending(run_store, admission.id)
            diags.append(f"{admission.id}:view-completed-pending-dispatch")
    for run in run_store.list_runs():
        status = getattr(run.status, "value", run.status)
        attempt = run_store.get_resume_attempt(run.id)
        active = attempt is not None and attempt.state == "ACTIVE"
        if status == StoredRunStatus.INTERRUPTED.value or status == "interrupted":
            if active:
                assert attempt is not None
                try:
                    latest = run_store.get_latest_checkpoint(run.id)
                    result_attempt = latest.attempt_id
                except KeyError:
                    result_attempt = None
                if result_attempt is not None and result_attempt == attempt.attempt_id:
                    from wf_artifacts.runs.models import ResumeAttempt

                    run_store.save_resume_attempt(
                        ResumeAttempt(
                            run_id=run.id,
                            attempt_id=attempt.attempt_id,
                            state="DONE",
                            created_at=attempt.created_at,
                            updated_at=now,
                        )
                    )
                    diags.append(f"{run.id}:fresh-result-resumable")
                else:
                    _fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
                    diags.append(f"{run.id}:failed-closed")
            else:
                diags.append(f"{run.id}:waiting-resumable")
        elif status == StoredRunStatus.ADMITTED.value or status == "admitted":
            if _is_pending(run_store, run.id):
                diags.append(f"{run.id}:pending-dispatch-kept")
                continue
            try:
                run_store.get_admission(run.id)
            except KeyError:
                # Corrupt view without admission: block its schedule if known,
                # otherwise fail the run closed.
                diags.append(f"{run.id}:corrupt-blocked")
                continue
            if active:
                _fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
            else:
                _fail_run(run_store, run, ABANDONED_REASON, now, record_history)
            diags.append(f"{run.id}:failed-closed")
        elif status == StoredRunStatus.COMPLETED.value or status == "completed":
            if active:
                assert attempt is not None
                try:
                    latest = run_store.get_latest_checkpoint(run.id)
                    completed_attempt = latest.attempt_id
                except KeyError:
                    completed_attempt = None
                if (
                    completed_attempt is not None
                    and completed_attempt == attempt.attempt_id
                ):
                    from wf_artifacts.runs.models import ResumeAttempt

                    run_store.save_resume_attempt(
                        ResumeAttempt(
                            run_id=run.id,
                            attempt_id=attempt.attempt_id,
                            state="DONE",
                            created_at=attempt.created_at,
                            updated_at=now,
                        )
                    )
                    diags.append(f"{run.id}:attempt-reconciled")
                else:
                    _fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
                    diags.append(f"{run.id}:failed-closed")
                    continue
            if record_history is not None and not _has_terminal(
                schedule_store, run.id, "completed"
            ):
                record_history(
                    kind="completed",
                    run_id=run.id,
                    reason="reconciled-on-recovery",
                )
                diags.append(f"{run.id}:terminal-reconciled")
    return diags


def _fail_run(
    run_store: Any, run: Any, reason: str, now: datetime, record_history: Any | None
) -> None:
    from wf_artifacts.runs.models import StoredRunStatus

    updated = run.model_copy(
        update={"status": StoredRunStatus.FAILED, "updated_at": now}
    )
    run_store.save_run(updated)
    if record_history is not None:
        try:
            admission = run_store.get_admission(run.id)
            sched_id = admission.schedule_id or ""
            intended = admission.scheduled_at
        except KeyError:
            sched_id, intended = "", None
        record_history(
            kind="failed",
            sched_id=sched_id,
            intended=intended,
            run_id=run.id,
            reason=reason,
        )


def _has_terminal(schedule_store: Any, run_id: str, kind: str) -> bool:
    # Scheduler history lives per schedule; without a schedule index, skip
    # dedup here (poll reconciliation in T08 already guards re-admission via
    # consumed watermarks). Kept as a seam for T13 inspection.
    return False


def _mark_pending(run_store: Any, run_id: str) -> None:
    run_store.mark_pending_dispatch(run_id)


def _is_pending(run_store: Any, run_id: str) -> bool:
    return bool(run_store.is_pending_dispatch(run_id))


def clear_pending(run_store: Any, run_id: str) -> None:
    """Clear the pending-dispatch marker after the poll sweep dispatches."""
    run_store.clear_pending_dispatch(run_id)