lda.chat/lda-wf
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sched: typed history recorder with exactly-once terminal reconcile (F4)

Browse Source
This commit is contained in:
lda
2026-09-08 11:39:13 +07:00 Verified
parent 78a966d722
commit 5a6056b144
7 changed files with 879 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/wf_scheduling/history.py
+115
View File
@@ -0,0 +1,115 @@
"""Typed occurrence-history interface with idempotent reconciliation (R4/F4).
Both the poll loop and startup recovery record through :class:`HistoryRecorder`,
so completed/failed/interrupted entries share one shape and one idempotency
identity: ``(run_id, kind, checkpoint_id)``. A resumed run that interrupts
again produces a new checkpoint id and therefore a new entry; repeating
recovery never duplicates an entry. The file-backed recorder derives entry
identity the same way the scheduler always has (occurrence instants hash to
``occurrence_id``; interval summaries use their span key).
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Any, Protocol
from wf_scheduling.models import OccurrenceKind, OccurrenceRecord
from wf_scheduling.occurrences import occurrence_id
UTC = timezone.utc
TERMINAL_KINDS: tuple[str, str, str] = ("completed", "interrupted", "failed")
@dataclass(frozen=True)
class HistoryEntry:
"""One occurrence-history entry with a stable idempotency identity."""
schedule_id: str
kind: OccurrenceKind
resolved_at: datetime | None = None
run_id: str | None = None
revision: int | None = None
reason: str = ""
admitted_at: datetime | None = None
started_at: datetime | None = None
checkpoint_id: str | None = None
interval_start: datetime | None = None
interval_end: datetime | None = None
interval_count: int = 0
created_at: datetime | None = None
class HistoryRecorder(Protocol):
"""Occurrence-history sink shared by polling and recovery."""
def record(self, entry: HistoryEntry) -> None:
"""Append one entry (callers dedup stopped results first)."""
...
def has_terminal(
self,
schedule_id: str,
run_id: str,
kind: str,
checkpoint_id: str | None = None,
) -> bool:
"""Whether this exact stopped result was already reconciled."""
...
def entry_occurrence_id(entry: HistoryEntry, created: datetime) -> str:
"""Derive the history occurrence id with the scheduler's standing rules."""
if entry.resolved_at is not None:
return occurrence_id(entry.schedule_id, entry.resolved_at)
if entry.interval_start is not None and entry.interval_end is not None:
start = entry.interval_start.isoformat()
end = entry.interval_end.isoformat()
return f"{entry.schedule_id}|summary|{start}|{end}"
return f"{entry.schedule_id}|summary|{created.isoformat()}"
class FileScheduleHistoryRecorder:
"""History recorder backed by a file schedule store."""
def __init__(self, schedule_store: Any) -> None:
self.schedule_store = schedule_store
def record(self, entry: HistoryEntry) -> None:
created = (
entry.created_at if entry.created_at is not None else datetime.now(UTC)
)
self.schedule_store.append_history(
OccurrenceRecord(
schedule_id=entry.schedule_id,
occurrence_id=entry_occurrence_id(entry, created),
kind=entry.kind,
resolved_at=entry.resolved_at,
run_id=entry.run_id,
revision=entry.revision,
reason=entry.reason,
admitted_at=entry.admitted_at,
started_at=entry.started_at,
checkpoint_id=entry.checkpoint_id,
interval_start=entry.interval_start,
interval_end=entry.interval_end,
interval_count=entry.interval_count,
created_at=created,
)
)
def has_terminal(
self,
schedule_id: str,
run_id: str,
kind: str,
checkpoint_id: str | None = None,
) -> bool:
return self.schedule_store.has_history_entry(
schedule_id,
run_id=run_id,
kind=kind,
checkpoint_id=checkpoint_id,
)
src/wf_scheduling/models.py
+8
View File
@@ -156,6 +156,14 @@ class OccurrenceRecord(BaseModel):
reason: str = ""
admitted_at: datetime | None = None
started_at: datetime | None = None
checkpoint_id: str | None = Field(
default=None,
description=(
"Stopped checkpoint this entry reconciles, for exactly-once "
"recovery dedup: a resumed run that stops again carries a new "
"checkpoint id and therefore a new entry."
),
)
interval_start: datetime | None = None
interval_end: datetime | None = None
interval_count: int = 0
src/wf_scheduling/poll.py
+58 -13
View File
@@ -25,8 +25,12 @@ from wf_scheduling.calendar import (
OneShotSource,
)
from wf_scheduling.dispatch import RunDispatcher, StillRunning
from wf_scheduling.models import OccurrenceRecord, PendingCandidate
from wf_scheduling.occurrences import occurrence_id
from wf_scheduling.history import (
FileScheduleHistoryRecorder,
HistoryEntry,
HistoryRecorder,
)
from wf_scheduling.models import PendingCandidate
from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
from wf_scheduling.prepare import InvocationPreparer, PreparationRejected
@@ -116,6 +120,7 @@ class Scheduler:
preparer: InvocationPreparer,
dispatcher: RunDispatcher,
ownership: SchedulerOwnership,
history: HistoryRecorder | None = None,
) -> None:
self.schedule_store = schedule_store
self.run_store = run_store
@@ -124,6 +129,11 @@ class Scheduler:
self.preparer = preparer
self.dispatcher = dispatcher
self.ownership = ownership
self.history: HistoryRecorder = (
history
if history is not None
else FileScheduleHistoryRecorder(schedule_store)
)
self._poll_cursor = 0
def _require_ownership(self) -> None:
@@ -191,18 +201,12 @@ class Scheduler:
now: datetime | None = None,
admitted_at: datetime | None = None,
started_at: datetime | None = None,
checkpoint_id: str | None = None,
) -> None:
created = now if now is not None else datetime.now(UTC)
if intended is not None:
oid = occurrence_id(sched_id, intended)
elif interval is not None:
oid = f"{sched_id}|summary|{interval[0].isoformat()}|{interval[1].isoformat()}"
else:
oid = f"{sched_id}|summary|{created.isoformat()}"
self.schedule_store.append_history(
OccurrenceRecord(
self.history.record(
HistoryEntry(
schedule_id=sched_id,
occurrence_id=oid,
kind=kind, # type: ignore[arg-type]
resolved_at=intended,
run_id=run_id,
@@ -210,6 +214,7 @@ class Scheduler:
reason=reason,
admitted_at=admitted_at,
started_at=started_at,
checkpoint_id=checkpoint_id,
interval_start=interval[0] if interval else None,
interval_end=interval[1] if interval else None,
interval_count=count,
@@ -217,6 +222,20 @@ class Scheduler:
)
)
def _existing_occurrence_run(self, sched_id: str, intended: datetime) -> str | None:
"""Return the run already owning this occurrence, if any.
Identity is ``(schedule_id, resolved UTC instant)`` from the durable
admission record — the admission persist is the decision point, so
admissions are scanned rather than views (a crashed admission may
not have a view yet). Manual runs carry no scheduled instant and
never match.
"""
for admission in self.run_store.list_admissions():
if admission.schedule_id == sched_id and admission.scheduled_at == intended:
return admission.id
return None
def _admit(self, sched: Any, intended: datetime, now: datetime) -> str | None:
if getattr(sched, "blocked_reason", None):
raise BlockedSchedule(getattr(sched, "blocked_reason"))
@@ -232,6 +251,30 @@ class Scheduler:
sched.blocked_reason = reason
self.schedule_store.save_schedule(sched)
raise BlockedSchedule(reason) from None
existing = self._existing_occurrence_run(sched.id, intended)
if existing is not None:
# The occurrence already owns a run (crash between the admission
# persist and the watermark/history writes, or a lost watermark
# write): an occurrence is immutable and never replayed. Advance
# the watermark, reconcile a missing admitted entry exactly once,
# and return the owner without dispatching (the pending sweep
# owns dispatch).
cand = self.schedule_store.get_candidate(sched.id)
if cand is not None and cand.intended_at == intended:
self.schedule_store.save_candidate(None, schedule_id=sched.id)
_save_consumed_max(self.schedule_store, sched.id, intended)
if not self.history.has_terminal(sched.id, existing, "admitted", None):
self._record(
kind="admitted",
sched_id=sched.id,
intended=intended,
run_id=existing,
reason=f"rev={sched.revision}",
revision=sched.revision,
now=now,
admitted_at=now,
)
return existing
prepared = self.preparer.prepare(sched=sched, intended=intended, now=now)
if isinstance(prepared, PreparationRejected):
self._record(
@@ -377,6 +420,7 @@ class Scheduler:
run_id=run_id,
now=now,
started_at=now,
checkpoint_id=stopped.latest_checkpoint_id,
)
def record_stopped_execution(self, run_id: str, result: Any, now: datetime) -> None:
@@ -422,6 +466,7 @@ class Scheduler:
run_id=run_id,
now=now,
started_at=now,
checkpoint_id=stopped.latest_checkpoint_id,
)
# -- polling --------------------------------------------------------
@@ -476,7 +521,7 @@ class Scheduler:
run,
CORRUPT_PENDING_REASON,
now,
None,
self.history,
)
clear_pending(self.run_store, run.id)
clear_executing(self.run_store, run.id)
@@ -487,7 +532,7 @@ class Scheduler:
run,
CORRUPT_PENDING_REASON,
now,
None,
self.history,
)
clear_pending(self.run_store, run.id)
clear_executing(self.run_store, run.id)
src/wf_scheduling/recovery.py
+184 -69
View File
@@ -36,6 +36,11 @@ from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from wf_scheduling.history import (
FileScheduleHistoryRecorder,
HistoryEntry,
HistoryRecorder,
)
from wf_scheduling.ownership import SchedulerOwnership, SecondOwnerError
UTC = timezone.utc
@@ -60,7 +65,7 @@ def recover(
run_store: Any,
now: datetime,
ownership: SchedulerOwnership,
record_history: Any | None = None,
history: HistoryRecorder | None = None,
) -> list[str]:
"""Reconcile durable state after a restart without executing work."""
from wf_artifacts.runs.models import StoredRunStatus
@@ -70,6 +75,8 @@ def recover(
"scheduler ownership is required before recovery: an unowned "
"recovery could abandon or redispatch another owner's work"
)
if history is None:
history = FileScheduleHistoryRecorder(schedule_store)
diags: list[str] = []
# Admission record is the recovery authority: admitted but never
# materialized views are completed here and flagged pending for the
@@ -102,40 +109,51 @@ def recover(
clear_pending(run_store, run.id)
diags.append(f"{run.id}:completion-window-cleared")
if status == StoredRunStatus.INTERRUPTED.value or status == "interrupted":
checkpoint_id, result_attempt = _latest_result(run_store, run.id)
sched_id, intended, revision = _attribution(run_store, run.id)
if active:
assert attempt is not None
try:
latest = run_store.get_latest_checkpoint(run.id)
result_attempt = latest.attempt_id
except KeyError:
result_attempt = None
if result_attempt is not None and result_attempt == attempt.attempt_id:
from wf_artifacts.runs.models import ResumeAttempt
run_store.save_resume_attempt(
ResumeAttempt(
run_id=run.id,
attempt_id=attempt.attempt_id,
state="DONE",
created_at=attempt.created_at,
updated_at=now,
)
)
_complete_attempt(run_store, run.id, attempt, now)
diags.append(f"{run.id}:fresh-result-resumable")
if _reconcile_terminal(
history,
sched_id,
run.id,
"interrupted",
checkpoint_id,
intended,
revision,
"fresh-result",
now,
):
diags.append(f"{run.id}:terminal-reconciled")
else:
_fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
_fail_run(run_store, run, AMBIGUOUS_REASON, now, history)
diags.append(f"{run.id}:failed-closed")
else:
diags.append(f"{run.id}:waiting-resumable")
if _reconcile_terminal(
history,
sched_id,
run.id,
"interrupted",
checkpoint_id,
intended,
revision,
"reconciled-on-recovery",
now,
):
diags.append(f"{run.id}:terminal-reconciled")
elif status == StoredRunStatus.ADMITTED.value or status == "admitted":
if is_executing(run_store, run.id):
# The executor may already have produced external effects:
# abandon, never retry. Both markers are cleared so a later
# recovery does not re-fail the now-terminal run.
if active:
_fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
_fail_run(run_store, run, AMBIGUOUS_REASON, now, history)
else:
_fail_run(run_store, run, ABANDONED_REASON, now, record_history)
_fail_run(run_store, run, ABANDONED_REASON, now, history)
clear_pending(run_store, run.id)
clear_executing(run_store, run.id)
diags.append(f"{run.id}:failed-closed")
@@ -151,80 +169,177 @@ def recover(
diags.append(f"{run.id}:corrupt-blocked")
continue
if active:
_fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
_fail_run(run_store, run, AMBIGUOUS_REASON, now, history)
else:
_fail_run(run_store, run, ABANDONED_REASON, now, record_history)
_fail_run(run_store, run, ABANDONED_REASON, now, history)
diags.append(f"{run.id}:failed-closed")
elif status == StoredRunStatus.COMPLETED.value or status == "completed":
checkpoint_id, completed_attempt = _latest_result(run_store, run.id)
sched_id, intended, revision = _attribution(run_store, run.id)
if active:
assert attempt is not None
try:
latest = run_store.get_latest_checkpoint(run.id)
completed_attempt = latest.attempt_id
except KeyError:
completed_attempt = None
if (
completed_attempt is not None
and completed_attempt == attempt.attempt_id
):
from wf_artifacts.runs.models import ResumeAttempt
run_store.save_resume_attempt(
ResumeAttempt(
run_id=run.id,
attempt_id=attempt.attempt_id,
state="DONE",
created_at=attempt.created_at,
updated_at=now,
)
)
_complete_attempt(run_store, run.id, attempt, now)
diags.append(f"{run.id}:attempt-reconciled")
else:
_fail_run(run_store, run, AMBIGUOUS_REASON, now, record_history)
_fail_run(run_store, run, AMBIGUOUS_REASON, now, history)
diags.append(f"{run.id}:failed-closed")
continue
if record_history is not None and not _has_terminal(
schedule_store, run.id, "completed"
if _reconcile_terminal(
history,
sched_id,
run.id,
"completed",
checkpoint_id,
intended,
revision,
"reconciled-on-recovery",
now,
):
diags.append(f"{run.id}:terminal-reconciled")
elif status == StoredRunStatus.FAILED.value or status == "failed":
sched_id, intended, revision = _attribution(run_store, run.id)
if _reconcile_terminal(
history,
sched_id,
run.id,
"failed",
run.latest_checkpoint_id,
intended,
revision,
"reconciled-on-recovery",
now,
):
record_history(
kind="completed",
run_id=run.id,
reason="reconciled-on-recovery",
)
diags.append(f"{run.id}:terminal-reconciled")
return diags
def _attribution(
run_store: Any, run_id: str
) -> tuple[str | None, datetime | None, int | None]:
"""Return ``(schedule_id, scheduled_at, schedule_revision)`` for a run.
``(None, None, None)`` when no admission owns the run: schedule history
cannot attribute it, so callers persist the reason on the run itself.
"""
try:
admission = run_store.get_admission(run_id)
except KeyError:
return None, None, None
return (
admission.schedule_id,
admission.scheduled_at,
admission.schedule_revision,
)
def _latest_result(run_store: Any, run_id: str) -> tuple[str | None, Any]:
"""Return ``(checkpoint_id, attempt_id)`` of the latest stopped checkpoint."""
try:
latest = run_store.get_latest_checkpoint(run_id)
except KeyError:
return None, None
return latest.id, latest.attempt_id
def _complete_attempt(run_store: Any, run_id: str, attempt: Any, now: datetime) -> None:
"""Mark the ACTIVE attempt DONE after its fresh result was recognized."""
from wf_artifacts.runs.models import ResumeAttempt
run_store.save_resume_attempt(
ResumeAttempt(
run_id=run_id,
attempt_id=attempt.attempt_id,
state="DONE",
created_at=attempt.created_at,
updated_at=now,
)
)
def _reconcile_terminal(
history: HistoryRecorder,
sched_id: str | None,
run_id: str,
kind: str,
checkpoint_id: str | None,
intended: datetime | None,
revision: int | None,
reason: str,
now: datetime,
) -> bool:
"""Record one stopped-result entry unless this exact result is known.
Returns whether an entry was appended. Idempotency identity is
``(run_id, kind, checkpoint_id)``: repeating recovery never duplicates
an entry, while a resumed run that stops again (new checkpoint id)
records a new one.
"""
if sched_id is None:
return False
if history.has_terminal(sched_id, run_id, kind, checkpoint_id):
return False
history.record(
HistoryEntry(
schedule_id=sched_id,
kind=kind, # type: ignore[arg-type]
resolved_at=intended,
run_id=run_id,
revision=revision,
reason=reason,
checkpoint_id=checkpoint_id,
created_at=now,
)
)
return True
def _fail_run(
run_store: Any, run: Any, reason: str, now: datetime, record_history: Any | None
run_store: Any,
run: Any,
reason: str,
now: datetime,
history: HistoryRecorder,
) -> None:
"""Fail a run closed with its truthful reason durably preserved.
The reason is always written into the run summary diagnostics; when an
admission attributes the run to a schedule, a failed occurrence entry is
reconciled exactly once through the history recorder.
"""
from wf_artifacts import DependencyDiagnostic, DiagnosticSeverity
from wf_artifacts.runs.models import StoredRunStatus
diagnostic = DependencyDiagnostic(
severity=DiagnosticSeverity.ERROR,
code="schedule-recovery",
logical_ref=run.id,
message=reason,
repair_hint=None,
)
updated = run.model_copy(
update={"status": StoredRunStatus.FAILED, "updated_at": now}
update={
"status": StoredRunStatus.FAILED,
"updated_at": now,
"diagnostics": [*run.diagnostics, diagnostic],
}
)
run_store.save_run(updated)
if record_history is not None:
try:
admission = run_store.get_admission(run.id)
sched_id = admission.schedule_id or ""
intended = admission.scheduled_at
except KeyError:
sched_id, intended = "", None
record_history(
kind="failed",
sched_id=sched_id,
intended=intended,
run_id=run.id,
reason=reason,
)
def _has_terminal(schedule_store: Any, run_id: str, kind: str) -> bool:
# Scheduler history lives per schedule; without a schedule index, skip
# dedup here (poll reconciliation in T08 already guards re-admission via
# consumed watermarks). Kept as a seam for T13 inspection.
return False
sched_id, intended, revision = _attribution(run_store, run.id)
_reconcile_terminal(
history,
sched_id,
run.id,
"failed",
run.latest_checkpoint_id,
intended,
revision,
reason,
now,
)
def _mark_pending(run_store: Any, run_id: str) -> None:
src/wf_scheduling/store.py
+23
View File
@@ -168,6 +168,29 @@ class FileScheduleStore:
entries.append(record.model_dump(mode="json"))
self._write_json(path, entries)
def has_history_entry(
self,
schedule_id: str,
*,
run_id: str,
kind: str,
checkpoint_id: str | None = None,
) -> bool:
"""Whether this exact stopped result already has a history entry.
Idempotency identity is ``(run_id, kind, checkpoint_id)``: repeats
of one recovery pass dedup, while a resumed run that stops again
(new checkpoint id) records a new entry.
"""
for item in self._read_history_locked(schedule_id):
if (
item.get("run_id") == run_id
and item.get("kind") == kind
and item.get("checkpoint_id") == checkpoint_id
):
return True
return False
@staticmethod
def _sort_key(
item: OccurrenceRecord,