lda.chat/lda-wf
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sched: one canonical lock identity per composition; checkpoint coherence and ordering authority (R4 wave 3)

Browse Source
This commit is contained in:
lda
2026-09-08 18:57:17 +07:00 Verified
parent 3f1d5d158b
commit d2c0867f76
4 changed files with 1030 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/wf_scheduling/ownership.py
+62 -19
View File
@@ -11,11 +11,22 @@ Locking design (Windows-tested, per the store transaction boundary): one
new module holding ``msvcrt.locking`` (Windows) / ``fcntl.flock`` (POSIX)
on ``<composition root>/scheduler.lock``. No ad-hoc per-run lock files.
One store composition has exactly one lock identity: the canonical
nearest common ancestor of its store roots (see
:func:`canonical_lock_root`). Any ancestor is NOT good enough — different
ancestors yield different lock files and therefore independent OS locks,
so ancestor containment can never prove exclusivity. Guards require the
held lock's frozen identity to equal the composition identity; overlapping
compositions that share no common ancestor are rejected instead of
claimed safe.
The lock is acquired on the composition root that contains the protected
schedule and run stores. Entry-point guards do not trust the caller to
have passed the right lock: they validate the held guard against the
actual store roots (canonicalized for platform aliases) before any write
or side effect.
or side effect. The acquired identity is frozen at acquisition: later
mutation of the public ``root`` attribute cannot redirect a held handle's
authority.
"""
from __future__ import annotations
@@ -48,12 +59,28 @@ def canonical_store_path(path: Path | str) -> str:
return os.path.normcase(os.path.realpath(os.fspath(path)))
def _is_within(candidate: str, base: str) -> bool:
"""Whether canonical ``candidate`` lies beneath canonical ``base``."""
if candidate == base:
return True
prefix = base if base.endswith(os.sep) else base + os.sep
return candidate.startswith(prefix)
def canonical_lock_root(*store_roots: Path | str | None) -> str | None:
"""Return the one canonical lock identity for a store composition.
The identity is the canonical nearest common ancestor of the store
roots; the lock file lives at ``<identity>/scheduler.lock``. Both the
held guard's frozen root and the guarded stores map through this
function, so exactly one lock file can authorize one composition.
Returns None when a root is missing or the roots share no common
ancestor (e.g. different drives): such compositions are unsupported
and guards reject them instead of claiming safety.
"""
canonical: list[str] = []
for store_root in store_roots:
if store_root is None:
return None
canonical.append(canonical_store_path(store_root))
if not canonical:
return None
try:
return os.path.commonpath(canonical)
except ValueError:
return None
class SchedulerOwnership:
@@ -64,6 +91,7 @@ class SchedulerOwnership:
self.owner = owner
self._handle: BinaryIO | None = None
self._locked = False
self._acquired_root: str | None = None
@property
def lock_path(self) -> Path:
@@ -80,25 +108,35 @@ class SchedulerOwnership:
return self._locked
def canonical_root(self) -> str:
"""Return the canonical composition root this lock was acquired on."""
"""Return the canonical composition root this lock was acquired on.
The identity is frozen at acquisition: mutating the public
``root`` afterwards cannot redirect a held handle's authority.
Before acquisition (or after release) it reports the live root.
"""
if self._acquired_root is not None:
return self._acquired_root
return canonical_store_path(self.root)
def covers(self, *store_roots: Path | str | None) -> bool:
"""Whether this held lock protects the given store roots.
"""Whether this held lock is the one lock for the given stores.
Coverage means each root canonicalizes to the lock's composition
root or beneath it. An unrelated (even held) lock, a released lock,
and rootless stores are never covered: guards reject those before
any write or side effect instead of trusting the caller's promise.
Coverage means the frozen acquisition identity equals the
composition's canonical lock identity
(:func:`canonical_lock_root`), not merely that the stores sit
beneath some held ancestor: a lock on a parent directory is an
independent OS lock from the composition's own lock and proves
nothing. An unrelated (even held) lock, a released lock, a
mutated handle, and rootless or ancestor-less stores are never
covered: guards reject those before any write or side effect
instead of trusting the caller's promise.
"""
if not self._locked or not store_roots:
return False
base = self.canonical_root()
return all(
store_root is not None
and _is_within(canonical_store_path(store_root), base)
for store_root in store_roots
)
if self._acquired_root is None:
return False
expected = canonical_lock_root(*store_roots)
return expected is not None and self._acquired_root == expected
def acquire(self) -> SchedulerOwnership:
"""Acquire the held lock non-blockingly or raise SecondOwnerError."""
@@ -120,6 +158,10 @@ class SchedulerOwnership:
raise StartupRejected(f"unsupported scheduler locking: {exc}") from exc
self._handle = handle
self._locked = True
# Freeze the authority identity now: the public ``root`` may be
# reassigned later, but a held handle must keep proving exactly the
# composition it locked, never the new value.
self._acquired_root = canonical_store_path(self.root)
return self
def release(self) -> None:
@@ -134,6 +176,7 @@ class SchedulerOwnership:
finally:
self._handle = None
self._locked = False
self._acquired_root = None
def __enter__(self) -> SchedulerOwnership:
return self.acquire()
src/wf_scheduling/recovery.py
+210 -44
View File
@@ -31,13 +31,25 @@ clear → executor → stopped persist → executing clear → history):
reconciliation across the torn ``save_checkpoint``/``save_run`` boundary
is owned by the F2 reconcile step, which runs before marker handling.
- stopped summary, no marks: existing terminal/attempt reconciliation.
- failed summary: a durable recovery decision is never re-processed. When
the checkpoint pointer still matches the decided state, history is
ensured without touching status, readiness, or diagnostics (repeat
recovery is silent). Only a genuinely newer checkpoint (pointer
mismatch) may reopen the run via the reconcile path; an older
checkpoint can never supersede the abandonment. Failed runs carry
not-applicable resume readiness.
- failed summary: a durable recovery decision is never re-processed. A
matching checkpoint pointer only ensures history; it never revalidates
the decision. A mismatched pointer reopens the run only for a
genuinely newer result: the referenced checkpoint must still exist, the
latest checkpoint must have a strictly greater sequence with attempt
provenance (an attributed result never yields to an older attempt, and
an unattributed result never supersedes an attributed decision), and
the newer checkpoint must be internally coherent (see below).
Anything else — a missing referenced checkpoint, an older stray, a
stale attempt, or an incoherent newer checkpoint — keeps the decision
with history ensured; corruption is additionally noted once as a
durable diagnostic. Failed runs carry not-applicable resume readiness.
- checkpoint authority: before any summary rewrite, the latest checkpoint
must prove coherence — same run identity, checkpoint id matching its
sequence, runtime state decodable through core validation, and the
outer reason agreeing with the decoded stopped state. The outer reason
enum alone is never trusted. Contradictory or corrupt checkpoints fail
closed with durable diagnostics; a successful result is never
fabricated from disagreement.
"""
from __future__ import annotations
@@ -121,19 +133,24 @@ def recover(
clear_executing(run_store, run.id)
clear_pending(run_store, run.id)
diags.append(f"{run.id}:completion-window-cleared")
# Durable-decision stability: a FAILED summary for the current
# checkpoint state is never re-processed. Only a genuinely newer
# checkpoint (pointer mismatch) may reopen it via the reconcile
# path below. History is ensured without mutating the summary.
# Durable-decision stability: a FAILED summary for the decided
# checkpoint state is never re-processed — history is ensured
# without mutating the summary. A mismatched pointer reopens the
# run only for a validated genuinely-newer result (referenced
# checkpoint present, strictly greater sequence, attempt
# provenance, coherent content); anything else keeps the decision.
if status == StoredRunStatus.FAILED.value or status == "failed":
if _failed_decision_stable(run_store, run, history, now, diags):
continue
# Checkpoint-first reconcile across the torn
# save_checkpoint/save_run boundary: a durable stopped checkpoint
# wins over a stale summary (status, checkpoint pointer, and
# readiness are rewritten together from the checkpoint's own
# persisted fields; nothing is fabricated). Attempt matching below
# still distinguishes fresh results from stale ones.
# save_checkpoint/save_run boundary: a validated durable stopped
# checkpoint wins over a stale summary (status, checkpoint pointer,
# and readiness are rewritten together from the checkpoint's own
# persisted fields; nothing is fabricated). Checkpoints that fail
# authority validation (wrong run, id/sequence disagreement,
# undecodable state, reason disagreeing with the decoded stopped
# state) raise below and fail closed. Attempt matching below still
# distinguishes fresh results from stale ones.
try:
changed, status = _reconcile_summary_from_checkpoint(run_store, run, now)
except (ValueError, OSError) as exc:
@@ -284,17 +301,57 @@ def recover(
return diags
def _check_checkpoint_coherence(run: Any, checkpoint: Any) -> None:
"""Prove a checkpoint may speak for a run summary, or raise.
Raises KeyError only from the caller's own loads; raises ValueError
(or OSError for unreadable state) when the checkpoint disagrees with
the run: a foreign run identity, a checkpoint id that does not match
its sequence, runtime state that core validation rejects, or an outer
reason that contradicts the decoded stopped state. The outer reason
enum alone is never authority: a COMPLETED label over an interrupted
runtime snapshot must fail closed, never release overlap.
"""
from wf_core import load_run_state_with_upgrade
if checkpoint.run_id != run.id:
raise ValueError(
f"checkpoint {checkpoint.id!r} names run {checkpoint.run_id!r}, "
f"not {run.id!r}"
)
if checkpoint.id != f"{run.id}.{checkpoint.sequence:06d}":
raise ValueError(
f"checkpoint {checkpoint.id!r} disagrees with its sequence "
f"{checkpoint.sequence}"
)
try:
decoded, _ = load_run_state_with_upgrade(
checkpoint.state.model_dump(mode="json")
)
except ValueError as exc:
raise ValueError(
f"checkpoint {checkpoint.id!r} has undecodable runtime state"
) from exc
if decoded.status.value != checkpoint.reason.value:
raise ValueError(
f"checkpoint {checkpoint.id!r} reason {checkpoint.reason.value!r} "
f"contradicts its runtime state {decoded.status.value!r}"
)
def _reconcile_summary_from_checkpoint(
run_store: Any, run: Any, now: datetime
) -> tuple[bool, str]:
"""Rewrite a stale summary from its durable stopped checkpoint.
"""Rewrite a stale summary from its validated durable stopped checkpoint.
Returns ``(changed, status)``. Only the checkpoint's own persisted
fields (status, checkpoint id, readiness derived exactly as
``persist_stopped_run`` derives it) are copied; creation time,
environment, and diagnostics are preserved. A run without checkpoints
is untouched (an admitted summary with no checkpoint is genuinely
undispatched-or-unknown, handled by the marker rules).
undispatched-or-unknown, handled by the marker rules). Authority
validation runs first: incoherent checkpoints raise and fail closed
in the caller instead of rewriting the summary.
"""
from wf_artifacts.runs.models import ResumeReadiness, StoredRunStatus
@@ -302,6 +359,7 @@ def _reconcile_summary_from_checkpoint(
latest = run_store.get_latest_checkpoint(run.id)
except KeyError:
return False, getattr(run.status, "value", run.status)
_check_checkpoint_coherence(run, latest)
expected = StoredRunStatus(latest.reason.value)
readiness = (
ResumeReadiness.READY
@@ -342,6 +400,87 @@ def _decision_reason(run: Any) -> str:
return ""
KEPT_DECISION_NOTE_CODE = "schedule-recovery-note"
def _ensure_failed_history(
run_store: Any,
run: Any,
history: HistoryRecorder,
now: datetime,
diags: list[str],
) -> None:
"""Ensure the failed terminal entry without touching the summary."""
sched_id, intended, revision = _attribution(run_store, run.id)
reason = _decision_reason(run) or "reconciled-on-recovery"
if _reconcile_terminal(
history,
sched_id,
run.id,
"failed",
run.latest_checkpoint_id,
intended,
revision,
reason,
now,
):
diags.append(f"{run.id}:terminal-reconciled")
def _note_decision_kept(
run_store: Any, run: Any, message: str, now: datetime, diags: list[str]
) -> None:
"""Durably note once that a decision survived unresolvable state.
Status and readiness are untouched; the note is appended only when no
identical note exists, so repeated recovery stays silent.
"""
from wf_artifacts import DependencyDiagnostic, DiagnosticSeverity
if any(
getattr(item, "code", None) == KEPT_DECISION_NOTE_CODE
and str(getattr(item, "message", "")) == message
for item in run.diagnostics
):
return
run_store.save_run(
run.model_copy(
update={
"updated_at": now,
"diagnostics": [
*run.diagnostics,
DependencyDiagnostic(
severity=DiagnosticSeverity.ERROR,
code=KEPT_DECISION_NOTE_CODE,
logical_ref=run.id,
message=message,
repair_hint=None,
),
],
}
)
)
diags.append(f"{run.id}:decision-kept-noted")
def _supersedes_decision(referenced: Any, latest: Any) -> bool:
"""Whether a newer checkpoint result may reopen a durable decision.
Ordering alone is not provenance: the latest checkpoint must carry a
strictly greater sequence AND attempt provenance — an attributed
result never yields to an older attempt, and an unattributed result
never supersedes an attributed decision (unattributed results predate
the resume protocol that produced the decision). Sequence ties and
regressions never supersede: a different checkpoint is not
necessarily a newer result.
"""
if latest.sequence <= referenced.sequence:
return False
if latest.attempt_id is None or referenced.attempt_id is None:
return referenced.attempt_id is None
return latest.attempt_id >= referenced.attempt_id
def _failed_decision_stable(
run_store: Any,
run: Any,
@@ -353,38 +492,65 @@ def _failed_decision_stable(
Returns True when the run needs no further processing this pass: the
summary already reflects a recovery decision (or a legacy/external
failure) for the current checkpoint state, so re-running reconcile or
re-failing would only duplicate diagnostics and rewrite readiness. A
genuinely newer checkpoint (pointer mismatch) returns False so the
reconcile path may repair the summary. Unreadable checkpoint state
likewise returns False: the reconcile path fails closed for undecided
runs and keeps decided ones via its own branch.
failure) and no validated newer result supersedes it, so re-running
reconcile or re-failing would only duplicate diagnostics and rewrite
readiness. A pointer match ensures history and stays silent. A
mismatch reopens the run only for a genuinely newer result —
referenced checkpoint present, strictly greater sequence, attempt
provenance, coherent content — via the reconcile path. A missing
referenced checkpoint, an older or unattributable newer checkpoint,
or an incoherent newer checkpoint keeps the decision: missing state
never rolls a run backward and never undoes a durable abandonment.
Corruption is noted once as a durable diagnostic. Unreadable
checkpoint state returns False: the reconcile path fails closed for
undecided runs and keeps decided ones via its own branch.
"""
try:
latest = run_store.get_latest_checkpoint(run.id)
except KeyError:
latest = None
checkpoints = run_store.list_checkpoints(run.id)
except ValueError, OSError:
return False
pointer = run.latest_checkpoint_id
latest_id = latest.id if latest is not None else None
if latest_id != pointer:
# Genuinely newer (or torn-away) checkpoint state: not stable.
referenced = (
next((c for c in checkpoints if c.id == pointer), None)
if pointer is not None
else None
)
latest = max(checkpoints, key=lambda c: c.sequence, default=None)
if pointer is not None and referenced is None:
_ensure_failed_history(run_store, run, history, now, diags)
_note_decision_kept(
run_store,
run,
f"durable recovery decision kept: referenced checkpoint "
f"{pointer!r} is missing",
now,
diags,
)
return True
if latest is None or latest.id == pointer:
_ensure_failed_history(run_store, run, history, now, diags)
return True
if referenced is None:
# A decision without a checkpoint pointer cannot order stray
# checkpoints: keep the decision with history ensured.
_ensure_failed_history(run_store, run, history, now, diags)
return True
if _supersedes_decision(referenced, latest):
try:
_check_checkpoint_coherence(run, latest)
except (ValueError, OSError) as exc:
_ensure_failed_history(run_store, run, history, now, diags)
_note_decision_kept(
run_store,
run,
f"durable recovery decision kept: newer checkpoint "
f"{latest.id!r} is incoherent: {exc}",
now,
diags,
)
return True
return False
sched_id, intended, revision = _attribution(run_store, run.id)
reason = _decision_reason(run) or "reconciled-on-recovery"
if _reconcile_terminal(
history,
sched_id,
run.id,
"failed",
pointer,
intended,
revision,
reason,
now,
):
diags.append(f"{run.id}:terminal-reconciled")
_ensure_failed_history(run_store, run, history, now, diags)
return True