475 lines
17 KiB
Python
475 lines
17 KiB
Python
|
|
"""Event-driven governed recovery reconciler with polling fallback.
|
|
|
|
The controller cannot create a task and cannot bypass admission. It resumes
|
|
only an exact, already-existing child through an injected governed executor.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
from typing import Mapping, Protocol
|
|
|
|
from .governance_admission import (
|
|
AdmissionOperation,
|
|
AdmissionPreflight,
|
|
GovernanceFacts,
|
|
)
|
|
from .governance_checkpoint import (
|
|
CheckpointStore,
|
|
RecoveryCheckpoint,
|
|
RetryBudget,
|
|
)
|
|
from .governance_resolver import ExactTaskResolver, ResolutionQuery, ResolutionState
|
|
|
|
|
|
def _canonical(value) -> str:
|
|
return json.dumps(
|
|
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
|
)
|
|
|
|
|
|
def _digest(value) -> str:
|
|
return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GovernanceEvent:
|
|
kind: str
|
|
revision: str
|
|
occurred_at: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RecoveryReceipt:
|
|
receipt_ref: str
|
|
task_id: str
|
|
accepted: bool
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReconcileResult:
|
|
status: str
|
|
next_legal_action: str
|
|
task_id: str | None
|
|
blocker_code: str | None
|
|
mechanism_repair_required: bool
|
|
receipt_ref: str | None = None
|
|
|
|
|
|
class GovernanceFactsProvider(Protocol):
|
|
def snapshot(
|
|
self,
|
|
*,
|
|
task_id: str | None,
|
|
actor_identity: str,
|
|
operation: AdmissionOperation,
|
|
) -> GovernanceFacts: ...
|
|
|
|
|
|
class ExistingChildRecoveryExecutor(Protocol):
|
|
def recover_existing_child(
|
|
self,
|
|
*,
|
|
task_id: str,
|
|
actor_identity: str,
|
|
idempotency_key: str,
|
|
facts: GovernanceFacts,
|
|
) -> RecoveryReceipt: ...
|
|
|
|
# Optional at runtime. If implemented, this is preferred after an
|
|
# unknown-result write so the controller reconciles before re-executing.
|
|
def reconcile_recovery_result(
|
|
self,
|
|
*,
|
|
task_id: str,
|
|
actor_identity: str,
|
|
idempotency_key: str,
|
|
) -> RecoveryReceipt | None: ...
|
|
|
|
|
|
class GovernanceRecoveryController:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
goal_id: str,
|
|
query: ResolutionQuery,
|
|
actor_identity: str,
|
|
resolver: ExactTaskResolver,
|
|
facts_provider: GovernanceFactsProvider,
|
|
executor: ExistingChildRecoveryExecutor,
|
|
checkpoint_store: CheckpointStore,
|
|
retry_budget: RetryBudget = RetryBudget(),
|
|
) -> None:
|
|
self.goal_id = goal_id
|
|
self.query = query
|
|
self.actor_identity = actor_identity
|
|
self.resolver = resolver
|
|
self.facts_provider = facts_provider
|
|
self.executor = executor
|
|
self.checkpoints = checkpoint_store
|
|
self.retry_budget = retry_budget
|
|
self.preflight = AdmissionPreflight()
|
|
|
|
def _timestamp(self, event: GovernanceEvent) -> str:
|
|
return event.occurred_at or datetime.now(timezone.utc).isoformat()
|
|
|
|
def _save_blocked(
|
|
self,
|
|
*,
|
|
event: GovernanceEvent,
|
|
task_id: str | None,
|
|
mission_id: str | None,
|
|
root_task_id: str | None,
|
|
parent_task_id: str | None,
|
|
principal_id: str | None,
|
|
blocker_code: str,
|
|
blocker_details: tuple[str, ...],
|
|
) -> ReconcileResult:
|
|
previous = self.checkpoints.load()
|
|
fingerprint = self.retry_budget.blocker_fingerprint(
|
|
blocker_code, blocker_details
|
|
)
|
|
cycles, repair = self.retry_budget.observe(
|
|
previous=previous,
|
|
blocker_code=blocker_code,
|
|
blocker_fingerprint=fingerprint,
|
|
)
|
|
next_action = "REPAIR_MECHANISM" if repair else "WAIT_DEPENDENCY"
|
|
checkpoint = RecoveryCheckpoint(
|
|
schema="aios.governance-recovery-checkpoint.v1",
|
|
goal_id=self.goal_id,
|
|
repository=self.query.repository,
|
|
source_oid=self.query.base_oid,
|
|
candidate_digest=(
|
|
self.query.patch_digest
|
|
or self.query.artifact_sha256
|
|
or self.query.candidate_oid
|
|
or ""
|
|
),
|
|
task_id=task_id,
|
|
mission_id=mission_id,
|
|
root_task_id=root_task_id,
|
|
parent_task_id=parent_task_id,
|
|
principal_id=principal_id,
|
|
blocker_code=blocker_code,
|
|
blocker_fingerprint=fingerprint,
|
|
next_legal_action=next_action,
|
|
last_verified_at=self._timestamp(event),
|
|
evidence_refs=(f"event:{event.kind}:{event.revision}",),
|
|
unchanged_cycles=cycles,
|
|
mechanism_repair_required=repair,
|
|
)
|
|
self.checkpoints.save(checkpoint)
|
|
return ReconcileResult(
|
|
status="BLOCKED",
|
|
next_legal_action=next_action,
|
|
task_id=task_id,
|
|
blocker_code=blocker_code,
|
|
mechanism_repair_required=repair,
|
|
)
|
|
|
|
def _candidate_digest(self) -> str:
|
|
return (
|
|
self.query.patch_digest
|
|
or self.query.artifact_sha256
|
|
or self.query.candidate_oid
|
|
or ""
|
|
)
|
|
|
|
def _recovery_context_digest(
|
|
self, *, task_id: str, facts: GovernanceFacts
|
|
) -> str:
|
|
return _digest({
|
|
"task_id": task_id,
|
|
"repository": self.query.repository,
|
|
"candidate": self._candidate_digest(),
|
|
"principal": facts.authenticated_principal_id,
|
|
"branch": facts.target_branch,
|
|
"branch_oid": facts.expected_branch_oid,
|
|
"generation": facts.generation,
|
|
"task_lease_id": facts.task_lease_id,
|
|
"task_principal_id": facts.task_principal_id,
|
|
"writer_lease_id": facts.writer_lease_id,
|
|
"acceptance_revision": facts.acceptance_revision,
|
|
"file_allowlist": sorted(facts.accepted_file_allowlist),
|
|
})
|
|
|
|
def _finish_recovered(
|
|
self,
|
|
*,
|
|
event: GovernanceEvent,
|
|
resolved,
|
|
facts: GovernanceFacts,
|
|
idempotency_key: str,
|
|
context_digest: str,
|
|
receipt: RecoveryReceipt,
|
|
) -> ReconcileResult:
|
|
checkpoint = RecoveryCheckpoint(
|
|
schema="aios.governance-recovery-checkpoint.v1",
|
|
goal_id=self.goal_id,
|
|
repository=self.query.repository,
|
|
source_oid=self.query.base_oid,
|
|
candidate_digest=self._candidate_digest(),
|
|
task_id=resolved.task_id,
|
|
mission_id=resolved.mission_id,
|
|
root_task_id=resolved.root_task_id,
|
|
parent_task_id=resolved.parent_task_id,
|
|
principal_id=facts.authenticated_principal_id,
|
|
blocker_code=None,
|
|
blocker_fingerprint=None,
|
|
next_legal_action="ROOT_REVIEW",
|
|
last_verified_at=self._timestamp(event),
|
|
evidence_refs=(
|
|
f"event:{event.kind}:{event.revision}",
|
|
receipt.receipt_ref,
|
|
),
|
|
unchanged_cycles=0,
|
|
mechanism_repair_required=False,
|
|
recovery_idempotency_key=idempotency_key,
|
|
recovery_receipt_ref=receipt.receipt_ref,
|
|
recovery_context_digest=context_digest,
|
|
)
|
|
self.checkpoints.save(checkpoint)
|
|
return ReconcileResult(
|
|
status="RECOVERED",
|
|
next_legal_action="ROOT_REVIEW",
|
|
task_id=resolved.task_id,
|
|
blocker_code=None,
|
|
mechanism_repair_required=False,
|
|
receipt_ref=receipt.receipt_ref,
|
|
)
|
|
|
|
def reconcile(self, event: GovernanceEvent) -> ReconcileResult:
|
|
resolution = self.resolver.resolve(
|
|
self.query, actor_identity=self.actor_identity
|
|
)
|
|
if resolution.state is not ResolutionState.RESOLVED:
|
|
code = {
|
|
ResolutionState.NOT_FOUND: "TASK_NOT_RESOLVED",
|
|
ResolutionState.AMBIGUOUS: "TASK_RESOLUTION_AMBIGUOUS",
|
|
ResolutionState.INVALID_QUERY: "TASK_RESOLUTION_INVALID",
|
|
}[resolution.state]
|
|
details = (
|
|
resolution.detail,
|
|
*resolution.candidate_task_ids,
|
|
)
|
|
return self._save_blocked(
|
|
event=event,
|
|
task_id=None,
|
|
mission_id=None,
|
|
root_task_id=None,
|
|
parent_task_id=None,
|
|
principal_id=None,
|
|
blocker_code=code,
|
|
blocker_details=tuple(str(v) for v in details if v),
|
|
)
|
|
|
|
resolved = resolution.resolved
|
|
assert resolved is not None
|
|
facts = self.facts_provider.snapshot(
|
|
task_id=resolved.task_id,
|
|
actor_identity=self.actor_identity,
|
|
operation=AdmissionOperation.RECOVER_EXISTING_CHILD,
|
|
)
|
|
admission = self.preflight.evaluate(
|
|
resolution=resolution,
|
|
facts=facts,
|
|
operation=AdmissionOperation.RECOVER_EXISTING_CHILD,
|
|
)
|
|
if not admission.ready:
|
|
primary = admission.blockers[0]
|
|
details = tuple(
|
|
f"{blocker.code}:{blocker.detail}"
|
|
for blocker in admission.blockers
|
|
)
|
|
return self._save_blocked(
|
|
event=event,
|
|
task_id=resolved.task_id,
|
|
mission_id=resolved.mission_id,
|
|
root_task_id=resolved.root_task_id,
|
|
parent_task_id=resolved.parent_task_id,
|
|
principal_id=facts.authenticated_principal_id,
|
|
blocker_code=primary.code,
|
|
blocker_details=details,
|
|
)
|
|
|
|
context_digest = self._recovery_context_digest(
|
|
task_id=resolved.task_id, facts=facts
|
|
)
|
|
previous = self.checkpoints.load()
|
|
candidate_digest = self._candidate_digest()
|
|
unknown_previous = (
|
|
previous is not None
|
|
and previous.blocker_code == "RECOVERY_RESULT_UNKNOWN"
|
|
and previous.task_id == resolved.task_id
|
|
and previous.candidate_digest == candidate_digest
|
|
and bool(previous.recovery_idempotency_key)
|
|
)
|
|
|
|
if unknown_previous:
|
|
idempotency_key = previous.recovery_idempotency_key
|
|
reconcile = getattr(
|
|
self.executor, "reconcile_recovery_result", None
|
|
)
|
|
if callable(reconcile):
|
|
receipt = reconcile(
|
|
task_id=resolved.task_id,
|
|
actor_identity=self.actor_identity,
|
|
idempotency_key=idempotency_key,
|
|
)
|
|
if receipt is not None:
|
|
if receipt.accepted:
|
|
return self._finish_recovered(
|
|
event=event,
|
|
resolved=resolved,
|
|
facts=facts,
|
|
idempotency_key=idempotency_key,
|
|
context_digest=(
|
|
previous.recovery_context_digest
|
|
or context_digest
|
|
),
|
|
receipt=receipt,
|
|
)
|
|
return self._save_blocked(
|
|
event=event,
|
|
task_id=resolved.task_id,
|
|
mission_id=resolved.mission_id,
|
|
root_task_id=resolved.root_task_id,
|
|
parent_task_id=resolved.parent_task_id,
|
|
principal_id=facts.authenticated_principal_id,
|
|
blocker_code="RECOVERY_NOT_ACCEPTED",
|
|
blocker_details=(receipt.receipt_ref,),
|
|
)
|
|
if (
|
|
previous.recovery_context_digest is not None
|
|
and previous.recovery_context_digest != context_digest
|
|
):
|
|
checkpoint = RecoveryCheckpoint(
|
|
schema="aios.governance-recovery-checkpoint.v1",
|
|
goal_id=self.goal_id,
|
|
repository=self.query.repository,
|
|
source_oid=self.query.base_oid,
|
|
candidate_digest=candidate_digest,
|
|
task_id=resolved.task_id,
|
|
mission_id=resolved.mission_id,
|
|
root_task_id=resolved.root_task_id,
|
|
parent_task_id=resolved.parent_task_id,
|
|
principal_id=facts.authenticated_principal_id,
|
|
blocker_code="RECOVERY_RESULT_UNKNOWN",
|
|
blocker_fingerprint=_digest({
|
|
"task_id": resolved.task_id,
|
|
"idempotency_key": idempotency_key,
|
|
"context_changed": True,
|
|
}),
|
|
next_legal_action="RECOVER_EXISTING_CHILD",
|
|
last_verified_at=self._timestamp(event),
|
|
evidence_refs=(f"event:{event.kind}:{event.revision}",),
|
|
unchanged_cycles=1,
|
|
mechanism_repair_required=False,
|
|
recovery_idempotency_key=idempotency_key,
|
|
recovery_context_digest=(
|
|
previous.recovery_context_digest
|
|
),
|
|
)
|
|
self.checkpoints.save(checkpoint)
|
|
return ReconcileResult(
|
|
status="RESULT_UNKNOWN",
|
|
next_legal_action="RECOVER_EXISTING_CHILD",
|
|
task_id=resolved.task_id,
|
|
blocker_code="RECOVERY_RESULT_UNKNOWN",
|
|
mechanism_repair_required=False,
|
|
)
|
|
else:
|
|
idempotency_key = "recover-" + _digest({
|
|
"task_id": resolved.task_id,
|
|
"repository": self.query.repository,
|
|
"candidate": candidate_digest,
|
|
"branch_oid": facts.expected_branch_oid,
|
|
"generation": facts.generation,
|
|
"writer_lease_id": facts.writer_lease_id,
|
|
})[:40]
|
|
|
|
try:
|
|
receipt = self.executor.recover_existing_child(
|
|
task_id=resolved.task_id,
|
|
actor_identity=self.actor_identity,
|
|
idempotency_key=idempotency_key,
|
|
facts=facts,
|
|
)
|
|
except Exception:
|
|
# Unknown-result write: retain the SAME idempotency key for the next
|
|
# reconcile. Never assume the side effect did not happen.
|
|
checkpoint = RecoveryCheckpoint(
|
|
schema="aios.governance-recovery-checkpoint.v1",
|
|
goal_id=self.goal_id,
|
|
repository=self.query.repository,
|
|
source_oid=self.query.base_oid,
|
|
candidate_digest=(
|
|
self.query.patch_digest
|
|
or self.query.artifact_sha256
|
|
or self.query.candidate_oid
|
|
or ""
|
|
),
|
|
task_id=resolved.task_id,
|
|
mission_id=resolved.mission_id,
|
|
root_task_id=resolved.root_task_id,
|
|
parent_task_id=resolved.parent_task_id,
|
|
principal_id=facts.authenticated_principal_id,
|
|
blocker_code="RECOVERY_RESULT_UNKNOWN",
|
|
blocker_fingerprint=_digest({
|
|
"task_id": resolved.task_id,
|
|
"idempotency_key": idempotency_key,
|
|
}),
|
|
next_legal_action="RECOVER_EXISTING_CHILD",
|
|
last_verified_at=self._timestamp(event),
|
|
evidence_refs=(f"event:{event.kind}:{event.revision}",),
|
|
unchanged_cycles=1,
|
|
mechanism_repair_required=False,
|
|
recovery_idempotency_key=idempotency_key,
|
|
recovery_context_digest=context_digest,
|
|
)
|
|
self.checkpoints.save(checkpoint)
|
|
return ReconcileResult(
|
|
status="RESULT_UNKNOWN",
|
|
next_legal_action="RECOVER_EXISTING_CHILD",
|
|
task_id=resolved.task_id,
|
|
blocker_code="RECOVERY_RESULT_UNKNOWN",
|
|
mechanism_repair_required=False,
|
|
)
|
|
|
|
if not receipt.accepted:
|
|
return self._save_blocked(
|
|
event=event,
|
|
task_id=resolved.task_id,
|
|
mission_id=resolved.mission_id,
|
|
root_task_id=resolved.root_task_id,
|
|
parent_task_id=resolved.parent_task_id,
|
|
principal_id=facts.authenticated_principal_id,
|
|
blocker_code="RECOVERY_NOT_ACCEPTED",
|
|
blocker_details=(receipt.receipt_ref,),
|
|
)
|
|
|
|
return self._finish_recovered(
|
|
event=event,
|
|
resolved=resolved,
|
|
facts=facts,
|
|
idempotency_key=idempotency_key,
|
|
context_digest=context_digest,
|
|
receipt=receipt,
|
|
)
|
|
|
|
def on_event(self, event: GovernanceEvent) -> ReconcileResult:
|
|
# Event-driven primary path: reconcile immediately in the same call.
|
|
return self.reconcile(event)
|
|
|
|
def poll_once(self, *, revision: str, occurred_at: str) -> ReconcileResult:
|
|
# Polling is only a fallback and uses exactly the same reconcile path.
|
|
return self.reconcile(GovernanceEvent(
|
|
kind="POLL_FALLBACK",
|
|
revision=revision,
|
|
occurred_at=occurred_at,
|
|
))
|