289 lines
10 KiB
Python
289 lines
10 KiB
Python
"""Unified autonomous control-plane primitives.
|
|
|
|
This is deliberately a library, not another daemon. Task API, workers and host
|
|
recovery code consume the same Admission -> Invariant -> Repair semantics.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import asdict, dataclass
|
|
from enum import Enum
|
|
import re
|
|
from typing import Callable, Iterable, Mapping, Sequence
|
|
|
|
from aios_capability_contract import (
|
|
capability_contract,
|
|
has_capabilities,
|
|
normalize_required_capabilities,
|
|
normalize_worker_capabilities,
|
|
registered_capability,
|
|
validate_capability_contract_registry,
|
|
)
|
|
|
|
|
|
class FailureClass(str, Enum):
|
|
CAPABILITY_CONTRACT_ERROR = "CAPABILITY_CONTRACT_ERROR"
|
|
CREDENTIAL_PROJECTION_ERROR = "CREDENTIAL_PROJECTION_ERROR"
|
|
DEPENDENCY_UNAVAILABLE = "DEPENDENCY_UNAVAILABLE"
|
|
POLICY_DENIED = "POLICY_DENIED"
|
|
RESOURCE_UNAVAILABLE = "RESOURCE_UNAVAILABLE"
|
|
WORKER_CRASH = "WORKER_CRASH"
|
|
LEASE_LOST = "LEASE_LOST"
|
|
BUSINESS_PERMANENT = "BUSINESS_PERMANENT"
|
|
UNKNOWN = "UNKNOWN"
|
|
|
|
|
|
class RepairLevel(int, Enum):
|
|
AUTO_LOCAL = 0
|
|
AUTO_REPAIR_TASK = 1
|
|
OWNER_APPROVAL = 2
|
|
MANUAL = 3
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class InvariantResult:
|
|
name: str
|
|
passed: bool
|
|
detail: str = ""
|
|
|
|
def as_dict(self) -> dict[str, object]:
|
|
return asdict(self)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AdmissionDecision:
|
|
admitted: bool
|
|
state: str
|
|
invariants: tuple[InvariantResult, ...]
|
|
failure_class: FailureClass | None = None
|
|
repair_level: RepairLevel | None = None
|
|
repair_action: str | None = None
|
|
|
|
def as_dict(self) -> dict[str, object]:
|
|
return {
|
|
"schema": "aios.runtime-admission.decision.v1",
|
|
"admitted": self.admitted,
|
|
"state": self.state,
|
|
"invariants": [item.as_dict() for item in self.invariants],
|
|
"failure_class": None if self.failure_class is None else self.failure_class.value,
|
|
"repair_level": None if self.repair_level is None else int(self.repair_level),
|
|
"repair_action": self.repair_action,
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RepairDecision:
|
|
failure_class: FailureClass
|
|
level: RepairLevel
|
|
action: str
|
|
consumes_business_attempt: bool
|
|
|
|
def as_dict(self) -> dict[str, object]:
|
|
return {
|
|
"failure_class": self.failure_class.value,
|
|
"level": int(self.level),
|
|
"action": self.action,
|
|
"consumes_business_attempt": self.consumes_business_attempt,
|
|
}
|
|
|
|
|
|
def classify_failure(error: object, *, phase: str = "") -> FailureClass:
|
|
text = (str(error) + " " + str(phase)).lower()
|
|
if any(token in text for token in (
|
|
"credential", "permission denied", "token registry", "secret", "unauthorized"
|
|
)):
|
|
return FailureClass.CREDENTIAL_PROJECTION_ERROR
|
|
if any(token in text for token in (
|
|
"transport", "unavailable", "timeout", "connection", "bad gateway", "http_5"
|
|
)):
|
|
return FailureClass.DEPENDENCY_UNAVAILABLE
|
|
if any(token in text for token in (
|
|
"capability", "eligible", "matcher", "contract"
|
|
)):
|
|
return FailureClass.CAPABILITY_CONTRACT_ERROR
|
|
if any(token in text for token in (
|
|
"policy", "forbidden", "approval required", "not authorized"
|
|
)):
|
|
return FailureClass.POLICY_DENIED
|
|
if any(token in text for token in ("lease lost", "lease_expired", "heartbeat_failed")):
|
|
return FailureClass.LEASE_LOST
|
|
if any(token in text for token in ("out of memory", "disk full", "resource")):
|
|
return FailureClass.RESOURCE_UNAVAILABLE
|
|
if any(token in text for token in ("worker crash", "process exited", "signal")):
|
|
return FailureClass.WORKER_CRASH
|
|
if any(token in text for token in ("invalid release", "business permanent")):
|
|
return FailureClass.BUSINESS_PERMANENT
|
|
return FailureClass.UNKNOWN
|
|
|
|
|
|
def repair_for_failure(failure: FailureClass) -> RepairDecision:
|
|
table = {
|
|
FailureClass.CAPABILITY_CONTRACT_ERROR: RepairDecision(
|
|
failure, RepairLevel.AUTO_REPAIR_TASK,
|
|
"create_same_lineage_capability_contract_repair", False,
|
|
),
|
|
FailureClass.CREDENTIAL_PROJECTION_ERROR: RepairDecision(
|
|
failure, RepairLevel.AUTO_REPAIR_TASK,
|
|
"restore_service_private_credential_projection", False,
|
|
),
|
|
FailureClass.DEPENDENCY_UNAVAILABLE: RepairDecision(
|
|
failure, RepairLevel.AUTO_LOCAL,
|
|
"retry_dependency_health_and_resume", False,
|
|
),
|
|
FailureClass.RESOURCE_UNAVAILABLE: RepairDecision(
|
|
failure, RepairLevel.AUTO_LOCAL,
|
|
"heal_runtime_resource_and_resume", False,
|
|
),
|
|
FailureClass.WORKER_CRASH: RepairDecision(
|
|
failure, RepairLevel.AUTO_LOCAL,
|
|
"restart_worker_and_resume_same_lineage", False,
|
|
),
|
|
FailureClass.LEASE_LOST: RepairDecision(
|
|
failure, RepairLevel.AUTO_LOCAL,
|
|
"recover_lease_and_resume_same_lineage", False,
|
|
),
|
|
FailureClass.POLICY_DENIED: RepairDecision(
|
|
failure, RepairLevel.OWNER_APPROVAL,
|
|
"request_bound_owner_authorization", False,
|
|
),
|
|
FailureClass.BUSINESS_PERMANENT: RepairDecision(
|
|
failure, RepairLevel.MANUAL,
|
|
"terminal_business_failure", True,
|
|
),
|
|
FailureClass.UNKNOWN: RepairDecision(
|
|
failure, RepairLevel.AUTO_REPAIR_TASK,
|
|
"create_diagnostic_repair_task", False,
|
|
),
|
|
}
|
|
return table[failure]
|
|
|
|
|
|
def _probe(
|
|
name: str,
|
|
probes: Mapping[str, Callable[[], bool]],
|
|
) -> InvariantResult:
|
|
fn = probes.get(name)
|
|
if fn is None:
|
|
return InvariantResult(name, False, "PROBE_MISSING")
|
|
try:
|
|
passed = bool(fn())
|
|
except Exception as exc:
|
|
return InvariantResult(name, False, "PROBE_ERROR:" + type(exc).__name__)
|
|
return InvariantResult(name, passed, "PASS" if passed else "NOT_READY")
|
|
|
|
|
|
def runtime_admission(
|
|
*,
|
|
worker_capabilities: Iterable[object],
|
|
required_capabilities: Iterable[object],
|
|
requested_operations: Sequence[str] = (),
|
|
probes: Mapping[str, Callable[[], bool]] | None = None,
|
|
) -> AdmissionDecision:
|
|
"""Admission is evaluated before claim; failures do not consume attempts."""
|
|
|
|
worker_values = tuple(worker_capabilities)
|
|
required_values = tuple(required_capabilities)
|
|
probe_map = probes or {}
|
|
invariants: list[InvariantResult] = []
|
|
|
|
registry = validate_capability_contract_registry()
|
|
invariants.append(InvariantResult(
|
|
"capability_contract_registry",
|
|
bool(registry.get("valid")),
|
|
"PASS" if registry.get("valid") else "REGISTRY_INVALID",
|
|
))
|
|
|
|
task_known = all(registered_capability(item) for item in required_values)
|
|
worker_known = all(registered_capability(item, worker=True) for item in worker_values)
|
|
invariants.extend((
|
|
InvariantResult("task_capabilities_registered", task_known, "PASS" if task_known else "UNKNOWN_TASK_CAPABILITY"),
|
|
InvariantResult("worker_capabilities_registered", worker_known, "PASS" if worker_known else "UNKNOWN_WORKER_CAPABILITY"),
|
|
))
|
|
|
|
capability_match = (
|
|
task_known
|
|
and worker_known
|
|
and has_capabilities(worker_values, required_values)
|
|
)
|
|
invariants.append(InvariantResult(
|
|
"capability_match",
|
|
capability_match,
|
|
"PASS" if capability_match else "WORKER_NOT_ELIGIBLE",
|
|
))
|
|
|
|
normalized = normalize_required_capabilities(required_values)
|
|
contracts = [
|
|
capability_contract(item)
|
|
for item in normalized
|
|
if not str(item).startswith("exact:")
|
|
]
|
|
contracts = [item for item in contracts if item is not None]
|
|
|
|
forbidden: set[str] = set()
|
|
dependencies: set[str] = set()
|
|
health: set[str] = set()
|
|
for contract in contracts:
|
|
forbidden.update(contract.forbidden)
|
|
dependencies.update(contract.dependencies)
|
|
health.update(contract.health_requirements)
|
|
|
|
requested = {str(item).strip().lower() for item in requested_operations if str(item).strip()}
|
|
forbidden_hit = sorted(requested & {item.lower() for item in forbidden})
|
|
invariants.append(InvariantResult(
|
|
"forbidden_operations_absent",
|
|
not forbidden_hit,
|
|
"PASS" if not forbidden_hit else "FORBIDDEN:" + ",".join(forbidden_hit),
|
|
))
|
|
|
|
for name in sorted(dependencies | health):
|
|
invariants.append(_probe(name, probe_map))
|
|
|
|
failed = [item for item in invariants if not item.passed]
|
|
if not failed:
|
|
return AdmissionDecision(
|
|
True,
|
|
"EXECUTION_READY",
|
|
tuple(invariants),
|
|
)
|
|
|
|
details = " ".join(item.name + ":" + item.detail for item in failed)
|
|
failure = classify_failure(details, phase="admission")
|
|
if any(item.name == "forbidden_operations_absent" and not item.passed for item in failed):
|
|
failure = FailureClass.POLICY_DENIED
|
|
elif any("credential" in item.name for item in failed):
|
|
failure = FailureClass.CREDENTIAL_PROJECTION_ERROR
|
|
elif any(item.name in {"capability_contract_registry", "task_capabilities_registered", "worker_capabilities_registered", "capability_match"} for item in failed):
|
|
failure = FailureClass.CAPABILITY_CONTRACT_ERROR
|
|
elif any(item.name in dependencies or item.name in health for item in failed):
|
|
failure = FailureClass.DEPENDENCY_UNAVAILABLE
|
|
|
|
repair = repair_for_failure(failure)
|
|
return AdmissionDecision(
|
|
False,
|
|
"EXECUTION_READY_FAILED",
|
|
tuple(invariants),
|
|
failure,
|
|
repair.level,
|
|
repair.action,
|
|
)
|
|
|
|
|
|
def system_invariant_snapshot(
|
|
*,
|
|
worker_capabilities: Iterable[object],
|
|
required_capabilities: Iterable[object],
|
|
requested_operations: Sequence[str] = (),
|
|
probes: Mapping[str, Callable[[], bool]] | None = None,
|
|
) -> dict[str, object]:
|
|
decision = runtime_admission(
|
|
worker_capabilities=worker_capabilities,
|
|
required_capabilities=required_capabilities,
|
|
requested_operations=requested_operations,
|
|
probes=probes,
|
|
)
|
|
return {
|
|
"schema": "aios.system-invariants.v1",
|
|
"status": "HEALTHY" if decision.admitted else "DEGRADED",
|
|
"execution_ready": decision.admitted,
|
|
"admission": decision.as_dict(),
|
|
}
|