104 lines
3.4 KiB
Python
104 lines
3.4 KiB
Python
"""Health policy and self-healing decisions for OpenClaw runtimes."""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from typing import Mapping
|
|
|
|
from .contract import RuntimeHealth
|
|
|
|
|
|
def _parse(value: str) -> datetime:
|
|
normalized = value[:-1] + "+00:00" if value.endswith("Z") else value
|
|
parsed = datetime.fromisoformat(normalized)
|
|
if parsed.tzinfo is None:
|
|
raise ValueError("heartbeat timestamp must include timezone")
|
|
return parsed.astimezone(timezone.utc)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class HealthPolicy:
|
|
max_heartbeat_age_seconds: int = 90
|
|
require_workspace_isolation: bool = True
|
|
require_credential_ref: bool = True
|
|
accepted_statuses: frozenset[str] = field(
|
|
default_factory=lambda: frozenset({"healthy", "ready"})
|
|
)
|
|
|
|
def __post_init__(self) -> None:
|
|
if self.max_heartbeat_age_seconds < 5:
|
|
raise ValueError("max_heartbeat_age_seconds must be >= 5")
|
|
if not self.accepted_statuses:
|
|
raise ValueError("accepted_statuses must not be empty")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class HealthAssessment:
|
|
healthy: bool
|
|
reasons: tuple[str, ...]
|
|
recoverable: bool
|
|
recommended_action: str
|
|
details: Mapping[str, object] = field(default_factory=dict)
|
|
|
|
|
|
class RuntimeHealthMonitor:
|
|
"""Evaluate runtime health and recommend bounded recovery actions."""
|
|
|
|
def __init__(self, policy: HealthPolicy | None = None) -> None:
|
|
self.policy = policy or HealthPolicy()
|
|
|
|
def assess(
|
|
self,
|
|
health: RuntimeHealth,
|
|
*,
|
|
now: datetime | None = None,
|
|
) -> HealthAssessment:
|
|
current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc)
|
|
heartbeat = _parse(health.heartbeat_at)
|
|
age = max(0.0, current.timestamp() - heartbeat.timestamp())
|
|
reasons: list[str] = []
|
|
|
|
if health.status.lower() not in self.policy.accepted_statuses:
|
|
reasons.append("runtime_status_unhealthy")
|
|
if age > self.policy.max_heartbeat_age_seconds:
|
|
reasons.append("heartbeat_stale")
|
|
if self.policy.require_workspace_isolation and not health.workspace_isolated:
|
|
reasons.append("workspace_not_isolated")
|
|
if self.policy.require_credential_ref and not health.credential_ref_configured:
|
|
reasons.append("credential_reference_missing")
|
|
|
|
if not reasons:
|
|
return HealthAssessment(
|
|
healthy=True,
|
|
reasons=(),
|
|
recoverable=True,
|
|
recommended_action="none",
|
|
details={"heartbeat_age_seconds": age},
|
|
)
|
|
|
|
non_recoverable = {
|
|
"workspace_not_isolated",
|
|
"credential_reference_missing",
|
|
}.intersection(reasons)
|
|
if non_recoverable:
|
|
action = "quarantine_runtime"
|
|
recoverable = False
|
|
elif "heartbeat_stale" in reasons:
|
|
action = "restart_runtime"
|
|
recoverable = True
|
|
else:
|
|
action = "drain_and_restart"
|
|
recoverable = True
|
|
|
|
return HealthAssessment(
|
|
healthy=False,
|
|
reasons=tuple(reasons),
|
|
recoverable=recoverable,
|
|
recommended_action=action,
|
|
details={
|
|
"heartbeat_age_seconds": age,
|
|
"runtime_id": health.runtime_id,
|
|
"active_tasks": health.active_tasks,
|
|
},
|
|
)
|