Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
189b655d04 | ||
|
|
5f2b5f16ba | ||
|
|
e2d9de3a1f | ||
|
|
83cbd64fb6 | ||
|
|
7852844338 | ||
|
|
46b9d3188d | ||
|
|
c69916442c | ||
|
|
781239bea7 | ||
|
|
193f6546b1 | ||
|
|
e063c6cb3c | ||
|
|
9f9f7be796 | ||
|
|
77d0d7176d | ||
|
|
0029980d74 | ||
|
|
0ed265e3cb | ||
|
|
c7a1cf5b2a | ||
|
|
d6a6491d2b | ||
|
|
e364432a1f | ||
|
|
343b6f1ba6 | ||
|
|
e1d4bfc77b | ||
|
|
f7cf7249f1 | ||
|
|
13846c222f | ||
|
|
749afecf08 | ||
|
|
b424de9d98 | ||
|
|
b761536f16 |
@@ -0,0 +1,38 @@
|
||||
"""Autonomous Controller health adapter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from .blocker_task_adapter import BlockerTaskAdapter
|
||||
from .git_permission_watchdog import GitPermissionWatchdog
|
||||
from .system_blocker_memory import SystemBlockerMemory
|
||||
from .system_health_center import SystemHealthCenter
|
||||
|
||||
|
||||
class AutonomousHealthAdapter:
|
||||
def __init__(self):
|
||||
self.git = GitPermissionWatchdog()
|
||||
self.memory = SystemBlockerMemory()
|
||||
self.task_adapter = BlockerTaskAdapter()
|
||||
self.health = SystemHealthCenter()
|
||||
|
||||
def check(self, previous_blockers: list[dict[str, Any]] | None = None) -> dict[str, Any]:
|
||||
git_report = self.git.scan().to_dict()
|
||||
blockers = list(previous_blockers or [])
|
||||
if git_report.get("status") != "healthy":
|
||||
blockers.append({
|
||||
"blocker_id": "git-permission",
|
||||
"category": "git_permission",
|
||||
"title": "Git object permission drift",
|
||||
"severity": "p0",
|
||||
"evidence": [str(git_report)],
|
||||
})
|
||||
return self.health.build(git_report=git_report, blockers=blockers)
|
||||
|
||||
def optimize_task(self, blocker: dict[str, Any]) -> dict[str, Any]:
|
||||
decision = self.memory.decide(blocker)
|
||||
return self.task_adapter.to_task(blocker, decision.to_dict())
|
||||
|
||||
|
||||
__all__ = ["AutonomousHealthAdapter"]
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Convert recurring blockers into Task Center tasks.
|
||||
|
||||
When a blocker becomes recurrent or critical, it should not remain only as an
|
||||
incident log. It becomes a visible task with owner/executor/progress/approval
|
||||
state in the Task Center.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
class BlockerTaskAdapter:
|
||||
def to_task(self, blocker: dict[str, Any], decision: dict[str, Any]) -> dict[str, Any]:
|
||||
priority = decision.get("priority", "P2")
|
||||
status = "waiting_approval" if decision.get("owner_gate_required") else "pending"
|
||||
if priority == "P0":
|
||||
status = "running"
|
||||
return {
|
||||
"task_id": f"blocker-{blocker.get('fingerprint', blocker.get('blocker_id', 'unknown'))}",
|
||||
"title": f"Resolve blocker: {blocker.get('title', 'unknown')}",
|
||||
"assigned_by": "system-blocker-memory",
|
||||
"executor": decision.get("assigned_to", "autonomous-controller"),
|
||||
"source": "system_blocker",
|
||||
"status": status,
|
||||
"priority": priority,
|
||||
"created_at": utc_now(),
|
||||
"updated_at": utc_now(),
|
||||
"progress": {
|
||||
"percent": 20 if status == "running" else 0,
|
||||
"current_step": decision.get("action", "record_and_monitor"),
|
||||
"completed_steps": ["blocker_recorded", "decision_created"],
|
||||
"remaining_steps": self._remaining_steps(decision),
|
||||
"blocked_reason": blocker.get("title") if status == "waiting_approval" else None,
|
||||
},
|
||||
"notification": {
|
||||
"required": True,
|
||||
"sent": False,
|
||||
"channel": "wechat",
|
||||
},
|
||||
"owner_gate_required": bool(decision.get("owner_gate_required")),
|
||||
"owner_gate_status": "pending" if decision.get("owner_gate_required") else None,
|
||||
"evidence": blocker.get("evidence", []),
|
||||
"metadata": {
|
||||
"blocker": blocker,
|
||||
"decision": decision,
|
||||
},
|
||||
}
|
||||
|
||||
def _remaining_steps(self, decision: dict[str, Any]) -> list[str]:
|
||||
steps = ["root_cause_analysis", "patch_or_repair", "test", "deploy", "verify", "close"]
|
||||
if decision.get("architect_review_required"):
|
||||
steps.insert(1, "architect_review")
|
||||
if decision.get("owner_gate_required"):
|
||||
steps.insert(2, "owner_approval")
|
||||
return steps
|
||||
|
||||
|
||||
__all__ = ["BlockerTaskAdapter"]
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Command watcher stall detector.
|
||||
|
||||
Detects commands that remain in active/running states beyond the configured
|
||||
threshold and emits blocker records so AIOS can self-optimize instead of waiting
|
||||
for manual progress checks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def parse_ts(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CommandStallRecord:
|
||||
command_id: str
|
||||
path: str
|
||||
bucket: str
|
||||
age_minutes: float
|
||||
threshold_minutes: int
|
||||
severity: str
|
||||
detected_at: str = field(default_factory=utc_now)
|
||||
|
||||
def to_blocker(self) -> dict[str, Any]:
|
||||
return {
|
||||
"blocker_id": f"command-stall-{self.command_id}",
|
||||
"category": "command_stuck",
|
||||
"title": f"Command stalled: {self.command_id}",
|
||||
"source": "command-stall-detector",
|
||||
"severity": self.severity,
|
||||
"status": "open",
|
||||
"first_seen_at": self.detected_at,
|
||||
"last_seen_at": self.detected_at,
|
||||
"occurrence_count": 1,
|
||||
"evidence": [self.path],
|
||||
"metadata": self.to_dict(),
|
||||
}
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
class CommandStallDetector:
|
||||
def __init__(self, root: str = "agents/commands", threshold_minutes: int = 15):
|
||||
self.root = Path(root)
|
||||
self.threshold_minutes = threshold_minutes
|
||||
|
||||
def detect(self) -> list[CommandStallRecord]:
|
||||
records: list[CommandStallRecord] = []
|
||||
for bucket in ("active", "running"):
|
||||
bucket_path = self.root / bucket
|
||||
if not bucket_path.exists():
|
||||
continue
|
||||
for path in bucket_path.glob("*.md"):
|
||||
age = self._age_minutes(path)
|
||||
if age >= self.threshold_minutes:
|
||||
records.append(
|
||||
CommandStallRecord(
|
||||
command_id=path.stem,
|
||||
path=str(path),
|
||||
bucket=bucket,
|
||||
age_minutes=round(age, 2),
|
||||
threshold_minutes=self.threshold_minutes,
|
||||
severity="p0" if bucket == "running" else "high",
|
||||
)
|
||||
)
|
||||
return records
|
||||
|
||||
def _age_minutes(self, path: Path) -> float:
|
||||
stat = path.stat()
|
||||
age_seconds = datetime.now(timezone.utc).timestamp() - stat.st_mtime
|
||||
return age_seconds / 60
|
||||
|
||||
|
||||
__all__ = ["CommandStallRecord", "CommandStallDetector"]
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Git object permission watchdog and self-healing.
|
||||
|
||||
Prevents AIOS autonomous operation from being blocked by recurring
|
||||
`.git/objects` ownership drift. The repair is intentionally conservative and
|
||||
limited to the workspace Git object database.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import pwd
|
||||
import grp
|
||||
import subprocess
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GitPermissionIssue:
|
||||
path: str
|
||||
issue: str
|
||||
actual_uid: int | None = None
|
||||
actual_gid: int | None = None
|
||||
expected_user: str = "aiman"
|
||||
expected_group: str = "gitops"
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GitPermissionReport:
|
||||
workspace: str
|
||||
expected_user: str
|
||||
expected_group: str
|
||||
status: str
|
||||
issues: list[GitPermissionIssue] = field(default_factory=list)
|
||||
checked_at: str = field(default_factory=utc_now)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"workspace": self.workspace,
|
||||
"expected_user": self.expected_user,
|
||||
"expected_group": self.expected_group,
|
||||
"status": self.status,
|
||||
"issues": [issue.to_dict() for issue in self.issues],
|
||||
"checked_at": self.checked_at,
|
||||
}
|
||||
|
||||
|
||||
class GitPermissionWatchdog:
|
||||
def __init__(
|
||||
self,
|
||||
workspace: str = "/data/aios-gitops/workspace",
|
||||
expected_user: str = "aiman",
|
||||
expected_group: str = "gitops",
|
||||
max_scan_depth: int = 2,
|
||||
):
|
||||
self.workspace = Path(workspace)
|
||||
self.expected_user = expected_user
|
||||
self.expected_group = expected_group
|
||||
self.max_scan_depth = max_scan_depth
|
||||
|
||||
@property
|
||||
def objects_dir(self) -> Path:
|
||||
return self.workspace / ".git" / "objects"
|
||||
|
||||
def scan(self) -> GitPermissionReport:
|
||||
issues: list[GitPermissionIssue] = []
|
||||
if not self.objects_dir.exists():
|
||||
issues.append(GitPermissionIssue(str(self.objects_dir), "missing_objects_dir"))
|
||||
return GitPermissionReport(str(self.workspace), self.expected_user, self.expected_group, "blocked", issues)
|
||||
|
||||
expected_uid = pwd.getpwnam(self.expected_user).pw_uid
|
||||
expected_gid = grp.getgrnam(self.expected_group).gr_gid
|
||||
|
||||
for path in self._walk_limited(self.objects_dir):
|
||||
try:
|
||||
stat = path.stat()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if stat.st_uid != expected_uid:
|
||||
issues.append(GitPermissionIssue(str(path), "owner_drift", stat.st_uid, stat.st_gid, self.expected_user, self.expected_group))
|
||||
if stat.st_gid != expected_gid:
|
||||
issues.append(GitPermissionIssue(str(path), "group_drift", stat.st_uid, stat.st_gid, self.expected_user, self.expected_group))
|
||||
if not os.access(path, os.W_OK):
|
||||
issues.append(GitPermissionIssue(str(path), "not_writable", stat.st_uid, stat.st_gid, self.expected_user, self.expected_group))
|
||||
if len(issues) >= 200:
|
||||
break
|
||||
|
||||
return GitPermissionReport(
|
||||
workspace=str(self.workspace),
|
||||
expected_user=self.expected_user,
|
||||
expected_group=self.expected_group,
|
||||
status="healthy" if not issues else "drift_detected",
|
||||
issues=issues,
|
||||
)
|
||||
|
||||
def repair(self, dry_run: bool = False) -> dict[str, Any]:
|
||||
report = self.scan()
|
||||
if report.status == "healthy":
|
||||
return {"status": "healthy", "report": report.to_dict(), "repaired": False}
|
||||
if dry_run:
|
||||
return {"status": "dry_run", "report": report.to_dict(), "repaired": False}
|
||||
commands = [
|
||||
["chown", "-R", f"{self.expected_user}:{self.expected_group}", str(self.objects_dir)],
|
||||
["chmod", "-R", "u+rwX,g+rwX", str(self.objects_dir)],
|
||||
]
|
||||
results = []
|
||||
for command in commands:
|
||||
proc = subprocess.run(command, text=True, capture_output=True, check=False)
|
||||
results.append({"cmd": command, "returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr})
|
||||
if proc.returncode != 0:
|
||||
return {"status": "repair_failed", "report": report.to_dict(), "results": results, "repaired": False}
|
||||
post = self.scan()
|
||||
return {"status": "repaired" if post.status == "healthy" else "repair_incomplete", "report": post.to_dict(), "results": results, "repaired": post.status == "healthy"}
|
||||
|
||||
def write_probe(self) -> dict[str, Any]:
|
||||
probe = self.objects_dir / f".aios-write-probe-{datetime.now(timezone.utc).timestamp()}"
|
||||
try:
|
||||
probe.write_text("probe\n", encoding="utf-8")
|
||||
probe.unlink(missing_ok=True)
|
||||
return {"status": "ok", "path": str(probe)}
|
||||
except Exception as exc: # pragma: no cover - depends on runtime perms
|
||||
return {"status": "failed", "path": str(probe), "error": str(exc)}
|
||||
|
||||
def _walk_limited(self, root: Path):
|
||||
yield root
|
||||
base_depth = len(root.parts)
|
||||
for path in root.rglob("*"):
|
||||
if len(path.parts) - base_depth <= self.max_scan_depth:
|
||||
yield path
|
||||
|
||||
|
||||
__all__ = ["GitPermissionIssue", "GitPermissionReport", "GitPermissionWatchdog"]
|
||||
@@ -0,0 +1,31 @@
|
||||
"""CLI entrypoint for Git Permission Watchdog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from .git_permission_watchdog import GitPermissionWatchdog
|
||||
|
||||
|
||||
def main() -> int:
|
||||
workspace = os.getenv("AIOS_GITOPS_WORKSPACE", "/data/aios-gitops/workspace")
|
||||
expected_user = os.getenv("AIOS_GIT_EXPECTED_USER", "aiman")
|
||||
expected_group = os.getenv("AIOS_GIT_EXPECTED_GROUP", "gitops")
|
||||
audit_dir = Path(os.getenv("AIOS_GIT_PERMISSION_AUDIT_DIR", "runtime/git-permission-watchdog"))
|
||||
audit_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
watchdog = GitPermissionWatchdog(workspace, expected_user, expected_group)
|
||||
result = watchdog.repair(dry_run=os.getenv("AIOS_GIT_PERMISSION_DRY_RUN") == "1")
|
||||
probe = watchdog.write_probe()
|
||||
payload = {"repair": result, "write_probe": probe}
|
||||
status = result.get("status")
|
||||
out = audit_dir / "latest.json"
|
||||
out.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(json.dumps(payload, ensure_ascii=False))
|
||||
return 0 if status in {"healthy", "repaired"} and probe.get("status") == "ok" else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Unified AIOS operations command center view model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
class OperationsCommandCenter:
|
||||
def build(
|
||||
self,
|
||||
tasks: list[dict[str, Any]] | None = None,
|
||||
health: dict[str, Any] | None = None,
|
||||
blockers: list[dict[str, Any]] | None = None,
|
||||
searches: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
tasks = tasks or []
|
||||
blockers = blockers or []
|
||||
searches = searches or []
|
||||
return {
|
||||
"module": "operations-command-center",
|
||||
"route": "/admin/ai-evolution/operations",
|
||||
"title": "AIOS 自治运营驾驶舱",
|
||||
"generated_at": utc_now(),
|
||||
"health": health or {},
|
||||
"tasks": {
|
||||
"active": len([t for t in tasks if t.get("status") in {"running", "pending", "waiting_approval"}]),
|
||||
"blocked": len([t for t in tasks if t.get("status") == "blocked"]),
|
||||
"completed": len([t for t in tasks if t.get("status") == "completed"]),
|
||||
},
|
||||
"blockers": {
|
||||
"total": len(blockers),
|
||||
"p0": len([b for b in blockers if b.get("severity") in {"p0", "critical"}]),
|
||||
"recurring": len([b for b in blockers if b.get("mechanism_gap")]),
|
||||
},
|
||||
"search": {
|
||||
"recent": len(searches),
|
||||
},
|
||||
"loops": {
|
||||
"task_loop": True,
|
||||
"health_loop": True,
|
||||
"optimization_loop": True,
|
||||
"notification_loop": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["OperationsCommandCenter"]
|
||||
@@ -0,0 +1,120 @@
|
||||
"""System blocker memory and recurring issue detection.
|
||||
|
||||
Records blocking issues, long-no-response incidents and repeated operational
|
||||
failures. Repeated blockers are escalated from incident handling into mechanism
|
||||
optimization work.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SystemBlockerRecord:
|
||||
blocker_id: str
|
||||
category: str
|
||||
title: str
|
||||
source: str
|
||||
severity: str
|
||||
status: str = "open"
|
||||
first_seen_at: str = field(default_factory=utc_now)
|
||||
last_seen_at: str = field(default_factory=utc_now)
|
||||
occurrence_count: int = 1
|
||||
evidence: list[str] = field(default_factory=list)
|
||||
mechanism_gap: bool = False
|
||||
recommended_owner: str = "autonomous-controller"
|
||||
|
||||
def fingerprint(self) -> str:
|
||||
return f"{self.category}:{self.title}".lower().replace(" ", "_")
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self) | {"fingerprint": self.fingerprint()}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BlockerEscalationDecision:
|
||||
status: str
|
||||
priority: str
|
||||
reason: str
|
||||
action: str
|
||||
assigned_to: str
|
||||
create_task: bool = True
|
||||
architect_review_required: bool = False
|
||||
owner_gate_required: bool = False
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
class SystemBlockerMemory:
|
||||
"""In-memory decision logic; production storage can wrap this with persistence."""
|
||||
|
||||
def record(self, existing: list[dict[str, Any]], incoming: SystemBlockerRecord) -> dict[str, Any]:
|
||||
records = [dict(item) for item in existing]
|
||||
fp = incoming.fingerprint()
|
||||
for item in records:
|
||||
if item.get("fingerprint") == fp or f"{item.get('category')}:{item.get('title')}".lower().replace(" ", "_") == fp:
|
||||
item["occurrence_count"] = int(item.get("occurrence_count", 1)) + 1
|
||||
item["last_seen_at"] = utc_now()
|
||||
item["status"] = "recurrent"
|
||||
evidence = list(item.get("evidence", [])) + incoming.evidence
|
||||
item["evidence"] = list(dict.fromkeys(evidence))
|
||||
item["mechanism_gap"] = self.is_mechanism_gap(item)
|
||||
item["fingerprint"] = fp
|
||||
return item
|
||||
item = incoming.to_dict()
|
||||
item["mechanism_gap"] = self.is_mechanism_gap(item)
|
||||
records.append(item)
|
||||
return item
|
||||
|
||||
def decide(self, blocker: dict[str, Any]) -> BlockerEscalationDecision:
|
||||
count = int(blocker.get("occurrence_count", 1))
|
||||
severity = blocker.get("severity", "medium")
|
||||
category = blocker.get("category", "unknown")
|
||||
if severity in {"critical", "p0"} or category in {"git_permission", "runtime_deadlock", "autonomous_controller_stall"}:
|
||||
return BlockerEscalationDecision(
|
||||
status="escalate_to_mechanism_fix",
|
||||
priority="P0",
|
||||
reason="blocking infrastructure issue affects autonomous operation",
|
||||
action="create_self_healing_mechanism",
|
||||
assigned_to="openclaw",
|
||||
architect_review_required=True,
|
||||
)
|
||||
if count >= 3:
|
||||
return BlockerEscalationDecision(
|
||||
status="escalate_to_mechanism_fix",
|
||||
priority="P1",
|
||||
reason="recurring issue exceeded threshold",
|
||||
action="create_root_cause_and_prevention_patch",
|
||||
assigned_to="autonomous-controller",
|
||||
architect_review_required=True,
|
||||
)
|
||||
if category == "long_no_response":
|
||||
return BlockerEscalationDecision(
|
||||
status="create_monitoring_task",
|
||||
priority="P1",
|
||||
reason="long no response requires watchdog and notification",
|
||||
action="create_stall_watchdog_task",
|
||||
assigned_to="openclaw",
|
||||
)
|
||||
return BlockerEscalationDecision(
|
||||
status="track_incident",
|
||||
priority="P2",
|
||||
reason="single incident tracked for recurrence",
|
||||
action="record_and_monitor",
|
||||
assigned_to="autonomous-controller",
|
||||
create_task=False,
|
||||
)
|
||||
|
||||
def is_mechanism_gap(self, blocker: dict[str, Any]) -> bool:
|
||||
return int(blocker.get("occurrence_count", 1)) >= 2 or blocker.get("severity") in {"critical", "p0"}
|
||||
|
||||
|
||||
__all__ = ["SystemBlockerRecord", "BlockerEscalationDecision", "SystemBlockerMemory"]
|
||||
@@ -0,0 +1,32 @@
|
||||
"""System Health Center API."""
|
||||
|
||||
try:
|
||||
from fastapi import APIRouter
|
||||
except Exception:
|
||||
APIRouter = None
|
||||
|
||||
from .system_health_center import SystemHealthCenter
|
||||
|
||||
|
||||
center = SystemHealthCenter()
|
||||
|
||||
|
||||
def get_system_health_overview():
|
||||
return center.build()
|
||||
|
||||
|
||||
def build_router():
|
||||
if APIRouter is None:
|
||||
return None
|
||||
router = APIRouter(prefix="/admin/ai-evolution/system-health", tags=["system-health"])
|
||||
|
||||
@router.get("/overview", operation_id="getSystemHealthOverview")
|
||||
def overview():
|
||||
return get_system_health_overview()
|
||||
|
||||
return router
|
||||
|
||||
|
||||
router = build_router()
|
||||
|
||||
__all__ = ["router", "get_system_health_overview"]
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Unified AIOS system health center model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
class SystemHealthCenter:
|
||||
def build(
|
||||
self,
|
||||
git_report: dict[str, Any] | None = None,
|
||||
blockers: list[dict[str, Any]] | None = None,
|
||||
tasks: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
blockers = blockers or []
|
||||
tasks = tasks or []
|
||||
return {
|
||||
"module": "system-health-center",
|
||||
"route": "/admin/ai-evolution/system-health",
|
||||
"title": "AIOS 系统健康与自优化中心",
|
||||
"generated_at": utc_now(),
|
||||
"status": self._status(git_report, blockers),
|
||||
"git": git_report or {},
|
||||
"blockers": {
|
||||
"total": len(blockers),
|
||||
"recurrent": len([b for b in blockers if b.get("mechanism_gap")]),
|
||||
"p0": len([b for b in blockers if b.get("severity") in {"p0", "critical"}]),
|
||||
},
|
||||
"tasks": {
|
||||
"optimization_tasks": len([t for t in tasks if t.get("source") == "system_blocker"]),
|
||||
"pending": len([t for t in tasks if t.get("status") in {"pending", "running", "waiting_approval"}]),
|
||||
},
|
||||
"optimization_loop": {
|
||||
"detect": True,
|
||||
"remember": True,
|
||||
"optimize": True,
|
||||
"verify": True,
|
||||
},
|
||||
}
|
||||
|
||||
def _status(self, git_report, blockers):
|
||||
if git_report and git_report.get("status") not in {"healthy", "ok"}:
|
||||
return "attention_required"
|
||||
if any(b.get("severity") in {"p0", "critical"} for b in blockers):
|
||||
return "attention_required"
|
||||
return "healthy"
|
||||
|
||||
|
||||
__all__ = ["SystemHealthCenter"]
|
||||
@@ -0,0 +1,35 @@
|
||||
git_permission_watchdog:
|
||||
enabled: true
|
||||
priority: P0
|
||||
workspace: /data/aios-gitops/workspace
|
||||
|
||||
expected_identity:
|
||||
user: aiman
|
||||
group: gitops
|
||||
|
||||
checks:
|
||||
- object_owner
|
||||
- object_group
|
||||
- object_write_permission
|
||||
- git_write_probe
|
||||
|
||||
repair:
|
||||
enabled: true
|
||||
allowed_paths:
|
||||
- .git/objects
|
||||
forbidden:
|
||||
- chmod_777
|
||||
- system_wide_permission_change
|
||||
|
||||
runtime:
|
||||
interval_minutes: 5
|
||||
create_task_on_drift: true
|
||||
send_notification_on_failure: true
|
||||
|
||||
audit:
|
||||
enabled: true
|
||||
record:
|
||||
- detection_time
|
||||
- affected_paths
|
||||
- repair_action
|
||||
- repair_result
|
||||
@@ -0,0 +1,37 @@
|
||||
system_blocker_optimization:
|
||||
enabled: true
|
||||
purpose: convert_recurring_failures_into_mechanism_improvements
|
||||
|
||||
tracked_categories:
|
||||
- git_permission
|
||||
- runtime_stall
|
||||
- command_stuck
|
||||
- provider_failure
|
||||
- deployment_failure
|
||||
- approval_wait
|
||||
- notification_failure
|
||||
- agent_timeout
|
||||
|
||||
thresholds:
|
||||
repeat_count_for_mechanism_review: 3
|
||||
critical_issue_immediate_review: true
|
||||
long_no_response_minutes: 15
|
||||
|
||||
actions:
|
||||
first_occurrence:
|
||||
- record_incident
|
||||
- monitor
|
||||
repeated_occurrence:
|
||||
- create_optimization_task
|
||||
- assign_autonomous_controller
|
||||
- require_architect_review
|
||||
critical_occurrence:
|
||||
- create_p0_task
|
||||
- notify_owner
|
||||
- self_healing_attempt
|
||||
|
||||
integration:
|
||||
task_center: true
|
||||
audit_plane: true
|
||||
event_bus: true
|
||||
architect_agent: true
|
||||
@@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=AIOS Git Permission Watchdog
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=aiman
|
||||
Group=gitops
|
||||
WorkingDirectory=/data/aios-gitops/workspace
|
||||
ExecStart=/usr/bin/python3 -m active_runtime.app.git_permission_watchdog_cli
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Run AIOS Git Permission Watchdog periodically
|
||||
|
||||
[Timer]
|
||||
OnBootSec=5min
|
||||
OnUnitActiveSec=5min
|
||||
Unit=aios-git-permission-watchdog.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,35 @@
|
||||
# CMD-20260709-enable-autonomous-operation-command-center
|
||||
|
||||
## Goal
|
||||
|
||||
Create unified AIOS autonomous operations command center.
|
||||
|
||||
## Integrate
|
||||
|
||||
- Autonomous Controller
|
||||
- System Health Center
|
||||
- Task Center
|
||||
- Blocker Memory
|
||||
- Search Gateway monitoring
|
||||
- Notification loop
|
||||
|
||||
## Page
|
||||
|
||||
/admin/ai-evolution/operations
|
||||
|
||||
## Display
|
||||
|
||||
- active tasks
|
||||
- blocked tasks
|
||||
- P0 issues
|
||||
- recurring problems
|
||||
- self-healing actions
|
||||
- search health
|
||||
- notification status
|
||||
|
||||
## Validation
|
||||
|
||||
- AIOS_OPERATIONS_COMMAND_CENTER_READY
|
||||
- AIOS_AUTONOMOUS_LOOP_VISIBLE
|
||||
- AIOS_BLOCKER_OPTIMIZATION_VISIBLE
|
||||
- AIOS_TASK_HEALTH_UNIFIED_READY
|
||||
@@ -0,0 +1,46 @@
|
||||
# CMD-20260709-enable-blocker-task-health-center
|
||||
|
||||
## Goal
|
||||
|
||||
Connect recurring issue detection with Task Center and System Health Center.
|
||||
|
||||
## Deploy
|
||||
|
||||
Components:
|
||||
|
||||
- System Blocker Memory
|
||||
- Blocker Task Adapter
|
||||
- System Health Center
|
||||
- Autonomous Controller integration
|
||||
|
||||
## Behavior
|
||||
|
||||
When an issue appears:
|
||||
|
||||
1. Record incident.
|
||||
2. Calculate recurrence.
|
||||
3. Decide if mechanism optimization is required.
|
||||
4. Create Task Center optimization task.
|
||||
5. Trigger Architect Agent review when required.
|
||||
6. Notify through configured channels.
|
||||
|
||||
## Web UI
|
||||
|
||||
Add:
|
||||
|
||||
/admin/ai-evolution/system-health
|
||||
|
||||
Display:
|
||||
|
||||
- current health
|
||||
- recurring blockers
|
||||
- P0 issues
|
||||
- optimization tasks
|
||||
- self-healing history
|
||||
|
||||
## Validation
|
||||
|
||||
- AIOS_BLOCKER_MEMORY_READY
|
||||
- AIOS_SELF_OPTIMIZATION_TASK_READY
|
||||
- AIOS_SYSTEM_HEALTH_CENTER_READY
|
||||
- AIOS_ARCHITECT_REVIEW_TRIGGER_READY
|
||||
@@ -0,0 +1,72 @@
|
||||
# CMD-20260709-enable-git-permission-self-healing
|
||||
|
||||
## Priority
|
||||
|
||||
P0
|
||||
|
||||
## Goal
|
||||
|
||||
Stop recurring `.git/objects` permission failures from blocking AIOS autonomous operation.
|
||||
|
||||
## Problem
|
||||
|
||||
GitOps autonomous writes have repeatedly failed with:
|
||||
|
||||
```text
|
||||
error: insufficient permission for adding an object to repository database .git/objects
|
||||
error: Error building trees
|
||||
```
|
||||
|
||||
Manual repair is not sufficient. AIOS needs a permanent mechanism.
|
||||
|
||||
## Required Mechanism
|
||||
|
||||
Implement all of the following:
|
||||
|
||||
1. Git permission preflight before every GitOps write.
|
||||
2. Git object ownership scanner.
|
||||
3. Safe auto-repair for workspace `.git/objects` ownership and mode.
|
||||
4. systemd timer or controller loop check.
|
||||
5. Task Center incident creation when permission drift is detected.
|
||||
6. WeChat notification when repair is performed or fails.
|
||||
7. Audit evidence for every scan and repair.
|
||||
|
||||
## Expected Owner/Group
|
||||
|
||||
```text
|
||||
user: aiman
|
||||
group: gitops
|
||||
```
|
||||
|
||||
## Check Commands
|
||||
|
||||
```bash
|
||||
cd /data/aios-gitops/workspace
|
||||
|
||||
id
|
||||
ls -ld .git .git/objects
|
||||
find .git/objects -maxdepth 2 ! -user aiman -ls | head -50
|
||||
find .git/objects -maxdepth 2 ! -group gitops -ls | head -50
|
||||
```
|
||||
|
||||
## Safe Repair
|
||||
|
||||
```bash
|
||||
cd /data/aios-gitops/workspace
|
||||
chown -R aiman:gitops .git/objects
|
||||
chmod -R u+rwX,g+rwX .git/objects
|
||||
```
|
||||
|
||||
Do not use chmod 777.
|
||||
|
||||
## Validation
|
||||
|
||||
After repair, run a real write smoke test on a test branch and remove the test file.
|
||||
|
||||
## Success Signals
|
||||
|
||||
- AIOS_GIT_PERMISSION_PREFLIGHT_READY
|
||||
- AIOS_GIT_OBJECT_PERMISSION_WATCHDOG_READY
|
||||
- AIOS_GIT_PERMISSION_SELF_HEALING_READY
|
||||
- AIOS_GIT_PERMISSION_DRIFT_TASK_READY
|
||||
- AIOS_GIT_PERMISSION_WECHAT_ALERT_READY
|
||||
@@ -0,0 +1,56 @@
|
||||
# CMD-20260709-enable-system-blocker-self-optimization
|
||||
|
||||
## Goal
|
||||
|
||||
Enable AIOS to learn from recurring blockers and convert them into mechanism improvements.
|
||||
|
||||
## Deploy
|
||||
|
||||
Components:
|
||||
|
||||
- System Blocker Memory
|
||||
- Recurrence Detector
|
||||
- Optimization Policy
|
||||
- Task Center integration
|
||||
- Architect Agent integration
|
||||
|
||||
## Rules
|
||||
|
||||
Single incident:
|
||||
|
||||
- record
|
||||
- monitor
|
||||
|
||||
Repeated incident:
|
||||
|
||||
- create optimization task
|
||||
- assign autonomous controller
|
||||
- require architecture review
|
||||
|
||||
Critical incident:
|
||||
|
||||
- create P0 task
|
||||
- attempt self healing
|
||||
- notify owner
|
||||
|
||||
## Tracked Problems
|
||||
|
||||
- Git permission drift
|
||||
- runtime stalls
|
||||
- command stuck
|
||||
- provider failure
|
||||
- deployment failure
|
||||
- approval waiting
|
||||
- notification failure
|
||||
- agent timeout
|
||||
|
||||
## Validation
|
||||
|
||||
- AIOS_BLOCKER_MEMORY_READY
|
||||
- AIOS_RECURRING_ISSUE_DETECTION_READY
|
||||
- AIOS_SELF_OPTIMIZATION_TASK_READY
|
||||
- AIOS_ARCHITECT_REVIEW_TRIGGER_READY
|
||||
|
||||
## Notification
|
||||
|
||||
Report optimization tasks through Task Center and configured notification channels.
|
||||
@@ -0,0 +1,41 @@
|
||||
# CMD-20260709-integrate-autonomous-health-center
|
||||
|
||||
## Goal
|
||||
|
||||
Integrate system blocker optimization and Git permission self healing into Autonomous Controller.
|
||||
|
||||
## Deploy
|
||||
|
||||
Components:
|
||||
|
||||
- Git Permission Watchdog
|
||||
- Autonomous Health Adapter
|
||||
- System Health Center API
|
||||
- Blocker Task Adapter
|
||||
- Task Center optimization tasks
|
||||
|
||||
## Routes
|
||||
|
||||
Enable:
|
||||
|
||||
/admin/ai-evolution/system-health/overview
|
||||
|
||||
## Controller Checks
|
||||
|
||||
Add checks:
|
||||
|
||||
- git permission health
|
||||
- recurring blocker count
|
||||
- unresolved P0 tasks
|
||||
- self healing result
|
||||
|
||||
## Validation
|
||||
|
||||
- AIOS_GIT_PERMISSION_WATCHDOG_READY
|
||||
- AIOS_AUTONOMOUS_HEALTH_CENTER_READY
|
||||
- AIOS_BLOCKER_TASK_INTEGRATION_READY
|
||||
- AIOS_SELF_OPTIMIZATION_LOOP_READY
|
||||
|
||||
## Report
|
||||
|
||||
Send completion report through configured notification channel.
|
||||
@@ -0,0 +1,16 @@
|
||||
from active_runtime.app.autonomous_health_adapter import AutonomousHealthAdapter
|
||||
|
||||
|
||||
def test_health_adapter_returns_center():
|
||||
result = AutonomousHealthAdapter().check([])
|
||||
assert result["module"] == "system-health-center"
|
||||
|
||||
|
||||
def test_health_adapter_can_create_blocker_task():
|
||||
task = AutonomousHealthAdapter().optimize_task({
|
||||
"fingerprint": "git_permission:test",
|
||||
"title": "git permission",
|
||||
"category": "git_permission",
|
||||
"severity": "p0",
|
||||
})
|
||||
assert task["source"] == "system_blocker"
|
||||
@@ -0,0 +1,6 @@
|
||||
from active_runtime.app.command_stall_detector import CommandStallDetector
|
||||
|
||||
|
||||
def test_command_stall_detector_exists():
|
||||
detector = CommandStallDetector('/tmp/nonexistent-aios-command-root')
|
||||
assert detector.detect() == []
|
||||
@@ -0,0 +1,12 @@
|
||||
from active_runtime.app.git_permission_watchdog import GitPermissionWatchdog
|
||||
|
||||
|
||||
def test_watchdog_has_write_probe():
|
||||
watchdog = GitPermissionWatchdog('/tmp/aios-test')
|
||||
result = watchdog.write_probe()
|
||||
assert result['status'] in {'ok', 'failed'}
|
||||
|
||||
|
||||
def test_watchdog_report_shape():
|
||||
report = GitPermissionWatchdog('/tmp/aios-test').scan()
|
||||
assert report.status in {'healthy', 'blocked', 'drift_detected'}
|
||||
@@ -0,0 +1,23 @@
|
||||
from active_runtime.app.system_blocker_memory import SystemBlockerMemory, SystemBlockerRecord
|
||||
|
||||
|
||||
def test_repeated_blocker_becomes_mechanism_gap():
|
||||
memory = SystemBlockerMemory()
|
||||
records = []
|
||||
record = SystemBlockerRecord(
|
||||
blocker_id="b1",
|
||||
category="git_permission",
|
||||
title="git object permission",
|
||||
source="watchdog",
|
||||
severity="p0",
|
||||
)
|
||||
first = memory.record(records, record)
|
||||
second = memory.record([first], record)
|
||||
decision = memory.decide(second)
|
||||
assert second["mechanism_gap"] is True
|
||||
assert decision.priority == "P0"
|
||||
|
||||
|
||||
def test_long_no_response_creates_watchdog_task():
|
||||
decision = SystemBlockerMemory().decide({"category": "long_no_response", "occurrence_count": 1})
|
||||
assert decision.action == "create_stall_watchdog_task"
|
||||
@@ -0,0 +1,10 @@
|
||||
from active_runtime.app.system_health_center import SystemHealthCenter
|
||||
|
||||
|
||||
def test_health_center_detects_p0_attention():
|
||||
result = SystemHealthCenter().build(
|
||||
git_report={"status": "healthy"},
|
||||
blockers=[{"severity": "p0", "mechanism_gap": True}],
|
||||
)
|
||||
assert result["status"] == "attention_required"
|
||||
assert result["blockers"]["p0"] == 1
|
||||
Reference in New Issue
Block a user