Compare commits

...
Author SHA1 Message Date
openclaw 189b655d04 Add autonomous operation command center deployment command 2026-07-09 15:57:18 +08:00
openclaw 5f2b5f16ba Add command stall detector tests 2026-07-09 15:57:13 +08:00
openclaw e2d9de3a1f Add unified operations command center model 2026-07-09 15:57:11 +08:00
openclaw 83cbd64fb6 Add Command Watcher stall detector 2026-07-09 15:57:05 +08:00
openclaw 7852844338 Add autonomous health center deployment command 2026-07-09 15:53:00 +08:00
openclaw 46b9d3188d Add autonomous health adapter tests 2026-07-09 15:52:46 +08:00
openclaw c69916442c Add system health API 2026-07-09 15:52:39 +08:00
openclaw 781239bea7 Add autonomous controller health adapter 2026-07-09 15:52:27 +08:00
openclaw 193f6546b1 Fix Git permission watchdog service entrypoint 2026-07-09 15:52:15 +08:00
openclaw e063c6cb3c Add Git permission watchdog CLI entrypoint 2026-07-09 15:52:07 +08:00
openclaw 9f9f7be796 Add blocker health center deployment command 2026-07-09 15:45:12 +08:00
openclaw 77d0d7176d Add system health center tests 2026-07-09 15:45:01 +08:00
openclaw 0029980d74 Add unified system health center 2026-07-09 15:44:58 +08:00
openclaw 0ed265e3cb Add blocker to Task Center adapter 2026-07-09 15:44:50 +08:00
openclaw c7a1cf5b2a Add OpenClaw command for system blocker self optimization 2026-07-09 15:37:32 +08:00
openclaw d6a6491d2b Add system blocker optimization tests 2026-07-09 15:37:27 +08:00
openclaw e364432a1f Add recurring blocker optimization policy 2026-07-09 15:37:22 +08:00
openclaw 343b6f1ba6 Add system blocker memory and recurrence detection 2026-07-09 15:37:15 +08:00
openclaw e1d4bfc77b Add Git permission watchdog systemd timer 2026-07-09 15:34:15 +08:00
openclaw f7cf7249f1 Add Git permission watchdog systemd service 2026-07-09 15:34:12 +08:00
openclaw 13846c222f Add Git permission watchdog tests 2026-07-09 15:34:09 +08:00
openclaw 749afecf08 Add Git permission watchdog policy 2026-07-09 15:34:05 +08:00
openclaw b424de9d98 Add Git permission watchdog self-healing module 2026-07-09 15:34:00 +08:00
openclaw b761536f16 Dispatch Git permission self-healing mechanism 2026-07-09 15:31:43 +08:00
23 changed files with 1034 additions and 0 deletions
@@ -0,0 +1,38 @@
"""Autonomous Controller health adapter."""
from __future__ import annotations
from typing import Any
from .blocker_task_adapter import BlockerTaskAdapter
from .git_permission_watchdog import GitPermissionWatchdog
from .system_blocker_memory import SystemBlockerMemory
from .system_health_center import SystemHealthCenter
class AutonomousHealthAdapter:
def __init__(self):
self.git = GitPermissionWatchdog()
self.memory = SystemBlockerMemory()
self.task_adapter = BlockerTaskAdapter()
self.health = SystemHealthCenter()
def check(self, previous_blockers: list[dict[str, Any]] | None = None) -> dict[str, Any]:
git_report = self.git.scan().to_dict()
blockers = list(previous_blockers or [])
if git_report.get("status") != "healthy":
blockers.append({
"blocker_id": "git-permission",
"category": "git_permission",
"title": "Git object permission drift",
"severity": "p0",
"evidence": [str(git_report)],
})
return self.health.build(git_report=git_report, blockers=blockers)
def optimize_task(self, blocker: dict[str, Any]) -> dict[str, Any]:
decision = self.memory.decide(blocker)
return self.task_adapter.to_task(blocker, decision.to_dict())
__all__ = ["AutonomousHealthAdapter"]
@@ -0,0 +1,64 @@
"""Convert recurring blockers into Task Center tasks.
When a blocker becomes recurrent or critical, it should not remain only as an
incident log. It becomes a visible task with owner/executor/progress/approval
state in the Task Center.
"""
from __future__ import annotations
from typing import Any
from datetime import datetime, timezone
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
class BlockerTaskAdapter:
def to_task(self, blocker: dict[str, Any], decision: dict[str, Any]) -> dict[str, Any]:
priority = decision.get("priority", "P2")
status = "waiting_approval" if decision.get("owner_gate_required") else "pending"
if priority == "P0":
status = "running"
return {
"task_id": f"blocker-{blocker.get('fingerprint', blocker.get('blocker_id', 'unknown'))}",
"title": f"Resolve blocker: {blocker.get('title', 'unknown')}",
"assigned_by": "system-blocker-memory",
"executor": decision.get("assigned_to", "autonomous-controller"),
"source": "system_blocker",
"status": status,
"priority": priority,
"created_at": utc_now(),
"updated_at": utc_now(),
"progress": {
"percent": 20 if status == "running" else 0,
"current_step": decision.get("action", "record_and_monitor"),
"completed_steps": ["blocker_recorded", "decision_created"],
"remaining_steps": self._remaining_steps(decision),
"blocked_reason": blocker.get("title") if status == "waiting_approval" else None,
},
"notification": {
"required": True,
"sent": False,
"channel": "wechat",
},
"owner_gate_required": bool(decision.get("owner_gate_required")),
"owner_gate_status": "pending" if decision.get("owner_gate_required") else None,
"evidence": blocker.get("evidence", []),
"metadata": {
"blocker": blocker,
"decision": decision,
},
}
def _remaining_steps(self, decision: dict[str, Any]) -> list[str]:
steps = ["root_cause_analysis", "patch_or_repair", "test", "deploy", "verify", "close"]
if decision.get("architect_review_required"):
steps.insert(1, "architect_review")
if decision.get("owner_gate_required"):
steps.insert(2, "owner_approval")
return steps
__all__ = ["BlockerTaskAdapter"]
@@ -0,0 +1,90 @@
"""Command watcher stall detector.
Detects commands that remain in active/running states beyond the configured
threshold and emits blocker records so AIOS can self-optimize instead of waiting
for manual progress checks.
"""
from __future__ import annotations
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def parse_ts(value: str | None) -> datetime | None:
if not value:
return None
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except Exception:
return None
@dataclass(frozen=True)
class CommandStallRecord:
command_id: str
path: str
bucket: str
age_minutes: float
threshold_minutes: int
severity: str
detected_at: str = field(default_factory=utc_now)
def to_blocker(self) -> dict[str, Any]:
return {
"blocker_id": f"command-stall-{self.command_id}",
"category": "command_stuck",
"title": f"Command stalled: {self.command_id}",
"source": "command-stall-detector",
"severity": self.severity,
"status": "open",
"first_seen_at": self.detected_at,
"last_seen_at": self.detected_at,
"occurrence_count": 1,
"evidence": [self.path],
"metadata": self.to_dict(),
}
def to_dict(self) -> dict[str, Any]:
return asdict(self)
class CommandStallDetector:
def __init__(self, root: str = "agents/commands", threshold_minutes: int = 15):
self.root = Path(root)
self.threshold_minutes = threshold_minutes
def detect(self) -> list[CommandStallRecord]:
records: list[CommandStallRecord] = []
for bucket in ("active", "running"):
bucket_path = self.root / bucket
if not bucket_path.exists():
continue
for path in bucket_path.glob("*.md"):
age = self._age_minutes(path)
if age >= self.threshold_minutes:
records.append(
CommandStallRecord(
command_id=path.stem,
path=str(path),
bucket=bucket,
age_minutes=round(age, 2),
threshold_minutes=self.threshold_minutes,
severity="p0" if bucket == "running" else "high",
)
)
return records
def _age_minutes(self, path: Path) -> float:
stat = path.stat()
age_seconds = datetime.now(timezone.utc).timestamp() - stat.st_mtime
return age_seconds / 60
__all__ = ["CommandStallRecord", "CommandStallDetector"]
@@ -0,0 +1,141 @@
"""Git object permission watchdog and self-healing.
Prevents AIOS autonomous operation from being blocked by recurring
`.git/objects` ownership drift. The repair is intentionally conservative and
limited to the workspace Git object database.
"""
from __future__ import annotations
import os
import pwd
import grp
import subprocess
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
@dataclass(frozen=True)
class GitPermissionIssue:
path: str
issue: str
actual_uid: int | None = None
actual_gid: int | None = None
expected_user: str = "aiman"
expected_group: str = "gitops"
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@dataclass(frozen=True)
class GitPermissionReport:
workspace: str
expected_user: str
expected_group: str
status: str
issues: list[GitPermissionIssue] = field(default_factory=list)
checked_at: str = field(default_factory=utc_now)
def to_dict(self) -> dict[str, Any]:
return {
"workspace": self.workspace,
"expected_user": self.expected_user,
"expected_group": self.expected_group,
"status": self.status,
"issues": [issue.to_dict() for issue in self.issues],
"checked_at": self.checked_at,
}
class GitPermissionWatchdog:
def __init__(
self,
workspace: str = "/data/aios-gitops/workspace",
expected_user: str = "aiman",
expected_group: str = "gitops",
max_scan_depth: int = 2,
):
self.workspace = Path(workspace)
self.expected_user = expected_user
self.expected_group = expected_group
self.max_scan_depth = max_scan_depth
@property
def objects_dir(self) -> Path:
return self.workspace / ".git" / "objects"
def scan(self) -> GitPermissionReport:
issues: list[GitPermissionIssue] = []
if not self.objects_dir.exists():
issues.append(GitPermissionIssue(str(self.objects_dir), "missing_objects_dir"))
return GitPermissionReport(str(self.workspace), self.expected_user, self.expected_group, "blocked", issues)
expected_uid = pwd.getpwnam(self.expected_user).pw_uid
expected_gid = grp.getgrnam(self.expected_group).gr_gid
for path in self._walk_limited(self.objects_dir):
try:
stat = path.stat()
except FileNotFoundError:
continue
if stat.st_uid != expected_uid:
issues.append(GitPermissionIssue(str(path), "owner_drift", stat.st_uid, stat.st_gid, self.expected_user, self.expected_group))
if stat.st_gid != expected_gid:
issues.append(GitPermissionIssue(str(path), "group_drift", stat.st_uid, stat.st_gid, self.expected_user, self.expected_group))
if not os.access(path, os.W_OK):
issues.append(GitPermissionIssue(str(path), "not_writable", stat.st_uid, stat.st_gid, self.expected_user, self.expected_group))
if len(issues) >= 200:
break
return GitPermissionReport(
workspace=str(self.workspace),
expected_user=self.expected_user,
expected_group=self.expected_group,
status="healthy" if not issues else "drift_detected",
issues=issues,
)
def repair(self, dry_run: bool = False) -> dict[str, Any]:
report = self.scan()
if report.status == "healthy":
return {"status": "healthy", "report": report.to_dict(), "repaired": False}
if dry_run:
return {"status": "dry_run", "report": report.to_dict(), "repaired": False}
commands = [
["chown", "-R", f"{self.expected_user}:{self.expected_group}", str(self.objects_dir)],
["chmod", "-R", "u+rwX,g+rwX", str(self.objects_dir)],
]
results = []
for command in commands:
proc = subprocess.run(command, text=True, capture_output=True, check=False)
results.append({"cmd": command, "returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr})
if proc.returncode != 0:
return {"status": "repair_failed", "report": report.to_dict(), "results": results, "repaired": False}
post = self.scan()
return {"status": "repaired" if post.status == "healthy" else "repair_incomplete", "report": post.to_dict(), "results": results, "repaired": post.status == "healthy"}
def write_probe(self) -> dict[str, Any]:
probe = self.objects_dir / f".aios-write-probe-{datetime.now(timezone.utc).timestamp()}"
try:
probe.write_text("probe\n", encoding="utf-8")
probe.unlink(missing_ok=True)
return {"status": "ok", "path": str(probe)}
except Exception as exc: # pragma: no cover - depends on runtime perms
return {"status": "failed", "path": str(probe), "error": str(exc)}
def _walk_limited(self, root: Path):
yield root
base_depth = len(root.parts)
for path in root.rglob("*"):
if len(path.parts) - base_depth <= self.max_scan_depth:
yield path
__all__ = ["GitPermissionIssue", "GitPermissionReport", "GitPermissionWatchdog"]
@@ -0,0 +1,31 @@
"""CLI entrypoint for Git Permission Watchdog."""
from __future__ import annotations
import json
import os
from pathlib import Path
from .git_permission_watchdog import GitPermissionWatchdog
def main() -> int:
workspace = os.getenv("AIOS_GITOPS_WORKSPACE", "/data/aios-gitops/workspace")
expected_user = os.getenv("AIOS_GIT_EXPECTED_USER", "aiman")
expected_group = os.getenv("AIOS_GIT_EXPECTED_GROUP", "gitops")
audit_dir = Path(os.getenv("AIOS_GIT_PERMISSION_AUDIT_DIR", "runtime/git-permission-watchdog"))
audit_dir.mkdir(parents=True, exist_ok=True)
watchdog = GitPermissionWatchdog(workspace, expected_user, expected_group)
result = watchdog.repair(dry_run=os.getenv("AIOS_GIT_PERMISSION_DRY_RUN") == "1")
probe = watchdog.write_probe()
payload = {"repair": result, "write_probe": probe}
status = result.get("status")
out = audit_dir / "latest.json"
out.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps(payload, ensure_ascii=False))
return 0 if status in {"healthy", "repaired"} and probe.get("status") == "ok" else 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,52 @@
"""Unified AIOS operations command center view model."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
class OperationsCommandCenter:
def build(
self,
tasks: list[dict[str, Any]] | None = None,
health: dict[str, Any] | None = None,
blockers: list[dict[str, Any]] | None = None,
searches: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
tasks = tasks or []
blockers = blockers or []
searches = searches or []
return {
"module": "operations-command-center",
"route": "/admin/ai-evolution/operations",
"title": "AIOS 自治运营驾驶舱",
"generated_at": utc_now(),
"health": health or {},
"tasks": {
"active": len([t for t in tasks if t.get("status") in {"running", "pending", "waiting_approval"}]),
"blocked": len([t for t in tasks if t.get("status") == "blocked"]),
"completed": len([t for t in tasks if t.get("status") == "completed"]),
},
"blockers": {
"total": len(blockers),
"p0": len([b for b in blockers if b.get("severity") in {"p0", "critical"}]),
"recurring": len([b for b in blockers if b.get("mechanism_gap")]),
},
"search": {
"recent": len(searches),
},
"loops": {
"task_loop": True,
"health_loop": True,
"optimization_loop": True,
"notification_loop": True,
},
}
__all__ = ["OperationsCommandCenter"]
+120
View File
@@ -0,0 +1,120 @@
"""System blocker memory and recurring issue detection.
Records blocking issues, long-no-response incidents and repeated operational
failures. Repeated blockers are escalated from incident handling into mechanism
optimization work.
"""
from __future__ import annotations
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from typing import Any
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
@dataclass(frozen=True)
class SystemBlockerRecord:
blocker_id: str
category: str
title: str
source: str
severity: str
status: str = "open"
first_seen_at: str = field(default_factory=utc_now)
last_seen_at: str = field(default_factory=utc_now)
occurrence_count: int = 1
evidence: list[str] = field(default_factory=list)
mechanism_gap: bool = False
recommended_owner: str = "autonomous-controller"
def fingerprint(self) -> str:
return f"{self.category}:{self.title}".lower().replace(" ", "_")
def to_dict(self) -> dict[str, Any]:
return asdict(self) | {"fingerprint": self.fingerprint()}
@dataclass(frozen=True)
class BlockerEscalationDecision:
status: str
priority: str
reason: str
action: str
assigned_to: str
create_task: bool = True
architect_review_required: bool = False
owner_gate_required: bool = False
def to_dict(self) -> dict[str, Any]:
return asdict(self)
class SystemBlockerMemory:
"""In-memory decision logic; production storage can wrap this with persistence."""
def record(self, existing: list[dict[str, Any]], incoming: SystemBlockerRecord) -> dict[str, Any]:
records = [dict(item) for item in existing]
fp = incoming.fingerprint()
for item in records:
if item.get("fingerprint") == fp or f"{item.get('category')}:{item.get('title')}".lower().replace(" ", "_") == fp:
item["occurrence_count"] = int(item.get("occurrence_count", 1)) + 1
item["last_seen_at"] = utc_now()
item["status"] = "recurrent"
evidence = list(item.get("evidence", [])) + incoming.evidence
item["evidence"] = list(dict.fromkeys(evidence))
item["mechanism_gap"] = self.is_mechanism_gap(item)
item["fingerprint"] = fp
return item
item = incoming.to_dict()
item["mechanism_gap"] = self.is_mechanism_gap(item)
records.append(item)
return item
def decide(self, blocker: dict[str, Any]) -> BlockerEscalationDecision:
count = int(blocker.get("occurrence_count", 1))
severity = blocker.get("severity", "medium")
category = blocker.get("category", "unknown")
if severity in {"critical", "p0"} or category in {"git_permission", "runtime_deadlock", "autonomous_controller_stall"}:
return BlockerEscalationDecision(
status="escalate_to_mechanism_fix",
priority="P0",
reason="blocking infrastructure issue affects autonomous operation",
action="create_self_healing_mechanism",
assigned_to="openclaw",
architect_review_required=True,
)
if count >= 3:
return BlockerEscalationDecision(
status="escalate_to_mechanism_fix",
priority="P1",
reason="recurring issue exceeded threshold",
action="create_root_cause_and_prevention_patch",
assigned_to="autonomous-controller",
architect_review_required=True,
)
if category == "long_no_response":
return BlockerEscalationDecision(
status="create_monitoring_task",
priority="P1",
reason="long no response requires watchdog and notification",
action="create_stall_watchdog_task",
assigned_to="openclaw",
)
return BlockerEscalationDecision(
status="track_incident",
priority="P2",
reason="single incident tracked for recurrence",
action="record_and_monitor",
assigned_to="autonomous-controller",
create_task=False,
)
def is_mechanism_gap(self, blocker: dict[str, Any]) -> bool:
return int(blocker.get("occurrence_count", 1)) >= 2 or blocker.get("severity") in {"critical", "p0"}
__all__ = ["SystemBlockerRecord", "BlockerEscalationDecision", "SystemBlockerMemory"]
+32
View File
@@ -0,0 +1,32 @@
"""System Health Center API."""
try:
from fastapi import APIRouter
except Exception:
APIRouter = None
from .system_health_center import SystemHealthCenter
center = SystemHealthCenter()
def get_system_health_overview():
return center.build()
def build_router():
if APIRouter is None:
return None
router = APIRouter(prefix="/admin/ai-evolution/system-health", tags=["system-health"])
@router.get("/overview", operation_id="getSystemHealthOverview")
def overview():
return get_system_health_overview()
return router
router = build_router()
__all__ = ["router", "get_system_health_overview"]
@@ -0,0 +1,54 @@
"""Unified AIOS system health center model."""
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
class SystemHealthCenter:
def build(
self,
git_report: dict[str, Any] | None = None,
blockers: list[dict[str, Any]] | None = None,
tasks: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
blockers = blockers or []
tasks = tasks or []
return {
"module": "system-health-center",
"route": "/admin/ai-evolution/system-health",
"title": "AIOS 系统健康与自优化中心",
"generated_at": utc_now(),
"status": self._status(git_report, blockers),
"git": git_report or {},
"blockers": {
"total": len(blockers),
"recurrent": len([b for b in blockers if b.get("mechanism_gap")]),
"p0": len([b for b in blockers if b.get("severity") in {"p0", "critical"}]),
},
"tasks": {
"optimization_tasks": len([t for t in tasks if t.get("source") == "system_blocker"]),
"pending": len([t for t in tasks if t.get("status") in {"pending", "running", "waiting_approval"}]),
},
"optimization_loop": {
"detect": True,
"remember": True,
"optimize": True,
"verify": True,
},
}
def _status(self, git_report, blockers):
if git_report and git_report.get("status") not in {"healthy", "ok"}:
return "attention_required"
if any(b.get("severity") in {"p0", "critical"} for b in blockers):
return "attention_required"
return "healthy"
__all__ = ["SystemHealthCenter"]
@@ -0,0 +1,35 @@
git_permission_watchdog:
enabled: true
priority: P0
workspace: /data/aios-gitops/workspace
expected_identity:
user: aiman
group: gitops
checks:
- object_owner
- object_group
- object_write_permission
- git_write_probe
repair:
enabled: true
allowed_paths:
- .git/objects
forbidden:
- chmod_777
- system_wide_permission_change
runtime:
interval_minutes: 5
create_task_on_drift: true
send_notification_on_failure: true
audit:
enabled: true
record:
- detection_time
- affected_paths
- repair_action
- repair_result
@@ -0,0 +1,37 @@
system_blocker_optimization:
enabled: true
purpose: convert_recurring_failures_into_mechanism_improvements
tracked_categories:
- git_permission
- runtime_stall
- command_stuck
- provider_failure
- deployment_failure
- approval_wait
- notification_failure
- agent_timeout
thresholds:
repeat_count_for_mechanism_review: 3
critical_issue_immediate_review: true
long_no_response_minutes: 15
actions:
first_occurrence:
- record_incident
- monitor
repeated_occurrence:
- create_optimization_task
- assign_autonomous_controller
- require_architect_review
critical_occurrence:
- create_p0_task
- notify_owner
- self_healing_attempt
integration:
task_center: true
audit_plane: true
event_bus: true
architect_agent: true
@@ -0,0 +1,13 @@
[Unit]
Description=AIOS Git Permission Watchdog
After=network.target
[Service]
Type=oneshot
User=aiman
Group=gitops
WorkingDirectory=/data/aios-gitops/workspace
ExecStart=/usr/bin/python3 -m active_runtime.app.git_permission_watchdog_cli
[Install]
WantedBy=multi-user.target
+10
View File
@@ -0,0 +1,10 @@
[Unit]
Description=Run AIOS Git Permission Watchdog periodically
[Timer]
OnBootSec=5min
OnUnitActiveSec=5min
Unit=aios-git-permission-watchdog.service
[Install]
WantedBy=timers.target
@@ -0,0 +1,35 @@
# CMD-20260709-enable-autonomous-operation-command-center
## Goal
Create unified AIOS autonomous operations command center.
## Integrate
- Autonomous Controller
- System Health Center
- Task Center
- Blocker Memory
- Search Gateway monitoring
- Notification loop
## Page
/admin/ai-evolution/operations
## Display
- active tasks
- blocked tasks
- P0 issues
- recurring problems
- self-healing actions
- search health
- notification status
## Validation
- AIOS_OPERATIONS_COMMAND_CENTER_READY
- AIOS_AUTONOMOUS_LOOP_VISIBLE
- AIOS_BLOCKER_OPTIMIZATION_VISIBLE
- AIOS_TASK_HEALTH_UNIFIED_READY
@@ -0,0 +1,46 @@
# CMD-20260709-enable-blocker-task-health-center
## Goal
Connect recurring issue detection with Task Center and System Health Center.
## Deploy
Components:
- System Blocker Memory
- Blocker Task Adapter
- System Health Center
- Autonomous Controller integration
## Behavior
When an issue appears:
1. Record incident.
2. Calculate recurrence.
3. Decide if mechanism optimization is required.
4. Create Task Center optimization task.
5. Trigger Architect Agent review when required.
6. Notify through configured channels.
## Web UI
Add:
/admin/ai-evolution/system-health
Display:
- current health
- recurring blockers
- P0 issues
- optimization tasks
- self-healing history
## Validation
- AIOS_BLOCKER_MEMORY_READY
- AIOS_SELF_OPTIMIZATION_TASK_READY
- AIOS_SYSTEM_HEALTH_CENTER_READY
- AIOS_ARCHITECT_REVIEW_TRIGGER_READY
@@ -0,0 +1,72 @@
# CMD-20260709-enable-git-permission-self-healing
## Priority
P0
## Goal
Stop recurring `.git/objects` permission failures from blocking AIOS autonomous operation.
## Problem
GitOps autonomous writes have repeatedly failed with:
```text
error: insufficient permission for adding an object to repository database .git/objects
error: Error building trees
```
Manual repair is not sufficient. AIOS needs a permanent mechanism.
## Required Mechanism
Implement all of the following:
1. Git permission preflight before every GitOps write.
2. Git object ownership scanner.
3. Safe auto-repair for workspace `.git/objects` ownership and mode.
4. systemd timer or controller loop check.
5. Task Center incident creation when permission drift is detected.
6. WeChat notification when repair is performed or fails.
7. Audit evidence for every scan and repair.
## Expected Owner/Group
```text
user: aiman
group: gitops
```
## Check Commands
```bash
cd /data/aios-gitops/workspace
id
ls -ld .git .git/objects
find .git/objects -maxdepth 2 ! -user aiman -ls | head -50
find .git/objects -maxdepth 2 ! -group gitops -ls | head -50
```
## Safe Repair
```bash
cd /data/aios-gitops/workspace
chown -R aiman:gitops .git/objects
chmod -R u+rwX,g+rwX .git/objects
```
Do not use chmod 777.
## Validation
After repair, run a real write smoke test on a test branch and remove the test file.
## Success Signals
- AIOS_GIT_PERMISSION_PREFLIGHT_READY
- AIOS_GIT_OBJECT_PERMISSION_WATCHDOG_READY
- AIOS_GIT_PERMISSION_SELF_HEALING_READY
- AIOS_GIT_PERMISSION_DRIFT_TASK_READY
- AIOS_GIT_PERMISSION_WECHAT_ALERT_READY
@@ -0,0 +1,56 @@
# CMD-20260709-enable-system-blocker-self-optimization
## Goal
Enable AIOS to learn from recurring blockers and convert them into mechanism improvements.
## Deploy
Components:
- System Blocker Memory
- Recurrence Detector
- Optimization Policy
- Task Center integration
- Architect Agent integration
## Rules
Single incident:
- record
- monitor
Repeated incident:
- create optimization task
- assign autonomous controller
- require architecture review
Critical incident:
- create P0 task
- attempt self healing
- notify owner
## Tracked Problems
- Git permission drift
- runtime stalls
- command stuck
- provider failure
- deployment failure
- approval waiting
- notification failure
- agent timeout
## Validation
- AIOS_BLOCKER_MEMORY_READY
- AIOS_RECURRING_ISSUE_DETECTION_READY
- AIOS_SELF_OPTIMIZATION_TASK_READY
- AIOS_ARCHITECT_REVIEW_TRIGGER_READY
## Notification
Report optimization tasks through Task Center and configured notification channels.
@@ -0,0 +1,41 @@
# CMD-20260709-integrate-autonomous-health-center
## Goal
Integrate system blocker optimization and Git permission self healing into Autonomous Controller.
## Deploy
Components:
- Git Permission Watchdog
- Autonomous Health Adapter
- System Health Center API
- Blocker Task Adapter
- Task Center optimization tasks
## Routes
Enable:
/admin/ai-evolution/system-health/overview
## Controller Checks
Add checks:
- git permission health
- recurring blocker count
- unresolved P0 tasks
- self healing result
## Validation
- AIOS_GIT_PERMISSION_WATCHDOG_READY
- AIOS_AUTONOMOUS_HEALTH_CENTER_READY
- AIOS_BLOCKER_TASK_INTEGRATION_READY
- AIOS_SELF_OPTIMIZATION_LOOP_READY
## Report
Send completion report through configured notification channel.
+16
View File
@@ -0,0 +1,16 @@
from active_runtime.app.autonomous_health_adapter import AutonomousHealthAdapter
def test_health_adapter_returns_center():
result = AutonomousHealthAdapter().check([])
assert result["module"] == "system-health-center"
def test_health_adapter_can_create_blocker_task():
task = AutonomousHealthAdapter().optimize_task({
"fingerprint": "git_permission:test",
"title": "git permission",
"category": "git_permission",
"severity": "p0",
})
assert task["source"] == "system_blocker"
+6
View File
@@ -0,0 +1,6 @@
from active_runtime.app.command_stall_detector import CommandStallDetector
def test_command_stall_detector_exists():
detector = CommandStallDetector('/tmp/nonexistent-aios-command-root')
assert detector.detect() == []
+12
View File
@@ -0,0 +1,12 @@
from active_runtime.app.git_permission_watchdog import GitPermissionWatchdog
def test_watchdog_has_write_probe():
watchdog = GitPermissionWatchdog('/tmp/aios-test')
result = watchdog.write_probe()
assert result['status'] in {'ok', 'failed'}
def test_watchdog_report_shape():
report = GitPermissionWatchdog('/tmp/aios-test').scan()
assert report.status in {'healthy', 'blocked', 'drift_detected'}
+23
View File
@@ -0,0 +1,23 @@
from active_runtime.app.system_blocker_memory import SystemBlockerMemory, SystemBlockerRecord
def test_repeated_blocker_becomes_mechanism_gap():
memory = SystemBlockerMemory()
records = []
record = SystemBlockerRecord(
blocker_id="b1",
category="git_permission",
title="git object permission",
source="watchdog",
severity="p0",
)
first = memory.record(records, record)
second = memory.record([first], record)
decision = memory.decide(second)
assert second["mechanism_gap"] is True
assert decision.priority == "P0"
def test_long_no_response_creates_watchdog_task():
decision = SystemBlockerMemory().decide({"category": "long_no_response", "occurrence_count": 1})
assert decision.action == "create_stall_watchdog_task"
+10
View File
@@ -0,0 +1,10 @@
from active_runtime.app.system_health_center import SystemHealthCenter
def test_health_center_detects_p0_attention():
result = SystemHealthCenter().build(
git_report={"status": "healthy"},
blockers=[{"severity": "p0", "mechanism_gap": True}],
)
assert result["status"] == "attention_required"
assert result["blockers"]["p0"] == 1