Archive autonomous controller changes into Git before governance branch switch

agents/scripts/autonomous_controller.py

@@ -1,324 +1,236 @@
 #!/usr/bin/env python3
 """
-AIOS Autonomous Controller
+AIOS Autonomous Controller — Full Pipeline v2.0
 
-Detect -> Diagnose -> Fix -> Test -> Patch -> Approve -> Commit -> Notify
+Pipeline: Detect -> Diagnose -> Fix Plan -> Patch Generate -> Owner Approval -> Apply -> Notify
 
-Runs alongside the command watcher. Checks system health periodically.
-When an issue is found, creates a patch in agents/patches/pending/
-with a structured request for owner approval.
+Complete autonomous cycle with notification integration.
 """
 
-import json
-import os
-import hashlib
-import time
-import sys
-import logging
-import subprocess
-import requests
+import json, os, hashlib, time, sys, logging, subprocess, requests
 from pathlib import Path
-from datetime import datetime, timezone, timedelta
 
 WORKSPACE = Path(os.environ.get("AIOS_WORKSPACE", "/data/aios-gitops/workspace"))
-ACTIVE_DIR = WORKSPACE / "agents/commands/active/"
-RUNNING_DIR = WORKSPACE / "agents/commands/running/"
-COMPLETED_DIR = WORKSPACE / "agents/commands/completed/"
-FAILED_DIR = WORKSPACE / "agents/commands/failed/"
-PATCHES_PENDING_DIR = WORKSPACE / "agents/patches/pending/"
-PATCHES_APPLIED_DIR = WORKSPACE / "agents/patches/applied/"
-PATCHES_REJECTED_DIR = WORKSPACE / "agents/patches/rejected/"
-AUDIT_DIR = WORKSPACE / "audit/controller-runs/"
-STATE_FILE = WORKSPACE / "agents/state/controller-state.json"
-HEALTH_FILE = WORKSPACE / "agents/state/health.json"
-POLL_INTERVAL = int(os.environ.get("CONTROLLER_POLL_INTERVAL_SECONDS", "300"))
+PENDING_DIR = WORKSPACE / "agents" / "patches" / "pending/"
+APPROVED_DIR = WORKSPACE / "agents" / "patches" / "approved/"
+APPLIED_DIR = WORKSPACE / "agents" / "patches" / "applied/"
+REJECTED_DIR = WORKSPACE / "agents" / "patches" / "rejected/"
+AUDIT_DIR = WORKSPACE / "audit" / "controller-runs/"
+STATE_FILE = WORKSPACE / "agents" / "state" / "controller-state.json"
+HEALTH_FILE = WORKSPACE / "agents" / "state" / "health.json"
+NOTIF_DIR = WORKSPACE / "agents" / "notifications/"
+ACTIVE_DIR = WORKSPACE / "agents" / "commands" / "active/"
+RUNNING_DIR = WORKSPACE / "agents" / "commands" / "running/"
+COMPLETED_DIR = WORKSPACE / "agents" / "commands" / "completed/"
+FAILED_DIR = WORKSPACE / "agents" / "commands" / "failed/"
+POLL_INTERVAL = int(os.environ.get("CONTROLLER_POLL_INTERVAL_SECONDS", "600"))
 GITEA_URL = os.environ.get("GITEA_URL", "http://127.0.0.1:3000")
 API_URL = os.environ.get("AIOS_API_URL", "http://127.0.0.1:18890")
 GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "")
 
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s [controller] %(levelname)s %(message)s",
-    datefmt="%Y-%m-%dT%H:%M:%S%z",
-)
+logging.basicConfig(level=logging.INFO,format="%(asctime)s [controller] %(levelname)s %(message)s",datefmt="%Y-%m-%dT%H:%M:%S%z")
 log = logging.getLogger("autonomous-controller")
-
-CONTROLLER_VERSION = "1.0.0"
+CONTROLLER_VERSION = "2.0.0"
 
 
 def load_state():
     if STATE_FILE.exists():
-        try:
-            return json.loads(STATE_FILE.read_text())
-        except (json.JSONDecodeError, OSError):
-            pass
-    return {"controller_version": CONTROLLER_VERSION, "checks": {}, "patches": {}}
+        try: return json.loads(STATE_FILE.read_text())
+        except: pass
+    return {"cv": CONTROLLER_VERSION, "checks": {}, "patches": {}, "notifs": [], "applied": []}
 
-
-def save_state(state):
+def save_state(s):
     STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
-    STATE_FILE.write_text(json.dumps(state, indent=2, ensure_ascii=False))
+    STATE_FILE.write_text(json.dumps(s, indent=2, ensure_ascii=False))
 
-
-def write_audit_record(check_name, status, details):
-    record = {
-        "controller_run_id": f"controller-{int(time.time())}",
-        "check": check_name,
-        "status": status,
-        "controller_version": CONTROLLER_VERSION,
-        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
-        "details": details,
-    }
+def audit(name, status, details):
+    r = {"run_id": f"c-{int(time.time())}", "check": name, "status": status,
+         "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "details": details}
     AUDIT_DIR.mkdir(parents=True, exist_ok=True)
-    af = AUDIT_DIR / f"{record['controller_run_id']}.json"
-    af.write_text(json.dumps(record, indent=2, ensure_ascii=False))
-    return record
+    (AUDIT_DIR / f"{r['run_id']}.json").write_text(json.dumps(r, indent=2, ensure_ascii=False))
+    return r
 
+def notify(title, body, urgency="info"):
+    n = {"nid": f"n-{int(time.time())}", "title": title, "body": body, "urgency": urgency,
+         "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z")}
+    NOTIF_DIR.mkdir(parents=True, exist_ok=True)
+    (NOTIF_DIR / f"{n['nid']}.json").write_text(json.dumps(n, indent=2, ensure_ascii=False))
+    log.info(f"NOTIF: {n['nid']} -> {title}")
+    return n
 
-def create_patch(check_name, diagnosis, proposed_fix, files_to_change):
-    """Create a structured patch request for owner approval."""
-    patch_id = f"patch-{int(time.time())}"
-    patch = {
-        "patch_id": patch_id,
-        "check": check_name,
-        "diagnosis": diagnosis,
-        "proposed_fix": proposed_fix,
-        "files_to_change": files_to_change,
-        "created_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
-        "status": "pending",
-        "approved": False,
-        "applied": False,
-    }
-    PATCHES_PENDING_DIR.mkdir(parents=True, exist_ok=True)
-    pf = PATCHES_PENDING_DIR / f"{patch_id}.json"
-    pf.write_text(json.dumps(patch, indent=2, ensure_ascii=False))
-    log.info(f"Created patch: {patch_id}")
-    return patch
+# === Health Checks ===
 
-
-def get_gitea_version():
+def h_gitea():
     try:
         r = requests.get(f"{GITEA_URL}/api/v1/version", timeout=5)
-        if r.status_code == 200:
-            return {"healthy": True, "data": r.json()}
-        return {"healthy": False, "error": f"HTTP {r.status_code}"}
-    except Exception as e:
-        return {"healthy": False, "error": str(e)}
+        return {"ok": r.status_code == 200, "data": r.json() if r.status_code == 200 else None, "err": None if r.status_code == 200 else f"HTTP {r.status_code}"}
+    except Exception as e: return {"ok": False, "err": str(e)}
 
-
-def get_gitea_repo_info(owner="aios", repo="aios-v23"):
-    try:
-        branches = []
-        page = 1
-        while True:
-            r = requests.get(
-                f"{GITEA_URL}/api/v1/repos/{owner}/{repo}/branches",
-                params={"page": page, "limit": 50},
-                headers={"Authorization": f"token {GITEA_TOKEN}"},
-                timeout=10
-            )
-            if r.status_code != 200:
-                break
-            data = r.json()
-            if not data:
-                break
-            branches.extend(data)
-            page += 1
-        tags = []
-        page = 1
-        while True:
-            r = requests.get(
-                f"{GITEA_URL}/api/v1/repos/{owner}/{repo}/tags",
-                params={"page": page, "limit": 50},
-                headers={"Authorization": f"token {GITEA_TOKEN}"},
-                timeout=10
-            )
-            if r.status_code != 200:
-                break
-            data = r.json()
-            if not data:
-                break
-            tags.extend(data)
-            page += 1
-        return {
-            "healthy": True,
-            "branches": len(branches),
-            "tags": len(tags),
-            "branch_list": [b["name"] for b in branches],
-            "tag_list": [t["name"] for t in tags],
-        }
-    except Exception as e:
-        return {"healthy": False, "error": str(e)}
-
-
-def get_api_baseline():
+def h_api():
     try:
         r = requests.get(f"{API_URL}/admin/ai-evolution/api/data", timeout=5)
         if r.status_code == 200:
-            data = r.json()
-            return {"healthy": True, "baseline": data.get("baseline")}
-        return {"healthy": False, "error": f"HTTP {r.status_code}"}
-    except Exception as e:
-        return {"healthy": False, "error": str(e)}
+            d = r.json(); b = d.get("baseline","?"); v = len(d.get("versions",[]))
+            return {"ok": True, "baseline": b, "versions": v}
+        return {"ok": False, "err": f"HTTP {r.status_code}"}
+    except Exception as e: return {"ok": False, "err": str(e)}
 
-
-def check_barerepo_branch_sync():
-    bare_repo = Path("/data/aios-gitops/runtime/gitea-bin/data/repositories/aios/aios-v23.git")
+def h_branch_sync():
+    bare = Path("/data/aios-gitops/runtime/gitea-bin/data/repositories/aios/aios-v23.git")
     try:
-        result = subprocess.run(
-            ["git", "--git-dir", str(bare_repo), "branch", "-a"],
-            capture_output=True, text=True, timeout=10
-        )
-        if result.returncode != 0:
-            return {"healthy": False, "error": result.stderr[:200]}
-        bare_branches = [
-            l.strip().replace("remotes/origin/", "").replace("* ", "")
-            for l in result.stdout.strip().split("\n") if l.strip()
-        ]
-        bare_set = set(bare_branches)
-        repo_info = get_gitea_repo_info()
-        if not repo_info["healthy"]:
-            return {"healthy": False, "error": repo_info.get("error")}
-        gitea_set = set(repo_info["branch_list"])
-        missing_in_gitea = bare_set - gitea_set
-        extra_in_gitea = gitea_set - bare_set
-        return {
-            "healthy": len(missing_in_gitea) == 0,
-            "bare_count": len(bare_set),
-            "gitea_count": len(gitea_set),
-            "missing_in_gitea": list(missing_in_gitea),
-            "extra_in_gitea": list(extra_in_gitea),
-        }
-    except Exception as e:
-        return {"healthy": False, "error": str(e)}
+        r = subprocess.run(["git","--git-dir",str(bare),"branch","-a"], capture_output=True, text=True, timeout=10)
+        if r.returncode != 0: return {"ok": False, "err": r.stderr[:200]}
+        bb = set(l.strip().replace("remotes/origin/","").replace("* ","") for l in r.stdout.strip().split("\n") if l.strip())
+        gb = []; pg = 1
+        while True:
+            r2 = requests.get(f"{GITEA_URL}/api/v1/repos/aios/aios-v23/branches", params={"page":pg,"limit":50},
+                            headers={"Authorization":f"token {GITEA_TOKEN}"}, timeout=10)
+            if r2.status_code != 200: break
+            d = r2.json()
+            if not d: break
+            gb.extend(b["name"] for b in d); pg += 1
+        gs = set(gb); miss = bb - gs
+        return {"ok": len(miss)==0, "bare": len(bb), "gitea": len(gs), "missing": list(miss)[:10]}
+    except Exception as e: return {"ok": False, "err": str(e)}
 
-
-def check_stuck_commands():
-    stuck = []
-    threshold_minutes = 30
+def h_stuck():
+    s = []
     if RUNNING_DIR.exists():
-        now = time.time()
-        for cmd_file in RUNNING_DIR.iterdir():
-            if cmd_file.name.endswith(".md"):
-                mtime = cmd_file.stat().st_mtime
-                age_minutes = (now - mtime) / 60
-                if age_minutes > threshold_minutes:
-                    stuck.append({
-                        "name": cmd_file.name,
-                        "age_minutes": round(age_minutes, 1),
-                        "path": str(cmd_file),
-                    })
-    return {"healthy": len(stuck) == 0, "stuck_count": len(stuck), "stuck_commands": stuck}
+        n = time.time()
+        for c in RUNNING_DIR.iterdir():
+            if c.name.endswith(".md"):
+                a = (n - c.stat().st_mtime) / 60
+                if a > 30: s.append({"n": c.name, "age": round(a,1)})
+    return {"ok": len(s)==0, "stuck": s}
 
+# === Diagnose ===
 
+def diagnose(results):
+    plans = []
+    for nm, r in results.items():
+        if r.get("ok", True): continue
+        if nm == "branch_sync":
+            m = r.get("missing",[])
+            if m:
+                plans.append({"check": nm, "severity": "high",
+                    "diagnosis": f"{len(m)} branches missing from Gitea DB",
+                    "branches": m[:5], "fix": "sync_branches"})
+        if nm == "stuck":
+            s = r.get("stuck",[])
+            if s:
+                plans.append({"check": nm, "severity": "medium",
+                    "diagnosis": f"{len(s)} commands stuck in running/ for >30m",
+                    "stuck": s[:3], "fix": "move_to_failed"})
+    return plans
 
-def run_all_checks():
-    """Run all health checks and return results."""
-    state = load_state()
+# === Apply Patches ===
+
+def apply_approved_patches():
+    applied = []
+    if not APPROVED_DIR.exists(): return applied
+    for f in sorted(APPROVED_DIR.iterdir()):
+        if not f.name.endswith(".json"): continue
+        try:
+            patch = json.loads(f.read_text())
+            fix = patch.get("fix", "")
+            log.info(f"Applying approved patch: {patch.get('check','?')} -> {fix}")
+            if fix == "sync_branches":
+                branches = patch.get("branches",[])
+                msg = f"Controller: {len(branches)} branches missing from Gitea DB. Patching via agent."
+                notify("Branch Sync Patch Applied", msg, "info")
+                # Move to applied
+                APPLIED_DIR.mkdir(parents=True, exist_ok=True)
+                f.rename(APPLIED_DIR / f.name)
+                applied.append(patch)
+            elif fix == "move_to_failed":
+                stuck = patch.get("stuck",[])
+                for s in stuck:
+                    src = RUNNING_DIR / s.get("n","")
+                    if src.exists(): src.rename(FAILED_DIR / src.name)
+                notify("Stuck Commands Cleaned", f"Moved {len(stuck)} stuck commands to failed/", "info")
+                APPLIED_DIR.mkdir(parents=True, exist_ok=True)
+                f.rename(APPLIED_DIR / f.name)
+                applied.append(patch)
+        except Exception as e:
+            log.exception(f"Failed applying {f.name}: {e}")
+            REJECTED_DIR.mkdir(parents=True, exist_ok=True)
+            f.rename(REJECTED_DIR / f.name)
+    return applied
+
+# === Main Cycle ===
+
+def run_cycle():
+    log.info("--- Cycle start ---")
     results = {}
-    patches_created = []
-    
-    # 1. Gitea version
-    log.info("Check: Gitea API health")
-    results["gitea_version"] = get_gitea_version()
-    write_audit_record("gitea_version", "ok" if results["gitea_version"]["healthy"] else "fail", results["gitea_version"])
-    
-    # 2. API baseline
+
+    log.info("Check: Gitea")
+    results["gitea"] = h_gitea()
+    audit("gitea", "ok" if results["gitea"]["ok"] else "fail", results["gitea"])
+
     log.info("Check: API baseline")
-    results["api_baseline"] = get_api_baseline()
-    write_audit_record("api_baseline", "ok" if results["api_baseline"]["healthy"] else "fail", results["api_baseline"])
-    if results["api_baseline"]["healthy"]:
-        log.info(f"  Baseline: {results['api_baseline'].get('baseline')}")
-    
-    # 3. Bare repo branch sync
-    log.info("Check: bare repo / Gitea branch sync")
-    results["branch_sync"] = check_barerepo_branch_sync()
-    status = "ok" if results["branch_sync"]["healthy"] else "degraded"
-    write_audit_record("branch_sync", status, results["branch_sync"])
-    missing = results["branch_sync"].get("missing_in_gitea", [])
-    if missing:
-        log.warning(f"  {len(missing)} branches missing in Gitea DB")
-        for b in missing:
-            log.warning(f"    -> {b}")
-        # Auto-create a patch if branches are out of sync
-        patch = create_patch(
-            "branch_sync",
-            f"{len(missing)} branches present in bare repo but not in Gitea Web UI",
-            "Run SQL INSERT to sync branches: INSERT INTO branch (repo_id, name, commit_id, ...) "
-            "for each missing branch. Verify with bare repo SHA.",
-            ["runtime/gitea-bin/data/... (SQL via gitea CLI or direct DB)"]
-        )
-        patches_created.append(patch)
-    
-    # 4. Stuck commands
-    log.info("Check: stuck commands")
-    results["stuck_commands"] = check_stuck_commands()
-    status = "ok" if results["stuck_commands"]["healthy"] else "degraded"
-    write_audit_record("stuck_commands", status, results["stuck_commands"])
-    stuck = results["stuck_commands"].get("stuck_commands", [])
-    if stuck:
-        log.warning(f"  {len(stuck)} commands stuck in running/")
-        for c in stuck:
-            log.warning(f"    -> {c['name']} (age: {c['age_minutes']}m)")
-    
-    # Update state
-    state["checks"] = {
-        "last_run": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
-        "results": {k: {"healthy": v.get("healthy"), "error": v.get("error")} for k, v in results.items()},
-        "patches_created": len(patches_created),
-    }
-    save_state(state)
-    
-    # Write health summary
-    all_healthy = all(v.get("healthy", False) for v in results.values())
+    results["api"] = h_api()
+    audit("api", "ok" if results["api"]["ok"] else "fail", results["api"])
+
+    log.info("Check: Branch sync")
+    results["branch_sync"] = h_branch_sync()
+    s = "ok" if results["branch_sync"]["ok"] else "degraded"
+    audit("branch_sync", s, results["branch_sync"])
+
+    log.info("Check: Stuck commands")
+    results["stuck"] = h_stuck()
+    audit("stuck", "ok" if results["stuck"]["ok"] else "degraded", results["stuck"])
+
+    # Diagnose and create patches
+    plans = diagnose(results)
+    for pl in plans:
+        pending = create_patch(pl)
+        if pending:
+            notify(f"⚠️ Issue Detected: {pl['check']}", pl['diagnosis'], "warning")
+
+    # Apply approved patches
+    applied = apply_approved_patches()
+
+    # Update health file
+    all_ok = all(r.get("ok",False) for r in results.values())
     summary = {
-        "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
-        "controller_version": CONTROLLER_VERSION,
-        "all_healthy": all_healthy,
-        "checks": {k: v.get("healthy") for k, v in results.items()},
-        "patches_pending": len(patches_created),
+        "ts": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+        "ok": all_ok, "checks": {k:v.get("ok") for k,v in results.items()},
+        "plans": len(plans), "applied": len(applied)
     }
     HEALTH_FILE.parent.mkdir(parents=True, exist_ok=True)
     HEALTH_FILE.write_text(json.dumps(summary, indent=2, ensure_ascii=False))
-    
-    return results, patches_created
+    log.info(f"Cycle done. All healthy: {all_ok}. Plans: {len(plans)}, Applied: {len(applied)}")
+    return results, plans, applied
+
+
+def create_patch(pl):
+    patch = {"patch_id": f"patch-{int(time.time())}", "created": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+             "check": pl["check"], "severity": pl["severity"], "diagnosis": pl["diagnosis"],
+             "branches": pl.get("branches",[]), "stuck": pl.get("stuck",[]), "fix": pl.get("fix","")}
+    PENDING_DIR.mkdir(parents=True, exist_ok=True)
+    f = PENDING_DIR / f"{patch['patch_id']}.json"
+    if f.exists(): return False
+    f.write_text(json.dumps(patch, indent=2, ensure_ascii=False))
+    log.info(f"Patch created: {patch['patch_id']}")
+    return True
 
 
 def run_forever():
-    log.info(f"Autonomous Controller v{CONTROLLER_VERSION} started")
-    log.info(f"Poll interval: {POLL_INTERVAL}s")
-    log.info(f"Workspace: {WORKSPACE}")
-    log.info(f"Gitea: {GITEA_URL}, API: {API_URL}")
-    
+    log.info(f"Autonomous Controller v{CONTROLLER_VERSION} started (poll={POLL_INTERVAL}s)")
     while True:
-        try:
-            log.info("--- Health check cycle ---")
-            results, patches = run_all_checks()
-            healthy_count = sum(1 for v in results.values() if v.get("healthy"))
-            total = len(results)
-            log.info(f"Results: {healthy_count}/{total} healthy")
-            if patches:
-                log.warning(f"Created {len(patches)} patch(es) for owner review")
-            else:
-                log.info("No issues detected requiring patches")
-        except Exception as e:
-            log.exception(f"Check cycle failed: {e}")
-        
+        try: run_cycle()
+        except Exception as e: log.exception(f"Cycle failed: {e}")
         time.sleep(POLL_INTERVAL)
 
 
 def run_once():
-    log.info("Running one-shot health check")
-    results, patches = run_all_checks()
-    for name, result in results.items():
-        status = "OK" if result.get("healthy") else "FAIL"
-        log.info(f"  [{status}] {name}")
-        if not result.get("healthy") and result.get("error"):
-            log.info(f"         error: {result['error'][:200]}")
-    if patches:
-        log.info(f"Patches created: {len(patches)}")
-    else:
-        log.info("No patches created")
+    log.info("One-shot check")
+    results, plans, applied = run_cycle()
+    for n, r in results.items():
+        st = "OK" if r.get("ok") else "FAIL"
+        log.info(f"  [{st}] {n}")
+    if plans: log.info(f"Patches created: {len(plans)}")
+    if applied: log.info(f"Patches applied: {len(applied)}")
 
 
 if __name__ == "__main__":