Compare commits
10
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d31267e89c | ||
|
|
f4022d3d1e | ||
|
|
008bea3324 | ||
|
|
52ef41e45f | ||
|
|
416e5e295d | ||
|
|
4f81344509 | ||
|
|
8efdaea22f | ||
|
|
3b48740b17 | ||
|
|
0c330afe03 | ||
|
|
764bb2f08e |
@@ -0,0 +1,722 @@
|
|||||||
|
"""Owner-controlled CAS-bound execution-envelope rebind for Online Agent Tasks.
|
||||||
|
|
||||||
|
This service exists for a narrow recovery case: a Task was projected/claimed with a read-only
|
||||||
|
source_capture envelope before its already-registered deterministic plan became authoritative.
|
||||||
|
|
||||||
|
It never creates Tasks/Jobs/branches and never accepts arbitrary metadata patches. It only replaces
|
||||||
|
metadata.execution + the derived execution_projection for the same governed binding, after an
|
||||||
|
official NEEDS_REWORK transition has returned the Task to QUEUED with no active lease.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
from typing import Any, Callable, Mapping, Sequence
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
validate_execution_envelope,
|
||||||
|
)
|
||||||
|
|
||||||
|
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,159}$")
|
||||||
|
SHA_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
MAX_BODY = 64 * 1024
|
||||||
|
AUTHORIZATION_ID_ENV = "AIOS_AGENT_TASK_EXECUTION_REBIND_AUTHORIZATION_ID"
|
||||||
|
|
||||||
|
|
||||||
|
class ExecutionRebindError(RuntimeError):
|
||||||
|
def __init__(self, code: str, status: int = 409) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.status = status
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _sha(value: object) -> str:
|
||||||
|
return hashlib.sha256(_canonical(value)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _require_string(
|
||||||
|
value: object,
|
||||||
|
code: str,
|
||||||
|
pattern: re.Pattern[str] | None = None,
|
||||||
|
) -> str:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
raise ExecutionRebindError(code, 400)
|
||||||
|
value = value.strip()
|
||||||
|
if pattern is not None and pattern.fullmatch(value) is None:
|
||||||
|
raise ExecutionRebindError(code, 400)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _required_authorization_id() -> str:
|
||||||
|
value = os.environ.get(AUTHORIZATION_ID_ENV, "").strip()
|
||||||
|
if not value:
|
||||||
|
raise RuntimeError("TASK_EXECUTION_REBIND_AUTHORIZATION_ID_MISSING")
|
||||||
|
if KEY_RE.fullmatch(value) is None:
|
||||||
|
raise RuntimeError("TASK_EXECUTION_REBIND_AUTHORIZATION_ID_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ExecutionRebindRequest:
|
||||||
|
expected_status: str
|
||||||
|
expected_revision: int
|
||||||
|
expected_attempt: int
|
||||||
|
previous_verification_sha256: str
|
||||||
|
expected_old_execution_digest: str
|
||||||
|
expected_new_execution_digest: str
|
||||||
|
authorization_id: str
|
||||||
|
idempotency_key: str
|
||||||
|
reason: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_mapping(cls, value: Mapping[str, Any]) -> "ExecutionRebindRequest":
|
||||||
|
allowed = {
|
||||||
|
"expected_status",
|
||||||
|
"expected_revision",
|
||||||
|
"expected_attempt",
|
||||||
|
"previous_verification_sha256",
|
||||||
|
"expected_old_execution_digest",
|
||||||
|
"expected_new_execution_digest",
|
||||||
|
"authorization_id",
|
||||||
|
"idempotency_key",
|
||||||
|
"reason",
|
||||||
|
}
|
||||||
|
if set(value) != allowed:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_FIELDS_INVALID", 400)
|
||||||
|
|
||||||
|
status = _require_string(value["expected_status"], "EXPECTED_STATUS_INVALID")
|
||||||
|
if status != "QUEUED":
|
||||||
|
raise ExecutionRebindError("EXPECTED_STATUS_INVALID", 400)
|
||||||
|
|
||||||
|
revision = value["expected_revision"]
|
||||||
|
attempt = value["expected_attempt"]
|
||||||
|
if (
|
||||||
|
not isinstance(revision, int)
|
||||||
|
or isinstance(revision, bool)
|
||||||
|
or revision < 0
|
||||||
|
or not isinstance(attempt, int)
|
||||||
|
or isinstance(attempt, bool)
|
||||||
|
or attempt < 1
|
||||||
|
):
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_INTEGER_INVALID", 400)
|
||||||
|
|
||||||
|
verification = _require_string(
|
||||||
|
value["previous_verification_sha256"], "VERIFICATION_SHA256_INVALID"
|
||||||
|
)
|
||||||
|
old_digest = _require_string(
|
||||||
|
value["expected_old_execution_digest"], "OLD_EXECUTION_DIGEST_INVALID"
|
||||||
|
)
|
||||||
|
new_digest = _require_string(
|
||||||
|
value["expected_new_execution_digest"], "NEW_EXECUTION_DIGEST_INVALID"
|
||||||
|
)
|
||||||
|
for digest, code in (
|
||||||
|
(verification, "VERIFICATION_SHA256_INVALID"),
|
||||||
|
(old_digest, "OLD_EXECUTION_DIGEST_INVALID"),
|
||||||
|
(new_digest, "NEW_EXECUTION_DIGEST_INVALID"),
|
||||||
|
):
|
||||||
|
if SHA_RE.fullmatch(digest) is None:
|
||||||
|
raise ExecutionRebindError(code, 400)
|
||||||
|
if old_digest == new_digest:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_DIGEST_UNCHANGED", 400)
|
||||||
|
|
||||||
|
authorization = _require_string(
|
||||||
|
value["authorization_id"], "AUTHORIZATION_ID_INVALID", KEY_RE
|
||||||
|
)
|
||||||
|
key = _require_string(value["idempotency_key"], "IDEMPOTENCY_KEY_INVALID", KEY_RE)
|
||||||
|
reason = _require_string(value["reason"], "EXECUTION_REBIND_REASON_INVALID")
|
||||||
|
if len(reason) > 512:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_REASON_INVALID", 400)
|
||||||
|
|
||||||
|
return cls(
|
||||||
|
status,
|
||||||
|
revision,
|
||||||
|
attempt,
|
||||||
|
verification,
|
||||||
|
old_digest,
|
||||||
|
new_digest,
|
||||||
|
authorization,
|
||||||
|
key,
|
||||||
|
reason,
|
||||||
|
)
|
||||||
|
|
||||||
|
def public(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"expected_status": self.expected_status,
|
||||||
|
"expected_revision": self.expected_revision,
|
||||||
|
"expected_attempt": self.expected_attempt,
|
||||||
|
"previous_verification_sha256": self.previous_verification_sha256,
|
||||||
|
"expected_old_execution_digest": self.expected_old_execution_digest,
|
||||||
|
"expected_new_execution_digest": self.expected_new_execution_digest,
|
||||||
|
"authorization_id": self.authorization_id,
|
||||||
|
"idempotency_key": self.idempotency_key,
|
||||||
|
"reason": self.reason,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class CanonicalBridgeExecutionResolver:
|
||||||
|
"""Resolve the already-registered plan from the canonical governed Bridge."""
|
||||||
|
|
||||||
|
def __init__(self, base_url: str, control_token: str) -> None:
|
||||||
|
self.base_url = base_url.rstrip("/")
|
||||||
|
self.control_token = control_token
|
||||||
|
|
||||||
|
def __call__(
|
||||||
|
self,
|
||||||
|
task_id: str,
|
||||||
|
required_capabilities: Sequence[str],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
body = json.dumps(
|
||||||
|
{
|
||||||
|
"task_id": task_id,
|
||||||
|
"required_capabilities": list(required_capabilities),
|
||||||
|
},
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
request = urllib.request.Request(
|
||||||
|
self.base_url + "/v2/execution-envelope",
|
||||||
|
method="POST",
|
||||||
|
data=body,
|
||||||
|
headers={
|
||||||
|
"Authorization": "Bearer " + self.control_token,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "application/json",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(request, timeout=15) as response:
|
||||||
|
raw = response.read()
|
||||||
|
status = response.status
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raw = exc.read()
|
||||||
|
status = exc.code
|
||||||
|
try:
|
||||||
|
value = json.loads(raw.decode("utf-8"))
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
raise ExecutionRebindError("CANONICAL_EXECUTION_RESPONSE_INVALID", 502) from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise ExecutionRebindError("CANONICAL_EXECUTION_RESPONSE_INVALID", 502)
|
||||||
|
if status != 200:
|
||||||
|
raise ExecutionRebindError(
|
||||||
|
str(value.get("error") or "CANONICAL_EXECUTION_RESOLUTION_FAILED"),
|
||||||
|
502 if status >= 500 else 409,
|
||||||
|
)
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
|
||||||
|
class ExecutionRebindStore:
|
||||||
|
"""Official owner-controlled mutation with CAS, plan resolution and durable audit."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
db_path: Path,
|
||||||
|
resolver: Callable[[str, Sequence[str]], Mapping[str, Any]],
|
||||||
|
) -> None:
|
||||||
|
self.db_path = Path(db_path)
|
||||||
|
self.resolver = resolver
|
||||||
|
|
||||||
|
def _connect(self) -> sqlite3.Connection:
|
||||||
|
connection = sqlite3.connect(self.db_path, timeout=30.0)
|
||||||
|
connection.row_factory = sqlite3.Row
|
||||||
|
connection.execute("PRAGMA foreign_keys=ON")
|
||||||
|
connection.execute("PRAGMA busy_timeout=30000")
|
||||||
|
return connection
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _columns(
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
table: str,
|
||||||
|
) -> dict[str, sqlite3.Row]:
|
||||||
|
return {
|
||||||
|
str(row["name"]): row
|
||||||
|
for row in connection.execute(f'PRAGMA table_info("{table}")')
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _json_value(raw: object, code: str) -> dict[str, Any]:
|
||||||
|
if isinstance(raw, Mapping):
|
||||||
|
return dict(raw)
|
||||||
|
if not isinstance(raw, str):
|
||||||
|
raise ExecutionRebindError(code, 409)
|
||||||
|
try:
|
||||||
|
value = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise ExecutionRebindError(code, 409) from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise ExecutionRebindError(code, 409)
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _verification_hash(raw: object) -> str:
|
||||||
|
if raw in (None, ""):
|
||||||
|
value: object = {}
|
||||||
|
elif isinstance(raw, str):
|
||||||
|
try:
|
||||||
|
value = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise ExecutionRebindError("TASK_VERIFICATION_INVALID", 409) from exc
|
||||||
|
else:
|
||||||
|
value = raw
|
||||||
|
return _sha(value)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _metadata_column(columns: Mapping[str, sqlite3.Row]) -> str:
|
||||||
|
for name in ("metadata_json", "metadata"):
|
||||||
|
if name in columns:
|
||||||
|
return name
|
||||||
|
raise ExecutionRebindError("TASK_METADATA_SCHEMA_UNSUPPORTED", 503)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _required_capabilities(
|
||||||
|
task: sqlite3.Row,
|
||||||
|
columns: Mapping[str, sqlite3.Row],
|
||||||
|
old_execution: Mapping[str, Any],
|
||||||
|
) -> list[str]:
|
||||||
|
for field in ("required_capabilities_json", "required_capabilities"):
|
||||||
|
if field not in columns:
|
||||||
|
continue
|
||||||
|
raw = task[field]
|
||||||
|
if raw in (None, ""):
|
||||||
|
continue
|
||||||
|
if isinstance(raw, str):
|
||||||
|
try:
|
||||||
|
value = json.loads(raw)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
value = [item for item in raw.split(",") if item.strip()]
|
||||||
|
else:
|
||||||
|
value = raw
|
||||||
|
if isinstance(value, list) and all(isinstance(item, str) for item in value):
|
||||||
|
return [item.strip() for item in value if item.strip()]
|
||||||
|
value = old_execution.get("capability_scope")
|
||||||
|
if isinstance(value, list) and all(isinstance(item, str) for item in value):
|
||||||
|
return [item for item in value if item]
|
||||||
|
raise ExecutionRebindError("TASK_CAPABILITIES_UNAVAILABLE", 503)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _event_table(
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
) -> tuple[str, dict[str, sqlite3.Row]]:
|
||||||
|
tables = {
|
||||||
|
row["name"]
|
||||||
|
for row in connection.execute(
|
||||||
|
"SELECT name FROM sqlite_master WHERE type='table'"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
for name in ("task_events", "events"):
|
||||||
|
if name not in tables:
|
||||||
|
continue
|
||||||
|
columns = ExecutionRebindStore._columns(connection, name)
|
||||||
|
if "task_id" in columns and ("event_type" in columns or "type" in columns):
|
||||||
|
return name, columns
|
||||||
|
raise ExecutionRebindError("TASK_EVENT_TABLE_UNSUPPORTED", 503)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _insert_event(
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
task_id: str,
|
||||||
|
payload: Mapping[str, Any],
|
||||||
|
) -> None:
|
||||||
|
table, columns = ExecutionRebindStore._event_table(connection)
|
||||||
|
values: dict[str, Any] = {"task_id": task_id}
|
||||||
|
if "event_type" in columns:
|
||||||
|
values["event_type"] = "TASK_EXECUTION_REBIND_AUTHORIZED"
|
||||||
|
else:
|
||||||
|
values["type"] = "TASK_EXECUTION_REBIND_AUTHORIZED"
|
||||||
|
payload_text = json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
||||||
|
if "payload_json" in columns:
|
||||||
|
values["payload_json"] = payload_text
|
||||||
|
elif "payload" in columns:
|
||||||
|
values["payload"] = payload_text
|
||||||
|
else:
|
||||||
|
raise ExecutionRebindError("TASK_EVENT_PAYLOAD_UNSUPPORTED", 503)
|
||||||
|
if "created_at" in columns:
|
||||||
|
values["created_at"] = _now()
|
||||||
|
if "agent_id" in columns:
|
||||||
|
values["agent_id"] = "owner-control"
|
||||||
|
if "actor_id" in columns:
|
||||||
|
values["actor_id"] = "owner-control"
|
||||||
|
names = list(values)
|
||||||
|
quoted = ",".join('"' + name + '"' for name in names)
|
||||||
|
placeholders = ",".join("?" for _ in names)
|
||||||
|
connection.execute(
|
||||||
|
f'INSERT INTO "{table}" ({quoted}) VALUES ({placeholders})',
|
||||||
|
tuple(values[name] for name in names),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _ensure_rebind_table(connection: sqlite3.Connection) -> None:
|
||||||
|
connection.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS task_execution_rebinds (
|
||||||
|
task_id TEXT NOT NULL,
|
||||||
|
idempotency_key TEXT NOT NULL,
|
||||||
|
request_sha256 TEXT NOT NULL,
|
||||||
|
authorization_id TEXT NOT NULL,
|
||||||
|
previous_verification_sha256 TEXT NOT NULL,
|
||||||
|
old_execution_digest TEXT NOT NULL,
|
||||||
|
new_execution_digest TEXT NOT NULL,
|
||||||
|
expected_revision INTEGER NOT NULL,
|
||||||
|
applied_revision INTEGER NOT NULL,
|
||||||
|
attempt INTEGER NOT NULL,
|
||||||
|
reason TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
PRIMARY KEY(task_id,idempotency_key)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
def _load_and_validate(
|
||||||
|
self,
|
||||||
|
connection: sqlite3.Connection,
|
||||||
|
task_id: str,
|
||||||
|
request: ExecutionRebindRequest,
|
||||||
|
) -> tuple[sqlite3.Row, dict[str, sqlite3.Row], str, dict[str, Any], dict[str, Any]]:
|
||||||
|
columns = self._columns(connection, "tasks")
|
||||||
|
required = {
|
||||||
|
"task_id",
|
||||||
|
"status",
|
||||||
|
"attempt",
|
||||||
|
"revision",
|
||||||
|
"claimed_by",
|
||||||
|
"lease_token_hash",
|
||||||
|
"lease_expires_at",
|
||||||
|
"verification_json",
|
||||||
|
}
|
||||||
|
if not required.issubset(columns):
|
||||||
|
raise ExecutionRebindError("TASK_SCHEMA_EXECUTION_REBIND_UNSUPPORTED", 503)
|
||||||
|
metadata_column = self._metadata_column(columns)
|
||||||
|
task = connection.execute(
|
||||||
|
"SELECT * FROM tasks WHERE task_id=?",
|
||||||
|
(task_id,),
|
||||||
|
).fetchone()
|
||||||
|
if task is None:
|
||||||
|
raise ExecutionRebindError("TASK_NOT_FOUND", 404)
|
||||||
|
|
||||||
|
if (
|
||||||
|
str(task["status"]) != request.expected_status
|
||||||
|
or int(task["revision"]) != request.expected_revision
|
||||||
|
or int(task["attempt"]) != request.expected_attempt
|
||||||
|
):
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_CAS_MISMATCH", 409)
|
||||||
|
if task["lease_token_hash"] not in (None, "") or task["lease_expires_at"] not in (None, ""):
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_ACTIVE_LEASE_DENY", 409)
|
||||||
|
|
||||||
|
verification_sha = self._verification_hash(task["verification_json"])
|
||||||
|
if verification_sha != request.previous_verification_sha256:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_VERIFICATION_DRIFT", 409)
|
||||||
|
|
||||||
|
metadata = self._json_value(
|
||||||
|
task[metadata_column],
|
||||||
|
"TASK_METADATA_INVALID",
|
||||||
|
)
|
||||||
|
execution = metadata.get("execution")
|
||||||
|
if not isinstance(execution, Mapping):
|
||||||
|
raise ExecutionRebindError("TASK_EXECUTION_METADATA_MISSING", 409)
|
||||||
|
old_execution = validate_execution_envelope(dict(execution))
|
||||||
|
if old_execution["execution_digest"] != request.expected_old_execution_digest:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_OLD_DIGEST_DRIFT", 409)
|
||||||
|
if old_execution.get("action") != "source_capture":
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_SOURCE_CAPTURE_REQUIRED", 409)
|
||||||
|
|
||||||
|
projection = metadata.get("execution_projection")
|
||||||
|
if isinstance(projection, Mapping):
|
||||||
|
projected_digest = str(projection.get("execution_digest") or "")
|
||||||
|
if projected_digest and projected_digest != request.expected_old_execution_digest:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_PROJECTION_DRIFT", 409)
|
||||||
|
return task, columns, metadata_column, metadata, old_execution
|
||||||
|
|
||||||
|
def rebind(
|
||||||
|
self,
|
||||||
|
task_id: str,
|
||||||
|
request: ExecutionRebindRequest,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
task_id = _require_string(task_id, "TASK_ID_INVALID", TASK_RE)
|
||||||
|
required_authorization = _required_authorization_id()
|
||||||
|
if request.authorization_id != required_authorization:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_AUTHORIZATION_ID_MISMATCH", 403)
|
||||||
|
|
||||||
|
request_sha = _sha(request.public())
|
||||||
|
|
||||||
|
# Idempotent replay must win before CAS validation because a successful
|
||||||
|
# rebind necessarily increments the Task revision and changes the persisted envelope.
|
||||||
|
with self._connect() as connection:
|
||||||
|
self._ensure_rebind_table(connection)
|
||||||
|
replay = connection.execute(
|
||||||
|
"SELECT * FROM task_execution_rebinds WHERE task_id=? AND idempotency_key=?",
|
||||||
|
(task_id, request.idempotency_key),
|
||||||
|
).fetchone()
|
||||||
|
if replay is not None:
|
||||||
|
if replay["request_sha256"] != request_sha:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_IDEMPOTENCY_CONFLICT", 409)
|
||||||
|
connection.commit()
|
||||||
|
return {
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "QUEUED",
|
||||||
|
"attempt": int(replay["attempt"]),
|
||||||
|
"revision": int(replay["applied_revision"]),
|
||||||
|
"old_execution_digest": replay["old_execution_digest"],
|
||||||
|
"new_execution_digest": replay["new_execution_digest"],
|
||||||
|
"authorization_id": replay["authorization_id"],
|
||||||
|
"idempotent": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Read/CAS snapshot first. Canonical Bridge resolution happens outside the DB write lock.
|
||||||
|
with self._connect() as connection:
|
||||||
|
task, columns, _, _, old_execution = self._load_and_validate(
|
||||||
|
connection, task_id, request
|
||||||
|
)
|
||||||
|
capabilities = self._required_capabilities(task, columns, old_execution)
|
||||||
|
|
||||||
|
binding = GovernedExecutionBinding.from_mapping(old_execution)
|
||||||
|
resolved = dict(self.resolver(task_id, capabilities))
|
||||||
|
new_execution = validate_execution_envelope(resolved, binding=binding)
|
||||||
|
if new_execution.get("action") != "deterministic":
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_DETERMINISTIC_PLAN_REQUIRED", 409)
|
||||||
|
if new_execution["execution_digest"] != request.expected_new_execution_digest:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_NEW_DIGEST_DRIFT", 409)
|
||||||
|
if new_execution["execution_digest"] == old_execution["execution_digest"]:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_DIGEST_UNCHANGED", 409)
|
||||||
|
|
||||||
|
# Re-read under BEGIN IMMEDIATE and apply only if the exact snapshot is unchanged.
|
||||||
|
with self._connect() as connection:
|
||||||
|
connection.execute("BEGIN IMMEDIATE")
|
||||||
|
self._ensure_rebind_table(connection)
|
||||||
|
|
||||||
|
replay = connection.execute(
|
||||||
|
"SELECT * FROM task_execution_rebinds WHERE task_id=? AND idempotency_key=?",
|
||||||
|
(task_id, request.idempotency_key),
|
||||||
|
).fetchone()
|
||||||
|
if replay is not None:
|
||||||
|
if replay["request_sha256"] != request_sha:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_IDEMPOTENCY_CONFLICT", 409)
|
||||||
|
connection.commit()
|
||||||
|
return {
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "QUEUED",
|
||||||
|
"attempt": int(replay["attempt"]),
|
||||||
|
"revision": int(replay["applied_revision"]),
|
||||||
|
"old_execution_digest": replay["old_execution_digest"],
|
||||||
|
"new_execution_digest": replay["new_execution_digest"],
|
||||||
|
"authorization_id": replay["authorization_id"],
|
||||||
|
"idempotent": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
task, columns, metadata_column, metadata, current_old = self._load_and_validate(
|
||||||
|
connection, task_id, request
|
||||||
|
)
|
||||||
|
if current_old["execution_digest"] != old_execution["execution_digest"]:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_OLD_DIGEST_DRIFT", 409)
|
||||||
|
|
||||||
|
new_metadata = dict(metadata)
|
||||||
|
new_metadata["execution"] = new_execution
|
||||||
|
new_metadata["execution_projection"] = {
|
||||||
|
"schema": "aios.online-agent.task-projection.v1",
|
||||||
|
"task_id": binding.task_id,
|
||||||
|
"job_id": binding.job_id,
|
||||||
|
"execution_digest": new_execution["execution_digest"],
|
||||||
|
}
|
||||||
|
applied_revision = request.expected_revision + 1
|
||||||
|
previous_claimed_by = task["claimed_by"]
|
||||||
|
|
||||||
|
cursor = connection.execute(
|
||||||
|
f"""
|
||||||
|
UPDATE tasks
|
||||||
|
SET "{metadata_column}"=?, revision=?, claimed_by=NULL,
|
||||||
|
lease_token_hash=NULL, lease_expires_at=NULL
|
||||||
|
WHERE task_id=? AND status=? AND revision=? AND attempt=?
|
||||||
|
AND claimed_by IS ? AND lease_token_hash IS NULL AND lease_expires_at IS NULL
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
json.dumps(new_metadata, sort_keys=True, separators=(",", ":")),
|
||||||
|
applied_revision,
|
||||||
|
task_id,
|
||||||
|
request.expected_status,
|
||||||
|
request.expected_revision,
|
||||||
|
request.expected_attempt,
|
||||||
|
previous_claimed_by,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if cursor.rowcount != 1:
|
||||||
|
raise ExecutionRebindError("EXECUTION_REBIND_CAS_MISMATCH", 409)
|
||||||
|
|
||||||
|
event_payload = {
|
||||||
|
"schema": "aios.task-execution-rebind.event.v1",
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": binding.job_id,
|
||||||
|
"authorization_id": request.authorization_id,
|
||||||
|
"idempotency_key": request.idempotency_key,
|
||||||
|
"reason": request.reason,
|
||||||
|
"previous_verification_sha256": request.previous_verification_sha256,
|
||||||
|
"old_execution_digest": old_execution["execution_digest"],
|
||||||
|
"new_execution_digest": new_execution["execution_digest"],
|
||||||
|
"old_action": old_execution.get("action"),
|
||||||
|
"new_action": new_execution.get("action"),
|
||||||
|
"previous_revision": request.expected_revision,
|
||||||
|
"applied_revision": applied_revision,
|
||||||
|
"attempt": request.expected_attempt,
|
||||||
|
"request_sha256": request_sha,
|
||||||
|
"previous_claimed_by": previous_claimed_by,
|
||||||
|
"stale_claim_normalized": previous_claimed_by not in (None, ""),
|
||||||
|
}
|
||||||
|
self._insert_event(connection, task_id, event_payload)
|
||||||
|
connection.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO task_execution_rebinds(
|
||||||
|
task_id,idempotency_key,request_sha256,authorization_id,
|
||||||
|
previous_verification_sha256,old_execution_digest,new_execution_digest,
|
||||||
|
expected_revision,applied_revision,attempt,reason,created_at
|
||||||
|
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
task_id,
|
||||||
|
request.idempotency_key,
|
||||||
|
request_sha,
|
||||||
|
request.authorization_id,
|
||||||
|
request.previous_verification_sha256,
|
||||||
|
old_execution["execution_digest"],
|
||||||
|
new_execution["execution_digest"],
|
||||||
|
request.expected_revision,
|
||||||
|
applied_revision,
|
||||||
|
request.expected_attempt,
|
||||||
|
request.reason,
|
||||||
|
_now(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
return {
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "QUEUED",
|
||||||
|
"attempt": request.expected_attempt,
|
||||||
|
"revision": applied_revision,
|
||||||
|
"old_execution_digest": old_execution["execution_digest"],
|
||||||
|
"new_execution_digest": new_execution["execution_digest"],
|
||||||
|
"authorization_id": request.authorization_id,
|
||||||
|
"idempotent": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class ExecutionRebindHandler(BaseHTTPRequestHandler):
|
||||||
|
store: ExecutionRebindStore | None = None
|
||||||
|
control_token: str = ""
|
||||||
|
|
||||||
|
def log_message(self, *_args: Any) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def _json(self, value: Mapping[str, Any], status: int) -> None:
|
||||||
|
data = json.dumps(
|
||||||
|
dict(value), sort_keys=True, separators=(",", ":")
|
||||||
|
).encode("utf-8")
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(data)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(data)
|
||||||
|
|
||||||
|
def _authorized(self) -> bool:
|
||||||
|
header = self.headers.get("Authorization", "")
|
||||||
|
return (
|
||||||
|
header.startswith("Bearer ")
|
||||||
|
and bool(self.control_token)
|
||||||
|
and hmac.compare_digest(header[7:], self.control_token)
|
||||||
|
)
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
if self.path == "/health":
|
||||||
|
self._json(
|
||||||
|
{"status": "ok", "service": "aios-agent-task-execution-rebind"},
|
||||||
|
200,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self._json({"error": "NOT_FOUND"}, 404)
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
match = re.fullmatch(r"/v1/tasks/([^/]+)/execution-rebind", self.path)
|
||||||
|
if match is None:
|
||||||
|
self._json({"error": "NOT_FOUND"}, 404)
|
||||||
|
return
|
||||||
|
if not self._authorized():
|
||||||
|
self._json({"error": "UNAUTHORIZED"}, 401)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get("Content-Length", "0"))
|
||||||
|
except ValueError:
|
||||||
|
self._json({"error": "CONTENT_LENGTH_INVALID"}, 400)
|
||||||
|
return
|
||||||
|
if length < 1 or length > MAX_BODY:
|
||||||
|
self._json({"error": "REQUEST_SIZE_INVALID"}, 413)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
value = json.loads(self.rfile.read(length).decode("utf-8"))
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise ValueError
|
||||||
|
request = ExecutionRebindRequest.from_mapping(value)
|
||||||
|
if self.store is None:
|
||||||
|
raise ExecutionRebindError("RUNTIME_UNAVAILABLE", 503)
|
||||||
|
result = self.store.rebind(match.group(1), request)
|
||||||
|
self._json(result, 200 if result["idempotent"] else 201)
|
||||||
|
except ExecutionRebindError as exc:
|
||||||
|
self._json({"error": exc.code}, exc.status)
|
||||||
|
except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
|
||||||
|
self._json({"error": "REQUEST_INVALID"}, 400)
|
||||||
|
|
||||||
|
|
||||||
|
def load_control_token(path: Path) -> str:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise RuntimeError("TASK_EXECUTION_REBIND_CONTROL_CREDENTIAL_UNAVAILABLE")
|
||||||
|
value = path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(value) < 32:
|
||||||
|
raise RuntimeError("TASK_EXECUTION_REBIND_CONTROL_CREDENTIAL_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def serve() -> None:
|
||||||
|
db = Path(os.environ.get("AIOS_AGENT_TASK_DB", "/var/lib/aios-agent-task/state.sqlite3"))
|
||||||
|
_required_authorization_id()
|
||||||
|
token_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
|
||||||
|
str(
|
||||||
|
Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
||||||
|
/ "agent_task_control_token"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
token = load_control_token(token_path)
|
||||||
|
bridge = os.environ.get(
|
||||||
|
"AIOS_GOVERNED_WRITE_API_URL",
|
||||||
|
"http://127.0.0.1:18892",
|
||||||
|
)
|
||||||
|
host = os.environ.get("AIOS_TASK_EXECUTION_REBIND_HOST", "127.0.0.1")
|
||||||
|
port = int(os.environ.get("AIOS_TASK_EXECUTION_REBIND_PORT", "18942"))
|
||||||
|
ExecutionRebindHandler.store = ExecutionRebindStore(
|
||||||
|
db,
|
||||||
|
CanonicalBridgeExecutionResolver(bridge, token),
|
||||||
|
)
|
||||||
|
ExecutionRebindHandler.control_token = token
|
||||||
|
ThreadingHTTPServer((host, port), ExecutionRebindHandler).serve_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
serve()
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
"""Canonical fail-closed capability semantics for task eligibility."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Iterable
|
||||||
|
|
||||||
|
|
||||||
|
_TASK_ALIASES: dict[str, frozenset[str]] = {
|
||||||
|
"repository_read": frozenset({"repo.read"}),
|
||||||
|
"git.read": frozenset({"repo.read"}),
|
||||||
|
"repo.read": frozenset({"repo.read"}),
|
||||||
|
"test_execution": frozenset({"test.execute"}),
|
||||||
|
"git.test": frozenset({"test.execute"}),
|
||||||
|
"test.execute": frozenset({"test.execute"}),
|
||||||
|
"governed_git_write": frozenset({"governed.git.write.branch", "governed.git.write.file"}),
|
||||||
|
"governed.git.write": frozenset({"governed.git.write.branch", "governed.git.write.file"}),
|
||||||
|
"governed.write.branch": frozenset({"governed.git.write.branch"}),
|
||||||
|
"governed.write.file": frozenset({"governed.git.write.file"}),
|
||||||
|
"shell": frozenset({"runtime.shell"}),
|
||||||
|
"git": frozenset({"runtime.git"}),
|
||||||
|
"python": frozenset({"runtime.python"}),
|
||||||
|
"code.read": frozenset({"legacy.code.read"}),
|
||||||
|
"code.review": frozenset({"legacy.code.review"}),
|
||||||
|
"code.write": frozenset({"legacy.code.write"}),
|
||||||
|
"code.write.patch": frozenset({"legacy.code.write.patch"}),
|
||||||
|
"test.python": frozenset({"legacy.test.python"}),
|
||||||
|
"test.shell.allowlisted": frozenset({"legacy.test.shell.allowlisted"}),
|
||||||
|
"git.commit": frozenset({"legacy.git.commit"}),
|
||||||
|
"git.diff.read": frozenset({"legacy.git.diff.read"}),
|
||||||
|
"git.force_push": frozenset({"legacy.git.force_push"}),
|
||||||
|
"deploy.production": frozenset({"legacy.deploy.production"}),
|
||||||
|
"secret.transform": frozenset({"legacy.secret.transform"}),
|
||||||
|
"secure_file_transformer": frozenset({"legacy.secure_file_transformer"}),
|
||||||
|
"runtime.shadow": frozenset({"legacy.runtime.shadow"}),
|
||||||
|
"review.security": frozenset({"legacy.review.security"}),
|
||||||
|
"review.api-contract": frozenset({"legacy.review.api-contract"}),
|
||||||
|
"gpu.train": frozenset({"legacy.gpu.train"}),
|
||||||
|
"agent.runtime.hermes": frozenset({"agent.runtime.hermes"}),
|
||||||
|
"agent.runtime.openclaw": frozenset({"agent.runtime.openclaw"}),
|
||||||
|
}
|
||||||
|
|
||||||
|
_WORKER_ALIASES: dict[str, frozenset[str]] = {
|
||||||
|
"git.read": frozenset({"repo.read"}),
|
||||||
|
"repo.read": frozenset({"repo.read"}),
|
||||||
|
"repository_read": frozenset({"repo.read"}),
|
||||||
|
"git.test": frozenset({"test.execute"}),
|
||||||
|
"test.execute": frozenset({"test.execute"}),
|
||||||
|
"test_execution": frozenset({"test.execute"}),
|
||||||
|
"governed.write.branch": frozenset({"governed.git.write.branch"}),
|
||||||
|
"governed.git.write.branch": frozenset({"governed.git.write.branch"}),
|
||||||
|
"governed.write.file": frozenset({"governed.git.write.file"}),
|
||||||
|
"governed.git.write.file": frozenset({"governed.git.write.file"}),
|
||||||
|
"shell": frozenset({"runtime.shell"}),
|
||||||
|
"git": frozenset({"runtime.git"}),
|
||||||
|
"python": frozenset({"runtime.python"}),
|
||||||
|
"code.read": frozenset({"legacy.code.read"}),
|
||||||
|
"code.review": frozenset({"legacy.code.review"}),
|
||||||
|
"code.write": frozenset({"legacy.code.write"}),
|
||||||
|
"code.write.patch": frozenset({"legacy.code.write.patch"}),
|
||||||
|
"test.python": frozenset({"legacy.test.python"}),
|
||||||
|
"test.shell.allowlisted": frozenset({"legacy.test.shell.allowlisted"}),
|
||||||
|
"git.commit": frozenset({"legacy.git.commit"}),
|
||||||
|
"git.diff.read": frozenset({"legacy.git.diff.read"}),
|
||||||
|
"git.force_push": frozenset({"legacy.git.force_push"}),
|
||||||
|
"deploy.production": frozenset({"legacy.deploy.production"}),
|
||||||
|
"secret.transform": frozenset({"legacy.secret.transform"}),
|
||||||
|
"secure_file_transformer": frozenset({"legacy.secure_file_transformer"}),
|
||||||
|
"runtime.shadow": frozenset({"legacy.runtime.shadow"}),
|
||||||
|
"review.security": frozenset({"legacy.review.security"}),
|
||||||
|
"review.api-contract": frozenset({"legacy.review.api-contract"}),
|
||||||
|
"gpu.train": frozenset({"legacy.gpu.train"}),
|
||||||
|
"agent.runtime.hermes": frozenset({"agent.runtime.hermes"}),
|
||||||
|
"agent.runtime.openclaw": frozenset({"agent.runtime.openclaw"}),
|
||||||
|
}
|
||||||
|
|
||||||
|
_UNKNOWN_PREFIX = "exact:"
|
||||||
|
|
||||||
|
|
||||||
|
def _clean(value: object) -> str:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise ValueError("capability must be a string")
|
||||||
|
text = value.strip().lower()
|
||||||
|
if not text:
|
||||||
|
raise ValueError("capability must not be empty")
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_required_capabilities(values: Iterable[object]) -> frozenset[str]:
|
||||||
|
"""Map task-facing names to one semantic vocabulary.
|
||||||
|
|
||||||
|
Unknown names remain explicit exact requirements. They are never treated
|
||||||
|
as wildcards or granted by a failed normalization.
|
||||||
|
"""
|
||||||
|
|
||||||
|
result: set[str] = set()
|
||||||
|
for value in values:
|
||||||
|
text = _clean(value)
|
||||||
|
if "*" in text:
|
||||||
|
result.add(_UNKNOWN_PREFIX + text)
|
||||||
|
else:
|
||||||
|
result.update(_TASK_ALIASES.get(text, frozenset({_UNKNOWN_PREFIX + text})))
|
||||||
|
return frozenset(result)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_worker_capabilities(values: Iterable[object]) -> frozenset[str]:
|
||||||
|
"""Map registered worker names to effective semantic capabilities."""
|
||||||
|
|
||||||
|
result: set[str] = set()
|
||||||
|
for value in values:
|
||||||
|
text = _clean(value)
|
||||||
|
if "*" in text:
|
||||||
|
continue
|
||||||
|
mapped = _WORKER_ALIASES.get(text)
|
||||||
|
if mapped is not None:
|
||||||
|
result.update(mapped)
|
||||||
|
if {
|
||||||
|
"governed.git.write.branch",
|
||||||
|
"governed.git.write.file",
|
||||||
|
}.issubset(result):
|
||||||
|
result.add("governed.git.write")
|
||||||
|
return frozenset(result)
|
||||||
|
|
||||||
|
|
||||||
|
def has_capabilities(
|
||||||
|
worker_capabilities: Iterable[object],
|
||||||
|
required_capabilities: Iterable[object],
|
||||||
|
) -> bool:
|
||||||
|
"""Return the single eligibility decision used by poll and claim."""
|
||||||
|
|
||||||
|
required = normalize_required_capabilities(required_capabilities)
|
||||||
|
effective = normalize_worker_capabilities(worker_capabilities)
|
||||||
|
if any(value.startswith(_UNKNOWN_PREFIX) for value in required):
|
||||||
|
return False
|
||||||
|
return required.issubset(effective)
|
||||||
|
|
||||||
|
|
||||||
|
def required_capabilities_subset(
|
||||||
|
granted_requirements: Iterable[object],
|
||||||
|
required_requirements: Iterable[object],
|
||||||
|
) -> bool:
|
||||||
|
"""Check semantic capability containment for delegation constraints."""
|
||||||
|
granted = normalize_required_capabilities(granted_requirements)
|
||||||
|
required = normalize_required_capabilities(required_requirements)
|
||||||
|
if any(value.startswith(_UNKNOWN_PREFIX) for value in granted | required):
|
||||||
|
return False
|
||||||
|
return required.issubset(granted)
|
||||||
|
|
||||||
|
|
||||||
|
def normalized_capability_matrix(
|
||||||
|
worker_capabilities: Iterable[object],
|
||||||
|
required_capabilities: Iterable[object],
|
||||||
|
) -> dict[str, object]:
|
||||||
|
"""Expose non-secret decision evidence for tests and diagnostics."""
|
||||||
|
|
||||||
|
worker_values = tuple(worker_capabilities)
|
||||||
|
required_values = tuple(required_capabilities)
|
||||||
|
worker = normalize_worker_capabilities(worker_values)
|
||||||
|
required = normalize_required_capabilities(required_values)
|
||||||
|
return {
|
||||||
|
"worker_effective": sorted(worker),
|
||||||
|
"task_required": sorted(required),
|
||||||
|
"eligible": has_capabilities(worker_values, required_values),
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,232 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Root-owned Model Governance configuration reconciler.
|
||||||
|
|
||||||
|
The config contains provider/model/policy metadata only. Raw credentials are rejected by
|
||||||
|
aios_model_governance and never stored in the governance database.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
from aios_model_governance import (
|
||||||
|
ModelGovernanceError,
|
||||||
|
ModelGovernanceStore,
|
||||||
|
ModelRecord,
|
||||||
|
ProviderRecord,
|
||||||
|
ScopePolicy,
|
||||||
|
)
|
||||||
|
|
||||||
|
SCHEMA = "aios.model-governance.bootstrap.v1"
|
||||||
|
MAX_CONFIG_BYTES = 512 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def fail(code: str) -> None:
|
||||||
|
raise SystemExit(code)
|
||||||
|
|
||||||
|
|
||||||
|
def _load(path: Path) -> dict[str, Any]:
|
||||||
|
if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_CONFIG_BYTES:
|
||||||
|
fail("MODEL_GOVERNANCE_CONFIG_INVALID")
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
fail("MODEL_GOVERNANCE_CONFIG_INVALID")
|
||||||
|
if not isinstance(value, Mapping) or value.get("schema") != SCHEMA:
|
||||||
|
fail("MODEL_GOVERNANCE_CONFIG_SCHEMA_INVALID")
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _provider_desired(value: Mapping[str, Any]) -> ProviderRecord:
|
||||||
|
return ProviderRecord(
|
||||||
|
provider_id=str(value.get("provider_id") or ""),
|
||||||
|
credential_ref=value.get("credential_ref"),
|
||||||
|
enabled=bool(value.get("enabled", True)),
|
||||||
|
failure_domain=str(value.get("failure_domain") or "default"),
|
||||||
|
max_failures=int(value.get("max_failures") or 3),
|
||||||
|
cooldown_seconds=int(value.get("cooldown_seconds") or 60),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _model_desired(value: Mapping[str, Any]) -> ModelRecord:
|
||||||
|
return ModelRecord(
|
||||||
|
model_id=str(value.get("model_id") or ""),
|
||||||
|
provider_id=str(value.get("provider_id") or ""),
|
||||||
|
version=str(value.get("version") or ""),
|
||||||
|
capabilities=tuple(value.get("capabilities") or ()),
|
||||||
|
input_cost_micros_per_1k=int(value.get("input_cost_micros_per_1k") or 0),
|
||||||
|
output_cost_micros_per_1k=int(value.get("output_cost_micros_per_1k") or 0),
|
||||||
|
max_input_tokens=int(value.get("max_input_tokens") or 0),
|
||||||
|
max_output_tokens=int(value.get("max_output_tokens") or 0),
|
||||||
|
evaluation_state=str(value.get("evaluation_state") or "APPROVED"),
|
||||||
|
evaluation_sha256=str(value.get("evaluation_sha256") or ""),
|
||||||
|
enabled=bool(value.get("enabled", True)),
|
||||||
|
priority=int(value.get("priority") or 100),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _policy_desired(value: Mapping[str, Any]) -> ScopePolicy:
|
||||||
|
return ScopePolicy(
|
||||||
|
scope_type=str(value.get("scope_type") or ""),
|
||||||
|
tenant_id=value.get("tenant_id"),
|
||||||
|
project_id=value.get("project_id"),
|
||||||
|
allowed_models=tuple(value.get("allowed_models") or ()),
|
||||||
|
denied_providers=tuple(value.get("denied_providers") or ()),
|
||||||
|
required_capabilities=tuple(value.get("required_capabilities") or ()),
|
||||||
|
pinned_model_id=value.get("pinned_model_id"),
|
||||||
|
pinned_model_version=value.get("pinned_model_version"),
|
||||||
|
max_input_tokens_per_request=int(value.get("max_input_tokens_per_request") or 0),
|
||||||
|
max_output_tokens_per_request=int(value.get("max_output_tokens_per_request") or 0),
|
||||||
|
daily_token_budget=int(value.get("daily_token_budget") or 0),
|
||||||
|
daily_cost_budget_micros=int(value.get("daily_cost_budget_micros") or 0),
|
||||||
|
fallback_limit=int(value.get("fallback_limit") or 0),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _row_dict(row: object) -> dict[str, Any] | None:
|
||||||
|
return dict(row) if row is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _provider_matches(row: Mapping[str, Any], desired: ProviderRecord) -> bool:
|
||||||
|
return (
|
||||||
|
row["credential_ref"] == desired.credential_ref
|
||||||
|
and bool(row["enabled"]) == desired.enabled
|
||||||
|
and row["failure_domain"] == desired.failure_domain
|
||||||
|
and int(row["max_failures"]) == desired.max_failures
|
||||||
|
and int(row["cooldown_seconds"]) == desired.cooldown_seconds
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _model_matches(row: Mapping[str, Any], desired: ModelRecord) -> bool:
|
||||||
|
return (
|
||||||
|
row["provider_id"] == desired.provider_id
|
||||||
|
and row["version"] == desired.version
|
||||||
|
and tuple(json.loads(row["capabilities_json"])) == tuple(sorted(set(desired.capabilities)))
|
||||||
|
and int(row["input_cost_micros_per_1k"]) == desired.input_cost_micros_per_1k
|
||||||
|
and int(row["output_cost_micros_per_1k"]) == desired.output_cost_micros_per_1k
|
||||||
|
and int(row["max_input_tokens"]) == desired.max_input_tokens
|
||||||
|
and int(row["max_output_tokens"]) == desired.max_output_tokens
|
||||||
|
and row["evaluation_state"] == desired.evaluation_state
|
||||||
|
and row["evaluation_sha256"] == desired.evaluation_sha256
|
||||||
|
and bool(row["enabled"]) == desired.enabled
|
||||||
|
and int(row["priority"]) == desired.priority
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _policy_matches(row: Mapping[str, Any], desired: ScopePolicy) -> bool:
|
||||||
|
return (
|
||||||
|
row["scope_type"] == desired.scope_type
|
||||||
|
and row["tenant_id"] == desired.tenant_id
|
||||||
|
and row["project_id"] == desired.project_id
|
||||||
|
and tuple(json.loads(row["allowed_models_json"])) == tuple(sorted(set(desired.allowed_models)))
|
||||||
|
and tuple(json.loads(row["denied_providers_json"])) == tuple(sorted(set(desired.denied_providers)))
|
||||||
|
and tuple(json.loads(row["required_capabilities_json"])) == tuple(sorted(set(desired.required_capabilities)))
|
||||||
|
and row["pinned_model_id"] == desired.pinned_model_id
|
||||||
|
and row["pinned_model_version"] == desired.pinned_model_version
|
||||||
|
and int(row["max_input_tokens_per_request"]) == desired.max_input_tokens_per_request
|
||||||
|
and int(row["max_output_tokens_per_request"]) == desired.max_output_tokens_per_request
|
||||||
|
and int(row["daily_token_budget"]) == desired.daily_token_budget
|
||||||
|
and int(row["daily_cost_budget_micros"]) == desired.daily_cost_budget_micros
|
||||||
|
and int(row["fallback_limit"]) == desired.fallback_limit
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_config(db_path: Path, config_path: Path) -> dict[str, Any]:
|
||||||
|
config = _load(config_path)
|
||||||
|
config_sha = hashlib.sha256(canonical(config)).hexdigest()
|
||||||
|
providers = config.get("providers") or []
|
||||||
|
models = config.get("models") or []
|
||||||
|
policies = config.get("policies") or []
|
||||||
|
if not all(isinstance(value, list) for value in (providers, models, policies)):
|
||||||
|
fail("MODEL_GOVERNANCE_CONFIG_COLLECTION_INVALID")
|
||||||
|
|
||||||
|
store = ModelGovernanceStore(db_path)
|
||||||
|
changes = {"providers": 0, "models": 0, "policies": 0, "unchanged": 0}
|
||||||
|
try:
|
||||||
|
for item in providers:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
fail("MODEL_GOVERNANCE_PROVIDER_CONFIG_INVALID")
|
||||||
|
desired = _provider_desired(item)
|
||||||
|
row = store.conn.execute(
|
||||||
|
"SELECT * FROM model_governance_providers WHERE provider_id=?",
|
||||||
|
(desired.provider_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None and _provider_matches(row, desired):
|
||||||
|
changes["unchanged"] += 1
|
||||||
|
continue
|
||||||
|
expected = int(row["revision"]) if row is not None else 0
|
||||||
|
store.register_provider(desired, expected_revision=expected)
|
||||||
|
changes["providers"] += 1
|
||||||
|
|
||||||
|
for item in models:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
fail("MODEL_GOVERNANCE_MODEL_CONFIG_INVALID")
|
||||||
|
desired = _model_desired(item)
|
||||||
|
row = store.conn.execute(
|
||||||
|
"SELECT * FROM model_governance_models WHERE model_id=?",
|
||||||
|
(desired.model_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None and _model_matches(row, desired):
|
||||||
|
changes["unchanged"] += 1
|
||||||
|
continue
|
||||||
|
expected = int(row["revision"]) if row is not None else 0
|
||||||
|
store.register_model(desired, expected_revision=expected)
|
||||||
|
changes["models"] += 1
|
||||||
|
|
||||||
|
for item in policies:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
fail("MODEL_GOVERNANCE_POLICY_CONFIG_INVALID")
|
||||||
|
desired = _policy_desired(item)
|
||||||
|
row = store.conn.execute(
|
||||||
|
"SELECT * FROM model_governance_scope_policies WHERE scope_key=?",
|
||||||
|
(desired.scope_key,),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None and _policy_matches(row, desired):
|
||||||
|
changes["unchanged"] += 1
|
||||||
|
continue
|
||||||
|
expected = int(row["revision"]) if row is not None else 0
|
||||||
|
store.set_scope_policy(desired, expected_revision=expected)
|
||||||
|
changes["policies"] += 1
|
||||||
|
|
||||||
|
if not store.verify_audit_chain():
|
||||||
|
fail("MODEL_GOVERNANCE_AUDIT_CHAIN_INVALID")
|
||||||
|
return {
|
||||||
|
"schema": "aios.model-governance.bootstrap-result.v1",
|
||||||
|
"database": str(db_path),
|
||||||
|
"config_sha256": config_sha,
|
||||||
|
"changes": changes,
|
||||||
|
"audit_chain_valid": True,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--db", required=True)
|
||||||
|
parser.add_argument("--config", required=True)
|
||||||
|
parser.add_argument("--output")
|
||||||
|
args = parser.parse_args()
|
||||||
|
try:
|
||||||
|
result = apply_config(Path(args.db), Path(args.config))
|
||||||
|
except ModelGovernanceError as exc:
|
||||||
|
print(json.dumps({"status": "FAILED", "error": exc.code}, sort_keys=True))
|
||||||
|
return 20
|
||||||
|
data = json.dumps(result, indent=2, sort_keys=True) + "\n"
|
||||||
|
if args.output:
|
||||||
|
Path(args.output).write_text(data, encoding="utf-8")
|
||||||
|
print(data, end="")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -17,6 +17,7 @@ PHASE_ORDER = {
|
|||||||
"claimed": 10,
|
"claimed": 10,
|
||||||
"workspace_prepared": 20,
|
"workspace_prepared": 20,
|
||||||
"verification_passed": 30,
|
"verification_passed": 30,
|
||||||
|
"agent_runtime_done": 40,
|
||||||
"source_capture_done": 40,
|
"source_capture_done": 40,
|
||||||
"commit_done": 40,
|
"commit_done": 40,
|
||||||
"reconciliation_required": 45,
|
"reconciliation_required": 45,
|
||||||
|
|||||||
@@ -24,7 +24,10 @@ BRANCH_RE = re.compile(r"^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$")
|
|||||||
KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,159}$")
|
KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,159}$")
|
||||||
PROTECTED_PREFIXES = ("main", "master", "refs/", "tags/", "tag/", "release/", "baseline/", "production/")
|
PROTECTED_PREFIXES = ("main", "master", "refs/", "tags/", "tag/", "release/", "baseline/", "production/")
|
||||||
MUTATION_ACTIONS = frozenset({"deterministic"})
|
MUTATION_ACTIONS = frozenset({"deterministic"})
|
||||||
SUPPORTED_ACTIONS = frozenset({"deterministic", "source_capture"})
|
READ_ONLY_ACTIONS = frozenset({"source_capture", "agent_runtime"})
|
||||||
|
SUPPORTED_ACTIONS = frozenset({"deterministic", "source_capture", "agent_runtime"})
|
||||||
|
AGENT_RUNTIME_ADAPTERS = frozenset({"hermes", "openclaw"})
|
||||||
|
MAX_AGENT_RUNTIME_INPUT_BYTES = 64 * 1024
|
||||||
|
|
||||||
|
|
||||||
class ExecutionProjectionError(RuntimeError):
|
class ExecutionProjectionError(RuntimeError):
|
||||||
@@ -172,6 +175,76 @@ def _source_signals(value: object) -> tuple[str, ...]:
|
|||||||
return tuple(result)
|
return tuple(result)
|
||||||
|
|
||||||
|
|
||||||
|
def _agent_runtime_input(value: object) -> dict[str, Any]:
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
_error("AGENT_RUNTIME_INPUT_REQUIRED")
|
||||||
|
raw = dict(value)
|
||||||
|
allowed = {
|
||||||
|
"goal", "instructions", "success_criteria", "constraints",
|
||||||
|
"scope", "trace_id",
|
||||||
|
}
|
||||||
|
if set(raw) - allowed:
|
||||||
|
_error("AGENT_RUNTIME_INPUT_FIELDS_INVALID")
|
||||||
|
goal = _string(raw.get("goal"), "AGENT_RUNTIME_GOAL_REQUIRED")
|
||||||
|
instructions = raw.get("instructions", "")
|
||||||
|
if not isinstance(instructions, str):
|
||||||
|
_error("AGENT_RUNTIME_INSTRUCTIONS_INVALID")
|
||||||
|
criteria = raw.get("success_criteria", [])
|
||||||
|
if not isinstance(criteria, list) or len(criteria) > 32:
|
||||||
|
_error("AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID")
|
||||||
|
normalized_criteria: list[dict[str, str]] = []
|
||||||
|
for item in criteria:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
_error("AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID")
|
||||||
|
criterion_id = _string(item.get("id"), "AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID")
|
||||||
|
description = _string(
|
||||||
|
item.get("description"), "AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID"
|
||||||
|
)
|
||||||
|
normalized_criteria.append(
|
||||||
|
{"id": criterion_id[:120], "description": description[:1000]}
|
||||||
|
)
|
||||||
|
constraints = raw.get("constraints", {})
|
||||||
|
if not isinstance(constraints, Mapping):
|
||||||
|
_error("AGENT_RUNTIME_CONSTRAINTS_INVALID")
|
||||||
|
sensitive_markers = ("token", "secret", "password", "credential", "authorization", "api_key", "apikey")
|
||||||
|
for key in constraints:
|
||||||
|
lowered = str(key).lower()
|
||||||
|
if any(marker in lowered for marker in sensitive_markers):
|
||||||
|
_error("AGENT_RUNTIME_SECRET_FIELD_DENY")
|
||||||
|
|
||||||
|
scope = raw.get("scope", {})
|
||||||
|
if not isinstance(scope, Mapping):
|
||||||
|
_error("AGENT_RUNTIME_SCOPE_INVALID")
|
||||||
|
scope = {
|
||||||
|
"scope_type": str(scope.get("scope_type") or "legacy"),
|
||||||
|
"tenant_id": scope.get("tenant_id"),
|
||||||
|
"project_id": scope.get("project_id"),
|
||||||
|
}
|
||||||
|
if scope["scope_type"] not in {"legacy", "system", "tenant"}:
|
||||||
|
_error("AGENT_RUNTIME_SCOPE_INVALID")
|
||||||
|
if scope["scope_type"] == "tenant":
|
||||||
|
if not isinstance(scope["tenant_id"], str) or not scope["tenant_id"].strip():
|
||||||
|
_error("AGENT_RUNTIME_SCOPE_INVALID")
|
||||||
|
elif scope["tenant_id"] not in (None, "") or scope["project_id"] not in (None, ""):
|
||||||
|
_error("AGENT_RUNTIME_SCOPE_INVALID")
|
||||||
|
|
||||||
|
trace_id = str(raw.get("trace_id") or "").strip()
|
||||||
|
if not trace_id or len(trace_id) > 255:
|
||||||
|
_error("AGENT_RUNTIME_TRACE_ID_INVALID")
|
||||||
|
|
||||||
|
normalized = {
|
||||||
|
"goal": goal[:12000],
|
||||||
|
"instructions": instructions[:32000],
|
||||||
|
"success_criteria": normalized_criteria,
|
||||||
|
"constraints": dict(constraints),
|
||||||
|
"scope": scope,
|
||||||
|
"trace_id": trace_id,
|
||||||
|
}
|
||||||
|
if len(_canonical(normalized).encode("utf-8")) > MAX_AGENT_RUNTIME_INPUT_BYTES:
|
||||||
|
_error("AGENT_RUNTIME_INPUT_TOO_LARGE")
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
def _capabilities(value: object) -> tuple[str, ...]:
|
def _capabilities(value: object) -> tuple[str, ...]:
|
||||||
if value is None:
|
if value is None:
|
||||||
return ()
|
return ()
|
||||||
@@ -288,6 +361,22 @@ def validate_execution_envelope(
|
|||||||
signals = list(_source_signals(envelope.get("source_signals") or []))
|
signals = list(_source_signals(envelope.get("source_signals") or []))
|
||||||
if envelope.get("source_signals") != signals:
|
if envelope.get("source_signals") != signals:
|
||||||
_error("SOURCE_CAPTURE_SIGNALS_INVALID")
|
_error("SOURCE_CAPTURE_SIGNALS_INVALID")
|
||||||
|
if envelope.get("runtime_input") not in (None, {}):
|
||||||
|
_error("SOURCE_CAPTURE_RUNTIME_INPUT_DENY")
|
||||||
|
elif action == "agent_runtime":
|
||||||
|
if changes or verification:
|
||||||
|
_error("AGENT_RUNTIME_MUTATION_DENY")
|
||||||
|
adapter = str(envelope.get("adapter") or "")
|
||||||
|
if adapter not in AGENT_RUNTIME_ADAPTERS:
|
||||||
|
_error("AGENT_RUNTIME_ADAPTER_INVALID")
|
||||||
|
runtime_input = _agent_runtime_input(envelope.get("runtime_input"))
|
||||||
|
if envelope.get("runtime_input") != runtime_input:
|
||||||
|
_error("AGENT_RUNTIME_INPUT_NOT_CANONICAL")
|
||||||
|
required_runtime_capability = f"agent.runtime.{adapter}"
|
||||||
|
if required_runtime_capability not in capability_scope:
|
||||||
|
_error("AGENT_RUNTIME_CAPABILITY_REQUIRED")
|
||||||
|
if envelope.get("source_signals") not in (None, []):
|
||||||
|
_error("AGENT_RUNTIME_SOURCE_SIGNALS_DENY")
|
||||||
elif not changes:
|
elif not changes:
|
||||||
_error("CHANGE_SET_INVALID")
|
_error("CHANGE_SET_INVALID")
|
||||||
declared = str(envelope.get("execution_digest") or "")
|
declared = str(envelope.get("execution_digest") or "")
|
||||||
@@ -329,11 +418,15 @@ def project_execution_envelope(
|
|||||||
"schema": EXECUTION_SCHEMA,
|
"schema": EXECUTION_SCHEMA,
|
||||||
**binding.public(),
|
**binding.public(),
|
||||||
"action": action,
|
"action": action,
|
||||||
"adapter": str(request.get("adapter") or ("deterministic" if action == "deterministic" else "source_capture")),
|
"adapter": str(
|
||||||
|
request.get("adapter")
|
||||||
|
or ("deterministic" if action == "deterministic" else "source_capture")
|
||||||
|
),
|
||||||
"capability_scope": list(capabilities),
|
"capability_scope": list(capabilities),
|
||||||
"change_set": list(request.get("change_set") or []),
|
"change_set": list(request.get("change_set") or []),
|
||||||
"verification": list(request.get("verification") or []),
|
"verification": list(request.get("verification") or []),
|
||||||
"source_signals": list(_source_signals(request.get("source_signals") or [])) if action == "source_capture" else [],
|
"source_signals": list(_source_signals(request.get("source_signals") or [])) if action == "source_capture" else [],
|
||||||
|
"runtime_input": _agent_runtime_input(request.get("runtime_input")) if action == "agent_runtime" else {},
|
||||||
"governance": governance,
|
"governance": governance,
|
||||||
"workspace": {
|
"workspace": {
|
||||||
"authority_git_dir": "/data/aios-gitops/repo.git",
|
"authority_git_dir": "/data/aios-gitops/repo.git",
|
||||||
@@ -352,7 +445,45 @@ def project_online_task(
|
|||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
request = dict(task_request)
|
request = dict(task_request)
|
||||||
required = _capabilities(request.get("required_capabilities"))
|
required = _capabilities(request.get("required_capabilities"))
|
||||||
execution = project_execution_envelope(binding, execution_request, required_capabilities=required)
|
execution_value = dict(execution_request or {})
|
||||||
|
if str(execution_value.get("action") or "source_capture") == "agent_runtime":
|
||||||
|
adapter = str(execution_value.get("adapter") or "")
|
||||||
|
if adapter not in AGENT_RUNTIME_ADAPTERS:
|
||||||
|
_error("AGENT_RUNTIME_ADAPTER_INVALID")
|
||||||
|
runtime_input = execution_value.get("runtime_input")
|
||||||
|
tenant_id = request.get("tenant_id")
|
||||||
|
project_id = request.get("project_id")
|
||||||
|
declared_scope = str(
|
||||||
|
request.get("scope_type")
|
||||||
|
or ("tenant" if isinstance(tenant_id, str) and tenant_id.strip() else "legacy")
|
||||||
|
)
|
||||||
|
if runtime_input is None:
|
||||||
|
runtime_input = {
|
||||||
|
"goal": request.get("goal"),
|
||||||
|
"instructions": request.get("instructions") or "",
|
||||||
|
"success_criteria": request.get("success_criteria") or [],
|
||||||
|
"constraints": request.get("constraints") or {},
|
||||||
|
}
|
||||||
|
elif not isinstance(runtime_input, Mapping):
|
||||||
|
_error("AGENT_RUNTIME_INPUT_REQUIRED")
|
||||||
|
runtime_input = dict(runtime_input)
|
||||||
|
# Scope/trace are authoritative Task identity, never caller-selected runtime hints.
|
||||||
|
runtime_input["scope"] = {
|
||||||
|
"scope_type": declared_scope,
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"project_id": project_id,
|
||||||
|
}
|
||||||
|
runtime_input["trace_id"] = str(
|
||||||
|
request.get("trace_id")
|
||||||
|
or request.get("run_id")
|
||||||
|
or binding.task_id
|
||||||
|
)
|
||||||
|
execution_value["runtime_input"] = runtime_input
|
||||||
|
execution = project_execution_envelope(
|
||||||
|
binding,
|
||||||
|
execution_value,
|
||||||
|
required_capabilities=required,
|
||||||
|
)
|
||||||
metadata = dict(request.get("metadata") or {})
|
metadata = dict(request.get("metadata") or {})
|
||||||
if "execution" in metadata:
|
if "execution" in metadata:
|
||||||
validate_execution_envelope(_object(metadata["execution"], "EXECUTION_ENVELOPE_INVALID"), binding=binding)
|
validate_execution_envelope(_object(metadata["execution"], "EXECUTION_ENVELOPE_INVALID"), binding=binding)
|
||||||
|
|||||||
@@ -44,6 +44,10 @@ from aios_online_agent_checkpoint import (
|
|||||||
CheckpointError,
|
CheckpointError,
|
||||||
CheckpointStore,
|
CheckpointStore,
|
||||||
)
|
)
|
||||||
|
from aios_external_agent_runtime import (
|
||||||
|
ExternalAgentExchange,
|
||||||
|
ExternalAgentRuntimeError,
|
||||||
|
)
|
||||||
from aios_online_agent_compact_map import (
|
from aios_online_agent_compact_map import (
|
||||||
CompactInterfaceMapError,
|
CompactInterfaceMapError,
|
||||||
build_compact_interface_map,
|
build_compact_interface_map,
|
||||||
@@ -83,6 +87,23 @@ WORKER_CAPABILITIES = tuple(
|
|||||||
if item.strip()
|
if item.strip()
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
WORKER_ALLOWED_ADAPTERS = frozenset(
|
||||||
|
item.strip()
|
||||||
|
for item in os.environ.get(
|
||||||
|
"AIOS_WORKER_ALLOWED_ADAPTERS",
|
||||||
|
"deterministic,source_capture,compact_interface_map,codex",
|
||||||
|
).split(",")
|
||||||
|
if item.strip()
|
||||||
|
)
|
||||||
|
EXTERNAL_AGENT_EXECUTOR_ROOT = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_EXTERNAL_AGENT_EXECUTOR_ROOT",
|
||||||
|
"/var/lib/aios-external-agent-executor",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
EXTERNAL_AGENT_EXECUTOR_TIMEOUT_SECONDS = int(
|
||||||
|
os.environ.get("AIOS_EXTERNAL_AGENT_EXECUTOR_TIMEOUT_SECONDS", "1800")
|
||||||
|
)
|
||||||
MAX_TASK_BYTES = 512 * 1024
|
MAX_TASK_BYTES = 512 * 1024
|
||||||
MAX_FILE_BYTES = 2 * 1024 * 1024
|
MAX_FILE_BYTES = 2 * 1024 * 1024
|
||||||
ISOLATED_EXECUTOR_ROOT = Path(
|
ISOLATED_EXECUTOR_ROOT = Path(
|
||||||
@@ -215,6 +236,64 @@ def atomic_json(path: Path, value: Mapping[str, Any], mode: int = 0o640) -> None
|
|||||||
os.unlink(temporary)
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
WORKER_LIVENESS_PATH = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_WORKER_LIVENESS_PATH",
|
||||||
|
str(STATE_ROOT / "worker-liveness.json"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
_LIVENESS_LOCK = threading.Lock()
|
||||||
|
_LIVENESS_SEQUENCE = 0
|
||||||
|
ENGINEERING_WRITE_CAPABILITIES = frozenset({
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def write_worker_liveness(
|
||||||
|
state: str,
|
||||||
|
*,
|
||||||
|
visible_count: int | None = None,
|
||||||
|
task_id: str | None = None,
|
||||||
|
stage: str | None = None,
|
||||||
|
error: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Write idle/busy liveness without touching Task/Job state."""
|
||||||
|
global _LIVENESS_SEQUENCE
|
||||||
|
with _LIVENESS_LOCK:
|
||||||
|
_LIVENESS_SEQUENCE += 1
|
||||||
|
epoch = time.time()
|
||||||
|
payload = {
|
||||||
|
"schema": "aios.online-agent.worker-liveness.v1",
|
||||||
|
"agent_id": AGENT_ID,
|
||||||
|
"agent_type": AGENT_TYPE,
|
||||||
|
"agent_version": AGENT_VERSION,
|
||||||
|
"pid": os.getpid(),
|
||||||
|
"state": state,
|
||||||
|
"sequence": _LIVENESS_SEQUENCE,
|
||||||
|
"last_seen_epoch": epoch,
|
||||||
|
"last_seen_at": datetime.fromtimestamp(epoch, timezone.utc).isoformat(),
|
||||||
|
"poll_seconds": POLL_SECONDS,
|
||||||
|
"heartbeat_seconds": HEARTBEAT_SECONDS,
|
||||||
|
"api_base": API_BASE,
|
||||||
|
"bridge_base": BRIDGE_BASE,
|
||||||
|
"visible_count": visible_count,
|
||||||
|
"task_id": task_id,
|
||||||
|
"stage": stage,
|
||||||
|
"error": error,
|
||||||
|
}
|
||||||
|
atomic_json(WORKER_LIVENESS_PATH, payload)
|
||||||
|
|
||||||
|
|
||||||
|
def source_capture_requires_deterministic_plan(envelope: "ExecutionEnvelope") -> bool:
|
||||||
|
"""Treat write-capable source_capture as a binding placeholder, never executable work."""
|
||||||
|
return (
|
||||||
|
envelope.action == "source_capture"
|
||||||
|
and bool(ENGINEERING_WRITE_CAPABILITIES.intersection(envelope.capability_scope))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def safe_relative_path(value: Any, *, field: str = "path") -> str:
|
def safe_relative_path(value: Any, *, field: str = "path") -> str:
|
||||||
if not isinstance(value, str) or not value or len(value) > 240:
|
if not isinstance(value, str) or not value or len(value) > 240:
|
||||||
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
|
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
|
||||||
@@ -359,6 +438,7 @@ class ExecutionEnvelope:
|
|||||||
governance: dict[str, Any]
|
governance: dict[str, Any]
|
||||||
workspace: dict[str, Any]
|
workspace: dict[str, Any]
|
||||||
digest: str
|
digest: str
|
||||||
|
runtime_input: dict[str, Any] | None = None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_task(
|
def from_task(
|
||||||
@@ -403,7 +483,7 @@ class ExecutionEnvelope:
|
|||||||
adapter = str(adapter_value.get("name") or "")
|
adapter = str(adapter_value.get("name") or "")
|
||||||
else:
|
else:
|
||||||
adapter = str(adapter_value or "")
|
adapter = str(adapter_value or "")
|
||||||
if adapter not in {"deterministic", "source_capture", "compact_interface_map", "codex", "hermes"}:
|
if adapter not in {"deterministic", "source_capture", "compact_interface_map", "codex", "hermes", "openclaw"}:
|
||||||
raise WorkerError("RUNTIME_ADAPTER_UNSUPPORTED", phase="validation")
|
raise WorkerError("RUNTIME_ADAPTER_UNSUPPORTED", phase="validation")
|
||||||
change_set = execution.get("change_set")
|
change_set = execution.get("change_set")
|
||||||
if not isinstance(change_set, list) or len(change_set) > 128 or (action == "deterministic" and not change_set):
|
if not isinstance(change_set, list) or len(change_set) > 128 or (action == "deterministic" and not change_set):
|
||||||
@@ -494,6 +574,7 @@ class ExecutionEnvelope:
|
|||||||
adapter=adapter,
|
adapter=adapter,
|
||||||
capability_scope=tuple(execution.get("capability_scope") or ()),
|
capability_scope=tuple(execution.get("capability_scope") or ()),
|
||||||
source_signals=tuple(execution.get("source_signals") or ()),
|
source_signals=tuple(execution.get("source_signals") or ()),
|
||||||
|
runtime_input=dict(execution.get("runtime_input") or {}),
|
||||||
change_set=tuple(normalized_changes),
|
change_set=tuple(normalized_changes),
|
||||||
verification=tuple(normalized_verification),
|
verification=tuple(normalized_verification),
|
||||||
governance=governance_value,
|
governance=governance_value,
|
||||||
@@ -502,11 +583,44 @@ class ExecutionEnvelope:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def external_agent_request_id(
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
runtime_name: str,
|
||||||
|
attempt: int,
|
||||||
|
) -> str:
|
||||||
|
if not isinstance(attempt, int) or isinstance(attempt, bool) or attempt < 1:
|
||||||
|
raise WorkerError("TASK_ATTEMPT_INVALID", phase="external_agent")
|
||||||
|
if runtime_name not in {"hermes", "openclaw"}:
|
||||||
|
raise WorkerError(
|
||||||
|
"EXTERNAL_AGENT_RUNTIME_UNSUPPORTED",
|
||||||
|
phase="external_agent",
|
||||||
|
details={"runtime": runtime_name},
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
f"external-{runtime_name}-"
|
||||||
|
+ digest_json(
|
||||||
|
{
|
||||||
|
"task_id": envelope.task_id,
|
||||||
|
"job_id": envelope.job_id,
|
||||||
|
"execution_digest": envelope.digest,
|
||||||
|
"runtime": runtime_name,
|
||||||
|
"attempt": attempt,
|
||||||
|
}
|
||||||
|
)[:32]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class RuntimeAdapter(ABC):
|
class RuntimeAdapter(ABC):
|
||||||
name: str
|
name: str
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
@@ -527,7 +641,13 @@ class DeterministicExecutor(RuntimeAdapter):
|
|||||||
if os.path.exists(temporary):
|
if os.path.exists(temporary):
|
||||||
os.unlink(temporary)
|
os.unlink(temporary)
|
||||||
|
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
changed: set[str] = set()
|
changed: set[str] = set()
|
||||||
for change in envelope.change_set:
|
for change in envelope.change_set:
|
||||||
relative = str(change["path"])
|
relative = str(change["path"])
|
||||||
@@ -582,7 +702,13 @@ class SourceCaptureExecutor(RuntimeAdapter):
|
|||||||
"governed", "trace", "tool", "credential", "scope", "message", "handoff",
|
"governed", "trace", "tool", "credential", "scope", "message", "handoff",
|
||||||
}
|
}
|
||||||
|
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
terms = set(self.defaults)
|
terms = set(self.defaults)
|
||||||
for signal in envelope.source_signals:
|
for signal in envelope.source_signals:
|
||||||
terms.update(token.lower() for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]{2,63}", signal))
|
terms.update(token.lower() for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]{2,63}", signal))
|
||||||
@@ -651,7 +777,13 @@ class CompactInterfaceMapExecutor(RuntimeAdapter):
|
|||||||
|
|
||||||
name = "compact_interface_map"
|
name = "compact_interface_map"
|
||||||
|
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
if envelope.action != "source_capture":
|
if envelope.action != "source_capture":
|
||||||
raise WorkerError("COMPACT_INTERFACE_MAP_ACTION_INVALID", phase="interface_map")
|
raise WorkerError("COMPACT_INTERFACE_MAP_ACTION_INVALID", phase="interface_map")
|
||||||
try:
|
try:
|
||||||
@@ -681,7 +813,13 @@ class CompactInterfaceMapExecutor(RuntimeAdapter):
|
|||||||
class CodexExecutor(RuntimeAdapter):
|
class CodexExecutor(RuntimeAdapter):
|
||||||
name = "codex"
|
name = "codex"
|
||||||
|
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
raise WorkerError(
|
raise WorkerError(
|
||||||
"CODEX_ADAPTER_REQUIRES_EXPLICIT_POLICY",
|
"CODEX_ADAPTER_REQUIRES_EXPLICIT_POLICY",
|
||||||
phase="adapter",
|
phase="adapter",
|
||||||
@@ -689,15 +827,78 @@ class CodexExecutor(RuntimeAdapter):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class HermesExecutor(RuntimeAdapter):
|
class ExternalAgentExecutor(RuntimeAdapter):
|
||||||
name = "hermes"
|
runtime_name: str
|
||||||
|
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
raise WorkerError(
|
self,
|
||||||
"HERMES_ENGINEERING_ADAPTER_UNAVAILABLE",
|
envelope: ExecutionEnvelope,
|
||||||
phase="adapter",
|
worktree: Path,
|
||||||
details={"reason": "The live Hermes service is a user-chat runtime, not an engineering executor"},
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if envelope.action != "agent_runtime":
|
||||||
|
raise WorkerError(
|
||||||
|
"EXTERNAL_AGENT_ACTION_INVALID",
|
||||||
|
phase="adapter",
|
||||||
|
details={"adapter": self.runtime_name, "action": envelope.action},
|
||||||
|
)
|
||||||
|
if envelope.adapter != self.runtime_name:
|
||||||
|
raise WorkerError(
|
||||||
|
"EXTERNAL_AGENT_ADAPTER_BINDING_INVALID",
|
||||||
|
phase="adapter",
|
||||||
|
)
|
||||||
|
if not envelope.runtime_input:
|
||||||
|
raise WorkerError("EXTERNAL_AGENT_RUNTIME_INPUT_MISSING", phase="adapter")
|
||||||
|
request_id = external_agent_request_id(
|
||||||
|
envelope,
|
||||||
|
self.runtime_name,
|
||||||
|
attempt,
|
||||||
)
|
)
|
||||||
|
exchange = ExternalAgentExchange(EXTERNAL_AGENT_EXECUTOR_ROOT)
|
||||||
|
try:
|
||||||
|
external = exchange.execute(
|
||||||
|
request_id=request_id,
|
||||||
|
runtime=self.runtime_name,
|
||||||
|
task_id=envelope.task_id,
|
||||||
|
attempt=attempt,
|
||||||
|
execution_digest=envelope.digest,
|
||||||
|
payload=envelope.runtime_input,
|
||||||
|
timeout_seconds=EXTERNAL_AGENT_EXECUTOR_TIMEOUT_SECONDS,
|
||||||
|
)
|
||||||
|
except ExternalAgentRuntimeError as exc:
|
||||||
|
raise WorkerError(
|
||||||
|
str(exc),
|
||||||
|
phase="external_agent",
|
||||||
|
details={"adapter": self.runtime_name},
|
||||||
|
) from exc
|
||||||
|
audit(
|
||||||
|
"EXTERNAL_AGENT_RUNTIME_COMPLETED",
|
||||||
|
task_id=envelope.task_id,
|
||||||
|
adapter=self.runtime_name,
|
||||||
|
request_id=request_id,
|
||||||
|
attempt=attempt,
|
||||||
|
output_sha256=external.get("output_sha256"),
|
||||||
|
duration_ms=external.get("duration_ms"),
|
||||||
|
credential_material_exposed=False,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"changed_files": [],
|
||||||
|
"commands": [],
|
||||||
|
"tests": [],
|
||||||
|
"adapter": self.runtime_name,
|
||||||
|
"external_agent": external,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class HermesExecutor(ExternalAgentExecutor):
|
||||||
|
name = "hermes"
|
||||||
|
runtime_name = "hermes"
|
||||||
|
|
||||||
|
|
||||||
|
class OpenClawExecutor(ExternalAgentExecutor):
|
||||||
|
name = "openclaw"
|
||||||
|
runtime_name = "openclaw"
|
||||||
|
|
||||||
|
|
||||||
def adapter_for(name: str) -> RuntimeAdapter:
|
def adapter_for(name: str) -> RuntimeAdapter:
|
||||||
@@ -707,6 +908,7 @@ def adapter_for(name: str) -> RuntimeAdapter:
|
|||||||
"compact_interface_map": CompactInterfaceMapExecutor,
|
"compact_interface_map": CompactInterfaceMapExecutor,
|
||||||
"codex": CodexExecutor,
|
"codex": CodexExecutor,
|
||||||
"hermes": HermesExecutor,
|
"hermes": HermesExecutor,
|
||||||
|
"openclaw": OpenClawExecutor,
|
||||||
}[name]()
|
}[name]()
|
||||||
|
|
||||||
|
|
||||||
@@ -805,6 +1007,11 @@ class LeaseHeartbeat:
|
|||||||
self.failed = str(response.get("error") or status)
|
self.failed = str(response.get("error") or status)
|
||||||
raise WorkerError("TASK_HEARTBEAT_FAILED", phase="heartbeat", details={"api_status": status, "error": self.failed})
|
raise WorkerError("TASK_HEARTBEAT_FAILED", phase="heartbeat", details={"api_status": status, "error": self.failed})
|
||||||
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=self.stage, api_status=status)
|
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=self.stage, api_status=status)
|
||||||
|
write_worker_liveness(
|
||||||
|
"BUSY",
|
||||||
|
task_id=self.task_id,
|
||||||
|
stage=self.stage,
|
||||||
|
)
|
||||||
|
|
||||||
def start(self) -> None:
|
def start(self) -> None:
|
||||||
self.thread.start()
|
self.thread.start()
|
||||||
@@ -824,6 +1031,11 @@ class LeaseHeartbeat:
|
|||||||
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, api_status=status, error=self.failed)
|
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, api_status=status, error=self.failed)
|
||||||
else:
|
else:
|
||||||
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=stage, api_status=status)
|
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=stage, api_status=status)
|
||||||
|
write_worker_liveness(
|
||||||
|
"BUSY",
|
||||||
|
task_id=self.task_id,
|
||||||
|
stage=stage,
|
||||||
|
)
|
||||||
except Exception as exc: # pragma: no cover - defensive runtime guard
|
except Exception as exc: # pragma: no cover - defensive runtime guard
|
||||||
self.failed = type(exc).__name__
|
self.failed = type(exc).__name__
|
||||||
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, error=self.failed)
|
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, error=self.failed)
|
||||||
@@ -1201,7 +1413,13 @@ class IsolatedDeterministicExecutor(RuntimeAdapter):
|
|||||||
if os.path.exists(temporary):
|
if os.path.exists(temporary):
|
||||||
os.unlink(temporary)
|
os.unlink(temporary)
|
||||||
|
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
if envelope.action != "deterministic":
|
if envelope.action != "deterministic":
|
||||||
raise WorkerError("ISOLATED_EXECUTOR_ACTION_INVALID", phase="adapter")
|
raise WorkerError("ISOLATED_EXECUTOR_ACTION_INVALID", phase="adapter")
|
||||||
root = ISOLATED_EXECUTOR_ROOT
|
root = ISOLATED_EXECUTOR_ROOT
|
||||||
@@ -1575,6 +1793,69 @@ def build_compact_interface_map_result(
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def build_agent_runtime_result(
|
||||||
|
task_id: str,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
execution: Mapping[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
external = execution.get("external_agent")
|
||||||
|
if not isinstance(external, Mapping):
|
||||||
|
raise WorkerError("EXTERNAL_AGENT_RESULT_INVALID", phase="result")
|
||||||
|
if external.get("runtime") != envelope.adapter:
|
||||||
|
raise WorkerError("EXTERNAL_AGENT_RESULT_RUNTIME_MISMATCH", phase="result")
|
||||||
|
if external.get("task_id") != task_id:
|
||||||
|
raise WorkerError("EXTERNAL_AGENT_RESULT_TASK_MISMATCH", phase="result")
|
||||||
|
if external.get("execution_digest") != envelope.digest:
|
||||||
|
raise WorkerError("EXTERNAL_AGENT_RESULT_DIGEST_MISMATCH", phase="result")
|
||||||
|
if external.get("credential_material_exposed") is not False:
|
||||||
|
raise WorkerError("EXTERNAL_AGENT_CREDENTIAL_ISOLATION_FAILED", phase="result")
|
||||||
|
result = {
|
||||||
|
"status": "SUCCEEDED",
|
||||||
|
"classification": "EXTERNAL_AGENT_RUNTIME_SUCCEEDED",
|
||||||
|
"verification_state": "VERIFIED_BY_WORKER_PENDING_SERVER_REVIEW",
|
||||||
|
"execution_mode": "agent_runtime",
|
||||||
|
"execution_backend": "generic-online-agent-worker-v2",
|
||||||
|
"agent_runtime": envelope.adapter,
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": envelope.job_id,
|
||||||
|
"repository": envelope.repository,
|
||||||
|
"base_oid": envelope.base_oid,
|
||||||
|
"expected_branch_oid": envelope.expected_branch_oid,
|
||||||
|
"target_branch": envelope.target_branch,
|
||||||
|
"generation": envelope.generation,
|
||||||
|
"execution_digest": envelope.digest,
|
||||||
|
"output_text": external.get("output_text"),
|
||||||
|
"output_json": external.get("output_json"),
|
||||||
|
"output_sha256": external.get("output_sha256"),
|
||||||
|
"runtime_duration_ms": external.get("duration_ms"),
|
||||||
|
"external_request_id": external.get("request_id"),
|
||||||
|
"changed_files": [],
|
||||||
|
"tests": [],
|
||||||
|
"workspace_cleanup": "PASS",
|
||||||
|
"writer_lease": "NONE",
|
||||||
|
"secret_material_exposed": False,
|
||||||
|
"credential_material_exposed": False,
|
||||||
|
"control_plane_access_granted_to_runtime": False,
|
||||||
|
"direct_main_write": False,
|
||||||
|
"force_push": False,
|
||||||
|
"production_write": False,
|
||||||
|
"protected_ref_bypass": False,
|
||||||
|
}
|
||||||
|
encoded = json.dumps(
|
||||||
|
sanitize(result),
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
if len(encoded) > 192 * 1024:
|
||||||
|
raise WorkerError(
|
||||||
|
"EXTERNAL_AGENT_TASK_RESULT_TOO_LARGE",
|
||||||
|
phase="result",
|
||||||
|
details={"result_bytes": len(encoded)},
|
||||||
|
)
|
||||||
|
result["result_bytes"] = len(encoded)
|
||||||
|
return result
|
||||||
|
|
||||||
def build_source_capture_result(
|
def build_source_capture_result(
|
||||||
task_id: str,
|
task_id: str,
|
||||||
envelope: ExecutionEnvelope,
|
envelope: ExecutionEnvelope,
|
||||||
@@ -1624,10 +1905,25 @@ class WorkerSupervisor:
|
|||||||
if not isinstance(execution, Mapping):
|
if not isinstance(execution, Mapping):
|
||||||
raise WorkerError("EXECUTION_ENVELOPE_SCHEMA_VALIDATION", phase="validation")
|
raise WorkerError("EXECUTION_ENVELOPE_SCHEMA_VALIDATION", phase="validation")
|
||||||
persisted = ExecutionEnvelope.from_task(task, worker_capabilities=WORKER_CAPABILITIES)
|
persisted = ExecutionEnvelope.from_task(task, worker_capabilities=WORKER_CAPABILITIES)
|
||||||
# The durable plan registry is authoritative for a task that was projected
|
# Read-only source_capture/agent_runtime tasks replay their persisted
|
||||||
# before its deterministic plan was registered. Re-resolve without sending
|
# payload. A write-capable source_capture is only a Bridge-created
|
||||||
# the stale persisted payload so the Bridge can apply the exact plan/CAS.
|
# engineering placeholder: never pass that stale payload back to the
|
||||||
status, authoritative = self.bridge.project_execution(task)
|
# resolver, and never claim it until a deterministic plan exists.
|
||||||
|
pending_plan = source_capture_requires_deterministic_plan(persisted)
|
||||||
|
projection_input = (
|
||||||
|
execution
|
||||||
|
if persisted.action in {"source_capture", "agent_runtime"} and not pending_plan
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
status, authoritative = self.bridge.project_execution(task, projection_input)
|
||||||
|
if pending_plan and (
|
||||||
|
status == 404
|
||||||
|
or authoritative.get("error") == "TASK_EXECUTION_PROJECTION_MISSING"
|
||||||
|
):
|
||||||
|
raise WorkerError(
|
||||||
|
"TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN",
|
||||||
|
phase="projection",
|
||||||
|
)
|
||||||
if not 200 <= status < 300:
|
if not 200 <= status < 300:
|
||||||
raise WorkerError("TASK_PERSISTED_BINDING_UNVERIFIED", phase="projection", details={"api_status": status, "error": authoritative.get("error")})
|
raise WorkerError("TASK_PERSISTED_BINDING_UNVERIFIED", phase="projection", details={"api_status": status, "error": authoritative.get("error")})
|
||||||
canonical = ExecutionEnvelope.from_task(task, authoritative, WORKER_CAPABILITIES)
|
canonical = ExecutionEnvelope.from_task(task, authoritative, WORKER_CAPABILITIES)
|
||||||
@@ -1656,6 +1952,11 @@ class WorkerSupervisor:
|
|||||||
execution_digest=canonical.digest,
|
execution_digest=canonical.digest,
|
||||||
)
|
)
|
||||||
return canonical, "plan_registry_verified"
|
return canonical, "plan_registry_verified"
|
||||||
|
if pending_plan:
|
||||||
|
raise WorkerError(
|
||||||
|
"TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN",
|
||||||
|
phase="projection",
|
||||||
|
)
|
||||||
return persisted, "persisted_verified"
|
return persisted, "persisted_verified"
|
||||||
status, projected = self.bridge.project_execution(task)
|
status, projected = self.bridge.project_execution(task)
|
||||||
if status == 404 or projected.get("error") == "TASK_EXECUTION_PROJECTION_MISSING":
|
if status == 404 or projected.get("error") == "TASK_EXECUTION_PROJECTION_MISSING":
|
||||||
@@ -1824,6 +2125,43 @@ class WorkerSupervisor:
|
|||||||
attempt=attempt,
|
attempt=attempt,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if envelope.action == "agent_runtime" and phase in {
|
||||||
|
"agent_runtime_done",
|
||||||
|
"result_pending",
|
||||||
|
"result_submitted",
|
||||||
|
}:
|
||||||
|
candidate = state.get("result")
|
||||||
|
if isinstance(candidate, Mapping):
|
||||||
|
result = dict(candidate)
|
||||||
|
audit(
|
||||||
|
"TASK_RESUMED_FROM_EXTERNAL_AGENT_RESULT",
|
||||||
|
task_id=task_id,
|
||||||
|
previous_attempt=state.get("resume_from_attempt"),
|
||||||
|
attempt=attempt,
|
||||||
|
phase=phase,
|
||||||
|
adapter=envelope.adapter,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
result is None
|
||||||
|
and envelope.action == "agent_runtime"
|
||||||
|
and phase == "verification_passed"
|
||||||
|
and isinstance(state.get("execution"), Mapping)
|
||||||
|
):
|
||||||
|
result = build_agent_runtime_result(
|
||||||
|
task_id, envelope, state["execution"]
|
||||||
|
)
|
||||||
|
state.update(
|
||||||
|
{"phase": "agent_runtime_done", "result": result, "attempt": attempt}
|
||||||
|
)
|
||||||
|
save_task_state(task_id, state)
|
||||||
|
save_checkpoint(task_id, state, envelope, attempt)
|
||||||
|
audit(
|
||||||
|
"TASK_RESUMED_AFTER_EXTERNAL_AGENT_EXECUTION",
|
||||||
|
task_id=task_id,
|
||||||
|
attempt=attempt,
|
||||||
|
adapter=envelope.adapter,
|
||||||
|
)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
result is None
|
result is None
|
||||||
and envelope.action == "deterministic"
|
and envelope.action == "deterministic"
|
||||||
@@ -1904,7 +2242,11 @@ class WorkerSupervisor:
|
|||||||
# re-run deterministic verification. No external side effect exists yet.
|
# re-run deterministic verification. No external side effect exists yet.
|
||||||
run_root = new_run_root()
|
run_root = new_run_root()
|
||||||
try:
|
try:
|
||||||
worktree = prepare_worktree(run_root, envelope)
|
if envelope.action == "agent_runtime":
|
||||||
|
worktree = run_root / "agent-runtime-workspace"
|
||||||
|
worktree.mkdir(mode=0o750, parents=True, exist_ok=True)
|
||||||
|
else:
|
||||||
|
worktree = prepare_worktree(run_root, envelope)
|
||||||
state.update(
|
state.update(
|
||||||
{
|
{
|
||||||
"phase": "workspace_prepared",
|
"phase": "workspace_prepared",
|
||||||
@@ -1916,7 +2258,11 @@ class WorkerSupervisor:
|
|||||||
save_task_state(task_id, state)
|
save_task_state(task_id, state)
|
||||||
save_checkpoint(task_id, state, envelope, attempt)
|
save_checkpoint(task_id, state, envelope, attempt)
|
||||||
adapter = adapter_for(envelope.adapter)
|
adapter = adapter_for(envelope.adapter)
|
||||||
execution = adapter.execute(envelope, worktree)
|
execution = adapter.execute(
|
||||||
|
envelope,
|
||||||
|
worktree,
|
||||||
|
attempt=attempt,
|
||||||
|
)
|
||||||
state.update(
|
state.update(
|
||||||
{
|
{
|
||||||
"phase": "verification_passed",
|
"phase": "verification_passed",
|
||||||
@@ -1938,6 +2284,17 @@ class WorkerSupervisor:
|
|||||||
"attempt": attempt,
|
"attempt": attempt,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
elif envelope.action == "agent_runtime":
|
||||||
|
result = build_agent_runtime_result(
|
||||||
|
task_id, envelope, execution
|
||||||
|
)
|
||||||
|
state.update(
|
||||||
|
{
|
||||||
|
"phase": "agent_runtime_done",
|
||||||
|
"result": result,
|
||||||
|
"attempt": attempt,
|
||||||
|
}
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
heartbeat.set_stage("committing")
|
heartbeat.set_stage("committing")
|
||||||
governed = governed_commit(
|
governed = governed_commit(
|
||||||
@@ -2088,8 +2445,13 @@ class WorkerSupervisor:
|
|||||||
def once(self) -> int:
|
def once(self) -> int:
|
||||||
try:
|
try:
|
||||||
tasks = self.agent.poll()
|
tasks = self.agent.poll()
|
||||||
|
write_worker_liveness("POLL_OK", visible_count=len(tasks))
|
||||||
audit("TASK_POLL", visible_count=len(tasks))
|
audit("TASK_POLL", visible_count=len(tasks))
|
||||||
except WorkerError as exc:
|
except WorkerError as exc:
|
||||||
|
write_worker_liveness(
|
||||||
|
"POLL_FAILED",
|
||||||
|
error=exc.code,
|
||||||
|
)
|
||||||
audit("TASK_POLL_FAILED", error=exc.code, phase=exc.phase, details=exc.details)
|
audit("TASK_POLL_FAILED", error=exc.code, phase=exc.phase, details=exc.details)
|
||||||
return 10
|
return 10
|
||||||
dynamic_tasks: list[tuple[dict[str, Any], ExecutionEnvelope]] = []
|
dynamic_tasks: list[tuple[dict[str, Any], ExecutionEnvelope]] = []
|
||||||
@@ -2098,13 +2460,25 @@ class WorkerSupervisor:
|
|||||||
envelope, source = self.resolve_envelope(task)
|
envelope, source = self.resolve_envelope(task)
|
||||||
except WorkerError as exc:
|
except WorkerError as exc:
|
||||||
event = "TASK_SKIPPED_ENVELOPE_INVALID"
|
event = "TASK_SKIPPED_ENVELOPE_INVALID"
|
||||||
if exc.code in {"TASK_SKIPPED_PROJECTION_MISSING", "TASK_SKIPPED_POLICY_DENY"}:
|
if exc.code in {
|
||||||
|
"TASK_SKIPPED_PROJECTION_MISSING",
|
||||||
|
"TASK_SKIPPED_POLICY_DENY",
|
||||||
|
"TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN",
|
||||||
|
}:
|
||||||
event = exc.code
|
event = exc.code
|
||||||
elif exc.code == "CAPABILITY_MISMATCH_DENY":
|
elif exc.code == "CAPABILITY_MISMATCH_DENY":
|
||||||
event = "TASK_SKIPPED_CAPABILITY_MISMATCH"
|
event = "TASK_SKIPPED_CAPABILITY_MISMATCH"
|
||||||
audit(event, task_id=task.get("task_id"), reason=exc.code, phase=exc.phase, details=exc.details)
|
audit(event, task_id=task.get("task_id"), reason=exc.code, phase=exc.phase, details=exc.details)
|
||||||
continue
|
continue
|
||||||
audit("TASK_VISIBLE_TO_WORKER_CLAIMABLE", task_id=task.get("task_id"), envelope_source=source, action=envelope.action)
|
audit("TASK_VISIBLE_TO_WORKER_CLAIMABLE", task_id=task.get("task_id"), envelope_source=source, action=envelope.action)
|
||||||
|
if envelope.adapter not in WORKER_ALLOWED_ADAPTERS:
|
||||||
|
audit(
|
||||||
|
"TASK_SKIPPED_ADAPTER_MISMATCH",
|
||||||
|
task_id=task.get("task_id"),
|
||||||
|
adapter=envelope.adapter,
|
||||||
|
allowed_adapters=sorted(WORKER_ALLOWED_ADAPTERS),
|
||||||
|
)
|
||||||
|
continue
|
||||||
dynamic_tasks.append((task, envelope))
|
dynamic_tasks.append((task, envelope))
|
||||||
dynamic_tasks.sort(key=_claim_order)
|
dynamic_tasks.sort(key=_claim_order)
|
||||||
for task, envelope in dynamic_tasks:
|
for task, envelope in dynamic_tasks:
|
||||||
@@ -2124,6 +2498,7 @@ class WorkerSupervisor:
|
|||||||
ensure_roots()
|
ensure_roots()
|
||||||
recover_stale_workspaces()
|
recover_stale_workspaces()
|
||||||
checkpoint_store()
|
checkpoint_store()
|
||||||
|
write_worker_liveness("STARTING")
|
||||||
audit("WORKER_STARTED", task_agnostic=True, api_base=API_BASE, bridge_base=BRIDGE_BASE, adapter_names=["isolated-deterministic", "source_capture", "codex", "hermes"], runtime_only_credentials=True, isolated_executor=True, durable_checkpoint=True)
|
audit("WORKER_STARTED", task_agnostic=True, api_base=API_BASE, bridge_base=BRIDGE_BASE, adapter_names=["isolated-deterministic", "source_capture", "codex", "hermes"], runtime_only_credentials=True, isolated_executor=True, durable_checkpoint=True)
|
||||||
while True:
|
while True:
|
||||||
self.once()
|
self.once()
|
||||||
|
|||||||
@@ -0,0 +1,462 @@
|
|||||||
|
"""Self-healing availability guard for persistent Online Agent workers.
|
||||||
|
|
||||||
|
The guard never claims Tasks and never writes Task/Job/Git databases. It only:
|
||||||
|
- ensures required worker/executor systemd units are enabled + active;
|
||||||
|
- checks worker-owned liveness JSON written by the real poll/heartbeat loop;
|
||||||
|
- restarts an active worker only when its liveness is stale;
|
||||||
|
- honors an explicit maintenance inhibit file;
|
||||||
|
- writes a bounded status JSON under /run for observability.
|
||||||
|
|
||||||
|
Idle liveness is the successful poll loop itself. Busy liveness is refreshed by the
|
||||||
|
Task lease-heartbeat thread, so long-running Tasks are not mistaken for hung workers.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import shlex
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from typing import Any, Callable, Mapping, Sequence
|
||||||
|
|
||||||
|
SCHEMA = "aios.worker-availability.status.v1"
|
||||||
|
DEFAULT_WORKER_UNITS = (
|
||||||
|
"aios-online-agent-worker-20260815.service",
|
||||||
|
"aios-hermes-online-agent-worker.service",
|
||||||
|
"aios-openclaw-online-agent-worker.service",
|
||||||
|
)
|
||||||
|
DEFAULT_DEPENDENCY_UNITS = (
|
||||||
|
"aios-external-agent-executor.service",
|
||||||
|
)
|
||||||
|
DEFAULT_RUN_ROOT = Path("/run/aios-worker-availability")
|
||||||
|
DEFAULT_STATUS = DEFAULT_RUN_ROOT / "status.json"
|
||||||
|
DEFAULT_RECOVERY = DEFAULT_RUN_ROOT / "recovery.json"
|
||||||
|
DEFAULT_INHIBIT = DEFAULT_RUN_ROOT / "maintenance.lock"
|
||||||
|
DEFAULT_MAX_STALE_SECONDS = 90
|
||||||
|
DEFAULT_RESTART_COOLDOWN_SECONDS = 60
|
||||||
|
DEFAULT_STARTUP_WAIT_SECONDS = 10
|
||||||
|
ACCEPTED_ENABLED_STATES = frozenset({"enabled", "enabled-runtime", "static", "indirect", "generated"})
|
||||||
|
|
||||||
|
|
||||||
|
class AvailabilityGuardError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class UnitSnapshot:
|
||||||
|
unit: str
|
||||||
|
role: str
|
||||||
|
load_state: str
|
||||||
|
active_state: str
|
||||||
|
unit_file_state: str
|
||||||
|
enabled_ok: bool
|
||||||
|
liveness_path: str | None = None
|
||||||
|
liveness_state: str | None = None
|
||||||
|
last_seen_epoch: float | None = None
|
||||||
|
liveness_age_seconds: float | None = None
|
||||||
|
fresh: bool | None = None
|
||||||
|
action: str = "NONE"
|
||||||
|
detail: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class Systemd:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
runner: Callable[..., subprocess.CompletedProcess[str]] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self.runner = runner or subprocess.run
|
||||||
|
|
||||||
|
def _run(self, args: Sequence[str], *, check: bool = False) -> subprocess.CompletedProcess[str]:
|
||||||
|
result = self.runner(
|
||||||
|
["/usr/bin/systemctl", *args],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
timeout=20,
|
||||||
|
)
|
||||||
|
if check and result.returncode != 0:
|
||||||
|
raise AvailabilityGuardError(
|
||||||
|
"SYSTEMD_COMMAND_FAILED:"
|
||||||
|
+ ":".join(args)
|
||||||
|
+ ":"
|
||||||
|
+ str(result.returncode)
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def property(self, unit: str, name: str) -> str:
|
||||||
|
result = self._run(["show", unit, f"--property={name}", "--value"])
|
||||||
|
if result.returncode != 0:
|
||||||
|
return ""
|
||||||
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
def environment(self, unit: str) -> dict[str, str]:
|
||||||
|
text = self.property(unit, "Environment")
|
||||||
|
result: dict[str, str] = {}
|
||||||
|
for item in shlex.split(text):
|
||||||
|
if "=" not in item:
|
||||||
|
continue
|
||||||
|
key, value = item.split("=", 1)
|
||||||
|
result[key] = value
|
||||||
|
return result
|
||||||
|
|
||||||
|
def enable(self, unit: str) -> None:
|
||||||
|
self._run(["enable", unit], check=True)
|
||||||
|
|
||||||
|
def start(self, unit: str) -> None:
|
||||||
|
self._run(["start", unit], check=True)
|
||||||
|
|
||||||
|
def restart(self, unit: str) -> None:
|
||||||
|
self._run(["restart", unit], check=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_json(path: Path, value: Mapping[str, Any]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o755)
|
||||||
|
if path.parent.is_symlink() or path.is_symlink():
|
||||||
|
raise AvailabilityGuardError("STATUS_PATH_SYMLINK_REJECTED")
|
||||||
|
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||||
|
try:
|
||||||
|
os.fchmod(fd, 0o644)
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(value, handle, ensure_ascii=False, sort_keys=True, indent=2)
|
||||||
|
handle.write("\n")
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
os.replace(temporary, path)
|
||||||
|
os.chmod(path, 0o644)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(temporary):
|
||||||
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_json(path: Path) -> dict[str, Any] | None:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return None
|
||||||
|
return dict(value) if isinstance(value, Mapping) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _state_root(environment: Mapping[str, str], unit: str) -> Path:
|
||||||
|
explicit = environment.get("AIOS_WORKER_STATE_ROOT")
|
||||||
|
if explicit:
|
||||||
|
return Path(explicit)
|
||||||
|
fallbacks = {
|
||||||
|
"aios-online-agent-worker-20260815.service": Path(
|
||||||
|
"/var/lib/aios-online-agent-worker-20260815"
|
||||||
|
),
|
||||||
|
"aios-hermes-online-agent-worker.service": Path(
|
||||||
|
"/var/lib/aios-online-agent-hermes"
|
||||||
|
),
|
||||||
|
"aios-openclaw-online-agent-worker.service": Path(
|
||||||
|
"/var/lib/aios-online-agent-openclaw"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
if unit not in fallbacks:
|
||||||
|
raise AvailabilityGuardError("WORKER_STATE_ROOT_UNAVAILABLE:" + unit)
|
||||||
|
return fallbacks[unit]
|
||||||
|
|
||||||
|
|
||||||
|
def _liveness_path(environment: Mapping[str, str], unit: str) -> Path:
|
||||||
|
explicit = environment.get("AIOS_WORKER_LIVENESS_PATH")
|
||||||
|
if explicit:
|
||||||
|
return Path(explicit)
|
||||||
|
return _state_root(environment, unit) / "worker-liveness.json"
|
||||||
|
|
||||||
|
|
||||||
|
class AvailabilityGuard:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
systemd: Systemd | None = None,
|
||||||
|
worker_units: Sequence[str] = DEFAULT_WORKER_UNITS,
|
||||||
|
dependency_units: Sequence[str] = DEFAULT_DEPENDENCY_UNITS,
|
||||||
|
status_path: Path = DEFAULT_STATUS,
|
||||||
|
recovery_path: Path = DEFAULT_RECOVERY,
|
||||||
|
inhibit_path: Path = DEFAULT_INHIBIT,
|
||||||
|
max_stale_seconds: int = DEFAULT_MAX_STALE_SECONDS,
|
||||||
|
restart_cooldown_seconds: int = DEFAULT_RESTART_COOLDOWN_SECONDS,
|
||||||
|
startup_wait_seconds: int = DEFAULT_STARTUP_WAIT_SECONDS,
|
||||||
|
now_fn: Callable[[], float] = time.time,
|
||||||
|
sleep_fn: Callable[[float], None] = time.sleep,
|
||||||
|
) -> None:
|
||||||
|
self.systemd = systemd or Systemd()
|
||||||
|
self.worker_units = tuple(worker_units)
|
||||||
|
self.dependency_units = tuple(dependency_units)
|
||||||
|
self.status_path = Path(status_path)
|
||||||
|
self.recovery_path = Path(recovery_path)
|
||||||
|
self.inhibit_path = Path(inhibit_path)
|
||||||
|
self.max_stale_seconds = int(max_stale_seconds)
|
||||||
|
self.restart_cooldown_seconds = int(restart_cooldown_seconds)
|
||||||
|
self.startup_wait_seconds = int(startup_wait_seconds)
|
||||||
|
self.now_fn = now_fn
|
||||||
|
self.sleep_fn = sleep_fn
|
||||||
|
if self.max_stale_seconds < 15:
|
||||||
|
raise AvailabilityGuardError("MAX_STALE_SECONDS_TOO_SMALL")
|
||||||
|
if self.restart_cooldown_seconds < 15:
|
||||||
|
raise AvailabilityGuardError("RESTART_COOLDOWN_TOO_SMALL")
|
||||||
|
if self.startup_wait_seconds < 0 or self.startup_wait_seconds > 60:
|
||||||
|
raise AvailabilityGuardError("STARTUP_WAIT_INVALID")
|
||||||
|
|
||||||
|
def _recovery_state(self) -> dict[str, float]:
|
||||||
|
value = _load_json(self.recovery_path) or {}
|
||||||
|
raw = value.get("last_restart_epoch")
|
||||||
|
if not isinstance(raw, Mapping):
|
||||||
|
return {}
|
||||||
|
result: dict[str, float] = {}
|
||||||
|
for unit, epoch in raw.items():
|
||||||
|
try:
|
||||||
|
result[str(unit)] = float(epoch)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _save_recovery_state(self, state: Mapping[str, float]) -> None:
|
||||||
|
_atomic_json(
|
||||||
|
self.recovery_path,
|
||||||
|
{
|
||||||
|
"schema": "aios.worker-availability.recovery.v1",
|
||||||
|
"last_restart_epoch": dict(state),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _basic_snapshot(self, unit: str, role: str) -> tuple[str, str, str, bool]:
|
||||||
|
load = self.systemd.property(unit, "LoadState")
|
||||||
|
active = self.systemd.property(unit, "ActiveState")
|
||||||
|
enabled = self.systemd.property(unit, "UnitFileState")
|
||||||
|
return load, active, enabled, enabled in ACCEPTED_ENABLED_STATES
|
||||||
|
|
||||||
|
def _ensure_enabled_active(self, unit: str, role: str) -> tuple[str, bool]:
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, role)
|
||||||
|
if load != "loaded":
|
||||||
|
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
|
||||||
|
action = "NONE"
|
||||||
|
if not enabled_ok:
|
||||||
|
self.systemd.enable(unit)
|
||||||
|
action = "ENABLE"
|
||||||
|
if active != "active":
|
||||||
|
self.systemd.start(unit)
|
||||||
|
action = "START" if action == "NONE" else "ENABLE_START"
|
||||||
|
return action, active == "active"
|
||||||
|
|
||||||
|
def _wait_for_liveness(self, path: Path) -> dict[str, Any] | None:
|
||||||
|
deadline = self.now_fn() + self.startup_wait_seconds
|
||||||
|
while True:
|
||||||
|
value = _load_json(path)
|
||||||
|
if value is not None:
|
||||||
|
try:
|
||||||
|
epoch = float(value.get("last_seen_epoch"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
epoch = 0.0
|
||||||
|
if epoch > 0 and self.now_fn() - epoch <= self.max_stale_seconds:
|
||||||
|
return value
|
||||||
|
if self.now_fn() >= deadline:
|
||||||
|
return value
|
||||||
|
self.sleep_fn(0.5)
|
||||||
|
|
||||||
|
def run(self, *, heal: bool) -> dict[str, Any]:
|
||||||
|
now = self.now_fn()
|
||||||
|
if self.inhibit_path.exists():
|
||||||
|
result = {
|
||||||
|
"schema": SCHEMA,
|
||||||
|
"status": "MAINTENANCE_INHIBITED",
|
||||||
|
"generated_at_epoch": now,
|
||||||
|
"maintenance_inhibit": str(self.inhibit_path),
|
||||||
|
"workers": [],
|
||||||
|
"dependencies": [],
|
||||||
|
"all_fresh": False,
|
||||||
|
"actions": [],
|
||||||
|
}
|
||||||
|
_atomic_json(self.status_path, result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
actions: list[dict[str, str]] = []
|
||||||
|
dependencies: list[UnitSnapshot] = []
|
||||||
|
for unit in self.dependency_units:
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "dependency")
|
||||||
|
action = "NONE"
|
||||||
|
if heal:
|
||||||
|
if load != "loaded":
|
||||||
|
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
|
||||||
|
if not enabled_ok:
|
||||||
|
self.systemd.enable(unit)
|
||||||
|
action = "ENABLE"
|
||||||
|
if active != "active":
|
||||||
|
self.systemd.start(unit)
|
||||||
|
action = "START" if action == "NONE" else "ENABLE_START"
|
||||||
|
if action != "NONE":
|
||||||
|
actions.append({"unit": unit, "action": action})
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "dependency")
|
||||||
|
dependencies.append(
|
||||||
|
UnitSnapshot(
|
||||||
|
unit=unit,
|
||||||
|
role="dependency",
|
||||||
|
load_state=load,
|
||||||
|
active_state=active,
|
||||||
|
unit_file_state=enabled,
|
||||||
|
enabled_ok=enabled_ok,
|
||||||
|
fresh=(active == "active"),
|
||||||
|
action=action,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
recovery = self._recovery_state()
|
||||||
|
workers: list[UnitSnapshot] = []
|
||||||
|
for unit in self.worker_units:
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
|
||||||
|
environment = self.systemd.environment(unit) if load == "loaded" else {}
|
||||||
|
path = _liveness_path(environment, unit) if load == "loaded" else None
|
||||||
|
action = "NONE"
|
||||||
|
|
||||||
|
if heal:
|
||||||
|
if load != "loaded":
|
||||||
|
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
|
||||||
|
if not enabled_ok:
|
||||||
|
self.systemd.enable(unit)
|
||||||
|
action = "ENABLE"
|
||||||
|
if active != "active":
|
||||||
|
self.systemd.start(unit)
|
||||||
|
action = "START" if action == "NONE" else "ENABLE_START"
|
||||||
|
actions.append({"unit": unit, "action": action})
|
||||||
|
if path is not None:
|
||||||
|
self._wait_for_liveness(path)
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
|
||||||
|
|
||||||
|
live = _load_json(path) if path is not None else None
|
||||||
|
last_seen: float | None = None
|
||||||
|
state: str | None = None
|
||||||
|
age: float | None = None
|
||||||
|
fresh = False
|
||||||
|
detail: str | None = None
|
||||||
|
if live is not None:
|
||||||
|
state = str(live.get("state") or "")
|
||||||
|
try:
|
||||||
|
last_seen = float(live.get("last_seen_epoch"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
last_seen = None
|
||||||
|
if last_seen is not None:
|
||||||
|
age = max(0.0, now - last_seen)
|
||||||
|
fresh = age <= self.max_stale_seconds
|
||||||
|
if live.get("agent_id") in (None, ""):
|
||||||
|
fresh = False
|
||||||
|
detail = "LIVENESS_AGENT_ID_MISSING"
|
||||||
|
else:
|
||||||
|
detail = "LIVENESS_MISSING"
|
||||||
|
|
||||||
|
if heal and active == "active" and not fresh:
|
||||||
|
previous = float(recovery.get(unit, 0.0))
|
||||||
|
if now - previous >= self.restart_cooldown_seconds:
|
||||||
|
self.systemd.restart(unit)
|
||||||
|
recovery[unit] = now
|
||||||
|
action = "RESTART_STALE"
|
||||||
|
actions.append({"unit": unit, "action": action})
|
||||||
|
if path is not None:
|
||||||
|
live = self._wait_for_liveness(path)
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
|
||||||
|
if live is not None:
|
||||||
|
state = str(live.get("state") or "")
|
||||||
|
try:
|
||||||
|
last_seen = float(live.get("last_seen_epoch"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
last_seen = None
|
||||||
|
if last_seen is not None:
|
||||||
|
age = max(0.0, self.now_fn() - last_seen)
|
||||||
|
fresh = age <= self.max_stale_seconds
|
||||||
|
detail = None if fresh else "LIVENESS_STALE_AFTER_RESTART"
|
||||||
|
else:
|
||||||
|
detail = "RESTART_COOLDOWN_ACTIVE"
|
||||||
|
|
||||||
|
workers.append(
|
||||||
|
UnitSnapshot(
|
||||||
|
unit=unit,
|
||||||
|
role="worker",
|
||||||
|
load_state=load,
|
||||||
|
active_state=active,
|
||||||
|
unit_file_state=enabled,
|
||||||
|
enabled_ok=enabled_ok,
|
||||||
|
liveness_path=str(path) if path is not None else None,
|
||||||
|
liveness_state=state,
|
||||||
|
last_seen_epoch=last_seen,
|
||||||
|
liveness_age_seconds=age,
|
||||||
|
fresh=bool(active == "active" and enabled_ok and fresh),
|
||||||
|
action=action,
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if heal:
|
||||||
|
self._save_recovery_state(recovery)
|
||||||
|
|
||||||
|
all_fresh = (
|
||||||
|
all(item.active_state == "active" and item.enabled_ok and item.fresh for item in workers)
|
||||||
|
and all(item.active_state == "active" for item in dependencies)
|
||||||
|
)
|
||||||
|
result = {
|
||||||
|
"schema": SCHEMA,
|
||||||
|
"status": "HEALTHY" if all_fresh else "DEGRADED",
|
||||||
|
"generated_at_epoch": self.now_fn(),
|
||||||
|
"max_stale_seconds": self.max_stale_seconds,
|
||||||
|
"maintenance_inhibit": str(self.inhibit_path),
|
||||||
|
"workers": [asdict(item) for item in workers],
|
||||||
|
"dependencies": [asdict(item) for item in dependencies],
|
||||||
|
"all_fresh": all_fresh,
|
||||||
|
"actions": actions,
|
||||||
|
}
|
||||||
|
_atomic_json(self.status_path, result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
mode = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
mode.add_argument("--heal", action="store_true")
|
||||||
|
mode.add_argument("--check", action="store_true")
|
||||||
|
parser.add_argument("--status-file", default=str(DEFAULT_STATUS))
|
||||||
|
parser.add_argument("--recovery-file", default=str(DEFAULT_RECOVERY))
|
||||||
|
parser.add_argument("--inhibit-file", default=str(DEFAULT_INHIBIT))
|
||||||
|
parser.add_argument(
|
||||||
|
"--max-stale-seconds",
|
||||||
|
type=int,
|
||||||
|
default=int(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS",
|
||||||
|
str(DEFAULT_MAX_STALE_SECONDS),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--restart-cooldown-seconds",
|
||||||
|
type=int,
|
||||||
|
default=int(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_WORKER_AVAILABILITY_RESTART_COOLDOWN_SECONDS",
|
||||||
|
str(DEFAULT_RESTART_COOLDOWN_SECONDS),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
guard = AvailabilityGuard(
|
||||||
|
status_path=Path(args.status_file),
|
||||||
|
recovery_path=Path(args.recovery_file),
|
||||||
|
inhibit_path=Path(args.inhibit_file),
|
||||||
|
max_stale_seconds=args.max_stale_seconds,
|
||||||
|
restart_cooldown_seconds=args.restart_cooldown_seconds,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
result = guard.run(heal=args.heal)
|
||||||
|
except AvailabilityGuardError as exc:
|
||||||
|
print(json.dumps({"status": "FAILED", "error": str(exc)}, sort_keys=True))
|
||||||
|
return 20
|
||||||
|
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
||||||
|
if args.check and not result.get("all_fresh"):
|
||||||
|
return 10
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
providers:
|
||||||
|
oneapi:
|
||||||
|
name: oneapi
|
||||||
|
base_url: http://172.18.0.2:3000/v1
|
||||||
|
key_env: OPENAI_API_KEY
|
||||||
|
default_model: deepseek-v4-flash
|
||||||
|
model:
|
||||||
|
provider: oneapi
|
||||||
|
name: deepseek-v4-flash
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"argv": [
|
||||||
|
"/usr/local/bin/hermes",
|
||||||
|
"run",
|
||||||
|
"--non-interactive"
|
||||||
|
],
|
||||||
|
"input_mode": "stdin_text",
|
||||||
|
"max_output_bytes": 131072,
|
||||||
|
"output_mode": "stdout_text",
|
||||||
|
"runtime": "hermes",
|
||||||
|
"schema": "aios.external-agent-runtime.profile.v1",
|
||||||
|
"timeout_seconds": 900
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
{
|
||||||
|
"argv": [
|
||||||
|
"/opt/aios-hermes-agent/hermes-agent/.venv/bin/hermes",
|
||||||
|
"-z",
|
||||||
|
"{prompt}",
|
||||||
|
"--provider",
|
||||||
|
"oneapi",
|
||||||
|
"-m",
|
||||||
|
"{model_id}",
|
||||||
|
"--no-restore-cwd"
|
||||||
|
],
|
||||||
|
"credential_env": {
|
||||||
|
"OPENAI_API_KEY": {
|
||||||
|
"credential": "aios-external-agent-oneapi-env",
|
||||||
|
"format": "dotenv",
|
||||||
|
"key": "ONEAPI_REGULAR_TOKEN"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"environment": {
|
||||||
|
"HERMES_HOME": "/etc/aios/external-agent-runtime.d/hermes-home"
|
||||||
|
},
|
||||||
|
"input_mode": "stdin_text",
|
||||||
|
"max_output_bytes": 131072,
|
||||||
|
"model_governance": {
|
||||||
|
"max_output_tokens": 512,
|
||||||
|
"model_capabilities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"provider_ids": [
|
||||||
|
"oneapi"
|
||||||
|
],
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"output_mode": "stdout_text",
|
||||||
|
"runtime": "hermes",
|
||||||
|
"schema": "aios.external-agent-runtime.profile.v1",
|
||||||
|
"timeout_seconds": 180
|
||||||
|
}
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"tools": {
|
||||||
|
"profile": "minimal"
|
||||||
|
},
|
||||||
|
"memory": {
|
||||||
|
"search": {
|
||||||
|
"enabled": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"agents": {
|
||||||
|
"defaults": {
|
||||||
|
"workspace": "/var/lib/aios-external-agent-executor/openclaw-workspace",
|
||||||
|
"model": {
|
||||||
|
"primary": "openai/deepseek-v4-flash"
|
||||||
|
},
|
||||||
|
"thinkingDefault": "off"
|
||||||
|
},
|
||||||
|
"entries": {
|
||||||
|
"main": {
|
||||||
|
"default": true,
|
||||||
|
"workspace": "/var/lib/aios-external-agent-executor/openclaw-workspace"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"models": {
|
||||||
|
"providers": {
|
||||||
|
"openai": {
|
||||||
|
"baseUrl": "http://172.18.0.2:3000/v1",
|
||||||
|
"api": "openai-completions",
|
||||||
|
"models": [
|
||||||
|
{
|
||||||
|
"id": "deepseek-v4-flash",
|
||||||
|
"name": "deepseek-v4-flash",
|
||||||
|
"contextWindow": 32768,
|
||||||
|
"maxTokens": 256
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
{
|
||||||
|
"argv": [
|
||||||
|
"/usr/local/bin/openclaw",
|
||||||
|
"run",
|
||||||
|
"--non-interactive"
|
||||||
|
],
|
||||||
|
"input_mode": "stdin_text",
|
||||||
|
"max_output_bytes": 131072,
|
||||||
|
"output_mode": "stdout_text",
|
||||||
|
"runtime": "openclaw",
|
||||||
|
"schema": "aios.external-agent-runtime.profile.v1",
|
||||||
|
"timeout_seconds": 900
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
{
|
||||||
|
"argv": [
|
||||||
|
"/usr/bin/node",
|
||||||
|
"/opt/aios/openclaw-engineering/v2026.7.2-beta.7/openclaw.mjs",
|
||||||
|
"agent",
|
||||||
|
"--local",
|
||||||
|
"--json",
|
||||||
|
"--agent",
|
||||||
|
"main",
|
||||||
|
"--thinking",
|
||||||
|
"off",
|
||||||
|
"--message",
|
||||||
|
"{prompt}",
|
||||||
|
"--model",
|
||||||
|
"{model_id}",
|
||||||
|
"--timeout",
|
||||||
|
"180"
|
||||||
|
],
|
||||||
|
"credential_env": {
|
||||||
|
"OPENAI_API_KEY": {
|
||||||
|
"credential": "aios-external-agent-oneapi-env",
|
||||||
|
"format": "dotenv",
|
||||||
|
"key": "ONEAPI_REGULAR_TOKEN"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"environment": {
|
||||||
|
"OPENCLAW_CONFIG_PATH": "/etc/aios/external-agent-runtime.d/openclaw-config.json",
|
||||||
|
"OPENCLAW_HOME": "/var/lib/aios-external-agent-executor/openclaw-home",
|
||||||
|
"OPENCLAW_STATE_DIR": "/var/lib/aios-external-agent-executor/openclaw-state"
|
||||||
|
},
|
||||||
|
"input_mode": "stdin_text",
|
||||||
|
"max_output_bytes": 131072,
|
||||||
|
"model_governance": {
|
||||||
|
"max_output_tokens": 256,
|
||||||
|
"model_capabilities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"provider_ids": [
|
||||||
|
"openai"
|
||||||
|
],
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"output_mode": "stdout_json",
|
||||||
|
"runtime": "openclaw",
|
||||||
|
"schema": "aios.external-agent-runtime.profile.v1",
|
||||||
|
"timeout_seconds": 240
|
||||||
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
{
|
||||||
|
"models": [
|
||||||
|
{
|
||||||
|
"capabilities": [
|
||||||
|
"agent.runtime.hermes",
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"enabled": true,
|
||||||
|
"evaluation_sha256": "43ae0db1640daeb6d33e2787769732844f366d36070065d3a2239905bd9a823b",
|
||||||
|
"evaluation_state": "APPROVED",
|
||||||
|
"input_cost_micros_per_1k": 0,
|
||||||
|
"max_input_tokens": 32768,
|
||||||
|
"max_output_tokens": 512,
|
||||||
|
"model_id": "deepseek-v4-flash",
|
||||||
|
"output_cost_micros_per_1k": 0,
|
||||||
|
"priority": 10,
|
||||||
|
"provider_id": "oneapi",
|
||||||
|
"version": "profile-sha:54615e533bfdb6a2e8b870ae74bb8cd7fc45177495bcf6e8c45a50837cefb85a"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"capabilities": [
|
||||||
|
"agent.runtime.openclaw",
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"enabled": true,
|
||||||
|
"evaluation_sha256": "7b1c89634eb9c0e7cb5a3d75323d904ebbd91b3ae5494ce938bd8316a4cf5b12",
|
||||||
|
"evaluation_state": "APPROVED",
|
||||||
|
"input_cost_micros_per_1k": 0,
|
||||||
|
"max_input_tokens": 32768,
|
||||||
|
"max_output_tokens": 256,
|
||||||
|
"model_id": "openai/deepseek-v4-flash",
|
||||||
|
"output_cost_micros_per_1k": 0,
|
||||||
|
"priority": 10,
|
||||||
|
"provider_id": "openai",
|
||||||
|
"version": "profile-sha:2a31c480226aa78d44ed9692bddcedc99c3aa03c23c0dd5f031a5cbbb8bbd7ba"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"policies": [
|
||||||
|
{
|
||||||
|
"allowed_models": [
|
||||||
|
"deepseek-v4-flash",
|
||||||
|
"openai/deepseek-v4-flash"
|
||||||
|
],
|
||||||
|
"daily_cost_budget_micros": 0,
|
||||||
|
"daily_token_budget": 250000,
|
||||||
|
"denied_providers": [],
|
||||||
|
"fallback_limit": 1,
|
||||||
|
"max_input_tokens_per_request": 16000,
|
||||||
|
"max_output_tokens_per_request": 512,
|
||||||
|
"project_id": null,
|
||||||
|
"required_capabilities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"scope_type": "legacy",
|
||||||
|
"tenant_id": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"allowed_models": [
|
||||||
|
"deepseek-v4-flash",
|
||||||
|
"openai/deepseek-v4-flash"
|
||||||
|
],
|
||||||
|
"daily_cost_budget_micros": 0,
|
||||||
|
"daily_token_budget": 250000,
|
||||||
|
"denied_providers": [],
|
||||||
|
"fallback_limit": 1,
|
||||||
|
"max_input_tokens_per_request": 16000,
|
||||||
|
"max_output_tokens_per_request": 512,
|
||||||
|
"project_id": "staging",
|
||||||
|
"required_capabilities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"scope_type": "tenant",
|
||||||
|
"tenant_id": "model-governance-live"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"providers": [
|
||||||
|
{
|
||||||
|
"cooldown_seconds": 60,
|
||||||
|
"credential_ref": "runtime-profile:hermes",
|
||||||
|
"enabled": true,
|
||||||
|
"failure_domain": "oneapi",
|
||||||
|
"max_failures": 2,
|
||||||
|
"provider_id": "oneapi"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cooldown_seconds": 60,
|
||||||
|
"credential_ref": "runtime-profile:openclaw",
|
||||||
|
"enabled": true,
|
||||||
|
"failure_domain": "oneapi",
|
||||||
|
"max_failures": 2,
|
||||||
|
"provider_id": "openai"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"schema": "aios.model-governance.bootstrap.v1"
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS owner-controlled Task execution-rebind API
|
||||||
|
After=aios-agent-task-api.service
|
||||||
|
Wants=aios-agent-task-api.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-agent-task
|
||||||
|
Group=aios-agent-task
|
||||||
|
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
|
||||||
|
EnvironmentFile=/etc/aios-gitops/agent-task-execution-rebind.env
|
||||||
|
Environment=AIOS_AGENT_TASK_DB=/var/lib/aios-agent-task/state.sqlite3
|
||||||
|
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
|
||||||
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
||||||
|
Environment=AIOS_TASK_EXECUTION_REBIND_HOST=127.0.0.1
|
||||||
|
Environment=AIOS_TASK_EXECUTION_REBIND_PORT=18942
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_agent_task_execution_rebind.py
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||||
|
ReadWritePaths=/var/lib/aios-agent-task
|
||||||
|
UMask=0077
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS credential-isolated external Agent runtime executor
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-engineering-executor
|
||||||
|
Group=aios-executor-exchange
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_external_agent_runtime.py --root /var/lib/aios-external-agent-executor --profile-root /etc/aios/external-agent-runtime.d
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
IPAddressDeny=127.0.0.0/8
|
||||||
|
IPAddressDeny=::1/128
|
||||||
|
IPAddressAllow=172.18.0.2/32
|
||||||
|
LoadCredential=aios-external-agent-oneapi-env:/data/agent/secrets/oneapi-model-ready.env
|
||||||
|
InaccessiblePaths=/etc/aios-gitops
|
||||||
|
InaccessiblePaths=/data/aios-gitops
|
||||||
|
InaccessiblePaths=/data/agent/secrets
|
||||||
|
ReadOnlyPaths=/etc/aios/external-agent-runtime.d
|
||||||
|
ReadOnlyPaths=/opt/aios-hermes-agent
|
||||||
|
ReadOnlyPaths=/opt/aios/openclaw-engineering
|
||||||
|
Environment=AIOS_MODEL_GOVERNANCE_DB=/var/lib/aios-model-governance/state.sqlite3
|
||||||
|
Environment=AIOS_MODEL_GOVERNANCE_ENABLED=1
|
||||||
|
ReadWritePaths=/var/lib/aios-external-agent-executor
|
||||||
|
ReadWritePaths=/var/lib/aios-model-governance
|
||||||
|
UMask=0027
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS Hermes Online Agent Worker
|
||||||
|
After=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
|
||||||
|
Wants=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-engineering-worker
|
||||||
|
Group=aios-engineering-worker
|
||||||
|
SupplementaryGroups=gitops aios-executor-exchange
|
||||||
|
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
||||||
|
Environment=AIOS_AGENT_ID=online-hermes-worker-cn-network-mkt
|
||||||
|
Environment=AIOS_AGENT_TYPE=external-agent-runtime-worker
|
||||||
|
Environment=AIOS_AGENT_VERSION=2026-08-18.1-hermes-runtime
|
||||||
|
Environment=AIOS_WORKER_CAPABILITIES=agent.runtime.hermes
|
||||||
|
Environment=AIOS_WORKER_ALLOWED_ADAPTERS=hermes
|
||||||
|
Environment=AIOS_AUTHORITY_GIT_DIR=/data/aios-gitops/repo.git
|
||||||
|
Environment=AIOS_EXTERNAL_AGENT_EXECUTOR_ROOT=/var/lib/aios-external-agent-executor
|
||||||
|
Environment=AIOS_WORKER_STATE_ROOT=/var/lib/aios-online-agent-hermes
|
||||||
|
Environment=AIOS_WORKER_CHECKPOINT_DB=/var/lib/aios-online-agent-hermes/checkpoints.sqlite3
|
||||||
|
Environment=AIOS_WORKER_WORKSPACE_ROOT=/var/lib/aios-online-agent-hermes/workspaces
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_online_agent_worker.py
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
TimeoutStopSec=20
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
ReadOnlyPaths=/data/aios-gitops/repo.git
|
||||||
|
ReadWritePaths=/var/lib/aios-online-agent-hermes
|
||||||
|
ReadWritePaths=/var/lib/aios-external-agent-executor
|
||||||
|
UMask=0027
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS OpenClaw Online Agent Worker
|
||||||
|
After=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
|
||||||
|
Wants=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=aios-engineering-worker
|
||||||
|
Group=aios-engineering-worker
|
||||||
|
SupplementaryGroups=gitops aios-executor-exchange
|
||||||
|
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
||||||
|
Environment=AIOS_AGENT_ID=online-openclaw-worker-cn-network-mkt
|
||||||
|
Environment=AIOS_AGENT_TYPE=external-agent-runtime-worker
|
||||||
|
Environment=AIOS_AGENT_VERSION=2026-08-18.1-openclaw-runtime
|
||||||
|
Environment=AIOS_WORKER_CAPABILITIES=agent.runtime.openclaw
|
||||||
|
Environment=AIOS_WORKER_ALLOWED_ADAPTERS=openclaw
|
||||||
|
Environment=AIOS_AUTHORITY_GIT_DIR=/data/aios-gitops/repo.git
|
||||||
|
Environment=AIOS_EXTERNAL_AGENT_EXECUTOR_ROOT=/var/lib/aios-external-agent-executor
|
||||||
|
Environment=AIOS_WORKER_STATE_ROOT=/var/lib/aios-online-agent-openclaw
|
||||||
|
Environment=AIOS_WORKER_CHECKPOINT_DB=/var/lib/aios-online-agent-openclaw/checkpoints.sqlite3
|
||||||
|
Environment=AIOS_WORKER_WORKSPACE_ROOT=/var/lib/aios-online-agent-openclaw/workspaces
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_online_agent_worker.py
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
TimeoutStopSec=20
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
ReadOnlyPaths=/data/aios-gitops/repo.git
|
||||||
|
ReadWritePaths=/var/lib/aios-online-agent-openclaw
|
||||||
|
ReadWritePaths=/var/lib/aios-external-agent-executor
|
||||||
|
UMask=0027
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS persistent Online Agent worker availability guard
|
||||||
|
After=local-fs.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=root
|
||||||
|
Group=root
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
Environment=AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS=90
|
||||||
|
Environment=AIOS_WORKER_AVAILABILITY_RESTART_COOLDOWN_SECONDS=60
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_worker_availability_guard.py --heal
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX
|
||||||
|
ReadWritePaths=/run/aios-worker-availability
|
||||||
|
UMask=0022
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Periodically ensure AIOS Online Agent workers remain available
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=15s
|
||||||
|
OnUnitActiveSec=30s
|
||||||
|
AccuracySec=5s
|
||||||
|
RandomizedDelaySec=3s
|
||||||
|
Persistent=true
|
||||||
|
Unit=aios-worker-availability-guard.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
d /var/lib/aios-external-agent-executor 2770 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-external-agent-executor/requests 2770 aios-engineering-worker aios-executor-exchange -
|
||||||
|
d /var/lib/aios-external-agent-executor/results 2770 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-external-agent-executor/failed 2770 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-external-agent-executor/openclaw-workspace 0750 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-external-agent-executor/openclaw-state 0750 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-external-agent-executor/openclaw-home 0750 aios-engineering-executor aios-executor-exchange -
|
||||||
|
d /var/lib/aios-online-agent-hermes 0750 aios-engineering-worker aios-engineering-worker -
|
||||||
|
d /var/lib/aios-online-agent-hermes/workspaces 0750 aios-engineering-worker aios-engineering-worker -
|
||||||
|
d /var/lib/aios-online-agent-openclaw 0750 aios-engineering-worker aios-engineering-worker -
|
||||||
|
d /var/lib/aios-online-agent-openclaw/workspaces 0750 aios-engineering-worker aios-engineering-worker -
|
||||||
|
d /etc/aios/external-agent-runtime.d 0755 root root -
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
d /var/lib/aios-model-governance 0750 aios-engineering-executor aios-executor-exchange -
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
d /run/aios-worker-availability 0755 root root -
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
# External Agent Attempt-Scoped Request Identity
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
External Agent execution previously derived its durable request ID from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
task_id
|
||||||
|
job_id
|
||||||
|
execution_digest
|
||||||
|
runtime
|
||||||
|
```
|
||||||
|
|
||||||
|
The Online Task attempt was missing.
|
||||||
|
|
||||||
|
That is incorrect for retryable external side effects. A failed Model Governance route reservation is
|
||||||
|
durable and terminal for that request ID. Reusing the same external request ID on a later official
|
||||||
|
Task attempt therefore reuses the old route decision/reservation and can fail settlement with:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MODEL_RESERVATION_STATE_INVALID
|
||||||
|
```
|
||||||
|
|
||||||
|
Archiving an exchange failed marker does not fix the root cause because the Model Governance
|
||||||
|
reservation remains intentionally durable.
|
||||||
|
|
||||||
|
## Contract
|
||||||
|
|
||||||
|
External request identity is now:
|
||||||
|
|
||||||
|
```text
|
||||||
|
hash(
|
||||||
|
task_id,
|
||||||
|
job_id,
|
||||||
|
execution_digest,
|
||||||
|
runtime,
|
||||||
|
attempt
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
The attempt is a **transient execution coordinate**. It is not added to the signed governed execution
|
||||||
|
envelope and does not alter the Task/Job binding or execution digest.
|
||||||
|
|
||||||
|
The worker gets the authoritative attempt from the successful claim response and passes it through:
|
||||||
|
|
||||||
|
```text
|
||||||
|
process_claim
|
||||||
|
→ RuntimeAdapter.execute(..., attempt=N)
|
||||||
|
→ ExternalAgentExecutor
|
||||||
|
→ ExternalAgentExchange
|
||||||
|
→ external request JSON
|
||||||
|
→ external result JSON
|
||||||
|
```
|
||||||
|
|
||||||
|
The external result must bind the same attempt.
|
||||||
|
|
||||||
|
## Idempotency
|
||||||
|
|
||||||
|
This gives the intended two-level behavior:
|
||||||
|
|
||||||
|
```text
|
||||||
|
same Task/Job/execution/runtime + same attempt
|
||||||
|
→ same external request ID
|
||||||
|
→ same external result/reservation can be replayed safely
|
||||||
|
|
||||||
|
same Task/Job/execution/runtime + next official attempt
|
||||||
|
→ different external request ID
|
||||||
|
→ different Model Governance request ID
|
||||||
|
→ new reservation
|
||||||
|
```
|
||||||
|
|
||||||
|
Old FAILED reservations are retained for audit. No Model Governance DB cleanup is required or
|
||||||
|
permitted.
|
||||||
|
|
||||||
|
## Model Governance
|
||||||
|
|
||||||
|
Model Governance already derives its route request ID from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
external_request_id + ":model"
|
||||||
|
```
|
||||||
|
|
||||||
|
Therefore making the external request ID attempt-scoped automatically makes route decisions and
|
||||||
|
reservations attempt-scoped without creating another state authority.
|
||||||
|
|
||||||
|
## Recovery
|
||||||
|
|
||||||
|
For the current Hermes live Task:
|
||||||
|
|
||||||
|
```text
|
||||||
|
attempt 6
|
||||||
|
→ old request/reservation remains FAILED
|
||||||
|
|
||||||
|
official NEEDS_REWORK
|
||||||
|
→ bounded retry extension 6 → 7
|
||||||
|
|
||||||
|
attempt 7
|
||||||
|
→ new external request ID
|
||||||
|
→ new route request ID
|
||||||
|
→ new reservation
|
||||||
|
→ provider execution
|
||||||
|
→ settlement
|
||||||
|
```
|
||||||
|
|
||||||
|
No replacement Task/Job, direct DB rewrite, failed-reservation deletion or marker deletion is
|
||||||
|
required.
|
||||||
|
|
||||||
|
## Worker Availability
|
||||||
|
|
||||||
|
The repair must preserve the already verified Worker Availability runtime. Runtime switches use the
|
||||||
|
maintenance inhibit and must finish with the guard healthy and all three workers fresh.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
Required regression coverage:
|
||||||
|
|
||||||
|
- request ID stable within one attempt;
|
||||||
|
- request ID different across attempts;
|
||||||
|
- request schema requires a valid attempt;
|
||||||
|
- external result binds attempt;
|
||||||
|
- old failed Model Governance reservation remains FAILED;
|
||||||
|
- next attempt gets a new reservation and can settle;
|
||||||
|
- existing external-agent and Model Governance runtime tests still pass.
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
# Hermes / OpenClaw Online Agent Runtime
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Promote Hermes and OpenClaw from shadow-only adapter descriptions to real Online Agent workers
|
||||||
|
without creating a second Task/claim protocol.
|
||||||
|
|
||||||
|
Both workers reuse the existing P1/P2 Online Agent lifecycle:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Online Task projected
|
||||||
|
→ capability-filtered poll
|
||||||
|
→ automatic claim
|
||||||
|
→ heartbeat
|
||||||
|
→ durable checkpoint
|
||||||
|
→ external Agent execution
|
||||||
|
→ attempt-aware result submission
|
||||||
|
→ official verification
|
||||||
|
```
|
||||||
|
|
||||||
|
They do **not** change the existing TaskPlane `OpenClawAdapter/HermesAdapter` shadow contract. Those
|
||||||
|
adapters remain side-effect-free compatibility surfaces.
|
||||||
|
|
||||||
|
## Execution action
|
||||||
|
|
||||||
|
The canonical execution envelope adds a read-only action:
|
||||||
|
|
||||||
|
```text
|
||||||
|
action=agent_runtime
|
||||||
|
adapter=hermes | openclaw
|
||||||
|
```
|
||||||
|
|
||||||
|
The action requires capability `agent.runtime.<adapter>`, has an empty change set, no verification
|
||||||
|
commands, and `writer_lease=false`.
|
||||||
|
|
||||||
|
`project_online_task()` copies only bounded task fields into the signed execution envelope:
|
||||||
|
|
||||||
|
- goal;
|
||||||
|
- instructions;
|
||||||
|
- success criteria;
|
||||||
|
- non-secret constraints.
|
||||||
|
|
||||||
|
The task may not choose a command, executable, URL, credential path, environment, or shell string.
|
||||||
|
|
||||||
|
## Worker separation
|
||||||
|
|
||||||
|
The generic engineering worker defaults to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
AIOS_WORKER_ALLOWED_ADAPTERS=deterministic,source_capture,compact_interface_map,codex
|
||||||
|
```
|
||||||
|
|
||||||
|
so it skips Hermes/OpenClaw tasks before claim.
|
||||||
|
|
||||||
|
Dedicated services advertise:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Hermes:
|
||||||
|
AIOS_WORKER_CAPABILITIES=agent.runtime.hermes
|
||||||
|
AIOS_WORKER_ALLOWED_ADAPTERS=hermes
|
||||||
|
|
||||||
|
OpenClaw:
|
||||||
|
AIOS_WORKER_CAPABILITIES=agent.runtime.openclaw
|
||||||
|
AIOS_WORKER_ALLOWED_ADAPTERS=openclaw
|
||||||
|
```
|
||||||
|
|
||||||
|
This makes adapter selection a pre-claim capability/fleet decision instead of a post-claim failure.
|
||||||
|
|
||||||
|
## Credential-isolated executor
|
||||||
|
|
||||||
|
The Online Agent worker itself still owns only Task API/Bridge coordination credentials. It writes
|
||||||
|
a hash-bound request to `/var/lib/aios-external-agent-executor`.
|
||||||
|
|
||||||
|
`aios-external-agent-executor.service` runs as the existing uncredentialed engineering executor user.
|
||||||
|
It has:
|
||||||
|
|
||||||
|
- one provider-only `LoadCredential` projection for the model-ready OneAPI environment;
|
||||||
|
- `/etc/aios-gitops`, `/data/aios-gitops`, and the original provider-secret directory inaccessible;
|
||||||
|
- no repository path;
|
||||||
|
- no shell execution;
|
||||||
|
- loopback denied, preventing direct access to the AIOS Task/Bridge control plane;
|
||||||
|
- only the staging OneAPI container address allowed for the installed Agent runtime/provider;
|
||||||
|
- only the exchange directory writable.
|
||||||
|
|
||||||
|
The root-owned runtime profile in `/etc/aios/external-agent-runtime.d/<runtime>.json` decides the
|
||||||
|
absolute executable, fixed non-secret environment, provider-only credential mapping, and I/O mode.
|
||||||
|
A task cannot override any of these values. The profile may select `OPENAI_API_KEY` only from the
|
||||||
|
systemd credential named `aios-external-agent-oneapi-env`; it cannot name a credential, inject an
|
||||||
|
environment variable, or carry an inline secret. The runtime executor parses only the selected
|
||||||
|
`ONEAPI_REGULAR_TOKEN` entry and does not expose the source file or other provider tiers to the
|
||||||
|
child.
|
||||||
|
|
||||||
|
## Runtime profile
|
||||||
|
|
||||||
|
A profile supports:
|
||||||
|
|
||||||
|
- `argv`: fixed argv, absolute allowlisted executable;
|
||||||
|
- `input_mode`: `stdin_text`, `stdin_json`, or `file_json`;
|
||||||
|
- `output_mode`: `stdout_text`, `stdout_json`, or `file_json`;
|
||||||
|
- timeout;
|
||||||
|
- bounded output size.
|
||||||
|
|
||||||
|
`{input_json}` and `{output_json}` placeholders are allowed only for file modes. `{prompt}` is a
|
||||||
|
root-profile-only argv placeholder populated from the bounded, signed task projection.
|
||||||
|
|
||||||
|
The repository includes the staging profile templates validated against the installed Hermes
|
||||||
|
v0.20.0 and OpenClaw v2026.7.2-beta.7 CLIs. Before activation, the Agent must re-run the actual
|
||||||
|
`--help`/version probes and install the profiles and static configs root-owned. If the installed
|
||||||
|
CLI differs, update only the root-owned profile/config projection and record the observed command;
|
||||||
|
do not modify the Task contract to compensate.
|
||||||
|
|
||||||
|
The provider projection is staging-only and uses the existing model-ready OneAPI source. It is not
|
||||||
|
an AIOS Task, Bridge, GitOps, SSH, or Writer credential. Live activation remains blocked if the
|
||||||
|
model-ready source cannot be projected through systemd or if the selected provider credential is
|
||||||
|
missing/quota-invalid. This worker never passes API keys through task metadata or arbitrary
|
||||||
|
environment variables.
|
||||||
|
|
||||||
|
## Retry/restart
|
||||||
|
|
||||||
|
The external request ID is deterministic over Task/Job/execution digest/runtime. The exchange keeps
|
||||||
|
the request/result across worker restart. A repeated attempt with the exact execution digest reuses
|
||||||
|
the durable external result instead of invoking the Agent twice.
|
||||||
|
|
||||||
|
The normal P1 result idempotency/checkpoint path remains authoritative for Task result submission.
|
||||||
|
|
||||||
|
## Result
|
||||||
|
|
||||||
|
Successful execution returns:
|
||||||
|
|
||||||
|
```text
|
||||||
|
status=SUCCEEDED
|
||||||
|
classification=EXTERNAL_AGENT_RUNTIME_SUCCEEDED
|
||||||
|
execution_mode=agent_runtime
|
||||||
|
agent_runtime=hermes | openclaw
|
||||||
|
writer_lease=NONE
|
||||||
|
changed_files=[]
|
||||||
|
credential_material_exposed=false
|
||||||
|
control_plane_access_granted_to_runtime=false
|
||||||
|
```
|
||||||
|
|
||||||
|
This first runtime version is intentionally read-only with respect to the repository. Engineering
|
||||||
|
changes continue to use assistant-authored deterministic plans and governed WriterLease/commit.
|
||||||
@@ -0,0 +1,211 @@
|
|||||||
|
# AIOS Model Governance Runtime
|
||||||
|
|
||||||
|
## Status and purpose
|
||||||
|
|
||||||
|
This component turns the previously `PARTIAL` Model Governance interface map into an enforceable
|
||||||
|
runtime policy for the already verified Hermes/OpenClaw Online Agent path.
|
||||||
|
|
||||||
|
It is intentionally a **single SQLite library**, not a new network service. The existing external
|
||||||
|
Agent executor calls the library immediately before launching a model-backed runtime and settles the
|
||||||
|
same reservation immediately after the result.
|
||||||
|
|
||||||
|
The authoritative execution chain remains:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Online Task scope / trace
|
||||||
|
→ dedicated worker claim + heartbeat
|
||||||
|
→ external Agent executor
|
||||||
|
→ ModelGovernanceStore.route()
|
||||||
|
→ root-owned runtime profile binds provider/model/version
|
||||||
|
→ Hermes / OpenClaw process
|
||||||
|
→ ModelGovernanceStore.record_success() / record_failure()
|
||||||
|
→ durable Online Agent checkpoint/result
|
||||||
|
```
|
||||||
|
|
||||||
|
P0 scope/trace/credential, P1 checkpoint/retry, P2 fleet scheduling and the existing external Agent
|
||||||
|
credential boundary remain authoritative.
|
||||||
|
|
||||||
|
## Ten Model Governance contracts
|
||||||
|
|
||||||
|
### MODEL_PROVIDER_REGISTRY
|
||||||
|
|
||||||
|
`model_governance_providers` stores provider identity, failure domain, circuit policy and an opaque
|
||||||
|
credential reference. Raw credentials are rejected.
|
||||||
|
|
||||||
|
Allowed reference prefixes:
|
||||||
|
|
||||||
|
- `systemd-credential:`
|
||||||
|
- `runtime-profile:`
|
||||||
|
- `credential-broker:`
|
||||||
|
|
||||||
|
### MODEL_SELECTION_POLICY
|
||||||
|
|
||||||
|
`model_governance_scope_policies` constrains eligible models/providers/capabilities. Routing is
|
||||||
|
deterministic: explicit preferred models, then configured priority, then stable provider/model IDs.
|
||||||
|
|
||||||
|
### MODEL_VERSION_PINNING
|
||||||
|
|
||||||
|
A registry row always includes a model version. A scope can additionally pin a model ID + exact
|
||||||
|
version. Every route decision persists both model ID and version plus model/policy revisions.
|
||||||
|
|
||||||
|
### MODEL_BUDGET_ACCOUNTING
|
||||||
|
|
||||||
|
A route first reserves estimated input/output tokens and estimated cost inside `BEGIN IMMEDIATE`.
|
||||||
|
Settled usage and outstanding reservations both count against the daily scope budget, preventing two
|
||||||
|
concurrent requests from oversubscribing the same budget.
|
||||||
|
|
||||||
|
Runtime-reported usage is preferred. When telemetry is absent, the reserved upper bound is charged
|
||||||
|
rather than silently under-counting.
|
||||||
|
|
||||||
|
OpenClaw's verified result shape is supported directly:
|
||||||
|
|
||||||
|
```text
|
||||||
|
meta.agentMeta.usage.input
|
||||||
|
meta.agentMeta.usage.output
|
||||||
|
meta.agentMeta.costUsd
|
||||||
|
```
|
||||||
|
|
||||||
|
Hermes currently has no structured usage in the verified output, so it uses conservative
|
||||||
|
reservation settlement.
|
||||||
|
|
||||||
|
### MODEL_FALLBACK_POLICY
|
||||||
|
|
||||||
|
Provider failures are durable. Repeated failures open a circuit for the configured cooldown. A
|
||||||
|
fallback request excludes the failed provider and increments `fallback_index`; the scope policy
|
||||||
|
bounds the number of fallback attempts.
|
||||||
|
|
||||||
|
### MODEL_SCOPE_ISOLATION
|
||||||
|
|
||||||
|
The signed `agent_runtime` execution envelope now carries authoritative Task scope + trace identity.
|
||||||
|
|
||||||
|
The projection layer always overwrites `runtime_input.scope` and `runtime_input.trace_id` from the
|
||||||
|
Online Task, even if an execution request attempts to supply different values.
|
||||||
|
|
||||||
|
Tenant/project lookup may fall back only to the same tenant's policy. It never falls back to a
|
||||||
|
system or legacy policy. `system` and `legacy` require their own explicit policies, avoiding fake
|
||||||
|
tenant identities.
|
||||||
|
|
||||||
|
### MODEL_CREDENTIAL_BOUNDARY
|
||||||
|
|
||||||
|
Model governance stores credential references only. The root-owned external-runtime profile still
|
||||||
|
owns the actual provider credential projection. Task payloads cannot select executable, argv,
|
||||||
|
environment, provider credential, or credential path.
|
||||||
|
|
||||||
|
The external child process remains unable to reach the AIOS control plane.
|
||||||
|
|
||||||
|
### MODEL_TRACE_AUDIT
|
||||||
|
|
||||||
|
Every route, failure, provider/model/policy update and settlement appends to a SHA-256 hash-chained
|
||||||
|
audit table with task ID, trace ID and scope key. `verify_audit_chain()` detects tampering.
|
||||||
|
|
||||||
|
### MODEL_EVALUATION_GATE
|
||||||
|
|
||||||
|
A model cannot enter the registry unless its state is `APPROVED` and it has a 64-hex evaluation
|
||||||
|
evidence digest.
|
||||||
|
|
||||||
|
The staging bootstrap uses the already verified Hermes/OpenClaw E2E external-result SHA256 values as
|
||||||
|
evaluation evidence, rather than a configuration-file hash.
|
||||||
|
|
||||||
|
### MODEL_ROUTING_RESILIENCE
|
||||||
|
|
||||||
|
Route request IDs are idempotent and bound to a canonical request hash. Conflicting replay is
|
||||||
|
rejected. Provider health, reservations, usage settlement and policy/provider/model updates use
|
||||||
|
transaction/CAS semantics.
|
||||||
|
|
||||||
|
## Runtime profile contract
|
||||||
|
|
||||||
|
A root-owned Hermes/OpenClaw profile may declare:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"model_governance": {
|
||||||
|
"required": true,
|
||||||
|
"provider_ids": ["provider-a"],
|
||||||
|
"model_capabilities": ["text"],
|
||||||
|
"max_output_tokens": 512
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
When `required=true`:
|
||||||
|
|
||||||
|
- `provider_ids` must be non-empty;
|
||||||
|
- `max_output_tokens` must be positive;
|
||||||
|
- argv/environment must contain at least one governance-controlled marker:
|
||||||
|
`{provider_id}`, `{model_id}`, or `{model_version}`;
|
||||||
|
- `AIOS_MODEL_GOVERNANCE_ENABLED` must be enabled;
|
||||||
|
- a matching scope policy/model/provider must exist;
|
||||||
|
- budget reservation must succeed before the child process is launched.
|
||||||
|
|
||||||
|
The task cannot provide these markers or their values.
|
||||||
|
|
||||||
|
## SQLite authority
|
||||||
|
|
||||||
|
Default database:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/var/lib/aios-model-governance/state.sqlite3
|
||||||
|
```
|
||||||
|
|
||||||
|
The executor process already owns the only runtime mutation path. The directory is created by
|
||||||
|
tmpfiles and is the only new writable path added to that executor.
|
||||||
|
|
||||||
|
No direct Task/Job/Git database writes are introduced.
|
||||||
|
|
||||||
|
## Bootstrap/reconciliation
|
||||||
|
|
||||||
|
`aios_model_governance_admin.py` applies a root-owned JSON bootstrap idempotently. It compares desired
|
||||||
|
state to current rows and uses the current revision as the CAS input only when a change is actually
|
||||||
|
required.
|
||||||
|
|
||||||
|
The repository staging bootstrap is not a Production policy. It exists only to reproduce and
|
||||||
|
govern the already verified non-production Hermes/OpenClaw runtimes.
|
||||||
|
|
||||||
|
## Cost caveat
|
||||||
|
|
||||||
|
The current verified OneAPI-backed runtime evidence reports zero authoritative provider cost for
|
||||||
|
OpenClaw and no structured cost for Hermes. The staging bootstrap therefore does **not** invent
|
||||||
|
pricing and sets cost rates/budget to zero/unbounded.
|
||||||
|
|
||||||
|
Token budgets remain enforced. Real monetary budget enforcement becomes effective as soon as
|
||||||
|
authoritative per-model pricing is entered into the root-owned registry.
|
||||||
|
|
||||||
|
## OpenClaw JIT/WASM compatibility
|
||||||
|
|
||||||
|
The previous live acceptance proved that OpenClaw Node/WASM requires
|
||||||
|
`MemoryDenyWriteExecute=no`. Model Governance must preserve that already-evidenced runtime exception;
|
||||||
|
it must not further weaken `NoNewPrivileges`, `ProtectSystem=strict`, `ProtectHome`, loopback deny,
|
||||||
|
credential isolation or repository isolation.
|
||||||
|
|
||||||
|
## Failure/recovery invariants
|
||||||
|
|
||||||
|
- Budget denial occurs before provider execution.
|
||||||
|
- Missing scope policy fails closed.
|
||||||
|
- Missing/disabled governance fails closed when a profile requires it.
|
||||||
|
- Actual usage cannot exceed the reservation.
|
||||||
|
- Provider failure releases that reservation into `FAILED` state before fallback.
|
||||||
|
- Settlement is idempotent.
|
||||||
|
- Route replay is idempotent only for the exact canonical request.
|
||||||
|
- Audit corruption is detectable.
|
||||||
|
- Online Agent checkpoint/result retry remains unchanged and remains the outer recovery authority.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
Required unit/integration coverage includes:
|
||||||
|
|
||||||
|
- registry/evaluation/CAS;
|
||||||
|
- scope isolation and no system→tenant fallback;
|
||||||
|
- model/version pinning;
|
||||||
|
- atomic budget reservation;
|
||||||
|
- settlement/replay;
|
||||||
|
- missing telemetry conservative settlement;
|
||||||
|
- provider circuit/fallback;
|
||||||
|
- trace/audit tamper detection;
|
||||||
|
- authoritative scope/trace projection;
|
||||||
|
- task runtime-input scope override attack;
|
||||||
|
- root-owned provider/model binding;
|
||||||
|
- OpenClaw nested usage extraction;
|
||||||
|
- real Hermes/OpenClaw staging live tasks with Model Governance evidence.
|
||||||
|
|
||||||
|
The governed implementation is not complete without the unified all-modified-files bundle for the
|
||||||
|
candidate delta and the cumulative lineage bundle.
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
# Owner-Controlled Same-Binding Execution Rebind
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
`aios_agent_task_execution_rebind.py` closes a projection/plan race without replacing the Task,
|
||||||
|
Governed Job, target branch, generation, or idempotency lineage.
|
||||||
|
|
||||||
|
The only supported transition is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
official NEEDS_REWORK
|
||||||
|
→ Task QUEUED with no active lease
|
||||||
|
→ persisted source_capture envelope
|
||||||
|
→ canonical Bridge resolves an already-registered deterministic plan
|
||||||
|
→ CAS-bound metadata.execution + execution_projection replacement
|
||||||
|
→ audit event
|
||||||
|
→ normal worker claim
|
||||||
|
```
|
||||||
|
|
||||||
|
The service rejects arbitrary metadata patches. It derives the new envelope from canonical Bridge
|
||||||
|
`/v2/execution-envelope`, validates every governed binding coordinate against the persisted envelope,
|
||||||
|
requires `source_capture → deterministic`, requires exact old/new execution digests supplied by the
|
||||||
|
Owner recovery request, and increments the Task revision under `BEGIN IMMEDIATE`.
|
||||||
|
|
||||||
|
It records `TASK_EXECUTION_REBIND_AUTHORIZED` and an idempotent `task_execution_rebinds` receipt.
|
||||||
|
No Task/Job/branch replacement is created and the worker still performs the real claim/heartbeat,
|
||||||
|
WriterLease and governed commit.
|
||||||
|
|
||||||
|
The endpoint is loopback-only:
|
||||||
|
|
||||||
|
`POST /v1/tasks/{task_id}/execution-rebind`
|
||||||
|
|
||||||
|
and requires the existing Owner control credential plus
|
||||||
|
`AIOS_AGENT_TASK_EXECUTION_REBIND_AUTHORIZATION_ID`.
|
||||||
|
|
||||||
|
Direct/manual Task database edits remain forbidden; the service is the audited mutation boundary.
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
# AIOS Worker Availability Runtime
|
||||||
|
|
||||||
|
## Priority
|
||||||
|
|
||||||
|
This runtime exists to make the already verified Online Agent execution path continuously usable
|
||||||
|
without repeated human intervention.
|
||||||
|
|
||||||
|
The system already proved that generic, Hermes and OpenClaw workers can claim and complete real
|
||||||
|
Online Tasks. The remaining availability defect was operational: evidence-freeze workflows left the
|
||||||
|
worker services stopped, while registry `ACTIVE` records remained stale.
|
||||||
|
|
||||||
|
The availability contract therefore separates:
|
||||||
|
|
||||||
|
```text
|
||||||
|
AUTO_CLAIM_MECHANISM=VERIFIED
|
||||||
|
CURRENT_WORKER_AVAILABILITY=<fresh runtime state>
|
||||||
|
```
|
||||||
|
|
||||||
|
A historical successful claim is not current liveness.
|
||||||
|
|
||||||
|
## Source of truth for idle liveness
|
||||||
|
|
||||||
|
An idle worker may have no Task lease heartbeat. Therefore Task heartbeat alone cannot prove that
|
||||||
|
the worker is continuously polling.
|
||||||
|
|
||||||
|
`aios_online_agent_worker.py` now writes:
|
||||||
|
|
||||||
|
```text
|
||||||
|
<AIOS_WORKER_STATE_ROOT>/worker-liveness.json
|
||||||
|
```
|
||||||
|
|
||||||
|
on:
|
||||||
|
|
||||||
|
- process start: `STARTING`;
|
||||||
|
- every successful poll: `POLL_OK`;
|
||||||
|
- poll failure: `POLL_FAILED`;
|
||||||
|
- every successful Task lease heartbeat while executing: `BUSY`.
|
||||||
|
|
||||||
|
The file contains:
|
||||||
|
|
||||||
|
- agent ID/type/version;
|
||||||
|
- process PID;
|
||||||
|
- monotonically increasing local sequence;
|
||||||
|
- `last_seen_epoch`;
|
||||||
|
- poll/heartbeat intervals;
|
||||||
|
- current Task/stage while busy.
|
||||||
|
|
||||||
|
The liveness file is worker-owned observability state only. It does not write or mutate Task, Job,
|
||||||
|
Git, WriterLease or registry databases.
|
||||||
|
|
||||||
|
## Self-healing guard
|
||||||
|
|
||||||
|
`aios-worker-availability-guard.timer` invokes a root-owned oneshot guard every 30 seconds.
|
||||||
|
|
||||||
|
The guard manages:
|
||||||
|
|
||||||
|
```text
|
||||||
|
aios-online-agent-worker-20260815.service
|
||||||
|
aios-hermes-online-agent-worker.service
|
||||||
|
aios-openclaw-online-agent-worker.service
|
||||||
|
```
|
||||||
|
|
||||||
|
and the required external runtime dependency:
|
||||||
|
|
||||||
|
```text
|
||||||
|
aios-external-agent-executor.service
|
||||||
|
```
|
||||||
|
|
||||||
|
For each worker:
|
||||||
|
|
||||||
|
1. missing systemd unit → fail closed;
|
||||||
|
2. disabled → `systemctl enable`;
|
||||||
|
3. inactive → `systemctl start`;
|
||||||
|
4. active but liveness older than 90 seconds → restart;
|
||||||
|
5. repeated stale restart within 60 seconds → do not restart again; report degraded.
|
||||||
|
|
||||||
|
Long-running Tasks are safe because the Task lease-heartbeat thread refreshes `BUSY` liveness.
|
||||||
|
|
||||||
|
The guard never claims a Task and never touches Task/Job/Git databases.
|
||||||
|
|
||||||
|
## Maintenance inhibit
|
||||||
|
|
||||||
|
Planned maintenance creates:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/run/aios-worker-availability/maintenance.lock
|
||||||
|
```
|
||||||
|
|
||||||
|
While this file exists the guard performs no start/restart actions and reports:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MAINTENANCE_INHIBITED
|
||||||
|
```
|
||||||
|
|
||||||
|
Maintenance tooling must remove the file when done. This prevents the self-healer from fighting
|
||||||
|
intentional service changes.
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
The guard writes:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/run/aios-worker-availability/status.json
|
||||||
|
```
|
||||||
|
|
||||||
|
with current worker/dependency states, liveness ages and recovery actions.
|
||||||
|
|
||||||
|
A worker is currently available only when:
|
||||||
|
|
||||||
|
```text
|
||||||
|
systemd load_state=loaded
|
||||||
|
systemd active_state=active
|
||||||
|
unit enabled/static
|
||||||
|
worker-liveness.json present
|
||||||
|
last_seen_age <= 90s
|
||||||
|
```
|
||||||
|
|
||||||
|
Registry `ACTIVE` without fresh poll/busy liveness is not sufficient.
|
||||||
|
|
||||||
|
## Boot/restart behavior
|
||||||
|
|
||||||
|
The timer is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
OnBootSec=15s
|
||||||
|
OnUnitActiveSec=30s
|
||||||
|
Persistent=true
|
||||||
|
```
|
||||||
|
|
||||||
|
Therefore reboot, manual stop left behind by another workflow, process crash, and hung poll loop are
|
||||||
|
all repaired without requiring a human to notice and intervene.
|
||||||
|
|
||||||
|
Normal worker units retain their own `Restart=always`; the availability guard handles the cases that
|
||||||
|
`Restart=always` cannot, especially a deliberate `systemctl stop` left after an evidence workflow.
|
||||||
|
|
||||||
|
## Acceptance
|
||||||
|
|
||||||
|
The live acceptance requires all of the following:
|
||||||
|
|
||||||
|
```text
|
||||||
|
guard timer active/enabled
|
||||||
|
external executor active/enabled
|
||||||
|
generic worker active/enabled + fresh liveness
|
||||||
|
Hermes worker active/enabled + fresh liveness
|
||||||
|
OpenClaw worker active/enabled + fresh liveness
|
||||||
|
```
|
||||||
|
|
||||||
|
Then a new read-only Online Task is projected through canonical Bridge `18892`.
|
||||||
|
|
||||||
|
Success requires:
|
||||||
|
|
||||||
|
```text
|
||||||
|
TASK_CREATED
|
||||||
|
→ generic worker auto claim
|
||||||
|
→ TASK_HEARTBEAT
|
||||||
|
→ RESULT_SUBMITTED
|
||||||
|
→ official TASK_VERIFIED(SUCCEEDED)
|
||||||
|
```
|
||||||
|
|
||||||
|
After the Task, the services must remain active and liveness must continue advancing. The acceptance
|
||||||
|
also stops a worker intentionally once and proves the guard starts it again with a new fresh
|
||||||
|
liveness sequence.
|
||||||
|
|
||||||
|
Only after these gates may current status be upgraded from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
OVERALL=PARTIAL
|
||||||
|
```
|
||||||
|
|
||||||
|
to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
PERSISTENT_WORKER_LIVENESS=VERIFIED
|
||||||
|
NEW_TASK_AUTO_CONSUMPTION=VERIFIED
|
||||||
|
WORKER_AVAILABILITY_RUNTIME=VERIFIED_CANDIDATE
|
||||||
|
```
|
||||||
|
|
||||||
|
Protected main and Production remain out of scope.
|
||||||
|
|
||||||
|
|
||||||
|
## Forced source-capture projection compatibility
|
||||||
|
|
||||||
|
The deployed canonical Bridge currently projects a default `source_capture` envelope even when a
|
||||||
|
governed binding request asks for binding-only behavior. This must not be allowed to make always-on
|
||||||
|
workers consume engineering Tasks before their deterministic plan is registered.
|
||||||
|
|
||||||
|
The worker therefore distinguishes a real read-only source capture from an engineering placeholder:
|
||||||
|
|
||||||
|
```text
|
||||||
|
source_capture
|
||||||
|
+ capability_scope contains governed.write.branch or governed.write.file
|
||||||
|
= PENDING_DETERMINISTIC_PLAN
|
||||||
|
```
|
||||||
|
|
||||||
|
For such a placeholder the worker:
|
||||||
|
|
||||||
|
1. re-resolves through the canonical plan registry **without replaying the stale source-capture
|
||||||
|
envelope**;
|
||||||
|
2. if no deterministic plan exists, emits `TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN` and does not
|
||||||
|
claim;
|
||||||
|
3. if the registered canonical envelope is deterministic and the governed binding still matches,
|
||||||
|
uses that deterministic envelope and claims normally.
|
||||||
|
|
||||||
|
A real read-only source-capture Task with only `git.read`/`git.test` still replays its exact persisted
|
||||||
|
payload and remains claimable.
|
||||||
|
|
||||||
|
This makes persistent workers safe with the currently deployed Bridge while preserving the same
|
||||||
|
Task/Job binding. No Task DB rewrite, cancellation, replacement Task, fake NEEDS_REWORK, or
|
||||||
|
pre-claim metadata mutation is required.
|
||||||
|
|
||||||
|
A later Bridge cleanup may implement true binding-only projection, but Worker Availability does not
|
||||||
|
depend on that cleanup.
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(ROOT / "active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_capability_contract import ( # noqa: E402
|
||||||
|
has_capabilities,
|
||||||
|
normalize_required_capabilities,
|
||||||
|
normalize_worker_capabilities,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("runtime", ["hermes", "openclaw"])
|
||||||
|
def test_runtime_capability_is_known_to_task_and_worker_vocabulary(runtime: str) -> None:
|
||||||
|
capability = f"agent.runtime.{runtime}"
|
||||||
|
assert normalize_required_capabilities([capability]) == frozenset({capability})
|
||||||
|
assert normalize_worker_capabilities([capability]) == frozenset({capability})
|
||||||
|
assert has_capabilities([capability], [capability]) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_capabilities_are_isolated_between_workers() -> None:
|
||||||
|
assert has_capabilities(["agent.runtime.hermes"], ["agent.runtime.openclaw"]) is False
|
||||||
|
assert has_capabilities(["agent.runtime.openclaw"], ["agent.runtime.hermes"]) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_runtime_capability_remains_fail_closed() -> None:
|
||||||
|
assert has_capabilities(["agent.runtime.hermes"], ["agent.runtime.unknown"]) is False
|
||||||
@@ -0,0 +1,293 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_agent_task_execution_rebind import (
|
||||||
|
ExecutionRebindError,
|
||||||
|
ExecutionRebindRequest,
|
||||||
|
ExecutionRebindStore,
|
||||||
|
)
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_execution_envelope,
|
||||||
|
)
|
||||||
|
|
||||||
|
AUTHORIZATION="owner-execution-rebind-20260818"
|
||||||
|
TASK_ID="task-rebind-a"
|
||||||
|
JOB_ID="job-rebind-a"
|
||||||
|
BASE="a"*40
|
||||||
|
BRANCH="agent/task-rebind-a"
|
||||||
|
IDEMPOTENCY="task-rebind-a-binding-v1"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _auth(monkeypatch):
|
||||||
|
monkeypatch.setenv(
|
||||||
|
"AIOS_AGENT_TASK_EXECUTION_REBIND_AUTHORIZATION_ID",
|
||||||
|
AUTHORIZATION,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def canon(value):
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(value,sort_keys=True,separators=(",",":")).encode()
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def binding():
|
||||||
|
return GovernedExecutionBinding.from_mapping({
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"job_id":JOB_ID,
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,
|
||||||
|
"expected_branch_oid":BASE,
|
||||||
|
"target_branch":BRANCH,
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"generation":1,
|
||||||
|
"idempotency_key":IDEMPOTENCY,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def old_envelope():
|
||||||
|
return project_execution_envelope(
|
||||||
|
binding(),
|
||||||
|
{
|
||||||
|
"action":"source_capture",
|
||||||
|
"adapter":"compact_interface_map",
|
||||||
|
"capability_scope":[
|
||||||
|
"git.read","git.test","governed.write.branch","governed.write.file"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def new_envelope():
|
||||||
|
return project_execution_envelope(
|
||||||
|
binding(),
|
||||||
|
{
|
||||||
|
"action":"deterministic",
|
||||||
|
"adapter":"deterministic",
|
||||||
|
"capability_scope":[
|
||||||
|
"git.read","git.test","governed.write.branch","governed.write.file"
|
||||||
|
],
|
||||||
|
"change_set":[
|
||||||
|
{
|
||||||
|
"path":"active-runtime/app/example.py",
|
||||||
|
"operation":"write_file",
|
||||||
|
"content":"VALUE=1\\n",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"verification":[
|
||||||
|
{
|
||||||
|
"label":"unit",
|
||||||
|
"argv":["/usr/bin/python3","-m","pytest","-q"],
|
||||||
|
"cwd":".",
|
||||||
|
"timeout_seconds":120,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"governance":{
|
||||||
|
"commit_idempotency_key":"task-rebind-a-commit-v1",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def make_db(tmp_path, *, status="QUEUED", revision=5, attempt=2, lease=False):
|
||||||
|
path=tmp_path/"state.sqlite3"
|
||||||
|
con=sqlite3.connect(path)
|
||||||
|
con.executescript("""
|
||||||
|
CREATE TABLE tasks(
|
||||||
|
task_id TEXT PRIMARY KEY,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
attempt INTEGER NOT NULL,
|
||||||
|
max_attempts INTEGER NOT NULL,
|
||||||
|
revision INTEGER NOT NULL,
|
||||||
|
claimed_by TEXT,
|
||||||
|
lease_token_hash TEXT,
|
||||||
|
lease_expires_at TEXT,
|
||||||
|
verification_json TEXT,
|
||||||
|
result_json TEXT,
|
||||||
|
metadata_json TEXT,
|
||||||
|
required_capabilities_json TEXT
|
||||||
|
);
|
||||||
|
CREATE TABLE task_events(
|
||||||
|
event_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
task_id TEXT,
|
||||||
|
event_type TEXT,
|
||||||
|
agent_id TEXT,
|
||||||
|
payload_json TEXT,
|
||||||
|
created_at TEXT
|
||||||
|
);
|
||||||
|
""")
|
||||||
|
verification={"decision":"NEEDS_REWORK","reason":"WRONG_ENVELOPE_BEFORE_PLAN"}
|
||||||
|
old=old_envelope()
|
||||||
|
metadata={
|
||||||
|
"execution":old,
|
||||||
|
"execution_projection":{
|
||||||
|
"schema":"aios.online-agent.task-projection.v1",
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"job_id":JOB_ID,
|
||||||
|
"execution_digest":old["execution_digest"],
|
||||||
|
},
|
||||||
|
"preserved":{"value":"unchanged"},
|
||||||
|
}
|
||||||
|
con.execute(
|
||||||
|
"INSERT INTO tasks VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||||
|
(
|
||||||
|
TASK_ID,status,attempt,3,revision,
|
||||||
|
"previous-worker" if not lease else "active-worker",
|
||||||
|
"lease-hash" if lease else None,
|
||||||
|
"2099-01-01T00:00:00Z" if lease else None,
|
||||||
|
json.dumps(verification),
|
||||||
|
json.dumps({"status":"SUCCEEDED"}),
|
||||||
|
json.dumps(metadata),
|
||||||
|
json.dumps([
|
||||||
|
"git.read","git.test","governed.write.branch","governed.write.file"
|
||||||
|
]),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
con.commit();con.close()
|
||||||
|
return path,verification
|
||||||
|
|
||||||
|
|
||||||
|
def request(verification, *, revision=5, old=None, new=None, key="rebind-task-a-v1"):
|
||||||
|
old=old or old_envelope()
|
||||||
|
new=new or new_envelope()
|
||||||
|
return ExecutionRebindRequest.from_mapping({
|
||||||
|
"expected_status":"QUEUED",
|
||||||
|
"expected_revision":revision,
|
||||||
|
"expected_attempt":2,
|
||||||
|
"previous_verification_sha256":canon(verification),
|
||||||
|
"expected_old_execution_digest":old["execution_digest"],
|
||||||
|
"expected_new_execution_digest":new["execution_digest"],
|
||||||
|
"authorization_id":AUTHORIZATION,
|
||||||
|
"idempotency_key":key,
|
||||||
|
"reason":"projection-before-plan race recovery",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def resolver(_task_id,_caps):
|
||||||
|
return new_envelope()
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_cas_updates_only_execution_projection_and_audits(tmp_path):
|
||||||
|
path,verification=make_db(tmp_path)
|
||||||
|
result=ExecutionRebindStore(path,resolver).rebind(
|
||||||
|
TASK_ID,request(verification)
|
||||||
|
)
|
||||||
|
assert result["status"]=="QUEUED"
|
||||||
|
assert result["revision"]==6
|
||||||
|
assert result["idempotent"] is False
|
||||||
|
|
||||||
|
con=sqlite3.connect(path)
|
||||||
|
row=con.execute(
|
||||||
|
"SELECT status,attempt,revision,claimed_by,metadata_json FROM tasks"
|
||||||
|
).fetchone()
|
||||||
|
assert row[:4]==("QUEUED",2,6,None)
|
||||||
|
metadata=json.loads(row[4])
|
||||||
|
assert metadata["preserved"]=={"value":"unchanged"}
|
||||||
|
assert metadata["execution"]["action"]=="deterministic"
|
||||||
|
assert metadata["execution"]["execution_digest"]==new_envelope()["execution_digest"]
|
||||||
|
assert metadata["execution_projection"]["execution_digest"]==new_envelope()["execution_digest"]
|
||||||
|
|
||||||
|
event=con.execute(
|
||||||
|
"SELECT event_type,payload_json FROM task_events"
|
||||||
|
).fetchone()
|
||||||
|
assert event[0]=="TASK_EXECUTION_REBIND_AUTHORIZED"
|
||||||
|
payload=json.loads(event[1])
|
||||||
|
assert payload["old_action"]=="source_capture"
|
||||||
|
assert payload["new_action"]=="deterministic"
|
||||||
|
assert payload["stale_claim_normalized"] is True
|
||||||
|
assert con.execute("SELECT COUNT(*) FROM task_execution_rebinds").fetchone()[0]==1
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_idempotent_replay(tmp_path):
|
||||||
|
path,verification=make_db(tmp_path)
|
||||||
|
store=ExecutionRebindStore(path,resolver)
|
||||||
|
req=request(verification)
|
||||||
|
first=store.rebind(TASK_ID,req)
|
||||||
|
second=store.rebind(TASK_ID,req)
|
||||||
|
assert first["idempotent"] is False
|
||||||
|
assert second["idempotent"] is True
|
||||||
|
assert second["new_execution_digest"]==first["new_execution_digest"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_rejects_active_lease(tmp_path):
|
||||||
|
path,verification=make_db(tmp_path,lease=True)
|
||||||
|
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_ACTIVE_LEASE_DENY"):
|
||||||
|
ExecutionRebindStore(path,resolver).rebind(TASK_ID,request(verification))
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_rejects_verification_drift(tmp_path):
|
||||||
|
path,_=make_db(tmp_path)
|
||||||
|
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_VERIFICATION_DRIFT"):
|
||||||
|
ExecutionRebindStore(path,resolver).rebind(
|
||||||
|
TASK_ID,request({"decision":"NEEDS_REWORK","reason":"other"})
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_rejects_nonqueued_task(tmp_path):
|
||||||
|
path,verification=make_db(tmp_path,status="VERIFYING")
|
||||||
|
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_CAS_MISMATCH"):
|
||||||
|
ExecutionRebindStore(path,resolver).rebind(TASK_ID,request(verification))
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_rejects_non_deterministic_canonical_resolution(tmp_path):
|
||||||
|
path,verification=make_db(tmp_path)
|
||||||
|
def wrong(_task,_caps):
|
||||||
|
return old_envelope()
|
||||||
|
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_DETERMINISTIC_PLAN_REQUIRED"):
|
||||||
|
ExecutionRebindStore(path,wrong).rebind(TASK_ID,request(verification))
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_rejects_binding_drift(tmp_path):
|
||||||
|
path,verification=make_db(tmp_path)
|
||||||
|
different=project_execution_envelope(
|
||||||
|
GovernedExecutionBinding.from_mapping({
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"job_id":"job-other",
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,
|
||||||
|
"expected_branch_oid":BASE,
|
||||||
|
"target_branch":BRANCH,
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"generation":1,
|
||||||
|
"idempotency_key":IDEMPOTENCY,
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
"action":"deterministic",
|
||||||
|
"capability_scope":["git.read"],
|
||||||
|
"change_set":[{"path":"x.py","operation":"write_file","content":"x=1\\n"}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
def wrong(_task,_caps):
|
||||||
|
return different
|
||||||
|
with pytest.raises(Exception):
|
||||||
|
ExecutionRebindStore(path,wrong).rebind(TASK_ID,request(verification))
|
||||||
|
|
||||||
|
|
||||||
|
def test_rebind_request_is_narrow_and_digest_bound():
|
||||||
|
verification={"decision":"NEEDS_REWORK"}
|
||||||
|
data=request(verification).public()
|
||||||
|
data["arbitrary_metadata"]={"x":1}
|
||||||
|
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_FIELDS_INVALID"):
|
||||||
|
ExecutionRebindRequest.from_mapping(data)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_rebind_unit_is_unprivileged_and_owner_credential_bound():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-agent-task-execution-rebind.service").read_text()
|
||||||
|
assert "User=aios-agent-task" in unit
|
||||||
|
assert "Group=aios-agent-task" in unit
|
||||||
|
assert "LoadCredential=agent_task_control_token:" in unit
|
||||||
|
assert "Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892" in unit
|
||||||
|
assert "RestrictAddressFamilies=AF_UNIX AF_INET" in unit
|
||||||
|
assert "User=root" not in unit
|
||||||
@@ -0,0 +1,232 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_online_agent_worker as worker
|
||||||
|
from aios_external_agent_runtime import (
|
||||||
|
ExternalAgentExchange,
|
||||||
|
ExternalAgentRuntimeError,
|
||||||
|
validate_request,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def envelope():
|
||||||
|
value=object.__new__(worker.ExecutionEnvelope)
|
||||||
|
object.__setattr__(value,"task_id","task-attempt-identity")
|
||||||
|
object.__setattr__(value,"job_id","job-attempt-identity")
|
||||||
|
object.__setattr__(value,"digest","d"*64)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_request_id_is_stable_within_attempt_and_changes_across_attempts():
|
||||||
|
env=envelope()
|
||||||
|
a1=worker.external_agent_request_id(env,"hermes",6)
|
||||||
|
a1_replay=worker.external_agent_request_id(env,"hermes",6)
|
||||||
|
a2=worker.external_agent_request_id(env,"hermes",7)
|
||||||
|
assert a1==a1_replay
|
||||||
|
assert a1!=a2
|
||||||
|
assert a1.startswith("external-hermes-")
|
||||||
|
assert a2.startswith("external-hermes-")
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_request_id_rejects_invalid_attempt():
|
||||||
|
env=envelope()
|
||||||
|
with pytest.raises(worker.WorkerError,match="TASK_ATTEMPT_INVALID"):
|
||||||
|
worker.external_agent_request_id(env,"hermes",0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_request_schema_binds_attempt():
|
||||||
|
value=validate_request({
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":"external-hermes-0123456789abcdef",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"task_id":"task-attempt-identity",
|
||||||
|
"attempt":7,
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":{
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
assert value["attempt"]==7
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_request_schema_rejects_missing_attempt():
|
||||||
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_ATTEMPT_INVALID"):
|
||||||
|
validate_request({
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":"external-hermes-0123456789abcdef",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"task_id":"task-attempt-identity",
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":{
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def _write_bound_result(path, request):
|
||||||
|
import hashlib
|
||||||
|
from aios_external_agent_runtime import canonical, sha256
|
||||||
|
material={
|
||||||
|
"schema":"aios.external-agent-runtime.result.v1",
|
||||||
|
"request_id":request["request_id"],
|
||||||
|
"request_sha256":sha256(canonical(request)),
|
||||||
|
"runtime":request["runtime"],
|
||||||
|
"task_id":request["task_id"],
|
||||||
|
"attempt":request["attempt"],
|
||||||
|
"execution_digest":request["execution_digest"],
|
||||||
|
"exit_code":0,
|
||||||
|
"duration_ms":1,
|
||||||
|
"output_text":"ok",
|
||||||
|
"output_sha256":hashlib.sha256(b"ok").hexdigest(),
|
||||||
|
"stderr_sha256":hashlib.sha256(b"").hexdigest(),
|
||||||
|
"credential_material_exposed":False,
|
||||||
|
"control_plane_access_granted":False,
|
||||||
|
}
|
||||||
|
material["result_sha256"]=sha256(canonical(material))
|
||||||
|
path.write_text(json.dumps(material,sort_keys=True),encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_exchange_result_binding_rejects_wrong_attempt(tmp_path):
|
||||||
|
exchange=ExternalAgentExchange(tmp_path)
|
||||||
|
request={
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":"external-hermes-0123456789abcdef",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"task_id":"task-attempt-identity",
|
||||||
|
"attempt":7,
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":{
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
request=validate_request(request)
|
||||||
|
result_path=tmp_path/"result.json"
|
||||||
|
_write_bound_result(result_path,request)
|
||||||
|
value=json.loads(result_path.read_text())
|
||||||
|
value["attempt"]=6
|
||||||
|
# Recompute digest so the binding check, not the digest check, is what fails.
|
||||||
|
from aios_external_agent_runtime import canonical,sha256
|
||||||
|
material=dict(value); material.pop("result_sha256")
|
||||||
|
value["result_sha256"]=sha256(canonical(material))
|
||||||
|
result_path.write_text(json.dumps(value),encoding="utf-8")
|
||||||
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_RESULT_BINDING_INVALID"):
|
||||||
|
exchange._validated_result(
|
||||||
|
result_path,
|
||||||
|
request,
|
||||||
|
sha256(canonical(request)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_governance_request_id_is_attempt_scoped():
|
||||||
|
env=envelope()
|
||||||
|
attempt6=worker.external_agent_request_id(env,"hermes",6)+":model"
|
||||||
|
attempt7=worker.external_agent_request_id(env,"hermes",7)+":model"
|
||||||
|
assert attempt6!=attempt7
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_previous_attempt_does_not_poison_next_model_governance_reservation(tmp_path):
|
||||||
|
from aios_model_governance import (
|
||||||
|
ModelGovernanceStore,
|
||||||
|
ModelRecord,
|
||||||
|
ProviderRecord,
|
||||||
|
RouteRequest,
|
||||||
|
ScopePolicy,
|
||||||
|
)
|
||||||
|
|
||||||
|
store=ModelGovernanceStore(tmp_path/"model-governance.sqlite3",now_fn=lambda:1_800_000_000.0)
|
||||||
|
store.register_provider(
|
||||||
|
ProviderRecord(
|
||||||
|
"oneapi",
|
||||||
|
"runtime-profile:hermes",
|
||||||
|
max_failures=5,
|
||||||
|
cooldown_seconds=60,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"deepseek-v4-flash",
|
||||||
|
"oneapi",
|
||||||
|
"v1",
|
||||||
|
("agent.runtime.hermes","text"),
|
||||||
|
input_cost_micros_per_1k=1,
|
||||||
|
output_cost_micros_per_1k=1,
|
||||||
|
max_input_tokens=10000,
|
||||||
|
max_output_tokens=1000,
|
||||||
|
evaluation_sha256="a"*64,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy(
|
||||||
|
"tenant",
|
||||||
|
"model-governance-live",
|
||||||
|
"staging",
|
||||||
|
allowed_models=("deepseek-v4-flash",),
|
||||||
|
required_capabilities=("text",),
|
||||||
|
daily_token_budget=100000,
|
||||||
|
daily_cost_budget_micros=100000,
|
||||||
|
fallback_limit=1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def route(request_id):
|
||||||
|
return store.route(
|
||||||
|
RouteRequest(
|
||||||
|
request_id=request_id,
|
||||||
|
task_id="r42-model-governance-hermes-live-20260818",
|
||||||
|
trace_id="trace-hermes-model-governance-live",
|
||||||
|
scope_type="tenant",
|
||||||
|
tenant_id="model-governance-live",
|
||||||
|
project_id="staging",
|
||||||
|
required_capabilities=("agent.runtime.hermes",),
|
||||||
|
estimated_input_tokens=100,
|
||||||
|
max_output_tokens=50,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
attempt6=route("external-hermes-attempt6:model")
|
||||||
|
store.record_failure(attempt6.request_id,"provider_process_failed")
|
||||||
|
row6=store.conn.execute(
|
||||||
|
"select state from model_governance_reservations where request_id=?",
|
||||||
|
(attempt6.request_id,),
|
||||||
|
).fetchone()
|
||||||
|
assert row6["state"]=="FAILED"
|
||||||
|
|
||||||
|
attempt7=route("external-hermes-attempt7:model")
|
||||||
|
assert attempt7.request_id!=attempt6.request_id
|
||||||
|
row7=store.conn.execute(
|
||||||
|
"select state from model_governance_reservations where request_id=?",
|
||||||
|
(attempt7.request_id,),
|
||||||
|
).fetchone()
|
||||||
|
assert row7["state"]=="RESERVED"
|
||||||
|
|
||||||
|
settled=store.record_success(
|
||||||
|
attempt7.request_id,
|
||||||
|
input_tokens=90,
|
||||||
|
output_tokens=40,
|
||||||
|
usage_source="runtime",
|
||||||
|
)
|
||||||
|
assert settled["state"]=="SETTLED"
|
||||||
|
assert store.conn.execute(
|
||||||
|
"select state from model_governance_reservations where request_id=?",
|
||||||
|
(attempt6.request_id,),
|
||||||
|
).fetchone()["state"]=="FAILED"
|
||||||
|
assert store.verify_audit_chain() is True
|
||||||
|
store.close()
|
||||||
@@ -0,0 +1,421 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
from aios_external_agent_runtime import (
|
||||||
|
ExternalAgentRuntimeError,
|
||||||
|
RuntimeProfile,
|
||||||
|
execute_request,
|
||||||
|
)
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
ExecutionProjectionError,
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_online_task,
|
||||||
|
validate_execution_envelope,
|
||||||
|
)
|
||||||
|
import aios_online_agent_worker as worker
|
||||||
|
|
||||||
|
|
||||||
|
BASE="a"*40
|
||||||
|
|
||||||
|
|
||||||
|
def binding(task_id="task-hermes-runtime"):
|
||||||
|
return GovernedExecutionBinding.from_mapping({
|
||||||
|
"task_id":task_id,
|
||||||
|
"job_id":"job-"+task_id,
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,
|
||||||
|
"expected_branch_oid":BASE,
|
||||||
|
"target_branch":"agent/"+task_id,
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"generation":1,
|
||||||
|
"idempotency_key":task_id+"-binding-v1",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def task_request(task_id, runtime):
|
||||||
|
return {
|
||||||
|
"task_id":task_id,
|
||||||
|
"goal":"Summarize the requested operation.",
|
||||||
|
"instructions":"Return a concise result.",
|
||||||
|
"success_criteria":[{"id":"done","description":"A result is returned."}],
|
||||||
|
"constraints":{"network":"runtime-managed"},
|
||||||
|
"required_capabilities":[f"agent.runtime.{runtime}"],
|
||||||
|
"max_attempts":3,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def projected(task_id="task-hermes-runtime", runtime="hermes"):
|
||||||
|
return project_online_task(
|
||||||
|
binding(task_id),
|
||||||
|
task_request(task_id,runtime),
|
||||||
|
{
|
||||||
|
"action":"agent_runtime",
|
||||||
|
"adapter":runtime,
|
||||||
|
"capability_scope":[f"agent.runtime.{runtime}"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def clear_secret_env(monkeypatch):
|
||||||
|
for key in list(os.environ):
|
||||||
|
upper=key.upper()
|
||||||
|
if any(marker in upper for marker in (
|
||||||
|
"TOKEN","SECRET","PASSWORD","CREDENTIAL","AUTHORIZATION","PRIVATE_KEY"
|
||||||
|
)):
|
||||||
|
monkeypatch.delenv(key,raising=False)
|
||||||
|
monkeypatch.setenv("CREDENTIALS_DIRECTORY","/definitely-not-present-aios-test")
|
||||||
|
|
||||||
|
|
||||||
|
def make_helper(tmp_path):
|
||||||
|
helper=tmp_path/"runtime_helper.py"
|
||||||
|
helper.write_text(
|
||||||
|
"import json,sys\n"
|
||||||
|
"value=json.load(sys.stdin)\n"
|
||||||
|
"print(json.dumps({'answer':'ok','goal':value['goal']},sort_keys=True))\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return helper
|
||||||
|
|
||||||
|
|
||||||
|
def make_profile(tmp_path,runtime,helper):
|
||||||
|
root=tmp_path/"profiles";root.mkdir(exist_ok=True)
|
||||||
|
(root/f"{runtime}.json").write_text(
|
||||||
|
json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":runtime,
|
||||||
|
"argv":["/usr/bin/python3",str(helper)],
|
||||||
|
"input_mode":"stdin_json",
|
||||||
|
"output_mode":"stdout_json",
|
||||||
|
"timeout_seconds":30,
|
||||||
|
"max_output_bytes":65536,
|
||||||
|
}),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def make_text_request(tmp_path, runtime, payload=None):
|
||||||
|
request={
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":f"external-{runtime}-0123456789abcdef",
|
||||||
|
"runtime":runtime,
|
||||||
|
"task_id":f"task-{runtime}-runtime",
|
||||||
|
"attempt":1,
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":payload or {
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"No side effects.",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
request_path=tmp_path/"request.json"
|
||||||
|
result_path=tmp_path/"result.json"
|
||||||
|
request_path.write_text(json.dumps(request),encoding="utf-8")
|
||||||
|
return request_path, result_path
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_projection_is_read_only_and_task_bound():
|
||||||
|
value=projected()
|
||||||
|
execution=value["metadata"]["execution"]
|
||||||
|
assert execution["action"]=="agent_runtime"
|
||||||
|
assert execution["adapter"]=="hermes"
|
||||||
|
assert execution["runtime_input"]["goal"]==value["goal"]
|
||||||
|
assert execution["runtime_input"]["instructions"]==value["instructions"]
|
||||||
|
assert execution["governance"]["writer_lease"] is False
|
||||||
|
assert execution["change_set"]==[]
|
||||||
|
assert execution["verification"]==[]
|
||||||
|
validate_execution_envelope(
|
||||||
|
execution,
|
||||||
|
binding=binding(),
|
||||||
|
worker_capabilities=["agent.runtime.hermes"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_projection_requires_runtime_capability():
|
||||||
|
with pytest.raises(ExecutionProjectionError,match="AGENT_RUNTIME_CAPABILITY_REQUIRED"):
|
||||||
|
project_online_task(
|
||||||
|
binding(),
|
||||||
|
task_request("task-hermes-runtime","hermes"),
|
||||||
|
{
|
||||||
|
"action":"agent_runtime",
|
||||||
|
"adapter":"hermes",
|
||||||
|
"capability_scope":["git.read"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_projection_rejects_mutation():
|
||||||
|
with pytest.raises(ExecutionProjectionError,match="AGENT_RUNTIME_MUTATION_DENY"):
|
||||||
|
project_online_task(
|
||||||
|
binding(),
|
||||||
|
task_request("task-hermes-runtime","hermes"),
|
||||||
|
{
|
||||||
|
"action":"agent_runtime",
|
||||||
|
"adapter":"hermes",
|
||||||
|
"capability_scope":["agent.runtime.hermes"],
|
||||||
|
"change_set":[{"path":"x.py","operation":"write_file","content":"x=1\n"}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_projection_rejects_secret_constraint_keys():
|
||||||
|
request=task_request("task-hermes-runtime","hermes")
|
||||||
|
request["constraints"]={"api_token":"do-not-project"}
|
||||||
|
with pytest.raises(ExecutionProjectionError,match="AGENT_RUNTIME_SECRET_FIELD_DENY"):
|
||||||
|
project_online_task(
|
||||||
|
binding(),
|
||||||
|
request,
|
||||||
|
{
|
||||||
|
"action":"agent_runtime",
|
||||||
|
"adapter":"hermes",
|
||||||
|
"capability_scope":["agent.runtime.hermes"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("runtime",["hermes","openclaw"])
|
||||||
|
def test_external_executor_runs_root_owned_profile_without_shell(tmp_path,monkeypatch,runtime):
|
||||||
|
clear_secret_env(monkeypatch)
|
||||||
|
helper=make_helper(tmp_path)
|
||||||
|
profile_root=make_profile(tmp_path,runtime,helper)
|
||||||
|
request={
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":f"external-{runtime}-0123456789abcdef",
|
||||||
|
"runtime":runtime,
|
||||||
|
"task_id":f"task-{runtime}-runtime",
|
||||||
|
"attempt":1,
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":{
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"No side effects.",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
request_path=tmp_path/"request.json"
|
||||||
|
result_path=tmp_path/"result.json"
|
||||||
|
request_path.write_text(json.dumps(request),encoding="utf-8")
|
||||||
|
execute_request(request_path,result_path,profile_root=profile_root)
|
||||||
|
result=json.loads(result_path.read_text())
|
||||||
|
assert result["runtime"]==runtime
|
||||||
|
assert result["exit_code"]==0
|
||||||
|
assert result["output_json"]["answer"]=="ok"
|
||||||
|
assert result["credential_material_exposed"] is False
|
||||||
|
assert result["control_plane_access_granted"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_profile_rejects_shell_mode(tmp_path):
|
||||||
|
root=tmp_path/"profiles";root.mkdir()
|
||||||
|
(root/"hermes.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3","-c","print('x')"],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
}))
|
||||||
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_SHELL_MODE_DENY"):
|
||||||
|
RuntimeProfile.load("hermes",root)
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_executor_projects_only_selected_provider_credential(tmp_path,monkeypatch):
|
||||||
|
clear_secret_env(monkeypatch)
|
||||||
|
helper=tmp_path/"credential_helper.py"
|
||||||
|
helper.write_text(
|
||||||
|
"import os\n"
|
||||||
|
"value=os.environ.get('OPENAI_API_KEY','')\n"
|
||||||
|
"print('AIOS_PROVIDER_PROJECTED_OK' if value == 'test-provider-key' else 'AIOS_PROVIDER_PROJECTED_FAIL')\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
profile_root=tmp_path/"profiles";profile_root.mkdir()
|
||||||
|
(profile_root/"hermes.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3",str(helper)],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
"environment":{"RUNTIME_PROFILE_MARKER":"staging"},
|
||||||
|
"credential_env":{
|
||||||
|
"OPENAI_API_KEY":{
|
||||||
|
"credential":"provider.env",
|
||||||
|
"format":"dotenv",
|
||||||
|
"key":"ONEAPI_REGULAR_TOKEN",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}),encoding="utf-8")
|
||||||
|
credential_dir=tmp_path/"credentials";credential_dir.mkdir()
|
||||||
|
(credential_dir/"provider.env").write_text(
|
||||||
|
"ONEAPI_REGULAR_TOKEN=test-provider-key\nOTHER_PROVIDER_TOKEN=not-projected\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
monkeypatch.setenv("CREDENTIALS_DIRECTORY",str(credential_dir))
|
||||||
|
request_path,result_path=make_text_request(tmp_path,"hermes")
|
||||||
|
execute_request(request_path,result_path,profile_root=profile_root)
|
||||||
|
result=json.loads(result_path.read_text())
|
||||||
|
assert "AIOS_PROVIDER_PROJECTED_OK" in result["output_text"]
|
||||||
|
assert "test-provider-key" not in result["output_text"]
|
||||||
|
assert result["credential_material_exposed"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_profile_rejects_credential_path_traversal(tmp_path):
|
||||||
|
root=tmp_path/"profiles";root.mkdir()
|
||||||
|
(root/"hermes.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3","-V"],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
"credential_env":{
|
||||||
|
"OPENAI_API_KEY":{
|
||||||
|
"credential":"../provider.env",
|
||||||
|
"format":"dotenv",
|
||||||
|
"key":"ONEAPI_REGULAR_TOKEN",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}),encoding="utf-8")
|
||||||
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_CREDENTIAL_NAME_INVALID"):
|
||||||
|
RuntimeProfile.load("hermes",root)
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_profile_prompt_placeholder_is_root_owned(tmp_path,monkeypatch):
|
||||||
|
clear_secret_env(monkeypatch)
|
||||||
|
helper=tmp_path/"prompt_helper.py"
|
||||||
|
helper.write_text(
|
||||||
|
"import sys\n"
|
||||||
|
"print('AIOS_PROMPT_PLACEHOLDER_OK' if 'Goal:' in sys.argv[1] else 'AIOS_PROMPT_PLACEHOLDER_FAIL')\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
profile_root=tmp_path/"profiles";profile_root.mkdir()
|
||||||
|
(profile_root/"hermes.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3",str(helper),"{prompt}"],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
}),encoding="utf-8")
|
||||||
|
request_path,result_path=make_text_request(tmp_path,"hermes")
|
||||||
|
execute_request(request_path,result_path,profile_root=profile_root)
|
||||||
|
result=json.loads(result_path.read_text())
|
||||||
|
assert "AIOS_PROMPT_PLACEHOLDER_OK" in result["output_text"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("runtime,capability",[
|
||||||
|
("hermes","agent.runtime.hermes"),
|
||||||
|
("openclaw","agent.runtime.openclaw"),
|
||||||
|
])
|
||||||
|
def test_worker_external_adapter_builds_readonly_result(monkeypatch,tmp_path,runtime,capability):
|
||||||
|
mapped=projected(f"task-{runtime}-runtime",runtime)
|
||||||
|
envelope=worker.ExecutionEnvelope.from_task(
|
||||||
|
mapped,
|
||||||
|
worker_capabilities=(capability,),
|
||||||
|
)
|
||||||
|
class FakeExchange:
|
||||||
|
def __init__(self,*args,**kwargs): pass
|
||||||
|
def execute(self,**kwargs):
|
||||||
|
return {
|
||||||
|
"schema":"aios.external-agent-runtime.result.v1",
|
||||||
|
"request_id":kwargs["request_id"],
|
||||||
|
"request_sha256":"e"*64,
|
||||||
|
"runtime":runtime,
|
||||||
|
"task_id":kwargs["task_id"],
|
||||||
|
"attempt":kwargs["attempt"],
|
||||||
|
"execution_digest":kwargs["execution_digest"],
|
||||||
|
"exit_code":0,
|
||||||
|
"duration_ms":10,
|
||||||
|
"output_text":"done",
|
||||||
|
"output_sha256":"f"*64,
|
||||||
|
"stderr_sha256":"0"*64,
|
||||||
|
"credential_material_exposed":False,
|
||||||
|
"control_plane_access_granted":False,
|
||||||
|
"result_sha256":"1"*64,
|
||||||
|
}
|
||||||
|
monkeypatch.setattr(worker,"ExternalAgentExchange",FakeExchange)
|
||||||
|
monkeypatch.setattr(worker,"audit",lambda *a,**k:None)
|
||||||
|
executor=worker.HermesExecutor() if runtime=="hermes" else worker.OpenClawExecutor()
|
||||||
|
execution=executor.execute(envelope,tmp_path,attempt=1)
|
||||||
|
result=worker.build_agent_runtime_result(envelope.task_id,envelope,execution)
|
||||||
|
assert result["status"]=="SUCCEEDED"
|
||||||
|
assert result["classification"]=="EXTERNAL_AGENT_RUNTIME_SUCCEEDED"
|
||||||
|
assert result["execution_mode"]=="agent_runtime"
|
||||||
|
assert result["agent_runtime"]==runtime
|
||||||
|
assert result["writer_lease"]=="NONE"
|
||||||
|
assert result["changed_files"]==[]
|
||||||
|
assert result["control_plane_access_granted_to_runtime"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_adapter_filter_defaults_exclude_external_runtimes():
|
||||||
|
assert "hermes" not in worker.WORKER_ALLOWED_ADAPTERS
|
||||||
|
assert "openclaw" not in worker.WORKER_ALLOWED_ADAPTERS
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_executor_systemd_has_no_control_credentials_or_repository_access():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-external-agent-executor.service").read_text()
|
||||||
|
assert "User=aios-engineering-executor" in unit
|
||||||
|
assert "LoadCredential=aios-external-agent-oneapi-env:/data/agent/secrets/oneapi-model-ready.env" in unit
|
||||||
|
assert "LoadCredentialEncrypted" not in unit
|
||||||
|
assert "InaccessiblePaths=/run/credentials" not in unit
|
||||||
|
assert "InaccessiblePaths=/etc/aios-gitops" in unit
|
||||||
|
assert "InaccessiblePaths=/data/aios-gitops" in unit
|
||||||
|
assert "InaccessiblePaths=/data/agent/secrets" in unit
|
||||||
|
assert "IPAddressDeny=127.0.0.0/8" in unit
|
||||||
|
assert "IPAddressDeny=::1/128" in unit
|
||||||
|
assert "IPAddressAllow=172.18.0.2/32" in unit
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"unit_name,capability,adapter,state_root",
|
||||||
|
[
|
||||||
|
(
|
||||||
|
"aios-hermes-online-agent-worker.service",
|
||||||
|
"agent.runtime.hermes",
|
||||||
|
"hermes",
|
||||||
|
"/var/lib/aios-online-agent-hermes",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"aios-openclaw-online-agent-worker.service",
|
||||||
|
"agent.runtime.openclaw",
|
||||||
|
"openclaw",
|
||||||
|
"/var/lib/aios-online-agent-openclaw",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_dedicated_worker_units_are_capability_and_adapter_scoped(
|
||||||
|
unit_name,capability,adapter,state_root
|
||||||
|
):
|
||||||
|
unit=(ROOT/"deploy/systemd"/unit_name).read_text()
|
||||||
|
assert f"Environment=AIOS_WORKER_CAPABILITIES={capability}" in unit
|
||||||
|
assert f"Environment=AIOS_WORKER_ALLOWED_ADAPTERS={adapter}" in unit
|
||||||
|
assert f"Environment=AIOS_WORKER_STATE_ROOT={state_root}" in unit
|
||||||
|
assert "AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892" in unit
|
||||||
|
assert "LoadCredentialEncrypted=agent_token:" in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_example_profiles_are_non_authoritative_examples_only():
|
||||||
|
for runtime in ("hermes","openclaw"):
|
||||||
|
value=json.loads(
|
||||||
|
(ROOT/"config/external-agent-runtime.d"/f"{runtime}.example.json").read_text()
|
||||||
|
)
|
||||||
|
assert value["schema"]=="aios.external-agent-runtime.profile.v1"
|
||||||
|
assert value["runtime"]==runtime
|
||||||
|
assert value["input_mode"]=="stdin_text"
|
||||||
|
assert value["output_mode"]=="stdout_text"
|
||||||
|
|
||||||
|
|
||||||
|
def test_staging_profiles_are_provider_only_and_root_profile_bound():
|
||||||
|
hermes=json.loads((ROOT/"config/external-agent-runtime.d/hermes.profile.json").read_text())
|
||||||
|
openclaw=json.loads((ROOT/"config/external-agent-runtime.d/openclaw.profile.json").read_text())
|
||||||
|
for runtime,value in (("hermes",hermes),("openclaw",openclaw)):
|
||||||
|
assert value["runtime"]==runtime
|
||||||
|
assert set(value["credential_env"])=={"OPENAI_API_KEY"}
|
||||||
|
assert value["credential_env"]["OPENAI_API_KEY"]["credential"]=="aios-external-agent-oneapi-env"
|
||||||
|
assert "OPENAI_API_KEY" not in value.get("environment",{})
|
||||||
@@ -0,0 +1,325 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_model_governance import (
|
||||||
|
APPROVED,
|
||||||
|
ModelGovernanceError,
|
||||||
|
ModelGovernanceStore,
|
||||||
|
ModelRecord,
|
||||||
|
ProviderRecord,
|
||||||
|
RouteRequest,
|
||||||
|
ScopePolicy,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def setup_store(tmp_path, now=1_800_000_000.0):
|
||||||
|
store=ModelGovernanceStore(tmp_path/"model-governance.sqlite3",now_fn=lambda:now)
|
||||||
|
store.register_provider(
|
||||||
|
ProviderRecord(
|
||||||
|
"provider-a",
|
||||||
|
"runtime-profile:provider-a",
|
||||||
|
max_failures=1,
|
||||||
|
cooldown_seconds=120,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_provider(
|
||||||
|
ProviderRecord(
|
||||||
|
"provider-b",
|
||||||
|
"runtime-profile:provider-b",
|
||||||
|
max_failures=2,
|
||||||
|
cooldown_seconds=60,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"model-a",
|
||||||
|
"provider-a",
|
||||||
|
"2026-08-18",
|
||||||
|
("agent.runtime.hermes","text"),
|
||||||
|
input_cost_micros_per_1k=100,
|
||||||
|
output_cost_micros_per_1k=200,
|
||||||
|
max_input_tokens=10000,
|
||||||
|
max_output_tokens=4000,
|
||||||
|
evaluation_state=APPROVED,
|
||||||
|
evaluation_sha256="a"*64,
|
||||||
|
priority=10,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"model-b",
|
||||||
|
"provider-b",
|
||||||
|
"2026-08-17",
|
||||||
|
("agent.runtime.hermes","text"),
|
||||||
|
input_cost_micros_per_1k=120,
|
||||||
|
output_cost_micros_per_1k=250,
|
||||||
|
max_input_tokens=10000,
|
||||||
|
max_output_tokens=4000,
|
||||||
|
evaluation_state=APPROVED,
|
||||||
|
evaluation_sha256="b"*64,
|
||||||
|
priority=20,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy(
|
||||||
|
"tenant",
|
||||||
|
"tenant-a",
|
||||||
|
"project-a",
|
||||||
|
allowed_models=("model-a","model-b"),
|
||||||
|
required_capabilities=("text",),
|
||||||
|
max_input_tokens_per_request=5000,
|
||||||
|
max_output_tokens_per_request=2000,
|
||||||
|
daily_token_budget=12000,
|
||||||
|
daily_cost_budget_micros=4000,
|
||||||
|
fallback_limit=2,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return store
|
||||||
|
|
||||||
|
|
||||||
|
def req(request_id="request-001", *, exclude=()):
|
||||||
|
return RouteRequest(
|
||||||
|
request_id=request_id,
|
||||||
|
task_id="task-a",
|
||||||
|
trace_id="trace-a",
|
||||||
|
scope_type="tenant",
|
||||||
|
tenant_id="tenant-a",
|
||||||
|
project_id="project-a",
|
||||||
|
required_capabilities=("agent.runtime.hermes",),
|
||||||
|
estimated_input_tokens=1000,
|
||||||
|
max_output_tokens=500,
|
||||||
|
preferred_models=(),
|
||||||
|
exclude_providers=exclude,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_registry_rejects_raw_secret(tmp_path):
|
||||||
|
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_CREDENTIAL"):
|
||||||
|
store.register_provider(
|
||||||
|
ProviderRecord("provider-a","Bearer raw-secret-value")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_registration_requires_evaluation_gate(tmp_path):
|
||||||
|
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
|
||||||
|
store.register_provider(ProviderRecord("provider-a"))
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_EVALUATION_GATE_DENY"):
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"model-a","provider-a","v1",("text",),
|
||||||
|
evaluation_state="PENDING",evaluation_sha256="a"*64
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_scope_policy_requires_version_with_pin(tmp_path):
|
||||||
|
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_POLICY_PIN_VERSION_REQUIRED"):
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy("tenant","t","p",pinned_model_id="model-a")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_route_is_deterministic_version_pinned_and_idempotent(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
first=store.route(req())
|
||||||
|
second=store.route(req())
|
||||||
|
assert first==second
|
||||||
|
assert first.model_id=="model-a"
|
||||||
|
assert first.model_version=="2026-08-18"
|
||||||
|
assert first.provider_credential_ref=="runtime-profile:provider-a"
|
||||||
|
assert first.policy_revision==1
|
||||||
|
assert first.model_revision==1
|
||||||
|
snapshot=store.usage_snapshot("tenant","tenant-a","project-a")
|
||||||
|
assert snapshot["reserved_input_tokens"]==1000
|
||||||
|
assert snapshot["reserved_output_tokens"]==500
|
||||||
|
assert snapshot["reserved_cost_micros"]>0
|
||||||
|
|
||||||
|
|
||||||
|
def test_route_idempotency_payload_conflict_fails(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
store.route(req())
|
||||||
|
changed=RouteRequest(
|
||||||
|
**{**req().__dict__,"estimated_input_tokens":2000}
|
||||||
|
)
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_ROUTE_IDEMPOTENCY_CONFLICT"):
|
||||||
|
store.route(changed)
|
||||||
|
|
||||||
|
|
||||||
|
def test_budget_reservation_is_atomic(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy(
|
||||||
|
"tenant","tenant-a","project-a",
|
||||||
|
allowed_models=("model-a",),
|
||||||
|
required_capabilities=("text",),
|
||||||
|
daily_token_budget=1600,
|
||||||
|
daily_cost_budget_micros=10000,
|
||||||
|
),
|
||||||
|
expected_revision=1,
|
||||||
|
)
|
||||||
|
store.route(req("request-1"))
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_BUDGET_DENY"):
|
||||||
|
store.route(req("request-2"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_usage_settlement_is_idempotent_and_releases_reservation(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
decision=store.route(req())
|
||||||
|
first=store.record_success(
|
||||||
|
decision.request_id,
|
||||||
|
input_tokens=900,
|
||||||
|
output_tokens=400,
|
||||||
|
usage_source="runtime",
|
||||||
|
)
|
||||||
|
second=store.record_success(
|
||||||
|
decision.request_id,
|
||||||
|
input_tokens=900,
|
||||||
|
output_tokens=400,
|
||||||
|
usage_source="runtime",
|
||||||
|
)
|
||||||
|
assert first["idempotent"] is False
|
||||||
|
assert second["idempotent"] is True
|
||||||
|
snapshot=store.usage_snapshot("tenant","tenant-a","project-a")
|
||||||
|
assert snapshot["settled_input_tokens"]==900
|
||||||
|
assert snapshot["settled_output_tokens"]==400
|
||||||
|
assert snapshot["reserved_input_tokens"]==0
|
||||||
|
assert snapshot["reserved_output_tokens"]==0
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_runtime_usage_falls_back_to_reserved_upper_bound(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
decision=store.route(req())
|
||||||
|
result=store.record_success(
|
||||||
|
decision.request_id,
|
||||||
|
input_tokens=None,
|
||||||
|
output_tokens=None,
|
||||||
|
)
|
||||||
|
assert result["usage_source"]=="reserved_upper_bound"
|
||||||
|
assert result["input_tokens"]==1000
|
||||||
|
assert result["output_tokens"]==500
|
||||||
|
|
||||||
|
|
||||||
|
def test_usage_cannot_exceed_reserved_budget(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
decision=store.route(req())
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_USAGE_EXCEEDS_RESERVATION"):
|
||||||
|
store.record_success(
|
||||||
|
decision.request_id,
|
||||||
|
input_tokens=1001,
|
||||||
|
output_tokens=500,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_provider_failure_opens_circuit_and_fallback_uses_other_provider(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
original=req()
|
||||||
|
first=store.route(original)
|
||||||
|
assert first.provider_id=="provider-a"
|
||||||
|
failure=store.record_failure(first.request_id,"provider_timeout")
|
||||||
|
assert failure["open_until"]>0
|
||||||
|
fallback=store.fallback(original,first.request_id)
|
||||||
|
assert fallback.provider_id=="provider-b"
|
||||||
|
assert fallback.fallback_index==1
|
||||||
|
|
||||||
|
|
||||||
|
def test_circuit_excludes_provider_for_new_request(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
first=store.route(req("request-first"))
|
||||||
|
store.record_failure(first.request_id,"provider_timeout")
|
||||||
|
next_decision=store.route(req("request-next"))
|
||||||
|
assert next_decision.provider_id=="provider-b"
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_version_pin_is_enforced(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy(
|
||||||
|
"tenant","tenant-a","project-a",
|
||||||
|
allowed_models=("model-a","model-b"),
|
||||||
|
pinned_model_id="model-b",
|
||||||
|
pinned_model_version="2026-08-17",
|
||||||
|
daily_token_budget=12000,
|
||||||
|
daily_cost_budget_micros=4000,
|
||||||
|
),
|
||||||
|
expected_revision=1,
|
||||||
|
)
|
||||||
|
decision=store.route(req())
|
||||||
|
assert decision.model_id=="model-b"
|
||||||
|
assert decision.model_version=="2026-08-17"
|
||||||
|
|
||||||
|
|
||||||
|
def test_scope_isolation_denies_missing_cross_scope_policy(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
cross=RouteRequest(
|
||||||
|
request_id="request-cross",
|
||||||
|
task_id="task-a",
|
||||||
|
trace_id="trace-a",
|
||||||
|
scope_type="tenant",
|
||||||
|
tenant_id="tenant-b",
|
||||||
|
project_id="project-a",
|
||||||
|
required_capabilities=("agent.runtime.hermes",),
|
||||||
|
estimated_input_tokens=100,
|
||||||
|
max_output_tokens=50,
|
||||||
|
)
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_SCOPE_POLICY_MISSING"):
|
||||||
|
store.route(cross)
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_chain_detects_tampering(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
decision=store.route(req())
|
||||||
|
store.record_success(decision.request_id,input_tokens=100,output_tokens=50)
|
||||||
|
assert store.verify_audit_chain() is True
|
||||||
|
store.conn.execute(
|
||||||
|
"UPDATE model_governance_audit SET payload_json='{}' WHERE sequence=1"
|
||||||
|
)
|
||||||
|
assert store.verify_audit_chain() is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_registry_and_policy_use_cas(tmp_path):
|
||||||
|
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
|
||||||
|
store.register_provider(ProviderRecord("p"))
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_PROVIDER_CAS_MISMATCH"):
|
||||||
|
store.register_provider(ProviderRecord("p"),expected_revision=7)
|
||||||
|
store.set_scope_policy(ScopePolicy("tenant","t","p"))
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_POLICY_CAS_MISMATCH"):
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy("tenant","t","p"),
|
||||||
|
expected_revision=7,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_system_policy_never_authorizes_missing_tenant_policy(tmp_path):
|
||||||
|
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
|
||||||
|
store.register_provider(ProviderRecord("provider-a"))
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"model-a","provider-a","v1",("text",),
|
||||||
|
evaluation_sha256="a"*64,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy("system",None,None,allowed_models=("model-a",))
|
||||||
|
)
|
||||||
|
request=RouteRequest(
|
||||||
|
request_id="request-tenant-no-policy",
|
||||||
|
task_id="task-a",
|
||||||
|
trace_id="trace-a",
|
||||||
|
scope_type="tenant",
|
||||||
|
tenant_id="tenant-a",
|
||||||
|
project_id="project-a",
|
||||||
|
required_capabilities=("text",),
|
||||||
|
estimated_input_tokens=10,
|
||||||
|
max_output_tokens=10,
|
||||||
|
)
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_SCOPE_POLICY_MISSING"):
|
||||||
|
store.route(request)
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_model_governance_admin import apply_config
|
||||||
|
from aios_model_governance import ModelGovernanceStore
|
||||||
|
|
||||||
|
|
||||||
|
def config():
|
||||||
|
return {
|
||||||
|
"schema":"aios.model-governance.bootstrap.v1",
|
||||||
|
"providers":[{
|
||||||
|
"provider_id":"provider-a",
|
||||||
|
"credential_ref":"runtime-profile:provider-a",
|
||||||
|
"max_failures":2,
|
||||||
|
"cooldown_seconds":60,
|
||||||
|
}],
|
||||||
|
"models":[{
|
||||||
|
"model_id":"model-a",
|
||||||
|
"provider_id":"provider-a",
|
||||||
|
"version":"v1",
|
||||||
|
"capabilities":["text"],
|
||||||
|
"input_cost_micros_per_1k":1,
|
||||||
|
"output_cost_micros_per_1k":2,
|
||||||
|
"max_input_tokens":1000,
|
||||||
|
"max_output_tokens":100,
|
||||||
|
"evaluation_state":"APPROVED",
|
||||||
|
"evaluation_sha256":"a"*64,
|
||||||
|
"priority":10,
|
||||||
|
}],
|
||||||
|
"policies":[{
|
||||||
|
"scope_type":"legacy",
|
||||||
|
"tenant_id":None,
|
||||||
|
"project_id":None,
|
||||||
|
"allowed_models":["model-a"],
|
||||||
|
"required_capabilities":["text"],
|
||||||
|
"daily_token_budget":10000,
|
||||||
|
"daily_cost_budget_micros":1000,
|
||||||
|
"fallback_limit":0,
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_admin_apply_is_idempotent(tmp_path):
|
||||||
|
cfg=tmp_path/"config.json"
|
||||||
|
cfg.write_text(json.dumps(config()),encoding="utf-8")
|
||||||
|
db=tmp_path/"db.sqlite3"
|
||||||
|
first=apply_config(db,cfg)
|
||||||
|
second=apply_config(db,cfg)
|
||||||
|
assert first["changes"]["providers"]==1
|
||||||
|
assert first["changes"]["models"]==1
|
||||||
|
assert first["changes"]["policies"]==1
|
||||||
|
assert second["changes"]["providers"]==0
|
||||||
|
assert second["changes"]["models"]==0
|
||||||
|
assert second["changes"]["policies"]==0
|
||||||
|
assert second["changes"]["unchanged"]==3
|
||||||
|
store=ModelGovernanceStore(db)
|
||||||
|
assert store.verify_audit_chain() is True
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_admin_config_change_uses_cas_revision(tmp_path):
|
||||||
|
value=config()
|
||||||
|
cfg=tmp_path/"config.json"
|
||||||
|
cfg.write_text(json.dumps(value),encoding="utf-8")
|
||||||
|
db=tmp_path/"db.sqlite3"
|
||||||
|
apply_config(db,cfg)
|
||||||
|
value["providers"][0]["cooldown_seconds"]=120
|
||||||
|
cfg.write_text(json.dumps(value),encoding="utf-8")
|
||||||
|
result=apply_config(db,cfg)
|
||||||
|
assert result["changes"]["providers"]==1
|
||||||
|
store=ModelGovernanceStore(db)
|
||||||
|
row=store.conn.execute(
|
||||||
|
"select revision,cooldown_seconds from model_governance_providers where provider_id='provider-a'"
|
||||||
|
).fetchone()
|
||||||
|
assert tuple(row)==(2,120)
|
||||||
|
store.close()
|
||||||
@@ -0,0 +1,363 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
from aios_model_governance import (
|
||||||
|
ModelGovernanceStore,
|
||||||
|
ModelRecord,
|
||||||
|
ProviderRecord,
|
||||||
|
ScopePolicy,
|
||||||
|
)
|
||||||
|
from aios_online_agent_execution import GovernedExecutionBinding, project_online_task
|
||||||
|
import aios_external_agent_runtime as runtime
|
||||||
|
|
||||||
|
|
||||||
|
BASE="a"*40
|
||||||
|
|
||||||
|
|
||||||
|
def binding(task_id):
|
||||||
|
return GovernedExecutionBinding.from_mapping({
|
||||||
|
"task_id":task_id,
|
||||||
|
"job_id":"job-"+task_id,
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,
|
||||||
|
"expected_branch_oid":BASE,
|
||||||
|
"target_branch":"agent/"+task_id,
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"generation":1,
|
||||||
|
"idempotency_key":task_id+"-binding-v1",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def projected(task_id="task-model-governed", *, scope_type="tenant", tenant_id="tenant-a", project_id="project-a"):
|
||||||
|
request={
|
||||||
|
"task_id":task_id,
|
||||||
|
"goal":"Return the requested sentinel.",
|
||||||
|
"instructions":"No repository or control-plane side effects.",
|
||||||
|
"success_criteria":[{"id":"done","description":"Return a result."}],
|
||||||
|
"constraints":{"repository_mutation":"forbidden"},
|
||||||
|
"scope_type":scope_type,
|
||||||
|
"tenant_id":tenant_id,
|
||||||
|
"project_id":project_id,
|
||||||
|
"trace_id":"trace-model-a",
|
||||||
|
"required_capabilities":["agent.runtime.hermes"],
|
||||||
|
}
|
||||||
|
return project_online_task(
|
||||||
|
binding(task_id),
|
||||||
|
request,
|
||||||
|
{
|
||||||
|
"action":"agent_runtime",
|
||||||
|
"adapter":"hermes",
|
||||||
|
"capability_scope":["agent.runtime.hermes"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def helper(tmp_path):
|
||||||
|
path=tmp_path/"model_runtime.py"
|
||||||
|
path.write_text(
|
||||||
|
"import json,sys\n"
|
||||||
|
"model=sys.argv[1]\n"
|
||||||
|
"provider=sys.argv[2]\n"
|
||||||
|
"payload=json.load(sys.stdin)\n"
|
||||||
|
"if model=='model-a' and payload.get('constraints',{}).get('force_fail_a'):\n"
|
||||||
|
" raise SystemExit(7)\n"
|
||||||
|
"print(json.dumps({'answer':'ok','model':model,'provider':provider,"
|
||||||
|
"'usage':{'input_tokens':20,'output_tokens':10,'cost_micros':2}},sort_keys=True))\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def write_profile(tmp_path, helper_path, *, providers=("provider-a","provider-b")):
|
||||||
|
root=tmp_path/"profiles";root.mkdir(exist_ok=True)
|
||||||
|
value={
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3",str(helper_path),"{model_id}","{provider_id}"],
|
||||||
|
"input_mode":"stdin_json",
|
||||||
|
"output_mode":"stdout_json",
|
||||||
|
"timeout_seconds":30,
|
||||||
|
"max_output_bytes":65536,
|
||||||
|
"environment":{},
|
||||||
|
"credential_env":{},
|
||||||
|
"model_governance":{
|
||||||
|
"required":True,
|
||||||
|
"provider_ids":list(providers),
|
||||||
|
"model_capabilities":["text"],
|
||||||
|
"max_output_tokens":100,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
(root/"hermes.json").write_text(json.dumps(value),encoding="utf-8")
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def seed(db_path, *, legacy=False, tiny_budget=False):
|
||||||
|
store=ModelGovernanceStore(db_path,now_fn=lambda:1_800_000_000.0)
|
||||||
|
for provider in ("provider-a","provider-b"):
|
||||||
|
store.register_provider(
|
||||||
|
ProviderRecord(
|
||||||
|
provider,
|
||||||
|
f"runtime-profile:{provider}",
|
||||||
|
max_failures=1 if provider=="provider-a" else 2,
|
||||||
|
cooldown_seconds=120,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"model-a","provider-a","v1",
|
||||||
|
("agent.runtime.hermes","text"),
|
||||||
|
input_cost_micros_per_1k=10,
|
||||||
|
output_cost_micros_per_1k=20,
|
||||||
|
max_input_tokens=5000,max_output_tokens=1000,
|
||||||
|
evaluation_sha256="a"*64,priority=1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"model-b","provider-b","v2",
|
||||||
|
("agent.runtime.hermes","text"),
|
||||||
|
input_cost_micros_per_1k=12,
|
||||||
|
output_cost_micros_per_1k=22,
|
||||||
|
max_input_tokens=5000,max_output_tokens=1000,
|
||||||
|
evaluation_sha256="b"*64,priority=2,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if legacy:
|
||||||
|
policy=ScopePolicy(
|
||||||
|
"legacy",None,None,
|
||||||
|
allowed_models=("model-a","model-b"),
|
||||||
|
required_capabilities=("text",),
|
||||||
|
daily_token_budget=10000,
|
||||||
|
daily_cost_budget_micros=10000,
|
||||||
|
fallback_limit=2,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
policy=ScopePolicy(
|
||||||
|
"tenant","tenant-a","project-a",
|
||||||
|
allowed_models=("model-a","model-b"),
|
||||||
|
required_capabilities=("text",),
|
||||||
|
daily_token_budget=1 if tiny_budget else 10000,
|
||||||
|
daily_cost_budget_micros=10000,
|
||||||
|
fallback_limit=2,
|
||||||
|
)
|
||||||
|
store.set_scope_policy(policy)
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def make_external_request(projected_task, *, force_fail=False, attempt=1):
|
||||||
|
execution=projected_task["metadata"]["execution"]
|
||||||
|
payload=dict(execution["runtime_input"])
|
||||||
|
if force_fail:
|
||||||
|
payload["constraints"]=dict(payload["constraints"])
|
||||||
|
payload["constraints"]["force_fail_a"]=True
|
||||||
|
return {
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":"external-model-governance-0123456789",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"task_id":projected_task["task_id"],
|
||||||
|
"attempt":attempt,
|
||||||
|
"execution_digest":execution["execution_digest"],
|
||||||
|
"payload":payload,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def clean_env(monkeypatch):
|
||||||
|
for key in list(os.environ):
|
||||||
|
upper=key.upper()
|
||||||
|
if any(marker in upper for marker in ("TOKEN","SECRET","PASSWORD","CREDENTIAL","AUTHORIZATION","PRIVATE_KEY","API_KEY")):
|
||||||
|
monkeypatch.delenv(key,raising=False)
|
||||||
|
monkeypatch.setenv("CREDENTIALS_DIRECTORY","/definitely-not-present-model-governance")
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_projection_binds_trusted_scope_and_trace():
|
||||||
|
task=projected()
|
||||||
|
runtime_input=task["metadata"]["execution"]["runtime_input"]
|
||||||
|
assert runtime_input["scope"]=={
|
||||||
|
"scope_type":"tenant",
|
||||||
|
"tenant_id":"tenant-a",
|
||||||
|
"project_id":"project-a",
|
||||||
|
}
|
||||||
|
assert runtime_input["trace_id"]=="trace-model-a"
|
||||||
|
|
||||||
|
|
||||||
|
def test_legacy_scope_remains_explicit_not_fake_tenant():
|
||||||
|
task=projected(scope_type="legacy",tenant_id=None,project_id=None)
|
||||||
|
scope=task["metadata"]["execution"]["runtime_input"]["scope"]
|
||||||
|
assert scope=={"scope_type":"legacy","tenant_id":None,"project_id":None}
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_requires_model_binding_when_governance_required(tmp_path):
|
||||||
|
path=tmp_path/"helper.py";path.write_text("print('ok')\n")
|
||||||
|
root=tmp_path/"profiles";root.mkdir()
|
||||||
|
(root/"hermes.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3",str(path)],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
"model_governance":{"required":True,"provider_ids":["provider-a"],"max_output_tokens":100},
|
||||||
|
}))
|
||||||
|
with pytest.raises(runtime.ExternalAgentRuntimeError,match="EXTERNAL_AGENT_MODEL_BINDING_REQUIRED"):
|
||||||
|
runtime.RuntimeProfile.load("hermes",root)
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_runtime_selects_model_and_settles_usage(tmp_path,monkeypatch):
|
||||||
|
helper_path=helper(tmp_path)
|
||||||
|
profile_root=write_profile(tmp_path,helper_path)
|
||||||
|
db=tmp_path/"governance.sqlite3";seed(db)
|
||||||
|
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
|
||||||
|
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
|
||||||
|
|
||||||
|
request_path=tmp_path/"request.json"
|
||||||
|
result_path=tmp_path/"result.json"
|
||||||
|
request_path.write_text(json.dumps(make_external_request(projected())),encoding="utf-8")
|
||||||
|
runtime.execute_request(request_path,result_path,profile_root=profile_root)
|
||||||
|
result=json.loads(result_path.read_text())
|
||||||
|
mg=result["model_governance"]
|
||||||
|
assert mg["model_id"]=="model-a"
|
||||||
|
assert mg["model_version"]=="v1"
|
||||||
|
assert mg["provider_id"]=="provider-a"
|
||||||
|
assert mg["settlement"]["usage_source"]=="runtime"
|
||||||
|
assert mg["settlement"]["input_tokens"]==20
|
||||||
|
assert mg["settlement"]["output_tokens"]==10
|
||||||
|
assert mg["audit_chain_valid"] is True
|
||||||
|
assert result["output_json"]["model"]=="model-a"
|
||||||
|
|
||||||
|
|
||||||
|
def test_budget_denies_before_runtime_process(tmp_path,monkeypatch):
|
||||||
|
helper_path=helper(tmp_path)
|
||||||
|
profile_root=write_profile(tmp_path,helper_path)
|
||||||
|
db=tmp_path/"governance.sqlite3";seed(db,tiny_budget=True)
|
||||||
|
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
|
||||||
|
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
|
||||||
|
|
||||||
|
req=tmp_path/"request.json";res=tmp_path/"result.json"
|
||||||
|
req.write_text(json.dumps(make_external_request(projected())),encoding="utf-8")
|
||||||
|
with pytest.raises(runtime.ExternalAgentRuntimeError,match="MODEL_BUDGET_DENY"):
|
||||||
|
runtime.execute_request(req,res,profile_root=profile_root)
|
||||||
|
assert not res.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_provider_failure_routes_to_fallback_provider(tmp_path,monkeypatch):
|
||||||
|
helper_path=helper(tmp_path)
|
||||||
|
profile_root=write_profile(tmp_path,helper_path)
|
||||||
|
db=tmp_path/"governance.sqlite3";seed(db)
|
||||||
|
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
|
||||||
|
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
|
||||||
|
|
||||||
|
req=tmp_path/"request.json";res=tmp_path/"result.json"
|
||||||
|
req.write_text(json.dumps(make_external_request(projected(),force_fail=True)),encoding="utf-8")
|
||||||
|
runtime.execute_request(req,res,profile_root=profile_root)
|
||||||
|
result=json.loads(res.read_text())
|
||||||
|
mg=result["model_governance"]
|
||||||
|
assert len(mg["attempts"])==2
|
||||||
|
assert mg["attempts"][0]["provider_id"]=="provider-a"
|
||||||
|
assert mg["attempts"][1]["provider_id"]=="provider-b"
|
||||||
|
assert mg["provider_id"]=="provider-b"
|
||||||
|
assert result["output_json"]["provider"]=="provider-b"
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_provider_allowlist_fails_closed(tmp_path,monkeypatch):
|
||||||
|
helper_path=helper(tmp_path)
|
||||||
|
profile_root=write_profile(tmp_path,helper_path,providers=("provider-b",))
|
||||||
|
db=tmp_path/"governance.sqlite3";seed(db)
|
||||||
|
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
|
||||||
|
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
|
||||||
|
|
||||||
|
req=tmp_path/"request.json";res=tmp_path/"result.json"
|
||||||
|
req.write_text(json.dumps(make_external_request(projected())),encoding="utf-8")
|
||||||
|
with pytest.raises(runtime.ExternalAgentRuntimeError,match="EXTERNAL_AGENT_MODEL_PROVIDER_PROFILE_DENY"):
|
||||||
|
runtime.execute_request(req,res,profile_root=profile_root)
|
||||||
|
|
||||||
|
|
||||||
|
def test_governance_required_but_runtime_disabled_fails_closed(tmp_path,monkeypatch):
|
||||||
|
helper_path=helper(tmp_path)
|
||||||
|
profile_root=write_profile(tmp_path,helper_path)
|
||||||
|
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",False)
|
||||||
|
req=tmp_path/"request.json";res=tmp_path/"result.json"
|
||||||
|
req.write_text(json.dumps(make_external_request(projected())),encoding="utf-8")
|
||||||
|
with pytest.raises(runtime.ExternalAgentRuntimeError,match="EXTERNAL_AGENT_MODEL_GOVERNANCE_DISABLED"):
|
||||||
|
runtime.execute_request(req,res,profile_root=profile_root)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_request_cannot_override_authoritative_task_scope():
|
||||||
|
task_id="task-scope-override"
|
||||||
|
request={
|
||||||
|
"task_id":task_id,
|
||||||
|
"goal":"Return result.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
"scope_type":"tenant",
|
||||||
|
"tenant_id":"tenant-a",
|
||||||
|
"project_id":"project-a",
|
||||||
|
"trace_id":"trace-authoritative",
|
||||||
|
"required_capabilities":["agent.runtime.hermes"],
|
||||||
|
}
|
||||||
|
task=project_online_task(
|
||||||
|
binding(task_id),
|
||||||
|
request,
|
||||||
|
{
|
||||||
|
"action":"agent_runtime",
|
||||||
|
"adapter":"hermes",
|
||||||
|
"capability_scope":["agent.runtime.hermes"],
|
||||||
|
"runtime_input":{
|
||||||
|
"goal":"Return result.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
"scope":{
|
||||||
|
"scope_type":"tenant",
|
||||||
|
"tenant_id":"tenant-b",
|
||||||
|
"project_id":"project-b",
|
||||||
|
},
|
||||||
|
"trace_id":"trace-forged",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
runtime_input=task["metadata"]["execution"]["runtime_input"]
|
||||||
|
assert runtime_input["scope"]["tenant_id"]=="tenant-a"
|
||||||
|
assert runtime_input["scope"]["project_id"]=="project-a"
|
||||||
|
assert runtime_input["trace_id"]=="trace-authoritative"
|
||||||
|
|
||||||
|
|
||||||
|
def test_openclaw_nested_usage_is_extracted_exactly():
|
||||||
|
value={
|
||||||
|
"meta":{
|
||||||
|
"agentMeta":{
|
||||||
|
"usage":{"input":7770,"output":13,"total":7783},
|
||||||
|
"lastCallUsage":{"input":7770,"output":13,"total":7783},
|
||||||
|
"costUsd":0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert runtime._extract_runtime_usage(value)==(7770,13,0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_governance_required_profile_needs_provider_allowlist_and_output_limit(tmp_path):
|
||||||
|
path=tmp_path/"helper.py";path.write_text("print('ok')\\n")
|
||||||
|
root=tmp_path/"profiles";root.mkdir()
|
||||||
|
(root/"hermes.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3",str(path),"{model_id}"],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
"model_governance":{"required":True},
|
||||||
|
}))
|
||||||
|
with pytest.raises(
|
||||||
|
runtime.ExternalAgentRuntimeError,
|
||||||
|
match="EXTERNAL_AGENT_MODEL_PROVIDER_IDS_REQUIRED",
|
||||||
|
):
|
||||||
|
runtime.RuntimeProfile.load("hermes",root)
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_executor_has_single_model_governance_sqlite_write_path():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-external-agent-executor.service").read_text()
|
||||||
|
assert "AIOS_MODEL_GOVERNANCE_DB=/var/lib/aios-model-governance/state.sqlite3" in unit
|
||||||
|
assert "AIOS_MODEL_GOVERNANCE_ENABLED=1" in unit
|
||||||
|
assert "ReadWritePaths=/var/lib/aios-model-governance" in unit
|
||||||
|
assert "ReadWritePaths=/var/lib/aios-external-agent-executor" in unit
|
||||||
|
assert "InaccessiblePaths=/etc/aios-gitops" in unit
|
||||||
|
assert "InaccessiblePaths=/data/aios-gitops" in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_tmpfiles_owner_matches_external_executor():
|
||||||
|
line=(ROOT/"deploy/tmpfiles.d/aios-model-governance.conf").read_text().strip()
|
||||||
|
assert line == (
|
||||||
|
"d /var/lib/aios-model-governance 0750 "
|
||||||
|
"aios-engineering-executor aios-executor-exchange -"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_staging_models_use_real_e2e_evaluation_evidence():
|
||||||
|
value=json.loads(
|
||||||
|
(ROOT/"config/model-governance/model-governance.staging.json").read_text()
|
||||||
|
)
|
||||||
|
assert value["schema"]=="aios.model-governance.bootstrap.v1"
|
||||||
|
models={item["model_id"]:item for item in value["models"]}
|
||||||
|
assert models["deepseek-v4-flash"]["evaluation_sha256"] == (
|
||||||
|
"43ae0db1640daeb6d33e2787769732844f366d36070065d3a2239905bd9a823b"
|
||||||
|
)
|
||||||
|
assert models["openai/deepseek-v4-flash"]["evaluation_sha256"] == (
|
||||||
|
"7b1c89634eb9c0e7cb5a3d75323d904ebbd91b3ae5494ce938bd8316a4cf5b12"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_profiles_bind_model_from_governance_not_task():
|
||||||
|
for name in ("hermes","openclaw"):
|
||||||
|
value=json.loads(
|
||||||
|
(ROOT/"config/external-agent-runtime.d"/f"{name}.profile.json").read_text()
|
||||||
|
)
|
||||||
|
governance=value["model_governance"]
|
||||||
|
assert governance["required"] is True
|
||||||
|
assert governance["provider_ids"]
|
||||||
|
assert governance["max_output_tokens"] > 0
|
||||||
|
serialized=json.dumps(value["argv"])+json.dumps(value.get("environment",{}))
|
||||||
|
assert "{model_id}" in serialized or "{provider_id}" in serialized
|
||||||
|
|
||||||
|
|
||||||
|
def test_staging_bootstrap_has_explicit_tenant_live_policy():
|
||||||
|
value=json.loads(
|
||||||
|
(ROOT/"config/model-governance/model-governance.staging.json").read_text()
|
||||||
|
)
|
||||||
|
keys={
|
||||||
|
(
|
||||||
|
item["scope_type"],
|
||||||
|
item.get("tenant_id"),
|
||||||
|
item.get("project_id"),
|
||||||
|
)
|
||||||
|
for item in value["policies"]
|
||||||
|
}
|
||||||
|
assert ("legacy",None,None) in keys
|
||||||
|
assert ("tenant","model-governance-live","staging") in keys
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(ROOT / "active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_online_agent_execution import ( # noqa: E402
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_execution_envelope,
|
||||||
|
)
|
||||||
|
import aios_online_agent_worker as worker # noqa: F401,E402
|
||||||
|
from aios_online_agent_worker import WorkerSupervisor # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
BASE = "a" * 40
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_replays_the_durable_envelope_to_bridge(monkeypatch) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
worker,
|
||||||
|
"WORKER_CAPABILITIES",
|
||||||
|
tuple(sorted((*worker.WORKER_CAPABILITIES, "agent.runtime.hermes"))),
|
||||||
|
)
|
||||||
|
binding = GovernedExecutionBinding.from_mapping(
|
||||||
|
{
|
||||||
|
"task_id": "r42-agent-runtime-replay-test-20260818",
|
||||||
|
"job_id": "job-agent-runtime-replay-test",
|
||||||
|
"repository": "aios/aios",
|
||||||
|
"base_oid": BASE,
|
||||||
|
"expected_branch_oid": BASE,
|
||||||
|
"target_branch": "agent/agent-runtime-replay-test",
|
||||||
|
"branch_mode": "CREATE_NEW",
|
||||||
|
"generation": 1,
|
||||||
|
"idempotency_key": "agent-runtime-replay-test-v1",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
execution = project_execution_envelope(
|
||||||
|
binding,
|
||||||
|
{
|
||||||
|
"action": "agent_runtime",
|
||||||
|
"adapter": "hermes",
|
||||||
|
"capability_scope": ["agent.runtime.hermes"],
|
||||||
|
"change_set": [],
|
||||||
|
"runtime_input": {
|
||||||
|
"goal": "replay the exact durable runtime envelope",
|
||||||
|
"instructions": "return the bounded sentinel",
|
||||||
|
"success_criteria": [
|
||||||
|
{
|
||||||
|
"id": "sentinel",
|
||||||
|
"description": "return the bounded sentinel",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"constraints": {
|
||||||
|
"provider_boundary": "provider-credential-only",
|
||||||
|
"change_set": [],
|
||||||
|
"writer_lease": False,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
task = {
|
||||||
|
**binding.public(),
|
||||||
|
"required_capabilities": ["agent.runtime.hermes"],
|
||||||
|
"metadata": {"execution": execution},
|
||||||
|
}
|
||||||
|
|
||||||
|
class Bridge:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.calls: list[object] = []
|
||||||
|
|
||||||
|
def project_execution(self, _task: object, projection_input: object = None):
|
||||||
|
self.calls.append(projection_input)
|
||||||
|
return 200, dict(projection_input or {})
|
||||||
|
|
||||||
|
supervisor = object.__new__(WorkerSupervisor)
|
||||||
|
supervisor.bridge = Bridge()
|
||||||
|
resolved, source = supervisor.resolve_envelope(task)
|
||||||
|
|
||||||
|
assert source == "persisted_verified"
|
||||||
|
assert resolved.action == "agent_runtime"
|
||||||
|
assert resolved.adapter == "hermes"
|
||||||
|
assert resolved.digest == execution["execution_digest"]
|
||||||
|
assert supervisor.bridge.calls == [execution]
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from aios_online_agent_checkpoint import CheckpointBinding, CheckpointStore
|
||||||
|
|
||||||
|
|
||||||
|
def _binding() -> CheckpointBinding:
|
||||||
|
return CheckpointBinding(
|
||||||
|
task_id="checkpoint-agent-runtime-test-20260818",
|
||||||
|
attempt=1,
|
||||||
|
execution_digest="a" * 64,
|
||||||
|
job_id="job-checkpoint-agent-runtime-test-20260818",
|
||||||
|
repository="aios/aios",
|
||||||
|
base_oid="b" * 40,
|
||||||
|
expected_branch_oid="b" * 40,
|
||||||
|
target_branch="agent/checkpoint-agent-runtime-test-20260818",
|
||||||
|
generation=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_checkpoint_phase_is_durable_before_result(tmp_path: Path) -> None:
|
||||||
|
store = CheckpointStore(tmp_path / "checkpoints.sqlite3")
|
||||||
|
binding = _binding()
|
||||||
|
phases = [
|
||||||
|
"claimed",
|
||||||
|
"workspace_prepared",
|
||||||
|
"verification_passed",
|
||||||
|
"agent_runtime_done",
|
||||||
|
"result_pending",
|
||||||
|
"result_submitted",
|
||||||
|
]
|
||||||
|
|
||||||
|
rows = [
|
||||||
|
store.append(binding, phase, {"phase": phase, "task_id": binding.task_id})
|
||||||
|
for phase in phases
|
||||||
|
]
|
||||||
|
|
||||||
|
assert [row["phase"] for row in rows] == phases
|
||||||
|
assert [int(row["phase_order"]) for row in rows] == [10, 20, 30, 40, 50, 60]
|
||||||
|
latest = store.latest_for_attempt(binding.task_id, binding.attempt)
|
||||||
|
assert latest is not None
|
||||||
|
assert latest["phase"] == "result_submitted"
|
||||||
|
assert latest["payload"] == {"phase": "result_submitted", "task_id": binding.task_id}
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_checkpoint_append_is_idempotent(tmp_path: Path) -> None:
|
||||||
|
store = CheckpointStore(tmp_path / "checkpoints.sqlite3")
|
||||||
|
binding = _binding()
|
||||||
|
payload = {"phase": "agent_runtime_done", "result": {"status": "SUCCEEDED"}}
|
||||||
|
|
||||||
|
first = store.append(binding, "agent_runtime_done", payload)
|
||||||
|
replay = store.append(binding, "agent_runtime_done", payload)
|
||||||
|
|
||||||
|
assert replay["sequence"] == first["sequence"]
|
||||||
|
assert replay["payload_sha256"] == first["payload_sha256"]
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
|
||||||
|
import aios_online_agent_worker as worker
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_execution_envelope,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _state_paths(monkeypatch, tmp_path):
|
||||||
|
root=tmp_path/"worker-state"
|
||||||
|
monkeypatch.setattr(worker,"STATE_ROOT",root)
|
||||||
|
monkeypatch.setattr(worker,"WORKER_LIVENESS_PATH",root/"worker-liveness.json")
|
||||||
|
|
||||||
|
BASE="a"*40
|
||||||
|
TASK_ID="task-worker-availability-placeholder"
|
||||||
|
JOB_ID="job-worker-availability-placeholder"
|
||||||
|
BRANCH="agent/task-worker-availability-placeholder"
|
||||||
|
IDEMPOTENCY="task-worker-availability-placeholder-binding-v1"
|
||||||
|
|
||||||
|
|
||||||
|
def binding():
|
||||||
|
return GovernedExecutionBinding.from_mapping({
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"job_id":JOB_ID,
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,
|
||||||
|
"expected_branch_oid":BASE,
|
||||||
|
"target_branch":BRANCH,
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"generation":1,
|
||||||
|
"idempotency_key":IDEMPOTENCY,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def placeholder():
|
||||||
|
return project_execution_envelope(
|
||||||
|
binding(),
|
||||||
|
{
|
||||||
|
"action":"source_capture",
|
||||||
|
"adapter":"source_capture",
|
||||||
|
"capability_scope":[
|
||||||
|
"git.read",
|
||||||
|
"git.test",
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def readonly_source_capture():
|
||||||
|
return project_execution_envelope(
|
||||||
|
binding(),
|
||||||
|
{
|
||||||
|
"action":"source_capture",
|
||||||
|
"adapter":"source_capture",
|
||||||
|
"capability_scope":["git.read","git.test"],
|
||||||
|
"source_signals":["worker availability"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def deterministic():
|
||||||
|
return project_execution_envelope(
|
||||||
|
binding(),
|
||||||
|
{
|
||||||
|
"action":"deterministic",
|
||||||
|
"adapter":"deterministic",
|
||||||
|
"capability_scope":[
|
||||||
|
"git.read",
|
||||||
|
"git.test",
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
],
|
||||||
|
"change_set":[{
|
||||||
|
"path":"active-runtime/app/example.py",
|
||||||
|
"operation":"write_file",
|
||||||
|
"content":"VALUE=1\\n",
|
||||||
|
}],
|
||||||
|
"verification":[{
|
||||||
|
"label":"unit",
|
||||||
|
"argv":["/usr/bin/python3","-m","pytest","-q"],
|
||||||
|
"cwd":".",
|
||||||
|
"timeout_seconds":120,
|
||||||
|
}],
|
||||||
|
"governance":{
|
||||||
|
"commit_idempotency_key":"task-worker-availability-placeholder-commit-v1",
|
||||||
|
"allowed_files":["active-runtime/app/example.py"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def task(execution):
|
||||||
|
return {
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"required_capabilities":list(execution["capability_scope"]),
|
||||||
|
"metadata":{
|
||||||
|
"execution":execution,
|
||||||
|
"execution_projection":{
|
||||||
|
"schema":"aios.online-agent.task-projection.v1",
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"job_id":JOB_ID,
|
||||||
|
"execution_digest":execution["execution_digest"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class FakeBridge:
|
||||||
|
def __init__(self,status,execution):
|
||||||
|
self.status=status
|
||||||
|
self.execution=execution
|
||||||
|
self.calls=[]
|
||||||
|
|
||||||
|
def project_execution(self,task_value,execution=None):
|
||||||
|
self.calls.append(execution)
|
||||||
|
return self.status,self.execution
|
||||||
|
|
||||||
|
|
||||||
|
class FakeAgent:
|
||||||
|
def __init__(self,tasks):
|
||||||
|
self.tasks=tasks
|
||||||
|
self.claim_calls=[]
|
||||||
|
|
||||||
|
def poll(self):
|
||||||
|
return list(self.tasks)
|
||||||
|
|
||||||
|
def claim(self,task_id):
|
||||||
|
self.claim_calls.append(task_id)
|
||||||
|
return 500,{"error":"SHOULD_NOT_CLAIM"}
|
||||||
|
|
||||||
|
|
||||||
|
def supervisor(bridge, agent=None):
|
||||||
|
value=object.__new__(worker.WorkerSupervisor)
|
||||||
|
value.bridge=bridge
|
||||||
|
value.agent=agent
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_capable_source_capture_is_classified_as_placeholder():
|
||||||
|
envelope=worker.ExecutionEnvelope.from_task(
|
||||||
|
task(placeholder()),
|
||||||
|
worker_capabilities=tuple(placeholder()["capability_scope"]),
|
||||||
|
)
|
||||||
|
assert worker.source_capture_requires_deterministic_plan(envelope) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_readonly_source_capture_is_not_placeholder():
|
||||||
|
envelope=worker.ExecutionEnvelope.from_task(
|
||||||
|
task(readonly_source_capture()),
|
||||||
|
worker_capabilities=tuple(readonly_source_capture()["capability_scope"]),
|
||||||
|
)
|
||||||
|
assert worker.source_capture_requires_deterministic_plan(envelope) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_placeholder_without_registered_plan_is_skipped_before_claim():
|
||||||
|
persisted=placeholder()
|
||||||
|
bridge=FakeBridge(200,persisted)
|
||||||
|
agent=FakeAgent([task(persisted)])
|
||||||
|
sup=supervisor(bridge,agent)
|
||||||
|
assert sup.once()==0
|
||||||
|
assert bridge.calls==[None]
|
||||||
|
assert agent.claim_calls==[]
|
||||||
|
|
||||||
|
|
||||||
|
def test_placeholder_with_missing_projection_is_skipped_before_claim():
|
||||||
|
persisted=placeholder()
|
||||||
|
bridge=FakeBridge(404,{"error":"TASK_EXECUTION_PROJECTION_MISSING"})
|
||||||
|
agent=FakeAgent([task(persisted)])
|
||||||
|
sup=supervisor(bridge,agent)
|
||||||
|
assert sup.once()==0
|
||||||
|
assert bridge.calls==[None]
|
||||||
|
assert agent.claim_calls==[]
|
||||||
|
|
||||||
|
|
||||||
|
def test_registered_deterministic_plan_replaces_placeholder():
|
||||||
|
persisted=placeholder()
|
||||||
|
resolved=deterministic()
|
||||||
|
bridge=FakeBridge(200,resolved)
|
||||||
|
sup=supervisor(bridge)
|
||||||
|
envelope,source=sup.resolve_envelope(task(persisted))
|
||||||
|
assert source=="plan_registry_verified"
|
||||||
|
assert envelope.action=="deterministic"
|
||||||
|
assert envelope.digest==resolved["execution_digest"]
|
||||||
|
assert bridge.calls==[None]
|
||||||
|
|
||||||
|
|
||||||
|
def test_real_readonly_source_capture_replays_exact_payload():
|
||||||
|
persisted=readonly_source_capture()
|
||||||
|
bridge=FakeBridge(200,persisted)
|
||||||
|
sup=supervisor(bridge)
|
||||||
|
envelope,source=sup.resolve_envelope(task(persisted))
|
||||||
|
assert source=="persisted_verified"
|
||||||
|
assert envelope.action=="source_capture"
|
||||||
|
assert envelope.digest==persisted["execution_digest"]
|
||||||
|
assert bridge.calls==[persisted]
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(ROOT / "active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_execution_envelope,
|
||||||
|
)
|
||||||
|
import aios_online_agent_worker as worker
|
||||||
|
from aios_online_agent_worker import WorkerSupervisor
|
||||||
|
|
||||||
|
|
||||||
|
BASE = "a" * 40
|
||||||
|
|
||||||
|
|
||||||
|
def _binding() -> GovernedExecutionBinding:
|
||||||
|
return GovernedExecutionBinding.from_mapping(
|
||||||
|
{
|
||||||
|
"task_id": "r42-source-capture-replay-test-20260818",
|
||||||
|
"job_id": "job-source-capture-replay-test",
|
||||||
|
"repository": "aios/aios",
|
||||||
|
"base_oid": BASE,
|
||||||
|
"expected_branch_oid": BASE,
|
||||||
|
"target_branch": "agent/source-capture-replay-test",
|
||||||
|
"branch_mode": "CREATE_NEW",
|
||||||
|
"generation": 1,
|
||||||
|
"idempotency_key": "source-capture-replay-test-v1",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_capture_replays_the_durable_envelope_to_bridge():
|
||||||
|
binding = _binding()
|
||||||
|
execution = project_execution_envelope(
|
||||||
|
binding,
|
||||||
|
{
|
||||||
|
"action": "source_capture",
|
||||||
|
"adapter": "compact_interface_map",
|
||||||
|
"capability_scope": ["git.read", "git.test"],
|
||||||
|
"source_signals": ["model provider registry version pinning"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
task = {
|
||||||
|
**binding.public(),
|
||||||
|
"required_capabilities": ["git.read", "git.test"],
|
||||||
|
"metadata": {"execution": execution},
|
||||||
|
}
|
||||||
|
|
||||||
|
class Bridge:
|
||||||
|
def __init__(self):
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def project_execution(self, _task, projection_input=None):
|
||||||
|
self.calls.append(projection_input)
|
||||||
|
return 200, dict(projection_input or {})
|
||||||
|
|
||||||
|
supervisor = object.__new__(WorkerSupervisor)
|
||||||
|
supervisor.bridge = Bridge()
|
||||||
|
resolved, source = supervisor.resolve_envelope(task)
|
||||||
|
|
||||||
|
assert source == "persisted_verified"
|
||||||
|
assert resolved.digest == execution["execution_digest"]
|
||||||
|
assert supervisor.bridge.calls == [execution]
|
||||||
|
|
||||||
|
|
||||||
|
def test_deterministic_tasks_still_use_plan_registry_without_payload_replay():
|
||||||
|
binding = _binding()
|
||||||
|
execution = project_execution_envelope(
|
||||||
|
binding,
|
||||||
|
{
|
||||||
|
"action": "deterministic",
|
||||||
|
"adapter": "deterministic",
|
||||||
|
"capability_scope": [
|
||||||
|
"git.read",
|
||||||
|
"git.test",
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
],
|
||||||
|
"change_set": [
|
||||||
|
{
|
||||||
|
"path": "active-runtime/app/example.py",
|
||||||
|
"operation": "write_file",
|
||||||
|
"content": "x",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
task = {
|
||||||
|
**binding.public(),
|
||||||
|
"required_capabilities": [
|
||||||
|
"git.read",
|
||||||
|
"git.test",
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
],
|
||||||
|
"metadata": {"execution": execution},
|
||||||
|
}
|
||||||
|
|
||||||
|
class Bridge:
|
||||||
|
def __init__(self):
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def project_execution(self, _task, projection_input=None):
|
||||||
|
self.calls.append(projection_input)
|
||||||
|
return 200, dict(execution)
|
||||||
|
|
||||||
|
supervisor = object.__new__(WorkerSupervisor)
|
||||||
|
supervisor.bridge = Bridge()
|
||||||
|
resolved, source = supervisor.resolve_envelope(task)
|
||||||
|
|
||||||
|
assert source == "persisted_verified"
|
||||||
|
assert resolved.digest == execution["execution_digest"]
|
||||||
|
assert supervisor.bridge.calls == [None]
|
||||||
@@ -0,0 +1,199 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_worker_availability_guard import AvailabilityGuard
|
||||||
|
|
||||||
|
|
||||||
|
class FakeSystemd:
|
||||||
|
def __init__(self, roots, now=1000.0):
|
||||||
|
self.now=now
|
||||||
|
self.actions=[]
|
||||||
|
self.units={}
|
||||||
|
for unit,root in roots.items():
|
||||||
|
self.units[unit]={
|
||||||
|
"LoadState":"loaded",
|
||||||
|
"ActiveState":"active",
|
||||||
|
"UnitFileState":"enabled",
|
||||||
|
"Environment":f"AIOS_WORKER_STATE_ROOT={root}",
|
||||||
|
}
|
||||||
|
self.units["aios-external-agent-executor.service"]={
|
||||||
|
"LoadState":"loaded",
|
||||||
|
"ActiveState":"active",
|
||||||
|
"UnitFileState":"enabled",
|
||||||
|
"Environment":"",
|
||||||
|
}
|
||||||
|
|
||||||
|
def property(self, unit, name):
|
||||||
|
return self.units.get(unit,{}).get(name,"")
|
||||||
|
|
||||||
|
def environment(self, unit):
|
||||||
|
raw=self.property(unit,"Environment")
|
||||||
|
result={}
|
||||||
|
for item in raw.split():
|
||||||
|
if "=" in item:
|
||||||
|
k,v=item.split("=",1)
|
||||||
|
result[k]=v
|
||||||
|
return result
|
||||||
|
|
||||||
|
def enable(self, unit):
|
||||||
|
self.actions.append((unit,"enable"))
|
||||||
|
self.units[unit]["UnitFileState"]="enabled"
|
||||||
|
|
||||||
|
def start(self, unit):
|
||||||
|
self.actions.append((unit,"start"))
|
||||||
|
self.units[unit]["ActiveState"]="active"
|
||||||
|
env=self.environment(unit)
|
||||||
|
if "AIOS_WORKER_STATE_ROOT" not in env:
|
||||||
|
return
|
||||||
|
root=Path(env["AIOS_WORKER_STATE_ROOT"])
|
||||||
|
root.mkdir(parents=True,exist_ok=True)
|
||||||
|
(root/"worker-liveness.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.online-agent.worker-liveness.v1",
|
||||||
|
"agent_id":"agent-"+unit,
|
||||||
|
"state":"POLL_OK",
|
||||||
|
"last_seen_epoch":self.now,
|
||||||
|
}),encoding="utf-8")
|
||||||
|
|
||||||
|
def restart(self, unit):
|
||||||
|
self.actions.append((unit,"restart"))
|
||||||
|
self.units[unit]["ActiveState"]="active"
|
||||||
|
env=self.environment(unit)
|
||||||
|
root=Path(env["AIOS_WORKER_STATE_ROOT"])
|
||||||
|
root.mkdir(parents=True,exist_ok=True)
|
||||||
|
(root/"worker-liveness.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.online-agent.worker-liveness.v1",
|
||||||
|
"agent_id":"agent-"+unit,
|
||||||
|
"state":"POLL_OK",
|
||||||
|
"last_seen_epoch":self.now,
|
||||||
|
}),encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def make_guard(tmp_path, fake):
|
||||||
|
worker_units=tuple(
|
||||||
|
unit for unit in fake.units
|
||||||
|
if unit!="aios-external-agent-executor.service"
|
||||||
|
)
|
||||||
|
return AvailabilityGuard(
|
||||||
|
systemd=fake,
|
||||||
|
worker_units=worker_units,
|
||||||
|
dependency_units=("aios-external-agent-executor.service",),
|
||||||
|
status_path=tmp_path/"run/status.json",
|
||||||
|
recovery_path=tmp_path/"run/recovery.json",
|
||||||
|
inhibit_path=tmp_path/"run/maintenance.lock",
|
||||||
|
max_stale_seconds=90,
|
||||||
|
restart_cooldown_seconds=60,
|
||||||
|
startup_wait_seconds=0,
|
||||||
|
now_fn=lambda:fake.now,
|
||||||
|
sleep_fn=lambda _seconds:None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def write_liveness(root, now, *, state="POLL_OK"):
|
||||||
|
root.mkdir(parents=True,exist_ok=True)
|
||||||
|
(root/"worker-liveness.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.online-agent.worker-liveness.v1",
|
||||||
|
"agent_id":"agent",
|
||||||
|
"state":state,
|
||||||
|
"last_seen_epoch":now,
|
||||||
|
}),encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_healthy_workers_need_no_action(tmp_path):
|
||||||
|
roots={
|
||||||
|
"generic.service":tmp_path/"generic",
|
||||||
|
"hermes.service":tmp_path/"hermes",
|
||||||
|
"openclaw.service":tmp_path/"openclaw",
|
||||||
|
}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
for root in roots.values():
|
||||||
|
write_liveness(root,fake.now)
|
||||||
|
result=make_guard(tmp_path,fake).run(heal=True)
|
||||||
|
assert result["status"]=="HEALTHY"
|
||||||
|
assert result["all_fresh"] is True
|
||||||
|
assert fake.actions==[]
|
||||||
|
|
||||||
|
|
||||||
|
def test_inactive_worker_is_started_and_becomes_fresh(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
fake.units["generic.service"]["ActiveState"]="inactive"
|
||||||
|
result=make_guard(tmp_path,fake).run(heal=True)
|
||||||
|
assert ("generic.service","start") in fake.actions
|
||||||
|
assert result["all_fresh"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_disabled_worker_is_enabled_and_started(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
fake.units["generic.service"]["UnitFileState"]="disabled"
|
||||||
|
fake.units["generic.service"]["ActiveState"]="inactive"
|
||||||
|
result=make_guard(tmp_path,fake).run(heal=True)
|
||||||
|
assert ("generic.service","enable") in fake.actions
|
||||||
|
assert ("generic.service","start") in fake.actions
|
||||||
|
assert result["all_fresh"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_active_but_stale_worker_is_restarted(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
write_liveness(roots["generic.service"],fake.now-500)
|
||||||
|
result=make_guard(tmp_path,fake).run(heal=True)
|
||||||
|
assert ("generic.service","restart") in fake.actions
|
||||||
|
assert result["all_fresh"] is True
|
||||||
|
assert result["workers"][0]["action"]=="RESTART_STALE"
|
||||||
|
|
||||||
|
|
||||||
|
def test_restart_cooldown_prevents_restart_storm(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
write_liveness(roots["generic.service"],fake.now-500)
|
||||||
|
guard=make_guard(tmp_path,fake)
|
||||||
|
guard.recovery_path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
guard.recovery_path.write_text(json.dumps({
|
||||||
|
"schema":"aios.worker-availability.recovery.v1",
|
||||||
|
"last_restart_epoch":{"generic.service":fake.now-10},
|
||||||
|
}),encoding="utf-8")
|
||||||
|
result=guard.run(heal=True)
|
||||||
|
assert ("generic.service","restart") not in fake.actions
|
||||||
|
assert result["status"]=="DEGRADED"
|
||||||
|
assert result["workers"][0]["detail"]=="RESTART_COOLDOWN_ACTIVE"
|
||||||
|
|
||||||
|
|
||||||
|
def test_maintenance_inhibit_disables_healing(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
fake.units["generic.service"]["ActiveState"]="inactive"
|
||||||
|
guard=make_guard(tmp_path,fake)
|
||||||
|
guard.inhibit_path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
guard.inhibit_path.write_text("planned maintenance\n")
|
||||||
|
result=guard.run(heal=True)
|
||||||
|
assert result["status"]=="MAINTENANCE_INHIBITED"
|
||||||
|
assert fake.actions==[]
|
||||||
|
|
||||||
|
|
||||||
|
def test_dependency_is_enabled_and_started(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
write_liveness(roots["generic.service"],fake.now)
|
||||||
|
fake.units["aios-external-agent-executor.service"]["UnitFileState"]="disabled"
|
||||||
|
fake.units["aios-external-agent-executor.service"]["ActiveState"]="inactive"
|
||||||
|
result=make_guard(tmp_path,fake).run(heal=True)
|
||||||
|
assert ("aios-external-agent-executor.service","enable") in fake.actions
|
||||||
|
assert ("aios-external-agent-executor.service","start") in fake.actions
|
||||||
|
assert result["all_fresh"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_reports_stale_without_mutation(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
write_liveness(roots["generic.service"],fake.now-500)
|
||||||
|
result=make_guard(tmp_path,fake).run(heal=False)
|
||||||
|
assert result["status"]=="DEGRADED"
|
||||||
|
assert result["all_fresh"] is False
|
||||||
|
assert fake.actions==[]
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_source_has_idle_and_busy_liveness():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_online_agent_worker.py").read_text()
|
||||||
|
assert "aios.online-agent.worker-liveness.v1" in source
|
||||||
|
assert 'write_worker_liveness("POLL_OK"' in source
|
||||||
|
assert '"POLL_FAILED"' in source
|
||||||
|
assert 'write_worker_liveness("STARTING")' in source
|
||||||
|
assert '"BUSY"' in source
|
||||||
|
assert "last_seen_epoch" in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_guard_timer_is_persistent_and_frequent():
|
||||||
|
timer=(ROOT/"deploy/systemd/aios-worker-availability-guard.timer").read_text()
|
||||||
|
assert "OnBootSec=15s" in timer
|
||||||
|
assert "OnUnitActiveSec=30s" in timer
|
||||||
|
assert "Persistent=true" in timer
|
||||||
|
assert "WantedBy=timers.target" in timer
|
||||||
|
|
||||||
|
|
||||||
|
def test_guard_service_has_no_network_or_credentials():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-worker-availability-guard.service").read_text()
|
||||||
|
assert "User=root" in unit
|
||||||
|
assert "LoadCredential" not in unit
|
||||||
|
assert "RestrictAddressFamilies=AF_UNIX" in unit
|
||||||
|
assert "NoNewPrivileges=true" in unit
|
||||||
|
assert "ProtectSystem=strict" in unit
|
||||||
|
assert "ReadWritePaths=/run/aios-worker-availability" in unit
|
||||||
|
assert "AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS=90" in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_guard_manages_all_three_workers_and_external_executor():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_worker_availability_guard.py").read_text()
|
||||||
|
for unit in (
|
||||||
|
"aios-online-agent-worker-20260815.service",
|
||||||
|
"aios-hermes-online-agent-worker.service",
|
||||||
|
"aios-openclaw-online-agent-worker.service",
|
||||||
|
"aios-external-agent-executor.service",
|
||||||
|
):
|
||||||
|
assert unit in source
|
||||||
|
assert "maintenance.lock" in source
|
||||||
|
assert "RESTART_STALE" in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_defers_write_capable_source_capture_until_plan_exists():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_online_agent_worker.py").read_text()
|
||||||
|
assert "ENGINEERING_WRITE_CAPABILITIES" in source
|
||||||
|
assert "source_capture_requires_deterministic_plan" in source
|
||||||
|
assert "TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN" in source
|
||||||
|
assert '"governed.write.branch"' in source
|
||||||
|
assert '"governed.write.file"' in source
|
||||||
Reference in New Issue
Block a user