Compare commits

...
Author SHA1 Message Date
AIOS Governed Continuation 69a51d50c7 feat(gpt-edge): add disabled-by-default TLS edge 2026-08-19 09:48:00 +08:00
AIOS Governed Continuation 7f12094e58 feat(online-task): add credential-isolated external GPT read-only ingress 2026-08-19 09:20:52 +08:00
AIOS Governed Continuation 2ae50a2a3d fix(model-governance): persist policy floor source of truth 2026-08-19 01:44:04 +08:00
AIOS Governed Continuation e9426dc0b8 fix(model-governance): reserve OpenClaw injected context 2026-08-18 20:02:52 +08:00
AIOS Governed Continuation d31267e89c fix(external-agent): scope request identity by task attempt 2026-08-18 17:01:38 +08:00
AIOS Governed Continuation f4022d3d1e feat(model-governance): enforce routed budgets and runtime policy 2026-08-18 13:18:20 +08:00
AIOS Governed Continuation 008bea3324 feat(online-agent): persist workers and defer engineering placeholders 2026-08-18 12:24:42 +08:00
AIOS Governed Continuation 52ef41e45f fix(online-agent): persist external runtime completion checkpoint phase 2026-08-18 08:46:05 +08:00
AIOS Governed Continuation 416e5e295d fix(online-agent): replay persisted external runtime envelope through canonical projection 2026-08-18 07:59:36 +08:00
AIOS Governed Continuation 4f81344509 fix(online-agent): normalize external runtime capabilities 2026-08-18 07:31:27 +08:00
AIOS Governed Continuation 8efdaea22f fix(online-agent): add provider-only Hermes/OpenClaw credential projection 2026-08-18 06:42:53 +08:00
AIOS Governed Continuation 3b48740b17 feat(online-agent): add Hermes and OpenClaw runtime workers 2026-08-18 05:13:52 +08:00
AIOS Governed Continuation 0c330afe03 fix(online-agent): preserve source-capture envelope during canonical replay 2026-08-18 04:16:54 +08:00
AIOS Governed Continuation 764bb2f08e feat(online-agent): add owner-controlled same-binding execution rebind 2026-08-18 03:45:27 +08:00
AIOS Governed Continuation 37814da581 feat(online-agent): generalize compact interface-map profiles 2026-08-18 03:27:02 +08:00
AIOS Governed Continuation 6de31e0f3e fix(task-plane): preserve fleet fairness through legacy oracle 2026-08-17 23:49:32 +08:00
AIOS Governed Continuation 51d7a356ca feat(task-plane): wire fleet scheduler into real pull dispatch 2026-08-17 22:04:21 +08:00
64 changed files with 11795 additions and 70 deletions
@@ -0,0 +1,722 @@
"""Owner-controlled CAS-bound execution-envelope rebind for Online Agent Tasks.
This service exists for a narrow recovery case: a Task was projected/claimed with a read-only
source_capture envelope before its already-registered deterministic plan became authoritative.
It never creates Tasks/Jobs/branches and never accepts arbitrary metadata patches. It only replaces
metadata.execution + the derived execution_projection for the same governed binding, after an
official NEEDS_REWORK transition has returned the Task to QUEUED with no active lease.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
import hashlib
import hmac
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import re
import sqlite3
from typing import Any, Callable, Mapping, Sequence
import urllib.error
import urllib.request
from aios_online_agent_execution import (
GovernedExecutionBinding,
validate_execution_envelope,
)
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,159}$")
SHA_RE = re.compile(r"^[0-9a-f]{64}$")
MAX_BODY = 64 * 1024
AUTHORIZATION_ID_ENV = "AIOS_AGENT_TASK_EXECUTION_REBIND_AUTHORIZATION_ID"
class ExecutionRebindError(RuntimeError):
def __init__(self, code: str, status: int = 409) -> None:
super().__init__(code)
self.code = code
self.status = status
def _canonical(value: object) -> bytes:
return json.dumps(
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False
).encode("utf-8")
def _sha(value: object) -> str:
return hashlib.sha256(_canonical(value)).hexdigest()
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _require_string(
value: object,
code: str,
pattern: re.Pattern[str] | None = None,
) -> str:
if not isinstance(value, str) or not value.strip():
raise ExecutionRebindError(code, 400)
value = value.strip()
if pattern is not None and pattern.fullmatch(value) is None:
raise ExecutionRebindError(code, 400)
return value
def _required_authorization_id() -> str:
value = os.environ.get(AUTHORIZATION_ID_ENV, "").strip()
if not value:
raise RuntimeError("TASK_EXECUTION_REBIND_AUTHORIZATION_ID_MISSING")
if KEY_RE.fullmatch(value) is None:
raise RuntimeError("TASK_EXECUTION_REBIND_AUTHORIZATION_ID_INVALID")
return value
@dataclass(frozen=True)
class ExecutionRebindRequest:
expected_status: str
expected_revision: int
expected_attempt: int
previous_verification_sha256: str
expected_old_execution_digest: str
expected_new_execution_digest: str
authorization_id: str
idempotency_key: str
reason: str
@classmethod
def from_mapping(cls, value: Mapping[str, Any]) -> "ExecutionRebindRequest":
allowed = {
"expected_status",
"expected_revision",
"expected_attempt",
"previous_verification_sha256",
"expected_old_execution_digest",
"expected_new_execution_digest",
"authorization_id",
"idempotency_key",
"reason",
}
if set(value) != allowed:
raise ExecutionRebindError("EXECUTION_REBIND_FIELDS_INVALID", 400)
status = _require_string(value["expected_status"], "EXPECTED_STATUS_INVALID")
if status != "QUEUED":
raise ExecutionRebindError("EXPECTED_STATUS_INVALID", 400)
revision = value["expected_revision"]
attempt = value["expected_attempt"]
if (
not isinstance(revision, int)
or isinstance(revision, bool)
or revision < 0
or not isinstance(attempt, int)
or isinstance(attempt, bool)
or attempt < 1
):
raise ExecutionRebindError("EXECUTION_REBIND_INTEGER_INVALID", 400)
verification = _require_string(
value["previous_verification_sha256"], "VERIFICATION_SHA256_INVALID"
)
old_digest = _require_string(
value["expected_old_execution_digest"], "OLD_EXECUTION_DIGEST_INVALID"
)
new_digest = _require_string(
value["expected_new_execution_digest"], "NEW_EXECUTION_DIGEST_INVALID"
)
for digest, code in (
(verification, "VERIFICATION_SHA256_INVALID"),
(old_digest, "OLD_EXECUTION_DIGEST_INVALID"),
(new_digest, "NEW_EXECUTION_DIGEST_INVALID"),
):
if SHA_RE.fullmatch(digest) is None:
raise ExecutionRebindError(code, 400)
if old_digest == new_digest:
raise ExecutionRebindError("EXECUTION_REBIND_DIGEST_UNCHANGED", 400)
authorization = _require_string(
value["authorization_id"], "AUTHORIZATION_ID_INVALID", KEY_RE
)
key = _require_string(value["idempotency_key"], "IDEMPOTENCY_KEY_INVALID", KEY_RE)
reason = _require_string(value["reason"], "EXECUTION_REBIND_REASON_INVALID")
if len(reason) > 512:
raise ExecutionRebindError("EXECUTION_REBIND_REASON_INVALID", 400)
return cls(
status,
revision,
attempt,
verification,
old_digest,
new_digest,
authorization,
key,
reason,
)
def public(self) -> dict[str, Any]:
return {
"expected_status": self.expected_status,
"expected_revision": self.expected_revision,
"expected_attempt": self.expected_attempt,
"previous_verification_sha256": self.previous_verification_sha256,
"expected_old_execution_digest": self.expected_old_execution_digest,
"expected_new_execution_digest": self.expected_new_execution_digest,
"authorization_id": self.authorization_id,
"idempotency_key": self.idempotency_key,
"reason": self.reason,
}
class CanonicalBridgeExecutionResolver:
"""Resolve the already-registered plan from the canonical governed Bridge."""
def __init__(self, base_url: str, control_token: str) -> None:
self.base_url = base_url.rstrip("/")
self.control_token = control_token
def __call__(
self,
task_id: str,
required_capabilities: Sequence[str],
) -> dict[str, Any]:
body = json.dumps(
{
"task_id": task_id,
"required_capabilities": list(required_capabilities),
},
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
request = urllib.request.Request(
self.base_url + "/v2/execution-envelope",
method="POST",
data=body,
headers={
"Authorization": "Bearer " + self.control_token,
"Content-Type": "application/json",
"Accept": "application/json",
},
)
try:
with urllib.request.urlopen(request, timeout=15) as response:
raw = response.read()
status = response.status
except urllib.error.HTTPError as exc:
raw = exc.read()
status = exc.code
try:
value = json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise ExecutionRebindError("CANONICAL_EXECUTION_RESPONSE_INVALID", 502) from exc
if not isinstance(value, Mapping):
raise ExecutionRebindError("CANONICAL_EXECUTION_RESPONSE_INVALID", 502)
if status != 200:
raise ExecutionRebindError(
str(value.get("error") or "CANONICAL_EXECUTION_RESOLUTION_FAILED"),
502 if status >= 500 else 409,
)
return dict(value)
class ExecutionRebindStore:
"""Official owner-controlled mutation with CAS, plan resolution and durable audit."""
def __init__(
self,
db_path: Path,
resolver: Callable[[str, Sequence[str]], Mapping[str, Any]],
) -> None:
self.db_path = Path(db_path)
self.resolver = resolver
def _connect(self) -> sqlite3.Connection:
connection = sqlite3.connect(self.db_path, timeout=30.0)
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA foreign_keys=ON")
connection.execute("PRAGMA busy_timeout=30000")
return connection
@staticmethod
def _columns(
connection: sqlite3.Connection,
table: str,
) -> dict[str, sqlite3.Row]:
return {
str(row["name"]): row
for row in connection.execute(f'PRAGMA table_info("{table}")')
}
@staticmethod
def _json_value(raw: object, code: str) -> dict[str, Any]:
if isinstance(raw, Mapping):
return dict(raw)
if not isinstance(raw, str):
raise ExecutionRebindError(code, 409)
try:
value = json.loads(raw)
except json.JSONDecodeError as exc:
raise ExecutionRebindError(code, 409) from exc
if not isinstance(value, Mapping):
raise ExecutionRebindError(code, 409)
return dict(value)
@staticmethod
def _verification_hash(raw: object) -> str:
if raw in (None, ""):
value: object = {}
elif isinstance(raw, str):
try:
value = json.loads(raw)
except json.JSONDecodeError as exc:
raise ExecutionRebindError("TASK_VERIFICATION_INVALID", 409) from exc
else:
value = raw
return _sha(value)
@staticmethod
def _metadata_column(columns: Mapping[str, sqlite3.Row]) -> str:
for name in ("metadata_json", "metadata"):
if name in columns:
return name
raise ExecutionRebindError("TASK_METADATA_SCHEMA_UNSUPPORTED", 503)
@staticmethod
def _required_capabilities(
task: sqlite3.Row,
columns: Mapping[str, sqlite3.Row],
old_execution: Mapping[str, Any],
) -> list[str]:
for field in ("required_capabilities_json", "required_capabilities"):
if field not in columns:
continue
raw = task[field]
if raw in (None, ""):
continue
if isinstance(raw, str):
try:
value = json.loads(raw)
except json.JSONDecodeError:
value = [item for item in raw.split(",") if item.strip()]
else:
value = raw
if isinstance(value, list) and all(isinstance(item, str) for item in value):
return [item.strip() for item in value if item.strip()]
value = old_execution.get("capability_scope")
if isinstance(value, list) and all(isinstance(item, str) for item in value):
return [item for item in value if item]
raise ExecutionRebindError("TASK_CAPABILITIES_UNAVAILABLE", 503)
@staticmethod
def _event_table(
connection: sqlite3.Connection,
) -> tuple[str, dict[str, sqlite3.Row]]:
tables = {
row["name"]
for row in connection.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
)
}
for name in ("task_events", "events"):
if name not in tables:
continue
columns = ExecutionRebindStore._columns(connection, name)
if "task_id" in columns and ("event_type" in columns or "type" in columns):
return name, columns
raise ExecutionRebindError("TASK_EVENT_TABLE_UNSUPPORTED", 503)
@staticmethod
def _insert_event(
connection: sqlite3.Connection,
task_id: str,
payload: Mapping[str, Any],
) -> None:
table, columns = ExecutionRebindStore._event_table(connection)
values: dict[str, Any] = {"task_id": task_id}
if "event_type" in columns:
values["event_type"] = "TASK_EXECUTION_REBIND_AUTHORIZED"
else:
values["type"] = "TASK_EXECUTION_REBIND_AUTHORIZED"
payload_text = json.dumps(payload, sort_keys=True, separators=(",", ":"))
if "payload_json" in columns:
values["payload_json"] = payload_text
elif "payload" in columns:
values["payload"] = payload_text
else:
raise ExecutionRebindError("TASK_EVENT_PAYLOAD_UNSUPPORTED", 503)
if "created_at" in columns:
values["created_at"] = _now()
if "agent_id" in columns:
values["agent_id"] = "owner-control"
if "actor_id" in columns:
values["actor_id"] = "owner-control"
names = list(values)
quoted = ",".join('"' + name + '"' for name in names)
placeholders = ",".join("?" for _ in names)
connection.execute(
f'INSERT INTO "{table}" ({quoted}) VALUES ({placeholders})',
tuple(values[name] for name in names),
)
@staticmethod
def _ensure_rebind_table(connection: sqlite3.Connection) -> None:
connection.execute(
"""
CREATE TABLE IF NOT EXISTS task_execution_rebinds (
task_id TEXT NOT NULL,
idempotency_key TEXT NOT NULL,
request_sha256 TEXT NOT NULL,
authorization_id TEXT NOT NULL,
previous_verification_sha256 TEXT NOT NULL,
old_execution_digest TEXT NOT NULL,
new_execution_digest TEXT NOT NULL,
expected_revision INTEGER NOT NULL,
applied_revision INTEGER NOT NULL,
attempt INTEGER NOT NULL,
reason TEXT NOT NULL,
created_at TEXT NOT NULL,
PRIMARY KEY(task_id,idempotency_key)
)
"""
)
def _load_and_validate(
self,
connection: sqlite3.Connection,
task_id: str,
request: ExecutionRebindRequest,
) -> tuple[sqlite3.Row, dict[str, sqlite3.Row], str, dict[str, Any], dict[str, Any]]:
columns = self._columns(connection, "tasks")
required = {
"task_id",
"status",
"attempt",
"revision",
"claimed_by",
"lease_token_hash",
"lease_expires_at",
"verification_json",
}
if not required.issubset(columns):
raise ExecutionRebindError("TASK_SCHEMA_EXECUTION_REBIND_UNSUPPORTED", 503)
metadata_column = self._metadata_column(columns)
task = connection.execute(
"SELECT * FROM tasks WHERE task_id=?",
(task_id,),
).fetchone()
if task is None:
raise ExecutionRebindError("TASK_NOT_FOUND", 404)
if (
str(task["status"]) != request.expected_status
or int(task["revision"]) != request.expected_revision
or int(task["attempt"]) != request.expected_attempt
):
raise ExecutionRebindError("EXECUTION_REBIND_CAS_MISMATCH", 409)
if task["lease_token_hash"] not in (None, "") or task["lease_expires_at"] not in (None, ""):
raise ExecutionRebindError("EXECUTION_REBIND_ACTIVE_LEASE_DENY", 409)
verification_sha = self._verification_hash(task["verification_json"])
if verification_sha != request.previous_verification_sha256:
raise ExecutionRebindError("EXECUTION_REBIND_VERIFICATION_DRIFT", 409)
metadata = self._json_value(
task[metadata_column],
"TASK_METADATA_INVALID",
)
execution = metadata.get("execution")
if not isinstance(execution, Mapping):
raise ExecutionRebindError("TASK_EXECUTION_METADATA_MISSING", 409)
old_execution = validate_execution_envelope(dict(execution))
if old_execution["execution_digest"] != request.expected_old_execution_digest:
raise ExecutionRebindError("EXECUTION_REBIND_OLD_DIGEST_DRIFT", 409)
if old_execution.get("action") != "source_capture":
raise ExecutionRebindError("EXECUTION_REBIND_SOURCE_CAPTURE_REQUIRED", 409)
projection = metadata.get("execution_projection")
if isinstance(projection, Mapping):
projected_digest = str(projection.get("execution_digest") or "")
if projected_digest and projected_digest != request.expected_old_execution_digest:
raise ExecutionRebindError("EXECUTION_REBIND_PROJECTION_DRIFT", 409)
return task, columns, metadata_column, metadata, old_execution
def rebind(
self,
task_id: str,
request: ExecutionRebindRequest,
) -> dict[str, Any]:
task_id = _require_string(task_id, "TASK_ID_INVALID", TASK_RE)
required_authorization = _required_authorization_id()
if request.authorization_id != required_authorization:
raise ExecutionRebindError("EXECUTION_REBIND_AUTHORIZATION_ID_MISMATCH", 403)
request_sha = _sha(request.public())
# Idempotent replay must win before CAS validation because a successful
# rebind necessarily increments the Task revision and changes the persisted envelope.
with self._connect() as connection:
self._ensure_rebind_table(connection)
replay = connection.execute(
"SELECT * FROM task_execution_rebinds WHERE task_id=? AND idempotency_key=?",
(task_id, request.idempotency_key),
).fetchone()
if replay is not None:
if replay["request_sha256"] != request_sha:
raise ExecutionRebindError("EXECUTION_REBIND_IDEMPOTENCY_CONFLICT", 409)
connection.commit()
return {
"task_id": task_id,
"status": "QUEUED",
"attempt": int(replay["attempt"]),
"revision": int(replay["applied_revision"]),
"old_execution_digest": replay["old_execution_digest"],
"new_execution_digest": replay["new_execution_digest"],
"authorization_id": replay["authorization_id"],
"idempotent": True,
}
# Read/CAS snapshot first. Canonical Bridge resolution happens outside the DB write lock.
with self._connect() as connection:
task, columns, _, _, old_execution = self._load_and_validate(
connection, task_id, request
)
capabilities = self._required_capabilities(task, columns, old_execution)
binding = GovernedExecutionBinding.from_mapping(old_execution)
resolved = dict(self.resolver(task_id, capabilities))
new_execution = validate_execution_envelope(resolved, binding=binding)
if new_execution.get("action") != "deterministic":
raise ExecutionRebindError("EXECUTION_REBIND_DETERMINISTIC_PLAN_REQUIRED", 409)
if new_execution["execution_digest"] != request.expected_new_execution_digest:
raise ExecutionRebindError("EXECUTION_REBIND_NEW_DIGEST_DRIFT", 409)
if new_execution["execution_digest"] == old_execution["execution_digest"]:
raise ExecutionRebindError("EXECUTION_REBIND_DIGEST_UNCHANGED", 409)
# Re-read under BEGIN IMMEDIATE and apply only if the exact snapshot is unchanged.
with self._connect() as connection:
connection.execute("BEGIN IMMEDIATE")
self._ensure_rebind_table(connection)
replay = connection.execute(
"SELECT * FROM task_execution_rebinds WHERE task_id=? AND idempotency_key=?",
(task_id, request.idempotency_key),
).fetchone()
if replay is not None:
if replay["request_sha256"] != request_sha:
raise ExecutionRebindError("EXECUTION_REBIND_IDEMPOTENCY_CONFLICT", 409)
connection.commit()
return {
"task_id": task_id,
"status": "QUEUED",
"attempt": int(replay["attempt"]),
"revision": int(replay["applied_revision"]),
"old_execution_digest": replay["old_execution_digest"],
"new_execution_digest": replay["new_execution_digest"],
"authorization_id": replay["authorization_id"],
"idempotent": True,
}
task, columns, metadata_column, metadata, current_old = self._load_and_validate(
connection, task_id, request
)
if current_old["execution_digest"] != old_execution["execution_digest"]:
raise ExecutionRebindError("EXECUTION_REBIND_OLD_DIGEST_DRIFT", 409)
new_metadata = dict(metadata)
new_metadata["execution"] = new_execution
new_metadata["execution_projection"] = {
"schema": "aios.online-agent.task-projection.v1",
"task_id": binding.task_id,
"job_id": binding.job_id,
"execution_digest": new_execution["execution_digest"],
}
applied_revision = request.expected_revision + 1
previous_claimed_by = task["claimed_by"]
cursor = connection.execute(
f"""
UPDATE tasks
SET "{metadata_column}"=?, revision=?, claimed_by=NULL,
lease_token_hash=NULL, lease_expires_at=NULL
WHERE task_id=? AND status=? AND revision=? AND attempt=?
AND claimed_by IS ? AND lease_token_hash IS NULL AND lease_expires_at IS NULL
""",
(
json.dumps(new_metadata, sort_keys=True, separators=(",", ":")),
applied_revision,
task_id,
request.expected_status,
request.expected_revision,
request.expected_attempt,
previous_claimed_by,
),
)
if cursor.rowcount != 1:
raise ExecutionRebindError("EXECUTION_REBIND_CAS_MISMATCH", 409)
event_payload = {
"schema": "aios.task-execution-rebind.event.v1",
"task_id": task_id,
"job_id": binding.job_id,
"authorization_id": request.authorization_id,
"idempotency_key": request.idempotency_key,
"reason": request.reason,
"previous_verification_sha256": request.previous_verification_sha256,
"old_execution_digest": old_execution["execution_digest"],
"new_execution_digest": new_execution["execution_digest"],
"old_action": old_execution.get("action"),
"new_action": new_execution.get("action"),
"previous_revision": request.expected_revision,
"applied_revision": applied_revision,
"attempt": request.expected_attempt,
"request_sha256": request_sha,
"previous_claimed_by": previous_claimed_by,
"stale_claim_normalized": previous_claimed_by not in (None, ""),
}
self._insert_event(connection, task_id, event_payload)
connection.execute(
"""
INSERT INTO task_execution_rebinds(
task_id,idempotency_key,request_sha256,authorization_id,
previous_verification_sha256,old_execution_digest,new_execution_digest,
expected_revision,applied_revision,attempt,reason,created_at
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
""",
(
task_id,
request.idempotency_key,
request_sha,
request.authorization_id,
request.previous_verification_sha256,
old_execution["execution_digest"],
new_execution["execution_digest"],
request.expected_revision,
applied_revision,
request.expected_attempt,
request.reason,
_now(),
),
)
connection.commit()
return {
"task_id": task_id,
"status": "QUEUED",
"attempt": request.expected_attempt,
"revision": applied_revision,
"old_execution_digest": old_execution["execution_digest"],
"new_execution_digest": new_execution["execution_digest"],
"authorization_id": request.authorization_id,
"idempotent": False,
}
class ExecutionRebindHandler(BaseHTTPRequestHandler):
store: ExecutionRebindStore | None = None
control_token: str = ""
def log_message(self, *_args: Any) -> None:
return
def _json(self, value: Mapping[str, Any], status: int) -> None:
data = json.dumps(
dict(value), sort_keys=True, separators=(",", ":")
).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
def _authorized(self) -> bool:
header = self.headers.get("Authorization", "")
return (
header.startswith("Bearer ")
and bool(self.control_token)
and hmac.compare_digest(header[7:], self.control_token)
)
def do_GET(self) -> None:
if self.path == "/health":
self._json(
{"status": "ok", "service": "aios-agent-task-execution-rebind"},
200,
)
else:
self._json({"error": "NOT_FOUND"}, 404)
def do_POST(self) -> None:
match = re.fullmatch(r"/v1/tasks/([^/]+)/execution-rebind", self.path)
if match is None:
self._json({"error": "NOT_FOUND"}, 404)
return
if not self._authorized():
self._json({"error": "UNAUTHORIZED"}, 401)
return
try:
length = int(self.headers.get("Content-Length", "0"))
except ValueError:
self._json({"error": "CONTENT_LENGTH_INVALID"}, 400)
return
if length < 1 or length > MAX_BODY:
self._json({"error": "REQUEST_SIZE_INVALID"}, 413)
return
try:
value = json.loads(self.rfile.read(length).decode("utf-8"))
if not isinstance(value, Mapping):
raise ValueError
request = ExecutionRebindRequest.from_mapping(value)
if self.store is None:
raise ExecutionRebindError("RUNTIME_UNAVAILABLE", 503)
result = self.store.rebind(match.group(1), request)
self._json(result, 200 if result["idempotent"] else 201)
except ExecutionRebindError as exc:
self._json({"error": exc.code}, exc.status)
except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
self._json({"error": "REQUEST_INVALID"}, 400)
def load_control_token(path: Path) -> str:
if path.is_symlink() or not path.is_file():
raise RuntimeError("TASK_EXECUTION_REBIND_CONTROL_CREDENTIAL_UNAVAILABLE")
value = path.read_text(encoding="utf-8").strip()
if len(value) < 32:
raise RuntimeError("TASK_EXECUTION_REBIND_CONTROL_CREDENTIAL_INVALID")
return value
def serve() -> None:
db = Path(os.environ.get("AIOS_AGENT_TASK_DB", "/var/lib/aios-agent-task/state.sqlite3"))
_required_authorization_id()
token_path = Path(
os.environ.get(
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
str(
Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
/ "agent_task_control_token"
),
)
)
token = load_control_token(token_path)
bridge = os.environ.get(
"AIOS_GOVERNED_WRITE_API_URL",
"http://127.0.0.1:18892",
)
host = os.environ.get("AIOS_TASK_EXECUTION_REBIND_HOST", "127.0.0.1")
port = int(os.environ.get("AIOS_TASK_EXECUTION_REBIND_PORT", "18942"))
ExecutionRebindHandler.store = ExecutionRebindStore(
db,
CanonicalBridgeExecutionResolver(bridge, token),
)
ExecutionRebindHandler.control_token = token
ThreadingHTTPServer((host, port), ExecutionRebindHandler).serve_forever()
if __name__ == "__main__":
serve()
@@ -0,0 +1,163 @@
"""Canonical fail-closed capability semantics for task eligibility."""
from __future__ import annotations
from collections.abc import Iterable
_TASK_ALIASES: dict[str, frozenset[str]] = {
"repository_read": frozenset({"repo.read"}),
"git.read": frozenset({"repo.read"}),
"repo.read": frozenset({"repo.read"}),
"test_execution": frozenset({"test.execute"}),
"git.test": frozenset({"test.execute"}),
"test.execute": frozenset({"test.execute"}),
"governed_git_write": frozenset({"governed.git.write.branch", "governed.git.write.file"}),
"governed.git.write": frozenset({"governed.git.write.branch", "governed.git.write.file"}),
"governed.write.branch": frozenset({"governed.git.write.branch"}),
"governed.write.file": frozenset({"governed.git.write.file"}),
"shell": frozenset({"runtime.shell"}),
"git": frozenset({"runtime.git"}),
"python": frozenset({"runtime.python"}),
"code.read": frozenset({"legacy.code.read"}),
"code.review": frozenset({"legacy.code.review"}),
"code.write": frozenset({"legacy.code.write"}),
"code.write.patch": frozenset({"legacy.code.write.patch"}),
"test.python": frozenset({"legacy.test.python"}),
"test.shell.allowlisted": frozenset({"legacy.test.shell.allowlisted"}),
"git.commit": frozenset({"legacy.git.commit"}),
"git.diff.read": frozenset({"legacy.git.diff.read"}),
"git.force_push": frozenset({"legacy.git.force_push"}),
"deploy.production": frozenset({"legacy.deploy.production"}),
"secret.transform": frozenset({"legacy.secret.transform"}),
"secure_file_transformer": frozenset({"legacy.secure_file_transformer"}),
"runtime.shadow": frozenset({"legacy.runtime.shadow"}),
"review.security": frozenset({"legacy.review.security"}),
"review.api-contract": frozenset({"legacy.review.api-contract"}),
"gpu.train": frozenset({"legacy.gpu.train"}),
"agent.runtime.hermes": frozenset({"agent.runtime.hermes"}),
"agent.runtime.openclaw": frozenset({"agent.runtime.openclaw"}),
}
_WORKER_ALIASES: dict[str, frozenset[str]] = {
"git.read": frozenset({"repo.read"}),
"repo.read": frozenset({"repo.read"}),
"repository_read": frozenset({"repo.read"}),
"git.test": frozenset({"test.execute"}),
"test.execute": frozenset({"test.execute"}),
"test_execution": frozenset({"test.execute"}),
"governed.write.branch": frozenset({"governed.git.write.branch"}),
"governed.git.write.branch": frozenset({"governed.git.write.branch"}),
"governed.write.file": frozenset({"governed.git.write.file"}),
"governed.git.write.file": frozenset({"governed.git.write.file"}),
"shell": frozenset({"runtime.shell"}),
"git": frozenset({"runtime.git"}),
"python": frozenset({"runtime.python"}),
"code.read": frozenset({"legacy.code.read"}),
"code.review": frozenset({"legacy.code.review"}),
"code.write": frozenset({"legacy.code.write"}),
"code.write.patch": frozenset({"legacy.code.write.patch"}),
"test.python": frozenset({"legacy.test.python"}),
"test.shell.allowlisted": frozenset({"legacy.test.shell.allowlisted"}),
"git.commit": frozenset({"legacy.git.commit"}),
"git.diff.read": frozenset({"legacy.git.diff.read"}),
"git.force_push": frozenset({"legacy.git.force_push"}),
"deploy.production": frozenset({"legacy.deploy.production"}),
"secret.transform": frozenset({"legacy.secret.transform"}),
"secure_file_transformer": frozenset({"legacy.secure_file_transformer"}),
"runtime.shadow": frozenset({"legacy.runtime.shadow"}),
"review.security": frozenset({"legacy.review.security"}),
"review.api-contract": frozenset({"legacy.review.api-contract"}),
"gpu.train": frozenset({"legacy.gpu.train"}),
"agent.runtime.hermes": frozenset({"agent.runtime.hermes"}),
"agent.runtime.openclaw": frozenset({"agent.runtime.openclaw"}),
}
_UNKNOWN_PREFIX = "exact:"
def _clean(value: object) -> str:
if not isinstance(value, str):
raise ValueError("capability must be a string")
text = value.strip().lower()
if not text:
raise ValueError("capability must not be empty")
return text
def normalize_required_capabilities(values: Iterable[object]) -> frozenset[str]:
"""Map task-facing names to one semantic vocabulary.
Unknown names remain explicit exact requirements. They are never treated
as wildcards or granted by a failed normalization.
"""
result: set[str] = set()
for value in values:
text = _clean(value)
if "*" in text:
result.add(_UNKNOWN_PREFIX + text)
else:
result.update(_TASK_ALIASES.get(text, frozenset({_UNKNOWN_PREFIX + text})))
return frozenset(result)
def normalize_worker_capabilities(values: Iterable[object]) -> frozenset[str]:
"""Map registered worker names to effective semantic capabilities."""
result: set[str] = set()
for value in values:
text = _clean(value)
if "*" in text:
continue
mapped = _WORKER_ALIASES.get(text)
if mapped is not None:
result.update(mapped)
if {
"governed.git.write.branch",
"governed.git.write.file",
}.issubset(result):
result.add("governed.git.write")
return frozenset(result)
def has_capabilities(
worker_capabilities: Iterable[object],
required_capabilities: Iterable[object],
) -> bool:
"""Return the single eligibility decision used by poll and claim."""
required = normalize_required_capabilities(required_capabilities)
effective = normalize_worker_capabilities(worker_capabilities)
if any(value.startswith(_UNKNOWN_PREFIX) for value in required):
return False
return required.issubset(effective)
def required_capabilities_subset(
granted_requirements: Iterable[object],
required_requirements: Iterable[object],
) -> bool:
"""Check semantic capability containment for delegation constraints."""
granted = normalize_required_capabilities(granted_requirements)
required = normalize_required_capabilities(required_requirements)
if any(value.startswith(_UNKNOWN_PREFIX) for value in granted | required):
return False
return required.issubset(granted)
def normalized_capability_matrix(
worker_capabilities: Iterable[object],
required_capabilities: Iterable[object],
) -> dict[str, object]:
"""Expose non-secret decision evidence for tests and diagnostics."""
worker_values = tuple(worker_capabilities)
required_values = tuple(required_capabilities)
worker = normalize_worker_capabilities(worker_values)
required = normalize_required_capabilities(required_values)
return {
"worker_effective": sorted(worker),
"task_required": sorted(required),
"eligible": has_capabilities(worker_values, required_values),
}
File diff suppressed because it is too large Load Diff
+456
View File
@@ -0,0 +1,456 @@
#!/usr/bin/env python3
"""Disabled-by-default TLS edge for the read-only GPT Online Task ingress.
The internal AIOS ingress remains loopback-only. This service is a separate edge
that may bind non-loopback only when the owner explicitly approves it. It
forwards only the tiny read-only ingress API and never proxies the canonical
Bridge or Task API.
"""
from __future__ import annotations
from collections import defaultdict, deque
from dataclasses import dataclass
import hashlib
import hmac
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import re
import secrets
import ssl
import sys
import threading
import time
import urllib.error
import urllib.request
from typing import Any, Mapping
from urllib.parse import urlsplit
UPSTREAM = "http://127.0.0.1:18945"
MAX_BODY = 64 * 1024
MAX_RESPONSE = 256 * 1024
MAX_CONCURRENT_REQUESTS = 32
REQUEST_TIMEOUT_SECONDS = 20
RATE_WINDOW_SECONDS = 60
RATE_LIMIT_REQUESTS = 60
RATE_BURST_REQUESTS = 15
RATE_MAX_CLIENTS = 4096
_AUDIT_SALT = secrets.token_bytes(32)
PUBLIC_APPROVAL_ENV = "AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED"
PUBLIC_DOMAIN_ENV = "AIOS_GPT_EDGE_PUBLIC_DOMAIN"
TASK_PATH_RE = re.compile(r"^/v1/read-only-tasks(?:/[A-Za-z0-9][A-Za-z0-9._:-]{7,127})?$")
def _client_pseudonym(value: str) -> str:
return hashlib.sha256(_AUDIT_SALT + value.encode("utf-8", "replace")).hexdigest()[:16]
def audit(event: str, **fields: object) -> None:
safe = {
"event": event,
"ts_unix_ms": int(time.time() * 1000),
}
for key, value in fields.items():
if key in {"authorization", "body", "token", "cookie", "client_ip"}:
continue
safe[key] = value
print(
json.dumps(safe, sort_keys=True, separators=(",", ":")),
file=sys.stderr,
flush=True,
)
class EdgeError(RuntimeError):
def __init__(self, code: str, status: int = 400) -> None:
super().__init__(code)
self.code = code
self.status = status
def canonical(value: object) -> bytes:
return json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
def public_bind_allowed(host: str, approval: str) -> str:
value = host.strip()
if value in {"127.0.0.1", "::1", "localhost"}:
return value
if approval != "1":
raise RuntimeError("GPT_EDGE_PUBLIC_BIND_OWNER_APPROVAL_REQUIRED")
if value not in {"0.0.0.0", "::"}:
raise RuntimeError("GPT_EDGE_BIND_ADDRESS_INVALID")
return value
def validated_public_domain(value: str) -> str:
domain = value.strip().lower()
if not domain:
return ""
if len(domain) > 253 or re.fullmatch(
r"(?=.{1,253}\Z)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?",
domain,
) is None:
raise RuntimeError("GPT_EDGE_PUBLIC_DOMAIN_INVALID")
return domain
def validated_edge_identity(host: str, approval: str, domain: str) -> tuple[str, str]:
bind_host = public_bind_allowed(host, approval)
public_domain = validated_public_domain(domain)
if bind_host not in {"127.0.0.1", "::1", "localhost"} and not public_domain:
raise RuntimeError("GPT_EDGE_PUBLIC_DOMAIN_REQUIRED")
return bind_host, public_domain
def validate_tls_file(path: Path, *, key: bool) -> Path:
if path.is_symlink() or not path.is_file():
raise RuntimeError("GPT_EDGE_TLS_FILE_INVALID")
size = path.stat().st_size
if size < 16 or size > 256 * 1024:
raise RuntimeError("GPT_EDGE_TLS_FILE_INVALID")
if key and (path.stat().st_mode & 0o077):
raise RuntimeError("GPT_EDGE_TLS_KEY_MODE_INVALID")
return path
def build_tls_context(cert_path: Path, key_path: Path) -> ssl.SSLContext:
cert = validate_tls_file(cert_path, key=False)
key = validate_tls_file(key_path, key=True)
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.minimum_version = ssl.TLSVersion.TLSv1_2
context.options |= ssl.OP_NO_COMPRESSION
context.load_cert_chain(certfile=str(cert), keyfile=str(key))
return context
def allowed_route(method: str, raw_path: str) -> str:
parsed = urlsplit(raw_path)
if parsed.query or parsed.fragment:
raise EdgeError("QUERY_OR_FRAGMENT_DENY", 400)
path = parsed.path
if method == "GET" and path in {"/health", "/openapi.json"}:
return path
if method == "POST" and path == "/v1/read-only-tasks":
return path
if method == "GET" and TASK_PATH_RE.fullmatch(path):
return path
raise EdgeError("ROUTE_DENY", 404)
def route_label(method: str, path: str) -> str:
if method == "GET" and path == "/health":
return "health"
if method == "GET" and path == "/openapi.json":
return "openapi"
if method == "POST" and path == "/v1/read-only-tasks":
return "task_create"
if method == "GET" and path.startswith("/v1/read-only-tasks/"):
return "task_status"
return "unknown"
def forwarded_headers(headers: Mapping[str, str], *, body: bool) -> dict[str, str]:
result = {"Accept": "application/json"}
authorization = headers.get("Authorization")
if not isinstance(authorization, str) or not authorization.startswith("Bearer "):
raise EdgeError("AUTHORIZATION_REQUIRED", 401)
token = authorization[7:]
if len(token) < 32 or len(token) > 4096 or any(ch.isspace() for ch in token):
raise EdgeError("AUTHORIZATION_INVALID", 401)
result["Authorization"] = authorization
if body:
content_type = headers.get("Content-Type", "")
if content_type.split(";", 1)[0].strip().lower() != "application/json":
raise EdgeError("CONTENT_TYPE_INVALID", 415)
result["Content-Type"] = "application/json"
return result
class RateLimiter:
def __init__(
self,
*,
window_seconds: int = RATE_WINDOW_SECONDS,
limit: int = RATE_LIMIT_REQUESTS,
burst: int = RATE_BURST_REQUESTS,
max_clients: int = RATE_MAX_CLIENTS,
now_fn=time.monotonic,
) -> None:
self.window_seconds = window_seconds
self.limit = limit
self.burst = burst
self.max_clients = max_clients
self.now_fn = now_fn
self._lock = threading.Lock()
self._events: dict[str, deque[float]] = defaultdict(deque)
def _prune_stale(self, cutoff: float) -> None:
stale = [
key
for key, events in self._events.items()
if not events or events[-1] <= cutoff
]
for key in stale:
self._events.pop(key, None)
def allowed(self, key: str) -> bool:
now = self.now_fn()
cutoff = now - self.window_seconds
with self._lock:
if key not in self._events and len(self._events) >= self.max_clients:
self._prune_stale(cutoff)
if len(self._events) >= self.max_clients:
return False
events = self._events[key]
while events and events[0] <= cutoff:
events.popleft()
if len(events) >= self.limit + self.burst:
return False
events.append(now)
return True
@dataclass(frozen=True)
class Proxy:
upstream: str = UPSTREAM
timeout: int = REQUEST_TIMEOUT_SECONDS
def request(
self,
method: str,
path: str,
headers: Mapping[str, str],
body: bytes | None,
) -> tuple[int, bytes, str]:
if self.upstream != UPSTREAM:
raise EdgeError("UPSTREAM_DRIFT_DENY", 503)
req = urllib.request.Request(
self.upstream + path,
method=method,
data=body,
headers=dict(headers),
)
try:
with urllib.request.urlopen(req, timeout=self.timeout) as response:
raw = response.read(MAX_RESPONSE + 1)
status = response.status
content_type = response.headers.get("Content-Type", "application/json")
except urllib.error.HTTPError as exc:
raw = exc.read(MAX_RESPONSE + 1)
status = exc.code
content_type = exc.headers.get("Content-Type", "application/json")
except Exception as exc:
raise EdgeError("UPSTREAM_UNAVAILABLE", 502) from exc
if len(raw) > MAX_RESPONSE:
raise EdgeError("UPSTREAM_RESPONSE_TOO_LARGE", 502)
if content_type.split(";", 1)[0].strip().lower() != "application/json":
raise EdgeError("UPSTREAM_CONTENT_TYPE_INVALID", 502)
return status, raw, "application/json"
class EdgeServer(ThreadingHTTPServer):
daemon_threads = True
request_queue_size = 64
allow_reuse_address = True
class Handler(BaseHTTPRequestHandler):
proxy = Proxy()
limiter = RateLimiter()
concurrency = threading.BoundedSemaphore(MAX_CONCURRENT_REQUESTS)
expected_domain = ""
server_version = "AIOSGPTIngressEdge/1.0"
sys_version = ""
def log_message(self, *_args: Any) -> None:
return
def respond_json(self, value: Mapping[str, Any], status: int) -> None:
raw = canonical(value)
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Cache-Control", "no-store")
self.send_header("X-Content-Type-Options", "nosniff")
self.send_header("Referrer-Policy", "no-referrer")
self.send_header("Content-Security-Policy", "default-src 'none'")
self.send_header("Strict-Transport-Security", "max-age=31536000")
self.send_header("Content-Length", str(len(raw)))
self.end_headers()
self.wfile.write(raw)
def send_proxy_response(self, status: int, raw: bytes, content_type: str) -> None:
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Cache-Control", "no-store")
self.send_header("X-Content-Type-Options", "nosniff")
self.send_header("Referrer-Policy", "no-referrer")
self.send_header("Content-Security-Policy", "default-src 'none'")
self.send_header("Strict-Transport-Security", "max-age=31536000")
self.send_header("Content-Length", str(len(raw)))
self.end_headers()
self.wfile.write(raw)
def client_key(self) -> str:
return str(self.client_address[0])
def validate_host(self) -> None:
if not self.expected_domain:
return
host = self.headers.get("Host", "").split(":", 1)[0].lower()
if not hmac.compare_digest(host, self.expected_domain):
raise EdgeError("HOST_HEADER_DENY", 421)
def body(self) -> bytes:
try:
length = int(self.headers.get("Content-Length", "-1"))
except ValueError as exc:
raise EdgeError("CONTENT_LENGTH_INVALID") from exc
if length < 1 or length > MAX_BODY:
raise EdgeError("REQUEST_SIZE_INVALID", 413)
raw = self.rfile.read(length)
if len(raw) != length:
raise EdgeError("REQUEST_BODY_TRUNCATED", 400)
# Validate JSON at the edge so malformed bodies do not consume upstream work.
try:
value = json.loads(raw.decode("utf-8"))
except Exception as exc:
raise EdgeError("REQUEST_JSON_INVALID", 400) from exc
if not isinstance(value, Mapping):
raise EdgeError("REQUEST_JSON_OBJECT_REQUIRED", 400)
return canonical(value)
def handle_proxy(self, method: str) -> None:
client_id = _client_pseudonym(self.client_key())
started = time.monotonic()
acquired = self.concurrency.acquire(blocking=False)
if not acquired:
audit("EDGE_BUSY", client_id=client_id, method=method)
self.respond_json({"error": "EDGE_BUSY"}, 503)
return
label = "unknown"
try:
self.validate_host()
path = allowed_route(method, self.path)
label = route_label(method, path)
if path == "/health" and method == "GET":
self.respond_json(
{
"status": "ok",
"service": "aios-gpt-https-edge",
"upstream": "loopback-readonly-ingress",
},
200,
)
audit(
"EDGE_REQUEST",
client_id=client_id,
method=method,
route=label,
status=200,
duration_ms=int((time.monotonic() - started) * 1000),
)
return
if not self.limiter.allowed(self.client_key()):
audit(
"EDGE_RATE_LIMIT",
client_id=client_id,
method=method,
route=label,
)
self.respond_json({"error": "RATE_LIMITED"}, 429)
return
has_body = method == "POST"
headers = forwarded_headers(self.headers, body=has_body)
body = self.body() if has_body else None
status, raw, content_type = self.proxy.request(
method,
path,
headers,
body,
)
self.send_proxy_response(status, raw, content_type)
audit(
"EDGE_REQUEST",
client_id=client_id,
method=method,
route=label,
status=status,
duration_ms=int((time.monotonic() - started) * 1000),
)
except EdgeError as exc:
audit(
"EDGE_REJECT",
client_id=client_id,
method=method,
route=label,
status=exc.status,
error=exc.code,
)
self.respond_json({"error": exc.code}, exc.status)
finally:
self.concurrency.release()
def do_GET(self) -> None:
self.handle_proxy("GET")
def do_POST(self) -> None:
self.handle_proxy("POST")
def do_PUT(self) -> None:
self.respond_json({"error": "METHOD_DENY"}, 405)
def do_PATCH(self) -> None:
self.respond_json({"error": "METHOD_DENY"}, 405)
def do_DELETE(self) -> None:
self.respond_json({"error": "METHOD_DENY"}, 405)
def do_CONNECT(self) -> None:
self.respond_json({"error": "METHOD_DENY"}, 405)
def do_OPTIONS(self) -> None:
self.respond_json({"error": "METHOD_DENY"}, 405)
def serve() -> None:
host, public_domain = validated_edge_identity(
os.environ.get("AIOS_GPT_EDGE_HOST", "127.0.0.1"),
os.environ.get(PUBLIC_APPROVAL_ENV, "0"),
os.environ.get(PUBLIC_DOMAIN_ENV, ""),
)
port = int(os.environ.get("AIOS_GPT_EDGE_PORT", "18443"))
if not 1024 <= port <= 65535:
raise RuntimeError("GPT_EDGE_PORT_INVALID")
Handler.expected_domain = public_domain
credentials = Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
cert_path = Path(
os.environ.get(
"AIOS_GPT_EDGE_TLS_CERT_FILE",
str(credentials / "tls_cert"),
)
)
key_path = Path(
os.environ.get(
"AIOS_GPT_EDGE_TLS_KEY_FILE",
str(credentials / "tls_key"),
)
)
context = build_tls_context(cert_path, key_path)
server = EdgeServer((host, port), Handler)
server.socket = context.wrap_socket(server.socket, server_side=True)
server.serve_forever()
if __name__ == "__main__":
serve()
@@ -0,0 +1,816 @@
#!/usr/bin/env python3
"""Credential-isolated host-side ingress for externally scheduled read-only Online Tasks.
This service is intentionally not a public AIOS control plane. It accepts a tiny,
authenticated external intent and translates it to the canonical loopback Bridge.
Callers cannot select repository coordinates, capabilities, execution action,
worker, branch policy, credentials, verification decisions, or write permissions.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
import hashlib
import hmac
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
from pathlib import Path
import re
import sqlite3
import threading
import urllib.error
import urllib.request
from typing import Any, Mapping
SCHEMA = "aios.gpt-online-task-ingress.request.v1"
RESPONSE_SCHEMA = "aios.gpt-online-task-ingress.response.v1"
OPENAPI_VERSION = "3.1.0"
REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$")
TASK_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
JOB_ID_RE = re.compile(r"^job-[0-9a-f]{32}$")
OID_RE = re.compile(r"^[0-9a-f]{40}$")
MAX_BODY = 64 * 1024
MAX_GOAL = 12_000
MAX_INSTRUCTIONS = 24_000
MAX_CRITERIA = 16
MAX_CRITERION_TEXT = 1_000
READONLY_CAPABILITIES = ("git.read", "git.test")
REPOSITORY = "aios/aios"
class IngressError(RuntimeError):
def __init__(self, code: str, status: int = 400) -> None:
super().__init__(code)
self.code = code
self.status = status
def canonical(value: object) -> bytes:
return json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
def digest(value: object) -> str:
return hashlib.sha256(canonical(value)).hexdigest()
def now() -> str:
return datetime.now(timezone.utc).isoformat()
def load_secret(path: Path, *, code: str) -> str:
if path.is_symlink() or not path.is_file():
raise RuntimeError(code + "_UNAVAILABLE")
value = path.read_text(encoding="utf-8").strip()
if len(value) < 32 or "\n" in value or "\r" in value:
raise RuntimeError(code + "_INVALID")
return value
def normalized_request(value: Mapping[str, Any]) -> dict[str, Any]:
allowed = {
"schema",
"request_id",
"goal",
"instructions",
"success_criteria",
"constraints",
}
if set(value) - allowed:
raise IngressError("UNKNOWN_FIELDS")
if value.get("schema") != SCHEMA:
raise IngressError("SCHEMA_INVALID")
request_id = value.get("request_id")
if not isinstance(request_id, str) or REQUEST_ID_RE.fullmatch(request_id) is None:
raise IngressError("REQUEST_ID_INVALID")
goal = value.get("goal")
if not isinstance(goal, str) or not goal.strip() or len(goal) > MAX_GOAL:
raise IngressError("GOAL_INVALID")
instructions = value.get("instructions", "")
if not isinstance(instructions, str) or len(instructions) > MAX_INSTRUCTIONS:
raise IngressError("INSTRUCTIONS_INVALID")
criteria = value.get("success_criteria") or []
if not isinstance(criteria, list) or len(criteria) > MAX_CRITERIA:
raise IngressError("SUCCESS_CRITERIA_INVALID")
normalized_criteria = []
for item in criteria:
if not isinstance(item, Mapping) or set(item) != {"id", "description"}:
raise IngressError("SUCCESS_CRITERIA_INVALID")
criterion_id = item.get("id")
description = item.get("description")
if (
not isinstance(criterion_id, str)
or not criterion_id.strip()
or len(criterion_id) > 120
or not isinstance(description, str)
or not description.strip()
or len(description) > MAX_CRITERION_TEXT
):
raise IngressError("SUCCESS_CRITERIA_INVALID")
normalized_criteria.append(
{"id": criterion_id.strip(), "description": description.strip()}
)
constraints = value.get("constraints") or {}
if not isinstance(constraints, Mapping):
raise IngressError("CONSTRAINTS_INVALID")
allowed_constraints = {
"max_files",
"focus_paths",
"notes",
}
if set(constraints) - allowed_constraints:
raise IngressError("CONSTRAINTS_FIELDS_INVALID")
normalized_constraints: dict[str, Any] = {}
if "max_files" in constraints:
max_files = constraints["max_files"]
if (
not isinstance(max_files, int)
or isinstance(max_files, bool)
or not 1 <= max_files <= 500
):
raise IngressError("MAX_FILES_INVALID")
normalized_constraints["max_files"] = max_files
if "focus_paths" in constraints:
paths = constraints["focus_paths"]
if (
not isinstance(paths, list)
or len(paths) > 32
or not all(
isinstance(path, str)
and path
and len(path) <= 512
and not path.startswith("/")
and ".." not in Path(path).parts
for path in paths
)
):
raise IngressError("FOCUS_PATHS_INVALID")
normalized_constraints["focus_paths"] = paths
if "notes" in constraints:
notes = constraints["notes"]
if not isinstance(notes, str) or len(notes) > 4_000:
raise IngressError("CONSTRAINT_NOTES_INVALID")
normalized_constraints["notes"] = notes
return {
"schema": SCHEMA,
"request_id": request_id,
"goal": goal.strip(),
"instructions": instructions,
"success_criteria": normalized_criteria,
"constraints": normalized_constraints,
}
def validated_bind_host(value: str) -> str:
host = value.strip()
if host not in {"127.0.0.1", "::1", "localhost"}:
raise RuntimeError("GPT_INGRESS_NON_LOOPBACK_BIND_DENY")
return host
def task_id_for(request_id: str) -> str:
suffix = hashlib.sha256(request_id.encode("utf-8")).hexdigest()[:24]
return "gpt-readonly-" + suffix
def branch_for(task_id: str) -> str:
return "agent/external-gpt-readonly/" + task_id[-24:]
@dataclass(frozen=True)
class Upstream:
bridge_base: str
task_base: str
control_token: str
timeout: int = 20
def _request(
self,
base: str,
method: str,
path: str,
body: Mapping[str, Any] | None = None,
) -> tuple[int, dict[str, Any]]:
raw_body = None if body is None else canonical(body)
headers = {
"Authorization": "Bearer " + self.control_token,
"Accept": "application/json",
}
if raw_body is not None:
headers["Content-Type"] = "application/json"
req = urllib.request.Request(
base.rstrip("/") + path,
method=method,
data=raw_body,
headers=headers,
)
try:
with urllib.request.urlopen(req, timeout=self.timeout) as response:
raw = response.read()
status = response.status
except urllib.error.HTTPError as exc:
raw = exc.read()
status = exc.code
except Exception as exc:
raise IngressError("UPSTREAM_TRANSPORT_FAILED", 503) from exc
try:
value = json.loads(raw.decode("utf-8"))
except Exception as exc:
raise IngressError("UPSTREAM_RESPONSE_INVALID", 502) from exc
if not isinstance(value, Mapping):
raise IngressError("UPSTREAM_RESPONSE_INVALID", 502)
return status, dict(value)
def control_snapshot(self) -> dict[str, Any]:
status, value = self._request(
self.bridge_base,
"GET",
"/v2/control-plane-snapshot",
)
if status != 200:
raise IngressError("CONTROL_SNAPSHOT_UNAVAILABLE", 503)
return value
def create_governed_task(
self,
body: Mapping[str, Any],
) -> tuple[int, dict[str, Any]]:
return self._request(
self.bridge_base,
"POST",
"/v2/governed-task",
body,
)
def execution_envelope(
self,
body: Mapping[str, Any],
) -> tuple[int, dict[str, Any]]:
return self._request(
self.bridge_base,
"POST",
"/v2/execution-envelope",
body,
)
def get_task(self, task_id: str) -> tuple[int, dict[str, Any]]:
return self._request(
self.task_base,
"GET",
"/v1/tasks/" + task_id,
)
class Store:
def __init__(self, path: Path) -> None:
self.path = Path(path)
self._lock = threading.RLock()
self.path.parent.mkdir(parents=True, exist_ok=True)
with self._connect() as con:
con.execute(
"""
CREATE TABLE IF NOT EXISTS external_task_requests(
request_id TEXT PRIMARY KEY,
request_sha256 TEXT NOT NULL,
task_id TEXT NOT NULL UNIQUE,
job_id TEXT,
base_oid TEXT,
state TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
response_json TEXT
)
"""
)
con.commit()
def _connect(self) -> sqlite3.Connection:
con = sqlite3.connect(self.path, timeout=30)
con.row_factory = sqlite3.Row
con.execute("PRAGMA busy_timeout=30000")
return con
def get(self, request_id: str) -> sqlite3.Row | None:
with self._connect() as con:
return con.execute(
"SELECT * FROM external_task_requests WHERE request_id=?",
(request_id,),
).fetchone()
def reserve(self, request: Mapping[str, Any], task_id: str) -> sqlite3.Row:
request_sha = digest(request)
timestamp = now()
with self._lock, self._connect() as con:
con.execute("BEGIN IMMEDIATE")
row = con.execute(
"SELECT * FROM external_task_requests WHERE request_id=?",
(request["request_id"],),
).fetchone()
if row is not None:
if row["request_sha256"] != request_sha:
raise IngressError("IDEMPOTENCY_CONFLICT", 409)
con.commit()
return row
con.execute(
"""
INSERT INTO external_task_requests(
request_id,request_sha256,task_id,state,created_at,updated_at
) VALUES(?,?,?,?,?,?)
""",
(
request["request_id"],
request_sha,
task_id,
"PENDING",
timestamp,
timestamp,
),
)
con.commit()
row = self.get(request["request_id"])
assert row is not None
return row
def bound(
self,
request_id: str,
*,
job_id: str,
base_oid: str,
response: Mapping[str, Any],
) -> None:
with self._lock, self._connect() as con:
con.execute(
"""
UPDATE external_task_requests
SET job_id=?,base_oid=?,state='BOUND',response_json=?,updated_at=?
WHERE request_id=?
""",
(
job_id,
base_oid,
json.dumps(response, sort_keys=True, separators=(",", ":")),
now(),
request_id,
),
)
con.commit()
class IngressService:
def __init__(self, upstream: Upstream, store: Store) -> None:
self.upstream = upstream
self.store = store
@staticmethod
def _main_oid(snapshot: Mapping[str, Any]) -> str:
candidates = [
snapshot.get("authoritative_head_oid"),
snapshot.get("head"),
(snapshot.get("aggregate") or {}).get("authoritative_head_oid")
if isinstance(snapshot.get("aggregate"), Mapping)
else None,
]
for candidate in candidates:
if isinstance(candidate, str) and OID_RE.fullmatch(candidate):
return candidate
raise IngressError("AUTHORITATIVE_HEAD_UNAVAILABLE", 503)
@staticmethod
def _public_response(
request_id: str,
task_id: str,
job_id: str,
state: str,
) -> dict[str, Any]:
return {
"schema": RESPONSE_SCHEMA,
"request_id": request_id,
"task_id": task_id,
"job_id": job_id,
"state": state,
"execution_mode": "source_capture",
"capabilities": list(READONLY_CAPABILITIES),
"write_access": False,
"official_verification_required": True,
}
def _recover_existing(
self,
request: Mapping[str, Any],
row: sqlite3.Row,
) -> dict[str, Any] | None:
task_id = str(row["task_id"])
status, task = self.upstream.get_task(task_id)
if status == 404:
return None
if status != 200:
raise IngressError("TASK_RECOVERY_FAILED", 503)
metadata = task.get("metadata")
execution = (
metadata.get("execution")
if isinstance(metadata, Mapping)
and isinstance(metadata.get("execution"), Mapping)
else {}
)
job_id = str(execution.get("job_id") or row["job_id"] or "")
base_oid = str(execution.get("base_oid") or row["base_oid"] or "")
if (
task.get("task_id") != task_id
or not JOB_ID_RE.fullmatch(job_id)
or not OID_RE.fullmatch(base_oid)
or execution.get("action") != "source_capture"
or sorted(execution.get("capability_scope") or [])
!= sorted(READONLY_CAPABILITIES)
or (execution.get("governance") or {}).get("writer_lease") is not False
):
raise IngressError("TASK_RECOVERY_BINDING_INVALID", 409)
response = self._public_response(
request["request_id"],
task_id,
job_id,
str(task.get("status") or "BOUND"),
)
self.store.bound(
request["request_id"],
job_id=job_id,
base_oid=base_oid,
response=response,
)
return response
def create(self, raw: Mapping[str, Any]) -> dict[str, Any]:
request = normalized_request(raw)
task_id = task_id_for(request["request_id"])
row = self.store.reserve(request, task_id)
if row["state"] == "BOUND" and row["response_json"]:
return json.loads(row["response_json"])
recovered = self._recover_existing(request, row)
if recovered is not None:
return recovered
snapshot = self.upstream.control_snapshot()
base_oid = self._main_oid(snapshot)
target_branch = branch_for(task_id)
binding_key = "gpt-readonly-binding-" + digest(
{"request_id": request["request_id"], "base_oid": base_oid}
)[:24]
constraints = {
**request["constraints"],
"repository_mutation": "forbidden",
"control_plane_side_effects": "forbidden",
"credential_access": "forbidden",
}
body = {
"task_id": task_id,
"repository": REPOSITORY,
"base_oid": base_oid,
"expected_branch_oid": base_oid,
"target_branch": target_branch,
"branch_mode": "CREATE_NEW",
"generation": 1,
"risk": "low",
"description": "Externally requested read-only governed Online Task.",
"idempotency_key": binding_key,
"priority": 0,
"max_attempts": 3,
"require_online_task_projection": True,
"online_task": {
"goal": request["goal"],
"instructions": request["instructions"],
"success_criteria": request["success_criteria"],
"constraints": constraints,
"required_capabilities": list(READONLY_CAPABILITIES),
"priority": 0,
"max_attempts": 3,
},
"execution": {
"action": "source_capture",
"adapter": "source_capture",
"capability_scope": list(READONLY_CAPABILITIES),
"source_signals": [
"external read-only source analysis request",
"no repository mutation no credential access no control-plane side effects",
],
},
}
status, value = self.upstream.create_governed_task(body)
if status not in {200, 201, 202}:
if status == 409:
recovered = self._recover_existing(request, row)
if recovered is not None:
return recovered
raise IngressError(
"GOVERNED_TASK_CREATE_FAILED",
409 if status == 409 else 503,
)
job_id = str(value.get("job_id") or "")
if value.get("task_id") != task_id or JOB_ID_RE.fullmatch(job_id) is None:
raise IngressError("GOVERNED_TASK_RESPONSE_INVALID", 502)
envelope_status, envelope = self.upstream.execution_envelope(
{
"task_id": task_id,
"job_id": job_id,
"required_capabilities": list(READONLY_CAPABILITIES),
"execution": body["execution"],
}
)
if envelope_status != 200:
raise IngressError("EXECUTION_PROJECTION_FAILED", 503)
if (
envelope.get("task_id") != task_id
or envelope.get("job_id") != job_id
or envelope.get("base_oid") != base_oid
or envelope.get("action") != "source_capture"
or sorted(envelope.get("capability_scope") or [])
!= sorted(READONLY_CAPABILITIES)
or (envelope.get("governance") or {}).get("writer_lease") is not False
):
raise IngressError("EXECUTION_PROJECTION_INVALID", 502)
response = self._public_response(
request["request_id"],
task_id,
job_id,
"BOUND",
)
self.store.bound(
request["request_id"],
job_id=job_id,
base_oid=base_oid,
response=response,
)
return response
def status(self, request_id: str) -> dict[str, Any]:
if REQUEST_ID_RE.fullmatch(request_id) is None:
raise IngressError("REQUEST_ID_INVALID")
row = self.store.get(request_id)
if row is None:
raise IngressError("REQUEST_NOT_FOUND", 404)
task_id = str(row["task_id"])
status, task = self.upstream.get_task(task_id)
if status == 200 and not row["job_id"]:
metadata = task.get("metadata")
execution = (
metadata.get("execution")
if isinstance(metadata, Mapping)
and isinstance(metadata.get("execution"), Mapping)
else {}
)
job_id = str(execution.get("job_id") or "")
base_oid = str(execution.get("base_oid") or "")
if (
JOB_ID_RE.fullmatch(job_id)
and OID_RE.fullmatch(base_oid)
and execution.get("action") == "source_capture"
and sorted(execution.get("capability_scope") or [])
== sorted(READONLY_CAPABILITIES)
and (execution.get("governance") or {}).get("writer_lease") is False
):
recovered = self._public_response(
request_id,
task_id,
job_id,
str(task.get("status") or "BOUND"),
)
self.store.bound(
request_id,
job_id=job_id,
base_oid=base_oid,
response=recovered,
)
row = self.store.get(request_id) or row
if status == 404:
return {
"schema": RESPONSE_SCHEMA,
"request_id": request_id,
"task_id": task_id,
"state": "PENDING",
"official_verification_required": True,
}
if status != 200:
raise IngressError("TASK_STATUS_UNAVAILABLE", 503)
result = task.get("result") if isinstance(task.get("result"), Mapping) else {}
verification = (
task.get("verification")
if isinstance(task.get("verification"), Mapping)
else {}
)
safe = {
"schema": RESPONSE_SCHEMA,
"request_id": request_id,
"task_id": task_id,
"job_id": str(row["job_id"] or ""),
"state": str(task.get("status") or "UNKNOWN"),
"attempt": int(task.get("attempt") or 0),
"claimed": bool(task.get("claimed_by")),
"result_status": result.get("status"),
"result_classification": result.get("classification"),
"official_verification": verification.get("decision"),
"official_classification": verification.get("classification"),
"write_access": False,
}
if result.get("execution_mode") == "source_capture":
snapshot = result.get("source_snapshot")
if isinstance(snapshot, Mapping):
safe["source_snapshot_sha256"] = snapshot.get("payload_sha256")
safe["source_snapshot_file_count"] = snapshot.get("file_count")
return safe
def openapi_document() -> dict[str, Any]:
return {
"openapi": OPENAPI_VERSION,
"info": {
"title": "AIOS Read-Only Online Task Ingress",
"version": "1.0.0",
},
"paths": {
"/v1/read-only-tasks": {
"post": {
"operationId": "createReadOnlyOnlineTask",
"security": [{"bearerAuth": []}],
"requestBody": {"required": True},
"responses": {"200": {"description": "Task accepted"}},
}
},
"/v1/read-only-tasks/{request_id}": {
"get": {
"operationId": "getReadOnlyOnlineTaskStatus",
"security": [{"bearerAuth": []}],
"responses": {"200": {"description": "Sanitized task status"}},
}
},
},
"components": {
"securitySchemes": {
"bearerAuth": {
"type": "http",
"scheme": "bearer",
}
}
},
}
class Handler(BaseHTTPRequestHandler):
service: IngressService | None = None
ingress_token: str = ""
def log_message(self, *_args: Any) -> None:
return
def respond(self, value: Mapping[str, Any], status: int) -> None:
raw = canonical(value)
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Cache-Control", "no-store")
self.send_header("Content-Length", str(len(raw)))
self.end_headers()
self.wfile.write(raw)
def authorized(self) -> bool:
header = self.headers.get("Authorization", "")
return (
header.startswith("Bearer ")
and bool(self.ingress_token)
and hmac.compare_digest(header[7:], self.ingress_token)
)
def body(self) -> dict[str, Any]:
try:
length = int(self.headers.get("Content-Length", "-1"))
except ValueError as exc:
raise IngressError("CONTENT_LENGTH_INVALID") from exc
if length < 1 or length > MAX_BODY:
raise IngressError("REQUEST_SIZE_INVALID", 413)
try:
value = json.loads(self.rfile.read(length).decode("utf-8"))
except Exception as exc:
raise IngressError("REQUEST_JSON_INVALID") from exc
if not isinstance(value, Mapping):
raise IngressError("REQUEST_OBJECT_REQUIRED")
return dict(value)
def do_GET(self) -> None:
if self.path == "/health":
self.respond(
{
"status": "ok",
"service": "aios-gpt-online-task-ingress",
"mode": "read-only",
},
200,
)
return
if self.path == "/openapi.json":
self.respond(openapi_document(), 200)
return
if not self.authorized():
self.respond({"error": "UNAUTHORIZED"}, 401)
return
match = re.fullmatch(r"/v1/read-only-tasks/([^/?#]+)", self.path)
if match is None:
self.respond({"error": "NOT_FOUND"}, 404)
return
try:
if self.service is None:
raise IngressError("RUNTIME_UNAVAILABLE", 503)
self.respond(self.service.status(match.group(1)), 200)
except IngressError as exc:
self.respond({"error": exc.code}, exc.status)
def do_POST(self) -> None:
if self.path != "/v1/read-only-tasks":
self.respond({"error": "NOT_FOUND"}, 404)
return
if not self.authorized():
self.respond({"error": "UNAUTHORIZED"}, 401)
return
try:
if self.service is None:
raise IngressError("RUNTIME_UNAVAILABLE", 503)
value = self.service.create(self.body())
self.respond(value, 200)
except IngressError as exc:
self.respond({"error": exc.code}, exc.status)
def serve() -> None:
ingress_token_path = Path(
os.environ.get(
"AIOS_GPT_INGRESS_TOKEN_FILE",
str(
Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
/ "gpt_ingress_token"
),
)
)
control_token_path = Path(
os.environ.get(
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
str(
Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
/ "agent_task_control_token"
),
)
)
ingress_token = load_secret(
ingress_token_path,
code="GPT_INGRESS_CREDENTIAL",
)
control_token = load_secret(
control_token_path,
code="CONTROL_PLANE_CREDENTIAL",
)
state_root = Path(
os.environ.get(
"AIOS_GPT_INGRESS_STATE_ROOT",
"/var/lib/aios-gpt-online-task-ingress",
)
)
host = validated_bind_host(
os.environ.get("AIOS_GPT_INGRESS_HOST", "127.0.0.1")
)
port = int(os.environ.get("AIOS_GPT_INGRESS_PORT", "18945"))
if not 1024 <= port <= 65535:
raise RuntimeError("GPT_INGRESS_PORT_INVALID")
Handler.service = IngressService(
Upstream(
bridge_base=os.environ.get(
"AIOS_GOVERNED_WRITE_API_URL",
"http://127.0.0.1:18892",
),
task_base=os.environ.get(
"AIOS_AGENT_TASK_API_URL",
"http://127.0.0.1:18940",
),
control_token=control_token,
),
Store(state_root / "state.sqlite3"),
)
Handler.ingress_token = ingress_token
ThreadingHTTPServer((host, port), Handler).serve_forever()
if __name__ == "__main__":
serve()
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,232 @@
#!/usr/bin/env python3
"""Root-owned Model Governance configuration reconciler.
The config contains provider/model/policy metadata only. Raw credentials are rejected by
aios_model_governance and never stored in the governance database.
"""
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
from typing import Any, Mapping
from aios_model_governance import (
ModelGovernanceError,
ModelGovernanceStore,
ModelRecord,
ProviderRecord,
ScopePolicy,
)
SCHEMA = "aios.model-governance.bootstrap.v1"
MAX_CONFIG_BYTES = 512 * 1024
def canonical(value: object) -> bytes:
return json.dumps(
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
).encode("utf-8")
def fail(code: str) -> None:
raise SystemExit(code)
def _load(path: Path) -> dict[str, Any]:
if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_CONFIG_BYTES:
fail("MODEL_GOVERNANCE_CONFIG_INVALID")
try:
value = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
fail("MODEL_GOVERNANCE_CONFIG_INVALID")
if not isinstance(value, Mapping) or value.get("schema") != SCHEMA:
fail("MODEL_GOVERNANCE_CONFIG_SCHEMA_INVALID")
return dict(value)
def _provider_desired(value: Mapping[str, Any]) -> ProviderRecord:
return ProviderRecord(
provider_id=str(value.get("provider_id") or ""),
credential_ref=value.get("credential_ref"),
enabled=bool(value.get("enabled", True)),
failure_domain=str(value.get("failure_domain") or "default"),
max_failures=int(value.get("max_failures") or 3),
cooldown_seconds=int(value.get("cooldown_seconds") or 60),
)
def _model_desired(value: Mapping[str, Any]) -> ModelRecord:
return ModelRecord(
model_id=str(value.get("model_id") or ""),
provider_id=str(value.get("provider_id") or ""),
version=str(value.get("version") or ""),
capabilities=tuple(value.get("capabilities") or ()),
input_cost_micros_per_1k=int(value.get("input_cost_micros_per_1k") or 0),
output_cost_micros_per_1k=int(value.get("output_cost_micros_per_1k") or 0),
max_input_tokens=int(value.get("max_input_tokens") or 0),
max_output_tokens=int(value.get("max_output_tokens") or 0),
evaluation_state=str(value.get("evaluation_state") or "APPROVED"),
evaluation_sha256=str(value.get("evaluation_sha256") or ""),
enabled=bool(value.get("enabled", True)),
priority=int(value.get("priority") or 100),
)
def _policy_desired(value: Mapping[str, Any]) -> ScopePolicy:
return ScopePolicy(
scope_type=str(value.get("scope_type") or ""),
tenant_id=value.get("tenant_id"),
project_id=value.get("project_id"),
allowed_models=tuple(value.get("allowed_models") or ()),
denied_providers=tuple(value.get("denied_providers") or ()),
required_capabilities=tuple(value.get("required_capabilities") or ()),
pinned_model_id=value.get("pinned_model_id"),
pinned_model_version=value.get("pinned_model_version"),
max_input_tokens_per_request=int(value.get("max_input_tokens_per_request") or 0),
max_output_tokens_per_request=int(value.get("max_output_tokens_per_request") or 0),
daily_token_budget=int(value.get("daily_token_budget") or 0),
daily_cost_budget_micros=int(value.get("daily_cost_budget_micros") or 0),
fallback_limit=int(value.get("fallback_limit") or 0),
)
def _row_dict(row: object) -> dict[str, Any] | None:
return dict(row) if row is not None else None
def _provider_matches(row: Mapping[str, Any], desired: ProviderRecord) -> bool:
return (
row["credential_ref"] == desired.credential_ref
and bool(row["enabled"]) == desired.enabled
and row["failure_domain"] == desired.failure_domain
and int(row["max_failures"]) == desired.max_failures
and int(row["cooldown_seconds"]) == desired.cooldown_seconds
)
def _model_matches(row: Mapping[str, Any], desired: ModelRecord) -> bool:
return (
row["provider_id"] == desired.provider_id
and row["version"] == desired.version
and tuple(json.loads(row["capabilities_json"])) == tuple(sorted(set(desired.capabilities)))
and int(row["input_cost_micros_per_1k"]) == desired.input_cost_micros_per_1k
and int(row["output_cost_micros_per_1k"]) == desired.output_cost_micros_per_1k
and int(row["max_input_tokens"]) == desired.max_input_tokens
and int(row["max_output_tokens"]) == desired.max_output_tokens
and row["evaluation_state"] == desired.evaluation_state
and row["evaluation_sha256"] == desired.evaluation_sha256
and bool(row["enabled"]) == desired.enabled
and int(row["priority"]) == desired.priority
)
def _policy_matches(row: Mapping[str, Any], desired: ScopePolicy) -> bool:
return (
row["scope_type"] == desired.scope_type
and row["tenant_id"] == desired.tenant_id
and row["project_id"] == desired.project_id
and tuple(json.loads(row["allowed_models_json"])) == tuple(sorted(set(desired.allowed_models)))
and tuple(json.loads(row["denied_providers_json"])) == tuple(sorted(set(desired.denied_providers)))
and tuple(json.loads(row["required_capabilities_json"])) == tuple(sorted(set(desired.required_capabilities)))
and row["pinned_model_id"] == desired.pinned_model_id
and row["pinned_model_version"] == desired.pinned_model_version
and int(row["max_input_tokens_per_request"]) == desired.max_input_tokens_per_request
and int(row["max_output_tokens_per_request"]) == desired.max_output_tokens_per_request
and int(row["daily_token_budget"]) == desired.daily_token_budget
and int(row["daily_cost_budget_micros"]) == desired.daily_cost_budget_micros
and int(row["fallback_limit"]) == desired.fallback_limit
)
def apply_config(db_path: Path, config_path: Path) -> dict[str, Any]:
config = _load(config_path)
config_sha = hashlib.sha256(canonical(config)).hexdigest()
providers = config.get("providers") or []
models = config.get("models") or []
policies = config.get("policies") or []
if not all(isinstance(value, list) for value in (providers, models, policies)):
fail("MODEL_GOVERNANCE_CONFIG_COLLECTION_INVALID")
store = ModelGovernanceStore(db_path)
changes = {"providers": 0, "models": 0, "policies": 0, "unchanged": 0}
try:
for item in providers:
if not isinstance(item, Mapping):
fail("MODEL_GOVERNANCE_PROVIDER_CONFIG_INVALID")
desired = _provider_desired(item)
row = store.conn.execute(
"SELECT * FROM model_governance_providers WHERE provider_id=?",
(desired.provider_id,),
).fetchone()
if row is not None and _provider_matches(row, desired):
changes["unchanged"] += 1
continue
expected = int(row["revision"]) if row is not None else 0
store.register_provider(desired, expected_revision=expected)
changes["providers"] += 1
for item in models:
if not isinstance(item, Mapping):
fail("MODEL_GOVERNANCE_MODEL_CONFIG_INVALID")
desired = _model_desired(item)
row = store.conn.execute(
"SELECT * FROM model_governance_models WHERE model_id=?",
(desired.model_id,),
).fetchone()
if row is not None and _model_matches(row, desired):
changes["unchanged"] += 1
continue
expected = int(row["revision"]) if row is not None else 0
store.register_model(desired, expected_revision=expected)
changes["models"] += 1
for item in policies:
if not isinstance(item, Mapping):
fail("MODEL_GOVERNANCE_POLICY_CONFIG_INVALID")
desired = _policy_desired(item)
row = store.conn.execute(
"SELECT * FROM model_governance_scope_policies WHERE scope_key=?",
(desired.scope_key,),
).fetchone()
if row is not None and _policy_matches(row, desired):
changes["unchanged"] += 1
continue
expected = int(row["revision"]) if row is not None else 0
store.set_scope_policy(desired, expected_revision=expected)
changes["policies"] += 1
if not store.verify_audit_chain():
fail("MODEL_GOVERNANCE_AUDIT_CHAIN_INVALID")
return {
"schema": "aios.model-governance.bootstrap-result.v1",
"database": str(db_path),
"config_sha256": config_sha,
"changes": changes,
"audit_chain_valid": True,
}
finally:
store.close()
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--db", required=True)
parser.add_argument("--config", required=True)
parser.add_argument("--output")
args = parser.parse_args()
try:
result = apply_config(Path(args.db), Path(args.config))
except ModelGovernanceError as exc:
print(json.dumps({"status": "FAILED", "error": exc.code}, sort_keys=True))
return 20
data = json.dumps(result, indent=2, sort_keys=True) + "\n"
if args.output:
Path(args.output).write_text(data, encoding="utf-8")
print(data, end="")
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -17,6 +17,7 @@ PHASE_ORDER = {
"claimed": 10,
"workspace_prepared": 20,
"verification_passed": 30,
"agent_runtime_done": 40,
"source_capture_done": 40,
"commit_done": 40,
"reconciliation_required": 45,
@@ -42,7 +42,7 @@ DEFAULT_TERMS = {
"backoff", "circuit", "health", "liveness", "scope", "tenant", "project", "trace",
"message", "checkpoint", "recovery", "idempotency", "dispatch", "assignment",
}
CONTRACT_TERMS: dict[str, tuple[str, ...]] = {
FLEET_CONTRACT_TERMS: dict[str, tuple[str, ...]] = {
"FLEET_AGENT_REGISTRATION": ("agent", "register", "worker", "capability", "version"),
"FLEET_CAPACITY_SCHEDULING": ("capacity", "slot", "concurrency", "scheduler", "queue"),
"FLEET_FAIRNESS": ("fair", "priority", "queue", "tenant", "project", "starvation"),
@@ -52,6 +52,30 @@ CONTRACT_TERMS: dict[str, tuple[str, ...]] = {
"FLEET_IDEMPOTENT_DISPATCH": ("idempot", "dispatch", "claim", "lease", "result", "attempt"),
"FLEET_FAILURE_RECOVERY": ("checkpoint", "resume", "recover", "retry", "cleanup", "receipt"),
}
MODEL_GOVERNANCE_CONTRACT_TERMS: dict[str, tuple[str, ...]] = {
"MODEL_PROVIDER_REGISTRY": ("model", "provider", "registry", "endpoint", "capability"),
"MODEL_SELECTION_POLICY": ("model", "select", "route", "policy", "capability"),
"MODEL_VERSION_PINNING": ("model", "version", "pin", "revision", "snapshot"),
"MODEL_BUDGET_ACCOUNTING": ("budget", "token", "cost", "quota", "usage"),
"MODEL_FALLBACK_POLICY": ("fallback", "retry", "route", "provider", "model"),
"MODEL_SCOPE_ISOLATION": ("tenant", "project", "scope", "policy", "model"),
"MODEL_CREDENTIAL_BOUNDARY": ("credential", "secret", "provider", "tool", "token"),
"MODEL_TRACE_AUDIT": ("trace", "audit", "model", "provider", "request"),
"MODEL_EVALUATION_GATE": ("eval", "evaluation", "approval", "quality", "gate"),
"MODEL_ROUTING_RESILIENCE": ("route", "retry", "fallback", "timeout", "health"),
}
PROFILE_CLASSIFICATION = {
"fleet": "P2_INTERFACE_MAP_CAPTURED",
"model_governance": "MODEL_GOVERNANCE_INTERFACE_MAP_CAPTURED",
}
MODEL_PROFILE_MARKERS = {
"model", "provider", "llm", "inference", "routing", "router", "budget",
"token", "cost", "quota", "fallback", "evaluation", "eval", "approval",
"version", "pinning",
}
SENSITIVE_RE = re.compile(
r"(?i)(authorization\s*:\s*bearer\s+\S+|"
r"(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*['\"]?[^'\"\s,;]+)"
@@ -92,6 +116,27 @@ def _signal_terms(signals: Sequence[str]) -> set[str]:
return set(sorted(terms)[:96])
def _profile(source_signals: Sequence[str]) -> str:
tokens: set[str] = set()
for signal in source_signals:
if not isinstance(signal, str):
continue
tokens.update(
token.lower()
for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]{2,63}", signal)
)
model_score = len(tokens & MODEL_PROFILE_MARKERS)
return "model_governance" if model_score >= 2 else "fleet"
def _contract_terms(profile: str) -> Mapping[str, tuple[str, ...]]:
if profile == "model_governance":
return MODEL_GOVERNANCE_CONTRACT_TERMS
if profile == "fleet":
return FLEET_CONTRACT_TERMS
raise CompactInterfaceMapError("INTERFACE_MAP_PROFILE_INVALID")
def _candidate_files(root: Path, terms: set[str]) -> list[dict[str, Any]]:
root = root.resolve()
candidates: list[dict[str, Any]] = []
@@ -190,10 +235,11 @@ def _surface_ref(symbol: Mapping[str, Any]) -> str:
def _contract_map(
symbols: Sequence[Mapping[str, Any]],
modules: Sequence[Mapping[str, Any]],
contract_terms: Mapping[str, tuple[str, ...]],
) -> dict[str, Any]:
result: dict[str, Any] = {}
module_paths = [str(item["path"]) for item in modules]
for contract, terms in CONTRACT_TERMS.items():
for contract, terms in contract_terms.items():
scored: list[tuple[int, str]] = []
for symbol in symbols:
haystack = _surface_ref(symbol).lower()
@@ -216,37 +262,67 @@ def _contract_map(
return result
def _gap_evidence(symbols: Sequence[Mapping[str, Any]]) -> tuple[list[str], list[str], list[str]]:
def _gap_evidence(
symbols: Sequence[Mapping[str, Any]],
profile: str,
) -> tuple[list[str], list[str], list[str]]:
haystack = "\n".join(_surface_ref(item).lower() for item in symbols)
invariants = [
"Online Task claim/heartbeat remains the authoritative worker ownership boundary.",
"Governed mutation remains separated from read-only analysis and requires WriterLease.",
"P1 attempt-aware result idempotency and durable checkpoint recovery are reusable fleet primitives.",
"Scope/trace/message lineage must remain attached when work is scheduled or reassigned.",
]
failure_windows: list[str] = []
if "scheduler" not in haystack:
failure_windows.append("No first-class durable fleet scheduler symbol detected; worker-local polling may still be the scheduling mechanism.")
if "capacity" not in haystack and "slot" not in haystack:
failure_windows.append("No explicit durable worker-capacity/slot primitive detected.")
if "fair" not in haystack and "starvation" not in haystack:
failure_windows.append("No explicit fairness/starvation policy symbol detected.")
if "circuit" not in haystack:
failure_windows.append("No explicit circuit-breaker primitive detected for repeated worker/task failures.")
if "reassign" not in haystack:
failure_windows.append("No explicit worker-reassignment primitive detected beyond lease expiry/retry semantics.")
if profile == "model_governance":
invariants = [
"Online Task claim/heartbeat remains the authoritative worker ownership boundary.",
"Provider/model credentials remain outside uncredentialed planner/executor processes.",
"P0-A tenant/project scope and P0-C trace/audit lineage must constrain model routing.",
"P1 retry/checkpoint semantics remain authoritative for model-call execution recovery.",
]
failure_windows: list[str] = []
if "provider" not in haystack and "model" not in haystack:
failure_windows.append("No first-class model/provider registry surface detected.")
if "budget" not in haystack and "cost" not in haystack and "quota" not in haystack:
failure_windows.append("No explicit durable model budget/cost/quota accounting surface detected.")
if "fallback" not in haystack and "route" not in haystack:
failure_windows.append("No explicit model fallback/routing policy surface detected.")
if "eval" not in haystack and "approval" not in haystack:
failure_windows.append("No explicit evaluation/approval gate surface detected for model changes.")
if "version" not in haystack and "pin" not in haystack:
failure_windows.append("No explicit model-version pinning surface detected.")
desired = (
("trace", "P0-C trace/audit lineage"),
("scope", "P0-A tenant/project scope isolation"),
("credential", "P0-C credential governance"),
("checkpoint", "P1 durable execution recovery"),
("message", "P0-B MessageEnvelope/handoff"),
("retry", "P1 retry/idempotency"),
)
else:
invariants = [
"Online Task claim/heartbeat remains the authoritative worker ownership boundary.",
"Governed mutation remains separated from read-only analysis and requires WriterLease.",
"P1 attempt-aware result idempotency and durable checkpoint recovery are reusable fleet primitives.",
"Scope/trace/message lineage must remain attached when work is scheduled or reassigned.",
]
failure_windows = []
if "scheduler" not in haystack:
failure_windows.append("No first-class durable fleet scheduler symbol detected; worker-local polling may still be the scheduling mechanism.")
if "capacity" not in haystack and "slot" not in haystack:
failure_windows.append("No explicit durable worker-capacity/slot primitive detected.")
if "fair" not in haystack and "starvation" not in haystack:
failure_windows.append("No explicit fairness/starvation policy symbol detected.")
if "circuit" not in haystack:
failure_windows.append("No explicit circuit-breaker primitive detected for repeated worker/task failures.")
if "reassign" not in haystack:
failure_windows.append("No explicit worker-reassignment primitive detected beyond lease expiry/retry semantics.")
desired = (
("checkpoint", "P1 durable checkpoint/resume"),
("message", "P0-B MessageEnvelope/handoff"),
("trace", "P0-C trace correlation"),
("scope", "P0-A scope isolation"),
("lease", "claim/WriterLease ownership"),
("heartbeat", "worker liveness"),
("idempot", "retry-safe dispatch/result"),
)
reuse_points: list[str] = []
desired = (
("checkpoint", "P1 durable checkpoint/resume"),
("message", "P0-B MessageEnvelope/handoff"),
("trace", "P0-C trace correlation"),
("scope", "P0-A scope isolation"),
("lease", "claim/WriterLease ownership"),
("heartbeat", "worker liveness"),
("idempot", "retry-safe dispatch/result"),
)
for token, label in desired:
match = next((item for item in symbols if token in _surface_ref(item).lower()), None)
if match is not None:
@@ -287,6 +363,7 @@ def build_compact_interface_map(
if re.fullmatch(r"[0-9a-f]{40}", source_commit) is None:
raise CompactInterfaceMapError("INTERFACE_MAP_SOURCE_COMMIT_INVALID")
profile = _profile(source_signals)
terms = _signal_terms(source_signals)
candidates = _candidate_files(root, terms)
if not candidates:
@@ -363,9 +440,11 @@ def build_compact_interface_map(
}
)
invariants, failure_windows, reuse_points = _gap_evidence(all_symbols)
invariants, failure_windows, reuse_points = _gap_evidence(all_symbols, profile)
result = {
"schema": SCHEMA,
"profile": profile,
"classification": PROFILE_CLASSIFICATION[profile],
"source_commit": source_commit,
"implementation_map": {
"modules": modules,
@@ -379,6 +458,6 @@ def build_compact_interface_map(
"reuse_points": reuse_points,
},
"bounded_excerpts": excerpts,
"contracts": _contract_map(all_symbols, modules),
"contracts": _contract_map(all_symbols, modules, _contract_terms(profile)),
}
return _trim_to_budget(result)
@@ -24,7 +24,10 @@ BRANCH_RE = re.compile(r"^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$")
KEY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,159}$")
PROTECTED_PREFIXES = ("main", "master", "refs/", "tags/", "tag/", "release/", "baseline/", "production/")
MUTATION_ACTIONS = frozenset({"deterministic"})
SUPPORTED_ACTIONS = frozenset({"deterministic", "source_capture"})
READ_ONLY_ACTIONS = frozenset({"source_capture", "agent_runtime"})
SUPPORTED_ACTIONS = frozenset({"deterministic", "source_capture", "agent_runtime"})
AGENT_RUNTIME_ADAPTERS = frozenset({"hermes", "openclaw"})
MAX_AGENT_RUNTIME_INPUT_BYTES = 64 * 1024
class ExecutionProjectionError(RuntimeError):
@@ -172,6 +175,76 @@ def _source_signals(value: object) -> tuple[str, ...]:
return tuple(result)
def _agent_runtime_input(value: object) -> dict[str, Any]:
if not isinstance(value, Mapping):
_error("AGENT_RUNTIME_INPUT_REQUIRED")
raw = dict(value)
allowed = {
"goal", "instructions", "success_criteria", "constraints",
"scope", "trace_id",
}
if set(raw) - allowed:
_error("AGENT_RUNTIME_INPUT_FIELDS_INVALID")
goal = _string(raw.get("goal"), "AGENT_RUNTIME_GOAL_REQUIRED")
instructions = raw.get("instructions", "")
if not isinstance(instructions, str):
_error("AGENT_RUNTIME_INSTRUCTIONS_INVALID")
criteria = raw.get("success_criteria", [])
if not isinstance(criteria, list) or len(criteria) > 32:
_error("AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID")
normalized_criteria: list[dict[str, str]] = []
for item in criteria:
if not isinstance(item, Mapping):
_error("AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID")
criterion_id = _string(item.get("id"), "AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID")
description = _string(
item.get("description"), "AGENT_RUNTIME_SUCCESS_CRITERIA_INVALID"
)
normalized_criteria.append(
{"id": criterion_id[:120], "description": description[:1000]}
)
constraints = raw.get("constraints", {})
if not isinstance(constraints, Mapping):
_error("AGENT_RUNTIME_CONSTRAINTS_INVALID")
sensitive_markers = ("token", "secret", "password", "credential", "authorization", "api_key", "apikey")
for key in constraints:
lowered = str(key).lower()
if any(marker in lowered for marker in sensitive_markers):
_error("AGENT_RUNTIME_SECRET_FIELD_DENY")
scope = raw.get("scope", {})
if not isinstance(scope, Mapping):
_error("AGENT_RUNTIME_SCOPE_INVALID")
scope = {
"scope_type": str(scope.get("scope_type") or "legacy"),
"tenant_id": scope.get("tenant_id"),
"project_id": scope.get("project_id"),
}
if scope["scope_type"] not in {"legacy", "system", "tenant"}:
_error("AGENT_RUNTIME_SCOPE_INVALID")
if scope["scope_type"] == "tenant":
if not isinstance(scope["tenant_id"], str) or not scope["tenant_id"].strip():
_error("AGENT_RUNTIME_SCOPE_INVALID")
elif scope["tenant_id"] not in (None, "") or scope["project_id"] not in (None, ""):
_error("AGENT_RUNTIME_SCOPE_INVALID")
trace_id = str(raw.get("trace_id") or "").strip()
if not trace_id or len(trace_id) > 255:
_error("AGENT_RUNTIME_TRACE_ID_INVALID")
normalized = {
"goal": goal[:12000],
"instructions": instructions[:32000],
"success_criteria": normalized_criteria,
"constraints": dict(constraints),
"scope": scope,
"trace_id": trace_id,
}
if len(_canonical(normalized).encode("utf-8")) > MAX_AGENT_RUNTIME_INPUT_BYTES:
_error("AGENT_RUNTIME_INPUT_TOO_LARGE")
return normalized
def _capabilities(value: object) -> tuple[str, ...]:
if value is None:
return ()
@@ -288,6 +361,22 @@ def validate_execution_envelope(
signals = list(_source_signals(envelope.get("source_signals") or []))
if envelope.get("source_signals") != signals:
_error("SOURCE_CAPTURE_SIGNALS_INVALID")
if envelope.get("runtime_input") not in (None, {}):
_error("SOURCE_CAPTURE_RUNTIME_INPUT_DENY")
elif action == "agent_runtime":
if changes or verification:
_error("AGENT_RUNTIME_MUTATION_DENY")
adapter = str(envelope.get("adapter") or "")
if adapter not in AGENT_RUNTIME_ADAPTERS:
_error("AGENT_RUNTIME_ADAPTER_INVALID")
runtime_input = _agent_runtime_input(envelope.get("runtime_input"))
if envelope.get("runtime_input") != runtime_input:
_error("AGENT_RUNTIME_INPUT_NOT_CANONICAL")
required_runtime_capability = f"agent.runtime.{adapter}"
if required_runtime_capability not in capability_scope:
_error("AGENT_RUNTIME_CAPABILITY_REQUIRED")
if envelope.get("source_signals") not in (None, []):
_error("AGENT_RUNTIME_SOURCE_SIGNALS_DENY")
elif not changes:
_error("CHANGE_SET_INVALID")
declared = str(envelope.get("execution_digest") or "")
@@ -329,11 +418,15 @@ def project_execution_envelope(
"schema": EXECUTION_SCHEMA,
**binding.public(),
"action": action,
"adapter": str(request.get("adapter") or ("deterministic" if action == "deterministic" else "source_capture")),
"adapter": str(
request.get("adapter")
or ("deterministic" if action == "deterministic" else "source_capture")
),
"capability_scope": list(capabilities),
"change_set": list(request.get("change_set") or []),
"verification": list(request.get("verification") or []),
"source_signals": list(_source_signals(request.get("source_signals") or [])) if action == "source_capture" else [],
"runtime_input": _agent_runtime_input(request.get("runtime_input")) if action == "agent_runtime" else {},
"governance": governance,
"workspace": {
"authority_git_dir": "/data/aios-gitops/repo.git",
@@ -352,7 +445,45 @@ def project_online_task(
) -> dict[str, Any]:
request = dict(task_request)
required = _capabilities(request.get("required_capabilities"))
execution = project_execution_envelope(binding, execution_request, required_capabilities=required)
execution_value = dict(execution_request or {})
if str(execution_value.get("action") or "source_capture") == "agent_runtime":
adapter = str(execution_value.get("adapter") or "")
if adapter not in AGENT_RUNTIME_ADAPTERS:
_error("AGENT_RUNTIME_ADAPTER_INVALID")
runtime_input = execution_value.get("runtime_input")
tenant_id = request.get("tenant_id")
project_id = request.get("project_id")
declared_scope = str(
request.get("scope_type")
or ("tenant" if isinstance(tenant_id, str) and tenant_id.strip() else "legacy")
)
if runtime_input is None:
runtime_input = {
"goal": request.get("goal"),
"instructions": request.get("instructions") or "",
"success_criteria": request.get("success_criteria") or [],
"constraints": request.get("constraints") or {},
}
elif not isinstance(runtime_input, Mapping):
_error("AGENT_RUNTIME_INPUT_REQUIRED")
runtime_input = dict(runtime_input)
# Scope/trace are authoritative Task identity, never caller-selected runtime hints.
runtime_input["scope"] = {
"scope_type": declared_scope,
"tenant_id": tenant_id,
"project_id": project_id,
}
runtime_input["trace_id"] = str(
request.get("trace_id")
or request.get("run_id")
or binding.task_id
)
execution_value["runtime_input"] = runtime_input
execution = project_execution_envelope(
binding,
execution_value,
required_capabilities=required,
)
metadata = dict(request.get("metadata") or {})
if "execution" in metadata:
validate_execution_envelope(_object(metadata["execution"], "EXECUTION_ENVELOPE_INVALID"), binding=binding)
+406 -22
View File
@@ -44,6 +44,10 @@ from aios_online_agent_checkpoint import (
CheckpointError,
CheckpointStore,
)
from aios_external_agent_runtime import (
ExternalAgentExchange,
ExternalAgentRuntimeError,
)
from aios_online_agent_compact_map import (
CompactInterfaceMapError,
build_compact_interface_map,
@@ -83,6 +87,23 @@ WORKER_CAPABILITIES = tuple(
if item.strip()
)
)
WORKER_ALLOWED_ADAPTERS = frozenset(
item.strip()
for item in os.environ.get(
"AIOS_WORKER_ALLOWED_ADAPTERS",
"deterministic,source_capture,compact_interface_map,codex",
).split(",")
if item.strip()
)
EXTERNAL_AGENT_EXECUTOR_ROOT = Path(
os.environ.get(
"AIOS_EXTERNAL_AGENT_EXECUTOR_ROOT",
"/var/lib/aios-external-agent-executor",
)
)
EXTERNAL_AGENT_EXECUTOR_TIMEOUT_SECONDS = int(
os.environ.get("AIOS_EXTERNAL_AGENT_EXECUTOR_TIMEOUT_SECONDS", "1800")
)
MAX_TASK_BYTES = 512 * 1024
MAX_FILE_BYTES = 2 * 1024 * 1024
ISOLATED_EXECUTOR_ROOT = Path(
@@ -215,6 +236,64 @@ def atomic_json(path: Path, value: Mapping[str, Any], mode: int = 0o640) -> None
os.unlink(temporary)
WORKER_LIVENESS_PATH = Path(
os.environ.get(
"AIOS_WORKER_LIVENESS_PATH",
str(STATE_ROOT / "worker-liveness.json"),
)
)
_LIVENESS_LOCK = threading.Lock()
_LIVENESS_SEQUENCE = 0
ENGINEERING_WRITE_CAPABILITIES = frozenset({
"governed.write.branch",
"governed.write.file",
})
def write_worker_liveness(
state: str,
*,
visible_count: int | None = None,
task_id: str | None = None,
stage: str | None = None,
error: str | None = None,
) -> None:
"""Write idle/busy liveness without touching Task/Job state."""
global _LIVENESS_SEQUENCE
with _LIVENESS_LOCK:
_LIVENESS_SEQUENCE += 1
epoch = time.time()
payload = {
"schema": "aios.online-agent.worker-liveness.v1",
"agent_id": AGENT_ID,
"agent_type": AGENT_TYPE,
"agent_version": AGENT_VERSION,
"pid": os.getpid(),
"state": state,
"sequence": _LIVENESS_SEQUENCE,
"last_seen_epoch": epoch,
"last_seen_at": datetime.fromtimestamp(epoch, timezone.utc).isoformat(),
"poll_seconds": POLL_SECONDS,
"heartbeat_seconds": HEARTBEAT_SECONDS,
"api_base": API_BASE,
"bridge_base": BRIDGE_BASE,
"visible_count": visible_count,
"task_id": task_id,
"stage": stage,
"error": error,
}
atomic_json(WORKER_LIVENESS_PATH, payload)
def source_capture_requires_deterministic_plan(envelope: "ExecutionEnvelope") -> bool:
"""Treat write-capable source_capture as a binding placeholder, never executable work."""
return (
envelope.action == "source_capture"
and bool(ENGINEERING_WRITE_CAPABILITIES.intersection(envelope.capability_scope))
)
def safe_relative_path(value: Any, *, field: str = "path") -> str:
if not isinstance(value, str) or not value or len(value) > 240:
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
@@ -359,6 +438,7 @@ class ExecutionEnvelope:
governance: dict[str, Any]
workspace: dict[str, Any]
digest: str
runtime_input: dict[str, Any] | None = None
@classmethod
def from_task(
@@ -403,7 +483,7 @@ class ExecutionEnvelope:
adapter = str(adapter_value.get("name") or "")
else:
adapter = str(adapter_value or "")
if adapter not in {"deterministic", "source_capture", "compact_interface_map", "codex", "hermes"}:
if adapter not in {"deterministic", "source_capture", "compact_interface_map", "codex", "hermes", "openclaw"}:
raise WorkerError("RUNTIME_ADAPTER_UNSUPPORTED", phase="validation")
change_set = execution.get("change_set")
if not isinstance(change_set, list) or len(change_set) > 128 or (action == "deterministic" and not change_set):
@@ -494,6 +574,7 @@ class ExecutionEnvelope:
adapter=adapter,
capability_scope=tuple(execution.get("capability_scope") or ()),
source_signals=tuple(execution.get("source_signals") or ()),
runtime_input=dict(execution.get("runtime_input") or {}),
change_set=tuple(normalized_changes),
verification=tuple(normalized_verification),
governance=governance_value,
@@ -502,11 +583,44 @@ class ExecutionEnvelope:
)
def external_agent_request_id(
envelope: ExecutionEnvelope,
runtime_name: str,
attempt: int,
) -> str:
if not isinstance(attempt, int) or isinstance(attempt, bool) or attempt < 1:
raise WorkerError("TASK_ATTEMPT_INVALID", phase="external_agent")
if runtime_name not in {"hermes", "openclaw"}:
raise WorkerError(
"EXTERNAL_AGENT_RUNTIME_UNSUPPORTED",
phase="external_agent",
details={"runtime": runtime_name},
)
return (
f"external-{runtime_name}-"
+ digest_json(
{
"task_id": envelope.task_id,
"job_id": envelope.job_id,
"execution_digest": envelope.digest,
"runtime": runtime_name,
"attempt": attempt,
}
)[:32]
)
class RuntimeAdapter(ABC):
name: str
@abstractmethod
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
raise NotImplementedError
@@ -527,7 +641,13 @@ class DeterministicExecutor(RuntimeAdapter):
if os.path.exists(temporary):
os.unlink(temporary)
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
changed: set[str] = set()
for change in envelope.change_set:
relative = str(change["path"])
@@ -582,7 +702,13 @@ class SourceCaptureExecutor(RuntimeAdapter):
"governed", "trace", "tool", "credential", "scope", "message", "handoff",
}
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
terms = set(self.defaults)
for signal in envelope.source_signals:
terms.update(token.lower() for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]{2,63}", signal))
@@ -651,7 +777,13 @@ class CompactInterfaceMapExecutor(RuntimeAdapter):
name = "compact_interface_map"
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
if envelope.action != "source_capture":
raise WorkerError("COMPACT_INTERFACE_MAP_ACTION_INVALID", phase="interface_map")
try:
@@ -681,7 +813,13 @@ class CompactInterfaceMapExecutor(RuntimeAdapter):
class CodexExecutor(RuntimeAdapter):
name = "codex"
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
raise WorkerError(
"CODEX_ADAPTER_REQUIRES_EXPLICIT_POLICY",
phase="adapter",
@@ -689,15 +827,78 @@ class CodexExecutor(RuntimeAdapter):
)
class HermesExecutor(RuntimeAdapter):
name = "hermes"
class ExternalAgentExecutor(RuntimeAdapter):
runtime_name: str
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
raise WorkerError(
"HERMES_ENGINEERING_ADAPTER_UNAVAILABLE",
phase="adapter",
details={"reason": "The live Hermes service is a user-chat runtime, not an engineering executor"},
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
if envelope.action != "agent_runtime":
raise WorkerError(
"EXTERNAL_AGENT_ACTION_INVALID",
phase="adapter",
details={"adapter": self.runtime_name, "action": envelope.action},
)
if envelope.adapter != self.runtime_name:
raise WorkerError(
"EXTERNAL_AGENT_ADAPTER_BINDING_INVALID",
phase="adapter",
)
if not envelope.runtime_input:
raise WorkerError("EXTERNAL_AGENT_RUNTIME_INPUT_MISSING", phase="adapter")
request_id = external_agent_request_id(
envelope,
self.runtime_name,
attempt,
)
exchange = ExternalAgentExchange(EXTERNAL_AGENT_EXECUTOR_ROOT)
try:
external = exchange.execute(
request_id=request_id,
runtime=self.runtime_name,
task_id=envelope.task_id,
attempt=attempt,
execution_digest=envelope.digest,
payload=envelope.runtime_input,
timeout_seconds=EXTERNAL_AGENT_EXECUTOR_TIMEOUT_SECONDS,
)
except ExternalAgentRuntimeError as exc:
raise WorkerError(
str(exc),
phase="external_agent",
details={"adapter": self.runtime_name},
) from exc
audit(
"EXTERNAL_AGENT_RUNTIME_COMPLETED",
task_id=envelope.task_id,
adapter=self.runtime_name,
request_id=request_id,
attempt=attempt,
output_sha256=external.get("output_sha256"),
duration_ms=external.get("duration_ms"),
credential_material_exposed=False,
)
return {
"changed_files": [],
"commands": [],
"tests": [],
"adapter": self.runtime_name,
"external_agent": external,
}
class HermesExecutor(ExternalAgentExecutor):
name = "hermes"
runtime_name = "hermes"
class OpenClawExecutor(ExternalAgentExecutor):
name = "openclaw"
runtime_name = "openclaw"
def adapter_for(name: str) -> RuntimeAdapter:
@@ -707,6 +908,7 @@ def adapter_for(name: str) -> RuntimeAdapter:
"compact_interface_map": CompactInterfaceMapExecutor,
"codex": CodexExecutor,
"hermes": HermesExecutor,
"openclaw": OpenClawExecutor,
}[name]()
@@ -805,6 +1007,11 @@ class LeaseHeartbeat:
self.failed = str(response.get("error") or status)
raise WorkerError("TASK_HEARTBEAT_FAILED", phase="heartbeat", details={"api_status": status, "error": self.failed})
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=self.stage, api_status=status)
write_worker_liveness(
"BUSY",
task_id=self.task_id,
stage=self.stage,
)
def start(self) -> None:
self.thread.start()
@@ -824,6 +1031,11 @@ class LeaseHeartbeat:
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, api_status=status, error=self.failed)
else:
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=stage, api_status=status)
write_worker_liveness(
"BUSY",
task_id=self.task_id,
stage=stage,
)
except Exception as exc: # pragma: no cover - defensive runtime guard
self.failed = type(exc).__name__
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, error=self.failed)
@@ -1201,7 +1413,13 @@ class IsolatedDeterministicExecutor(RuntimeAdapter):
if os.path.exists(temporary):
os.unlink(temporary)
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
if envelope.action != "deterministic":
raise WorkerError("ISOLATED_EXECUTOR_ACTION_INVALID", phase="adapter")
root = ISOLATED_EXECUTOR_ROOT
@@ -1526,9 +1744,18 @@ def build_compact_interface_map_result(
contracts = interface_map.get("contracts")
if not isinstance(implementation_map, Mapping) or not isinstance(excerpts, list) or not isinstance(contracts, Mapping):
raise WorkerError("COMPACT_INTERFACE_MAP_RESULT_INVALID", phase="result")
profile = str(interface_map.get("profile") or "fleet")
classification = str(interface_map.get("classification") or "")
allowed_classifications = {
"fleet": "P2_INTERFACE_MAP_CAPTURED",
"model_governance": "MODEL_GOVERNANCE_INTERFACE_MAP_CAPTURED",
}
if allowed_classifications.get(profile) != classification:
raise WorkerError("COMPACT_INTERFACE_MAP_PROFILE_BINDING_INVALID", phase="result")
result = {
"status": "SUCCEEDED",
"classification": "P2_INTERFACE_MAP_CAPTURED",
"classification": classification,
"interface_map_profile": profile,
"verification_state": "VERIFIED_BY_WORKER_PENDING_SERVER_REVIEW",
"execution_mode": "compact_interface_map",
"execution_backend": "generic-online-agent-worker-v2",
@@ -1566,6 +1793,69 @@ def build_compact_interface_map_result(
return result
def build_agent_runtime_result(
task_id: str,
envelope: ExecutionEnvelope,
execution: Mapping[str, Any],
) -> dict[str, Any]:
external = execution.get("external_agent")
if not isinstance(external, Mapping):
raise WorkerError("EXTERNAL_AGENT_RESULT_INVALID", phase="result")
if external.get("runtime") != envelope.adapter:
raise WorkerError("EXTERNAL_AGENT_RESULT_RUNTIME_MISMATCH", phase="result")
if external.get("task_id") != task_id:
raise WorkerError("EXTERNAL_AGENT_RESULT_TASK_MISMATCH", phase="result")
if external.get("execution_digest") != envelope.digest:
raise WorkerError("EXTERNAL_AGENT_RESULT_DIGEST_MISMATCH", phase="result")
if external.get("credential_material_exposed") is not False:
raise WorkerError("EXTERNAL_AGENT_CREDENTIAL_ISOLATION_FAILED", phase="result")
result = {
"status": "SUCCEEDED",
"classification": "EXTERNAL_AGENT_RUNTIME_SUCCEEDED",
"verification_state": "VERIFIED_BY_WORKER_PENDING_SERVER_REVIEW",
"execution_mode": "agent_runtime",
"execution_backend": "generic-online-agent-worker-v2",
"agent_runtime": envelope.adapter,
"task_id": task_id,
"job_id": envelope.job_id,
"repository": envelope.repository,
"base_oid": envelope.base_oid,
"expected_branch_oid": envelope.expected_branch_oid,
"target_branch": envelope.target_branch,
"generation": envelope.generation,
"execution_digest": envelope.digest,
"output_text": external.get("output_text"),
"output_json": external.get("output_json"),
"output_sha256": external.get("output_sha256"),
"runtime_duration_ms": external.get("duration_ms"),
"external_request_id": external.get("request_id"),
"changed_files": [],
"tests": [],
"workspace_cleanup": "PASS",
"writer_lease": "NONE",
"secret_material_exposed": False,
"credential_material_exposed": False,
"control_plane_access_granted_to_runtime": False,
"direct_main_write": False,
"force_push": False,
"production_write": False,
"protected_ref_bypass": False,
}
encoded = json.dumps(
sanitize(result),
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
if len(encoded) > 192 * 1024:
raise WorkerError(
"EXTERNAL_AGENT_TASK_RESULT_TOO_LARGE",
phase="result",
details={"result_bytes": len(encoded)},
)
result["result_bytes"] = len(encoded)
return result
def build_source_capture_result(
task_id: str,
envelope: ExecutionEnvelope,
@@ -1615,10 +1905,25 @@ class WorkerSupervisor:
if not isinstance(execution, Mapping):
raise WorkerError("EXECUTION_ENVELOPE_SCHEMA_VALIDATION", phase="validation")
persisted = ExecutionEnvelope.from_task(task, worker_capabilities=WORKER_CAPABILITIES)
# The durable plan registry is authoritative for a task that was projected
# before its deterministic plan was registered. Re-resolve without sending
# the stale persisted payload so the Bridge can apply the exact plan/CAS.
status, authoritative = self.bridge.project_execution(task)
# Read-only source_capture/agent_runtime tasks replay their persisted
# payload. A write-capable source_capture is only a Bridge-created
# engineering placeholder: never pass that stale payload back to the
# resolver, and never claim it until a deterministic plan exists.
pending_plan = source_capture_requires_deterministic_plan(persisted)
projection_input = (
execution
if persisted.action in {"source_capture", "agent_runtime"} and not pending_plan
else None
)
status, authoritative = self.bridge.project_execution(task, projection_input)
if pending_plan and (
status == 404
or authoritative.get("error") == "TASK_EXECUTION_PROJECTION_MISSING"
):
raise WorkerError(
"TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN",
phase="projection",
)
if not 200 <= status < 300:
raise WorkerError("TASK_PERSISTED_BINDING_UNVERIFIED", phase="projection", details={"api_status": status, "error": authoritative.get("error")})
canonical = ExecutionEnvelope.from_task(task, authoritative, WORKER_CAPABILITIES)
@@ -1647,6 +1952,11 @@ class WorkerSupervisor:
execution_digest=canonical.digest,
)
return canonical, "plan_registry_verified"
if pending_plan:
raise WorkerError(
"TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN",
phase="projection",
)
return persisted, "persisted_verified"
status, projected = self.bridge.project_execution(task)
if status == 404 or projected.get("error") == "TASK_EXECUTION_PROJECTION_MISSING":
@@ -1815,6 +2125,43 @@ class WorkerSupervisor:
attempt=attempt,
)
if envelope.action == "agent_runtime" and phase in {
"agent_runtime_done",
"result_pending",
"result_submitted",
}:
candidate = state.get("result")
if isinstance(candidate, Mapping):
result = dict(candidate)
audit(
"TASK_RESUMED_FROM_EXTERNAL_AGENT_RESULT",
task_id=task_id,
previous_attempt=state.get("resume_from_attempt"),
attempt=attempt,
phase=phase,
adapter=envelope.adapter,
)
if (
result is None
and envelope.action == "agent_runtime"
and phase == "verification_passed"
and isinstance(state.get("execution"), Mapping)
):
result = build_agent_runtime_result(
task_id, envelope, state["execution"]
)
state.update(
{"phase": "agent_runtime_done", "result": result, "attempt": attempt}
)
save_task_state(task_id, state)
save_checkpoint(task_id, state, envelope, attempt)
audit(
"TASK_RESUMED_AFTER_EXTERNAL_AGENT_EXECUTION",
task_id=task_id,
attempt=attempt,
adapter=envelope.adapter,
)
if (
result is None
and envelope.action == "deterministic"
@@ -1895,7 +2242,11 @@ class WorkerSupervisor:
# re-run deterministic verification. No external side effect exists yet.
run_root = new_run_root()
try:
worktree = prepare_worktree(run_root, envelope)
if envelope.action == "agent_runtime":
worktree = run_root / "agent-runtime-workspace"
worktree.mkdir(mode=0o750, parents=True, exist_ok=True)
else:
worktree = prepare_worktree(run_root, envelope)
state.update(
{
"phase": "workspace_prepared",
@@ -1907,7 +2258,11 @@ class WorkerSupervisor:
save_task_state(task_id, state)
save_checkpoint(task_id, state, envelope, attempt)
adapter = adapter_for(envelope.adapter)
execution = adapter.execute(envelope, worktree)
execution = adapter.execute(
envelope,
worktree,
attempt=attempt,
)
state.update(
{
"phase": "verification_passed",
@@ -1929,6 +2284,17 @@ class WorkerSupervisor:
"attempt": attempt,
}
)
elif envelope.action == "agent_runtime":
result = build_agent_runtime_result(
task_id, envelope, execution
)
state.update(
{
"phase": "agent_runtime_done",
"result": result,
"attempt": attempt,
}
)
else:
heartbeat.set_stage("committing")
governed = governed_commit(
@@ -2079,8 +2445,13 @@ class WorkerSupervisor:
def once(self) -> int:
try:
tasks = self.agent.poll()
write_worker_liveness("POLL_OK", visible_count=len(tasks))
audit("TASK_POLL", visible_count=len(tasks))
except WorkerError as exc:
write_worker_liveness(
"POLL_FAILED",
error=exc.code,
)
audit("TASK_POLL_FAILED", error=exc.code, phase=exc.phase, details=exc.details)
return 10
dynamic_tasks: list[tuple[dict[str, Any], ExecutionEnvelope]] = []
@@ -2089,13 +2460,25 @@ class WorkerSupervisor:
envelope, source = self.resolve_envelope(task)
except WorkerError as exc:
event = "TASK_SKIPPED_ENVELOPE_INVALID"
if exc.code in {"TASK_SKIPPED_PROJECTION_MISSING", "TASK_SKIPPED_POLICY_DENY"}:
if exc.code in {
"TASK_SKIPPED_PROJECTION_MISSING",
"TASK_SKIPPED_POLICY_DENY",
"TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN",
}:
event = exc.code
elif exc.code == "CAPABILITY_MISMATCH_DENY":
event = "TASK_SKIPPED_CAPABILITY_MISMATCH"
audit(event, task_id=task.get("task_id"), reason=exc.code, phase=exc.phase, details=exc.details)
continue
audit("TASK_VISIBLE_TO_WORKER_CLAIMABLE", task_id=task.get("task_id"), envelope_source=source, action=envelope.action)
if envelope.adapter not in WORKER_ALLOWED_ADAPTERS:
audit(
"TASK_SKIPPED_ADAPTER_MISMATCH",
task_id=task.get("task_id"),
adapter=envelope.adapter,
allowed_adapters=sorted(WORKER_ALLOWED_ADAPTERS),
)
continue
dynamic_tasks.append((task, envelope))
dynamic_tasks.sort(key=_claim_order)
for task, envelope in dynamic_tasks:
@@ -2115,6 +2498,7 @@ class WorkerSupervisor:
ensure_roots()
recover_stale_workspaces()
checkpoint_store()
write_worker_liveness("STARTING")
audit("WORKER_STARTED", task_agnostic=True, api_base=API_BASE, bridge_base=BRIDGE_BASE, adapter_names=["isolated-deterministic", "source_capture", "codex", "hermes"], runtime_only_credentials=True, isolated_executor=True, durable_checkpoint=True)
while True:
self.once()
@@ -0,0 +1,462 @@
"""Self-healing availability guard for persistent Online Agent workers.
The guard never claims Tasks and never writes Task/Job/Git databases. It only:
- ensures required worker/executor systemd units are enabled + active;
- checks worker-owned liveness JSON written by the real poll/heartbeat loop;
- restarts an active worker only when its liveness is stale;
- honors an explicit maintenance inhibit file;
- writes a bounded status JSON under /run for observability.
Idle liveness is the successful poll loop itself. Busy liveness is refreshed by the
Task lease-heartbeat thread, so long-running Tasks are not mistaken for hung workers.
"""
from __future__ import annotations
from dataclasses import asdict, dataclass
import argparse
import json
import os
from pathlib import Path
import shlex
import subprocess
import tempfile
import time
from typing import Any, Callable, Mapping, Sequence
SCHEMA = "aios.worker-availability.status.v1"
DEFAULT_WORKER_UNITS = (
"aios-online-agent-worker-20260815.service",
"aios-hermes-online-agent-worker.service",
"aios-openclaw-online-agent-worker.service",
)
DEFAULT_DEPENDENCY_UNITS = (
"aios-external-agent-executor.service",
)
DEFAULT_RUN_ROOT = Path("/run/aios-worker-availability")
DEFAULT_STATUS = DEFAULT_RUN_ROOT / "status.json"
DEFAULT_RECOVERY = DEFAULT_RUN_ROOT / "recovery.json"
DEFAULT_INHIBIT = DEFAULT_RUN_ROOT / "maintenance.lock"
DEFAULT_MAX_STALE_SECONDS = 90
DEFAULT_RESTART_COOLDOWN_SECONDS = 60
DEFAULT_STARTUP_WAIT_SECONDS = 10
ACCEPTED_ENABLED_STATES = frozenset({"enabled", "enabled-runtime", "static", "indirect", "generated"})
class AvailabilityGuardError(RuntimeError):
pass
@dataclass(frozen=True)
class UnitSnapshot:
unit: str
role: str
load_state: str
active_state: str
unit_file_state: str
enabled_ok: bool
liveness_path: str | None = None
liveness_state: str | None = None
last_seen_epoch: float | None = None
liveness_age_seconds: float | None = None
fresh: bool | None = None
action: str = "NONE"
detail: str | None = None
class Systemd:
def __init__(
self,
runner: Callable[..., subprocess.CompletedProcess[str]] | None = None,
) -> None:
self.runner = runner or subprocess.run
def _run(self, args: Sequence[str], *, check: bool = False) -> subprocess.CompletedProcess[str]:
result = self.runner(
["/usr/bin/systemctl", *args],
text=True,
capture_output=True,
check=False,
timeout=20,
)
if check and result.returncode != 0:
raise AvailabilityGuardError(
"SYSTEMD_COMMAND_FAILED:"
+ ":".join(args)
+ ":"
+ str(result.returncode)
)
return result
def property(self, unit: str, name: str) -> str:
result = self._run(["show", unit, f"--property={name}", "--value"])
if result.returncode != 0:
return ""
return result.stdout.strip()
def environment(self, unit: str) -> dict[str, str]:
text = self.property(unit, "Environment")
result: dict[str, str] = {}
for item in shlex.split(text):
if "=" not in item:
continue
key, value = item.split("=", 1)
result[key] = value
return result
def enable(self, unit: str) -> None:
self._run(["enable", unit], check=True)
def start(self, unit: str) -> None:
self._run(["start", unit], check=True)
def restart(self, unit: str) -> None:
self._run(["restart", unit], check=True)
def _atomic_json(path: Path, value: Mapping[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True, mode=0o755)
if path.parent.is_symlink() or path.is_symlink():
raise AvailabilityGuardError("STATUS_PATH_SYMLINK_REJECTED")
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
try:
os.fchmod(fd, 0o644)
with os.fdopen(fd, "w", encoding="utf-8") as handle:
json.dump(value, handle, ensure_ascii=False, sort_keys=True, indent=2)
handle.write("\n")
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
os.chmod(path, 0o644)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
def _load_json(path: Path) -> dict[str, Any] | None:
if path.is_symlink() or not path.is_file():
return None
try:
value = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return None
return dict(value) if isinstance(value, Mapping) else None
def _state_root(environment: Mapping[str, str], unit: str) -> Path:
explicit = environment.get("AIOS_WORKER_STATE_ROOT")
if explicit:
return Path(explicit)
fallbacks = {
"aios-online-agent-worker-20260815.service": Path(
"/var/lib/aios-online-agent-worker-20260815"
),
"aios-hermes-online-agent-worker.service": Path(
"/var/lib/aios-online-agent-hermes"
),
"aios-openclaw-online-agent-worker.service": Path(
"/var/lib/aios-online-agent-openclaw"
),
}
if unit not in fallbacks:
raise AvailabilityGuardError("WORKER_STATE_ROOT_UNAVAILABLE:" + unit)
return fallbacks[unit]
def _liveness_path(environment: Mapping[str, str], unit: str) -> Path:
explicit = environment.get("AIOS_WORKER_LIVENESS_PATH")
if explicit:
return Path(explicit)
return _state_root(environment, unit) / "worker-liveness.json"
class AvailabilityGuard:
def __init__(
self,
*,
systemd: Systemd | None = None,
worker_units: Sequence[str] = DEFAULT_WORKER_UNITS,
dependency_units: Sequence[str] = DEFAULT_DEPENDENCY_UNITS,
status_path: Path = DEFAULT_STATUS,
recovery_path: Path = DEFAULT_RECOVERY,
inhibit_path: Path = DEFAULT_INHIBIT,
max_stale_seconds: int = DEFAULT_MAX_STALE_SECONDS,
restart_cooldown_seconds: int = DEFAULT_RESTART_COOLDOWN_SECONDS,
startup_wait_seconds: int = DEFAULT_STARTUP_WAIT_SECONDS,
now_fn: Callable[[], float] = time.time,
sleep_fn: Callable[[float], None] = time.sleep,
) -> None:
self.systemd = systemd or Systemd()
self.worker_units = tuple(worker_units)
self.dependency_units = tuple(dependency_units)
self.status_path = Path(status_path)
self.recovery_path = Path(recovery_path)
self.inhibit_path = Path(inhibit_path)
self.max_stale_seconds = int(max_stale_seconds)
self.restart_cooldown_seconds = int(restart_cooldown_seconds)
self.startup_wait_seconds = int(startup_wait_seconds)
self.now_fn = now_fn
self.sleep_fn = sleep_fn
if self.max_stale_seconds < 15:
raise AvailabilityGuardError("MAX_STALE_SECONDS_TOO_SMALL")
if self.restart_cooldown_seconds < 15:
raise AvailabilityGuardError("RESTART_COOLDOWN_TOO_SMALL")
if self.startup_wait_seconds < 0 or self.startup_wait_seconds > 60:
raise AvailabilityGuardError("STARTUP_WAIT_INVALID")
def _recovery_state(self) -> dict[str, float]:
value = _load_json(self.recovery_path) or {}
raw = value.get("last_restart_epoch")
if not isinstance(raw, Mapping):
return {}
result: dict[str, float] = {}
for unit, epoch in raw.items():
try:
result[str(unit)] = float(epoch)
except (TypeError, ValueError):
continue
return result
def _save_recovery_state(self, state: Mapping[str, float]) -> None:
_atomic_json(
self.recovery_path,
{
"schema": "aios.worker-availability.recovery.v1",
"last_restart_epoch": dict(state),
},
)
def _basic_snapshot(self, unit: str, role: str) -> tuple[str, str, str, bool]:
load = self.systemd.property(unit, "LoadState")
active = self.systemd.property(unit, "ActiveState")
enabled = self.systemd.property(unit, "UnitFileState")
return load, active, enabled, enabled in ACCEPTED_ENABLED_STATES
def _ensure_enabled_active(self, unit: str, role: str) -> tuple[str, bool]:
load, active, enabled, enabled_ok = self._basic_snapshot(unit, role)
if load != "loaded":
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
action = "NONE"
if not enabled_ok:
self.systemd.enable(unit)
action = "ENABLE"
if active != "active":
self.systemd.start(unit)
action = "START" if action == "NONE" else "ENABLE_START"
return action, active == "active"
def _wait_for_liveness(self, path: Path) -> dict[str, Any] | None:
deadline = self.now_fn() + self.startup_wait_seconds
while True:
value = _load_json(path)
if value is not None:
try:
epoch = float(value.get("last_seen_epoch"))
except (TypeError, ValueError):
epoch = 0.0
if epoch > 0 and self.now_fn() - epoch <= self.max_stale_seconds:
return value
if self.now_fn() >= deadline:
return value
self.sleep_fn(0.5)
def run(self, *, heal: bool) -> dict[str, Any]:
now = self.now_fn()
if self.inhibit_path.exists():
result = {
"schema": SCHEMA,
"status": "MAINTENANCE_INHIBITED",
"generated_at_epoch": now,
"maintenance_inhibit": str(self.inhibit_path),
"workers": [],
"dependencies": [],
"all_fresh": False,
"actions": [],
}
_atomic_json(self.status_path, result)
return result
actions: list[dict[str, str]] = []
dependencies: list[UnitSnapshot] = []
for unit in self.dependency_units:
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "dependency")
action = "NONE"
if heal:
if load != "loaded":
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
if not enabled_ok:
self.systemd.enable(unit)
action = "ENABLE"
if active != "active":
self.systemd.start(unit)
action = "START" if action == "NONE" else "ENABLE_START"
if action != "NONE":
actions.append({"unit": unit, "action": action})
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "dependency")
dependencies.append(
UnitSnapshot(
unit=unit,
role="dependency",
load_state=load,
active_state=active,
unit_file_state=enabled,
enabled_ok=enabled_ok,
fresh=(active == "active"),
action=action,
)
)
recovery = self._recovery_state()
workers: list[UnitSnapshot] = []
for unit in self.worker_units:
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
environment = self.systemd.environment(unit) if load == "loaded" else {}
path = _liveness_path(environment, unit) if load == "loaded" else None
action = "NONE"
if heal:
if load != "loaded":
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
if not enabled_ok:
self.systemd.enable(unit)
action = "ENABLE"
if active != "active":
self.systemd.start(unit)
action = "START" if action == "NONE" else "ENABLE_START"
actions.append({"unit": unit, "action": action})
if path is not None:
self._wait_for_liveness(path)
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
live = _load_json(path) if path is not None else None
last_seen: float | None = None
state: str | None = None
age: float | None = None
fresh = False
detail: str | None = None
if live is not None:
state = str(live.get("state") or "")
try:
last_seen = float(live.get("last_seen_epoch"))
except (TypeError, ValueError):
last_seen = None
if last_seen is not None:
age = max(0.0, now - last_seen)
fresh = age <= self.max_stale_seconds
if live.get("agent_id") in (None, ""):
fresh = False
detail = "LIVENESS_AGENT_ID_MISSING"
else:
detail = "LIVENESS_MISSING"
if heal and active == "active" and not fresh:
previous = float(recovery.get(unit, 0.0))
if now - previous >= self.restart_cooldown_seconds:
self.systemd.restart(unit)
recovery[unit] = now
action = "RESTART_STALE"
actions.append({"unit": unit, "action": action})
if path is not None:
live = self._wait_for_liveness(path)
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
if live is not None:
state = str(live.get("state") or "")
try:
last_seen = float(live.get("last_seen_epoch"))
except (TypeError, ValueError):
last_seen = None
if last_seen is not None:
age = max(0.0, self.now_fn() - last_seen)
fresh = age <= self.max_stale_seconds
detail = None if fresh else "LIVENESS_STALE_AFTER_RESTART"
else:
detail = "RESTART_COOLDOWN_ACTIVE"
workers.append(
UnitSnapshot(
unit=unit,
role="worker",
load_state=load,
active_state=active,
unit_file_state=enabled,
enabled_ok=enabled_ok,
liveness_path=str(path) if path is not None else None,
liveness_state=state,
last_seen_epoch=last_seen,
liveness_age_seconds=age,
fresh=bool(active == "active" and enabled_ok and fresh),
action=action,
detail=detail,
)
)
if heal:
self._save_recovery_state(recovery)
all_fresh = (
all(item.active_state == "active" and item.enabled_ok and item.fresh for item in workers)
and all(item.active_state == "active" for item in dependencies)
)
result = {
"schema": SCHEMA,
"status": "HEALTHY" if all_fresh else "DEGRADED",
"generated_at_epoch": self.now_fn(),
"max_stale_seconds": self.max_stale_seconds,
"maintenance_inhibit": str(self.inhibit_path),
"workers": [asdict(item) for item in workers],
"dependencies": [asdict(item) for item in dependencies],
"all_fresh": all_fresh,
"actions": actions,
}
_atomic_json(self.status_path, result)
return result
def main() -> int:
parser = argparse.ArgumentParser()
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--heal", action="store_true")
mode.add_argument("--check", action="store_true")
parser.add_argument("--status-file", default=str(DEFAULT_STATUS))
parser.add_argument("--recovery-file", default=str(DEFAULT_RECOVERY))
parser.add_argument("--inhibit-file", default=str(DEFAULT_INHIBIT))
parser.add_argument(
"--max-stale-seconds",
type=int,
default=int(
os.environ.get(
"AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS",
str(DEFAULT_MAX_STALE_SECONDS),
)
),
)
parser.add_argument(
"--restart-cooldown-seconds",
type=int,
default=int(
os.environ.get(
"AIOS_WORKER_AVAILABILITY_RESTART_COOLDOWN_SECONDS",
str(DEFAULT_RESTART_COOLDOWN_SECONDS),
)
),
)
args = parser.parse_args()
guard = AvailabilityGuard(
status_path=Path(args.status_file),
recovery_path=Path(args.recovery_file),
inhibit_path=Path(args.inhibit_file),
max_stale_seconds=args.max_stale_seconds,
restart_cooldown_seconds=args.restart_cooldown_seconds,
)
try:
result = guard.run(heal=args.heal)
except AvailabilityGuardError as exc:
print(json.dumps({"status": "FAILED", "error": str(exc)}, sort_keys=True))
return 20
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
if args.check and not result.get("all_fresh"):
return 10
return 0
if __name__ == "__main__":
raise SystemExit(main())
+97 -14
View File
@@ -363,7 +363,7 @@ class FleetStateStore:
row = self.conn.execute(
"""
SELECT COUNT(*) FROM fleet_dispatches
WHERE worker_id=? AND state IN ('RESERVED','CLAIMED') AND reserved_until>?
WHERE worker_id=? AND state='RESERVED' AND reserved_until>?
""",
(worker_id, now),
).fetchone()
@@ -412,6 +412,13 @@ class FleetStateStore:
""",
(task_id,),
).fetchone()
# Capacity is rechecked inside the same durable reservation transaction.
# This closes the cross-thread/process race between eligibility scoring and reserve.
policy_row = self.conn.execute(
"SELECT max_slots FROM fleet_worker_capacity WHERE worker_id=?",
(worker_id,),
).fetchone()
max_slots = int(policy_row[0]) if policy_row else 1
if existing and str(existing[6]) in {"RESERVED", "CLAIMED"} and float(existing[5]) > now:
if str(existing[1]) != worker_id or int(existing[2]) != task_revision:
raise FleetConflict("FLEET_TASK_ALREADY_RESERVED")
@@ -419,6 +426,32 @@ class FleetStateStore:
str(existing[0]), task_id, worker_id, task_revision, str(existing[3]),
str(existing[4]), float(existing[5]), score
)
claim_columns = {
str(row[1])
for row in self.conn.execute("PRAGMA table_info(claims)").fetchall()
}
active_claims = 0
if {"worker_id", "status"}.issubset(claim_columns):
sql = "SELECT COUNT(*) FROM claims WHERE worker_id=? AND status='ACTIVE'"
params: list[Any] = [worker_id]
if "lease_expires_at" in claim_columns:
sql += " AND lease_expires_at>?"
params.append(now)
row = self.conn.execute(sql, tuple(params)).fetchone()
active_claims = int(row[0] if row else 0)
row = self.conn.execute(
"""
SELECT COUNT(*) FROM fleet_dispatches
WHERE worker_id=? AND task_id<>?
AND state='RESERVED' AND reserved_until>?
""",
(worker_id, task_id, now),
).fetchone()
active_reservations = int(row[0] if row else 0)
if active_claims + active_reservations >= max_slots:
raise FleetConflict("FLEET_WORKER_CAPACITY_EXHAUSTED")
self.conn.execute(
"""
INSERT INTO fleet_dispatches(
@@ -682,19 +715,23 @@ class FleetScheduler:
return getter(worker_id)
def _active_claims(self, worker_id: str, now: float) -> int:
columns = {
str(row[1])
for row in self.store.conn.execute("PRAGMA table_info(claims)").fetchall()
}
required = {"worker_id", "status"}
if not required.issubset(columns):
raise FleetError("FLEET_CLAIMS_SCHEMA_UNSUPPORTED")
sql = "SELECT COUNT(*) FROM claims WHERE worker_id=? AND status='ACTIVE'"
params: list[Any] = [worker_id]
if "lease_expires_at" in columns:
sql += " AND lease_expires_at>?"
params.append(now)
row = self.store.conn.execute(sql, tuple(params)).fetchone()
# The TaskPlane store owns one SQLite connection in the current runtime.
# Fleet reads must share its lock just like fleet writes do, otherwise two
# concurrent pullers can issue overlapping operations on the same handle.
with self.state.lock:
columns = {
str(row[1])
for row in self.store.conn.execute("PRAGMA table_info(claims)").fetchall()
}
required = {"worker_id", "status"}
if not required.issubset(columns):
raise FleetError("FLEET_CLAIMS_SCHEMA_UNSUPPORTED")
sql = "SELECT COUNT(*) FROM claims WHERE worker_id=? AND status='ACTIVE'"
params: list[Any] = [worker_id]
if "lease_expires_at" in columns:
sql += " AND lease_expires_at>?"
params.append(now)
row = self.store.conn.execute(sql, tuple(params)).fetchone()
return int(row[0] if row else 0)
@staticmethod
@@ -764,6 +801,52 @@ class FleetScheduler:
and _task_id(task)
]
def rank_for_worker(
self,
worker_id: str,
*,
actor_context: Any = None,
now: float | None = None,
) -> list[object]:
"""Return fleet-ranked tasks without reserving or claiming anything."""
now = float(self.now_fn() if now is None else now)
worker_id = _text(worker_id)
if not worker_id:
return []
ranked: list[object] = []
for task in self._tasks(actor_context):
eligible, _, _ = self._eligible_worker(worker_id, task, now)
if eligible:
ranked.append(task)
return sorted(ranked, key=lambda task: self._task_score(task, now))
def reserve_for_worker(
self,
worker_id: str,
task: object,
*,
now: float | None = None,
) -> FleetDispatchDecision:
"""Reserve exactly one already-ranked task, rechecking eligibility and capacity."""
now = float(self.now_fn() if now is None else now)
worker_id = _text(worker_id)
eligible, _, policy = self._eligible_worker(worker_id, task, now)
if not eligible:
raise FleetConflict("FLEET_WORKER_NOT_ELIGIBLE")
metadata = _task_metadata(task)
scope_weight = _float(metadata.get("fleet_scope_weight"), 1.0, 0.05, 100.0)
return self.state.reserve(
task_id=_task_id(task),
worker_id=worker_id,
task_revision=_task_revision(task),
scope_key=_task_scope_key(task),
failure_domain=policy.failure_domain,
score=self._task_score(task, now),
reservation_seconds=self.reservation_seconds,
now=now,
scope_weight=scope_weight,
)
def select(
self,
worker_ids: Sequence[str],
+190
View File
@@ -84,3 +84,193 @@ from .fleet import (
FleetStateStore,
FleetWorkerPolicy,
)
# R4.2 P2-B fleet-aware pull scheduling.
#
# The original TaskScheduler remains the security oracle and claim-CAS implementation.
# Fleet ordering only controls the order presented to that oracle.
import inspect as _fleet_inspect
import threading as _fleet_threading
from collections.abc import Mapping as _FleetMapping
from .worker_registry import WorkerRegistry as _FleetWorkerRegistry
from .errors import NotFoundError as _FleetNotFoundError
_LegacyTaskScheduler = TaskScheduler
_LegacyScheduleParameters = _fleet_inspect.signature(
_LegacyTaskScheduler.schedule_for_worker
).parameters
_LegacyNextParameters = _fleet_inspect.signature(
_LegacyTaskScheduler.next_queued_task
).parameters
class _FleetOrderedStoreProxy:
def __init__(self, delegate, tasks):
self._delegate = delegate
self._tasks = tuple(tasks)
def list_tasks(self, *args, **kwargs):
return list(self._tasks)
def __getattr__(self, name):
return getattr(self._delegate, name)
def _fleet_claim_field(claim, name, default=None):
if isinstance(claim, _FleetMapping):
return claim.get(name, default)
return getattr(claim, name, default)
class TaskScheduler(_LegacyTaskScheduler):
"""Fleet-integrated pull scheduler preserving legacy security and claim semantics."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
store = getattr(self, "store", args[0] if args else None)
if store is None:
raise FleetError("FLEET_SCHEDULER_STORE_REQUIRED")
registry = getattr(self, "workers", None)
if registry is None:
registry = _FleetWorkerRegistry(store)
self.fleet_scheduler = FleetScheduler(store, registry)
self._fleet_local = _fleet_threading.local()
def _fleet_legacy_next(self, worker_id, tasks, *args, **kwargs):
proxy = _FleetOrderedStoreProxy(self.store, tasks)
legacy = _LegacyTaskScheduler(proxy)
legacy_kwargs = dict(kwargs)
if "actor_context" not in _LegacyNextParameters:
legacy_kwargs.pop("actor_context", None)
return legacy.next_queued_task(worker_id, *args, **legacy_kwargs)
def next_queued_task(self, worker_id, *args, **kwargs):
actor_context = kwargs.get(
"actor_context", getattr(self._fleet_local, "actor_context", None)
)
# Preserve legacy fail-closed worker checks before fleet ordering.
probe_kwargs = dict(kwargs)
if "actor_context" not in _LegacyNextParameters:
probe_kwargs.pop("actor_context", None)
legacy_probe = _LegacyTaskScheduler(self.store).next_queued_task(
worker_id, *args, **probe_kwargs
)
if legacy_probe is None:
return None
ranked = self.fleet_scheduler.rank_for_worker(
worker_id, actor_context=actor_context
)
# Preserve FleetScheduler fairness order. The legacy scheduler is used
# only as a fail-closed security oracle: present one fleet-ranked task
# at a time so its internal sort cannot reorder across scopes.
# Keep the legacy priority dimension ahead of Fleet fairness for
# compatibility with existing TaskPlane scheduling contracts; the
# Fleet order remains authoritative among equal-priority tasks.
def _legacy_priority(task):
plan = getattr(task, "task_plan", task)
metadata = getattr(plan, "metadata", {}) or {}
if not isinstance(metadata, _FleetMapping):
return 0
try:
return int(metadata.get("priority", 0))
except (TypeError, ValueError):
return 0
ranked = sorted(ranked, key=lambda task: -_legacy_priority(task))
for ranked_candidate in ranked:
try:
candidate = self._fleet_legacy_next(
worker_id, [ranked_candidate], *args, **kwargs
)
except _FleetNotFoundError:
continue
if candidate is None:
continue
ranked_id = getattr(
ranked_candidate,
"task_id",
getattr(getattr(ranked_candidate, "task_plan", None), "task_id", None),
)
candidate_id = getattr(
candidate,
"task_id",
getattr(getattr(candidate, "task_plan", None), "task_id", None),
)
if candidate_id != ranked_id:
raise FleetConflict("FLEET_LEGACY_ORACLE_ORDER_VIOLATION")
try:
decision = self.fleet_scheduler.reserve_for_worker(
worker_id, candidate
)
except FleetConflict:
continue
self._fleet_local.pending = decision
return candidate
return None
def schedule_for_worker(self, worker_id, *args, **kwargs):
actor_context = kwargs.get("actor_context")
legacy_kwargs = dict(kwargs)
if "actor_context" not in _LegacyScheduleParameters:
legacy_kwargs.pop("actor_context", None)
self._fleet_local.actor_context = actor_context
self._fleet_local.pending = None
try:
claim = _LegacyTaskScheduler.schedule_for_worker(
self, worker_id, *args, **legacy_kwargs
)
decision = getattr(self._fleet_local, "pending", None)
if claim is None:
if decision is not None:
self.fleet_scheduler.state.release(
decision.task_id, state="RELEASED"
)
return None
if decision is not None:
claim_id = _fleet_claim_field(claim, "claim_id")
claim_task_id = _fleet_claim_field(claim, "task_id", decision.task_id)
if not claim_id:
self.fleet_scheduler.state.release(
decision.task_id, state="RELEASED"
)
raise FleetError("FLEET_CLAIM_ID_REQUIRED")
if claim_task_id != decision.task_id:
self.fleet_scheduler.state.release(
decision.task_id, state="RELEASED"
)
raise FleetConflict("FLEET_CLAIM_TASK_MISMATCH")
self.fleet_scheduler.state.bind_claim(
decision.reservation_id, str(claim_id)
)
return claim
except Exception:
decision = getattr(self._fleet_local, "pending", None)
if decision is not None:
self.fleet_scheduler.state.release(
decision.task_id, state="RELEASED"
)
raise
finally:
self._fleet_local.pending = None
self._fleet_local.actor_context = None
def record_worker_failure(self, worker_id, failure_class, **kwargs):
return self.fleet_scheduler.state.record_worker_failure(
worker_id, failure_class, **kwargs
)
def record_worker_success(self, worker_id, **kwargs):
return self.fleet_scheduler.state.record_worker_success(
worker_id, **kwargs
)
def quarantine_task_worker(
self, task_id, worker_id, failure_class, **kwargs
):
return self.fleet_scheduler.state.record_task_worker_failure(
task_id, worker_id, failure_class, **kwargs
)
@@ -0,0 +1,9 @@
providers:
oneapi:
name: oneapi
base_url: http://172.18.0.2:3000/v1
key_env: OPENAI_API_KEY
default_model: deepseek-v4-flash
model:
provider: oneapi
name: deepseek-v4-flash
@@ -0,0 +1,13 @@
{
"argv": [
"/usr/local/bin/hermes",
"run",
"--non-interactive"
],
"input_mode": "stdin_text",
"max_output_bytes": 131072,
"output_mode": "stdout_text",
"runtime": "hermes",
"schema": "aios.external-agent-runtime.profile.v1",
"timeout_seconds": 900
}
@@ -0,0 +1,38 @@
{
"argv": [
"/opt/aios-hermes-agent/hermes-agent/.venv/bin/hermes",
"-z",
"{prompt}",
"--provider",
"oneapi",
"-m",
"{model_id}",
"--no-restore-cwd"
],
"credential_env": {
"OPENAI_API_KEY": {
"credential": "aios-external-agent-oneapi-env",
"format": "dotenv",
"key": "ONEAPI_REGULAR_TOKEN"
}
},
"environment": {
"HERMES_HOME": "/etc/aios/external-agent-runtime.d/hermes-home"
},
"input_mode": "stdin_text",
"max_output_bytes": 131072,
"model_governance": {
"max_output_tokens": 512,
"model_capabilities": [
"text"
],
"provider_ids": [
"oneapi"
],
"required": true
},
"output_mode": "stdout_text",
"runtime": "hermes",
"schema": "aios.external-agent-runtime.profile.v1",
"timeout_seconds": 180
}
@@ -0,0 +1,41 @@
{
"tools": {
"profile": "minimal"
},
"memory": {
"search": {
"enabled": false
}
},
"agents": {
"defaults": {
"workspace": "/var/lib/aios-external-agent-executor/openclaw-workspace",
"model": {
"primary": "openai/deepseek-v4-flash"
},
"thinkingDefault": "off"
},
"entries": {
"main": {
"default": true,
"workspace": "/var/lib/aios-external-agent-executor/openclaw-workspace"
}
}
},
"models": {
"providers": {
"openai": {
"baseUrl": "http://172.18.0.2:3000/v1",
"api": "openai-completions",
"models": [
{
"id": "deepseek-v4-flash",
"name": "deepseek-v4-flash",
"contextWindow": 32768,
"maxTokens": 256
}
]
}
}
}
}
@@ -0,0 +1,13 @@
{
"argv": [
"/usr/local/bin/openclaw",
"run",
"--non-interactive"
],
"input_mode": "stdin_text",
"max_output_bytes": 131072,
"output_mode": "stdout_text",
"runtime": "openclaw",
"schema": "aios.external-agent-runtime.profile.v1",
"timeout_seconds": 900
}
@@ -0,0 +1,48 @@
{
"argv": [
"/usr/bin/node",
"/opt/aios/openclaw-engineering/v2026.7.2-beta.7/openclaw.mjs",
"agent",
"--local",
"--json",
"--agent",
"main",
"--thinking",
"off",
"--message",
"{prompt}",
"--model",
"{model_id}",
"--timeout",
"180"
],
"credential_env": {
"OPENAI_API_KEY": {
"credential": "aios-external-agent-oneapi-env",
"format": "dotenv",
"key": "ONEAPI_REGULAR_TOKEN"
}
},
"environment": {
"OPENCLAW_CONFIG_PATH": "/etc/aios/external-agent-runtime.d/openclaw-config.json",
"OPENCLAW_HOME": "/var/lib/aios-external-agent-executor/openclaw-home",
"OPENCLAW_STATE_DIR": "/var/lib/aios-external-agent-executor/openclaw-state"
},
"input_mode": "stdin_text",
"max_output_bytes": 131072,
"model_governance": {
"input_reservation_floor_tokens": 16384,
"max_output_tokens": 256,
"model_capabilities": [
"text"
],
"provider_ids": [
"openai"
],
"required": true
},
"output_mode": "stdout_json",
"runtime": "openclaw",
"schema": "aios.external-agent-runtime.profile.v1",
"timeout_seconds": 240
}
@@ -0,0 +1,95 @@
{
"models": [
{
"capabilities": [
"agent.runtime.hermes",
"text"
],
"enabled": true,
"evaluation_sha256": "43ae0db1640daeb6d33e2787769732844f366d36070065d3a2239905bd9a823b",
"evaluation_state": "APPROVED",
"input_cost_micros_per_1k": 0,
"max_input_tokens": 32768,
"max_output_tokens": 512,
"model_id": "deepseek-v4-flash",
"output_cost_micros_per_1k": 0,
"priority": 10,
"provider_id": "oneapi",
"version": "profile-sha:54615e533bfdb6a2e8b870ae74bb8cd7fc45177495bcf6e8c45a50837cefb85a"
},
{
"capabilities": [
"agent.runtime.openclaw",
"text"
],
"enabled": true,
"evaluation_sha256": "7b1c89634eb9c0e7cb5a3d75323d904ebbd91b3ae5494ce938bd8316a4cf5b12",
"evaluation_state": "APPROVED",
"input_cost_micros_per_1k": 0,
"max_input_tokens": 32768,
"max_output_tokens": 256,
"model_id": "openai/deepseek-v4-flash",
"output_cost_micros_per_1k": 0,
"priority": 10,
"provider_id": "openai",
"version": "profile-sha:2a31c480226aa78d44ed9692bddcedc99c3aa03c23c0dd5f031a5cbbb8bbd7ba"
}
],
"policies": [
{
"allowed_models": [
"deepseek-v4-flash",
"openai/deepseek-v4-flash"
],
"daily_cost_budget_micros": 0,
"daily_token_budget": 250000,
"denied_providers": [],
"fallback_limit": 1,
"max_input_tokens_per_request": 16384,
"max_output_tokens_per_request": 512,
"project_id": null,
"required_capabilities": [
"text"
],
"scope_type": "legacy",
"tenant_id": null
},
{
"allowed_models": [
"deepseek-v4-flash",
"openai/deepseek-v4-flash"
],
"daily_cost_budget_micros": 0,
"daily_token_budget": 250000,
"denied_providers": [],
"fallback_limit": 1,
"max_input_tokens_per_request": 16384,
"max_output_tokens_per_request": 512,
"project_id": "staging",
"required_capabilities": [
"text"
],
"scope_type": "tenant",
"tenant_id": "model-governance-live"
}
],
"providers": [
{
"cooldown_seconds": 60,
"credential_ref": "runtime-profile:hermes",
"enabled": true,
"failure_domain": "oneapi",
"max_failures": 2,
"provider_id": "oneapi"
},
{
"cooldown_seconds": 60,
"credential_ref": "runtime-profile:openclaw",
"enabled": true,
"failure_domain": "oneapi",
"max_failures": 2,
"provider_id": "openai"
}
],
"schema": "aios.model-governance.bootstrap.v1"
}
@@ -0,0 +1,37 @@
[Unit]
Description=AIOS owner-controlled Task execution-rebind API
After=aios-agent-task-api.service
Wants=aios-agent-task-api.service
[Service]
Type=simple
User=aios-agent-task
Group=aios-agent-task
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
EnvironmentFile=/etc/aios-gitops/agent-task-execution-rebind.env
Environment=AIOS_AGENT_TASK_DB=/var/lib/aios-agent-task/state.sqlite3
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
Environment=AIOS_TASK_EXECUTION_REBIND_HOST=127.0.0.1
Environment=AIOS_TASK_EXECUTION_REBIND_PORT=18942
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_agent_task_execution_rebind.py
Restart=on-failure
RestartSec=2
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
LockPersonality=true
RestrictSUIDSGID=true
RestrictRealtime=true
RestrictNamespaces=true
MemoryDenyWriteExecute=true
RestrictAddressFamilies=AF_UNIX AF_INET
ReadWritePaths=/var/lib/aios-agent-task
UMask=0077
[Install]
WantedBy=multi-user.target
@@ -0,0 +1,43 @@
[Unit]
Description=AIOS credential-isolated external Agent runtime executor
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=aios-engineering-executor
Group=aios-executor-exchange
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_external_agent_runtime.py --root /var/lib/aios-external-agent-executor --profile-root /etc/aios/external-agent-runtime.d
Restart=always
RestartSec=2
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
LockPersonality=true
RestrictSUIDSGID=true
RestrictRealtime=true
RestrictNamespaces=true
MemoryDenyWriteExecute=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
IPAddressDeny=127.0.0.0/8
IPAddressDeny=::1/128
IPAddressAllow=172.18.0.2/32
LoadCredential=aios-external-agent-oneapi-env:/data/agent/secrets/oneapi-model-ready.env
InaccessiblePaths=/etc/aios-gitops
InaccessiblePaths=/data/aios-gitops
InaccessiblePaths=/data/agent/secrets
ReadOnlyPaths=/etc/aios/external-agent-runtime.d
ReadOnlyPaths=/opt/aios-hermes-agent
ReadOnlyPaths=/opt/aios/openclaw-engineering
Environment=AIOS_MODEL_GOVERNANCE_DB=/var/lib/aios-model-governance/state.sqlite3
Environment=AIOS_MODEL_GOVERNANCE_ENABLED=1
ReadWritePaths=/var/lib/aios-external-agent-executor
ReadWritePaths=/var/lib/aios-model-governance
UMask=0027
[Install]
WantedBy=multi-user.target
@@ -0,0 +1,41 @@
[Unit]
Description=AIOS disabled-by-default HTTPS edge for GPT read-only Online Task ingress
After=network-online.target aios-gpt-online-task-ingress.service
Wants=network-online.target aios-gpt-online-task-ingress.service
ConditionPathExists=/etc/aios-gitops/gpt-https-edge.approved
[Service]
Type=simple
DynamicUser=yes
LoadCredential=tls_cert:/etc/aios-gitops/gpt-https-edge-tls.crt
LoadCredential=tls_key:/etc/aios-gitops/gpt-https-edge-tls.key
Environment=AIOS_GPT_EDGE_TLS_CERT_FILE=%d/tls_cert
Environment=AIOS_GPT_EDGE_TLS_KEY_FILE=%d/tls_key
Environment=AIOS_GPT_EDGE_HOST=127.0.0.1
Environment=AIOS_GPT_EDGE_PORT=18443
Environment=AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=0
Environment=AIOS_GPT_EDGE_PUBLIC_DOMAIN=
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_gpt_https_edge.py
Restart=on-failure
RestartSec=2
NoNewPrivileges=yes
PrivateTmp=yes
PrivateDevices=yes
ProtectSystem=strict
ProtectHome=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectControlGroups=yes
ProtectKernelLogs=yes
ProtectClock=yes
LockPersonality=yes
RestrictSUIDSGID=yes
RestrictRealtime=yes
RestrictNamespaces=yes
MemoryDenyWriteExecute=yes
RestrictAddressFamilies=AF_INET AF_INET6
InaccessiblePaths=/data/aios-gitops /etc/aios-gitops /home /var/lib/aios-agent-task
UMask=0077
[Install]
WantedBy=multi-user.target
@@ -0,0 +1,45 @@
[Unit]
Description=AIOS external GPT read-only Online Task ingress
After=network.target aios-chatgpt-bridge-v2-staging.service aios-agent-task-api.service
Wants=aios-chatgpt-bridge-v2-staging.service aios-agent-task-api.service
[Service]
Type=simple
DynamicUser=yes
StateDirectory=aios-gpt-online-task-ingress
StateDirectoryMode=0750
LoadCredential=gpt_ingress_token:/etc/aios-gitops/gpt-online-task-ingress.token
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
Environment=AIOS_GPT_INGRESS_TOKEN_FILE=%d/gpt_ingress_token
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
Environment=AIOS_GPT_INGRESS_STATE_ROOT=%S/aios-gpt-online-task-ingress
Environment=AIOS_GPT_INGRESS_HOST=127.0.0.1
Environment=AIOS_GPT_INGRESS_PORT=18945
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_gpt_online_task_ingress.py
Restart=on-failure
RestartSec=2
NoNewPrivileges=yes
PrivateTmp=yes
PrivateDevices=yes
ProtectSystem=strict
ProtectHome=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectControlGroups=yes
ProtectKernelLogs=yes
ProtectClock=yes
LockPersonality=yes
RestrictSUIDSGID=yes
RestrictRealtime=yes
RestrictNamespaces=yes
MemoryDenyWriteExecute=yes
RestrictAddressFamilies=AF_UNIX AF_INET
IPAddressDeny=any
IPAddressAllow=127.0.0.0/8
InaccessiblePaths=/data/aios-gitops /etc/aios-gitops /home
UMask=0077
[Install]
WantedBy=multi-user.target
@@ -0,0 +1,48 @@
[Unit]
Description=AIOS Hermes Online Agent Worker
After=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
Wants=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
[Service]
Type=simple
User=aios-engineering-worker
Group=aios-engineering-worker
SupplementaryGroups=gitops aios-executor-exchange
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token
Environment=PYTHONUNBUFFERED=1
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
Environment=AIOS_AGENT_ID=online-hermes-worker-cn-network-mkt
Environment=AIOS_AGENT_TYPE=external-agent-runtime-worker
Environment=AIOS_AGENT_VERSION=2026-08-18.1-hermes-runtime
Environment=AIOS_WORKER_CAPABILITIES=agent.runtime.hermes
Environment=AIOS_WORKER_ALLOWED_ADAPTERS=hermes
Environment=AIOS_AUTHORITY_GIT_DIR=/data/aios-gitops/repo.git
Environment=AIOS_EXTERNAL_AGENT_EXECUTOR_ROOT=/var/lib/aios-external-agent-executor
Environment=AIOS_WORKER_STATE_ROOT=/var/lib/aios-online-agent-hermes
Environment=AIOS_WORKER_CHECKPOINT_DB=/var/lib/aios-online-agent-hermes/checkpoints.sqlite3
Environment=AIOS_WORKER_WORKSPACE_ROOT=/var/lib/aios-online-agent-hermes/workspaces
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_online_agent_worker.py
Restart=always
RestartSec=5
TimeoutStopSec=20
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
LockPersonality=true
RestrictSUIDSGID=true
RestrictRealtime=true
RestrictNamespaces=true
MemoryDenyWriteExecute=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
ReadOnlyPaths=/data/aios-gitops/repo.git
ReadWritePaths=/var/lib/aios-online-agent-hermes
ReadWritePaths=/var/lib/aios-external-agent-executor
UMask=0027
[Install]
WantedBy=multi-user.target
@@ -0,0 +1,48 @@
[Unit]
Description=AIOS OpenClaw Online Agent Worker
After=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
Wants=aios-agent-task-api.service aios-chatgpt-bridge-v2-staging.service aios-external-agent-executor.service
[Service]
Type=simple
User=aios-engineering-worker
Group=aios-engineering-worker
SupplementaryGroups=gitops aios-executor-exchange
LoadCredentialEncrypted=agent_token:/etc/credstore.encrypted/aios-online-agent-worker-20260815/agent_token
Environment=PYTHONUNBUFFERED=1
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
Environment=AIOS_AGENT_ID=online-openclaw-worker-cn-network-mkt
Environment=AIOS_AGENT_TYPE=external-agent-runtime-worker
Environment=AIOS_AGENT_VERSION=2026-08-18.1-openclaw-runtime
Environment=AIOS_WORKER_CAPABILITIES=agent.runtime.openclaw
Environment=AIOS_WORKER_ALLOWED_ADAPTERS=openclaw
Environment=AIOS_AUTHORITY_GIT_DIR=/data/aios-gitops/repo.git
Environment=AIOS_EXTERNAL_AGENT_EXECUTOR_ROOT=/var/lib/aios-external-agent-executor
Environment=AIOS_WORKER_STATE_ROOT=/var/lib/aios-online-agent-openclaw
Environment=AIOS_WORKER_CHECKPOINT_DB=/var/lib/aios-online-agent-openclaw/checkpoints.sqlite3
Environment=AIOS_WORKER_WORKSPACE_ROOT=/var/lib/aios-online-agent-openclaw/workspaces
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_online_agent_worker.py
Restart=always
RestartSec=5
TimeoutStopSec=20
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
LockPersonality=true
RestrictSUIDSGID=true
RestrictRealtime=true
RestrictNamespaces=true
MemoryDenyWriteExecute=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
ReadOnlyPaths=/data/aios-gitops/repo.git
ReadWritePaths=/var/lib/aios-online-agent-openclaw
ReadWritePaths=/var/lib/aios-external-agent-executor
UMask=0027
[Install]
WantedBy=multi-user.target
@@ -0,0 +1,27 @@
[Unit]
Description=AIOS persistent Online Agent worker availability guard
After=local-fs.target
[Service]
Type=oneshot
User=root
Group=root
Environment=PYTHONUNBUFFERED=1
Environment=AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS=90
Environment=AIOS_WORKER_AVAILABILITY_RESTART_COOLDOWN_SECONDS=60
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_worker_availability_guard.py --heal
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ProtectKernelTunables=true
ProtectKernelModules=true
ProtectControlGroups=true
LockPersonality=true
RestrictSUIDSGID=true
RestrictRealtime=true
RestrictNamespaces=true
MemoryDenyWriteExecute=true
RestrictAddressFamilies=AF_UNIX
ReadWritePaths=/run/aios-worker-availability
UMask=0022
@@ -0,0 +1,13 @@
[Unit]
Description=Periodically ensure AIOS Online Agent workers remain available
[Timer]
OnBootSec=15s
OnUnitActiveSec=30s
AccuracySec=5s
RandomizedDelaySec=3s
Persistent=true
Unit=aios-worker-availability-guard.service
[Install]
WantedBy=timers.target
@@ -0,0 +1,12 @@
d /var/lib/aios-external-agent-executor 2770 aios-engineering-executor aios-executor-exchange -
d /var/lib/aios-external-agent-executor/requests 2770 aios-engineering-worker aios-executor-exchange -
d /var/lib/aios-external-agent-executor/results 2770 aios-engineering-executor aios-executor-exchange -
d /var/lib/aios-external-agent-executor/failed 2770 aios-engineering-executor aios-executor-exchange -
d /var/lib/aios-external-agent-executor/openclaw-workspace 0750 aios-engineering-executor aios-executor-exchange -
d /var/lib/aios-external-agent-executor/openclaw-state 0750 aios-engineering-executor aios-executor-exchange -
d /var/lib/aios-external-agent-executor/openclaw-home 0750 aios-engineering-executor aios-executor-exchange -
d /var/lib/aios-online-agent-hermes 0750 aios-engineering-worker aios-engineering-worker -
d /var/lib/aios-online-agent-hermes/workspaces 0750 aios-engineering-worker aios-engineering-worker -
d /var/lib/aios-online-agent-openclaw 0750 aios-engineering-worker aios-engineering-worker -
d /var/lib/aios-online-agent-openclaw/workspaces 0750 aios-engineering-worker aios-engineering-worker -
d /etc/aios/external-agent-runtime.d 0755 root root -
@@ -0,0 +1 @@
d /var/lib/aios-model-governance 0750 aios-engineering-executor aios-executor-exchange -
@@ -0,0 +1 @@
d /run/aios-worker-availability 0755 root root -
@@ -0,0 +1,17 @@
# R4.2 P2-B Fleet Order Preservation Repair
P2-C live acceptance exposed a real integration defect in the verified P2-B candidate. The fleet
scheduler correctly produced durable cross-scope fairness order, but the integration passed the
entire ranked list to the legacy scheduler. The legacy scheduler then applied its own priority /
created_at / task_id sort and silently discarded fleet ordering.
The repair preserves the legacy scheduler as the fail-closed security oracle while changing how it
is called: one fleet-ranked candidate is presented at a time. A candidate rejected by legacy
security/policy advances to the next fleet candidate. A candidate returned by the oracle must have
the same task ID as the candidate presented, otherwise execution fails closed.
Task ownership remains unchanged: the original schedule_for_worker/claim_task CAS remains
authoritative, and a fleet reservation is still only bound after the claim succeeds.
This repair does not modify FleetScheduler state semantics, capacity, WriterLease, checkpoint,
protected refs or Production behavior.
@@ -0,0 +1,23 @@
# R4.2 P2-B Fleet Runtime Integration
P2-B connects the verified P2 fleet primitives to the existing TaskPlane pull scheduler without
replacing its security or ownership logic.
The integration uses two stages. `FleetScheduler.rank_for_worker()` performs only read-only
capacity/fairness/fault-isolation ranking. The original `TaskScheduler.next_queued_task()` is then
run over that ordered list and remains the fail-closed security oracle for worker liveness,
capabilities, data classification, execution zone and other existing policy checks. Only a task
accepted by the original scheduler receives a short durable fleet reservation.
The original `TaskScheduler.schedule_for_worker()` remains the sole claim path and continues to call
the existing TaskPlane `claim_task` CAS. After a successful claim, the claim ID is bound to the
fleet reservation. A claim error releases the reservation. A fleet reservation is never treated as
task ownership.
P2-B also makes the reservation capacity check atomic inside the fleet SQLite transaction so two
concurrent pullers cannot oversubscribe the same worker slot. Circuit and task/worker quarantine
state are enforced on the real pull path. Explicit execution-failure signals can be recorded through
the integrated scheduler's failure/success/quarantine methods; policy/claim failures are not
automatically mislabeled as worker crashes.
No new daemon, database or production write path is introduced.
@@ -0,0 +1,123 @@
# External Agent Attempt-Scoped Request Identity
## Problem
External Agent execution previously derived its durable request ID from:
```text
task_id
job_id
execution_digest
runtime
```
The Online Task attempt was missing.
That is incorrect for retryable external side effects. A failed Model Governance route reservation is
durable and terminal for that request ID. Reusing the same external request ID on a later official
Task attempt therefore reuses the old route decision/reservation and can fail settlement with:
```text
MODEL_RESERVATION_STATE_INVALID
```
Archiving an exchange failed marker does not fix the root cause because the Model Governance
reservation remains intentionally durable.
## Contract
External request identity is now:
```text
hash(
task_id,
job_id,
execution_digest,
runtime,
attempt
)
```
The attempt is a **transient execution coordinate**. It is not added to the signed governed execution
envelope and does not alter the Task/Job binding or execution digest.
The worker gets the authoritative attempt from the successful claim response and passes it through:
```text
process_claim
→ RuntimeAdapter.execute(..., attempt=N)
→ ExternalAgentExecutor
→ ExternalAgentExchange
→ external request JSON
→ external result JSON
```
The external result must bind the same attempt.
## Idempotency
This gives the intended two-level behavior:
```text
same Task/Job/execution/runtime + same attempt
→ same external request ID
→ same external result/reservation can be replayed safely
same Task/Job/execution/runtime + next official attempt
→ different external request ID
→ different Model Governance request ID
→ new reservation
```
Old FAILED reservations are retained for audit. No Model Governance DB cleanup is required or
permitted.
## Model Governance
Model Governance already derives its route request ID from:
```text
external_request_id + ":model"
```
Therefore making the external request ID attempt-scoped automatically makes route decisions and
reservations attempt-scoped without creating another state authority.
## Recovery
For the current Hermes live Task:
```text
attempt 6
→ old request/reservation remains FAILED
official NEEDS_REWORK
→ bounded retry extension 6 → 7
attempt 7
→ new external request ID
→ new route request ID
→ new reservation
→ provider execution
→ settlement
```
No replacement Task/Job, direct DB rewrite, failed-reservation deletion or marker deletion is
required.
## Worker Availability
The repair must preserve the already verified Worker Availability runtime. Runtime switches use the
maintenance inhibit and must finish with the guard healthy and all three workers fresh.
## Verification
Required regression coverage:
- request ID stable within one attempt;
- request ID different across attempts;
- request schema requires a valid attempt;
- external result binds attempt;
- old failed Model Governance reservation remains FAILED;
- next attempt gets a new reservation and can settle;
- existing external-agent and Model Governance runtime tests still pass.
@@ -0,0 +1,18 @@
# Generic Compact Interface-Map Profiles
The compact interface-map runtime is a reusable read-only control-plane capability. It keeps the
canonical `source_capture` action and `compact_interface_map` adapter, but selects a bounded semantic
profile from the task's source signals.
The existing `fleet` profile remains backward compatible and returns
`P2_INTERFACE_MAP_CAPTURED` with the `FLEET_*` contracts.
The new `model_governance` profile is selected when model/provider/routing/budget/evaluation terms
are present. It returns `MODEL_GOVERNANCE_INTERFACE_MAP_CAPTURED` and model-specific contracts for
provider registry, selection policy, version pinning, budget accounting, fallback, scope isolation,
credential boundary, trace/audit, evaluation gates and routing resilience.
Profile and classification are emitted by the compact-map module and revalidated by the worker
before result submission. Unknown or mismatched profile/classification pairs fail closed.
No new Bridge action, daemon, database, WriterLease, mutation path or credential surface is added.
@@ -0,0 +1,142 @@
# Hermes / OpenClaw Online Agent Runtime
## Goal
Promote Hermes and OpenClaw from shadow-only adapter descriptions to real Online Agent workers
without creating a second Task/claim protocol.
Both workers reuse the existing P1/P2 Online Agent lifecycle:
```text
Online Task projected
→ capability-filtered poll
→ automatic claim
→ heartbeat
→ durable checkpoint
→ external Agent execution
→ attempt-aware result submission
→ official verification
```
They do **not** change the existing TaskPlane `OpenClawAdapter/HermesAdapter` shadow contract. Those
adapters remain side-effect-free compatibility surfaces.
## Execution action
The canonical execution envelope adds a read-only action:
```text
action=agent_runtime
adapter=hermes | openclaw
```
The action requires capability `agent.runtime.<adapter>`, has an empty change set, no verification
commands, and `writer_lease=false`.
`project_online_task()` copies only bounded task fields into the signed execution envelope:
- goal;
- instructions;
- success criteria;
- non-secret constraints.
The task may not choose a command, executable, URL, credential path, environment, or shell string.
## Worker separation
The generic engineering worker defaults to:
```text
AIOS_WORKER_ALLOWED_ADAPTERS=deterministic,source_capture,compact_interface_map,codex
```
so it skips Hermes/OpenClaw tasks before claim.
Dedicated services advertise:
```text
Hermes:
AIOS_WORKER_CAPABILITIES=agent.runtime.hermes
AIOS_WORKER_ALLOWED_ADAPTERS=hermes
OpenClaw:
AIOS_WORKER_CAPABILITIES=agent.runtime.openclaw
AIOS_WORKER_ALLOWED_ADAPTERS=openclaw
```
This makes adapter selection a pre-claim capability/fleet decision instead of a post-claim failure.
## Credential-isolated executor
The Online Agent worker itself still owns only Task API/Bridge coordination credentials. It writes
a hash-bound request to `/var/lib/aios-external-agent-executor`.
`aios-external-agent-executor.service` runs as the existing uncredentialed engineering executor user.
It has:
- one provider-only `LoadCredential` projection for the model-ready OneAPI environment;
- `/etc/aios-gitops`, `/data/aios-gitops`, and the original provider-secret directory inaccessible;
- no repository path;
- no shell execution;
- loopback denied, preventing direct access to the AIOS Task/Bridge control plane;
- only the staging OneAPI container address allowed for the installed Agent runtime/provider;
- only the exchange directory writable.
The root-owned runtime profile in `/etc/aios/external-agent-runtime.d/<runtime>.json` decides the
absolute executable, fixed non-secret environment, provider-only credential mapping, and I/O mode.
A task cannot override any of these values. The profile may select `OPENAI_API_KEY` only from the
systemd credential named `aios-external-agent-oneapi-env`; it cannot name a credential, inject an
environment variable, or carry an inline secret. The runtime executor parses only the selected
`ONEAPI_REGULAR_TOKEN` entry and does not expose the source file or other provider tiers to the
child.
## Runtime profile
A profile supports:
- `argv`: fixed argv, absolute allowlisted executable;
- `input_mode`: `stdin_text`, `stdin_json`, or `file_json`;
- `output_mode`: `stdout_text`, `stdout_json`, or `file_json`;
- timeout;
- bounded output size.
`{input_json}` and `{output_json}` placeholders are allowed only for file modes. `{prompt}` is a
root-profile-only argv placeholder populated from the bounded, signed task projection.
The repository includes the staging profile templates validated against the installed Hermes
v0.20.0 and OpenClaw v2026.7.2-beta.7 CLIs. Before activation, the Agent must re-run the actual
`--help`/version probes and install the profiles and static configs root-owned. If the installed
CLI differs, update only the root-owned profile/config projection and record the observed command;
do not modify the Task contract to compensate.
The provider projection is staging-only and uses the existing model-ready OneAPI source. It is not
an AIOS Task, Bridge, GitOps, SSH, or Writer credential. Live activation remains blocked if the
model-ready source cannot be projected through systemd or if the selected provider credential is
missing/quota-invalid. This worker never passes API keys through task metadata or arbitrary
environment variables.
## Retry/restart
The external request ID is deterministic over Task/Job/execution digest/runtime. The exchange keeps
the request/result across worker restart. A repeated attempt with the exact execution digest reuses
the durable external result instead of invoking the Agent twice.
The normal P1 result idempotency/checkpoint path remains authoritative for Task result submission.
## Result
Successful execution returns:
```text
status=SUCCEEDED
classification=EXTERNAL_AGENT_RUNTIME_SUCCEEDED
execution_mode=agent_runtime
agent_runtime=hermes | openclaw
writer_lease=NONE
changed_files=[]
credential_material_exposed=false
control_plane_access_granted_to_runtime=false
```
This first runtime version is intentionally read-only with respect to the repository. Engineering
changes continue to use assistant-authored deterministic plans and governed WriterLease/commit.
@@ -0,0 +1,109 @@
# Model Governance Policy / Profile Floor Compatibility
## Problem
The root-owned OpenClaw runtime profile reserves a minimum of 16,384 input tokens so Model
Governance can cover OpenClaw-injected workspace, skills and system context before provider
execution.
The staging scope policies still limited input to 16,000 tokens per request. Route selection
therefore failed before reservation with:
```text
MODEL_INPUT_TOKEN_LIMIT_DENY
```
This was a configuration invariant violation:
```text
scope max_input_tokens_per_request
<
root-owned profile input_reservation_floor_tokens
```
The registered OpenClaw model itself supports 32,768 input tokens, so the model capability was not
the blocker.
## Invariant
For every Model Governance scope that allows a model/runtime whose root-owned profile declares an
input reservation floor:
```text
scope.max_input_tokens_per_request
>=
profile.model_governance.input_reservation_floor_tokens
```
The profile floor remains root-owned. Task payloads cannot alter it.
## Staging policy
The two staging policies that allow `openai/deepseek-v4-flash` are updated from:
```text
16000
```
to:
```text
16384
```
This is the minimum compatible value. It intentionally remains below the model's 32,768-token
technical input ceiling.
The daily token budget is unchanged.
## Governance path
The live SQLite policy must be changed only through the repository bootstrap and
`aios_model_governance_admin.py`.
The admin reconciler:
- reads the governed repository config;
- uses current policy revision as CAS;
- calls `ModelGovernanceStore.set_scope_policy`;
- appends normal Model Governance audit evidence;
- verifies the audit chain.
Direct SQL updates remain forbidden.
## Exhausted OpenClaw recovery
The fixed OpenClaw live Task has exhausted its normal three attempts.
After the governed policy candidate is officially verified and the live policy is reconciled:
```text
attempt 3 VERIFYING/FAILED result
→ official NEEDS_REWORK
→ Task becomes FAILED 3/3
→ official bounded retry extension 3 → 4
→ same Task / same Job returns QUEUED
→ persistent OpenClaw worker auto-claims attempt 4
```
AR1 attempt-scoped external request identity ensures attempt 4 receives a new external request and a
new Model Governance reservation.
No historical reservation is deleted or rewritten.
## Acceptance
Attempt 4 must prove:
```text
scope max_input_tokens_per_request=16384
profile input_reservation_floor_tokens=16384
reserved_estimated_input_tokens>=16384
actual input <= reserved input
settlement=SETTLED
usage_source=runtime
audit_chain_valid=true
```
If attempt 4 exposes a new failure, stop. No additional retry extension is authorized by this
repair.
@@ -0,0 +1,211 @@
# AIOS Model Governance Runtime
## Status and purpose
This component turns the previously `PARTIAL` Model Governance interface map into an enforceable
runtime policy for the already verified Hermes/OpenClaw Online Agent path.
It is intentionally a **single SQLite library**, not a new network service. The existing external
Agent executor calls the library immediately before launching a model-backed runtime and settles the
same reservation immediately after the result.
The authoritative execution chain remains:
```text
Online Task scope / trace
→ dedicated worker claim + heartbeat
→ external Agent executor
→ ModelGovernanceStore.route()
→ root-owned runtime profile binds provider/model/version
→ Hermes / OpenClaw process
→ ModelGovernanceStore.record_success() / record_failure()
→ durable Online Agent checkpoint/result
```
P0 scope/trace/credential, P1 checkpoint/retry, P2 fleet scheduling and the existing external Agent
credential boundary remain authoritative.
## Ten Model Governance contracts
### MODEL_PROVIDER_REGISTRY
`model_governance_providers` stores provider identity, failure domain, circuit policy and an opaque
credential reference. Raw credentials are rejected.
Allowed reference prefixes:
- `systemd-credential:`
- `runtime-profile:`
- `credential-broker:`
### MODEL_SELECTION_POLICY
`model_governance_scope_policies` constrains eligible models/providers/capabilities. Routing is
deterministic: explicit preferred models, then configured priority, then stable provider/model IDs.
### MODEL_VERSION_PINNING
A registry row always includes a model version. A scope can additionally pin a model ID + exact
version. Every route decision persists both model ID and version plus model/policy revisions.
### MODEL_BUDGET_ACCOUNTING
A route first reserves estimated input/output tokens and estimated cost inside `BEGIN IMMEDIATE`.
Settled usage and outstanding reservations both count against the daily scope budget, preventing two
concurrent requests from oversubscribing the same budget.
Runtime-reported usage is preferred. When telemetry is absent, the reserved upper bound is charged
rather than silently under-counting.
OpenClaw's verified result shape is supported directly:
```text
meta.agentMeta.usage.input
meta.agentMeta.usage.output
meta.agentMeta.costUsd
```
Hermes currently has no structured usage in the verified output, so it uses conservative
reservation settlement.
### MODEL_FALLBACK_POLICY
Provider failures are durable. Repeated failures open a circuit for the configured cooldown. A
fallback request excludes the failed provider and increments `fallback_index`; the scope policy
bounds the number of fallback attempts.
### MODEL_SCOPE_ISOLATION
The signed `agent_runtime` execution envelope now carries authoritative Task scope + trace identity.
The projection layer always overwrites `runtime_input.scope` and `runtime_input.trace_id` from the
Online Task, even if an execution request attempts to supply different values.
Tenant/project lookup may fall back only to the same tenant's policy. It never falls back to a
system or legacy policy. `system` and `legacy` require their own explicit policies, avoiding fake
tenant identities.
### MODEL_CREDENTIAL_BOUNDARY
Model governance stores credential references only. The root-owned external-runtime profile still
owns the actual provider credential projection. Task payloads cannot select executable, argv,
environment, provider credential, or credential path.
The external child process remains unable to reach the AIOS control plane.
### MODEL_TRACE_AUDIT
Every route, failure, provider/model/policy update and settlement appends to a SHA-256 hash-chained
audit table with task ID, trace ID and scope key. `verify_audit_chain()` detects tampering.
### MODEL_EVALUATION_GATE
A model cannot enter the registry unless its state is `APPROVED` and it has a 64-hex evaluation
evidence digest.
The staging bootstrap uses the already verified Hermes/OpenClaw E2E external-result SHA256 values as
evaluation evidence, rather than a configuration-file hash.
### MODEL_ROUTING_RESILIENCE
Route request IDs are idempotent and bound to a canonical request hash. Conflicting replay is
rejected. Provider health, reservations, usage settlement and policy/provider/model updates use
transaction/CAS semantics.
## Runtime profile contract
A root-owned Hermes/OpenClaw profile may declare:
```json
{
"model_governance": {
"required": true,
"provider_ids": ["provider-a"],
"model_capabilities": ["text"],
"max_output_tokens": 512
}
}
```
When `required=true`:
- `provider_ids` must be non-empty;
- `max_output_tokens` must be positive;
- argv/environment must contain at least one governance-controlled marker:
`{provider_id}`, `{model_id}`, or `{model_version}`;
- `AIOS_MODEL_GOVERNANCE_ENABLED` must be enabled;
- a matching scope policy/model/provider must exist;
- budget reservation must succeed before the child process is launched.
The task cannot provide these markers or their values.
## SQLite authority
Default database:
```text
/var/lib/aios-model-governance/state.sqlite3
```
The executor process already owns the only runtime mutation path. The directory is created by
tmpfiles and is the only new writable path added to that executor.
No direct Task/Job/Git database writes are introduced.
## Bootstrap/reconciliation
`aios_model_governance_admin.py` applies a root-owned JSON bootstrap idempotently. It compares desired
state to current rows and uses the current revision as the CAS input only when a change is actually
required.
The repository staging bootstrap is not a Production policy. It exists only to reproduce and
govern the already verified non-production Hermes/OpenClaw runtimes.
## Cost caveat
The current verified OneAPI-backed runtime evidence reports zero authoritative provider cost for
OpenClaw and no structured cost for Hermes. The staging bootstrap therefore does **not** invent
pricing and sets cost rates/budget to zero/unbounded.
Token budgets remain enforced. Real monetary budget enforcement becomes effective as soon as
authoritative per-model pricing is entered into the root-owned registry.
## OpenClaw JIT/WASM compatibility
The previous live acceptance proved that OpenClaw Node/WASM requires
`MemoryDenyWriteExecute=no`. Model Governance must preserve that already-evidenced runtime exception;
it must not further weaken `NoNewPrivileges`, `ProtectSystem=strict`, `ProtectHome`, loopback deny,
credential isolation or repository isolation.
## Failure/recovery invariants
- Budget denial occurs before provider execution.
- Missing scope policy fails closed.
- Missing/disabled governance fails closed when a profile requires it.
- Actual usage cannot exceed the reservation.
- Provider failure releases that reservation into `FAILED` state before fallback.
- Settlement is idempotent.
- Route replay is idempotent only for the exact canonical request.
- Audit corruption is detectable.
- Online Agent checkpoint/result retry remains unchanged and remains the outer recovery authority.
## Verification
Required unit/integration coverage includes:
- registry/evaluation/CAS;
- scope isolation and no system→tenant fallback;
- model/version pinning;
- atomic budget reservation;
- settlement/replay;
- missing telemetry conservative settlement;
- provider circuit/fallback;
- trace/audit tamper detection;
- authoritative scope/trace projection;
- task runtime-input scope override attack;
- root-owned provider/model binding;
- OpenClaw nested usage extraction;
- real Hermes/OpenClaw staging live tasks with Model Governance evidence.
The governed implementation is not complete without the unified all-modified-files bundle for the
candidate delta and the cumulative lineage bundle.
@@ -0,0 +1,80 @@
# OpenClaw Input Reservation Floor
## Problem
Model Governance originally estimated input reservation from the user-visible prompt only.
The verified OpenClaw runtime injects additional root-owned/system-controlled context including
workspace, skills and system instructions. In live acceptance the prompt estimate reserved only
153 input tokens while OpenClaw reported approximately 7,770 actual input tokens.
The settlement correctly failed closed with:
```text
MODEL_USAGE_EXCEEDS_RESERVATION
```
The provider call itself had succeeded. The defect was therefore pre-execution reservation
underestimation, not provider connectivity.
## Contract
A root-owned external Agent profile may now declare:
```json
{
"model_governance": {
"input_reservation_floor_tokens": 16384
}
}
```
The effective input reservation is:
```text
max(
byte-based prompt estimate,
profile input_reservation_floor_tokens
)
```
Only the root-owned runtime profile can choose this floor. Online Task payloads cannot lower or
override it.
The OpenClaw staging profile uses 16,384 tokens. This is greater than twice the observed 7,770-token
live input high-water mark while remaining below the registered 32,768-token model input limit.
Hermes has no floor unless explicitly configured and therefore preserves its existing behavior.
## Fail-closed behavior
The floor is not an unlimited post-execution top-up.
If actual runtime usage exceeds the reserved amount, settlement still fails with
`MODEL_USAGE_EXCEEDS_RESERVATION`.
This keeps budget enforcement pre-execution and avoids silently authorizing usage after it has
already happened.
## Evidence
External results expose:
```text
prompt_estimated_input_tokens
input_reservation_floor_tokens
reserved_estimated_input_tokens
```
alongside the normal settlement and audit evidence.
## Retry
The current OpenClaw attempt 1 failure remains preserved. After this code repair is governed and
activated, official `NEEDS_REWORK` returns the same fixed Task to the normal retry path. Because AR1
already made external request identity attempt-scoped, attempt 2 receives a new external request ID
and a new reservation.
No retry-extension is required because the Task is currently attempt 1 of 3.
No Model Governance reservation is deleted or rewritten.
@@ -0,0 +1,36 @@
# Owner-Controlled Same-Binding Execution Rebind
## Purpose
`aios_agent_task_execution_rebind.py` closes a projection/plan race without replacing the Task,
Governed Job, target branch, generation, or idempotency lineage.
The only supported transition is:
```text
official NEEDS_REWORK
→ Task QUEUED with no active lease
→ persisted source_capture envelope
→ canonical Bridge resolves an already-registered deterministic plan
→ CAS-bound metadata.execution + execution_projection replacement
→ audit event
→ normal worker claim
```
The service rejects arbitrary metadata patches. It derives the new envelope from canonical Bridge
`/v2/execution-envelope`, validates every governed binding coordinate against the persisted envelope,
requires `source_capture → deterministic`, requires exact old/new execution digests supplied by the
Owner recovery request, and increments the Task revision under `BEGIN IMMEDIATE`.
It records `TASK_EXECUTION_REBIND_AUTHORIZED` and an idempotent `task_execution_rebinds` receipt.
No Task/Job/branch replacement is created and the worker still performs the real claim/heartbeat,
WriterLease and governed commit.
The endpoint is loopback-only:
`POST /v1/tasks/{task_id}/execution-rebind`
and requires the existing Owner control credential plus
`AIOS_AGENT_TASK_EXECUTION_REBIND_AUTHORIZATION_ID`.
Direct/manual Task database edits remain forbidden; the service is the audited mutation boundary.
@@ -0,0 +1,212 @@
# AIOS Worker Availability Runtime
## Priority
This runtime exists to make the already verified Online Agent execution path continuously usable
without repeated human intervention.
The system already proved that generic, Hermes and OpenClaw workers can claim and complete real
Online Tasks. The remaining availability defect was operational: evidence-freeze workflows left the
worker services stopped, while registry `ACTIVE` records remained stale.
The availability contract therefore separates:
```text
AUTO_CLAIM_MECHANISM=VERIFIED
CURRENT_WORKER_AVAILABILITY=<fresh runtime state>
```
A historical successful claim is not current liveness.
## Source of truth for idle liveness
An idle worker may have no Task lease heartbeat. Therefore Task heartbeat alone cannot prove that
the worker is continuously polling.
`aios_online_agent_worker.py` now writes:
```text
<AIOS_WORKER_STATE_ROOT>/worker-liveness.json
```
on:
- process start: `STARTING`;
- every successful poll: `POLL_OK`;
- poll failure: `POLL_FAILED`;
- every successful Task lease heartbeat while executing: `BUSY`.
The file contains:
- agent ID/type/version;
- process PID;
- monotonically increasing local sequence;
- `last_seen_epoch`;
- poll/heartbeat intervals;
- current Task/stage while busy.
The liveness file is worker-owned observability state only. It does not write or mutate Task, Job,
Git, WriterLease or registry databases.
## Self-healing guard
`aios-worker-availability-guard.timer` invokes a root-owned oneshot guard every 30 seconds.
The guard manages:
```text
aios-online-agent-worker-20260815.service
aios-hermes-online-agent-worker.service
aios-openclaw-online-agent-worker.service
```
and the required external runtime dependency:
```text
aios-external-agent-executor.service
```
For each worker:
1. missing systemd unit → fail closed;
2. disabled → `systemctl enable`;
3. inactive → `systemctl start`;
4. active but liveness older than 90 seconds → restart;
5. repeated stale restart within 60 seconds → do not restart again; report degraded.
Long-running Tasks are safe because the Task lease-heartbeat thread refreshes `BUSY` liveness.
The guard never claims a Task and never touches Task/Job/Git databases.
## Maintenance inhibit
Planned maintenance creates:
```text
/run/aios-worker-availability/maintenance.lock
```
While this file exists the guard performs no start/restart actions and reports:
```text
MAINTENANCE_INHIBITED
```
Maintenance tooling must remove the file when done. This prevents the self-healer from fighting
intentional service changes.
## Status
The guard writes:
```text
/run/aios-worker-availability/status.json
```
with current worker/dependency states, liveness ages and recovery actions.
A worker is currently available only when:
```text
systemd load_state=loaded
systemd active_state=active
unit enabled/static
worker-liveness.json present
last_seen_age <= 90s
```
Registry `ACTIVE` without fresh poll/busy liveness is not sufficient.
## Boot/restart behavior
The timer is:
```text
OnBootSec=15s
OnUnitActiveSec=30s
Persistent=true
```
Therefore reboot, manual stop left behind by another workflow, process crash, and hung poll loop are
all repaired without requiring a human to notice and intervene.
Normal worker units retain their own `Restart=always`; the availability guard handles the cases that
`Restart=always` cannot, especially a deliberate `systemctl stop` left after an evidence workflow.
## Acceptance
The live acceptance requires all of the following:
```text
guard timer active/enabled
external executor active/enabled
generic worker active/enabled + fresh liveness
Hermes worker active/enabled + fresh liveness
OpenClaw worker active/enabled + fresh liveness
```
Then a new read-only Online Task is projected through canonical Bridge `18892`.
Success requires:
```text
TASK_CREATED
→ generic worker auto claim
→ TASK_HEARTBEAT
→ RESULT_SUBMITTED
→ official TASK_VERIFIED(SUCCEEDED)
```
After the Task, the services must remain active and liveness must continue advancing. The acceptance
also stops a worker intentionally once and proves the guard starts it again with a new fresh
liveness sequence.
Only after these gates may current status be upgraded from:
```text
OVERALL=PARTIAL
```
to:
```text
PERSISTENT_WORKER_LIVENESS=VERIFIED
NEW_TASK_AUTO_CONSUMPTION=VERIFIED
WORKER_AVAILABILITY_RUNTIME=VERIFIED_CANDIDATE
```
Protected main and Production remain out of scope.
## Forced source-capture projection compatibility
The deployed canonical Bridge currently projects a default `source_capture` envelope even when a
governed binding request asks for binding-only behavior. This must not be allowed to make always-on
workers consume engineering Tasks before their deterministic plan is registered.
The worker therefore distinguishes a real read-only source capture from an engineering placeholder:
```text
source_capture
+ capability_scope contains governed.write.branch or governed.write.file
= PENDING_DETERMINISTIC_PLAN
```
For such a placeholder the worker:
1. re-resolves through the canonical plan registry **without replaying the stale source-capture
envelope**;
2. if no deterministic plan exists, emits `TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN` and does not
claim;
3. if the registered canonical envelope is deterministic and the governed binding still matches,
uses that deterministic envelope and claims normally.
A real read-only source-capture Task with only `git.read`/`git.test` still replays its exact persisted
payload and remains claimable.
This makes persistent workers safe with the currently deployed Bridge while preserving the same
Task/Job binding. No Task DB rewrite, cancellation, replacement Task, fake NEEDS_REWORK, or
pre-claim metadata mutation is required.
A later Bridge cleanup may implement true binding-only projection, but Worker Availability does not
depend on that cleanup.
@@ -0,0 +1,120 @@
# Disabled-by-default HTTPS edge for external GPT Online Task access
## Purpose
The internal ingress (`127.0.0.1:18945`) remains loopback-only. This edge is the only component
allowed to become Internet-facing, and only after a separate owner approval.
It forwards only:
```text
POST /v1/read-only-tasks
GET /v1/read-only-tasks/{request_id}
GET /openapi.json
GET /health
```
It never proxies:
```text
18892 canonical Bridge
18940 Task API
WriterLease
commit
verification
release
tag
baseline
arbitrary URLs
```
## Default state
The systemd unit is disabled by policy unless the owner creates:
```text
/etc/aios-gitops/gpt-https-edge.approved
```
The base unit still binds only to:
```text
127.0.0.1:18443
AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=0
```
Non-loopback bind is rejected by code unless an owner-controlled drop-in sets:
```text
AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=1
AIOS_GPT_EDGE_HOST=0.0.0.0
AIOS_GPT_EDGE_PUBLIC_DOMAIN=<approved hostname>
```
This repository candidate does not create that drop-in or approval marker.
## TLS
TLS certificate/key are owner-managed systemd credentials:
```text
/etc/aios-gitops/gpt-https-edge-tls.crt
/etc/aios-gitops/gpt-https-edge-tls.key
```
The service requires TLS 1.2 or newer, disables TLS compression, and never logs or returns key
material.
Certificate/key issuance and renewal are external owner responsibilities until a separately
governed ACME workflow is implemented.
## Request protections
The edge provides:
- 64 KiB request cap;
- 256 KiB upstream response cap;
- JSON validation before proxying;
- fixed upstream `http://127.0.0.1:18945`;
- Authorization-only header forwarding;
- no Cookie/X-Forwarded-* trust;
- route/method allowlist;
- per-source-IP in-memory rate limiting;
- bounded concurrent request handling;
- optional exact Host-header enforcement;
- HSTS / no-store / nosniff / no-referrer response headers.
The rate limiter is intentionally simple and local. Multi-node/global abuse protection is outside
the current one-host architecture.
## Authentication
The HTTPS edge does not create a new authentication scheme. It forwards the external Bearer
credential to the loopback GPT ingress, which verifies `gpt_ingress_token`.
The edge never receives the AIOS control-plane token.
## Deployment boundary
Public activation is a high-risk network operation and requires explicit owner approval. Before
activation verify:
```text
host-side GPT ingress E2E=VERIFIED
owner ingress token provisioned
TLS certificate/key provisioned
public hostname owned and DNS configured
firewall/network exposure reviewed
explicit public-bind approval recorded
rollback plan available
```
Only after those gates may an owner-authorized runtime drop-in bind this service publicly.
Until then:
```text
GPT_HTTPS_EDGE_IMPLEMENTATION=candidate
PUBLIC_NETWORK_EXPOSURE=NONE
PUBLIC_CUSTOM_GPT_DIRECT_AI0S_ACCESS=NOT_VERIFIED
```
@@ -0,0 +1,138 @@
# External GPT → Online Task ingress
## Scope
This component closes the current architectural gap between an external Custom GPT and AIOS
without exposing the internal Task API (`18940`) or canonical Bridge (`18892`) directly.
The first supported public-facing capability is intentionally read-only:
```text
authenticated external intent
→ host-side ingress
→ canonical /v2/governed-task
→ canonical /v2/execution-envelope
→ persistent generic worker
→ source_capture result
→ official control-plane verification
```
The ingress itself never claims, heartbeats, verifies, commits, acquires WriterLease, or writes
protected refs.
## Security boundary
The service binds only to loopback (`127.0.0.1:18945`). A future owner-approved TLS reverse proxy
or tunnel may expose this service externally. Direct non-loopback binding is denied by code.
Two independent credentials are projected by systemd:
- `gpt_ingress_token`: authenticates the external caller to this ingress.
- `agent_task_control_token`: lets the ingress call the loopback AIOS control plane.
Neither credential is stored in Git, SQLite, task payloads, results, logs, or API responses.
The ingress has no repository filesystem access.
## Caller-controlled fields
Allowed:
```text
request_id
goal
instructions
success_criteria
constraints.max_files
constraints.focus_paths
constraints.notes
```
Not caller-controlled:
```text
repository
base_oid
expected_branch_oid
target_branch
branch_mode
generation
risk
priority
max_attempts
capabilities
action/adapter
WriterLease
worker identity
credentials
verification decision
main/production/tag/baseline permissions
```
All external tasks are hard-bound to:
```text
repository=aios/aios
capabilities=[git.read, git.test]
action=source_capture
writer_lease=false
repository_mutation=forbidden
control_plane_side_effects=forbidden
credential_access=forbidden
```
## Idempotency
`request_id` is the durable external idempotency boundary.
The ingress stores only:
```text
request hash
derived AIOS task id
canonical job id
base oid
sanitized receipt
state
```
An exact retry returns the same receipt. Reusing a `request_id` with different content returns
`IDEMPOTENCY_CONFLICT`.
A crash after canonical task creation can recover the same binding from the Task API rather than
creating a replacement Task.
## Status
`GET /v1/read-only-tasks/{request_id}` returns only sanitized state:
```text
task/job ids
status
attempt
whether claimed
result classification
official verification decision/classification
source snapshot hash/count when available
```
It does not proxy raw Task metadata, execution envelopes, result text, credentials, or source files.
## Public exposure
This candidate does **not** make AIOS publicly reachable. Public exposure requires a separately
approved HTTPS ingress/reverse proxy with:
- TLS;
- external authentication lifecycle;
- rate limiting / abuse controls;
- request size limits;
- observability;
- explicit owner approval for the network exposure.
Until that step is verified:
```text
HOST_SIDE_EXTERNAL_GPT_ADAPTER=implemented/staging-candidate
PUBLIC_CUSTOM_GPT_DIRECT_AI0S_ACCESS=not verified
```
@@ -0,0 +1,70 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[3]
ACTIVE_RUNTIME = ROOT / "active-runtime"
if str(ACTIVE_RUNTIME) not in sys.path:
sys.path.insert(0, str(ACTIVE_RUNTIME))
from app.task_plane.scheduler import TaskScheduler
def _security_contract():
path = Path(__file__).with_name("test_task_plane_security_contract.py")
spec = importlib.util.spec_from_file_location("p2b_order_security_contract", path)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _create_task(service, contract, task_id, tenant, project):
return service.create_task(
{
"task_id": task_id,
"issuer": {"type": "agent", "id": contract.ADMIN_IDENTITY},
"required_capabilities": ["code.read"],
"scope_type": "tenant",
"tenant_id": tenant,
"project_id": project,
"metadata": {"fleet_priority": 0},
}
)
def _task_id(claim):
return getattr(
claim,
"task_id",
claim.get("task_id") if isinstance(claim, dict) else None,
)
def test_P2B_FLEET_ORDER_is_preserved_through_legacy_claim_oracle():
contract = _security_contract()
service = contract._service()
service.register_worker(contract._worker_payload(max_concurrency=3))
worker_id = contract.WORKER_ID
scheduler = TaskScheduler(service.store)
scheduler.fleet_scheduler.configure_worker(
worker_id, max_slots=3, failure_domain="zone-a"
)
_create_task(service, contract, "p2b-order-a1", "tenant-a", "project-a")
_create_task(service, contract, "p2b-order-a2", "tenant-a", "project-a")
_create_task(service, contract, "p2b-order-b1", "tenant-b", "project-b")
first = scheduler.schedule_for_worker(worker_id)
second = scheduler.schedule_for_worker(worker_id)
assert _task_id(first) == "p2b-order-a1"
assert _task_id(second) == "p2b-order-b1"
rows = service.store.conn.execute(
"SELECT scope_key,dispatch_count FROM fleet_scope_state ORDER BY scope_key"
).fetchall()
assert len(rows) >= 2
@@ -0,0 +1,181 @@
from __future__ import annotations
from dataclasses import dataclass, field
import importlib.util
from pathlib import Path
import sqlite3
import sys
import threading
import time
import pytest
ROOT = Path(__file__).resolve().parents[3]
ACTIVE_RUNTIME = ROOT / "active-runtime"
if str(ACTIVE_RUNTIME) not in sys.path:
sys.path.insert(0, str(ACTIVE_RUNTIME))
from app.task_plane.fleet import FleetConflict, FleetScheduler
from app.task_plane.errors import NotFoundError
from app.task_plane.scheduler import TaskScheduler
def _security_contract():
path = Path(__file__).with_name("test_task_plane_security_contract.py")
spec = importlib.util.spec_from_file_location("p2b_security_contract", path)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_P2B_REAL_DISPATCH_claim_binds_fleet_reservation_and_circuit():
contract = _security_contract()
service = contract._service()
service.register_worker(contract._worker_payload(max_concurrency=2))
worker_id = contract.WORKER_ID
service.create_task(
{
"task_id": "task-p2b-real-1",
"issuer": {"type": "agent", "id": contract.ADMIN_IDENTITY},
"required_capabilities": ["code.read"],
}
)
scheduler = TaskScheduler(service.store)
scheduler.fleet_scheduler.configure_worker(
worker_id,
max_slots=2,
failure_domain="zone-real",
labels={"runtime": "task-plane"},
)
first = scheduler.schedule_for_worker(worker_id)
assert first is not None
assert first.task_id == "task-p2b-real-1"
row = service.store.conn.execute(
"SELECT state,claim_id,worker_id FROM fleet_dispatches WHERE task_id=?",
("task-p2b-real-1",),
).fetchone()
assert tuple(row) == ("CLAIMED", first.claim_id, worker_id)
# A real circuit blocks the actual scheduler even though one capacity slot remains.
failure_now = time.time()
scheduler.record_worker_failure(
worker_id, "runtime_crash", threshold=1, now=failure_now
)
service.create_task(
{
"task_id": "task-p2b-real-2",
"issuer": {"type": "agent", "id": contract.ADMIN_IDENTITY},
"required_capabilities": ["code.read"],
}
)
with pytest.raises(NotFoundError, match="no queued tasks available"):
scheduler.schedule_for_worker(worker_id)
scheduler.record_worker_success(worker_id, now=failure_now + 1.0)
second = scheduler.schedule_for_worker(worker_id)
assert second is not None
assert second.task_id == "task-p2b-real-2"
@dataclass
class Plan:
task_id: str
required_capabilities: list[str] = field(default_factory=lambda: ["code.read"])
metadata: dict = field(default_factory=dict)
scope_type: str = "tenant"
tenant_id: str = "tenant-a"
project_id: str | None = "project-a"
user_id: str | None = None
@dataclass
class Task:
task_id: str
revision: int = 1
status: str = "QUEUED"
created_at: float = 1.0
task_plan: Plan | None = None
def __post_init__(self):
if self.task_plan is None:
self.task_plan = Plan(self.task_id)
@dataclass
class Worker:
worker_id: str
capabilities: list[str] = field(default_factory=lambda: ["code.read"])
class Store:
def __init__(self):
self.conn = sqlite3.connect(":memory:", check_same_thread=False)
self._lock = threading.RLock()
self.conn.execute(
"CREATE TABLE claims("
"claim_id TEXT PRIMARY KEY,task_id TEXT,worker_id TEXT,status TEXT,"
"lease_expires_at REAL)"
)
self.tasks = []
self.workers = {}
def list_tasks(self, _ctx=None):
return list(self.tasks)
def get_worker(self, worker_id):
return self.workers[worker_id]
class Registry:
def __init__(self):
self.online = set()
def is_online(self, worker_id, now=None):
return worker_id in self.online
def _fleet():
store = Store()
registry = Registry()
store.workers["w-a"] = Worker("w-a")
registry.online.add("w-a")
fleet = FleetScheduler(store, registry, now_fn=lambda: 1000.0)
fleet.configure_worker("w-a", max_slots=1, failure_domain="zone-a")
return store, fleet
def test_P2B_CAPACITY_RESERVATION_is_atomic_under_concurrent_pullers():
store, fleet = _fleet()
tasks = [Task("task-a"), Task("task-b")]
store.tasks = tasks
barrier = threading.Barrier(2)
outcomes = []
def reserve(task):
barrier.wait()
try:
fleet.reserve_for_worker("w-a", task, now=1000.0)
outcomes.append("reserved")
except FleetConflict:
outcomes.append("conflict")
threads = [threading.Thread(target=reserve, args=(task,)) for task in tasks]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
assert sorted(outcomes) == ["conflict", "reserved"]
assert fleet.state.reserved_slots("w-a", 1000.0) == 1
def test_P2B_RANK_is_read_only_until_legacy_security_accepts_candidate():
store, fleet = _fleet()
store.tasks = [Task("task-a"), Task("task-b")]
ranked = fleet.rank_for_worker("w-a", now=1000.0)
assert [task.task_id for task in ranked] == ["task-a", "task-b"]
assert fleet.state.reserved_slots("w-a", 1000.0) == 0
@@ -0,0 +1,33 @@
from __future__ import annotations
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "active-runtime/app"))
from aios_capability_contract import ( # noqa: E402
has_capabilities,
normalize_required_capabilities,
normalize_worker_capabilities,
)
@pytest.mark.parametrize("runtime", ["hermes", "openclaw"])
def test_runtime_capability_is_known_to_task_and_worker_vocabulary(runtime: str) -> None:
capability = f"agent.runtime.{runtime}"
assert normalize_required_capabilities([capability]) == frozenset({capability})
assert normalize_worker_capabilities([capability]) == frozenset({capability})
assert has_capabilities([capability], [capability]) is True
def test_runtime_capabilities_are_isolated_between_workers() -> None:
assert has_capabilities(["agent.runtime.hermes"], ["agent.runtime.openclaw"]) is False
assert has_capabilities(["agent.runtime.openclaw"], ["agent.runtime.hermes"]) is False
def test_unknown_runtime_capability_remains_fail_closed() -> None:
assert has_capabilities(["agent.runtime.hermes"], ["agent.runtime.unknown"]) is False
+293
View File
@@ -0,0 +1,293 @@
from __future__ import annotations
from pathlib import Path
import hashlib
import json
import sqlite3
import sys
import pytest
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/"active-runtime/app"))
from aios_agent_task_execution_rebind import (
ExecutionRebindError,
ExecutionRebindRequest,
ExecutionRebindStore,
)
from aios_online_agent_execution import (
GovernedExecutionBinding,
project_execution_envelope,
)
AUTHORIZATION="owner-execution-rebind-20260818"
TASK_ID="task-rebind-a"
JOB_ID="job-rebind-a"
BASE="a"*40
BRANCH="agent/task-rebind-a"
IDEMPOTENCY="task-rebind-a-binding-v1"
@pytest.fixture(autouse=True)
def _auth(monkeypatch):
monkeypatch.setenv(
"AIOS_AGENT_TASK_EXECUTION_REBIND_AUTHORIZATION_ID",
AUTHORIZATION,
)
def canon(value):
return hashlib.sha256(
json.dumps(value,sort_keys=True,separators=(",",":")).encode()
).hexdigest()
def binding():
return GovernedExecutionBinding.from_mapping({
"task_id":TASK_ID,
"job_id":JOB_ID,
"repository":"aios/aios",
"base_oid":BASE,
"expected_branch_oid":BASE,
"target_branch":BRANCH,
"branch_mode":"CREATE_NEW",
"generation":1,
"idempotency_key":IDEMPOTENCY,
})
def old_envelope():
return project_execution_envelope(
binding(),
{
"action":"source_capture",
"adapter":"compact_interface_map",
"capability_scope":[
"git.read","git.test","governed.write.branch","governed.write.file"
],
},
)
def new_envelope():
return project_execution_envelope(
binding(),
{
"action":"deterministic",
"adapter":"deterministic",
"capability_scope":[
"git.read","git.test","governed.write.branch","governed.write.file"
],
"change_set":[
{
"path":"active-runtime/app/example.py",
"operation":"write_file",
"content":"VALUE=1\\n",
}
],
"verification":[
{
"label":"unit",
"argv":["/usr/bin/python3","-m","pytest","-q"],
"cwd":".",
"timeout_seconds":120,
}
],
"governance":{
"commit_idempotency_key":"task-rebind-a-commit-v1",
},
},
)
def make_db(tmp_path, *, status="QUEUED", revision=5, attempt=2, lease=False):
path=tmp_path/"state.sqlite3"
con=sqlite3.connect(path)
con.executescript("""
CREATE TABLE tasks(
task_id TEXT PRIMARY KEY,
status TEXT NOT NULL,
attempt INTEGER NOT NULL,
max_attempts INTEGER NOT NULL,
revision INTEGER NOT NULL,
claimed_by TEXT,
lease_token_hash TEXT,
lease_expires_at TEXT,
verification_json TEXT,
result_json TEXT,
metadata_json TEXT,
required_capabilities_json TEXT
);
CREATE TABLE task_events(
event_id INTEGER PRIMARY KEY AUTOINCREMENT,
task_id TEXT,
event_type TEXT,
agent_id TEXT,
payload_json TEXT,
created_at TEXT
);
""")
verification={"decision":"NEEDS_REWORK","reason":"WRONG_ENVELOPE_BEFORE_PLAN"}
old=old_envelope()
metadata={
"execution":old,
"execution_projection":{
"schema":"aios.online-agent.task-projection.v1",
"task_id":TASK_ID,
"job_id":JOB_ID,
"execution_digest":old["execution_digest"],
},
"preserved":{"value":"unchanged"},
}
con.execute(
"INSERT INTO tasks VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
(
TASK_ID,status,attempt,3,revision,
"previous-worker" if not lease else "active-worker",
"lease-hash" if lease else None,
"2099-01-01T00:00:00Z" if lease else None,
json.dumps(verification),
json.dumps({"status":"SUCCEEDED"}),
json.dumps(metadata),
json.dumps([
"git.read","git.test","governed.write.branch","governed.write.file"
]),
),
)
con.commit();con.close()
return path,verification
def request(verification, *, revision=5, old=None, new=None, key="rebind-task-a-v1"):
old=old or old_envelope()
new=new or new_envelope()
return ExecutionRebindRequest.from_mapping({
"expected_status":"QUEUED",
"expected_revision":revision,
"expected_attempt":2,
"previous_verification_sha256":canon(verification),
"expected_old_execution_digest":old["execution_digest"],
"expected_new_execution_digest":new["execution_digest"],
"authorization_id":AUTHORIZATION,
"idempotency_key":key,
"reason":"projection-before-plan race recovery",
})
def resolver(_task_id,_caps):
return new_envelope()
def test_rebind_cas_updates_only_execution_projection_and_audits(tmp_path):
path,verification=make_db(tmp_path)
result=ExecutionRebindStore(path,resolver).rebind(
TASK_ID,request(verification)
)
assert result["status"]=="QUEUED"
assert result["revision"]==6
assert result["idempotent"] is False
con=sqlite3.connect(path)
row=con.execute(
"SELECT status,attempt,revision,claimed_by,metadata_json FROM tasks"
).fetchone()
assert row[:4]==("QUEUED",2,6,None)
metadata=json.loads(row[4])
assert metadata["preserved"]=={"value":"unchanged"}
assert metadata["execution"]["action"]=="deterministic"
assert metadata["execution"]["execution_digest"]==new_envelope()["execution_digest"]
assert metadata["execution_projection"]["execution_digest"]==new_envelope()["execution_digest"]
event=con.execute(
"SELECT event_type,payload_json FROM task_events"
).fetchone()
assert event[0]=="TASK_EXECUTION_REBIND_AUTHORIZED"
payload=json.loads(event[1])
assert payload["old_action"]=="source_capture"
assert payload["new_action"]=="deterministic"
assert payload["stale_claim_normalized"] is True
assert con.execute("SELECT COUNT(*) FROM task_execution_rebinds").fetchone()[0]==1
def test_rebind_idempotent_replay(tmp_path):
path,verification=make_db(tmp_path)
store=ExecutionRebindStore(path,resolver)
req=request(verification)
first=store.rebind(TASK_ID,req)
second=store.rebind(TASK_ID,req)
assert first["idempotent"] is False
assert second["idempotent"] is True
assert second["new_execution_digest"]==first["new_execution_digest"]
def test_rebind_rejects_active_lease(tmp_path):
path,verification=make_db(tmp_path,lease=True)
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_ACTIVE_LEASE_DENY"):
ExecutionRebindStore(path,resolver).rebind(TASK_ID,request(verification))
def test_rebind_rejects_verification_drift(tmp_path):
path,_=make_db(tmp_path)
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_VERIFICATION_DRIFT"):
ExecutionRebindStore(path,resolver).rebind(
TASK_ID,request({"decision":"NEEDS_REWORK","reason":"other"})
)
def test_rebind_rejects_nonqueued_task(tmp_path):
path,verification=make_db(tmp_path,status="VERIFYING")
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_CAS_MISMATCH"):
ExecutionRebindStore(path,resolver).rebind(TASK_ID,request(verification))
def test_rebind_rejects_non_deterministic_canonical_resolution(tmp_path):
path,verification=make_db(tmp_path)
def wrong(_task,_caps):
return old_envelope()
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_DETERMINISTIC_PLAN_REQUIRED"):
ExecutionRebindStore(path,wrong).rebind(TASK_ID,request(verification))
def test_rebind_rejects_binding_drift(tmp_path):
path,verification=make_db(tmp_path)
different=project_execution_envelope(
GovernedExecutionBinding.from_mapping({
"task_id":TASK_ID,
"job_id":"job-other",
"repository":"aios/aios",
"base_oid":BASE,
"expected_branch_oid":BASE,
"target_branch":BRANCH,
"branch_mode":"CREATE_NEW",
"generation":1,
"idempotency_key":IDEMPOTENCY,
}),
{
"action":"deterministic",
"capability_scope":["git.read"],
"change_set":[{"path":"x.py","operation":"write_file","content":"x=1\\n"}],
},
)
def wrong(_task,_caps):
return different
with pytest.raises(Exception):
ExecutionRebindStore(path,wrong).rebind(TASK_ID,request(verification))
def test_rebind_request_is_narrow_and_digest_bound():
verification={"decision":"NEEDS_REWORK"}
data=request(verification).public()
data["arbitrary_metadata"]={"x":1}
with pytest.raises(ExecutionRebindError,match="EXECUTION_REBIND_FIELDS_INVALID"):
ExecutionRebindRequest.from_mapping(data)
def test_execution_rebind_unit_is_unprivileged_and_owner_credential_bound():
unit=(ROOT/"deploy/systemd/aios-agent-task-execution-rebind.service").read_text()
assert "User=aios-agent-task" in unit
assert "Group=aios-agent-task" in unit
assert "LoadCredential=agent_task_control_token:" in unit
assert "Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892" in unit
assert "RestrictAddressFamilies=AF_UNIX AF_INET" in unit
assert "User=root" not in unit
@@ -0,0 +1,232 @@
from __future__ import annotations
import json
from pathlib import Path
import sys
import pytest
ROOT=Path(__file__).resolve().parents[1]
APP=ROOT/"active-runtime/app"
sys.path.insert(0,str(APP))
import aios_online_agent_worker as worker
from aios_external_agent_runtime import (
ExternalAgentExchange,
ExternalAgentRuntimeError,
validate_request,
)
def envelope():
value=object.__new__(worker.ExecutionEnvelope)
object.__setattr__(value,"task_id","task-attempt-identity")
object.__setattr__(value,"job_id","job-attempt-identity")
object.__setattr__(value,"digest","d"*64)
return value
def test_external_request_id_is_stable_within_attempt_and_changes_across_attempts():
env=envelope()
a1=worker.external_agent_request_id(env,"hermes",6)
a1_replay=worker.external_agent_request_id(env,"hermes",6)
a2=worker.external_agent_request_id(env,"hermes",7)
assert a1==a1_replay
assert a1!=a2
assert a1.startswith("external-hermes-")
assert a2.startswith("external-hermes-")
def test_external_request_id_rejects_invalid_attempt():
env=envelope()
with pytest.raises(worker.WorkerError,match="TASK_ATTEMPT_INVALID"):
worker.external_agent_request_id(env,"hermes",0)
def test_external_request_schema_binds_attempt():
value=validate_request({
"schema":"aios.external-agent-runtime.request.v1",
"request_id":"external-hermes-0123456789abcdef",
"runtime":"hermes",
"task_id":"task-attempt-identity",
"attempt":7,
"execution_digest":"d"*64,
"payload":{
"goal":"Return ok.",
"instructions":"",
"success_criteria":[],
"constraints":{},
},
})
assert value["attempt"]==7
def test_external_request_schema_rejects_missing_attempt():
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_ATTEMPT_INVALID"):
validate_request({
"schema":"aios.external-agent-runtime.request.v1",
"request_id":"external-hermes-0123456789abcdef",
"runtime":"hermes",
"task_id":"task-attempt-identity",
"execution_digest":"d"*64,
"payload":{
"goal":"Return ok.",
"instructions":"",
"success_criteria":[],
"constraints":{},
},
})
def _write_bound_result(path, request):
import hashlib
from aios_external_agent_runtime import canonical, sha256
material={
"schema":"aios.external-agent-runtime.result.v1",
"request_id":request["request_id"],
"request_sha256":sha256(canonical(request)),
"runtime":request["runtime"],
"task_id":request["task_id"],
"attempt":request["attempt"],
"execution_digest":request["execution_digest"],
"exit_code":0,
"duration_ms":1,
"output_text":"ok",
"output_sha256":hashlib.sha256(b"ok").hexdigest(),
"stderr_sha256":hashlib.sha256(b"").hexdigest(),
"credential_material_exposed":False,
"control_plane_access_granted":False,
}
material["result_sha256"]=sha256(canonical(material))
path.write_text(json.dumps(material,sort_keys=True),encoding="utf-8")
def test_exchange_result_binding_rejects_wrong_attempt(tmp_path):
exchange=ExternalAgentExchange(tmp_path)
request={
"schema":"aios.external-agent-runtime.request.v1",
"request_id":"external-hermes-0123456789abcdef",
"runtime":"hermes",
"task_id":"task-attempt-identity",
"attempt":7,
"execution_digest":"d"*64,
"payload":{
"goal":"Return ok.",
"instructions":"",
"success_criteria":[],
"constraints":{},
},
}
request=validate_request(request)
result_path=tmp_path/"result.json"
_write_bound_result(result_path,request)
value=json.loads(result_path.read_text())
value["attempt"]=6
# Recompute digest so the binding check, not the digest check, is what fails.
from aios_external_agent_runtime import canonical,sha256
material=dict(value); material.pop("result_sha256")
value["result_sha256"]=sha256(canonical(material))
result_path.write_text(json.dumps(value),encoding="utf-8")
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_RESULT_BINDING_INVALID"):
exchange._validated_result(
result_path,
request,
sha256(canonical(request)),
)
def test_model_governance_request_id_is_attempt_scoped():
env=envelope()
attempt6=worker.external_agent_request_id(env,"hermes",6)+":model"
attempt7=worker.external_agent_request_id(env,"hermes",7)+":model"
assert attempt6!=attempt7
def test_failed_previous_attempt_does_not_poison_next_model_governance_reservation(tmp_path):
from aios_model_governance import (
ModelGovernanceStore,
ModelRecord,
ProviderRecord,
RouteRequest,
ScopePolicy,
)
store=ModelGovernanceStore(tmp_path/"model-governance.sqlite3",now_fn=lambda:1_800_000_000.0)
store.register_provider(
ProviderRecord(
"oneapi",
"runtime-profile:hermes",
max_failures=5,
cooldown_seconds=60,
)
)
store.register_model(
ModelRecord(
"deepseek-v4-flash",
"oneapi",
"v1",
("agent.runtime.hermes","text"),
input_cost_micros_per_1k=1,
output_cost_micros_per_1k=1,
max_input_tokens=10000,
max_output_tokens=1000,
evaluation_sha256="a"*64,
)
)
store.set_scope_policy(
ScopePolicy(
"tenant",
"model-governance-live",
"staging",
allowed_models=("deepseek-v4-flash",),
required_capabilities=("text",),
daily_token_budget=100000,
daily_cost_budget_micros=100000,
fallback_limit=1,
)
)
def route(request_id):
return store.route(
RouteRequest(
request_id=request_id,
task_id="r42-model-governance-hermes-live-20260818",
trace_id="trace-hermes-model-governance-live",
scope_type="tenant",
tenant_id="model-governance-live",
project_id="staging",
required_capabilities=("agent.runtime.hermes",),
estimated_input_tokens=100,
max_output_tokens=50,
)
)
attempt6=route("external-hermes-attempt6:model")
store.record_failure(attempt6.request_id,"provider_process_failed")
row6=store.conn.execute(
"select state from model_governance_reservations where request_id=?",
(attempt6.request_id,),
).fetchone()
assert row6["state"]=="FAILED"
attempt7=route("external-hermes-attempt7:model")
assert attempt7.request_id!=attempt6.request_id
row7=store.conn.execute(
"select state from model_governance_reservations where request_id=?",
(attempt7.request_id,),
).fetchone()
assert row7["state"]=="RESERVED"
settled=store.record_success(
attempt7.request_id,
input_tokens=90,
output_tokens=40,
usage_source="runtime",
)
assert settled["state"]=="SETTLED"
assert store.conn.execute(
"select state from model_governance_reservations where request_id=?",
(attempt6.request_id,),
).fetchone()["state"]=="FAILED"
assert store.verify_audit_chain() is True
store.close()
+421
View File
@@ -0,0 +1,421 @@
from __future__ import annotations
import json
import os
from pathlib import Path
import sys
import pytest
ROOT=Path(__file__).resolve().parents[1]
APP=ROOT/"active-runtime/app"
sys.path.insert(0,str(APP))
from aios_external_agent_runtime import (
ExternalAgentRuntimeError,
RuntimeProfile,
execute_request,
)
from aios_online_agent_execution import (
ExecutionProjectionError,
GovernedExecutionBinding,
project_online_task,
validate_execution_envelope,
)
import aios_online_agent_worker as worker
BASE="a"*40
def binding(task_id="task-hermes-runtime"):
return GovernedExecutionBinding.from_mapping({
"task_id":task_id,
"job_id":"job-"+task_id,
"repository":"aios/aios",
"base_oid":BASE,
"expected_branch_oid":BASE,
"target_branch":"agent/"+task_id,
"branch_mode":"CREATE_NEW",
"generation":1,
"idempotency_key":task_id+"-binding-v1",
})
def task_request(task_id, runtime):
return {
"task_id":task_id,
"goal":"Summarize the requested operation.",
"instructions":"Return a concise result.",
"success_criteria":[{"id":"done","description":"A result is returned."}],
"constraints":{"network":"runtime-managed"},
"required_capabilities":[f"agent.runtime.{runtime}"],
"max_attempts":3,
}
def projected(task_id="task-hermes-runtime", runtime="hermes"):
return project_online_task(
binding(task_id),
task_request(task_id,runtime),
{
"action":"agent_runtime",
"adapter":runtime,
"capability_scope":[f"agent.runtime.{runtime}"],
},
)
def clear_secret_env(monkeypatch):
for key in list(os.environ):
upper=key.upper()
if any(marker in upper for marker in (
"TOKEN","SECRET","PASSWORD","CREDENTIAL","AUTHORIZATION","PRIVATE_KEY"
)):
monkeypatch.delenv(key,raising=False)
monkeypatch.setenv("CREDENTIALS_DIRECTORY","/definitely-not-present-aios-test")
def make_helper(tmp_path):
helper=tmp_path/"runtime_helper.py"
helper.write_text(
"import json,sys\n"
"value=json.load(sys.stdin)\n"
"print(json.dumps({'answer':'ok','goal':value['goal']},sort_keys=True))\n",
encoding="utf-8",
)
return helper
def make_profile(tmp_path,runtime,helper):
root=tmp_path/"profiles";root.mkdir(exist_ok=True)
(root/f"{runtime}.json").write_text(
json.dumps({
"schema":"aios.external-agent-runtime.profile.v1",
"runtime":runtime,
"argv":["/usr/bin/python3",str(helper)],
"input_mode":"stdin_json",
"output_mode":"stdout_json",
"timeout_seconds":30,
"max_output_bytes":65536,
}),
encoding="utf-8",
)
return root
def make_text_request(tmp_path, runtime, payload=None):
request={
"schema":"aios.external-agent-runtime.request.v1",
"request_id":f"external-{runtime}-0123456789abcdef",
"runtime":runtime,
"task_id":f"task-{runtime}-runtime",
"attempt":1,
"execution_digest":"d"*64,
"payload":payload or {
"goal":"Return ok.",
"instructions":"No side effects.",
"success_criteria":[],
"constraints":{},
},
}
request_path=tmp_path/"request.json"
result_path=tmp_path/"result.json"
request_path.write_text(json.dumps(request),encoding="utf-8")
return request_path, result_path
def test_agent_runtime_projection_is_read_only_and_task_bound():
value=projected()
execution=value["metadata"]["execution"]
assert execution["action"]=="agent_runtime"
assert execution["adapter"]=="hermes"
assert execution["runtime_input"]["goal"]==value["goal"]
assert execution["runtime_input"]["instructions"]==value["instructions"]
assert execution["governance"]["writer_lease"] is False
assert execution["change_set"]==[]
assert execution["verification"]==[]
validate_execution_envelope(
execution,
binding=binding(),
worker_capabilities=["agent.runtime.hermes"],
)
def test_agent_runtime_projection_requires_runtime_capability():
with pytest.raises(ExecutionProjectionError,match="AGENT_RUNTIME_CAPABILITY_REQUIRED"):
project_online_task(
binding(),
task_request("task-hermes-runtime","hermes"),
{
"action":"agent_runtime",
"adapter":"hermes",
"capability_scope":["git.read"],
},
)
def test_agent_runtime_projection_rejects_mutation():
with pytest.raises(ExecutionProjectionError,match="AGENT_RUNTIME_MUTATION_DENY"):
project_online_task(
binding(),
task_request("task-hermes-runtime","hermes"),
{
"action":"agent_runtime",
"adapter":"hermes",
"capability_scope":["agent.runtime.hermes"],
"change_set":[{"path":"x.py","operation":"write_file","content":"x=1\n"}],
},
)
def test_agent_runtime_projection_rejects_secret_constraint_keys():
request=task_request("task-hermes-runtime","hermes")
request["constraints"]={"api_token":"do-not-project"}
with pytest.raises(ExecutionProjectionError,match="AGENT_RUNTIME_SECRET_FIELD_DENY"):
project_online_task(
binding(),
request,
{
"action":"agent_runtime",
"adapter":"hermes",
"capability_scope":["agent.runtime.hermes"],
},
)
@pytest.mark.parametrize("runtime",["hermes","openclaw"])
def test_external_executor_runs_root_owned_profile_without_shell(tmp_path,monkeypatch,runtime):
clear_secret_env(monkeypatch)
helper=make_helper(tmp_path)
profile_root=make_profile(tmp_path,runtime,helper)
request={
"schema":"aios.external-agent-runtime.request.v1",
"request_id":f"external-{runtime}-0123456789abcdef",
"runtime":runtime,
"task_id":f"task-{runtime}-runtime",
"attempt":1,
"execution_digest":"d"*64,
"payload":{
"goal":"Return ok.",
"instructions":"No side effects.",
"success_criteria":[],
"constraints":{},
},
}
request_path=tmp_path/"request.json"
result_path=tmp_path/"result.json"
request_path.write_text(json.dumps(request),encoding="utf-8")
execute_request(request_path,result_path,profile_root=profile_root)
result=json.loads(result_path.read_text())
assert result["runtime"]==runtime
assert result["exit_code"]==0
assert result["output_json"]["answer"]=="ok"
assert result["credential_material_exposed"] is False
assert result["control_plane_access_granted"] is False
def test_runtime_profile_rejects_shell_mode(tmp_path):
root=tmp_path/"profiles";root.mkdir()
(root/"hermes.json").write_text(json.dumps({
"schema":"aios.external-agent-runtime.profile.v1",
"runtime":"hermes",
"argv":["/usr/bin/python3","-c","print('x')"],
"input_mode":"stdin_text",
"output_mode":"stdout_text",
}))
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_SHELL_MODE_DENY"):
RuntimeProfile.load("hermes",root)
def test_external_executor_projects_only_selected_provider_credential(tmp_path,monkeypatch):
clear_secret_env(monkeypatch)
helper=tmp_path/"credential_helper.py"
helper.write_text(
"import os\n"
"value=os.environ.get('OPENAI_API_KEY','')\n"
"print('AIOS_PROVIDER_PROJECTED_OK' if value == 'test-provider-key' else 'AIOS_PROVIDER_PROJECTED_FAIL')\n",
encoding="utf-8",
)
profile_root=tmp_path/"profiles";profile_root.mkdir()
(profile_root/"hermes.json").write_text(json.dumps({
"schema":"aios.external-agent-runtime.profile.v1",
"runtime":"hermes",
"argv":["/usr/bin/python3",str(helper)],
"input_mode":"stdin_text",
"output_mode":"stdout_text",
"environment":{"RUNTIME_PROFILE_MARKER":"staging"},
"credential_env":{
"OPENAI_API_KEY":{
"credential":"provider.env",
"format":"dotenv",
"key":"ONEAPI_REGULAR_TOKEN",
}
},
}),encoding="utf-8")
credential_dir=tmp_path/"credentials";credential_dir.mkdir()
(credential_dir/"provider.env").write_text(
"ONEAPI_REGULAR_TOKEN=test-provider-key\nOTHER_PROVIDER_TOKEN=not-projected\n",
encoding="utf-8",
)
monkeypatch.setenv("CREDENTIALS_DIRECTORY",str(credential_dir))
request_path,result_path=make_text_request(tmp_path,"hermes")
execute_request(request_path,result_path,profile_root=profile_root)
result=json.loads(result_path.read_text())
assert "AIOS_PROVIDER_PROJECTED_OK" in result["output_text"]
assert "test-provider-key" not in result["output_text"]
assert result["credential_material_exposed"] is False
def test_runtime_profile_rejects_credential_path_traversal(tmp_path):
root=tmp_path/"profiles";root.mkdir()
(root/"hermes.json").write_text(json.dumps({
"schema":"aios.external-agent-runtime.profile.v1",
"runtime":"hermes",
"argv":["/usr/bin/python3","-V"],
"input_mode":"stdin_text",
"output_mode":"stdout_text",
"credential_env":{
"OPENAI_API_KEY":{
"credential":"../provider.env",
"format":"dotenv",
"key":"ONEAPI_REGULAR_TOKEN",
}
},
}),encoding="utf-8")
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_CREDENTIAL_NAME_INVALID"):
RuntimeProfile.load("hermes",root)
def test_runtime_profile_prompt_placeholder_is_root_owned(tmp_path,monkeypatch):
clear_secret_env(monkeypatch)
helper=tmp_path/"prompt_helper.py"
helper.write_text(
"import sys\n"
"print('AIOS_PROMPT_PLACEHOLDER_OK' if 'Goal:' in sys.argv[1] else 'AIOS_PROMPT_PLACEHOLDER_FAIL')\n",
encoding="utf-8",
)
profile_root=tmp_path/"profiles";profile_root.mkdir()
(profile_root/"hermes.json").write_text(json.dumps({
"schema":"aios.external-agent-runtime.profile.v1",
"runtime":"hermes",
"argv":["/usr/bin/python3",str(helper),"{prompt}"],
"input_mode":"stdin_text",
"output_mode":"stdout_text",
}),encoding="utf-8")
request_path,result_path=make_text_request(tmp_path,"hermes")
execute_request(request_path,result_path,profile_root=profile_root)
result=json.loads(result_path.read_text())
assert "AIOS_PROMPT_PLACEHOLDER_OK" in result["output_text"]
@pytest.mark.parametrize("runtime,capability",[
("hermes","agent.runtime.hermes"),
("openclaw","agent.runtime.openclaw"),
])
def test_worker_external_adapter_builds_readonly_result(monkeypatch,tmp_path,runtime,capability):
mapped=projected(f"task-{runtime}-runtime",runtime)
envelope=worker.ExecutionEnvelope.from_task(
mapped,
worker_capabilities=(capability,),
)
class FakeExchange:
def __init__(self,*args,**kwargs): pass
def execute(self,**kwargs):
return {
"schema":"aios.external-agent-runtime.result.v1",
"request_id":kwargs["request_id"],
"request_sha256":"e"*64,
"runtime":runtime,
"task_id":kwargs["task_id"],
"attempt":kwargs["attempt"],
"execution_digest":kwargs["execution_digest"],
"exit_code":0,
"duration_ms":10,
"output_text":"done",
"output_sha256":"f"*64,
"stderr_sha256":"0"*64,
"credential_material_exposed":False,
"control_plane_access_granted":False,
"result_sha256":"1"*64,
}
monkeypatch.setattr(worker,"ExternalAgentExchange",FakeExchange)
monkeypatch.setattr(worker,"audit",lambda *a,**k:None)
executor=worker.HermesExecutor() if runtime=="hermes" else worker.OpenClawExecutor()
execution=executor.execute(envelope,tmp_path,attempt=1)
result=worker.build_agent_runtime_result(envelope.task_id,envelope,execution)
assert result["status"]=="SUCCEEDED"
assert result["classification"]=="EXTERNAL_AGENT_RUNTIME_SUCCEEDED"
assert result["execution_mode"]=="agent_runtime"
assert result["agent_runtime"]==runtime
assert result["writer_lease"]=="NONE"
assert result["changed_files"]==[]
assert result["control_plane_access_granted_to_runtime"] is False
def test_worker_adapter_filter_defaults_exclude_external_runtimes():
assert "hermes" not in worker.WORKER_ALLOWED_ADAPTERS
assert "openclaw" not in worker.WORKER_ALLOWED_ADAPTERS
def test_external_executor_systemd_has_no_control_credentials_or_repository_access():
unit=(ROOT/"deploy/systemd/aios-external-agent-executor.service").read_text()
assert "User=aios-engineering-executor" in unit
assert "LoadCredential=aios-external-agent-oneapi-env:/data/agent/secrets/oneapi-model-ready.env" in unit
assert "LoadCredentialEncrypted" not in unit
assert "InaccessiblePaths=/run/credentials" not in unit
assert "InaccessiblePaths=/etc/aios-gitops" in unit
assert "InaccessiblePaths=/data/aios-gitops" in unit
assert "InaccessiblePaths=/data/agent/secrets" in unit
assert "IPAddressDeny=127.0.0.0/8" in unit
assert "IPAddressDeny=::1/128" in unit
assert "IPAddressAllow=172.18.0.2/32" in unit
@pytest.mark.parametrize(
"unit_name,capability,adapter,state_root",
[
(
"aios-hermes-online-agent-worker.service",
"agent.runtime.hermes",
"hermes",
"/var/lib/aios-online-agent-hermes",
),
(
"aios-openclaw-online-agent-worker.service",
"agent.runtime.openclaw",
"openclaw",
"/var/lib/aios-online-agent-openclaw",
),
],
)
def test_dedicated_worker_units_are_capability_and_adapter_scoped(
unit_name,capability,adapter,state_root
):
unit=(ROOT/"deploy/systemd"/unit_name).read_text()
assert f"Environment=AIOS_WORKER_CAPABILITIES={capability}" in unit
assert f"Environment=AIOS_WORKER_ALLOWED_ADAPTERS={adapter}" in unit
assert f"Environment=AIOS_WORKER_STATE_ROOT={state_root}" in unit
assert "AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892" in unit
assert "LoadCredentialEncrypted=agent_token:" in unit
def test_example_profiles_are_non_authoritative_examples_only():
for runtime in ("hermes","openclaw"):
value=json.loads(
(ROOT/"config/external-agent-runtime.d"/f"{runtime}.example.json").read_text()
)
assert value["schema"]=="aios.external-agent-runtime.profile.v1"
assert value["runtime"]==runtime
assert value["input_mode"]=="stdin_text"
assert value["output_mode"]=="stdout_text"
def test_staging_profiles_are_provider_only_and_root_profile_bound():
hermes=json.loads((ROOT/"config/external-agent-runtime.d/hermes.profile.json").read_text())
openclaw=json.loads((ROOT/"config/external-agent-runtime.d/openclaw.profile.json").read_text())
for runtime,value in (("hermes",hermes),("openclaw",openclaw)):
assert value["runtime"]==runtime
assert set(value["credential_env"])=={"OPENAI_API_KEY"}
assert value["credential_env"]["OPENAI_API_KEY"]["credential"]=="aios-external-agent-oneapi-env"
assert "OPENAI_API_KEY" not in value.get("environment",{})
+205
View File
@@ -0,0 +1,205 @@
from __future__ import annotations
from email.message import Message
import json
from pathlib import Path
import stat
import sys
import pytest
ROOT=Path(__file__).resolve().parents[1]
APP=ROOT/"active-runtime/app"
sys.path.insert(0,str(APP))
import aios_gpt_https_edge as edge
class Headers(dict):
def get(self,key,default=None):
return super().get(key,default)
def test_default_and_public_bind_require_explicit_owner_approval():
assert edge.public_bind_allowed("127.0.0.1","0")=="127.0.0.1"
assert edge.public_bind_allowed("::1","0")=="::1"
with pytest.raises(RuntimeError,match="OWNER_APPROVAL_REQUIRED"):
edge.public_bind_allowed("0.0.0.0","0")
with pytest.raises(RuntimeError,match="OWNER_APPROVAL_REQUIRED"):
edge.public_bind_allowed("::","")
assert edge.public_bind_allowed("0.0.0.0","1")=="0.0.0.0"
def test_arbitrary_public_bind_address_is_denied_even_with_approval():
with pytest.raises(RuntimeError,match="BIND_ADDRESS_INVALID"):
edge.public_bind_allowed("192.0.2.10","1")
def test_public_domain_validation():
assert edge.validated_public_domain("tasks.example.com")=="tasks.example.com"
assert edge.validated_public_domain("")==""
with pytest.raises(RuntimeError,match="PUBLIC_DOMAIN_INVALID"):
edge.validated_public_domain("https://example.com")
with pytest.raises(RuntimeError,match="PUBLIC_DOMAIN_INVALID"):
edge.validated_public_domain("localhost")
def test_route_allowlist_accepts_only_readonly_api():
assert edge.allowed_route("POST","/v1/read-only-tasks")=="/v1/read-only-tasks"
assert edge.allowed_route("GET","/v1/read-only-tasks/external-request-0001")==(
"/v1/read-only-tasks/external-request-0001"
)
assert edge.allowed_route("GET","/health")=="/health"
assert edge.allowed_route("GET","/openapi.json")=="/openapi.json"
for method,path in [
("POST","/v2/governed-task"),
("GET","/v1/tasks/x"),
("POST","/v1/read-only-tasks/x"),
("DELETE","/v1/read-only-tasks/x"),
]:
with pytest.raises(edge.EdgeError,match="ROUTE_DENY"):
edge.allowed_route(method,path)
def test_query_and_fragment_are_denied():
with pytest.raises(edge.EdgeError,match="QUERY_OR_FRAGMENT_DENY"):
edge.allowed_route("GET","/health?x=1")
with pytest.raises(edge.EdgeError,match="QUERY_OR_FRAGMENT_DENY"):
edge.allowed_route("GET","/openapi.json#x")
def test_forwarded_headers_are_minimal_and_require_bearer():
incoming=Headers({
"Authorization":"Bearer "+"a"*40,
"Content-Type":"application/json; charset=utf-8",
"Cookie":"secret=1",
"X-Forwarded-For":"203.0.113.10",
"X-Api-Key":"do-not-forward",
})
forwarded=edge.forwarded_headers(incoming,body=True)
assert forwarded=={
"Accept":"application/json",
"Authorization":"Bearer "+"a"*40,
"Content-Type":"application/json",
}
with pytest.raises(edge.EdgeError,match="AUTHORIZATION_REQUIRED"):
edge.forwarded_headers(Headers({}),body=False)
def test_invalid_content_type_is_denied():
with pytest.raises(edge.EdgeError,match="CONTENT_TYPE_INVALID"):
edge.forwarded_headers(
Headers({
"Authorization":"Bearer "+"a"*40,
"Content-Type":"text/plain",
}),
body=True,
)
def test_rate_limiter_caps_window_and_allows_expiry():
now=[100.0]
limiter=edge.RateLimiter(window_seconds=60,limit=2,burst=1,now_fn=lambda:now[0])
assert limiter.allowed("client")
assert limiter.allowed("client")
assert limiter.allowed("client")
assert not limiter.allowed("client")
now[0]=161.0
assert limiter.allowed("client")
def test_rate_limiter_client_table_is_bounded():
now=[100.0]
limiter=edge.RateLimiter(
window_seconds=60,
limit=10,
burst=0,
max_clients=2,
now_fn=lambda:now[0],
)
assert limiter.allowed("a")
assert limiter.allowed("b")
assert not limiter.allowed("c")
now[0]=161.0
assert limiter.allowed("c")
assert len(limiter._events) <= 2
def test_audit_source_does_not_log_sensitive_request_material():
source=(ROOT/"active-runtime/app/aios_gpt_https_edge.py").read_text()
assert '"authorization", "body", "token", "cookie", "client_ip"' in source
assert "_client_pseudonym" in source
def test_proxy_upstream_is_fixed():
proxy=edge.Proxy()
assert proxy.upstream=="http://127.0.0.1:18945"
bad=edge.Proxy(upstream="http://127.0.0.1:18892")
with pytest.raises(edge.EdgeError,match="UPSTREAM_DRIFT_DENY"):
bad.request("GET","/health",{"Authorization":"Bearer "+"a"*40},None)
def test_public_bind_requires_approved_domain():
assert edge.validated_edge_identity("127.0.0.1","0","")==("127.0.0.1","")
with pytest.raises(RuntimeError,match="PUBLIC_DOMAIN_REQUIRED"):
edge.validated_edge_identity("0.0.0.0","1","")
assert edge.validated_edge_identity(
"0.0.0.0","1","tasks.example.com"
)==("0.0.0.0","tasks.example.com")
def test_tls_key_requires_no_group_or_other_permissions(tmp_path):
key=tmp_path/"key.pem"
key.write_text("x"*64,encoding="utf-8")
key.chmod(0o600)
assert edge.validate_tls_file(key,key=True)==key
key.chmod(0o644)
with pytest.raises(RuntimeError,match="TLS_KEY_MODE_INVALID"):
edge.validate_tls_file(key,key=True)
def test_tls_symlink_is_denied(tmp_path):
real=tmp_path/"cert.pem"
real.write_text("x"*64,encoding="utf-8")
link=tmp_path/"link.pem"
link.symlink_to(real)
with pytest.raises(RuntimeError,match="TLS_FILE_INVALID"):
edge.validate_tls_file(link,key=False)
def test_systemd_unit_is_disabled_by_owner_approval_condition_and_loopback_default():
unit=(ROOT/"deploy/systemd/aios-gpt-https-edge.service").read_text()
assert "ConditionPathExists=/etc/aios-gitops/gpt-https-edge.approved" in unit
assert "AIOS_GPT_EDGE_HOST=127.0.0.1" in unit
assert "AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=0" in unit
assert "AIOS_GPT_EDGE_PORT=18443" in unit
assert "LoadCredential=tls_cert:" in unit
assert "LoadCredential=tls_key:" in unit
assert "InaccessiblePaths=/data/aios-gitops" in unit
assert "InaccessiblePaths=" in unit and "/var/lib/aios-agent-task" in unit
assert "NoNewPrivileges=yes" in unit
assert "ProtectSystem=strict" in unit
def test_systemd_unit_does_not_reference_internal_control_credentials():
unit=(ROOT/"deploy/systemd/aios-gpt-https-edge.service").read_text()
assert "agent_task_control" not in unit
assert "gitea" not in unit.lower()
assert "18892" not in unit
assert "18940" not in unit
def test_edge_source_does_not_proxy_internal_bridge_or_task_api():
source=(ROOT/"active-runtime/app/aios_gpt_https_edge.py").read_text()
assert 'UPSTREAM = "http://127.0.0.1:18945"' in source
assert "18892" not in source
assert "18940" not in source
assert "/v2/governed-task" not in source
assert "/v1/tasks/" not in source
def test_docs_keep_public_activation_separately_approved():
doc=(ROOT/"docs/superpowers/specs/2026-08-19-gpt-https-edge.md").read_text()
assert "explicit owner approval" in doc.lower()
assert "PUBLIC_NETWORK_EXPOSURE=NONE" in doc
assert "PUBLIC_CUSTOM_GPT_DIRECT_AI0S_ACCESS=NOT_VERIFIED" in doc
+342
View File
@@ -0,0 +1,342 @@
from __future__ import annotations
import json
from pathlib import Path
import sqlite3
import sys
import pytest
ROOT=Path(__file__).resolve().parents[1]
APP=ROOT/"active-runtime/app"
sys.path.insert(0,str(APP))
from aios_gpt_online_task_ingress import (
IngressError,
IngressService,
Store,
READONLY_CAPABILITIES,
branch_for,
normalized_request,
openapi_document,
task_id_for,
validated_bind_host,
)
MAIN="3244f0f17f8845592ac94d585f5ebd65693c9f29"
JOB="job-0123456789abcdef0123456789abcdef"
def request(request_id="external-request-0001", **overrides):
value={
"schema":"aios.gpt-online-task-ingress.request.v1",
"request_id":request_id,
"goal":"Inspect the repository for durable worker state.",
"instructions":"Read only. Return a concise source analysis.",
"success_criteria":[
{"id":"analysis","description":"Return a source-backed analysis."}
],
"constraints":{
"max_files":50,
"focus_paths":["active-runtime/app","tests"],
"notes":"Do not mutate repository state.",
},
}
value.update(overrides)
return value
class FakeUpstream:
def __init__(self):
self.created=[]
self.projected=[]
self.tasks={}
self.snapshot={"authoritative_head_oid":MAIN}
def control_snapshot(self):
return dict(self.snapshot)
def create_governed_task(self, body):
self.created.append(dict(body))
task_id=body["task_id"]
job_id=JOB
self.tasks[task_id]={
"task_id":task_id,
"status":"PROJECTED",
"attempt":0,
"metadata":{
"execution":{
"task_id":task_id,
"job_id":job_id,
"base_oid":body["base_oid"],
"action":"source_capture",
"capability_scope":list(READONLY_CAPABILITIES),
"governance":{"writer_lease":False},
}
},
}
return 201,{"task_id":task_id,"job_id":job_id,"state":"BOUND"}
def execution_envelope(self, body):
self.projected.append(dict(body))
task_id=body["task_id"]
return 200,{
"schema":"aios.online-agent.execution.v2",
"task_id":task_id,
"job_id":body["job_id"],
"repository":"aios/aios",
"base_oid":MAIN,
"action":"source_capture",
"capability_scope":list(READONLY_CAPABILITIES),
"governance":{"writer_lease":False},
}
def get_task(self, task_id):
value=self.tasks.get(task_id)
return (404,{}) if value is None else (200,dict(value))
def service(tmp_path):
upstream=FakeUpstream()
store=Store(tmp_path/"state.sqlite3")
return IngressService(upstream,store),upstream,store
def test_request_schema_is_narrow_and_rejects_capability_or_execution_control():
value=normalized_request(request())
assert value["goal"].startswith("Inspect")
for field in ("capability_scope","required_capabilities","action","adapter","repository","base_oid"):
bad=request()
bad[field]="evil"
with pytest.raises(IngressError,match="UNKNOWN_FIELDS"):
normalized_request(bad)
def test_constraints_are_bounded_and_path_traversal_is_denied():
bad=request(constraints={"focus_paths":["../etc/passwd"]})
with pytest.raises(IngressError,match="FOCUS_PATHS_INVALID"):
normalized_request(bad)
bad=request(constraints={"max_files":1000})
with pytest.raises(IngressError,match="MAX_FILES_INVALID"):
normalized_request(bad)
def test_non_loopback_bind_is_denied():
assert validated_bind_host("127.0.0.1")=="127.0.0.1"
with pytest.raises(RuntimeError,match="GPT_INGRESS_NON_LOOPBACK_BIND_DENY"):
validated_bind_host("0.0.0.0")
with pytest.raises(RuntimeError,match="GPT_INGRESS_NON_LOOPBACK_BIND_DENY"):
validated_bind_host("192.0.2.10")
def test_task_and_branch_ids_are_deterministic_and_bounded():
a=task_id_for("external-request-0001")
b=task_id_for("external-request-0001")
c=task_id_for("external-request-0002")
assert a==b
assert a!=c
assert a.startswith("gpt-readonly-")
branch=branch_for(a)
assert branch.startswith("agent/external-gpt-readonly/")
assert len(a)<180
def test_create_translates_to_canonical_readonly_governed_task(tmp_path):
svc,upstream,store=service(tmp_path)
response=svc.create(request())
assert response["state"]=="BOUND"
assert response["write_access"] is False
assert response["official_verification_required"] is True
assert response["capabilities"]==list(READONLY_CAPABILITIES)
assert len(upstream.created)==1
body=upstream.created[0]
assert body["repository"]=="aios/aios"
assert body["base_oid"]==MAIN
assert body["expected_branch_oid"]==MAIN
assert body["branch_mode"]=="CREATE_NEW"
assert body["risk"]=="low"
assert body["priority"]==0
assert body["max_attempts"]==3
assert body["require_online_task_projection"] is True
assert body["online_task"]["required_capabilities"]==list(READONLY_CAPABILITIES)
assert body["online_task"]["constraints"]["repository_mutation"]=="forbidden"
assert body["online_task"]["constraints"]["credential_access"]=="forbidden"
assert body["execution"]["action"]=="source_capture"
assert body["execution"]["adapter"]=="source_capture"
assert body["execution"]["capability_scope"]==list(READONLY_CAPABILITIES)
assert "writer_lease" not in body["execution"]
assert len(upstream.projected)==1
projection=upstream.projected[0]
assert projection["task_id"]==response["task_id"]
assert projection["job_id"]==JOB
assert projection["required_capabilities"]==list(READONLY_CAPABILITIES)
row=store.get("external-request-0001")
assert row is not None
assert row["state"]=="BOUND"
assert row["job_id"]==JOB
assert row["base_oid"]==MAIN
def test_exact_retry_is_idempotent_without_duplicate_bridge_call(tmp_path):
svc,upstream,_=service(tmp_path)
first=svc.create(request())
second=svc.create(request())
assert second==first
assert len(upstream.created)==1
assert len(upstream.projected)==1
def test_same_request_id_with_different_content_is_conflict(tmp_path):
svc,upstream,_=service(tmp_path)
svc.create(request())
with pytest.raises(IngressError,match="IDEMPOTENCY_CONFLICT"):
svc.create(request(goal="Different goal."))
assert len(upstream.created)==1
def test_pending_request_recovers_existing_canonical_task_after_crash(tmp_path):
svc,upstream,store=service(tmp_path)
raw=normalized_request(request())
task_id=task_id_for(raw["request_id"])
store.reserve(raw,task_id)
upstream.tasks[task_id]={
"task_id":task_id,
"status":"PROJECTED",
"attempt":0,
"metadata":{
"execution":{
"task_id":task_id,
"job_id":JOB,
"base_oid":MAIN,
"action":"source_capture",
"capability_scope":list(READONLY_CAPABILITIES),
"governance":{"writer_lease":False},
}
},
}
result=svc.create(request())
assert result["job_id"]==JOB
assert result["task_id"]==task_id
assert len(upstream.created)==0
row=store.get(raw["request_id"])
assert row["state"]=="BOUND"
def test_status_is_sanitized_and_never_proxies_raw_execution_or_result_text(tmp_path):
svc,upstream,store=service(tmp_path)
created=svc.create(request())
task_id=created["task_id"]
upstream.tasks[task_id]={
"task_id":task_id,
"status":"SUCCEEDED",
"attempt":1,
"claimed_by":"online-engineering-worker-cn-network-mkt-generic-20260815",
"metadata":{
"execution":{
"job_id":JOB,
"base_oid":MAIN,
"action":"source_capture",
"capability_scope":list(READONLY_CAPABILITIES),
"governance":{"writer_lease":False},
"secret_marker":"do-not-return",
}
},
"result":{
"status":"SUCCEEDED",
"classification":"NEEDS_REWORK_SOURCE_CAPTURE",
"execution_mode":"source_capture",
"output_text":"raw source text that must not be proxied",
"source_snapshot":{
"payload_sha256":"a"*64,
"file_count":42,
"files":["secret/path"],
},
},
"verification":{
"decision":"SUCCEEDED",
"classification":"GENERIC_ONLINE_TASK_AUTO_CONSUMPTION_VERIFIED",
"private":"do-not-return",
},
}
status=svc.status("external-request-0001")
assert status["state"]=="SUCCEEDED"
assert status["attempt"]==1
assert status["claimed"] is True
assert status["official_verification"]=="SUCCEEDED"
assert status["source_snapshot_sha256"]=="a"*64
assert status["source_snapshot_file_count"]==42
serialized=json.dumps(status)
assert "raw source text" not in serialized
assert "secret_marker" not in serialized
assert "secret/path" not in serialized
assert "do-not-return" not in serialized
def test_pending_status_can_recover_job_binding(tmp_path):
svc,upstream,store=service(tmp_path)
raw=normalized_request(request())
task_id=task_id_for(raw["request_id"])
store.reserve(raw,task_id)
upstream.tasks[task_id]={
"task_id":task_id,
"status":"PROJECTED",
"attempt":0,
"metadata":{
"execution":{
"job_id":JOB,
"base_oid":MAIN,
"action":"source_capture",
"capability_scope":list(READONLY_CAPABILITIES),
"governance":{"writer_lease":False},
}
},
}
value=svc.status(raw["request_id"])
assert value["job_id"]==JOB
assert store.get(raw["request_id"])["state"]=="BOUND"
def test_openapi_exposes_only_create_readonly_and_status_operations():
doc=openapi_document()
assert doc["openapi"]=="3.1.0"
paths=doc["paths"]
assert set(paths)=={
"/v1/read-only-tasks",
"/v1/read-only-tasks/{request_id}",
}
rendered=json.dumps(doc)
for forbidden in ("writer-lease","commit","verify","production","tag","baseline"):
assert forbidden not in rendered.lower()
def test_systemd_unit_is_loopback_credential_isolated_and_has_no_repository_access():
unit=(ROOT/"deploy/systemd/aios-gpt-online-task-ingress.service").read_text()
assert "DynamicUser=yes" in unit
assert "LoadCredential=gpt_ingress_token:" in unit
assert "LoadCredential=agent_task_control_token:" in unit
assert "AIOS_GPT_INGRESS_HOST=127.0.0.1" in unit
assert "AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892" in unit
assert "AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940" in unit
assert "IPAddressDeny=any" in unit
assert "IPAddressAllow=127.0.0.0/8" in unit
assert "InaccessiblePaths=/data/aios-gitops /etc/aios-gitops /home" in unit
assert "ProtectSystem=strict" in unit
assert "NoNewPrivileges=yes" in unit
assert "MemoryDenyWriteExecute=yes" in unit
assert "0.0.0.0" not in unit
def test_ingress_sqlite_contains_no_request_body_or_credentials(tmp_path):
svc,upstream,store=service(tmp_path)
svc.create(request(instructions="sensitive-looking but non-secret user text"))
con=sqlite3.connect(store.path)
columns=[row[1] for row in con.execute("pragma table_info(external_task_requests)")]
assert "request_json" not in columns
assert "goal" not in columns
assert "instructions" not in columns
assert "token" not in " ".join(columns).lower()
+325
View File
@@ -0,0 +1,325 @@
from __future__ import annotations
from pathlib import Path
import sys
import pytest
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/"active-runtime/app"))
from aios_model_governance import (
APPROVED,
ModelGovernanceError,
ModelGovernanceStore,
ModelRecord,
ProviderRecord,
RouteRequest,
ScopePolicy,
)
def setup_store(tmp_path, now=1_800_000_000.0):
store=ModelGovernanceStore(tmp_path/"model-governance.sqlite3",now_fn=lambda:now)
store.register_provider(
ProviderRecord(
"provider-a",
"runtime-profile:provider-a",
max_failures=1,
cooldown_seconds=120,
)
)
store.register_provider(
ProviderRecord(
"provider-b",
"runtime-profile:provider-b",
max_failures=2,
cooldown_seconds=60,
)
)
store.register_model(
ModelRecord(
"model-a",
"provider-a",
"2026-08-18",
("agent.runtime.hermes","text"),
input_cost_micros_per_1k=100,
output_cost_micros_per_1k=200,
max_input_tokens=10000,
max_output_tokens=4000,
evaluation_state=APPROVED,
evaluation_sha256="a"*64,
priority=10,
)
)
store.register_model(
ModelRecord(
"model-b",
"provider-b",
"2026-08-17",
("agent.runtime.hermes","text"),
input_cost_micros_per_1k=120,
output_cost_micros_per_1k=250,
max_input_tokens=10000,
max_output_tokens=4000,
evaluation_state=APPROVED,
evaluation_sha256="b"*64,
priority=20,
)
)
store.set_scope_policy(
ScopePolicy(
"tenant",
"tenant-a",
"project-a",
allowed_models=("model-a","model-b"),
required_capabilities=("text",),
max_input_tokens_per_request=5000,
max_output_tokens_per_request=2000,
daily_token_budget=12000,
daily_cost_budget_micros=4000,
fallback_limit=2,
)
)
return store
def req(request_id="request-001", *, exclude=()):
return RouteRequest(
request_id=request_id,
task_id="task-a",
trace_id="trace-a",
scope_type="tenant",
tenant_id="tenant-a",
project_id="project-a",
required_capabilities=("agent.runtime.hermes",),
estimated_input_tokens=1000,
max_output_tokens=500,
preferred_models=(),
exclude_providers=exclude,
)
def test_registry_rejects_raw_secret(tmp_path):
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
with pytest.raises(ModelGovernanceError,match="MODEL_CREDENTIAL"):
store.register_provider(
ProviderRecord("provider-a","Bearer raw-secret-value")
)
def test_model_registration_requires_evaluation_gate(tmp_path):
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
store.register_provider(ProviderRecord("provider-a"))
with pytest.raises(ModelGovernanceError,match="MODEL_EVALUATION_GATE_DENY"):
store.register_model(
ModelRecord(
"model-a","provider-a","v1",("text",),
evaluation_state="PENDING",evaluation_sha256="a"*64
)
)
def test_scope_policy_requires_version_with_pin(tmp_path):
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
with pytest.raises(ModelGovernanceError,match="MODEL_POLICY_PIN_VERSION_REQUIRED"):
store.set_scope_policy(
ScopePolicy("tenant","t","p",pinned_model_id="model-a")
)
def test_route_is_deterministic_version_pinned_and_idempotent(tmp_path):
store=setup_store(tmp_path)
first=store.route(req())
second=store.route(req())
assert first==second
assert first.model_id=="model-a"
assert first.model_version=="2026-08-18"
assert first.provider_credential_ref=="runtime-profile:provider-a"
assert first.policy_revision==1
assert first.model_revision==1
snapshot=store.usage_snapshot("tenant","tenant-a","project-a")
assert snapshot["reserved_input_tokens"]==1000
assert snapshot["reserved_output_tokens"]==500
assert snapshot["reserved_cost_micros"]>0
def test_route_idempotency_payload_conflict_fails(tmp_path):
store=setup_store(tmp_path)
store.route(req())
changed=RouteRequest(
**{**req().__dict__,"estimated_input_tokens":2000}
)
with pytest.raises(ModelGovernanceError,match="MODEL_ROUTE_IDEMPOTENCY_CONFLICT"):
store.route(changed)
def test_budget_reservation_is_atomic(tmp_path):
store=setup_store(tmp_path)
store.set_scope_policy(
ScopePolicy(
"tenant","tenant-a","project-a",
allowed_models=("model-a",),
required_capabilities=("text",),
daily_token_budget=1600,
daily_cost_budget_micros=10000,
),
expected_revision=1,
)
store.route(req("request-1"))
with pytest.raises(ModelGovernanceError,match="MODEL_BUDGET_DENY"):
store.route(req("request-2"))
def test_usage_settlement_is_idempotent_and_releases_reservation(tmp_path):
store=setup_store(tmp_path)
decision=store.route(req())
first=store.record_success(
decision.request_id,
input_tokens=900,
output_tokens=400,
usage_source="runtime",
)
second=store.record_success(
decision.request_id,
input_tokens=900,
output_tokens=400,
usage_source="runtime",
)
assert first["idempotent"] is False
assert second["idempotent"] is True
snapshot=store.usage_snapshot("tenant","tenant-a","project-a")
assert snapshot["settled_input_tokens"]==900
assert snapshot["settled_output_tokens"]==400
assert snapshot["reserved_input_tokens"]==0
assert snapshot["reserved_output_tokens"]==0
def test_missing_runtime_usage_falls_back_to_reserved_upper_bound(tmp_path):
store=setup_store(tmp_path)
decision=store.route(req())
result=store.record_success(
decision.request_id,
input_tokens=None,
output_tokens=None,
)
assert result["usage_source"]=="reserved_upper_bound"
assert result["input_tokens"]==1000
assert result["output_tokens"]==500
def test_usage_cannot_exceed_reserved_budget(tmp_path):
store=setup_store(tmp_path)
decision=store.route(req())
with pytest.raises(ModelGovernanceError,match="MODEL_USAGE_EXCEEDS_RESERVATION"):
store.record_success(
decision.request_id,
input_tokens=1001,
output_tokens=500,
)
def test_provider_failure_opens_circuit_and_fallback_uses_other_provider(tmp_path):
store=setup_store(tmp_path)
original=req()
first=store.route(original)
assert first.provider_id=="provider-a"
failure=store.record_failure(first.request_id,"provider_timeout")
assert failure["open_until"]>0
fallback=store.fallback(original,first.request_id)
assert fallback.provider_id=="provider-b"
assert fallback.fallback_index==1
def test_circuit_excludes_provider_for_new_request(tmp_path):
store=setup_store(tmp_path)
first=store.route(req("request-first"))
store.record_failure(first.request_id,"provider_timeout")
next_decision=store.route(req("request-next"))
assert next_decision.provider_id=="provider-b"
def test_model_version_pin_is_enforced(tmp_path):
store=setup_store(tmp_path)
store.set_scope_policy(
ScopePolicy(
"tenant","tenant-a","project-a",
allowed_models=("model-a","model-b"),
pinned_model_id="model-b",
pinned_model_version="2026-08-17",
daily_token_budget=12000,
daily_cost_budget_micros=4000,
),
expected_revision=1,
)
decision=store.route(req())
assert decision.model_id=="model-b"
assert decision.model_version=="2026-08-17"
def test_scope_isolation_denies_missing_cross_scope_policy(tmp_path):
store=setup_store(tmp_path)
cross=RouteRequest(
request_id="request-cross",
task_id="task-a",
trace_id="trace-a",
scope_type="tenant",
tenant_id="tenant-b",
project_id="project-a",
required_capabilities=("agent.runtime.hermes",),
estimated_input_tokens=100,
max_output_tokens=50,
)
with pytest.raises(ModelGovernanceError,match="MODEL_SCOPE_POLICY_MISSING"):
store.route(cross)
def test_audit_chain_detects_tampering(tmp_path):
store=setup_store(tmp_path)
decision=store.route(req())
store.record_success(decision.request_id,input_tokens=100,output_tokens=50)
assert store.verify_audit_chain() is True
store.conn.execute(
"UPDATE model_governance_audit SET payload_json='{}' WHERE sequence=1"
)
assert store.verify_audit_chain() is False
def test_registry_and_policy_use_cas(tmp_path):
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
store.register_provider(ProviderRecord("p"))
with pytest.raises(ModelGovernanceError,match="MODEL_PROVIDER_CAS_MISMATCH"):
store.register_provider(ProviderRecord("p"),expected_revision=7)
store.set_scope_policy(ScopePolicy("tenant","t","p"))
with pytest.raises(ModelGovernanceError,match="MODEL_POLICY_CAS_MISMATCH"):
store.set_scope_policy(
ScopePolicy("tenant","t","p"),
expected_revision=7,
)
def test_system_policy_never_authorizes_missing_tenant_policy(tmp_path):
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
store.register_provider(ProviderRecord("provider-a"))
store.register_model(
ModelRecord(
"model-a","provider-a","v1",("text",),
evaluation_sha256="a"*64,
)
)
store.set_scope_policy(
ScopePolicy("system",None,None,allowed_models=("model-a",))
)
request=RouteRequest(
request_id="request-tenant-no-policy",
task_id="task-a",
trace_id="trace-a",
scope_type="tenant",
tenant_id="tenant-a",
project_id="project-a",
required_capabilities=("text",),
estimated_input_tokens=10,
max_output_tokens=10,
)
with pytest.raises(ModelGovernanceError,match="MODEL_SCOPE_POLICY_MISSING"):
store.route(request)
+82
View File
@@ -0,0 +1,82 @@
from __future__ import annotations
import json
from pathlib import Path
import sys
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/"active-runtime/app"))
from aios_model_governance_admin import apply_config
from aios_model_governance import ModelGovernanceStore
def config():
return {
"schema":"aios.model-governance.bootstrap.v1",
"providers":[{
"provider_id":"provider-a",
"credential_ref":"runtime-profile:provider-a",
"max_failures":2,
"cooldown_seconds":60,
}],
"models":[{
"model_id":"model-a",
"provider_id":"provider-a",
"version":"v1",
"capabilities":["text"],
"input_cost_micros_per_1k":1,
"output_cost_micros_per_1k":2,
"max_input_tokens":1000,
"max_output_tokens":100,
"evaluation_state":"APPROVED",
"evaluation_sha256":"a"*64,
"priority":10,
}],
"policies":[{
"scope_type":"legacy",
"tenant_id":None,
"project_id":None,
"allowed_models":["model-a"],
"required_capabilities":["text"],
"daily_token_budget":10000,
"daily_cost_budget_micros":1000,
"fallback_limit":0,
}],
}
def test_admin_apply_is_idempotent(tmp_path):
cfg=tmp_path/"config.json"
cfg.write_text(json.dumps(config()),encoding="utf-8")
db=tmp_path/"db.sqlite3"
first=apply_config(db,cfg)
second=apply_config(db,cfg)
assert first["changes"]["providers"]==1
assert first["changes"]["models"]==1
assert first["changes"]["policies"]==1
assert second["changes"]["providers"]==0
assert second["changes"]["models"]==0
assert second["changes"]["policies"]==0
assert second["changes"]["unchanged"]==3
store=ModelGovernanceStore(db)
assert store.verify_audit_chain() is True
store.close()
def test_admin_config_change_uses_cas_revision(tmp_path):
value=config()
cfg=tmp_path/"config.json"
cfg.write_text(json.dumps(value),encoding="utf-8")
db=tmp_path/"db.sqlite3"
apply_config(db,cfg)
value["providers"][0]["cooldown_seconds"]=120
cfg.write_text(json.dumps(value),encoding="utf-8")
result=apply_config(db,cfg)
assert result["changes"]["providers"]==1
store=ModelGovernanceStore(db)
row=store.conn.execute(
"select revision,cooldown_seconds from model_governance_providers where provider_id='provider-a'"
).fetchone()
assert tuple(row)==(2,120)
store.close()
@@ -0,0 +1,554 @@
from __future__ import annotations
import json
import os
from pathlib import Path
import sys
import pytest
ROOT=Path(__file__).resolve().parents[1]
APP=ROOT/"active-runtime/app"
sys.path.insert(0,str(APP))
from aios_model_governance import (
ModelGovernanceStore,
ModelRecord,
ProviderRecord,
ScopePolicy,
)
from aios_online_agent_execution import GovernedExecutionBinding, project_online_task
import aios_external_agent_runtime as runtime
BASE="a"*40
def binding(task_id):
return GovernedExecutionBinding.from_mapping({
"task_id":task_id,
"job_id":"job-"+task_id,
"repository":"aios/aios",
"base_oid":BASE,
"expected_branch_oid":BASE,
"target_branch":"agent/"+task_id,
"branch_mode":"CREATE_NEW",
"generation":1,
"idempotency_key":task_id+"-binding-v1",
})
def projected(task_id="task-model-governed", *, scope_type="tenant", tenant_id="tenant-a", project_id="project-a"):
request={
"task_id":task_id,
"goal":"Return the requested sentinel.",
"instructions":"No repository or control-plane side effects.",
"success_criteria":[{"id":"done","description":"Return a result."}],
"constraints":{"repository_mutation":"forbidden"},
"scope_type":scope_type,
"tenant_id":tenant_id,
"project_id":project_id,
"trace_id":"trace-model-a",
"required_capabilities":["agent.runtime.hermes"],
}
return project_online_task(
binding(task_id),
request,
{
"action":"agent_runtime",
"adapter":"hermes",
"capability_scope":["agent.runtime.hermes"],
},
)
def helper(tmp_path):
path=tmp_path/"model_runtime.py"
path.write_text(
"import json,sys\n"
"model=sys.argv[1]\n"
"provider=sys.argv[2]\n"
"payload=json.load(sys.stdin)\n"
"if model=='model-a' and payload.get('constraints',{}).get('force_fail_a'):\n"
" raise SystemExit(7)\n"
"print(json.dumps({'answer':'ok','model':model,'provider':provider,"
"'usage':{'input_tokens':20,'output_tokens':10,'cost_micros':2}},sort_keys=True))\n",
encoding="utf-8",
)
return path
def write_profile(tmp_path, helper_path, *, providers=("provider-a","provider-b")):
root=tmp_path/"profiles";root.mkdir(exist_ok=True)
value={
"schema":"aios.external-agent-runtime.profile.v1",
"runtime":"hermes",
"argv":["/usr/bin/python3",str(helper_path),"{model_id}","{provider_id}"],
"input_mode":"stdin_json",
"output_mode":"stdout_json",
"timeout_seconds":30,
"max_output_bytes":65536,
"environment":{},
"credential_env":{},
"model_governance":{
"required":True,
"provider_ids":list(providers),
"model_capabilities":["text"],
"max_output_tokens":100,
},
}
(root/"hermes.json").write_text(json.dumps(value),encoding="utf-8")
return root
def seed(db_path, *, legacy=False, tiny_budget=False):
store=ModelGovernanceStore(db_path,now_fn=lambda:1_800_000_000.0)
for provider in ("provider-a","provider-b"):
store.register_provider(
ProviderRecord(
provider,
f"runtime-profile:{provider}",
max_failures=1 if provider=="provider-a" else 2,
cooldown_seconds=120,
)
)
store.register_model(
ModelRecord(
"model-a","provider-a","v1",
("agent.runtime.hermes","text"),
input_cost_micros_per_1k=10,
output_cost_micros_per_1k=20,
max_input_tokens=5000,max_output_tokens=1000,
evaluation_sha256="a"*64,priority=1,
)
)
store.register_model(
ModelRecord(
"model-b","provider-b","v2",
("agent.runtime.hermes","text"),
input_cost_micros_per_1k=12,
output_cost_micros_per_1k=22,
max_input_tokens=5000,max_output_tokens=1000,
evaluation_sha256="b"*64,priority=2,
)
)
if legacy:
policy=ScopePolicy(
"legacy",None,None,
allowed_models=("model-a","model-b"),
required_capabilities=("text",),
daily_token_budget=10000,
daily_cost_budget_micros=10000,
fallback_limit=2,
)
else:
policy=ScopePolicy(
"tenant","tenant-a","project-a",
allowed_models=("model-a","model-b"),
required_capabilities=("text",),
daily_token_budget=1 if tiny_budget else 10000,
daily_cost_budget_micros=10000,
fallback_limit=2,
)
store.set_scope_policy(policy)
store.close()
def make_external_request(projected_task, *, force_fail=False, attempt=1):
execution=projected_task["metadata"]["execution"]
payload=dict(execution["runtime_input"])
if force_fail:
payload["constraints"]=dict(payload["constraints"])
payload["constraints"]["force_fail_a"]=True
return {
"schema":"aios.external-agent-runtime.request.v1",
"request_id":"external-model-governance-0123456789",
"runtime":"hermes",
"task_id":projected_task["task_id"],
"attempt":attempt,
"execution_digest":execution["execution_digest"],
"payload":payload,
}
@pytest.fixture(autouse=True)
def clean_env(monkeypatch):
for key in list(os.environ):
upper=key.upper()
if any(marker in upper for marker in ("TOKEN","SECRET","PASSWORD","CREDENTIAL","AUTHORIZATION","PRIVATE_KEY","API_KEY")):
monkeypatch.delenv(key,raising=False)
monkeypatch.setenv("CREDENTIALS_DIRECTORY","/definitely-not-present-model-governance")
def test_execution_projection_binds_trusted_scope_and_trace():
task=projected()
runtime_input=task["metadata"]["execution"]["runtime_input"]
assert runtime_input["scope"]=={
"scope_type":"tenant",
"tenant_id":"tenant-a",
"project_id":"project-a",
}
assert runtime_input["trace_id"]=="trace-model-a"
def test_legacy_scope_remains_explicit_not_fake_tenant():
task=projected(scope_type="legacy",tenant_id=None,project_id=None)
scope=task["metadata"]["execution"]["runtime_input"]["scope"]
assert scope=={"scope_type":"legacy","tenant_id":None,"project_id":None}
def test_profile_requires_model_binding_when_governance_required(tmp_path):
path=tmp_path/"helper.py";path.write_text("print('ok')\n")
root=tmp_path/"profiles";root.mkdir()
(root/"hermes.json").write_text(json.dumps({
"schema":"aios.external-agent-runtime.profile.v1",
"runtime":"hermes",
"argv":["/usr/bin/python3",str(path)],
"input_mode":"stdin_text",
"output_mode":"stdout_text",
"model_governance":{"required":True,"provider_ids":["provider-a"],"max_output_tokens":100},
}))
with pytest.raises(runtime.ExternalAgentRuntimeError,match="EXTERNAL_AGENT_MODEL_BINDING_REQUIRED"):
runtime.RuntimeProfile.load("hermes",root)
def test_governed_runtime_selects_model_and_settles_usage(tmp_path,monkeypatch):
helper_path=helper(tmp_path)
profile_root=write_profile(tmp_path,helper_path)
db=tmp_path/"governance.sqlite3";seed(db)
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
request_path=tmp_path/"request.json"
result_path=tmp_path/"result.json"
request_path.write_text(json.dumps(make_external_request(projected())),encoding="utf-8")
runtime.execute_request(request_path,result_path,profile_root=profile_root)
result=json.loads(result_path.read_text())
mg=result["model_governance"]
assert mg["model_id"]=="model-a"
assert mg["model_version"]=="v1"
assert mg["provider_id"]=="provider-a"
assert mg["settlement"]["usage_source"]=="runtime"
assert mg["settlement"]["input_tokens"]==20
assert mg["settlement"]["output_tokens"]==10
assert mg["audit_chain_valid"] is True
assert result["output_json"]["model"]=="model-a"
def test_budget_denies_before_runtime_process(tmp_path,monkeypatch):
helper_path=helper(tmp_path)
profile_root=write_profile(tmp_path,helper_path)
db=tmp_path/"governance.sqlite3";seed(db,tiny_budget=True)
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
req=tmp_path/"request.json";res=tmp_path/"result.json"
req.write_text(json.dumps(make_external_request(projected())),encoding="utf-8")
with pytest.raises(runtime.ExternalAgentRuntimeError,match="MODEL_BUDGET_DENY"):
runtime.execute_request(req,res,profile_root=profile_root)
assert not res.exists()
def test_provider_failure_routes_to_fallback_provider(tmp_path,monkeypatch):
helper_path=helper(tmp_path)
profile_root=write_profile(tmp_path,helper_path)
db=tmp_path/"governance.sqlite3";seed(db)
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
req=tmp_path/"request.json";res=tmp_path/"result.json"
req.write_text(json.dumps(make_external_request(projected(),force_fail=True)),encoding="utf-8")
runtime.execute_request(req,res,profile_root=profile_root)
result=json.loads(res.read_text())
mg=result["model_governance"]
assert len(mg["attempts"])==2
assert mg["attempts"][0]["provider_id"]=="provider-a"
assert mg["attempts"][1]["provider_id"]=="provider-b"
assert mg["provider_id"]=="provider-b"
assert result["output_json"]["provider"]=="provider-b"
def test_profile_provider_allowlist_fails_closed(tmp_path,monkeypatch):
helper_path=helper(tmp_path)
profile_root=write_profile(tmp_path,helper_path,providers=("provider-b",))
db=tmp_path/"governance.sqlite3";seed(db)
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
req=tmp_path/"request.json";res=tmp_path/"result.json"
req.write_text(json.dumps(make_external_request(projected())),encoding="utf-8")
with pytest.raises(runtime.ExternalAgentRuntimeError,match="EXTERNAL_AGENT_MODEL_PROVIDER_PROFILE_DENY"):
runtime.execute_request(req,res,profile_root=profile_root)
def test_governance_required_but_runtime_disabled_fails_closed(tmp_path,monkeypatch):
helper_path=helper(tmp_path)
profile_root=write_profile(tmp_path,helper_path)
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",False)
req=tmp_path/"request.json";res=tmp_path/"result.json"
req.write_text(json.dumps(make_external_request(projected())),encoding="utf-8")
with pytest.raises(runtime.ExternalAgentRuntimeError,match="EXTERNAL_AGENT_MODEL_GOVERNANCE_DISABLED"):
runtime.execute_request(req,res,profile_root=profile_root)
def test_execution_request_cannot_override_authoritative_task_scope():
task_id="task-scope-override"
request={
"task_id":task_id,
"goal":"Return result.",
"instructions":"",
"success_criteria":[],
"constraints":{},
"scope_type":"tenant",
"tenant_id":"tenant-a",
"project_id":"project-a",
"trace_id":"trace-authoritative",
"required_capabilities":["agent.runtime.hermes"],
}
task=project_online_task(
binding(task_id),
request,
{
"action":"agent_runtime",
"adapter":"hermes",
"capability_scope":["agent.runtime.hermes"],
"runtime_input":{
"goal":"Return result.",
"instructions":"",
"success_criteria":[],
"constraints":{},
"scope":{
"scope_type":"tenant",
"tenant_id":"tenant-b",
"project_id":"project-b",
},
"trace_id":"trace-forged",
},
},
)
runtime_input=task["metadata"]["execution"]["runtime_input"]
assert runtime_input["scope"]["tenant_id"]=="tenant-a"
assert runtime_input["scope"]["project_id"]=="project-a"
assert runtime_input["trace_id"]=="trace-authoritative"
def test_openclaw_nested_usage_is_extracted_exactly():
value={
"meta":{
"agentMeta":{
"usage":{"input":7770,"output":13,"total":7783},
"lastCallUsage":{"input":7770,"output":13,"total":7783},
"costUsd":0,
}
}
}
assert runtime._extract_runtime_usage(value)==(7770,13,0)
def test_governance_required_profile_needs_provider_allowlist_and_output_limit(tmp_path):
path=tmp_path/"helper.py";path.write_text("print('ok')\\n")
root=tmp_path/"profiles";root.mkdir()
(root/"hermes.json").write_text(json.dumps({
"schema":"aios.external-agent-runtime.profile.v1",
"runtime":"hermes",
"argv":["/usr/bin/python3",str(path),"{model_id}"],
"input_mode":"stdin_text",
"output_mode":"stdout_text",
"model_governance":{"required":True},
}))
with pytest.raises(
runtime.ExternalAgentRuntimeError,
match="EXTERNAL_AGENT_MODEL_PROVIDER_IDS_REQUIRED",
):
runtime.RuntimeProfile.load("hermes",root)
def _openclaw_context_helper(tmp_path, *, input_tokens=7770, output_tokens=13):
path=tmp_path/"openclaw_context_helper.py"
path.write_text(
"import json,sys\n"
"model=sys.argv[1]\n"
"provider=sys.argv[2]\n"
"json.load(sys.stdin)\n"
f"print(json.dumps({{'answer':'ok','model':model,'provider':provider,"
f"'meta':{{'agentMeta':{{'usage':{{'input':{input_tokens},'output':{output_tokens},"
f"'total':{input_tokens+output_tokens}}},'costUsd':0}}}}}},sort_keys=True))\n",
encoding="utf-8",
)
return path
def _openclaw_floor_profile(tmp_path, helper_path, *, floor=16384):
root=tmp_path/"openclaw-profiles"
root.mkdir(exist_ok=True)
value={
"schema":"aios.external-agent-runtime.profile.v1",
"runtime":"openclaw",
"argv":["/usr/bin/python3",str(helper_path),"{model_id}","{provider_id}"],
"input_mode":"stdin_json",
"output_mode":"stdout_json",
"timeout_seconds":30,
"max_output_bytes":65536,
"environment":{},
"credential_env":{},
"model_governance":{
"required":True,
"provider_ids":["openai"],
"model_capabilities":["text"],
"max_output_tokens":256,
"input_reservation_floor_tokens":floor,
},
}
(root/"openclaw.json").write_text(json.dumps(value),encoding="utf-8")
return root
def _seed_openclaw_floor_db(db_path):
store=ModelGovernanceStore(db_path,now_fn=lambda:1_800_000_000.0)
store.register_provider(
ProviderRecord(
"openai",
"runtime-profile:openclaw",
max_failures=2,
cooldown_seconds=60,
)
)
store.register_model(
ModelRecord(
"openai/deepseek-v4-flash",
"openai",
"v1",
("agent.runtime.openclaw","text"),
input_cost_micros_per_1k=0,
output_cost_micros_per_1k=0,
max_input_tokens=32768,
max_output_tokens=256,
evaluation_sha256="c"*64,
priority=10,
)
)
store.set_scope_policy(
ScopePolicy(
"tenant",
"model-governance-live",
"staging",
allowed_models=("openai/deepseek-v4-flash",),
required_capabilities=("text",),
max_input_tokens_per_request=32768,
max_output_tokens_per_request=256,
daily_token_budget=250000,
daily_cost_budget_micros=0,
fallback_limit=1,
)
)
store.close()
def _openclaw_floor_request(*, attempt=2):
return {
"schema":"aios.external-agent-runtime.request.v1",
"request_id":f"external-openclaw-floor-attempt-{attempt:02d}",
"runtime":"openclaw",
"task_id":"r42-model-governance-openclaw-live-20260818",
"attempt":attempt,
"execution_digest":"d"*64,
"payload":{
"goal":"Return AIOS_MODEL_GOVERNANCE_OPENCLAW_OK.",
"instructions":"No side effects.",
"success_criteria":[],
"constraints":{},
"scope":{
"scope_type":"tenant",
"tenant_id":"model-governance-live",
"project_id":"staging",
},
"trace_id":"trace-openclaw-model-governance-live",
},
}
def test_openclaw_profile_reservation_floor_covers_injected_context(
tmp_path,monkeypatch
):
helper_path=_openclaw_context_helper(tmp_path,input_tokens=7770,output_tokens=13)
profile_root=_openclaw_floor_profile(tmp_path,helper_path,floor=16384)
db=tmp_path/"openclaw-governance.sqlite3"
_seed_openclaw_floor_db(db)
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
request_path=tmp_path/"request.json"
result_path=tmp_path/"result.json"
request_path.write_text(json.dumps(_openclaw_floor_request()),encoding="utf-8")
runtime.execute_request(request_path,result_path,profile_root=profile_root)
result=json.loads(result_path.read_text())
mg=result["model_governance"]
assert mg["input_reservation_floor_tokens"]==16384
assert mg["prompt_estimated_input_tokens"] < 16384
assert mg["reserved_estimated_input_tokens"]==16384
assert mg["settlement"]["state"]=="SETTLED"
assert mg["settlement"]["usage_source"]=="runtime"
assert mg["settlement"]["input_tokens"]==7770
assert mg["settlement"]["output_tokens"]==13
store=ModelGovernanceStore(db)
row=store.conn.execute(
"""
select reserved_input_tokens,state,actual_input_tokens,actual_output_tokens
from model_governance_reservations
where request_id=?
""",
(result["request_id"]+":model",),
).fetchone()
assert int(row["reserved_input_tokens"])==16384
assert row["state"]=="SETTLED"
assert int(row["actual_input_tokens"])==7770
assert int(row["actual_output_tokens"])==13
store.close()
def test_openclaw_reservation_floor_still_fails_closed_above_floor(
tmp_path,monkeypatch
):
helper_path=_openclaw_context_helper(tmp_path,input_tokens=17000,output_tokens=13)
profile_root=_openclaw_floor_profile(tmp_path,helper_path,floor=16384)
db=tmp_path/"openclaw-governance.sqlite3"
_seed_openclaw_floor_db(db)
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
request_path=tmp_path/"request.json"
result_path=tmp_path/"result.json"
request_path.write_text(json.dumps(_openclaw_floor_request()),encoding="utf-8")
with pytest.raises(
runtime.ExternalAgentRuntimeError,
match="MODEL_USAGE_EXCEEDS_RESERVATION",
):
runtime.execute_request(request_path,result_path,profile_root=profile_root)
assert not result_path.exists()
def test_model_governance_profile_rejects_invalid_input_reservation_floor(tmp_path):
path=tmp_path/"helper.py"
path.write_text("print('ok')\n",encoding="utf-8")
root=tmp_path/"profiles"
root.mkdir()
(root/"openclaw.json").write_text(json.dumps({
"schema":"aios.external-agent-runtime.profile.v1",
"runtime":"openclaw",
"argv":["/usr/bin/python3",str(path),"{model_id}"],
"input_mode":"stdin_text",
"output_mode":"stdout_text",
"model_governance":{
"required":True,
"provider_ids":["openai"],
"max_output_tokens":256,
"input_reservation_floor_tokens":262145,
},
}))
with pytest.raises(
runtime.ExternalAgentRuntimeError,
match="EXTERNAL_AGENT_MODEL_INPUT_RESERVATION_FLOOR_INVALID",
):
runtime.RuntimeProfile.load("openclaw",root)
@@ -0,0 +1,99 @@
from __future__ import annotations
import json
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
def test_executor_has_single_model_governance_sqlite_write_path():
unit=(ROOT/"deploy/systemd/aios-external-agent-executor.service").read_text()
assert "AIOS_MODEL_GOVERNANCE_DB=/var/lib/aios-model-governance/state.sqlite3" in unit
assert "AIOS_MODEL_GOVERNANCE_ENABLED=1" in unit
assert "ReadWritePaths=/var/lib/aios-model-governance" in unit
assert "ReadWritePaths=/var/lib/aios-external-agent-executor" in unit
assert "InaccessiblePaths=/etc/aios-gitops" in unit
assert "InaccessiblePaths=/data/aios-gitops" in unit
def test_tmpfiles_owner_matches_external_executor():
line=(ROOT/"deploy/tmpfiles.d/aios-model-governance.conf").read_text().strip()
assert line == (
"d /var/lib/aios-model-governance 0750 "
"aios-engineering-executor aios-executor-exchange -"
)
def test_staging_models_use_real_e2e_evaluation_evidence():
value=json.loads(
(ROOT/"config/model-governance/model-governance.staging.json").read_text()
)
assert value["schema"]=="aios.model-governance.bootstrap.v1"
models={item["model_id"]:item for item in value["models"]}
assert models["deepseek-v4-flash"]["evaluation_sha256"] == (
"43ae0db1640daeb6d33e2787769732844f366d36070065d3a2239905bd9a823b"
)
assert models["openai/deepseek-v4-flash"]["evaluation_sha256"] == (
"7b1c89634eb9c0e7cb5a3d75323d904ebbd91b3ae5494ce938bd8316a4cf5b12"
)
def test_runtime_profiles_bind_model_from_governance_not_task():
for name in ("hermes","openclaw"):
value=json.loads(
(ROOT/"config/external-agent-runtime.d"/f"{name}.profile.json").read_text()
)
governance=value["model_governance"]
assert governance["required"] is True
assert governance["provider_ids"]
assert governance["max_output_tokens"] > 0
serialized=json.dumps(value["argv"])+json.dumps(value.get("environment",{}))
assert "{model_id}" in serialized or "{provider_id}" in serialized
def test_staging_bootstrap_has_explicit_tenant_live_policy():
value=json.loads(
(ROOT/"config/model-governance/model-governance.staging.json").read_text()
)
keys={
(
item["scope_type"],
item.get("tenant_id"),
item.get("project_id"),
)
for item in value["policies"]
}
assert ("legacy",None,None) in keys
assert ("tenant","model-governance-live","staging") in keys
def test_staging_policy_input_limit_covers_root_owned_openclaw_reservation_floor():
config=json.loads(
(ROOT/"config/model-governance/model-governance.staging.json").read_text()
)
profile=json.loads(
(ROOT/"config/external-agent-runtime.d/openclaw.profile.json").read_text()
)
floor=int(profile["model_governance"]["input_reservation_floor_tokens"])
assert floor==16384
openclaw_model="openai/deepseek-v4-flash"
relevant=[
policy for policy in config["policies"]
if openclaw_model in policy["allowed_models"]
]
assert relevant
for policy in relevant:
assert int(policy["max_input_tokens_per_request"]) >= floor
def test_staging_openclaw_scope_limit_is_least_privilege_floor_not_model_ceiling():
config=json.loads(
(ROOT/"config/model-governance/model-governance.staging.json").read_text()
)
models={item["model_id"]:item for item in config["models"]}
model=models["openai/deepseek-v4-flash"]
assert int(model["max_input_tokens"])==32768
for policy in config["policies"]:
if "openai/deepseek-v4-flash" in policy["allowed_models"]:
assert int(policy["max_input_tokens_per_request"])==16384
assert int(policy["max_input_tokens_per_request"]) < int(model["max_input_tokens"])
@@ -0,0 +1,85 @@
from __future__ import annotations
from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "active-runtime/app"))
from aios_online_agent_execution import ( # noqa: E402
GovernedExecutionBinding,
project_execution_envelope,
)
import aios_online_agent_worker as worker # noqa: F401,E402
from aios_online_agent_worker import WorkerSupervisor # noqa: E402
BASE = "a" * 40
def test_agent_runtime_replays_the_durable_envelope_to_bridge(monkeypatch) -> None:
monkeypatch.setattr(
worker,
"WORKER_CAPABILITIES",
tuple(sorted((*worker.WORKER_CAPABILITIES, "agent.runtime.hermes"))),
)
binding = GovernedExecutionBinding.from_mapping(
{
"task_id": "r42-agent-runtime-replay-test-20260818",
"job_id": "job-agent-runtime-replay-test",
"repository": "aios/aios",
"base_oid": BASE,
"expected_branch_oid": BASE,
"target_branch": "agent/agent-runtime-replay-test",
"branch_mode": "CREATE_NEW",
"generation": 1,
"idempotency_key": "agent-runtime-replay-test-v1",
}
)
execution = project_execution_envelope(
binding,
{
"action": "agent_runtime",
"adapter": "hermes",
"capability_scope": ["agent.runtime.hermes"],
"change_set": [],
"runtime_input": {
"goal": "replay the exact durable runtime envelope",
"instructions": "return the bounded sentinel",
"success_criteria": [
{
"id": "sentinel",
"description": "return the bounded sentinel",
}
],
"constraints": {
"provider_boundary": "provider-credential-only",
"change_set": [],
"writer_lease": False,
},
},
},
)
task = {
**binding.public(),
"required_capabilities": ["agent.runtime.hermes"],
"metadata": {"execution": execution},
}
class Bridge:
def __init__(self) -> None:
self.calls: list[object] = []
def project_execution(self, _task: object, projection_input: object = None):
self.calls.append(projection_input)
return 200, dict(projection_input or {})
supervisor = object.__new__(WorkerSupervisor)
supervisor.bridge = Bridge()
resolved, source = supervisor.resolve_envelope(task)
assert source == "persisted_verified"
assert resolved.action == "agent_runtime"
assert resolved.adapter == "hermes"
assert resolved.digest == execution["execution_digest"]
assert supervisor.bridge.calls == [execution]
@@ -0,0 +1,56 @@
from __future__ import annotations
from pathlib import Path
from aios_online_agent_checkpoint import CheckpointBinding, CheckpointStore
def _binding() -> CheckpointBinding:
return CheckpointBinding(
task_id="checkpoint-agent-runtime-test-20260818",
attempt=1,
execution_digest="a" * 64,
job_id="job-checkpoint-agent-runtime-test-20260818",
repository="aios/aios",
base_oid="b" * 40,
expected_branch_oid="b" * 40,
target_branch="agent/checkpoint-agent-runtime-test-20260818",
generation=1,
)
def test_agent_runtime_checkpoint_phase_is_durable_before_result(tmp_path: Path) -> None:
store = CheckpointStore(tmp_path / "checkpoints.sqlite3")
binding = _binding()
phases = [
"claimed",
"workspace_prepared",
"verification_passed",
"agent_runtime_done",
"result_pending",
"result_submitted",
]
rows = [
store.append(binding, phase, {"phase": phase, "task_id": binding.task_id})
for phase in phases
]
assert [row["phase"] for row in rows] == phases
assert [int(row["phase_order"]) for row in rows] == [10, 20, 30, 40, 50, 60]
latest = store.latest_for_attempt(binding.task_id, binding.attempt)
assert latest is not None
assert latest["phase"] == "result_submitted"
assert latest["payload"] == {"phase": "result_submitted", "task_id": binding.task_id}
def test_agent_runtime_checkpoint_append_is_idempotent(tmp_path: Path) -> None:
store = CheckpointStore(tmp_path / "checkpoints.sqlite3")
binding = _binding()
payload = {"phase": "agent_runtime_done", "result": {"status": "SUCCEEDED"}}
first = store.append(binding, "agent_runtime_done", payload)
replay = store.append(binding, "agent_runtime_done", payload)
assert replay["sequence"] == first["sequence"]
assert replay["payload_sha256"] == first["payload_sha256"]
@@ -0,0 +1,139 @@
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
import json
import sys
import pytest
ROOT = Path(__file__).resolve().parents[1]
APP = ROOT / "active-runtime/app"
sys.path.insert(0, str(APP))
from aios_online_agent_compact_map import build_compact_interface_map
import aios_online_agent_worker as worker
from aios_online_agent_worker import build_compact_interface_map_result
def _repo(tmp_path: Path) -> Path:
root = tmp_path / "repo"
(root / "active-runtime/app").mkdir(parents=True)
(root / "tests").mkdir(parents=True)
(root / "active-runtime/app/model_router.py").write_text(
"""
class ProviderRegistry:
def register_provider(self, provider_id, model_version):
return provider_id, model_version
class ModelRouter:
def select_model(self, tenant_id, budget_tokens, fallback_provider):
return fallback_provider
def record_model_cost(trace_id, token_count, cost_usd):
return trace_id, token_count, cost_usd
def evaluate_model_version(model_id, version, approval_gate):
return approval_gate
""".strip() + "\\n",
encoding="utf-8",
)
(root / "tests/test_model_router.py").write_text(
"""
def test_model_budget_and_fallback():
assert True
""".strip() + "\\n",
encoding="utf-8",
)
return root
def _envelope(base_oid: str):
return SimpleNamespace(
base_oid=base_oid,
job_id="job-model-governance",
repository="aios/aios",
expected_branch_oid=base_oid,
target_branch="agent/model-governance-map",
generation=1,
digest="e" * 64,
)
def test_model_governance_profile_contracts_and_classification(tmp_path):
root = _repo(tmp_path)
value = build_compact_interface_map(
root,
[
"model provider registry routing budget token cost fallback",
"evaluation approval version pinning tenant project credential trace",
],
source_commit="a" * 40,
)
assert value["profile"] == "model_governance"
assert value["classification"] == "MODEL_GOVERNANCE_INTERFACE_MAP_CAPTURED"
assert set(value["contracts"]) == {
"MODEL_PROVIDER_REGISTRY",
"MODEL_SELECTION_POLICY",
"MODEL_VERSION_PINNING",
"MODEL_BUDGET_ACCOUNTING",
"MODEL_FALLBACK_POLICY",
"MODEL_SCOPE_ISOLATION",
"MODEL_CREDENTIAL_BOUNDARY",
"MODEL_TRACE_AUDIT",
"MODEL_EVALUATION_GATE",
"MODEL_ROUTING_RESILIENCE",
}
assert "FLEET_FAIRNESS" not in value["contracts"]
def test_fleet_profile_is_backward_compatible(tmp_path):
root = _repo(tmp_path)
value = build_compact_interface_map(
root,
["agent fleet scheduler worker capacity fairness"],
source_commit="b" * 40,
)
assert value["profile"] == "fleet"
assert value["classification"] == "P2_INTERFACE_MAP_CAPTURED"
assert "FLEET_AGENT_REGISTRATION" in value["contracts"]
def test_worker_binds_model_profile_to_model_classification():
interface_map = {
"schema": "aios.engineering.interface-map.v1",
"profile": "model_governance",
"classification": "MODEL_GOVERNANCE_INTERFACE_MAP_CAPTURED",
"source_commit": "c" * 40,
"implementation_map": {"modules": [], "classes": [], "functions": [], "tests": []},
"bounded_excerpts": [],
"contracts": {"MODEL_PROVIDER_REGISTRY": {"status": "GAP", "surfaces": []}},
"result_bytes": 512,
}
result = build_compact_interface_map_result(
"r42-model-governance-interface-map-20260818",
_envelope("c" * 40),
{"interface_map": interface_map},
)
assert result["classification"] == "MODEL_GOVERNANCE_INTERFACE_MAP_CAPTURED"
assert result["interface_map_profile"] == "model_governance"
assert result["writer_lease"] == "NONE"
def test_worker_rejects_profile_classification_mismatch():
interface_map = {
"schema": "aios.engineering.interface-map.v1",
"profile": "model_governance",
"classification": "P2_INTERFACE_MAP_CAPTURED",
"source_commit": "c" * 40,
"implementation_map": {"modules": [], "classes": [], "functions": [], "tests": []},
"bounded_excerpts": [],
"contracts": {},
"result_bytes": 512,
}
with pytest.raises(worker.WorkerError, match="COMPACT_INTERFACE_MAP_PROFILE_BINDING_INVALID"):
build_compact_interface_map_result(
"r42-model-governance-interface-map-20260818",
_envelope("c" * 40),
{"interface_map": interface_map},
)
@@ -0,0 +1,209 @@
from __future__ import annotations
from pathlib import Path
import sys
import pytest
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/"active-runtime/app"))
import aios_online_agent_worker as worker
from aios_online_agent_execution import (
GovernedExecutionBinding,
project_execution_envelope,
)
@pytest.fixture(autouse=True)
def _state_paths(monkeypatch, tmp_path):
root=tmp_path/"worker-state"
monkeypatch.setattr(worker,"STATE_ROOT",root)
monkeypatch.setattr(worker,"WORKER_LIVENESS_PATH",root/"worker-liveness.json")
BASE="a"*40
TASK_ID="task-worker-availability-placeholder"
JOB_ID="job-worker-availability-placeholder"
BRANCH="agent/task-worker-availability-placeholder"
IDEMPOTENCY="task-worker-availability-placeholder-binding-v1"
def binding():
return GovernedExecutionBinding.from_mapping({
"task_id":TASK_ID,
"job_id":JOB_ID,
"repository":"aios/aios",
"base_oid":BASE,
"expected_branch_oid":BASE,
"target_branch":BRANCH,
"branch_mode":"CREATE_NEW",
"generation":1,
"idempotency_key":IDEMPOTENCY,
})
def placeholder():
return project_execution_envelope(
binding(),
{
"action":"source_capture",
"adapter":"source_capture",
"capability_scope":[
"git.read",
"git.test",
"governed.write.branch",
"governed.write.file",
],
},
)
def readonly_source_capture():
return project_execution_envelope(
binding(),
{
"action":"source_capture",
"adapter":"source_capture",
"capability_scope":["git.read","git.test"],
"source_signals":["worker availability"],
},
)
def deterministic():
return project_execution_envelope(
binding(),
{
"action":"deterministic",
"adapter":"deterministic",
"capability_scope":[
"git.read",
"git.test",
"governed.write.branch",
"governed.write.file",
],
"change_set":[{
"path":"active-runtime/app/example.py",
"operation":"write_file",
"content":"VALUE=1\\n",
}],
"verification":[{
"label":"unit",
"argv":["/usr/bin/python3","-m","pytest","-q"],
"cwd":".",
"timeout_seconds":120,
}],
"governance":{
"commit_idempotency_key":"task-worker-availability-placeholder-commit-v1",
"allowed_files":["active-runtime/app/example.py"],
},
},
)
def task(execution):
return {
"task_id":TASK_ID,
"required_capabilities":list(execution["capability_scope"]),
"metadata":{
"execution":execution,
"execution_projection":{
"schema":"aios.online-agent.task-projection.v1",
"task_id":TASK_ID,
"job_id":JOB_ID,
"execution_digest":execution["execution_digest"],
},
},
}
class FakeBridge:
def __init__(self,status,execution):
self.status=status
self.execution=execution
self.calls=[]
def project_execution(self,task_value,execution=None):
self.calls.append(execution)
return self.status,self.execution
class FakeAgent:
def __init__(self,tasks):
self.tasks=tasks
self.claim_calls=[]
def poll(self):
return list(self.tasks)
def claim(self,task_id):
self.claim_calls.append(task_id)
return 500,{"error":"SHOULD_NOT_CLAIM"}
def supervisor(bridge, agent=None):
value=object.__new__(worker.WorkerSupervisor)
value.bridge=bridge
value.agent=agent
return value
def test_write_capable_source_capture_is_classified_as_placeholder():
envelope=worker.ExecutionEnvelope.from_task(
task(placeholder()),
worker_capabilities=tuple(placeholder()["capability_scope"]),
)
assert worker.source_capture_requires_deterministic_plan(envelope) is True
def test_readonly_source_capture_is_not_placeholder():
envelope=worker.ExecutionEnvelope.from_task(
task(readonly_source_capture()),
worker_capabilities=tuple(readonly_source_capture()["capability_scope"]),
)
assert worker.source_capture_requires_deterministic_plan(envelope) is False
def test_placeholder_without_registered_plan_is_skipped_before_claim():
persisted=placeholder()
bridge=FakeBridge(200,persisted)
agent=FakeAgent([task(persisted)])
sup=supervisor(bridge,agent)
assert sup.once()==0
assert bridge.calls==[None]
assert agent.claim_calls==[]
def test_placeholder_with_missing_projection_is_skipped_before_claim():
persisted=placeholder()
bridge=FakeBridge(404,{"error":"TASK_EXECUTION_PROJECTION_MISSING"})
agent=FakeAgent([task(persisted)])
sup=supervisor(bridge,agent)
assert sup.once()==0
assert bridge.calls==[None]
assert agent.claim_calls==[]
def test_registered_deterministic_plan_replaces_placeholder():
persisted=placeholder()
resolved=deterministic()
bridge=FakeBridge(200,resolved)
sup=supervisor(bridge)
envelope,source=sup.resolve_envelope(task(persisted))
assert source=="plan_registry_verified"
assert envelope.action=="deterministic"
assert envelope.digest==resolved["execution_digest"]
assert bridge.calls==[None]
def test_real_readonly_source_capture_replays_exact_payload():
persisted=readonly_source_capture()
bridge=FakeBridge(200,persisted)
sup=supervisor(bridge)
envelope,source=sup.resolve_envelope(task(persisted))
assert source=="persisted_verified"
assert envelope.action=="source_capture"
assert envelope.digest==persisted["execution_digest"]
assert bridge.calls==[persisted]
@@ -0,0 +1,117 @@
from __future__ import annotations
from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "active-runtime/app"))
from aios_online_agent_execution import (
GovernedExecutionBinding,
project_execution_envelope,
)
import aios_online_agent_worker as worker
from aios_online_agent_worker import WorkerSupervisor
BASE = "a" * 40
def _binding() -> GovernedExecutionBinding:
return GovernedExecutionBinding.from_mapping(
{
"task_id": "r42-source-capture-replay-test-20260818",
"job_id": "job-source-capture-replay-test",
"repository": "aios/aios",
"base_oid": BASE,
"expected_branch_oid": BASE,
"target_branch": "agent/source-capture-replay-test",
"branch_mode": "CREATE_NEW",
"generation": 1,
"idempotency_key": "source-capture-replay-test-v1",
}
)
def test_source_capture_replays_the_durable_envelope_to_bridge():
binding = _binding()
execution = project_execution_envelope(
binding,
{
"action": "source_capture",
"adapter": "compact_interface_map",
"capability_scope": ["git.read", "git.test"],
"source_signals": ["model provider registry version pinning"],
},
)
task = {
**binding.public(),
"required_capabilities": ["git.read", "git.test"],
"metadata": {"execution": execution},
}
class Bridge:
def __init__(self):
self.calls = []
def project_execution(self, _task, projection_input=None):
self.calls.append(projection_input)
return 200, dict(projection_input or {})
supervisor = object.__new__(WorkerSupervisor)
supervisor.bridge = Bridge()
resolved, source = supervisor.resolve_envelope(task)
assert source == "persisted_verified"
assert resolved.digest == execution["execution_digest"]
assert supervisor.bridge.calls == [execution]
def test_deterministic_tasks_still_use_plan_registry_without_payload_replay():
binding = _binding()
execution = project_execution_envelope(
binding,
{
"action": "deterministic",
"adapter": "deterministic",
"capability_scope": [
"git.read",
"git.test",
"governed.write.branch",
"governed.write.file",
],
"change_set": [
{
"path": "active-runtime/app/example.py",
"operation": "write_file",
"content": "x",
}
],
},
)
task = {
**binding.public(),
"required_capabilities": [
"git.read",
"git.test",
"governed.write.branch",
"governed.write.file",
],
"metadata": {"execution": execution},
}
class Bridge:
def __init__(self):
self.calls = []
def project_execution(self, _task, projection_input=None):
self.calls.append(projection_input)
return 200, dict(execution)
supervisor = object.__new__(WorkerSupervisor)
supervisor.bridge = Bridge()
resolved, source = supervisor.resolve_envelope(task)
assert source == "persisted_verified"
assert resolved.digest == execution["execution_digest"]
assert supervisor.bridge.calls == [None]
@@ -0,0 +1,16 @@
from __future__ import annotations
import json
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
def test_openclaw_profile_uses_conservative_input_reservation_floor():
value=json.loads(
(ROOT/"config/external-agent-runtime.d/openclaw.profile.json").read_text()
)
mg=value["model_governance"]
assert mg["required"] is True
assert mg["input_reservation_floor_tokens"]==16384
assert mg["max_output_tokens"]==256
+199
View File
@@ -0,0 +1,199 @@
from __future__ import annotations
import json
from pathlib import Path
import sys
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/"active-runtime/app"))
from aios_worker_availability_guard import AvailabilityGuard
class FakeSystemd:
def __init__(self, roots, now=1000.0):
self.now=now
self.actions=[]
self.units={}
for unit,root in roots.items():
self.units[unit]={
"LoadState":"loaded",
"ActiveState":"active",
"UnitFileState":"enabled",
"Environment":f"AIOS_WORKER_STATE_ROOT={root}",
}
self.units["aios-external-agent-executor.service"]={
"LoadState":"loaded",
"ActiveState":"active",
"UnitFileState":"enabled",
"Environment":"",
}
def property(self, unit, name):
return self.units.get(unit,{}).get(name,"")
def environment(self, unit):
raw=self.property(unit,"Environment")
result={}
for item in raw.split():
if "=" in item:
k,v=item.split("=",1)
result[k]=v
return result
def enable(self, unit):
self.actions.append((unit,"enable"))
self.units[unit]["UnitFileState"]="enabled"
def start(self, unit):
self.actions.append((unit,"start"))
self.units[unit]["ActiveState"]="active"
env=self.environment(unit)
if "AIOS_WORKER_STATE_ROOT" not in env:
return
root=Path(env["AIOS_WORKER_STATE_ROOT"])
root.mkdir(parents=True,exist_ok=True)
(root/"worker-liveness.json").write_text(json.dumps({
"schema":"aios.online-agent.worker-liveness.v1",
"agent_id":"agent-"+unit,
"state":"POLL_OK",
"last_seen_epoch":self.now,
}),encoding="utf-8")
def restart(self, unit):
self.actions.append((unit,"restart"))
self.units[unit]["ActiveState"]="active"
env=self.environment(unit)
root=Path(env["AIOS_WORKER_STATE_ROOT"])
root.mkdir(parents=True,exist_ok=True)
(root/"worker-liveness.json").write_text(json.dumps({
"schema":"aios.online-agent.worker-liveness.v1",
"agent_id":"agent-"+unit,
"state":"POLL_OK",
"last_seen_epoch":self.now,
}),encoding="utf-8")
def make_guard(tmp_path, fake):
worker_units=tuple(
unit for unit in fake.units
if unit!="aios-external-agent-executor.service"
)
return AvailabilityGuard(
systemd=fake,
worker_units=worker_units,
dependency_units=("aios-external-agent-executor.service",),
status_path=tmp_path/"run/status.json",
recovery_path=tmp_path/"run/recovery.json",
inhibit_path=tmp_path/"run/maintenance.lock",
max_stale_seconds=90,
restart_cooldown_seconds=60,
startup_wait_seconds=0,
now_fn=lambda:fake.now,
sleep_fn=lambda _seconds:None,
)
def write_liveness(root, now, *, state="POLL_OK"):
root.mkdir(parents=True,exist_ok=True)
(root/"worker-liveness.json").write_text(json.dumps({
"schema":"aios.online-agent.worker-liveness.v1",
"agent_id":"agent",
"state":state,
"last_seen_epoch":now,
}),encoding="utf-8")
def test_healthy_workers_need_no_action(tmp_path):
roots={
"generic.service":tmp_path/"generic",
"hermes.service":tmp_path/"hermes",
"openclaw.service":tmp_path/"openclaw",
}
fake=FakeSystemd(roots)
for root in roots.values():
write_liveness(root,fake.now)
result=make_guard(tmp_path,fake).run(heal=True)
assert result["status"]=="HEALTHY"
assert result["all_fresh"] is True
assert fake.actions==[]
def test_inactive_worker_is_started_and_becomes_fresh(tmp_path):
roots={"generic.service":tmp_path/"generic"}
fake=FakeSystemd(roots)
fake.units["generic.service"]["ActiveState"]="inactive"
result=make_guard(tmp_path,fake).run(heal=True)
assert ("generic.service","start") in fake.actions
assert result["all_fresh"] is True
def test_disabled_worker_is_enabled_and_started(tmp_path):
roots={"generic.service":tmp_path/"generic"}
fake=FakeSystemd(roots)
fake.units["generic.service"]["UnitFileState"]="disabled"
fake.units["generic.service"]["ActiveState"]="inactive"
result=make_guard(tmp_path,fake).run(heal=True)
assert ("generic.service","enable") in fake.actions
assert ("generic.service","start") in fake.actions
assert result["all_fresh"] is True
def test_active_but_stale_worker_is_restarted(tmp_path):
roots={"generic.service":tmp_path/"generic"}
fake=FakeSystemd(roots)
write_liveness(roots["generic.service"],fake.now-500)
result=make_guard(tmp_path,fake).run(heal=True)
assert ("generic.service","restart") in fake.actions
assert result["all_fresh"] is True
assert result["workers"][0]["action"]=="RESTART_STALE"
def test_restart_cooldown_prevents_restart_storm(tmp_path):
roots={"generic.service":tmp_path/"generic"}
fake=FakeSystemd(roots)
write_liveness(roots["generic.service"],fake.now-500)
guard=make_guard(tmp_path,fake)
guard.recovery_path.parent.mkdir(parents=True,exist_ok=True)
guard.recovery_path.write_text(json.dumps({
"schema":"aios.worker-availability.recovery.v1",
"last_restart_epoch":{"generic.service":fake.now-10},
}),encoding="utf-8")
result=guard.run(heal=True)
assert ("generic.service","restart") not in fake.actions
assert result["status"]=="DEGRADED"
assert result["workers"][0]["detail"]=="RESTART_COOLDOWN_ACTIVE"
def test_maintenance_inhibit_disables_healing(tmp_path):
roots={"generic.service":tmp_path/"generic"}
fake=FakeSystemd(roots)
fake.units["generic.service"]["ActiveState"]="inactive"
guard=make_guard(tmp_path,fake)
guard.inhibit_path.parent.mkdir(parents=True,exist_ok=True)
guard.inhibit_path.write_text("planned maintenance\n")
result=guard.run(heal=True)
assert result["status"]=="MAINTENANCE_INHIBITED"
assert fake.actions==[]
def test_dependency_is_enabled_and_started(tmp_path):
roots={"generic.service":tmp_path/"generic"}
fake=FakeSystemd(roots)
write_liveness(roots["generic.service"],fake.now)
fake.units["aios-external-agent-executor.service"]["UnitFileState"]="disabled"
fake.units["aios-external-agent-executor.service"]["ActiveState"]="inactive"
result=make_guard(tmp_path,fake).run(heal=True)
assert ("aios-external-agent-executor.service","enable") in fake.actions
assert ("aios-external-agent-executor.service","start") in fake.actions
assert result["all_fresh"] is True
def test_check_reports_stale_without_mutation(tmp_path):
roots={"generic.service":tmp_path/"generic"}
fake=FakeSystemd(roots)
write_liveness(roots["generic.service"],fake.now-500)
result=make_guard(tmp_path,fake).run(heal=False)
assert result["status"]=="DEGRADED"
assert result["all_fresh"] is False
assert fake.actions==[]
@@ -0,0 +1,56 @@
from __future__ import annotations
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
def test_worker_source_has_idle_and_busy_liveness():
source=(ROOT/"active-runtime/app/aios_online_agent_worker.py").read_text()
assert "aios.online-agent.worker-liveness.v1" in source
assert 'write_worker_liveness("POLL_OK"' in source
assert '"POLL_FAILED"' in source
assert 'write_worker_liveness("STARTING")' in source
assert '"BUSY"' in source
assert "last_seen_epoch" in source
def test_guard_timer_is_persistent_and_frequent():
timer=(ROOT/"deploy/systemd/aios-worker-availability-guard.timer").read_text()
assert "OnBootSec=15s" in timer
assert "OnUnitActiveSec=30s" in timer
assert "Persistent=true" in timer
assert "WantedBy=timers.target" in timer
def test_guard_service_has_no_network_or_credentials():
unit=(ROOT/"deploy/systemd/aios-worker-availability-guard.service").read_text()
assert "User=root" in unit
assert "LoadCredential" not in unit
assert "RestrictAddressFamilies=AF_UNIX" in unit
assert "NoNewPrivileges=true" in unit
assert "ProtectSystem=strict" in unit
assert "ReadWritePaths=/run/aios-worker-availability" in unit
assert "AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS=90" in unit
def test_guard_manages_all_three_workers_and_external_executor():
source=(ROOT/"active-runtime/app/aios_worker_availability_guard.py").read_text()
for unit in (
"aios-online-agent-worker-20260815.service",
"aios-hermes-online-agent-worker.service",
"aios-openclaw-online-agent-worker.service",
"aios-external-agent-executor.service",
):
assert unit in source
assert "maintenance.lock" in source
assert "RESTART_STALE" in source
def test_worker_defers_write_capable_source_capture_until_plan_exists():
source=(ROOT/"active-runtime/app/aios_online_agent_worker.py").read_text()
assert "ENGINEERING_WRITE_CAPABILITIES" in source
assert "source_capture_requires_deterministic_plan" in source
assert "TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN" in source
assert '"governed.write.branch"' in source
assert '"governed.write.file"' in source