Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
57218959b0 | ||
|
|
867d4a5173 | ||
|
|
3a02a25d79 | ||
|
|
69a51d50c7 | ||
|
|
7f12094e58 | ||
|
|
2ae50a2a3d | ||
|
|
e9426dc0b8 | ||
|
|
d31267e89c | ||
|
|
f4022d3d1e | ||
|
|
008bea3324 | ||
|
|
52ef41e45f | ||
|
|
416e5e295d |
@@ -0,0 +1,470 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Canonical AIOS all-modified-files evidence packager.
|
||||||
|
|
||||||
|
The bundle contract is intentionally compatible with the historical
|
||||||
|
aios_package_all_modified_files_v1.sh output while keeping secret scanning in
|
||||||
|
version-controlled, tested code.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import zipfile
|
||||||
|
from typing import Iterable
|
||||||
|
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
|
||||||
|
# A PEM private-key finding requires the complete PEM header. Source code,
|
||||||
|
# tests, docs, or error text containing only "BEGIN PRIVATE KEY" are not
|
||||||
|
# credentials and must not be rejected.
|
||||||
|
PEM_PRIVATE_KEY_RE = re.compile(
|
||||||
|
rb"-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
BEARER_RE = re.compile(
|
||||||
|
rb"authorization\s*:\s*bearer\s+[A-Za-z0-9._~+/=-]{20,}",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
KEY_VALUE_SECRET_RE = re.compile(
|
||||||
|
rb"(?:password|passwd|secret|api[_-]?key)\s*[:=]\s*[^\s,;]{12,}",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
DOCUMENT_SUFFIXES = {".md", ".rst", ".adoc", ".txt"}
|
||||||
|
DOCUMENT_PREFIXES = (
|
||||||
|
"README",
|
||||||
|
"CHANGELOG",
|
||||||
|
"RUNBOOK",
|
||||||
|
"ADR",
|
||||||
|
"ARCHITECTURE",
|
||||||
|
"DEPLOYMENT",
|
||||||
|
"TROUBLESHOOTING",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class PackagerError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def sha256(data: bytes) -> str:
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def secret_finding(data: bytes) -> str | None:
|
||||||
|
if PEM_PRIVATE_KEY_RE.search(data):
|
||||||
|
return "PEM_PRIVATE_KEY"
|
||||||
|
if BEARER_RE.search(data):
|
||||||
|
return "AUTHORIZATION_BEARER"
|
||||||
|
if KEY_VALUE_SECRET_RE.search(data):
|
||||||
|
return "KEY_VALUE_SECRET"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def scan_secret(data: bytes, label: str) -> None:
|
||||||
|
finding = secret_finding(data)
|
||||||
|
if finding is not None:
|
||||||
|
raise PackagerError(
|
||||||
|
f"MODIFIED_FILES_SECRET_SCAN_MATCH:{finding}:{label}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def git_bytes(git_dir: Path, oid: str, path: str) -> bytes:
|
||||||
|
try:
|
||||||
|
return subprocess.check_output(
|
||||||
|
[
|
||||||
|
"/usr/bin/git",
|
||||||
|
f"--git-dir={git_dir}",
|
||||||
|
"show",
|
||||||
|
f"{oid}:{path}",
|
||||||
|
],
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
raise PackagerError(
|
||||||
|
"MODIFIED_FILES_GIT_SHOW_FAILED:" + path
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def git_text(git_dir: Path, *args: str) -> str:
|
||||||
|
try:
|
||||||
|
return subprocess.check_output(
|
||||||
|
["/usr/bin/git", f"--git-dir={git_dir}", *args],
|
||||||
|
text=True,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
raise PackagerError(
|
||||||
|
"MODIFIED_FILES_GIT_COMMAND_FAILED:" + " ".join(args)
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def validate_lineage(git_dir: Path, base: str, candidate: str) -> None:
|
||||||
|
if OID_RE.fullmatch(base) is None or OID_RE.fullmatch(candidate) is None:
|
||||||
|
raise PackagerError("MODIFIED_FILES_OID_INVALID")
|
||||||
|
for oid in (base, candidate):
|
||||||
|
try:
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
"/usr/bin/git",
|
||||||
|
f"--git-dir={git_dir}",
|
||||||
|
"cat-file",
|
||||||
|
"-e",
|
||||||
|
f"{oid}^{{commit}}",
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
raise PackagerError("MODIFIED_FILES_COMMIT_MISSING:" + oid) from exc
|
||||||
|
if subprocess.run(
|
||||||
|
[
|
||||||
|
"/usr/bin/git",
|
||||||
|
f"--git-dir={git_dir}",
|
||||||
|
"merge-base",
|
||||||
|
"--is-ancestor",
|
||||||
|
base,
|
||||||
|
candidate,
|
||||||
|
],
|
||||||
|
check=False,
|
||||||
|
timeout=30,
|
||||||
|
).returncode != 0:
|
||||||
|
raise PackagerError("MODIFIED_FILES_LINEAGE_INVALID")
|
||||||
|
|
||||||
|
|
||||||
|
def safe_destination(root: Path, relative: str) -> Path:
|
||||||
|
path = Path(relative)
|
||||||
|
if path.is_absolute() or ".." in path.parts:
|
||||||
|
raise PackagerError("MODIFIED_FILE_PATH_INVALID:" + relative)
|
||||||
|
destination = root / path
|
||||||
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
return destination
|
||||||
|
|
||||||
|
|
||||||
|
def is_document_path(path: str) -> bool:
|
||||||
|
value = Path(path)
|
||||||
|
if path.startswith("docs/"):
|
||||||
|
return True
|
||||||
|
if value.suffix.lower() in DOCUMENT_SUFFIXES:
|
||||||
|
return True
|
||||||
|
upper = value.name.upper()
|
||||||
|
return any(upper.startswith(prefix) for prefix in DOCUMENT_PREFIXES)
|
||||||
|
|
||||||
|
|
||||||
|
def diff_entries(
|
||||||
|
git_dir: Path,
|
||||||
|
base: str,
|
||||||
|
candidate: str,
|
||||||
|
before_root: Path,
|
||||||
|
after_root: Path,
|
||||||
|
) -> list[dict[str, object]]:
|
||||||
|
raw = git_text(
|
||||||
|
git_dir,
|
||||||
|
"diff",
|
||||||
|
"--name-status",
|
||||||
|
"-M",
|
||||||
|
"-C",
|
||||||
|
base,
|
||||||
|
candidate,
|
||||||
|
)
|
||||||
|
entries: list[dict[str, object]] = []
|
||||||
|
for line in raw.splitlines():
|
||||||
|
if not line.strip():
|
||||||
|
continue
|
||||||
|
parts = line.split("\t")
|
||||||
|
status = parts[0]
|
||||||
|
code = status[0]
|
||||||
|
old_path: str | None = None
|
||||||
|
new_path: str | None = None
|
||||||
|
if code in {"R", "C"}:
|
||||||
|
if len(parts) != 3:
|
||||||
|
raise PackagerError(
|
||||||
|
"MODIFIED_FILES_DIFF_PARSE_INVALID:" + line
|
||||||
|
)
|
||||||
|
old_path, new_path = parts[1], parts[2]
|
||||||
|
else:
|
||||||
|
if len(parts) != 2:
|
||||||
|
raise PackagerError(
|
||||||
|
"MODIFIED_FILES_DIFF_PARSE_INVALID:" + line
|
||||||
|
)
|
||||||
|
new_path = parts[1] if code != "D" else None
|
||||||
|
old_path = parts[1] if code != "A" else None
|
||||||
|
|
||||||
|
entry: dict[str, object] = {"status": status}
|
||||||
|
if old_path is not None:
|
||||||
|
entry["previous_path"] = old_path
|
||||||
|
before = git_bytes(git_dir, base, old_path)
|
||||||
|
scan_secret(before, "before:" + old_path)
|
||||||
|
safe_destination(before_root, old_path).write_bytes(before)
|
||||||
|
entry["before_sha256"] = sha256(before)
|
||||||
|
entry["before_bytes"] = len(before)
|
||||||
|
if new_path is not None:
|
||||||
|
entry["path"] = new_path
|
||||||
|
after = git_bytes(git_dir, candidate, new_path)
|
||||||
|
scan_secret(after, "after:" + new_path)
|
||||||
|
safe_destination(after_root, new_path).write_bytes(after)
|
||||||
|
entry["after_sha256"] = sha256(after)
|
||||||
|
entry["after_bytes"] = len(after)
|
||||||
|
entry["deleted"] = code == "D"
|
||||||
|
entry["added"] = code == "A"
|
||||||
|
entry["renamed"] = code == "R"
|
||||||
|
entry["copied"] = code == "C"
|
||||||
|
entries.append(entry)
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def execution_entries(
|
||||||
|
extra_root: Path | None,
|
||||||
|
execution_root: Path,
|
||||||
|
) -> list[dict[str, object]]:
|
||||||
|
if extra_root is None or not extra_root.exists():
|
||||||
|
return []
|
||||||
|
if extra_root.is_symlink() or not extra_root.is_dir():
|
||||||
|
raise PackagerError("EXTRA_FILES_DIR_INVALID")
|
||||||
|
|
||||||
|
entries: list[dict[str, object]] = []
|
||||||
|
for path in sorted(extra_root.rglob("*")):
|
||||||
|
if path.is_symlink():
|
||||||
|
raise PackagerError(
|
||||||
|
"EXTRA_FILE_SYMLINK_REJECTED:" + str(path)
|
||||||
|
)
|
||||||
|
if not path.is_file():
|
||||||
|
continue
|
||||||
|
relative = path.relative_to(extra_root).as_posix()
|
||||||
|
data = path.read_bytes()
|
||||||
|
scan_secret(data, "execution:" + relative)
|
||||||
|
safe_destination(execution_root, relative).write_bytes(data)
|
||||||
|
entries.append(
|
||||||
|
{
|
||||||
|
"path": relative,
|
||||||
|
"sha256": sha256(data),
|
||||||
|
"bytes": len(data),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def write_checksums(bundle: Path) -> None:
|
||||||
|
checks: list[tuple[str, str]] = []
|
||||||
|
for path in sorted(bundle.rglob("*")):
|
||||||
|
if path.is_file() and path.name != "checksums.sha256":
|
||||||
|
checks.append(
|
||||||
|
(
|
||||||
|
sha256(path.read_bytes()),
|
||||||
|
path.relative_to(bundle).as_posix(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
(bundle / "checksums.sha256").write_text(
|
||||||
|
"".join(f"{digest} {name}\n" for digest, name in checks),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def verify_checksums(bundle: Path) -> None:
|
||||||
|
checksum_path = bundle / "checksums.sha256"
|
||||||
|
for line in checksum_path.read_text(encoding="utf-8").splitlines():
|
||||||
|
digest, relative = line.split(" ", 1)
|
||||||
|
path = bundle / relative
|
||||||
|
if not path.is_file() or sha256(path.read_bytes()) != digest:
|
||||||
|
raise PackagerError(
|
||||||
|
"MODIFIED_FILES_CHECKSUM_CLOSURE_INVALID:" + relative
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_bundle(
|
||||||
|
*,
|
||||||
|
git_dir: Path,
|
||||||
|
base: str,
|
||||||
|
candidate: str,
|
||||||
|
task_id: str,
|
||||||
|
extra_dir: Path | None = None,
|
||||||
|
output_root: Path | None = None,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
if TASK_RE.fullmatch(task_id) is None:
|
||||||
|
raise PackagerError("TASK_ID_INVALID")
|
||||||
|
validate_lineage(git_dir, base, candidate)
|
||||||
|
|
||||||
|
timestamp = (now or datetime.now(timezone.utc)).strftime(
|
||||||
|
"%Y%m%dT%H%M%SZ"
|
||||||
|
)
|
||||||
|
safe_task = re.sub(r"[^A-Za-z0-9._-]+", "-", task_id).strip("-")
|
||||||
|
if output_root is None:
|
||||||
|
output_root = Path(
|
||||||
|
tempfile.mkdtemp(
|
||||||
|
prefix=f"aios-modified-files-bundle-{safe_task}-{timestamp}-",
|
||||||
|
dir="/var/tmp",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
output_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
bundle = output_root / "bundle"
|
||||||
|
before_root = bundle / "repo" / "before"
|
||||||
|
after_root = bundle / "repo" / "after"
|
||||||
|
execution_root = bundle / "execution"
|
||||||
|
before_root.mkdir(parents=True)
|
||||||
|
after_root.mkdir(parents=True)
|
||||||
|
execution_root.mkdir(parents=True)
|
||||||
|
|
||||||
|
changes = diff_entries(
|
||||||
|
git_dir,
|
||||||
|
base,
|
||||||
|
candidate,
|
||||||
|
before_root,
|
||||||
|
after_root,
|
||||||
|
)
|
||||||
|
execution_files = execution_entries(extra_dir, execution_root)
|
||||||
|
|
||||||
|
document_changes = [
|
||||||
|
entry
|
||||||
|
for entry in changes
|
||||||
|
if is_document_path(
|
||||||
|
str(entry.get("path") or entry.get("previous_path") or "")
|
||||||
|
)
|
||||||
|
]
|
||||||
|
execution_documents = [
|
||||||
|
entry
|
||||||
|
for entry in execution_files
|
||||||
|
if is_document_path(str(entry["path"]))
|
||||||
|
]
|
||||||
|
|
||||||
|
documents_index = {
|
||||||
|
"schema": "aios.modified-documents.index.v1",
|
||||||
|
"task_id": task_id,
|
||||||
|
"base_oid": base,
|
||||||
|
"candidate_oid": candidate,
|
||||||
|
"repository_documents": document_changes,
|
||||||
|
"execution_documents": execution_documents,
|
||||||
|
"repository_document_count": len(document_changes),
|
||||||
|
"execution_document_count": len(execution_documents),
|
||||||
|
}
|
||||||
|
(bundle / "documents-index.json").write_text(
|
||||||
|
json.dumps(documents_index, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
manifest = {
|
||||||
|
"schema": "aios.all-modified-files.bundle.v1",
|
||||||
|
"task_id": task_id,
|
||||||
|
"base_oid": base,
|
||||||
|
"candidate_oid": candidate,
|
||||||
|
"generated_at": (
|
||||||
|
now or datetime.now(timezone.utc)
|
||||||
|
).isoformat(),
|
||||||
|
"changed_file_count": len(changes),
|
||||||
|
"execution_file_count": len(execution_files),
|
||||||
|
"repository_document_count": len(document_changes),
|
||||||
|
"execution_document_count": len(execution_documents),
|
||||||
|
"changes": changes,
|
||||||
|
"execution_files": execution_files,
|
||||||
|
}
|
||||||
|
(bundle / "all-modified-files-manifest.json").write_text(
|
||||||
|
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
write_checksums(bundle)
|
||||||
|
verify_checksums(bundle)
|
||||||
|
|
||||||
|
zip_name = (
|
||||||
|
f"AIOS_{task_id}_ALL_MODIFIED_FILES_{candidate[:12]}.zip"
|
||||||
|
.replace("/", "-")
|
||||||
|
)
|
||||||
|
zip_path = output_root / zip_name
|
||||||
|
with zipfile.ZipFile(
|
||||||
|
zip_path,
|
||||||
|
"w",
|
||||||
|
compression=zipfile.ZIP_DEFLATED,
|
||||||
|
) as archive:
|
||||||
|
for path in sorted(bundle.rglob("*")):
|
||||||
|
if path.is_symlink():
|
||||||
|
raise PackagerError(
|
||||||
|
"BUNDLE_SYMLINK_REJECTED:" + str(path)
|
||||||
|
)
|
||||||
|
if path.is_file():
|
||||||
|
archive.write(
|
||||||
|
path,
|
||||||
|
arcname=path.relative_to(bundle).as_posix(),
|
||||||
|
)
|
||||||
|
|
||||||
|
zip_sha = sha256(zip_path.read_bytes())
|
||||||
|
summary = {
|
||||||
|
"schema": "aios.all-modified-files.bundle-summary.v1",
|
||||||
|
"task_id": task_id,
|
||||||
|
"base_oid": base,
|
||||||
|
"candidate_oid": candidate,
|
||||||
|
"bundle_path": str(zip_path),
|
||||||
|
"bundle_sha256": zip_sha,
|
||||||
|
"changed_file_count": len(changes),
|
||||||
|
"execution_file_count": len(execution_files),
|
||||||
|
"repository_document_count": len(document_changes),
|
||||||
|
"execution_document_count": len(execution_documents),
|
||||||
|
}
|
||||||
|
(output_root / "ALL_MODIFIED_FILES_BUNDLE.json").write_text(
|
||||||
|
json.dumps(summary, indent=2, sort_keys=True) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("base")
|
||||||
|
parser.add_argument("candidate")
|
||||||
|
parser.add_argument("task_id")
|
||||||
|
parser.add_argument("extra_dir", nargs="?")
|
||||||
|
parser.add_argument(
|
||||||
|
"--authority-git-dir",
|
||||||
|
default=os.environ.get(
|
||||||
|
"AIOS_AUTHORITY_GIT_DIR",
|
||||||
|
"/data/aios-gitops/repo.git",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument("--output-root")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
summary = build_bundle(
|
||||||
|
git_dir=Path(args.authority_git_dir),
|
||||||
|
base=args.base,
|
||||||
|
candidate=args.candidate,
|
||||||
|
task_id=args.task_id,
|
||||||
|
extra_dir=Path(args.extra_dir) if args.extra_dir else None,
|
||||||
|
output_root=(
|
||||||
|
Path(args.output_root) if args.output_root else None
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except PackagerError as exc:
|
||||||
|
print("FATAL:" + str(exc))
|
||||||
|
return 20
|
||||||
|
|
||||||
|
print("ALL_MODIFIED_FILES_BUNDLE=PASS")
|
||||||
|
print("ALL_MODIFIED_FILES_BUNDLE_PATH=" + str(summary["bundle_path"]))
|
||||||
|
print("ALL_MODIFIED_FILES_BUNDLE_SHA256=" + str(summary["bundle_sha256"]))
|
||||||
|
print("CHANGED_FILE_COUNT=" + str(summary["changed_file_count"]))
|
||||||
|
print("EXECUTION_FILE_COUNT=" + str(summary["execution_file_count"]))
|
||||||
|
print(
|
||||||
|
"REPOSITORY_DOCUMENT_COUNT="
|
||||||
|
+ str(summary["repository_document_count"])
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"EXECUTION_DOCUMENT_COUNT="
|
||||||
|
+ str(summary["execution_document_count"])
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"ALL_MODIFIED_FILES_EVIDENCE="
|
||||||
|
+ str(Path(str(summary["bundle_path"])).parent)
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,770 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Durable Owner Authorization broker for unattended AIOS preparation work.
|
||||||
|
|
||||||
|
This module deliberately separates durable Owner intent authorization from
|
||||||
|
cryptographic release approval. A durable grant may authorize preparation
|
||||||
|
operations such as governed-task projection and release-candidate creation,
|
||||||
|
but it can never authorize deployment, tag publication, baseline finalization
|
||||||
|
or runtime switching.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
from http.server import BaseHTTPRequestHandler
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import socketserver
|
||||||
|
import stat
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
POLICY_SCHEMA = "aios.owner-durable-authorization.v1"
|
||||||
|
RECEIPT_SCHEMA = "aios.owner-durable-authorization-receipt.v1"
|
||||||
|
DECISION_SCHEMA = "aios.owner-durable-authorization-decision.v1"
|
||||||
|
|
||||||
|
ALLOWED_PERSISTENT_CAPABILITIES = frozenset({
|
||||||
|
"governed-task:project",
|
||||||
|
"release:candidate",
|
||||||
|
})
|
||||||
|
SAFE_PROJECTED_TASK_CAPABILITIES = frozenset({
|
||||||
|
"git.read",
|
||||||
|
"git.test",
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
})
|
||||||
|
FORBIDDEN_PERSISTENT_CAPABILITIES = frozenset({
|
||||||
|
"submitOwnerApproval",
|
||||||
|
"startDeployment",
|
||||||
|
"tag_publish",
|
||||||
|
"baseline_registry_finalize",
|
||||||
|
"runtime_switch",
|
||||||
|
"main_write",
|
||||||
|
"production_deploy",
|
||||||
|
"force_push",
|
||||||
|
"credential_admin",
|
||||||
|
})
|
||||||
|
ACCOUNT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{3,255}$")
|
||||||
|
GRANT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/-]{7,255}$")
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
VERSION_RE = re.compile(r"^V[0-9]+R[0-9]+C[0-9]+$")
|
||||||
|
TASK_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
BRANCH_RE = re.compile(r"^agent/[A-Za-z0-9][A-Za-z0-9._/-]{2,179}$")
|
||||||
|
JOB_RE = re.compile(r"^job-[0-9a-f]{32}$")
|
||||||
|
MAX_BODY = 512 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
class AuthorizationError(RuntimeError):
|
||||||
|
def __init__(self, code: str, status: int = 400) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.status = status
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def sha256(value: bytes) -> str:
|
||||||
|
return hashlib.sha256(value).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def parse_time(value: object, *, field: str) -> datetime:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
raise AuthorizationError("POLICY_" + field.upper() + "_INVALID")
|
||||||
|
text = value.strip()
|
||||||
|
if text.endswith("Z"):
|
||||||
|
text = text[:-1] + "+00:00"
|
||||||
|
try:
|
||||||
|
result = datetime.fromisoformat(text)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise AuthorizationError("POLICY_" + field.upper() + "_INVALID") from exc
|
||||||
|
if result.tzinfo is None:
|
||||||
|
raise AuthorizationError("POLICY_" + field.upper() + "_INVALID")
|
||||||
|
return result.astimezone(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_root_file(path: Path, *, max_size: int = 256 * 1024) -> Path:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise AuthorizationError("POLICY_FILE_INVALID", 503)
|
||||||
|
info = path.stat(follow_symlinks=False)
|
||||||
|
if info.st_uid != 0 or info.st_gid != 0:
|
||||||
|
raise AuthorizationError("POLICY_FILE_OWNERSHIP_INVALID", 503)
|
||||||
|
if stat.S_IMODE(info.st_mode) != 0o600:
|
||||||
|
raise AuthorizationError("POLICY_FILE_MODE_INVALID", 503)
|
||||||
|
if info.st_size < 16 or info.st_size > max_size:
|
||||||
|
raise AuthorizationError("POLICY_FILE_SIZE_INVALID", 503)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Constraints:
|
||||||
|
repository: str
|
||||||
|
account: str
|
||||||
|
allowed_target_prefix: str
|
||||||
|
release_version_prefix: str
|
||||||
|
max_task_risk: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DurablePolicy:
|
||||||
|
grant_id: str
|
||||||
|
owner_identity: str
|
||||||
|
account: str
|
||||||
|
capabilities: frozenset[str]
|
||||||
|
issued_at: datetime
|
||||||
|
not_before: datetime
|
||||||
|
persistent_until_revoked: bool
|
||||||
|
constraints: Constraints
|
||||||
|
policy_sha256: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load(
|
||||||
|
cls,
|
||||||
|
policy_path: Path,
|
||||||
|
revocation_path: Path | None = None,
|
||||||
|
*,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> "DurablePolicy":
|
||||||
|
validate_root_file(policy_path)
|
||||||
|
try:
|
||||||
|
value = json.loads(policy_path.read_text(encoding="utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
raise AuthorizationError("POLICY_JSON_INVALID", 503) from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise AuthorizationError("POLICY_JSON_INVALID", 503)
|
||||||
|
allowed_fields = {
|
||||||
|
"schema",
|
||||||
|
"grant_id",
|
||||||
|
"owner_identity",
|
||||||
|
"account",
|
||||||
|
"capabilities",
|
||||||
|
"issued_at",
|
||||||
|
"not_before",
|
||||||
|
"persistent_until_revoked",
|
||||||
|
"constraints",
|
||||||
|
}
|
||||||
|
if set(value) != allowed_fields or value.get("schema") != POLICY_SCHEMA:
|
||||||
|
raise AuthorizationError("POLICY_FIELDS_INVALID", 503)
|
||||||
|
|
||||||
|
grant_id = str(value.get("grant_id") or "")
|
||||||
|
owner_identity = str(value.get("owner_identity") or "")
|
||||||
|
account = str(value.get("account") or "")
|
||||||
|
if GRANT_RE.fullmatch(grant_id) is None:
|
||||||
|
raise AuthorizationError("POLICY_GRANT_ID_INVALID", 503)
|
||||||
|
if ACCOUNT_RE.fullmatch(owner_identity) is None:
|
||||||
|
raise AuthorizationError("POLICY_OWNER_IDENTITY_INVALID", 503)
|
||||||
|
if ACCOUNT_RE.fullmatch(account) is None:
|
||||||
|
raise AuthorizationError("POLICY_ACCOUNT_INVALID", 503)
|
||||||
|
|
||||||
|
raw_caps = value.get("capabilities")
|
||||||
|
if not isinstance(raw_caps, list) or not raw_caps:
|
||||||
|
raise AuthorizationError("POLICY_CAPABILITIES_INVALID", 503)
|
||||||
|
capabilities = frozenset(str(item) for item in raw_caps)
|
||||||
|
if len(capabilities) != len(raw_caps):
|
||||||
|
raise AuthorizationError("POLICY_CAPABILITIES_DUPLICATE", 503)
|
||||||
|
if not capabilities.issubset(ALLOWED_PERSISTENT_CAPABILITIES):
|
||||||
|
raise AuthorizationError("POLICY_CAPABILITY_NOT_PERSISTABLE", 403)
|
||||||
|
if capabilities & FORBIDDEN_PERSISTENT_CAPABILITIES:
|
||||||
|
raise AuthorizationError("POLICY_FORBIDDEN_CAPABILITY", 403)
|
||||||
|
|
||||||
|
issued_at = parse_time(value.get("issued_at"), field="issued_at")
|
||||||
|
not_before = parse_time(value.get("not_before"), field="not_before")
|
||||||
|
current = now or datetime.now(timezone.utc)
|
||||||
|
if issued_at > not_before or current < not_before:
|
||||||
|
raise AuthorizationError("POLICY_NOT_ACTIVE", 403)
|
||||||
|
if value.get("persistent_until_revoked") is not True:
|
||||||
|
raise AuthorizationError("POLICY_PERSISTENCE_INVALID", 503)
|
||||||
|
|
||||||
|
raw_constraints = value.get("constraints")
|
||||||
|
if not isinstance(raw_constraints, Mapping) or set(raw_constraints) != {
|
||||||
|
"repository",
|
||||||
|
"allowed_target_prefix",
|
||||||
|
"release_version_prefix",
|
||||||
|
"max_task_risk",
|
||||||
|
}:
|
||||||
|
raise AuthorizationError("POLICY_CONSTRAINTS_INVALID", 503)
|
||||||
|
if raw_constraints.get("repository") != "aios/aios":
|
||||||
|
raise AuthorizationError("POLICY_REPOSITORY_INVALID", 503)
|
||||||
|
prefix = str(raw_constraints.get("allowed_target_prefix") or "")
|
||||||
|
if prefix != "agent/":
|
||||||
|
raise AuthorizationError("POLICY_TARGET_PREFIX_INVALID", 503)
|
||||||
|
version_prefix = str(raw_constraints.get("release_version_prefix") or "")
|
||||||
|
if version_prefix != "V2R2C":
|
||||||
|
raise AuthorizationError("POLICY_RELEASE_VERSION_PREFIX_INVALID", 503)
|
||||||
|
max_risk = str(raw_constraints.get("max_task_risk") or "")
|
||||||
|
if max_risk not in {"low", "engineering"}:
|
||||||
|
raise AuthorizationError("POLICY_MAX_RISK_INVALID", 503)
|
||||||
|
|
||||||
|
if revocation_path is not None and revocation_path.exists():
|
||||||
|
validate_root_file(revocation_path, max_size=64 * 1024)
|
||||||
|
try:
|
||||||
|
revoked = json.loads(revocation_path.read_text(encoding="utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
raise AuthorizationError("REVOCATION_JSON_INVALID", 503) from exc
|
||||||
|
if not isinstance(revoked, Mapping):
|
||||||
|
raise AuthorizationError("REVOCATION_JSON_INVALID", 503)
|
||||||
|
grants = revoked.get("revoked_grant_ids")
|
||||||
|
if not isinstance(grants, list) or not all(isinstance(x, str) for x in grants):
|
||||||
|
raise AuthorizationError("REVOCATION_FIELDS_INVALID", 503)
|
||||||
|
if grant_id in grants:
|
||||||
|
raise AuthorizationError("OWNER_GRANT_REVOKED", 403)
|
||||||
|
|
||||||
|
return cls(
|
||||||
|
grant_id=grant_id,
|
||||||
|
owner_identity=owner_identity,
|
||||||
|
account=account,
|
||||||
|
capabilities=capabilities,
|
||||||
|
issued_at=issued_at,
|
||||||
|
not_before=not_before,
|
||||||
|
persistent_until_revoked=True,
|
||||||
|
constraints=Constraints(
|
||||||
|
repository="aios/aios",
|
||||||
|
account=account,
|
||||||
|
allowed_target_prefix=prefix,
|
||||||
|
release_version_prefix=version_prefix,
|
||||||
|
max_task_risk=max_risk,
|
||||||
|
),
|
||||||
|
policy_sha256=sha256(policy_path.read_bytes()),
|
||||||
|
)
|
||||||
|
|
||||||
|
def authorize(self, capability: str, request: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
if capability not in self.capabilities:
|
||||||
|
raise AuthorizationError("CAPABILITY_NOT_AUTHORIZED", 403)
|
||||||
|
if capability in FORBIDDEN_PERSISTENT_CAPABILITIES:
|
||||||
|
raise AuthorizationError("CAPABILITY_REQUIRES_SEPARATE_OWNER_APPROVAL", 403)
|
||||||
|
|
||||||
|
if capability == "governed-task:project":
|
||||||
|
required = {
|
||||||
|
"task_id",
|
||||||
|
"repository",
|
||||||
|
"base_oid",
|
||||||
|
"expected_branch_oid",
|
||||||
|
"target_branch",
|
||||||
|
"generation",
|
||||||
|
"risk",
|
||||||
|
"description",
|
||||||
|
"idempotency_key",
|
||||||
|
"priority",
|
||||||
|
"max_attempts",
|
||||||
|
"require_online_task_projection",
|
||||||
|
"online_task",
|
||||||
|
"execution",
|
||||||
|
}
|
||||||
|
optional = {"branch_mode"}
|
||||||
|
if set(request) - (required | optional) or not required.issubset(request):
|
||||||
|
raise AuthorizationError("GOVERNED_TASK_FIELDS_INVALID")
|
||||||
|
if request["repository"] != self.constraints.repository:
|
||||||
|
raise AuthorizationError("REPOSITORY_NOT_AUTHORIZED", 403)
|
||||||
|
if not TASK_RE.fullmatch(str(request["task_id"])):
|
||||||
|
raise AuthorizationError("TASK_ID_INVALID")
|
||||||
|
base = str(request["base_oid"])
|
||||||
|
expected = str(request["expected_branch_oid"])
|
||||||
|
if not OID_RE.fullmatch(base) or expected != base:
|
||||||
|
raise AuthorizationError("BASE_BINDING_INVALID")
|
||||||
|
branch = str(request["target_branch"])
|
||||||
|
if BRANCH_RE.fullmatch(branch) is None or not branch.startswith(
|
||||||
|
self.constraints.allowed_target_prefix
|
||||||
|
):
|
||||||
|
raise AuthorizationError("TARGET_BRANCH_NOT_AUTHORIZED", 403)
|
||||||
|
if int(request["generation"]) != 1:
|
||||||
|
raise AuthorizationError("GENERATION_NOT_AUTHORIZED", 403)
|
||||||
|
risk = str(request["risk"])
|
||||||
|
allowed_risks = {"low"} if self.constraints.max_task_risk == "low" else {
|
||||||
|
"low",
|
||||||
|
"engineering",
|
||||||
|
}
|
||||||
|
if risk not in allowed_risks:
|
||||||
|
raise AuthorizationError("TASK_RISK_NOT_AUTHORIZED", 403)
|
||||||
|
if request.get("branch_mode", "CREATE_NEW") != "CREATE_NEW":
|
||||||
|
raise AuthorizationError("BRANCH_MODE_NOT_AUTHORIZED", 403)
|
||||||
|
priority = request["priority"]
|
||||||
|
if (
|
||||||
|
not isinstance(priority, int)
|
||||||
|
or isinstance(priority, bool)
|
||||||
|
or not -100 <= priority <= 100
|
||||||
|
):
|
||||||
|
raise AuthorizationError("TASK_PRIORITY_INVALID")
|
||||||
|
max_attempts = request["max_attempts"]
|
||||||
|
if (
|
||||||
|
not isinstance(max_attempts, int)
|
||||||
|
or isinstance(max_attempts, bool)
|
||||||
|
or not 1 <= max_attempts <= 3
|
||||||
|
):
|
||||||
|
raise AuthorizationError("TASK_MAX_ATTEMPTS_INVALID")
|
||||||
|
if request["require_online_task_projection"] is not True:
|
||||||
|
raise AuthorizationError("ONLINE_TASK_PROJECTION_REQUIRED", 403)
|
||||||
|
|
||||||
|
online = request["online_task"]
|
||||||
|
execution = request["execution"]
|
||||||
|
if not isinstance(online, Mapping) or not isinstance(execution, Mapping):
|
||||||
|
raise AuthorizationError("ONLINE_TASK_ENVELOPE_INVALID")
|
||||||
|
online_allowed = {
|
||||||
|
"goal",
|
||||||
|
"instructions",
|
||||||
|
"success_criteria",
|
||||||
|
"constraints",
|
||||||
|
"required_capabilities",
|
||||||
|
"priority",
|
||||||
|
"max_attempts",
|
||||||
|
}
|
||||||
|
if set(online) != online_allowed:
|
||||||
|
raise AuthorizationError("ONLINE_TASK_FIELDS_INVALID")
|
||||||
|
if (
|
||||||
|
not isinstance(online["goal"], str)
|
||||||
|
or not online["goal"].strip()
|
||||||
|
or len(online["goal"]) > 12_000
|
||||||
|
or not isinstance(online["instructions"], str)
|
||||||
|
or len(online["instructions"]) > 24_000
|
||||||
|
or not isinstance(online["success_criteria"], list)
|
||||||
|
or len(online["success_criteria"]) > 32
|
||||||
|
or not isinstance(online["constraints"], Mapping)
|
||||||
|
):
|
||||||
|
raise AuthorizationError("ONLINE_TASK_CONTENT_INVALID")
|
||||||
|
projected_caps = online["required_capabilities"]
|
||||||
|
if (
|
||||||
|
not isinstance(projected_caps, list)
|
||||||
|
or not projected_caps
|
||||||
|
or len(projected_caps) != len(set(projected_caps))
|
||||||
|
or not frozenset(projected_caps).issubset(
|
||||||
|
SAFE_PROJECTED_TASK_CAPABILITIES
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise AuthorizationError("ONLINE_TASK_CAPABILITIES_NOT_AUTHORIZED", 403)
|
||||||
|
if online["priority"] != priority or online["max_attempts"] != max_attempts:
|
||||||
|
raise AuthorizationError("ONLINE_TASK_LIMITS_MISMATCH")
|
||||||
|
|
||||||
|
execution_allowed = {
|
||||||
|
"action",
|
||||||
|
"adapter",
|
||||||
|
"capability_scope",
|
||||||
|
"source_signals",
|
||||||
|
}
|
||||||
|
if set(execution) != execution_allowed:
|
||||||
|
raise AuthorizationError("EXECUTION_FIELDS_INVALID")
|
||||||
|
if execution.get("action") != "source_capture" or execution.get(
|
||||||
|
"adapter"
|
||||||
|
) != "source_capture":
|
||||||
|
raise AuthorizationError("INITIAL_EXECUTION_ACTION_NOT_AUTHORIZED", 403)
|
||||||
|
scope = execution.get("capability_scope")
|
||||||
|
if (
|
||||||
|
not isinstance(scope, list)
|
||||||
|
or sorted(scope) != sorted(projected_caps)
|
||||||
|
):
|
||||||
|
raise AuthorizationError("EXECUTION_CAPABILITY_SCOPE_MISMATCH")
|
||||||
|
signals = execution.get("source_signals")
|
||||||
|
if (
|
||||||
|
not isinstance(signals, list)
|
||||||
|
or len(signals) > 32
|
||||||
|
or not all(isinstance(item, str) and len(item) <= 1_000 for item in signals)
|
||||||
|
):
|
||||||
|
raise AuthorizationError("SOURCE_SIGNALS_INVALID")
|
||||||
|
|
||||||
|
elif capability == "release:candidate":
|
||||||
|
required = {"candidate_id", "source_commit", "version"}
|
||||||
|
if set(request) != required:
|
||||||
|
raise AuthorizationError("RELEASE_CANDIDATE_FIELDS_INVALID")
|
||||||
|
if not GRANT_RE.fullmatch(str(request["candidate_id"])):
|
||||||
|
raise AuthorizationError("RELEASE_CANDIDATE_ID_INVALID")
|
||||||
|
if OID_RE.fullmatch(str(request["source_commit"])) is None:
|
||||||
|
raise AuthorizationError("RELEASE_SOURCE_COMMIT_INVALID")
|
||||||
|
version = str(request["version"])
|
||||||
|
if VERSION_RE.fullmatch(version) is None or not version.startswith(
|
||||||
|
self.constraints.release_version_prefix
|
||||||
|
):
|
||||||
|
raise AuthorizationError("RELEASE_VERSION_NOT_AUTHORIZED", 403)
|
||||||
|
else:
|
||||||
|
raise AuthorizationError("CAPABILITY_NOT_SUPPORTED", 403)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"schema": DECISION_SCHEMA,
|
||||||
|
"decision": "ALLOW",
|
||||||
|
"grant_id": self.grant_id,
|
||||||
|
"owner_identity": self.owner_identity,
|
||||||
|
"account": self.account,
|
||||||
|
"capability": capability,
|
||||||
|
"policy_sha256": self.policy_sha256,
|
||||||
|
"request_sha256": sha256(canonical(request)),
|
||||||
|
"persistent_until_revoked": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class ReceiptStore:
|
||||||
|
def __init__(self, path: Path) -> None:
|
||||||
|
self.path = path
|
||||||
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with sqlite3.connect(self.path) as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS receipts(
|
||||||
|
idempotency_key TEXT PRIMARY KEY,
|
||||||
|
capability TEXT NOT NULL,
|
||||||
|
request_sha256 TEXT NOT NULL,
|
||||||
|
response_sha256 TEXT NOT NULL,
|
||||||
|
response_json TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
|
||||||
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
|
with sqlite3.connect(self.path) as con:
|
||||||
|
row = con.execute(
|
||||||
|
"SELECT request_sha256,response_json FROM receipts WHERE idempotency_key=?",
|
||||||
|
(key,),
|
||||||
|
).fetchone()
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
return {"request_sha256": row[0], "response": json.loads(row[1])}
|
||||||
|
|
||||||
|
def put(
|
||||||
|
self,
|
||||||
|
key: str,
|
||||||
|
capability: str,
|
||||||
|
request_sha256: str,
|
||||||
|
response: Mapping[str, Any],
|
||||||
|
) -> None:
|
||||||
|
encoded = json.dumps(response, sort_keys=True, separators=(",", ":"))
|
||||||
|
with self._lock, sqlite3.connect(self.path) as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO receipts(
|
||||||
|
idempotency_key,capability,request_sha256,response_sha256,
|
||||||
|
response_json,created_at
|
||||||
|
) VALUES(?,?,?,?,?,?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
key,
|
||||||
|
capability,
|
||||||
|
request_sha256,
|
||||||
|
sha256(encoded.encode()),
|
||||||
|
encoded,
|
||||||
|
datetime.now(timezone.utc).isoformat(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Upstream:
|
||||||
|
bridge_url: str
|
||||||
|
bridge_token: str
|
||||||
|
|
||||||
|
def post_bridge(
|
||||||
|
self,
|
||||||
|
path: str,
|
||||||
|
body: Mapping[str, Any],
|
||||||
|
) -> tuple[int, dict[str, Any]]:
|
||||||
|
req = urllib.request.Request(
|
||||||
|
self.bridge_url.rstrip("/") + path,
|
||||||
|
method="POST",
|
||||||
|
data=canonical(body),
|
||||||
|
headers={
|
||||||
|
"Authorization": "Bearer " + self.bridge_token,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "application/json",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as response:
|
||||||
|
raw, status = response.read(), response.status
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raw, status = exc.read(), exc.code
|
||||||
|
except Exception as exc:
|
||||||
|
raise AuthorizationError("UPSTREAM_TRANSPORT_FAILED", 503) from exc
|
||||||
|
try:
|
||||||
|
value = json.loads(raw.decode())
|
||||||
|
except Exception as exc:
|
||||||
|
raise AuthorizationError("UPSTREAM_RESPONSE_INVALID", 502) from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise AuthorizationError("UPSTREAM_RESPONSE_INVALID", 502)
|
||||||
|
return status, dict(value)
|
||||||
|
|
||||||
|
def project_governed_task(
|
||||||
|
self,
|
||||||
|
body: Mapping[str, Any],
|
||||||
|
) -> tuple[int, dict[str, Any]]:
|
||||||
|
return self.post_bridge("/v2/governed-task", body)
|
||||||
|
|
||||||
|
|
||||||
|
class Broker:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
policy_path: Path,
|
||||||
|
revocation_path: Path,
|
||||||
|
store: ReceiptStore,
|
||||||
|
upstream: Upstream,
|
||||||
|
expected_account: str,
|
||||||
|
) -> None:
|
||||||
|
self.policy_path = policy_path
|
||||||
|
self.revocation_path = revocation_path
|
||||||
|
self.store = store
|
||||||
|
self.upstream = upstream
|
||||||
|
self.expected_account = expected_account
|
||||||
|
|
||||||
|
def policy(self) -> DurablePolicy:
|
||||||
|
policy = DurablePolicy.load(self.policy_path, self.revocation_path)
|
||||||
|
if policy.account != self.expected_account:
|
||||||
|
raise AuthorizationError("POLICY_ACCOUNT_MISMATCH", 403)
|
||||||
|
return policy
|
||||||
|
|
||||||
|
def execute(
|
||||||
|
self,
|
||||||
|
capability: str,
|
||||||
|
request: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
idempotency_key: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if not GRANT_RE.fullmatch(idempotency_key):
|
||||||
|
raise AuthorizationError("IDEMPOTENCY_KEY_INVALID")
|
||||||
|
policy = self.policy()
|
||||||
|
decision = policy.authorize(capability, request)
|
||||||
|
request_sha = decision["request_sha256"]
|
||||||
|
|
||||||
|
prior = self.store.get(idempotency_key)
|
||||||
|
if prior is not None:
|
||||||
|
if prior["request_sha256"] != request_sha:
|
||||||
|
raise AuthorizationError("IDEMPOTENCY_CONFLICT", 409)
|
||||||
|
return prior["response"]
|
||||||
|
|
||||||
|
if capability == "governed-task:project":
|
||||||
|
status, value = self.upstream.project_governed_task(request)
|
||||||
|
if status not in {200, 201, 202}:
|
||||||
|
raise AuthorizationError(
|
||||||
|
"GOVERNED_TASK_PROJECTION_FAILED",
|
||||||
|
409 if status == 409 else 503,
|
||||||
|
)
|
||||||
|
if value.get("task_id") != request["task_id"]:
|
||||||
|
raise AuthorizationError("GOVERNED_TASK_RESPONSE_INVALID", 502)
|
||||||
|
operation_state = "EXECUTED"
|
||||||
|
upstream_status: int | None = status
|
||||||
|
upstream_response: Mapping[str, Any] | None = value
|
||||||
|
elif capability == "release:candidate":
|
||||||
|
# The trusted release builder lives outside the repository control
|
||||||
|
# plane. This broker persists/attests Owner intent but does not
|
||||||
|
# invent or proxy a release-builder endpoint. The external
|
||||||
|
# connector must project the account's release:candidate scope and
|
||||||
|
# may consume this grant/decision as authorization evidence.
|
||||||
|
operation_state = "AUTHORIZED_PENDING_EXTERNAL_CAPABILITY_PROJECTION"
|
||||||
|
upstream_status = None
|
||||||
|
upstream_response = None
|
||||||
|
else:
|
||||||
|
raise AuthorizationError("CAPABILITY_NOT_SUPPORTED", 403)
|
||||||
|
|
||||||
|
response = {
|
||||||
|
"schema": RECEIPT_SCHEMA,
|
||||||
|
"grant_id": policy.grant_id,
|
||||||
|
"owner_identity": policy.owner_identity,
|
||||||
|
"account": policy.account,
|
||||||
|
"capability": capability,
|
||||||
|
"policy_sha256": policy.policy_sha256,
|
||||||
|
"request_sha256": request_sha,
|
||||||
|
"idempotency_key": idempotency_key,
|
||||||
|
"operation_state": operation_state,
|
||||||
|
"upstream_status": upstream_status,
|
||||||
|
"upstream_response": upstream_response,
|
||||||
|
"external_capability_projection_required": (
|
||||||
|
capability == "release:candidate"
|
||||||
|
),
|
||||||
|
"separate_owner_release_approval_still_required": True,
|
||||||
|
}
|
||||||
|
self.store.put(idempotency_key, capability, request_sha, response)
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
def load_secret(path: Path, *, code: str) -> str:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise RuntimeError(code + "_UNAVAILABLE")
|
||||||
|
value = path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(value) < 32 or any(ch.isspace() for ch in value):
|
||||||
|
raise RuntimeError(code + "_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
broker: Broker | None = None
|
||||||
|
|
||||||
|
def log_message(self, *_args: Any) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def send_json(self, value: Mapping[str, Any], status: int) -> None:
|
||||||
|
raw = canonical(value)
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.send_header("Content-Length", str(len(raw)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(raw)
|
||||||
|
|
||||||
|
def caller_account(self) -> str:
|
||||||
|
return self.headers.get("X-AIOS-Account", "").strip()
|
||||||
|
|
||||||
|
def account_authorized(self) -> bool:
|
||||||
|
if self.broker is None:
|
||||||
|
return False
|
||||||
|
return self.caller_account() == self.broker.expected_account
|
||||||
|
|
||||||
|
def body(self) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get("Content-Length", "-1"))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise AuthorizationError("CONTENT_LENGTH_INVALID") from exc
|
||||||
|
if length < 1 or length > MAX_BODY:
|
||||||
|
raise AuthorizationError("REQUEST_SIZE_INVALID", 413)
|
||||||
|
try:
|
||||||
|
value = json.loads(self.rfile.read(length).decode())
|
||||||
|
except Exception as exc:
|
||||||
|
raise AuthorizationError("REQUEST_JSON_INVALID") from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise AuthorizationError("REQUEST_OBJECT_REQUIRED")
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
if self.path == "/health":
|
||||||
|
self.send_json(
|
||||||
|
{
|
||||||
|
"status": "ok",
|
||||||
|
"service": "aios-owner-durable-authorization-broker",
|
||||||
|
"transport": "unix",
|
||||||
|
},
|
||||||
|
200,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if not self.account_authorized():
|
||||||
|
self.send_json({"error": "ACCOUNT_NOT_AUTHORIZED"}, 403)
|
||||||
|
return
|
||||||
|
if self.path != "/v1/capabilities" or self.broker is None:
|
||||||
|
self.send_json({"error": "NOT_FOUND"}, 404)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
policy = self.broker.policy()
|
||||||
|
self.send_json(
|
||||||
|
{
|
||||||
|
"schema": POLICY_SCHEMA,
|
||||||
|
"grant_id": policy.grant_id,
|
||||||
|
"account": policy.account,
|
||||||
|
"capabilities": sorted(policy.capabilities),
|
||||||
|
"persistent_until_revoked": True,
|
||||||
|
"separate_owner_release_approval_still_required": True,
|
||||||
|
},
|
||||||
|
200,
|
||||||
|
)
|
||||||
|
except AuthorizationError as exc:
|
||||||
|
self.send_json({"error": exc.code}, exc.status)
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
if not self.account_authorized():
|
||||||
|
self.send_json({"error": "ACCOUNT_NOT_AUTHORIZED"}, 403)
|
||||||
|
return
|
||||||
|
routes = {
|
||||||
|
"/v1/project-governed-task": "governed-task:project",
|
||||||
|
"/v1/authorize-release-candidate": "release:candidate",
|
||||||
|
}
|
||||||
|
capability = routes.get(self.path)
|
||||||
|
if capability is None or self.broker is None:
|
||||||
|
self.send_json({"error": "NOT_FOUND"}, 404)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
body = self.body()
|
||||||
|
if set(body) != {"idempotency_key", "request"}:
|
||||||
|
raise AuthorizationError("BROKER_FIELDS_INVALID")
|
||||||
|
if not isinstance(body["request"], Mapping):
|
||||||
|
raise AuthorizationError("BROKER_REQUEST_INVALID")
|
||||||
|
value = self.broker.execute(
|
||||||
|
capability,
|
||||||
|
dict(body["request"]),
|
||||||
|
idempotency_key=str(body["idempotency_key"]),
|
||||||
|
)
|
||||||
|
self.send_json(value, 200)
|
||||||
|
except AuthorizationError as exc:
|
||||||
|
self.send_json({"error": exc.code}, exc.status)
|
||||||
|
|
||||||
|
|
||||||
|
class UnixHTTPServer(socketserver.ThreadingUnixStreamServer):
|
||||||
|
daemon_threads = True
|
||||||
|
|
||||||
|
def serve() -> None:
|
||||||
|
credentials = Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
||||||
|
policy_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_OWNER_DURABLE_AUTHORIZATION_POLICY",
|
||||||
|
"/etc/aios-gitops/owner-durable-authorization.json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
revocation_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_OWNER_DURABLE_AUTHORIZATION_REVOCATIONS",
|
||||||
|
"/etc/aios-gitops/owner-durable-authorization-revocations.json",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
bridge_token = load_secret(
|
||||||
|
Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
|
||||||
|
str(credentials / "agent_task_control_token"),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
code="BRIDGE_CONTROL_CREDENTIAL",
|
||||||
|
)
|
||||||
|
account = os.environ.get(
|
||||||
|
"AIOS_OWNER_AUTH_BROKER_ACCOUNT",
|
||||||
|
"chatgpt-governed-implementer-singlehost",
|
||||||
|
).strip()
|
||||||
|
state_root = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_OWNER_AUTH_BROKER_STATE_ROOT",
|
||||||
|
"/var/lib/aios-owner-authorization-broker",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
Handler.broker = Broker(
|
||||||
|
policy_path=policy_path,
|
||||||
|
revocation_path=revocation_path,
|
||||||
|
store=ReceiptStore(state_root / "receipts.sqlite3"),
|
||||||
|
upstream=Upstream(
|
||||||
|
bridge_url="http://127.0.0.1:18892",
|
||||||
|
bridge_token=bridge_token,
|
||||||
|
),
|
||||||
|
expected_account=account,
|
||||||
|
)
|
||||||
|
socket_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_OWNER_AUTH_BROKER_SOCKET",
|
||||||
|
"/run/aios-owner-authorization-broker/broker.sock",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not socket_path.is_absolute() or socket_path.is_symlink():
|
||||||
|
raise RuntimeError("BROKER_SOCKET_PATH_INVALID")
|
||||||
|
socket_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
if socket_path.exists():
|
||||||
|
if not stat.S_ISSOCK(socket_path.stat().st_mode):
|
||||||
|
raise RuntimeError("BROKER_SOCKET_CONFLICT")
|
||||||
|
socket_path.unlink()
|
||||||
|
server = UnixHTTPServer(str(socket_path), Handler)
|
||||||
|
os.chmod(socket_path, 0o660)
|
||||||
|
try:
|
||||||
|
server.serve_forever()
|
||||||
|
finally:
|
||||||
|
server.server_close()
|
||||||
|
socket_path.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
serve()
|
||||||
@@ -16,6 +16,11 @@ import subprocess
|
|||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
from typing import Any, Mapping, Sequence
|
from typing import Any, Mapping, Sequence
|
||||||
|
from aios_model_governance import (
|
||||||
|
ModelGovernanceError,
|
||||||
|
ModelGovernanceStore,
|
||||||
|
RouteRequest,
|
||||||
|
)
|
||||||
|
|
||||||
REQUEST_SCHEMA = "aios.external-agent-runtime.request.v1"
|
REQUEST_SCHEMA = "aios.external-agent-runtime.request.v1"
|
||||||
RESULT_SCHEMA = "aios.external-agent-runtime.result.v1"
|
RESULT_SCHEMA = "aios.external-agent-runtime.result.v1"
|
||||||
@@ -62,6 +67,15 @@ MAX_PROFILE_BYTES = 64 * 1024
|
|||||||
MAX_CREDENTIAL_BYTES = 64 * 1024
|
MAX_CREDENTIAL_BYTES = 64 * 1024
|
||||||
DEFAULT_PROFILE_ROOT = Path("/etc/aios/external-agent-runtime.d")
|
DEFAULT_PROFILE_ROOT = Path("/etc/aios/external-agent-runtime.d")
|
||||||
DEFAULT_EXCHANGE_ROOT = Path("/var/lib/aios-external-agent-executor")
|
DEFAULT_EXCHANGE_ROOT = Path("/var/lib/aios-external-agent-executor")
|
||||||
|
DEFAULT_MODEL_GOVERNANCE_DB = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_MODEL_GOVERNANCE_DB",
|
||||||
|
"/var/lib/aios-model-governance/state.sqlite3",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
MODEL_GOVERNANCE_ENABLED = os.environ.get(
|
||||||
|
"AIOS_MODEL_GOVERNANCE_ENABLED", "0"
|
||||||
|
).strip().lower() in {"1", "true", "yes", "on"}
|
||||||
ALLOWED_EXECUTABLE_ROOTS = (
|
ALLOWED_EXECUTABLE_ROOTS = (
|
||||||
Path("/usr/bin"),
|
Path("/usr/bin"),
|
||||||
Path("/usr/local/bin"),
|
Path("/usr/local/bin"),
|
||||||
@@ -208,6 +222,172 @@ def _profile_credential_environment(
|
|||||||
return tuple(sorted(result))
|
return tuple(sorted(result))
|
||||||
|
|
||||||
|
|
||||||
|
def _profile_model_governance(
|
||||||
|
value: object,
|
||||||
|
) -> tuple[bool, tuple[str, ...], tuple[str, ...], int, int]:
|
||||||
|
if value is None:
|
||||||
|
return False, (), (), 0, 0
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_GOVERNANCE_PROFILE_INVALID")
|
||||||
|
allowed = {
|
||||||
|
"required",
|
||||||
|
"provider_ids",
|
||||||
|
"model_capabilities",
|
||||||
|
"max_output_tokens",
|
||||||
|
"input_reservation_floor_tokens",
|
||||||
|
}
|
||||||
|
if set(value) - allowed:
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_GOVERNANCE_PROFILE_FIELDS_INVALID")
|
||||||
|
required = bool(value.get("required", False))
|
||||||
|
providers = value.get("provider_ids") or []
|
||||||
|
capabilities = value.get("model_capabilities") or []
|
||||||
|
if (
|
||||||
|
not isinstance(providers, Sequence)
|
||||||
|
or isinstance(providers, (str, bytes))
|
||||||
|
or not all(isinstance(item, str) and item.strip() for item in providers)
|
||||||
|
):
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_PROVIDER_IDS_INVALID")
|
||||||
|
if (
|
||||||
|
not isinstance(capabilities, Sequence)
|
||||||
|
or isinstance(capabilities, (str, bytes))
|
||||||
|
or not all(isinstance(item, str) and item.strip() for item in capabilities)
|
||||||
|
):
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_CAPABILITIES_INVALID")
|
||||||
|
max_output_tokens = int(value.get("max_output_tokens") or 0)
|
||||||
|
input_reservation_floor_tokens = int(
|
||||||
|
value.get("input_reservation_floor_tokens") or 0
|
||||||
|
)
|
||||||
|
if max_output_tokens < 0 or max_output_tokens > 262144:
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_OUTPUT_LIMIT_INVALID")
|
||||||
|
if (
|
||||||
|
input_reservation_floor_tokens < 0
|
||||||
|
or input_reservation_floor_tokens > 262144
|
||||||
|
):
|
||||||
|
raise ExternalAgentRuntimeError(
|
||||||
|
"EXTERNAL_AGENT_MODEL_INPUT_RESERVATION_FLOOR_INVALID"
|
||||||
|
)
|
||||||
|
if required and not providers:
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_PROVIDER_IDS_REQUIRED")
|
||||||
|
if required and max_output_tokens < 1:
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_OUTPUT_LIMIT_REQUIRED")
|
||||||
|
return (
|
||||||
|
required,
|
||||||
|
tuple(sorted({item.strip() for item in providers})),
|
||||||
|
tuple(sorted({item.strip() for item in capabilities})),
|
||||||
|
max_output_tokens,
|
||||||
|
input_reservation_floor_tokens,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _estimate_tokens(text: str) -> int:
|
||||||
|
# Conservative byte-based estimate; runtime-reported usage supersedes this at settlement.
|
||||||
|
data = text.encode("utf-8", "replace")
|
||||||
|
return max(1, (len(data) + 3) // 4)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_runtime_usage(
|
||||||
|
output_json: object,
|
||||||
|
) -> tuple[int | None, int | None, int | None]:
|
||||||
|
if not isinstance(output_json, Mapping):
|
||||||
|
return None, None, None
|
||||||
|
|
||||||
|
candidates: list[Mapping[str, Any]] = []
|
||||||
|
direct = output_json.get("usage")
|
||||||
|
if isinstance(direct, Mapping):
|
||||||
|
candidates.append(direct)
|
||||||
|
|
||||||
|
meta = output_json.get("meta")
|
||||||
|
if isinstance(meta, Mapping):
|
||||||
|
agent_meta = meta.get("agentMeta")
|
||||||
|
if isinstance(agent_meta, Mapping):
|
||||||
|
nested = agent_meta.get("usage")
|
||||||
|
if isinstance(nested, Mapping):
|
||||||
|
candidates.append(nested)
|
||||||
|
last_call = agent_meta.get("lastCallUsage")
|
||||||
|
if isinstance(last_call, Mapping):
|
||||||
|
candidates.append(last_call)
|
||||||
|
|
||||||
|
def number(mapping: Mapping[str, Any], *names: str) -> int | None:
|
||||||
|
for name in names:
|
||||||
|
value = mapping.get(name)
|
||||||
|
if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
|
||||||
|
return value
|
||||||
|
return None
|
||||||
|
|
||||||
|
input_tokens = None
|
||||||
|
output_tokens = None
|
||||||
|
cost_micros = None
|
||||||
|
for usage in candidates:
|
||||||
|
if input_tokens is None:
|
||||||
|
input_tokens = number(usage, "input_tokens", "prompt_tokens", "input")
|
||||||
|
if output_tokens is None:
|
||||||
|
output_tokens = number(
|
||||||
|
usage, "output_tokens", "completion_tokens", "output"
|
||||||
|
)
|
||||||
|
if cost_micros is None:
|
||||||
|
cost_micros = number(usage, "cost_micros", "total_cost_micros")
|
||||||
|
|
||||||
|
if cost_micros is None and isinstance(meta, Mapping):
|
||||||
|
agent_meta = meta.get("agentMeta")
|
||||||
|
if isinstance(agent_meta, Mapping):
|
||||||
|
cost_usd = agent_meta.get("costUsd")
|
||||||
|
if (
|
||||||
|
isinstance(cost_usd, (int, float))
|
||||||
|
and not isinstance(cost_usd, bool)
|
||||||
|
and cost_usd >= 0
|
||||||
|
):
|
||||||
|
cost_micros = int(round(float(cost_usd) * 1_000_000))
|
||||||
|
|
||||||
|
return input_tokens, output_tokens, cost_micros
|
||||||
|
|
||||||
|
|
||||||
|
def _scope_from_payload(payload: Mapping[str, Any]) -> tuple[str, str | None, str | None, str]:
|
||||||
|
scope = payload.get("scope")
|
||||||
|
if not isinstance(scope, Mapping):
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_SCOPE_MISSING")
|
||||||
|
scope_type = str(scope.get("scope_type") or "legacy")
|
||||||
|
tenant_id = scope.get("tenant_id")
|
||||||
|
project_id = scope.get("project_id")
|
||||||
|
trace_id = str(payload.get("trace_id") or "").strip()
|
||||||
|
if not trace_id:
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_TRACE_ID_MISSING")
|
||||||
|
return (
|
||||||
|
scope_type,
|
||||||
|
str(tenant_id) if tenant_id not in (None, "") else None,
|
||||||
|
str(project_id) if project_id not in (None, "") else None,
|
||||||
|
trace_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _model_replacements(decision: object | None) -> dict[str, str]:
|
||||||
|
if decision is None:
|
||||||
|
return {}
|
||||||
|
return {
|
||||||
|
"{provider_id}": str(getattr(decision, "provider_id")),
|
||||||
|
"{model_id}": str(getattr(decision, "model_id")),
|
||||||
|
"{model_version}": str(getattr(decision, "model_version")),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _bind_model_values(
|
||||||
|
values: Sequence[tuple[str, str]],
|
||||||
|
decision: object | None,
|
||||||
|
) -> tuple[tuple[str, str], ...]:
|
||||||
|
replacements = _model_replacements(decision)
|
||||||
|
result: list[tuple[str, str]] = []
|
||||||
|
for key, value in values:
|
||||||
|
bound = value
|
||||||
|
for marker, replacement in replacements.items():
|
||||||
|
bound = bound.replace(marker, replacement)
|
||||||
|
result.append((key, bound))
|
||||||
|
return tuple(result)
|
||||||
|
|
||||||
|
|
||||||
|
def _runtime_model_binding_present(profile: "RuntimeProfile") -> bool:
|
||||||
|
markers = ("{model_id}", "{model_version}", "{provider_id}")
|
||||||
|
return any(marker in item for item in (*profile.argv, *(value for _, value in profile.environment)) for marker in markers)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class RuntimeProfile:
|
class RuntimeProfile:
|
||||||
runtime: str
|
runtime: str
|
||||||
@@ -218,6 +398,11 @@ class RuntimeProfile:
|
|||||||
max_output_bytes: int
|
max_output_bytes: int
|
||||||
environment: tuple[tuple[str, str], ...]
|
environment: tuple[tuple[str, str], ...]
|
||||||
credential_env: tuple[tuple[str, str, str, str], ...]
|
credential_env: tuple[tuple[str, str, str, str], ...]
|
||||||
|
model_governance_required: bool
|
||||||
|
model_provider_ids: tuple[str, ...]
|
||||||
|
model_capabilities: tuple[str, ...]
|
||||||
|
model_max_output_tokens: int
|
||||||
|
model_input_reservation_floor_tokens: int
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def load(
|
def load(
|
||||||
@@ -275,7 +460,14 @@ class RuntimeProfile:
|
|||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_OUTPUT_LIMIT_INVALID")
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_OUTPUT_LIMIT_INVALID")
|
||||||
environment = _profile_environment(value.get("environment"))
|
environment = _profile_environment(value.get("environment"))
|
||||||
credential_env = _profile_credential_environment(value.get("credential_env"))
|
credential_env = _profile_credential_environment(value.get("credential_env"))
|
||||||
return cls(
|
(
|
||||||
|
model_governance_required,
|
||||||
|
model_provider_ids,
|
||||||
|
model_capabilities,
|
||||||
|
model_max_output_tokens,
|
||||||
|
model_input_reservation_floor_tokens,
|
||||||
|
) = _profile_model_governance(value.get("model_governance"))
|
||||||
|
profile = cls(
|
||||||
runtime,
|
runtime,
|
||||||
tuple(normalized),
|
tuple(normalized),
|
||||||
input_mode,
|
input_mode,
|
||||||
@@ -284,7 +476,15 @@ class RuntimeProfile:
|
|||||||
max_output,
|
max_output,
|
||||||
environment,
|
environment,
|
||||||
credential_env,
|
credential_env,
|
||||||
|
model_governance_required,
|
||||||
|
model_provider_ids,
|
||||||
|
model_capabilities,
|
||||||
|
model_max_output_tokens,
|
||||||
|
model_input_reservation_floor_tokens,
|
||||||
)
|
)
|
||||||
|
if profile.model_governance_required and not _runtime_model_binding_present(profile):
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_BINDING_REQUIRED")
|
||||||
|
return profile
|
||||||
|
|
||||||
|
|
||||||
def validate_request(value: Mapping[str, Any]) -> dict[str, Any]:
|
def validate_request(value: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
@@ -295,9 +495,16 @@ def validate_request(value: Mapping[str, Any]) -> dict[str, Any]:
|
|||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_REQUEST_ID_INVALID")
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_REQUEST_ID_INVALID")
|
||||||
runtime = _safe_runtime(value.get("runtime"))
|
runtime = _safe_runtime(value.get("runtime"))
|
||||||
task_id = str(value.get("task_id") or "")
|
task_id = str(value.get("task_id") or "")
|
||||||
|
attempt = value.get("attempt")
|
||||||
execution_digest = str(value.get("execution_digest") or "")
|
execution_digest = str(value.get("execution_digest") or "")
|
||||||
if not task_id or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{2,179}", task_id):
|
if not task_id or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{2,179}", task_id):
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_TASK_ID_INVALID")
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_TASK_ID_INVALID")
|
||||||
|
if (
|
||||||
|
not isinstance(attempt, int)
|
||||||
|
or isinstance(attempt, bool)
|
||||||
|
or not 1 <= attempt <= 1000
|
||||||
|
):
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_ATTEMPT_INVALID")
|
||||||
if re.fullmatch(r"[0-9a-f]{64}", execution_digest) is None:
|
if re.fullmatch(r"[0-9a-f]{64}", execution_digest) is None:
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_EXECUTION_DIGEST_INVALID")
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_EXECUTION_DIGEST_INVALID")
|
||||||
payload = value.get("payload")
|
payload = value.get("payload")
|
||||||
@@ -317,6 +524,7 @@ def validate_request(value: Mapping[str, Any]) -> dict[str, Any]:
|
|||||||
"request_id": request_id,
|
"request_id": request_id,
|
||||||
"runtime": runtime,
|
"runtime": runtime,
|
||||||
"task_id": task_id,
|
"task_id": task_id,
|
||||||
|
"attempt": attempt,
|
||||||
"execution_digest": execution_digest,
|
"execution_digest": execution_digest,
|
||||||
"payload": dict(payload),
|
"payload": dict(payload),
|
||||||
}
|
}
|
||||||
@@ -351,11 +559,13 @@ def _argv_with_files(
|
|||||||
input_path: Path,
|
input_path: Path,
|
||||||
output_path: Path,
|
output_path: Path,
|
||||||
prompt: str,
|
prompt: str,
|
||||||
|
model_decision: object | None = None,
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
replacements = {
|
replacements = {
|
||||||
"{input_json}": str(input_path),
|
"{input_json}": str(input_path),
|
||||||
"{output_json}": str(output_path),
|
"{output_json}": str(output_path),
|
||||||
"{prompt}": prompt,
|
"{prompt}": prompt,
|
||||||
|
**_model_replacements(model_decision),
|
||||||
}
|
}
|
||||||
result = []
|
result = []
|
||||||
for item in profile.argv:
|
for item in profile.argv:
|
||||||
@@ -471,7 +681,12 @@ def execute_request(
|
|||||||
request = validate_request(json.loads(raw_request.decode("utf-8")))
|
request = validate_request(json.loads(raw_request.decode("utf-8")))
|
||||||
except json.JSONDecodeError as exc:
|
except json.JSONDecodeError as exc:
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_REQUEST_JSON_INVALID") from exc
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_REQUEST_JSON_INVALID") from exc
|
||||||
|
|
||||||
profile = RuntimeProfile.load(request["runtime"], profile_root)
|
profile = RuntimeProfile.load(request["runtime"], profile_root)
|
||||||
|
governance_store: ModelGovernanceStore | None = None
|
||||||
|
route_request: RouteRequest | None = None
|
||||||
|
decision = None
|
||||||
|
governance_attempts: list[dict[str, Any]] = []
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory(prefix=f"aios-{request['runtime']}-") as temporary:
|
with tempfile.TemporaryDirectory(prefix=f"aios-{request['runtime']}-") as temporary:
|
||||||
temp_root = Path(temporary)
|
temp_root = Path(temporary)
|
||||||
@@ -480,78 +695,203 @@ def execute_request(
|
|||||||
output_path = temp_root / "output.json"
|
output_path = temp_root / "output.json"
|
||||||
input_path.write_bytes(canonical(request["payload"]))
|
input_path.write_bytes(canonical(request["payload"]))
|
||||||
prompt = render_prompt(request["payload"])
|
prompt = render_prompt(request["payload"])
|
||||||
argv = _argv_with_files(profile, input_path, output_path, prompt)
|
|
||||||
|
|
||||||
stdin_data = None
|
if profile.model_governance_required:
|
||||||
if profile.input_mode == "stdin_json":
|
if not MODEL_GOVERNANCE_ENABLED:
|
||||||
stdin_data = canonical(request["payload"]).decode("utf-8")
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_GOVERNANCE_DISABLED")
|
||||||
elif profile.input_mode == "stdin_text":
|
scope_type, tenant_id, project_id, trace_id = _scope_from_payload(request["payload"])
|
||||||
stdin_data = prompt
|
governance_store = ModelGovernanceStore(DEFAULT_MODEL_GOVERNANCE_DB)
|
||||||
|
prompt_estimated_input_tokens = _estimate_tokens(prompt)
|
||||||
child_env = {
|
reserved_estimated_input_tokens = max(
|
||||||
"PATH": "/usr/bin:/usr/local/bin:/bin",
|
prompt_estimated_input_tokens,
|
||||||
"LC_ALL": "C.UTF-8",
|
profile.model_input_reservation_floor_tokens,
|
||||||
"LANG": "C.UTF-8",
|
|
||||||
"HOME": str(temp_root / "home"),
|
|
||||||
"NO_COLOR": "1",
|
|
||||||
}
|
|
||||||
child_env.update(dict(profile.environment))
|
|
||||||
child_env.update(_credential_environment(profile))
|
|
||||||
|
|
||||||
started = time.monotonic()
|
|
||||||
try:
|
|
||||||
completed = subprocess.run(
|
|
||||||
argv,
|
|
||||||
input=stdin_data,
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
check=False,
|
|
||||||
shell=False,
|
|
||||||
timeout=profile.timeout_seconds,
|
|
||||||
cwd=str(temp_root),
|
|
||||||
env=child_env,
|
|
||||||
)
|
)
|
||||||
except subprocess.TimeoutExpired as exc:
|
route_request = RouteRequest(
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_TIMEOUT") from exc
|
request_id=request["request_id"] + ":model",
|
||||||
|
task_id=request["task_id"],
|
||||||
if completed.returncode != 0:
|
trace_id=trace_id,
|
||||||
raise ExternalAgentRuntimeError(
|
scope_type=scope_type,
|
||||||
f"EXTERNAL_AGENT_PROCESS_FAILED:{completed.returncode}:"
|
tenant_id=tenant_id,
|
||||||
+ _bounded_text(completed.stderr, 2048)
|
project_id=project_id,
|
||||||
|
required_capabilities=tuple(
|
||||||
|
sorted(
|
||||||
|
{
|
||||||
|
f"agent.runtime.{request['runtime']}",
|
||||||
|
*profile.model_capabilities,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
),
|
||||||
|
estimated_input_tokens=reserved_estimated_input_tokens,
|
||||||
|
max_output_tokens=profile.model_max_output_tokens,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
decision = governance_store.route(route_request)
|
||||||
|
except ModelGovernanceError as exc:
|
||||||
|
raise ExternalAgentRuntimeError(exc.code) from exc
|
||||||
|
if profile.model_provider_ids and decision.provider_id not in profile.model_provider_ids:
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_PROVIDER_PROFILE_DENY")
|
||||||
|
governance_attempts.append(
|
||||||
|
{
|
||||||
|
"request_id": decision.request_id,
|
||||||
|
"provider_id": decision.provider_id,
|
||||||
|
"model_id": decision.model_id,
|
||||||
|
"model_version": decision.model_version,
|
||||||
|
"fallback_index": decision.fallback_index,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
output_text, output_json = _parse_output(profile, completed.stdout, output_path)
|
|
||||||
result: dict[str, Any] = {
|
|
||||||
"schema": RESULT_SCHEMA,
|
|
||||||
"request_id": request["request_id"],
|
|
||||||
"request_sha256": sha256(canonical(request)),
|
|
||||||
"runtime": request["runtime"],
|
|
||||||
"task_id": request["task_id"],
|
|
||||||
"execution_digest": request["execution_digest"],
|
|
||||||
"exit_code": completed.returncode,
|
|
||||||
"duration_ms": int((time.monotonic() - started) * 1000),
|
|
||||||
"output_text": output_text,
|
|
||||||
"output_sha256": sha256(output_text.encode("utf-8", "replace")),
|
|
||||||
"stderr_sha256": sha256(completed.stderr.encode("utf-8", "replace")),
|
|
||||||
"credential_material_exposed": False,
|
|
||||||
"control_plane_access_granted": False,
|
|
||||||
}
|
|
||||||
if output_json is not None:
|
|
||||||
result["output_json"] = output_json
|
|
||||||
result["result_sha256"] = sha256(canonical(result))
|
|
||||||
result_path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
fd, name = tempfile.mkstemp(prefix=f".{result_path.name}.", dir=result_path.parent)
|
|
||||||
try:
|
try:
|
||||||
os.fchmod(fd, 0o640)
|
while True:
|
||||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
argv = _argv_with_files(
|
||||||
json.dump(result, handle, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
profile,
|
||||||
handle.write("\n")
|
input_path,
|
||||||
handle.flush()
|
output_path,
|
||||||
os.fsync(handle.fileno())
|
prompt,
|
||||||
os.replace(name, result_path)
|
decision,
|
||||||
|
)
|
||||||
|
stdin_data = None
|
||||||
|
if profile.input_mode == "stdin_json":
|
||||||
|
stdin_data = canonical(request["payload"]).decode("utf-8")
|
||||||
|
elif profile.input_mode == "stdin_text":
|
||||||
|
stdin_data = prompt
|
||||||
|
|
||||||
|
child_env = {
|
||||||
|
"PATH": "/usr/bin:/usr/local/bin:/bin",
|
||||||
|
"LC_ALL": "C.UTF-8",
|
||||||
|
"LANG": "C.UTF-8",
|
||||||
|
"HOME": str(temp_root / "home"),
|
||||||
|
"NO_COLOR": "1",
|
||||||
|
}
|
||||||
|
child_env.update(dict(_bind_model_values(profile.environment, decision)))
|
||||||
|
child_env.update(_credential_environment(profile))
|
||||||
|
|
||||||
|
started = time.monotonic()
|
||||||
|
failure_class = None
|
||||||
|
try:
|
||||||
|
completed = subprocess.run(
|
||||||
|
argv,
|
||||||
|
input=stdin_data,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
shell=False,
|
||||||
|
timeout=profile.timeout_seconds,
|
||||||
|
cwd=str(temp_root),
|
||||||
|
env=child_env,
|
||||||
|
)
|
||||||
|
if completed.returncode != 0:
|
||||||
|
failure_class = f"process_exit_{completed.returncode}"
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
completed = None
|
||||||
|
failure_class = "timeout"
|
||||||
|
|
||||||
|
if failure_class is None and completed is not None:
|
||||||
|
break
|
||||||
|
|
||||||
|
if governance_store is None or decision is None or route_request is None:
|
||||||
|
if failure_class == "timeout":
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_TIMEOUT")
|
||||||
|
raise ExternalAgentRuntimeError(
|
||||||
|
f"EXTERNAL_AGENT_PROCESS_FAILED:{completed.returncode}:"
|
||||||
|
+ _bounded_text(completed.stderr, 2048)
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
governance_store.record_failure(decision.request_id, failure_class)
|
||||||
|
decision = governance_store.fallback(route_request, decision.request_id)
|
||||||
|
except ModelGovernanceError as exc:
|
||||||
|
raise ExternalAgentRuntimeError(exc.code) from exc
|
||||||
|
if profile.model_provider_ids and decision.provider_id not in profile.model_provider_ids:
|
||||||
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_MODEL_PROVIDER_PROFILE_DENY")
|
||||||
|
governance_attempts.append(
|
||||||
|
{
|
||||||
|
"request_id": decision.request_id,
|
||||||
|
"provider_id": decision.provider_id,
|
||||||
|
"model_id": decision.model_id,
|
||||||
|
"model_version": decision.model_version,
|
||||||
|
"fallback_index": decision.fallback_index,
|
||||||
|
"previous_failure_class": failure_class,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
output_text, output_json = _parse_output(profile, completed.stdout, output_path)
|
||||||
|
model_governance: dict[str, Any] | None = None
|
||||||
|
if governance_store is not None and decision is not None:
|
||||||
|
input_tokens, output_tokens, actual_cost = _extract_runtime_usage(output_json)
|
||||||
|
try:
|
||||||
|
settlement = governance_store.record_success(
|
||||||
|
decision.request_id,
|
||||||
|
input_tokens=input_tokens,
|
||||||
|
output_tokens=output_tokens,
|
||||||
|
actual_cost_micros=actual_cost,
|
||||||
|
usage_source="runtime",
|
||||||
|
)
|
||||||
|
except ModelGovernanceError as exc:
|
||||||
|
raise ExternalAgentRuntimeError(exc.code) from exc
|
||||||
|
model_governance = {
|
||||||
|
"decision_id": decision.decision_id,
|
||||||
|
"request_id": decision.request_id,
|
||||||
|
"provider_id": decision.provider_id,
|
||||||
|
"model_id": decision.model_id,
|
||||||
|
"model_version": decision.model_version,
|
||||||
|
"policy_revision": decision.policy_revision,
|
||||||
|
"model_revision": decision.model_revision,
|
||||||
|
"fallback_index": decision.fallback_index,
|
||||||
|
"prompt_estimated_input_tokens": prompt_estimated_input_tokens,
|
||||||
|
"input_reservation_floor_tokens": (
|
||||||
|
profile.model_input_reservation_floor_tokens
|
||||||
|
),
|
||||||
|
"reserved_estimated_input_tokens": (
|
||||||
|
reserved_estimated_input_tokens
|
||||||
|
),
|
||||||
|
"settlement": settlement,
|
||||||
|
"attempts": governance_attempts,
|
||||||
|
"audit_chain_valid": governance_store.verify_audit_chain(),
|
||||||
|
}
|
||||||
|
|
||||||
|
result: dict[str, Any] = {
|
||||||
|
"schema": RESULT_SCHEMA,
|
||||||
|
"request_id": request["request_id"],
|
||||||
|
"request_sha256": sha256(canonical(request)),
|
||||||
|
"runtime": request["runtime"],
|
||||||
|
"task_id": request["task_id"],
|
||||||
|
"attempt": request["attempt"],
|
||||||
|
"execution_digest": request["execution_digest"],
|
||||||
|
"exit_code": completed.returncode,
|
||||||
|
"duration_ms": int((time.monotonic() - started) * 1000),
|
||||||
|
"output_text": output_text,
|
||||||
|
"output_sha256": sha256(output_text.encode("utf-8", "replace")),
|
||||||
|
"stderr_sha256": sha256(completed.stderr.encode("utf-8", "replace")),
|
||||||
|
"credential_material_exposed": False,
|
||||||
|
"control_plane_access_granted": False,
|
||||||
|
}
|
||||||
|
if output_json is not None:
|
||||||
|
result["output_json"] = output_json
|
||||||
|
if model_governance is not None:
|
||||||
|
result["model_governance"] = model_governance
|
||||||
|
result["result_sha256"] = sha256(canonical(result))
|
||||||
|
result_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, name = tempfile.mkstemp(prefix=f".{result_path.name}.", dir=result_path.parent)
|
||||||
|
try:
|
||||||
|
os.fchmod(fd, 0o640)
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(
|
||||||
|
result,
|
||||||
|
handle,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
)
|
||||||
|
handle.write("\n")
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
os.replace(name, result_path)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(name):
|
||||||
|
os.unlink(name)
|
||||||
finally:
|
finally:
|
||||||
if os.path.exists(name):
|
if governance_store is not None:
|
||||||
os.unlink(name)
|
governance_store.close()
|
||||||
|
|
||||||
|
|
||||||
class ExternalAgentExchange:
|
class ExternalAgentExchange:
|
||||||
@@ -566,6 +906,7 @@ class ExternalAgentExchange:
|
|||||||
request_id: str,
|
request_id: str,
|
||||||
runtime: str,
|
runtime: str,
|
||||||
task_id: str,
|
task_id: str,
|
||||||
|
attempt: int,
|
||||||
execution_digest: str,
|
execution_digest: str,
|
||||||
payload: Mapping[str, Any],
|
payload: Mapping[str, Any],
|
||||||
timeout_seconds: int = 1800,
|
timeout_seconds: int = 1800,
|
||||||
@@ -576,6 +917,7 @@ class ExternalAgentExchange:
|
|||||||
"request_id": request_id,
|
"request_id": request_id,
|
||||||
"runtime": runtime,
|
"runtime": runtime,
|
||||||
"task_id": task_id,
|
"task_id": task_id,
|
||||||
|
"attempt": attempt,
|
||||||
"execution_digest": execution_digest,
|
"execution_digest": execution_digest,
|
||||||
"payload": dict(payload),
|
"payload": dict(payload),
|
||||||
}
|
}
|
||||||
@@ -651,6 +993,7 @@ class ExternalAgentExchange:
|
|||||||
or value.get("request_sha256") != request_sha
|
or value.get("request_sha256") != request_sha
|
||||||
or value.get("runtime") != request["runtime"]
|
or value.get("runtime") != request["runtime"]
|
||||||
or value.get("task_id") != request["task_id"]
|
or value.get("task_id") != request["task_id"]
|
||||||
|
or value.get("attempt") != request["attempt"]
|
||||||
or value.get("execution_digest") != request["execution_digest"]
|
or value.get("execution_digest") != request["execution_digest"]
|
||||||
):
|
):
|
||||||
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_RESULT_BINDING_INVALID")
|
raise ExternalAgentRuntimeError("EXTERNAL_AGENT_RESULT_BINDING_INVALID")
|
||||||
|
|||||||
@@ -0,0 +1,456 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Disabled-by-default TLS edge for the read-only GPT Online Task ingress.
|
||||||
|
|
||||||
|
The internal AIOS ingress remains loopback-only. This service is a separate edge
|
||||||
|
that may bind non-loopback only when the owner explicitly approves it. It
|
||||||
|
forwards only the tiny read-only ingress API and never proxies the canonical
|
||||||
|
Bridge or Task API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import defaultdict, deque
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import secrets
|
||||||
|
import ssl
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Mapping
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
UPSTREAM = "http://127.0.0.1:18945"
|
||||||
|
MAX_BODY = 64 * 1024
|
||||||
|
MAX_RESPONSE = 256 * 1024
|
||||||
|
MAX_CONCURRENT_REQUESTS = 32
|
||||||
|
REQUEST_TIMEOUT_SECONDS = 20
|
||||||
|
RATE_WINDOW_SECONDS = 60
|
||||||
|
RATE_LIMIT_REQUESTS = 60
|
||||||
|
RATE_BURST_REQUESTS = 15
|
||||||
|
RATE_MAX_CLIENTS = 4096
|
||||||
|
_AUDIT_SALT = secrets.token_bytes(32)
|
||||||
|
PUBLIC_APPROVAL_ENV = "AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED"
|
||||||
|
PUBLIC_DOMAIN_ENV = "AIOS_GPT_EDGE_PUBLIC_DOMAIN"
|
||||||
|
TASK_PATH_RE = re.compile(r"^/v1/read-only-tasks(?:/[A-Za-z0-9][A-Za-z0-9._:-]{7,127})?$")
|
||||||
|
|
||||||
|
|
||||||
|
def _client_pseudonym(value: str) -> str:
|
||||||
|
return hashlib.sha256(_AUDIT_SALT + value.encode("utf-8", "replace")).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def audit(event: str, **fields: object) -> None:
|
||||||
|
safe = {
|
||||||
|
"event": event,
|
||||||
|
"ts_unix_ms": int(time.time() * 1000),
|
||||||
|
}
|
||||||
|
for key, value in fields.items():
|
||||||
|
if key in {"authorization", "body", "token", "cookie", "client_ip"}:
|
||||||
|
continue
|
||||||
|
safe[key] = value
|
||||||
|
print(
|
||||||
|
json.dumps(safe, sort_keys=True, separators=(",", ":")),
|
||||||
|
file=sys.stderr,
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class EdgeError(RuntimeError):
|
||||||
|
def __init__(self, code: str, status: int = 400) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.status = status
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def public_bind_allowed(host: str, approval: str) -> str:
|
||||||
|
value = host.strip()
|
||||||
|
if value in {"127.0.0.1", "::1", "localhost"}:
|
||||||
|
return value
|
||||||
|
if approval != "1":
|
||||||
|
raise RuntimeError("GPT_EDGE_PUBLIC_BIND_OWNER_APPROVAL_REQUIRED")
|
||||||
|
if value not in {"0.0.0.0", "::"}:
|
||||||
|
raise RuntimeError("GPT_EDGE_BIND_ADDRESS_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def validated_public_domain(value: str) -> str:
|
||||||
|
domain = value.strip().lower()
|
||||||
|
if not domain:
|
||||||
|
return ""
|
||||||
|
if len(domain) > 253 or re.fullmatch(
|
||||||
|
r"(?=.{1,253}\Z)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?",
|
||||||
|
domain,
|
||||||
|
) is None:
|
||||||
|
raise RuntimeError("GPT_EDGE_PUBLIC_DOMAIN_INVALID")
|
||||||
|
return domain
|
||||||
|
|
||||||
|
|
||||||
|
def validated_edge_identity(host: str, approval: str, domain: str) -> tuple[str, str]:
|
||||||
|
bind_host = public_bind_allowed(host, approval)
|
||||||
|
public_domain = validated_public_domain(domain)
|
||||||
|
if bind_host not in {"127.0.0.1", "::1", "localhost"} and not public_domain:
|
||||||
|
raise RuntimeError("GPT_EDGE_PUBLIC_DOMAIN_REQUIRED")
|
||||||
|
return bind_host, public_domain
|
||||||
|
|
||||||
|
|
||||||
|
def validate_tls_file(path: Path, *, key: bool) -> Path:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise RuntimeError("GPT_EDGE_TLS_FILE_INVALID")
|
||||||
|
size = path.stat().st_size
|
||||||
|
if size < 16 or size > 256 * 1024:
|
||||||
|
raise RuntimeError("GPT_EDGE_TLS_FILE_INVALID")
|
||||||
|
if key and (path.stat().st_mode & 0o077):
|
||||||
|
raise RuntimeError("GPT_EDGE_TLS_KEY_MODE_INVALID")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def build_tls_context(cert_path: Path, key_path: Path) -> ssl.SSLContext:
|
||||||
|
cert = validate_tls_file(cert_path, key=False)
|
||||||
|
key = validate_tls_file(key_path, key=True)
|
||||||
|
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||||
|
context.minimum_version = ssl.TLSVersion.TLSv1_2
|
||||||
|
context.options |= ssl.OP_NO_COMPRESSION
|
||||||
|
context.load_cert_chain(certfile=str(cert), keyfile=str(key))
|
||||||
|
return context
|
||||||
|
|
||||||
|
|
||||||
|
def allowed_route(method: str, raw_path: str) -> str:
|
||||||
|
parsed = urlsplit(raw_path)
|
||||||
|
if parsed.query or parsed.fragment:
|
||||||
|
raise EdgeError("QUERY_OR_FRAGMENT_DENY", 400)
|
||||||
|
path = parsed.path
|
||||||
|
if method == "GET" and path in {"/health", "/openapi.json"}:
|
||||||
|
return path
|
||||||
|
if method == "POST" and path == "/v1/read-only-tasks":
|
||||||
|
return path
|
||||||
|
if method == "GET" and TASK_PATH_RE.fullmatch(path):
|
||||||
|
return path
|
||||||
|
raise EdgeError("ROUTE_DENY", 404)
|
||||||
|
|
||||||
|
|
||||||
|
def route_label(method: str, path: str) -> str:
|
||||||
|
if method == "GET" and path == "/health":
|
||||||
|
return "health"
|
||||||
|
if method == "GET" and path == "/openapi.json":
|
||||||
|
return "openapi"
|
||||||
|
if method == "POST" and path == "/v1/read-only-tasks":
|
||||||
|
return "task_create"
|
||||||
|
if method == "GET" and path.startswith("/v1/read-only-tasks/"):
|
||||||
|
return "task_status"
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def forwarded_headers(headers: Mapping[str, str], *, body: bool) -> dict[str, str]:
|
||||||
|
result = {"Accept": "application/json"}
|
||||||
|
authorization = headers.get("Authorization")
|
||||||
|
if not isinstance(authorization, str) or not authorization.startswith("Bearer "):
|
||||||
|
raise EdgeError("AUTHORIZATION_REQUIRED", 401)
|
||||||
|
token = authorization[7:]
|
||||||
|
if len(token) < 32 or len(token) > 4096 or any(ch.isspace() for ch in token):
|
||||||
|
raise EdgeError("AUTHORIZATION_INVALID", 401)
|
||||||
|
result["Authorization"] = authorization
|
||||||
|
if body:
|
||||||
|
content_type = headers.get("Content-Type", "")
|
||||||
|
if content_type.split(";", 1)[0].strip().lower() != "application/json":
|
||||||
|
raise EdgeError("CONTENT_TYPE_INVALID", 415)
|
||||||
|
result["Content-Type"] = "application/json"
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
class RateLimiter:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
window_seconds: int = RATE_WINDOW_SECONDS,
|
||||||
|
limit: int = RATE_LIMIT_REQUESTS,
|
||||||
|
burst: int = RATE_BURST_REQUESTS,
|
||||||
|
max_clients: int = RATE_MAX_CLIENTS,
|
||||||
|
now_fn=time.monotonic,
|
||||||
|
) -> None:
|
||||||
|
self.window_seconds = window_seconds
|
||||||
|
self.limit = limit
|
||||||
|
self.burst = burst
|
||||||
|
self.max_clients = max_clients
|
||||||
|
self.now_fn = now_fn
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._events: dict[str, deque[float]] = defaultdict(deque)
|
||||||
|
|
||||||
|
def _prune_stale(self, cutoff: float) -> None:
|
||||||
|
stale = [
|
||||||
|
key
|
||||||
|
for key, events in self._events.items()
|
||||||
|
if not events or events[-1] <= cutoff
|
||||||
|
]
|
||||||
|
for key in stale:
|
||||||
|
self._events.pop(key, None)
|
||||||
|
|
||||||
|
def allowed(self, key: str) -> bool:
|
||||||
|
now = self.now_fn()
|
||||||
|
cutoff = now - self.window_seconds
|
||||||
|
with self._lock:
|
||||||
|
if key not in self._events and len(self._events) >= self.max_clients:
|
||||||
|
self._prune_stale(cutoff)
|
||||||
|
if len(self._events) >= self.max_clients:
|
||||||
|
return False
|
||||||
|
events = self._events[key]
|
||||||
|
while events and events[0] <= cutoff:
|
||||||
|
events.popleft()
|
||||||
|
if len(events) >= self.limit + self.burst:
|
||||||
|
return False
|
||||||
|
events.append(now)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Proxy:
|
||||||
|
upstream: str = UPSTREAM
|
||||||
|
timeout: int = REQUEST_TIMEOUT_SECONDS
|
||||||
|
|
||||||
|
def request(
|
||||||
|
self,
|
||||||
|
method: str,
|
||||||
|
path: str,
|
||||||
|
headers: Mapping[str, str],
|
||||||
|
body: bytes | None,
|
||||||
|
) -> tuple[int, bytes, str]:
|
||||||
|
if self.upstream != UPSTREAM:
|
||||||
|
raise EdgeError("UPSTREAM_DRIFT_DENY", 503)
|
||||||
|
req = urllib.request.Request(
|
||||||
|
self.upstream + path,
|
||||||
|
method=method,
|
||||||
|
data=body,
|
||||||
|
headers=dict(headers),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=self.timeout) as response:
|
||||||
|
raw = response.read(MAX_RESPONSE + 1)
|
||||||
|
status = response.status
|
||||||
|
content_type = response.headers.get("Content-Type", "application/json")
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raw = exc.read(MAX_RESPONSE + 1)
|
||||||
|
status = exc.code
|
||||||
|
content_type = exc.headers.get("Content-Type", "application/json")
|
||||||
|
except Exception as exc:
|
||||||
|
raise EdgeError("UPSTREAM_UNAVAILABLE", 502) from exc
|
||||||
|
if len(raw) > MAX_RESPONSE:
|
||||||
|
raise EdgeError("UPSTREAM_RESPONSE_TOO_LARGE", 502)
|
||||||
|
if content_type.split(";", 1)[0].strip().lower() != "application/json":
|
||||||
|
raise EdgeError("UPSTREAM_CONTENT_TYPE_INVALID", 502)
|
||||||
|
return status, raw, "application/json"
|
||||||
|
|
||||||
|
|
||||||
|
class EdgeServer(ThreadingHTTPServer):
|
||||||
|
daemon_threads = True
|
||||||
|
request_queue_size = 64
|
||||||
|
allow_reuse_address = True
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
proxy = Proxy()
|
||||||
|
limiter = RateLimiter()
|
||||||
|
concurrency = threading.BoundedSemaphore(MAX_CONCURRENT_REQUESTS)
|
||||||
|
expected_domain = ""
|
||||||
|
|
||||||
|
server_version = "AIOSGPTIngressEdge/1.0"
|
||||||
|
sys_version = ""
|
||||||
|
|
||||||
|
def log_message(self, *_args: Any) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def respond_json(self, value: Mapping[str, Any], status: int) -> None:
|
||||||
|
raw = canonical(value)
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.send_header("X-Content-Type-Options", "nosniff")
|
||||||
|
self.send_header("Referrer-Policy", "no-referrer")
|
||||||
|
self.send_header("Content-Security-Policy", "default-src 'none'")
|
||||||
|
self.send_header("Strict-Transport-Security", "max-age=31536000")
|
||||||
|
self.send_header("Content-Length", str(len(raw)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(raw)
|
||||||
|
|
||||||
|
def send_proxy_response(self, status: int, raw: bytes, content_type: str) -> None:
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", content_type)
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.send_header("X-Content-Type-Options", "nosniff")
|
||||||
|
self.send_header("Referrer-Policy", "no-referrer")
|
||||||
|
self.send_header("Content-Security-Policy", "default-src 'none'")
|
||||||
|
self.send_header("Strict-Transport-Security", "max-age=31536000")
|
||||||
|
self.send_header("Content-Length", str(len(raw)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(raw)
|
||||||
|
|
||||||
|
def client_key(self) -> str:
|
||||||
|
return str(self.client_address[0])
|
||||||
|
|
||||||
|
def validate_host(self) -> None:
|
||||||
|
if not self.expected_domain:
|
||||||
|
return
|
||||||
|
host = self.headers.get("Host", "").split(":", 1)[0].lower()
|
||||||
|
if not hmac.compare_digest(host, self.expected_domain):
|
||||||
|
raise EdgeError("HOST_HEADER_DENY", 421)
|
||||||
|
|
||||||
|
def body(self) -> bytes:
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get("Content-Length", "-1"))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise EdgeError("CONTENT_LENGTH_INVALID") from exc
|
||||||
|
if length < 1 or length > MAX_BODY:
|
||||||
|
raise EdgeError("REQUEST_SIZE_INVALID", 413)
|
||||||
|
raw = self.rfile.read(length)
|
||||||
|
if len(raw) != length:
|
||||||
|
raise EdgeError("REQUEST_BODY_TRUNCATED", 400)
|
||||||
|
# Validate JSON at the edge so malformed bodies do not consume upstream work.
|
||||||
|
try:
|
||||||
|
value = json.loads(raw.decode("utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
raise EdgeError("REQUEST_JSON_INVALID", 400) from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise EdgeError("REQUEST_JSON_OBJECT_REQUIRED", 400)
|
||||||
|
return canonical(value)
|
||||||
|
|
||||||
|
def handle_proxy(self, method: str) -> None:
|
||||||
|
client_id = _client_pseudonym(self.client_key())
|
||||||
|
started = time.monotonic()
|
||||||
|
acquired = self.concurrency.acquire(blocking=False)
|
||||||
|
if not acquired:
|
||||||
|
audit("EDGE_BUSY", client_id=client_id, method=method)
|
||||||
|
self.respond_json({"error": "EDGE_BUSY"}, 503)
|
||||||
|
return
|
||||||
|
label = "unknown"
|
||||||
|
try:
|
||||||
|
self.validate_host()
|
||||||
|
path = allowed_route(method, self.path)
|
||||||
|
label = route_label(method, path)
|
||||||
|
if path == "/health" and method == "GET":
|
||||||
|
self.respond_json(
|
||||||
|
{
|
||||||
|
"status": "ok",
|
||||||
|
"service": "aios-gpt-https-edge",
|
||||||
|
"upstream": "loopback-readonly-ingress",
|
||||||
|
},
|
||||||
|
200,
|
||||||
|
)
|
||||||
|
audit(
|
||||||
|
"EDGE_REQUEST",
|
||||||
|
client_id=client_id,
|
||||||
|
method=method,
|
||||||
|
route=label,
|
||||||
|
status=200,
|
||||||
|
duration_ms=int((time.monotonic() - started) * 1000),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if not self.limiter.allowed(self.client_key()):
|
||||||
|
audit(
|
||||||
|
"EDGE_RATE_LIMIT",
|
||||||
|
client_id=client_id,
|
||||||
|
method=method,
|
||||||
|
route=label,
|
||||||
|
)
|
||||||
|
self.respond_json({"error": "RATE_LIMITED"}, 429)
|
||||||
|
return
|
||||||
|
has_body = method == "POST"
|
||||||
|
headers = forwarded_headers(self.headers, body=has_body)
|
||||||
|
body = self.body() if has_body else None
|
||||||
|
status, raw, content_type = self.proxy.request(
|
||||||
|
method,
|
||||||
|
path,
|
||||||
|
headers,
|
||||||
|
body,
|
||||||
|
)
|
||||||
|
self.send_proxy_response(status, raw, content_type)
|
||||||
|
audit(
|
||||||
|
"EDGE_REQUEST",
|
||||||
|
client_id=client_id,
|
||||||
|
method=method,
|
||||||
|
route=label,
|
||||||
|
status=status,
|
||||||
|
duration_ms=int((time.monotonic() - started) * 1000),
|
||||||
|
)
|
||||||
|
except EdgeError as exc:
|
||||||
|
audit(
|
||||||
|
"EDGE_REJECT",
|
||||||
|
client_id=client_id,
|
||||||
|
method=method,
|
||||||
|
route=label,
|
||||||
|
status=exc.status,
|
||||||
|
error=exc.code,
|
||||||
|
)
|
||||||
|
self.respond_json({"error": exc.code}, exc.status)
|
||||||
|
finally:
|
||||||
|
self.concurrency.release()
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
self.handle_proxy("GET")
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
self.handle_proxy("POST")
|
||||||
|
|
||||||
|
def do_PUT(self) -> None:
|
||||||
|
self.respond_json({"error": "METHOD_DENY"}, 405)
|
||||||
|
|
||||||
|
def do_PATCH(self) -> None:
|
||||||
|
self.respond_json({"error": "METHOD_DENY"}, 405)
|
||||||
|
|
||||||
|
def do_DELETE(self) -> None:
|
||||||
|
self.respond_json({"error": "METHOD_DENY"}, 405)
|
||||||
|
|
||||||
|
def do_CONNECT(self) -> None:
|
||||||
|
self.respond_json({"error": "METHOD_DENY"}, 405)
|
||||||
|
|
||||||
|
def do_OPTIONS(self) -> None:
|
||||||
|
self.respond_json({"error": "METHOD_DENY"}, 405)
|
||||||
|
|
||||||
|
|
||||||
|
def serve() -> None:
|
||||||
|
host, public_domain = validated_edge_identity(
|
||||||
|
os.environ.get("AIOS_GPT_EDGE_HOST", "127.0.0.1"),
|
||||||
|
os.environ.get(PUBLIC_APPROVAL_ENV, "0"),
|
||||||
|
os.environ.get(PUBLIC_DOMAIN_ENV, ""),
|
||||||
|
)
|
||||||
|
port = int(os.environ.get("AIOS_GPT_EDGE_PORT", "18443"))
|
||||||
|
if not 1024 <= port <= 65535:
|
||||||
|
raise RuntimeError("GPT_EDGE_PORT_INVALID")
|
||||||
|
Handler.expected_domain = public_domain
|
||||||
|
|
||||||
|
credentials = Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
||||||
|
cert_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_GPT_EDGE_TLS_CERT_FILE",
|
||||||
|
str(credentials / "tls_cert"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
key_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_GPT_EDGE_TLS_KEY_FILE",
|
||||||
|
str(credentials / "tls_key"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
context = build_tls_context(cert_path, key_path)
|
||||||
|
|
||||||
|
server = EdgeServer((host, port), Handler)
|
||||||
|
server.socket = context.wrap_socket(server.socket, server_side=True)
|
||||||
|
server.serve_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
serve()
|
||||||
@@ -0,0 +1,561 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Owner-approved activation/rollback controller for the AIOS GPT HTTPS edge.
|
||||||
|
|
||||||
|
This controller never creates approval, TLS, or ingress credentials. It consumes
|
||||||
|
owner-managed inputs, applies one exact public-bind overlay, verifies the live
|
||||||
|
edge, and provides an idempotent rollback path.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import socket
|
||||||
|
import ssl
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
APPROVAL_SCHEMA = "aios.gpt-https-edge.public-approval.v1"
|
||||||
|
STATE_SCHEMA = "aios.gpt-https-edge.activation-state.v1"
|
||||||
|
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
SHA_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
APPROVAL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$")
|
||||||
|
DOMAIN_RE = re.compile(
|
||||||
|
r"(?=.{1,253}\Z)"
|
||||||
|
r"(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+"
|
||||||
|
r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?"
|
||||||
|
)
|
||||||
|
|
||||||
|
EDGE_UNIT = "aios-gpt-https-edge.service"
|
||||||
|
INGRESS_UNIT = "aios-gpt-online-task-ingress.service"
|
||||||
|
PUBLIC_HOST = "0.0.0.0"
|
||||||
|
PUBLIC_PORT = 18443
|
||||||
|
INGRESS_HEALTH = "http://127.0.0.1:18945/health"
|
||||||
|
DROPIN_NAME = "zzzzzzzzzz-owner-approved-public.conf"
|
||||||
|
|
||||||
|
|
||||||
|
class ActivationError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_bytes(data: bytes) -> str:
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_file(path: Path) -> str:
|
||||||
|
return sha256_bytes(path.read_bytes())
|
||||||
|
|
||||||
|
|
||||||
|
def parse_time(value: object, *, field: str) -> datetime:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
raise ActivationError("APPROVAL_" + field.upper() + "_INVALID")
|
||||||
|
text = value.strip()
|
||||||
|
if text.endswith("Z"):
|
||||||
|
text = text[:-1] + "+00:00"
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(text)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise ActivationError("APPROVAL_" + field.upper() + "_INVALID") from exc
|
||||||
|
if parsed.tzinfo is None:
|
||||||
|
raise ActivationError("APPROVAL_" + field.upper() + "_INVALID")
|
||||||
|
return parsed.astimezone(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_domain(value: object) -> str:
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise ActivationError("APPROVAL_DOMAIN_INVALID")
|
||||||
|
domain = value.strip().lower()
|
||||||
|
if DOMAIN_RE.fullmatch(domain) is None:
|
||||||
|
raise ActivationError("APPROVAL_DOMAIN_INVALID")
|
||||||
|
return domain
|
||||||
|
|
||||||
|
|
||||||
|
def validate_root_file(
|
||||||
|
path: Path,
|
||||||
|
*,
|
||||||
|
exact_mode: int | None = None,
|
||||||
|
max_size: int = 256 * 1024,
|
||||||
|
) -> Path:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise ActivationError("OWNER_FILE_INVALID")
|
||||||
|
info = path.stat()
|
||||||
|
if info.st_uid != 0 or info.st_gid != 0:
|
||||||
|
raise ActivationError("OWNER_FILE_OWNERSHIP_INVALID")
|
||||||
|
mode = stat.S_IMODE(info.st_mode)
|
||||||
|
if exact_mode is not None and mode != exact_mode:
|
||||||
|
raise ActivationError("OWNER_FILE_MODE_INVALID")
|
||||||
|
if info.st_size < 16 or info.st_size > max_size:
|
||||||
|
raise ActivationError("OWNER_FILE_SIZE_INVALID")
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def certificate_dns_names(cert_path: Path) -> tuple[str, ...]:
|
||||||
|
try:
|
||||||
|
decoded = ssl._ssl._test_decode_cert(str(cert_path))
|
||||||
|
except Exception as exc:
|
||||||
|
raise ActivationError("TLS_CERT_DECODE_INVALID") from exc
|
||||||
|
names = [
|
||||||
|
value
|
||||||
|
for kind, value in decoded.get("subjectAltName", ())
|
||||||
|
if kind == "DNS"
|
||||||
|
]
|
||||||
|
if not names:
|
||||||
|
for rdn in decoded.get("subject", ()):
|
||||||
|
for key, value in rdn:
|
||||||
|
if key == "commonName":
|
||||||
|
names.append(value)
|
||||||
|
return tuple(names)
|
||||||
|
|
||||||
|
|
||||||
|
def cert_matches_domain(cert_path: Path, domain: str) -> bool:
|
||||||
|
names = certificate_dns_names(cert_path)
|
||||||
|
try:
|
||||||
|
return bool(names) and any(ssl._dnsname_match(name, domain) for name in names)
|
||||||
|
except ssl.CertificateError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def validate_tls_pair(
|
||||||
|
cert_path: Path,
|
||||||
|
key_path: Path,
|
||||||
|
domain: str,
|
||||||
|
*,
|
||||||
|
now_fn=time.time,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
validate_root_file(cert_path)
|
||||||
|
validate_root_file(key_path, exact_mode=0o600)
|
||||||
|
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||||
|
context.minimum_version = ssl.TLSVersion.TLSv1_2
|
||||||
|
context.options |= ssl.OP_NO_COMPRESSION
|
||||||
|
try:
|
||||||
|
context.load_cert_chain(str(cert_path), str(key_path))
|
||||||
|
except Exception as exc:
|
||||||
|
raise ActivationError("TLS_CERT_KEY_PAIR_INVALID") from exc
|
||||||
|
if not cert_matches_domain(cert_path, domain):
|
||||||
|
raise ActivationError("TLS_CERT_DOMAIN_MISMATCH")
|
||||||
|
decoded = ssl._ssl._test_decode_cert(str(cert_path))
|
||||||
|
not_after = decoded.get("notAfter")
|
||||||
|
if not not_after:
|
||||||
|
raise ActivationError("TLS_CERT_EXPIRY_MISSING")
|
||||||
|
remaining = ssl.cert_time_to_seconds(not_after) - now_fn()
|
||||||
|
if remaining < 7 * 86400:
|
||||||
|
raise ActivationError("TLS_CERT_EXPIRY_TOO_CLOSE")
|
||||||
|
return {
|
||||||
|
"cert_sha256": sha256_file(cert_path),
|
||||||
|
"remaining_days": int(remaining // 86400),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Approval:
|
||||||
|
approval_id: str
|
||||||
|
candidate_oid: str
|
||||||
|
domain: str
|
||||||
|
bind_host: str
|
||||||
|
port: int
|
||||||
|
tls_cert_sha256: str
|
||||||
|
owner: str
|
||||||
|
issued_at: datetime
|
||||||
|
not_before: datetime
|
||||||
|
expires_at: datetime
|
||||||
|
raw_sha256: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load(
|
||||||
|
cls,
|
||||||
|
path: Path,
|
||||||
|
*,
|
||||||
|
candidate_oid: str,
|
||||||
|
cert_path: Path,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> "Approval":
|
||||||
|
validate_root_file(path, exact_mode=0o600, max_size=64 * 1024)
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
raise ActivationError("APPROVAL_JSON_INVALID") from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise ActivationError("APPROVAL_JSON_INVALID")
|
||||||
|
allowed = {
|
||||||
|
"schema",
|
||||||
|
"approval_id",
|
||||||
|
"candidate_oid",
|
||||||
|
"domain",
|
||||||
|
"public_bind",
|
||||||
|
"bind_host",
|
||||||
|
"port",
|
||||||
|
"tls_cert_sha256",
|
||||||
|
"issued_at",
|
||||||
|
"not_before",
|
||||||
|
"expires_at",
|
||||||
|
"owner",
|
||||||
|
}
|
||||||
|
if set(value) != allowed or value.get("schema") != APPROVAL_SCHEMA:
|
||||||
|
raise ActivationError("APPROVAL_FIELDS_INVALID")
|
||||||
|
|
||||||
|
approval_id = value.get("approval_id")
|
||||||
|
if not isinstance(approval_id, str) or APPROVAL_ID_RE.fullmatch(approval_id) is None:
|
||||||
|
raise ActivationError("APPROVAL_ID_INVALID")
|
||||||
|
if not OID_RE.fullmatch(str(value.get("candidate_oid") or "")):
|
||||||
|
raise ActivationError("APPROVAL_CANDIDATE_INVALID")
|
||||||
|
if value["candidate_oid"] != candidate_oid:
|
||||||
|
raise ActivationError("APPROVAL_CANDIDATE_MISMATCH")
|
||||||
|
if value.get("public_bind") is not True:
|
||||||
|
raise ActivationError("APPROVAL_PUBLIC_BIND_REQUIRED")
|
||||||
|
if value.get("bind_host") != PUBLIC_HOST or value.get("port") != PUBLIC_PORT:
|
||||||
|
raise ActivationError("APPROVAL_BINDING_INVALID")
|
||||||
|
|
||||||
|
cert_sha = str(value.get("tls_cert_sha256") or "")
|
||||||
|
if SHA_RE.fullmatch(cert_sha) is None:
|
||||||
|
raise ActivationError("APPROVAL_CERT_SHA_INVALID")
|
||||||
|
if not cert_path.is_file() or sha256_file(cert_path) != cert_sha:
|
||||||
|
raise ActivationError("APPROVAL_CERT_SHA_MISMATCH")
|
||||||
|
|
||||||
|
domain = validate_domain(value.get("domain"))
|
||||||
|
owner = value.get("owner")
|
||||||
|
if not isinstance(owner, str) or not owner.strip() or len(owner) > 255:
|
||||||
|
raise ActivationError("APPROVAL_OWNER_INVALID")
|
||||||
|
|
||||||
|
issued_at = parse_time(value.get("issued_at"), field="issued_at")
|
||||||
|
not_before = parse_time(value.get("not_before"), field="not_before")
|
||||||
|
expires_at = parse_time(value.get("expires_at"), field="expires_at")
|
||||||
|
current = now or datetime.now(timezone.utc)
|
||||||
|
if not (issued_at <= not_before <= expires_at):
|
||||||
|
raise ActivationError("APPROVAL_TIME_ORDER_INVALID")
|
||||||
|
if current < not_before:
|
||||||
|
raise ActivationError("APPROVAL_NOT_YET_VALID")
|
||||||
|
if current >= expires_at:
|
||||||
|
raise ActivationError("APPROVAL_EXPIRED")
|
||||||
|
if expires_at - issued_at > __import__("datetime").timedelta(days=7):
|
||||||
|
raise ActivationError("APPROVAL_VALIDITY_TOO_LONG")
|
||||||
|
|
||||||
|
return cls(
|
||||||
|
approval_id=approval_id,
|
||||||
|
candidate_oid=candidate_oid,
|
||||||
|
domain=domain,
|
||||||
|
bind_host=PUBLIC_HOST,
|
||||||
|
port=PUBLIC_PORT,
|
||||||
|
tls_cert_sha256=cert_sha,
|
||||||
|
owner=owner.strip(),
|
||||||
|
issued_at=issued_at,
|
||||||
|
not_before=not_before,
|
||||||
|
expires_at=expires_at,
|
||||||
|
raw_sha256=sha256_file(path),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_release(release_root: Path) -> dict[str, Path]:
|
||||||
|
if release_root.is_symlink() or not release_root.is_dir():
|
||||||
|
raise ActivationError("IMMUTABLE_RELEASE_INVALID")
|
||||||
|
edge = release_root / "active-runtime/app/aios_gpt_https_edge.py"
|
||||||
|
controller = release_root / "active-runtime/app/aios_gpt_https_edge_activation.py"
|
||||||
|
for path in (edge, controller):
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise ActivationError("IMMUTABLE_RELEASE_INVALID")
|
||||||
|
return {"edge": edge, "controller": controller}
|
||||||
|
|
||||||
|
|
||||||
|
def render_dropin(approval: Approval, release_root: Path) -> str:
|
||||||
|
app = release_root / "active-runtime/app/aios_gpt_https_edge.py"
|
||||||
|
return (
|
||||||
|
"[Service]\n"
|
||||||
|
f"Environment=AIOS_GPT_EDGE_HOST={approval.bind_host}\n"
|
||||||
|
f"Environment=AIOS_GPT_EDGE_PORT={approval.port}\n"
|
||||||
|
"Environment=AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=1\n"
|
||||||
|
f"Environment=AIOS_GPT_EDGE_PUBLIC_DOMAIN={approval.domain}\n"
|
||||||
|
"ExecStart=\n"
|
||||||
|
f"ExecStart=/usr/bin/python3 -B {app}\n"
|
||||||
|
f"ReadOnlyPaths={release_root}\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def atomic_write(path: Path, data: str, mode: int = 0o600) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, name = tempfile.mkstemp(prefix="." + path.name + ".", dir=path.parent)
|
||||||
|
try:
|
||||||
|
os.fchmod(fd, mode)
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
handle.write(data)
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
os.replace(name, path)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(name):
|
||||||
|
os.unlink(name)
|
||||||
|
|
||||||
|
|
||||||
|
class Systemd:
|
||||||
|
def run(self, *args: str, check: bool = True) -> subprocess.CompletedProcess[str]:
|
||||||
|
return subprocess.run(
|
||||||
|
["/usr/bin/systemctl", *args],
|
||||||
|
check=check,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
|
||||||
|
def active(self, unit: str) -> bool:
|
||||||
|
return self.run("is-active", "--quiet", unit, check=False).returncode == 0
|
||||||
|
|
||||||
|
def enabled(self, unit: str) -> bool:
|
||||||
|
return self.run("is-enabled", "--quiet", unit, check=False).returncode == 0
|
||||||
|
|
||||||
|
def show(self, unit: str, prop: str) -> str:
|
||||||
|
return self.run("show", unit, f"--property={prop}", "--value").stdout.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def require_ingress_ready(systemd: Systemd) -> None:
|
||||||
|
if not systemd.active(INGRESS_UNIT):
|
||||||
|
raise ActivationError("INGRESS_SERVICE_NOT_ACTIVE")
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(INGRESS_HEALTH, timeout=5) as response:
|
||||||
|
if response.status != 200:
|
||||||
|
raise ActivationError("INGRESS_HEALTH_FAILED")
|
||||||
|
value = json.loads(response.read().decode())
|
||||||
|
except ActivationError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
raise ActivationError("INGRESS_HEALTH_FAILED") from exc
|
||||||
|
if value.get("status") != "ok":
|
||||||
|
raise ActivationError("INGRESS_HEALTH_FAILED")
|
||||||
|
|
||||||
|
|
||||||
|
def tls_probe(domain: str, cert_path: Path, *, port: int = PUBLIC_PORT) -> None:
|
||||||
|
context = ssl.create_default_context(cafile=str(cert_path))
|
||||||
|
context.check_hostname = True
|
||||||
|
context.verify_mode = ssl.CERT_REQUIRED
|
||||||
|
try:
|
||||||
|
with socket.create_connection(("127.0.0.1", port), timeout=5) as sock:
|
||||||
|
with context.wrap_socket(sock, server_hostname=domain) as tls_sock:
|
||||||
|
if tls_sock.version() not in {"TLSv1.2", "TLSv1.3"}:
|
||||||
|
raise ActivationError("EDGE_TLS_VERSION_INVALID")
|
||||||
|
except ActivationError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
raise ActivationError("EDGE_TLS_PROBE_FAILED") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def receipt_for(
|
||||||
|
approval: Approval,
|
||||||
|
release_root: Path,
|
||||||
|
*,
|
||||||
|
state: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"schema": STATE_SCHEMA,
|
||||||
|
"state": state,
|
||||||
|
"approval_id": approval.approval_id,
|
||||||
|
"approval_sha256": approval.raw_sha256,
|
||||||
|
"candidate_oid": approval.candidate_oid,
|
||||||
|
"domain": approval.domain,
|
||||||
|
"bind_host": approval.bind_host,
|
||||||
|
"port": approval.port,
|
||||||
|
"tls_cert_sha256": approval.tls_cert_sha256,
|
||||||
|
"release_root": str(release_root),
|
||||||
|
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def activate(
|
||||||
|
*,
|
||||||
|
approval: Approval,
|
||||||
|
release_root: Path,
|
||||||
|
cert_path: Path,
|
||||||
|
key_path: Path,
|
||||||
|
dropin_path: Path,
|
||||||
|
state_path: Path,
|
||||||
|
systemd: Systemd,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if os.geteuid() != 0:
|
||||||
|
raise ActivationError("ROOT_REQUIRED")
|
||||||
|
release = validate_release(release_root)
|
||||||
|
app = release["edge"]
|
||||||
|
require_ingress_ready(systemd)
|
||||||
|
validate_tls_pair(cert_path, key_path, approval.domain)
|
||||||
|
|
||||||
|
if state_path.is_file():
|
||||||
|
try:
|
||||||
|
existing = json.loads(state_path.read_text(encoding="utf-8"))
|
||||||
|
except Exception:
|
||||||
|
existing = {}
|
||||||
|
if (
|
||||||
|
existing.get("schema") == STATE_SCHEMA
|
||||||
|
and existing.get("state") == "ACTIVE"
|
||||||
|
and existing.get("approval_sha256") == approval.raw_sha256
|
||||||
|
and existing.get("candidate_oid") == approval.candidate_oid
|
||||||
|
and systemd.active(EDGE_UNIT)
|
||||||
|
):
|
||||||
|
return existing
|
||||||
|
|
||||||
|
dropin = render_dropin(approval, release_root)
|
||||||
|
modified = False
|
||||||
|
try:
|
||||||
|
atomic_write(dropin_path, dropin, mode=0o644)
|
||||||
|
modified = True
|
||||||
|
systemd.run("daemon-reload")
|
||||||
|
systemd.run("enable", EDGE_UNIT)
|
||||||
|
systemd.run("restart", EDGE_UNIT)
|
||||||
|
if not systemd.active(EDGE_UNIT):
|
||||||
|
raise ActivationError("EDGE_SERVICE_NOT_ACTIVE")
|
||||||
|
exec_start = systemd.show(EDGE_UNIT, "ExecStart")
|
||||||
|
environment = systemd.show(EDGE_UNIT, "Environment")
|
||||||
|
if str(app) not in exec_start:
|
||||||
|
raise ActivationError("EDGE_EFFECTIVE_EXECSTART_INVALID")
|
||||||
|
for expected in (
|
||||||
|
f"AIOS_GPT_EDGE_HOST={approval.bind_host}",
|
||||||
|
f"AIOS_GPT_EDGE_PORT={approval.port}",
|
||||||
|
"AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=1",
|
||||||
|
f"AIOS_GPT_EDGE_PUBLIC_DOMAIN={approval.domain}",
|
||||||
|
):
|
||||||
|
if expected not in environment:
|
||||||
|
raise ActivationError("EDGE_EFFECTIVE_ENVIRONMENT_INVALID")
|
||||||
|
tls_probe(approval.domain, cert_path, port=approval.port)
|
||||||
|
receipt = receipt_for(approval, release_root, state="ACTIVE")
|
||||||
|
atomic_write(state_path, json.dumps(receipt, indent=2, sort_keys=True) + "\n")
|
||||||
|
return receipt
|
||||||
|
except Exception:
|
||||||
|
if modified:
|
||||||
|
rollback(
|
||||||
|
dropin_path=dropin_path,
|
||||||
|
state_path=state_path,
|
||||||
|
systemd=systemd,
|
||||||
|
reason="activation_failed",
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def rollback(
|
||||||
|
*,
|
||||||
|
dropin_path: Path,
|
||||||
|
state_path: Path,
|
||||||
|
systemd: Systemd,
|
||||||
|
reason: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if os.geteuid() != 0:
|
||||||
|
raise ActivationError("ROOT_REQUIRED")
|
||||||
|
systemd.run("stop", EDGE_UNIT, check=False)
|
||||||
|
systemd.run("disable", EDGE_UNIT, check=False)
|
||||||
|
dropin_path.unlink(missing_ok=True)
|
||||||
|
systemd.run("daemon-reload")
|
||||||
|
if systemd.active(EDGE_UNIT):
|
||||||
|
raise ActivationError("EDGE_ROLLBACK_SERVICE_STILL_ACTIVE")
|
||||||
|
previous: dict[str, Any] = {}
|
||||||
|
if state_path.is_file():
|
||||||
|
try:
|
||||||
|
previous = json.loads(state_path.read_text(encoding="utf-8"))
|
||||||
|
except Exception:
|
||||||
|
previous = {}
|
||||||
|
receipt = {
|
||||||
|
"schema": STATE_SCHEMA,
|
||||||
|
"state": "ROLLED_BACK",
|
||||||
|
"reason": reason[:255],
|
||||||
|
"candidate_oid": previous.get("candidate_oid"),
|
||||||
|
"approval_id": previous.get("approval_id"),
|
||||||
|
"domain": previous.get("domain"),
|
||||||
|
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
}
|
||||||
|
atomic_write(state_path, json.dumps(receipt, indent=2, sort_keys=True) + "\n")
|
||||||
|
return receipt
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("command", choices=("check", "activate", "rollback"))
|
||||||
|
parser.add_argument("--candidate")
|
||||||
|
parser.add_argument("--approval", default="/etc/aios-gitops/gpt-https-edge.approved")
|
||||||
|
parser.add_argument("--tls-cert", default="/etc/aios-gitops/gpt-https-edge-tls.crt")
|
||||||
|
parser.add_argument("--tls-key", default="/etc/aios-gitops/gpt-https-edge-tls.key")
|
||||||
|
parser.add_argument("--release-root")
|
||||||
|
parser.add_argument(
|
||||||
|
"--dropin",
|
||||||
|
default=f"/etc/systemd/system/{EDGE_UNIT}.d/{DROPIN_NAME}",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--state",
|
||||||
|
default="/var/lib/aios-gpt-https-edge-activation/state.json",
|
||||||
|
)
|
||||||
|
parser.add_argument("--reason", default="owner_requested")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
systemd = Systemd()
|
||||||
|
if args.command == "rollback":
|
||||||
|
value = rollback(
|
||||||
|
dropin_path=Path(args.dropin),
|
||||||
|
state_path=Path(args.state),
|
||||||
|
systemd=systemd,
|
||||||
|
reason=args.reason,
|
||||||
|
)
|
||||||
|
print(json.dumps(value, indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
candidate = str(args.candidate or "")
|
||||||
|
if OID_RE.fullmatch(candidate) is None:
|
||||||
|
raise SystemExit("CANDIDATE_REQUIRED")
|
||||||
|
release_root = (
|
||||||
|
Path(args.release_root)
|
||||||
|
if args.release_root
|
||||||
|
else Path(f"/opt/aios/releases/{candidate}-gpt-https-edge-public")
|
||||||
|
)
|
||||||
|
cert_path = Path(args.tls_cert)
|
||||||
|
key_path = Path(args.tls_key)
|
||||||
|
approval = Approval.load(
|
||||||
|
Path(args.approval),
|
||||||
|
candidate_oid=candidate,
|
||||||
|
cert_path=cert_path,
|
||||||
|
)
|
||||||
|
tls = validate_tls_pair(cert_path, key_path, approval.domain)
|
||||||
|
release = validate_release(release_root)
|
||||||
|
require_ingress_ready(systemd)
|
||||||
|
|
||||||
|
if args.command == "check":
|
||||||
|
value = {
|
||||||
|
"status": "READY",
|
||||||
|
"candidate_oid": candidate,
|
||||||
|
"approval_id": approval.approval_id,
|
||||||
|
"approval_sha256": approval.raw_sha256,
|
||||||
|
"domain": approval.domain,
|
||||||
|
"bind_host": approval.bind_host,
|
||||||
|
"port": approval.port,
|
||||||
|
"edge_sha256": sha256_file(release["edge"]),
|
||||||
|
"controller_sha256": sha256_file(release["controller"]),
|
||||||
|
**tls,
|
||||||
|
"public_network_mutation": False,
|
||||||
|
}
|
||||||
|
print(json.dumps(value, indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
value = activate(
|
||||||
|
approval=approval,
|
||||||
|
release_root=release_root,
|
||||||
|
cert_path=cert_path,
|
||||||
|
key_path=key_path,
|
||||||
|
dropin_path=Path(args.dropin),
|
||||||
|
state_path=Path(args.state),
|
||||||
|
systemd=systemd,
|
||||||
|
)
|
||||||
|
print(json.dumps(value, indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,816 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Credential-isolated host-side ingress for externally scheduled read-only Online Tasks.
|
||||||
|
|
||||||
|
This service is intentionally not a public AIOS control plane. It accepts a tiny,
|
||||||
|
authenticated external intent and translates it to the canonical loopback Bridge.
|
||||||
|
Callers cannot select repository coordinates, capabilities, execution action,
|
||||||
|
worker, branch policy, credentials, verification decisions, or write permissions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
SCHEMA = "aios.gpt-online-task-ingress.request.v1"
|
||||||
|
RESPONSE_SCHEMA = "aios.gpt-online-task-ingress.response.v1"
|
||||||
|
OPENAPI_VERSION = "3.1.0"
|
||||||
|
|
||||||
|
REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$")
|
||||||
|
TASK_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
|
||||||
|
JOB_ID_RE = re.compile(r"^job-[0-9a-f]{32}$")
|
||||||
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
MAX_BODY = 64 * 1024
|
||||||
|
MAX_GOAL = 12_000
|
||||||
|
MAX_INSTRUCTIONS = 24_000
|
||||||
|
MAX_CRITERIA = 16
|
||||||
|
MAX_CRITERION_TEXT = 1_000
|
||||||
|
|
||||||
|
READONLY_CAPABILITIES = ("git.read", "git.test")
|
||||||
|
REPOSITORY = "aios/aios"
|
||||||
|
|
||||||
|
|
||||||
|
class IngressError(RuntimeError):
|
||||||
|
def __init__(self, code: str, status: int = 400) -> None:
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
self.status = status
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value,
|
||||||
|
ensure_ascii=False,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def digest(value: object) -> str:
|
||||||
|
return hashlib.sha256(canonical(value)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def load_secret(path: Path, *, code: str) -> str:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
raise RuntimeError(code + "_UNAVAILABLE")
|
||||||
|
value = path.read_text(encoding="utf-8").strip()
|
||||||
|
if len(value) < 32 or "\n" in value or "\r" in value:
|
||||||
|
raise RuntimeError(code + "_INVALID")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def normalized_request(value: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
allowed = {
|
||||||
|
"schema",
|
||||||
|
"request_id",
|
||||||
|
"goal",
|
||||||
|
"instructions",
|
||||||
|
"success_criteria",
|
||||||
|
"constraints",
|
||||||
|
}
|
||||||
|
if set(value) - allowed:
|
||||||
|
raise IngressError("UNKNOWN_FIELDS")
|
||||||
|
if value.get("schema") != SCHEMA:
|
||||||
|
raise IngressError("SCHEMA_INVALID")
|
||||||
|
|
||||||
|
request_id = value.get("request_id")
|
||||||
|
if not isinstance(request_id, str) or REQUEST_ID_RE.fullmatch(request_id) is None:
|
||||||
|
raise IngressError("REQUEST_ID_INVALID")
|
||||||
|
|
||||||
|
goal = value.get("goal")
|
||||||
|
if not isinstance(goal, str) or not goal.strip() or len(goal) > MAX_GOAL:
|
||||||
|
raise IngressError("GOAL_INVALID")
|
||||||
|
instructions = value.get("instructions", "")
|
||||||
|
if not isinstance(instructions, str) or len(instructions) > MAX_INSTRUCTIONS:
|
||||||
|
raise IngressError("INSTRUCTIONS_INVALID")
|
||||||
|
|
||||||
|
criteria = value.get("success_criteria") or []
|
||||||
|
if not isinstance(criteria, list) or len(criteria) > MAX_CRITERIA:
|
||||||
|
raise IngressError("SUCCESS_CRITERIA_INVALID")
|
||||||
|
normalized_criteria = []
|
||||||
|
for item in criteria:
|
||||||
|
if not isinstance(item, Mapping) or set(item) != {"id", "description"}:
|
||||||
|
raise IngressError("SUCCESS_CRITERIA_INVALID")
|
||||||
|
criterion_id = item.get("id")
|
||||||
|
description = item.get("description")
|
||||||
|
if (
|
||||||
|
not isinstance(criterion_id, str)
|
||||||
|
or not criterion_id.strip()
|
||||||
|
or len(criterion_id) > 120
|
||||||
|
or not isinstance(description, str)
|
||||||
|
or not description.strip()
|
||||||
|
or len(description) > MAX_CRITERION_TEXT
|
||||||
|
):
|
||||||
|
raise IngressError("SUCCESS_CRITERIA_INVALID")
|
||||||
|
normalized_criteria.append(
|
||||||
|
{"id": criterion_id.strip(), "description": description.strip()}
|
||||||
|
)
|
||||||
|
|
||||||
|
constraints = value.get("constraints") or {}
|
||||||
|
if not isinstance(constraints, Mapping):
|
||||||
|
raise IngressError("CONSTRAINTS_INVALID")
|
||||||
|
allowed_constraints = {
|
||||||
|
"max_files",
|
||||||
|
"focus_paths",
|
||||||
|
"notes",
|
||||||
|
}
|
||||||
|
if set(constraints) - allowed_constraints:
|
||||||
|
raise IngressError("CONSTRAINTS_FIELDS_INVALID")
|
||||||
|
normalized_constraints: dict[str, Any] = {}
|
||||||
|
if "max_files" in constraints:
|
||||||
|
max_files = constraints["max_files"]
|
||||||
|
if (
|
||||||
|
not isinstance(max_files, int)
|
||||||
|
or isinstance(max_files, bool)
|
||||||
|
or not 1 <= max_files <= 500
|
||||||
|
):
|
||||||
|
raise IngressError("MAX_FILES_INVALID")
|
||||||
|
normalized_constraints["max_files"] = max_files
|
||||||
|
if "focus_paths" in constraints:
|
||||||
|
paths = constraints["focus_paths"]
|
||||||
|
if (
|
||||||
|
not isinstance(paths, list)
|
||||||
|
or len(paths) > 32
|
||||||
|
or not all(
|
||||||
|
isinstance(path, str)
|
||||||
|
and path
|
||||||
|
and len(path) <= 512
|
||||||
|
and not path.startswith("/")
|
||||||
|
and ".." not in Path(path).parts
|
||||||
|
for path in paths
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise IngressError("FOCUS_PATHS_INVALID")
|
||||||
|
normalized_constraints["focus_paths"] = paths
|
||||||
|
if "notes" in constraints:
|
||||||
|
notes = constraints["notes"]
|
||||||
|
if not isinstance(notes, str) or len(notes) > 4_000:
|
||||||
|
raise IngressError("CONSTRAINT_NOTES_INVALID")
|
||||||
|
normalized_constraints["notes"] = notes
|
||||||
|
|
||||||
|
return {
|
||||||
|
"schema": SCHEMA,
|
||||||
|
"request_id": request_id,
|
||||||
|
"goal": goal.strip(),
|
||||||
|
"instructions": instructions,
|
||||||
|
"success_criteria": normalized_criteria,
|
||||||
|
"constraints": normalized_constraints,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def validated_bind_host(value: str) -> str:
|
||||||
|
host = value.strip()
|
||||||
|
if host not in {"127.0.0.1", "::1", "localhost"}:
|
||||||
|
raise RuntimeError("GPT_INGRESS_NON_LOOPBACK_BIND_DENY")
|
||||||
|
return host
|
||||||
|
|
||||||
|
|
||||||
|
def task_id_for(request_id: str) -> str:
|
||||||
|
suffix = hashlib.sha256(request_id.encode("utf-8")).hexdigest()[:24]
|
||||||
|
return "gpt-readonly-" + suffix
|
||||||
|
|
||||||
|
|
||||||
|
def branch_for(task_id: str) -> str:
|
||||||
|
return "agent/external-gpt-readonly/" + task_id[-24:]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Upstream:
|
||||||
|
bridge_base: str
|
||||||
|
task_base: str
|
||||||
|
control_token: str
|
||||||
|
timeout: int = 20
|
||||||
|
|
||||||
|
def _request(
|
||||||
|
self,
|
||||||
|
base: str,
|
||||||
|
method: str,
|
||||||
|
path: str,
|
||||||
|
body: Mapping[str, Any] | None = None,
|
||||||
|
) -> tuple[int, dict[str, Any]]:
|
||||||
|
raw_body = None if body is None else canonical(body)
|
||||||
|
headers = {
|
||||||
|
"Authorization": "Bearer " + self.control_token,
|
||||||
|
"Accept": "application/json",
|
||||||
|
}
|
||||||
|
if raw_body is not None:
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
|
req = urllib.request.Request(
|
||||||
|
base.rstrip("/") + path,
|
||||||
|
method=method,
|
||||||
|
data=raw_body,
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=self.timeout) as response:
|
||||||
|
raw = response.read()
|
||||||
|
status = response.status
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raw = exc.read()
|
||||||
|
status = exc.code
|
||||||
|
except Exception as exc:
|
||||||
|
raise IngressError("UPSTREAM_TRANSPORT_FAILED", 503) from exc
|
||||||
|
try:
|
||||||
|
value = json.loads(raw.decode("utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
raise IngressError("UPSTREAM_RESPONSE_INVALID", 502) from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise IngressError("UPSTREAM_RESPONSE_INVALID", 502)
|
||||||
|
return status, dict(value)
|
||||||
|
|
||||||
|
def control_snapshot(self) -> dict[str, Any]:
|
||||||
|
status, value = self._request(
|
||||||
|
self.bridge_base,
|
||||||
|
"GET",
|
||||||
|
"/v2/control-plane-snapshot",
|
||||||
|
)
|
||||||
|
if status != 200:
|
||||||
|
raise IngressError("CONTROL_SNAPSHOT_UNAVAILABLE", 503)
|
||||||
|
return value
|
||||||
|
|
||||||
|
def create_governed_task(
|
||||||
|
self,
|
||||||
|
body: Mapping[str, Any],
|
||||||
|
) -> tuple[int, dict[str, Any]]:
|
||||||
|
return self._request(
|
||||||
|
self.bridge_base,
|
||||||
|
"POST",
|
||||||
|
"/v2/governed-task",
|
||||||
|
body,
|
||||||
|
)
|
||||||
|
|
||||||
|
def execution_envelope(
|
||||||
|
self,
|
||||||
|
body: Mapping[str, Any],
|
||||||
|
) -> tuple[int, dict[str, Any]]:
|
||||||
|
return self._request(
|
||||||
|
self.bridge_base,
|
||||||
|
"POST",
|
||||||
|
"/v2/execution-envelope",
|
||||||
|
body,
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_task(self, task_id: str) -> tuple[int, dict[str, Any]]:
|
||||||
|
return self._request(
|
||||||
|
self.task_base,
|
||||||
|
"GET",
|
||||||
|
"/v1/tasks/" + task_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Store:
|
||||||
|
def __init__(self, path: Path) -> None:
|
||||||
|
self.path = Path(path)
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with self._connect() as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS external_task_requests(
|
||||||
|
request_id TEXT PRIMARY KEY,
|
||||||
|
request_sha256 TEXT NOT NULL,
|
||||||
|
task_id TEXT NOT NULL UNIQUE,
|
||||||
|
job_id TEXT,
|
||||||
|
base_oid TEXT,
|
||||||
|
state TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL,
|
||||||
|
response_json TEXT
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
|
||||||
|
def _connect(self) -> sqlite3.Connection:
|
||||||
|
con = sqlite3.connect(self.path, timeout=30)
|
||||||
|
con.row_factory = sqlite3.Row
|
||||||
|
con.execute("PRAGMA busy_timeout=30000")
|
||||||
|
return con
|
||||||
|
|
||||||
|
def get(self, request_id: str) -> sqlite3.Row | None:
|
||||||
|
with self._connect() as con:
|
||||||
|
return con.execute(
|
||||||
|
"SELECT * FROM external_task_requests WHERE request_id=?",
|
||||||
|
(request_id,),
|
||||||
|
).fetchone()
|
||||||
|
|
||||||
|
def reserve(self, request: Mapping[str, Any], task_id: str) -> sqlite3.Row:
|
||||||
|
request_sha = digest(request)
|
||||||
|
timestamp = now()
|
||||||
|
with self._lock, self._connect() as con:
|
||||||
|
con.execute("BEGIN IMMEDIATE")
|
||||||
|
row = con.execute(
|
||||||
|
"SELECT * FROM external_task_requests WHERE request_id=?",
|
||||||
|
(request["request_id"],),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None:
|
||||||
|
if row["request_sha256"] != request_sha:
|
||||||
|
raise IngressError("IDEMPOTENCY_CONFLICT", 409)
|
||||||
|
con.commit()
|
||||||
|
return row
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO external_task_requests(
|
||||||
|
request_id,request_sha256,task_id,state,created_at,updated_at
|
||||||
|
) VALUES(?,?,?,?,?,?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
request["request_id"],
|
||||||
|
request_sha,
|
||||||
|
task_id,
|
||||||
|
"PENDING",
|
||||||
|
timestamp,
|
||||||
|
timestamp,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
row = self.get(request["request_id"])
|
||||||
|
assert row is not None
|
||||||
|
return row
|
||||||
|
|
||||||
|
def bound(
|
||||||
|
self,
|
||||||
|
request_id: str,
|
||||||
|
*,
|
||||||
|
job_id: str,
|
||||||
|
base_oid: str,
|
||||||
|
response: Mapping[str, Any],
|
||||||
|
) -> None:
|
||||||
|
with self._lock, self._connect() as con:
|
||||||
|
con.execute(
|
||||||
|
"""
|
||||||
|
UPDATE external_task_requests
|
||||||
|
SET job_id=?,base_oid=?,state='BOUND',response_json=?,updated_at=?
|
||||||
|
WHERE request_id=?
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
job_id,
|
||||||
|
base_oid,
|
||||||
|
json.dumps(response, sort_keys=True, separators=(",", ":")),
|
||||||
|
now(),
|
||||||
|
request_id,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
|
||||||
|
|
||||||
|
class IngressService:
|
||||||
|
def __init__(self, upstream: Upstream, store: Store) -> None:
|
||||||
|
self.upstream = upstream
|
||||||
|
self.store = store
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _main_oid(snapshot: Mapping[str, Any]) -> str:
|
||||||
|
candidates = [
|
||||||
|
snapshot.get("authoritative_head_oid"),
|
||||||
|
snapshot.get("head"),
|
||||||
|
(snapshot.get("aggregate") or {}).get("authoritative_head_oid")
|
||||||
|
if isinstance(snapshot.get("aggregate"), Mapping)
|
||||||
|
else None,
|
||||||
|
]
|
||||||
|
for candidate in candidates:
|
||||||
|
if isinstance(candidate, str) and OID_RE.fullmatch(candidate):
|
||||||
|
return candidate
|
||||||
|
raise IngressError("AUTHORITATIVE_HEAD_UNAVAILABLE", 503)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _public_response(
|
||||||
|
request_id: str,
|
||||||
|
task_id: str,
|
||||||
|
job_id: str,
|
||||||
|
state: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"schema": RESPONSE_SCHEMA,
|
||||||
|
"request_id": request_id,
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": job_id,
|
||||||
|
"state": state,
|
||||||
|
"execution_mode": "source_capture",
|
||||||
|
"capabilities": list(READONLY_CAPABILITIES),
|
||||||
|
"write_access": False,
|
||||||
|
"official_verification_required": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _recover_existing(
|
||||||
|
self,
|
||||||
|
request: Mapping[str, Any],
|
||||||
|
row: sqlite3.Row,
|
||||||
|
) -> dict[str, Any] | None:
|
||||||
|
task_id = str(row["task_id"])
|
||||||
|
status, task = self.upstream.get_task(task_id)
|
||||||
|
if status == 404:
|
||||||
|
return None
|
||||||
|
if status != 200:
|
||||||
|
raise IngressError("TASK_RECOVERY_FAILED", 503)
|
||||||
|
metadata = task.get("metadata")
|
||||||
|
execution = (
|
||||||
|
metadata.get("execution")
|
||||||
|
if isinstance(metadata, Mapping)
|
||||||
|
and isinstance(metadata.get("execution"), Mapping)
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
job_id = str(execution.get("job_id") or row["job_id"] or "")
|
||||||
|
base_oid = str(execution.get("base_oid") or row["base_oid"] or "")
|
||||||
|
if (
|
||||||
|
task.get("task_id") != task_id
|
||||||
|
or not JOB_ID_RE.fullmatch(job_id)
|
||||||
|
or not OID_RE.fullmatch(base_oid)
|
||||||
|
or execution.get("action") != "source_capture"
|
||||||
|
or sorted(execution.get("capability_scope") or [])
|
||||||
|
!= sorted(READONLY_CAPABILITIES)
|
||||||
|
or (execution.get("governance") or {}).get("writer_lease") is not False
|
||||||
|
):
|
||||||
|
raise IngressError("TASK_RECOVERY_BINDING_INVALID", 409)
|
||||||
|
response = self._public_response(
|
||||||
|
request["request_id"],
|
||||||
|
task_id,
|
||||||
|
job_id,
|
||||||
|
str(task.get("status") or "BOUND"),
|
||||||
|
)
|
||||||
|
self.store.bound(
|
||||||
|
request["request_id"],
|
||||||
|
job_id=job_id,
|
||||||
|
base_oid=base_oid,
|
||||||
|
response=response,
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
def create(self, raw: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
request = normalized_request(raw)
|
||||||
|
task_id = task_id_for(request["request_id"])
|
||||||
|
row = self.store.reserve(request, task_id)
|
||||||
|
if row["state"] == "BOUND" and row["response_json"]:
|
||||||
|
return json.loads(row["response_json"])
|
||||||
|
|
||||||
|
recovered = self._recover_existing(request, row)
|
||||||
|
if recovered is not None:
|
||||||
|
return recovered
|
||||||
|
|
||||||
|
snapshot = self.upstream.control_snapshot()
|
||||||
|
base_oid = self._main_oid(snapshot)
|
||||||
|
target_branch = branch_for(task_id)
|
||||||
|
binding_key = "gpt-readonly-binding-" + digest(
|
||||||
|
{"request_id": request["request_id"], "base_oid": base_oid}
|
||||||
|
)[:24]
|
||||||
|
constraints = {
|
||||||
|
**request["constraints"],
|
||||||
|
"repository_mutation": "forbidden",
|
||||||
|
"control_plane_side_effects": "forbidden",
|
||||||
|
"credential_access": "forbidden",
|
||||||
|
}
|
||||||
|
body = {
|
||||||
|
"task_id": task_id,
|
||||||
|
"repository": REPOSITORY,
|
||||||
|
"base_oid": base_oid,
|
||||||
|
"expected_branch_oid": base_oid,
|
||||||
|
"target_branch": target_branch,
|
||||||
|
"branch_mode": "CREATE_NEW",
|
||||||
|
"generation": 1,
|
||||||
|
"risk": "low",
|
||||||
|
"description": "Externally requested read-only governed Online Task.",
|
||||||
|
"idempotency_key": binding_key,
|
||||||
|
"priority": 0,
|
||||||
|
"max_attempts": 3,
|
||||||
|
"require_online_task_projection": True,
|
||||||
|
"online_task": {
|
||||||
|
"goal": request["goal"],
|
||||||
|
"instructions": request["instructions"],
|
||||||
|
"success_criteria": request["success_criteria"],
|
||||||
|
"constraints": constraints,
|
||||||
|
"required_capabilities": list(READONLY_CAPABILITIES),
|
||||||
|
"priority": 0,
|
||||||
|
"max_attempts": 3,
|
||||||
|
},
|
||||||
|
"execution": {
|
||||||
|
"action": "source_capture",
|
||||||
|
"adapter": "source_capture",
|
||||||
|
"capability_scope": list(READONLY_CAPABILITIES),
|
||||||
|
"source_signals": [
|
||||||
|
"external read-only source analysis request",
|
||||||
|
"no repository mutation no credential access no control-plane side effects",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
status, value = self.upstream.create_governed_task(body)
|
||||||
|
if status not in {200, 201, 202}:
|
||||||
|
if status == 409:
|
||||||
|
recovered = self._recover_existing(request, row)
|
||||||
|
if recovered is not None:
|
||||||
|
return recovered
|
||||||
|
raise IngressError(
|
||||||
|
"GOVERNED_TASK_CREATE_FAILED",
|
||||||
|
409 if status == 409 else 503,
|
||||||
|
)
|
||||||
|
job_id = str(value.get("job_id") or "")
|
||||||
|
if value.get("task_id") != task_id or JOB_ID_RE.fullmatch(job_id) is None:
|
||||||
|
raise IngressError("GOVERNED_TASK_RESPONSE_INVALID", 502)
|
||||||
|
|
||||||
|
envelope_status, envelope = self.upstream.execution_envelope(
|
||||||
|
{
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": job_id,
|
||||||
|
"required_capabilities": list(READONLY_CAPABILITIES),
|
||||||
|
"execution": body["execution"],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if envelope_status != 200:
|
||||||
|
raise IngressError("EXECUTION_PROJECTION_FAILED", 503)
|
||||||
|
if (
|
||||||
|
envelope.get("task_id") != task_id
|
||||||
|
or envelope.get("job_id") != job_id
|
||||||
|
or envelope.get("base_oid") != base_oid
|
||||||
|
or envelope.get("action") != "source_capture"
|
||||||
|
or sorted(envelope.get("capability_scope") or [])
|
||||||
|
!= sorted(READONLY_CAPABILITIES)
|
||||||
|
or (envelope.get("governance") or {}).get("writer_lease") is not False
|
||||||
|
):
|
||||||
|
raise IngressError("EXECUTION_PROJECTION_INVALID", 502)
|
||||||
|
|
||||||
|
response = self._public_response(
|
||||||
|
request["request_id"],
|
||||||
|
task_id,
|
||||||
|
job_id,
|
||||||
|
"BOUND",
|
||||||
|
)
|
||||||
|
self.store.bound(
|
||||||
|
request["request_id"],
|
||||||
|
job_id=job_id,
|
||||||
|
base_oid=base_oid,
|
||||||
|
response=response,
|
||||||
|
)
|
||||||
|
return response
|
||||||
|
|
||||||
|
def status(self, request_id: str) -> dict[str, Any]:
|
||||||
|
if REQUEST_ID_RE.fullmatch(request_id) is None:
|
||||||
|
raise IngressError("REQUEST_ID_INVALID")
|
||||||
|
row = self.store.get(request_id)
|
||||||
|
if row is None:
|
||||||
|
raise IngressError("REQUEST_NOT_FOUND", 404)
|
||||||
|
task_id = str(row["task_id"])
|
||||||
|
status, task = self.upstream.get_task(task_id)
|
||||||
|
if status == 200 and not row["job_id"]:
|
||||||
|
metadata = task.get("metadata")
|
||||||
|
execution = (
|
||||||
|
metadata.get("execution")
|
||||||
|
if isinstance(metadata, Mapping)
|
||||||
|
and isinstance(metadata.get("execution"), Mapping)
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
job_id = str(execution.get("job_id") or "")
|
||||||
|
base_oid = str(execution.get("base_oid") or "")
|
||||||
|
if (
|
||||||
|
JOB_ID_RE.fullmatch(job_id)
|
||||||
|
and OID_RE.fullmatch(base_oid)
|
||||||
|
and execution.get("action") == "source_capture"
|
||||||
|
and sorted(execution.get("capability_scope") or [])
|
||||||
|
== sorted(READONLY_CAPABILITIES)
|
||||||
|
and (execution.get("governance") or {}).get("writer_lease") is False
|
||||||
|
):
|
||||||
|
recovered = self._public_response(
|
||||||
|
request_id,
|
||||||
|
task_id,
|
||||||
|
job_id,
|
||||||
|
str(task.get("status") or "BOUND"),
|
||||||
|
)
|
||||||
|
self.store.bound(
|
||||||
|
request_id,
|
||||||
|
job_id=job_id,
|
||||||
|
base_oid=base_oid,
|
||||||
|
response=recovered,
|
||||||
|
)
|
||||||
|
row = self.store.get(request_id) or row
|
||||||
|
if status == 404:
|
||||||
|
return {
|
||||||
|
"schema": RESPONSE_SCHEMA,
|
||||||
|
"request_id": request_id,
|
||||||
|
"task_id": task_id,
|
||||||
|
"state": "PENDING",
|
||||||
|
"official_verification_required": True,
|
||||||
|
}
|
||||||
|
if status != 200:
|
||||||
|
raise IngressError("TASK_STATUS_UNAVAILABLE", 503)
|
||||||
|
|
||||||
|
result = task.get("result") if isinstance(task.get("result"), Mapping) else {}
|
||||||
|
verification = (
|
||||||
|
task.get("verification")
|
||||||
|
if isinstance(task.get("verification"), Mapping)
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
safe = {
|
||||||
|
"schema": RESPONSE_SCHEMA,
|
||||||
|
"request_id": request_id,
|
||||||
|
"task_id": task_id,
|
||||||
|
"job_id": str(row["job_id"] or ""),
|
||||||
|
"state": str(task.get("status") or "UNKNOWN"),
|
||||||
|
"attempt": int(task.get("attempt") or 0),
|
||||||
|
"claimed": bool(task.get("claimed_by")),
|
||||||
|
"result_status": result.get("status"),
|
||||||
|
"result_classification": result.get("classification"),
|
||||||
|
"official_verification": verification.get("decision"),
|
||||||
|
"official_classification": verification.get("classification"),
|
||||||
|
"write_access": False,
|
||||||
|
}
|
||||||
|
if result.get("execution_mode") == "source_capture":
|
||||||
|
snapshot = result.get("source_snapshot")
|
||||||
|
if isinstance(snapshot, Mapping):
|
||||||
|
safe["source_snapshot_sha256"] = snapshot.get("payload_sha256")
|
||||||
|
safe["source_snapshot_file_count"] = snapshot.get("file_count")
|
||||||
|
return safe
|
||||||
|
|
||||||
|
|
||||||
|
def openapi_document() -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"openapi": OPENAPI_VERSION,
|
||||||
|
"info": {
|
||||||
|
"title": "AIOS Read-Only Online Task Ingress",
|
||||||
|
"version": "1.0.0",
|
||||||
|
},
|
||||||
|
"paths": {
|
||||||
|
"/v1/read-only-tasks": {
|
||||||
|
"post": {
|
||||||
|
"operationId": "createReadOnlyOnlineTask",
|
||||||
|
"security": [{"bearerAuth": []}],
|
||||||
|
"requestBody": {"required": True},
|
||||||
|
"responses": {"200": {"description": "Task accepted"}},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"/v1/read-only-tasks/{request_id}": {
|
||||||
|
"get": {
|
||||||
|
"operationId": "getReadOnlyOnlineTaskStatus",
|
||||||
|
"security": [{"bearerAuth": []}],
|
||||||
|
"responses": {"200": {"description": "Sanitized task status"}},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"components": {
|
||||||
|
"securitySchemes": {
|
||||||
|
"bearerAuth": {
|
||||||
|
"type": "http",
|
||||||
|
"scheme": "bearer",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
service: IngressService | None = None
|
||||||
|
ingress_token: str = ""
|
||||||
|
|
||||||
|
def log_message(self, *_args: Any) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
def respond(self, value: Mapping[str, Any], status: int) -> None:
|
||||||
|
raw = canonical(value)
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.send_header("Content-Length", str(len(raw)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(raw)
|
||||||
|
|
||||||
|
def authorized(self) -> bool:
|
||||||
|
header = self.headers.get("Authorization", "")
|
||||||
|
return (
|
||||||
|
header.startswith("Bearer ")
|
||||||
|
and bool(self.ingress_token)
|
||||||
|
and hmac.compare_digest(header[7:], self.ingress_token)
|
||||||
|
)
|
||||||
|
|
||||||
|
def body(self) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get("Content-Length", "-1"))
|
||||||
|
except ValueError as exc:
|
||||||
|
raise IngressError("CONTENT_LENGTH_INVALID") from exc
|
||||||
|
if length < 1 or length > MAX_BODY:
|
||||||
|
raise IngressError("REQUEST_SIZE_INVALID", 413)
|
||||||
|
try:
|
||||||
|
value = json.loads(self.rfile.read(length).decode("utf-8"))
|
||||||
|
except Exception as exc:
|
||||||
|
raise IngressError("REQUEST_JSON_INVALID") from exc
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise IngressError("REQUEST_OBJECT_REQUIRED")
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
if self.path == "/health":
|
||||||
|
self.respond(
|
||||||
|
{
|
||||||
|
"status": "ok",
|
||||||
|
"service": "aios-gpt-online-task-ingress",
|
||||||
|
"mode": "read-only",
|
||||||
|
},
|
||||||
|
200,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
if self.path == "/openapi.json":
|
||||||
|
self.respond(openapi_document(), 200)
|
||||||
|
return
|
||||||
|
if not self.authorized():
|
||||||
|
self.respond({"error": "UNAUTHORIZED"}, 401)
|
||||||
|
return
|
||||||
|
match = re.fullmatch(r"/v1/read-only-tasks/([^/?#]+)", self.path)
|
||||||
|
if match is None:
|
||||||
|
self.respond({"error": "NOT_FOUND"}, 404)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
if self.service is None:
|
||||||
|
raise IngressError("RUNTIME_UNAVAILABLE", 503)
|
||||||
|
self.respond(self.service.status(match.group(1)), 200)
|
||||||
|
except IngressError as exc:
|
||||||
|
self.respond({"error": exc.code}, exc.status)
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
if self.path != "/v1/read-only-tasks":
|
||||||
|
self.respond({"error": "NOT_FOUND"}, 404)
|
||||||
|
return
|
||||||
|
if not self.authorized():
|
||||||
|
self.respond({"error": "UNAUTHORIZED"}, 401)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
if self.service is None:
|
||||||
|
raise IngressError("RUNTIME_UNAVAILABLE", 503)
|
||||||
|
value = self.service.create(self.body())
|
||||||
|
self.respond(value, 200)
|
||||||
|
except IngressError as exc:
|
||||||
|
self.respond({"error": exc.code}, exc.status)
|
||||||
|
|
||||||
|
|
||||||
|
def serve() -> None:
|
||||||
|
ingress_token_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_GPT_INGRESS_TOKEN_FILE",
|
||||||
|
str(
|
||||||
|
Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
||||||
|
/ "gpt_ingress_token"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
control_token_path = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_AGENT_TASK_CONTROL_TOKEN_FILE",
|
||||||
|
str(
|
||||||
|
Path(os.environ.get("CREDENTIALS_DIRECTORY", "/run/credentials"))
|
||||||
|
/ "agent_task_control_token"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
ingress_token = load_secret(
|
||||||
|
ingress_token_path,
|
||||||
|
code="GPT_INGRESS_CREDENTIAL",
|
||||||
|
)
|
||||||
|
control_token = load_secret(
|
||||||
|
control_token_path,
|
||||||
|
code="CONTROL_PLANE_CREDENTIAL",
|
||||||
|
)
|
||||||
|
|
||||||
|
state_root = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_GPT_INGRESS_STATE_ROOT",
|
||||||
|
"/var/lib/aios-gpt-online-task-ingress",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
host = validated_bind_host(
|
||||||
|
os.environ.get("AIOS_GPT_INGRESS_HOST", "127.0.0.1")
|
||||||
|
)
|
||||||
|
port = int(os.environ.get("AIOS_GPT_INGRESS_PORT", "18945"))
|
||||||
|
if not 1024 <= port <= 65535:
|
||||||
|
raise RuntimeError("GPT_INGRESS_PORT_INVALID")
|
||||||
|
|
||||||
|
Handler.service = IngressService(
|
||||||
|
Upstream(
|
||||||
|
bridge_base=os.environ.get(
|
||||||
|
"AIOS_GOVERNED_WRITE_API_URL",
|
||||||
|
"http://127.0.0.1:18892",
|
||||||
|
),
|
||||||
|
task_base=os.environ.get(
|
||||||
|
"AIOS_AGENT_TASK_API_URL",
|
||||||
|
"http://127.0.0.1:18940",
|
||||||
|
),
|
||||||
|
control_token=control_token,
|
||||||
|
),
|
||||||
|
Store(state_root / "state.sqlite3"),
|
||||||
|
)
|
||||||
|
Handler.ingress_token = ingress_token
|
||||||
|
ThreadingHTTPServer((host, port), Handler).serve_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
serve()
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,232 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Root-owned Model Governance configuration reconciler.
|
||||||
|
|
||||||
|
The config contains provider/model/policy metadata only. Raw credentials are rejected by
|
||||||
|
aios_model_governance and never stored in the governance database.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
from aios_model_governance import (
|
||||||
|
ModelGovernanceError,
|
||||||
|
ModelGovernanceStore,
|
||||||
|
ModelRecord,
|
||||||
|
ProviderRecord,
|
||||||
|
ScopePolicy,
|
||||||
|
)
|
||||||
|
|
||||||
|
SCHEMA = "aios.model-governance.bootstrap.v1"
|
||||||
|
MAX_CONFIG_BYTES = 512 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
def canonical(value: object) -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def fail(code: str) -> None:
|
||||||
|
raise SystemExit(code)
|
||||||
|
|
||||||
|
|
||||||
|
def _load(path: Path) -> dict[str, Any]:
|
||||||
|
if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_CONFIG_BYTES:
|
||||||
|
fail("MODEL_GOVERNANCE_CONFIG_INVALID")
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
fail("MODEL_GOVERNANCE_CONFIG_INVALID")
|
||||||
|
if not isinstance(value, Mapping) or value.get("schema") != SCHEMA:
|
||||||
|
fail("MODEL_GOVERNANCE_CONFIG_SCHEMA_INVALID")
|
||||||
|
return dict(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _provider_desired(value: Mapping[str, Any]) -> ProviderRecord:
|
||||||
|
return ProviderRecord(
|
||||||
|
provider_id=str(value.get("provider_id") or ""),
|
||||||
|
credential_ref=value.get("credential_ref"),
|
||||||
|
enabled=bool(value.get("enabled", True)),
|
||||||
|
failure_domain=str(value.get("failure_domain") or "default"),
|
||||||
|
max_failures=int(value.get("max_failures") or 3),
|
||||||
|
cooldown_seconds=int(value.get("cooldown_seconds") or 60),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _model_desired(value: Mapping[str, Any]) -> ModelRecord:
|
||||||
|
return ModelRecord(
|
||||||
|
model_id=str(value.get("model_id") or ""),
|
||||||
|
provider_id=str(value.get("provider_id") or ""),
|
||||||
|
version=str(value.get("version") or ""),
|
||||||
|
capabilities=tuple(value.get("capabilities") or ()),
|
||||||
|
input_cost_micros_per_1k=int(value.get("input_cost_micros_per_1k") or 0),
|
||||||
|
output_cost_micros_per_1k=int(value.get("output_cost_micros_per_1k") or 0),
|
||||||
|
max_input_tokens=int(value.get("max_input_tokens") or 0),
|
||||||
|
max_output_tokens=int(value.get("max_output_tokens") or 0),
|
||||||
|
evaluation_state=str(value.get("evaluation_state") or "APPROVED"),
|
||||||
|
evaluation_sha256=str(value.get("evaluation_sha256") or ""),
|
||||||
|
enabled=bool(value.get("enabled", True)),
|
||||||
|
priority=int(value.get("priority") or 100),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _policy_desired(value: Mapping[str, Any]) -> ScopePolicy:
|
||||||
|
return ScopePolicy(
|
||||||
|
scope_type=str(value.get("scope_type") or ""),
|
||||||
|
tenant_id=value.get("tenant_id"),
|
||||||
|
project_id=value.get("project_id"),
|
||||||
|
allowed_models=tuple(value.get("allowed_models") or ()),
|
||||||
|
denied_providers=tuple(value.get("denied_providers") or ()),
|
||||||
|
required_capabilities=tuple(value.get("required_capabilities") or ()),
|
||||||
|
pinned_model_id=value.get("pinned_model_id"),
|
||||||
|
pinned_model_version=value.get("pinned_model_version"),
|
||||||
|
max_input_tokens_per_request=int(value.get("max_input_tokens_per_request") or 0),
|
||||||
|
max_output_tokens_per_request=int(value.get("max_output_tokens_per_request") or 0),
|
||||||
|
daily_token_budget=int(value.get("daily_token_budget") or 0),
|
||||||
|
daily_cost_budget_micros=int(value.get("daily_cost_budget_micros") or 0),
|
||||||
|
fallback_limit=int(value.get("fallback_limit") or 0),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _row_dict(row: object) -> dict[str, Any] | None:
|
||||||
|
return dict(row) if row is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _provider_matches(row: Mapping[str, Any], desired: ProviderRecord) -> bool:
|
||||||
|
return (
|
||||||
|
row["credential_ref"] == desired.credential_ref
|
||||||
|
and bool(row["enabled"]) == desired.enabled
|
||||||
|
and row["failure_domain"] == desired.failure_domain
|
||||||
|
and int(row["max_failures"]) == desired.max_failures
|
||||||
|
and int(row["cooldown_seconds"]) == desired.cooldown_seconds
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _model_matches(row: Mapping[str, Any], desired: ModelRecord) -> bool:
|
||||||
|
return (
|
||||||
|
row["provider_id"] == desired.provider_id
|
||||||
|
and row["version"] == desired.version
|
||||||
|
and tuple(json.loads(row["capabilities_json"])) == tuple(sorted(set(desired.capabilities)))
|
||||||
|
and int(row["input_cost_micros_per_1k"]) == desired.input_cost_micros_per_1k
|
||||||
|
and int(row["output_cost_micros_per_1k"]) == desired.output_cost_micros_per_1k
|
||||||
|
and int(row["max_input_tokens"]) == desired.max_input_tokens
|
||||||
|
and int(row["max_output_tokens"]) == desired.max_output_tokens
|
||||||
|
and row["evaluation_state"] == desired.evaluation_state
|
||||||
|
and row["evaluation_sha256"] == desired.evaluation_sha256
|
||||||
|
and bool(row["enabled"]) == desired.enabled
|
||||||
|
and int(row["priority"]) == desired.priority
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _policy_matches(row: Mapping[str, Any], desired: ScopePolicy) -> bool:
|
||||||
|
return (
|
||||||
|
row["scope_type"] == desired.scope_type
|
||||||
|
and row["tenant_id"] == desired.tenant_id
|
||||||
|
and row["project_id"] == desired.project_id
|
||||||
|
and tuple(json.loads(row["allowed_models_json"])) == tuple(sorted(set(desired.allowed_models)))
|
||||||
|
and tuple(json.loads(row["denied_providers_json"])) == tuple(sorted(set(desired.denied_providers)))
|
||||||
|
and tuple(json.loads(row["required_capabilities_json"])) == tuple(sorted(set(desired.required_capabilities)))
|
||||||
|
and row["pinned_model_id"] == desired.pinned_model_id
|
||||||
|
and row["pinned_model_version"] == desired.pinned_model_version
|
||||||
|
and int(row["max_input_tokens_per_request"]) == desired.max_input_tokens_per_request
|
||||||
|
and int(row["max_output_tokens_per_request"]) == desired.max_output_tokens_per_request
|
||||||
|
and int(row["daily_token_budget"]) == desired.daily_token_budget
|
||||||
|
and int(row["daily_cost_budget_micros"]) == desired.daily_cost_budget_micros
|
||||||
|
and int(row["fallback_limit"]) == desired.fallback_limit
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_config(db_path: Path, config_path: Path) -> dict[str, Any]:
|
||||||
|
config = _load(config_path)
|
||||||
|
config_sha = hashlib.sha256(canonical(config)).hexdigest()
|
||||||
|
providers = config.get("providers") or []
|
||||||
|
models = config.get("models") or []
|
||||||
|
policies = config.get("policies") or []
|
||||||
|
if not all(isinstance(value, list) for value in (providers, models, policies)):
|
||||||
|
fail("MODEL_GOVERNANCE_CONFIG_COLLECTION_INVALID")
|
||||||
|
|
||||||
|
store = ModelGovernanceStore(db_path)
|
||||||
|
changes = {"providers": 0, "models": 0, "policies": 0, "unchanged": 0}
|
||||||
|
try:
|
||||||
|
for item in providers:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
fail("MODEL_GOVERNANCE_PROVIDER_CONFIG_INVALID")
|
||||||
|
desired = _provider_desired(item)
|
||||||
|
row = store.conn.execute(
|
||||||
|
"SELECT * FROM model_governance_providers WHERE provider_id=?",
|
||||||
|
(desired.provider_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None and _provider_matches(row, desired):
|
||||||
|
changes["unchanged"] += 1
|
||||||
|
continue
|
||||||
|
expected = int(row["revision"]) if row is not None else 0
|
||||||
|
store.register_provider(desired, expected_revision=expected)
|
||||||
|
changes["providers"] += 1
|
||||||
|
|
||||||
|
for item in models:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
fail("MODEL_GOVERNANCE_MODEL_CONFIG_INVALID")
|
||||||
|
desired = _model_desired(item)
|
||||||
|
row = store.conn.execute(
|
||||||
|
"SELECT * FROM model_governance_models WHERE model_id=?",
|
||||||
|
(desired.model_id,),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None and _model_matches(row, desired):
|
||||||
|
changes["unchanged"] += 1
|
||||||
|
continue
|
||||||
|
expected = int(row["revision"]) if row is not None else 0
|
||||||
|
store.register_model(desired, expected_revision=expected)
|
||||||
|
changes["models"] += 1
|
||||||
|
|
||||||
|
for item in policies:
|
||||||
|
if not isinstance(item, Mapping):
|
||||||
|
fail("MODEL_GOVERNANCE_POLICY_CONFIG_INVALID")
|
||||||
|
desired = _policy_desired(item)
|
||||||
|
row = store.conn.execute(
|
||||||
|
"SELECT * FROM model_governance_scope_policies WHERE scope_key=?",
|
||||||
|
(desired.scope_key,),
|
||||||
|
).fetchone()
|
||||||
|
if row is not None and _policy_matches(row, desired):
|
||||||
|
changes["unchanged"] += 1
|
||||||
|
continue
|
||||||
|
expected = int(row["revision"]) if row is not None else 0
|
||||||
|
store.set_scope_policy(desired, expected_revision=expected)
|
||||||
|
changes["policies"] += 1
|
||||||
|
|
||||||
|
if not store.verify_audit_chain():
|
||||||
|
fail("MODEL_GOVERNANCE_AUDIT_CHAIN_INVALID")
|
||||||
|
return {
|
||||||
|
"schema": "aios.model-governance.bootstrap-result.v1",
|
||||||
|
"database": str(db_path),
|
||||||
|
"config_sha256": config_sha,
|
||||||
|
"changes": changes,
|
||||||
|
"audit_chain_valid": True,
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--db", required=True)
|
||||||
|
parser.add_argument("--config", required=True)
|
||||||
|
parser.add_argument("--output")
|
||||||
|
args = parser.parse_args()
|
||||||
|
try:
|
||||||
|
result = apply_config(Path(args.db), Path(args.config))
|
||||||
|
except ModelGovernanceError as exc:
|
||||||
|
print(json.dumps({"status": "FAILED", "error": exc.code}, sort_keys=True))
|
||||||
|
return 20
|
||||||
|
data = json.dumps(result, indent=2, sort_keys=True) + "\n"
|
||||||
|
if args.output:
|
||||||
|
Path(args.output).write_text(data, encoding="utf-8")
|
||||||
|
print(data, end="")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -17,6 +17,7 @@ PHASE_ORDER = {
|
|||||||
"claimed": 10,
|
"claimed": 10,
|
||||||
"workspace_prepared": 20,
|
"workspace_prepared": 20,
|
||||||
"verification_passed": 30,
|
"verification_passed": 30,
|
||||||
|
"agent_runtime_done": 40,
|
||||||
"source_capture_done": 40,
|
"source_capture_done": 40,
|
||||||
"commit_done": 40,
|
"commit_done": 40,
|
||||||
"reconciliation_required": 45,
|
"reconciliation_required": 45,
|
||||||
|
|||||||
@@ -179,7 +179,10 @@ def _agent_runtime_input(value: object) -> dict[str, Any]:
|
|||||||
if not isinstance(value, Mapping):
|
if not isinstance(value, Mapping):
|
||||||
_error("AGENT_RUNTIME_INPUT_REQUIRED")
|
_error("AGENT_RUNTIME_INPUT_REQUIRED")
|
||||||
raw = dict(value)
|
raw = dict(value)
|
||||||
allowed = {"goal", "instructions", "success_criteria", "constraints"}
|
allowed = {
|
||||||
|
"goal", "instructions", "success_criteria", "constraints",
|
||||||
|
"scope", "trace_id",
|
||||||
|
}
|
||||||
if set(raw) - allowed:
|
if set(raw) - allowed:
|
||||||
_error("AGENT_RUNTIME_INPUT_FIELDS_INVALID")
|
_error("AGENT_RUNTIME_INPUT_FIELDS_INVALID")
|
||||||
goal = _string(raw.get("goal"), "AGENT_RUNTIME_GOAL_REQUIRED")
|
goal = _string(raw.get("goal"), "AGENT_RUNTIME_GOAL_REQUIRED")
|
||||||
@@ -208,11 +211,34 @@ def _agent_runtime_input(value: object) -> dict[str, Any]:
|
|||||||
lowered = str(key).lower()
|
lowered = str(key).lower()
|
||||||
if any(marker in lowered for marker in sensitive_markers):
|
if any(marker in lowered for marker in sensitive_markers):
|
||||||
_error("AGENT_RUNTIME_SECRET_FIELD_DENY")
|
_error("AGENT_RUNTIME_SECRET_FIELD_DENY")
|
||||||
|
|
||||||
|
scope = raw.get("scope", {})
|
||||||
|
if not isinstance(scope, Mapping):
|
||||||
|
_error("AGENT_RUNTIME_SCOPE_INVALID")
|
||||||
|
scope = {
|
||||||
|
"scope_type": str(scope.get("scope_type") or "legacy"),
|
||||||
|
"tenant_id": scope.get("tenant_id"),
|
||||||
|
"project_id": scope.get("project_id"),
|
||||||
|
}
|
||||||
|
if scope["scope_type"] not in {"legacy", "system", "tenant"}:
|
||||||
|
_error("AGENT_RUNTIME_SCOPE_INVALID")
|
||||||
|
if scope["scope_type"] == "tenant":
|
||||||
|
if not isinstance(scope["tenant_id"], str) or not scope["tenant_id"].strip():
|
||||||
|
_error("AGENT_RUNTIME_SCOPE_INVALID")
|
||||||
|
elif scope["tenant_id"] not in (None, "") or scope["project_id"] not in (None, ""):
|
||||||
|
_error("AGENT_RUNTIME_SCOPE_INVALID")
|
||||||
|
|
||||||
|
trace_id = str(raw.get("trace_id") or "").strip()
|
||||||
|
if not trace_id or len(trace_id) > 255:
|
||||||
|
_error("AGENT_RUNTIME_TRACE_ID_INVALID")
|
||||||
|
|
||||||
normalized = {
|
normalized = {
|
||||||
"goal": goal[:12000],
|
"goal": goal[:12000],
|
||||||
"instructions": instructions[:32000],
|
"instructions": instructions[:32000],
|
||||||
"success_criteria": normalized_criteria,
|
"success_criteria": normalized_criteria,
|
||||||
"constraints": dict(constraints),
|
"constraints": dict(constraints),
|
||||||
|
"scope": scope,
|
||||||
|
"trace_id": trace_id,
|
||||||
}
|
}
|
||||||
if len(_canonical(normalized).encode("utf-8")) > MAX_AGENT_RUNTIME_INPUT_BYTES:
|
if len(_canonical(normalized).encode("utf-8")) > MAX_AGENT_RUNTIME_INPUT_BYTES:
|
||||||
_error("AGENT_RUNTIME_INPUT_TOO_LARGE")
|
_error("AGENT_RUNTIME_INPUT_TOO_LARGE")
|
||||||
@@ -425,6 +451,12 @@ def project_online_task(
|
|||||||
if adapter not in AGENT_RUNTIME_ADAPTERS:
|
if adapter not in AGENT_RUNTIME_ADAPTERS:
|
||||||
_error("AGENT_RUNTIME_ADAPTER_INVALID")
|
_error("AGENT_RUNTIME_ADAPTER_INVALID")
|
||||||
runtime_input = execution_value.get("runtime_input")
|
runtime_input = execution_value.get("runtime_input")
|
||||||
|
tenant_id = request.get("tenant_id")
|
||||||
|
project_id = request.get("project_id")
|
||||||
|
declared_scope = str(
|
||||||
|
request.get("scope_type")
|
||||||
|
or ("tenant" if isinstance(tenant_id, str) and tenant_id.strip() else "legacy")
|
||||||
|
)
|
||||||
if runtime_input is None:
|
if runtime_input is None:
|
||||||
runtime_input = {
|
runtime_input = {
|
||||||
"goal": request.get("goal"),
|
"goal": request.get("goal"),
|
||||||
@@ -432,6 +464,20 @@ def project_online_task(
|
|||||||
"success_criteria": request.get("success_criteria") or [],
|
"success_criteria": request.get("success_criteria") or [],
|
||||||
"constraints": request.get("constraints") or {},
|
"constraints": request.get("constraints") or {},
|
||||||
}
|
}
|
||||||
|
elif not isinstance(runtime_input, Mapping):
|
||||||
|
_error("AGENT_RUNTIME_INPUT_REQUIRED")
|
||||||
|
runtime_input = dict(runtime_input)
|
||||||
|
# Scope/trace are authoritative Task identity, never caller-selected runtime hints.
|
||||||
|
runtime_input["scope"] = {
|
||||||
|
"scope_type": declared_scope,
|
||||||
|
"tenant_id": tenant_id,
|
||||||
|
"project_id": project_id,
|
||||||
|
}
|
||||||
|
runtime_input["trace_id"] = str(
|
||||||
|
request.get("trace_id")
|
||||||
|
or request.get("run_id")
|
||||||
|
or binding.task_id
|
||||||
|
)
|
||||||
execution_value["runtime_input"] = runtime_input
|
execution_value["runtime_input"] = runtime_input
|
||||||
execution = project_execution_envelope(
|
execution = project_execution_envelope(
|
||||||
binding,
|
binding,
|
||||||
|
|||||||
@@ -236,6 +236,64 @@ def atomic_json(path: Path, value: Mapping[str, Any], mode: int = 0o640) -> None
|
|||||||
os.unlink(temporary)
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
WORKER_LIVENESS_PATH = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_WORKER_LIVENESS_PATH",
|
||||||
|
str(STATE_ROOT / "worker-liveness.json"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
_LIVENESS_LOCK = threading.Lock()
|
||||||
|
_LIVENESS_SEQUENCE = 0
|
||||||
|
ENGINEERING_WRITE_CAPABILITIES = frozenset({
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def write_worker_liveness(
|
||||||
|
state: str,
|
||||||
|
*,
|
||||||
|
visible_count: int | None = None,
|
||||||
|
task_id: str | None = None,
|
||||||
|
stage: str | None = None,
|
||||||
|
error: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Write idle/busy liveness without touching Task/Job state."""
|
||||||
|
global _LIVENESS_SEQUENCE
|
||||||
|
with _LIVENESS_LOCK:
|
||||||
|
_LIVENESS_SEQUENCE += 1
|
||||||
|
epoch = time.time()
|
||||||
|
payload = {
|
||||||
|
"schema": "aios.online-agent.worker-liveness.v1",
|
||||||
|
"agent_id": AGENT_ID,
|
||||||
|
"agent_type": AGENT_TYPE,
|
||||||
|
"agent_version": AGENT_VERSION,
|
||||||
|
"pid": os.getpid(),
|
||||||
|
"state": state,
|
||||||
|
"sequence": _LIVENESS_SEQUENCE,
|
||||||
|
"last_seen_epoch": epoch,
|
||||||
|
"last_seen_at": datetime.fromtimestamp(epoch, timezone.utc).isoformat(),
|
||||||
|
"poll_seconds": POLL_SECONDS,
|
||||||
|
"heartbeat_seconds": HEARTBEAT_SECONDS,
|
||||||
|
"api_base": API_BASE,
|
||||||
|
"bridge_base": BRIDGE_BASE,
|
||||||
|
"visible_count": visible_count,
|
||||||
|
"task_id": task_id,
|
||||||
|
"stage": stage,
|
||||||
|
"error": error,
|
||||||
|
}
|
||||||
|
atomic_json(WORKER_LIVENESS_PATH, payload)
|
||||||
|
|
||||||
|
|
||||||
|
def source_capture_requires_deterministic_plan(envelope: "ExecutionEnvelope") -> bool:
|
||||||
|
"""Treat write-capable source_capture as a binding placeholder, never executable work."""
|
||||||
|
return (
|
||||||
|
envelope.action == "source_capture"
|
||||||
|
and bool(ENGINEERING_WRITE_CAPABILITIES.intersection(envelope.capability_scope))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def safe_relative_path(value: Any, *, field: str = "path") -> str:
|
def safe_relative_path(value: Any, *, field: str = "path") -> str:
|
||||||
if not isinstance(value, str) or not value or len(value) > 240:
|
if not isinstance(value, str) or not value or len(value) > 240:
|
||||||
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
|
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
|
||||||
@@ -525,11 +583,44 @@ class ExecutionEnvelope:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def external_agent_request_id(
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
runtime_name: str,
|
||||||
|
attempt: int,
|
||||||
|
) -> str:
|
||||||
|
if not isinstance(attempt, int) or isinstance(attempt, bool) or attempt < 1:
|
||||||
|
raise WorkerError("TASK_ATTEMPT_INVALID", phase="external_agent")
|
||||||
|
if runtime_name not in {"hermes", "openclaw"}:
|
||||||
|
raise WorkerError(
|
||||||
|
"EXTERNAL_AGENT_RUNTIME_UNSUPPORTED",
|
||||||
|
phase="external_agent",
|
||||||
|
details={"runtime": runtime_name},
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
f"external-{runtime_name}-"
|
||||||
|
+ digest_json(
|
||||||
|
{
|
||||||
|
"task_id": envelope.task_id,
|
||||||
|
"job_id": envelope.job_id,
|
||||||
|
"execution_digest": envelope.digest,
|
||||||
|
"runtime": runtime_name,
|
||||||
|
"attempt": attempt,
|
||||||
|
}
|
||||||
|
)[:32]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class RuntimeAdapter(ABC):
|
class RuntimeAdapter(ABC):
|
||||||
name: str
|
name: str
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
@@ -550,7 +641,13 @@ class DeterministicExecutor(RuntimeAdapter):
|
|||||||
if os.path.exists(temporary):
|
if os.path.exists(temporary):
|
||||||
os.unlink(temporary)
|
os.unlink(temporary)
|
||||||
|
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
changed: set[str] = set()
|
changed: set[str] = set()
|
||||||
for change in envelope.change_set:
|
for change in envelope.change_set:
|
||||||
relative = str(change["path"])
|
relative = str(change["path"])
|
||||||
@@ -605,7 +702,13 @@ class SourceCaptureExecutor(RuntimeAdapter):
|
|||||||
"governed", "trace", "tool", "credential", "scope", "message", "handoff",
|
"governed", "trace", "tool", "credential", "scope", "message", "handoff",
|
||||||
}
|
}
|
||||||
|
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
terms = set(self.defaults)
|
terms = set(self.defaults)
|
||||||
for signal in envelope.source_signals:
|
for signal in envelope.source_signals:
|
||||||
terms.update(token.lower() for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]{2,63}", signal))
|
terms.update(token.lower() for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]{2,63}", signal))
|
||||||
@@ -674,7 +777,13 @@ class CompactInterfaceMapExecutor(RuntimeAdapter):
|
|||||||
|
|
||||||
name = "compact_interface_map"
|
name = "compact_interface_map"
|
||||||
|
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
if envelope.action != "source_capture":
|
if envelope.action != "source_capture":
|
||||||
raise WorkerError("COMPACT_INTERFACE_MAP_ACTION_INVALID", phase="interface_map")
|
raise WorkerError("COMPACT_INTERFACE_MAP_ACTION_INVALID", phase="interface_map")
|
||||||
try:
|
try:
|
||||||
@@ -704,7 +813,13 @@ class CompactInterfaceMapExecutor(RuntimeAdapter):
|
|||||||
class CodexExecutor(RuntimeAdapter):
|
class CodexExecutor(RuntimeAdapter):
|
||||||
name = "codex"
|
name = "codex"
|
||||||
|
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
raise WorkerError(
|
raise WorkerError(
|
||||||
"CODEX_ADAPTER_REQUIRES_EXPLICIT_POLICY",
|
"CODEX_ADAPTER_REQUIRES_EXPLICIT_POLICY",
|
||||||
phase="adapter",
|
phase="adapter",
|
||||||
@@ -715,7 +830,13 @@ class CodexExecutor(RuntimeAdapter):
|
|||||||
class ExternalAgentExecutor(RuntimeAdapter):
|
class ExternalAgentExecutor(RuntimeAdapter):
|
||||||
runtime_name: str
|
runtime_name: str
|
||||||
|
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
if envelope.action != "agent_runtime":
|
if envelope.action != "agent_runtime":
|
||||||
raise WorkerError(
|
raise WorkerError(
|
||||||
"EXTERNAL_AGENT_ACTION_INVALID",
|
"EXTERNAL_AGENT_ACTION_INVALID",
|
||||||
@@ -729,16 +850,10 @@ class ExternalAgentExecutor(RuntimeAdapter):
|
|||||||
)
|
)
|
||||||
if not envelope.runtime_input:
|
if not envelope.runtime_input:
|
||||||
raise WorkerError("EXTERNAL_AGENT_RUNTIME_INPUT_MISSING", phase="adapter")
|
raise WorkerError("EXTERNAL_AGENT_RUNTIME_INPUT_MISSING", phase="adapter")
|
||||||
request_id = (
|
request_id = external_agent_request_id(
|
||||||
f"external-{self.runtime_name}-"
|
envelope,
|
||||||
+ digest_json(
|
self.runtime_name,
|
||||||
{
|
attempt,
|
||||||
"task_id": envelope.task_id,
|
|
||||||
"job_id": envelope.job_id,
|
|
||||||
"execution_digest": envelope.digest,
|
|
||||||
"runtime": self.runtime_name,
|
|
||||||
}
|
|
||||||
)[:32]
|
|
||||||
)
|
)
|
||||||
exchange = ExternalAgentExchange(EXTERNAL_AGENT_EXECUTOR_ROOT)
|
exchange = ExternalAgentExchange(EXTERNAL_AGENT_EXECUTOR_ROOT)
|
||||||
try:
|
try:
|
||||||
@@ -746,6 +861,7 @@ class ExternalAgentExecutor(RuntimeAdapter):
|
|||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
runtime=self.runtime_name,
|
runtime=self.runtime_name,
|
||||||
task_id=envelope.task_id,
|
task_id=envelope.task_id,
|
||||||
|
attempt=attempt,
|
||||||
execution_digest=envelope.digest,
|
execution_digest=envelope.digest,
|
||||||
payload=envelope.runtime_input,
|
payload=envelope.runtime_input,
|
||||||
timeout_seconds=EXTERNAL_AGENT_EXECUTOR_TIMEOUT_SECONDS,
|
timeout_seconds=EXTERNAL_AGENT_EXECUTOR_TIMEOUT_SECONDS,
|
||||||
@@ -761,6 +877,7 @@ class ExternalAgentExecutor(RuntimeAdapter):
|
|||||||
task_id=envelope.task_id,
|
task_id=envelope.task_id,
|
||||||
adapter=self.runtime_name,
|
adapter=self.runtime_name,
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
|
attempt=attempt,
|
||||||
output_sha256=external.get("output_sha256"),
|
output_sha256=external.get("output_sha256"),
|
||||||
duration_ms=external.get("duration_ms"),
|
duration_ms=external.get("duration_ms"),
|
||||||
credential_material_exposed=False,
|
credential_material_exposed=False,
|
||||||
@@ -890,6 +1007,11 @@ class LeaseHeartbeat:
|
|||||||
self.failed = str(response.get("error") or status)
|
self.failed = str(response.get("error") or status)
|
||||||
raise WorkerError("TASK_HEARTBEAT_FAILED", phase="heartbeat", details={"api_status": status, "error": self.failed})
|
raise WorkerError("TASK_HEARTBEAT_FAILED", phase="heartbeat", details={"api_status": status, "error": self.failed})
|
||||||
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=self.stage, api_status=status)
|
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=self.stage, api_status=status)
|
||||||
|
write_worker_liveness(
|
||||||
|
"BUSY",
|
||||||
|
task_id=self.task_id,
|
||||||
|
stage=self.stage,
|
||||||
|
)
|
||||||
|
|
||||||
def start(self) -> None:
|
def start(self) -> None:
|
||||||
self.thread.start()
|
self.thread.start()
|
||||||
@@ -909,6 +1031,11 @@ class LeaseHeartbeat:
|
|||||||
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, api_status=status, error=self.failed)
|
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, api_status=status, error=self.failed)
|
||||||
else:
|
else:
|
||||||
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=stage, api_status=status)
|
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=stage, api_status=status)
|
||||||
|
write_worker_liveness(
|
||||||
|
"BUSY",
|
||||||
|
task_id=self.task_id,
|
||||||
|
stage=stage,
|
||||||
|
)
|
||||||
except Exception as exc: # pragma: no cover - defensive runtime guard
|
except Exception as exc: # pragma: no cover - defensive runtime guard
|
||||||
self.failed = type(exc).__name__
|
self.failed = type(exc).__name__
|
||||||
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, error=self.failed)
|
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, error=self.failed)
|
||||||
@@ -1286,7 +1413,13 @@ class IsolatedDeterministicExecutor(RuntimeAdapter):
|
|||||||
if os.path.exists(temporary):
|
if os.path.exists(temporary):
|
||||||
os.unlink(temporary)
|
os.unlink(temporary)
|
||||||
|
|
||||||
def execute(self, envelope: ExecutionEnvelope, worktree: Path) -> dict[str, Any]:
|
def execute(
|
||||||
|
self,
|
||||||
|
envelope: ExecutionEnvelope,
|
||||||
|
worktree: Path,
|
||||||
|
*,
|
||||||
|
attempt: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
if envelope.action != "deterministic":
|
if envelope.action != "deterministic":
|
||||||
raise WorkerError("ISOLATED_EXECUTOR_ACTION_INVALID", phase="adapter")
|
raise WorkerError("ISOLATED_EXECUTOR_ACTION_INVALID", phase="adapter")
|
||||||
root = ISOLATED_EXECUTOR_ROOT
|
root = ISOLATED_EXECUTOR_ROOT
|
||||||
@@ -1772,12 +1905,25 @@ class WorkerSupervisor:
|
|||||||
if not isinstance(execution, Mapping):
|
if not isinstance(execution, Mapping):
|
||||||
raise WorkerError("EXECUTION_ENVELOPE_SCHEMA_VALIDATION", phase="validation")
|
raise WorkerError("EXECUTION_ENVELOPE_SCHEMA_VALIDATION", phase="validation")
|
||||||
persisted = ExecutionEnvelope.from_task(task, worker_capabilities=WORKER_CAPABILITIES)
|
persisted = ExecutionEnvelope.from_task(task, worker_capabilities=WORKER_CAPABILITIES)
|
||||||
# Deterministic tasks must re-resolve through the plan registry. A
|
# Read-only source_capture/agent_runtime tasks replay their persisted
|
||||||
# read-only source-capture task has no plan to replace its payload: replay
|
# payload. A write-capable source_capture is only a Bridge-created
|
||||||
# the already validated durable envelope so source signals remain part of
|
# engineering placeholder: never pass that stale payload back to the
|
||||||
# the canonical digest and the worker does not invent a different capture.
|
# resolver, and never claim it until a deterministic plan exists.
|
||||||
projection_input = execution if persisted.action == "source_capture" else None
|
pending_plan = source_capture_requires_deterministic_plan(persisted)
|
||||||
|
projection_input = (
|
||||||
|
execution
|
||||||
|
if persisted.action in {"source_capture", "agent_runtime"} and not pending_plan
|
||||||
|
else None
|
||||||
|
)
|
||||||
status, authoritative = self.bridge.project_execution(task, projection_input)
|
status, authoritative = self.bridge.project_execution(task, projection_input)
|
||||||
|
if pending_plan and (
|
||||||
|
status == 404
|
||||||
|
or authoritative.get("error") == "TASK_EXECUTION_PROJECTION_MISSING"
|
||||||
|
):
|
||||||
|
raise WorkerError(
|
||||||
|
"TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN",
|
||||||
|
phase="projection",
|
||||||
|
)
|
||||||
if not 200 <= status < 300:
|
if not 200 <= status < 300:
|
||||||
raise WorkerError("TASK_PERSISTED_BINDING_UNVERIFIED", phase="projection", details={"api_status": status, "error": authoritative.get("error")})
|
raise WorkerError("TASK_PERSISTED_BINDING_UNVERIFIED", phase="projection", details={"api_status": status, "error": authoritative.get("error")})
|
||||||
canonical = ExecutionEnvelope.from_task(task, authoritative, WORKER_CAPABILITIES)
|
canonical = ExecutionEnvelope.from_task(task, authoritative, WORKER_CAPABILITIES)
|
||||||
@@ -1806,6 +1952,11 @@ class WorkerSupervisor:
|
|||||||
execution_digest=canonical.digest,
|
execution_digest=canonical.digest,
|
||||||
)
|
)
|
||||||
return canonical, "plan_registry_verified"
|
return canonical, "plan_registry_verified"
|
||||||
|
if pending_plan:
|
||||||
|
raise WorkerError(
|
||||||
|
"TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN",
|
||||||
|
phase="projection",
|
||||||
|
)
|
||||||
return persisted, "persisted_verified"
|
return persisted, "persisted_verified"
|
||||||
status, projected = self.bridge.project_execution(task)
|
status, projected = self.bridge.project_execution(task)
|
||||||
if status == 404 or projected.get("error") == "TASK_EXECUTION_PROJECTION_MISSING":
|
if status == 404 or projected.get("error") == "TASK_EXECUTION_PROJECTION_MISSING":
|
||||||
@@ -2107,7 +2258,11 @@ class WorkerSupervisor:
|
|||||||
save_task_state(task_id, state)
|
save_task_state(task_id, state)
|
||||||
save_checkpoint(task_id, state, envelope, attempt)
|
save_checkpoint(task_id, state, envelope, attempt)
|
||||||
adapter = adapter_for(envelope.adapter)
|
adapter = adapter_for(envelope.adapter)
|
||||||
execution = adapter.execute(envelope, worktree)
|
execution = adapter.execute(
|
||||||
|
envelope,
|
||||||
|
worktree,
|
||||||
|
attempt=attempt,
|
||||||
|
)
|
||||||
state.update(
|
state.update(
|
||||||
{
|
{
|
||||||
"phase": "verification_passed",
|
"phase": "verification_passed",
|
||||||
@@ -2290,8 +2445,13 @@ class WorkerSupervisor:
|
|||||||
def once(self) -> int:
|
def once(self) -> int:
|
||||||
try:
|
try:
|
||||||
tasks = self.agent.poll()
|
tasks = self.agent.poll()
|
||||||
|
write_worker_liveness("POLL_OK", visible_count=len(tasks))
|
||||||
audit("TASK_POLL", visible_count=len(tasks))
|
audit("TASK_POLL", visible_count=len(tasks))
|
||||||
except WorkerError as exc:
|
except WorkerError as exc:
|
||||||
|
write_worker_liveness(
|
||||||
|
"POLL_FAILED",
|
||||||
|
error=exc.code,
|
||||||
|
)
|
||||||
audit("TASK_POLL_FAILED", error=exc.code, phase=exc.phase, details=exc.details)
|
audit("TASK_POLL_FAILED", error=exc.code, phase=exc.phase, details=exc.details)
|
||||||
return 10
|
return 10
|
||||||
dynamic_tasks: list[tuple[dict[str, Any], ExecutionEnvelope]] = []
|
dynamic_tasks: list[tuple[dict[str, Any], ExecutionEnvelope]] = []
|
||||||
@@ -2300,7 +2460,11 @@ class WorkerSupervisor:
|
|||||||
envelope, source = self.resolve_envelope(task)
|
envelope, source = self.resolve_envelope(task)
|
||||||
except WorkerError as exc:
|
except WorkerError as exc:
|
||||||
event = "TASK_SKIPPED_ENVELOPE_INVALID"
|
event = "TASK_SKIPPED_ENVELOPE_INVALID"
|
||||||
if exc.code in {"TASK_SKIPPED_PROJECTION_MISSING", "TASK_SKIPPED_POLICY_DENY"}:
|
if exc.code in {
|
||||||
|
"TASK_SKIPPED_PROJECTION_MISSING",
|
||||||
|
"TASK_SKIPPED_POLICY_DENY",
|
||||||
|
"TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN",
|
||||||
|
}:
|
||||||
event = exc.code
|
event = exc.code
|
||||||
elif exc.code == "CAPABILITY_MISMATCH_DENY":
|
elif exc.code == "CAPABILITY_MISMATCH_DENY":
|
||||||
event = "TASK_SKIPPED_CAPABILITY_MISMATCH"
|
event = "TASK_SKIPPED_CAPABILITY_MISMATCH"
|
||||||
@@ -2334,6 +2498,7 @@ class WorkerSupervisor:
|
|||||||
ensure_roots()
|
ensure_roots()
|
||||||
recover_stale_workspaces()
|
recover_stale_workspaces()
|
||||||
checkpoint_store()
|
checkpoint_store()
|
||||||
|
write_worker_liveness("STARTING")
|
||||||
audit("WORKER_STARTED", task_agnostic=True, api_base=API_BASE, bridge_base=BRIDGE_BASE, adapter_names=["isolated-deterministic", "source_capture", "codex", "hermes"], runtime_only_credentials=True, isolated_executor=True, durable_checkpoint=True)
|
audit("WORKER_STARTED", task_agnostic=True, api_base=API_BASE, bridge_base=BRIDGE_BASE, adapter_names=["isolated-deterministic", "source_capture", "codex", "hermes"], runtime_only_credentials=True, isolated_executor=True, durable_checkpoint=True)
|
||||||
while True:
|
while True:
|
||||||
self.once()
|
self.once()
|
||||||
|
|||||||
@@ -0,0 +1,462 @@
|
|||||||
|
"""Self-healing availability guard for persistent Online Agent workers.
|
||||||
|
|
||||||
|
The guard never claims Tasks and never writes Task/Job/Git databases. It only:
|
||||||
|
- ensures required worker/executor systemd units are enabled + active;
|
||||||
|
- checks worker-owned liveness JSON written by the real poll/heartbeat loop;
|
||||||
|
- restarts an active worker only when its liveness is stale;
|
||||||
|
- honors an explicit maintenance inhibit file;
|
||||||
|
- writes a bounded status JSON under /run for observability.
|
||||||
|
|
||||||
|
Idle liveness is the successful poll loop itself. Busy liveness is refreshed by the
|
||||||
|
Task lease-heartbeat thread, so long-running Tasks are not mistaken for hung workers.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import shlex
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from typing import Any, Callable, Mapping, Sequence
|
||||||
|
|
||||||
|
SCHEMA = "aios.worker-availability.status.v1"
|
||||||
|
DEFAULT_WORKER_UNITS = (
|
||||||
|
"aios-online-agent-worker-20260815.service",
|
||||||
|
"aios-hermes-online-agent-worker.service",
|
||||||
|
"aios-openclaw-online-agent-worker.service",
|
||||||
|
)
|
||||||
|
DEFAULT_DEPENDENCY_UNITS = (
|
||||||
|
"aios-external-agent-executor.service",
|
||||||
|
)
|
||||||
|
DEFAULT_RUN_ROOT = Path("/run/aios-worker-availability")
|
||||||
|
DEFAULT_STATUS = DEFAULT_RUN_ROOT / "status.json"
|
||||||
|
DEFAULT_RECOVERY = DEFAULT_RUN_ROOT / "recovery.json"
|
||||||
|
DEFAULT_INHIBIT = DEFAULT_RUN_ROOT / "maintenance.lock"
|
||||||
|
DEFAULT_MAX_STALE_SECONDS = 90
|
||||||
|
DEFAULT_RESTART_COOLDOWN_SECONDS = 60
|
||||||
|
DEFAULT_STARTUP_WAIT_SECONDS = 10
|
||||||
|
ACCEPTED_ENABLED_STATES = frozenset({"enabled", "enabled-runtime", "static", "indirect", "generated"})
|
||||||
|
|
||||||
|
|
||||||
|
class AvailabilityGuardError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class UnitSnapshot:
|
||||||
|
unit: str
|
||||||
|
role: str
|
||||||
|
load_state: str
|
||||||
|
active_state: str
|
||||||
|
unit_file_state: str
|
||||||
|
enabled_ok: bool
|
||||||
|
liveness_path: str | None = None
|
||||||
|
liveness_state: str | None = None
|
||||||
|
last_seen_epoch: float | None = None
|
||||||
|
liveness_age_seconds: float | None = None
|
||||||
|
fresh: bool | None = None
|
||||||
|
action: str = "NONE"
|
||||||
|
detail: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class Systemd:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
runner: Callable[..., subprocess.CompletedProcess[str]] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self.runner = runner or subprocess.run
|
||||||
|
|
||||||
|
def _run(self, args: Sequence[str], *, check: bool = False) -> subprocess.CompletedProcess[str]:
|
||||||
|
result = self.runner(
|
||||||
|
["/usr/bin/systemctl", *args],
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
timeout=20,
|
||||||
|
)
|
||||||
|
if check and result.returncode != 0:
|
||||||
|
raise AvailabilityGuardError(
|
||||||
|
"SYSTEMD_COMMAND_FAILED:"
|
||||||
|
+ ":".join(args)
|
||||||
|
+ ":"
|
||||||
|
+ str(result.returncode)
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def property(self, unit: str, name: str) -> str:
|
||||||
|
result = self._run(["show", unit, f"--property={name}", "--value"])
|
||||||
|
if result.returncode != 0:
|
||||||
|
return ""
|
||||||
|
return result.stdout.strip()
|
||||||
|
|
||||||
|
def environment(self, unit: str) -> dict[str, str]:
|
||||||
|
text = self.property(unit, "Environment")
|
||||||
|
result: dict[str, str] = {}
|
||||||
|
for item in shlex.split(text):
|
||||||
|
if "=" not in item:
|
||||||
|
continue
|
||||||
|
key, value = item.split("=", 1)
|
||||||
|
result[key] = value
|
||||||
|
return result
|
||||||
|
|
||||||
|
def enable(self, unit: str) -> None:
|
||||||
|
self._run(["enable", unit], check=True)
|
||||||
|
|
||||||
|
def start(self, unit: str) -> None:
|
||||||
|
self._run(["start", unit], check=True)
|
||||||
|
|
||||||
|
def restart(self, unit: str) -> None:
|
||||||
|
self._run(["restart", unit], check=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_json(path: Path, value: Mapping[str, Any]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o755)
|
||||||
|
if path.parent.is_symlink() or path.is_symlink():
|
||||||
|
raise AvailabilityGuardError("STATUS_PATH_SYMLINK_REJECTED")
|
||||||
|
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||||
|
try:
|
||||||
|
os.fchmod(fd, 0o644)
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(value, handle, ensure_ascii=False, sort_keys=True, indent=2)
|
||||||
|
handle.write("\n")
|
||||||
|
handle.flush()
|
||||||
|
os.fsync(handle.fileno())
|
||||||
|
os.replace(temporary, path)
|
||||||
|
os.chmod(path, 0o644)
|
||||||
|
finally:
|
||||||
|
if os.path.exists(temporary):
|
||||||
|
os.unlink(temporary)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_json(path: Path) -> dict[str, Any] | None:
|
||||||
|
if path.is_symlink() or not path.is_file():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
value = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return None
|
||||||
|
return dict(value) if isinstance(value, Mapping) else None
|
||||||
|
|
||||||
|
|
||||||
|
def _state_root(environment: Mapping[str, str], unit: str) -> Path:
|
||||||
|
explicit = environment.get("AIOS_WORKER_STATE_ROOT")
|
||||||
|
if explicit:
|
||||||
|
return Path(explicit)
|
||||||
|
fallbacks = {
|
||||||
|
"aios-online-agent-worker-20260815.service": Path(
|
||||||
|
"/var/lib/aios-online-agent-worker-20260815"
|
||||||
|
),
|
||||||
|
"aios-hermes-online-agent-worker.service": Path(
|
||||||
|
"/var/lib/aios-online-agent-hermes"
|
||||||
|
),
|
||||||
|
"aios-openclaw-online-agent-worker.service": Path(
|
||||||
|
"/var/lib/aios-online-agent-openclaw"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
if unit not in fallbacks:
|
||||||
|
raise AvailabilityGuardError("WORKER_STATE_ROOT_UNAVAILABLE:" + unit)
|
||||||
|
return fallbacks[unit]
|
||||||
|
|
||||||
|
|
||||||
|
def _liveness_path(environment: Mapping[str, str], unit: str) -> Path:
|
||||||
|
explicit = environment.get("AIOS_WORKER_LIVENESS_PATH")
|
||||||
|
if explicit:
|
||||||
|
return Path(explicit)
|
||||||
|
return _state_root(environment, unit) / "worker-liveness.json"
|
||||||
|
|
||||||
|
|
||||||
|
class AvailabilityGuard:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
systemd: Systemd | None = None,
|
||||||
|
worker_units: Sequence[str] = DEFAULT_WORKER_UNITS,
|
||||||
|
dependency_units: Sequence[str] = DEFAULT_DEPENDENCY_UNITS,
|
||||||
|
status_path: Path = DEFAULT_STATUS,
|
||||||
|
recovery_path: Path = DEFAULT_RECOVERY,
|
||||||
|
inhibit_path: Path = DEFAULT_INHIBIT,
|
||||||
|
max_stale_seconds: int = DEFAULT_MAX_STALE_SECONDS,
|
||||||
|
restart_cooldown_seconds: int = DEFAULT_RESTART_COOLDOWN_SECONDS,
|
||||||
|
startup_wait_seconds: int = DEFAULT_STARTUP_WAIT_SECONDS,
|
||||||
|
now_fn: Callable[[], float] = time.time,
|
||||||
|
sleep_fn: Callable[[float], None] = time.sleep,
|
||||||
|
) -> None:
|
||||||
|
self.systemd = systemd or Systemd()
|
||||||
|
self.worker_units = tuple(worker_units)
|
||||||
|
self.dependency_units = tuple(dependency_units)
|
||||||
|
self.status_path = Path(status_path)
|
||||||
|
self.recovery_path = Path(recovery_path)
|
||||||
|
self.inhibit_path = Path(inhibit_path)
|
||||||
|
self.max_stale_seconds = int(max_stale_seconds)
|
||||||
|
self.restart_cooldown_seconds = int(restart_cooldown_seconds)
|
||||||
|
self.startup_wait_seconds = int(startup_wait_seconds)
|
||||||
|
self.now_fn = now_fn
|
||||||
|
self.sleep_fn = sleep_fn
|
||||||
|
if self.max_stale_seconds < 15:
|
||||||
|
raise AvailabilityGuardError("MAX_STALE_SECONDS_TOO_SMALL")
|
||||||
|
if self.restart_cooldown_seconds < 15:
|
||||||
|
raise AvailabilityGuardError("RESTART_COOLDOWN_TOO_SMALL")
|
||||||
|
if self.startup_wait_seconds < 0 or self.startup_wait_seconds > 60:
|
||||||
|
raise AvailabilityGuardError("STARTUP_WAIT_INVALID")
|
||||||
|
|
||||||
|
def _recovery_state(self) -> dict[str, float]:
|
||||||
|
value = _load_json(self.recovery_path) or {}
|
||||||
|
raw = value.get("last_restart_epoch")
|
||||||
|
if not isinstance(raw, Mapping):
|
||||||
|
return {}
|
||||||
|
result: dict[str, float] = {}
|
||||||
|
for unit, epoch in raw.items():
|
||||||
|
try:
|
||||||
|
result[str(unit)] = float(epoch)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _save_recovery_state(self, state: Mapping[str, float]) -> None:
|
||||||
|
_atomic_json(
|
||||||
|
self.recovery_path,
|
||||||
|
{
|
||||||
|
"schema": "aios.worker-availability.recovery.v1",
|
||||||
|
"last_restart_epoch": dict(state),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _basic_snapshot(self, unit: str, role: str) -> tuple[str, str, str, bool]:
|
||||||
|
load = self.systemd.property(unit, "LoadState")
|
||||||
|
active = self.systemd.property(unit, "ActiveState")
|
||||||
|
enabled = self.systemd.property(unit, "UnitFileState")
|
||||||
|
return load, active, enabled, enabled in ACCEPTED_ENABLED_STATES
|
||||||
|
|
||||||
|
def _ensure_enabled_active(self, unit: str, role: str) -> tuple[str, bool]:
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, role)
|
||||||
|
if load != "loaded":
|
||||||
|
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
|
||||||
|
action = "NONE"
|
||||||
|
if not enabled_ok:
|
||||||
|
self.systemd.enable(unit)
|
||||||
|
action = "ENABLE"
|
||||||
|
if active != "active":
|
||||||
|
self.systemd.start(unit)
|
||||||
|
action = "START" if action == "NONE" else "ENABLE_START"
|
||||||
|
return action, active == "active"
|
||||||
|
|
||||||
|
def _wait_for_liveness(self, path: Path) -> dict[str, Any] | None:
|
||||||
|
deadline = self.now_fn() + self.startup_wait_seconds
|
||||||
|
while True:
|
||||||
|
value = _load_json(path)
|
||||||
|
if value is not None:
|
||||||
|
try:
|
||||||
|
epoch = float(value.get("last_seen_epoch"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
epoch = 0.0
|
||||||
|
if epoch > 0 and self.now_fn() - epoch <= self.max_stale_seconds:
|
||||||
|
return value
|
||||||
|
if self.now_fn() >= deadline:
|
||||||
|
return value
|
||||||
|
self.sleep_fn(0.5)
|
||||||
|
|
||||||
|
def run(self, *, heal: bool) -> dict[str, Any]:
|
||||||
|
now = self.now_fn()
|
||||||
|
if self.inhibit_path.exists():
|
||||||
|
result = {
|
||||||
|
"schema": SCHEMA,
|
||||||
|
"status": "MAINTENANCE_INHIBITED",
|
||||||
|
"generated_at_epoch": now,
|
||||||
|
"maintenance_inhibit": str(self.inhibit_path),
|
||||||
|
"workers": [],
|
||||||
|
"dependencies": [],
|
||||||
|
"all_fresh": False,
|
||||||
|
"actions": [],
|
||||||
|
}
|
||||||
|
_atomic_json(self.status_path, result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
actions: list[dict[str, str]] = []
|
||||||
|
dependencies: list[UnitSnapshot] = []
|
||||||
|
for unit in self.dependency_units:
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "dependency")
|
||||||
|
action = "NONE"
|
||||||
|
if heal:
|
||||||
|
if load != "loaded":
|
||||||
|
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
|
||||||
|
if not enabled_ok:
|
||||||
|
self.systemd.enable(unit)
|
||||||
|
action = "ENABLE"
|
||||||
|
if active != "active":
|
||||||
|
self.systemd.start(unit)
|
||||||
|
action = "START" if action == "NONE" else "ENABLE_START"
|
||||||
|
if action != "NONE":
|
||||||
|
actions.append({"unit": unit, "action": action})
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "dependency")
|
||||||
|
dependencies.append(
|
||||||
|
UnitSnapshot(
|
||||||
|
unit=unit,
|
||||||
|
role="dependency",
|
||||||
|
load_state=load,
|
||||||
|
active_state=active,
|
||||||
|
unit_file_state=enabled,
|
||||||
|
enabled_ok=enabled_ok,
|
||||||
|
fresh=(active == "active"),
|
||||||
|
action=action,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
recovery = self._recovery_state()
|
||||||
|
workers: list[UnitSnapshot] = []
|
||||||
|
for unit in self.worker_units:
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
|
||||||
|
environment = self.systemd.environment(unit) if load == "loaded" else {}
|
||||||
|
path = _liveness_path(environment, unit) if load == "loaded" else None
|
||||||
|
action = "NONE"
|
||||||
|
|
||||||
|
if heal:
|
||||||
|
if load != "loaded":
|
||||||
|
raise AvailabilityGuardError("REQUIRED_UNIT_NOT_LOADED:" + unit)
|
||||||
|
if not enabled_ok:
|
||||||
|
self.systemd.enable(unit)
|
||||||
|
action = "ENABLE"
|
||||||
|
if active != "active":
|
||||||
|
self.systemd.start(unit)
|
||||||
|
action = "START" if action == "NONE" else "ENABLE_START"
|
||||||
|
actions.append({"unit": unit, "action": action})
|
||||||
|
if path is not None:
|
||||||
|
self._wait_for_liveness(path)
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
|
||||||
|
|
||||||
|
live = _load_json(path) if path is not None else None
|
||||||
|
last_seen: float | None = None
|
||||||
|
state: str | None = None
|
||||||
|
age: float | None = None
|
||||||
|
fresh = False
|
||||||
|
detail: str | None = None
|
||||||
|
if live is not None:
|
||||||
|
state = str(live.get("state") or "")
|
||||||
|
try:
|
||||||
|
last_seen = float(live.get("last_seen_epoch"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
last_seen = None
|
||||||
|
if last_seen is not None:
|
||||||
|
age = max(0.0, now - last_seen)
|
||||||
|
fresh = age <= self.max_stale_seconds
|
||||||
|
if live.get("agent_id") in (None, ""):
|
||||||
|
fresh = False
|
||||||
|
detail = "LIVENESS_AGENT_ID_MISSING"
|
||||||
|
else:
|
||||||
|
detail = "LIVENESS_MISSING"
|
||||||
|
|
||||||
|
if heal and active == "active" and not fresh:
|
||||||
|
previous = float(recovery.get(unit, 0.0))
|
||||||
|
if now - previous >= self.restart_cooldown_seconds:
|
||||||
|
self.systemd.restart(unit)
|
||||||
|
recovery[unit] = now
|
||||||
|
action = "RESTART_STALE"
|
||||||
|
actions.append({"unit": unit, "action": action})
|
||||||
|
if path is not None:
|
||||||
|
live = self._wait_for_liveness(path)
|
||||||
|
load, active, enabled, enabled_ok = self._basic_snapshot(unit, "worker")
|
||||||
|
if live is not None:
|
||||||
|
state = str(live.get("state") or "")
|
||||||
|
try:
|
||||||
|
last_seen = float(live.get("last_seen_epoch"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
last_seen = None
|
||||||
|
if last_seen is not None:
|
||||||
|
age = max(0.0, self.now_fn() - last_seen)
|
||||||
|
fresh = age <= self.max_stale_seconds
|
||||||
|
detail = None if fresh else "LIVENESS_STALE_AFTER_RESTART"
|
||||||
|
else:
|
||||||
|
detail = "RESTART_COOLDOWN_ACTIVE"
|
||||||
|
|
||||||
|
workers.append(
|
||||||
|
UnitSnapshot(
|
||||||
|
unit=unit,
|
||||||
|
role="worker",
|
||||||
|
load_state=load,
|
||||||
|
active_state=active,
|
||||||
|
unit_file_state=enabled,
|
||||||
|
enabled_ok=enabled_ok,
|
||||||
|
liveness_path=str(path) if path is not None else None,
|
||||||
|
liveness_state=state,
|
||||||
|
last_seen_epoch=last_seen,
|
||||||
|
liveness_age_seconds=age,
|
||||||
|
fresh=bool(active == "active" and enabled_ok and fresh),
|
||||||
|
action=action,
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if heal:
|
||||||
|
self._save_recovery_state(recovery)
|
||||||
|
|
||||||
|
all_fresh = (
|
||||||
|
all(item.active_state == "active" and item.enabled_ok and item.fresh for item in workers)
|
||||||
|
and all(item.active_state == "active" for item in dependencies)
|
||||||
|
)
|
||||||
|
result = {
|
||||||
|
"schema": SCHEMA,
|
||||||
|
"status": "HEALTHY" if all_fresh else "DEGRADED",
|
||||||
|
"generated_at_epoch": self.now_fn(),
|
||||||
|
"max_stale_seconds": self.max_stale_seconds,
|
||||||
|
"maintenance_inhibit": str(self.inhibit_path),
|
||||||
|
"workers": [asdict(item) for item in workers],
|
||||||
|
"dependencies": [asdict(item) for item in dependencies],
|
||||||
|
"all_fresh": all_fresh,
|
||||||
|
"actions": actions,
|
||||||
|
}
|
||||||
|
_atomic_json(self.status_path, result)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
mode = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
mode.add_argument("--heal", action="store_true")
|
||||||
|
mode.add_argument("--check", action="store_true")
|
||||||
|
parser.add_argument("--status-file", default=str(DEFAULT_STATUS))
|
||||||
|
parser.add_argument("--recovery-file", default=str(DEFAULT_RECOVERY))
|
||||||
|
parser.add_argument("--inhibit-file", default=str(DEFAULT_INHIBIT))
|
||||||
|
parser.add_argument(
|
||||||
|
"--max-stale-seconds",
|
||||||
|
type=int,
|
||||||
|
default=int(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS",
|
||||||
|
str(DEFAULT_MAX_STALE_SECONDS),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--restart-cooldown-seconds",
|
||||||
|
type=int,
|
||||||
|
default=int(
|
||||||
|
os.environ.get(
|
||||||
|
"AIOS_WORKER_AVAILABILITY_RESTART_COOLDOWN_SECONDS",
|
||||||
|
str(DEFAULT_RESTART_COOLDOWN_SECONDS),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
guard = AvailabilityGuard(
|
||||||
|
status_path=Path(args.status_file),
|
||||||
|
recovery_path=Path(args.recovery_file),
|
||||||
|
inhibit_path=Path(args.inhibit_file),
|
||||||
|
max_stale_seconds=args.max_stale_seconds,
|
||||||
|
restart_cooldown_seconds=args.restart_cooldown_seconds,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
result = guard.run(heal=args.heal)
|
||||||
|
except AvailabilityGuardError as exc:
|
||||||
|
print(json.dumps({"status": "FAILED", "error": str(exc)}, sort_keys=True))
|
||||||
|
return 20
|
||||||
|
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
||||||
|
if args.check and not result.get("all_fresh"):
|
||||||
|
return 10
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -6,7 +6,7 @@
|
|||||||
"--provider",
|
"--provider",
|
||||||
"oneapi",
|
"oneapi",
|
||||||
"-m",
|
"-m",
|
||||||
"deepseek-v4-flash",
|
"{model_id}",
|
||||||
"--no-restore-cwd"
|
"--no-restore-cwd"
|
||||||
],
|
],
|
||||||
"credential_env": {
|
"credential_env": {
|
||||||
@@ -21,6 +21,16 @@
|
|||||||
},
|
},
|
||||||
"input_mode": "stdin_text",
|
"input_mode": "stdin_text",
|
||||||
"max_output_bytes": 131072,
|
"max_output_bytes": 131072,
|
||||||
|
"model_governance": {
|
||||||
|
"max_output_tokens": 512,
|
||||||
|
"model_capabilities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"provider_ids": [
|
||||||
|
"oneapi"
|
||||||
|
],
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
"output_mode": "stdout_text",
|
"output_mode": "stdout_text",
|
||||||
"runtime": "hermes",
|
"runtime": "hermes",
|
||||||
"schema": "aios.external-agent-runtime.profile.v1",
|
"schema": "aios.external-agent-runtime.profile.v1",
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
"--message",
|
"--message",
|
||||||
"{prompt}",
|
"{prompt}",
|
||||||
"--model",
|
"--model",
|
||||||
"openai/deepseek-v4-flash",
|
"{model_id}",
|
||||||
"--timeout",
|
"--timeout",
|
||||||
"180"
|
"180"
|
||||||
],
|
],
|
||||||
@@ -30,6 +30,17 @@
|
|||||||
},
|
},
|
||||||
"input_mode": "stdin_text",
|
"input_mode": "stdin_text",
|
||||||
"max_output_bytes": 131072,
|
"max_output_bytes": 131072,
|
||||||
|
"model_governance": {
|
||||||
|
"input_reservation_floor_tokens": 16384,
|
||||||
|
"max_output_tokens": 256,
|
||||||
|
"model_capabilities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"provider_ids": [
|
||||||
|
"openai"
|
||||||
|
],
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
"output_mode": "stdout_json",
|
"output_mode": "stdout_json",
|
||||||
"runtime": "openclaw",
|
"runtime": "openclaw",
|
||||||
"schema": "aios.external-agent-runtime.profile.v1",
|
"schema": "aios.external-agent-runtime.profile.v1",
|
||||||
|
|||||||
@@ -0,0 +1,95 @@
|
|||||||
|
{
|
||||||
|
"models": [
|
||||||
|
{
|
||||||
|
"capabilities": [
|
||||||
|
"agent.runtime.hermes",
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"enabled": true,
|
||||||
|
"evaluation_sha256": "43ae0db1640daeb6d33e2787769732844f366d36070065d3a2239905bd9a823b",
|
||||||
|
"evaluation_state": "APPROVED",
|
||||||
|
"input_cost_micros_per_1k": 0,
|
||||||
|
"max_input_tokens": 32768,
|
||||||
|
"max_output_tokens": 512,
|
||||||
|
"model_id": "deepseek-v4-flash",
|
||||||
|
"output_cost_micros_per_1k": 0,
|
||||||
|
"priority": 10,
|
||||||
|
"provider_id": "oneapi",
|
||||||
|
"version": "profile-sha:54615e533bfdb6a2e8b870ae74bb8cd7fc45177495bcf6e8c45a50837cefb85a"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"capabilities": [
|
||||||
|
"agent.runtime.openclaw",
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"enabled": true,
|
||||||
|
"evaluation_sha256": "7b1c89634eb9c0e7cb5a3d75323d904ebbd91b3ae5494ce938bd8316a4cf5b12",
|
||||||
|
"evaluation_state": "APPROVED",
|
||||||
|
"input_cost_micros_per_1k": 0,
|
||||||
|
"max_input_tokens": 32768,
|
||||||
|
"max_output_tokens": 256,
|
||||||
|
"model_id": "openai/deepseek-v4-flash",
|
||||||
|
"output_cost_micros_per_1k": 0,
|
||||||
|
"priority": 10,
|
||||||
|
"provider_id": "openai",
|
||||||
|
"version": "profile-sha:2a31c480226aa78d44ed9692bddcedc99c3aa03c23c0dd5f031a5cbbb8bbd7ba"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"policies": [
|
||||||
|
{
|
||||||
|
"allowed_models": [
|
||||||
|
"deepseek-v4-flash",
|
||||||
|
"openai/deepseek-v4-flash"
|
||||||
|
],
|
||||||
|
"daily_cost_budget_micros": 0,
|
||||||
|
"daily_token_budget": 250000,
|
||||||
|
"denied_providers": [],
|
||||||
|
"fallback_limit": 1,
|
||||||
|
"max_input_tokens_per_request": 16384,
|
||||||
|
"max_output_tokens_per_request": 512,
|
||||||
|
"project_id": null,
|
||||||
|
"required_capabilities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"scope_type": "legacy",
|
||||||
|
"tenant_id": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"allowed_models": [
|
||||||
|
"deepseek-v4-flash",
|
||||||
|
"openai/deepseek-v4-flash"
|
||||||
|
],
|
||||||
|
"daily_cost_budget_micros": 0,
|
||||||
|
"daily_token_budget": 250000,
|
||||||
|
"denied_providers": [],
|
||||||
|
"fallback_limit": 1,
|
||||||
|
"max_input_tokens_per_request": 16384,
|
||||||
|
"max_output_tokens_per_request": 512,
|
||||||
|
"project_id": "staging",
|
||||||
|
"required_capabilities": [
|
||||||
|
"text"
|
||||||
|
],
|
||||||
|
"scope_type": "tenant",
|
||||||
|
"tenant_id": "model-governance-live"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"providers": [
|
||||||
|
{
|
||||||
|
"cooldown_seconds": 60,
|
||||||
|
"credential_ref": "runtime-profile:hermes",
|
||||||
|
"enabled": true,
|
||||||
|
"failure_domain": "oneapi",
|
||||||
|
"max_failures": 2,
|
||||||
|
"provider_id": "oneapi"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cooldown_seconds": 60,
|
||||||
|
"credential_ref": "runtime-profile:openclaw",
|
||||||
|
"enabled": true,
|
||||||
|
"failure_domain": "oneapi",
|
||||||
|
"max_failures": 2,
|
||||||
|
"provider_id": "openai"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"schema": "aios.model-governance.bootstrap.v1"
|
||||||
|
}
|
||||||
@@ -33,7 +33,10 @@ InaccessiblePaths=/data/agent/secrets
|
|||||||
ReadOnlyPaths=/etc/aios/external-agent-runtime.d
|
ReadOnlyPaths=/etc/aios/external-agent-runtime.d
|
||||||
ReadOnlyPaths=/opt/aios-hermes-agent
|
ReadOnlyPaths=/opt/aios-hermes-agent
|
||||||
ReadOnlyPaths=/opt/aios/openclaw-engineering
|
ReadOnlyPaths=/opt/aios/openclaw-engineering
|
||||||
|
Environment=AIOS_MODEL_GOVERNANCE_DB=/var/lib/aios-model-governance/state.sqlite3
|
||||||
|
Environment=AIOS_MODEL_GOVERNANCE_ENABLED=1
|
||||||
ReadWritePaths=/var/lib/aios-external-agent-executor
|
ReadWritePaths=/var/lib/aios-external-agent-executor
|
||||||
|
ReadWritePaths=/var/lib/aios-model-governance
|
||||||
UMask=0027
|
UMask=0027
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS disabled-by-default HTTPS edge for GPT read-only Online Task ingress
|
||||||
|
After=network-online.target aios-gpt-online-task-ingress.service
|
||||||
|
Wants=network-online.target aios-gpt-online-task-ingress.service
|
||||||
|
ConditionPathExists=/etc/aios-gitops/gpt-https-edge.approved
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
DynamicUser=yes
|
||||||
|
LoadCredential=tls_cert:/etc/aios-gitops/gpt-https-edge-tls.crt
|
||||||
|
LoadCredential=tls_key:/etc/aios-gitops/gpt-https-edge-tls.key
|
||||||
|
Environment=AIOS_GPT_EDGE_TLS_CERT_FILE=%d/tls_cert
|
||||||
|
Environment=AIOS_GPT_EDGE_TLS_KEY_FILE=%d/tls_key
|
||||||
|
Environment=AIOS_GPT_EDGE_HOST=127.0.0.1
|
||||||
|
Environment=AIOS_GPT_EDGE_PORT=18443
|
||||||
|
Environment=AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=0
|
||||||
|
Environment=AIOS_GPT_EDGE_PUBLIC_DOMAIN=
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_gpt_https_edge.py
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
PrivateTmp=yes
|
||||||
|
PrivateDevices=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
ProtectKernelLogs=yes
|
||||||
|
ProtectClock=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
RestrictNamespaces=yes
|
||||||
|
MemoryDenyWriteExecute=yes
|
||||||
|
RestrictAddressFamilies=AF_INET AF_INET6
|
||||||
|
InaccessiblePaths=/data/aios-gitops /etc/aios-gitops /home /var/lib/aios-agent-task
|
||||||
|
UMask=0077
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS external GPT read-only Online Task ingress
|
||||||
|
After=network.target aios-chatgpt-bridge-v2-staging.service aios-agent-task-api.service
|
||||||
|
Wants=aios-chatgpt-bridge-v2-staging.service aios-agent-task-api.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
DynamicUser=yes
|
||||||
|
StateDirectory=aios-gpt-online-task-ingress
|
||||||
|
StateDirectoryMode=0750
|
||||||
|
LoadCredential=gpt_ingress_token:/etc/aios-gitops/gpt-online-task-ingress.token
|
||||||
|
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
|
||||||
|
Environment=AIOS_GPT_INGRESS_TOKEN_FILE=%d/gpt_ingress_token
|
||||||
|
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
|
||||||
|
Environment=AIOS_GPT_INGRESS_STATE_ROOT=%S/aios-gpt-online-task-ingress
|
||||||
|
Environment=AIOS_GPT_INGRESS_HOST=127.0.0.1
|
||||||
|
Environment=AIOS_GPT_INGRESS_PORT=18945
|
||||||
|
Environment=AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892
|
||||||
|
Environment=AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_gpt_online_task_ingress.py
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
PrivateTmp=yes
|
||||||
|
PrivateDevices=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
ProtectKernelLogs=yes
|
||||||
|
ProtectClock=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
RestrictNamespaces=yes
|
||||||
|
MemoryDenyWriteExecute=yes
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||||
|
IPAddressDeny=any
|
||||||
|
IPAddressAllow=127.0.0.0/8
|
||||||
|
InaccessiblePaths=/data/aios-gitops /etc/aios-gitops /home
|
||||||
|
UMask=0077
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS durable Owner authorization broker
|
||||||
|
After=network.target aios-chatgpt-bridge-v2-staging.service
|
||||||
|
Wants=aios-chatgpt-bridge-v2-staging.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=root
|
||||||
|
Group=aios-executor-exchange
|
||||||
|
RuntimeDirectory=aios-owner-authorization-broker
|
||||||
|
RuntimeDirectoryMode=0750
|
||||||
|
StateDirectory=aios-owner-authorization-broker
|
||||||
|
StateDirectoryMode=0750
|
||||||
|
LoadCredential=agent_task_control_token:/etc/aios-gitops/agent-task-control.token
|
||||||
|
Environment=AIOS_AGENT_TASK_CONTROL_TOKEN_FILE=%d/agent_task_control_token
|
||||||
|
Environment=AIOS_OWNER_DURABLE_AUTHORIZATION_POLICY=/etc/aios-gitops/owner-durable-authorization.json
|
||||||
|
Environment=AIOS_OWNER_DURABLE_AUTHORIZATION_REVOCATIONS=/etc/aios-gitops/owner-durable-authorization-revocations.json
|
||||||
|
Environment=AIOS_OWNER_AUTH_BROKER_ACCOUNT=chatgpt-governed-implementer-singlehost
|
||||||
|
Environment=AIOS_OWNER_AUTH_BROKER_SOCKET=%t/aios-owner-authorization-broker/broker.sock
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_durable_owner_authorization.py
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=2
|
||||||
|
NoNewPrivileges=yes
|
||||||
|
PrivateTmp=yes
|
||||||
|
PrivateDevices=yes
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=yes
|
||||||
|
ProtectKernelTunables=yes
|
||||||
|
ProtectKernelModules=yes
|
||||||
|
ProtectControlGroups=yes
|
||||||
|
ProtectKernelLogs=yes
|
||||||
|
ProtectClock=yes
|
||||||
|
LockPersonality=yes
|
||||||
|
RestrictSUIDSGID=yes
|
||||||
|
RestrictRealtime=yes
|
||||||
|
RestrictNamespaces=yes
|
||||||
|
MemoryDenyWriteExecute=yes
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET
|
||||||
|
IPAddressDeny=any
|
||||||
|
IPAddressAllow=127.0.0.0/8
|
||||||
|
ReadOnlyPaths=/etc/aios-gitops/owner-durable-authorization.json
|
||||||
|
ReadOnlyPaths=/etc/aios-gitops/owner-durable-authorization-revocations.json
|
||||||
|
InaccessiblePaths=/data/aios-gitops /home
|
||||||
|
UMask=0007
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=AIOS persistent Online Agent worker availability guard
|
||||||
|
After=local-fs.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=root
|
||||||
|
Group=root
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
Environment=AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS=90
|
||||||
|
Environment=AIOS_WORKER_AVAILABILITY_RESTART_COOLDOWN_SECONDS=60
|
||||||
|
ExecStart=/usr/bin/python3 -B /opt/aios/current/active-runtime/app/aios_worker_availability_guard.py --heal
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
MemoryDenyWriteExecute=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX
|
||||||
|
ReadWritePaths=/run/aios-worker-availability
|
||||||
|
UMask=0022
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Periodically ensure AIOS Online Agent workers remain available
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=15s
|
||||||
|
OnUnitActiveSec=30s
|
||||||
|
AccuracySec=5s
|
||||||
|
RandomizedDelaySec=3s
|
||||||
|
Persistent=true
|
||||||
|
Unit=aios-worker-availability-guard.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
d /var/lib/aios-model-governance 0750 aios-engineering-executor aios-executor-exchange -
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
d /run/aios-worker-availability 0755 root root -
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
# External Agent Attempt-Scoped Request Identity
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
External Agent execution previously derived its durable request ID from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
task_id
|
||||||
|
job_id
|
||||||
|
execution_digest
|
||||||
|
runtime
|
||||||
|
```
|
||||||
|
|
||||||
|
The Online Task attempt was missing.
|
||||||
|
|
||||||
|
That is incorrect for retryable external side effects. A failed Model Governance route reservation is
|
||||||
|
durable and terminal for that request ID. Reusing the same external request ID on a later official
|
||||||
|
Task attempt therefore reuses the old route decision/reservation and can fail settlement with:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MODEL_RESERVATION_STATE_INVALID
|
||||||
|
```
|
||||||
|
|
||||||
|
Archiving an exchange failed marker does not fix the root cause because the Model Governance
|
||||||
|
reservation remains intentionally durable.
|
||||||
|
|
||||||
|
## Contract
|
||||||
|
|
||||||
|
External request identity is now:
|
||||||
|
|
||||||
|
```text
|
||||||
|
hash(
|
||||||
|
task_id,
|
||||||
|
job_id,
|
||||||
|
execution_digest,
|
||||||
|
runtime,
|
||||||
|
attempt
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
The attempt is a **transient execution coordinate**. It is not added to the signed governed execution
|
||||||
|
envelope and does not alter the Task/Job binding or execution digest.
|
||||||
|
|
||||||
|
The worker gets the authoritative attempt from the successful claim response and passes it through:
|
||||||
|
|
||||||
|
```text
|
||||||
|
process_claim
|
||||||
|
→ RuntimeAdapter.execute(..., attempt=N)
|
||||||
|
→ ExternalAgentExecutor
|
||||||
|
→ ExternalAgentExchange
|
||||||
|
→ external request JSON
|
||||||
|
→ external result JSON
|
||||||
|
```
|
||||||
|
|
||||||
|
The external result must bind the same attempt.
|
||||||
|
|
||||||
|
## Idempotency
|
||||||
|
|
||||||
|
This gives the intended two-level behavior:
|
||||||
|
|
||||||
|
```text
|
||||||
|
same Task/Job/execution/runtime + same attempt
|
||||||
|
→ same external request ID
|
||||||
|
→ same external result/reservation can be replayed safely
|
||||||
|
|
||||||
|
same Task/Job/execution/runtime + next official attempt
|
||||||
|
→ different external request ID
|
||||||
|
→ different Model Governance request ID
|
||||||
|
→ new reservation
|
||||||
|
```
|
||||||
|
|
||||||
|
Old FAILED reservations are retained for audit. No Model Governance DB cleanup is required or
|
||||||
|
permitted.
|
||||||
|
|
||||||
|
## Model Governance
|
||||||
|
|
||||||
|
Model Governance already derives its route request ID from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
external_request_id + ":model"
|
||||||
|
```
|
||||||
|
|
||||||
|
Therefore making the external request ID attempt-scoped automatically makes route decisions and
|
||||||
|
reservations attempt-scoped without creating another state authority.
|
||||||
|
|
||||||
|
## Recovery
|
||||||
|
|
||||||
|
For the current Hermes live Task:
|
||||||
|
|
||||||
|
```text
|
||||||
|
attempt 6
|
||||||
|
→ old request/reservation remains FAILED
|
||||||
|
|
||||||
|
official NEEDS_REWORK
|
||||||
|
→ bounded retry extension 6 → 7
|
||||||
|
|
||||||
|
attempt 7
|
||||||
|
→ new external request ID
|
||||||
|
→ new route request ID
|
||||||
|
→ new reservation
|
||||||
|
→ provider execution
|
||||||
|
→ settlement
|
||||||
|
```
|
||||||
|
|
||||||
|
No replacement Task/Job, direct DB rewrite, failed-reservation deletion or marker deletion is
|
||||||
|
required.
|
||||||
|
|
||||||
|
## Worker Availability
|
||||||
|
|
||||||
|
The repair must preserve the already verified Worker Availability runtime. Runtime switches use the
|
||||||
|
maintenance inhibit and must finish with the guard healthy and all three workers fresh.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
Required regression coverage:
|
||||||
|
|
||||||
|
- request ID stable within one attempt;
|
||||||
|
- request ID different across attempts;
|
||||||
|
- request schema requires a valid attempt;
|
||||||
|
- external result binds attempt;
|
||||||
|
- old failed Model Governance reservation remains FAILED;
|
||||||
|
- next attempt gets a new reservation and can settle;
|
||||||
|
- existing external-agent and Model Governance runtime tests still pass.
|
||||||
+109
@@ -0,0 +1,109 @@
|
|||||||
|
# Model Governance Policy / Profile Floor Compatibility
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
The root-owned OpenClaw runtime profile reserves a minimum of 16,384 input tokens so Model
|
||||||
|
Governance can cover OpenClaw-injected workspace, skills and system context before provider
|
||||||
|
execution.
|
||||||
|
|
||||||
|
The staging scope policies still limited input to 16,000 tokens per request. Route selection
|
||||||
|
therefore failed before reservation with:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MODEL_INPUT_TOKEN_LIMIT_DENY
|
||||||
|
```
|
||||||
|
|
||||||
|
This was a configuration invariant violation:
|
||||||
|
|
||||||
|
```text
|
||||||
|
scope max_input_tokens_per_request
|
||||||
|
<
|
||||||
|
root-owned profile input_reservation_floor_tokens
|
||||||
|
```
|
||||||
|
|
||||||
|
The registered OpenClaw model itself supports 32,768 input tokens, so the model capability was not
|
||||||
|
the blocker.
|
||||||
|
|
||||||
|
## Invariant
|
||||||
|
|
||||||
|
For every Model Governance scope that allows a model/runtime whose root-owned profile declares an
|
||||||
|
input reservation floor:
|
||||||
|
|
||||||
|
```text
|
||||||
|
scope.max_input_tokens_per_request
|
||||||
|
>=
|
||||||
|
profile.model_governance.input_reservation_floor_tokens
|
||||||
|
```
|
||||||
|
|
||||||
|
The profile floor remains root-owned. Task payloads cannot alter it.
|
||||||
|
|
||||||
|
## Staging policy
|
||||||
|
|
||||||
|
The two staging policies that allow `openai/deepseek-v4-flash` are updated from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
16000
|
||||||
|
```
|
||||||
|
|
||||||
|
to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
16384
|
||||||
|
```
|
||||||
|
|
||||||
|
This is the minimum compatible value. It intentionally remains below the model's 32,768-token
|
||||||
|
technical input ceiling.
|
||||||
|
|
||||||
|
The daily token budget is unchanged.
|
||||||
|
|
||||||
|
## Governance path
|
||||||
|
|
||||||
|
The live SQLite policy must be changed only through the repository bootstrap and
|
||||||
|
`aios_model_governance_admin.py`.
|
||||||
|
|
||||||
|
The admin reconciler:
|
||||||
|
|
||||||
|
- reads the governed repository config;
|
||||||
|
- uses current policy revision as CAS;
|
||||||
|
- calls `ModelGovernanceStore.set_scope_policy`;
|
||||||
|
- appends normal Model Governance audit evidence;
|
||||||
|
- verifies the audit chain.
|
||||||
|
|
||||||
|
Direct SQL updates remain forbidden.
|
||||||
|
|
||||||
|
## Exhausted OpenClaw recovery
|
||||||
|
|
||||||
|
The fixed OpenClaw live Task has exhausted its normal three attempts.
|
||||||
|
|
||||||
|
After the governed policy candidate is officially verified and the live policy is reconciled:
|
||||||
|
|
||||||
|
```text
|
||||||
|
attempt 3 VERIFYING/FAILED result
|
||||||
|
→ official NEEDS_REWORK
|
||||||
|
→ Task becomes FAILED 3/3
|
||||||
|
→ official bounded retry extension 3 → 4
|
||||||
|
→ same Task / same Job returns QUEUED
|
||||||
|
→ persistent OpenClaw worker auto-claims attempt 4
|
||||||
|
```
|
||||||
|
|
||||||
|
AR1 attempt-scoped external request identity ensures attempt 4 receives a new external request and a
|
||||||
|
new Model Governance reservation.
|
||||||
|
|
||||||
|
No historical reservation is deleted or rewritten.
|
||||||
|
|
||||||
|
## Acceptance
|
||||||
|
|
||||||
|
Attempt 4 must prove:
|
||||||
|
|
||||||
|
```text
|
||||||
|
scope max_input_tokens_per_request=16384
|
||||||
|
profile input_reservation_floor_tokens=16384
|
||||||
|
reserved_estimated_input_tokens>=16384
|
||||||
|
actual input <= reserved input
|
||||||
|
settlement=SETTLED
|
||||||
|
usage_source=runtime
|
||||||
|
audit_chain_valid=true
|
||||||
|
```
|
||||||
|
|
||||||
|
If attempt 4 exposes a new failure, stop. No additional retry extension is authorized by this
|
||||||
|
repair.
|
||||||
@@ -0,0 +1,211 @@
|
|||||||
|
# AIOS Model Governance Runtime
|
||||||
|
|
||||||
|
## Status and purpose
|
||||||
|
|
||||||
|
This component turns the previously `PARTIAL` Model Governance interface map into an enforceable
|
||||||
|
runtime policy for the already verified Hermes/OpenClaw Online Agent path.
|
||||||
|
|
||||||
|
It is intentionally a **single SQLite library**, not a new network service. The existing external
|
||||||
|
Agent executor calls the library immediately before launching a model-backed runtime and settles the
|
||||||
|
same reservation immediately after the result.
|
||||||
|
|
||||||
|
The authoritative execution chain remains:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Online Task scope / trace
|
||||||
|
→ dedicated worker claim + heartbeat
|
||||||
|
→ external Agent executor
|
||||||
|
→ ModelGovernanceStore.route()
|
||||||
|
→ root-owned runtime profile binds provider/model/version
|
||||||
|
→ Hermes / OpenClaw process
|
||||||
|
→ ModelGovernanceStore.record_success() / record_failure()
|
||||||
|
→ durable Online Agent checkpoint/result
|
||||||
|
```
|
||||||
|
|
||||||
|
P0 scope/trace/credential, P1 checkpoint/retry, P2 fleet scheduling and the existing external Agent
|
||||||
|
credential boundary remain authoritative.
|
||||||
|
|
||||||
|
## Ten Model Governance contracts
|
||||||
|
|
||||||
|
### MODEL_PROVIDER_REGISTRY
|
||||||
|
|
||||||
|
`model_governance_providers` stores provider identity, failure domain, circuit policy and an opaque
|
||||||
|
credential reference. Raw credentials are rejected.
|
||||||
|
|
||||||
|
Allowed reference prefixes:
|
||||||
|
|
||||||
|
- `systemd-credential:`
|
||||||
|
- `runtime-profile:`
|
||||||
|
- `credential-broker:`
|
||||||
|
|
||||||
|
### MODEL_SELECTION_POLICY
|
||||||
|
|
||||||
|
`model_governance_scope_policies` constrains eligible models/providers/capabilities. Routing is
|
||||||
|
deterministic: explicit preferred models, then configured priority, then stable provider/model IDs.
|
||||||
|
|
||||||
|
### MODEL_VERSION_PINNING
|
||||||
|
|
||||||
|
A registry row always includes a model version. A scope can additionally pin a model ID + exact
|
||||||
|
version. Every route decision persists both model ID and version plus model/policy revisions.
|
||||||
|
|
||||||
|
### MODEL_BUDGET_ACCOUNTING
|
||||||
|
|
||||||
|
A route first reserves estimated input/output tokens and estimated cost inside `BEGIN IMMEDIATE`.
|
||||||
|
Settled usage and outstanding reservations both count against the daily scope budget, preventing two
|
||||||
|
concurrent requests from oversubscribing the same budget.
|
||||||
|
|
||||||
|
Runtime-reported usage is preferred. When telemetry is absent, the reserved upper bound is charged
|
||||||
|
rather than silently under-counting.
|
||||||
|
|
||||||
|
OpenClaw's verified result shape is supported directly:
|
||||||
|
|
||||||
|
```text
|
||||||
|
meta.agentMeta.usage.input
|
||||||
|
meta.agentMeta.usage.output
|
||||||
|
meta.agentMeta.costUsd
|
||||||
|
```
|
||||||
|
|
||||||
|
Hermes currently has no structured usage in the verified output, so it uses conservative
|
||||||
|
reservation settlement.
|
||||||
|
|
||||||
|
### MODEL_FALLBACK_POLICY
|
||||||
|
|
||||||
|
Provider failures are durable. Repeated failures open a circuit for the configured cooldown. A
|
||||||
|
fallback request excludes the failed provider and increments `fallback_index`; the scope policy
|
||||||
|
bounds the number of fallback attempts.
|
||||||
|
|
||||||
|
### MODEL_SCOPE_ISOLATION
|
||||||
|
|
||||||
|
The signed `agent_runtime` execution envelope now carries authoritative Task scope + trace identity.
|
||||||
|
|
||||||
|
The projection layer always overwrites `runtime_input.scope` and `runtime_input.trace_id` from the
|
||||||
|
Online Task, even if an execution request attempts to supply different values.
|
||||||
|
|
||||||
|
Tenant/project lookup may fall back only to the same tenant's policy. It never falls back to a
|
||||||
|
system or legacy policy. `system` and `legacy` require their own explicit policies, avoiding fake
|
||||||
|
tenant identities.
|
||||||
|
|
||||||
|
### MODEL_CREDENTIAL_BOUNDARY
|
||||||
|
|
||||||
|
Model governance stores credential references only. The root-owned external-runtime profile still
|
||||||
|
owns the actual provider credential projection. Task payloads cannot select executable, argv,
|
||||||
|
environment, provider credential, or credential path.
|
||||||
|
|
||||||
|
The external child process remains unable to reach the AIOS control plane.
|
||||||
|
|
||||||
|
### MODEL_TRACE_AUDIT
|
||||||
|
|
||||||
|
Every route, failure, provider/model/policy update and settlement appends to a SHA-256 hash-chained
|
||||||
|
audit table with task ID, trace ID and scope key. `verify_audit_chain()` detects tampering.
|
||||||
|
|
||||||
|
### MODEL_EVALUATION_GATE
|
||||||
|
|
||||||
|
A model cannot enter the registry unless its state is `APPROVED` and it has a 64-hex evaluation
|
||||||
|
evidence digest.
|
||||||
|
|
||||||
|
The staging bootstrap uses the already verified Hermes/OpenClaw E2E external-result SHA256 values as
|
||||||
|
evaluation evidence, rather than a configuration-file hash.
|
||||||
|
|
||||||
|
### MODEL_ROUTING_RESILIENCE
|
||||||
|
|
||||||
|
Route request IDs are idempotent and bound to a canonical request hash. Conflicting replay is
|
||||||
|
rejected. Provider health, reservations, usage settlement and policy/provider/model updates use
|
||||||
|
transaction/CAS semantics.
|
||||||
|
|
||||||
|
## Runtime profile contract
|
||||||
|
|
||||||
|
A root-owned Hermes/OpenClaw profile may declare:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"model_governance": {
|
||||||
|
"required": true,
|
||||||
|
"provider_ids": ["provider-a"],
|
||||||
|
"model_capabilities": ["text"],
|
||||||
|
"max_output_tokens": 512
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
When `required=true`:
|
||||||
|
|
||||||
|
- `provider_ids` must be non-empty;
|
||||||
|
- `max_output_tokens` must be positive;
|
||||||
|
- argv/environment must contain at least one governance-controlled marker:
|
||||||
|
`{provider_id}`, `{model_id}`, or `{model_version}`;
|
||||||
|
- `AIOS_MODEL_GOVERNANCE_ENABLED` must be enabled;
|
||||||
|
- a matching scope policy/model/provider must exist;
|
||||||
|
- budget reservation must succeed before the child process is launched.
|
||||||
|
|
||||||
|
The task cannot provide these markers or their values.
|
||||||
|
|
||||||
|
## SQLite authority
|
||||||
|
|
||||||
|
Default database:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/var/lib/aios-model-governance/state.sqlite3
|
||||||
|
```
|
||||||
|
|
||||||
|
The executor process already owns the only runtime mutation path. The directory is created by
|
||||||
|
tmpfiles and is the only new writable path added to that executor.
|
||||||
|
|
||||||
|
No direct Task/Job/Git database writes are introduced.
|
||||||
|
|
||||||
|
## Bootstrap/reconciliation
|
||||||
|
|
||||||
|
`aios_model_governance_admin.py` applies a root-owned JSON bootstrap idempotently. It compares desired
|
||||||
|
state to current rows and uses the current revision as the CAS input only when a change is actually
|
||||||
|
required.
|
||||||
|
|
||||||
|
The repository staging bootstrap is not a Production policy. It exists only to reproduce and
|
||||||
|
govern the already verified non-production Hermes/OpenClaw runtimes.
|
||||||
|
|
||||||
|
## Cost caveat
|
||||||
|
|
||||||
|
The current verified OneAPI-backed runtime evidence reports zero authoritative provider cost for
|
||||||
|
OpenClaw and no structured cost for Hermes. The staging bootstrap therefore does **not** invent
|
||||||
|
pricing and sets cost rates/budget to zero/unbounded.
|
||||||
|
|
||||||
|
Token budgets remain enforced. Real monetary budget enforcement becomes effective as soon as
|
||||||
|
authoritative per-model pricing is entered into the root-owned registry.
|
||||||
|
|
||||||
|
## OpenClaw JIT/WASM compatibility
|
||||||
|
|
||||||
|
The previous live acceptance proved that OpenClaw Node/WASM requires
|
||||||
|
`MemoryDenyWriteExecute=no`. Model Governance must preserve that already-evidenced runtime exception;
|
||||||
|
it must not further weaken `NoNewPrivileges`, `ProtectSystem=strict`, `ProtectHome`, loopback deny,
|
||||||
|
credential isolation or repository isolation.
|
||||||
|
|
||||||
|
## Failure/recovery invariants
|
||||||
|
|
||||||
|
- Budget denial occurs before provider execution.
|
||||||
|
- Missing scope policy fails closed.
|
||||||
|
- Missing/disabled governance fails closed when a profile requires it.
|
||||||
|
- Actual usage cannot exceed the reservation.
|
||||||
|
- Provider failure releases that reservation into `FAILED` state before fallback.
|
||||||
|
- Settlement is idempotent.
|
||||||
|
- Route replay is idempotent only for the exact canonical request.
|
||||||
|
- Audit corruption is detectable.
|
||||||
|
- Online Agent checkpoint/result retry remains unchanged and remains the outer recovery authority.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
Required unit/integration coverage includes:
|
||||||
|
|
||||||
|
- registry/evaluation/CAS;
|
||||||
|
- scope isolation and no system→tenant fallback;
|
||||||
|
- model/version pinning;
|
||||||
|
- atomic budget reservation;
|
||||||
|
- settlement/replay;
|
||||||
|
- missing telemetry conservative settlement;
|
||||||
|
- provider circuit/fallback;
|
||||||
|
- trace/audit tamper detection;
|
||||||
|
- authoritative scope/trace projection;
|
||||||
|
- task runtime-input scope override attack;
|
||||||
|
- root-owned provider/model binding;
|
||||||
|
- OpenClaw nested usage extraction;
|
||||||
|
- real Hermes/OpenClaw staging live tasks with Model Governance evidence.
|
||||||
|
|
||||||
|
The governed implementation is not complete without the unified all-modified-files bundle for the
|
||||||
|
candidate delta and the cumulative lineage bundle.
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
# OpenClaw Input Reservation Floor
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
Model Governance originally estimated input reservation from the user-visible prompt only.
|
||||||
|
|
||||||
|
The verified OpenClaw runtime injects additional root-owned/system-controlled context including
|
||||||
|
workspace, skills and system instructions. In live acceptance the prompt estimate reserved only
|
||||||
|
153 input tokens while OpenClaw reported approximately 7,770 actual input tokens.
|
||||||
|
|
||||||
|
The settlement correctly failed closed with:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MODEL_USAGE_EXCEEDS_RESERVATION
|
||||||
|
```
|
||||||
|
|
||||||
|
The provider call itself had succeeded. The defect was therefore pre-execution reservation
|
||||||
|
underestimation, not provider connectivity.
|
||||||
|
|
||||||
|
## Contract
|
||||||
|
|
||||||
|
A root-owned external Agent profile may now declare:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"model_governance": {
|
||||||
|
"input_reservation_floor_tokens": 16384
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The effective input reservation is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
max(
|
||||||
|
byte-based prompt estimate,
|
||||||
|
profile input_reservation_floor_tokens
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Only the root-owned runtime profile can choose this floor. Online Task payloads cannot lower or
|
||||||
|
override it.
|
||||||
|
|
||||||
|
The OpenClaw staging profile uses 16,384 tokens. This is greater than twice the observed 7,770-token
|
||||||
|
live input high-water mark while remaining below the registered 32,768-token model input limit.
|
||||||
|
|
||||||
|
Hermes has no floor unless explicitly configured and therefore preserves its existing behavior.
|
||||||
|
|
||||||
|
## Fail-closed behavior
|
||||||
|
|
||||||
|
The floor is not an unlimited post-execution top-up.
|
||||||
|
|
||||||
|
If actual runtime usage exceeds the reserved amount, settlement still fails with
|
||||||
|
`MODEL_USAGE_EXCEEDS_RESERVATION`.
|
||||||
|
|
||||||
|
This keeps budget enforcement pre-execution and avoids silently authorizing usage after it has
|
||||||
|
already happened.
|
||||||
|
|
||||||
|
## Evidence
|
||||||
|
|
||||||
|
External results expose:
|
||||||
|
|
||||||
|
```text
|
||||||
|
prompt_estimated_input_tokens
|
||||||
|
input_reservation_floor_tokens
|
||||||
|
reserved_estimated_input_tokens
|
||||||
|
```
|
||||||
|
|
||||||
|
alongside the normal settlement and audit evidence.
|
||||||
|
|
||||||
|
## Retry
|
||||||
|
|
||||||
|
The current OpenClaw attempt 1 failure remains preserved. After this code repair is governed and
|
||||||
|
activated, official `NEEDS_REWORK` returns the same fixed Task to the normal retry path. Because AR1
|
||||||
|
already made external request identity attempt-scoped, attempt 2 receives a new external request ID
|
||||||
|
and a new reservation.
|
||||||
|
|
||||||
|
No retry-extension is required because the Task is currently attempt 1 of 3.
|
||||||
|
|
||||||
|
No Model Governance reservation is deleted or rewritten.
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
# AIOS Worker Availability Runtime
|
||||||
|
|
||||||
|
## Priority
|
||||||
|
|
||||||
|
This runtime exists to make the already verified Online Agent execution path continuously usable
|
||||||
|
without repeated human intervention.
|
||||||
|
|
||||||
|
The system already proved that generic, Hermes and OpenClaw workers can claim and complete real
|
||||||
|
Online Tasks. The remaining availability defect was operational: evidence-freeze workflows left the
|
||||||
|
worker services stopped, while registry `ACTIVE` records remained stale.
|
||||||
|
|
||||||
|
The availability contract therefore separates:
|
||||||
|
|
||||||
|
```text
|
||||||
|
AUTO_CLAIM_MECHANISM=VERIFIED
|
||||||
|
CURRENT_WORKER_AVAILABILITY=<fresh runtime state>
|
||||||
|
```
|
||||||
|
|
||||||
|
A historical successful claim is not current liveness.
|
||||||
|
|
||||||
|
## Source of truth for idle liveness
|
||||||
|
|
||||||
|
An idle worker may have no Task lease heartbeat. Therefore Task heartbeat alone cannot prove that
|
||||||
|
the worker is continuously polling.
|
||||||
|
|
||||||
|
`aios_online_agent_worker.py` now writes:
|
||||||
|
|
||||||
|
```text
|
||||||
|
<AIOS_WORKER_STATE_ROOT>/worker-liveness.json
|
||||||
|
```
|
||||||
|
|
||||||
|
on:
|
||||||
|
|
||||||
|
- process start: `STARTING`;
|
||||||
|
- every successful poll: `POLL_OK`;
|
||||||
|
- poll failure: `POLL_FAILED`;
|
||||||
|
- every successful Task lease heartbeat while executing: `BUSY`.
|
||||||
|
|
||||||
|
The file contains:
|
||||||
|
|
||||||
|
- agent ID/type/version;
|
||||||
|
- process PID;
|
||||||
|
- monotonically increasing local sequence;
|
||||||
|
- `last_seen_epoch`;
|
||||||
|
- poll/heartbeat intervals;
|
||||||
|
- current Task/stage while busy.
|
||||||
|
|
||||||
|
The liveness file is worker-owned observability state only. It does not write or mutate Task, Job,
|
||||||
|
Git, WriterLease or registry databases.
|
||||||
|
|
||||||
|
## Self-healing guard
|
||||||
|
|
||||||
|
`aios-worker-availability-guard.timer` invokes a root-owned oneshot guard every 30 seconds.
|
||||||
|
|
||||||
|
The guard manages:
|
||||||
|
|
||||||
|
```text
|
||||||
|
aios-online-agent-worker-20260815.service
|
||||||
|
aios-hermes-online-agent-worker.service
|
||||||
|
aios-openclaw-online-agent-worker.service
|
||||||
|
```
|
||||||
|
|
||||||
|
and the required external runtime dependency:
|
||||||
|
|
||||||
|
```text
|
||||||
|
aios-external-agent-executor.service
|
||||||
|
```
|
||||||
|
|
||||||
|
For each worker:
|
||||||
|
|
||||||
|
1. missing systemd unit → fail closed;
|
||||||
|
2. disabled → `systemctl enable`;
|
||||||
|
3. inactive → `systemctl start`;
|
||||||
|
4. active but liveness older than 90 seconds → restart;
|
||||||
|
5. repeated stale restart within 60 seconds → do not restart again; report degraded.
|
||||||
|
|
||||||
|
Long-running Tasks are safe because the Task lease-heartbeat thread refreshes `BUSY` liveness.
|
||||||
|
|
||||||
|
The guard never claims a Task and never touches Task/Job/Git databases.
|
||||||
|
|
||||||
|
## Maintenance inhibit
|
||||||
|
|
||||||
|
Planned maintenance creates:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/run/aios-worker-availability/maintenance.lock
|
||||||
|
```
|
||||||
|
|
||||||
|
While this file exists the guard performs no start/restart actions and reports:
|
||||||
|
|
||||||
|
```text
|
||||||
|
MAINTENANCE_INHIBITED
|
||||||
|
```
|
||||||
|
|
||||||
|
Maintenance tooling must remove the file when done. This prevents the self-healer from fighting
|
||||||
|
intentional service changes.
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
The guard writes:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/run/aios-worker-availability/status.json
|
||||||
|
```
|
||||||
|
|
||||||
|
with current worker/dependency states, liveness ages and recovery actions.
|
||||||
|
|
||||||
|
A worker is currently available only when:
|
||||||
|
|
||||||
|
```text
|
||||||
|
systemd load_state=loaded
|
||||||
|
systemd active_state=active
|
||||||
|
unit enabled/static
|
||||||
|
worker-liveness.json present
|
||||||
|
last_seen_age <= 90s
|
||||||
|
```
|
||||||
|
|
||||||
|
Registry `ACTIVE` without fresh poll/busy liveness is not sufficient.
|
||||||
|
|
||||||
|
## Boot/restart behavior
|
||||||
|
|
||||||
|
The timer is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
OnBootSec=15s
|
||||||
|
OnUnitActiveSec=30s
|
||||||
|
Persistent=true
|
||||||
|
```
|
||||||
|
|
||||||
|
Therefore reboot, manual stop left behind by another workflow, process crash, and hung poll loop are
|
||||||
|
all repaired without requiring a human to notice and intervene.
|
||||||
|
|
||||||
|
Normal worker units retain their own `Restart=always`; the availability guard handles the cases that
|
||||||
|
`Restart=always` cannot, especially a deliberate `systemctl stop` left after an evidence workflow.
|
||||||
|
|
||||||
|
## Acceptance
|
||||||
|
|
||||||
|
The live acceptance requires all of the following:
|
||||||
|
|
||||||
|
```text
|
||||||
|
guard timer active/enabled
|
||||||
|
external executor active/enabled
|
||||||
|
generic worker active/enabled + fresh liveness
|
||||||
|
Hermes worker active/enabled + fresh liveness
|
||||||
|
OpenClaw worker active/enabled + fresh liveness
|
||||||
|
```
|
||||||
|
|
||||||
|
Then a new read-only Online Task is projected through canonical Bridge `18892`.
|
||||||
|
|
||||||
|
Success requires:
|
||||||
|
|
||||||
|
```text
|
||||||
|
TASK_CREATED
|
||||||
|
→ generic worker auto claim
|
||||||
|
→ TASK_HEARTBEAT
|
||||||
|
→ RESULT_SUBMITTED
|
||||||
|
→ official TASK_VERIFIED(SUCCEEDED)
|
||||||
|
```
|
||||||
|
|
||||||
|
After the Task, the services must remain active and liveness must continue advancing. The acceptance
|
||||||
|
also stops a worker intentionally once and proves the guard starts it again with a new fresh
|
||||||
|
liveness sequence.
|
||||||
|
|
||||||
|
Only after these gates may current status be upgraded from:
|
||||||
|
|
||||||
|
```text
|
||||||
|
OVERALL=PARTIAL
|
||||||
|
```
|
||||||
|
|
||||||
|
to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
PERSISTENT_WORKER_LIVENESS=VERIFIED
|
||||||
|
NEW_TASK_AUTO_CONSUMPTION=VERIFIED
|
||||||
|
WORKER_AVAILABILITY_RUNTIME=VERIFIED_CANDIDATE
|
||||||
|
```
|
||||||
|
|
||||||
|
Protected main and Production remain out of scope.
|
||||||
|
|
||||||
|
|
||||||
|
## Forced source-capture projection compatibility
|
||||||
|
|
||||||
|
The deployed canonical Bridge currently projects a default `source_capture` envelope even when a
|
||||||
|
governed binding request asks for binding-only behavior. This must not be allowed to make always-on
|
||||||
|
workers consume engineering Tasks before their deterministic plan is registered.
|
||||||
|
|
||||||
|
The worker therefore distinguishes a real read-only source capture from an engineering placeholder:
|
||||||
|
|
||||||
|
```text
|
||||||
|
source_capture
|
||||||
|
+ capability_scope contains governed.write.branch or governed.write.file
|
||||||
|
= PENDING_DETERMINISTIC_PLAN
|
||||||
|
```
|
||||||
|
|
||||||
|
For such a placeholder the worker:
|
||||||
|
|
||||||
|
1. re-resolves through the canonical plan registry **without replaying the stale source-capture
|
||||||
|
envelope**;
|
||||||
|
2. if no deterministic plan exists, emits `TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN` and does not
|
||||||
|
claim;
|
||||||
|
3. if the registered canonical envelope is deterministic and the governed binding still matches,
|
||||||
|
uses that deterministic envelope and claims normally.
|
||||||
|
|
||||||
|
A real read-only source-capture Task with only `git.read`/`git.test` still replays its exact persisted
|
||||||
|
payload and remains claimable.
|
||||||
|
|
||||||
|
This makes persistent workers safe with the currently deployed Bridge while preserving the same
|
||||||
|
Task/Job binding. No Task DB rewrite, cancellation, replacement Task, fake NEEDS_REWORK, or
|
||||||
|
pre-claim metadata mutation is required.
|
||||||
|
|
||||||
|
A later Bridge cleanup may implement true binding-only projection, but Worker Availability does not
|
||||||
|
depend on that cleanup.
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
# Canonical all-modified-files packager and secret scanning
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
The historical all-modified-files evidence packager lived only as a copied handoff script.
|
||||||
|
|
||||||
|
Its private-key scanner matched the fragment:
|
||||||
|
|
||||||
|
```text
|
||||||
|
BEGIN PRIVATE KEY
|
||||||
|
```
|
||||||
|
|
||||||
|
instead of a complete PEM header. A regression test containing that literal therefore caused a
|
||||||
|
false secret finding even though the candidate contained no credential.
|
||||||
|
|
||||||
|
A task had to use a temporary evidence-only compatibility packager to complete its bundle. That is
|
||||||
|
acceptable as an emergency workaround but not as a durable unattended path.
|
||||||
|
|
||||||
|
## Repository source of truth
|
||||||
|
|
||||||
|
The canonical implementation is now:
|
||||||
|
|
||||||
|
```text
|
||||||
|
active-runtime/app/aios_all_modified_files_packager.py
|
||||||
|
```
|
||||||
|
|
||||||
|
It preserves the historical output/bundle schemas:
|
||||||
|
|
||||||
|
```text
|
||||||
|
aios.all-modified-files.bundle.v1
|
||||||
|
aios.all-modified-files.bundle-summary.v1
|
||||||
|
aios.modified-documents.index.v1
|
||||||
|
```
|
||||||
|
|
||||||
|
and the host-visible status lines:
|
||||||
|
|
||||||
|
```text
|
||||||
|
ALL_MODIFIED_FILES_BUNDLE=PASS
|
||||||
|
ALL_MODIFIED_FILES_BUNDLE_PATH=...
|
||||||
|
ALL_MODIFIED_FILES_BUNDLE_SHA256=...
|
||||||
|
```
|
||||||
|
|
||||||
|
Future governed tasks should use this repository version instead of copying an unversioned scanner.
|
||||||
|
|
||||||
|
## Secret scanning invariant
|
||||||
|
|
||||||
|
Private-key detection requires a complete PEM header:
|
||||||
|
|
||||||
|
```text
|
||||||
|
-----BEGIN [RSA|EC|DSA|OPENSSH ]PRIVATE KEY-----
|
||||||
|
```
|
||||||
|
|
||||||
|
The following source/test/document text is not secret material and must remain packageable:
|
||||||
|
|
||||||
|
```text
|
||||||
|
BEGIN PRIVATE KEY
|
||||||
|
PRIVATE KEY
|
||||||
|
PEM private-key header
|
||||||
|
```
|
||||||
|
|
||||||
|
Bearer and key/value credential detection remains fail-closed:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Authorization: Bearer <20+ token chars>
|
||||||
|
password=<12+ non-space chars>
|
||||||
|
passwd=<12+ non-space chars>
|
||||||
|
secret=<12+ non-space chars>
|
||||||
|
api_key=<12+ non-space chars>
|
||||||
|
api-key=<12+ non-space chars>
|
||||||
|
```
|
||||||
|
|
||||||
|
Tests construct the real PEM header dynamically so the scanner's own test source does not contain a
|
||||||
|
literal credential-looking header.
|
||||||
|
|
||||||
|
## Bundle integrity
|
||||||
|
|
||||||
|
The packager continues to enforce:
|
||||||
|
|
||||||
|
- exact base→candidate ancestry;
|
||||||
|
- before/after file capture;
|
||||||
|
- rename/copy/delete handling;
|
||||||
|
- path traversal denial;
|
||||||
|
- execution evidence symlink denial;
|
||||||
|
- secret scan over repository before/after and execution evidence;
|
||||||
|
- document index;
|
||||||
|
- SHA-256 closure;
|
||||||
|
- ZIP without symlinks.
|
||||||
|
|
||||||
|
No secret finding is downgraded to a warning.
|
||||||
|
|
||||||
|
## Migration
|
||||||
|
|
||||||
|
The old handoff script may remain for historical reproducibility, but new handoffs should call the
|
||||||
|
repository packager from the verified candidate lineage.
|
||||||
|
|
||||||
|
A temporary compatibility scanner is no longer needed once this candidate is in the lineage.
|
||||||
@@ -0,0 +1,173 @@
|
|||||||
|
# Durable Owner Authorization and capability projection
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
AIOS already supports unattended worker execution after a Task is executable, but Owner intent was
|
||||||
|
not durable at the capability layer. The same Owner repeatedly had to authorize or repair:
|
||||||
|
|
||||||
|
- `governed-task:project`;
|
||||||
|
- `release:candidate`.
|
||||||
|
|
||||||
|
This caused valid one-time Owner intent to be lost between conversations and forced repeated human
|
||||||
|
interaction.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
Durable authorization is represented by a root-owned mode `0600` policy outside Git. The policy is
|
||||||
|
persistent until explicitly revoked.
|
||||||
|
|
||||||
|
The first persistable capabilities are deliberately narrow:
|
||||||
|
|
||||||
|
```text
|
||||||
|
governed-task:project
|
||||||
|
release:candidate
|
||||||
|
```
|
||||||
|
|
||||||
|
They cover preparation work only.
|
||||||
|
|
||||||
|
The following capabilities can never be inherited from this durable grant:
|
||||||
|
|
||||||
|
```text
|
||||||
|
submitOwnerApproval
|
||||||
|
startDeployment
|
||||||
|
tag_publish
|
||||||
|
baseline_registry_finalize
|
||||||
|
runtime_switch
|
||||||
|
main_write
|
||||||
|
production_deploy
|
||||||
|
force_push
|
||||||
|
credential_admin
|
||||||
|
```
|
||||||
|
|
||||||
|
Those remain separate high-risk release/Owner-signature gates.
|
||||||
|
|
||||||
|
## Policy
|
||||||
|
|
||||||
|
Example schema:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"schema": "aios.owner-durable-authorization.v1",
|
||||||
|
"grant_id": "owner-permanent-preparation-20260819",
|
||||||
|
"owner_identity": "owner@example",
|
||||||
|
"account": "chatgpt-governed-implementer-singlehost",
|
||||||
|
"capabilities": [
|
||||||
|
"governed-task:project",
|
||||||
|
"release:candidate"
|
||||||
|
],
|
||||||
|
"issued_at": "2026-08-19T05:00:00Z",
|
||||||
|
"not_before": "2026-08-19T05:00:00Z",
|
||||||
|
"persistent_until_revoked": true,
|
||||||
|
"constraints": {
|
||||||
|
"repository": "aios/aios",
|
||||||
|
"allowed_target_prefix": "agent/",
|
||||||
|
"release_version_prefix": "V2R2C",
|
||||||
|
"max_task_risk": "engineering"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The repository never stores the live Owner policy or release credential.
|
||||||
|
|
||||||
|
## Revocation
|
||||||
|
|
||||||
|
A root-owned mode `0600` revocation file lists `revoked_grant_ids`.
|
||||||
|
|
||||||
|
Revocation is checked on every operation, not only at service startup. A running broker therefore
|
||||||
|
fails closed immediately after revocation without needing a new conversation.
|
||||||
|
|
||||||
|
## Broker boundary
|
||||||
|
|
||||||
|
The broker exposes only an AF_UNIX socket:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/run/aios-owner-authorization-broker/broker.sock
|
||||||
|
```
|
||||||
|
|
||||||
|
The socket is mode `0660` and owned by the service's `aios-executor-exchange` group. This avoids introducing another long-lived bearer secret solely for local broker access.
|
||||||
|
|
||||||
|
It exposes only:
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /v1/capabilities
|
||||||
|
POST /v1/project-governed-task
|
||||||
|
POST /v1/authorize-release-candidate
|
||||||
|
```
|
||||||
|
|
||||||
|
`project-governed-task` calls only the canonical Bridge.
|
||||||
|
|
||||||
|
`authorize-release-candidate` persists an Owner authorization decision only. It deliberately does
|
||||||
|
**not** invent a local trusted-builder endpoint. The external release connector remains responsible
|
||||||
|
for projecting the account's real `release:candidate` scope. Until that external projection exists,
|
||||||
|
the decision state is `AUTHORIZED_PENDING_EXTERNAL_CAPABILITY_PROJECTION`.
|
||||||
|
|
||||||
|
The broker holds only the existing host control credential needed for canonical governed-task
|
||||||
|
projection. It never holds an Owner signing key or a release/deployment credential. The caller also supplies the expected account identifier over the Unix transport; the OS socket permission is the local access boundary.
|
||||||
|
|
||||||
|
|
||||||
|
## Canonical Online Task projection
|
||||||
|
|
||||||
|
A durable `governed-task:project` grant may create a complete canonical Online Task rather than a
|
||||||
|
bare Task. It requires:
|
||||||
|
|
||||||
|
```text
|
||||||
|
require_online_task_projection=true
|
||||||
|
generation=1
|
||||||
|
branch_mode=CREATE_NEW
|
||||||
|
max_attempts<=3
|
||||||
|
risk<=engineering
|
||||||
|
initial action=source_capture
|
||||||
|
```
|
||||||
|
|
||||||
|
Projected task capabilities are limited to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
git.read
|
||||||
|
git.test
|
||||||
|
governed.write.branch
|
||||||
|
governed.write.file
|
||||||
|
```
|
||||||
|
|
||||||
|
A write-capable task still begins as `source_capture`. Existing placeholder-deferral prevents the
|
||||||
|
generic worker from claiming it until a deterministic plan is registered. Repository mutation
|
||||||
|
therefore still requires the normal deterministic plan, WriterLease and governed commit path.
|
||||||
|
|
||||||
|
This fixes the previous split-brain behavior where a bare Task could exist without the canonical
|
||||||
|
governed execution projection.
|
||||||
|
|
||||||
|
## Idempotency and audit
|
||||||
|
|
||||||
|
Every broker operation requires an idempotency key. A receipt SQLite database stores only:
|
||||||
|
|
||||||
|
- idempotency key;
|
||||||
|
- capability;
|
||||||
|
- request SHA-256;
|
||||||
|
- response SHA-256;
|
||||||
|
- sanitized upstream response;
|
||||||
|
- timestamp.
|
||||||
|
|
||||||
|
An exact retry returns the same receipt. Reusing an idempotency key with different content fails.
|
||||||
|
|
||||||
|
## Unattended invariant
|
||||||
|
|
||||||
|
Once the Owner policy and narrow service credentials are provisioned:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Owner durable grant
|
||||||
|
→ broker decision
|
||||||
|
→ canonical governed task projection / release candidate preparation
|
||||||
|
→ worker / trusted builder
|
||||||
|
```
|
||||||
|
|
||||||
|
No repeated Owner interaction is required for those preparation capabilities.
|
||||||
|
|
||||||
|
Production execution remains:
|
||||||
|
|
||||||
|
```text
|
||||||
|
release candidate
|
||||||
|
→ immutable approval envelope
|
||||||
|
→ Owner signature
|
||||||
|
→ deployment
|
||||||
|
```
|
||||||
|
|
||||||
|
The durable grant never substitutes for the Owner signature.
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
# Owner-approved GPT HTTPS edge activation and rollback
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This controller is the final host-side control mechanism before the disabled-by-default GPT HTTPS
|
||||||
|
edge may bind publicly.
|
||||||
|
|
||||||
|
Repository implementation and public activation remain separate states:
|
||||||
|
|
||||||
|
```text
|
||||||
|
IMPLEMENTED
|
||||||
|
≠
|
||||||
|
OWNER_APPROVED
|
||||||
|
≠
|
||||||
|
ACTIVATED
|
||||||
|
≠
|
||||||
|
RUNTIME_VERIFIED
|
||||||
|
```
|
||||||
|
|
||||||
|
The controller never creates owner approval, TLS assets, DNS, firewall rules or ingress bearer
|
||||||
|
credentials.
|
||||||
|
|
||||||
|
## Approval binding
|
||||||
|
|
||||||
|
The owner approval file is root-owned mode `0600` and has schema:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"schema": "aios.gpt-https-edge.public-approval.v1",
|
||||||
|
"approval_id": "owner-public-edge-...",
|
||||||
|
"candidate_oid": "<exact governed edge candidate>",
|
||||||
|
"domain": "tasks.example.com",
|
||||||
|
"public_bind": true,
|
||||||
|
"bind_host": "0.0.0.0",
|
||||||
|
"port": 18443,
|
||||||
|
"tls_cert_sha256": "<exact certificate sha256>",
|
||||||
|
"issued_at": "2026-08-19T00:00:00Z",
|
||||||
|
"not_before": "2026-08-19T00:00:00Z",
|
||||||
|
"expires_at": "2026-08-20T00:00:00Z",
|
||||||
|
"owner": "<owner identity>"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Approval lifetime is at most seven days. Candidate/domain/bind/port/certificate are immutable within
|
||||||
|
one approval.
|
||||||
|
|
||||||
|
Changing any of those requires a new owner approval.
|
||||||
|
|
||||||
|
## Check
|
||||||
|
|
||||||
|
```text
|
||||||
|
aios_gpt_https_edge_activation.py check --candidate <GE>
|
||||||
|
```
|
||||||
|
|
||||||
|
Checks:
|
||||||
|
|
||||||
|
- approval ownership/mode/schema/time window;
|
||||||
|
- exact candidate binding;
|
||||||
|
- exact certificate hash;
|
||||||
|
- certificate/private-key pair;
|
||||||
|
- certificate hostname;
|
||||||
|
- certificate remaining lifetime;
|
||||||
|
- loopback GPT ingress service health.
|
||||||
|
|
||||||
|
It performs no public network mutation.
|
||||||
|
|
||||||
|
## Activate
|
||||||
|
|
||||||
|
`activate` additionally:
|
||||||
|
|
||||||
|
- writes one exact systemd public-bind drop-in atomically;
|
||||||
|
- enables/restarts only the HTTPS edge unit;
|
||||||
|
- verifies effective ExecStart points at the immutable candidate release;
|
||||||
|
- verifies effective public-bind/domain environment;
|
||||||
|
- performs a TLS handshake to `127.0.0.1:18443` using the approved public hostname;
|
||||||
|
- writes a durable activation receipt.
|
||||||
|
|
||||||
|
Any failure after drop-in mutation automatically invokes rollback.
|
||||||
|
|
||||||
|
Exact replay of an already-active matching approval returns the durable receipt without restarting
|
||||||
|
or creating duplicate state.
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
`rollback`:
|
||||||
|
|
||||||
|
- stops and disables only the HTTPS edge;
|
||||||
|
- removes only the controller-owned public-bind drop-in;
|
||||||
|
- daemon-reloads systemd;
|
||||||
|
- verifies edge inactive;
|
||||||
|
- writes a durable rollback receipt.
|
||||||
|
|
||||||
|
It does not delete:
|
||||||
|
|
||||||
|
- owner approval;
|
||||||
|
- TLS certificate/private key;
|
||||||
|
- ingress bearer token;
|
||||||
|
- internal loopback ingress;
|
||||||
|
- Task/Job state;
|
||||||
|
- Git branches.
|
||||||
|
|
||||||
|
Rollback is therefore replay-safe and does not require reconstructing secrets.
|
||||||
|
|
||||||
|
## Not covered
|
||||||
|
|
||||||
|
This controller intentionally does not own:
|
||||||
|
|
||||||
|
- DNS records;
|
||||||
|
- host firewall/NAT/security-group changes;
|
||||||
|
- certificate issuance/renewal;
|
||||||
|
- OpenAI Custom GPT domain verification;
|
||||||
|
- public Internet acceptance from outside the host.
|
||||||
|
|
||||||
|
Those remain separate owner/environment gates.
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
# Disabled-by-default HTTPS edge for external GPT Online Task access
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
The internal ingress (`127.0.0.1:18945`) remains loopback-only. This edge is the only component
|
||||||
|
allowed to become Internet-facing, and only after a separate owner approval.
|
||||||
|
|
||||||
|
It forwards only:
|
||||||
|
|
||||||
|
```text
|
||||||
|
POST /v1/read-only-tasks
|
||||||
|
GET /v1/read-only-tasks/{request_id}
|
||||||
|
GET /openapi.json
|
||||||
|
GET /health
|
||||||
|
```
|
||||||
|
|
||||||
|
It never proxies:
|
||||||
|
|
||||||
|
```text
|
||||||
|
18892 canonical Bridge
|
||||||
|
18940 Task API
|
||||||
|
WriterLease
|
||||||
|
commit
|
||||||
|
verification
|
||||||
|
release
|
||||||
|
tag
|
||||||
|
baseline
|
||||||
|
arbitrary URLs
|
||||||
|
```
|
||||||
|
|
||||||
|
## Default state
|
||||||
|
|
||||||
|
The systemd unit is disabled by policy unless the owner creates:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/etc/aios-gitops/gpt-https-edge.approved
|
||||||
|
```
|
||||||
|
|
||||||
|
The base unit still binds only to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
127.0.0.1:18443
|
||||||
|
AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=0
|
||||||
|
```
|
||||||
|
|
||||||
|
Non-loopback bind is rejected by code unless an owner-controlled drop-in sets:
|
||||||
|
|
||||||
|
```text
|
||||||
|
AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=1
|
||||||
|
AIOS_GPT_EDGE_HOST=0.0.0.0
|
||||||
|
AIOS_GPT_EDGE_PUBLIC_DOMAIN=<approved hostname>
|
||||||
|
```
|
||||||
|
|
||||||
|
This repository candidate does not create that drop-in or approval marker.
|
||||||
|
|
||||||
|
## TLS
|
||||||
|
|
||||||
|
TLS certificate/key are owner-managed systemd credentials:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/etc/aios-gitops/gpt-https-edge-tls.crt
|
||||||
|
/etc/aios-gitops/gpt-https-edge-tls.key
|
||||||
|
```
|
||||||
|
|
||||||
|
The service requires TLS 1.2 or newer, disables TLS compression, and never logs or returns key
|
||||||
|
material.
|
||||||
|
|
||||||
|
Certificate/key issuance and renewal are external owner responsibilities until a separately
|
||||||
|
governed ACME workflow is implemented.
|
||||||
|
|
||||||
|
## Request protections
|
||||||
|
|
||||||
|
The edge provides:
|
||||||
|
|
||||||
|
- 64 KiB request cap;
|
||||||
|
- 256 KiB upstream response cap;
|
||||||
|
- JSON validation before proxying;
|
||||||
|
- fixed upstream `http://127.0.0.1:18945`;
|
||||||
|
- Authorization-only header forwarding;
|
||||||
|
- no Cookie/X-Forwarded-* trust;
|
||||||
|
- route/method allowlist;
|
||||||
|
- per-source-IP in-memory rate limiting;
|
||||||
|
- bounded concurrent request handling;
|
||||||
|
- optional exact Host-header enforcement;
|
||||||
|
- HSTS / no-store / nosniff / no-referrer response headers.
|
||||||
|
|
||||||
|
The rate limiter is intentionally simple and local. Multi-node/global abuse protection is outside
|
||||||
|
the current one-host architecture.
|
||||||
|
|
||||||
|
## Authentication
|
||||||
|
|
||||||
|
The HTTPS edge does not create a new authentication scheme. It forwards the external Bearer
|
||||||
|
credential to the loopback GPT ingress, which verifies `gpt_ingress_token`.
|
||||||
|
|
||||||
|
The edge never receives the AIOS control-plane token.
|
||||||
|
|
||||||
|
## Deployment boundary
|
||||||
|
|
||||||
|
Public activation is a high-risk network operation and requires explicit owner approval. Before
|
||||||
|
activation verify:
|
||||||
|
|
||||||
|
```text
|
||||||
|
host-side GPT ingress E2E=VERIFIED
|
||||||
|
owner ingress token provisioned
|
||||||
|
TLS certificate/key provisioned
|
||||||
|
public hostname owned and DNS configured
|
||||||
|
firewall/network exposure reviewed
|
||||||
|
explicit public-bind approval recorded
|
||||||
|
rollback plan available
|
||||||
|
```
|
||||||
|
|
||||||
|
Only after those gates may an owner-authorized runtime drop-in bind this service publicly.
|
||||||
|
|
||||||
|
Until then:
|
||||||
|
|
||||||
|
```text
|
||||||
|
GPT_HTTPS_EDGE_IMPLEMENTATION=candidate
|
||||||
|
PUBLIC_NETWORK_EXPOSURE=NONE
|
||||||
|
PUBLIC_CUSTOM_GPT_DIRECT_AI0S_ACCESS=NOT_VERIFIED
|
||||||
|
```
|
||||||
@@ -0,0 +1,138 @@
|
|||||||
|
# External GPT → Online Task ingress
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
This component closes the current architectural gap between an external Custom GPT and AIOS
|
||||||
|
without exposing the internal Task API (`18940`) or canonical Bridge (`18892`) directly.
|
||||||
|
|
||||||
|
The first supported public-facing capability is intentionally read-only:
|
||||||
|
|
||||||
|
```text
|
||||||
|
authenticated external intent
|
||||||
|
→ host-side ingress
|
||||||
|
→ canonical /v2/governed-task
|
||||||
|
→ canonical /v2/execution-envelope
|
||||||
|
→ persistent generic worker
|
||||||
|
→ source_capture result
|
||||||
|
→ official control-plane verification
|
||||||
|
```
|
||||||
|
|
||||||
|
The ingress itself never claims, heartbeats, verifies, commits, acquires WriterLease, or writes
|
||||||
|
protected refs.
|
||||||
|
|
||||||
|
## Security boundary
|
||||||
|
|
||||||
|
The service binds only to loopback (`127.0.0.1:18945`). A future owner-approved TLS reverse proxy
|
||||||
|
or tunnel may expose this service externally. Direct non-loopback binding is denied by code.
|
||||||
|
|
||||||
|
Two independent credentials are projected by systemd:
|
||||||
|
|
||||||
|
- `gpt_ingress_token`: authenticates the external caller to this ingress.
|
||||||
|
- `agent_task_control_token`: lets the ingress call the loopback AIOS control plane.
|
||||||
|
|
||||||
|
Neither credential is stored in Git, SQLite, task payloads, results, logs, or API responses.
|
||||||
|
|
||||||
|
The ingress has no repository filesystem access.
|
||||||
|
|
||||||
|
## Caller-controlled fields
|
||||||
|
|
||||||
|
Allowed:
|
||||||
|
|
||||||
|
```text
|
||||||
|
request_id
|
||||||
|
goal
|
||||||
|
instructions
|
||||||
|
success_criteria
|
||||||
|
constraints.max_files
|
||||||
|
constraints.focus_paths
|
||||||
|
constraints.notes
|
||||||
|
```
|
||||||
|
|
||||||
|
Not caller-controlled:
|
||||||
|
|
||||||
|
```text
|
||||||
|
repository
|
||||||
|
base_oid
|
||||||
|
expected_branch_oid
|
||||||
|
target_branch
|
||||||
|
branch_mode
|
||||||
|
generation
|
||||||
|
risk
|
||||||
|
priority
|
||||||
|
max_attempts
|
||||||
|
capabilities
|
||||||
|
action/adapter
|
||||||
|
WriterLease
|
||||||
|
worker identity
|
||||||
|
credentials
|
||||||
|
verification decision
|
||||||
|
main/production/tag/baseline permissions
|
||||||
|
```
|
||||||
|
|
||||||
|
All external tasks are hard-bound to:
|
||||||
|
|
||||||
|
```text
|
||||||
|
repository=aios/aios
|
||||||
|
capabilities=[git.read, git.test]
|
||||||
|
action=source_capture
|
||||||
|
writer_lease=false
|
||||||
|
repository_mutation=forbidden
|
||||||
|
control_plane_side_effects=forbidden
|
||||||
|
credential_access=forbidden
|
||||||
|
```
|
||||||
|
|
||||||
|
## Idempotency
|
||||||
|
|
||||||
|
`request_id` is the durable external idempotency boundary.
|
||||||
|
|
||||||
|
The ingress stores only:
|
||||||
|
|
||||||
|
```text
|
||||||
|
request hash
|
||||||
|
derived AIOS task id
|
||||||
|
canonical job id
|
||||||
|
base oid
|
||||||
|
sanitized receipt
|
||||||
|
state
|
||||||
|
```
|
||||||
|
|
||||||
|
An exact retry returns the same receipt. Reusing a `request_id` with different content returns
|
||||||
|
`IDEMPOTENCY_CONFLICT`.
|
||||||
|
|
||||||
|
A crash after canonical task creation can recover the same binding from the Task API rather than
|
||||||
|
creating a replacement Task.
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
`GET /v1/read-only-tasks/{request_id}` returns only sanitized state:
|
||||||
|
|
||||||
|
```text
|
||||||
|
task/job ids
|
||||||
|
status
|
||||||
|
attempt
|
||||||
|
whether claimed
|
||||||
|
result classification
|
||||||
|
official verification decision/classification
|
||||||
|
source snapshot hash/count when available
|
||||||
|
```
|
||||||
|
|
||||||
|
It does not proxy raw Task metadata, execution envelopes, result text, credentials, or source files.
|
||||||
|
|
||||||
|
## Public exposure
|
||||||
|
|
||||||
|
This candidate does **not** make AIOS publicly reachable. Public exposure requires a separately
|
||||||
|
approved HTTPS ingress/reverse proxy with:
|
||||||
|
|
||||||
|
- TLS;
|
||||||
|
- external authentication lifecycle;
|
||||||
|
- rate limiting / abuse controls;
|
||||||
|
- request size limits;
|
||||||
|
- observability;
|
||||||
|
- explicit owner approval for the network exposure.
|
||||||
|
|
||||||
|
Until that step is verified:
|
||||||
|
|
||||||
|
```text
|
||||||
|
HOST_SIDE_EXTERNAL_GPT_ADAPTER=implemented/staging-candidate
|
||||||
|
PUBLIC_CUSTOM_GPT_DIRECT_AI0S_ACCESS=not verified
|
||||||
|
```
|
||||||
@@ -0,0 +1,252 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_all_modified_files_packager as packager
|
||||||
|
|
||||||
|
|
||||||
|
def init_repo(tmp_path: Path) -> tuple[Path, Path, str]:
|
||||||
|
bare=tmp_path/"repo.git"
|
||||||
|
work=tmp_path/"work"
|
||||||
|
subprocess.run(["git","init","--bare",str(bare)],check=True,capture_output=True)
|
||||||
|
subprocess.run(["git","clone",str(bare),str(work)],check=True,capture_output=True)
|
||||||
|
subprocess.run(["git","-C",str(work),"config","user.email","test@example.invalid"],check=True)
|
||||||
|
subprocess.run(["git","-C",str(work),"config","user.name","AIOS Test"],check=True)
|
||||||
|
(work/"README.md").write_text("base\n",encoding="utf-8")
|
||||||
|
subprocess.run(["git","-C",str(work),"add","README.md"],check=True)
|
||||||
|
subprocess.run(["git","-C",str(work),"commit","-m","base"],check=True,capture_output=True)
|
||||||
|
subprocess.run(["git","-C",str(work),"push","origin","HEAD:refs/heads/main"],check=True,capture_output=True)
|
||||||
|
base=subprocess.check_output(["git","-C",str(work),"rev-parse","HEAD"],text=True).strip()
|
||||||
|
return bare,work,base
|
||||||
|
|
||||||
|
|
||||||
|
def commit(work: Path, message: str) -> str:
|
||||||
|
subprocess.run(["git","-C",str(work),"add","-A"],check=True)
|
||||||
|
subprocess.run(["git","-C",str(work),"commit","-m",message],check=True,capture_output=True)
|
||||||
|
candidate=subprocess.check_output(["git","-C",str(work),"rev-parse","HEAD"],text=True).strip()
|
||||||
|
subprocess.run(["git","-C",str(work),"push","origin","HEAD:refs/heads/main"],check=True,capture_output=True)
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
|
||||||
|
def real_pem_header() -> bytes:
|
||||||
|
return b"-----" + b"BEGIN " + b"PRIVATE KEY" + b"-----"
|
||||||
|
|
||||||
|
|
||||||
|
def bearer_value() -> bytes:
|
||||||
|
return b"Authorization" + b": Bearer " + (b"a"*40)
|
||||||
|
|
||||||
|
|
||||||
|
def key_value_secret() -> bytes:
|
||||||
|
return b"api_key" + b"=" + (b"x"*24)
|
||||||
|
|
||||||
|
|
||||||
|
def test_plain_private_key_words_are_not_secret_findings():
|
||||||
|
for value in (
|
||||||
|
b"BEGIN PRIVATE KEY",
|
||||||
|
b"PRIVATE KEY",
|
||||||
|
b"PEM private-key header",
|
||||||
|
b"assert 'BEGIN PRIVATE KEY' in message",
|
||||||
|
):
|
||||||
|
assert packager.secret_finding(value) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_complete_pem_private_key_header_is_rejected():
|
||||||
|
assert packager.secret_finding(real_pem_header())=="PEM_PRIVATE_KEY"
|
||||||
|
assert packager.secret_finding(
|
||||||
|
b"-----BEGIN OPENSSH " + b"PRIVATE KEY-----"
|
||||||
|
)=="PEM_PRIVATE_KEY"
|
||||||
|
|
||||||
|
|
||||||
|
def test_bearer_and_key_value_secret_detection_remains_fail_closed():
|
||||||
|
assert packager.secret_finding(bearer_value())=="AUTHORIZATION_BEARER"
|
||||||
|
assert packager.secret_finding(key_value_secret())=="KEY_VALUE_SECRET"
|
||||||
|
|
||||||
|
|
||||||
|
def test_short_or_descriptive_secret_words_do_not_false_positive():
|
||||||
|
assert packager.secret_finding(b"secret=short") is None
|
||||||
|
assert packager.secret_finding(b"password field is validated") is None
|
||||||
|
assert packager.secret_finding(b"api_key example placeholder") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_end_to_end_bundle_accepts_literal_begin_private_key_text(tmp_path):
|
||||||
|
bare,work,base=init_repo(tmp_path)
|
||||||
|
(work/"tests").mkdir()
|
||||||
|
(work/"tests"/"test_text.py").write_text(
|
||||||
|
'ERROR = "BEGIN PRIVATE KEY is not a complete PEM header"\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
(work/"docs").mkdir()
|
||||||
|
(work/"docs"/"note.md").write_text("Scanner regression.\n",encoding="utf-8")
|
||||||
|
candidate=commit(work,"candidate")
|
||||||
|
|
||||||
|
extra=tmp_path/"execution"
|
||||||
|
extra.mkdir()
|
||||||
|
(extra/"evidence.json").write_text('{"status":"PASS"}\n',encoding="utf-8")
|
||||||
|
out=tmp_path/"out"
|
||||||
|
|
||||||
|
summary=packager.build_bundle(
|
||||||
|
git_dir=bare,
|
||||||
|
base=base,
|
||||||
|
candidate=candidate,
|
||||||
|
task_id="r42-packager-test",
|
||||||
|
extra_dir=extra,
|
||||||
|
output_root=out,
|
||||||
|
now=datetime(2026,8,19,tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert summary["changed_file_count"]==2
|
||||||
|
assert summary["execution_file_count"]==1
|
||||||
|
assert summary["repository_document_count"]==1
|
||||||
|
assert summary["execution_document_count"]==0
|
||||||
|
bundle_path=Path(summary["bundle_path"])
|
||||||
|
assert bundle_path.is_file()
|
||||||
|
assert packager.sha256(bundle_path.read_bytes())==summary["bundle_sha256"]
|
||||||
|
|
||||||
|
with zipfile.ZipFile(bundle_path) as archive:
|
||||||
|
names=set(archive.namelist())
|
||||||
|
assert "repo/after/tests/test_text.py" in names
|
||||||
|
assert "repo/after/docs/note.md" in names
|
||||||
|
assert "execution/evidence.json" in names
|
||||||
|
assert "documents-index.json" in names
|
||||||
|
assert "all-modified-files-manifest.json" in names
|
||||||
|
assert "checksums.sha256" in names
|
||||||
|
manifest=json.loads(archive.read("all-modified-files-manifest.json"))
|
||||||
|
assert manifest["schema"]=="aios.all-modified-files.bundle.v1"
|
||||||
|
assert manifest["candidate_oid"]==candidate
|
||||||
|
|
||||||
|
packager.verify_checksums(out/"bundle")
|
||||||
|
|
||||||
|
|
||||||
|
def test_end_to_end_bundle_rejects_real_pem_header(tmp_path):
|
||||||
|
bare,work,base=init_repo(tmp_path)
|
||||||
|
(work/"credential.txt").write_bytes(
|
||||||
|
real_pem_header()+b"\nnot-a-real-key-body\n"
|
||||||
|
)
|
||||||
|
candidate=commit(work,"credential-like candidate")
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
packager.PackagerError,
|
||||||
|
match="PEM_PRIVATE_KEY:after:credential.txt",
|
||||||
|
):
|
||||||
|
packager.build_bundle(
|
||||||
|
git_dir=bare,
|
||||||
|
base=base,
|
||||||
|
candidate=candidate,
|
||||||
|
task_id="r42-packager-pem-reject",
|
||||||
|
output_root=tmp_path/"out",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_evidence_bearer_is_rejected(tmp_path):
|
||||||
|
bare,work,base=init_repo(tmp_path)
|
||||||
|
(work/"normal.txt").write_text("normal\n",encoding="utf-8")
|
||||||
|
candidate=commit(work,"normal")
|
||||||
|
extra=tmp_path/"execution"
|
||||||
|
extra.mkdir()
|
||||||
|
(extra/"unsafe.txt").write_bytes(bearer_value()+b"\n")
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
packager.PackagerError,
|
||||||
|
match="AUTHORIZATION_BEARER:execution:unsafe.txt",
|
||||||
|
):
|
||||||
|
packager.build_bundle(
|
||||||
|
git_dir=bare,
|
||||||
|
base=base,
|
||||||
|
candidate=candidate,
|
||||||
|
task_id="r42-packager-bearer-reject",
|
||||||
|
extra_dir=extra,
|
||||||
|
output_root=tmp_path/"out",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_symlink_is_rejected(tmp_path):
|
||||||
|
bare,work,base=init_repo(tmp_path)
|
||||||
|
(work/"normal.txt").write_text("normal\n",encoding="utf-8")
|
||||||
|
candidate=commit(work,"normal")
|
||||||
|
extra=tmp_path/"execution"
|
||||||
|
extra.mkdir()
|
||||||
|
target=extra/"target.txt"
|
||||||
|
target.write_text("safe\n",encoding="utf-8")
|
||||||
|
(extra/"link.txt").symlink_to(target)
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
packager.PackagerError,
|
||||||
|
match="EXTRA_FILE_SYMLINK_REJECTED",
|
||||||
|
):
|
||||||
|
packager.build_bundle(
|
||||||
|
git_dir=bare,
|
||||||
|
base=base,
|
||||||
|
candidate=candidate,
|
||||||
|
task_id="r42-packager-symlink-reject",
|
||||||
|
extra_dir=extra,
|
||||||
|
output_root=tmp_path/"out",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_ancestor_lineage_is_rejected(tmp_path):
|
||||||
|
bare,work,base=init_repo(tmp_path)
|
||||||
|
(work/"one.txt").write_text("one\n",encoding="utf-8")
|
||||||
|
first=commit(work,"first")
|
||||||
|
|
||||||
|
subprocess.run(["git","-C",str(work),"checkout","--detach",base],check=True,capture_output=True)
|
||||||
|
(work/"other.txt").write_text("other\n",encoding="utf-8")
|
||||||
|
subprocess.run(["git","-C",str(work),"add","other.txt"],check=True)
|
||||||
|
subprocess.run(["git","-C",str(work),"commit","-m","other"],check=True,capture_output=True)
|
||||||
|
other=subprocess.check_output(["git","-C",str(work),"rev-parse","HEAD"],text=True).strip()
|
||||||
|
subprocess.run(
|
||||||
|
["git","-C",str(work),"push","origin","HEAD:refs/heads/other"],
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(packager.PackagerError,match="LINEAGE_INVALID"):
|
||||||
|
packager.validate_lineage(bare,first,other)
|
||||||
|
|
||||||
|
|
||||||
|
def test_bundle_checksum_closure_detects_tamper(tmp_path):
|
||||||
|
bare,work,base=init_repo(tmp_path)
|
||||||
|
(work/"normal.txt").write_text("normal\n",encoding="utf-8")
|
||||||
|
candidate=commit(work,"normal")
|
||||||
|
out=tmp_path/"out"
|
||||||
|
packager.build_bundle(
|
||||||
|
git_dir=bare,
|
||||||
|
base=base,
|
||||||
|
candidate=candidate,
|
||||||
|
task_id="r42-packager-checksum",
|
||||||
|
output_root=out,
|
||||||
|
)
|
||||||
|
(out/"bundle"/"repo"/"after"/"normal.txt").write_text(
|
||||||
|
"tampered\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
with pytest.raises(
|
||||||
|
packager.PackagerError,
|
||||||
|
match="CHECKSUM_CLOSURE_INVALID",
|
||||||
|
):
|
||||||
|
packager.verify_checksums(out/"bundle")
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_does_not_contain_complete_pem_header_literal():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_all_modified_files_packager.py").read_bytes()
|
||||||
|
tests=(ROOT/"tests/test_all_modified_files_packager.py").read_bytes()
|
||||||
|
complete=real_pem_header()
|
||||||
|
assert complete not in source
|
||||||
|
assert complete not in tests
|
||||||
|
|
||||||
|
|
||||||
|
def test_document_path_contract():
|
||||||
|
assert packager.is_document_path("docs/spec.md")
|
||||||
|
assert packager.is_document_path("README")
|
||||||
|
assert packager.is_document_path("CHANGELOG.md")
|
||||||
|
assert not packager.is_document_path("active-runtime/app/main.py")
|
||||||
@@ -0,0 +1,442 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_durable_owner_authorization as auth
|
||||||
|
|
||||||
|
|
||||||
|
NOW=datetime(2026,8,19,6,0,0,tzinfo=timezone.utc)
|
||||||
|
ACCOUNT="chatgpt-governed-implementer-singlehost"
|
||||||
|
|
||||||
|
|
||||||
|
def write_root_file(path: Path, value: object) -> Path:
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_text(json.dumps(value,sort_keys=True)+"\n",encoding="utf-8")
|
||||||
|
path.chmod(0o600)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def policy_value(**overrides):
|
||||||
|
value={
|
||||||
|
"schema":auth.POLICY_SCHEMA,
|
||||||
|
"grant_id":"owner-permanent-preparation-20260819",
|
||||||
|
"owner_identity":"owner-singlehost",
|
||||||
|
"account":ACCOUNT,
|
||||||
|
"capabilities":[
|
||||||
|
"governed-task:project",
|
||||||
|
"release:candidate",
|
||||||
|
],
|
||||||
|
"issued_at":"2026-08-19T05:00:00Z",
|
||||||
|
"not_before":"2026-08-19T05:00:00Z",
|
||||||
|
"persistent_until_revoked":True,
|
||||||
|
"constraints":{
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"allowed_target_prefix":"agent/",
|
||||||
|
"release_version_prefix":"V2R2C",
|
||||||
|
"max_task_risk":"engineering",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
value.update(overrides)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def load_policy(tmp_path,monkeypatch,**overrides):
|
||||||
|
path=write_root_file(tmp_path/"policy.json",policy_value(**overrides))
|
||||||
|
revocations=write_root_file(
|
||||||
|
tmp_path/"revocations.json",
|
||||||
|
{"revoked_grant_ids":[]},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(auth,"validate_root_file",lambda path,**kwargs:path)
|
||||||
|
return auth.DurablePolicy.load(
|
||||||
|
path,
|
||||||
|
revocations,
|
||||||
|
now=NOW,
|
||||||
|
),path,revocations
|
||||||
|
|
||||||
|
|
||||||
|
def governed_request(**overrides):
|
||||||
|
value={
|
||||||
|
"task_id":"r42-owner-auth-projection-test-20260819",
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":"8"*40,
|
||||||
|
"expected_branch_oid":"8"*40,
|
||||||
|
"target_branch":"agent/r42-owner-auth-projection-test-20260819",
|
||||||
|
"generation":1,
|
||||||
|
"risk":"engineering",
|
||||||
|
"description":"project existing governed task",
|
||||||
|
"idempotency_key":"binding-owner-auth-projection-v1",
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"priority":100,
|
||||||
|
"max_attempts":3,
|
||||||
|
"require_online_task_projection":True,
|
||||||
|
"online_task":{
|
||||||
|
"goal":"Analyze source read-only.",
|
||||||
|
"instructions":"Use source capture first.",
|
||||||
|
"success_criteria":[
|
||||||
|
{"id":"source","description":"Return source snapshot."}
|
||||||
|
],
|
||||||
|
"constraints":{"repository_mutation":"forbidden"},
|
||||||
|
"required_capabilities":["git.read","git.test"],
|
||||||
|
"priority":100,
|
||||||
|
"max_attempts":3,
|
||||||
|
},
|
||||||
|
"execution":{
|
||||||
|
"action":"source_capture",
|
||||||
|
"adapter":"source_capture",
|
||||||
|
"capability_scope":["git.read","git.test"],
|
||||||
|
"source_signals":["durable authorization projection"],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
value.update(overrides)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def release_request(**overrides):
|
||||||
|
value={
|
||||||
|
"candidate_id":"r42-pf1-release-candidate-20260819",
|
||||||
|
"source_commit":"8"*40,
|
||||||
|
"version":"V2R2C2",
|
||||||
|
}
|
||||||
|
value.update(overrides)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class FakeUpstream:
|
||||||
|
def __init__(self):
|
||||||
|
self.project_calls=[]
|
||||||
|
|
||||||
|
def project_governed_task(self,body):
|
||||||
|
self.project_calls.append(dict(body))
|
||||||
|
return 200,{
|
||||||
|
"task_id":body["task_id"],
|
||||||
|
"job_id":"job-"+"a"*32,
|
||||||
|
"state":"BOUND",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def broker(tmp_path,monkeypatch):
|
||||||
|
policy,path,revocations=load_policy(tmp_path,monkeypatch)
|
||||||
|
store=auth.ReceiptStore(tmp_path/"receipts.sqlite3")
|
||||||
|
upstream=FakeUpstream()
|
||||||
|
instance=auth.Broker(
|
||||||
|
policy_path=path,
|
||||||
|
revocation_path=revocations,
|
||||||
|
store=store,
|
||||||
|
upstream=upstream,
|
||||||
|
expected_account=ACCOUNT,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
auth.DurablePolicy,
|
||||||
|
"load",
|
||||||
|
classmethod(lambda cls,*args,**kwargs:policy),
|
||||||
|
)
|
||||||
|
return instance,upstream,store,policy
|
||||||
|
|
||||||
|
|
||||||
|
def test_allowed_persistent_capabilities_are_only_preparation_scopes():
|
||||||
|
assert auth.ALLOWED_PERSISTENT_CAPABILITIES=={
|
||||||
|
"governed-task:project",
|
||||||
|
"release:candidate",
|
||||||
|
}
|
||||||
|
assert auth.ALLOWED_PERSISTENT_CAPABILITIES.isdisjoint(
|
||||||
|
auth.FORBIDDEN_PERSISTENT_CAPABILITIES
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"capability",
|
||||||
|
sorted(auth.FORBIDDEN_PERSISTENT_CAPABILITIES),
|
||||||
|
)
|
||||||
|
def test_policy_rejects_high_risk_persistent_capability(
|
||||||
|
tmp_path,monkeypatch,capability
|
||||||
|
):
|
||||||
|
bad=policy_value(
|
||||||
|
capabilities=["governed-task:project",capability]
|
||||||
|
)
|
||||||
|
path=write_root_file(tmp_path/"policy.json",bad)
|
||||||
|
revocations=write_root_file(
|
||||||
|
tmp_path/"revocations.json",
|
||||||
|
{"revoked_grant_ids":[]},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(auth,"validate_root_file",lambda path,**kwargs:path)
|
||||||
|
with pytest.raises(
|
||||||
|
auth.AuthorizationError,
|
||||||
|
match="POLICY_CAPABILITY_NOT_PERSISTABLE",
|
||||||
|
):
|
||||||
|
auth.DurablePolicy.load(path,revocations,now=NOW)
|
||||||
|
|
||||||
|
|
||||||
|
def test_policy_is_persistent_until_revoked(tmp_path,monkeypatch):
|
||||||
|
policy,_,_=load_policy(tmp_path,monkeypatch)
|
||||||
|
assert policy.persistent_until_revoked is True
|
||||||
|
assert policy.account==ACCOUNT
|
||||||
|
assert policy.capabilities=={
|
||||||
|
"governed-task:project",
|
||||||
|
"release:candidate",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_revocation_fails_closed(tmp_path,monkeypatch):
|
||||||
|
path=write_root_file(tmp_path/"policy.json",policy_value())
|
||||||
|
revocations=write_root_file(
|
||||||
|
tmp_path/"revocations.json",
|
||||||
|
{"revoked_grant_ids":["owner-permanent-preparation-20260819"]},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(auth,"validate_root_file",lambda path,**kwargs:path)
|
||||||
|
with pytest.raises(auth.AuthorizationError,match="OWNER_GRANT_REVOKED"):
|
||||||
|
auth.DurablePolicy.load(path,revocations,now=NOW)
|
||||||
|
|
||||||
|
|
||||||
|
def test_future_policy_is_not_active(tmp_path,monkeypatch):
|
||||||
|
with pytest.raises(auth.AuthorizationError,match="POLICY_NOT_ACTIVE"):
|
||||||
|
load_policy(
|
||||||
|
tmp_path,
|
||||||
|
monkeypatch,
|
||||||
|
issued_at="2026-08-20T05:00:00Z",
|
||||||
|
not_before="2026-08-20T05:00:00Z",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_projection_request_is_tightly_constrained(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
policy,_,_=load_policy(tmp_path,monkeypatch)
|
||||||
|
decision=policy.authorize(
|
||||||
|
"governed-task:project",
|
||||||
|
governed_request(),
|
||||||
|
)
|
||||||
|
assert decision["decision"]=="ALLOW"
|
||||||
|
assert decision["persistent_until_revoked"] is True
|
||||||
|
|
||||||
|
bad_caps=governed_request()
|
||||||
|
bad_caps["online_task"]={
|
||||||
|
**bad_caps["online_task"],
|
||||||
|
"required_capabilities":["release:candidate"],
|
||||||
|
}
|
||||||
|
bad_action=governed_request()
|
||||||
|
bad_action["execution"]={
|
||||||
|
**bad_action["execution"],
|
||||||
|
"action":"deterministic",
|
||||||
|
"adapter":"deterministic",
|
||||||
|
}
|
||||||
|
bad_scope=governed_request()
|
||||||
|
bad_scope["execution"]={
|
||||||
|
**bad_scope["execution"],
|
||||||
|
"capability_scope":["git.read"],
|
||||||
|
}
|
||||||
|
for bad in (
|
||||||
|
governed_request(repository="other/repo"),
|
||||||
|
governed_request(target_branch="main"),
|
||||||
|
governed_request(generation=2),
|
||||||
|
governed_request(risk="high"),
|
||||||
|
governed_request(expected_branch_oid="9"*40),
|
||||||
|
governed_request(branch_mode="CONTINUE_EXISTING"),
|
||||||
|
governed_request(priority=101),
|
||||||
|
governed_request(max_attempts=4),
|
||||||
|
governed_request(require_online_task_projection=False),
|
||||||
|
bad_caps,
|
||||||
|
bad_action,
|
||||||
|
bad_scope,
|
||||||
|
):
|
||||||
|
with pytest.raises(auth.AuthorizationError):
|
||||||
|
policy.authorize("governed-task:project",bad)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_capable_projection_is_allowed_only_as_source_capture_placeholder(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
policy,_,_=load_policy(tmp_path,monkeypatch)
|
||||||
|
request=governed_request()
|
||||||
|
request["online_task"]={
|
||||||
|
**request["online_task"],
|
||||||
|
"required_capabilities":[
|
||||||
|
"git.read",
|
||||||
|
"git.test",
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
request["execution"]={
|
||||||
|
**request["execution"],
|
||||||
|
"capability_scope":[
|
||||||
|
"git.read",
|
||||||
|
"git.test",
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
],
|
||||||
|
}
|
||||||
|
decision=policy.authorize("governed-task:project",request)
|
||||||
|
assert decision["decision"]=="ALLOW"
|
||||||
|
assert request["execution"]["action"]=="source_capture"
|
||||||
|
|
||||||
|
def test_release_candidate_request_is_tightly_constrained(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
policy,_,_=load_policy(tmp_path,monkeypatch)
|
||||||
|
decision=policy.authorize("release:candidate",release_request())
|
||||||
|
assert decision["decision"]=="ALLOW"
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
auth.AuthorizationError,
|
||||||
|
match="RELEASE_VERSION_NOT_AUTHORIZED",
|
||||||
|
):
|
||||||
|
policy.authorize(
|
||||||
|
"release:candidate",
|
||||||
|
release_request(version="V3R1C1"),
|
||||||
|
)
|
||||||
|
with pytest.raises(
|
||||||
|
auth.AuthorizationError,
|
||||||
|
match="RELEASE_CANDIDATE_FIELDS_INVALID",
|
||||||
|
):
|
||||||
|
policy.authorize(
|
||||||
|
"release:candidate",
|
||||||
|
{**release_request(),"execute":True},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_task_projection_executes_canonical_upstream(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,upstream,store,_=broker(tmp_path,monkeypatch)
|
||||||
|
request=governed_request()
|
||||||
|
response=instance.execute(
|
||||||
|
"governed-task:project",
|
||||||
|
request,
|
||||||
|
idempotency_key="owner-auth-projection-operation-0001",
|
||||||
|
)
|
||||||
|
assert len(upstream.project_calls)==1
|
||||||
|
assert upstream.project_calls[0]==request
|
||||||
|
assert response["operation_state"]=="EXECUTED"
|
||||||
|
assert response["external_capability_projection_required"] is False
|
||||||
|
assert response["separate_owner_release_approval_still_required"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_candidate_authorization_does_not_invent_builder_upstream(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,upstream,store,_=broker(tmp_path,monkeypatch)
|
||||||
|
response=instance.execute(
|
||||||
|
"release:candidate",
|
||||||
|
release_request(),
|
||||||
|
idempotency_key="owner-auth-release-candidate-operation-0001",
|
||||||
|
)
|
||||||
|
assert upstream.project_calls==[]
|
||||||
|
assert response["operation_state"]==(
|
||||||
|
"AUTHORIZED_PENDING_EXTERNAL_CAPABILITY_PROJECTION"
|
||||||
|
)
|
||||||
|
assert response["upstream_status"] is None
|
||||||
|
assert response["upstream_response"] is None
|
||||||
|
assert response["external_capability_projection_required"] is True
|
||||||
|
assert response["separate_owner_release_approval_still_required"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_exact_retry_returns_same_receipt_without_second_projection(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,upstream,_,_=broker(tmp_path,monkeypatch)
|
||||||
|
request=governed_request()
|
||||||
|
first=instance.execute(
|
||||||
|
"governed-task:project",
|
||||||
|
request,
|
||||||
|
idempotency_key="owner-auth-projection-operation-0002",
|
||||||
|
)
|
||||||
|
second=instance.execute(
|
||||||
|
"governed-task:project",
|
||||||
|
request,
|
||||||
|
idempotency_key="owner-auth-projection-operation-0002",
|
||||||
|
)
|
||||||
|
assert second==first
|
||||||
|
assert len(upstream.project_calls)==1
|
||||||
|
|
||||||
|
|
||||||
|
def test_idempotency_conflict_fails_closed(tmp_path,monkeypatch):
|
||||||
|
instance,_,_,_=broker(tmp_path,monkeypatch)
|
||||||
|
instance.execute(
|
||||||
|
"release:candidate",
|
||||||
|
release_request(),
|
||||||
|
idempotency_key="owner-auth-release-operation-0002",
|
||||||
|
)
|
||||||
|
with pytest.raises(auth.AuthorizationError,match="IDEMPOTENCY_CONFLICT"):
|
||||||
|
instance.execute(
|
||||||
|
"release:candidate",
|
||||||
|
release_request(version="V2R2C3"),
|
||||||
|
idempotency_key="owner-auth-release-operation-0002",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_receipt_store_contains_no_owner_policy_or_credentials(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
instance,_,store,_=broker(tmp_path,monkeypatch)
|
||||||
|
instance.execute(
|
||||||
|
"release:candidate",
|
||||||
|
release_request(),
|
||||||
|
idempotency_key="owner-auth-release-operation-0003",
|
||||||
|
)
|
||||||
|
con=sqlite3.connect(store.path)
|
||||||
|
columns=[row[1] for row in con.execute("pragma table_info(receipts)")]
|
||||||
|
joined=" ".join(columns).lower()
|
||||||
|
assert "token" not in joined
|
||||||
|
assert "credential" not in joined
|
||||||
|
assert "policy_json" not in columns
|
||||||
|
|
||||||
|
|
||||||
|
def test_systemd_broker_uses_unix_socket_and_only_host_control_credential():
|
||||||
|
unit=(
|
||||||
|
ROOT/"deploy/systemd/aios-owner-durable-authorization-broker.service"
|
||||||
|
).read_text()
|
||||||
|
assert "LoadCredential=agent_task_control_token:" in unit
|
||||||
|
assert "release_candidate_token" not in unit
|
||||||
|
assert "18947" not in unit
|
||||||
|
assert "AIOS_RELEASE_CANDIDATE_API_URL" not in unit
|
||||||
|
assert "AIOS_OWNER_AUTH_BROKER_SOCKET=%t/aios-owner-authorization-broker/broker.sock" in unit
|
||||||
|
assert "Group=aios-executor-exchange" in unit
|
||||||
|
assert "RestrictAddressFamilies=AF_UNIX AF_INET" in unit
|
||||||
|
assert "IPAddressDeny=any" in unit
|
||||||
|
assert "IPAddressAllow=127.0.0.0/8" in unit
|
||||||
|
assert "InaccessiblePaths=/data/aios-gitops /home" in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_contains_no_release_builder_endpoint_or_high_risk_execution():
|
||||||
|
source=(
|
||||||
|
ROOT/"active-runtime/app/aios_durable_owner_authorization.py"
|
||||||
|
).read_text()
|
||||||
|
assert "/createReleaseCandidate" not in source
|
||||||
|
assert "18947" not in source
|
||||||
|
assert "ThreadingHTTPServer" not in source
|
||||||
|
assert "ThreadingUnixStreamServer" in source
|
||||||
|
assert "startDeployment" in source
|
||||||
|
assert "tag_publish" in source
|
||||||
|
assert "baseline_registry_finalize" in source
|
||||||
|
assert "runtime_switch" in source
|
||||||
|
assert "AUTHORIZED_PENDING_EXTERNAL_CAPABILITY_PROJECTION" in source
|
||||||
|
assert "SAFE_PROJECTED_TASK_CAPABILITIES" in source
|
||||||
|
assert "INITIAL_EXECUTION_ACTION_NOT_AUTHORIZED" in source
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_docs_avoid_new_broker_bearer_secret():
|
||||||
|
doc=(
|
||||||
|
ROOT/"docs/superpowers/specs/2026-08-19-durable-owner-authorization.md"
|
||||||
|
).read_text()
|
||||||
|
assert "AF_UNIX" in doc
|
||||||
|
assert "avoids introducing another long-lived bearer secret" in doc
|
||||||
|
|
||||||
|
def test_docs_preserve_separate_owner_release_approval():
|
||||||
|
doc=(
|
||||||
|
ROOT/"docs/superpowers/specs/2026-08-19-durable-owner-authorization.md"
|
||||||
|
).read_text()
|
||||||
|
assert "persistent until explicitly revoked" in doc.lower()
|
||||||
|
assert "Owner signature" in doc
|
||||||
|
assert "AUTHORIZED_PENDING_EXTERNAL_CAPABILITY_PROJECTION" in doc
|
||||||
@@ -0,0 +1,232 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_online_agent_worker as worker
|
||||||
|
from aios_external_agent_runtime import (
|
||||||
|
ExternalAgentExchange,
|
||||||
|
ExternalAgentRuntimeError,
|
||||||
|
validate_request,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def envelope():
|
||||||
|
value=object.__new__(worker.ExecutionEnvelope)
|
||||||
|
object.__setattr__(value,"task_id","task-attempt-identity")
|
||||||
|
object.__setattr__(value,"job_id","job-attempt-identity")
|
||||||
|
object.__setattr__(value,"digest","d"*64)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_request_id_is_stable_within_attempt_and_changes_across_attempts():
|
||||||
|
env=envelope()
|
||||||
|
a1=worker.external_agent_request_id(env,"hermes",6)
|
||||||
|
a1_replay=worker.external_agent_request_id(env,"hermes",6)
|
||||||
|
a2=worker.external_agent_request_id(env,"hermes",7)
|
||||||
|
assert a1==a1_replay
|
||||||
|
assert a1!=a2
|
||||||
|
assert a1.startswith("external-hermes-")
|
||||||
|
assert a2.startswith("external-hermes-")
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_request_id_rejects_invalid_attempt():
|
||||||
|
env=envelope()
|
||||||
|
with pytest.raises(worker.WorkerError,match="TASK_ATTEMPT_INVALID"):
|
||||||
|
worker.external_agent_request_id(env,"hermes",0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_request_schema_binds_attempt():
|
||||||
|
value=validate_request({
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":"external-hermes-0123456789abcdef",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"task_id":"task-attempt-identity",
|
||||||
|
"attempt":7,
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":{
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
assert value["attempt"]==7
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_request_schema_rejects_missing_attempt():
|
||||||
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_ATTEMPT_INVALID"):
|
||||||
|
validate_request({
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":"external-hermes-0123456789abcdef",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"task_id":"task-attempt-identity",
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":{
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def _write_bound_result(path, request):
|
||||||
|
import hashlib
|
||||||
|
from aios_external_agent_runtime import canonical, sha256
|
||||||
|
material={
|
||||||
|
"schema":"aios.external-agent-runtime.result.v1",
|
||||||
|
"request_id":request["request_id"],
|
||||||
|
"request_sha256":sha256(canonical(request)),
|
||||||
|
"runtime":request["runtime"],
|
||||||
|
"task_id":request["task_id"],
|
||||||
|
"attempt":request["attempt"],
|
||||||
|
"execution_digest":request["execution_digest"],
|
||||||
|
"exit_code":0,
|
||||||
|
"duration_ms":1,
|
||||||
|
"output_text":"ok",
|
||||||
|
"output_sha256":hashlib.sha256(b"ok").hexdigest(),
|
||||||
|
"stderr_sha256":hashlib.sha256(b"").hexdigest(),
|
||||||
|
"credential_material_exposed":False,
|
||||||
|
"control_plane_access_granted":False,
|
||||||
|
}
|
||||||
|
material["result_sha256"]=sha256(canonical(material))
|
||||||
|
path.write_text(json.dumps(material,sort_keys=True),encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_exchange_result_binding_rejects_wrong_attempt(tmp_path):
|
||||||
|
exchange=ExternalAgentExchange(tmp_path)
|
||||||
|
request={
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":"external-hermes-0123456789abcdef",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"task_id":"task-attempt-identity",
|
||||||
|
"attempt":7,
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":{
|
||||||
|
"goal":"Return ok.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
request=validate_request(request)
|
||||||
|
result_path=tmp_path/"result.json"
|
||||||
|
_write_bound_result(result_path,request)
|
||||||
|
value=json.loads(result_path.read_text())
|
||||||
|
value["attempt"]=6
|
||||||
|
# Recompute digest so the binding check, not the digest check, is what fails.
|
||||||
|
from aios_external_agent_runtime import canonical,sha256
|
||||||
|
material=dict(value); material.pop("result_sha256")
|
||||||
|
value["result_sha256"]=sha256(canonical(material))
|
||||||
|
result_path.write_text(json.dumps(value),encoding="utf-8")
|
||||||
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_RESULT_BINDING_INVALID"):
|
||||||
|
exchange._validated_result(
|
||||||
|
result_path,
|
||||||
|
request,
|
||||||
|
sha256(canonical(request)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_governance_request_id_is_attempt_scoped():
|
||||||
|
env=envelope()
|
||||||
|
attempt6=worker.external_agent_request_id(env,"hermes",6)+":model"
|
||||||
|
attempt7=worker.external_agent_request_id(env,"hermes",7)+":model"
|
||||||
|
assert attempt6!=attempt7
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_previous_attempt_does_not_poison_next_model_governance_reservation(tmp_path):
|
||||||
|
from aios_model_governance import (
|
||||||
|
ModelGovernanceStore,
|
||||||
|
ModelRecord,
|
||||||
|
ProviderRecord,
|
||||||
|
RouteRequest,
|
||||||
|
ScopePolicy,
|
||||||
|
)
|
||||||
|
|
||||||
|
store=ModelGovernanceStore(tmp_path/"model-governance.sqlite3",now_fn=lambda:1_800_000_000.0)
|
||||||
|
store.register_provider(
|
||||||
|
ProviderRecord(
|
||||||
|
"oneapi",
|
||||||
|
"runtime-profile:hermes",
|
||||||
|
max_failures=5,
|
||||||
|
cooldown_seconds=60,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"deepseek-v4-flash",
|
||||||
|
"oneapi",
|
||||||
|
"v1",
|
||||||
|
("agent.runtime.hermes","text"),
|
||||||
|
input_cost_micros_per_1k=1,
|
||||||
|
output_cost_micros_per_1k=1,
|
||||||
|
max_input_tokens=10000,
|
||||||
|
max_output_tokens=1000,
|
||||||
|
evaluation_sha256="a"*64,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy(
|
||||||
|
"tenant",
|
||||||
|
"model-governance-live",
|
||||||
|
"staging",
|
||||||
|
allowed_models=("deepseek-v4-flash",),
|
||||||
|
required_capabilities=("text",),
|
||||||
|
daily_token_budget=100000,
|
||||||
|
daily_cost_budget_micros=100000,
|
||||||
|
fallback_limit=1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def route(request_id):
|
||||||
|
return store.route(
|
||||||
|
RouteRequest(
|
||||||
|
request_id=request_id,
|
||||||
|
task_id="r42-model-governance-hermes-live-20260818",
|
||||||
|
trace_id="trace-hermes-model-governance-live",
|
||||||
|
scope_type="tenant",
|
||||||
|
tenant_id="model-governance-live",
|
||||||
|
project_id="staging",
|
||||||
|
required_capabilities=("agent.runtime.hermes",),
|
||||||
|
estimated_input_tokens=100,
|
||||||
|
max_output_tokens=50,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
attempt6=route("external-hermes-attempt6:model")
|
||||||
|
store.record_failure(attempt6.request_id,"provider_process_failed")
|
||||||
|
row6=store.conn.execute(
|
||||||
|
"select state from model_governance_reservations where request_id=?",
|
||||||
|
(attempt6.request_id,),
|
||||||
|
).fetchone()
|
||||||
|
assert row6["state"]=="FAILED"
|
||||||
|
|
||||||
|
attempt7=route("external-hermes-attempt7:model")
|
||||||
|
assert attempt7.request_id!=attempt6.request_id
|
||||||
|
row7=store.conn.execute(
|
||||||
|
"select state from model_governance_reservations where request_id=?",
|
||||||
|
(attempt7.request_id,),
|
||||||
|
).fetchone()
|
||||||
|
assert row7["state"]=="RESERVED"
|
||||||
|
|
||||||
|
settled=store.record_success(
|
||||||
|
attempt7.request_id,
|
||||||
|
input_tokens=90,
|
||||||
|
output_tokens=40,
|
||||||
|
usage_source="runtime",
|
||||||
|
)
|
||||||
|
assert settled["state"]=="SETTLED"
|
||||||
|
assert store.conn.execute(
|
||||||
|
"select state from model_governance_reservations where request_id=?",
|
||||||
|
(attempt6.request_id,),
|
||||||
|
).fetchone()["state"]=="FAILED"
|
||||||
|
assert store.verify_audit_chain() is True
|
||||||
|
store.close()
|
||||||
@@ -110,6 +110,7 @@ def make_text_request(tmp_path, runtime, payload=None):
|
|||||||
"request_id":f"external-{runtime}-0123456789abcdef",
|
"request_id":f"external-{runtime}-0123456789abcdef",
|
||||||
"runtime":runtime,
|
"runtime":runtime,
|
||||||
"task_id":f"task-{runtime}-runtime",
|
"task_id":f"task-{runtime}-runtime",
|
||||||
|
"attempt":1,
|
||||||
"execution_digest":"d"*64,
|
"execution_digest":"d"*64,
|
||||||
"payload":payload or {
|
"payload":payload or {
|
||||||
"goal":"Return ok.",
|
"goal":"Return ok.",
|
||||||
@@ -193,6 +194,7 @@ def test_external_executor_runs_root_owned_profile_without_shell(tmp_path,monkey
|
|||||||
"request_id":f"external-{runtime}-0123456789abcdef",
|
"request_id":f"external-{runtime}-0123456789abcdef",
|
||||||
"runtime":runtime,
|
"runtime":runtime,
|
||||||
"task_id":f"task-{runtime}-runtime",
|
"task_id":f"task-{runtime}-runtime",
|
||||||
|
"attempt":1,
|
||||||
"execution_digest":"d"*64,
|
"execution_digest":"d"*64,
|
||||||
"payload":{
|
"payload":{
|
||||||
"goal":"Return ok.",
|
"goal":"Return ok.",
|
||||||
@@ -326,6 +328,7 @@ def test_worker_external_adapter_builds_readonly_result(monkeypatch,tmp_path,run
|
|||||||
"request_sha256":"e"*64,
|
"request_sha256":"e"*64,
|
||||||
"runtime":runtime,
|
"runtime":runtime,
|
||||||
"task_id":kwargs["task_id"],
|
"task_id":kwargs["task_id"],
|
||||||
|
"attempt":kwargs["attempt"],
|
||||||
"execution_digest":kwargs["execution_digest"],
|
"execution_digest":kwargs["execution_digest"],
|
||||||
"exit_code":0,
|
"exit_code":0,
|
||||||
"duration_ms":10,
|
"duration_ms":10,
|
||||||
@@ -339,7 +342,7 @@ def test_worker_external_adapter_builds_readonly_result(monkeypatch,tmp_path,run
|
|||||||
monkeypatch.setattr(worker,"ExternalAgentExchange",FakeExchange)
|
monkeypatch.setattr(worker,"ExternalAgentExchange",FakeExchange)
|
||||||
monkeypatch.setattr(worker,"audit",lambda *a,**k:None)
|
monkeypatch.setattr(worker,"audit",lambda *a,**k:None)
|
||||||
executor=worker.HermesExecutor() if runtime=="hermes" else worker.OpenClawExecutor()
|
executor=worker.HermesExecutor() if runtime=="hermes" else worker.OpenClawExecutor()
|
||||||
execution=executor.execute(envelope,tmp_path)
|
execution=executor.execute(envelope,tmp_path,attempt=1)
|
||||||
result=worker.build_agent_runtime_result(envelope.task_id,envelope,execution)
|
result=worker.build_agent_runtime_result(envelope.task_id,envelope,execution)
|
||||||
assert result["status"]=="SUCCEEDED"
|
assert result["status"]=="SUCCEEDED"
|
||||||
assert result["classification"]=="EXTERNAL_AGENT_RUNTIME_SUCCEEDED"
|
assert result["classification"]=="EXTERNAL_AGENT_RUNTIME_SUCCEEDED"
|
||||||
|
|||||||
@@ -0,0 +1,205 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from email.message import Message
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import stat
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_gpt_https_edge as edge
|
||||||
|
|
||||||
|
|
||||||
|
class Headers(dict):
|
||||||
|
def get(self,key,default=None):
|
||||||
|
return super().get(key,default)
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_and_public_bind_require_explicit_owner_approval():
|
||||||
|
assert edge.public_bind_allowed("127.0.0.1","0")=="127.0.0.1"
|
||||||
|
assert edge.public_bind_allowed("::1","0")=="::1"
|
||||||
|
with pytest.raises(RuntimeError,match="OWNER_APPROVAL_REQUIRED"):
|
||||||
|
edge.public_bind_allowed("0.0.0.0","0")
|
||||||
|
with pytest.raises(RuntimeError,match="OWNER_APPROVAL_REQUIRED"):
|
||||||
|
edge.public_bind_allowed("::","")
|
||||||
|
assert edge.public_bind_allowed("0.0.0.0","1")=="0.0.0.0"
|
||||||
|
|
||||||
|
|
||||||
|
def test_arbitrary_public_bind_address_is_denied_even_with_approval():
|
||||||
|
with pytest.raises(RuntimeError,match="BIND_ADDRESS_INVALID"):
|
||||||
|
edge.public_bind_allowed("192.0.2.10","1")
|
||||||
|
|
||||||
|
|
||||||
|
def test_public_domain_validation():
|
||||||
|
assert edge.validated_public_domain("tasks.example.com")=="tasks.example.com"
|
||||||
|
assert edge.validated_public_domain("")==""
|
||||||
|
with pytest.raises(RuntimeError,match="PUBLIC_DOMAIN_INVALID"):
|
||||||
|
edge.validated_public_domain("https://example.com")
|
||||||
|
with pytest.raises(RuntimeError,match="PUBLIC_DOMAIN_INVALID"):
|
||||||
|
edge.validated_public_domain("localhost")
|
||||||
|
|
||||||
|
|
||||||
|
def test_route_allowlist_accepts_only_readonly_api():
|
||||||
|
assert edge.allowed_route("POST","/v1/read-only-tasks")=="/v1/read-only-tasks"
|
||||||
|
assert edge.allowed_route("GET","/v1/read-only-tasks/external-request-0001")==(
|
||||||
|
"/v1/read-only-tasks/external-request-0001"
|
||||||
|
)
|
||||||
|
assert edge.allowed_route("GET","/health")=="/health"
|
||||||
|
assert edge.allowed_route("GET","/openapi.json")=="/openapi.json"
|
||||||
|
for method,path in [
|
||||||
|
("POST","/v2/governed-task"),
|
||||||
|
("GET","/v1/tasks/x"),
|
||||||
|
("POST","/v1/read-only-tasks/x"),
|
||||||
|
("DELETE","/v1/read-only-tasks/x"),
|
||||||
|
]:
|
||||||
|
with pytest.raises(edge.EdgeError,match="ROUTE_DENY"):
|
||||||
|
edge.allowed_route(method,path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_query_and_fragment_are_denied():
|
||||||
|
with pytest.raises(edge.EdgeError,match="QUERY_OR_FRAGMENT_DENY"):
|
||||||
|
edge.allowed_route("GET","/health?x=1")
|
||||||
|
with pytest.raises(edge.EdgeError,match="QUERY_OR_FRAGMENT_DENY"):
|
||||||
|
edge.allowed_route("GET","/openapi.json#x")
|
||||||
|
|
||||||
|
|
||||||
|
def test_forwarded_headers_are_minimal_and_require_bearer():
|
||||||
|
incoming=Headers({
|
||||||
|
"Authorization":"Bearer "+"a"*40,
|
||||||
|
"Content-Type":"application/json; charset=utf-8",
|
||||||
|
"Cookie":"secret=1",
|
||||||
|
"X-Forwarded-For":"203.0.113.10",
|
||||||
|
"X-Api-Key":"do-not-forward",
|
||||||
|
})
|
||||||
|
forwarded=edge.forwarded_headers(incoming,body=True)
|
||||||
|
assert forwarded=={
|
||||||
|
"Accept":"application/json",
|
||||||
|
"Authorization":"Bearer "+"a"*40,
|
||||||
|
"Content-Type":"application/json",
|
||||||
|
}
|
||||||
|
with pytest.raises(edge.EdgeError,match="AUTHORIZATION_REQUIRED"):
|
||||||
|
edge.forwarded_headers(Headers({}),body=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_content_type_is_denied():
|
||||||
|
with pytest.raises(edge.EdgeError,match="CONTENT_TYPE_INVALID"):
|
||||||
|
edge.forwarded_headers(
|
||||||
|
Headers({
|
||||||
|
"Authorization":"Bearer "+"a"*40,
|
||||||
|
"Content-Type":"text/plain",
|
||||||
|
}),
|
||||||
|
body=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_rate_limiter_caps_window_and_allows_expiry():
|
||||||
|
now=[100.0]
|
||||||
|
limiter=edge.RateLimiter(window_seconds=60,limit=2,burst=1,now_fn=lambda:now[0])
|
||||||
|
assert limiter.allowed("client")
|
||||||
|
assert limiter.allowed("client")
|
||||||
|
assert limiter.allowed("client")
|
||||||
|
assert not limiter.allowed("client")
|
||||||
|
now[0]=161.0
|
||||||
|
assert limiter.allowed("client")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_rate_limiter_client_table_is_bounded():
|
||||||
|
now=[100.0]
|
||||||
|
limiter=edge.RateLimiter(
|
||||||
|
window_seconds=60,
|
||||||
|
limit=10,
|
||||||
|
burst=0,
|
||||||
|
max_clients=2,
|
||||||
|
now_fn=lambda:now[0],
|
||||||
|
)
|
||||||
|
assert limiter.allowed("a")
|
||||||
|
assert limiter.allowed("b")
|
||||||
|
assert not limiter.allowed("c")
|
||||||
|
now[0]=161.0
|
||||||
|
assert limiter.allowed("c")
|
||||||
|
assert len(limiter._events) <= 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_source_does_not_log_sensitive_request_material():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_gpt_https_edge.py").read_text()
|
||||||
|
assert '"authorization", "body", "token", "cookie", "client_ip"' in source
|
||||||
|
assert "_client_pseudonym" in source
|
||||||
|
|
||||||
|
def test_proxy_upstream_is_fixed():
|
||||||
|
proxy=edge.Proxy()
|
||||||
|
assert proxy.upstream=="http://127.0.0.1:18945"
|
||||||
|
bad=edge.Proxy(upstream="http://127.0.0.1:18892")
|
||||||
|
with pytest.raises(edge.EdgeError,match="UPSTREAM_DRIFT_DENY"):
|
||||||
|
bad.request("GET","/health",{"Authorization":"Bearer "+"a"*40},None)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_public_bind_requires_approved_domain():
|
||||||
|
assert edge.validated_edge_identity("127.0.0.1","0","")==("127.0.0.1","")
|
||||||
|
with pytest.raises(RuntimeError,match="PUBLIC_DOMAIN_REQUIRED"):
|
||||||
|
edge.validated_edge_identity("0.0.0.0","1","")
|
||||||
|
assert edge.validated_edge_identity(
|
||||||
|
"0.0.0.0","1","tasks.example.com"
|
||||||
|
)==("0.0.0.0","tasks.example.com")
|
||||||
|
|
||||||
|
def test_tls_key_requires_no_group_or_other_permissions(tmp_path):
|
||||||
|
key=tmp_path/"key.pem"
|
||||||
|
key.write_text("x"*64,encoding="utf-8")
|
||||||
|
key.chmod(0o600)
|
||||||
|
assert edge.validate_tls_file(key,key=True)==key
|
||||||
|
key.chmod(0o644)
|
||||||
|
with pytest.raises(RuntimeError,match="TLS_KEY_MODE_INVALID"):
|
||||||
|
edge.validate_tls_file(key,key=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_tls_symlink_is_denied(tmp_path):
|
||||||
|
real=tmp_path/"cert.pem"
|
||||||
|
real.write_text("x"*64,encoding="utf-8")
|
||||||
|
link=tmp_path/"link.pem"
|
||||||
|
link.symlink_to(real)
|
||||||
|
with pytest.raises(RuntimeError,match="TLS_FILE_INVALID"):
|
||||||
|
edge.validate_tls_file(link,key=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_systemd_unit_is_disabled_by_owner_approval_condition_and_loopback_default():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-gpt-https-edge.service").read_text()
|
||||||
|
assert "ConditionPathExists=/etc/aios-gitops/gpt-https-edge.approved" in unit
|
||||||
|
assert "AIOS_GPT_EDGE_HOST=127.0.0.1" in unit
|
||||||
|
assert "AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=0" in unit
|
||||||
|
assert "AIOS_GPT_EDGE_PORT=18443" in unit
|
||||||
|
assert "LoadCredential=tls_cert:" in unit
|
||||||
|
assert "LoadCredential=tls_key:" in unit
|
||||||
|
assert "InaccessiblePaths=/data/aios-gitops" in unit
|
||||||
|
assert "InaccessiblePaths=" in unit and "/var/lib/aios-agent-task" in unit
|
||||||
|
assert "NoNewPrivileges=yes" in unit
|
||||||
|
assert "ProtectSystem=strict" in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_systemd_unit_does_not_reference_internal_control_credentials():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-gpt-https-edge.service").read_text()
|
||||||
|
assert "agent_task_control" not in unit
|
||||||
|
assert "gitea" not in unit.lower()
|
||||||
|
assert "18892" not in unit
|
||||||
|
assert "18940" not in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_edge_source_does_not_proxy_internal_bridge_or_task_api():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_gpt_https_edge.py").read_text()
|
||||||
|
assert 'UPSTREAM = "http://127.0.0.1:18945"' in source
|
||||||
|
assert "18892" not in source
|
||||||
|
assert "18940" not in source
|
||||||
|
assert "/v2/governed-task" not in source
|
||||||
|
assert "/v1/tasks/" not in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_docs_keep_public_activation_separately_approved():
|
||||||
|
doc=(ROOT/"docs/superpowers/specs/2026-08-19-gpt-https-edge.md").read_text()
|
||||||
|
assert "explicit owner approval" in doc.lower()
|
||||||
|
assert "PUBLIC_NETWORK_EXPOSURE=NONE" in doc
|
||||||
|
assert "PUBLIC_CUSTOM_GPT_DIRECT_AI0S_ACCESS=NOT_VERIFIED" in doc
|
||||||
@@ -0,0 +1,362 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
import aios_gpt_https_edge_activation as activation
|
||||||
|
|
||||||
|
|
||||||
|
CANDIDATE="69a51d50c7f6cd9045346d41e338acebe4b6f1e8"
|
||||||
|
NOW=datetime(2026,8,19,2,0,0,tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def approval_value(cert_sha: str, **overrides):
|
||||||
|
value={
|
||||||
|
"schema":"aios.gpt-https-edge.public-approval.v1",
|
||||||
|
"approval_id":"owner-edge-approval-0001",
|
||||||
|
"candidate_oid":CANDIDATE,
|
||||||
|
"domain":"tasks.example.com",
|
||||||
|
"public_bind":True,
|
||||||
|
"bind_host":"0.0.0.0",
|
||||||
|
"port":18443,
|
||||||
|
"tls_cert_sha256":cert_sha,
|
||||||
|
"issued_at":"2026-08-19T01:00:00Z",
|
||||||
|
"not_before":"2026-08-19T01:00:00Z",
|
||||||
|
"expires_at":"2026-08-20T01:00:00Z",
|
||||||
|
"owner":"owner@example",
|
||||||
|
}
|
||||||
|
value.update(overrides)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def write_owner_file(path: Path, data: bytes, mode=0o600):
|
||||||
|
path.write_bytes(data)
|
||||||
|
path.chmod(mode)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def load_approval(tmp_path, monkeypatch, **overrides):
|
||||||
|
tmp_path.mkdir(parents=True,exist_ok=True)
|
||||||
|
cert=write_owner_file(tmp_path/"cert.pem",b"certificate-material-"*4,0o600)
|
||||||
|
value=approval_value(hashlib.sha256(cert.read_bytes()).hexdigest(),**overrides)
|
||||||
|
approval=write_owner_file(
|
||||||
|
tmp_path/"approval.json",
|
||||||
|
(json.dumps(value,sort_keys=True)+"\n").encode(),
|
||||||
|
0o600,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
activation,
|
||||||
|
"validate_root_file",
|
||||||
|
lambda path, **kwargs: path,
|
||||||
|
)
|
||||||
|
return activation.Approval.load(
|
||||||
|
approval,
|
||||||
|
candidate_oid=CANDIDATE,
|
||||||
|
cert_path=cert,
|
||||||
|
now=NOW,
|
||||||
|
),approval,cert
|
||||||
|
|
||||||
|
|
||||||
|
class FakeSystemd:
|
||||||
|
def __init__(self):
|
||||||
|
self.active_units={activation.INGRESS_UNIT}
|
||||||
|
self.enabled_units=set()
|
||||||
|
self.calls=[]
|
||||||
|
self.exec_start=""
|
||||||
|
self.environment=""
|
||||||
|
self.fail_restart=False
|
||||||
|
|
||||||
|
def run(self,*args,check=True):
|
||||||
|
self.calls.append(args)
|
||||||
|
class Result:
|
||||||
|
returncode=0
|
||||||
|
stdout=""
|
||||||
|
if args[:2]==("restart",activation.EDGE_UNIT):
|
||||||
|
if self.fail_restart:
|
||||||
|
raise subprocess.CalledProcessError(1,args)
|
||||||
|
self.active_units.add(activation.EDGE_UNIT)
|
||||||
|
elif args[:2]==("stop",activation.EDGE_UNIT):
|
||||||
|
self.active_units.discard(activation.EDGE_UNIT)
|
||||||
|
elif args[:2]==("enable",activation.EDGE_UNIT):
|
||||||
|
self.enabled_units.add(activation.EDGE_UNIT)
|
||||||
|
elif args[:2]==("disable",activation.EDGE_UNIT):
|
||||||
|
self.enabled_units.discard(activation.EDGE_UNIT)
|
||||||
|
return Result()
|
||||||
|
|
||||||
|
def active(self,unit):
|
||||||
|
return unit in self.active_units
|
||||||
|
|
||||||
|
def enabled(self,unit):
|
||||||
|
return unit in self.enabled_units
|
||||||
|
|
||||||
|
def show(self,unit,prop):
|
||||||
|
if prop=="ExecStart":
|
||||||
|
return self.exec_start
|
||||||
|
if prop=="Environment":
|
||||||
|
return self.environment
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_approval_binds_candidate_domain_bind_port_and_cert(tmp_path,monkeypatch):
|
||||||
|
approval,_,cert=load_approval(tmp_path,monkeypatch)
|
||||||
|
assert approval.candidate_oid==CANDIDATE
|
||||||
|
assert approval.domain=="tasks.example.com"
|
||||||
|
assert approval.bind_host=="0.0.0.0"
|
||||||
|
assert approval.port==18443
|
||||||
|
assert approval.tls_cert_sha256==hashlib.sha256(cert.read_bytes()).hexdigest()
|
||||||
|
assert approval.expires_at>approval.not_before
|
||||||
|
|
||||||
|
|
||||||
|
def test_approval_rejects_candidate_mismatch(tmp_path,monkeypatch):
|
||||||
|
cert=write_owner_file(tmp_path/"cert.pem",b"certificate-material-"*4)
|
||||||
|
value=approval_value(hashlib.sha256(cert.read_bytes()).hexdigest())
|
||||||
|
approval=write_owner_file(tmp_path/"approval.json",json.dumps(value).encode())
|
||||||
|
monkeypatch.setattr(activation,"validate_root_file",lambda path,**kwargs:path)
|
||||||
|
with pytest.raises(activation.ActivationError,match="CANDIDATE_MISMATCH"):
|
||||||
|
activation.Approval.load(
|
||||||
|
approval,
|
||||||
|
candidate_oid="0"*40,
|
||||||
|
cert_path=cert,
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_approval_rejects_cert_hash_mismatch(tmp_path,monkeypatch):
|
||||||
|
cert=write_owner_file(tmp_path/"cert.pem",b"certificate-material-"*4)
|
||||||
|
value=approval_value("a"*64)
|
||||||
|
approval=write_owner_file(tmp_path/"approval.json",json.dumps(value).encode())
|
||||||
|
monkeypatch.setattr(activation,"validate_root_file",lambda path,**kwargs:path)
|
||||||
|
with pytest.raises(activation.ActivationError,match="CERT_SHA_MISMATCH"):
|
||||||
|
activation.Approval.load(
|
||||||
|
approval,
|
||||||
|
candidate_oid=CANDIDATE,
|
||||||
|
cert_path=cert,
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_approval_rejects_expired_not_yet_valid_and_long_lived(tmp_path,monkeypatch):
|
||||||
|
with pytest.raises(activation.ActivationError,match="APPROVAL_EXPIRED"):
|
||||||
|
load_approval(
|
||||||
|
tmp_path/"expired",
|
||||||
|
monkeypatch,
|
||||||
|
issued_at="2026-08-17T01:00:00Z",
|
||||||
|
not_before="2026-08-17T01:00:00Z",
|
||||||
|
expires_at="2026-08-18T01:00:00Z",
|
||||||
|
)
|
||||||
|
# Use distinct temp roots because helper writes fixed names.
|
||||||
|
|
||||||
|
with pytest.raises(activation.ActivationError,match="NOT_YET_VALID"):
|
||||||
|
load_approval(
|
||||||
|
tmp_path/"future",
|
||||||
|
monkeypatch,
|
||||||
|
issued_at="2026-08-19T03:00:00Z",
|
||||||
|
not_before="2026-08-19T03:00:00Z",
|
||||||
|
expires_at="2026-08-20T03:00:00Z",
|
||||||
|
)
|
||||||
|
with pytest.raises(activation.ActivationError,match="VALIDITY_TOO_LONG"):
|
||||||
|
load_approval(
|
||||||
|
tmp_path/"long",
|
||||||
|
monkeypatch,
|
||||||
|
issued_at="2026-08-18T01:00:00Z",
|
||||||
|
not_before="2026-08-18T01:00:00Z",
|
||||||
|
expires_at="2026-08-26T01:00:00Z",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_release_requires_edge_and_controller_and_denies_symlink(tmp_path):
|
||||||
|
release=tmp_path/"release"
|
||||||
|
app=release/"active-runtime/app"
|
||||||
|
app.mkdir(parents=True)
|
||||||
|
edge_file=app/"aios_gpt_https_edge.py"
|
||||||
|
controller=app/"aios_gpt_https_edge_activation.py"
|
||||||
|
edge_file.write_text("edge\n")
|
||||||
|
controller.write_text("controller\n")
|
||||||
|
value=activation.validate_release(release)
|
||||||
|
assert value["edge"]==edge_file
|
||||||
|
assert value["controller"]==controller
|
||||||
|
controller.unlink()
|
||||||
|
with pytest.raises(activation.ActivationError,match="IMMUTABLE_RELEASE_INVALID"):
|
||||||
|
activation.validate_release(release)
|
||||||
|
|
||||||
|
def test_render_dropin_is_exact_and_contains_no_secret(tmp_path,monkeypatch):
|
||||||
|
approval,_,_=load_approval(tmp_path,monkeypatch)
|
||||||
|
release=Path("/opt/aios/releases")/(CANDIDATE+"-gpt-https-edge-public")
|
||||||
|
text=activation.render_dropin(approval,release)
|
||||||
|
assert "AIOS_GPT_EDGE_HOST=0.0.0.0" in text
|
||||||
|
assert "AIOS_GPT_EDGE_PORT=18443" in text
|
||||||
|
assert "AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=1" in text
|
||||||
|
assert "AIOS_GPT_EDGE_PUBLIC_DOMAIN=tasks.example.com" in text
|
||||||
|
assert CANDIDATE in text
|
||||||
|
for forbidden in ("Bearer ","agent_task_control","tls_key","gpt_ingress_token"):
|
||||||
|
assert forbidden not in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_receipt_contains_binding_but_no_owner_secret_material(tmp_path,monkeypatch):
|
||||||
|
approval,_,_=load_approval(tmp_path,monkeypatch)
|
||||||
|
receipt=activation.receipt_for(
|
||||||
|
approval,
|
||||||
|
Path("/opt/aios/releases/test"),
|
||||||
|
state="ACTIVE",
|
||||||
|
)
|
||||||
|
assert receipt["approval_id"]==approval.approval_id
|
||||||
|
assert receipt["candidate_oid"]==CANDIDATE
|
||||||
|
assert receipt["domain"]=="tasks.example.com"
|
||||||
|
serialized=json.dumps(receipt)
|
||||||
|
assert "Bearer " not in serialized
|
||||||
|
assert "private" not in serialized.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_activate_is_idempotent_for_same_active_approval(tmp_path,monkeypatch):
|
||||||
|
approval,_,cert=load_approval(tmp_path,monkeypatch)
|
||||||
|
release=tmp_path/"release"
|
||||||
|
app=release/"active-runtime/app/aios_gpt_https_edge.py"
|
||||||
|
controller=release/"active-runtime/app/aios_gpt_https_edge_activation.py"
|
||||||
|
app.parent.mkdir(parents=True)
|
||||||
|
app.write_text("print('x')\n")
|
||||||
|
controller.write_text("print('controller')\n")
|
||||||
|
key=write_owner_file(tmp_path/"key.pem",b"private-key-material-"*4,0o600)
|
||||||
|
dropin=tmp_path/"edge.d"/activation.DROPIN_NAME
|
||||||
|
state=tmp_path/"state.json"
|
||||||
|
systemd=FakeSystemd()
|
||||||
|
|
||||||
|
monkeypatch.setattr(activation.os,"geteuid",lambda:0)
|
||||||
|
monkeypatch.setattr(activation,"require_ingress_ready",lambda _:None)
|
||||||
|
monkeypatch.setattr(activation,"validate_tls_pair",lambda *a,**k:{"cert_sha256":approval.tls_cert_sha256})
|
||||||
|
monkeypatch.setattr(activation,"tls_probe",lambda *a,**k:None)
|
||||||
|
|
||||||
|
systemd.exec_start=str(app)
|
||||||
|
systemd.environment=(
|
||||||
|
"AIOS_GPT_EDGE_HOST=0.0.0.0 "
|
||||||
|
"AIOS_GPT_EDGE_PORT=18443 "
|
||||||
|
"AIOS_GPT_EDGE_PUBLIC_BIND_APPROVED=1 "
|
||||||
|
"AIOS_GPT_EDGE_PUBLIC_DOMAIN=tasks.example.com"
|
||||||
|
)
|
||||||
|
first=activation.activate(
|
||||||
|
approval=approval,
|
||||||
|
release_root=release,
|
||||||
|
cert_path=cert,
|
||||||
|
key_path=key,
|
||||||
|
dropin_path=dropin,
|
||||||
|
state_path=state,
|
||||||
|
systemd=systemd,
|
||||||
|
)
|
||||||
|
calls_after_first=len(systemd.calls)
|
||||||
|
second=activation.activate(
|
||||||
|
approval=approval,
|
||||||
|
release_root=release,
|
||||||
|
cert_path=cert,
|
||||||
|
key_path=key,
|
||||||
|
dropin_path=dropin,
|
||||||
|
state_path=state,
|
||||||
|
systemd=systemd,
|
||||||
|
)
|
||||||
|
assert first["state"]=="ACTIVE"
|
||||||
|
assert second==json.loads(state.read_text())
|
||||||
|
assert len(systemd.calls)==calls_after_first
|
||||||
|
|
||||||
|
|
||||||
|
def test_activate_failure_after_mutation_rolls_back(tmp_path,monkeypatch):
|
||||||
|
approval,_,cert=load_approval(tmp_path,monkeypatch)
|
||||||
|
release=tmp_path/"release"
|
||||||
|
app=release/"active-runtime/app/aios_gpt_https_edge.py"
|
||||||
|
controller=release/"active-runtime/app/aios_gpt_https_edge_activation.py"
|
||||||
|
app.parent.mkdir(parents=True)
|
||||||
|
app.write_text("print('x')\n")
|
||||||
|
controller.write_text("print('controller')\n")
|
||||||
|
key=write_owner_file(tmp_path/"key.pem",b"private-key-material-"*4,0o600)
|
||||||
|
dropin=tmp_path/"edge.d"/activation.DROPIN_NAME
|
||||||
|
state=tmp_path/"state.json"
|
||||||
|
systemd=FakeSystemd()
|
||||||
|
systemd.fail_restart=True
|
||||||
|
|
||||||
|
monkeypatch.setattr(activation.os,"geteuid",lambda:0)
|
||||||
|
monkeypatch.setattr(activation,"require_ingress_ready",lambda _:None)
|
||||||
|
monkeypatch.setattr(activation,"validate_tls_pair",lambda *a,**k:{"cert_sha256":approval.tls_cert_sha256})
|
||||||
|
monkeypatch.setattr(activation,"tls_probe",lambda *a,**k:None)
|
||||||
|
|
||||||
|
with pytest.raises(subprocess.CalledProcessError):
|
||||||
|
activation.activate(
|
||||||
|
approval=approval,
|
||||||
|
release_root=release,
|
||||||
|
cert_path=cert,
|
||||||
|
key_path=key,
|
||||||
|
dropin_path=dropin,
|
||||||
|
state_path=state,
|
||||||
|
systemd=systemd,
|
||||||
|
)
|
||||||
|
assert not dropin.exists()
|
||||||
|
assert not systemd.active(activation.EDGE_UNIT)
|
||||||
|
assert json.loads(state.read_text())["state"]=="ROLLED_BACK"
|
||||||
|
|
||||||
|
|
||||||
|
def test_rollback_is_replay_safe_and_leaves_owner_assets_alone(tmp_path,monkeypatch):
|
||||||
|
approval_file=tmp_path/"approval.json"
|
||||||
|
cert=tmp_path/"cert.pem"
|
||||||
|
key=tmp_path/"key.pem"
|
||||||
|
for path in (approval_file,cert,key):
|
||||||
|
path.write_text("owner-asset"*4)
|
||||||
|
dropin=tmp_path/"edge.d"/activation.DROPIN_NAME
|
||||||
|
dropin.parent.mkdir(parents=True)
|
||||||
|
dropin.write_text("[Service]\n")
|
||||||
|
state=tmp_path/"state.json"
|
||||||
|
state.write_text(json.dumps({
|
||||||
|
"schema":activation.STATE_SCHEMA,
|
||||||
|
"state":"ACTIVE",
|
||||||
|
"candidate_oid":CANDIDATE,
|
||||||
|
"approval_id":"owner-edge-approval-0001",
|
||||||
|
"domain":"tasks.example.com",
|
||||||
|
}))
|
||||||
|
systemd=FakeSystemd()
|
||||||
|
systemd.active_units.add(activation.EDGE_UNIT)
|
||||||
|
systemd.enabled_units.add(activation.EDGE_UNIT)
|
||||||
|
monkeypatch.setattr(activation.os,"geteuid",lambda:0)
|
||||||
|
|
||||||
|
first=activation.rollback(
|
||||||
|
dropin_path=dropin,
|
||||||
|
state_path=state,
|
||||||
|
systemd=systemd,
|
||||||
|
reason="owner_requested",
|
||||||
|
)
|
||||||
|
second=activation.rollback(
|
||||||
|
dropin_path=dropin,
|
||||||
|
state_path=state,
|
||||||
|
systemd=systemd,
|
||||||
|
reason="owner_requested",
|
||||||
|
)
|
||||||
|
assert first["state"]=="ROLLED_BACK"
|
||||||
|
assert second["state"]=="ROLLED_BACK"
|
||||||
|
assert not dropin.exists()
|
||||||
|
assert not systemd.active(activation.EDGE_UNIT)
|
||||||
|
assert not systemd.enabled(activation.EDGE_UNIT)
|
||||||
|
for path in (approval_file,cert,key):
|
||||||
|
assert path.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_activation_source_never_creates_owner_approval_or_tls_secret():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_gpt_https_edge_activation.py").read_text()
|
||||||
|
assert "secrets.token" not in source
|
||||||
|
assert "openssl" not in source.lower()
|
||||||
|
assert "BEGIN PRIVATE KEY" not in source
|
||||||
|
assert "gpt-https-edge.approved" in source
|
||||||
|
assert "gpt-https-edge-tls.key" in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_docs_separate_implementation_activation_and_runtime_verification():
|
||||||
|
doc=(ROOT/"docs/superpowers/specs/2026-08-19-gpt-https-edge-activation.md").read_text()
|
||||||
|
assert "IMPLEMENTED" in doc
|
||||||
|
assert "OWNER_APPROVED" in doc
|
||||||
|
assert "ACTIVATED" in doc
|
||||||
|
assert "RUNTIME_VERIFIED" in doc
|
||||||
|
assert "does not own" in doc.lower()
|
||||||
@@ -0,0 +1,342 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
from aios_gpt_online_task_ingress import (
|
||||||
|
IngressError,
|
||||||
|
IngressService,
|
||||||
|
Store,
|
||||||
|
READONLY_CAPABILITIES,
|
||||||
|
branch_for,
|
||||||
|
normalized_request,
|
||||||
|
openapi_document,
|
||||||
|
task_id_for,
|
||||||
|
validated_bind_host,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
MAIN="3244f0f17f8845592ac94d585f5ebd65693c9f29"
|
||||||
|
JOB="job-0123456789abcdef0123456789abcdef"
|
||||||
|
|
||||||
|
|
||||||
|
def request(request_id="external-request-0001", **overrides):
|
||||||
|
value={
|
||||||
|
"schema":"aios.gpt-online-task-ingress.request.v1",
|
||||||
|
"request_id":request_id,
|
||||||
|
"goal":"Inspect the repository for durable worker state.",
|
||||||
|
"instructions":"Read only. Return a concise source analysis.",
|
||||||
|
"success_criteria":[
|
||||||
|
{"id":"analysis","description":"Return a source-backed analysis."}
|
||||||
|
],
|
||||||
|
"constraints":{
|
||||||
|
"max_files":50,
|
||||||
|
"focus_paths":["active-runtime/app","tests"],
|
||||||
|
"notes":"Do not mutate repository state.",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
value.update(overrides)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class FakeUpstream:
|
||||||
|
def __init__(self):
|
||||||
|
self.created=[]
|
||||||
|
self.projected=[]
|
||||||
|
self.tasks={}
|
||||||
|
self.snapshot={"authoritative_head_oid":MAIN}
|
||||||
|
|
||||||
|
def control_snapshot(self):
|
||||||
|
return dict(self.snapshot)
|
||||||
|
|
||||||
|
def create_governed_task(self, body):
|
||||||
|
self.created.append(dict(body))
|
||||||
|
task_id=body["task_id"]
|
||||||
|
job_id=JOB
|
||||||
|
self.tasks[task_id]={
|
||||||
|
"task_id":task_id,
|
||||||
|
"status":"PROJECTED",
|
||||||
|
"attempt":0,
|
||||||
|
"metadata":{
|
||||||
|
"execution":{
|
||||||
|
"task_id":task_id,
|
||||||
|
"job_id":job_id,
|
||||||
|
"base_oid":body["base_oid"],
|
||||||
|
"action":"source_capture",
|
||||||
|
"capability_scope":list(READONLY_CAPABILITIES),
|
||||||
|
"governance":{"writer_lease":False},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return 201,{"task_id":task_id,"job_id":job_id,"state":"BOUND"}
|
||||||
|
|
||||||
|
def execution_envelope(self, body):
|
||||||
|
self.projected.append(dict(body))
|
||||||
|
task_id=body["task_id"]
|
||||||
|
return 200,{
|
||||||
|
"schema":"aios.online-agent.execution.v2",
|
||||||
|
"task_id":task_id,
|
||||||
|
"job_id":body["job_id"],
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":MAIN,
|
||||||
|
"action":"source_capture",
|
||||||
|
"capability_scope":list(READONLY_CAPABILITIES),
|
||||||
|
"governance":{"writer_lease":False},
|
||||||
|
}
|
||||||
|
|
||||||
|
def get_task(self, task_id):
|
||||||
|
value=self.tasks.get(task_id)
|
||||||
|
return (404,{}) if value is None else (200,dict(value))
|
||||||
|
|
||||||
|
|
||||||
|
def service(tmp_path):
|
||||||
|
upstream=FakeUpstream()
|
||||||
|
store=Store(tmp_path/"state.sqlite3")
|
||||||
|
return IngressService(upstream,store),upstream,store
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_schema_is_narrow_and_rejects_capability_or_execution_control():
|
||||||
|
value=normalized_request(request())
|
||||||
|
assert value["goal"].startswith("Inspect")
|
||||||
|
for field in ("capability_scope","required_capabilities","action","adapter","repository","base_oid"):
|
||||||
|
bad=request()
|
||||||
|
bad[field]="evil"
|
||||||
|
with pytest.raises(IngressError,match="UNKNOWN_FIELDS"):
|
||||||
|
normalized_request(bad)
|
||||||
|
|
||||||
|
|
||||||
|
def test_constraints_are_bounded_and_path_traversal_is_denied():
|
||||||
|
bad=request(constraints={"focus_paths":["../etc/passwd"]})
|
||||||
|
with pytest.raises(IngressError,match="FOCUS_PATHS_INVALID"):
|
||||||
|
normalized_request(bad)
|
||||||
|
bad=request(constraints={"max_files":1000})
|
||||||
|
with pytest.raises(IngressError,match="MAX_FILES_INVALID"):
|
||||||
|
normalized_request(bad)
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_loopback_bind_is_denied():
|
||||||
|
assert validated_bind_host("127.0.0.1")=="127.0.0.1"
|
||||||
|
with pytest.raises(RuntimeError,match="GPT_INGRESS_NON_LOOPBACK_BIND_DENY"):
|
||||||
|
validated_bind_host("0.0.0.0")
|
||||||
|
with pytest.raises(RuntimeError,match="GPT_INGRESS_NON_LOOPBACK_BIND_DENY"):
|
||||||
|
validated_bind_host("192.0.2.10")
|
||||||
|
|
||||||
|
|
||||||
|
def test_task_and_branch_ids_are_deterministic_and_bounded():
|
||||||
|
a=task_id_for("external-request-0001")
|
||||||
|
b=task_id_for("external-request-0001")
|
||||||
|
c=task_id_for("external-request-0002")
|
||||||
|
assert a==b
|
||||||
|
assert a!=c
|
||||||
|
assert a.startswith("gpt-readonly-")
|
||||||
|
branch=branch_for(a)
|
||||||
|
assert branch.startswith("agent/external-gpt-readonly/")
|
||||||
|
assert len(a)<180
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_translates_to_canonical_readonly_governed_task(tmp_path):
|
||||||
|
svc,upstream,store=service(tmp_path)
|
||||||
|
response=svc.create(request())
|
||||||
|
assert response["state"]=="BOUND"
|
||||||
|
assert response["write_access"] is False
|
||||||
|
assert response["official_verification_required"] is True
|
||||||
|
assert response["capabilities"]==list(READONLY_CAPABILITIES)
|
||||||
|
|
||||||
|
assert len(upstream.created)==1
|
||||||
|
body=upstream.created[0]
|
||||||
|
assert body["repository"]=="aios/aios"
|
||||||
|
assert body["base_oid"]==MAIN
|
||||||
|
assert body["expected_branch_oid"]==MAIN
|
||||||
|
assert body["branch_mode"]=="CREATE_NEW"
|
||||||
|
assert body["risk"]=="low"
|
||||||
|
assert body["priority"]==0
|
||||||
|
assert body["max_attempts"]==3
|
||||||
|
assert body["require_online_task_projection"] is True
|
||||||
|
assert body["online_task"]["required_capabilities"]==list(READONLY_CAPABILITIES)
|
||||||
|
assert body["online_task"]["constraints"]["repository_mutation"]=="forbidden"
|
||||||
|
assert body["online_task"]["constraints"]["credential_access"]=="forbidden"
|
||||||
|
assert body["execution"]["action"]=="source_capture"
|
||||||
|
assert body["execution"]["adapter"]=="source_capture"
|
||||||
|
assert body["execution"]["capability_scope"]==list(READONLY_CAPABILITIES)
|
||||||
|
assert "writer_lease" not in body["execution"]
|
||||||
|
|
||||||
|
assert len(upstream.projected)==1
|
||||||
|
projection=upstream.projected[0]
|
||||||
|
assert projection["task_id"]==response["task_id"]
|
||||||
|
assert projection["job_id"]==JOB
|
||||||
|
assert projection["required_capabilities"]==list(READONLY_CAPABILITIES)
|
||||||
|
|
||||||
|
row=store.get("external-request-0001")
|
||||||
|
assert row is not None
|
||||||
|
assert row["state"]=="BOUND"
|
||||||
|
assert row["job_id"]==JOB
|
||||||
|
assert row["base_oid"]==MAIN
|
||||||
|
|
||||||
|
|
||||||
|
def test_exact_retry_is_idempotent_without_duplicate_bridge_call(tmp_path):
|
||||||
|
svc,upstream,_=service(tmp_path)
|
||||||
|
first=svc.create(request())
|
||||||
|
second=svc.create(request())
|
||||||
|
assert second==first
|
||||||
|
assert len(upstream.created)==1
|
||||||
|
assert len(upstream.projected)==1
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_request_id_with_different_content_is_conflict(tmp_path):
|
||||||
|
svc,upstream,_=service(tmp_path)
|
||||||
|
svc.create(request())
|
||||||
|
with pytest.raises(IngressError,match="IDEMPOTENCY_CONFLICT"):
|
||||||
|
svc.create(request(goal="Different goal."))
|
||||||
|
assert len(upstream.created)==1
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_request_recovers_existing_canonical_task_after_crash(tmp_path):
|
||||||
|
svc,upstream,store=service(tmp_path)
|
||||||
|
raw=normalized_request(request())
|
||||||
|
task_id=task_id_for(raw["request_id"])
|
||||||
|
store.reserve(raw,task_id)
|
||||||
|
|
||||||
|
upstream.tasks[task_id]={
|
||||||
|
"task_id":task_id,
|
||||||
|
"status":"PROJECTED",
|
||||||
|
"attempt":0,
|
||||||
|
"metadata":{
|
||||||
|
"execution":{
|
||||||
|
"task_id":task_id,
|
||||||
|
"job_id":JOB,
|
||||||
|
"base_oid":MAIN,
|
||||||
|
"action":"source_capture",
|
||||||
|
"capability_scope":list(READONLY_CAPABILITIES),
|
||||||
|
"governance":{"writer_lease":False},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
result=svc.create(request())
|
||||||
|
assert result["job_id"]==JOB
|
||||||
|
assert result["task_id"]==task_id
|
||||||
|
assert len(upstream.created)==0
|
||||||
|
row=store.get(raw["request_id"])
|
||||||
|
assert row["state"]=="BOUND"
|
||||||
|
|
||||||
|
|
||||||
|
def test_status_is_sanitized_and_never_proxies_raw_execution_or_result_text(tmp_path):
|
||||||
|
svc,upstream,store=service(tmp_path)
|
||||||
|
created=svc.create(request())
|
||||||
|
task_id=created["task_id"]
|
||||||
|
upstream.tasks[task_id]={
|
||||||
|
"task_id":task_id,
|
||||||
|
"status":"SUCCEEDED",
|
||||||
|
"attempt":1,
|
||||||
|
"claimed_by":"online-engineering-worker-cn-network-mkt-generic-20260815",
|
||||||
|
"metadata":{
|
||||||
|
"execution":{
|
||||||
|
"job_id":JOB,
|
||||||
|
"base_oid":MAIN,
|
||||||
|
"action":"source_capture",
|
||||||
|
"capability_scope":list(READONLY_CAPABILITIES),
|
||||||
|
"governance":{"writer_lease":False},
|
||||||
|
"secret_marker":"do-not-return",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"result":{
|
||||||
|
"status":"SUCCEEDED",
|
||||||
|
"classification":"NEEDS_REWORK_SOURCE_CAPTURE",
|
||||||
|
"execution_mode":"source_capture",
|
||||||
|
"output_text":"raw source text that must not be proxied",
|
||||||
|
"source_snapshot":{
|
||||||
|
"payload_sha256":"a"*64,
|
||||||
|
"file_count":42,
|
||||||
|
"files":["secret/path"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"verification":{
|
||||||
|
"decision":"SUCCEEDED",
|
||||||
|
"classification":"GENERIC_ONLINE_TASK_AUTO_CONSUMPTION_VERIFIED",
|
||||||
|
"private":"do-not-return",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
status=svc.status("external-request-0001")
|
||||||
|
assert status["state"]=="SUCCEEDED"
|
||||||
|
assert status["attempt"]==1
|
||||||
|
assert status["claimed"] is True
|
||||||
|
assert status["official_verification"]=="SUCCEEDED"
|
||||||
|
assert status["source_snapshot_sha256"]=="a"*64
|
||||||
|
assert status["source_snapshot_file_count"]==42
|
||||||
|
serialized=json.dumps(status)
|
||||||
|
assert "raw source text" not in serialized
|
||||||
|
assert "secret_marker" not in serialized
|
||||||
|
assert "secret/path" not in serialized
|
||||||
|
assert "do-not-return" not in serialized
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_status_can_recover_job_binding(tmp_path):
|
||||||
|
svc,upstream,store=service(tmp_path)
|
||||||
|
raw=normalized_request(request())
|
||||||
|
task_id=task_id_for(raw["request_id"])
|
||||||
|
store.reserve(raw,task_id)
|
||||||
|
upstream.tasks[task_id]={
|
||||||
|
"task_id":task_id,
|
||||||
|
"status":"PROJECTED",
|
||||||
|
"attempt":0,
|
||||||
|
"metadata":{
|
||||||
|
"execution":{
|
||||||
|
"job_id":JOB,
|
||||||
|
"base_oid":MAIN,
|
||||||
|
"action":"source_capture",
|
||||||
|
"capability_scope":list(READONLY_CAPABILITIES),
|
||||||
|
"governance":{"writer_lease":False},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
value=svc.status(raw["request_id"])
|
||||||
|
assert value["job_id"]==JOB
|
||||||
|
assert store.get(raw["request_id"])["state"]=="BOUND"
|
||||||
|
|
||||||
|
|
||||||
|
def test_openapi_exposes_only_create_readonly_and_status_operations():
|
||||||
|
doc=openapi_document()
|
||||||
|
assert doc["openapi"]=="3.1.0"
|
||||||
|
paths=doc["paths"]
|
||||||
|
assert set(paths)=={
|
||||||
|
"/v1/read-only-tasks",
|
||||||
|
"/v1/read-only-tasks/{request_id}",
|
||||||
|
}
|
||||||
|
rendered=json.dumps(doc)
|
||||||
|
for forbidden in ("writer-lease","commit","verify","production","tag","baseline"):
|
||||||
|
assert forbidden not in rendered.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_systemd_unit_is_loopback_credential_isolated_and_has_no_repository_access():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-gpt-online-task-ingress.service").read_text()
|
||||||
|
assert "DynamicUser=yes" in unit
|
||||||
|
assert "LoadCredential=gpt_ingress_token:" in unit
|
||||||
|
assert "LoadCredential=agent_task_control_token:" in unit
|
||||||
|
assert "AIOS_GPT_INGRESS_HOST=127.0.0.1" in unit
|
||||||
|
assert "AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892" in unit
|
||||||
|
assert "AIOS_AGENT_TASK_API_URL=http://127.0.0.1:18940" in unit
|
||||||
|
assert "IPAddressDeny=any" in unit
|
||||||
|
assert "IPAddressAllow=127.0.0.0/8" in unit
|
||||||
|
assert "InaccessiblePaths=/data/aios-gitops /etc/aios-gitops /home" in unit
|
||||||
|
assert "ProtectSystem=strict" in unit
|
||||||
|
assert "NoNewPrivileges=yes" in unit
|
||||||
|
assert "MemoryDenyWriteExecute=yes" in unit
|
||||||
|
assert "0.0.0.0" not in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_ingress_sqlite_contains_no_request_body_or_credentials(tmp_path):
|
||||||
|
svc,upstream,store=service(tmp_path)
|
||||||
|
svc.create(request(instructions="sensitive-looking but non-secret user text"))
|
||||||
|
con=sqlite3.connect(store.path)
|
||||||
|
columns=[row[1] for row in con.execute("pragma table_info(external_task_requests)")]
|
||||||
|
assert "request_json" not in columns
|
||||||
|
assert "goal" not in columns
|
||||||
|
assert "instructions" not in columns
|
||||||
|
assert "token" not in " ".join(columns).lower()
|
||||||
@@ -0,0 +1,325 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_model_governance import (
|
||||||
|
APPROVED,
|
||||||
|
ModelGovernanceError,
|
||||||
|
ModelGovernanceStore,
|
||||||
|
ModelRecord,
|
||||||
|
ProviderRecord,
|
||||||
|
RouteRequest,
|
||||||
|
ScopePolicy,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def setup_store(tmp_path, now=1_800_000_000.0):
|
||||||
|
store=ModelGovernanceStore(tmp_path/"model-governance.sqlite3",now_fn=lambda:now)
|
||||||
|
store.register_provider(
|
||||||
|
ProviderRecord(
|
||||||
|
"provider-a",
|
||||||
|
"runtime-profile:provider-a",
|
||||||
|
max_failures=1,
|
||||||
|
cooldown_seconds=120,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_provider(
|
||||||
|
ProviderRecord(
|
||||||
|
"provider-b",
|
||||||
|
"runtime-profile:provider-b",
|
||||||
|
max_failures=2,
|
||||||
|
cooldown_seconds=60,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"model-a",
|
||||||
|
"provider-a",
|
||||||
|
"2026-08-18",
|
||||||
|
("agent.runtime.hermes","text"),
|
||||||
|
input_cost_micros_per_1k=100,
|
||||||
|
output_cost_micros_per_1k=200,
|
||||||
|
max_input_tokens=10000,
|
||||||
|
max_output_tokens=4000,
|
||||||
|
evaluation_state=APPROVED,
|
||||||
|
evaluation_sha256="a"*64,
|
||||||
|
priority=10,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"model-b",
|
||||||
|
"provider-b",
|
||||||
|
"2026-08-17",
|
||||||
|
("agent.runtime.hermes","text"),
|
||||||
|
input_cost_micros_per_1k=120,
|
||||||
|
output_cost_micros_per_1k=250,
|
||||||
|
max_input_tokens=10000,
|
||||||
|
max_output_tokens=4000,
|
||||||
|
evaluation_state=APPROVED,
|
||||||
|
evaluation_sha256="b"*64,
|
||||||
|
priority=20,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy(
|
||||||
|
"tenant",
|
||||||
|
"tenant-a",
|
||||||
|
"project-a",
|
||||||
|
allowed_models=("model-a","model-b"),
|
||||||
|
required_capabilities=("text",),
|
||||||
|
max_input_tokens_per_request=5000,
|
||||||
|
max_output_tokens_per_request=2000,
|
||||||
|
daily_token_budget=12000,
|
||||||
|
daily_cost_budget_micros=4000,
|
||||||
|
fallback_limit=2,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return store
|
||||||
|
|
||||||
|
|
||||||
|
def req(request_id="request-001", *, exclude=()):
|
||||||
|
return RouteRequest(
|
||||||
|
request_id=request_id,
|
||||||
|
task_id="task-a",
|
||||||
|
trace_id="trace-a",
|
||||||
|
scope_type="tenant",
|
||||||
|
tenant_id="tenant-a",
|
||||||
|
project_id="project-a",
|
||||||
|
required_capabilities=("agent.runtime.hermes",),
|
||||||
|
estimated_input_tokens=1000,
|
||||||
|
max_output_tokens=500,
|
||||||
|
preferred_models=(),
|
||||||
|
exclude_providers=exclude,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_registry_rejects_raw_secret(tmp_path):
|
||||||
|
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_CREDENTIAL"):
|
||||||
|
store.register_provider(
|
||||||
|
ProviderRecord("provider-a","Bearer raw-secret-value")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_registration_requires_evaluation_gate(tmp_path):
|
||||||
|
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
|
||||||
|
store.register_provider(ProviderRecord("provider-a"))
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_EVALUATION_GATE_DENY"):
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"model-a","provider-a","v1",("text",),
|
||||||
|
evaluation_state="PENDING",evaluation_sha256="a"*64
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_scope_policy_requires_version_with_pin(tmp_path):
|
||||||
|
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_POLICY_PIN_VERSION_REQUIRED"):
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy("tenant","t","p",pinned_model_id="model-a")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_route_is_deterministic_version_pinned_and_idempotent(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
first=store.route(req())
|
||||||
|
second=store.route(req())
|
||||||
|
assert first==second
|
||||||
|
assert first.model_id=="model-a"
|
||||||
|
assert first.model_version=="2026-08-18"
|
||||||
|
assert first.provider_credential_ref=="runtime-profile:provider-a"
|
||||||
|
assert first.policy_revision==1
|
||||||
|
assert first.model_revision==1
|
||||||
|
snapshot=store.usage_snapshot("tenant","tenant-a","project-a")
|
||||||
|
assert snapshot["reserved_input_tokens"]==1000
|
||||||
|
assert snapshot["reserved_output_tokens"]==500
|
||||||
|
assert snapshot["reserved_cost_micros"]>0
|
||||||
|
|
||||||
|
|
||||||
|
def test_route_idempotency_payload_conflict_fails(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
store.route(req())
|
||||||
|
changed=RouteRequest(
|
||||||
|
**{**req().__dict__,"estimated_input_tokens":2000}
|
||||||
|
)
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_ROUTE_IDEMPOTENCY_CONFLICT"):
|
||||||
|
store.route(changed)
|
||||||
|
|
||||||
|
|
||||||
|
def test_budget_reservation_is_atomic(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy(
|
||||||
|
"tenant","tenant-a","project-a",
|
||||||
|
allowed_models=("model-a",),
|
||||||
|
required_capabilities=("text",),
|
||||||
|
daily_token_budget=1600,
|
||||||
|
daily_cost_budget_micros=10000,
|
||||||
|
),
|
||||||
|
expected_revision=1,
|
||||||
|
)
|
||||||
|
store.route(req("request-1"))
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_BUDGET_DENY"):
|
||||||
|
store.route(req("request-2"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_usage_settlement_is_idempotent_and_releases_reservation(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
decision=store.route(req())
|
||||||
|
first=store.record_success(
|
||||||
|
decision.request_id,
|
||||||
|
input_tokens=900,
|
||||||
|
output_tokens=400,
|
||||||
|
usage_source="runtime",
|
||||||
|
)
|
||||||
|
second=store.record_success(
|
||||||
|
decision.request_id,
|
||||||
|
input_tokens=900,
|
||||||
|
output_tokens=400,
|
||||||
|
usage_source="runtime",
|
||||||
|
)
|
||||||
|
assert first["idempotent"] is False
|
||||||
|
assert second["idempotent"] is True
|
||||||
|
snapshot=store.usage_snapshot("tenant","tenant-a","project-a")
|
||||||
|
assert snapshot["settled_input_tokens"]==900
|
||||||
|
assert snapshot["settled_output_tokens"]==400
|
||||||
|
assert snapshot["reserved_input_tokens"]==0
|
||||||
|
assert snapshot["reserved_output_tokens"]==0
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_runtime_usage_falls_back_to_reserved_upper_bound(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
decision=store.route(req())
|
||||||
|
result=store.record_success(
|
||||||
|
decision.request_id,
|
||||||
|
input_tokens=None,
|
||||||
|
output_tokens=None,
|
||||||
|
)
|
||||||
|
assert result["usage_source"]=="reserved_upper_bound"
|
||||||
|
assert result["input_tokens"]==1000
|
||||||
|
assert result["output_tokens"]==500
|
||||||
|
|
||||||
|
|
||||||
|
def test_usage_cannot_exceed_reserved_budget(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
decision=store.route(req())
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_USAGE_EXCEEDS_RESERVATION"):
|
||||||
|
store.record_success(
|
||||||
|
decision.request_id,
|
||||||
|
input_tokens=1001,
|
||||||
|
output_tokens=500,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_provider_failure_opens_circuit_and_fallback_uses_other_provider(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
original=req()
|
||||||
|
first=store.route(original)
|
||||||
|
assert first.provider_id=="provider-a"
|
||||||
|
failure=store.record_failure(first.request_id,"provider_timeout")
|
||||||
|
assert failure["open_until"]>0
|
||||||
|
fallback=store.fallback(original,first.request_id)
|
||||||
|
assert fallback.provider_id=="provider-b"
|
||||||
|
assert fallback.fallback_index==1
|
||||||
|
|
||||||
|
|
||||||
|
def test_circuit_excludes_provider_for_new_request(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
first=store.route(req("request-first"))
|
||||||
|
store.record_failure(first.request_id,"provider_timeout")
|
||||||
|
next_decision=store.route(req("request-next"))
|
||||||
|
assert next_decision.provider_id=="provider-b"
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_version_pin_is_enforced(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy(
|
||||||
|
"tenant","tenant-a","project-a",
|
||||||
|
allowed_models=("model-a","model-b"),
|
||||||
|
pinned_model_id="model-b",
|
||||||
|
pinned_model_version="2026-08-17",
|
||||||
|
daily_token_budget=12000,
|
||||||
|
daily_cost_budget_micros=4000,
|
||||||
|
),
|
||||||
|
expected_revision=1,
|
||||||
|
)
|
||||||
|
decision=store.route(req())
|
||||||
|
assert decision.model_id=="model-b"
|
||||||
|
assert decision.model_version=="2026-08-17"
|
||||||
|
|
||||||
|
|
||||||
|
def test_scope_isolation_denies_missing_cross_scope_policy(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
cross=RouteRequest(
|
||||||
|
request_id="request-cross",
|
||||||
|
task_id="task-a",
|
||||||
|
trace_id="trace-a",
|
||||||
|
scope_type="tenant",
|
||||||
|
tenant_id="tenant-b",
|
||||||
|
project_id="project-a",
|
||||||
|
required_capabilities=("agent.runtime.hermes",),
|
||||||
|
estimated_input_tokens=100,
|
||||||
|
max_output_tokens=50,
|
||||||
|
)
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_SCOPE_POLICY_MISSING"):
|
||||||
|
store.route(cross)
|
||||||
|
|
||||||
|
|
||||||
|
def test_audit_chain_detects_tampering(tmp_path):
|
||||||
|
store=setup_store(tmp_path)
|
||||||
|
decision=store.route(req())
|
||||||
|
store.record_success(decision.request_id,input_tokens=100,output_tokens=50)
|
||||||
|
assert store.verify_audit_chain() is True
|
||||||
|
store.conn.execute(
|
||||||
|
"UPDATE model_governance_audit SET payload_json='{}' WHERE sequence=1"
|
||||||
|
)
|
||||||
|
assert store.verify_audit_chain() is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_registry_and_policy_use_cas(tmp_path):
|
||||||
|
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
|
||||||
|
store.register_provider(ProviderRecord("p"))
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_PROVIDER_CAS_MISMATCH"):
|
||||||
|
store.register_provider(ProviderRecord("p"),expected_revision=7)
|
||||||
|
store.set_scope_policy(ScopePolicy("tenant","t","p"))
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_POLICY_CAS_MISMATCH"):
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy("tenant","t","p"),
|
||||||
|
expected_revision=7,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_system_policy_never_authorizes_missing_tenant_policy(tmp_path):
|
||||||
|
store=ModelGovernanceStore(tmp_path/"db.sqlite3")
|
||||||
|
store.register_provider(ProviderRecord("provider-a"))
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"model-a","provider-a","v1",("text",),
|
||||||
|
evaluation_sha256="a"*64,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy("system",None,None,allowed_models=("model-a",))
|
||||||
|
)
|
||||||
|
request=RouteRequest(
|
||||||
|
request_id="request-tenant-no-policy",
|
||||||
|
task_id="task-a",
|
||||||
|
trace_id="trace-a",
|
||||||
|
scope_type="tenant",
|
||||||
|
tenant_id="tenant-a",
|
||||||
|
project_id="project-a",
|
||||||
|
required_capabilities=("text",),
|
||||||
|
estimated_input_tokens=10,
|
||||||
|
max_output_tokens=10,
|
||||||
|
)
|
||||||
|
with pytest.raises(ModelGovernanceError,match="MODEL_SCOPE_POLICY_MISSING"):
|
||||||
|
store.route(request)
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_model_governance_admin import apply_config
|
||||||
|
from aios_model_governance import ModelGovernanceStore
|
||||||
|
|
||||||
|
|
||||||
|
def config():
|
||||||
|
return {
|
||||||
|
"schema":"aios.model-governance.bootstrap.v1",
|
||||||
|
"providers":[{
|
||||||
|
"provider_id":"provider-a",
|
||||||
|
"credential_ref":"runtime-profile:provider-a",
|
||||||
|
"max_failures":2,
|
||||||
|
"cooldown_seconds":60,
|
||||||
|
}],
|
||||||
|
"models":[{
|
||||||
|
"model_id":"model-a",
|
||||||
|
"provider_id":"provider-a",
|
||||||
|
"version":"v1",
|
||||||
|
"capabilities":["text"],
|
||||||
|
"input_cost_micros_per_1k":1,
|
||||||
|
"output_cost_micros_per_1k":2,
|
||||||
|
"max_input_tokens":1000,
|
||||||
|
"max_output_tokens":100,
|
||||||
|
"evaluation_state":"APPROVED",
|
||||||
|
"evaluation_sha256":"a"*64,
|
||||||
|
"priority":10,
|
||||||
|
}],
|
||||||
|
"policies":[{
|
||||||
|
"scope_type":"legacy",
|
||||||
|
"tenant_id":None,
|
||||||
|
"project_id":None,
|
||||||
|
"allowed_models":["model-a"],
|
||||||
|
"required_capabilities":["text"],
|
||||||
|
"daily_token_budget":10000,
|
||||||
|
"daily_cost_budget_micros":1000,
|
||||||
|
"fallback_limit":0,
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_admin_apply_is_idempotent(tmp_path):
|
||||||
|
cfg=tmp_path/"config.json"
|
||||||
|
cfg.write_text(json.dumps(config()),encoding="utf-8")
|
||||||
|
db=tmp_path/"db.sqlite3"
|
||||||
|
first=apply_config(db,cfg)
|
||||||
|
second=apply_config(db,cfg)
|
||||||
|
assert first["changes"]["providers"]==1
|
||||||
|
assert first["changes"]["models"]==1
|
||||||
|
assert first["changes"]["policies"]==1
|
||||||
|
assert second["changes"]["providers"]==0
|
||||||
|
assert second["changes"]["models"]==0
|
||||||
|
assert second["changes"]["policies"]==0
|
||||||
|
assert second["changes"]["unchanged"]==3
|
||||||
|
store=ModelGovernanceStore(db)
|
||||||
|
assert store.verify_audit_chain() is True
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_admin_config_change_uses_cas_revision(tmp_path):
|
||||||
|
value=config()
|
||||||
|
cfg=tmp_path/"config.json"
|
||||||
|
cfg.write_text(json.dumps(value),encoding="utf-8")
|
||||||
|
db=tmp_path/"db.sqlite3"
|
||||||
|
apply_config(db,cfg)
|
||||||
|
value["providers"][0]["cooldown_seconds"]=120
|
||||||
|
cfg.write_text(json.dumps(value),encoding="utf-8")
|
||||||
|
result=apply_config(db,cfg)
|
||||||
|
assert result["changes"]["providers"]==1
|
||||||
|
store=ModelGovernanceStore(db)
|
||||||
|
row=store.conn.execute(
|
||||||
|
"select revision,cooldown_seconds from model_governance_providers where provider_id='provider-a'"
|
||||||
|
).fetchone()
|
||||||
|
assert tuple(row)==(2,120)
|
||||||
|
store.close()
|
||||||
@@ -0,0 +1,554 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
APP=ROOT/"active-runtime/app"
|
||||||
|
sys.path.insert(0,str(APP))
|
||||||
|
|
||||||
|
from aios_model_governance import (
|
||||||
|
ModelGovernanceStore,
|
||||||
|
ModelRecord,
|
||||||
|
ProviderRecord,
|
||||||
|
ScopePolicy,
|
||||||
|
)
|
||||||
|
from aios_online_agent_execution import GovernedExecutionBinding, project_online_task
|
||||||
|
import aios_external_agent_runtime as runtime
|
||||||
|
|
||||||
|
|
||||||
|
BASE="a"*40
|
||||||
|
|
||||||
|
|
||||||
|
def binding(task_id):
|
||||||
|
return GovernedExecutionBinding.from_mapping({
|
||||||
|
"task_id":task_id,
|
||||||
|
"job_id":"job-"+task_id,
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,
|
||||||
|
"expected_branch_oid":BASE,
|
||||||
|
"target_branch":"agent/"+task_id,
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"generation":1,
|
||||||
|
"idempotency_key":task_id+"-binding-v1",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def projected(task_id="task-model-governed", *, scope_type="tenant", tenant_id="tenant-a", project_id="project-a"):
|
||||||
|
request={
|
||||||
|
"task_id":task_id,
|
||||||
|
"goal":"Return the requested sentinel.",
|
||||||
|
"instructions":"No repository or control-plane side effects.",
|
||||||
|
"success_criteria":[{"id":"done","description":"Return a result."}],
|
||||||
|
"constraints":{"repository_mutation":"forbidden"},
|
||||||
|
"scope_type":scope_type,
|
||||||
|
"tenant_id":tenant_id,
|
||||||
|
"project_id":project_id,
|
||||||
|
"trace_id":"trace-model-a",
|
||||||
|
"required_capabilities":["agent.runtime.hermes"],
|
||||||
|
}
|
||||||
|
return project_online_task(
|
||||||
|
binding(task_id),
|
||||||
|
request,
|
||||||
|
{
|
||||||
|
"action":"agent_runtime",
|
||||||
|
"adapter":"hermes",
|
||||||
|
"capability_scope":["agent.runtime.hermes"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def helper(tmp_path):
|
||||||
|
path=tmp_path/"model_runtime.py"
|
||||||
|
path.write_text(
|
||||||
|
"import json,sys\n"
|
||||||
|
"model=sys.argv[1]\n"
|
||||||
|
"provider=sys.argv[2]\n"
|
||||||
|
"payload=json.load(sys.stdin)\n"
|
||||||
|
"if model=='model-a' and payload.get('constraints',{}).get('force_fail_a'):\n"
|
||||||
|
" raise SystemExit(7)\n"
|
||||||
|
"print(json.dumps({'answer':'ok','model':model,'provider':provider,"
|
||||||
|
"'usage':{'input_tokens':20,'output_tokens':10,'cost_micros':2}},sort_keys=True))\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def write_profile(tmp_path, helper_path, *, providers=("provider-a","provider-b")):
|
||||||
|
root=tmp_path/"profiles";root.mkdir(exist_ok=True)
|
||||||
|
value={
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3",str(helper_path),"{model_id}","{provider_id}"],
|
||||||
|
"input_mode":"stdin_json",
|
||||||
|
"output_mode":"stdout_json",
|
||||||
|
"timeout_seconds":30,
|
||||||
|
"max_output_bytes":65536,
|
||||||
|
"environment":{},
|
||||||
|
"credential_env":{},
|
||||||
|
"model_governance":{
|
||||||
|
"required":True,
|
||||||
|
"provider_ids":list(providers),
|
||||||
|
"model_capabilities":["text"],
|
||||||
|
"max_output_tokens":100,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
(root/"hermes.json").write_text(json.dumps(value),encoding="utf-8")
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def seed(db_path, *, legacy=False, tiny_budget=False):
|
||||||
|
store=ModelGovernanceStore(db_path,now_fn=lambda:1_800_000_000.0)
|
||||||
|
for provider in ("provider-a","provider-b"):
|
||||||
|
store.register_provider(
|
||||||
|
ProviderRecord(
|
||||||
|
provider,
|
||||||
|
f"runtime-profile:{provider}",
|
||||||
|
max_failures=1 if provider=="provider-a" else 2,
|
||||||
|
cooldown_seconds=120,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"model-a","provider-a","v1",
|
||||||
|
("agent.runtime.hermes","text"),
|
||||||
|
input_cost_micros_per_1k=10,
|
||||||
|
output_cost_micros_per_1k=20,
|
||||||
|
max_input_tokens=5000,max_output_tokens=1000,
|
||||||
|
evaluation_sha256="a"*64,priority=1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"model-b","provider-b","v2",
|
||||||
|
("agent.runtime.hermes","text"),
|
||||||
|
input_cost_micros_per_1k=12,
|
||||||
|
output_cost_micros_per_1k=22,
|
||||||
|
max_input_tokens=5000,max_output_tokens=1000,
|
||||||
|
evaluation_sha256="b"*64,priority=2,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if legacy:
|
||||||
|
policy=ScopePolicy(
|
||||||
|
"legacy",None,None,
|
||||||
|
allowed_models=("model-a","model-b"),
|
||||||
|
required_capabilities=("text",),
|
||||||
|
daily_token_budget=10000,
|
||||||
|
daily_cost_budget_micros=10000,
|
||||||
|
fallback_limit=2,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
policy=ScopePolicy(
|
||||||
|
"tenant","tenant-a","project-a",
|
||||||
|
allowed_models=("model-a","model-b"),
|
||||||
|
required_capabilities=("text",),
|
||||||
|
daily_token_budget=1 if tiny_budget else 10000,
|
||||||
|
daily_cost_budget_micros=10000,
|
||||||
|
fallback_limit=2,
|
||||||
|
)
|
||||||
|
store.set_scope_policy(policy)
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def make_external_request(projected_task, *, force_fail=False, attempt=1):
|
||||||
|
execution=projected_task["metadata"]["execution"]
|
||||||
|
payload=dict(execution["runtime_input"])
|
||||||
|
if force_fail:
|
||||||
|
payload["constraints"]=dict(payload["constraints"])
|
||||||
|
payload["constraints"]["force_fail_a"]=True
|
||||||
|
return {
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":"external-model-governance-0123456789",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"task_id":projected_task["task_id"],
|
||||||
|
"attempt":attempt,
|
||||||
|
"execution_digest":execution["execution_digest"],
|
||||||
|
"payload":payload,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def clean_env(monkeypatch):
|
||||||
|
for key in list(os.environ):
|
||||||
|
upper=key.upper()
|
||||||
|
if any(marker in upper for marker in ("TOKEN","SECRET","PASSWORD","CREDENTIAL","AUTHORIZATION","PRIVATE_KEY","API_KEY")):
|
||||||
|
monkeypatch.delenv(key,raising=False)
|
||||||
|
monkeypatch.setenv("CREDENTIALS_DIRECTORY","/definitely-not-present-model-governance")
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_projection_binds_trusted_scope_and_trace():
|
||||||
|
task=projected()
|
||||||
|
runtime_input=task["metadata"]["execution"]["runtime_input"]
|
||||||
|
assert runtime_input["scope"]=={
|
||||||
|
"scope_type":"tenant",
|
||||||
|
"tenant_id":"tenant-a",
|
||||||
|
"project_id":"project-a",
|
||||||
|
}
|
||||||
|
assert runtime_input["trace_id"]=="trace-model-a"
|
||||||
|
|
||||||
|
|
||||||
|
def test_legacy_scope_remains_explicit_not_fake_tenant():
|
||||||
|
task=projected(scope_type="legacy",tenant_id=None,project_id=None)
|
||||||
|
scope=task["metadata"]["execution"]["runtime_input"]["scope"]
|
||||||
|
assert scope=={"scope_type":"legacy","tenant_id":None,"project_id":None}
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_requires_model_binding_when_governance_required(tmp_path):
|
||||||
|
path=tmp_path/"helper.py";path.write_text("print('ok')\n")
|
||||||
|
root=tmp_path/"profiles";root.mkdir()
|
||||||
|
(root/"hermes.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3",str(path)],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
"model_governance":{"required":True,"provider_ids":["provider-a"],"max_output_tokens":100},
|
||||||
|
}))
|
||||||
|
with pytest.raises(runtime.ExternalAgentRuntimeError,match="EXTERNAL_AGENT_MODEL_BINDING_REQUIRED"):
|
||||||
|
runtime.RuntimeProfile.load("hermes",root)
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_runtime_selects_model_and_settles_usage(tmp_path,monkeypatch):
|
||||||
|
helper_path=helper(tmp_path)
|
||||||
|
profile_root=write_profile(tmp_path,helper_path)
|
||||||
|
db=tmp_path/"governance.sqlite3";seed(db)
|
||||||
|
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
|
||||||
|
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
|
||||||
|
|
||||||
|
request_path=tmp_path/"request.json"
|
||||||
|
result_path=tmp_path/"result.json"
|
||||||
|
request_path.write_text(json.dumps(make_external_request(projected())),encoding="utf-8")
|
||||||
|
runtime.execute_request(request_path,result_path,profile_root=profile_root)
|
||||||
|
result=json.loads(result_path.read_text())
|
||||||
|
mg=result["model_governance"]
|
||||||
|
assert mg["model_id"]=="model-a"
|
||||||
|
assert mg["model_version"]=="v1"
|
||||||
|
assert mg["provider_id"]=="provider-a"
|
||||||
|
assert mg["settlement"]["usage_source"]=="runtime"
|
||||||
|
assert mg["settlement"]["input_tokens"]==20
|
||||||
|
assert mg["settlement"]["output_tokens"]==10
|
||||||
|
assert mg["audit_chain_valid"] is True
|
||||||
|
assert result["output_json"]["model"]=="model-a"
|
||||||
|
|
||||||
|
|
||||||
|
def test_budget_denies_before_runtime_process(tmp_path,monkeypatch):
|
||||||
|
helper_path=helper(tmp_path)
|
||||||
|
profile_root=write_profile(tmp_path,helper_path)
|
||||||
|
db=tmp_path/"governance.sqlite3";seed(db,tiny_budget=True)
|
||||||
|
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
|
||||||
|
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
|
||||||
|
|
||||||
|
req=tmp_path/"request.json";res=tmp_path/"result.json"
|
||||||
|
req.write_text(json.dumps(make_external_request(projected())),encoding="utf-8")
|
||||||
|
with pytest.raises(runtime.ExternalAgentRuntimeError,match="MODEL_BUDGET_DENY"):
|
||||||
|
runtime.execute_request(req,res,profile_root=profile_root)
|
||||||
|
assert not res.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_provider_failure_routes_to_fallback_provider(tmp_path,monkeypatch):
|
||||||
|
helper_path=helper(tmp_path)
|
||||||
|
profile_root=write_profile(tmp_path,helper_path)
|
||||||
|
db=tmp_path/"governance.sqlite3";seed(db)
|
||||||
|
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
|
||||||
|
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
|
||||||
|
|
||||||
|
req=tmp_path/"request.json";res=tmp_path/"result.json"
|
||||||
|
req.write_text(json.dumps(make_external_request(projected(),force_fail=True)),encoding="utf-8")
|
||||||
|
runtime.execute_request(req,res,profile_root=profile_root)
|
||||||
|
result=json.loads(res.read_text())
|
||||||
|
mg=result["model_governance"]
|
||||||
|
assert len(mg["attempts"])==2
|
||||||
|
assert mg["attempts"][0]["provider_id"]=="provider-a"
|
||||||
|
assert mg["attempts"][1]["provider_id"]=="provider-b"
|
||||||
|
assert mg["provider_id"]=="provider-b"
|
||||||
|
assert result["output_json"]["provider"]=="provider-b"
|
||||||
|
|
||||||
|
|
||||||
|
def test_profile_provider_allowlist_fails_closed(tmp_path,monkeypatch):
|
||||||
|
helper_path=helper(tmp_path)
|
||||||
|
profile_root=write_profile(tmp_path,helper_path,providers=("provider-b",))
|
||||||
|
db=tmp_path/"governance.sqlite3";seed(db)
|
||||||
|
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
|
||||||
|
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
|
||||||
|
|
||||||
|
req=tmp_path/"request.json";res=tmp_path/"result.json"
|
||||||
|
req.write_text(json.dumps(make_external_request(projected())),encoding="utf-8")
|
||||||
|
with pytest.raises(runtime.ExternalAgentRuntimeError,match="EXTERNAL_AGENT_MODEL_PROVIDER_PROFILE_DENY"):
|
||||||
|
runtime.execute_request(req,res,profile_root=profile_root)
|
||||||
|
|
||||||
|
|
||||||
|
def test_governance_required_but_runtime_disabled_fails_closed(tmp_path,monkeypatch):
|
||||||
|
helper_path=helper(tmp_path)
|
||||||
|
profile_root=write_profile(tmp_path,helper_path)
|
||||||
|
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",False)
|
||||||
|
req=tmp_path/"request.json";res=tmp_path/"result.json"
|
||||||
|
req.write_text(json.dumps(make_external_request(projected())),encoding="utf-8")
|
||||||
|
with pytest.raises(runtime.ExternalAgentRuntimeError,match="EXTERNAL_AGENT_MODEL_GOVERNANCE_DISABLED"):
|
||||||
|
runtime.execute_request(req,res,profile_root=profile_root)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_request_cannot_override_authoritative_task_scope():
|
||||||
|
task_id="task-scope-override"
|
||||||
|
request={
|
||||||
|
"task_id":task_id,
|
||||||
|
"goal":"Return result.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
"scope_type":"tenant",
|
||||||
|
"tenant_id":"tenant-a",
|
||||||
|
"project_id":"project-a",
|
||||||
|
"trace_id":"trace-authoritative",
|
||||||
|
"required_capabilities":["agent.runtime.hermes"],
|
||||||
|
}
|
||||||
|
task=project_online_task(
|
||||||
|
binding(task_id),
|
||||||
|
request,
|
||||||
|
{
|
||||||
|
"action":"agent_runtime",
|
||||||
|
"adapter":"hermes",
|
||||||
|
"capability_scope":["agent.runtime.hermes"],
|
||||||
|
"runtime_input":{
|
||||||
|
"goal":"Return result.",
|
||||||
|
"instructions":"",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
"scope":{
|
||||||
|
"scope_type":"tenant",
|
||||||
|
"tenant_id":"tenant-b",
|
||||||
|
"project_id":"project-b",
|
||||||
|
},
|
||||||
|
"trace_id":"trace-forged",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
runtime_input=task["metadata"]["execution"]["runtime_input"]
|
||||||
|
assert runtime_input["scope"]["tenant_id"]=="tenant-a"
|
||||||
|
assert runtime_input["scope"]["project_id"]=="project-a"
|
||||||
|
assert runtime_input["trace_id"]=="trace-authoritative"
|
||||||
|
|
||||||
|
|
||||||
|
def test_openclaw_nested_usage_is_extracted_exactly():
|
||||||
|
value={
|
||||||
|
"meta":{
|
||||||
|
"agentMeta":{
|
||||||
|
"usage":{"input":7770,"output":13,"total":7783},
|
||||||
|
"lastCallUsage":{"input":7770,"output":13,"total":7783},
|
||||||
|
"costUsd":0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert runtime._extract_runtime_usage(value)==(7770,13,0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_governance_required_profile_needs_provider_allowlist_and_output_limit(tmp_path):
|
||||||
|
path=tmp_path/"helper.py";path.write_text("print('ok')\\n")
|
||||||
|
root=tmp_path/"profiles";root.mkdir()
|
||||||
|
(root/"hermes.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"hermes",
|
||||||
|
"argv":["/usr/bin/python3",str(path),"{model_id}"],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
"model_governance":{"required":True},
|
||||||
|
}))
|
||||||
|
with pytest.raises(
|
||||||
|
runtime.ExternalAgentRuntimeError,
|
||||||
|
match="EXTERNAL_AGENT_MODEL_PROVIDER_IDS_REQUIRED",
|
||||||
|
):
|
||||||
|
runtime.RuntimeProfile.load("hermes",root)
|
||||||
|
|
||||||
|
|
||||||
|
def _openclaw_context_helper(tmp_path, *, input_tokens=7770, output_tokens=13):
|
||||||
|
path=tmp_path/"openclaw_context_helper.py"
|
||||||
|
path.write_text(
|
||||||
|
"import json,sys\n"
|
||||||
|
"model=sys.argv[1]\n"
|
||||||
|
"provider=sys.argv[2]\n"
|
||||||
|
"json.load(sys.stdin)\n"
|
||||||
|
f"print(json.dumps({{'answer':'ok','model':model,'provider':provider,"
|
||||||
|
f"'meta':{{'agentMeta':{{'usage':{{'input':{input_tokens},'output':{output_tokens},"
|
||||||
|
f"'total':{input_tokens+output_tokens}}},'costUsd':0}}}}}},sort_keys=True))\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _openclaw_floor_profile(tmp_path, helper_path, *, floor=16384):
|
||||||
|
root=tmp_path/"openclaw-profiles"
|
||||||
|
root.mkdir(exist_ok=True)
|
||||||
|
value={
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"openclaw",
|
||||||
|
"argv":["/usr/bin/python3",str(helper_path),"{model_id}","{provider_id}"],
|
||||||
|
"input_mode":"stdin_json",
|
||||||
|
"output_mode":"stdout_json",
|
||||||
|
"timeout_seconds":30,
|
||||||
|
"max_output_bytes":65536,
|
||||||
|
"environment":{},
|
||||||
|
"credential_env":{},
|
||||||
|
"model_governance":{
|
||||||
|
"required":True,
|
||||||
|
"provider_ids":["openai"],
|
||||||
|
"model_capabilities":["text"],
|
||||||
|
"max_output_tokens":256,
|
||||||
|
"input_reservation_floor_tokens":floor,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
(root/"openclaw.json").write_text(json.dumps(value),encoding="utf-8")
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_openclaw_floor_db(db_path):
|
||||||
|
store=ModelGovernanceStore(db_path,now_fn=lambda:1_800_000_000.0)
|
||||||
|
store.register_provider(
|
||||||
|
ProviderRecord(
|
||||||
|
"openai",
|
||||||
|
"runtime-profile:openclaw",
|
||||||
|
max_failures=2,
|
||||||
|
cooldown_seconds=60,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.register_model(
|
||||||
|
ModelRecord(
|
||||||
|
"openai/deepseek-v4-flash",
|
||||||
|
"openai",
|
||||||
|
"v1",
|
||||||
|
("agent.runtime.openclaw","text"),
|
||||||
|
input_cost_micros_per_1k=0,
|
||||||
|
output_cost_micros_per_1k=0,
|
||||||
|
max_input_tokens=32768,
|
||||||
|
max_output_tokens=256,
|
||||||
|
evaluation_sha256="c"*64,
|
||||||
|
priority=10,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.set_scope_policy(
|
||||||
|
ScopePolicy(
|
||||||
|
"tenant",
|
||||||
|
"model-governance-live",
|
||||||
|
"staging",
|
||||||
|
allowed_models=("openai/deepseek-v4-flash",),
|
||||||
|
required_capabilities=("text",),
|
||||||
|
max_input_tokens_per_request=32768,
|
||||||
|
max_output_tokens_per_request=256,
|
||||||
|
daily_token_budget=250000,
|
||||||
|
daily_cost_budget_micros=0,
|
||||||
|
fallback_limit=1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _openclaw_floor_request(*, attempt=2):
|
||||||
|
return {
|
||||||
|
"schema":"aios.external-agent-runtime.request.v1",
|
||||||
|
"request_id":f"external-openclaw-floor-attempt-{attempt:02d}",
|
||||||
|
"runtime":"openclaw",
|
||||||
|
"task_id":"r42-model-governance-openclaw-live-20260818",
|
||||||
|
"attempt":attempt,
|
||||||
|
"execution_digest":"d"*64,
|
||||||
|
"payload":{
|
||||||
|
"goal":"Return AIOS_MODEL_GOVERNANCE_OPENCLAW_OK.",
|
||||||
|
"instructions":"No side effects.",
|
||||||
|
"success_criteria":[],
|
||||||
|
"constraints":{},
|
||||||
|
"scope":{
|
||||||
|
"scope_type":"tenant",
|
||||||
|
"tenant_id":"model-governance-live",
|
||||||
|
"project_id":"staging",
|
||||||
|
},
|
||||||
|
"trace_id":"trace-openclaw-model-governance-live",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_openclaw_profile_reservation_floor_covers_injected_context(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
helper_path=_openclaw_context_helper(tmp_path,input_tokens=7770,output_tokens=13)
|
||||||
|
profile_root=_openclaw_floor_profile(tmp_path,helper_path,floor=16384)
|
||||||
|
db=tmp_path/"openclaw-governance.sqlite3"
|
||||||
|
_seed_openclaw_floor_db(db)
|
||||||
|
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
|
||||||
|
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
|
||||||
|
|
||||||
|
request_path=tmp_path/"request.json"
|
||||||
|
result_path=tmp_path/"result.json"
|
||||||
|
request_path.write_text(json.dumps(_openclaw_floor_request()),encoding="utf-8")
|
||||||
|
runtime.execute_request(request_path,result_path,profile_root=profile_root)
|
||||||
|
|
||||||
|
result=json.loads(result_path.read_text())
|
||||||
|
mg=result["model_governance"]
|
||||||
|
assert mg["input_reservation_floor_tokens"]==16384
|
||||||
|
assert mg["prompt_estimated_input_tokens"] < 16384
|
||||||
|
assert mg["reserved_estimated_input_tokens"]==16384
|
||||||
|
assert mg["settlement"]["state"]=="SETTLED"
|
||||||
|
assert mg["settlement"]["usage_source"]=="runtime"
|
||||||
|
assert mg["settlement"]["input_tokens"]==7770
|
||||||
|
assert mg["settlement"]["output_tokens"]==13
|
||||||
|
|
||||||
|
store=ModelGovernanceStore(db)
|
||||||
|
row=store.conn.execute(
|
||||||
|
"""
|
||||||
|
select reserved_input_tokens,state,actual_input_tokens,actual_output_tokens
|
||||||
|
from model_governance_reservations
|
||||||
|
where request_id=?
|
||||||
|
""",
|
||||||
|
(result["request_id"]+":model",),
|
||||||
|
).fetchone()
|
||||||
|
assert int(row["reserved_input_tokens"])==16384
|
||||||
|
assert row["state"]=="SETTLED"
|
||||||
|
assert int(row["actual_input_tokens"])==7770
|
||||||
|
assert int(row["actual_output_tokens"])==13
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_openclaw_reservation_floor_still_fails_closed_above_floor(
|
||||||
|
tmp_path,monkeypatch
|
||||||
|
):
|
||||||
|
helper_path=_openclaw_context_helper(tmp_path,input_tokens=17000,output_tokens=13)
|
||||||
|
profile_root=_openclaw_floor_profile(tmp_path,helper_path,floor=16384)
|
||||||
|
db=tmp_path/"openclaw-governance.sqlite3"
|
||||||
|
_seed_openclaw_floor_db(db)
|
||||||
|
monkeypatch.setattr(runtime,"MODEL_GOVERNANCE_ENABLED",True)
|
||||||
|
monkeypatch.setattr(runtime,"DEFAULT_MODEL_GOVERNANCE_DB",db)
|
||||||
|
|
||||||
|
request_path=tmp_path/"request.json"
|
||||||
|
result_path=tmp_path/"result.json"
|
||||||
|
request_path.write_text(json.dumps(_openclaw_floor_request()),encoding="utf-8")
|
||||||
|
with pytest.raises(
|
||||||
|
runtime.ExternalAgentRuntimeError,
|
||||||
|
match="MODEL_USAGE_EXCEEDS_RESERVATION",
|
||||||
|
):
|
||||||
|
runtime.execute_request(request_path,result_path,profile_root=profile_root)
|
||||||
|
assert not result_path.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_governance_profile_rejects_invalid_input_reservation_floor(tmp_path):
|
||||||
|
path=tmp_path/"helper.py"
|
||||||
|
path.write_text("print('ok')\n",encoding="utf-8")
|
||||||
|
root=tmp_path/"profiles"
|
||||||
|
root.mkdir()
|
||||||
|
(root/"openclaw.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.external-agent-runtime.profile.v1",
|
||||||
|
"runtime":"openclaw",
|
||||||
|
"argv":["/usr/bin/python3",str(path),"{model_id}"],
|
||||||
|
"input_mode":"stdin_text",
|
||||||
|
"output_mode":"stdout_text",
|
||||||
|
"model_governance":{
|
||||||
|
"required":True,
|
||||||
|
"provider_ids":["openai"],
|
||||||
|
"max_output_tokens":256,
|
||||||
|
"input_reservation_floor_tokens":262145,
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
with pytest.raises(
|
||||||
|
runtime.ExternalAgentRuntimeError,
|
||||||
|
match="EXTERNAL_AGENT_MODEL_INPUT_RESERVATION_FLOOR_INVALID",
|
||||||
|
):
|
||||||
|
runtime.RuntimeProfile.load("openclaw",root)
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_executor_has_single_model_governance_sqlite_write_path():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-external-agent-executor.service").read_text()
|
||||||
|
assert "AIOS_MODEL_GOVERNANCE_DB=/var/lib/aios-model-governance/state.sqlite3" in unit
|
||||||
|
assert "AIOS_MODEL_GOVERNANCE_ENABLED=1" in unit
|
||||||
|
assert "ReadWritePaths=/var/lib/aios-model-governance" in unit
|
||||||
|
assert "ReadWritePaths=/var/lib/aios-external-agent-executor" in unit
|
||||||
|
assert "InaccessiblePaths=/etc/aios-gitops" in unit
|
||||||
|
assert "InaccessiblePaths=/data/aios-gitops" in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_tmpfiles_owner_matches_external_executor():
|
||||||
|
line=(ROOT/"deploy/tmpfiles.d/aios-model-governance.conf").read_text().strip()
|
||||||
|
assert line == (
|
||||||
|
"d /var/lib/aios-model-governance 0750 "
|
||||||
|
"aios-engineering-executor aios-executor-exchange -"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_staging_models_use_real_e2e_evaluation_evidence():
|
||||||
|
value=json.loads(
|
||||||
|
(ROOT/"config/model-governance/model-governance.staging.json").read_text()
|
||||||
|
)
|
||||||
|
assert value["schema"]=="aios.model-governance.bootstrap.v1"
|
||||||
|
models={item["model_id"]:item for item in value["models"]}
|
||||||
|
assert models["deepseek-v4-flash"]["evaluation_sha256"] == (
|
||||||
|
"43ae0db1640daeb6d33e2787769732844f366d36070065d3a2239905bd9a823b"
|
||||||
|
)
|
||||||
|
assert models["openai/deepseek-v4-flash"]["evaluation_sha256"] == (
|
||||||
|
"7b1c89634eb9c0e7cb5a3d75323d904ebbd91b3ae5494ce938bd8316a4cf5b12"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_profiles_bind_model_from_governance_not_task():
|
||||||
|
for name in ("hermes","openclaw"):
|
||||||
|
value=json.loads(
|
||||||
|
(ROOT/"config/external-agent-runtime.d"/f"{name}.profile.json").read_text()
|
||||||
|
)
|
||||||
|
governance=value["model_governance"]
|
||||||
|
assert governance["required"] is True
|
||||||
|
assert governance["provider_ids"]
|
||||||
|
assert governance["max_output_tokens"] > 0
|
||||||
|
serialized=json.dumps(value["argv"])+json.dumps(value.get("environment",{}))
|
||||||
|
assert "{model_id}" in serialized or "{provider_id}" in serialized
|
||||||
|
|
||||||
|
|
||||||
|
def test_staging_bootstrap_has_explicit_tenant_live_policy():
|
||||||
|
value=json.loads(
|
||||||
|
(ROOT/"config/model-governance/model-governance.staging.json").read_text()
|
||||||
|
)
|
||||||
|
keys={
|
||||||
|
(
|
||||||
|
item["scope_type"],
|
||||||
|
item.get("tenant_id"),
|
||||||
|
item.get("project_id"),
|
||||||
|
)
|
||||||
|
for item in value["policies"]
|
||||||
|
}
|
||||||
|
assert ("legacy",None,None) in keys
|
||||||
|
assert ("tenant","model-governance-live","staging") in keys
|
||||||
|
|
||||||
|
|
||||||
|
def test_staging_policy_input_limit_covers_root_owned_openclaw_reservation_floor():
|
||||||
|
config=json.loads(
|
||||||
|
(ROOT/"config/model-governance/model-governance.staging.json").read_text()
|
||||||
|
)
|
||||||
|
profile=json.loads(
|
||||||
|
(ROOT/"config/external-agent-runtime.d/openclaw.profile.json").read_text()
|
||||||
|
)
|
||||||
|
floor=int(profile["model_governance"]["input_reservation_floor_tokens"])
|
||||||
|
assert floor==16384
|
||||||
|
openclaw_model="openai/deepseek-v4-flash"
|
||||||
|
relevant=[
|
||||||
|
policy for policy in config["policies"]
|
||||||
|
if openclaw_model in policy["allowed_models"]
|
||||||
|
]
|
||||||
|
assert relevant
|
||||||
|
for policy in relevant:
|
||||||
|
assert int(policy["max_input_tokens_per_request"]) >= floor
|
||||||
|
|
||||||
|
|
||||||
|
def test_staging_openclaw_scope_limit_is_least_privilege_floor_not_model_ceiling():
|
||||||
|
config=json.loads(
|
||||||
|
(ROOT/"config/model-governance/model-governance.staging.json").read_text()
|
||||||
|
)
|
||||||
|
models={item["model_id"]:item for item in config["models"]}
|
||||||
|
model=models["openai/deepseek-v4-flash"]
|
||||||
|
assert int(model["max_input_tokens"])==32768
|
||||||
|
for policy in config["policies"]:
|
||||||
|
if "openai/deepseek-v4-flash" in policy["allowed_models"]:
|
||||||
|
assert int(policy["max_input_tokens_per_request"])==16384
|
||||||
|
assert int(policy["max_input_tokens_per_request"]) < int(model["max_input_tokens"])
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(ROOT / "active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_online_agent_execution import ( # noqa: E402
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_execution_envelope,
|
||||||
|
)
|
||||||
|
import aios_online_agent_worker as worker # noqa: F401,E402
|
||||||
|
from aios_online_agent_worker import WorkerSupervisor # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
BASE = "a" * 40
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_replays_the_durable_envelope_to_bridge(monkeypatch) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
worker,
|
||||||
|
"WORKER_CAPABILITIES",
|
||||||
|
tuple(sorted((*worker.WORKER_CAPABILITIES, "agent.runtime.hermes"))),
|
||||||
|
)
|
||||||
|
binding = GovernedExecutionBinding.from_mapping(
|
||||||
|
{
|
||||||
|
"task_id": "r42-agent-runtime-replay-test-20260818",
|
||||||
|
"job_id": "job-agent-runtime-replay-test",
|
||||||
|
"repository": "aios/aios",
|
||||||
|
"base_oid": BASE,
|
||||||
|
"expected_branch_oid": BASE,
|
||||||
|
"target_branch": "agent/agent-runtime-replay-test",
|
||||||
|
"branch_mode": "CREATE_NEW",
|
||||||
|
"generation": 1,
|
||||||
|
"idempotency_key": "agent-runtime-replay-test-v1",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
execution = project_execution_envelope(
|
||||||
|
binding,
|
||||||
|
{
|
||||||
|
"action": "agent_runtime",
|
||||||
|
"adapter": "hermes",
|
||||||
|
"capability_scope": ["agent.runtime.hermes"],
|
||||||
|
"change_set": [],
|
||||||
|
"runtime_input": {
|
||||||
|
"goal": "replay the exact durable runtime envelope",
|
||||||
|
"instructions": "return the bounded sentinel",
|
||||||
|
"success_criteria": [
|
||||||
|
{
|
||||||
|
"id": "sentinel",
|
||||||
|
"description": "return the bounded sentinel",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"constraints": {
|
||||||
|
"provider_boundary": "provider-credential-only",
|
||||||
|
"change_set": [],
|
||||||
|
"writer_lease": False,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
task = {
|
||||||
|
**binding.public(),
|
||||||
|
"required_capabilities": ["agent.runtime.hermes"],
|
||||||
|
"metadata": {"execution": execution},
|
||||||
|
}
|
||||||
|
|
||||||
|
class Bridge:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.calls: list[object] = []
|
||||||
|
|
||||||
|
def project_execution(self, _task: object, projection_input: object = None):
|
||||||
|
self.calls.append(projection_input)
|
||||||
|
return 200, dict(projection_input or {})
|
||||||
|
|
||||||
|
supervisor = object.__new__(WorkerSupervisor)
|
||||||
|
supervisor.bridge = Bridge()
|
||||||
|
resolved, source = supervisor.resolve_envelope(task)
|
||||||
|
|
||||||
|
assert source == "persisted_verified"
|
||||||
|
assert resolved.action == "agent_runtime"
|
||||||
|
assert resolved.adapter == "hermes"
|
||||||
|
assert resolved.digest == execution["execution_digest"]
|
||||||
|
assert supervisor.bridge.calls == [execution]
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from aios_online_agent_checkpoint import CheckpointBinding, CheckpointStore
|
||||||
|
|
||||||
|
|
||||||
|
def _binding() -> CheckpointBinding:
|
||||||
|
return CheckpointBinding(
|
||||||
|
task_id="checkpoint-agent-runtime-test-20260818",
|
||||||
|
attempt=1,
|
||||||
|
execution_digest="a" * 64,
|
||||||
|
job_id="job-checkpoint-agent-runtime-test-20260818",
|
||||||
|
repository="aios/aios",
|
||||||
|
base_oid="b" * 40,
|
||||||
|
expected_branch_oid="b" * 40,
|
||||||
|
target_branch="agent/checkpoint-agent-runtime-test-20260818",
|
||||||
|
generation=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_checkpoint_phase_is_durable_before_result(tmp_path: Path) -> None:
|
||||||
|
store = CheckpointStore(tmp_path / "checkpoints.sqlite3")
|
||||||
|
binding = _binding()
|
||||||
|
phases = [
|
||||||
|
"claimed",
|
||||||
|
"workspace_prepared",
|
||||||
|
"verification_passed",
|
||||||
|
"agent_runtime_done",
|
||||||
|
"result_pending",
|
||||||
|
"result_submitted",
|
||||||
|
]
|
||||||
|
|
||||||
|
rows = [
|
||||||
|
store.append(binding, phase, {"phase": phase, "task_id": binding.task_id})
|
||||||
|
for phase in phases
|
||||||
|
]
|
||||||
|
|
||||||
|
assert [row["phase"] for row in rows] == phases
|
||||||
|
assert [int(row["phase_order"]) for row in rows] == [10, 20, 30, 40, 50, 60]
|
||||||
|
latest = store.latest_for_attempt(binding.task_id, binding.attempt)
|
||||||
|
assert latest is not None
|
||||||
|
assert latest["phase"] == "result_submitted"
|
||||||
|
assert latest["payload"] == {"phase": "result_submitted", "task_id": binding.task_id}
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_runtime_checkpoint_append_is_idempotent(tmp_path: Path) -> None:
|
||||||
|
store = CheckpointStore(tmp_path / "checkpoints.sqlite3")
|
||||||
|
binding = _binding()
|
||||||
|
payload = {"phase": "agent_runtime_done", "result": {"status": "SUCCEEDED"}}
|
||||||
|
|
||||||
|
first = store.append(binding, "agent_runtime_done", payload)
|
||||||
|
replay = store.append(binding, "agent_runtime_done", payload)
|
||||||
|
|
||||||
|
assert replay["sequence"] == first["sequence"]
|
||||||
|
assert replay["payload_sha256"] == first["payload_sha256"]
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
|
||||||
|
import aios_online_agent_worker as worker
|
||||||
|
from aios_online_agent_execution import (
|
||||||
|
GovernedExecutionBinding,
|
||||||
|
project_execution_envelope,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _state_paths(monkeypatch, tmp_path):
|
||||||
|
root=tmp_path/"worker-state"
|
||||||
|
monkeypatch.setattr(worker,"STATE_ROOT",root)
|
||||||
|
monkeypatch.setattr(worker,"WORKER_LIVENESS_PATH",root/"worker-liveness.json")
|
||||||
|
|
||||||
|
BASE="a"*40
|
||||||
|
TASK_ID="task-worker-availability-placeholder"
|
||||||
|
JOB_ID="job-worker-availability-placeholder"
|
||||||
|
BRANCH="agent/task-worker-availability-placeholder"
|
||||||
|
IDEMPOTENCY="task-worker-availability-placeholder-binding-v1"
|
||||||
|
|
||||||
|
|
||||||
|
def binding():
|
||||||
|
return GovernedExecutionBinding.from_mapping({
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"job_id":JOB_ID,
|
||||||
|
"repository":"aios/aios",
|
||||||
|
"base_oid":BASE,
|
||||||
|
"expected_branch_oid":BASE,
|
||||||
|
"target_branch":BRANCH,
|
||||||
|
"branch_mode":"CREATE_NEW",
|
||||||
|
"generation":1,
|
||||||
|
"idempotency_key":IDEMPOTENCY,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def placeholder():
|
||||||
|
return project_execution_envelope(
|
||||||
|
binding(),
|
||||||
|
{
|
||||||
|
"action":"source_capture",
|
||||||
|
"adapter":"source_capture",
|
||||||
|
"capability_scope":[
|
||||||
|
"git.read",
|
||||||
|
"git.test",
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def readonly_source_capture():
|
||||||
|
return project_execution_envelope(
|
||||||
|
binding(),
|
||||||
|
{
|
||||||
|
"action":"source_capture",
|
||||||
|
"adapter":"source_capture",
|
||||||
|
"capability_scope":["git.read","git.test"],
|
||||||
|
"source_signals":["worker availability"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def deterministic():
|
||||||
|
return project_execution_envelope(
|
||||||
|
binding(),
|
||||||
|
{
|
||||||
|
"action":"deterministic",
|
||||||
|
"adapter":"deterministic",
|
||||||
|
"capability_scope":[
|
||||||
|
"git.read",
|
||||||
|
"git.test",
|
||||||
|
"governed.write.branch",
|
||||||
|
"governed.write.file",
|
||||||
|
],
|
||||||
|
"change_set":[{
|
||||||
|
"path":"active-runtime/app/example.py",
|
||||||
|
"operation":"write_file",
|
||||||
|
"content":"VALUE=1\\n",
|
||||||
|
}],
|
||||||
|
"verification":[{
|
||||||
|
"label":"unit",
|
||||||
|
"argv":["/usr/bin/python3","-m","pytest","-q"],
|
||||||
|
"cwd":".",
|
||||||
|
"timeout_seconds":120,
|
||||||
|
}],
|
||||||
|
"governance":{
|
||||||
|
"commit_idempotency_key":"task-worker-availability-placeholder-commit-v1",
|
||||||
|
"allowed_files":["active-runtime/app/example.py"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def task(execution):
|
||||||
|
return {
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"required_capabilities":list(execution["capability_scope"]),
|
||||||
|
"metadata":{
|
||||||
|
"execution":execution,
|
||||||
|
"execution_projection":{
|
||||||
|
"schema":"aios.online-agent.task-projection.v1",
|
||||||
|
"task_id":TASK_ID,
|
||||||
|
"job_id":JOB_ID,
|
||||||
|
"execution_digest":execution["execution_digest"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class FakeBridge:
|
||||||
|
def __init__(self,status,execution):
|
||||||
|
self.status=status
|
||||||
|
self.execution=execution
|
||||||
|
self.calls=[]
|
||||||
|
|
||||||
|
def project_execution(self,task_value,execution=None):
|
||||||
|
self.calls.append(execution)
|
||||||
|
return self.status,self.execution
|
||||||
|
|
||||||
|
|
||||||
|
class FakeAgent:
|
||||||
|
def __init__(self,tasks):
|
||||||
|
self.tasks=tasks
|
||||||
|
self.claim_calls=[]
|
||||||
|
|
||||||
|
def poll(self):
|
||||||
|
return list(self.tasks)
|
||||||
|
|
||||||
|
def claim(self,task_id):
|
||||||
|
self.claim_calls.append(task_id)
|
||||||
|
return 500,{"error":"SHOULD_NOT_CLAIM"}
|
||||||
|
|
||||||
|
|
||||||
|
def supervisor(bridge, agent=None):
|
||||||
|
value=object.__new__(worker.WorkerSupervisor)
|
||||||
|
value.bridge=bridge
|
||||||
|
value.agent=agent
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_capable_source_capture_is_classified_as_placeholder():
|
||||||
|
envelope=worker.ExecutionEnvelope.from_task(
|
||||||
|
task(placeholder()),
|
||||||
|
worker_capabilities=tuple(placeholder()["capability_scope"]),
|
||||||
|
)
|
||||||
|
assert worker.source_capture_requires_deterministic_plan(envelope) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_readonly_source_capture_is_not_placeholder():
|
||||||
|
envelope=worker.ExecutionEnvelope.from_task(
|
||||||
|
task(readonly_source_capture()),
|
||||||
|
worker_capabilities=tuple(readonly_source_capture()["capability_scope"]),
|
||||||
|
)
|
||||||
|
assert worker.source_capture_requires_deterministic_plan(envelope) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_placeholder_without_registered_plan_is_skipped_before_claim():
|
||||||
|
persisted=placeholder()
|
||||||
|
bridge=FakeBridge(200,persisted)
|
||||||
|
agent=FakeAgent([task(persisted)])
|
||||||
|
sup=supervisor(bridge,agent)
|
||||||
|
assert sup.once()==0
|
||||||
|
assert bridge.calls==[None]
|
||||||
|
assert agent.claim_calls==[]
|
||||||
|
|
||||||
|
|
||||||
|
def test_placeholder_with_missing_projection_is_skipped_before_claim():
|
||||||
|
persisted=placeholder()
|
||||||
|
bridge=FakeBridge(404,{"error":"TASK_EXECUTION_PROJECTION_MISSING"})
|
||||||
|
agent=FakeAgent([task(persisted)])
|
||||||
|
sup=supervisor(bridge,agent)
|
||||||
|
assert sup.once()==0
|
||||||
|
assert bridge.calls==[None]
|
||||||
|
assert agent.claim_calls==[]
|
||||||
|
|
||||||
|
|
||||||
|
def test_registered_deterministic_plan_replaces_placeholder():
|
||||||
|
persisted=placeholder()
|
||||||
|
resolved=deterministic()
|
||||||
|
bridge=FakeBridge(200,resolved)
|
||||||
|
sup=supervisor(bridge)
|
||||||
|
envelope,source=sup.resolve_envelope(task(persisted))
|
||||||
|
assert source=="plan_registry_verified"
|
||||||
|
assert envelope.action=="deterministic"
|
||||||
|
assert envelope.digest==resolved["execution_digest"]
|
||||||
|
assert bridge.calls==[None]
|
||||||
|
|
||||||
|
|
||||||
|
def test_real_readonly_source_capture_replays_exact_payload():
|
||||||
|
persisted=readonly_source_capture()
|
||||||
|
bridge=FakeBridge(200,persisted)
|
||||||
|
sup=supervisor(bridge)
|
||||||
|
envelope,source=sup.resolve_envelope(task(persisted))
|
||||||
|
assert source=="persisted_verified"
|
||||||
|
assert envelope.action=="source_capture"
|
||||||
|
assert envelope.digest==persisted["execution_digest"]
|
||||||
|
assert bridge.calls==[persisted]
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_openclaw_profile_uses_conservative_input_reservation_floor():
|
||||||
|
value=json.loads(
|
||||||
|
(ROOT/"config/external-agent-runtime.d/openclaw.profile.json").read_text()
|
||||||
|
)
|
||||||
|
mg=value["model_governance"]
|
||||||
|
assert mg["required"] is True
|
||||||
|
assert mg["input_reservation_floor_tokens"]==16384
|
||||||
|
assert mg["max_output_tokens"]==256
|
||||||
@@ -0,0 +1,199 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0,str(ROOT/"active-runtime/app"))
|
||||||
|
|
||||||
|
from aios_worker_availability_guard import AvailabilityGuard
|
||||||
|
|
||||||
|
|
||||||
|
class FakeSystemd:
|
||||||
|
def __init__(self, roots, now=1000.0):
|
||||||
|
self.now=now
|
||||||
|
self.actions=[]
|
||||||
|
self.units={}
|
||||||
|
for unit,root in roots.items():
|
||||||
|
self.units[unit]={
|
||||||
|
"LoadState":"loaded",
|
||||||
|
"ActiveState":"active",
|
||||||
|
"UnitFileState":"enabled",
|
||||||
|
"Environment":f"AIOS_WORKER_STATE_ROOT={root}",
|
||||||
|
}
|
||||||
|
self.units["aios-external-agent-executor.service"]={
|
||||||
|
"LoadState":"loaded",
|
||||||
|
"ActiveState":"active",
|
||||||
|
"UnitFileState":"enabled",
|
||||||
|
"Environment":"",
|
||||||
|
}
|
||||||
|
|
||||||
|
def property(self, unit, name):
|
||||||
|
return self.units.get(unit,{}).get(name,"")
|
||||||
|
|
||||||
|
def environment(self, unit):
|
||||||
|
raw=self.property(unit,"Environment")
|
||||||
|
result={}
|
||||||
|
for item in raw.split():
|
||||||
|
if "=" in item:
|
||||||
|
k,v=item.split("=",1)
|
||||||
|
result[k]=v
|
||||||
|
return result
|
||||||
|
|
||||||
|
def enable(self, unit):
|
||||||
|
self.actions.append((unit,"enable"))
|
||||||
|
self.units[unit]["UnitFileState"]="enabled"
|
||||||
|
|
||||||
|
def start(self, unit):
|
||||||
|
self.actions.append((unit,"start"))
|
||||||
|
self.units[unit]["ActiveState"]="active"
|
||||||
|
env=self.environment(unit)
|
||||||
|
if "AIOS_WORKER_STATE_ROOT" not in env:
|
||||||
|
return
|
||||||
|
root=Path(env["AIOS_WORKER_STATE_ROOT"])
|
||||||
|
root.mkdir(parents=True,exist_ok=True)
|
||||||
|
(root/"worker-liveness.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.online-agent.worker-liveness.v1",
|
||||||
|
"agent_id":"agent-"+unit,
|
||||||
|
"state":"POLL_OK",
|
||||||
|
"last_seen_epoch":self.now,
|
||||||
|
}),encoding="utf-8")
|
||||||
|
|
||||||
|
def restart(self, unit):
|
||||||
|
self.actions.append((unit,"restart"))
|
||||||
|
self.units[unit]["ActiveState"]="active"
|
||||||
|
env=self.environment(unit)
|
||||||
|
root=Path(env["AIOS_WORKER_STATE_ROOT"])
|
||||||
|
root.mkdir(parents=True,exist_ok=True)
|
||||||
|
(root/"worker-liveness.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.online-agent.worker-liveness.v1",
|
||||||
|
"agent_id":"agent-"+unit,
|
||||||
|
"state":"POLL_OK",
|
||||||
|
"last_seen_epoch":self.now,
|
||||||
|
}),encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def make_guard(tmp_path, fake):
|
||||||
|
worker_units=tuple(
|
||||||
|
unit for unit in fake.units
|
||||||
|
if unit!="aios-external-agent-executor.service"
|
||||||
|
)
|
||||||
|
return AvailabilityGuard(
|
||||||
|
systemd=fake,
|
||||||
|
worker_units=worker_units,
|
||||||
|
dependency_units=("aios-external-agent-executor.service",),
|
||||||
|
status_path=tmp_path/"run/status.json",
|
||||||
|
recovery_path=tmp_path/"run/recovery.json",
|
||||||
|
inhibit_path=tmp_path/"run/maintenance.lock",
|
||||||
|
max_stale_seconds=90,
|
||||||
|
restart_cooldown_seconds=60,
|
||||||
|
startup_wait_seconds=0,
|
||||||
|
now_fn=lambda:fake.now,
|
||||||
|
sleep_fn=lambda _seconds:None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def write_liveness(root, now, *, state="POLL_OK"):
|
||||||
|
root.mkdir(parents=True,exist_ok=True)
|
||||||
|
(root/"worker-liveness.json").write_text(json.dumps({
|
||||||
|
"schema":"aios.online-agent.worker-liveness.v1",
|
||||||
|
"agent_id":"agent",
|
||||||
|
"state":state,
|
||||||
|
"last_seen_epoch":now,
|
||||||
|
}),encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_healthy_workers_need_no_action(tmp_path):
|
||||||
|
roots={
|
||||||
|
"generic.service":tmp_path/"generic",
|
||||||
|
"hermes.service":tmp_path/"hermes",
|
||||||
|
"openclaw.service":tmp_path/"openclaw",
|
||||||
|
}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
for root in roots.values():
|
||||||
|
write_liveness(root,fake.now)
|
||||||
|
result=make_guard(tmp_path,fake).run(heal=True)
|
||||||
|
assert result["status"]=="HEALTHY"
|
||||||
|
assert result["all_fresh"] is True
|
||||||
|
assert fake.actions==[]
|
||||||
|
|
||||||
|
|
||||||
|
def test_inactive_worker_is_started_and_becomes_fresh(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
fake.units["generic.service"]["ActiveState"]="inactive"
|
||||||
|
result=make_guard(tmp_path,fake).run(heal=True)
|
||||||
|
assert ("generic.service","start") in fake.actions
|
||||||
|
assert result["all_fresh"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_disabled_worker_is_enabled_and_started(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
fake.units["generic.service"]["UnitFileState"]="disabled"
|
||||||
|
fake.units["generic.service"]["ActiveState"]="inactive"
|
||||||
|
result=make_guard(tmp_path,fake).run(heal=True)
|
||||||
|
assert ("generic.service","enable") in fake.actions
|
||||||
|
assert ("generic.service","start") in fake.actions
|
||||||
|
assert result["all_fresh"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_active_but_stale_worker_is_restarted(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
write_liveness(roots["generic.service"],fake.now-500)
|
||||||
|
result=make_guard(tmp_path,fake).run(heal=True)
|
||||||
|
assert ("generic.service","restart") in fake.actions
|
||||||
|
assert result["all_fresh"] is True
|
||||||
|
assert result["workers"][0]["action"]=="RESTART_STALE"
|
||||||
|
|
||||||
|
|
||||||
|
def test_restart_cooldown_prevents_restart_storm(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
write_liveness(roots["generic.service"],fake.now-500)
|
||||||
|
guard=make_guard(tmp_path,fake)
|
||||||
|
guard.recovery_path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
guard.recovery_path.write_text(json.dumps({
|
||||||
|
"schema":"aios.worker-availability.recovery.v1",
|
||||||
|
"last_restart_epoch":{"generic.service":fake.now-10},
|
||||||
|
}),encoding="utf-8")
|
||||||
|
result=guard.run(heal=True)
|
||||||
|
assert ("generic.service","restart") not in fake.actions
|
||||||
|
assert result["status"]=="DEGRADED"
|
||||||
|
assert result["workers"][0]["detail"]=="RESTART_COOLDOWN_ACTIVE"
|
||||||
|
|
||||||
|
|
||||||
|
def test_maintenance_inhibit_disables_healing(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
fake.units["generic.service"]["ActiveState"]="inactive"
|
||||||
|
guard=make_guard(tmp_path,fake)
|
||||||
|
guard.inhibit_path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
guard.inhibit_path.write_text("planned maintenance\n")
|
||||||
|
result=guard.run(heal=True)
|
||||||
|
assert result["status"]=="MAINTENANCE_INHIBITED"
|
||||||
|
assert fake.actions==[]
|
||||||
|
|
||||||
|
|
||||||
|
def test_dependency_is_enabled_and_started(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
write_liveness(roots["generic.service"],fake.now)
|
||||||
|
fake.units["aios-external-agent-executor.service"]["UnitFileState"]="disabled"
|
||||||
|
fake.units["aios-external-agent-executor.service"]["ActiveState"]="inactive"
|
||||||
|
result=make_guard(tmp_path,fake).run(heal=True)
|
||||||
|
assert ("aios-external-agent-executor.service","enable") in fake.actions
|
||||||
|
assert ("aios-external-agent-executor.service","start") in fake.actions
|
||||||
|
assert result["all_fresh"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_reports_stale_without_mutation(tmp_path):
|
||||||
|
roots={"generic.service":tmp_path/"generic"}
|
||||||
|
fake=FakeSystemd(roots)
|
||||||
|
write_liveness(roots["generic.service"],fake.now-500)
|
||||||
|
result=make_guard(tmp_path,fake).run(heal=False)
|
||||||
|
assert result["status"]=="DEGRADED"
|
||||||
|
assert result["all_fresh"] is False
|
||||||
|
assert fake.actions==[]
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT=Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_source_has_idle_and_busy_liveness():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_online_agent_worker.py").read_text()
|
||||||
|
assert "aios.online-agent.worker-liveness.v1" in source
|
||||||
|
assert 'write_worker_liveness("POLL_OK"' in source
|
||||||
|
assert '"POLL_FAILED"' in source
|
||||||
|
assert 'write_worker_liveness("STARTING")' in source
|
||||||
|
assert '"BUSY"' in source
|
||||||
|
assert "last_seen_epoch" in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_guard_timer_is_persistent_and_frequent():
|
||||||
|
timer=(ROOT/"deploy/systemd/aios-worker-availability-guard.timer").read_text()
|
||||||
|
assert "OnBootSec=15s" in timer
|
||||||
|
assert "OnUnitActiveSec=30s" in timer
|
||||||
|
assert "Persistent=true" in timer
|
||||||
|
assert "WantedBy=timers.target" in timer
|
||||||
|
|
||||||
|
|
||||||
|
def test_guard_service_has_no_network_or_credentials():
|
||||||
|
unit=(ROOT/"deploy/systemd/aios-worker-availability-guard.service").read_text()
|
||||||
|
assert "User=root" in unit
|
||||||
|
assert "LoadCredential" not in unit
|
||||||
|
assert "RestrictAddressFamilies=AF_UNIX" in unit
|
||||||
|
assert "NoNewPrivileges=true" in unit
|
||||||
|
assert "ProtectSystem=strict" in unit
|
||||||
|
assert "ReadWritePaths=/run/aios-worker-availability" in unit
|
||||||
|
assert "AIOS_WORKER_AVAILABILITY_MAX_STALE_SECONDS=90" in unit
|
||||||
|
|
||||||
|
|
||||||
|
def test_guard_manages_all_three_workers_and_external_executor():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_worker_availability_guard.py").read_text()
|
||||||
|
for unit in (
|
||||||
|
"aios-online-agent-worker-20260815.service",
|
||||||
|
"aios-hermes-online-agent-worker.service",
|
||||||
|
"aios-openclaw-online-agent-worker.service",
|
||||||
|
"aios-external-agent-executor.service",
|
||||||
|
):
|
||||||
|
assert unit in source
|
||||||
|
assert "maintenance.lock" in source
|
||||||
|
assert "RESTART_STALE" in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_defers_write_capable_source_capture_until_plan_exists():
|
||||||
|
source=(ROOT/"active-runtime/app/aios_online_agent_worker.py").read_text()
|
||||||
|
assert "ENGINEERING_WRITE_CAPABILITIES" in source
|
||||||
|
assert "source_capture_requires_deterministic_plan" in source
|
||||||
|
assert "TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN" in source
|
||||||
|
assert '"governed.write.branch"' in source
|
||||||
|
assert '"governed.write.file"' in source
|
||||||
Reference in New Issue
Block a user