Files
aios/active-runtime/app/aios_online_agent_worker.py
T

2523 lines
106 KiB
Python

#!/usr/bin/env python3
"""Generic, restartable Online Agent Worker.
The worker is deliberately task-agnostic. A task's signed/durable payload is
the only source of repository, branch, change, verification, and governance
parameters. There is no task id, base OID, branch name, or plan file in the
runtime configuration.
The worker performs local work only in an ephemeral detached worktree. All
authoritative ref updates go through the governed Bridge API; direct branch,
update-ref, push, main, production, tag, and baseline mutations are rejected.
"""
from __future__ import annotations
import base64
from dataclasses import dataclass
from datetime import datetime, timezone
from abc import ABC, abstractmethod
import hashlib
import secrets
import io
import json
import os
from pathlib import Path
import re
import shutil
import subprocess
import tempfile
import tarfile
import threading
import time
import urllib.error
import urllib.request
from typing import Any, Mapping, Sequence
from urllib.parse import unquote
from aios_online_agent_execution import (
ExecutionProjectionError,
validate_execution_envelope,
)
from aios_online_agent_checkpoint import (
CheckpointBinding,
CheckpointError,
CheckpointStore,
)
from aios_external_agent_runtime import (
ExternalAgentExchange,
ExternalAgentRuntimeError,
)
from aios_online_agent_compact_map import (
CompactInterfaceMapError,
build_compact_interface_map,
)
API_BASE = os.environ.get("AIOS_AGENT_TASK_API_URL", "http://127.0.0.1:18940").rstrip("/")
BRIDGE_BASE = os.environ.get("AIOS_GOVERNED_WRITE_API_URL", "http://127.0.0.1:18892").rstrip("/")
AGENT_ID = os.environ.get(
"AIOS_AGENT_ID", "online-engineering-worker-cn-network-mkt-generic-20260815"
)
AGENT_TYPE = os.environ.get("AIOS_AGENT_TYPE", "trusted-online-engineering-worker")
AGENT_VERSION = os.environ.get("AIOS_AGENT_VERSION", "2026-08-17.2-durable")
AUTHORITY_GIT_DIR = Path(
os.environ.get("AIOS_AUTHORITY_GIT_DIR", "/data/aios-gitops/repo.git")
)
STATE_ROOT = Path(
os.environ.get("AIOS_WORKER_STATE_ROOT", "/var/lib/aios-online-agent-worker-20260815")
)
WORKSPACE_ROOT = Path(
os.environ.get(
"AIOS_WORKER_WORKSPACE_ROOT", "/var/lib/aios-online-agent-worker-20260815/workspaces"
)
)
GIT_CONFIG_GLOBAL = STATE_ROOT / "gitconfig"
CHECKPOINT_DB = Path(os.environ.get("AIOS_WORKER_CHECKPOINT_DB", str(STATE_ROOT / "checkpoints.sqlite3")))
POLL_SECONDS = int(os.environ.get("AIOS_WORKER_POLL_SECONDS", "5"))
HEARTBEAT_SECONDS = int(os.environ.get("AIOS_WORKER_HEARTBEAT_SECONDS", "20"))
LEASE_SECONDS = int(os.environ.get("AIOS_WORKER_LEASE_SECONDS", "300"))
WORKER_CAPABILITIES = tuple(
sorted(
item.strip()
for item in os.environ.get(
"AIOS_WORKER_CAPABILITIES",
"git.read,git.test,governed.write.branch,governed.write.file",
).split(",")
if item.strip()
)
)
WORKER_ALLOWED_ADAPTERS = frozenset(
item.strip()
for item in os.environ.get(
"AIOS_WORKER_ALLOWED_ADAPTERS",
"deterministic,source_capture,compact_interface_map,codex",
).split(",")
if item.strip()
)
EXTERNAL_AGENT_EXECUTOR_ROOT = Path(
os.environ.get(
"AIOS_EXTERNAL_AGENT_EXECUTOR_ROOT",
"/var/lib/aios-external-agent-executor",
)
)
EXTERNAL_AGENT_EXECUTOR_TIMEOUT_SECONDS = int(
os.environ.get("AIOS_EXTERNAL_AGENT_EXECUTOR_TIMEOUT_SECONDS", "1800")
)
MAX_TASK_BYTES = 512 * 1024
MAX_FILE_BYTES = 2 * 1024 * 1024
ISOLATED_EXECUTOR_ROOT = Path(
os.environ.get("AIOS_ISOLATED_EXECUTOR_ROOT", "/var/lib/aios-online-agent-executor")
)
ISOLATED_EXECUTOR_TIMEOUT_SECONDS = int(
os.environ.get("AIOS_ISOLATED_EXECUTOR_TIMEOUT_SECONDS", "1800")
)
ISOLATED_EXECUTOR_TIMEOUT_HARD_MAX_SECONDS = 7200
GIT_VISIBILITY_TIMEOUT_SECONDS = int(
os.environ.get("AIOS_GIT_VISIBILITY_TIMEOUT_SECONDS", "45")
)
OID_RE = re.compile(r"^[0-9a-f]{40}$")
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{2,179}$")
REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{15,254}$")
BRANCH_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]*$")
PATH_RE = re.compile(r"^[^\\\x00-\x1f\x7f]+$")
SENSITIVE_RE = re.compile(
r"(?i)(authorization\s*:\s*bearer\s+\S+|(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*\S+)"
)
class WorkerError(RuntimeError):
def __init__(self, code: str, *, phase: str, details: Mapping[str, Any] | None = None) -> None:
super().__init__(code)
self.code = code
self.phase = phase
self.details = dict(details or {})
def _isolated_executor_wait_seconds(verification: Sequence[Mapping[str, Any]]) -> int:
verification_budget = sum(int(item["timeout_seconds"]) for item in verification)
required_wait_seconds = max(30, verification_budget + 30)
if not 30 <= ISOLATED_EXECUTOR_TIMEOUT_SECONDS <= ISOLATED_EXECUTOR_TIMEOUT_HARD_MAX_SECONDS:
raise WorkerError("ISOLATED_EXECUTOR_TIMEOUT_CONFIG_INVALID", phase="adapter")
if required_wait_seconds > ISOLATED_EXECUTOR_TIMEOUT_SECONDS:
raise WorkerError(
"ISOLATED_EXECUTOR_TIMEOUT_BUDGET_EXCEEDED",
phase="adapter",
details={
"required_seconds": required_wait_seconds,
"configured_seconds": ISOLATED_EXECUTOR_TIMEOUT_SECONDS,
},
)
return required_wait_seconds
def _claim_order(item: tuple[Mapping[str, Any], "ExecutionEnvelope"]) -> tuple[int, int, str, str]:
task, envelope = item
try:
priority = int(task.get("priority") or 0)
except (TypeError, ValueError):
priority = 0
action_rank = 0 if envelope.action == "deterministic" else 1
return (-priority, action_rank, str(task.get("created_at") or ""), envelope.task_id)
def now() -> str:
return datetime.now(timezone.utc).isoformat()
def digest_bytes(value: bytes) -> str:
return hashlib.sha256(value).hexdigest()
def digest_json(value: Any) -> str:
return digest_bytes(
json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
)
def sanitize(value: Any) -> Any:
if isinstance(value, Mapping):
cleaned: dict[str, Any] = {}
for key, inner in value.items():
lowered = str(key).lower()
if any(marker in lowered for marker in ("token", "secret", "credential", "password", "authorization")):
cleaned[str(key)] = "[REDACTED]"
else:
cleaned[str(key)] = sanitize(inner)
return cleaned
if isinstance(value, list):
return [sanitize(item) for item in value]
if isinstance(value, tuple):
return [sanitize(item) for item in value]
if isinstance(value, str):
return SENSITIVE_RE.sub("[REDACTED]", value)
return value
def excerpt(value: str, limit: int = 800) -> str:
value = str(sanitize(value))
return value if len(value) <= limit else value[: limit // 2] + "...[TRUNCATED]..." + value[-limit // 2 :]
def audit(event_type: str, **fields: Any) -> None:
STATE_ROOT.mkdir(mode=0o750, parents=True, exist_ok=True)
path = STATE_ROOT / "events.jsonl"
record = {
"event_type": event_type,
"created_at": now(),
"agent_id": AGENT_ID,
"credential_material_exposed": False,
"lease_material_exposed": False,
**sanitize(fields),
}
with path.open("a", encoding="utf-8") as handle:
handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
os.chmod(path, 0o640)
print(json.dumps(record, ensure_ascii=False, sort_keys=True), flush=True)
def atomic_json(path: Path, value: Mapping[str, Any], mode: int = 0o640) -> None:
path.parent.mkdir(mode=0o750, parents=True, exist_ok=True)
if path.parent.is_symlink() or path.is_symlink():
raise WorkerError("STATE_PATH_SYMLINK_REJECTED", phase="state")
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
try:
os.fchmod(fd, mode)
with os.fdopen(fd, "w", encoding="utf-8") as handle:
json.dump(sanitize(dict(value)), handle, ensure_ascii=False, sort_keys=True)
handle.write("\n")
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
os.chmod(path, mode)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
WORKER_LIVENESS_PATH = Path(
os.environ.get(
"AIOS_WORKER_LIVENESS_PATH",
str(STATE_ROOT / "worker-liveness.json"),
)
)
_LIVENESS_LOCK = threading.Lock()
_LIVENESS_SEQUENCE = 0
ENGINEERING_WRITE_CAPABILITIES = frozenset({
"governed.write.branch",
"governed.write.file",
})
def write_worker_liveness(
state: str,
*,
visible_count: int | None = None,
task_id: str | None = None,
stage: str | None = None,
error: str | None = None,
) -> None:
"""Write idle/busy liveness without touching Task/Job state."""
global _LIVENESS_SEQUENCE
with _LIVENESS_LOCK:
_LIVENESS_SEQUENCE += 1
epoch = time.time()
payload = {
"schema": "aios.online-agent.worker-liveness.v1",
"agent_id": AGENT_ID,
"agent_type": AGENT_TYPE,
"agent_version": AGENT_VERSION,
"pid": os.getpid(),
"state": state,
"sequence": _LIVENESS_SEQUENCE,
"last_seen_epoch": epoch,
"last_seen_at": datetime.fromtimestamp(epoch, timezone.utc).isoformat(),
"poll_seconds": POLL_SECONDS,
"heartbeat_seconds": HEARTBEAT_SECONDS,
"api_base": API_BASE,
"bridge_base": BRIDGE_BASE,
"visible_count": visible_count,
"task_id": task_id,
"stage": stage,
"error": error,
}
atomic_json(WORKER_LIVENESS_PATH, payload)
def source_capture_requires_deterministic_plan(envelope: "ExecutionEnvelope") -> bool:
"""Treat write-capable source_capture as a binding placeholder, never executable work."""
return (
envelope.action == "source_capture"
and bool(ENGINEERING_WRITE_CAPABILITIES.intersection(envelope.capability_scope))
)
def safe_relative_path(value: Any, *, field: str = "path") -> str:
if not isinstance(value, str) or not value or len(value) > 240:
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
if unquote(value) != value or not PATH_RE.fullmatch(value):
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
if value.startswith(("/", "~")) or re.match(r"^[A-Za-z]:", value):
raise WorkerError(f"{field.upper()}_ABSOLUTE", phase="validation")
parts = value.replace("\\", "/").split("/")
if any(part in {"", ".", ".."} for part in parts):
raise WorkerError(f"{field.upper()}_TRAVERSAL", phase="validation")
return "/".join(parts)
def resolve_under(root: Path, relative: str, *, write: bool = False) -> Path:
relative = safe_relative_path(relative)
root = root.resolve()
candidate = (root / relative).resolve(strict=False)
if candidate != root and root not in candidate.parents:
raise WorkerError("WORKSPACE_ESCAPE", phase="validation")
current = root
for part in Path(relative).parts:
current /= part
if current.exists() and current.is_symlink():
raise WorkerError("WORKSPACE_SYMLINK_REJECTED", phase="validation")
if write and candidate.exists() and candidate.is_symlink():
raise WorkerError("WRITE_SYMLINK_REJECTED", phase="validation")
return candidate
def validate_oid(value: Any, field: str = "oid") -> str:
if not isinstance(value, str) or not OID_RE.fullmatch(value):
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
return value
def validate_sha(value: Any, field: str = "sha256") -> str:
if not isinstance(value, str) or not SHA256_RE.fullmatch(value):
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
return value
def validate_branch(value: Any) -> str:
if not isinstance(value, str) or not 3 <= len(value) <= 180:
raise WorkerError("TARGET_BRANCH_INVALID", phase="validation")
if value in {"main", "master", "HEAD"} or value.startswith(
("main/", "master/", "refs/", "tags/", "tag/", "release/", "releases/", "baseline/", "production/")
):
raise WorkerError("PROTECTED_BRANCH_REJECTED", phase="validation")
if (
value.startswith("/")
or value.endswith(("/", ".", ".lock"))
or ".." in value
or "//" in value
or "@{" in value
or any(ch in value for ch in " ~^:?*[\\")
or any(part in {"", ".", ".."} for part in value.split("/"))
or not BRANCH_RE.fullmatch(value)
):
raise WorkerError("TARGET_BRANCH_INVALID", phase="validation")
return value
def validate_program(argv: Any, *, internal_git: bool = False) -> list[str]:
if not isinstance(argv, list) or not argv or not all(isinstance(item, str) and item for item in argv):
raise WorkerError("STRUCTURED_ARGV_REQUIRED", phase="validation")
command = argv[0]
if command not in {"/usr/bin/git", "/usr/bin/python3"}:
raise WorkerError("PROGRAM_NOT_ALLOWLISTED", phase="validation", details={"program": Path(command).name})
if command.endswith("python3"):
if any(item in {"-c", "-", "--command"} for item in argv):
raise WorkerError("PYTHON_INLINE_CODE_REJECTED", phase="validation")
if "-m" in argv:
index = argv.index("-m")
if index + 1 >= len(argv) or argv[index + 1] not in {"pytest", "py_compile"}:
raise WorkerError("PYTHON_MODULE_NOT_ALLOWLISTED", phase="validation")
if command.endswith("git") and not internal_git:
rejected = {"push", "update-ref", "receive-pack", "send-pack", "reset", "clean", "branch", "commit", "merge", "rebase"}
if any(item in rejected for item in argv[1:]):
raise WorkerError("GIT_MUTATION_NOT_ALLOWLISTED", phase="validation")
return list(argv)
def run_command(
label: str,
argv: list[str],
*,
cwd: Path,
timeout: int,
internal_git: bool = False,
allowed_returncodes: tuple[int, ...] = (0,),
) -> dict[str, Any]:
validate_program(argv, internal_git=internal_git)
if not 1 <= timeout <= 3600:
raise WorkerError("COMMAND_TIMEOUT_INVALID", phase=label)
started = time.monotonic()
try:
completed = subprocess.run(
argv,
cwd=str(cwd),
text=True,
capture_output=True,
check=False,
shell=False,
timeout=timeout,
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C.UTF-8", "GIT_CONFIG_GLOBAL": str(GIT_CONFIG_GLOBAL)},
)
except subprocess.TimeoutExpired as exc:
raise WorkerError("COMMAND_TIMEOUT", phase=label, details={"timeout_seconds": timeout}) from exc
result = {
"label": label,
"argv": [Path(item).name if index == 0 else item for index, item in enumerate(argv)],
"exit_code": completed.returncode,
"stdout_sha256": digest_bytes(completed.stdout.encode("utf-8", "replace")),
"stderr_sha256": digest_bytes(completed.stderr.encode("utf-8", "replace")),
"stdout_excerpt": excerpt(completed.stdout),
"stderr_excerpt": excerpt(completed.stderr),
"duration_ms": int((time.monotonic() - started) * 1000),
}
audit("COMMAND_RESULT", **result)
if completed.returncode not in allowed_returncodes:
raise WorkerError("COMMAND_FAILED", phase=label, details={"exit_code": completed.returncode, "stderr": excerpt(completed.stderr, 400)})
return result
@dataclass(frozen=True)
class ExecutionEnvelope:
task_id: str
job_id: str
repository: str
base_oid: str
expected_branch_oid: str
target_branch: str
branch_mode: str
generation: int
idempotency_key: str
action: str
adapter: str
capability_scope: tuple[str, ...]
source_signals: tuple[str, ...]
change_set: tuple[dict[str, Any], ...]
verification: tuple[dict[str, Any], ...]
governance: dict[str, Any]
workspace: dict[str, Any]
digest: str
runtime_input: dict[str, Any] | None = None
@classmethod
def from_task(
cls,
task: Mapping[str, Any],
execution_override: Mapping[str, Any] | None = None,
worker_capabilities: tuple[str, ...] | None = None,
) -> "ExecutionEnvelope":
task_id = str(task.get("task_id") or "")
if not ID_RE.fullmatch(task_id):
raise WorkerError("TASK_ID_INVALID", phase="validation")
metadata = task.get("metadata")
if not isinstance(metadata, Mapping):
raise WorkerError("TASK_METADATA_REQUIRED", phase="validation")
execution = execution_override if execution_override is not None else metadata.get("execution")
if not isinstance(execution, Mapping):
raise WorkerError("DYNAMIC_EXECUTION_PAYLOAD_REQUIRED", phase="validation")
try:
execution = validate_execution_envelope(execution, worker_capabilities=worker_capabilities)
except ExecutionProjectionError as exc:
raise WorkerError(str(exc), phase="validation") from exc
if execution.get("task_id") != task_id:
raise WorkerError("TASK_BINDING_CONFLICT", phase="validation")
action = str(execution["action"])
repository = execution.get("repository")
if not isinstance(repository, str) or not repository:
raise WorkerError("REPOSITORY_INVALID", phase="validation")
base_oid = validate_oid(execution.get("base_oid"), "base_oid")
target_branch = validate_branch(execution.get("target_branch"))
workspace = execution.get("workspace")
if not isinstance(workspace, Mapping):
raise WorkerError("WORKSPACE_POLICY_REQUIRED", phase="validation")
workspace_value = dict(workspace)
if workspace_value.get("authority_git_dir") != str(AUTHORITY_GIT_DIR):
raise WorkerError("AUTHORITY_ROOT_INVALID", phase="validation")
if workspace_value.get("read_only_repository") is not True:
raise WorkerError("REPOSITORY_WRITE_POLICY_INVALID", phase="validation")
if workspace_value.get("cleanup_policy") not in {"always", "cleanup_on_success"}:
raise WorkerError("WORKSPACE_CLEANUP_POLICY_INVALID", phase="validation")
adapter_value = execution.get("adapter")
if isinstance(adapter_value, Mapping):
adapter = str(adapter_value.get("name") or "")
else:
adapter = str(adapter_value or "")
if adapter not in {"deterministic", "source_capture", "compact_interface_map", "codex", "hermes", "openclaw"}:
raise WorkerError("RUNTIME_ADAPTER_UNSUPPORTED", phase="validation")
change_set = execution.get("change_set")
if not isinstance(change_set, list) or len(change_set) > 128 or (action == "deterministic" and not change_set):
raise WorkerError("CHANGE_SET_INVALID", phase="validation")
normalized_changes: list[dict[str, Any]] = []
for item in change_set:
if not isinstance(item, Mapping):
raise WorkerError("CHANGE_OPERATION_INVALID", phase="validation")
operation = str(item.get("operation") or "")
relative = safe_relative_path(item.get("path"))
normalized = dict(item)
normalized["operation"] = operation
normalized["path"] = relative
if operation == "write_file":
content = item.get("content")
if not isinstance(content, str) or len(content.encode("utf-8")) > MAX_FILE_BYTES:
raise WorkerError("WRITE_CONTENT_INVALID", phase="validation")
elif operation == "replace_text":
if not isinstance(item.get("old"), str) or not item.get("old") or not isinstance(item.get("new"), str):
raise WorkerError("REPLACE_TEXT_INVALID", phase="validation")
if len(str(item["old"]).encode("utf-8")) > MAX_FILE_BYTES:
raise WorkerError("REPLACE_TEXT_INVALID", phase="validation")
elif operation == "delete_file":
pass
else:
raise WorkerError("CHANGE_OPERATION_UNSUPPORTED", phase="validation", details={"operation": operation})
expected = item.get("expected_sha256")
if expected is not None:
validate_sha(expected, "expected_sha256")
normalized_changes.append(normalized)
verification = execution.get("verification") or []
if not isinstance(verification, list) or len(verification) > 128:
raise WorkerError("VERIFICATION_INVALID", phase="validation")
normalized_verification: list[dict[str, Any]] = []
for item in verification:
if not isinstance(item, Mapping):
raise WorkerError("VERIFICATION_COMMAND_INVALID", phase="validation")
label = str(item.get("label") or "verification")
argv = validate_program(item.get("argv"))
cwd_value = item.get("cwd")
cwd = "." if cwd_value in {None, "", "."} else safe_relative_path(cwd_value, field="cwd")
timeout = item.get("timeout_seconds", 120)
if not isinstance(timeout, int) or isinstance(timeout, bool) or not 1 <= timeout <= 3600:
raise WorkerError("VERIFICATION_TIMEOUT_INVALID", phase="validation")
normalized_verification.append({"label": label, "argv": argv, "cwd": cwd, "timeout_seconds": timeout})
governance = execution.get("governance")
if not isinstance(governance, Mapping):
raise WorkerError("GOVERNANCE_POLICY_REQUIRED", phase="validation")
governance_value = dict(governance)
governed_task_id = str(governance_value.get("task_id") or f"{task_id}-writer")
if not ID_RE.fullmatch(governed_task_id):
raise WorkerError("GOVERNED_TASK_ID_INVALID", phase="validation")
idempotency_key = str(governance_value.get("idempotency_key") or f"{task_id}-governed-v1")
commit_key = str(governance_value.get("commit_idempotency_key") or f"{task_id}-commit-v1")
for field, value in (("idempotency_key", idempotency_key), ("commit_idempotency_key", commit_key)):
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{7,159}", value):
raise WorkerError(f"{field.upper()}_INVALID", phase="validation")
governance_value.update(
{
"task_id": governed_task_id,
"idempotency_key": idempotency_key,
"commit_idempotency_key": commit_key,
"message": str(governance_value.get("message") or f"chore(gitops): execute task {task_id}"),
}
)
for field in ("main_write", "force_push", "production_write", "tag_write", "baseline_write", "protected_ref_bypass"):
if governance_value.get(field, False) is not False:
raise WorkerError("GOVERNANCE_POLICY_REJECTED", phase="validation", details={"field": field})
allowed_files = governance_value.get("allowed_files")
if allowed_files is not None:
if not isinstance(allowed_files, list):
raise WorkerError("ALLOWED_FILES_INVALID", phase="validation")
allowed = {safe_relative_path(item, field="allowed_file") for item in allowed_files}
if not {item["path"] for item in normalized_changes}.issubset(allowed):
raise WorkerError("CHANGE_OUTSIDE_ALLOWED_FILES", phase="validation")
governance_value["allowed_files"] = sorted(allowed)
return cls(
task_id=task_id,
job_id=str(execution["job_id"]),
repository=repository,
base_oid=base_oid,
expected_branch_oid=str(execution["expected_branch_oid"]),
target_branch=target_branch,
branch_mode=str(execution["branch_mode"]),
generation=int(execution["generation"]),
idempotency_key=str(execution["idempotency_key"]),
action=action,
adapter=adapter,
capability_scope=tuple(execution.get("capability_scope") or ()),
source_signals=tuple(execution.get("source_signals") or ()),
runtime_input=dict(execution.get("runtime_input") or {}),
change_set=tuple(normalized_changes),
verification=tuple(normalized_verification),
governance=governance_value,
workspace=workspace_value,
digest=str(execution["execution_digest"]),
)
def external_agent_request_id(
envelope: ExecutionEnvelope,
runtime_name: str,
attempt: int,
) -> str:
if not isinstance(attempt, int) or isinstance(attempt, bool) or attempt < 1:
raise WorkerError("TASK_ATTEMPT_INVALID", phase="external_agent")
if runtime_name not in {"hermes", "openclaw"}:
raise WorkerError(
"EXTERNAL_AGENT_RUNTIME_UNSUPPORTED",
phase="external_agent",
details={"runtime": runtime_name},
)
return (
f"external-{runtime_name}-"
+ digest_json(
{
"task_id": envelope.task_id,
"job_id": envelope.job_id,
"execution_digest": envelope.digest,
"runtime": runtime_name,
"attempt": attempt,
}
)[:32]
)
class RuntimeAdapter(ABC):
name: str
@abstractmethod
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
raise NotImplementedError
class DeterministicExecutor(RuntimeAdapter):
name = "deterministic"
@staticmethod
def _write_atomic(path: Path, content: str) -> None:
path.parent.mkdir(mode=0o750, parents=True, exist_ok=True)
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
handle.write(content)
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
changed: set[str] = set()
for change in envelope.change_set:
relative = str(change["path"])
path = resolve_under(worktree, relative, write=True)
expected = change.get("expected_sha256")
current = path.read_bytes() if path.is_file() else None
if expected is not None and (current is None or digest_bytes(current) != expected):
raise WorkerError("WORKTREE_SOURCE_DRIFT", phase="implementing", details={"path": relative})
operation = change["operation"]
if operation == "write_file":
self._write_atomic(path, str(change["content"]))
elif operation == "replace_text":
if current is None:
raise WorkerError("REPLACE_SOURCE_MISSING", phase="implementing", details={"path": relative})
original = current.decode("utf-8")
old = str(change["old"])
count = original.count(old)
expected_count = int(change.get("count", 1))
if count != expected_count:
raise WorkerError("REPLACE_TEXT_MATCH_COUNT_INVALID", phase="implementing", details={"path": relative, "count": count})
self._write_atomic(path, original.replace(old, str(change["new"])))
elif operation == "delete_file":
if path.exists():
path.unlink()
changed.add(relative)
audit("REPOSITORY_CHANGE_APPLIED", operation=operation, path=relative, sha256=digest_bytes(path.read_bytes()) if path.exists() else None)
commands: list[dict[str, Any]] = []
if changed:
commands.append(run_command("git_intent_to_add", ["/usr/bin/git", "-C", str(worktree), "add", "-N", "--", *sorted(changed)], cwd=worktree, timeout=120, internal_git=True))
commands.append(run_command("git_diff_check", ["/usr/bin/git", "-C", str(worktree), "diff", "--check"], cwd=worktree, timeout=120))
commands.append(run_command("git_diff_stat", ["/usr/bin/git", "-C", str(worktree), "diff", "--no-ext-diff", "--stat"], cwd=worktree, timeout=120))
tests: list[dict[str, Any]] = []
for item in envelope.verification:
cwd = worktree if item["cwd"] == "." else resolve_under(worktree, item["cwd"])
tests.append(run_command(item["label"], item["argv"], cwd=cwd, timeout=item["timeout_seconds"]))
return {
"changed_files": sorted(changed),
"commands": commands,
"tests": tests,
"adapter": self.name,
}
class SourceCaptureExecutor(RuntimeAdapter):
"""Read-only repository source capture for a first implementation attempt."""
name = "source_capture"
roots = ("active-runtime/app", "tests", "migrations", "deploy/systemd")
suffixes = {".py", ".sql", ".yaml", ".yml", ".json", ".md", ".service"}
defaults = {
"agent", "task", "worker", "execution", "projection", "claim", "heartbeat",
"governed", "trace", "tool", "credential", "scope", "message", "handoff",
}
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
terms = set(self.defaults)
for signal in envelope.source_signals:
terms.update(token.lower() for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]{2,63}", signal))
terms = set(sorted(terms)[:64])
selected: list[tuple[str, bytes]] = []
total = 0
root = worktree.resolve()
for relative_root in self.roots:
candidate_root = resolve_under(root, relative_root)
if not candidate_root.is_dir() or candidate_root.is_symlink():
continue
for path in sorted(candidate_root.rglob("*")):
if len(selected) >= 96 or total >= 3 * 1024 * 1024:
break
if path.is_symlink() or not path.is_file() or path.suffix.lower() not in self.suffixes:
continue
resolved = path.resolve(strict=True)
if root not in resolved.parents:
raise WorkerError("SOURCE_CAPTURE_PATH_ESCAPE", phase="source_capture")
data = path.read_bytes()
if len(data) > MAX_FILE_BYTES:
continue
relative = path.relative_to(root).as_posix()
searchable = (relative + "\n" + data[:256000].decode("utf-8", "ignore")).lower()
if not any(term in searchable for term in terms):
continue
selected.append((relative, data))
total += len(data)
if not selected:
raise WorkerError("SOURCE_CAPTURE_EMPTY", phase="source_capture")
payload = io.BytesIO()
with tarfile.open(fileobj=payload, mode="w:gz") as archive:
for relative, data in selected:
info = tarfile.TarInfo(relative)
info.size = len(data)
info.mode = 0o640
info.mtime = 0
archive.addfile(info, io.BytesIO(data))
compressed = payload.getvalue()
if len(compressed) > MAX_TASK_BYTES:
raise WorkerError("SOURCE_CAPTURE_TOO_LARGE", phase="source_capture", details={"compressed_bytes": len(compressed)})
manifest = [
{"path": relative, "bytes": len(data), "sha256": digest_bytes(data), "source": "repository"}
for relative, data in selected
]
audit("ENGINEERING_SOURCE_CAPTURED", file_count=len(manifest), compressed_bytes=len(compressed))
return {
"changed_files": [],
"commands": [],
"tests": [],
"adapter": self.name,
"source_snapshot": {
"schema": "aios.engineering.source-snapshot.v2",
"encoding": "tar.gz+base64",
"payload_b64": base64.b64encode(compressed).decode("ascii"),
"payload_sha256": digest_bytes(compressed),
"compressed_bytes": len(compressed),
"file_count": len(manifest),
"files": manifest,
},
}
class CompactInterfaceMapExecutor(RuntimeAdapter):
"""Bounded read-only source analysis carried by the canonical source_capture action."""
name = "compact_interface_map"
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
if envelope.action != "source_capture":
raise WorkerError("COMPACT_INTERFACE_MAP_ACTION_INVALID", phase="interface_map")
try:
interface_map = build_compact_interface_map(
worktree,
envelope.source_signals,
source_commit=envelope.base_oid,
)
except CompactInterfaceMapError as exc:
raise WorkerError(str(exc), phase="interface_map") from exc
audit(
"ENGINEERING_INTERFACE_MAP_CAPTURED",
module_count=len(interface_map.get("implementation_map", {}).get("modules", [])),
excerpt_count=len(interface_map.get("bounded_excerpts", [])),
result_bytes=interface_map.get("result_bytes"),
source_commit=envelope.base_oid,
)
return {
"changed_files": [],
"commands": [],
"tests": [],
"adapter": self.name,
"interface_map": interface_map,
}
class CodexExecutor(RuntimeAdapter):
name = "codex"
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
raise WorkerError(
"CODEX_ADAPTER_REQUIRES_EXPLICIT_POLICY",
phase="adapter",
details={"reason": "Codex is not allowed to mutate a governed worktree without an explicit file/command policy"},
)
class ExternalAgentExecutor(RuntimeAdapter):
runtime_name: str
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
if envelope.action != "agent_runtime":
raise WorkerError(
"EXTERNAL_AGENT_ACTION_INVALID",
phase="adapter",
details={"adapter": self.runtime_name, "action": envelope.action},
)
if envelope.adapter != self.runtime_name:
raise WorkerError(
"EXTERNAL_AGENT_ADAPTER_BINDING_INVALID",
phase="adapter",
)
if not envelope.runtime_input:
raise WorkerError("EXTERNAL_AGENT_RUNTIME_INPUT_MISSING", phase="adapter")
request_id = external_agent_request_id(
envelope,
self.runtime_name,
attempt,
)
exchange = ExternalAgentExchange(EXTERNAL_AGENT_EXECUTOR_ROOT)
try:
external = exchange.execute(
request_id=request_id,
runtime=self.runtime_name,
task_id=envelope.task_id,
attempt=attempt,
execution_digest=envelope.digest,
payload=envelope.runtime_input,
timeout_seconds=EXTERNAL_AGENT_EXECUTOR_TIMEOUT_SECONDS,
)
except ExternalAgentRuntimeError as exc:
raise WorkerError(
str(exc),
phase="external_agent",
details={"adapter": self.runtime_name},
) from exc
audit(
"EXTERNAL_AGENT_RUNTIME_COMPLETED",
task_id=envelope.task_id,
adapter=self.runtime_name,
request_id=request_id,
attempt=attempt,
output_sha256=external.get("output_sha256"),
duration_ms=external.get("duration_ms"),
credential_material_exposed=False,
)
return {
"changed_files": [],
"commands": [],
"tests": [],
"adapter": self.runtime_name,
"external_agent": external,
}
class HermesExecutor(ExternalAgentExecutor):
name = "hermes"
runtime_name = "hermes"
class OpenClawExecutor(ExternalAgentExecutor):
name = "openclaw"
runtime_name = "openclaw"
def adapter_for(name: str) -> RuntimeAdapter:
return {
"deterministic": IsolatedDeterministicExecutor,
"source_capture": SourceCaptureExecutor,
"compact_interface_map": CompactInterfaceMapExecutor,
"codex": CodexExecutor,
"hermes": HermesExecutor,
"openclaw": OpenClawExecutor,
}[name]()
class HttpClient:
@staticmethod
def call(base: str, method: str, route: str, token: str | None, body: Mapping[str, Any] | None = None, timeout: int = 30) -> tuple[int, dict[str, Any]]:
payload = None if body is None else json.dumps(body, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
headers = {"Accept": "application/json"}
if token:
headers["Authorization"] = "Bearer " + token
if payload is not None:
headers["Content-Type"] = "application/json"
request = urllib.request.Request(base + route, data=payload, headers=headers, method=method)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
value = json.loads(response.read().decode("utf-8"))
return response.status, value if isinstance(value, dict) else {}
except urllib.error.HTTPError as exc:
raw = exc.read(8192).decode("utf-8", "replace")
try:
value = json.loads(raw)
except json.JSONDecodeError:
value = {"error": "HTTP_ERROR"}
return exc.code, value if isinstance(value, dict) else {"error": "HTTP_ERROR"}
except (urllib.error.URLError, TimeoutError):
return 599, {"error": "SERVICE_UNAVAILABLE"}
class AgentTaskClient:
def __init__(self, token: str) -> None:
self.token = token
def poll(self) -> list[dict[str, Any]]:
status, response = HttpClient.call(API_BASE, "POST", "/v1/tasks/poll", self.token, {"limit": 100})
if status != 200:
raise WorkerError("TASK_POLL_FAILED", phase="poll", details={"api_status": status, "error": response.get("error")})
tasks = response.get("tasks")
return [dict(item) for item in tasks if isinstance(item, Mapping)] if isinstance(tasks, list) else []
def claim(self, task_id: str) -> tuple[int, dict[str, Any]]:
return HttpClient.call(API_BASE, "POST", f"/v1/tasks/{task_id}/claim", self.token, {"lease_seconds": LEASE_SECONDS})
def get(self, task_id: str) -> tuple[int, dict[str, Any]]:
return HttpClient.call(API_BASE, "GET", f"/v1/tasks/{task_id}", self.token)
def heartbeat(self, task_id: str, lease_token: str, stage: str) -> tuple[int, dict[str, Any]]:
return HttpClient.call(
API_BASE,
"POST",
f"/v1/tasks/{task_id}/heartbeat",
self.token,
{"lease_token": lease_token, "lease_seconds": LEASE_SECONDS, "progress": {"stage": stage, "worker_version": AGENT_VERSION}},
)
def result(self, task_id: str, lease_token: str, idempotency_key: str, result: Mapping[str, Any]) -> tuple[int, dict[str, Any]]:
return HttpClient.call(
API_BASE,
"POST",
f"/v1/tasks/{task_id}/result",
self.token,
{"lease_token": lease_token, "idempotency_key": idempotency_key, "result": dict(result)},
)
class BridgeClient:
def __init__(self, token: str) -> None:
self.token = token
def call(self, route: str, body: Mapping[str, Any]) -> tuple[int, dict[str, Any]]:
return HttpClient.call(BRIDGE_BASE, "POST", route, self.token, body, timeout=60)
def project_execution(self, task: Mapping[str, Any], execution: Mapping[str, Any] | None = None) -> tuple[int, dict[str, Any]]:
request: dict[str, Any] = {
"task_id": str(task.get("task_id") or ""),
"required_capabilities": list(task.get("required_capabilities") or []),
}
if execution is not None:
request["execution"] = dict(execution)
return self.call("/v2/execution-envelope", request)
class LeaseHeartbeat:
def __init__(self, client: AgentTaskClient, task_id: str, lease_token: str) -> None:
self.client = client
self.task_id = task_id
self.lease_token = lease_token
self.stage = "claimed"
self.failed: str | None = None
self.stop_event = threading.Event()
self.lock = threading.Lock()
self.thread = threading.Thread(target=self._run, name="online-agent-heartbeat", daemon=True)
def first(self) -> None:
status, response = self.client.heartbeat(self.task_id, self.lease_token, self.stage)
if not 200 <= status < 300:
self.failed = str(response.get("error") or status)
raise WorkerError("TASK_HEARTBEAT_FAILED", phase="heartbeat", details={"api_status": status, "error": self.failed})
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=self.stage, api_status=status)
write_worker_liveness(
"BUSY",
task_id=self.task_id,
stage=self.stage,
)
def start(self) -> None:
self.thread.start()
def set_stage(self, stage: str) -> None:
with self.lock:
self.stage = stage
def _run(self) -> None:
while not self.stop_event.wait(HEARTBEAT_SECONDS):
with self.lock:
stage = self.stage
try:
status, response = self.client.heartbeat(self.task_id, self.lease_token, stage)
if not 200 <= status < 300:
self.failed = str(response.get("error") or status)
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, api_status=status, error=self.failed)
else:
audit("TASK_HEARTBEAT", task_id=self.task_id, stage=stage, api_status=status)
write_worker_liveness(
"BUSY",
task_id=self.task_id,
stage=stage,
)
except Exception as exc: # pragma: no cover - defensive runtime guard
self.failed = type(exc).__name__
audit("TASK_HEARTBEAT_FAILED", task_id=self.task_id, stage=stage, error=self.failed)
def ensure(self) -> None:
if self.failed:
raise WorkerError("TASK_LEASE_LOST", phase="heartbeat", details={"error": self.failed})
def close(self) -> None:
self.stop_event.set()
if self.thread.ident is not None:
self.thread.join(timeout=2)
def load_agent_token() -> str:
directory = Path(os.environ.get("CREDENTIALS_DIRECTORY", ""))
path = directory / "agent_token"
if not path.is_file() or path.is_symlink():
raise WorkerError("AGENT_RUNTIME_CREDENTIAL_UNAVAILABLE", phase="startup")
value = path.read_text(encoding="utf-8").strip()
if len(value) < 24:
raise WorkerError("AGENT_RUNTIME_CREDENTIAL_INVALID", phase="startup")
return value
def ensure_roots() -> None:
if AUTHORITY_GIT_DIR.is_symlink() or STATE_ROOT.is_symlink() or WORKSPACE_ROOT.is_symlink():
raise WorkerError("ROOT_SYMLINK_REJECTED", phase="startup")
if not AUTHORITY_GIT_DIR.is_dir():
raise WorkerError("AUTHORITY_GIT_DIR_UNAVAILABLE", phase="startup")
STATE_ROOT.mkdir(mode=0o750, parents=True, exist_ok=True)
WORKSPACE_ROOT.mkdir(mode=0o750, parents=True, exist_ok=True)
if GIT_CONFIG_GLOBAL.is_symlink():
raise WorkerError("GIT_CONFIG_SYMLINK_REJECTED", phase="startup")
expected = f"[safe]\n\tdirectory = {AUTHORITY_GIT_DIR}\n"
if GIT_CONFIG_GLOBAL.exists() and GIT_CONFIG_GLOBAL.read_text(encoding="utf-8") != expected:
raise WorkerError("GIT_CONFIG_BINDING_CONFLICT", phase="startup")
if not GIT_CONFIG_GLOBAL.exists():
GIT_CONFIG_GLOBAL.write_text(expected, encoding="utf-8")
os.chmod(GIT_CONFIG_GLOBAL, 0o640)
def task_state_path(task_id: str) -> Path:
if not ID_RE.fullmatch(task_id):
raise WorkerError("TASK_ID_INVALID", phase="state")
return STATE_ROOT / "tasks" / f"{task_id}.json"
def load_task_state(
task_id: str,
execution_digest: str,
*,
attempt: int | None = None,
continuation_head_oid: str | None = None,
continuation_job_id: str | None = None,
commit_idempotency_key: str | None = None,
) -> dict[str, Any] | None:
path = task_state_path(task_id)
if not path.is_file() or path.is_symlink():
return None
try:
value = json.loads(path.read_text(encoding="utf-8"))
except Exception:
raise WorkerError("TASK_STATE_INVALID", phase="state")
if not isinstance(value, dict):
raise WorkerError("TASK_STATE_INVALID", phase="state")
if value.get("execution_digest") not in {None, execution_digest}:
if value.get("phase") in {"claimed", "workspace_prepared", "source_capture_done"} and not value.get("governed"):
return None
if (
value.get("phase") == "result_submitted"
and not value.get("governed")
and attempt is not None
and int(value.get("attempt") or 0) < int(attempt)
):
# A prior read-only/source-capture or pre-commit failure may leave a
# result_submitted runtime marker with the old envelope digest. It has
# no governed side effect, so a later official attempt may start from
# the newly CAS-bound deterministic plan without erasing the audit.
return None
if (
value.get("phase") == "verification_passed"
and not value.get("governed")
and attempt is not None
and int(value.get("attempt") or 0) < int(attempt)
):
# The prior official attempt completed deterministic validation but failed before
# a WriterLease or governed commit. A later CAS-authorized retry may use its new
# digest; the same attempt can never erase or replace this state.
return None
if value.get("phase") == "reconciliation_required" and not value.get("governed"):
stored_attempt = value.get("attempt")
if (
stored_attempt is not None
and attempt is not None
and int(stored_attempt) < int(attempt)
and value.get("job_id") == continuation_job_id
and isinstance(continuation_head_oid, str)
and OID_RE.fullmatch(continuation_head_oid)
):
return None
details = value.get("details")
if (
stored_attempt is None
and attempt is not None
and isinstance(details, Mapping)
and details.get("api_status") == 503
and details.get("error") == "WORKTREE_DIRTY_SCOPE_MISMATCH"
and value.get("job_id") == continuation_job_id
and isinstance(continuation_head_oid, str)
and OID_RE.fullmatch(continuation_head_oid)
):
# Compatibility for the pre-fix recovery record written by attempt 18,
# which omitted its attempt number. This exact pre-commit failure has no
# governed side effect; plan-registry CAS and the continuation head still
# bind the later official claim before the stale state can be discarded.
return None
if (
stored_attempt is not None
and attempt is not None
and int(stored_attempt) < int(attempt)
and value.get("job_id") == continuation_job_id
and continuation_head_oid is None
and isinstance(details, Mapping)
and details.get("api_status") == 503
and details.get("error") == "SECRET_LIKE_PAYLOAD_REJECTED"
):
# SECRET_LIKE_PAYLOAD_REJECTED is a deterministic policy denial from the
# governed bridge, not an ambiguous remote write. It cannot create a branch,
# receipt, or commit, so a later CAS-authorized CREATE_NEW attempt may safely
# discard this stale reconciliation marker after the same Task/Job binding is
# re-established by the official execution plan.
return None
governed = value.get("governed")
if (
continuation_head_oid
and isinstance(governed, Mapping)
and governed.get("commit_oid") == continuation_head_oid
and governed.get("commit_idempotency_key") != commit_idempotency_key
):
# A previous governed commit is the exact continuation head; a new digest/key is a
# legitimate next deterministic plan on the same Task/Job lineage.
return None
raise WorkerError("TASK_PAYLOAD_DRIFT", phase="recovery")
return value
def save_task_state(task_id: str, value: Mapping[str, Any]) -> None:
atomic_json(task_state_path(task_id), value)
def checkpoint_binding(envelope: ExecutionEnvelope, attempt: int) -> CheckpointBinding:
return CheckpointBinding(
task_id=envelope.task_id,
attempt=attempt,
execution_digest=envelope.digest,
job_id=envelope.job_id,
repository=envelope.repository,
base_oid=envelope.base_oid,
expected_branch_oid=envelope.expected_branch_oid,
target_branch=envelope.target_branch,
generation=envelope.generation,
)
_CHECKPOINT_STORE: CheckpointStore | None = None
def checkpoint_store() -> CheckpointStore:
global _CHECKPOINT_STORE
if _CHECKPOINT_STORE is None:
_CHECKPOINT_STORE = CheckpointStore(CHECKPOINT_DB)
return _CHECKPOINT_STORE
def save_checkpoint(
task_id: str,
state: Mapping[str, Any],
envelope: ExecutionEnvelope,
attempt: int,
) -> dict[str, Any]:
phase = str(state.get("phase") or "")
try:
return checkpoint_store().append(
checkpoint_binding(envelope, attempt),
phase,
dict(state),
)
except CheckpointError as exc:
raise WorkerError(str(exc), phase="checkpoint") from exc
def latest_reusable_checkpoint(
envelope: ExecutionEnvelope,
attempt: int,
) -> dict[str, Any] | None:
try:
return checkpoint_store().verify_latest_binding(
checkpoint_binding(envelope, attempt)
)
except CheckpointError as exc:
raise WorkerError(str(exc), phase="checkpoint") from exc
def result_idempotency_key(
task_id: str,
attempt: int,
envelope: ExecutionEnvelope,
result: Mapping[str, Any],
) -> str:
governed = result.get("governed")
commit_key = (
governed.get("commit_idempotency_key")
if isinstance(governed, Mapping)
else None
)
material = {
"task_id": task_id,
"attempt": attempt,
"execution_digest": envelope.digest,
"commit_idempotency_key": commit_key,
}
readable = re.sub(r"[^A-Za-z0-9._:-]+", "-", task_id).strip("-")[:72] or "task"
return f"{readable}-a{attempt}-result-{digest_json(material)[:24]}"
def submit_result_replay_safe(
client: AgentTaskClient,
task_id: str,
lease_token: str,
result_key: str,
result: Mapping[str, Any],
) -> tuple[int, dict[str, Any]]:
expected_digest = digest_json(result)
delay = 0.5
last_status = 599
last_response: dict[str, Any] = {"error": "SERVICE_UNAVAILABLE"}
for _ in range(4):
status, response = client.result(task_id, lease_token, result_key, result)
last_status, last_response = status, response
if 200 <= status < 300:
return status, response
if status == 409 and response.get("error") == "IDEMPOTENCY_CONFLICT":
get_status, current = client.get(task_id)
current_result = current.get("result") if isinstance(current, Mapping) else None
if (
get_status == 200
and isinstance(current_result, Mapping)
and digest_json(current_result) == expected_digest
and current.get("status") in {"VERIFYING", "SUCCEEDED"}
):
return 200, {
"idempotent": True,
"reconciled": True,
"status": current.get("status"),
}
return status, response
if status != 599 and status < 500:
return status, response
time.sleep(delay)
delay = min(delay * 2.0, 4.0)
return last_status, last_response
def recover_stale_workspaces() -> int:
if not WORKSPACE_ROOT.is_dir() or WORKSPACE_ROOT.is_symlink():
return 0
removed = 0
for path in sorted(WORKSPACE_ROOT.glob("task-*")):
if path.is_symlink() or not path.is_dir():
continue
resolved = path.resolve(strict=False)
root = WORKSPACE_ROOT.resolve()
if resolved == root or root not in resolved.parents:
raise WorkerError("WORKSPACE_RECOVERY_SCOPE_INVALID", phase="startup")
shutil.rmtree(resolved)
removed += 1
if removed:
audit("WORKSPACE_RECOVERY_CLEANUP", removed=removed)
return removed
def new_run_root() -> Path:
ensure_roots()
root = Path(tempfile.mkdtemp(prefix="task-", dir=str(WORKSPACE_ROOT)))
os.chmod(root, 0o750)
return root
def cleanup_run_root(path: Path) -> None:
root = WORKSPACE_ROOT.resolve()
candidate = path.resolve(strict=False)
if candidate == root or root not in candidate.parents:
raise WorkerError("CLEANUP_SCOPE_INVALID", phase="cleanup")
if candidate.exists() or candidate.is_symlink():
shutil.rmtree(candidate)
def prepare_worktree(run_root: Path, envelope: ExecutionEnvelope) -> Path:
clone = run_root / "repository"
worktree = run_root / "worktree"
worktree_oid = (
envelope.expected_branch_oid
if envelope.branch_mode == "CONTINUE_EXISTING"
else envelope.base_oid
)
run_command("workspace_clone", ["/usr/bin/git", "clone", "--no-local", "--no-hardlinks", str(AUTHORITY_GIT_DIR), str(clone)], cwd=run_root, timeout=300, internal_git=True)
run_command("workspace_base_probe", ["/usr/bin/git", "-C", str(clone), "rev-parse", "--verify", worktree_oid + "^{commit}"], cwd=run_root, timeout=60)
run_command("workspace_worktree_create", ["/usr/bin/git", "-C", str(clone), "worktree", "add", "--detach", str(worktree), worktree_oid], cwd=run_root, timeout=300, internal_git=True)
return worktree
def _archive_worktree(worktree: Path, archive_path: Path) -> str:
archive_path.parent.mkdir(parents=True, exist_ok=True)
descriptor, temporary_name = tempfile.mkstemp(
prefix=f".{archive_path.name}.",
dir=str(archive_path.parent),
)
os.close(descriptor)
temporary_path = Path(temporary_name)
try:
with tarfile.open(temporary_path, mode="w:gz") as archive:
for path in sorted(worktree.rglob("*")):
relative = path.relative_to(worktree)
if ".git" in relative.parts or path.is_symlink() or not path.is_file():
continue
data = path.read_bytes()
info = tarfile.TarInfo(relative.as_posix())
info.size = len(data)
info.mode = 0o640
info.mtime = 0
archive.addfile(info, io.BytesIO(data))
# Validate the closed temporary archive before it becomes visible to the
# uncredentialed executor. A killed coordinator must leave no final-path
# artifact that the executor can mistake for a complete source archive.
with tarfile.open(temporary_path, mode="r:gz") as verification:
for _ in verification:
pass
with temporary_path.open("rb") as handle:
os.fsync(handle.fileno())
temporary_path.chmod(0o640)
os.replace(temporary_path, archive_path)
try:
directory_fd = os.open(archive_path.parent, os.O_RDONLY | os.O_DIRECTORY)
except (AttributeError, OSError):
directory_fd = None
if directory_fd is not None:
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
return digest_bytes(archive_path.read_bytes())
finally:
temporary_path.unlink(missing_ok=True)
class IsolatedDeterministicExecutor(RuntimeAdapter):
"""Coordinator-side client for the uncredentialed executor exchange."""
name = "isolated-deterministic"
@staticmethod
def _atomic(path: Path, data: bytes) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
try:
with os.fdopen(fd, "wb") as handle:
handle.write(data)
handle.flush()
os.fsync(handle.fileno())
os.chmod(temporary, 0o640)
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
def execute(
self,
envelope: ExecutionEnvelope,
worktree: Path,
*,
attempt: int,
) -> dict[str, Any]:
if envelope.action != "deterministic":
raise WorkerError("ISOLATED_EXECUTOR_ACTION_INVALID", phase="adapter")
root = ISOLATED_EXECUTOR_ROOT
for child in ("requests", "results", "failed", "sources"):
candidate = root / child
if candidate.is_symlink() or not candidate.is_dir():
raise WorkerError("ISOLATED_EXECUTOR_UNAVAILABLE", phase="adapter", details={"path": str(candidate)})
request_id = f"{envelope.task_id}.{secrets.token_hex(16)}"
if not REQUEST_ID_RE.fullmatch(request_id):
raise WorkerError("ISOLATED_EXECUTOR_REQUEST_ID_INVALID", phase="adapter")
archive = root / "sources" / f"{request_id}.tgz"
request_path = root / "requests" / f"{request_id}.json"
result_path = root / "results" / f"{request_id}.json"
failed_path = root / "failed" / f"{request_id}.json"
for stale in (request_path, result_path, failed_path, archive):
if stale.exists() or stale.is_symlink():
raise WorkerError(
"ISOLATED_EXECUTOR_COLLISION_DETECTED", phase="adapter", details={"request_id": request_id, "path": str(stale)}
)
archive_sha = _archive_worktree(worktree, archive)
request = {
"schema": "aios.isolated-execution.request.v1",
"request_id": request_id,
"task_id": envelope.task_id,
"job_id": envelope.job_id,
"execution_digest": envelope.digest,
"source_archive": str(archive),
"source_archive_sha256": archive_sha,
"change_set": list(envelope.change_set),
"verification": list(envelope.verification),
}
request_sha = digest_json(request)
self._atomic(request_path, json.dumps(request, sort_keys=True, separators=(",", ":")).encode("utf-8"))
wait_seconds = _isolated_executor_wait_seconds(envelope.verification)
deadline = time.monotonic() + wait_seconds
try:
while time.monotonic() < deadline:
if failed_path.is_file():
details = json.loads(failed_path.read_text(encoding="utf-8"))
raise WorkerError("ISOLATED_EXECUTOR_FAILED", phase="adapter", details={"error": details.get("error")})
if result_path.is_file():
result = json.loads(result_path.read_text(encoding="utf-8"))
if result.get("request_id") != request_id:
raise WorkerError("ISOLATED_EXECUTOR_REQUEST_ID_MISMATCH", phase="adapter")
if result.get("schema") != "aios.isolated-execution.result.v1":
raise WorkerError("ISOLATED_EXECUTOR_RESULT_INVALID", phase="adapter")
if result.get("request_sha256") != request_sha:
raise WorkerError("ISOLATED_EXECUTOR_REQUEST_BINDING_INVALID", phase="adapter")
material = {key: value for key, value in result.items() if key != "result_sha256"}
if result.get("result_sha256") != digest_json(material):
raise WorkerError("ISOLATED_EXECUTOR_RESULT_DIGEST_INVALID", phase="adapter")
isolation = result.get("credential_isolation")
if not isinstance(isolation, Mapping) or not all(isolation.get(k) is True for k in (
"credential_env_absent", "credential_mount_inaccessible", "governed_bridge_unreachable"
)):
raise WorkerError("CREDENTIAL_ISOLATION_VERIFY", phase="adapter")
expected_paths = sorted({str(item["path"]) for item in envelope.change_set})
returned = result.get("changed_files")
if not isinstance(returned, list) or sorted(str(item.get("path")) for item in returned) != expected_paths:
raise WorkerError("ISOLATED_EXECUTOR_CHANGED_FILESET_INVALID", phase="adapter")
for item in returned:
relative = safe_relative_path(item.get("path"), field="isolated_changed_file")
path = resolve_under(worktree, relative, write=True)
if item.get("deleted") is True:
path.unlink(missing_ok=True)
continue
raw = base64.b64decode(str(item.get("content_b64") or ""), validate=True)
if digest_bytes(raw) != item.get("sha256"):
raise WorkerError("ISOLATED_EXECUTOR_FILE_DIGEST_INVALID", phase="adapter")
self._atomic(path, raw)
commands = [
run_command("git_diff_check", ["/usr/bin/git", "-C", str(worktree), "diff", "--check"], cwd=worktree, timeout=120),
run_command("git_diff_stat", ["/usr/bin/git", "-C", str(worktree), "diff", "--no-ext-diff", "--stat"], cwd=worktree, timeout=120),
]
return {
"changed_files": expected_paths,
"commands": commands,
"tests": list(result.get("verification") or []),
"adapter": self.name,
"credential_isolation": dict(isolation),
}
time.sleep(0.25)
raise WorkerError("ISOLATED_EXECUTOR_TIMEOUT", phase="adapter")
finally:
request_path.unlink(missing_ok=True)
result_path.unlink(missing_ok=True)
failed_path.unlink(missing_ok=True)
archive.unlink(missing_ok=True)
def _verify_converged_job(
response: Mapping[str, Any],
envelope: ExecutionEnvelope,
commit_oid: str,
) -> None:
if response.get("state") != "COMPLETED":
raise WorkerError("GOVERNED_RECEIPT_STATE_INVALID", phase="verification")
receipt = response.get("receipt")
if not isinstance(receipt, Mapping) or receipt.get("remote_verified") is not True:
raise WorkerError("GOVERNED_RECEIPT_REQUIRED", phase="verification")
for field in ("authority_oid", "api_branch_oid", "git_remote_oid"):
if receipt.get(field) != commit_oid:
raise WorkerError("GOVERNED_RECEIPT_OID_MISMATCH", phase="verification", details={"field": field})
leases = response.get("branch_leases")
if not isinstance(leases, list):
raise WorkerError("WRITER_LEASE_RECOVERY_EVIDENCE_MISSING", phase="verification")
active = [
item for item in leases
if isinstance(item, Mapping)
and item.get("state") == "ACTIVE"
and item.get("job_id") == envelope.job_id
and item.get("branch") == envelope.target_branch
]
if len(active) != 1:
raise WorkerError("WRITER_LEASE_RECOVERY_EVIDENCE_INVALID", phase="verification")
lease = active[0]
if (
int(lease.get("generation") or 0) != envelope.generation
or lease.get("current_head_oid") != commit_oid
):
raise WorkerError("WRITER_LEASE_CURRENT_HEAD_MISMATCH", phase="verification")
def acquire_writer_lease_replay_safe(
envelope: ExecutionEnvelope,
bridge: BridgeClient,
heartbeat: LeaseHeartbeat,
) -> dict[str, Any]:
governance = envelope.governance
body = {
"schema": "aios.task-writer-lease.v2",
"task_id": envelope.task_id,
"job_id": envelope.job_id,
"repository": envelope.repository,
"branch_mode": envelope.branch_mode,
"base_oid": envelope.base_oid,
"expected_branch_oid": envelope.expected_branch_oid,
"target_branch": envelope.target_branch,
"generation": envelope.generation,
"lease_seconds": int(governance.get("lease_seconds") or 900),
}
delay = 0.5
last_status = 599
last_response: dict[str, Any] = {"error": "SERVICE_UNAVAILABLE"}
for attempt_index in range(4):
heartbeat.ensure()
status, response = bridge.call("/v2/writer-lease", body)
last_status, last_response = status, response
if 200 <= status < 300:
if (
response.get("job_id") != envelope.job_id
or response.get("task_id") != envelope.task_id
or response.get("repository") != envelope.repository
or response.get("base_oid") != envelope.base_oid
or response.get("expected_branch_oid") != envelope.expected_branch_oid
or response.get("branch_mode") != envelope.branch_mode
or int(response.get("generation") or 0) != envelope.generation
or response.get("target_branch") != envelope.target_branch
or response.get("state") not in {None, "ACTIVE"}
):
raise WorkerError(
"WRITER_LEASE_RESPONSE_BINDING_INVALID",
phase="committing",
)
if attempt_index:
audit(
"WRITER_LEASE_RECOVERED",
task_id=envelope.task_id,
job_id=envelope.job_id,
attempts=attempt_index + 1,
lease_id=response.get("lease_id"),
)
return response
if status != 599 and status < 500:
break
time.sleep(delay)
delay = min(delay * 2.0, 4.0)
raise WorkerError(
"WRITER_LEASE_ACQUIRE_FAILED",
phase="committing",
details={
"api_status": last_status,
"error": last_response.get("error"),
},
)
def governed_commit(envelope: ExecutionEnvelope, worktree: Path, bridge: BridgeClient, heartbeat: LeaseHeartbeat) -> dict[str, Any]:
governance = envelope.governance
if governance.get("task_id") != envelope.task_id or governance.get("job_id") != envelope.job_id:
raise WorkerError("GOVERNED_LINEAGE_INVALID", phase="committing")
generation = envelope.generation
expected_branch_oid = envelope.expected_branch_oid
branch_mode = envelope.branch_mode
lease_response = acquire_writer_lease_replay_safe(
envelope,
bridge,
heartbeat,
)
heartbeat.ensure()
files: dict[str, str] = {}
paths = governance.get("allowed_files") or [item["path"] for item in envelope.change_set]
for relative in sorted({safe_relative_path(item, field="allowed_file") for item in paths}):
path = resolve_under(worktree, relative)
if not path.is_file():
raise WorkerError("COMMIT_FILE_MISSING", phase="committing", details={"path": relative})
files[relative] = base64.b64encode(path.read_bytes()).decode("ascii")
commit_body = {
"task_id": envelope.task_id,
"job_id": envelope.job_id,
"repository": envelope.repository,
"expected_branch_oid": expected_branch_oid,
"target_branch": envelope.target_branch,
"generation": generation,
"idempotency_key": governance["commit_idempotency_key"],
"files": files,
"message": governance["message"],
}
status, commit_response = bridge.call("/v2/commit-files", commit_body)
if not 200 <= status < 300:
error = commit_response.get("error")
ambiguous = status == 599 or (
status >= 500 and error != "SECRET_LIKE_PAYLOAD_REJECTED"
)
code = "RECONCILIATION_REQUIRED" if ambiguous else (
str(error) if isinstance(error, str) and error else "GOVERNED_COMMIT_FAILED"
)
raise WorkerError(code, phase="committing", details={"api_status": status, "error": commit_response.get("error"), "job_id": envelope.job_id})
if (
commit_response.get("job_id") != envelope.job_id
or commit_response.get("target_branch") != envelope.target_branch
or int(commit_response.get("generation") or 0) != generation
or commit_response.get("branch_mode") not in {"CREATE_NEW", "CONTINUE_EXISTING"}
):
raise WorkerError("GOVERNED_COMMIT_RESPONSE_BINDING_INVALID", phase="committing")
commit_oid = commit_response.get("commit_oid")
if not isinstance(commit_oid, str) or not OID_RE.fullmatch(commit_oid):
raise WorkerError("GOVERNED_COMMIT_OID_INVALID", phase="committing")
receipt = commit_response.get("receipt")
if not isinstance(receipt, Mapping) or receipt.get("remote_verified") is not True:
raise WorkerError("GOVERNED_COMMIT_RECEIPT_REQUIRED", phase="verification")
visible_oid: str | None = None
deadline = time.monotonic() + max(5, min(GIT_VISIBILITY_TIMEOUT_SECONDS, 120))
delay = 0.5
while time.monotonic() < deadline:
probe = subprocess.run(
["/usr/bin/git", "--git-dir", str(AUTHORITY_GIT_DIR), "rev-parse", "--verify", f"refs/heads/{envelope.target_branch}"],
text=True,
capture_output=True,
check=False,
env={"PATH": "/usr/bin:/bin", "LC_ALL": "C.UTF-8", "GIT_CONFIG_GLOBAL": str(GIT_CONFIG_GLOBAL)},
timeout=30,
)
if probe.returncode == 0:
visible_oid = probe.stdout.strip()
if visible_oid == commit_oid:
break
time.sleep(delay)
delay = min(delay * 1.7, 5.0)
if visible_oid != commit_oid:
raise WorkerError("RECONCILIATION_REQUIRED", phase="verification", details={"commit_oid": commit_oid, "visible_oid": visible_oid, "job_id": envelope.job_id})
job_status, job_response = bridge.call("/v2/job", {"job_id": envelope.job_id})
if not 200 <= job_status < 300:
raise WorkerError("RECONCILIATION_REQUIRED", phase="verification", details={"api_status": job_status, "job_id": envelope.job_id})
_verify_converged_job(job_response, envelope, commit_oid)
return {
"governed_task_id": envelope.task_id,
"job_id": envelope.job_id,
"branch_mode": branch_mode,
"generation": generation,
"target_branch": envelope.target_branch,
"commit_oid": commit_oid,
"visible_oid": visible_oid,
"commit_idempotency_key": governance["commit_idempotency_key"],
"api_commit_visible": commit_response.get("commit_visible"),
"receipt": dict(commit_response.get("receipt")) if isinstance(commit_response.get("receipt"), Mapping) else None,
"direct_main_write": False,
"force_push": False,
"production_write": False,
"protected_ref_bypass": False,
}
def build_result(task_id: str, envelope: ExecutionEnvelope, execution: Mapping[str, Any], governed: Mapping[str, Any]) -> dict[str, Any]:
return {
"status": "SUCCEEDED",
"verification_state": "VERIFIED_BY_WORKER_PENDING_SERVER_REVIEW",
"task_id": task_id,
"execution_backend": "generic-online-agent-worker-v2",
"adapter": envelope.adapter,
"execution_digest": envelope.digest,
"repository": envelope.repository,
"base_oid": envelope.base_oid,
"target_branch": envelope.target_branch,
"changed_files": execution.get("changed_files", []),
"commands": [{k: item.get(k) for k in ("label", "exit_code", "stdout_sha256", "stderr_sha256", "duration_ms")} for item in execution.get("commands", [])],
"tests": [{k: item.get(k) for k in ("label", "exit_code", "stdout_sha256", "stderr_sha256", "duration_ms")} for item in execution.get("tests", [])],
"governed": dict(governed),
"workspace_cleanup": "PASS",
"secret_material_exposed": False,
"direct_main_write": False,
"force_push": False,
"production_write": False,
"protected_ref_bypass": False,
}
def build_compact_interface_map_result(
task_id: str,
envelope: ExecutionEnvelope,
execution: Mapping[str, Any],
) -> dict[str, Any]:
interface_map = execution.get("interface_map")
if not isinstance(interface_map, Mapping):
raise WorkerError("COMPACT_INTERFACE_MAP_RESULT_INVALID", phase="result")
if interface_map.get("schema") != "aios.engineering.interface-map.v1":
raise WorkerError("COMPACT_INTERFACE_MAP_RESULT_INVALID", phase="result")
if interface_map.get("source_commit") != envelope.base_oid:
raise WorkerError("COMPACT_INTERFACE_MAP_SOURCE_BINDING_INVALID", phase="result")
implementation_map = interface_map.get("implementation_map")
excerpts = interface_map.get("bounded_excerpts")
contracts = interface_map.get("contracts")
if not isinstance(implementation_map, Mapping) or not isinstance(excerpts, list) or not isinstance(contracts, Mapping):
raise WorkerError("COMPACT_INTERFACE_MAP_RESULT_INVALID", phase="result")
profile = str(interface_map.get("profile") or "fleet")
classification = str(interface_map.get("classification") or "")
allowed_classifications = {
"fleet": "P2_INTERFACE_MAP_CAPTURED",
"model_governance": "MODEL_GOVERNANCE_INTERFACE_MAP_CAPTURED",
}
if allowed_classifications.get(profile) != classification:
raise WorkerError("COMPACT_INTERFACE_MAP_PROFILE_BINDING_INVALID", phase="result")
result = {
"status": "SUCCEEDED",
"classification": classification,
"interface_map_profile": profile,
"verification_state": "VERIFIED_BY_WORKER_PENDING_SERVER_REVIEW",
"execution_mode": "compact_interface_map",
"execution_backend": "generic-online-agent-worker-v2",
"task_id": task_id,
"job_id": envelope.job_id,
"repository": envelope.repository,
"base_oid": envelope.base_oid,
"expected_branch_oid": envelope.expected_branch_oid,
"target_branch": envelope.target_branch,
"generation": envelope.generation,
"execution_digest": envelope.digest,
"source_commit": interface_map["source_commit"],
"implementation_map": dict(implementation_map),
"bounded_excerpts": list(excerpts),
"contracts": dict(contracts),
"interface_map_result_bytes": interface_map.get("result_bytes"),
"changed_files": [],
"tests": [],
"workspace_cleanup": "PASS",
"writer_lease": "NONE",
"secret_material_exposed": False,
"direct_main_write": False,
"force_push": False,
"production_write": False,
"protected_ref_bypass": False,
}
encoded = json.dumps(result, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
if len(encoded) > 120 * 1024:
raise WorkerError(
"COMPACT_INTERFACE_MAP_RESULT_TOO_LARGE",
phase="result",
details={"result_bytes": len(encoded)},
)
result["result_bytes"] = len(encoded)
return result
def build_agent_runtime_result(
task_id: str,
envelope: ExecutionEnvelope,
execution: Mapping[str, Any],
) -> dict[str, Any]:
external = execution.get("external_agent")
if not isinstance(external, Mapping):
raise WorkerError("EXTERNAL_AGENT_RESULT_INVALID", phase="result")
if external.get("runtime") != envelope.adapter:
raise WorkerError("EXTERNAL_AGENT_RESULT_RUNTIME_MISMATCH", phase="result")
if external.get("task_id") != task_id:
raise WorkerError("EXTERNAL_AGENT_RESULT_TASK_MISMATCH", phase="result")
if external.get("execution_digest") != envelope.digest:
raise WorkerError("EXTERNAL_AGENT_RESULT_DIGEST_MISMATCH", phase="result")
if external.get("credential_material_exposed") is not False:
raise WorkerError("EXTERNAL_AGENT_CREDENTIAL_ISOLATION_FAILED", phase="result")
result = {
"status": "SUCCEEDED",
"classification": "EXTERNAL_AGENT_RUNTIME_SUCCEEDED",
"verification_state": "VERIFIED_BY_WORKER_PENDING_SERVER_REVIEW",
"execution_mode": "agent_runtime",
"execution_backend": "generic-online-agent-worker-v2",
"agent_runtime": envelope.adapter,
"task_id": task_id,
"job_id": envelope.job_id,
"repository": envelope.repository,
"base_oid": envelope.base_oid,
"expected_branch_oid": envelope.expected_branch_oid,
"target_branch": envelope.target_branch,
"generation": envelope.generation,
"execution_digest": envelope.digest,
"output_text": external.get("output_text"),
"output_json": external.get("output_json"),
"output_sha256": external.get("output_sha256"),
"runtime_duration_ms": external.get("duration_ms"),
"external_request_id": external.get("request_id"),
"changed_files": [],
"tests": [],
"workspace_cleanup": "PASS",
"writer_lease": "NONE",
"secret_material_exposed": False,
"credential_material_exposed": False,
"control_plane_access_granted_to_runtime": False,
"direct_main_write": False,
"force_push": False,
"production_write": False,
"protected_ref_bypass": False,
}
encoded = json.dumps(
sanitize(result),
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
if len(encoded) > 192 * 1024:
raise WorkerError(
"EXTERNAL_AGENT_TASK_RESULT_TOO_LARGE",
phase="result",
details={"result_bytes": len(encoded)},
)
result["result_bytes"] = len(encoded)
return result
def build_source_capture_result(
task_id: str,
envelope: ExecutionEnvelope,
execution: Mapping[str, Any],
) -> dict[str, Any]:
if envelope.adapter == "compact_interface_map":
return build_compact_interface_map_result(task_id, envelope, execution)
snapshot = execution.get("source_snapshot")
if not isinstance(snapshot, Mapping):
raise WorkerError("SOURCE_CAPTURE_RESULT_INVALID", phase="result")
return {
"status": "SUCCEEDED",
"classification": "NEEDS_REWORK_SOURCE_CAPTURE",
"verification_state": "NEEDS_REWORK",
"execution_mode": "source_capture",
"execution_backend": "generic-online-agent-worker-v2",
"task_id": task_id,
"job_id": envelope.job_id,
"repository": envelope.repository,
"base_oid": envelope.base_oid,
"expected_branch_oid": envelope.expected_branch_oid,
"target_branch": envelope.target_branch,
"generation": envelope.generation,
"execution_digest": envelope.digest,
"changed_files": [],
"tests": [],
"source_snapshot": dict(snapshot),
"workspace_cleanup": "PASS",
"writer_lease": "NONE",
"secret_material_exposed": False,
"direct_main_write": False,
"force_push": False,
"production_write": False,
"protected_ref_bypass": False,
}
class WorkerSupervisor:
def __init__(self, token: str) -> None:
self.agent = AgentTaskClient(token)
self.bridge = BridgeClient(token)
def resolve_envelope(self, task: Mapping[str, Any]) -> tuple[ExecutionEnvelope, str]:
metadata = task.get("metadata")
execution = metadata.get("execution") if isinstance(metadata, Mapping) else None
if isinstance(metadata, Mapping) and "execution" in metadata:
if not isinstance(execution, Mapping):
raise WorkerError("EXECUTION_ENVELOPE_SCHEMA_VALIDATION", phase="validation")
persisted = ExecutionEnvelope.from_task(task, worker_capabilities=WORKER_CAPABILITIES)
# Read-only source_capture/agent_runtime tasks replay their persisted
# payload. A write-capable source_capture is only a Bridge-created
# engineering placeholder: never pass that stale payload back to the
# resolver, and never claim it until a deterministic plan exists.
pending_plan = source_capture_requires_deterministic_plan(persisted)
projection_input = (
execution
if persisted.action in {"source_capture", "agent_runtime"} and not pending_plan
else None
)
status, authoritative = self.bridge.project_execution(task, projection_input)
if pending_plan and (
status == 404
or authoritative.get("error") == "TASK_EXECUTION_PROJECTION_MISSING"
):
raise WorkerError(
"TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN",
phase="projection",
)
if not 200 <= status < 300:
raise WorkerError("TASK_PERSISTED_BINDING_UNVERIFIED", phase="projection", details={"api_status": status, "error": authoritative.get("error")})
canonical = ExecutionEnvelope.from_task(task, authoritative, WORKER_CAPABILITIES)
binding_fields = (
"task_id",
"job_id",
"repository",
"base_oid",
"expected_branch_oid",
"target_branch",
"branch_mode",
"generation",
)
if any(getattr(canonical, field) != getattr(persisted, field) for field in binding_fields):
raise WorkerError("TASK_PERSISTED_BINDING_CONFLICT", phase="projection")
if canonical.digest != persisted.digest:
if canonical.action != "deterministic":
raise WorkerError("TASK_PERSISTED_BINDING_CONFLICT", phase="projection")
audit(
"TASK_PERSISTED_ENVELOPE_REPLACED_BY_PLAN",
task_id=canonical.task_id,
job_id=canonical.job_id,
previous_action=persisted.action,
action=canonical.action,
previous_execution_digest=persisted.digest,
execution_digest=canonical.digest,
)
return canonical, "plan_registry_verified"
if pending_plan:
raise WorkerError(
"TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN",
phase="projection",
)
return persisted, "persisted_verified"
status, projected = self.bridge.project_execution(task)
if status == 404 or projected.get("error") == "TASK_EXECUTION_PROJECTION_MISSING":
raise WorkerError("TASK_SKIPPED_PROJECTION_MISSING", phase="projection")
if not 200 <= status < 300:
raise WorkerError(
"TASK_SKIPPED_POLICY_DENY",
phase="projection",
details={"api_status": status, "error": projected.get("error")},
)
envelope = ExecutionEnvelope.from_task(task, projected, WORKER_CAPABILITIES)
audit(
"TASK_EXECUTION_ENVELOPE_PROJECTED",
task_id=envelope.task_id,
job_id=envelope.job_id,
generation=envelope.generation,
action=envelope.action,
execution_digest=envelope.digest,
)
return envelope, "recovered_projection"
def process_claim(
self,
task: Mapping[str, Any],
claim: Mapping[str, Any],
envelope: ExecutionEnvelope,
) -> None:
task_id = str(claim.get("task_id") or task.get("task_id") or "")
attempt = int(claim.get("attempt") or 0)
if attempt < 1:
raise WorkerError("TASK_ATTEMPT_INVALID", phase="claim")
lease_token = claim.get("lease_token")
if not isinstance(lease_token, str) or len(lease_token) < 24:
raise WorkerError("LEASE_TOKEN_NOT_RETURNED", phase="claim")
audit("TASK_CLAIMED", task_id=task_id, attempt=attempt, revision=claim.get("revision"))
heartbeat = LeaseHeartbeat(self.agent, task_id, lease_token)
state: dict[str, Any] = {
"task_id": task_id,
"execution_digest": envelope.digest,
"attempt": attempt,
"phase": "claimed",
"job_id": envelope.job_id,
}
try:
heartbeat.first()
heartbeat.set_stage("started")
heartbeat.start()
audit("TASK_STARTED", task_id=task_id, attempt=attempt)
status, current_task = self.agent.get(task_id)
if status != 200:
raise WorkerError(
"TASK_READ_AFTER_CLAIM_FAILED",
phase="claim",
details={"api_status": status},
)
current_envelope, _ = self.resolve_envelope(current_task)
if current_envelope.digest != envelope.digest or current_envelope.job_id != envelope.job_id:
audit(
"TASK_PAYLOAD_DRIFT_DETAILS",
task_id=task_id,
expected_digest=envelope.digest,
current_digest=current_envelope.digest,
expected_job_id=envelope.job_id,
current_job_id=current_envelope.job_id,
expected_action=envelope.action,
current_action=current_envelope.action,
expected_branch_mode=envelope.branch_mode,
current_branch_mode=current_envelope.branch_mode,
expected_generation=envelope.generation,
current_generation=current_envelope.generation,
)
raise WorkerError("TASK_PAYLOAD_DRIFT", phase="claim")
cached = load_task_state(
task_id,
envelope.digest,
attempt=attempt,
continuation_head_oid=(
envelope.expected_branch_oid
if envelope.branch_mode == "CONTINUE_EXISTING"
else None
),
continuation_job_id=envelope.job_id,
commit_idempotency_key=str(
envelope.governance.get("commit_idempotency_key") or ""
),
)
durable = latest_reusable_checkpoint(envelope, attempt)
if cached is None and isinstance(durable, Mapping):
payload = durable.get("payload")
if isinstance(payload, Mapping):
cached = dict(payload)
cached["recovered_from_checkpoint"] = True
audit(
"TASK_CHECKPOINT_RECOVERED",
task_id=task_id,
checkpoint_attempt=durable.get("attempt"),
checkpoint_phase=durable.get("phase"),
checkpoint_sequence=durable.get("sequence"),
)
if cached is not None:
state = dict(cached)
state["task_id"] = task_id
state["execution_digest"] = envelope.digest
state["job_id"] = envelope.job_id
previous_attempt = int(state.get("attempt") or 0)
state["resume_from_attempt"] = previous_attempt or None
state["attempt"] = attempt
current_attempt_checkpoint = checkpoint_store().latest_for_attempt(task_id, attempt)
if current_attempt_checkpoint is None:
save_checkpoint(
task_id,
{
"task_id": task_id,
"execution_digest": envelope.digest,
"job_id": envelope.job_id,
"attempt": attempt,
"phase": "claimed",
"resume_from_phase": state.get("phase"),
},
envelope,
attempt,
)
governed = (
state.get("governed")
if isinstance(state.get("governed"), Mapping)
else None
)
phase = str(state.get("phase") or "claimed")
result: Mapping[str, Any] | None = None
# Read-only source capture is safe to reuse across official attempts when the exact
# execution/scope digest is unchanged. This is the key restart/retry fast path.
if envelope.action == "source_capture" and phase in {
"source_capture_done",
"result_pending",
"result_submitted",
}:
candidate = state.get("result")
if isinstance(candidate, Mapping):
result = dict(candidate)
audit(
"TASK_RESUMED_FROM_SOURCE_CAPTURE",
task_id=task_id,
previous_attempt=state.get("resume_from_attempt"),
attempt=attempt,
phase=phase,
)
if (
result is None
and envelope.action == "source_capture"
and phase == "verification_passed"
and isinstance(state.get("execution"), Mapping)
):
result = build_source_capture_result(
task_id, envelope, state["execution"]
)
state.update({"phase": "source_capture_done", "result": result})
save_task_state(task_id, state)
save_checkpoint(task_id, state, envelope, attempt)
audit(
"TASK_RESUMED_AFTER_SOURCE_CAPTURE_VERIFICATION",
task_id=task_id,
attempt=attempt,
)
if envelope.action == "agent_runtime" and phase in {
"agent_runtime_done",
"result_pending",
"result_submitted",
}:
candidate = state.get("result")
if isinstance(candidate, Mapping):
result = dict(candidate)
audit(
"TASK_RESUMED_FROM_EXTERNAL_AGENT_RESULT",
task_id=task_id,
previous_attempt=state.get("resume_from_attempt"),
attempt=attempt,
phase=phase,
adapter=envelope.adapter,
)
if (
result is None
and envelope.action == "agent_runtime"
and phase == "verification_passed"
and isinstance(state.get("execution"), Mapping)
):
result = build_agent_runtime_result(
task_id, envelope, state["execution"]
)
state.update(
{"phase": "agent_runtime_done", "result": result, "attempt": attempt}
)
save_task_state(task_id, state)
save_checkpoint(task_id, state, envelope, attempt)
audit(
"TASK_RESUMED_AFTER_EXTERNAL_AGENT_EXECUTION",
task_id=task_id,
attempt=attempt,
adapter=envelope.adapter,
)
if (
result is None
and envelope.action == "deterministic"
and isinstance(governed, Mapping)
and isinstance(governed.get("commit_oid"), str)
and OID_RE.fullmatch(governed["commit_oid"])
):
if (
governed.get("job_id") != envelope.job_id
or governed.get("target_branch") != envelope.target_branch
or int(governed.get("generation") or 0) != envelope.generation
or not isinstance(governed.get("receipt"), Mapping)
):
raise WorkerError(
"RECOVERY_EVIDENCE_BINDING_INVALID", phase="recovery"
)
job_status, job_receipt = self.bridge.call(
"/v2/job", {"job_id": envelope.job_id}
)
if not 200 <= job_status < 300:
raise WorkerError(
"RECOVERY_RECEIPT_UNVERIFIED",
phase="recovery",
details={"api_status": job_status},
)
_verify_converged_job(
job_receipt, envelope, str(governed["commit_oid"])
)
probe = subprocess.run(
[
"/usr/bin/git",
"--git-dir",
str(AUTHORITY_GIT_DIR),
"rev-parse",
"--verify",
f"refs/heads/{envelope.target_branch}",
],
text=True,
capture_output=True,
check=False,
env={
"PATH": "/usr/bin:/bin",
"LC_ALL": "C.UTF-8",
"GIT_CONFIG_GLOBAL": str(GIT_CONFIG_GLOBAL),
},
timeout=30,
)
if (
probe.returncode != 0
or probe.stdout.strip() != governed["commit_oid"]
):
raise WorkerError("RECOVERY_HEAD_UNVERIFIED", phase="recovery")
candidate = state.get("result")
if isinstance(candidate, Mapping):
result = dict(candidate)
else:
result = {
"status": "SUCCEEDED",
"verification_state": "VERIFIED_BY_WORKER_PENDING_SERVER_REVIEW",
"task_id": task_id,
"execution_digest": envelope.digest,
"governed": dict(governed),
"recovered_without_duplicate_commit": True,
"direct_main_write": False,
"production_write": False,
"force_push": False,
"protected_ref_bypass": False,
}
audit(
"TASK_RECOVERED_COMMIT",
task_id=task_id,
commit_oid=governed["commit_oid"],
duplicate_commit=False,
)
if result is None:
# Any pre-commit crash is replay-safe: rebuild a clean detached worktree and
# re-run deterministic verification. No external side effect exists yet.
run_root = new_run_root()
try:
if envelope.action == "agent_runtime":
worktree = run_root / "agent-runtime-workspace"
worktree.mkdir(mode=0o750, parents=True, exist_ok=True)
else:
worktree = prepare_worktree(run_root, envelope)
state.update(
{
"phase": "workspace_prepared",
"execution_digest": envelope.digest,
"attempt": attempt,
"job_id": envelope.job_id,
}
)
save_task_state(task_id, state)
save_checkpoint(task_id, state, envelope, attempt)
adapter = adapter_for(envelope.adapter)
execution = adapter.execute(
envelope,
worktree,
attempt=attempt,
)
state.update(
{
"phase": "verification_passed",
"execution": execution,
"attempt": attempt,
}
)
save_task_state(task_id, state)
save_checkpoint(task_id, state, envelope, attempt)
heartbeat.ensure()
if envelope.action == "source_capture":
result = build_source_capture_result(
task_id, envelope, execution
)
state.update(
{
"phase": "source_capture_done",
"result": result,
"attempt": attempt,
}
)
elif envelope.action == "agent_runtime":
result = build_agent_runtime_result(
task_id, envelope, execution
)
state.update(
{
"phase": "agent_runtime_done",
"result": result,
"attempt": attempt,
}
)
else:
heartbeat.set_stage("committing")
governed = governed_commit(
envelope, worktree, self.bridge, heartbeat
)
result = build_result(
task_id, envelope, execution, governed
)
state.update(
{
"phase": "commit_done",
"governed": governed,
"result": result,
"attempt": attempt,
}
)
save_task_state(task_id, state)
save_checkpoint(task_id, state, envelope, attempt)
finally:
cleanup_run_root(run_root)
audit(
"WORKSPACE_CLEANED",
task_id=task_id,
cleanup="PASS",
)
if not isinstance(result, Mapping):
raise WorkerError("TASK_RESULT_MISSING", phase="result")
heartbeat.ensure()
heartbeat.set_stage("result")
result = dict(result)
result_key = result_idempotency_key(
task_id, attempt, envelope, result
)
state.update(
{
"phase": "result_pending",
"attempt": attempt,
"result": result,
"result_idempotency_key": result_key,
"result_sha256": digest_json(result),
}
)
save_task_state(task_id, state)
save_checkpoint(task_id, state, envelope, attempt)
status, response = submit_result_replay_safe(
self.agent, task_id, lease_token, result_key, result
)
if not 200 <= status < 300:
raise WorkerError(
"TASK_RESULT_SUBMIT_FAILED",
phase="result",
details={
"api_status": status,
"error": response.get("error"),
"result_idempotency_key": result_key,
},
)
state.update(
{
"phase": "result_submitted",
"attempt": attempt,
"result_idempotency_key": result_key,
"result_submit_api_status": status,
"result_submit_reconciled": bool(response.get("reconciled")),
}
)
save_task_state(task_id, state)
save_checkpoint(task_id, state, envelope, attempt)
audit(
"TASK_RESULT_SUBMITTED",
task_id=task_id,
attempt=attempt,
api_status=status,
result_idempotency_key=result_key,
commit_oid=result.get("governed", {}).get("commit_oid"),
replay_reconciled=bool(response.get("reconciled")),
)
except WorkerError as exc:
audit(
"TASK_FAILED",
task_id=task_id,
attempt=attempt,
error=exc.code,
phase=exc.phase,
details=exc.details,
)
if exc.code == "RECONCILIATION_REQUIRED":
state = {
"task_id": task_id,
"execution_digest": envelope.digest,
"phase": "reconciliation_required",
"attempt": attempt,
"job_id": envelope.job_id,
"details": exc.details,
}
save_task_state(task_id, state)
save_checkpoint(task_id, state, envelope, attempt)
audit(
"TASK_RECONCILIATION_REQUIRED",
task_id=task_id,
job_id=envelope.job_id,
details=exc.details,
)
return
failure_key = f"{task_id}-attempt-{attempt}-failure"
failure = {
"status": "FAILED",
"verification_state": "UNVERIFIED",
"task_id": task_id,
"attempt": attempt,
"execution_backend": "generic-online-agent-worker-v2",
"error": exc.code,
"phase": exc.phase,
"details": exc.details,
"direct_main_write": False,
"force_push": False,
"production_write": False,
"protected_ref_bypass": False,
"secret_material_exposed": False,
}
try:
status, response = submit_result_replay_safe(
self.agent,
task_id,
lease_token,
failure_key,
failure,
)
audit(
"TASK_FAILURE_RESULT_SUBMITTED",
task_id=task_id,
attempt=attempt,
api_status=status,
error=response.get("error"),
)
except Exception as submit_exc: # pragma: no cover - defensive runtime guard
audit(
"TASK_FAILURE_RESULT_SUBMIT_FAILED",
task_id=task_id,
attempt=attempt,
error=type(submit_exc).__name__,
)
finally:
heartbeat.close()
def once(self) -> int:
try:
tasks = self.agent.poll()
write_worker_liveness("POLL_OK", visible_count=len(tasks))
audit("TASK_POLL", visible_count=len(tasks))
except WorkerError as exc:
write_worker_liveness(
"POLL_FAILED",
error=exc.code,
)
audit("TASK_POLL_FAILED", error=exc.code, phase=exc.phase, details=exc.details)
return 10
dynamic_tasks: list[tuple[dict[str, Any], ExecutionEnvelope]] = []
for task in tasks:
try:
envelope, source = self.resolve_envelope(task)
except WorkerError as exc:
event = "TASK_SKIPPED_ENVELOPE_INVALID"
if exc.code in {
"TASK_SKIPPED_PROJECTION_MISSING",
"TASK_SKIPPED_POLICY_DENY",
"TASK_SKIPPED_PENDING_DETERMINISTIC_PLAN",
}:
event = exc.code
elif exc.code == "CAPABILITY_MISMATCH_DENY":
event = "TASK_SKIPPED_CAPABILITY_MISMATCH"
audit(event, task_id=task.get("task_id"), reason=exc.code, phase=exc.phase, details=exc.details)
continue
audit("TASK_VISIBLE_TO_WORKER_CLAIMABLE", task_id=task.get("task_id"), envelope_source=source, action=envelope.action)
if envelope.adapter not in WORKER_ALLOWED_ADAPTERS:
audit(
"TASK_SKIPPED_ADAPTER_MISMATCH",
task_id=task.get("task_id"),
adapter=envelope.adapter,
allowed_adapters=sorted(WORKER_ALLOWED_ADAPTERS),
)
continue
dynamic_tasks.append((task, envelope))
dynamic_tasks.sort(key=_claim_order)
for task, envelope in dynamic_tasks:
task_id = str(task.get("task_id") or "")
status, claim = self.agent.claim(task_id)
if status == 409:
audit("TASK_CLAIM_SKIPPED", task_id=task_id, api_status=status, error=claim.get("error"))
continue
if status != 200:
audit("TASK_CLAIM_FAILED", task_id=task_id, api_status=status, error=claim.get("error"))
continue
self.process_claim(task, claim, envelope)
return 0
return 0
def run_forever(self) -> int:
ensure_roots()
recover_stale_workspaces()
checkpoint_store()
write_worker_liveness("STARTING")
audit("WORKER_STARTED", task_agnostic=True, api_base=API_BASE, bridge_base=BRIDGE_BASE, adapter_names=["isolated-deterministic", "source_capture", "codex", "hermes"], runtime_only_credentials=True, isolated_executor=True, durable_checkpoint=True)
while True:
self.once()
time.sleep(POLL_SECONDS)
def main() -> int:
try:
token = load_agent_token()
supervisor = WorkerSupervisor(token)
if os.environ.get("AIOS_WORKER_ONCE") == "1":
ensure_roots()
return supervisor.once()
return supervisor.run_forever()
except WorkerError as exc:
audit("WORKER_START_FAILED", error=exc.code, phase=exc.phase, details=exc.details)
return 20
if __name__ == "__main__":
raise SystemExit(main())