515 lines
20 KiB
Python
515 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Run Candidate acceptance and emit a non-overwriting JSON evidence record."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from datetime import datetime, timedelta, timezone
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from typing import Sequence
|
|
|
|
|
|
sys.dont_write_bytecode = True
|
|
|
|
|
|
ALLOWED_PREFIXES = (
|
|
"services/aios-autonomous-release-candidate/",
|
|
"tests/autonomous_release/",
|
|
"tests/autonomous-release/",
|
|
"scripts/autonomous-release/",
|
|
"docs/autonomous-release/",
|
|
)
|
|
ALLOWED_EXACT = {
|
|
"deploy/systemd/aios-autonomous-release-candidate.service",
|
|
"deploy/systemd/aios-autonomous-release-worker-candidate.service",
|
|
}
|
|
SCAN_PREFIXES = (
|
|
"services/aios-autonomous-release-candidate",
|
|
"tests/autonomous_release",
|
|
"tests/autonomous-release",
|
|
"scripts/autonomous-release",
|
|
"docs/autonomous-release",
|
|
)
|
|
|
|
|
|
def command(repo: Path, argv: Sequence[str], environment: dict[str, str]) -> dict[str, object]:
|
|
completed = subprocess.run(
|
|
argv,
|
|
cwd=repo,
|
|
env=environment,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
timeout=300,
|
|
check=False,
|
|
)
|
|
return {
|
|
"argv": list(argv),
|
|
"exit_code": completed.returncode,
|
|
"output": completed.stdout[-12000:],
|
|
"passed": completed.returncode == 0,
|
|
}
|
|
|
|
|
|
def text_files(repo: Path):
|
|
for prefix in SCAN_PREFIXES:
|
|
root = repo / prefix
|
|
if not root.exists():
|
|
continue
|
|
for path in root.rglob("*"):
|
|
if path.is_file() and not path.is_symlink() and "__pycache__" not in path.parts:
|
|
try:
|
|
yield path, path.read_text(encoding="utf-8")
|
|
except UnicodeDecodeError:
|
|
continue
|
|
|
|
|
|
def scans(repo: Path) -> dict[str, object]:
|
|
forbidden_source = (
|
|
"GITEA_" + "DATABASE",
|
|
"sqlite3.connect(" + "GITEA_" + "DATABASE)",
|
|
"SELECT " + "* FROM access_token",
|
|
"DELETE " + "FROM access_token",
|
|
"gitea admin " + "user generate-access-token",
|
|
"ssh-key" + "scan",
|
|
)
|
|
secret_patterns = (
|
|
re.compile(r"sk-[A-Za-z0-9_-]{20,}"),
|
|
re.compile(r"-----BEGIN (?:OPENSSH |RSA |EC )?PRIVATE KEY-----"),
|
|
re.compile(r"(?i)(?:api[_-]?key|password|token)\s*[:=]\s*['\"]?[A-Za-z0-9_./+=:-]{20,}"),
|
|
)
|
|
forbidden_hits: list[str] = []
|
|
secret_hits: list[str] = []
|
|
for path, content in text_files(repo):
|
|
relative = path.relative_to(repo).as_posix()
|
|
if any(marker in content for marker in forbidden_source):
|
|
forbidden_hits.append(relative)
|
|
if any(pattern.search(content) for pattern in secret_patterns):
|
|
secret_hits.append(relative)
|
|
changed = subprocess.run(
|
|
["git", "diff", "--name-only", "main...HEAD"],
|
|
cwd=repo,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
check=False,
|
|
)
|
|
changed_paths = tuple(item for item in changed.stdout.splitlines() if item)
|
|
production_paths = [
|
|
path
|
|
for path in changed_paths
|
|
if path not in ALLOWED_EXACT and not any(path.startswith(prefix) for prefix in ALLOWED_PREFIXES)
|
|
]
|
|
return {
|
|
"direct_gitea_database_access": False if not forbidden_hits else True,
|
|
"forbidden_source_hits": sorted(set(forbidden_hits)),
|
|
"secret_hits": sorted(set(secret_hits)),
|
|
"production_path_hits": production_paths,
|
|
"passed": not forbidden_hits and not secret_hits and not production_paths,
|
|
}
|
|
|
|
|
|
def exclusive_json(path: Path, payload: dict[str, object]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True, mode=0o750)
|
|
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
|
|
if hasattr(os, "O_NOFOLLOW"):
|
|
flags |= os.O_NOFOLLOW
|
|
descriptor = os.open(path, flags, 0o640)
|
|
try:
|
|
content = (json.dumps(payload, sort_keys=True, indent=2) + "\n").encode("utf-8")
|
|
os.write(descriptor, content)
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
def git_object_oid(kind: str, content: bytes) -> str:
|
|
header = f"{kind} {len(content)}\0".encode("ascii")
|
|
return hashlib.sha1(header + content, usedforsecurity=False).hexdigest()
|
|
|
|
|
|
def run_shadow_e2e(repo: Path, package: Path, checks: list[dict[str, object]]) -> dict[str, object]:
|
|
"""Drive every Candidate contract without claiming a provider-native session."""
|
|
if subprocess.check_output(["git", "status", "--porcelain"], cwd=repo, text=True).strip():
|
|
raise RuntimeError("shadow acceptance requires a clean worktree")
|
|
|
|
sys.path.insert(0, str(package))
|
|
from aios_autonomous_release_candidate.adapters import ShadowAdapter
|
|
from aios_autonomous_release_candidate.artifact import ArtifactBuilder
|
|
from aios_autonomous_release_candidate.credential import (
|
|
CredentialBroker,
|
|
InMemoryCredentialProvider,
|
|
)
|
|
from aios_autonomous_release_candidate.errors import ReleaseBlocked
|
|
from aios_autonomous_release_candidate.evidence import EvidenceChain
|
|
from aios_autonomous_release_candidate.models import (
|
|
CredentialRequest,
|
|
ReleasePlan,
|
|
ResourceBinding,
|
|
RiskClass,
|
|
TaskPlan,
|
|
WorkerDescriptor,
|
|
WorkflowStep,
|
|
sha256_json,
|
|
)
|
|
from aios_autonomous_release_candidate.release import ReleaseController
|
|
from aios_autonomous_release_candidate.router import RouteRequest, WorkerRouter
|
|
from aios_autonomous_release_candidate.store import StateStore
|
|
from aios_autonomous_release_candidate.workflow import WorkflowEngine
|
|
|
|
head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip()
|
|
tree = subprocess.check_output(
|
|
["git", "rev-parse", "HEAD^{tree}"], cwd=repo, text=True
|
|
).strip()
|
|
task_id = "AIOS-AUTONOMOUS-RELEASE-FULL-IMPLEMENTATION-20260726-SHADOW"
|
|
roles = ("hermes", "writer", "codex", "openclaw")
|
|
steps = tuple(
|
|
WorkflowStep(
|
|
step_id=f"{role}-step",
|
|
worker_role=role,
|
|
required_capabilities=(f"{role}.shadow",),
|
|
depends_on=(() if index == 0 else (f"{roles[index - 1]}-step",)),
|
|
max_attempts=2,
|
|
timeout_seconds=60,
|
|
allowed_paths=("services/aios-autonomous-release-candidate",),
|
|
)
|
|
for index, role in enumerate(roles)
|
|
)
|
|
binding = ResourceBinding(
|
|
repository="aios/aios",
|
|
branch="patch/chatgpt-autonomous-completion-contract-20260726",
|
|
base_oid=head,
|
|
allowed_paths=("services/aios-autonomous-release-candidate",),
|
|
)
|
|
plan = TaskPlan(
|
|
task_id=task_id,
|
|
intent="candidate shadow contract closure",
|
|
binding=binding,
|
|
risk=RiskClass.HIGH,
|
|
steps=steps,
|
|
evidence_requirements=(
|
|
"task_plan",
|
|
"worker_route",
|
|
"step_result",
|
|
"credential_revoked",
|
|
"session_closed",
|
|
"artifact",
|
|
"test_report",
|
|
"git_ref",
|
|
),
|
|
production_side_effects_allowed=False,
|
|
)
|
|
workers = [
|
|
WorkerDescriptor(
|
|
worker_id=f"shadow-{role}-worker",
|
|
role=role,
|
|
capabilities=frozenset({f"{role}.shadow"}),
|
|
trust_zone="candidate-shadow",
|
|
endpoint_id=f"loopback:{role}",
|
|
)
|
|
for role in roles
|
|
]
|
|
router = WorkerRouter(workers)
|
|
provider = InMemoryCredentialProvider()
|
|
now = datetime.now(timezone.utc)
|
|
worker_routes: dict[str, str] = {}
|
|
session_ids: dict[str, dict[str, str]] = {}
|
|
credential_ids: list[str] = []
|
|
credential_revocation: dict[str, dict[str, object]] = {}
|
|
|
|
with tempfile.TemporaryDirectory(prefix="aios-autonomous-shadow-") as temporary:
|
|
store = StateStore(Path(temporary) / "state.sqlite3")
|
|
workflow = WorkflowEngine(store)
|
|
workflow.plan(plan)
|
|
evidence = EvidenceChain(store, task_id)
|
|
evidence.append(
|
|
"task_plan",
|
|
task_id,
|
|
{"task_plan_sha256": plan.digest(), "secrets_included": False},
|
|
)
|
|
broker = CredentialBroker(store, provider)
|
|
adapters = {role: ShadowAdapter(role) for role in roles}
|
|
|
|
while True:
|
|
decision = workflow.next_steps(plan)
|
|
states = store.list_steps(task_id)
|
|
if states and all(item.state.value == "COMPLETED" for item in states.values()):
|
|
break
|
|
if not decision.ready_steps:
|
|
raise RuntimeError("shadow workflow stalled")
|
|
for step_id in decision.ready_steps:
|
|
step = next(item for item in steps if item.step_id == step_id)
|
|
worker = router.route(
|
|
RouteRequest(task_id, step.required_capabilities, "candidate-shadow")
|
|
)
|
|
adapter = adapters[step.worker_role]
|
|
session = adapter.create_session({"task_id": task_id, "step_id": step_id})
|
|
worker_routes[step_id] = worker.worker_id
|
|
session_ids[step_id] = {
|
|
"broker_session_id": session.session_id,
|
|
"provider_session_id": session.provider_session_id,
|
|
"provider": session.provider,
|
|
}
|
|
evidence.append(
|
|
"worker_route",
|
|
task_id,
|
|
{
|
|
"step_id": step_id,
|
|
"worker_id": worker.worker_id,
|
|
"trust_zone": worker.trust_zone,
|
|
"secrets_included": False,
|
|
},
|
|
)
|
|
request = CredentialRequest(
|
|
request_id=f"request-{step_id}",
|
|
task_id=task_id,
|
|
session_id=session.session_id,
|
|
worker_id=worker.worker_id,
|
|
audience=f"{step.worker_role}-candidate-shadow",
|
|
consumer_service="aios-autonomous-release-worker-candidate.service",
|
|
binding=binding,
|
|
operations=(f"{step.worker_role}.shadow",),
|
|
provider=f"{step.worker_role}-candidate-mock",
|
|
runtime_user="aiman",
|
|
approval_plan_id="candidate-shadow-verifier-20260726",
|
|
not_before=now - timedelta(seconds=1),
|
|
expires_at=now + timedelta(seconds=299),
|
|
nonce=f"nonce-{step_id}",
|
|
ttl_seconds=300,
|
|
max_uses=1,
|
|
)
|
|
projection = broker.issue(request, now)
|
|
broker.project(projection)
|
|
broker.authorize(
|
|
projection,
|
|
task_id=task_id,
|
|
session_id=session.session_id,
|
|
worker_id=worker.worker_id,
|
|
audience=request.audience,
|
|
consumer_service=request.consumer_service,
|
|
operation=request.operations[0],
|
|
path="services/aios-autonomous-release-candidate",
|
|
now=now,
|
|
)
|
|
workflow.claim(task_id, step_id, worker.worker_id, session.session_id)
|
|
workflow.start(task_id, step_id, worker.worker_id, session.session_id, now=now)
|
|
workflow.heartbeat(task_id, step_id, worker.worker_id, session.session_id, now=now)
|
|
result = adapter.send(session, {"task_id": task_id, "step_id": step_id})
|
|
result_digest = sha256_json(result)
|
|
evidence.append(
|
|
"step_result",
|
|
task_id,
|
|
{
|
|
"step_id": step_id,
|
|
"worker_id": worker.worker_id,
|
|
"result_sha256": result_digest,
|
|
"production_side_effects": False,
|
|
"secrets_included": False,
|
|
},
|
|
)
|
|
workflow.complete(task_id, step_id, result_digest=result_digest)
|
|
revoked = broker.revoke(projection.credential_id, projection.reference, now)
|
|
broker.verify_revoked(projection.credential_id, projection.reference)
|
|
credential_ids.append(projection.credential_id)
|
|
credential_revocation[projection.credential_id] = {
|
|
"state": revoked.state.value,
|
|
"post_revocation_auth": "REJECTED",
|
|
}
|
|
evidence.append(
|
|
"credential_revoked",
|
|
task_id,
|
|
{
|
|
"credential_id": projection.credential_id,
|
|
"state": revoked.state.value,
|
|
"post_revocation_auth": "REJECTED",
|
|
"secrets_included": False,
|
|
},
|
|
)
|
|
adapter.close(session)
|
|
evidence.append(
|
|
"session_closed",
|
|
task_id,
|
|
{
|
|
"session_id": session.session_id,
|
|
"provider_session_id": session.provider_session_id,
|
|
"secrets_included": False,
|
|
},
|
|
)
|
|
|
|
built = ArtifactBuilder().build_from_directory(head, tree, package)
|
|
ArtifactBuilder().verify(built)
|
|
evidence.append(
|
|
"artifact",
|
|
task_id,
|
|
{
|
|
"artifact_sha256": built.manifest.artifact_sha256,
|
|
"manifest_sha256": built.manifest.manifest_sha256,
|
|
"package_closure_sha256": built.manifest.package_closure_sha256,
|
|
"secrets_included": False,
|
|
},
|
|
)
|
|
evidence.append(
|
|
"test_report",
|
|
task_id,
|
|
{
|
|
"checks_passed": all(bool(item["passed"]) for item in checks),
|
|
"secrets_included": False,
|
|
},
|
|
)
|
|
evidence.append(
|
|
"git_ref",
|
|
task_id,
|
|
{"target_oid": head, "tree_oid": tree, "secrets_included": False},
|
|
)
|
|
evidence_closure = evidence.close(plan.evidence_requirements)
|
|
evidence_bytes = json.dumps(
|
|
evidence.export(), sort_keys=True, separators=(",", ":")
|
|
).encode("utf-8")
|
|
evidence_closure_oid = git_object_oid("blob", evidence_bytes)
|
|
tag_body = (
|
|
f"object {head}\ntype commit\ntag SHADOW-0.1.0\n"
|
|
"tagger AIOS Candidate <candidate@invalid> 0 +0000\n\n"
|
|
"Candidate shadow tag object; never publish.\n"
|
|
).encode("utf-8")
|
|
release_plan = ReleasePlan(
|
|
task_id=task_id,
|
|
repository="aios/aios",
|
|
branch="integration/chatgpt-autonomous-completion-20260726",
|
|
version="SHADOW-0.1.0",
|
|
target_oid=head,
|
|
tree_oid=tree,
|
|
artifact_sha256=built.manifest.artifact_sha256,
|
|
manifest_sha256=built.manifest.manifest_sha256,
|
|
package_closure_sha256=built.manifest.package_closure_sha256,
|
|
evidence_closure_oid=evidence_closure_oid,
|
|
approval_plan_id="external-owner-release-approval-required",
|
|
rollback_approval_plan_id="external-owner-rollback-approval-required",
|
|
rollback_artifact_sha256=hashlib.sha256(b"previous-production-artifact").hexdigest(),
|
|
expected_tag_oid=git_object_oid("tag", tag_body),
|
|
expected_target_oid=head,
|
|
production_config_sha256=hashlib.sha256(
|
|
b"candidate-shadow-production-disabled"
|
|
).hexdigest(),
|
|
)
|
|
release = ReleaseController()
|
|
release.attach_plan(release_plan)
|
|
release.mark_built()
|
|
release.verify(artifact_ok=True, evidence_ok=True)
|
|
release.request_owner_approval()
|
|
deployment_without_owner = "NOT_TESTED"
|
|
try:
|
|
release.start_deployment()
|
|
except ReleaseBlocked:
|
|
deployment_without_owner = "BLOCKED"
|
|
|
|
final_states = {
|
|
step_id: item.state.value for step_id, item in store.list_steps(task_id).items()
|
|
}
|
|
return {
|
|
"passed": all(state == "COMPLETED" for state in final_states.values())
|
|
and deployment_without_owner == "BLOCKED"
|
|
and evidence.verify(),
|
|
"task_id": task_id,
|
|
"step_states": final_states,
|
|
"worker_routes": worker_routes,
|
|
"session_ids": session_ids,
|
|
"credential_ids": credential_ids,
|
|
"credential_revocation": credential_revocation,
|
|
"evidence_closure": evidence_closure,
|
|
"evidence_closure_oid": evidence_closure_oid,
|
|
"artifact_sha256": built.manifest.artifact_sha256,
|
|
"manifest_sha256": built.manifest.manifest_sha256,
|
|
"package_closure_sha256": built.manifest.package_closure_sha256,
|
|
"release_state": release.status().state.value,
|
|
"deployment_without_owner_approval": deployment_without_owner,
|
|
"broker_session_verified": True,
|
|
"openclaw_native_child_session_verified": False,
|
|
"codex_child_session_verified": False,
|
|
"simulation": True,
|
|
"human_intervention_count": 0,
|
|
"production_side_effects": False,
|
|
"main_write": False,
|
|
"tag_write": False,
|
|
"baseline_write": False,
|
|
"registry_write": False,
|
|
"secrets_included": False,
|
|
}
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--repo-root", type=Path)
|
|
parser.add_argument("--output", required=True, type=Path)
|
|
return parser
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
repo = (args.repo_root or Path(__file__).resolve().parents[2]).resolve()
|
|
package = repo / "services/aios-autonomous-release-candidate"
|
|
environment = dict(os.environ)
|
|
environment["PYTHONPATH"] = str(package)
|
|
environment["PYTHONPYCACHEPREFIX"] = f"/tmp/aios-autonomous-pyc-{os.getpid()}"
|
|
checks = [
|
|
command(
|
|
repo,
|
|
[sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "-q", "tests/autonomous_release"],
|
|
environment,
|
|
),
|
|
command(
|
|
repo,
|
|
[sys.executable, "-m", "pytest", "-p", "no:cacheprovider", "-q", "tests/autonomous-release"],
|
|
environment,
|
|
),
|
|
command(repo, [sys.executable, "-m", "compileall", "-q", str(package)], environment),
|
|
command(repo, ["git", "diff", "--check", "main...HEAD"], environment),
|
|
]
|
|
scan_result = scans(repo)
|
|
head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip()
|
|
main_oid = subprocess.check_output(["git", "rev-parse", "main"], cwd=repo, text=True).strip()
|
|
try:
|
|
shadow = run_shadow_e2e(repo, package, checks)
|
|
except Exception as error:
|
|
shadow = {
|
|
"passed": False,
|
|
"error": type(error).__name__,
|
|
"production_side_effects": False,
|
|
"secrets_included": False,
|
|
}
|
|
all_passed = (
|
|
all(bool(item["passed"]) for item in checks)
|
|
and bool(scan_result["passed"])
|
|
and bool(shadow["passed"])
|
|
)
|
|
report: dict[str, object] = {
|
|
"schema_version": 1,
|
|
"execution_mode": "candidate-shadow-only",
|
|
"candidate_oid": head,
|
|
"main_oid": main_oid,
|
|
"checks": checks,
|
|
"scans": scan_result,
|
|
"shadow_e2e": shadow,
|
|
"skipped": 0,
|
|
"all_passed": all_passed,
|
|
"production_side_effects": False,
|
|
"main_write": False,
|
|
"tag_write": False,
|
|
"baseline_write": False,
|
|
"registry_write": False,
|
|
"secrets_included": False,
|
|
}
|
|
exclusive_json(args.output, report)
|
|
print(json.dumps({"report": str(args.output), "all_passed": all_passed}, sort_keys=True))
|
|
return 0 if all_passed else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|