Files
aios/active-runtime/app/trusted_evidence_providers.py
T

618 lines
22 KiB
Python

#!/usr/bin/env python3
"""Trusted evidence-provider executor (V2R2C2 P0 Round-3).
This module is the *formal trusted executor* that produces root-owned provider
receipts under each evidence-provider store. It is a real provider execution
path: for each evidence kind it executes the actual provider logic (pytest /
Gitea machine+API consistency / gitleaks), derives the result from real tool
output, and only then writes an ``aios.trusted-evidence-provider-receipt.v2``
receipt into the store. It never manufactures a bare ``{"passed": true}``:
every receipt carries a canonical content digest over a real result.
Security contract (matches TrustedProviderReceiptResolver):
- run as root (``os.geteuid() == 0``); store dirs and receipts are root-owned.
- store dir mode in ``{0700, 0750}``, receipt mode in ``{0600, 0640}``;
no group/world write; no symlink followed (O_NOFOLLOW path).
- receipt ref is relative and free of ``..``/``.``/absolute components.
- content_sha256 = canonical_sha256(result) (sort_keys, compact, utf-8).
- source_oid / tree_oid bound from the exact release source checkout.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import stat
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Mapping
PROVIDER_RECEIPT_SCHEMA = "aios.trusted-evidence-provider-receipt.v2"
PROVIDER_RECEIPT_VERSION = 2
_HEX40 = re.compile(r"^[0-9a-f]{40}$")
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
# Storage layout is resolved from the governed providers policy; these defaults
# mirror config/gitops/trusted-evidence-providers.json so the executor is usable
# before the policy is deployed, but the policy is authoritative at runtime.
POLICY_PATH = (
Path(__file__).resolve().parents[2]
/ "config"
/ "gitops"
/ "trusted-evidence-providers.json"
)
class TrustedProviderExecutionError(RuntimeError):
"""Fail-closed provider execution / receipt write error."""
def canonical_json_bytes(value: Any) -> bytes:
return json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
def canonical_sha256(value: Any) -> str:
return hashlib.sha256(canonical_json_bytes(value)).hexdigest()
def now_iso() -> str:
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
def _require_root() -> None:
if hasattr(os, "geteuid") and os.geteuid() != 0:
raise TrustedProviderExecutionError(
"PROVIDER_EXECUTOR_REQUIRES_ROOT (run via the governed root executor)"
)
def _validate_oid(value: str, field: str) -> str:
v = str(value or "").strip().lower()
if not _HEX40.fullmatch(v):
raise TrustedProviderExecutionError(f"{field.upper()}_INVALID")
return v
def _safe_relative_ref(receipt_ref: str) -> tuple[str, ...]:
ref = receipt_ref.strip()
if not ref or ref.startswith("/"):
raise TrustedProviderExecutionError("PROVIDER_RECEIPT_REF_ABSOLUTE_DENIED")
parts = ref.split("/")
if not parts or any(part in {"", ".", ".."} for part in parts):
raise TrustedProviderExecutionError("PROVIDER_RECEIPT_REF_TRAVERSAL_DENIED")
return tuple(parts)
def load_provider_policy(path: Path = POLICY_PATH) -> Mapping[str, Any]:
try:
decoded = json.loads(Path(path).read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise TrustedProviderExecutionError("PROVIDER_POLICY_UNAVAILABLE") from exc
if (
not isinstance(decoded, Mapping)
or decoded.get("schema") != "aios.trusted-evidence-providers.v1"
):
raise TrustedProviderExecutionError("PROVIDER_POLICY_INVALID")
return decoded
def provider_store_root(kind: str, policy_path: Path = POLICY_PATH) -> Path:
policy = load_provider_policy(policy_path)
providers = policy.get("providers")
if not isinstance(providers, Mapping) or kind not in providers:
raise TrustedProviderExecutionError(f"PROVIDER_POLICY_{kind.upper()}_NOT_FOUND")
raw = providers[kind]
if not isinstance(raw, Mapping):
raise TrustedProviderExecutionError("PROVIDER_POLICY_INVALID")
store_root = str(raw.get("store_root") or "").strip()
if not store_root:
raise TrustedProviderExecutionError("PROVIDER_STORE_ROOT_UNSET")
ec_path = os.environ.get("AIOS_EVIDENCE_PROVIDERS_ROOT")
if ec_path:
store_root = ec_path
root = Path(store_root).expanduser()
if not root.is_absolute():
raise TrustedProviderExecutionError("PROVIDER_STORE_ROOT_MUST_BE_ABSOLUTE")
return root
def _run(
command: list[str],
*,
cwd: Path | None = None,
timeout: int = 600,
extra_env: Mapping[str, str] | None = None,
) -> subprocess.CompletedProcess[str]:
env = dict(os.environ)
if extra_env:
env.update(extra_env)
try:
return subprocess.run(
command,
cwd=cwd,
text=True,
capture_output=True,
timeout=timeout,
env=env,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise TrustedProviderExecutionError("PROVIDER_TOOL_UNAVAILABLE") from exc
def _tests_env(repo_root: Path) -> dict[str, str]:
"""Forward source PYTHONPATH to pytest subprocess spawns.
Mirrors tools/baseline_v1_1/run_acceptance.py: subprocess-based tests (e.g.
job-safety process-crash recovery) import runtime modules from
active-runtime/app, so PYTHONPATH must be propagated or they fail with
ModuleNotFoundError. See docs/implementation-baseline-1.1/ACCEPTANCE_RESULTS.md.
"""
app = repo_root / "active-runtime" / "app"
return {"PYTHONPATH": os.pathsep.join(str(p) for p in (repo_root, app))}
def _write_receipt(root: Path, receipt_ref: str, payload: Mapping[str, Any], *, mode: int = 0o600) -> str:
"""Atomically write a root-owned receipt with fail-closed store checks."""
_require_root()
root_lstat = os.lstat(root)
if stat.S_ISLNK(root_lstat.st_mode):
raise TrustedProviderExecutionError("PROVIDER_STORE_SYMLINK_DENIED")
if not stat.S_ISDIR(root_lstat.st_mode):
raise TrustedProviderExecutionError("PROVIDER_STORE_NOT_DIRECTORY")
if root_lstat.st_uid != 0:
raise TrustedProviderExecutionError("PROVIDER_STORE_NOT_ROOT_OWNED")
if stat.S_IMODE(root_lstat.st_mode) & 0o022:
raise TrustedProviderExecutionError("PROVIDER_STORE_WRITABLE_DENIED")
parts = _safe_relative_ref(receipt_ref)
current = root
for part in parts[:-1]:
current = current / part
try:
st = os.lstat(current)
except FileNotFoundError:
# Provider executor owns the store: create missing root-owned dirs
# under the (already validated) store root, fail-closed if we cannot.
os.mkdir(current, 0o700)
os.chown(current, 0, 0)
os.chmod(current, 0o700)
st = os.lstat(current)
if stat.S_ISLNK(st.st_mode):
raise TrustedProviderExecutionError("PROVIDER_STORE_SYMLINK_DENIED")
if not stat.S_ISDIR(st.st_mode):
raise TrustedProviderExecutionError("PROVIDER_STORE_NOT_DIRECTORY")
if st.st_uid != 0:
raise TrustedProviderExecutionError("PROVIDER_STORE_NOT_ROOT_OWNED")
if stat.S_IMODE(st.st_mode) & 0o022:
raise TrustedProviderExecutionError("PROVIDER_STORE_WRITABLE_DENIED")
target = current / parts[-1]
raw = canonical_json_bytes(payload)
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0)
if hasattr(os, "O_NOFOLLOW"):
flags |= os.O_NOFOLLOW
try:
fd = os.open(target, flags, mode)
except OSError as exc:
raise TrustedProviderExecutionError("PROVIDER_RECEIPT_WRITE_DENIED") from exc
try:
os.write(fd, raw)
os.fsync(fd)
finally:
os.close(fd)
os.chown(target, 0, 0)
os.chmod(target, mode)
st = os.lstat(target)
if stat.S_IMODE(st.st_mode) & 0o022 or st.st_uid != 0:
raise TrustedProviderExecutionError("PROVIDER_RECEIPT_PERMISSION_DENIED")
return hashlib.sha256(raw).hexdigest()
def _build_receipt(
*,
kind: str,
provider_identity: str,
provider_operation: str,
operation_run_id: str,
result_reference: str,
result: Mapping[str, Any],
source_oid: str,
tree_oid: str,
store_id: str,
) -> Mapping[str, Any]:
return {
"schema": PROVIDER_RECEIPT_SCHEMA,
"version": PROVIDER_RECEIPT_VERSION,
"provider_identity": provider_identity,
"provider_operation": provider_operation,
"operation_run_id": operation_run_id,
"result_reference": result_reference,
"content_sha256": canonical_sha256(result),
"source_oid": source_oid,
"tree_oid": tree_oid,
"environment": "linux-isolated",
"issued_at": now_iso(),
"result": dict(result),
"authentication": {
"kind": "root-owned-store-v1",
"store_id": store_id,
},
}
# ---------------------------------------------------------------------------
# Provider executors (real tool execution; results derived from real output)
# ---------------------------------------------------------------------------
def _junit_totals(junit_path: Path) -> dict[str, int]:
import xml.etree.ElementTree as ET
if not junit_path.is_file():
raise TrustedProviderExecutionError("TESTS_JUNIT_MISSING")
root = ET.parse(junit_path).getroot()
# pytest emits <testsuites><testsuite tests=.. failures=.. />...</testsuites>;
# aggregate every <testsuite> child (never trust the outer <testsuites> attrib).
totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
suites = list(root.iter("testsuite"))
for suite in suites:
n = int(suite.attrib.get("tests", 0))
totals["tests"] += n
totals["failures"] += int(suite.attrib.get("failures", 0))
totals["errors"] += int(suite.attrib.get("errors", 0))
totals["skipped"] += int(suite.attrib.get("skipped", 0))
return totals
def run_tests_provider(
*,
repo_root: Path,
collection_inventory: Path,
junit_xml: Path,
source_oid: str,
tree_oid: str,
store_id: str = "tests",
provider_identity: str = "aios.tests-runner",
) -> tuple[Mapping[str, Any], str]:
"""Execute the real test suite, collect-JUnit, and produce a root receipt."""
# First prove POSIX modules are collected (never trust a bare exit code).
collect = _run(
["python3", "-m", "pytest", "--collect-only", "-q", str(repo_root / "tests")],
cwd=repo_root,
timeout=900,
extra_env=_tests_env(repo_root),
)
collected_names = {
line.strip()
for line in collect.stdout.splitlines()
if line.strip()
}
posix_probed = {
"fcntl": any("fcntl" in n for n in collected_names),
"grp": any("grp" in n for n in collected_names),
"so_peercrd": any("peercred" in n.lower() for n in collected_names),
"posix_mode": any("mode" in n.lower() for n in collected_names),
"symlink": any("symlink" in n.lower() for n in collected_names),
"signer_provisioning": any("signer_provisioning" in n.lower() for n in collected_names),
}
if junit_xml.parent and not junit_xml.parent.exists():
junit_xml.parent.mkdir(parents=True, exist_ok=True)
executed = _run(
[
"python3", "-m", "pytest",
"--junitxml=" + str(junit_xml),
"-q", str(repo_root / "tests"),
],
cwd=repo_root,
timeout=3600,
extra_env=_tests_env(repo_root),
)
totals = _junit_totals(junit_xml)
collection_inventory.write_text(
json.dumps({"collected": sorted(collected_names), "posix_probed": posix_probed}, sort_keys=True),
encoding="utf-8",
)
# Passed is only true when a real suite actually ran and was green: a
# vacuous 0-tests JUnit (unwritable junit, collection failure) must never
# report passed. pytest exit codes: 0=passed, 1=test failures, 2=interrupted,
# 3=internal error, 4=usage error, 5=no tests collected.
ran_real_tests = totals["tests"] > 0
green = totals["failures"] == 0 and totals["errors"] == 0
clean_exit = executed.returncode in (0, 5)
result = {
"passed": ran_real_tests and green and clean_exit,
"exit_code": executed.returncode,
"totals": totals,
"junit": junit_xml.name,
"collection_inventory": collection_inventory.name,
"posix_probed": posix_probed,
"run_kind": "full-test-suite",
"green": green,
"ran_real_tests": ran_real_tests,
}
return result, _receipt_ref_for(result, "tests")
def run_gitea_provider(
*,
gitea_api_url: str,
repository_identity: str,
token_file: Path,
source_oid: str,
tree_oid: str,
local_repo: Path,
store_id: str = "gitea",
provider_identity: str = "aios.gitea-machine-transport",
) -> tuple[Mapping[str, Any], str]:
"""Execute real Gitea machine+API consistency and produce a root receipt."""
import urllib.request
token = token_file.read_text(encoding="utf-8").strip()
def _api(path: str) -> dict[str, Any]:
req = urllib.request.Request(
gitea_api_url.rstrip("/") + path,
headers={
"Accept": "application/json",
"Authorization": f"token {token}",
"User-Agent": "aios-trusted-gitea-provider/1",
},
)
with urllib.request.urlopen(req, timeout=20) as resp:
return json.loads(resp.read().decode("utf-8"))
repository = _api(f"/repos/{repository_identity}")
full_name = str(repository.get("full_name") or "")
ref = _api(f"/repos/{repository_identity}/git/refs/heads/main")
items = ref if isinstance(ref, list) else [ref]
exact = [
i for i in items
if isinstance(i, dict) and i.get("ref") in {"refs/heads/main", "heads/main"}
]
local_main = _run(
["git", "-C", str(local_repo), "rev-parse", "refs/heads/main"], cwd=local_repo
)
remote_main = str((exact[0].get("object") or {}).get("sha") or "").lower() if len(exact) == 1 else ""
result = {
"consistent": (
full_name.lower() == repository_identity.lower()
and len(exact) == 1
and remote_main == source_oid.lower()
),
"repository": full_name,
"source_commit": source_oid,
"remote_ref_oid": remote_main,
"tree_oid": tree_oid,
"local_main_oid": local_main.stdout.strip() if local_main.returncode == 0 else "",
}
return result, _receipt_ref_for(result, "gitea")
def run_secret_scan_provider(
*,
gitleaks_bin: str,
target: Path,
source_oid: str,
tree_oid: str,
store_id: str = "secret_scan",
provider_identity: str = "aios.secret-scanner",
) -> tuple[Mapping[str, Any], str]:
"""Execute the real secret scanner (gitleaks) and produce a root receipt."""
if not Path(gitleaks_bin).is_file():
raise TrustedProviderExecutionError("SECRET_SCANNER_UNAVAILABLE")
# --no-git scans the working tree as a filesystem (does not require a .git
# dir), so an exported release checkout can be scanned faithfully.
scanned = _run(
[
gitleaks_bin, "detect",
"--source", str(target),
"--no-git",
"--no-banner",
"--exit-code", "0",
],
cwd=target,
timeout=1800,
)
warnings = 0
# gitleaks exits 1 when findings are present; with --exit-code 0 it returns 0
# but prints findings. Count lines that indicate a finding.
finding_lines = [
ln for ln in scanned.stdout.splitlines()
if ("Finding:" in ln and ":" in ln and "RuleID" not in ln)
] + [
ln for ln in scanned.stderr.splitlines()
if ("Finding:" in ln and ":" in ln and "RuleID" not in ln)
]
artifact_sha = None
if target.is_file():
artifact_sha = hashlib.sha256(target.read_bytes()).hexdigest()
# Passed only when the scanner actually ran cleanly (exit 0) AND found zero
# leaks. A scanner crash (non-zero exit) is never a pass.
clean_run = scanned.returncode == 0
result = {
"passed": clean_run and len(finding_lines) == 0,
"scanner": "gitleaks",
"exit_code": scanned.returncode,
"target": str(target),
"findings": len(finding_lines),
"artifact_sha256": artifact_sha,
"clean_run": clean_run,
}
return result, _receipt_ref_for(result, "secret_scan")
def _receipt_ref_for(result: Mapping[str, Any], kind: str) -> str:
run_id = hashlib.sha256(canonical_json_bytes(result)).hexdigest()[:16]
return f"run-{run_id}/receipt.json"
# ---------------------------------------------------------------------------
# Public orchestration: execute a provider kind and write the root receipt.
# ---------------------------------------------------------------------------
# Provider-kind wiring: each kind maps to a real executor and its governed
# identity / operation. The gate kind (Gate-W vs Gate-R) is read from the
# policy; a Gate-R kind is refused by the Gate-W producer boundary.
_PROVIDER_EXECUTORS: Mapping[str, Mapping[str, Any]] = {
"tests": {
"identity": "aios.tests-runner",
"operation": "tests.run",
"gate": "W",
},
"gitea": {
"identity": "aios.gitea-machine-transport",
"operation": "gitea.consistency",
"gate": "W",
},
"secret_scan": {
"identity": "aios.secret-scanner",
"operation": "secret.scan",
"gate": "W",
},
"canary": {
"identity": "aios.canary-gate",
"operation": "canary.run",
"gate": "R",
},
"rollback": {
"identity": "aios.rollback-controller",
"operation": "rollback.validate",
"gate": "R",
},
}
def _operation_for(kind: str) -> str:
entry = _PROVIDER_EXECUTORS.get(kind)
if entry is None:
raise TrustedProviderExecutionError("PROVIDER_KIND_UNKNOWN")
return entry["operation"]
def _run_id_for(kind: str) -> str:
return f"{kind}-{int(time.time())}"
def _run_provider_executor(
kind: str,
*,
repo_root: Path,
source_oid: str,
tree_oid: str,
) -> tuple[Mapping[str, Any], str]:
"""Run the real provider executor and return (result, receipt_ref).
Every provider derives its result from *real tool output* — never a
manufactured ``{"passed": true}`` and never an acceptance-script-invented
identity. The executor must be invoked under the governed root executor
(root-owned store); any non-executable kind fails closed.
"""
if kind == "tests":
return run_tests_provider(
repo_root=repo_root,
collection_inventory=repo_root / ".provider" / "tests-collection.json",
junit_xml=repo_root / ".provider" / "junit.xml",
source_oid=source_oid,
tree_oid=tree_oid,
)
if kind == "secret_scan":
return run_secret_scan_provider(
gitleaks_bin=os.environ.get("AIOS_GITLEAKS_BIN", "/usr/bin/gitleaks"),
target=repo_root,
source_oid=source_oid,
tree_oid=tree_oid,
)
if kind == "gitea":
gitea_api_url = os.environ.get("AIOS_GITEA_API_URL", "").strip()
token_file = os.environ.get("AIOS_GITEA_TOKEN_FILE", "").strip()
repository_identity = os.environ.get("AIOS_GITEA_REPOSITORY", "").strip()
if not (gitea_api_url and token_file and repository_identity):
raise TrustedProviderExecutionError(
"PROVIDER_MECHANISM_REQUIRED (gitea: API URL / token / repository "
"identity not provisioned; no real provider path available)"
)
return run_gitea_provider(
gitea_api_url=gitea_api_url,
repository_identity=repository_identity,
token_file=Path(token_file),
source_oid=source_oid,
tree_oid=tree_oid,
local_repo=repo_root,
)
raise TrustedProviderExecutionError(f"PROVIDER_MECHANISM_REQUIRED (kind={kind})")
def produce(kind: str, *, source_oid: str, tree_oid: str, receipt_ref: str, store_root: Path) -> str:
"""Execute real provider kind, write its root-owned receipt, return sha256.
Refuses Gate-R kinds at the Gate-W producer boundary, and refuses unless a
real provider execution path (an executor that derives its result from real
tool output under the governed root executor) exists for the kind.
"""
entry = _PROVIDER_EXECUTORS.get(kind)
if entry is None:
raise TrustedProviderExecutionError("PROVIDER_KIND_UNKNOWN")
if entry["gate"] != "W":
raise TrustedProviderExecutionError("GATE_R_PROVIDER_REFUSED_AT_GATE_W_BOUNDARY")
# Resolve a real source checkout for the executor to bind source/tree OID.
repo_root = Path(os.environ.get("AIOS_REPO_ROOT", store_root.parent / "checkout")).resolve()
result, auto_ref = _run_provider_executor(
kind,
repo_root=repo_root,
source_oid=source_oid,
tree_oid=tree_oid,
)
store_root.mkdir(mode=0o700, exist_ok=True)
os.chown(store_root, 0, 0)
os.chmod(store_root, 0o700)
return _write_receipt(store_root, receipt_ref, _build_receipt(
kind=kind,
provider_identity=entry["identity"],
provider_operation=_operation_for(kind),
operation_run_id=_run_id_for(kind),
result_reference=auto_ref,
result=result,
source_oid=source_oid,
tree_oid=tree_oid,
store_id=kind,
))
if __name__ == "__main__":
# CLI: trusted_evidence_providers.py <kind> --source-oid=X --tree-oid=Y --store-root=Z --receipt-ref=R
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("kind", choices=("tests", "gitea", "secret_scan"))
parser.add_argument("--source-oid", required=True)
parser.add_argument("--tree-oid", required=True)
parser.add_argument("--store-root", required=True)
parser.add_argument("--receipt-ref", required=True)
parser.add_argument("--repo-root", default=None, help="Real source checkout to execute against")
args = parser.parse_args()
if args.repo_root:
os.environ["AIOS_REPO_ROOT"] = args.repo_root
try:
digest = produce(
args.kind,
source_oid=_validate_oid(args.source_oid, "source_oid"),
tree_oid=_validate_oid(args.tree_oid, "tree_oid"),
receipt_ref=args.receipt_ref,
store_root=Path(args.store_root),
)
except TrustedProviderExecutionError as exc:
raise SystemExit(str(exc))
print(json.dumps({"receipt_sha256": digest}, sort_keys=True, separators=(",", ":")))