565 lines
18 KiB
Python
565 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Execution-readiness adapters and the fail-closed Shadow controller service."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import dataclasses
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from http import HTTPStatus
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Mapping, Protocol
|
|
from urllib.parse import parse_qs, urlparse
|
|
|
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import (
|
|
Ed25519PrivateKey,
|
|
)
|
|
|
|
from silent_release_controller import _sanitize, utc_now
|
|
from silent_release_hardening import (
|
|
BuilderAttestation,
|
|
CapabilityAuthorizationError,
|
|
ReleaseRecoveryWorker,
|
|
SignedApprovalVerifier,
|
|
SignedReleasePlanApproval,
|
|
SilentReleaseActionService,
|
|
TrustedCandidateBundle,
|
|
_canonical_json,
|
|
_durable_json,
|
|
_safe_child,
|
|
)
|
|
|
|
|
|
class BuilderSigner(Protocol):
|
|
@property
|
|
def key_id(self) -> str:
|
|
...
|
|
|
|
def sign(self, payload: bytes) -> bytes:
|
|
...
|
|
|
|
|
|
class LocalTestBuilderSigner:
|
|
"""Ephemeral in-memory signer. It is forbidden for production execution."""
|
|
|
|
test_only = True
|
|
|
|
def __init__(self, key_id: str, private_key: Ed25519PrivateKey):
|
|
self._key_id = str(key_id)
|
|
self._private_key = private_key
|
|
|
|
@property
|
|
def key_id(self) -> str:
|
|
return self._key_id
|
|
|
|
@classmethod
|
|
def generate(
|
|
cls, key_id: str = "shadow-builder-test-key"
|
|
) -> "LocalTestBuilderSigner":
|
|
return cls(key_id, Ed25519PrivateKey.generate())
|
|
|
|
@property
|
|
def public_key(self):
|
|
return self._private_key.public_key()
|
|
|
|
def sign(self, payload: bytes) -> bytes:
|
|
return self._private_key.sign(payload)
|
|
|
|
|
|
class ExternalKmsBuilderSigner:
|
|
"""Signer facade whose private operation remains in KMS/HSM."""
|
|
|
|
def __init__(
|
|
self,
|
|
key_id: str,
|
|
credential_reference: str,
|
|
sign_callback: Callable[[str, bytes], bytes],
|
|
):
|
|
if not str(credential_reference).startswith(
|
|
("kms://", "hsm://", "secret-manager://")
|
|
):
|
|
raise ValueError("production builder requires a KMS/HSM reference")
|
|
self._key_id = str(key_id)
|
|
self.credential_reference = str(credential_reference)
|
|
self._sign_callback = sign_callback
|
|
|
|
@property
|
|
def key_id(self) -> str:
|
|
return self._key_id
|
|
|
|
def sign(self, payload: bytes) -> bytes:
|
|
return bytes(self._sign_callback(self.credential_reference, payload))
|
|
|
|
|
|
class SignedTrustedBuilderAdapter:
|
|
"""Signs evidence produced by a trusted collector, never caller evidence."""
|
|
|
|
def __init__(
|
|
self,
|
|
collector: Callable[[Mapping[str, Any]], TrustedCandidateBundle],
|
|
signer: BuilderSigner,
|
|
):
|
|
self.collector = collector
|
|
self.signer = signer
|
|
|
|
def build(self, intent: Mapping[str, Any]) -> TrustedCandidateBundle:
|
|
forbidden = {
|
|
"artifact_sha256",
|
|
"canary",
|
|
"checks",
|
|
"manifest_sha256",
|
|
"tests",
|
|
"tree_oid",
|
|
}
|
|
if forbidden.intersection(intent):
|
|
raise CapabilityAuthorizationError(
|
|
"trusted builder rejects caller-supplied evidence"
|
|
)
|
|
unsigned = self.collector(dict(intent))
|
|
if unsigned.attestation is not None:
|
|
raise CapabilityAuthorizationError(
|
|
"collector cannot self-attest its evidence"
|
|
)
|
|
candidate = unsigned.candidate
|
|
test_sha = hashlib.sha256(
|
|
_canonical_json(_sanitize(candidate.tests))
|
|
).hexdigest()
|
|
canary_sha = hashlib.sha256(
|
|
_canonical_json(_sanitize(candidate.canary))
|
|
).hexdigest()
|
|
attestation_id = "builder-attestation-" + hashlib.sha256(
|
|
(
|
|
candidate.candidate_id
|
|
+ "\0"
|
|
+ candidate.source_commit
|
|
+ "\0"
|
|
+ candidate.artifact_sha256
|
|
).encode("utf-8")
|
|
).hexdigest()[:24]
|
|
attestation = BuilderAttestation(
|
|
builder_attestation_id=attestation_id,
|
|
builder_key_id=self.signer.key_id,
|
|
source_commit=candidate.source_commit,
|
|
tree_oid=candidate.tree_oid,
|
|
artifact_sha256=candidate.artifact_sha256,
|
|
manifest_sha256=candidate.manifest_sha256,
|
|
package_closure_sha256=(
|
|
unsigned.release_plan.deployment.package_closure_sha256
|
|
),
|
|
test_evidence_sha256=test_sha,
|
|
canary_evidence_sha256=canary_sha,
|
|
issued_at=utc_now(),
|
|
signature="",
|
|
)
|
|
attestation = dataclasses.replace(
|
|
attestation,
|
|
signature=base64.b64encode(
|
|
self.signer.sign(attestation.canonical_bytes())
|
|
).decode("ascii"),
|
|
)
|
|
return dataclasses.replace(unsigned, attestation=attestation)
|
|
|
|
|
|
class ApprovalBackendAdapter(Protocol):
|
|
@property
|
|
def available(self) -> bool:
|
|
...
|
|
|
|
def verify(
|
|
self,
|
|
approval: SignedReleasePlanApproval,
|
|
*,
|
|
candidate,
|
|
release_plan,
|
|
policy_sha256: str,
|
|
) -> SignedReleasePlanApproval:
|
|
...
|
|
|
|
|
|
class LocalTestApprovalSigner:
|
|
"""Test-only approval signer used solely for Shadow acceptance."""
|
|
|
|
test_only = True
|
|
|
|
def __init__(self, key_id: str, private_key: Ed25519PrivateKey):
|
|
self.key_id = str(key_id)
|
|
self.private_key = private_key
|
|
|
|
@classmethod
|
|
def generate(
|
|
cls, key_id: str = "shadow-owner-test-key"
|
|
) -> "LocalTestApprovalSigner":
|
|
return cls(key_id, Ed25519PrivateKey.generate())
|
|
|
|
def sign(
|
|
self, approval: SignedReleasePlanApproval
|
|
) -> SignedReleasePlanApproval:
|
|
unsigned = dataclasses.replace(
|
|
approval, key_id=self.key_id, signature=""
|
|
)
|
|
return dataclasses.replace(
|
|
unsigned,
|
|
signature=base64.b64encode(
|
|
self.private_key.sign(unsigned.canonical_bytes())
|
|
).decode("ascii"),
|
|
)
|
|
|
|
|
|
class ProductionApprovalBackendAdapter:
|
|
"""Verification-only production adapter configured by a secret/KMS ref."""
|
|
|
|
def __init__(
|
|
self,
|
|
verifier: SignedApprovalVerifier,
|
|
credential_reference: str,
|
|
):
|
|
if not str(credential_reference).startswith(
|
|
("kms://", "hsm://", "secret-manager://")
|
|
):
|
|
raise ValueError(
|
|
"production approval backend requires a KMS/secret reference"
|
|
)
|
|
self.verifier = verifier
|
|
self.credential_reference = str(credential_reference)
|
|
|
|
@property
|
|
def available(self) -> bool:
|
|
return True
|
|
|
|
def verify(
|
|
self,
|
|
approval: SignedReleasePlanApproval,
|
|
*,
|
|
candidate,
|
|
release_plan,
|
|
policy_sha256: str,
|
|
) -> SignedReleasePlanApproval:
|
|
return self.verifier.verify_release_plan(
|
|
approval, candidate, release_plan, policy_sha256
|
|
)
|
|
|
|
|
|
class UnavailableProductionApprovalBackend:
|
|
credential_reference = None
|
|
|
|
@property
|
|
def available(self) -> bool:
|
|
return False
|
|
|
|
def verify(self, *args, **kwargs):
|
|
raise CapabilityAuthorizationError(
|
|
"production approval backend is not configured"
|
|
)
|
|
|
|
|
|
class LeaderLock:
|
|
def __init__(self, path: Path):
|
|
self.path = Path(path)
|
|
self.handle = None
|
|
|
|
def __enter__(self):
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
os.chmod(self.path.parent, 0o700)
|
|
self.handle = self.path.open("a+", encoding="utf-8")
|
|
os.chmod(self.path, 0o600)
|
|
try:
|
|
import fcntl
|
|
|
|
fcntl.flock(
|
|
self.handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB
|
|
)
|
|
except BlockingIOError as exc:
|
|
self.handle.close()
|
|
self.handle = None
|
|
raise RuntimeError("silent release leader is already active") from exc
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, tb):
|
|
if self.handle is not None:
|
|
try:
|
|
import fcntl
|
|
|
|
fcntl.flock(self.handle.fileno(), fcntl.LOCK_UN)
|
|
finally:
|
|
self.handle.close()
|
|
self.handle = None
|
|
|
|
|
|
class PersistentSilentReleaseController:
|
|
"""Idempotent main-event controller with durable startup/recovery reports."""
|
|
|
|
def __init__(
|
|
self,
|
|
root: Path,
|
|
service: SilentReleaseActionService,
|
|
*,
|
|
main_probe: Callable[[], str],
|
|
recovery_worker: ReleaseRecoveryWorker | None = None,
|
|
):
|
|
self.root = Path(root)
|
|
self.service = service
|
|
self.main_probe = main_probe
|
|
self.recovery_worker = recovery_worker
|
|
self.leader_path = self.root / "controller.leader.lock"
|
|
self.status_path = self.root / "controller-status.json"
|
|
|
|
def startup_verify(self) -> dict[str, Any]:
|
|
checked = 0
|
|
failed = 0
|
|
for journal in sorted(self.service.sagas.root.glob("*.events.jsonl")):
|
|
candidate_id = journal.name[: -len(".events.jsonl")]
|
|
saga = self.service.sagas.load(candidate_id)
|
|
checked += 1
|
|
if not saga.journal_integrity:
|
|
failed += 1
|
|
status = {
|
|
"checked_sagas": checked,
|
|
"execution_enabled": self.service.execution_enabled,
|
|
"failed_closed": failed,
|
|
"shadow_mode": not self.service.execution_enabled,
|
|
"side_effects_executed": False,
|
|
"status": "READY" if failed == 0 else "FAIL_CLOSED",
|
|
}
|
|
_durable_json(self.status_path, status)
|
|
return status
|
|
|
|
def run_once(self) -> dict[str, Any]:
|
|
with LeaderLock(self.leader_path):
|
|
startup = self.startup_verify()
|
|
if startup["status"] != "READY":
|
|
result = {
|
|
"execution_enabled": self.service.execution_enabled,
|
|
"intent": None,
|
|
"notification": None,
|
|
"recovery": None,
|
|
"shadow_mode": not self.service.execution_enabled,
|
|
"side_effects_executed": False,
|
|
"startup": startup,
|
|
"status": "FAIL_CLOSED",
|
|
}
|
|
_durable_json(self.status_path, result)
|
|
return _sanitize(result)
|
|
source_commit = str(self.main_probe())
|
|
event_id = f"main-{source_commit}"
|
|
intent = self.service.create_release_intent(
|
|
{"event_id": event_id, "source_commit": source_commit}
|
|
)
|
|
notification = None
|
|
if intent.get("candidate_id"):
|
|
notification = self.service.ensure_owner_notification(
|
|
str(intent["candidate_id"])
|
|
)
|
|
recovery = (
|
|
self.recovery_worker.run_once()
|
|
if self.recovery_worker is not None
|
|
else self.service.get_recovery_status()
|
|
)
|
|
result = {
|
|
"execution_enabled": self.service.execution_enabled,
|
|
"intent": intent,
|
|
"notification": notification,
|
|
"recovery": recovery,
|
|
"shadow_mode": not self.service.execution_enabled,
|
|
"side_effects_executed": False,
|
|
"startup": startup,
|
|
}
|
|
_durable_json(self.status_path, result)
|
|
return _sanitize(result)
|
|
|
|
|
|
class ShadowActionHandler(BaseHTTPRequestHandler):
|
|
service: SilentReleaseActionService
|
|
controller: PersistentSilentReleaseController
|
|
|
|
def log_message(self, fmt, *args):
|
|
return
|
|
|
|
def _json(self, payload: Mapping[str, Any], status=HTTPStatus.OK):
|
|
body = json.dumps(_sanitize(dict(payload)), sort_keys=True).encode()
|
|
self.send_response(int(status))
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def _body(self) -> dict[str, Any]:
|
|
size = int(self.headers.get("Content-Length", "0"))
|
|
if size > 1024 * 1024:
|
|
raise ValueError("request body is too large")
|
|
return json.loads(self.rfile.read(size) or b"{}")
|
|
|
|
def do_GET(self):
|
|
parsed = urlparse(self.path)
|
|
query = parse_qs(parsed.query)
|
|
candidate_id = str((query.get("candidate_id") or [""])[0])
|
|
try:
|
|
if parsed.path == "/healthz":
|
|
return self._json(
|
|
{
|
|
"execution_enabled": False,
|
|
"shadow_mode": True,
|
|
"side_effects_executed": False,
|
|
"status": "ok",
|
|
}
|
|
)
|
|
routes = {
|
|
"/getReleaseCandidate": self.service.get_release_candidate,
|
|
"/getReleasePlan": self.service.get_release_plan,
|
|
"/getReleaseAcceptance": self.service.get_acceptance,
|
|
"/getReleaseSaga": self.service.get_release_saga,
|
|
"/getCapabilityStatus": self.service.get_capability_status,
|
|
}
|
|
if parsed.path in routes:
|
|
return self._json(routes[parsed.path](candidate_id))
|
|
if parsed.path == "/getRecoveryStatus":
|
|
return self._json(self.service.get_recovery_status())
|
|
self._json({"error": "not_found"}, HTTPStatus.NOT_FOUND)
|
|
except (ValueError, FileNotFoundError) as exc:
|
|
self._json(
|
|
{"error": type(exc).__name__}, HTTPStatus.BAD_REQUEST
|
|
)
|
|
|
|
def do_POST(self):
|
|
parsed = urlparse(self.path)
|
|
try:
|
|
body = self._body()
|
|
if parsed.path == "/createReleaseIntent":
|
|
return self._json(self.service.create_release_intent(body))
|
|
if parsed.path in {
|
|
"/submitSignedApproval",
|
|
"/submitOwnerApproval",
|
|
}:
|
|
return self._json(self.service.submit_owner_approval(body))
|
|
if parsed.path == "/startDeployment":
|
|
return self._json(
|
|
self.service.start_deployment(
|
|
str(body.get("candidate_id") or "")
|
|
)
|
|
)
|
|
self._json({"error": "not_found"}, HTTPStatus.NOT_FOUND)
|
|
except (ValueError, FileNotFoundError, CapabilityAuthorizationError) as exc:
|
|
self._json(
|
|
{"error": type(exc).__name__}, HTTPStatus.BAD_REQUEST
|
|
)
|
|
|
|
|
|
def git_main_probe(repository: Path) -> Callable[[], str]:
|
|
repository = Path(repository)
|
|
|
|
def probe() -> str:
|
|
completed = subprocess.run(
|
|
[
|
|
"git",
|
|
f"--git-dir={repository}",
|
|
"rev-parse",
|
|
"refs/heads/main",
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=15,
|
|
env={"PATH": os.environ.get("PATH", "")},
|
|
)
|
|
return completed.stdout.strip()
|
|
|
|
return probe
|
|
|
|
|
|
def serve_shadow(
|
|
*,
|
|
state_root: Path,
|
|
repository: Path,
|
|
host: str,
|
|
port: int,
|
|
poll_seconds: float,
|
|
) -> None:
|
|
service = SilentReleaseActionService.shadow(state_root)
|
|
controller = PersistentSilentReleaseController(
|
|
state_root,
|
|
service,
|
|
main_probe=git_main_probe(repository),
|
|
)
|
|
controller.run_once()
|
|
stop = threading.Event()
|
|
|
|
def controller_loop() -> None:
|
|
while not stop.wait(poll_seconds):
|
|
try:
|
|
controller.run_once()
|
|
except Exception as exc: # fail closed and keep the API available
|
|
_durable_json(
|
|
controller.status_path,
|
|
{
|
|
"error": type(exc).__name__,
|
|
"execution_enabled": False,
|
|
"shadow_mode": True,
|
|
"side_effects_executed": False,
|
|
"status": "FAIL_CLOSED",
|
|
},
|
|
)
|
|
|
|
handler = type(
|
|
"ConfiguredShadowActionHandler",
|
|
(ShadowActionHandler,),
|
|
{"service": service, "controller": controller},
|
|
)
|
|
server = ThreadingHTTPServer((host, port), handler)
|
|
api_thread = threading.Thread(
|
|
target=server.serve_forever,
|
|
name="silent-release-shadow-api",
|
|
daemon=False,
|
|
)
|
|
controller_thread = threading.Thread(
|
|
target=controller_loop,
|
|
name="silent-release-shadow-controller",
|
|
daemon=True,
|
|
)
|
|
api_thread.start()
|
|
controller_thread.start()
|
|
try:
|
|
while api_thread.is_alive():
|
|
api_thread.join(timeout=1)
|
|
finally:
|
|
stop.set()
|
|
server.shutdown()
|
|
server.server_close()
|
|
controller_thread.join(timeout=max(1.0, poll_seconds + 1.0))
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--state-root", required=True, type=Path)
|
|
parser.add_argument("--repository", required=True, type=Path)
|
|
parser.add_argument("--host", default="127.0.0.1")
|
|
parser.add_argument("--port", default=18891, type=int)
|
|
parser.add_argument("--poll-seconds", default=30.0, type=float)
|
|
arguments = parser.parse_args()
|
|
if arguments.poll_seconds <= 0:
|
|
raise SystemExit("poll seconds must be positive")
|
|
if os.environ.get("AIOS_GITOPS_WRITES_ENABLED", "false").lower() != "false":
|
|
raise SystemExit("GitOps writes must remain disabled")
|
|
if (
|
|
os.environ.get(
|
|
"AIOS_SILENT_RELEASE_EXECUTION_ENABLED", "false"
|
|
).lower()
|
|
!= "false"
|
|
):
|
|
raise SystemExit("Silent Release execution must remain disabled")
|
|
serve_shadow(
|
|
state_root=arguments.state_root,
|
|
repository=arguments.repository,
|
|
host=arguments.host,
|
|
port=arguments.port,
|
|
poll_seconds=arguments.poll_seconds,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|