#!/usr/bin/env python3 """Fail-closed orchestration for build-to-runtime release candidates. Runtime deployment and production tag publication intentionally use separate controllers and separate owner approvals. """ from __future__ import annotations import hashlib import json import os import re import subprocess import tempfile import time import urllib.error import urllib.parse import urllib.request from dataclasses import asdict, dataclass, field from datetime import datetime, timezone from enum import Enum from pathlib import Path from typing import Any, Protocol OID_RE = re.compile(r"^[0-9a-f]{40}$") SHA256_RE = re.compile(r"^[0-9a-f]{64}$") CANDIDATE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{2,127}$") VERSION_RE = re.compile(r"^V\d+R\d+C\d+(?:-[A-Za-z0-9_.-]+)?$") SECRET_PATTERNS = ( re.compile(r"(?i)(authorization\s*[:=]\s*)(?:bearer|token)\s+\S+"), re.compile(r"(?i)((?:access_)?token\s*[:=]\s*)\S+"), re.compile(r"(?i)(https?://[^/\s:@]+:)[^@\s/]+@"), re.compile(r"(?i)((?:password|secret)\s*[:=]\s*)\S+"), ) def utc_now() -> str: return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) def redact(value: Any) -> str: text = str(value or "") for pattern in SECRET_PATTERNS: text = pattern.sub(r"\1REDACTED", text) return text class EnvironmentAllowList: """Build a subprocess environment without inheriting process secrets.""" SENSITIVE_NAME_RE = re.compile( r"(?i)(authorization|credential|password|secret|token|api[_-]?key|" r"askpass|auth[_-]?sock|ld_preload|pythonpath|database_url)" ) def __init__(self, allowed_names: set[str] | frozenset[str]): self.allowed_names = frozenset(str(name) for name in allowed_names) def build(self, values: dict[str, str]) -> dict[str, str]: result: dict[str, str] = {} for name, raw in values.items(): if ( name not in self.allowed_names or self.SENSITIVE_NAME_RE.search(name) ): raise ValueError(f"environment variable is not allowed: {name}") value = str(raw) if redact(value) != value: raise ValueError( f"environment value is not allowed: {name}" ) result[name] = value return result SAFE_PROCESS_ENVIRONMENT = EnvironmentAllowList( { "PATH", "LANG", "LC_ALL", "TZ", "HOME", "GIT_OPTIONAL_LOCKS", "PYTHONDONTWRITEBYTECODE", } ).build( { "PATH": "/usr/bin:/bin", "LANG": "C.UTF-8", "LC_ALL": "C.UTF-8", "TZ": "UTC", "HOME": "/var/empty", "GIT_OPTIONAL_LOCKS": "0", "PYTHONDONTWRITEBYTECODE": "1", } ) def _sanitize(value: Any) -> Any: if isinstance(value, dict): return { str(key): ( "REDACTED" if re.search( r"(?i)(authorization|credential|password|secret|token)", str(key), ) else _sanitize(item) ) for key, item in value.items() } if isinstance(value, list): return [_sanitize(item) for item in value] if isinstance(value, tuple): return [_sanitize(item) for item in value] return redact(value) if isinstance(value, str) else value def _validate_oid(name: str, value: str) -> str: normalized = str(value or "").lower() if not OID_RE.fullmatch(normalized): raise ValueError(f"{name} must be a full 40-character OID") return normalized def _validate_sha256(name: str, value: str) -> str: normalized = str(value or "").lower() if not SHA256_RE.fullmatch(normalized): raise ValueError(f"{name} must be a SHA-256 digest") return normalized def _parse_utc(value: str) -> datetime: parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) if parsed.tzinfo is None: raise ValueError("approval expiry must include a timezone") return parsed.astimezone(timezone.utc) class ReleaseState(str, Enum): BUILDING = "BUILDING" VERIFYING = "VERIFYING" READY_FOR_APPROVAL = "READY_FOR_APPROVAL" APPROVED = "APPROVED" DEPLOYING = "DEPLOYING" VERIFIED = "VERIFIED" FAILED = "FAILED" ROLLED_BACK = "ROLLED_BACK" ALLOWED_TRANSITIONS = { ReleaseState.BUILDING: { ReleaseState.VERIFYING, ReleaseState.FAILED, }, ReleaseState.VERIFYING: { ReleaseState.READY_FOR_APPROVAL, ReleaseState.FAILED, }, ReleaseState.READY_FOR_APPROVAL: { ReleaseState.APPROVED, ReleaseState.FAILED, }, ReleaseState.APPROVED: { ReleaseState.DEPLOYING, ReleaseState.FAILED, }, ReleaseState.DEPLOYING: { ReleaseState.VERIFIED, ReleaseState.FAILED, ReleaseState.ROLLED_BACK, }, ReleaseState.VERIFIED: {ReleaseState.ROLLED_BACK}, ReleaseState.FAILED: set(), ReleaseState.ROLLED_BACK: set(), } @dataclass class ReleaseCandidate: candidate_id: str source_commit: str tree_oid: str artifact_sha256: str manifest_sha256: str tests: dict[str, Any] = field(default_factory=dict) canary: dict[str, Any] = field(default_factory=dict) rollback_ready: bool = False git_consistency: dict[str, Any] = field(default_factory=dict) gitea_consistency: dict[str, Any] = field(default_factory=dict) approval_state: str = "PENDING" state: ReleaseState = ReleaseState.BUILDING failure_reason: str | None = None created_at: str = field(default_factory=utc_now) updated_at: str = field(default_factory=utc_now) def __post_init__(self) -> None: if not CANDIDATE_ID_RE.fullmatch(str(self.candidate_id or "")): raise ValueError("invalid candidate_id") self.source_commit = _validate_oid("source_commit", self.source_commit) self.tree_oid = _validate_oid("tree_oid", self.tree_oid) self.artifact_sha256 = _validate_sha256( "artifact_sha256", self.artifact_sha256 ) self.manifest_sha256 = _validate_sha256( "manifest_sha256", self.manifest_sha256 ) if not isinstance(self.state, ReleaseState): self.state = ReleaseState(str(self.state)) @property def fingerprint(self) -> str: payload = "\0".join( ( self.candidate_id, self.source_commit, self.tree_oid, self.artifact_sha256, self.manifest_sha256, ) ).encode() return hashlib.sha256(payload).hexdigest() def transition(self, target: ReleaseState) -> None: target = ReleaseState(target) if target not in ALLOWED_TRANSITIONS[self.state]: raise ValueError( f"invalid release transition: {self.state.value} -> {target.value}" ) self.state = target self.updated_at = utc_now() def to_dict(self) -> dict[str, Any]: payload = asdict(self) payload["state"] = self.state.value return _sanitize(payload) @classmethod def from_dict(cls, payload: dict[str, Any]) -> "ReleaseCandidate": data = dict(payload) data["state"] = ReleaseState(data.get("state", ReleaseState.BUILDING)) return cls(**data) class ReleaseCandidateStore: def __init__(self, root: Path): self.root = Path(root) def path(self, candidate_id: str) -> Path: if not CANDIDATE_ID_RE.fullmatch(str(candidate_id or "")): raise ValueError("invalid candidate_id") root = self.root.resolve() candidate = (root / f"{candidate_id}.json").resolve() if candidate.parent != root: raise ValueError("candidate path escapes store") return candidate def save(self, candidate: ReleaseCandidate) -> Path: path = self.path(candidate.candidate_id) path.parent.mkdir(parents=True, exist_ok=True) _atomic_json(path, candidate.to_dict()) return path def load(self, candidate_id: str) -> ReleaseCandidate: return ReleaseCandidate.from_dict( json.loads(self.path(candidate_id).read_text(encoding="utf-8")) ) @dataclass(frozen=True) class AcceptanceEvidence: local_main_oid: str bare_main_oid: str gitea_main_oid: str repository_identity: str artifact_sha256: str manifest_sha256: str manifest_source_commit: str manifest_tree_oid: str tests: dict[str, Any] canary: dict[str, Any] rollback_ready: bool authoritative_git_dir: str authoritative_git_readable: bool @dataclass(frozen=True) class AcceptanceResult: ready: bool checks: dict[str, bool] failed_checks: tuple[str, ...] error: str | None = None class ReleaseProbe(Protocol): def collect(self, release_candidate: ReleaseCandidate) -> AcceptanceEvidence: ... class ReleaseAcceptanceEngine: """Deep module that converts raw probe evidence into one approval gate.""" def __init__( self, probe: ReleaseProbe, *, expected_repository: str, expected_authoritative_git_dir: str = "/data/aios-gitops/repo.git", ): self.probe = probe self.expected_repository = expected_repository self.expected_authoritative_git_dir = expected_authoritative_git_dir def evaluate(self, candidate: ReleaseCandidate) -> AcceptanceResult: if candidate.state == ReleaseState.BUILDING: candidate.transition(ReleaseState.VERIFYING) if candidate.state != ReleaseState.VERIFYING: raise ValueError("candidate is not eligible for verification") try: evidence = self.probe.collect(candidate) checks = { "local_main": evidence.local_main_oid == candidate.source_commit, "bare_main": evidence.bare_main_oid == candidate.source_commit, "gitea_main": evidence.gitea_main_oid == candidate.source_commit, "repository_identity": ( evidence.repository_identity == self.expected_repository ), "artifact_sha256": ( evidence.artifact_sha256 == candidate.artifact_sha256 ), "manifest_sha256": ( evidence.manifest_sha256 == candidate.manifest_sha256 ), "manifest_source_commit": ( evidence.manifest_source_commit == candidate.source_commit ), "manifest_tree_oid": ( evidence.manifest_tree_oid == candidate.tree_oid ), "tests": evidence.tests.get("passed") is True, "canary": evidence.canary.get("passed") is True, "runtime_without_git": ( evidence.canary.get("runtime_without_git") is True ), "writer_disabled": ( evidence.canary.get("writes_enabled") is False ), "rollback_ready": evidence.rollback_ready is True, "authoritative_git_binding": ( evidence.authoritative_git_readable is True and evidence.authoritative_git_dir == self.expected_authoritative_git_dir ), } candidate.tests = _sanitize(evidence.tests) candidate.canary = _sanitize(evidence.canary) candidate.rollback_ready = evidence.rollback_ready candidate.git_consistency = { "local_main_oid": evidence.local_main_oid, "bare_main_oid": evidence.bare_main_oid, "authoritative_git_dir": evidence.authoritative_git_dir, "authoritative_git_readable": evidence.authoritative_git_readable, } candidate.gitea_consistency = { "main_oid": evidence.gitea_main_oid, "repository_identity": evidence.repository_identity, } failed = tuple(name for name, passed in checks.items() if not passed) if failed: candidate.failure_reason = "acceptance failed: " + ", ".join(failed) candidate.transition(ReleaseState.FAILED) return AcceptanceResult(False, checks, failed) candidate.transition(ReleaseState.READY_FOR_APPROVAL) return AcceptanceResult(True, checks, ()) except Exception as exc: candidate.failure_reason = redact(exc) candidate.transition(ReleaseState.FAILED) return AcceptanceResult( False, {"probe": False}, ("probe",), candidate.failure_reason, ) @dataclass(frozen=True) class ApprovalEnvelope: approval_id: str owner_id: str candidate_id: str candidate_fingerprint: str source_commit: str tree_oid: str artifact_sha256: str manifest_sha256: str action: str expires_at: str issued_at: str = field(default_factory=utc_now) @classmethod def for_candidate( cls, candidate: ReleaseCandidate, *, approval_id: str, owner_id: str, expires_at: str, ) -> "ApprovalEnvelope": return cls( approval_id=approval_id, owner_id=owner_id, candidate_id=candidate.candidate_id, candidate_fingerprint=candidate.fingerprint, source_commit=candidate.source_commit, tree_oid=candidate.tree_oid, artifact_sha256=candidate.artifact_sha256, manifest_sha256=candidate.manifest_sha256, action="production_runtime_switch", expires_at=expires_at, ) def validate(self, candidate: ReleaseCandidate) -> None: if self.action != "production_runtime_switch": raise ApprovalError("approval does not authorize runtime switch") if not self.owner_id or not self.approval_id: raise ApprovalError("owner and approval id are required") if _parse_utc(self.expires_at) <= datetime.now(timezone.utc): raise ApprovalError("approval has expired") expected = ( candidate.candidate_id, candidate.fingerprint, candidate.source_commit, candidate.tree_oid, candidate.artifact_sha256, candidate.manifest_sha256, ) actual = ( self.candidate_id, self.candidate_fingerprint, self.source_commit, self.tree_oid, self.artifact_sha256, self.manifest_sha256, ) if actual != expected: raise ApprovalError("approval fingerprint mismatch") class ApprovalError(RuntimeError): pass class ApprovalLedger: def __init__(self, root: Path): self.root = Path(root) def path(self, approval_id: str) -> Path: if not CANDIDATE_ID_RE.fullmatch(str(approval_id or "")): raise ApprovalError("invalid approval id") root = self.root.resolve() path = (root / f"{approval_id}.json").resolve() if path.parent != root: raise ApprovalError("approval path escapes ledger") return path def consume(self, envelope: ApprovalEnvelope) -> None: path = self.path(envelope.approval_id) path.parent.mkdir(parents=True, exist_ok=True) try: descriptor = os.open( path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600, ) except FileExistsError as exc: raise ApprovalError("approval already consumed") from exc payload = { "approval_id": envelope.approval_id, "candidate_id": envelope.candidate_id, "candidate_fingerprint": envelope.candidate_fingerprint, "action": envelope.action, "consumed_at": utc_now(), } with os.fdopen(descriptor, "w", encoding="utf-8") as output: json.dump(payload, output, sort_keys=True) output.write("\n") output.flush() os.fsync(output.fileno()) @dataclass(frozen=True) class SilentReleasePolicy: automatic: frozenset[str] owner: frozenset[str] forbidden: frozenset[str] @classmethod def default(cls) -> "SilentReleasePolicy": return cls( automatic=frozenset( {"build", "test", "artifact", "canary", "evidence"} ), owner=frozenset( {"production_runtime_switch", "production_tag"} ), forbidden=frozenset( { "force_push", "tag_rewrite", "direct_main_push", "baseline_mutation", "registry_mutation", } ), ) @classmethod def from_file(cls, path: Path) -> "SilentReleasePolicy": payload = json.loads(Path(path).read_text(encoding="utf-8")) return cls( frozenset(payload["automatic"]), frozenset(payload["owner"]), frozenset(payload["forbidden"]), ) def is_automatic(self, action: str) -> bool: return action in self.automatic def requires_owner(self, action: str) -> bool: return action in self.owner def is_forbidden(self, action: str) -> bool: return action in self.forbidden class DeploymentAdapter(Protocol): def atomic_switch( self, release_candidate: ReleaseCandidate ) -> dict[str, Any]: ... def restart(self) -> None: ... def health_check( self, release_candidate: ReleaseCandidate ) -> dict[str, Any]: ... def rollback( self, switch_evidence: dict[str, Any] ) -> dict[str, Any]: ... class DeploymentController: def __init__( self, deployment: DeploymentAdapter, ledger: ApprovalLedger, policy: SilentReleasePolicy, store: ReleaseCandidateStore | None = None, *, allow_legacy_unsigned_approval: bool = False, ): self.deployment = deployment self.ledger = ledger self.policy = policy self.store = store self.allow_legacy_unsigned_approval = ( allow_legacy_unsigned_approval ) def _save(self, candidate: ReleaseCandidate) -> None: if self.store: self.store.save(candidate) def deploy( self, candidate: ReleaseCandidate, approval: ApprovalEnvelope, ) -> dict[str, Any]: if not self.allow_legacy_unsigned_approval: raise ApprovalError( "legacy unsigned runtime approval is disabled; " "use SignedReleasePlanApproval" ) if candidate.state != ReleaseState.READY_FOR_APPROVAL: raise ApprovalError("candidate is not ready for approval") if not self.policy.requires_owner("production_runtime_switch"): raise ApprovalError("runtime switch is not owner governed") for forbidden in ( "force_push", "tag_rewrite", "direct_main_push", "baseline_mutation", "registry_mutation", ): if not self.policy.is_forbidden(forbidden): raise ApprovalError("deployment policy is incomplete") approval.validate(candidate) self.ledger.consume(approval) candidate.approval_state = "APPROVED" candidate.transition(ReleaseState.APPROVED) self._save(candidate) candidate.transition(ReleaseState.DEPLOYING) self._save(candidate) switch_evidence: dict[str, Any] | None = None try: switch_evidence = self.deployment.atomic_switch(candidate) self.deployment.restart() health = self.deployment.health_check(candidate) passed = ( health.get("passed") is True and health.get("runtime_commit") == candidate.source_commit and health.get("writes_enabled") is False and health.get("authoritative_head_oid") == candidate.source_commit and health.get("repository_identity") == candidate.gitea_consistency.get( "repository_identity" ) ) if not passed: raise RuntimeError("post-deployment health check failed") candidate.transition(ReleaseState.VERIFIED) self._save(candidate) return { "verified": True, "state": candidate.state.value, "health": _sanitize(health), "switch": _sanitize(switch_evidence), } except Exception as exc: candidate.failure_reason = redact(exc) if switch_evidence is not None: try: rollback = self.deployment.rollback(switch_evidence) if ( rollback.get("passed") is not True or rollback.get("writes_enabled") is not False ): raise RuntimeError( "post-rollback health check failed" ) candidate.transition(ReleaseState.ROLLED_BACK) except Exception as rollback_exc: candidate.failure_reason = ( candidate.failure_reason + "; rollback failed: " + redact(rollback_exc) ) candidate.transition(ReleaseState.FAILED) else: candidate.transition(ReleaseState.FAILED) self._save(candidate) return { "verified": False, "state": candidate.state.value, "error": candidate.failure_reason, "rolled_back": candidate.state == ReleaseState.ROLLED_BACK, } class ReleaseBuilder(Protocol): def build(self, source_commit: str) -> ReleaseCandidate: ... class SilentReleaseController: """Small external interface for prepare -> approve -> deploy.""" def __init__( self, *, builder: ReleaseBuilder, acceptance: ReleaseAcceptanceEngine, deployment: DeploymentController, store: ReleaseCandidateStore, policy: SilentReleasePolicy, ): self.builder = builder self.acceptance = acceptance self.deployment = deployment self.store = store self.policy = policy def prepare(self, source_commit: str) -> ReleaseCandidate: required = {"build", "test", "artifact", "canary", "evidence"} if not all(self.policy.is_automatic(action) for action in required): raise RuntimeError("automatic preparation policy is incomplete") candidate = self.builder.build( _validate_oid("source_commit", source_commit) ) if candidate.source_commit != source_commit: candidate.failure_reason = "builder source commit mismatch" candidate.transition(ReleaseState.FAILED) self.store.save(candidate) return candidate self.store.save(candidate) self.acceptance.evaluate(candidate) self.store.save(candidate) return candidate def deploy( self, candidate_id: str, approval: ApprovalEnvelope, ) -> dict[str, Any]: candidate = self.store.load(candidate_id) return self.deployment.deploy(candidate, approval) @dataclass(frozen=True) class CommandReleaseBuilderConfig: command: tuple[str, ...] cwd: Path candidate_output: Path timeout: float = 900 class CommandReleaseBuilder: """Adapter for a deterministic build pipeline that emits candidate JSON.""" def __init__(self, config: CommandReleaseBuilderConfig): self.config = config def build(self, source_commit: str) -> ReleaseCandidate: if not self.config.command: raise RuntimeError("release build command is not configured") try: result = subprocess.run( [*self.config.command, source_commit], cwd=self.config.cwd, text=True, capture_output=True, timeout=self.config.timeout, env=dict(SAFE_PROCESS_ENVIRONMENT), ) except (OSError, subprocess.TimeoutExpired) as exc: raise RuntimeError("release build unavailable") from exc if result.returncode: raise RuntimeError( "release build failed: " + redact(result.stderr.strip() or result.stdout.strip()) ) candidate = ReleaseCandidate.from_dict( json.loads( self.config.candidate_output.read_text(encoding="utf-8") ) ) if candidate.source_commit != source_commit: raise RuntimeError("release build source commit mismatch") return candidate @dataclass(frozen=True) class TagApprovalEnvelope: approval_id: str owner_id: str version: str tag_oid: str target_oid: str action: str expires_at: str issued_at: str = field(default_factory=utc_now) def validate(self) -> None: if self.action != "production_tag": raise ApprovalError("approval does not authorize production tag") if not VERSION_RE.fullmatch(self.version): raise ApprovalError("invalid release version") _validate_oid("tag_oid", self.tag_oid) _validate_oid("target_oid", self.target_oid) if _parse_utc(self.expires_at) <= datetime.now(timezone.utc): raise ApprovalError("approval has expired") if not self.owner_id or not self.approval_id: raise ApprovalError("owner and approval id are required") class DuplicateTagConflict(RuntimeError): pass class TagAdapter(Protocol): def lookup(self, version: str) -> tuple[str, str] | None: ... def create(self, version: str, tag_oid: str, target_oid: str) -> None: ... def verify(self, version: str) -> tuple[str, str] | None: ... class TagController: """Separate approval, execution, and verification for production tags.""" def __init__( self, adapter: TagAdapter, policy: SilentReleasePolicy, ledger: ApprovalLedger | None, *, allow_legacy_unsigned_approval: bool = False, ): self.adapter = adapter self.policy = policy self.ledger = ledger self.allow_legacy_unsigned_approval = ( allow_legacy_unsigned_approval ) def _require_legacy_test_mode(self) -> None: if not self.allow_legacy_unsigned_approval: raise ApprovalError( "legacy unsigned tag approval is disabled; " "use SignedReleasePlanApproval" ) def approve(self, envelope: TagApprovalEnvelope) -> TagApprovalEnvelope: self._require_legacy_test_mode() if not self.policy.requires_owner("production_tag"): raise ApprovalError("production tag is not owner governed") if not self.policy.is_forbidden("tag_rewrite"): raise ApprovalError("tag rewrite must be forbidden") envelope.validate() return envelope def execute( self, approved: TagApprovalEnvelope ) -> dict[str, Any]: self._require_legacy_test_mode() approved.validate() existing = self.adapter.lookup(approved.version) expected = (approved.tag_oid, approved.target_oid) if existing and existing != expected: raise DuplicateTagConflict( "existing production tag has a different object or target" ) if self.ledger: runtime_envelope = ApprovalEnvelope( approval_id=approved.approval_id, owner_id=approved.owner_id, candidate_id=approved.version, candidate_fingerprint=hashlib.sha256( "\0".join(expected).encode() ).hexdigest(), source_commit=approved.target_oid, tree_oid=approved.target_oid, artifact_sha256="0" * 64, manifest_sha256="0" * 64, action=approved.action, expires_at=approved.expires_at, issued_at=approved.issued_at, ) self.ledger.consume(runtime_envelope) if existing is None: self.adapter.create( approved.version, approved.tag_oid, approved.target_oid, ) verified = self.verify(approved) return {"executed": existing is None, **verified} def verify( self, approved: TagApprovalEnvelope ) -> dict[str, Any]: self._require_legacy_test_mode() actual = self.adapter.verify(approved.version) expected = (approved.tag_oid, approved.target_oid) if actual != expected: raise DuplicateTagConflict("production tag verification failed") return { "verified": True, "version": approved.version, "tag_oid": actual[0], "target_oid": actual[1], } @dataclass(frozen=True) class ReleaseProbeConfig: local_repository: Path authoritative_git_dir: Path release_directory: Path artifact_path: Path manifest_path: Path rollback_artifact: Path gitea_api_url: str repository_identity: str gitea_token_file: Path gitea_token_line: int = 2 test_command: tuple[str, ...] = () canary_command: tuple[str, ...] = () class CommandReleaseProbe: """Production adapter for ReleaseAcceptanceEngine; never writes Git refs.""" def __init__(self, config: ReleaseProbeConfig): self.config = config def _run( self, command: list[str], *, cwd: Path | None = None ) -> subprocess.CompletedProcess[str]: try: return subprocess.run( command, cwd=cwd, text=True, capture_output=True, timeout=300, env=dict(SAFE_PROCESS_ENVIRONMENT), ) except (OSError, subprocess.TimeoutExpired) as exc: raise RuntimeError("COMMAND_UNAVAILABLE") from exc def _git(self, command: list[str]) -> str: result = self._run(command) if result.returncode: raise RuntimeError("AUTHORITATIVE_GIT_UNAVAILABLE") return result.stdout.strip() def _api(self, path: str) -> Any: lines = self.config.gitea_token_file.read_text( encoding="utf-8" ).splitlines() if not 0 < self.config.gitea_token_line <= len(lines): raise RuntimeError("GITEA_CREDENTIAL_REFERENCE_INVALID") token = lines[self.config.gitea_token_line - 1].strip() request = urllib.request.Request( self.config.gitea_api_url.rstrip("/") + path, headers={ "Accept": "application/json", "Authorization": f"token {token}", "User-Agent": "aios-silent-release-controller/1", }, ) try: with urllib.request.urlopen(request, timeout=15) as response: return json.loads(response.read().decode("utf-8")) except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as exc: raise RuntimeError("GITEA_UNAVAILABLE") from exc def _command_evidence( self, command: tuple[str, ...], *, cwd: Path ) -> dict[str, Any]: if not command: return {"passed": False, "error": "command not configured"} result = self._run(list(command), cwd=cwd) raw = result.stdout.strip() parsed: dict[str, Any] = {} if raw: try: value = json.loads(raw) if isinstance(value, dict): parsed = _sanitize(value) except json.JSONDecodeError: pass summary = redact((raw or result.stderr.strip())[-500:]) evidence = { **parsed, "passed": ( result.returncode == 0 and parsed.get("passed", True) is True ), "exit_code": result.returncode, "summary": summary, } return evidence def collect(self, candidate: ReleaseCandidate) -> AcceptanceEvidence: cfg = self.config local_main = self._git( [ "git", "-C", str(cfg.local_repository), "rev-parse", "refs/heads/main", ] ) bare_main = self._git( [ "git", f"--git-dir={cfg.authoritative_git_dir}", "rev-parse", "refs/heads/main", ] ) bare = self._git( [ "git", f"--git-dir={cfg.authoritative_git_dir}", "rev-parse", "--is-bare-repository", ] ) if bare.lower() != "true": raise RuntimeError("AUTHORITATIVE_GIT_UNAVAILABLE") remote_url = self._git( [ "git", f"--git-dir={cfg.authoritative_git_dir}", "config", "--get", "remote.gitea.url", ] ) if ( _repository_identity_from_remote(remote_url) != cfg.repository_identity.lower() ): raise RuntimeError("REPOSITORY_IDENTITY_MISMATCH") repository = self._api(f"/repos/{cfg.repository_identity}") ref = self._api( f"/repos/{cfg.repository_identity}/git/refs/heads/main" ) items = ref if isinstance(ref, list) else [ref] exact = [ item for item in items if isinstance(item, dict) and item.get("ref") in {"refs/heads/main", "heads/main"} ] if len(exact) != 1: raise RuntimeError("GITEA_MAIN_UNAVAILABLE") gitea_main = str( ((exact[0].get("object") or {}).get("sha")) or "" ).lower() manifest = json.loads(cfg.manifest_path.read_text(encoding="utf-8")) canary = self._command_evidence( cfg.canary_command, cwd=cfg.release_directory, ) canary["runtime_without_git"] = not any( path.name == ".git" for path in cfg.release_directory.rglob(".git") ) return AcceptanceEvidence( local_main_oid=local_main, bare_main_oid=bare_main, gitea_main_oid=gitea_main, repository_identity=str(repository.get("full_name") or ""), artifact_sha256=_sha256(cfg.artifact_path), manifest_sha256=_sha256(cfg.manifest_path), manifest_source_commit=str( manifest.get("target_commit") or "" ).lower(), manifest_tree_oid=str( manifest.get("git_tree_oid") or "" ).lower(), tests=self._command_evidence( cfg.test_command, cwd=cfg.release_directory, ), canary=canary, rollback_ready=cfg.rollback_artifact.is_file(), authoritative_git_dir=str( cfg.authoritative_git_dir.resolve() ), authoritative_git_readable=os.access( cfg.authoritative_git_dir, os.R_OK | os.X_OK ), ) @dataclass(frozen=True) class SystemdDeploymentConfig: release_root: Path cutover_link: Path service_name: str health_url: str command_prefix: tuple[str, ...] = () dropin_name: str = "aios-gitops-model-api-production.conf" service_allowlist: tuple[str, ...] = ("aios-gitops-model-api",) dropin_allowlist: tuple[str, ...] = ( "aios-gitops-model-api-production.conf", ) deployment_lock: Path = Path( "/run/lock/aios-silent-release-deployment.lock" ) class SystemdDeploymentAdapter: """Atomic symlink switch. It has no tag, Baseline, or Registry methods.""" def __init__(self, config: SystemdDeploymentConfig): self.config = config self._lock_handle = None self._restart_evidence: dict[str, Any] = {} if self.config.service_name not in self.config.service_allowlist: raise ValueError("systemd service is not allowlisted") if ( self.config.dropin_name not in self.config.dropin_allowlist or "/" in self.config.dropin_name or "\\" in self.config.dropin_name ): raise ValueError("systemd drop-in is not allowlisted") def _acquire_lock(self) -> None: if self._lock_handle is not None: raise RuntimeError("deployment lock is already held") path = self.config.deployment_lock path.parent.mkdir(parents=True, exist_ok=True) handle = path.open("a+", encoding="utf-8") try: import fcntl fcntl.flock(handle.fileno(), fcntl.LOCK_EX) except ImportError: pass self._lock_handle = handle def _release_lock(self) -> None: handle = self._lock_handle self._lock_handle = None if handle is None: return try: import fcntl fcntl.flock(handle.fileno(), fcntl.LOCK_UN) except ImportError: pass handle.close() def _service_identity(self) -> dict[str, Any]: result = subprocess.run( [ *self.config.command_prefix, "systemctl", "show", self.config.service_name, "--property=MainPID", "--property=ActiveEnterTimestampMonotonic", "--value", ], check=True, capture_output=True, text=True, env=dict(SAFE_PROCESS_ENVIRONMENT), ) values = result.stdout.splitlines() if len(values) < 2: raise RuntimeError("systemd process identity unavailable") try: pid = int(values[0]) started = int(values[1]) except ValueError as exc: raise RuntimeError( "systemd process identity is invalid" ) from exc return {"main_pid": pid, "started_monotonic": started} def atomic_switch( self, candidate: ReleaseCandidate ) -> dict[str, Any]: self._acquire_lock() try: release_root = self.config.release_root.resolve(strict=True) raw_target = ( release_root / candidate.source_commit / "deployment" / self.config.dropin_name ) if raw_target.is_symlink(): raise RuntimeError("release drop-in must not be a symlink") target = raw_target.resolve(strict=True) if target.parent.parent.parent != release_root: raise RuntimeError("release drop-in escapes release root") link = self.config.cutover_link if link.exists() and not link.is_symlink(): raise RuntimeError("cutover path is not a managed symlink") previous = os.readlink(link) if link.is_symlink() else None if previous: previous_path = Path(previous) if not previous_path.is_absolute(): previous_path = link.parent / previous_path try: previous_path.resolve().relative_to(release_root) except ValueError as exc: raise RuntimeError( "existing cutover symlink escapes release root" ) from exc link.parent.mkdir(parents=True, exist_ok=True) temporary = link.with_name(f".{link.name}.{os.getpid()}.tmp") if temporary.exists() or temporary.is_symlink(): temporary.unlink() os.symlink(str(target), temporary) os.replace(temporary, link) return { "previous_target": previous, "new_target": str(target), "cutover_link": str(link), } except Exception: self._release_lock() raise def restart(self) -> None: if self._lock_handle is None: raise RuntimeError("deployment lock is not held") before = self._service_identity() command = [ *self.config.command_prefix, "systemctl", "daemon-reload", ] subprocess.run( command, check=True, capture_output=True, text=True, env=dict(SAFE_PROCESS_ENVIRONMENT), ) subprocess.run( [ *self.config.command_prefix, "systemctl", "restart", self.config.service_name, ], check=True, capture_output=True, text=True, env=dict(SAFE_PROCESS_ENVIRONMENT), ) after = self._service_identity() if ( after["main_pid"] <= 0 or after["started_monotonic"] <= before["started_monotonic"] ): raise RuntimeError( "systemd restart did not produce a new start identity" ) self._restart_evidence = {"before": before, "after": after} def health_check( self, candidate: ReleaseCandidate ) -> dict[str, Any]: try: with urllib.request.urlopen( self.config.health_url, timeout=15 ) as response: payload = json.loads(response.read().decode("utf-8")) except Exception as exc: raise RuntimeError("runtime health check unavailable") from exc try: return { "passed": response.status == 200, "runtime_commit": payload.get("runtime_release_commit"), "writes_enabled": payload.get("writes_enabled"), "authoritative_head_oid": payload.get( "authoritative_head_oid" ), "repository_identity": payload.get("repository_identity"), "systemd_restart": _sanitize(self._restart_evidence), } finally: self._release_lock() def rollback( self, switch_evidence: dict[str, Any] ) -> dict[str, Any]: try: if self._lock_handle is None: self._acquire_lock() link = self.config.cutover_link previous = switch_evidence.get("previous_target") temporary = link.with_name( f".{link.name}.{os.getpid()}.rollback" ) if previous: if temporary.exists() or temporary.is_symlink(): temporary.unlink() os.symlink(str(previous), temporary) os.replace(temporary, link) elif link.is_symlink(): link.unlink() self.restart() try: with urllib.request.urlopen( self.config.health_url, timeout=15 ) as response: payload = json.loads( response.read().decode("utf-8") ) except Exception as exc: raise RuntimeError( "rollback health check unavailable" ) from exc return { "passed": response.status == 200, "writes_enabled": payload.get("writes_enabled"), "runtime_commit": payload.get("runtime_release_commit"), "systemd_restart": _sanitize(self._restart_evidence), } finally: self._release_lock() def _sha256(path: Path) -> str: digest = hashlib.sha256() with Path(path).open("rb") as source: for chunk in iter(lambda: source.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def _repository_identity_from_remote(remote_url: str) -> str: value = str(remote_url or "").strip().split("?", 1)[0].rstrip("/") if "://" in value: value = urllib.parse.urlparse(value).path elif ":" in value: value = value.rsplit(":", 1)[1] value = re.sub(r"\.git$", "", value.strip("/"), flags=re.IGNORECASE) parts = [part for part in value.split("/") if part] return "/".join(parts[-2:]).lower() if len(parts) >= 2 else "" def _atomic_json(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) descriptor, temporary = tempfile.mkstemp( prefix=f".{path.name}.", suffix=".tmp", dir=path.parent, ) try: with os.fdopen(descriptor, "w", encoding="utf-8") as output: json.dump(payload, output, indent=2, sort_keys=True) output.write("\n") output.flush() os.fsync(output.fileno()) os.chmod(temporary, 0o600) os.replace(temporary, path) finally: if os.path.exists(temporary): os.unlink(temporary)