Files
aios/active-runtime/app/silent_release_controller.py
T

1348 lines
45 KiB
Python

#!/usr/bin/env python3
"""Fail-closed orchestration for build-to-runtime release candidates.
Runtime deployment and production tag publication intentionally use separate
controllers and separate owner approvals.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import subprocess
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from enum import Enum
from pathlib import Path
from typing import Any, Protocol
from release_version import validate_release_version
OID_RE = re.compile(r"^[0-9a-f]{40}$")
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
CANDIDATE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{2,127}$")
SECRET_PATTERNS = (
re.compile(r"(?i)(authorization\s*[:=]\s*)(?:bearer|token)\s+\S+"),
re.compile(r"(?i)((?:access_)?token\s*[:=]\s*)\S+"),
re.compile(r"(?i)(https?://[^/\s:@]+:)[^@\s/]+@"),
re.compile(r"(?i)((?:password|secret)\s*[:=]\s*)\S+"),
)
def utc_now() -> str:
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
def redact(value: Any) -> str:
text = str(value or "")
for pattern in SECRET_PATTERNS:
text = pattern.sub(r"\1REDACTED", text)
return text
class EnvironmentAllowList:
"""Build a subprocess environment without inheriting process secrets."""
SENSITIVE_NAME_RE = re.compile(
r"(?i)(authorization|credential|password|secret|token|api[_-]?key|"
r"askpass|auth[_-]?sock|ld_preload|pythonpath|database_url)"
)
def __init__(self, allowed_names: set[str] | frozenset[str]):
self.allowed_names = frozenset(str(name) for name in allowed_names)
def build(self, values: dict[str, str]) -> dict[str, str]:
result: dict[str, str] = {}
for name, raw in values.items():
if (
name not in self.allowed_names
or self.SENSITIVE_NAME_RE.search(name)
):
raise ValueError(f"environment variable is not allowed: {name}")
value = str(raw)
if redact(value) != value:
raise ValueError(
f"environment value is not allowed: {name}"
)
result[name] = value
return result
SAFE_PROCESS_ENVIRONMENT = EnvironmentAllowList(
{
"PATH",
"LANG",
"LC_ALL",
"TZ",
"HOME",
"GIT_OPTIONAL_LOCKS",
"PYTHONDONTWRITEBYTECODE",
}
).build(
{
"PATH": "/usr/bin:/bin",
"LANG": "C.UTF-8",
"LC_ALL": "C.UTF-8",
"TZ": "UTC",
"HOME": "/var/empty",
"GIT_OPTIONAL_LOCKS": "0",
"PYTHONDONTWRITEBYTECODE": "1",
}
)
def _sanitize(value: Any) -> Any:
if isinstance(value, dict):
return {
str(key): (
"REDACTED"
if re.search(
r"(?i)(authorization|credential|password|secret|token)",
str(key),
)
else _sanitize(item)
)
for key, item in value.items()
}
if isinstance(value, list):
return [_sanitize(item) for item in value]
if isinstance(value, tuple):
return [_sanitize(item) for item in value]
return redact(value) if isinstance(value, str) else value
def _validate_oid(name: str, value: str) -> str:
normalized = str(value or "").lower()
if not OID_RE.fullmatch(normalized):
raise ValueError(f"{name} must be a full 40-character OID")
return normalized
def _validate_sha256(name: str, value: str) -> str:
normalized = str(value or "").lower()
if not SHA256_RE.fullmatch(normalized):
raise ValueError(f"{name} must be a SHA-256 digest")
return normalized
def _parse_utc(value: str) -> datetime:
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
if parsed.tzinfo is None:
raise ValueError("approval expiry must include a timezone")
return parsed.astimezone(timezone.utc)
class ReleaseState(str, Enum):
BUILDING = "BUILDING"
VERIFYING = "VERIFYING"
READY_FOR_APPROVAL = "READY_FOR_APPROVAL"
APPROVED = "APPROVED"
DEPLOYING = "DEPLOYING"
VERIFIED = "VERIFIED"
FAILED = "FAILED"
ROLLED_BACK = "ROLLED_BACK"
ALLOWED_TRANSITIONS = {
ReleaseState.BUILDING: {
ReleaseState.VERIFYING,
ReleaseState.FAILED,
},
ReleaseState.VERIFYING: {
ReleaseState.READY_FOR_APPROVAL,
ReleaseState.FAILED,
},
ReleaseState.READY_FOR_APPROVAL: {
ReleaseState.APPROVED,
ReleaseState.FAILED,
},
ReleaseState.APPROVED: {
ReleaseState.DEPLOYING,
ReleaseState.FAILED,
},
ReleaseState.DEPLOYING: {
ReleaseState.VERIFIED,
ReleaseState.FAILED,
ReleaseState.ROLLED_BACK,
},
ReleaseState.VERIFIED: {ReleaseState.ROLLED_BACK},
ReleaseState.FAILED: set(),
ReleaseState.ROLLED_BACK: set(),
}
@dataclass
class ReleaseCandidate:
candidate_id: str
source_commit: str
tree_oid: str
artifact_sha256: str
manifest_sha256: str
tests: dict[str, Any] = field(default_factory=dict)
canary: dict[str, Any] = field(default_factory=dict)
rollback_ready: bool = False
git_consistency: dict[str, Any] = field(default_factory=dict)
gitea_consistency: dict[str, Any] = field(default_factory=dict)
approval_state: str = "PENDING"
state: ReleaseState = ReleaseState.BUILDING
failure_reason: str | None = None
created_at: str = field(default_factory=utc_now)
updated_at: str = field(default_factory=utc_now)
def __post_init__(self) -> None:
if not CANDIDATE_ID_RE.fullmatch(str(self.candidate_id or "")):
raise ValueError("invalid candidate_id")
self.source_commit = _validate_oid("source_commit", self.source_commit)
self.tree_oid = _validate_oid("tree_oid", self.tree_oid)
self.artifact_sha256 = _validate_sha256(
"artifact_sha256", self.artifact_sha256
)
self.manifest_sha256 = _validate_sha256(
"manifest_sha256", self.manifest_sha256
)
if not isinstance(self.state, ReleaseState):
self.state = ReleaseState(str(self.state))
@property
def fingerprint(self) -> str:
payload = "\0".join(
(
self.candidate_id,
self.source_commit,
self.tree_oid,
self.artifact_sha256,
self.manifest_sha256,
)
).encode()
return hashlib.sha256(payload).hexdigest()
def transition(self, target: ReleaseState) -> None:
target = ReleaseState(target)
if target not in ALLOWED_TRANSITIONS[self.state]:
raise ValueError(
f"invalid release transition: {self.state.value} -> {target.value}"
)
self.state = target
self.updated_at = utc_now()
def to_dict(self) -> dict[str, Any]:
payload = asdict(self)
payload["state"] = self.state.value
return _sanitize(payload)
@classmethod
def from_dict(cls, payload: dict[str, Any]) -> "ReleaseCandidate":
data = dict(payload)
data["state"] = ReleaseState(data.get("state", ReleaseState.BUILDING))
return cls(**data)
class ReleaseCandidateStore:
def __init__(self, root: Path):
self.root = Path(root)
def path(self, candidate_id: str) -> Path:
if not CANDIDATE_ID_RE.fullmatch(str(candidate_id or "")):
raise ValueError("invalid candidate_id")
root = self.root.resolve()
candidate = (root / f"{candidate_id}.json").resolve()
if candidate.parent != root:
raise ValueError("candidate path escapes store")
return candidate
def save(self, candidate: ReleaseCandidate) -> Path:
path = self.path(candidate.candidate_id)
path.parent.mkdir(parents=True, exist_ok=True)
_atomic_json(path, candidate.to_dict())
return path
def load(self, candidate_id: str) -> ReleaseCandidate:
return ReleaseCandidate.from_dict(
json.loads(self.path(candidate_id).read_text(encoding="utf-8"))
)
@dataclass(frozen=True)
class AcceptanceEvidence:
local_main_oid: str
bare_main_oid: str
gitea_main_oid: str
repository_identity: str
artifact_sha256: str
manifest_sha256: str
manifest_source_commit: str
manifest_tree_oid: str
tests: dict[str, Any]
canary: dict[str, Any]
rollback_ready: bool
authoritative_git_dir: str
authoritative_git_readable: bool
@dataclass(frozen=True)
class AcceptanceResult:
ready: bool
checks: dict[str, bool]
failed_checks: tuple[str, ...]
error: str | None = None
class ReleaseProbe(Protocol):
def collect(self, release_candidate: ReleaseCandidate) -> AcceptanceEvidence:
...
class ReleaseAcceptanceEngine:
"""Deep module that converts raw probe evidence into one approval gate."""
def __init__(
self,
probe: ReleaseProbe,
*,
expected_repository: str,
expected_authoritative_git_dir: str = "/data/aios-gitops/repo.git",
):
self.probe = probe
self.expected_repository = expected_repository
self.expected_authoritative_git_dir = expected_authoritative_git_dir
def evaluate(self, candidate: ReleaseCandidate) -> AcceptanceResult:
if candidate.state == ReleaseState.BUILDING:
candidate.transition(ReleaseState.VERIFYING)
if candidate.state != ReleaseState.VERIFYING:
raise ValueError("candidate is not eligible for verification")
try:
evidence = self.probe.collect(candidate)
checks = {
"local_main": evidence.local_main_oid == candidate.source_commit,
"bare_main": evidence.bare_main_oid == candidate.source_commit,
"gitea_main": evidence.gitea_main_oid == candidate.source_commit,
"repository_identity": (
evidence.repository_identity == self.expected_repository
),
"artifact_sha256": (
evidence.artifact_sha256 == candidate.artifact_sha256
),
"manifest_sha256": (
evidence.manifest_sha256 == candidate.manifest_sha256
),
"manifest_source_commit": (
evidence.manifest_source_commit == candidate.source_commit
),
"manifest_tree_oid": (
evidence.manifest_tree_oid == candidate.tree_oid
),
"tests": evidence.tests.get("passed") is True,
"canary": evidence.canary.get("passed") is True,
"runtime_without_git": (
evidence.canary.get("runtime_without_git") is True
),
"writer_disabled": (
evidence.canary.get("writes_enabled") is False
),
"rollback_ready": evidence.rollback_ready is True,
"authoritative_git_binding": (
evidence.authoritative_git_readable is True
and evidence.authoritative_git_dir
== self.expected_authoritative_git_dir
),
}
candidate.tests = _sanitize(evidence.tests)
candidate.canary = _sanitize(evidence.canary)
candidate.rollback_ready = evidence.rollback_ready
candidate.git_consistency = {
"local_main_oid": evidence.local_main_oid,
"bare_main_oid": evidence.bare_main_oid,
"authoritative_git_dir": evidence.authoritative_git_dir,
"authoritative_git_readable": evidence.authoritative_git_readable,
}
candidate.gitea_consistency = {
"main_oid": evidence.gitea_main_oid,
"repository_identity": evidence.repository_identity,
}
failed = tuple(name for name, passed in checks.items() if not passed)
if failed:
candidate.failure_reason = "acceptance failed: " + ", ".join(failed)
candidate.transition(ReleaseState.FAILED)
return AcceptanceResult(False, checks, failed)
candidate.transition(ReleaseState.READY_FOR_APPROVAL)
return AcceptanceResult(True, checks, ())
except Exception as exc:
candidate.failure_reason = redact(exc)
candidate.transition(ReleaseState.FAILED)
return AcceptanceResult(
False,
{"probe": False},
("probe",),
candidate.failure_reason,
)
@dataclass(frozen=True)
class ApprovalEnvelope:
approval_id: str
owner_id: str
candidate_id: str
candidate_fingerprint: str
source_commit: str
tree_oid: str
artifact_sha256: str
manifest_sha256: str
action: str
expires_at: str
issued_at: str = field(default_factory=utc_now)
@classmethod
def for_candidate(
cls,
candidate: ReleaseCandidate,
*,
approval_id: str,
owner_id: str,
expires_at: str,
) -> "ApprovalEnvelope":
return cls(
approval_id=approval_id,
owner_id=owner_id,
candidate_id=candidate.candidate_id,
candidate_fingerprint=candidate.fingerprint,
source_commit=candidate.source_commit,
tree_oid=candidate.tree_oid,
artifact_sha256=candidate.artifact_sha256,
manifest_sha256=candidate.manifest_sha256,
action="production_runtime_switch",
expires_at=expires_at,
)
def validate(self, candidate: ReleaseCandidate) -> None:
if self.action != "production_runtime_switch":
raise ApprovalError("approval does not authorize runtime switch")
if not self.owner_id or not self.approval_id:
raise ApprovalError("owner and approval id are required")
if _parse_utc(self.expires_at) <= datetime.now(timezone.utc):
raise ApprovalError("approval has expired")
expected = (
candidate.candidate_id,
candidate.fingerprint,
candidate.source_commit,
candidate.tree_oid,
candidate.artifact_sha256,
candidate.manifest_sha256,
)
actual = (
self.candidate_id,
self.candidate_fingerprint,
self.source_commit,
self.tree_oid,
self.artifact_sha256,
self.manifest_sha256,
)
if actual != expected:
raise ApprovalError("approval fingerprint mismatch")
class ApprovalError(RuntimeError):
pass
class ApprovalLedger:
def __init__(self, root: Path):
self.root = Path(root)
def path(self, approval_id: str) -> Path:
if not CANDIDATE_ID_RE.fullmatch(str(approval_id or "")):
raise ApprovalError("invalid approval id")
root = self.root.resolve()
path = (root / f"{approval_id}.json").resolve()
if path.parent != root:
raise ApprovalError("approval path escapes ledger")
return path
def consume(self, envelope: ApprovalEnvelope) -> None:
path = self.path(envelope.approval_id)
path.parent.mkdir(parents=True, exist_ok=True)
try:
descriptor = os.open(
path,
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
0o600,
)
except FileExistsError as exc:
raise ApprovalError("approval already consumed") from exc
payload = {
"approval_id": envelope.approval_id,
"candidate_id": envelope.candidate_id,
"candidate_fingerprint": envelope.candidate_fingerprint,
"action": envelope.action,
"consumed_at": utc_now(),
}
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
json.dump(payload, output, sort_keys=True)
output.write("\n")
output.flush()
os.fsync(output.fileno())
@dataclass(frozen=True)
class SilentReleasePolicy:
automatic: frozenset[str]
owner: frozenset[str]
forbidden: frozenset[str]
@classmethod
def default(cls) -> "SilentReleasePolicy":
return cls(
automatic=frozenset(
{"build", "test", "artifact", "canary", "evidence"}
),
owner=frozenset(
{"production_runtime_switch", "production_tag"}
),
forbidden=frozenset(
{
"force_push",
"tag_rewrite",
"direct_main_push",
"baseline_mutation",
"registry_mutation",
}
),
)
@classmethod
def from_file(cls, path: Path) -> "SilentReleasePolicy":
payload = json.loads(Path(path).read_text(encoding="utf-8"))
return cls(
frozenset(payload["automatic"]),
frozenset(payload["owner"]),
frozenset(payload["forbidden"]),
)
def is_automatic(self, action: str) -> bool:
return action in self.automatic
def requires_owner(self, action: str) -> bool:
return action in self.owner
def is_forbidden(self, action: str) -> bool:
return action in self.forbidden
class DeploymentAdapter(Protocol):
def atomic_switch(
self, release_candidate: ReleaseCandidate
) -> dict[str, Any]:
...
def restart(self) -> None:
...
def health_check(
self, release_candidate: ReleaseCandidate
) -> dict[str, Any]:
...
def rollback(
self, switch_evidence: dict[str, Any]
) -> dict[str, Any]:
...
class DeploymentController:
def __init__(
self,
deployment: DeploymentAdapter,
ledger: ApprovalLedger,
policy: SilentReleasePolicy,
store: ReleaseCandidateStore | None = None,
*,
allow_legacy_unsigned_approval: bool = False,
):
self.deployment = deployment
self.ledger = ledger
self.policy = policy
self.store = store
self.allow_legacy_unsigned_approval = (
allow_legacy_unsigned_approval
)
def _save(self, candidate: ReleaseCandidate) -> None:
if self.store:
self.store.save(candidate)
def deploy(
self,
candidate: ReleaseCandidate,
approval: ApprovalEnvelope,
) -> dict[str, Any]:
if not self.allow_legacy_unsigned_approval:
raise ApprovalError(
"legacy unsigned runtime approval is disabled; "
"use SignedReleasePlanApproval"
)
if candidate.state != ReleaseState.READY_FOR_APPROVAL:
raise ApprovalError("candidate is not ready for approval")
if not self.policy.requires_owner("production_runtime_switch"):
raise ApprovalError("runtime switch is not owner governed")
for forbidden in (
"force_push",
"tag_rewrite",
"direct_main_push",
"baseline_mutation",
"registry_mutation",
):
if not self.policy.is_forbidden(forbidden):
raise ApprovalError("deployment policy is incomplete")
approval.validate(candidate)
self.ledger.consume(approval)
candidate.approval_state = "APPROVED"
candidate.transition(ReleaseState.APPROVED)
self._save(candidate)
candidate.transition(ReleaseState.DEPLOYING)
self._save(candidate)
switch_evidence: dict[str, Any] | None = None
try:
switch_evidence = self.deployment.atomic_switch(candidate)
self.deployment.restart()
health = self.deployment.health_check(candidate)
passed = (
health.get("passed") is True
and health.get("runtime_commit") == candidate.source_commit
and health.get("writes_enabled") is False
and health.get("authoritative_head_oid")
== candidate.source_commit
and health.get("repository_identity")
== candidate.gitea_consistency.get(
"repository_identity"
)
)
if not passed:
raise RuntimeError("post-deployment health check failed")
candidate.transition(ReleaseState.VERIFIED)
self._save(candidate)
return {
"verified": True,
"state": candidate.state.value,
"health": _sanitize(health),
"switch": _sanitize(switch_evidence),
}
except Exception as exc:
candidate.failure_reason = redact(exc)
if switch_evidence is not None:
try:
rollback = self.deployment.rollback(switch_evidence)
if (
rollback.get("passed") is not True
or rollback.get("writes_enabled") is not False
):
raise RuntimeError(
"post-rollback health check failed"
)
candidate.transition(ReleaseState.ROLLED_BACK)
except Exception as rollback_exc:
candidate.failure_reason = (
candidate.failure_reason
+ "; rollback failed: "
+ redact(rollback_exc)
)
candidate.transition(ReleaseState.FAILED)
else:
candidate.transition(ReleaseState.FAILED)
self._save(candidate)
return {
"verified": False,
"state": candidate.state.value,
"error": candidate.failure_reason,
"rolled_back": candidate.state == ReleaseState.ROLLED_BACK,
}
class ReleaseBuilder(Protocol):
def build(self, source_commit: str) -> ReleaseCandidate:
...
class SilentReleaseController:
"""Small external interface for prepare -> approve -> deploy."""
def __init__(
self,
*,
builder: ReleaseBuilder,
acceptance: ReleaseAcceptanceEngine,
deployment: DeploymentController,
store: ReleaseCandidateStore,
policy: SilentReleasePolicy,
):
self.builder = builder
self.acceptance = acceptance
self.deployment = deployment
self.store = store
self.policy = policy
def prepare(self, source_commit: str) -> ReleaseCandidate:
required = {"build", "test", "artifact", "canary", "evidence"}
if not all(self.policy.is_automatic(action) for action in required):
raise RuntimeError("automatic preparation policy is incomplete")
candidate = self.builder.build(
_validate_oid("source_commit", source_commit)
)
if candidate.source_commit != source_commit:
candidate.failure_reason = "builder source commit mismatch"
candidate.transition(ReleaseState.FAILED)
self.store.save(candidate)
return candidate
self.store.save(candidate)
self.acceptance.evaluate(candidate)
self.store.save(candidate)
return candidate
def deploy(
self,
candidate_id: str,
approval: ApprovalEnvelope,
) -> dict[str, Any]:
candidate = self.store.load(candidate_id)
return self.deployment.deploy(candidate, approval)
@dataclass(frozen=True)
class CommandReleaseBuilderConfig:
command: tuple[str, ...]
cwd: Path
candidate_output: Path
timeout: float = 900
class CommandReleaseBuilder:
"""Adapter for a deterministic build pipeline that emits candidate JSON."""
def __init__(self, config: CommandReleaseBuilderConfig):
self.config = config
def build(self, source_commit: str) -> ReleaseCandidate:
if not self.config.command:
raise RuntimeError("release build command is not configured")
try:
result = subprocess.run(
[*self.config.command, source_commit],
cwd=self.config.cwd,
text=True,
capture_output=True,
timeout=self.config.timeout,
env=dict(SAFE_PROCESS_ENVIRONMENT),
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise RuntimeError("release build unavailable") from exc
if result.returncode:
raise RuntimeError(
"release build failed: "
+ redact(result.stderr.strip() or result.stdout.strip())
)
candidate = ReleaseCandidate.from_dict(
json.loads(
self.config.candidate_output.read_text(encoding="utf-8")
)
)
if candidate.source_commit != source_commit:
raise RuntimeError("release build source commit mismatch")
return candidate
@dataclass(frozen=True)
class TagApprovalEnvelope:
approval_id: str
owner_id: str
version: str
tag_oid: str
target_oid: str
action: str
expires_at: str
issued_at: str = field(default_factory=utc_now)
def validate(self) -> None:
if self.action != "production_tag":
raise ApprovalError("approval does not authorize production tag")
try:
validate_release_version(self.version)
except ValueError as exc:
raise ApprovalError("invalid release version") from exc
_validate_oid("tag_oid", self.tag_oid)
_validate_oid("target_oid", self.target_oid)
if _parse_utc(self.expires_at) <= datetime.now(timezone.utc):
raise ApprovalError("approval has expired")
if not self.owner_id or not self.approval_id:
raise ApprovalError("owner and approval id are required")
class DuplicateTagConflict(RuntimeError):
pass
class TagAdapter(Protocol):
def lookup(self, version: str) -> tuple[str, str] | None:
...
def create(self, version: str, tag_oid: str, target_oid: str) -> None:
...
def verify(self, version: str) -> tuple[str, str] | None:
...
class TagController:
"""Separate approval, execution, and verification for production tags."""
def __init__(
self,
adapter: TagAdapter,
policy: SilentReleasePolicy,
ledger: ApprovalLedger | None,
*,
allow_legacy_unsigned_approval: bool = False,
):
self.adapter = adapter
self.policy = policy
self.ledger = ledger
self.allow_legacy_unsigned_approval = (
allow_legacy_unsigned_approval
)
def _require_legacy_test_mode(self) -> None:
if not self.allow_legacy_unsigned_approval:
raise ApprovalError(
"legacy unsigned tag approval is disabled; "
"use SignedReleasePlanApproval"
)
def approve(self, envelope: TagApprovalEnvelope) -> TagApprovalEnvelope:
self._require_legacy_test_mode()
if not self.policy.requires_owner("production_tag"):
raise ApprovalError("production tag is not owner governed")
if not self.policy.is_forbidden("tag_rewrite"):
raise ApprovalError("tag rewrite must be forbidden")
envelope.validate()
return envelope
def execute(
self, approved: TagApprovalEnvelope
) -> dict[str, Any]:
self._require_legacy_test_mode()
approved.validate()
existing = self.adapter.lookup(approved.version)
expected = (approved.tag_oid, approved.target_oid)
if existing and existing != expected:
raise DuplicateTagConflict(
"existing production tag has a different object or target"
)
if self.ledger:
runtime_envelope = ApprovalEnvelope(
approval_id=approved.approval_id,
owner_id=approved.owner_id,
candidate_id=approved.version,
candidate_fingerprint=hashlib.sha256(
"\0".join(expected).encode()
).hexdigest(),
source_commit=approved.target_oid,
tree_oid=approved.target_oid,
artifact_sha256="0" * 64,
manifest_sha256="0" * 64,
action=approved.action,
expires_at=approved.expires_at,
issued_at=approved.issued_at,
)
self.ledger.consume(runtime_envelope)
if existing is None:
self.adapter.create(
approved.version,
approved.tag_oid,
approved.target_oid,
)
verified = self.verify(approved)
return {"executed": existing is None, **verified}
def verify(
self, approved: TagApprovalEnvelope
) -> dict[str, Any]:
self._require_legacy_test_mode()
actual = self.adapter.verify(approved.version)
expected = (approved.tag_oid, approved.target_oid)
if actual != expected:
raise DuplicateTagConflict("production tag verification failed")
return {
"verified": True,
"version": approved.version,
"tag_oid": actual[0],
"target_oid": actual[1],
}
@dataclass(frozen=True)
class ReleaseProbeConfig:
local_repository: Path
authoritative_git_dir: Path
release_directory: Path
artifact_path: Path
manifest_path: Path
rollback_artifact: Path
gitea_api_url: str
repository_identity: str
gitea_token_file: Path
gitea_token_line: int = 2
test_command: tuple[str, ...] = ()
canary_command: tuple[str, ...] = ()
class CommandReleaseProbe:
"""Production adapter for ReleaseAcceptanceEngine; never writes Git refs."""
def __init__(self, config: ReleaseProbeConfig):
self.config = config
def _run(
self, command: list[str], *, cwd: Path | None = None
) -> subprocess.CompletedProcess[str]:
try:
return subprocess.run(
command,
cwd=cwd,
text=True,
capture_output=True,
timeout=300,
env=dict(SAFE_PROCESS_ENVIRONMENT),
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise RuntimeError("COMMAND_UNAVAILABLE") from exc
def _git(self, command: list[str]) -> str:
result = self._run(command)
if result.returncode:
raise RuntimeError("AUTHORITATIVE_GIT_UNAVAILABLE")
return result.stdout.strip()
def _api(self, path: str) -> Any:
lines = self.config.gitea_token_file.read_text(
encoding="utf-8"
).splitlines()
if not 0 < self.config.gitea_token_line <= len(lines):
raise RuntimeError("GITEA_CREDENTIAL_REFERENCE_INVALID")
token = lines[self.config.gitea_token_line - 1].strip()
request = urllib.request.Request(
self.config.gitea_api_url.rstrip("/") + path,
headers={
"Accept": "application/json",
"Authorization": f"token {token}",
"User-Agent": "aios-silent-release-controller/1",
},
)
try:
with urllib.request.urlopen(request, timeout=15) as response:
return json.loads(response.read().decode("utf-8"))
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as exc:
raise RuntimeError("GITEA_UNAVAILABLE") from exc
def _command_evidence(
self, command: tuple[str, ...], *, cwd: Path
) -> dict[str, Any]:
if not command:
return {"passed": False, "error": "command not configured"}
result = self._run(list(command), cwd=cwd)
raw = result.stdout.strip()
parsed: dict[str, Any] = {}
if raw:
try:
value = json.loads(raw)
if isinstance(value, dict):
parsed = _sanitize(value)
except json.JSONDecodeError:
pass
summary = redact((raw or result.stderr.strip())[-500:])
evidence = {
**parsed,
"passed": (
result.returncode == 0
and parsed.get("passed", True) is True
),
"exit_code": result.returncode,
"summary": summary,
}
return evidence
def collect(self, candidate: ReleaseCandidate) -> AcceptanceEvidence:
cfg = self.config
local_main = self._git(
[
"git",
"-C",
str(cfg.local_repository),
"rev-parse",
"refs/heads/main",
]
)
bare_main = self._git(
[
"git",
f"--git-dir={cfg.authoritative_git_dir}",
"rev-parse",
"refs/heads/main",
]
)
bare = self._git(
[
"git",
f"--git-dir={cfg.authoritative_git_dir}",
"rev-parse",
"--is-bare-repository",
]
)
if bare.lower() != "true":
raise RuntimeError("AUTHORITATIVE_GIT_UNAVAILABLE")
remote_url = self._git(
[
"git",
f"--git-dir={cfg.authoritative_git_dir}",
"config",
"--get",
"remote.gitea.url",
]
)
if (
_repository_identity_from_remote(remote_url)
!= cfg.repository_identity.lower()
):
raise RuntimeError("REPOSITORY_IDENTITY_MISMATCH")
repository = self._api(f"/repos/{cfg.repository_identity}")
ref = self._api(
f"/repos/{cfg.repository_identity}/git/refs/heads/main"
)
items = ref if isinstance(ref, list) else [ref]
exact = [
item
for item in items
if isinstance(item, dict)
and item.get("ref") in {"refs/heads/main", "heads/main"}
]
if len(exact) != 1:
raise RuntimeError("GITEA_MAIN_UNAVAILABLE")
gitea_main = str(
((exact[0].get("object") or {}).get("sha")) or ""
).lower()
manifest = json.loads(cfg.manifest_path.read_text(encoding="utf-8"))
canary = self._command_evidence(
cfg.canary_command,
cwd=cfg.release_directory,
)
canary["runtime_without_git"] = not any(
path.name == ".git"
for path in cfg.release_directory.rglob(".git")
)
return AcceptanceEvidence(
local_main_oid=local_main,
bare_main_oid=bare_main,
gitea_main_oid=gitea_main,
repository_identity=str(repository.get("full_name") or ""),
artifact_sha256=_sha256(cfg.artifact_path),
manifest_sha256=_sha256(cfg.manifest_path),
manifest_source_commit=str(
manifest.get("target_commit") or ""
).lower(),
manifest_tree_oid=str(
manifest.get("git_tree_oid") or ""
).lower(),
tests=self._command_evidence(
cfg.test_command,
cwd=cfg.release_directory,
),
canary=canary,
rollback_ready=cfg.rollback_artifact.is_file(),
authoritative_git_dir=str(
cfg.authoritative_git_dir.resolve()
),
authoritative_git_readable=os.access(
cfg.authoritative_git_dir, os.R_OK | os.X_OK
),
)
@dataclass(frozen=True)
class SystemdDeploymentConfig:
release_root: Path
cutover_link: Path
service_name: str
health_url: str
command_prefix: tuple[str, ...] = ()
dropin_name: str = "aios-gitops-model-api-production.conf"
service_allowlist: tuple[str, ...] = ("aios-gitops-model-api",)
dropin_allowlist: tuple[str, ...] = (
"aios-gitops-model-api-production.conf",
)
deployment_lock: Path = Path(
"/run/lock/aios-silent-release-deployment.lock"
)
class SystemdDeploymentAdapter:
"""Atomic symlink switch. It has no tag, Baseline, or Registry methods."""
def __init__(self, config: SystemdDeploymentConfig):
self.config = config
self._lock_handle = None
self._restart_evidence: dict[str, Any] = {}
if self.config.service_name not in self.config.service_allowlist:
raise ValueError("systemd service is not allowlisted")
if (
self.config.dropin_name not in self.config.dropin_allowlist
or "/" in self.config.dropin_name
or "\\" in self.config.dropin_name
):
raise ValueError("systemd drop-in is not allowlisted")
def _acquire_lock(self) -> None:
if self._lock_handle is not None:
raise RuntimeError("deployment lock is already held")
path = self.config.deployment_lock
path.parent.mkdir(parents=True, exist_ok=True)
handle = path.open("a+", encoding="utf-8")
try:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
except ImportError:
pass
self._lock_handle = handle
def _release_lock(self) -> None:
handle = self._lock_handle
self._lock_handle = None
if handle is None:
return
try:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
except ImportError:
pass
handle.close()
def _service_identity(self) -> dict[str, Any]:
result = subprocess.run(
[
*self.config.command_prefix,
"systemctl",
"show",
self.config.service_name,
"--property=MainPID",
"--property=ActiveEnterTimestampMonotonic",
"--value",
],
check=True,
capture_output=True,
text=True,
env=dict(SAFE_PROCESS_ENVIRONMENT),
)
values = result.stdout.splitlines()
if len(values) < 2:
raise RuntimeError("systemd process identity unavailable")
try:
pid = int(values[0])
started = int(values[1])
except ValueError as exc:
raise RuntimeError(
"systemd process identity is invalid"
) from exc
return {"main_pid": pid, "started_monotonic": started}
def atomic_switch(
self, candidate: ReleaseCandidate
) -> dict[str, Any]:
self._acquire_lock()
try:
release_root = self.config.release_root.resolve(strict=True)
raw_target = (
release_root
/ candidate.source_commit
/ "deployment"
/ self.config.dropin_name
)
if raw_target.is_symlink():
raise RuntimeError("release drop-in must not be a symlink")
target = raw_target.resolve(strict=True)
if target.parent.parent.parent != release_root:
raise RuntimeError("release drop-in escapes release root")
link = self.config.cutover_link
if link.exists() and not link.is_symlink():
raise RuntimeError("cutover path is not a managed symlink")
previous = os.readlink(link) if link.is_symlink() else None
if previous:
previous_path = Path(previous)
if not previous_path.is_absolute():
previous_path = link.parent / previous_path
try:
previous_path.resolve().relative_to(release_root)
except ValueError as exc:
raise RuntimeError(
"existing cutover symlink escapes release root"
) from exc
link.parent.mkdir(parents=True, exist_ok=True)
temporary = link.with_name(f".{link.name}.{os.getpid()}.tmp")
if temporary.exists() or temporary.is_symlink():
temporary.unlink()
os.symlink(str(target), temporary)
os.replace(temporary, link)
return {
"previous_target": previous,
"new_target": str(target),
"cutover_link": str(link),
}
except Exception:
self._release_lock()
raise
def restart(self) -> None:
if self._lock_handle is None:
raise RuntimeError("deployment lock is not held")
before = self._service_identity()
command = [
*self.config.command_prefix,
"systemctl",
"daemon-reload",
]
subprocess.run(
command,
check=True,
capture_output=True,
text=True,
env=dict(SAFE_PROCESS_ENVIRONMENT),
)
subprocess.run(
[
*self.config.command_prefix,
"systemctl",
"restart",
self.config.service_name,
],
check=True,
capture_output=True,
text=True,
env=dict(SAFE_PROCESS_ENVIRONMENT),
)
after = self._service_identity()
if (
after["main_pid"] <= 0
or after["started_monotonic"]
<= before["started_monotonic"]
):
raise RuntimeError(
"systemd restart did not produce a new start identity"
)
self._restart_evidence = {"before": before, "after": after}
def health_check(
self, candidate: ReleaseCandidate
) -> dict[str, Any]:
try:
with urllib.request.urlopen(
self.config.health_url, timeout=15
) as response:
payload = json.loads(response.read().decode("utf-8"))
except Exception as exc:
raise RuntimeError("runtime health check unavailable") from exc
try:
return {
"passed": response.status == 200,
"runtime_commit": payload.get("runtime_release_commit"),
"writes_enabled": payload.get("writes_enabled"),
"authoritative_head_oid": payload.get(
"authoritative_head_oid"
),
"repository_identity": payload.get("repository_identity"),
"systemd_restart": _sanitize(self._restart_evidence),
}
finally:
self._release_lock()
def rollback(
self, switch_evidence: dict[str, Any]
) -> dict[str, Any]:
try:
if self._lock_handle is None:
self._acquire_lock()
link = self.config.cutover_link
previous = switch_evidence.get("previous_target")
temporary = link.with_name(
f".{link.name}.{os.getpid()}.rollback"
)
if previous:
if temporary.exists() or temporary.is_symlink():
temporary.unlink()
os.symlink(str(previous), temporary)
os.replace(temporary, link)
elif link.is_symlink():
link.unlink()
self.restart()
try:
with urllib.request.urlopen(
self.config.health_url, timeout=15
) as response:
payload = json.loads(
response.read().decode("utf-8")
)
except Exception as exc:
raise RuntimeError(
"rollback health check unavailable"
) from exc
return {
"passed": response.status == 200,
"writes_enabled": payload.get("writes_enabled"),
"runtime_commit": payload.get("runtime_release_commit"),
"systemd_restart": _sanitize(self._restart_evidence),
}
finally:
self._release_lock()
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with Path(path).open("rb") as source:
for chunk in iter(lambda: source.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _repository_identity_from_remote(remote_url: str) -> str:
value = str(remote_url or "").strip().split("?", 1)[0].rstrip("/")
if "://" in value:
value = urllib.parse.urlparse(value).path
elif ":" in value:
value = value.rsplit(":", 1)[1]
value = re.sub(r"\.git$", "", value.strip("/"), flags=re.IGNORECASE)
parts = [part for part in value.split("/") if part]
return "/".join(parts[-2:]).lower() if len(parts) >= 2 else ""
def _atomic_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
descriptor, temporary = tempfile.mkstemp(
prefix=f".{path.name}.",
suffix=".tmp",
dir=path.parent,
)
try:
with os.fdopen(descriptor, "w", encoding="utf-8") as output:
json.dump(payload, output, indent=2, sort_keys=True)
output.write("\n")
output.flush()
os.fsync(output.fileno())
os.chmod(temporary, 0o600)
os.replace(temporary, path)
finally:
if os.path.exists(temporary):
os.unlink(temporary)