532 lines
19 KiB
Python
532 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Governed, fail-closed projection of verified release evidence into Builder storage.
|
|
|
|
This module never invents candidate/release evidence. A separate governed release
|
|
producer must first write a complete, root-owned evidence closure below
|
|
``source_root/<version>``. The materializer re-validates that closure with the
|
|
same ``ServerOwnedEvidenceSource``/``DeterministicEvidenceCollector`` code used by
|
|
Trusted Candidate Builder and atomically projects the exact bytes into the
|
|
Builder-owned evidence root.
|
|
|
|
Readiness is deliberately orthogonal: an empty Builder evidence root does not make
|
|
Builder infrastructure globally unavailable, but every BuildTrustedCandidateV1
|
|
request still requires a valid materialized closure.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from dataclasses import replace
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import secrets
|
|
import shutil
|
|
import stat
|
|
from typing import Any, Callable, Mapping
|
|
import subprocess
|
|
|
|
from aios_trusted_candidate_builder import (
|
|
BuilderError,
|
|
DeterministicEvidenceCollector,
|
|
ServerOwnedEvidenceSource,
|
|
TrustedCandidateBuilderConfig,
|
|
load_builder_policy,
|
|
)
|
|
from release_migration_plan import FileMigrationPlanResolver
|
|
from release_source_admission import (
|
|
ReleaseSourceAdmissionStore,
|
|
open_source_admission_store,
|
|
source_admission_policy_sha256,
|
|
)
|
|
from release_version import validate_release_version
|
|
|
|
|
|
MATERIALIZATION_SCHEMA = "aios.trusted-builder-evidence-materialization.v1"
|
|
SOURCE_ADMISSION_CREDENTIAL = "source_admission_hmac"
|
|
|
|
|
|
class EvidenceMaterializationError(RuntimeError):
|
|
"""Stable, non-sensitive materialization failure."""
|
|
|
|
|
|
def _canonical_json(value: Mapping[str, Any]) -> bytes:
|
|
return json.dumps(
|
|
value,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
ensure_ascii=True,
|
|
).encode("utf-8")
|
|
|
|
|
|
def _file_sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with Path(path).open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _validate_oid(value: object, name: str) -> str:
|
|
normalized = str(value or "").strip().lower()
|
|
if (
|
|
len(normalized) != 40
|
|
or any(char not in "0123456789abcdef" for char in normalized)
|
|
):
|
|
raise EvidenceMaterializationError(f"{name.upper()}_INVALID")
|
|
return normalized
|
|
|
|
|
|
def _validate_identifier(value: object, name: str) -> str:
|
|
normalized = str(value or "").strip()
|
|
if (
|
|
not normalized
|
|
or len(normalized) > 255
|
|
or normalized[0] in ".-"
|
|
or any(
|
|
char not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._:-"
|
|
for char in normalized
|
|
)
|
|
):
|
|
raise EvidenceMaterializationError(f"{name.upper()}_INVALID")
|
|
return normalized
|
|
|
|
|
|
def _safe_directory(path: Path, *, expected_owner_uid: int | None) -> Path:
|
|
candidate = Path(path)
|
|
try:
|
|
metadata = candidate.stat(follow_symlinks=False)
|
|
resolved = candidate.resolve(strict=True)
|
|
except OSError as exc:
|
|
raise EvidenceMaterializationError("EVIDENCE_DIRECTORY_UNAVAILABLE") from exc
|
|
if candidate.is_symlink() or not stat.S_ISDIR(metadata.st_mode):
|
|
raise EvidenceMaterializationError("EVIDENCE_DIRECTORY_INVALID")
|
|
if os.name != "nt" and metadata.st_mode & 0o022:
|
|
raise EvidenceMaterializationError("EVIDENCE_DIRECTORY_PERMISSIONS_INVALID")
|
|
if (
|
|
os.name != "nt"
|
|
and expected_owner_uid is not None
|
|
and metadata.st_uid != expected_owner_uid
|
|
):
|
|
raise EvidenceMaterializationError("EVIDENCE_DIRECTORY_OWNER_INVALID")
|
|
return resolved
|
|
|
|
|
|
def _safe_regular(
|
|
root: Path,
|
|
relative: str,
|
|
*,
|
|
expected_owner_uid: int | None,
|
|
) -> Path:
|
|
root = _safe_directory(root, expected_owner_uid=expected_owner_uid)
|
|
relative_path = Path(relative)
|
|
if relative_path.is_absolute() or any(
|
|
part in {"", ".", ".."} for part in relative_path.parts
|
|
):
|
|
raise EvidenceMaterializationError("EVIDENCE_PATH_INVALID")
|
|
current = root
|
|
for part in relative_path.parts:
|
|
current = current / part
|
|
if current.is_symlink():
|
|
raise EvidenceMaterializationError("EVIDENCE_SYMLINK_DENIED")
|
|
try:
|
|
resolved = (root / relative_path).resolve(strict=True)
|
|
resolved.relative_to(root)
|
|
metadata = resolved.stat(follow_symlinks=False)
|
|
except (OSError, ValueError) as exc:
|
|
raise EvidenceMaterializationError("EVIDENCE_INPUT_UNAVAILABLE") from exc
|
|
if not stat.S_ISREG(metadata.st_mode):
|
|
raise EvidenceMaterializationError("EVIDENCE_SPECIAL_FILE_DENIED")
|
|
if os.name != "nt" and metadata.st_mode & 0o022:
|
|
raise EvidenceMaterializationError("EVIDENCE_INPUT_PERMISSIONS_INVALID")
|
|
if (
|
|
os.name != "nt"
|
|
and expected_owner_uid is not None
|
|
and metadata.st_uid != expected_owner_uid
|
|
):
|
|
raise EvidenceMaterializationError("EVIDENCE_INPUT_OWNER_INVALID")
|
|
return resolved
|
|
|
|
|
|
def _fsync_directory(path: Path) -> None:
|
|
if os.name == "nt":
|
|
return
|
|
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
|
try:
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
def _copy_exact(source: Path, target: Path, *, mode: int = 0o640) -> None:
|
|
target.parent.mkdir(parents=True, exist_ok=True, mode=0o750)
|
|
descriptor = os.open(
|
|
target,
|
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
|
|
| getattr(os, "O_BINARY", 0),
|
|
mode,
|
|
)
|
|
try:
|
|
with source.open("rb") as src:
|
|
while True:
|
|
chunk = src.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
os.write(descriptor, chunk)
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
os.chmod(target, mode)
|
|
|
|
|
|
def _write_json_exclusive(path: Path, payload: Mapping[str, Any]) -> str:
|
|
raw = _canonical_json(payload) + b"\n"
|
|
descriptor = os.open(
|
|
path,
|
|
os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
|
|
| getattr(os, "O_BINARY", 0),
|
|
0o640,
|
|
)
|
|
try:
|
|
os.write(descriptor, raw)
|
|
os.fsync(descriptor)
|
|
finally:
|
|
os.close(descriptor)
|
|
os.chmod(path, 0o640)
|
|
return hashlib.sha256(raw).hexdigest()
|
|
|
|
|
|
class TrustedBuilderEvidenceMaterializer:
|
|
"""Verify a governed evidence closure and atomically project exact bytes."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
builder_policy: Path,
|
|
authority_git_dir: Path,
|
|
source_root: Path,
|
|
output_root: Path,
|
|
source_admission_root: Path | None = None,
|
|
source_admission_hmac: bytes | None = None,
|
|
migration_plan_root: Path | None = None,
|
|
migration_profile_root: Path | None = None,
|
|
expected_owner_uid: int | None = 0,
|
|
git_runner: Callable[..., Any] = subprocess.run,
|
|
source_admission_resolver: Callable[[str, str | None, str], Mapping[str, str]] | None = None,
|
|
migration_plan_resolver: Callable[[str, str, str], Any] | None = None,
|
|
) -> None:
|
|
self.builder_policy = Path(builder_policy)
|
|
self.authority_git_dir = Path(authority_git_dir)
|
|
self.source_root = Path(source_root)
|
|
self.output_root = Path(output_root)
|
|
self.expected_owner_uid = expected_owner_uid
|
|
self.git_runner = git_runner
|
|
|
|
try:
|
|
policy = load_builder_policy(self.builder_policy)
|
|
config = TrustedCandidateBuilderConfig.from_policy(policy)
|
|
except BuilderError as exc:
|
|
raise EvidenceMaterializationError(str(exc)) from exc
|
|
|
|
self.source_admission_store: ReleaseSourceAdmissionStore | None = None
|
|
if source_admission_resolver is not None:
|
|
config = replace(
|
|
config,
|
|
source_admission_resolver=source_admission_resolver,
|
|
)
|
|
elif source_admission_root is not None:
|
|
if not source_admission_hmac or len(source_admission_hmac) < 32:
|
|
raise EvidenceMaterializationError("SOURCE_ADMISSION_HMAC_UNAVAILABLE")
|
|
self.source_admission_store = open_source_admission_store(
|
|
Path(source_admission_root),
|
|
repository=config.repository,
|
|
policy=self.builder_policy,
|
|
hmac_key=source_admission_hmac,
|
|
expected_owner_uid=expected_owner_uid,
|
|
writable=False,
|
|
)
|
|
config = replace(
|
|
config,
|
|
source_admission_resolver=self.source_admission_store.resolve_recipe,
|
|
)
|
|
|
|
if migration_plan_resolver is not None:
|
|
config = replace(config, migration_plan_resolver=migration_plan_resolver)
|
|
elif migration_plan_root is not None or migration_profile_root is not None:
|
|
if migration_plan_root is None or migration_profile_root is None:
|
|
raise EvidenceMaterializationError(
|
|
"MIGRATION_PLAN_CONFIGURATION_INVALID"
|
|
)
|
|
resolver = FileMigrationPlanResolver(
|
|
Path(migration_plan_root),
|
|
profile_root=Path(migration_profile_root),
|
|
expected_owner_uid=expected_owner_uid,
|
|
)
|
|
config = replace(config, migration_plan_resolver=resolver.resolve)
|
|
|
|
self.config = config
|
|
|
|
@property
|
|
def policy_file_sha256(self) -> str:
|
|
return source_admission_policy_sha256(self.builder_policy)
|
|
|
|
def _collector_for(self, evidence_root: Path) -> DeterministicEvidenceCollector:
|
|
source = ServerOwnedEvidenceSource(
|
|
self.config,
|
|
self.authority_git_dir,
|
|
evidence_root,
|
|
expected_owner_uid=self.expected_owner_uid,
|
|
git_runner=self.git_runner,
|
|
)
|
|
return DeterministicEvidenceCollector(
|
|
self.config,
|
|
evidence_source=source,
|
|
)
|
|
|
|
def _relative_files(self, version: str) -> tuple[str, ...]:
|
|
relative = [f"{version}/receipt.json"]
|
|
relative.extend(
|
|
template.format(version=version)
|
|
for template in self.config.evidence_layout.values()
|
|
)
|
|
return tuple(dict.fromkeys(relative))
|
|
|
|
def _validate_existing(
|
|
self,
|
|
*,
|
|
version: str,
|
|
candidate_id: str,
|
|
source_commit: str,
|
|
expected_bundle: Mapping[str, Any],
|
|
) -> dict[str, Any]:
|
|
try:
|
|
observed = self._collector_for(self.output_root).collect(
|
|
version,
|
|
self.config.resolve_recipe(version, candidate_id, source_commit),
|
|
)
|
|
except (BuilderError, RuntimeError, ValueError) as exc:
|
|
raise EvidenceMaterializationError("EVIDENCE_DESTINATION_CONFLICT") from exc
|
|
if _canonical_json(dict(observed)) != _canonical_json(dict(expected_bundle)):
|
|
raise EvidenceMaterializationError("EVIDENCE_DESTINATION_CONFLICT")
|
|
record_path = self.output_root / version / "materialization.json"
|
|
if not record_path.is_file():
|
|
raise EvidenceMaterializationError("EVIDENCE_DESTINATION_CONFLICT")
|
|
try:
|
|
record = json.loads(record_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise EvidenceMaterializationError("EVIDENCE_DESTINATION_CONFLICT") from exc
|
|
if not isinstance(record, Mapping):
|
|
raise EvidenceMaterializationError("EVIDENCE_DESTINATION_CONFLICT")
|
|
return dict(record)
|
|
|
|
def materialize(
|
|
self,
|
|
*,
|
|
version: str,
|
|
candidate_id: str,
|
|
source_commit: str,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
version = validate_release_version(version)
|
|
except ValueError as exc:
|
|
raise EvidenceMaterializationError("VERSION_INVALID") from exc
|
|
candidate_id = _validate_identifier(candidate_id, "candidate_id")
|
|
source_commit = _validate_oid(source_commit, "source_commit")
|
|
|
|
_safe_directory(
|
|
self.source_root,
|
|
expected_owner_uid=self.expected_owner_uid,
|
|
)
|
|
_safe_directory(
|
|
self.output_root,
|
|
expected_owner_uid=self.expected_owner_uid,
|
|
)
|
|
|
|
try:
|
|
recipe = self.config.resolve_recipe(
|
|
version,
|
|
candidate_id,
|
|
source_commit,
|
|
)
|
|
bundle = self._collector_for(self.source_root).collect(
|
|
version,
|
|
recipe,
|
|
)
|
|
except BuilderError as exc:
|
|
raise EvidenceMaterializationError(str(exc)) from exc
|
|
except (RuntimeError, ValueError) as exc:
|
|
raise EvidenceMaterializationError("EVIDENCE_SOURCE_INVALID") from exc
|
|
|
|
admission_id = ""
|
|
if self.source_admission_store is not None:
|
|
try:
|
|
admission_id = str(
|
|
self.source_admission_store.lookup_source(source_commit)[
|
|
"admission_id"
|
|
]
|
|
)
|
|
except Exception as exc:
|
|
raise EvidenceMaterializationError(
|
|
"SOURCE_ADMISSION_INVALID"
|
|
) from exc
|
|
|
|
receipt_path = _safe_regular(
|
|
self.source_root,
|
|
f"{version}/receipt.json",
|
|
expected_owner_uid=self.expected_owner_uid,
|
|
)
|
|
receipt_sha256 = _file_sha256(receipt_path)
|
|
release_plan = dict(bundle["release_plan"])
|
|
migration = release_plan.get("migration")
|
|
migration_sha256 = ""
|
|
if isinstance(migration, Mapping):
|
|
migration_sha256 = str(migration.get("sha256") or "")
|
|
|
|
materialization_payload = {
|
|
"schema": MATERIALIZATION_SCHEMA,
|
|
"repository": self.config.repository,
|
|
"version": version,
|
|
"candidate_id": candidate_id,
|
|
"source_commit": source_commit,
|
|
"tree_oid": str(bundle["candidate"]["tree_oid"]),
|
|
"source_admission_id": admission_id,
|
|
"builder_policy_file_sha256": self.policy_file_sha256,
|
|
"receipt_sha256": receipt_sha256,
|
|
"migration_sha256": migration_sha256,
|
|
"evidence_digests": dict(bundle["evidence_digests"]),
|
|
}
|
|
materialization_id = hashlib.sha256(
|
|
_canonical_json(materialization_payload)
|
|
).hexdigest()
|
|
materialization_payload["materialization_id"] = (
|
|
"evidence-materialization-" + materialization_id[:24]
|
|
)
|
|
|
|
target = self.output_root / version
|
|
if target.exists() or target.is_symlink():
|
|
existing = self._validate_existing(
|
|
version=version,
|
|
candidate_id=candidate_id,
|
|
source_commit=source_commit,
|
|
expected_bundle=bundle,
|
|
)
|
|
if (
|
|
existing.get("materialization_id")
|
|
!= materialization_payload["materialization_id"]
|
|
):
|
|
raise EvidenceMaterializationError(
|
|
"EVIDENCE_DESTINATION_CONFLICT"
|
|
)
|
|
return {
|
|
**materialization_payload,
|
|
"status": "EVIDENCE_MATERIALIZED",
|
|
"idempotent": True,
|
|
}
|
|
|
|
temporary = self.output_root / (
|
|
f".{version}.{os.getpid()}.{secrets.token_hex(8)}.tmp"
|
|
)
|
|
if temporary.exists() or temporary.is_symlink():
|
|
raise EvidenceMaterializationError("EVIDENCE_TEMP_CONFLICT")
|
|
temporary.mkdir(mode=0o750)
|
|
try:
|
|
for relative in self._relative_files(version):
|
|
source = _safe_regular(
|
|
self.source_root,
|
|
relative,
|
|
expected_owner_uid=self.expected_owner_uid,
|
|
)
|
|
destination_relative = Path(relative).relative_to(version)
|
|
destination = temporary / destination_relative
|
|
_copy_exact(source, destination)
|
|
if _file_sha256(destination) != _file_sha256(source):
|
|
raise EvidenceMaterializationError(
|
|
"EVIDENCE_COPY_DIGEST_MISMATCH"
|
|
)
|
|
materialization_sha256 = _write_json_exclusive(
|
|
temporary / "materialization.json",
|
|
materialization_payload,
|
|
)
|
|
_fsync_directory(temporary)
|
|
os.replace(temporary, target)
|
|
_fsync_directory(self.output_root)
|
|
except Exception:
|
|
shutil.rmtree(temporary, ignore_errors=True)
|
|
raise
|
|
|
|
try:
|
|
observed = self._collector_for(self.output_root).collect(
|
|
version,
|
|
recipe,
|
|
)
|
|
except (BuilderError, RuntimeError, ValueError) as exc:
|
|
raise EvidenceMaterializationError(
|
|
"EVIDENCE_POST_WRITE_VERIFY_FAILED"
|
|
) from exc
|
|
if _canonical_json(dict(observed)) != _canonical_json(dict(bundle)):
|
|
raise EvidenceMaterializationError(
|
|
"EVIDENCE_POST_WRITE_VERIFY_FAILED"
|
|
)
|
|
|
|
return {
|
|
**materialization_payload,
|
|
"materialization_sha256": materialization_sha256,
|
|
"status": "EVIDENCE_MATERIALIZED",
|
|
"idempotent": False,
|
|
}
|
|
|
|
|
|
def _credential(name: str) -> bytes:
|
|
root = os.environ.get("CREDENTIALS_DIRECTORY", "").strip()
|
|
if not root:
|
|
raise EvidenceMaterializationError("CREDENTIALS_DIRECTORY_REQUIRED")
|
|
path = Path(root) / name
|
|
try:
|
|
value = path.read_bytes().strip()
|
|
except OSError as exc:
|
|
raise EvidenceMaterializationError(
|
|
f"{name.upper()}_UNAVAILABLE"
|
|
) from exc
|
|
if len(value) < 32:
|
|
raise EvidenceMaterializationError(f"{name.upper()}_INVALID")
|
|
return value
|
|
|
|
|
|
def _main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("operation", choices=("materialize",))
|
|
parser.add_argument("--version", required=True)
|
|
parser.add_argument("--candidate-id", required=True)
|
|
parser.add_argument("--source-commit", required=True)
|
|
parser.add_argument("--builder-policy", required=True, type=Path)
|
|
parser.add_argument("--authority-git-dir", required=True, type=Path)
|
|
parser.add_argument("--source-root", required=True, type=Path)
|
|
parser.add_argument("--output-root", required=True, type=Path)
|
|
parser.add_argument("--source-admission-root", required=True, type=Path)
|
|
parser.add_argument("--migration-plan-root", required=True, type=Path)
|
|
parser.add_argument("--migration-profile-root", required=True, type=Path)
|
|
arguments = parser.parse_args()
|
|
|
|
try:
|
|
result = TrustedBuilderEvidenceMaterializer(
|
|
builder_policy=arguments.builder_policy,
|
|
authority_git_dir=arguments.authority_git_dir,
|
|
source_root=arguments.source_root,
|
|
output_root=arguments.output_root,
|
|
source_admission_root=arguments.source_admission_root,
|
|
source_admission_hmac=_credential(SOURCE_ADMISSION_CREDENTIAL),
|
|
migration_plan_root=arguments.migration_plan_root,
|
|
migration_profile_root=arguments.migration_profile_root,
|
|
).materialize(
|
|
version=arguments.version,
|
|
candidate_id=arguments.candidate_id,
|
|
source_commit=arguments.source_commit,
|
|
)
|
|
except EvidenceMaterializationError as exc:
|
|
raise SystemExit(str(exc)) from exc
|
|
print(json.dumps(result, sort_keys=True, separators=(",", ":")))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
_main()
|