269 lines
8.3 KiB
Python
269 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Build a deterministic, secret-scanned Silent Release evidence package."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tarfile
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
|
AUTHORIZATION_PATTERN = re.compile(
|
|
rb"Authorization:\s*(?:Bearer|token)\s+([A-Za-z0-9._~+/=-]{16,})",
|
|
re.I,
|
|
)
|
|
SECRET_PATTERNS = (
|
|
re.compile(rb"(?:ghp|glpat)_[A-Za-z0-9_-]{20,}", re.I),
|
|
re.compile(
|
|
rb"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----\r?\n"
|
|
rb"[A-Za-z0-9+/=\r\n]{64,}\r?\n"
|
|
rb"-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"
|
|
),
|
|
)
|
|
ALLOWED_AUTHORIZATION_PLACEHOLDER_SHA256 = frozenset(
|
|
{
|
|
"009f7191815ec615d09d0698457d048432e9a73f62ab4f26af1875009d0afbbc",
|
|
"599a7f359d1e11124054f8afeae201f2d265b8988d91cb5b7d4dc4c9e2225c30",
|
|
"f39dac6cbaba535e2c207cd0cd8f154974223c848f727f98b3564cea569b41cf",
|
|
}
|
|
)
|
|
TARGET_TESTS = (
|
|
"tests/test_silent_release_execution_readiness.py",
|
|
"tests/test_silent_release_controller.py",
|
|
"tests/test_silent_release_hardening.py",
|
|
"tests/test_silent_release_production_contract.py",
|
|
"tests/test_chatgpt_actions_tag.py",
|
|
"tests/test_gitea_tag_publication.py",
|
|
"tests/test_authoritative_git_context.py",
|
|
)
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def atomic_json(path: Path, payload: dict[str, Any]) -> None:
|
|
temporary = path.with_suffix(path.suffix + ".tmp")
|
|
with temporary.open("w", encoding="utf-8", newline="\n") as output:
|
|
json.dump(payload, output, sort_keys=True, separators=(",", ":"))
|
|
output.write("\n")
|
|
output.flush()
|
|
os.fsync(output.fileno())
|
|
os.replace(temporary, path)
|
|
|
|
|
|
def redact_output(value: str) -> str:
|
|
value = re.sub(
|
|
r"(?i)(Authorization:\s*(?:Bearer|token)\s+)"
|
|
r"[A-Za-z0-9._~+/=-]{8,}",
|
|
r"\1<redacted>",
|
|
str(value),
|
|
)
|
|
value = re.sub(
|
|
r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----.*?"
|
|
r"-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----",
|
|
"<redacted-private-key>",
|
|
value,
|
|
flags=re.DOTALL,
|
|
)
|
|
return value
|
|
|
|
|
|
def git(git_dir: Path, *arguments: str) -> str:
|
|
completed = subprocess.run(
|
|
["git", f"--git-dir={git_dir}", *arguments],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=60,
|
|
env={"PATH": os.environ.get("PATH", "")},
|
|
)
|
|
return completed.stdout.strip()
|
|
|
|
|
|
def run_evidence(
|
|
command: list[str],
|
|
*,
|
|
cwd: Path,
|
|
output: Path,
|
|
timeout: int,
|
|
) -> dict[str, Any]:
|
|
completed = subprocess.run(
|
|
command,
|
|
cwd=cwd,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
env={
|
|
"HOME": str(cwd / ".builder-home"),
|
|
"PATH": os.environ.get("PATH", ""),
|
|
"PYTHONDONTWRITEBYTECODE": "1",
|
|
},
|
|
)
|
|
record = {
|
|
"command": command,
|
|
"exit_code": completed.returncode,
|
|
"stderr": redact_output(completed.stderr[-20000:]),
|
|
"stdout": redact_output(completed.stdout[-20000:]),
|
|
}
|
|
atomic_json(output, record)
|
|
if completed.returncode != 0:
|
|
raise RuntimeError(f"trusted builder command failed: {command[0]}")
|
|
return record
|
|
|
|
|
|
def secret_scan(root: Path) -> dict[str, Any]:
|
|
findings: list[str] = []
|
|
for path in sorted(item for item in root.rglob("*") if item.is_file()):
|
|
if path.stat().st_size > 5 * 1024 * 1024:
|
|
continue
|
|
data = path.read_bytes()
|
|
authorization_tokens = AUTHORIZATION_PATTERN.findall(data)
|
|
unsafe_authorization = any(
|
|
hashlib.sha256(token.lower()).hexdigest()
|
|
not in ALLOWED_AUTHORIZATION_PLACEHOLDER_SHA256
|
|
for token in authorization_tokens
|
|
)
|
|
if unsafe_authorization or any(
|
|
pattern.search(data) for pattern in SECRET_PATTERNS
|
|
):
|
|
findings.append(path.relative_to(root).as_posix())
|
|
return {"findings": findings, "passed": not findings}
|
|
|
|
|
|
def build(git_dir: Path, commit: str, output: Path) -> dict[str, Any]:
|
|
commit = git(git_dir, "rev-parse", f"{commit}^{{commit}}")
|
|
if not OID_RE.fullmatch(commit):
|
|
raise ValueError("commit must resolve to a full OID")
|
|
tree_oid = git(git_dir, "rev-parse", f"{commit}^{{tree}}")
|
|
if output.exists():
|
|
raise FileExistsError("artifact output already exists")
|
|
output.mkdir(parents=True, mode=0o700)
|
|
source_tar = output / "source.tar"
|
|
subprocess.run(
|
|
[
|
|
"git",
|
|
f"--git-dir={git_dir}",
|
|
"archive",
|
|
"--format=tar",
|
|
f"--output={source_tar}",
|
|
commit,
|
|
],
|
|
check=True,
|
|
timeout=120,
|
|
env={"PATH": os.environ.get("PATH", "")},
|
|
)
|
|
canary = output / "canary-src"
|
|
canary.mkdir(mode=0o700)
|
|
with tarfile.open(source_tar, "r") as archive:
|
|
archive.extractall(canary, filter="data")
|
|
if any(path.name == ".git" for path in canary.rglob(".git")):
|
|
raise RuntimeError("no-.git canary contains Git metadata")
|
|
|
|
test_record = run_evidence(
|
|
["python3", "-m", "pytest", "-q", *TARGET_TESTS],
|
|
cwd=canary,
|
|
output=output / "test-evidence.json",
|
|
timeout=600,
|
|
)
|
|
fixture_record = run_evidence(
|
|
["bash", "tests/run_live_gitea_fixture.sh"],
|
|
cwd=canary,
|
|
output=output / "live-fixture-evidence.json",
|
|
timeout=600,
|
|
)
|
|
canary_record = {
|
|
"git_directory_present": False,
|
|
"live_fixture_exit_code": fixture_record["exit_code"],
|
|
"passed": True,
|
|
"target_test_exit_code": test_record["exit_code"],
|
|
}
|
|
atomic_json(output / "canary-evidence.json", canary_record)
|
|
|
|
scan = secret_scan(canary)
|
|
atomic_json(output / "secret-scan.json", scan)
|
|
if not scan["passed"]:
|
|
raise RuntimeError("secret scan failed")
|
|
|
|
manifest_files: list[dict[str, Any]] = []
|
|
tracked = git(git_dir, "ls-tree", "-r", "--name-only", commit).splitlines()
|
|
for relative in tracked:
|
|
path = canary / relative
|
|
if path.is_file():
|
|
manifest_files.append(
|
|
{
|
|
"path": relative,
|
|
"sha256": sha256(path),
|
|
"size": path.stat().st_size,
|
|
}
|
|
)
|
|
manifest = {
|
|
"commit": commit,
|
|
"file_count": len(manifest_files),
|
|
"files": manifest_files,
|
|
"tree_oid": tree_oid,
|
|
}
|
|
atomic_json(output / "manifest.json", manifest)
|
|
shutil.copyfile(source_tar, output / "rollback-source.tar")
|
|
|
|
closure = {
|
|
"artifact_sha256": sha256(source_tar),
|
|
"canary_evidence_sha256": sha256(output / "canary-evidence.json"),
|
|
"commit": commit,
|
|
"live_fixture_evidence_sha256": sha256(
|
|
output / "live-fixture-evidence.json"
|
|
),
|
|
"manifest_sha256": sha256(output / "manifest.json"),
|
|
"no_git_canary": True,
|
|
"rollback_artifact_sha256": sha256(
|
|
output / "rollback-source.tar"
|
|
),
|
|
"secret_scan_sha256": sha256(output / "secret-scan.json"),
|
|
"test_evidence_sha256": sha256(output / "test-evidence.json"),
|
|
"tree_oid": tree_oid,
|
|
}
|
|
atomic_json(output / "package-closure.json", closure)
|
|
closure["package_closure_sha256"] = sha256(
|
|
output / "package-closure.json"
|
|
)
|
|
checksummed = sorted(
|
|
path
|
|
for path in output.iterdir()
|
|
if path.is_file() and path.name != "checksums.sha256"
|
|
)
|
|
with (output / "checksums.sha256").open(
|
|
"w", encoding="utf-8", newline="\n"
|
|
) as checksums:
|
|
for path in checksummed:
|
|
checksums.write(f"{sha256(path)} {path.name}\n")
|
|
return closure
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--git-dir", required=True, type=Path)
|
|
parser.add_argument("--commit", required=True)
|
|
parser.add_argument("--output", required=True, type=Path)
|
|
arguments = parser.parse_args()
|
|
result = build(
|
|
arguments.git_dir.resolve(),
|
|
arguments.commit,
|
|
arguments.output.resolve(),
|
|
)
|
|
print(json.dumps(result, sort_keys=True))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|