aios/aios
1
0
Fork 0
Code Issues Pull Requests 5 Actions Packages Projects Releases Wiki Activity

feat(gitops): add persistent shadow acceptance worker

Browse Source
This commit is contained in:
chatgpt-gitops-bot
2026-07-24 03:06:20 +08:00
parent d23aafc04c
commit 58511395c1
3 changed files with 506 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
active-runtime/app/silent_release_execution.py
+35 -4
View File
@@ -477,6 +477,7 @@ def serve_shadow(
repository: Path,
host: str,
port: int,
poll_seconds: float,
) -> None:
service = SilentReleaseActionService.shadow(state_root)
controller = PersistentSilentReleaseController(
@@ -485,24 +486,50 @@ def serve_shadow(
main_probe=git_main_probe(repository),
)
controller.run_once()
stop = threading.Event()
def controller_loop() -> None:
while not stop.wait(poll_seconds):
try:
controller.run_once()
except Exception as exc: # fail closed and keep the API available
_durable_json(
controller.status_path,
{
"error": type(exc).__name__,
"execution_enabled": False,
"shadow_mode": True,
"side_effects_executed": False,
"status": "FAIL_CLOSED",
},
)
handler = type(
"ConfiguredShadowActionHandler",
(ShadowActionHandler,),
{"service": service, "controller": controller},
)
server = ThreadingHTTPServer((host, port), handler)
thread = threading.Thread(
api_thread = threading.Thread(
target=server.serve_forever,
name="silent-release-shadow-api",
daemon=False,
)
thread.start()
controller_thread = threading.Thread(
target=controller_loop,
name="silent-release-shadow-controller",
daemon=True,
)
api_thread.start()
controller_thread.start()
try:
while thread.is_alive():
thread.join(timeout=1)
while api_thread.is_alive():
api_thread.join(timeout=1)
finally:
stop.set()
server.shutdown()
server.server_close()
controller_thread.join(timeout=max(1.0, poll_seconds + 1.0))
def main() -> None:
@@ -511,7 +538,10 @@ def main() -> None:
parser.add_argument("--repository", required=True, type=Path)
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", default=18891, type=int)
parser.add_argument("--poll-seconds", default=30.0, type=float)
arguments = parser.parse_args()
if arguments.poll_seconds <= 0:
raise SystemExit("poll seconds must be positive")
if os.environ.get("AIOS_GITOPS_WRITES_ENABLED", "false").lower() != "false":
raise SystemExit("GitOps writes must remain disabled")
if (
@@ -526,6 +556,7 @@ def main() -> None:
repository=arguments.repository,
host=arguments.host,
port=arguments.port,
poll_seconds=arguments.poll_seconds,
)
deploy/systemd/aios-silent-release-controller.service
+1 -1
View File
@@ -11,7 +11,7 @@ UMask=0077
Environment=AIOS_GITOPS_WRITES_ENABLED=false
Environment=AIOS_SILENT_RELEASE_EXECUTION_ENABLED=false
Environment=PYTHONDONTWRITEBYTECODE=1
ExecStart=/usr/bin/python3 /usr/libexec/aios-silent-release-shadow/current/active-runtime/app/silent_release_execution.py --state-root /var/lib/aios-silent-release-shadow --repository /data/aios-gitops/repo.git --host 127.0.0.1 --port 18891
ExecStart=/usr/bin/python3 /usr/libexec/aios-silent-release-shadow/current/active-runtime/app/silent_release_execution.py --state-root /var/lib/aios-silent-release-shadow --repository /data/aios-gitops/repo.git --host 127.0.0.1 --port 18891 --poll-seconds 30
Restart=on-failure
RestartSec=5
NoNewPrivileges=true
scripts/run_silent_release_shadow_acceptance.py
+470
View File
@@ -0,0 +1,470 @@
#!/usr/bin/env python3
"""Run two zero-side-effect Silent Release candidates in Shadow mode."""
from __future__ import annotations
import argparse
import dataclasses
import hashlib
import json
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import (
Ed25519PrivateKey,
)
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "active-runtime" / "app"))
from silent_release_controller import ReleaseCandidate, ReleaseState # noqa: E402
from silent_release_execution import ( # noqa: E402
LocalTestApprovalSigner,
LocalTestBuilderSigner,
PersistentSilentReleaseController,
SignedTrustedBuilderAdapter,
)
from silent_release_hardening import ( # noqa: E402
ApprovalKeyRing,
ApprovalRevocationStore,
BuilderAttestationVerifier,
DeploymentPlan,
FinalizationPlan,
ReleasePlan,
SignedApprovalVerifier,
SignedReleasePlanApproval,
SilentReleaseActionService,
TagPlan,
TagPublicationProbeEvidence,
TrustedCandidateBundle,
TrustedReleasePolicy,
classify_tag_publication,
)
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as source:
for block in iter(lambda: source.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def write_json(path: Path, value: object) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(
json.dumps(value, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
temporary.replace(path)
def oid(seed: str) -> str:
return hashlib.sha1(seed.encode("utf-8")).hexdigest()
def candidate_id(event_id: str, source_commit: str) -> str:
return "rc-shadow-" + hashlib.sha256(
f"{event_id}\0{source_commit}".encode("utf-8")
).hexdigest()[:20]
def make_plan(arguments: argparse.Namespace, candidate: str) -> ReleasePlan:
version = "shadow-" + candidate.removeprefix("rc-shadow-")
tag_oid = oid("tag:" + candidate)
approval_plan_id = "shadow-plan-" + candidate.removeprefix("rc-shadow-")
release_path = str(Path(arguments.release_path))
previous_baseline = arguments.baseline_before_sha256
return ReleasePlan(
deployment=DeploymentPlan(
service="aios-silent-release-controller",
release_path=release_path,
release_root=str(Path(release_path).parent.parent),
cutover_link=str(Path(release_path).parent.parent / "current"),
dropin_path=str(
Path(release_path)
/ "deploy"
/ "systemd"
/ "aios-silent-release-controller.service"
),
dropin_sha256=arguments.systemd_unit_sha256,
systemd_unit_sha256=arguments.systemd_unit_sha256,
rollback_path=str(Path(arguments.rollback_artifact)),
rollback_sha256=arguments.artifact_sha256,
source_commit=arguments.source_commit,
tree_oid=arguments.tree_oid,
artifact_sha256=arguments.artifact_sha256,
manifest_sha256=arguments.manifest_sha256,
package_closure_sha256=arguments.package_closure_sha256,
authoritative_git_dir=str(Path(arguments.repository)),
repository_identity=arguments.repository_identity,
writer_expected=False,
baseline_before_sha256=previous_baseline,
registry_before_sha256=arguments.registry_before_sha256,
production_tag_refset_before_sha256=(
arguments.tag_refset_before_sha256
),
health_contract_version="shadow-health/v1",
timeout_policy={"health_seconds": 10},
retry_policy={"health_attempts": 1},
policy_sha256=arguments.policy_sha256,
health_contract={
"execution_enabled": False,
"shadow_mode": True,
"side_effects_executed": False,
},
),
tag=TagPlan(
version=version,
tag_type="annotated",
expected_tag_oid=tag_oid,
expected_target_oid=arguments.source_commit,
evidence_closure_oid=arguments.source_commit,
approval_plan_id=approval_plan_id,
repository_identity=arguments.repository_identity,
gitea_owner="aios",
gitea_repository="aios",
current_remote_tag_oid=None,
remote_absent_evidence_sha256=hashlib.sha256(
f"shadow-only:{candidate}".encode("utf-8")
).hexdigest(),
verification_contract="shadow-no-tag-write/v1",
),
finalization=FinalizationPlan(
version=version,
expected_tag_oid=tag_oid,
expected_target_oid=arguments.source_commit,
expected_previous_baseline_sha256=previous_baseline,
registry_expected_record={
"shadow_only": True,
"target_oid": arguments.source_commit,
},
baseline_expected_record={
"shadow_only": True,
"target_oid": arguments.source_commit,
},
final_consistency_contract="shadow-no-finalization/v1",
),
policy_sha256=arguments.policy_sha256,
)
def make_candidate(
arguments: argparse.Namespace, event_id: str
) -> ReleaseCandidate:
return ReleaseCandidate(
candidate_id=candidate_id(event_id, arguments.source_commit),
source_commit=arguments.source_commit,
tree_oid=arguments.tree_oid,
artifact_sha256=arguments.artifact_sha256,
manifest_sha256=arguments.manifest_sha256,
tests={"passed": True, "source": "trusted-artifact-evidence"},
canary={"passed": True, "without_git": True},
rollback_ready=True,
git_consistency={"consistent": True},
gitea_consistency={"consistent": True},
approval_state="PENDING",
state=ReleaseState.READY_FOR_APPROVAL,
)
def sign_approval(
service: SilentReleaseActionService,
signer: LocalTestApprovalSigner,
candidate: str,
nonce_suffix: str,
) -> SignedReleasePlanApproval:
release_candidate = service.candidates.load(candidate)
plan = service.release_plans.load(candidate)
now = datetime.now(timezone.utc)
unsigned = SignedReleasePlanApproval(
release_plan_id="shadow-release-" + candidate.removeprefix("rc-"),
candidate_id=candidate,
source_commit=release_candidate.source_commit,
tree_oid=release_candidate.tree_oid,
artifact_sha256=release_candidate.artifact_sha256,
manifest_sha256=release_candidate.manifest_sha256,
deployment_plan_sha256=plan.deployment.sha256,
release_plan_sha256=plan.sha256,
policy_sha256=plan.policy_sha256,
approval_plan_id=plan.tag.approval_plan_id,
evidence_closure_oid=plan.tag.evidence_closure_oid,
owner_approval_record_oid=oid("shadow-owner:" + candidate),
capabilities=(
"runtime_switch",
"tag_publish",
"baseline_registry_finalize",
),
owner_identity="owner:shadow-acceptance-test",
trust_domain="aios.shadow",
issued_at=now.isoformat().replace("+00:00", "Z"),
not_before=now.isoformat().replace("+00:00", "Z"),
expires_at=(now + timedelta(minutes=10))
.isoformat()
.replace("+00:00", "Z"),
nonce="shadow-nonce-" + nonce_suffix,
key_id=signer.key_id,
signature="",
)
return signer.sign(unsigned)
def tag_probe_results(plan: TagPlan) -> dict[str, str]:
exact = TagPublicationProbeEvidence(
repository_identity=plan.repository_identity,
remote_reachable=True,
gitea_api_reachable=True,
gitea_listing_reachable=True,
remote_tag_oid=plan.expected_tag_oid,
annotated_tag_object_oid=plan.expected_tag_oid,
remote_peeled_target_oid=plan.expected_target_oid,
gitea_api_tag_oid=plan.expected_tag_oid,
gitea_api_peeled_target_oid=plan.expected_target_oid,
gitea_listing_visible=True,
gitea_listing_tag_oid=plan.expected_tag_oid,
gitea_listing_peeled_target_oid=plan.expected_target_oid,
observed_at=1,
)
absent = dataclasses.replace(
exact,
remote_tag_oid=None,
annotated_tag_object_oid=None,
remote_peeled_target_oid=None,
gitea_api_tag_oid=None,
gitea_api_peeled_target_oid=None,
gitea_listing_visible=False,
gitea_listing_tag_oid=None,
gitea_listing_peeled_target_oid=None,
)
indeterminate = dataclasses.replace(exact, gitea_api_reachable=False)
conflict = dataclasses.replace(exact, remote_tag_oid=oid("conflict"))
return {
"APPLIED_VERIFIED": classify_tag_publication(exact, plan).value,
"CONFLICT": classify_tag_publication(conflict, plan).value,
"INDETERMINATE": classify_tag_publication(
indeterminate, plan
).value,
"NOT_APPLIED": classify_tag_publication(absent, plan).value,
}
def parser() -> argparse.ArgumentParser:
value = argparse.ArgumentParser()
value.add_argument("--state-root", required=True, type=Path)
value.add_argument("--repository", required=True, type=Path)
value.add_argument("--release-path", required=True, type=Path)
value.add_argument("--rollback-artifact", required=True, type=Path)
value.add_argument("--source-commit", required=True)
value.add_argument("--tree-oid", required=True)
value.add_argument("--artifact-sha256", required=True)
value.add_argument("--manifest-sha256", required=True)
value.add_argument("--package-closure-sha256", required=True)
value.add_argument("--policy-sha256", required=True)
value.add_argument("--systemd-unit-sha256", required=True)
value.add_argument("--baseline-before-sha256", required=True)
value.add_argument("--registry-before-sha256", required=True)
value.add_argument("--tag-refset-before-sha256", required=True)
value.add_argument("--repository-identity", default="aios/aios")
value.add_argument("--output", required=True, type=Path)
return value
def main() -> None:
arguments = parser().parse_args()
if sha256_file(arguments.rollback_artifact) != arguments.artifact_sha256:
raise SystemExit("rollback artifact hash mismatch")
unit_path = (
arguments.release_path
/ "deploy"
/ "systemd"
/ "aios-silent-release-controller.service"
)
if sha256_file(unit_path) != arguments.systemd_unit_sha256:
raise SystemExit("systemd unit hash mismatch")
builder_signer = LocalTestBuilderSigner.generate()
builder_public = builder_signer.public_key.public_bytes(
serialization.Encoding.Raw,
serialization.PublicFormat.Raw,
)
owner_private = Ed25519PrivateKey.generate()
owner_public = owner_private.public_key().public_bytes(
serialization.Encoding.Raw,
serialization.PublicFormat.Raw,
)
owner_signer = LocalTestApprovalSigner(
"shadow-owner-ephemeral", owner_private
)
def collector(intent):
event_id = str(intent["event_id"])
candidate = make_candidate(arguments, event_id)
return TrustedCandidateBundle(
candidate=candidate,
release_plan=make_plan(arguments, candidate.candidate_id),
checks={
"authoritative_git": True,
"artifact": True,
"canary": True,
"gitea": True,
"manifest": True,
"package_closure": True,
"rollback": True,
"secret_scan": True,
"tests": True,
},
)
release_root = str(arguments.release_path.parent.parent)
dropin = str(
arguments.release_path
/ "deploy"
/ "systemd"
/ "aios-silent-release-controller.service"
)
policy = TrustedReleasePolicy(
services=frozenset({"aios-silent-release-controller"}),
release_roots=frozenset({release_root}),
cutover_links=frozenset({str(Path(release_root) / "current")}),
dropin_paths=frozenset({dropin}),
authoritative_git_dirs=frozenset({str(arguments.repository)}),
repository_identities=frozenset(
{arguments.repository_identity}
),
)
verifier = SignedApprovalVerifier(
ApprovalKeyRing({"shadow-owner-ephemeral": owner_public}),
ApprovalRevocationStore(arguments.state_root / "revocations"),
allowed_trust_domains={"aios.shadow"},
)
service = SilentReleaseActionService(
arguments.state_root,
execution_enabled=False,
approval_verifier=verifier,
candidate_builder=SignedTrustedBuilderAdapter(
collector, builder_signer
),
builder_attestation_verifier=BuilderAttestationVerifier(
{builder_signer.key_id: builder_public}
),
trusted_policy=policy,
expected_repository=arguments.repository_identity,
policy_sha256=arguments.policy_sha256,
)
first_event = "main-" + arguments.source_commit
first = service.create_release_intent(
{
"event_id": first_event,
"source_commit": arguments.source_commit,
}
)
first_duplicate = service.create_release_intent(
{
"event_id": first_event,
"source_commit": arguments.source_commit,
}
)
first_notification = service.ensure_owner_notification(
first["candidate_id"]
)
repeated_notification = service.ensure_owner_notification(
first["candidate_id"]
)
first_approval = sign_approval(
service, owner_signer, first["candidate_id"], "candidate-one"
)
first_submit = service.submit_owner_approval(
first_approval.to_dict()
)
first_submit_repeated = service.submit_owner_approval(
first_approval.to_dict()
)
first_start = service.start_deployment(first["candidate_id"])
second = service.create_release_intent(
{
"event_id": "shadow-indeterminate-" + arguments.source_commit,
"source_commit": arguments.source_commit,
}
)
second_plan = service.release_plans.load(second["candidate_id"])
probe_results = tag_probe_results(second_plan.tag)
first_controller = PersistentSilentReleaseController(
arguments.state_root,
service,
main_probe=lambda: arguments.source_commit,
)
controller_first = first_controller.run_once()
controller_restarted = PersistentSilentReleaseController(
arguments.state_root,
service,
main_probe=lambda: arguments.source_commit,
).run_once()
report = {
"approval_backend": "ephemeral-shadow-test-only",
"artifact_sha256": arguments.artifact_sha256,
"builder_attestation": "VERIFIED",
"candidate_count": 2,
"candidates": [
{
"candidate_id": first["candidate_id"],
"duplicate_intent_idempotent": first_duplicate["idempotent"],
"notification_idempotent": (
repeated_notification["idempotent"]
),
"notification_initial_idempotent": (
first_notification["idempotent"]
),
"replayed_approval_status": (
first_submit_repeated["status"]
),
"shadow_start": first_start,
"signed_approval_status": first_submit["status"],
},
{
"candidate_id": second["candidate_id"],
"tag_publication_probe": "INDETERMINATE",
},
],
"controller_restart_idempotent": (
controller_restarted["intent"]["idempotent"]
),
"controller_restart_notification_idempotent": (
controller_restarted["notification"]["idempotent"]
),
"controller_status": controller_first["startup"]["status"],
"execution_enabled": False,
"manifest_sha256": arguments.manifest_sha256,
"package_closure_sha256": arguments.package_closure_sha256,
"private_key_files_written": False,
"production_tag": "NOT_EXECUTED",
"repository_identity": arguments.repository_identity,
"shadow_mode": True,
"side_effects_executed": False,
"source_commit": arguments.source_commit,
"tag_probe_outcomes": probe_results,
"tree_oid": arguments.tree_oid,
}
write_json(arguments.output, report)
print(
json.dumps(
{
"candidate_count": 2,
"execution_enabled": False,
"output": str(arguments.output),
"shadow_mode": True,
"side_effects_executed": False,
"status": "SHADOW_ACCEPTANCE_PASSED",
},
sort_keys=True,
)
)
if __name__ == "__main__":
main()