Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b26ff010fc | ||
|
|
4b49f2ba4b | ||
|
|
b4b8fe8828 | ||
|
|
4e8ab051a0 | ||
|
|
f61e35b218 | ||
|
|
b2f90f24a3 | ||
|
|
6721eb4752 | ||
|
|
a9e4fd1e40 | ||
|
|
621750976a | ||
|
|
5359e88806 | ||
|
|
76636931ba | ||
|
|
180c1fc66e | ||
|
|
c0ff052dab | ||
|
|
94cffb6ba3 | ||
|
|
9ad5628e69 | ||
|
|
6d397a7894 | ||
|
|
165fea6ba9 | ||
|
|
d079e2087c | ||
|
|
e3b36ebdcc | ||
|
|
5e791b11c0 | ||
|
|
50be65b76e | ||
|
|
7bb86e30bf | ||
|
|
ea56b45b7e | ||
|
|
f53682da7a | ||
|
|
71884ac088 | ||
|
|
969c278cb0 | ||
|
|
6fe2a1e036 | ||
|
|
9a60e0c68e | ||
|
|
46f553ca11 | ||
|
|
4f0a0c2184 | ||
|
|
c107b193d9 | ||
|
|
2db7ced7a4 | ||
|
|
6361d9a334 | ||
|
|
2cf54f04ef | ||
|
|
f840ce5c94 | ||
|
|
35e7b55490 | ||
|
|
1f8d1f23b3 | ||
|
|
e93011791e | ||
|
|
4460564a48 | ||
|
|
4d6658136e | ||
|
|
5767e665c2 |
@@ -1,76 +0,0 @@
|
||||
{
|
||||
"contract_commit": "cbe33ecab8e9794ba5f318bea43aeaf447a56fbc",
|
||||
"created_at": "2026-07-18T04:58:00+00:00",
|
||||
"schema_version": 2,
|
||||
"state_writer": "gateway",
|
||||
"work_units": [
|
||||
{
|
||||
"id": "R0-DISCOVER-SECRET-PROVIDER",
|
||||
"track": "R_RUNNER",
|
||||
"gate": "R0_RUNNER_LIVE_RECOVERY",
|
||||
"primary": "openclaw",
|
||||
"fallback": "codex",
|
||||
"status": "READY",
|
||||
"quorum": "requires_gpt_plus_one_executor"
|
||||
},
|
||||
{
|
||||
"id": "P2-INVENTORY-REAL-EVIDENCE",
|
||||
"track": "P_PERSONAL_WEIXIN",
|
||||
"gate": "P2_REAL_B3_EVIDENCE",
|
||||
"primary": "openclaw",
|
||||
"fallback": "codex",
|
||||
"status": "READY",
|
||||
"quorum": "requires_gpt_plus_one_executor"
|
||||
},
|
||||
{
|
||||
"id": "P1-POST-CONTRACT-REACK",
|
||||
"track": "P_PERSONAL_WEIXIN",
|
||||
"gate": "P1_REPAIR3_CODE",
|
||||
"primary": "codex",
|
||||
"fallback": "openclaw",
|
||||
"status": "READY",
|
||||
"quorum": "requires_gpt_plus_one_executor"
|
||||
},
|
||||
{
|
||||
"id": "Q0-GPT-HEALTH-INIT",
|
||||
"track": "_QUORUM",
|
||||
"gate": "Q0_GPT_AND_EXECUTOR_HEALTH",
|
||||
"primary": "gateway",
|
||||
"fallback": null,
|
||||
"status": "READY",
|
||||
"quorum": "gateway_initiated"
|
||||
},
|
||||
{
|
||||
"id": "Q1-ACTIVE-EXECUTOR-ELECTION",
|
||||
"track": "_QUORUM",
|
||||
"gate": "Q1_ELECT_ACTIVE_EXECUTOR",
|
||||
"primary": "gateway",
|
||||
"fallback": null,
|
||||
"status": "PENDING",
|
||||
"depends_on": "Q0-GPT-HEALTH-INIT",
|
||||
"quorum": "gateway_initiated"
|
||||
},
|
||||
{
|
||||
"id": "Q2-FAILOVER-HEALTHCHECK",
|
||||
"track": "_QUORUM",
|
||||
"gate": "Q2_STANDBY_HEALTH_MONITOR",
|
||||
"primary": "gateway",
|
||||
"fallback": null,
|
||||
"status": "PENDING",
|
||||
"depends_on": "Q1-ACTIVE-EXECUTOR-ELECTION",
|
||||
"quorum": "log_only_if_active_ok"
|
||||
},
|
||||
{
|
||||
"id": "Q3-UNATTENDED-ACCEPTANCE-MATRIX",
|
||||
"track": "_QUORUM",
|
||||
"gate": "A0_AUTOMATION_CLOSED",
|
||||
"primary": "gateway",
|
||||
"fallback": null,
|
||||
"status": "PENDING_LIVE_EVIDENCE",
|
||||
"depends_on": "Q2-FAILOVER-HEALTHCHECK",
|
||||
"quorum": "requires_gpt_plus_one_executor",
|
||||
"required_scenarios": 16,
|
||||
"may_mark_pass_without_live_evidence": false
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,248 +0,0 @@
|
||||
{
|
||||
"adaptive_dispatch": {
|
||||
"codex_first": [
|
||||
"complex_code",
|
||||
"security_boundary",
|
||||
"architecture",
|
||||
"high_risk_review",
|
||||
"strong_model_materially_better"
|
||||
],
|
||||
"default_orchestrator": "openclaw",
|
||||
"dynamic_executor_selection": {
|
||||
"large_repetitive_infrastructure_monitoring": "openclaw",
|
||||
"complex_code_security_architecture": "codex",
|
||||
"both_healthy_active": "assign_by_preference",
|
||||
"one_healthy_active": "use_healthy_only",
|
||||
"both_unhealthy_blocked": true
|
||||
},
|
||||
"max_attempts_per_work_unit": 2,
|
||||
"on_timeout": [
|
||||
"mark_TIMEOUT",
|
||||
"preserve_logs",
|
||||
"switch_agent"
|
||||
],
|
||||
"openclaw_first": [
|
||||
"quality_close",
|
||||
"large_or_repetitive",
|
||||
"long_running",
|
||||
"infra_or_git_inventory",
|
||||
"cost_sensitive"
|
||||
],
|
||||
"single_state_writer_lease_required": true,
|
||||
"strict_role_partition": false,
|
||||
"state_writer": "gateway",
|
||||
"timeout_can_pass": false
|
||||
},
|
||||
"authoritative_state": {
|
||||
"b3": "BLOCKED_EVIDENCE_REQUIRED",
|
||||
"b4": "NOT_STARTED",
|
||||
"baseline": "V1R4C6",
|
||||
"production_release": "NOT_STARTED",
|
||||
"release": "paused",
|
||||
"repair3": {
|
||||
"base": "8d9c6da90fc8caa69b691d243c6fefb40b6b0575",
|
||||
"candidate": "PASS_CANDIDATE_PREPARED",
|
||||
"commit": "e110cda9968cadc5c7b7179ec5003b05aad11ce4",
|
||||
"exact4": "PASS",
|
||||
"result": "49a9c2e0cc898ecf2bbd45f454e007644d7f3045"
|
||||
},
|
||||
"runner": {
|
||||
"artifact_sha256": "85c472c05a1e55a5899b215d3cd41df53912787de529ac9cb3d5308da4158130",
|
||||
"commit": "bbd27cf06acf1f589eef291eab54d4b127aa3aea",
|
||||
"live": "BLOCKED_GATEWAY_SECRET_REF_UNAVAILABLE",
|
||||
"offline": "PASS",
|
||||
"result": "e789632e359e9232d8ae081e7c4f23ed7b615a9d"
|
||||
}
|
||||
},
|
||||
"central_gpt": {
|
||||
"required": "HEALTHY",
|
||||
"health_definition": {
|
||||
"live_probe_required": true,
|
||||
"workspace_agent_validated_live_or_responses_validated_live": true,
|
||||
"configuration_or_credential_presence_is_insufficient": true,
|
||||
"unknown_is_not_healthy": true
|
||||
},
|
||||
"independent_review": {
|
||||
"single_executor_mode": true,
|
||||
"required_evidence": [
|
||||
"committed_git_tree",
|
||||
"full_commit_oid",
|
||||
"test_logs_with_rc",
|
||||
"gitleaks",
|
||||
"artifact_digest",
|
||||
"worktree_bare_gitea_oid",
|
||||
"live_runtime_evidence"
|
||||
],
|
||||
"model_judgment_insufficient": true
|
||||
}
|
||||
},
|
||||
"contract_id": "V2R1-PERSONAL-WEIXIN-PRODUCTION-EXECUTION-CONTRACT",
|
||||
"executor_quorum": {
|
||||
"minimum_healthy": 1,
|
||||
"candidates": [
|
||||
"openclaw",
|
||||
"codex"
|
||||
],
|
||||
"quorum_pass": [
|
||||
"GPT_HEALTHY_AND_OPENCLAW_HEALTHY",
|
||||
"GPT_HEALTHY_AND_CODEX_HEALTHY"
|
||||
],
|
||||
"quorum_block": [
|
||||
"GPT_UNHEALTHY",
|
||||
"GPT_HEALTHY_BUT_BOTH_EXECUTORS_UNAVAILABLE",
|
||||
"GPT_OR_SOLE_EXECUTOR_UNKNOWN"
|
||||
]
|
||||
},
|
||||
"automation_closed_success_conditions": [
|
||||
"CENTRAL_GPT_HEALTH=PASS",
|
||||
"EXECUTOR_QUORUM=PASS",
|
||||
"ACTIVE_EXECUTOR=openclaw|codex",
|
||||
"EVENT_TRIGGER=PASS",
|
||||
"POLLING=PASS",
|
||||
"FAILOVER=PASS",
|
||||
"LIFECYCLE_SYNC=PASS",
|
||||
"TWO_CYCLE_UNATTENDED_LOOP=PASS",
|
||||
"RESTART_RECOVERY=PASS",
|
||||
"AUTOMATION_CLOSED=PASS"
|
||||
],
|
||||
"gateway_lifecycle_events": [
|
||||
"TASK_DISPATCHED",
|
||||
"TASK_STARTED",
|
||||
"TASK_ENDED",
|
||||
"DECISION_APPLIED",
|
||||
"PROJECT_SUMMARY",
|
||||
"PROJECT_COMPLETED"
|
||||
],
|
||||
"facts_policy": {
|
||||
"accept": [
|
||||
"committed_git_tree",
|
||||
"full_oid",
|
||||
"worktree_bare_gitea_oid_match",
|
||||
"test_log_with_rc",
|
||||
"immutable_digest",
|
||||
"result_only_commit",
|
||||
"live_evidence"
|
||||
],
|
||||
"reject": [
|
||||
"verbal_claim",
|
||||
"readme_only",
|
||||
"local_only_commit",
|
||||
"environment_variable_as_secret_ref",
|
||||
"compile_only",
|
||||
"timeout_as_pass",
|
||||
"unrelated_repo",
|
||||
"fabricated_evidence"
|
||||
]
|
||||
},
|
||||
"forbidden_git": [
|
||||
"reset",
|
||||
"amend",
|
||||
"rebase",
|
||||
"force_push",
|
||||
"direct_main_push",
|
||||
"direct_bare_ref_mutation",
|
||||
"unapproved_gitea_db_mutation",
|
||||
"reuse_accident_worktree",
|
||||
"delete_live_worktree_to_clear_collision"
|
||||
],
|
||||
"generated_at": "2026-07-18T04:58:00+00:00",
|
||||
"handoff": {
|
||||
"ack": "CONTRACT_ACK=<full_contract_commit_oid>",
|
||||
"advance": "independent_verification_required",
|
||||
"all_agents_match": true
|
||||
},
|
||||
"join_gate": {
|
||||
"requires": [
|
||||
"AUTOMATION_CLOSED",
|
||||
"RUNNER_RECOVERED",
|
||||
"B3_PASS",
|
||||
"B4_PASS",
|
||||
"INTEGRATION_PASS"
|
||||
]
|
||||
},
|
||||
"model_policy": {
|
||||
"codex": "strongest_model_actually_available_in_runtime",
|
||||
"gpt_5_6": "use_only_if_exposed",
|
||||
"no_unverified_model_claims": true,
|
||||
"openclaw_default": "DeepSeek V4 Flash"
|
||||
},
|
||||
"owner_authorization": {
|
||||
"adaptive_agent_assignment_authorized": true,
|
||||
"agent_substitution_authorized": true,
|
||||
"autonomous_execution": true,
|
||||
"may_bypass_release_controls": false,
|
||||
"may_bypass_secret_controls": false,
|
||||
"may_fabricate_evidence": false,
|
||||
"user_step_by_step_confirmation_required": false
|
||||
},
|
||||
"release_gates": [
|
||||
"AUTOMATION_CLOSED",
|
||||
"MAIN_MERGE",
|
||||
"STAGING_AND_CANARY",
|
||||
"PRODUCTION"
|
||||
],
|
||||
"schema_version": 4,
|
||||
"self_healing": {
|
||||
"priorities": {
|
||||
"p0": "central_gpt_failure",
|
||||
"p1": "both_executors_unavailable",
|
||||
"p2": "active_executor_failed_with_healthy_standby",
|
||||
"p3": "standby_executor_alone_failed"
|
||||
},
|
||||
"rules": {
|
||||
"standby_failure_when_active_ok": {
|
||||
"action": "log_only",
|
||||
"block_business": false,
|
||||
"create_blocking_gate": false,
|
||||
"pause_release": false
|
||||
},
|
||||
"active_executor_failure": {
|
||||
"action": "failover_to_standby",
|
||||
"inherit_contract_commit": true,
|
||||
"inherit_event_id": true,
|
||||
"inherit_state_version": true,
|
||||
"mark_original_task": "INTERRUPTED_AGENT_FAILOVER",
|
||||
"new_executor_continues_from_checkpoint": true,
|
||||
"block_duplicate_side_effects": true
|
||||
},
|
||||
"both_unavailable_block": "BLOCKED_NO_HEALTHY_EXECUTOR"
|
||||
}
|
||||
},
|
||||
"state_rules": [
|
||||
"tracks_run_parallel",
|
||||
"runner_block_does_not_block_offline_work_or_evidence_inventory",
|
||||
"join_requires_both_tracks",
|
||||
"failure_stops_dependents",
|
||||
"production_remains_V1R4C6_paused_until_production_pass",
|
||||
"gateway_persists_authoritative_state",
|
||||
"lifecycle_events_persisted_regardless_of_openclaw_online"
|
||||
],
|
||||
"status": "ACTIVE",
|
||||
"tracks": {
|
||||
"P_PERSONAL_WEIXIN": {
|
||||
"b3": "BLOCKED_EVIDENCE_REQUIRED",
|
||||
"b4": "NOT_STARTED",
|
||||
"parallel": true,
|
||||
"repair3": "PASS_CANDIDATE_PREPARED",
|
||||
"repair3_commit": "e110cda9968cadc5c7b7179ec5003b05aad11ce4"
|
||||
},
|
||||
"R_RUNNER": {
|
||||
"commit": "bbd27cf06acf1f589eef291eab54d4b127aa3aea",
|
||||
"current": "BLOCKED_GATEWAY_SECRET_REF_UNAVAILABLE",
|
||||
"fallback": "codex",
|
||||
"parallel": true,
|
||||
"primary": "openclaw",
|
||||
"required": [
|
||||
"artifact_verified",
|
||||
"approved_secret_provider_contract",
|
||||
"owner_probe_verified",
|
||||
"service_healthy",
|
||||
"heartbeat_healthy",
|
||||
"gateway_auth",
|
||||
"durable_task_create_read_cancel",
|
||||
"execution_channel",
|
||||
"live_rollback"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
{
|
||||
"b3": "BLOCKED_EVIDENCE_REQUIRED",
|
||||
"b4": "NOT_STARTED",
|
||||
"baseline": "V1R4C6",
|
||||
"contract_branch": "task/governance/v2r1-personal-weixin-production-contract-20260717",
|
||||
"contract_commit": "cbe33ecab8e9794ba5f318bea43aeaf447a56fbc",
|
||||
"dispatch_commit": null,
|
||||
"executor_quorum": {
|
||||
"candidates": ["openclaw", "codex"],
|
||||
"minimum_healthy": 1,
|
||||
"openclaw": "UNKNOWN",
|
||||
"codex": "UNKNOWN"
|
||||
},
|
||||
"active_executor": null,
|
||||
"standby_executor": null,
|
||||
"central_gpt": "UNKNOWN",
|
||||
"automation_closed": {
|
||||
"central_gpt_health": "UNKNOWN",
|
||||
"executor_quorum": "UNKNOWN",
|
||||
"active_executor": null,
|
||||
"event_trigger": "UNKNOWN",
|
||||
"polling": "UNKNOWN",
|
||||
"failover": "UNKNOWN",
|
||||
"lifecycle_sync": "UNKNOWN",
|
||||
"two_cycle_unattended_loop": "UNKNOWN",
|
||||
"restart_recovery": "UNKNOWN",
|
||||
"result": "NOT_STARTED"
|
||||
},
|
||||
"lifecycle_events": [],
|
||||
"production_release": "NOT_STARTED",
|
||||
"release": "paused",
|
||||
"repair3": "PASS_CANDIDATE_PREPARED",
|
||||
"runner": "BLOCKED_GATEWAY_SECRET_REF_UNAVAILABLE",
|
||||
"schema_version": 2,
|
||||
"single_state_writer": "gateway",
|
||||
"single_state_writer_lease_required": true,
|
||||
"quorum_controller": {
|
||||
"implementation": "CODE_COMMITTED_TESTED_PENDING_LIVE_ACCEPTANCE",
|
||||
"live_acceptance": "NOT_STARTED",
|
||||
"health_source": "ROOT_OWNED_METADATA_ONLY_LIVE_PROBE_RECEIPTS",
|
||||
"unknown_counts_as_healthy": false
|
||||
},
|
||||
"updated_at": "2026-07-18T04:58:00+00:00"
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
# Codex task: apply and verify AIOS Orchestrator Gateway v1
|
||||
|
||||
Contract: `cbe33ecab8e9794ba5f318bea43aeaf447a56fbc`
|
||||
|
||||
Use the strongest actually available Codex model for the initial security review.
|
||||
If unavailable or timed out, record TIMEOUT/NOT_EXECUTED and let OpenClaw run
|
||||
the deterministic test path. Do not count a timeout as PASS.
|
||||
|
||||
## Apply
|
||||
|
||||
1. Create a new clean append-only branch and isolated worktree.
|
||||
2. Validate the delivery tarball and `SHA256SUMS`.
|
||||
3. Run the external `scripts/apply_gateway_patch.sh` with `SOURCE_DIR` and
|
||||
`WORKTREE`; do not alter unrelated paths.
|
||||
4. Review all added code, especially:
|
||||
- exact contract and optimistic state-version checks;
|
||||
- event/decision idempotency;
|
||||
- SQLite leases and retry behavior;
|
||||
- Workspace Agent `202` asynchronous callback flow;
|
||||
- Responses background creation and polling;
|
||||
- file-inbox and Git-ref polling;
|
||||
- traversal/symlink protections;
|
||||
- OpenClaw lifecycle synchronization;
|
||||
- systemd sandbox;
|
||||
- install and rollback transactions.
|
||||
|
||||
## Tests
|
||||
|
||||
Run from the final source tree:
|
||||
|
||||
- Python compile for all gateway and helper files;
|
||||
- unit tests;
|
||||
- JSON schema/config parsing;
|
||||
- Bash syntax;
|
||||
- `systemd-analyze verify`;
|
||||
- event duplicate and payload-conflict tests;
|
||||
- stale state-version rejection;
|
||||
- lease/double-claim test;
|
||||
- Workspace trigger failure and Responses fallback;
|
||||
- response queued/in-progress/completed polling;
|
||||
- file-inbox recovery after HTTP failure;
|
||||
- Git-ref initial baseline and subsequent-change event;
|
||||
- task start/end and project summary synchronization;
|
||||
- dispatch traversal and symlink rejection;
|
||||
- retry/backoff/dead-letter;
|
||||
- install dry-run;
|
||||
- rollback test in an isolated fixture;
|
||||
- `git diff --check`;
|
||||
- full repository tests;
|
||||
- final Gitleaks.
|
||||
|
||||
## Commit and review
|
||||
|
||||
1. Create an append-only implementation commit.
|
||||
2. Push without force.
|
||||
3. Confirm `WORKTREE_OID == BARE_OID == GITEA_OID`.
|
||||
4. Create an independent RESULT-only branch and exact-one result commit.
|
||||
5. Record the source bundle SHA-256, implementation OID, RESULT OID, tests,
|
||||
Gitleaks, systemd validation and remaining external configuration blockers.
|
||||
|
||||
## Forbidden
|
||||
|
||||
No reset, amend, rebase, force push, direct main push, controller mutation,
|
||||
secret disclosure, fabricated test result, production deployment or B3 evidence
|
||||
fabrication.
|
||||
|
||||
Success:
|
||||
|
||||
`GATEWAY_CODE_REVIEW=PASS IMPLEMENTATION=<oid> RESULT=<oid> OID_MATCH=YES`
|
||||
@@ -1,13 +0,0 @@
|
||||
# Codex V2R1 adaptive execution
|
||||
CONTRACT_ACK=cbe33ecab8e9794ba5f318bea43aeaf447a56fbc
|
||||
Read `.agent/policies/V2R1-PERSONAL-WEIXIN-PRODUCTION-EXECUTION-CONTRACT.yaml`.
|
||||
Execute P1-POST-CONTRACT-REACK now:
|
||||
- read-only verify Repair3 e110cda9968cadc5c7b7179ec5003b05aad11ce4
|
||||
- read-only verify RESULT 49a9c2e0cc898ecf2bbd45f454e007644d7f3045
|
||||
- verify lineage from 8d9c6da90fc8caa69b691d243c6fefb40b6b0575, exact4, tests, Gitleaks and RESULT-only scope
|
||||
- do not modify Repair3 code in this unit
|
||||
Act as fallback for OpenClaw only on timeout/unavailability. Use GPT-5.6 only if actually exposed; otherwise strongest visible model. Prefer OpenClaw when quality is comparable or volume is high; prefer Codex for complex/security-critical work.
|
||||
Forbidden: reset, amend, rebase, force push, direct main push, controller mutation, secret disclosure, fabricated evidence, skipped gates.
|
||||
Report:
|
||||
CONTRACT=cbe33ecab8e9794ba5f318bea43aeaf447a56fbc WORK_UNIT=<id> TRACK=<id> GATE=<id> AGENT=codex STATUS=<status> IMPLEMENTATION=<oid-or-NONE> RESULT=<oid-or-NONE> OID_MATCH=<YES-or-NO> BLOCKER=<reason-or-NONE>
|
||||
Keep BASELINE=V1R4C6 RELEASE=paused PRODUCTION_RELEASE=NOT_STARTED until production gate passes.
|
||||
-119
@@ -1,119 +0,0 @@
|
||||
# OpenClaw A0 Three-Party Self-Healing and Release Supervision
|
||||
|
||||
CONTRACT_ACK=cbe33ecab8e9794ba5f318bea43aeaf447a56fbc
|
||||
|
||||
## Mission
|
||||
|
||||
Close the GPT/OpenClaw/Codex unattended loop before any main merge or production release. OpenClaw is the default state writer and long-running supervisor, but work may be reassigned dynamically under the shared contract.
|
||||
|
||||
## Authoritative inputs
|
||||
|
||||
- Governance contract: `cbe33ecab8e9794ba5f318bea43aeaf447a56fbc`
|
||||
- Governance dispatch head before this task: `9195470ceb78fd57ffface3dcc38f40b75efa9d8`
|
||||
- Gateway integration implementation: `c3984ca2b4388875d2875b4fdfdf95a2984f6ab3`
|
||||
- Gateway integration RESULT: `4e63b1559aa3fc07b1384154c97b2c90b84353ed`
|
||||
- Three-party self-healer candidate branch: `patch/a0-three-party-self-healing-20260718-20260718T043256`
|
||||
- Production baseline: `V1R4C6`
|
||||
- Release status: `paused`
|
||||
|
||||
## Priority invariant
|
||||
|
||||
Continuously evaluate exactly three parties:
|
||||
|
||||
1. `central_gpt` — Orchestrator Gateway plus a valid Workspace Agent or Responses API decision channel.
|
||||
2. `openclaw` — OpenClaw watcher, lifecycle sync, durable task execution and heartbeat.
|
||||
3. `codex` — Codex watcher, auto-claim, lifecycle sync, durable task execution and heartbeat.
|
||||
|
||||
When exactly two parties are HEALTHY and the third is explicitly UNHEALTHY, create a P0 repair work unit for the unique unhealthy party before every other task. UNKNOWN is not UNHEALTHY and must fail closed. Do not repair when fewer than two are healthy or any health result is ambiguous.
|
||||
|
||||
## A0-SH1: supervise implementation completion
|
||||
|
||||
Codex owns the high-risk code changes. Independently verify that the final implementation contains:
|
||||
|
||||
- strict root-owned configuration validation and no symlink chain;
|
||||
- fixed built-in party adapters rather than arbitrary untrusted repair commands;
|
||||
- bounded stdout/stderr redaction so tokens and secrets cannot enter audit logs;
|
||||
- exclusive lease, cooldown, circuit breaker and stale lease recovery;
|
||||
- event trigger plus periodic polling;
|
||||
- OpenClaw lifecycle events: `TASK_DISPATCHED`, `TASK_STARTED`, `TASK_ENDED`, `DECISION_APPLIED`, `PROJECT_SUMMARY`, `PROJECT_COMPLETED`;
|
||||
- durable file outbox fallback when HTTP callback is unavailable;
|
||||
- systemd service and timer;
|
||||
- health and repair adapters for central GPT, OpenClaw and Codex;
|
||||
- install, verify and rollback scripts;
|
||||
- deterministic tests for all health combinations and failures.
|
||||
|
||||
Reject the current candidate if it only contains `automation/three_party_self_healer.py` without these controls.
|
||||
|
||||
## A0-SH2: live three-party validation
|
||||
|
||||
After Codex implementation RESULT passes, deploy only to a non-production acceptance context and run:
|
||||
|
||||
1. All-three healthy steady state for at least three poll intervals.
|
||||
2. GPT unhealthy while OpenClaw and Codex healthy: GPT repair becomes P0 and recovers.
|
||||
3. OpenClaw unhealthy while GPT and Codex healthy: OpenClaw repair becomes P0 and recovers.
|
||||
4. Codex unhealthy while GPT and OpenClaw healthy: Codex repair becomes P0 and recovers.
|
||||
5. One healthy plus two unhealthy: no repair; fail closed.
|
||||
6. Any UNKNOWN health: no repair; fail closed.
|
||||
7. Repair timeout: retry with backoff, then circuit open.
|
||||
8. Service restart: state, lease and pending work recover.
|
||||
9. HTTP event path unavailable: file inbox polling recovers.
|
||||
10. Git webhook unavailable: Git ref polling detects change.
|
||||
11. Duplicate event ID: no duplicate dispatch.
|
||||
12. Main Workspace Agent channel unavailable: Responses fallback takes over only with a valid approved credential.
|
||||
13. OpenClaw receives every task start/end and project summary notification.
|
||||
14. Two consecutive unattended decision/execution/result cycles complete with no chat message or manual dispatch.
|
||||
|
||||
Do not use dummy API keys or environment-variable presence as proof of a valid central decision channel.
|
||||
|
||||
## A0-SH3: authoritative closure
|
||||
|
||||
Only after independent verification of the above create one append-only state update and an exact-one RESULT file. Required output:
|
||||
|
||||
```text
|
||||
AUTOMATION_CLOSED=PASS
|
||||
CENTRAL_GPT_HEALTH=PASS
|
||||
OPENCLAW_AUTO_CLAIM=PASS
|
||||
CODEX_AUTO_CLAIM=PASS
|
||||
THREE_PARTY_SELF_HEALING=PASS
|
||||
EVENT_TRIGGER=PASS
|
||||
POLLING=PASS
|
||||
OPENCLAW_LIFECYCLE_SYNC=PASS
|
||||
TWO_CYCLE_UNATTENDED_LOOP=PASS
|
||||
RESTART_RECOVERY=PASS
|
||||
ROLLBACK=PASS
|
||||
```
|
||||
|
||||
Any missing item keeps `AUTOMATION_CLOSED=NO`.
|
||||
|
||||
## Release continuation
|
||||
|
||||
After and only after `AUTOMATION_CLOSED=PASS`, automatically resume the governed release sequence:
|
||||
|
||||
1. Real B3 evidence intake and independent review.
|
||||
2. B4 implementation and validation.
|
||||
3. Full integration, regression, security and migration tests.
|
||||
4. Normal PR/GitOps merge to main; never direct-push main.
|
||||
5. Immutable release artifact, digest, SBOM and signature.
|
||||
6. Staging and rollback exercise.
|
||||
7. Canary and monitoring validation.
|
||||
8. Owner/admin GitOps production deployment using existing approved release controls.
|
||||
9. Production smoke tests and production RESULT-only commit.
|
||||
|
||||
External facts that cannot be fabricated — real WeChat credentials/raw-wire, valid secret references and owner-only release approval — remain fail-closed blockers. The loop must monitor them and resume automatically when they become available.
|
||||
|
||||
## Forbidden
|
||||
|
||||
- reset, amend, rebase, force push;
|
||||
- direct push to main;
|
||||
- controller mutation;
|
||||
- secret disclosure;
|
||||
- dummy credentials presented as production credentials;
|
||||
- fabricated B3 evidence;
|
||||
- timeout or skipped checks recorded as PASS;
|
||||
- production release before `AUTOMATION_CLOSED=PASS`.
|
||||
|
||||
Progress format:
|
||||
|
||||
```text
|
||||
CONTRACT=cbe33ecab8e9794ba5f318bea43aeaf447a56fbc WORK_UNIT=<id> AGENT=openclaw STATUS=<status> COMMIT=<oid-or-NONE> BLOCKER=<stable-reason-or-NONE>
|
||||
```
|
||||
@@ -1,68 +0,0 @@
|
||||
# OpenClaw task: deploy and supervise AIOS Orchestrator Gateway v1
|
||||
|
||||
Contract: `cbe33ecab8e9794ba5f318bea43aeaf447a56fbc`
|
||||
|
||||
You are the default orchestrator and deployment supervisor.
|
||||
|
||||
## Preconditions
|
||||
|
||||
- Codex has applied and independently reviewed this source bundle on a clean,
|
||||
append-only branch.
|
||||
- Implementation and RESULT OIDs are identical across worktree, bare repo and
|
||||
Gitea.
|
||||
- Final Gitleaks and repository tests are PASS.
|
||||
- No force push, reset, amend or rebase occurred.
|
||||
|
||||
## Responsibilities
|
||||
|
||||
1. Validate the source bundle SHA-256 and `SHA256SUMS`.
|
||||
2. Review the generated systemd service, poll timer, tmpfiles and sysusers files.
|
||||
3. Install through `scripts/install_aios_orchestrator_gateway.sh`.
|
||||
4. Configure one or both central-model channels:
|
||||
- published Workspace Agent API channel with `agtch_...` trigger ID and a
|
||||
Workspace Agent access token stored outside Git; or
|
||||
- Responses API fallback with an approved model and API key stored outside Git.
|
||||
5. Publish the callback endpoint securely and configure the Workspace Agent
|
||||
action from `workspace-agent/orchestrator-decision-action.openapi.yaml`.
|
||||
6. Confirm that the Workspace Agent instructions match
|
||||
`workspace-agent/CENTRAL_AGENT_INSTRUCTIONS.md`.
|
||||
7. Enable both:
|
||||
- the continuously running gateway service;
|
||||
- the independent `aios-orchestrator-gateway-poll.timer`.
|
||||
8. Verify HTTP event ingestion, file-inbox polling, Git-ref polling,
|
||||
Workspace Agent trigger acceptance, callback decision, Responses background
|
||||
polling fallback, dispatch queue generation, retry leases and loop guards.
|
||||
9. Verify OpenClaw receives and synchronizes:
|
||||
- TASK_DISPATCHED
|
||||
- TASK_STARTED
|
||||
- TASK_ENDED
|
||||
- PROJECT_SUMMARY
|
||||
- PROJECT_COMPLETED
|
||||
10. Run a harmless bootstrap work unit through OpenClaw and Codex and confirm
|
||||
duplicate event submission is idempotent.
|
||||
11. Run rollback, reinstall and heartbeat/health verification.
|
||||
12. Create one RESULT-only review recording exact commits, artifact digest,
|
||||
tests, systemd security, polling recovery, lifecycle synchronization and
|
||||
rollback.
|
||||
|
||||
## Fail-closed boundaries
|
||||
|
||||
- Do not put Workspace Agent tokens, API keys, callback tokens or gateway
|
||||
credentials in Git, logs, command arguments or RESULT files.
|
||||
- Do not claim the gateway resolves the existing B3 credential/raw-wire blocker.
|
||||
- Do not advance B4 or production merely because this gateway is healthy.
|
||||
- Do not bypass owner/admin production release approvals.
|
||||
- Do not use reset, amend, rebase, force push, direct main push or bare-ref edits.
|
||||
|
||||
## Required final status
|
||||
|
||||
Success:
|
||||
|
||||
`ORCHESTRATOR_GATEWAY_LIVE=PASS POLLING=PASS OPENCLAW_SYNC=PASS ROLLBACK=PASS`
|
||||
|
||||
Blocker:
|
||||
|
||||
`ORCHESTRATOR_GATEWAY_LIVE=BLOCKED reason=<stable_reason>`
|
||||
|
||||
Keep `BASELINE=V1R4C6 RELEASE=paused PRODUCTION_RELEASE=NOT_STARTED` until the
|
||||
existing production contract advances all release gates.
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
# OpenClaw V2R1 adaptive orchestration
|
||||
CONTRACT_ACK=cbe33ecab8e9794ba5f318bea43aeaf447a56fbc
|
||||
Read `.agent/policies/V2R1-PERSONAL-WEIXIN-PRODUCTION-EXECUTION-CONTRACT.yaml`.
|
||||
You are default orchestrator and sole authoritative state writer.
|
||||
Execute immediately:
|
||||
1. R0-DISCOVER-SECRET-PROVIDER: discover approved provider metadata and owner-probe source without reading or outputting secrets. Environment variables alone are not approved secret refs.
|
||||
2. P2-INVENTORY-REAL-EVIDENCE: inventory approved restricted B3 evidence locations; validate existing facts; never synthesize evidence.
|
||||
3. Independently verify Codex results before advancing.
|
||||
Prefer OpenClaw for comparable-quality, large, repetitive, long-running, infrastructure and cost-sensitive work. Prefer Codex strongest actually available model for complex code, security boundaries, architecture and high-risk review. On timeout mark TIMEOUT and switch agent; never infer PASS.
|
||||
Forbidden: reset, amend, rebase, force push, direct main push, controller mutation, secret disclosure, fabricated evidence, skipped gates.
|
||||
Report:
|
||||
CONTRACT=cbe33ecab8e9794ba5f318bea43aeaf447a56fbc WORK_UNIT=<id> TRACK=<id> GATE=<id> AGENT=openclaw STATUS=<status> COMMIT=<oid-or-NONE> BLOCKER=<reason-or-NONE>
|
||||
Keep BASELINE=V1R4C6 RELEASE=paused PRODUCTION_RELEASE=NOT_STARTED until production gate passes.
|
||||
@@ -1,11 +0,0 @@
|
||||
# AIOS Codex automatic command watcher
|
||||
|
||||
The watcher polls the governed `commands/active` queue without a chat trigger. It uses SQLite transactions for exclusive leases, records the exact command digest, rejects contract drift, retries transient failures, dead-letters exhausted work, and resumes from the last durable phase so an end-event retry does not execute a work unit twice.
|
||||
|
||||
Commands are JSON documents validated by `schemas/codex-command.schema.json`. `acceptance_probe` is the only built-in action and has no external side effect. Production `codex` work requires an administrator-provisioned absolute executor path; the instruction is passed on standard input, never argv or environment. The executor receives a sanitized environment and must obtain credentials through its approved file or exec provider. If the executor is absent, the task fails closed and follows the retry/dead-letter policy.
|
||||
|
||||
The watcher provides at-most-once execution. It writes the `EXECUTING` phase before launching an external executor; if the host dies after an external effect but before a result is durably recorded, recovery does not replay the work and instead dead-letters the ambiguity for human review. Start and end event JSON is persisted before transmission, so response-loss retries reuse the exact event ID and payload and are deduplicated by the Gateway.
|
||||
|
||||
Lifecycle evidence is appended to `/var/log/aios-codex-watcher/audit.jsonl` as `CODEX_AUTO_CLAIM`, `TASK_STARTED`, `TASK_EXECUTED`, `TASK_ENDED`, and `EVENT_POSTED`. Inputs, tokens, stdout, and stderr are not logged; only result digests are retained.
|
||||
|
||||
Install from a reviewed commit with `sudo scripts/install_aios_codex_watcher.sh`. The installer creates a content-addressed release, atomically updates `current`, writes a receipt, and enables the five-second systemd timer. The existing Gateway inbound token file is referenced in place and never copied.
|
||||
@@ -1,107 +0,0 @@
|
||||
# AIOS Orchestrator Gateway v0.2
|
||||
|
||||
This component turns the current passive GPT orchestration pattern into a
|
||||
durable, unattended loop:
|
||||
|
||||
```text
|
||||
Gitea/OpenClaw/Codex event
|
||||
-> HTTP event endpoint or polled file inbox
|
||||
-> durable SQLite state machine
|
||||
-> Workspace Agent API trigger
|
||||
-> strict callback decision
|
||||
-> Responses API background fallback and polling
|
||||
-> OpenClaw/Codex dispatch queues
|
||||
-> start/end/result events
|
||||
-> next orchestration cycle
|
||||
```
|
||||
|
||||
## Reliability mechanisms
|
||||
|
||||
The gateway does not depend on one timer, webhook or model call:
|
||||
|
||||
- HTTP event ingestion.
|
||||
- Periodic file-inbox polling for missed webhooks and offline producers.
|
||||
- Local bare-Git ref polling for `main`, governance, patch and result refs.
|
||||
- Workspace Agent retries with a stable event `Idempotency-Key`.
|
||||
- Workspace Agent callback timeout and Responses API background fallback.
|
||||
- Responses status polling until completion or fail-closed terminal status.
|
||||
- SQLite optimistic state versions, idempotency records and processing leases.
|
||||
- Exponential retry with a maximum retry count and dead-letter state.
|
||||
- Same-blocker and maximum-hop guards to stop non-progress loops.
|
||||
- A systemd timer runs `--poll-once` as an independent reliability watchdog.
|
||||
|
||||
Workspace Agent trigger runs are asynchronous and do not expose a retrievable
|
||||
final response. The central Workspace Agent must therefore call the
|
||||
`submit_orchestration_decision` action.
|
||||
|
||||
## OpenClaw synchronization
|
||||
|
||||
Every lifecycle stage is durably written to the OpenClaw synchronization queue:
|
||||
|
||||
- `TASK_DISPATCHED`
|
||||
- `TASK_STARTED`
|
||||
- `TASK_ENDED`
|
||||
- `DECISION_APPLIED`
|
||||
- `PROJECT_SUMMARY`
|
||||
- `PROJECT_COMPLETED`
|
||||
|
||||
An optional authenticated OpenClaw HTTP endpoint can receive the same records.
|
||||
A failed HTTP delivery remains in the lifecycle retry queue. OpenClaw remains
|
||||
the default project-status synchronizer and authoritative progress presenter.
|
||||
|
||||
## Agent execution contract
|
||||
|
||||
Each dispatched JSON envelope contains:
|
||||
|
||||
- exact contract commit;
|
||||
- source event;
|
||||
- clean base commit;
|
||||
- primary and fallback agent;
|
||||
- allowed paths;
|
||||
- required checks;
|
||||
- forbidden operations;
|
||||
- completion event callback format.
|
||||
|
||||
OpenClaw and Codex must send a `work_unit_started` event before execution and a
|
||||
`work_unit_completed`, `work_unit_blocked`, or `work_unit_failed` event when
|
||||
they stop. This guarantees that OpenClaw can synchronize every task start,
|
||||
finish, and overall project summary.
|
||||
|
||||
## Security
|
||||
|
||||
- Model and callback credentials are read only from root-owned secret files.
|
||||
- Credentials never enter Git, dispatch payloads or logs.
|
||||
- The gateway runs as a dedicated unprivileged user.
|
||||
- Dispatch and sync paths are protected against traversal and symlink targets.
|
||||
- All state changes require the exact contract commit and current state version.
|
||||
- Production evidence, credentials and approvals stay fail-closed.
|
||||
- Git history operations remain controlled by the committed governance contract.
|
||||
|
||||
## Deployment sequence
|
||||
|
||||
1. Apply this source bundle to a clean append-only branch.
|
||||
2. Run unit, repository, shell, JSON, systemd and Gitleaks checks.
|
||||
3. Configure one or both model channels:
|
||||
- published Workspace Agent API channel and Workspace Agent access token;
|
||||
- Responses API key and approved fallback model.
|
||||
4. Publish the decision callback as a Workspace Agent action or MCP tool.
|
||||
5. Install the service through `scripts/install_aios_orchestrator_gateway.sh`.
|
||||
6. Run `scripts/verify_aios_orchestrator_gateway.sh`.
|
||||
7. Submit a harmless event and verify:
|
||||
- trigger acceptance;
|
||||
- strict decision callback;
|
||||
- dispatch file;
|
||||
- OpenClaw task-start/task-end/project-summary synchronization;
|
||||
- duplicate idempotency;
|
||||
- polling recovery after suppressing the HTTP webhook.
|
||||
|
||||
The installer copies an explicit gateway-only allowlist into a release directory
|
||||
named by its SHA-256 digest, atomically updates `/opt/aios-orchestrator-gateway/current`,
|
||||
and persists `/var/lib/aios-orchestrator-gateway/install-receipt.json` with the
|
||||
current and previous release. `--dry-run` performs source validation and emits
|
||||
`AIOS_ORCHESTRATOR_INSTALL_DRY_RUN=PASS` without changing installed state.
|
||||
|
||||
Rollback reads that root-owned `0600` receipt, verifies both release directory
|
||||
names and `SOURCE_DIGEST` files, coordinates the gateway and poll units, and
|
||||
atomically switches `current` to the previous release. Repeating the same
|
||||
rollback is a no-op and emits `NOOP_ALREADY_ROLLED_BACK`.
|
||||
@@ -10,22 +10,11 @@ from orchestrator.action_monitor import ActionMonitor
|
||||
|
||||
ROOT = Path(os.getenv('AIOS_GITOPS_ROOT', '/data/aios-gitops'))
|
||||
WS = Path(os.getenv('AIOS_GITOPS_WORKSPACE', str(ROOT / 'workspace')))
|
||||
sys.path.insert(0, str(WS))
|
||||
from services.aios_orchestrator_gateway.action_bridge import (
|
||||
CallbackBridgeError,
|
||||
submit_orchestration_decision,
|
||||
)
|
||||
|
||||
SECRETS = Path(os.getenv('AIOS_GITOPS_SECRETS_DIR', str(ROOT / 'runtime' / 'secrets')))
|
||||
TOKEN_FILE = Path(os.getenv('AIOS_GITOPS_TOKEN_FILE', str(SECRETS / 'gitops.tokens')))
|
||||
HOST = os.getenv('AIOS_GITOPS_HOST', '127.0.0.1')
|
||||
PORT = int(os.getenv('AIOS_GITOPS_PORT', '18890'))
|
||||
OPENAPI = WS / 'docs' / 'gitops' / 'openapi-chatgpt-actions.yaml'
|
||||
ORCHESTRATOR_OPENAPI = WS / 'workspace-agent' / 'orchestrator-decision-action.openapi.yaml'
|
||||
ORCHESTRATOR_CONTRACT_COMMIT = os.getenv('AIOS_ORCHESTRATOR_CONTRACT_COMMIT', '')
|
||||
ORCHESTRATOR_CALLBACK_SECRET_REF = os.getenv('AIOS_ORCHESTRATOR_CALLBACK_SECRET_REF', '')
|
||||
_resolver = os.getenv('AIOS_ORCHESTRATOR_SECRET_REF_RESOLVER', '')
|
||||
ORCHESTRATOR_SECRET_REF_RESOLVER = Path(_resolver) if _resolver else None
|
||||
BARE_REPO = Path(os.getenv('AIOS_GITOPS_BARE_REPO', str(ROOT / 'runtime' / 'gitea-bin' / 'data' / 'repositories' / 'aios' / 'aios.git')))
|
||||
monitor = ActionMonitor()
|
||||
|
||||
@@ -195,8 +184,6 @@ class H(BaseHTTPRequestHandler):
|
||||
if path in ('/health','/api/health'): return self.sendj({'status':'ok','service':'chatgpt-gitops-adapter','workspace':str(WS)})
|
||||
if path in ('/api/openapi.yaml','/api/openapi-chatgpt-actions.yaml','/api/schema'):
|
||||
return self.sendt(OPENAPI.read_text(encoding='utf-8'), 'application/yaml; charset=utf-8') if OPENAPI.exists() else self.sendj({'error':'schema_missing'},404)
|
||||
if path == '/api/orchestrator-decision.openapi.yaml':
|
||||
return self.sendt(ORCHESTRATOR_OPENAPI.read_text(encoding='utf-8'), 'application/yaml; charset=utf-8') if ORCHESTRATOR_OPENAPI.exists() else self.sendj({'error':'schema_missing'},404)
|
||||
if path in ('/api','/api/status'): return self.sendj(api_spec())
|
||||
if path in ('/api/base','/api/baseline','/base'):
|
||||
data=jload(WS/'baseline'/'current.json', {}); data.setdefault('git', sh(['rev-parse','--short','HEAD'])['stdout']); return self.sendj(data)
|
||||
@@ -229,7 +216,6 @@ class H(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
path=urllib.parse.urlparse(self.path).path.rstrip('/'); data=self.body()
|
||||
try:
|
||||
if path == '/v1/orchestrator/decisions': return self.orchestration_decision(data)
|
||||
if path in ('/api/branch','/api/create-branch'): return self.branch(data)
|
||||
if path in ('/api/file','/api/commit-file'): return self.file(data)
|
||||
if path == '/api/commit': return self.commit(data)
|
||||
@@ -243,25 +229,6 @@ class H(BaseHTTPRequestHandler):
|
||||
except Exception as e:
|
||||
monitor.record_failure('do_POST', str(e), {'path':path})
|
||||
return self.sendj({'error':str(e)},500)
|
||||
def orchestration_decision(self,data):
|
||||
a=self.need('orchestrator:decision')
|
||||
if not a: return
|
||||
if not ORCHESTRATOR_CONTRACT_COMMIT or ORCHESTRATOR_SECRET_REF_RESOLVER is None:
|
||||
return self.sendj({'error':'CALLBACK_CONFIGURATION_UNAVAILABLE'},503)
|
||||
try:
|
||||
result=submit_orchestration_decision(
|
||||
data,
|
||||
contract_commit=ORCHESTRATOR_CONTRACT_COMMIT,
|
||||
secret_ref=ORCHESTRATOR_CALLBACK_SECRET_REF,
|
||||
resolver=ORCHESTRATOR_SECRET_REF_RESOLVER,
|
||||
)
|
||||
except CallbackBridgeError as exc:
|
||||
return self.sendj({'error':exc.category},exc.status_code)
|
||||
except Exception:
|
||||
# The outer request handler records exception strings. Keep this
|
||||
# secret-bearing boundary metadata-only even for unexpected faults.
|
||||
return self.sendj({'error':'CALLBACK_INTERNAL_ERROR'},500)
|
||||
return self.sendj(result,202)
|
||||
def branch(self,data):
|
||||
a=self.need('branch:create');
|
||||
if not a: return
|
||||
@@ -423,9 +390,9 @@ return{vs,cmds,ar,cmd,cdet,tot,sok,sfail,runs,baseline,refreshMsg,loadCmd,selVer
|
||||
|
||||
|
||||
def api_spec():
|
||||
return {'version':'V1R1C0-chatgpt-actions','openapi':'/api/openapi.yaml','orchestrator_openapi':'/api/orchestrator-decision.openapi.yaml','read':['GET /api/base','GET /api/ver','GET /api/diff','GET /api/patch','GET /api/model','GET /api/run','GET /admin/ai-evolution'],'write':['POST /v1/orchestrator/decisions','POST /api/branch','POST /api/file','POST /api/commit','POST /api/patch','POST /api/tag','POST /api/rollback','POST /api/model','PUT /api/model-priority','POST /api/nl'],'policy':{'bearer_required_for_write':True,'orchestrator_decision_scope':'orchestrator:decision','orchestrator_callback_credential':'secret_ref_only','direct_main_push':False,'delete_tag':False,'owner_approval_required_for_v_level':True,'api_keys_in_git':False,'git_secret_field':'secret_ref','online_git_sync':True}}
|
||||
return {'version':'V1R1C0-chatgpt-actions','openapi':'/api/openapi.yaml','read':['GET /api/base','GET /api/ver','GET /api/diff','GET /api/patch','GET /api/model','GET /api/run','GET /admin/ai-evolution'],'write':['POST /api/branch','POST /api/file','POST /api/commit','POST /api/patch','POST /api/tag','POST /api/rollback','POST /api/model','PUT /api/model-priority','POST /api/nl'],'policy':{'bearer_required_for_write':True,'direct_main_push':False,'delete_tag':False,'owner_approval_required_for_v_level':True,'api_keys_in_git':False,'git_secret_field':'secret_ref','online_git_sync':True}}
|
||||
|
||||
def main():
|
||||
print(f'AIOS ChatGPT GitOps adapter listening on {HOST}:{PORT}')
|
||||
ThreadingHTTPServer((HOST,PORT), H).serve_forever()
|
||||
if __name__ == '__main__': main()
|
||||
if __name__ == '__main__': main()
|
||||
@@ -0,0 +1,25 @@
|
||||
"""AIOS-managed Hermes runtime boundary."""
|
||||
|
||||
from .adapter import (
|
||||
HermesAdapter,
|
||||
HermesConflict,
|
||||
HermesNotFound,
|
||||
HermesRuntimeError,
|
||||
InMemoryHermesStore,
|
||||
)
|
||||
from .contract import HermesHealth, HermesJob, HermesJobState, HermesResult
|
||||
from .worker import HermesWorker, WorkerRunResult
|
||||
|
||||
__all__ = [
|
||||
"HermesAdapter",
|
||||
"HermesConflict",
|
||||
"HermesHealth",
|
||||
"HermesJob",
|
||||
"HermesJobState",
|
||||
"HermesNotFound",
|
||||
"HermesResult",
|
||||
"HermesRuntimeError",
|
||||
"HermesWorker",
|
||||
"InMemoryHermesStore",
|
||||
"WorkerRunResult",
|
||||
]
|
||||
@@ -0,0 +1,437 @@
|
||||
"""Tenant-partitioned Hermes queue and worker lease boundary."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import threading
|
||||
from dataclasses import replace
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Iterable, Mapping
|
||||
|
||||
from .contract import (
|
||||
ALLOWED_TRANSITIONS,
|
||||
TERMINAL_STATES,
|
||||
HermesHealth,
|
||||
HermesJob,
|
||||
HermesJobState,
|
||||
HermesResult,
|
||||
utc_now,
|
||||
)
|
||||
|
||||
|
||||
class HermesRuntimeError(RuntimeError):
|
||||
"""Base Hermes runtime-boundary failure."""
|
||||
|
||||
|
||||
class HermesNotFound(HermesRuntimeError):
|
||||
"""A tenant-scoped job was not found."""
|
||||
|
||||
|
||||
class HermesConflict(HermesRuntimeError):
|
||||
"""A state, lease, idempotency, or revision conflict occurred."""
|
||||
|
||||
|
||||
class InMemoryHermesStore:
|
||||
"""Thread-safe reference store; production backends implement this contract."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.lock = threading.RLock()
|
||||
self.jobs: dict[str, HermesJob] = {}
|
||||
self.results: dict[str, HermesResult] = {}
|
||||
self.idempotency: dict[tuple[str, str], tuple[str, str]] = {}
|
||||
self.worker_heartbeats: dict[str, tuple[str, str | None]] = {}
|
||||
|
||||
|
||||
class HermesAdapter:
|
||||
"""Queue and reconcile Hermes jobs without owning AIOS policy."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: InMemoryHermesStore | None = None,
|
||||
*,
|
||||
runtime_id: str = "hermes-runtime",
|
||||
runtime_version: str = "v1",
|
||||
default_lease_seconds: int = 60,
|
||||
) -> None:
|
||||
if default_lease_seconds < 5:
|
||||
raise ValueError("default_lease_seconds must be >= 5")
|
||||
self.store = store or InMemoryHermesStore()
|
||||
self.runtime_id = runtime_id
|
||||
self.runtime_version = runtime_version
|
||||
self.default_lease_seconds = int(default_lease_seconds)
|
||||
|
||||
@staticmethod
|
||||
def _digest(job: HermesJob) -> str:
|
||||
raw = json.dumps(
|
||||
{
|
||||
"tenant_id": job.tenant_id,
|
||||
"cell_id": job.cell_id,
|
||||
"profile_id": job.profile_id,
|
||||
"task_id": job.task_id,
|
||||
"trace_id": job.trace_id,
|
||||
"action": job.action,
|
||||
"payload": dict(job.payload),
|
||||
"priority": job.priority,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
default=str,
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _parse(value: str | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value[:-1] + "+00:00" if value.endswith("Z") else value
|
||||
parsed = datetime.fromisoformat(normalized)
|
||||
if parsed.tzinfo is None:
|
||||
raise ValueError("timestamp must include timezone")
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
@staticmethod
|
||||
def _format(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
@staticmethod
|
||||
def _assert_tenant(job: HermesJob, tenant_id: str) -> None:
|
||||
if job.tenant_id != tenant_id:
|
||||
raise HermesNotFound("job not found")
|
||||
|
||||
def enqueue(self, job: HermesJob) -> HermesJob:
|
||||
if job.state != HermesJobState.QUEUED or job.revision != 1:
|
||||
raise HermesConflict("new job must be QUEUED at revision 1")
|
||||
digest = self._digest(job)
|
||||
idem_ref = (job.tenant_id, job.idempotency_key)
|
||||
with self.store.lock:
|
||||
existing = self.store.jobs.get(job.job_id)
|
||||
if existing is not None:
|
||||
if self._digest(existing) != digest:
|
||||
raise HermesConflict("job_id reused with different content")
|
||||
return existing
|
||||
prior = self.store.idempotency.get(idem_ref)
|
||||
if prior is not None:
|
||||
prior_id, prior_digest = prior
|
||||
if prior_digest != digest:
|
||||
raise HermesConflict("idempotency key reused with different content")
|
||||
return self.store.jobs[prior_id]
|
||||
self.store.jobs[job.job_id] = job
|
||||
self.store.idempotency[idem_ref] = (job.job_id, digest)
|
||||
return job
|
||||
|
||||
def get(self, job_id: str, *, tenant_id: str) -> HermesJob:
|
||||
with self.store.lock:
|
||||
job = self.store.jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HermesNotFound("job not found")
|
||||
self._assert_tenant(job, tenant_id)
|
||||
return job
|
||||
|
||||
def list_jobs(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
cell_id: str | None = None,
|
||||
profile_id: str | None = None,
|
||||
states: Iterable[HermesJobState | str] | None = None,
|
||||
) -> list[HermesJob]:
|
||||
state_filter = {HermesJobState(item) for item in states} if states else None
|
||||
with self.store.lock:
|
||||
self._requeue_expired_locked()
|
||||
jobs = [
|
||||
job
|
||||
for job in self.store.jobs.values()
|
||||
if job.tenant_id == tenant_id
|
||||
and (cell_id is None or job.cell_id == cell_id)
|
||||
and (profile_id is None or job.profile_id == profile_id)
|
||||
and (state_filter is None or job.state in state_filter)
|
||||
]
|
||||
return sorted(jobs, key=lambda item: (item.priority, item.created_at, item.job_id))
|
||||
|
||||
def claim(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
cell_id: str,
|
||||
profile_id: str,
|
||||
worker_id: str,
|
||||
lease_seconds: int | None = None,
|
||||
) -> HermesJob | None:
|
||||
worker_id = str(worker_id or "").strip()
|
||||
if not worker_id:
|
||||
raise ValueError("worker_id is required")
|
||||
duration = int(lease_seconds or self.default_lease_seconds)
|
||||
if duration < 5:
|
||||
raise ValueError("lease_seconds must be >= 5")
|
||||
with self.store.lock:
|
||||
self._requeue_expired_locked()
|
||||
candidates = [
|
||||
job
|
||||
for job in self.store.jobs.values()
|
||||
if job.tenant_id == tenant_id
|
||||
and job.cell_id == cell_id
|
||||
and job.profile_id == profile_id
|
||||
and job.state == HermesJobState.QUEUED
|
||||
]
|
||||
self.store.worker_heartbeats[worker_id] = (utc_now(), None)
|
||||
if not candidates:
|
||||
return None
|
||||
current = sorted(
|
||||
candidates,
|
||||
key=lambda item: (item.priority, item.created_at, item.job_id),
|
||||
)[0]
|
||||
claimed = self._transition_locked(
|
||||
current,
|
||||
HermesJobState.CLAIMED,
|
||||
assigned_worker_id=worker_id,
|
||||
lease_expires_at=self._format(
|
||||
datetime.now(timezone.utc) + timedelta(seconds=duration)
|
||||
),
|
||||
)
|
||||
self.store.worker_heartbeats[worker_id] = (utc_now(), claimed.job_id)
|
||||
return claimed
|
||||
|
||||
def start(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
tenant_id: str,
|
||||
worker_id: str,
|
||||
expected_revision: int | None = None,
|
||||
) -> HermesJob:
|
||||
with self.store.lock:
|
||||
current = self._owned_job_locked(job_id, tenant_id, worker_id)
|
||||
self._assert_revision(current, expected_revision)
|
||||
return self._transition_locked(current, HermesJobState.RUNNING)
|
||||
|
||||
def heartbeat(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
tenant_id: str,
|
||||
worker_id: str,
|
||||
lease_seconds: int | None = None,
|
||||
) -> HermesJob:
|
||||
duration = int(lease_seconds or self.default_lease_seconds)
|
||||
with self.store.lock:
|
||||
current = self._owned_job_locked(job_id, tenant_id, worker_id)
|
||||
if current.state not in {HermesJobState.CLAIMED, HermesJobState.RUNNING}:
|
||||
raise HermesConflict("job does not accept heartbeat")
|
||||
updated = replace(
|
||||
current,
|
||||
revision=current.revision + 1,
|
||||
lease_expires_at=self._format(
|
||||
datetime.now(timezone.utc) + timedelta(seconds=duration)
|
||||
),
|
||||
updated_at=utc_now(),
|
||||
)
|
||||
self.store.jobs[job_id] = updated
|
||||
self.store.worker_heartbeats[worker_id] = (utc_now(), job_id)
|
||||
return updated
|
||||
|
||||
def complete(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
tenant_id: str,
|
||||
worker_id: str,
|
||||
output: Mapping[str, object],
|
||||
expected_revision: int | None = None,
|
||||
) -> HermesResult:
|
||||
return self._finish(
|
||||
job_id,
|
||||
tenant_id=tenant_id,
|
||||
worker_id=worker_id,
|
||||
state=HermesJobState.COMPLETED,
|
||||
output=output,
|
||||
expected_revision=expected_revision,
|
||||
)
|
||||
|
||||
def fail(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
tenant_id: str,
|
||||
worker_id: str,
|
||||
error: Mapping[str, object],
|
||||
expected_revision: int | None = None,
|
||||
) -> HermesResult:
|
||||
return self._finish(
|
||||
job_id,
|
||||
tenant_id=tenant_id,
|
||||
worker_id=worker_id,
|
||||
state=HermesJobState.FAILED,
|
||||
error=error,
|
||||
expected_revision=expected_revision,
|
||||
)
|
||||
|
||||
def cancel(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
tenant_id: str,
|
||||
expected_revision: int | None = None,
|
||||
) -> HermesResult:
|
||||
with self.store.lock:
|
||||
current = self.store.jobs.get(job_id)
|
||||
if current is None:
|
||||
raise HermesNotFound("job not found")
|
||||
self._assert_tenant(current, tenant_id)
|
||||
self._assert_revision(current, expected_revision)
|
||||
if current.state in TERMINAL_STATES:
|
||||
result = self.store.results.get(job_id)
|
||||
if result is None:
|
||||
raise HermesConflict("terminal result missing")
|
||||
return result
|
||||
updated = self._transition_locked(current, HermesJobState.CANCELLED)
|
||||
result = HermesResult(
|
||||
job_id=updated.job_id,
|
||||
tenant_id=updated.tenant_id,
|
||||
task_id=updated.task_id,
|
||||
trace_id=updated.trace_id,
|
||||
state=HermesJobState.CANCELLED,
|
||||
worker_id=updated.assigned_worker_id,
|
||||
error={"code": "cancelled", "message": "cancelled by AIOS"},
|
||||
finished_at=utc_now(),
|
||||
)
|
||||
self.store.results[job_id] = result
|
||||
return result
|
||||
|
||||
def result(self, job_id: str, *, tenant_id: str) -> HermesResult:
|
||||
with self.store.lock:
|
||||
job = self.store.jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HermesNotFound("job not found")
|
||||
self._assert_tenant(job, tenant_id)
|
||||
result = self.store.results.get(job_id)
|
||||
if result is None:
|
||||
raise HermesConflict("job has no terminal result")
|
||||
return result
|
||||
|
||||
def health(self) -> HermesHealth:
|
||||
with self.store.lock:
|
||||
self._requeue_expired_locked()
|
||||
active_jobs = sum(
|
||||
1 for job in self.store.jobs.values()
|
||||
if job.state in {HermesJobState.CLAIMED, HermesJobState.RUNNING}
|
||||
)
|
||||
queued_jobs = sum(
|
||||
1 for job in self.store.jobs.values()
|
||||
if job.state == HermesJobState.QUEUED
|
||||
)
|
||||
partitions = {
|
||||
(job.tenant_id, job.cell_id, job.profile_id)
|
||||
for job in self.store.jobs.values()
|
||||
}
|
||||
active_workers = sum(
|
||||
1 for heartbeat, _ in self.store.worker_heartbeats.values()
|
||||
if self._parse(heartbeat)
|
||||
and self._parse(heartbeat) >= datetime.now(timezone.utc) - timedelta(seconds=120)
|
||||
)
|
||||
return HermesHealth(
|
||||
runtime_id=self.runtime_id,
|
||||
status="healthy",
|
||||
version=self.runtime_version,
|
||||
heartbeat_at=utc_now(),
|
||||
active_workers=active_workers,
|
||||
active_jobs=active_jobs,
|
||||
queued_jobs=queued_jobs,
|
||||
tenant_partitions=len(partitions),
|
||||
profile_isolation_enforced=True,
|
||||
details={"store": type(self.store).__name__},
|
||||
)
|
||||
|
||||
def _finish(
|
||||
self,
|
||||
job_id: str,
|
||||
*,
|
||||
tenant_id: str,
|
||||
worker_id: str,
|
||||
state: HermesJobState,
|
||||
output: Mapping[str, object] | None = None,
|
||||
error: Mapping[str, object] | None = None,
|
||||
expected_revision: int | None,
|
||||
) -> HermesResult:
|
||||
with self.store.lock:
|
||||
current = self._owned_job_locked(job_id, tenant_id, worker_id)
|
||||
self._assert_revision(current, expected_revision)
|
||||
updated = self._transition_locked(current, state)
|
||||
result = HermesResult(
|
||||
job_id=updated.job_id,
|
||||
tenant_id=updated.tenant_id,
|
||||
task_id=updated.task_id,
|
||||
trace_id=updated.trace_id,
|
||||
state=state,
|
||||
worker_id=worker_id,
|
||||
output=output,
|
||||
error=error,
|
||||
started_at=current.updated_at,
|
||||
finished_at=utc_now(),
|
||||
)
|
||||
self.store.results[job_id] = result
|
||||
self.store.worker_heartbeats[worker_id] = (utc_now(), None)
|
||||
return result
|
||||
|
||||
def _owned_job_locked(self, job_id: str, tenant_id: str, worker_id: str) -> HermesJob:
|
||||
current = self.store.jobs.get(job_id)
|
||||
if current is None:
|
||||
raise HermesNotFound("job not found")
|
||||
self._assert_tenant(current, tenant_id)
|
||||
if current.assigned_worker_id != worker_id:
|
||||
raise HermesConflict("worker does not own job lease")
|
||||
return current
|
||||
|
||||
@staticmethod
|
||||
def _assert_revision(job: HermesJob, expected: int | None) -> None:
|
||||
if expected is not None and job.revision != expected:
|
||||
raise HermesConflict("job revision changed")
|
||||
|
||||
def _transition_locked(
|
||||
self,
|
||||
current: HermesJob,
|
||||
target: HermesJobState,
|
||||
*,
|
||||
assigned_worker_id: str | None = None,
|
||||
lease_expires_at: str | None = None,
|
||||
) -> HermesJob:
|
||||
if target not in ALLOWED_TRANSITIONS[current.state]:
|
||||
raise HermesConflict(
|
||||
f"invalid transition {current.state.value} -> {target.value}"
|
||||
)
|
||||
updated = replace(
|
||||
current,
|
||||
state=target,
|
||||
revision=current.revision + 1,
|
||||
assigned_worker_id=(
|
||||
assigned_worker_id
|
||||
if assigned_worker_id is not None
|
||||
else current.assigned_worker_id
|
||||
),
|
||||
lease_expires_at=(
|
||||
lease_expires_at
|
||||
if lease_expires_at is not None
|
||||
else (None if target in TERMINAL_STATES else current.lease_expires_at)
|
||||
),
|
||||
updated_at=utc_now(),
|
||||
)
|
||||
self.store.jobs[current.job_id] = updated
|
||||
return updated
|
||||
|
||||
def _requeue_expired_locked(self) -> None:
|
||||
now = datetime.now(timezone.utc)
|
||||
for current in list(self.store.jobs.values()):
|
||||
expiry = self._parse(current.lease_expires_at)
|
||||
if (
|
||||
current.state in {HermesJobState.CLAIMED, HermesJobState.RUNNING}
|
||||
and expiry is not None
|
||||
and expiry <= now
|
||||
):
|
||||
self.store.jobs[current.job_id] = replace(
|
||||
current,
|
||||
state=HermesJobState.QUEUED,
|
||||
revision=current.revision + 1,
|
||||
assigned_worker_id=None,
|
||||
lease_expires_at=None,
|
||||
updated_at=utc_now(),
|
||||
)
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Tenant-scoped Hermes runtime contracts."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Mapping
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _required(value: str, label: str) -> str:
|
||||
normalized = str(value or "").strip()
|
||||
if not normalized:
|
||||
raise ValueError(f"{label} is required")
|
||||
return normalized
|
||||
|
||||
|
||||
def _safe_mapping(value: Mapping[str, Any] | None) -> dict[str, Any]:
|
||||
return json.loads(json.dumps(dict(value or {}), ensure_ascii=False, sort_keys=True, default=str))
|
||||
|
||||
|
||||
class HermesJobState(str, Enum):
|
||||
QUEUED = "QUEUED"
|
||||
CLAIMED = "CLAIMED"
|
||||
RUNNING = "RUNNING"
|
||||
COMPLETED = "COMPLETED"
|
||||
FAILED = "FAILED"
|
||||
CANCELLED = "CANCELLED"
|
||||
|
||||
|
||||
ALLOWED_TRANSITIONS: Mapping[HermesJobState, frozenset[HermesJobState]] = {
|
||||
HermesJobState.QUEUED: frozenset({HermesJobState.CLAIMED, HermesJobState.CANCELLED}),
|
||||
HermesJobState.CLAIMED: frozenset({HermesJobState.RUNNING, HermesJobState.CANCELLED}),
|
||||
HermesJobState.RUNNING: frozenset(
|
||||
{HermesJobState.COMPLETED, HermesJobState.FAILED, HermesJobState.CANCELLED}
|
||||
),
|
||||
HermesJobState.COMPLETED: frozenset(),
|
||||
HermesJobState.FAILED: frozenset(),
|
||||
HermesJobState.CANCELLED: frozenset(),
|
||||
}
|
||||
TERMINAL_STATES = frozenset(
|
||||
{HermesJobState.COMPLETED, HermesJobState.FAILED, HermesJobState.CANCELLED}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HermesJob:
|
||||
job_id: str
|
||||
tenant_id: str
|
||||
cell_id: str
|
||||
profile_id: str
|
||||
task_id: str
|
||||
trace_id: str
|
||||
action: str
|
||||
payload: Mapping[str, Any] = field(default_factory=dict)
|
||||
idempotency_key: str = ""
|
||||
priority: int = 100
|
||||
state: HermesJobState = HermesJobState.QUEUED
|
||||
revision: int = 1
|
||||
assigned_worker_id: str | None = None
|
||||
lease_expires_at: str | None = None
|
||||
created_at: str = field(default_factory=utc_now)
|
||||
updated_at: str = field(default_factory=utc_now)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name in (
|
||||
"job_id", "tenant_id", "cell_id", "profile_id", "task_id",
|
||||
"trace_id", "action", "idempotency_key", "created_at", "updated_at",
|
||||
):
|
||||
object.__setattr__(self, name, _required(getattr(self, name), name))
|
||||
object.__setattr__(self, "payload", _safe_mapping(self.payload))
|
||||
object.__setattr__(self, "state", HermesJobState(self.state))
|
||||
if not 0 <= int(self.priority) <= 1000:
|
||||
raise ValueError("priority must be between 0 and 1000")
|
||||
if self.revision < 1:
|
||||
raise ValueError("revision must be >= 1")
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls,
|
||||
*,
|
||||
tenant_id: str,
|
||||
cell_id: str,
|
||||
profile_id: str,
|
||||
task_id: str,
|
||||
trace_id: str,
|
||||
action: str,
|
||||
payload: Mapping[str, Any] | None,
|
||||
idempotency_key: str,
|
||||
priority: int = 100,
|
||||
) -> "HermesJob":
|
||||
return cls(
|
||||
job_id=f"hermes_{uuid.uuid4().hex}",
|
||||
tenant_id=tenant_id,
|
||||
cell_id=cell_id,
|
||||
profile_id=profile_id,
|
||||
task_id=task_id,
|
||||
trace_id=trace_id,
|
||||
action=action,
|
||||
payload=_safe_mapping(payload),
|
||||
idempotency_key=idempotency_key,
|
||||
priority=priority,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HermesResult:
|
||||
job_id: str
|
||||
tenant_id: str
|
||||
task_id: str
|
||||
trace_id: str
|
||||
state: HermesJobState
|
||||
worker_id: str | None = None
|
||||
output: Mapping[str, Any] | None = None
|
||||
error: Mapping[str, Any] | None = None
|
||||
started_at: str | None = None
|
||||
finished_at: str | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name in ("job_id", "tenant_id", "task_id", "trace_id"):
|
||||
object.__setattr__(self, name, _required(getattr(self, name), name))
|
||||
object.__setattr__(self, "state", HermesJobState(self.state))
|
||||
if self.output is not None:
|
||||
object.__setattr__(self, "output", _safe_mapping(self.output))
|
||||
if self.error is not None:
|
||||
object.__setattr__(self, "error", _safe_mapping(self.error))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HermesHealth:
|
||||
runtime_id: str
|
||||
status: str
|
||||
version: str
|
||||
heartbeat_at: str
|
||||
active_workers: int = 0
|
||||
active_jobs: int = 0
|
||||
queued_jobs: int = 0
|
||||
tenant_partitions: int = 0
|
||||
profile_isolation_enforced: bool = True
|
||||
details: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name in ("runtime_id", "status", "version", "heartbeat_at"):
|
||||
object.__setattr__(self, name, _required(getattr(self, name), name))
|
||||
for name in ("active_workers", "active_jobs", "queued_jobs", "tenant_partitions"):
|
||||
if int(getattr(self, name)) < 0:
|
||||
raise ValueError(f"{name} must be >= 0")
|
||||
object.__setattr__(self, "details", _safe_mapping(self.details))
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Hermes worker loop with lease renewal and bounded recovery."""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Mapping
|
||||
|
||||
from .adapter import HermesAdapter, HermesConflict
|
||||
from .contract import HermesJob, HermesJobState
|
||||
|
||||
|
||||
JobHandler = Callable[[HermesJob], Mapping[str, Any]]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkerRunResult:
|
||||
claimed: bool
|
||||
job_id: str | None = None
|
||||
state: str | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class HermesWorker:
|
||||
"""Poll one tenant/cell/profile partition and execute jobs safely."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
adapter: HermesAdapter,
|
||||
handler: JobHandler,
|
||||
*,
|
||||
worker_id: str,
|
||||
tenant_id: str,
|
||||
cell_id: str,
|
||||
profile_id: str,
|
||||
poll_seconds: float = 1.0,
|
||||
heartbeat_seconds: float = 10.0,
|
||||
) -> None:
|
||||
if poll_seconds <= 0 or heartbeat_seconds <= 0:
|
||||
raise ValueError("poll_seconds and heartbeat_seconds must be positive")
|
||||
for name, value in {
|
||||
"worker_id": worker_id,
|
||||
"tenant_id": tenant_id,
|
||||
"cell_id": cell_id,
|
||||
"profile_id": profile_id,
|
||||
}.items():
|
||||
if not str(value or "").strip():
|
||||
raise ValueError(f"{name} is required")
|
||||
self.adapter = adapter
|
||||
self.handler = handler
|
||||
self.worker_id = worker_id
|
||||
self.tenant_id = tenant_id
|
||||
self.cell_id = cell_id
|
||||
self.profile_id = profile_id
|
||||
self.poll_seconds = float(poll_seconds)
|
||||
self.heartbeat_seconds = float(heartbeat_seconds)
|
||||
self._stop = threading.Event()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
|
||||
def run_once(self) -> WorkerRunResult:
|
||||
job = self.adapter.claim(
|
||||
tenant_id=self.tenant_id,
|
||||
cell_id=self.cell_id,
|
||||
profile_id=self.profile_id,
|
||||
worker_id=self.worker_id,
|
||||
)
|
||||
if job is None:
|
||||
return WorkerRunResult(claimed=False)
|
||||
|
||||
running = self.adapter.start(
|
||||
job.job_id,
|
||||
tenant_id=self.tenant_id,
|
||||
worker_id=self.worker_id,
|
||||
expected_revision=job.revision,
|
||||
)
|
||||
|
||||
heartbeat_stop = threading.Event()
|
||||
heartbeat_thread = threading.Thread(
|
||||
target=self._heartbeat_loop,
|
||||
args=(running.job_id, heartbeat_stop),
|
||||
name=f"hermes-heartbeat-{self.worker_id}",
|
||||
daemon=True,
|
||||
)
|
||||
heartbeat_thread.start()
|
||||
|
||||
output: Mapping[str, Any] | None = None
|
||||
execution_error: Exception | None = None
|
||||
try:
|
||||
output = dict(self.handler(running))
|
||||
except Exception as exc: # runtime boundary: normalize below
|
||||
execution_error = exc
|
||||
finally:
|
||||
# Stop the lease heartbeat before reading the final revision. This
|
||||
# prevents a revision increment racing with complete/fail.
|
||||
heartbeat_stop.set()
|
||||
heartbeat_thread.join(timeout=max(self.heartbeat_seconds * 2, 1.0))
|
||||
|
||||
try:
|
||||
current = self.adapter.get(running.job_id, tenant_id=self.tenant_id)
|
||||
if execution_error is None:
|
||||
result = self.adapter.complete(
|
||||
running.job_id,
|
||||
tenant_id=self.tenant_id,
|
||||
worker_id=self.worker_id,
|
||||
output=dict(output or {}),
|
||||
expected_revision=current.revision,
|
||||
)
|
||||
return WorkerRunResult(
|
||||
claimed=True,
|
||||
job_id=result.job_id,
|
||||
state=result.state.value,
|
||||
)
|
||||
|
||||
result = self.adapter.fail(
|
||||
running.job_id,
|
||||
tenant_id=self.tenant_id,
|
||||
worker_id=self.worker_id,
|
||||
error={
|
||||
"code": "worker_error",
|
||||
"message": str(execution_error),
|
||||
},
|
||||
expected_revision=current.revision,
|
||||
)
|
||||
return WorkerRunResult(
|
||||
claimed=True,
|
||||
job_id=result.job_id,
|
||||
state=result.state.value,
|
||||
error=str(execution_error),
|
||||
)
|
||||
except HermesConflict as exc:
|
||||
return WorkerRunResult(
|
||||
claimed=True,
|
||||
job_id=running.job_id,
|
||||
state=HermesJobState.FAILED.value,
|
||||
error=f"reconciliation_conflict: {exc}",
|
||||
)
|
||||
|
||||
def run_forever(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
result = self.run_once()
|
||||
if not result.claimed:
|
||||
self._stop.wait(self.poll_seconds)
|
||||
|
||||
def _heartbeat_loop(self, job_id: str, stop_event: threading.Event) -> None:
|
||||
while not stop_event.wait(self.heartbeat_seconds):
|
||||
try:
|
||||
self.adapter.heartbeat(
|
||||
job_id,
|
||||
tenant_id=self.tenant_id,
|
||||
worker_id=self.worker_id,
|
||||
)
|
||||
except Exception:
|
||||
stop_event.set()
|
||||
return
|
||||
@@ -0,0 +1,37 @@
|
||||
"""AIOS-managed OpenClaw execution adapter."""
|
||||
|
||||
from .adapter import (
|
||||
AdapterError,
|
||||
CommandConflict,
|
||||
CommandNotFound,
|
||||
CommandRejected,
|
||||
InMemoryCallbackSink,
|
||||
OpenClawAdapter,
|
||||
)
|
||||
from .contract import (
|
||||
CommandState,
|
||||
OpenClawCommand,
|
||||
OpenClawResult,
|
||||
RuntimeHealth,
|
||||
sign_command,
|
||||
verify_command_signature,
|
||||
)
|
||||
from .transport import HTTPRuntimeTransport, InProcessRuntimeTransport, RuntimeTransport
|
||||
|
||||
__all__ = [
|
||||
"AdapterError",
|
||||
"CommandConflict",
|
||||
"CommandNotFound",
|
||||
"CommandRejected",
|
||||
"CommandState",
|
||||
"HTTPRuntimeTransport",
|
||||
"InMemoryCallbackSink",
|
||||
"InProcessRuntimeTransport",
|
||||
"OpenClawAdapter",
|
||||
"OpenClawCommand",
|
||||
"OpenClawResult",
|
||||
"RuntimeHealth",
|
||||
"RuntimeTransport",
|
||||
"sign_command",
|
||||
"verify_command_signature",
|
||||
]
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Signed, tenant-isolated execution adapter for OpenClaw runtimes."""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from dataclasses import replace
|
||||
from typing import Callable, Mapping, Protocol
|
||||
|
||||
from .contract import (
|
||||
CommandState,
|
||||
OpenClawCommand,
|
||||
OpenClawResult,
|
||||
RuntimeHealth,
|
||||
utc_now,
|
||||
verify_command_signature,
|
||||
)
|
||||
|
||||
|
||||
class AdapterError(RuntimeError):
|
||||
"""Base adapter failure."""
|
||||
|
||||
|
||||
class CommandRejected(AdapterError):
|
||||
"""A command failed signature or contract validation."""
|
||||
|
||||
|
||||
class CommandNotFound(AdapterError):
|
||||
"""A tenant-scoped command is not visible."""
|
||||
|
||||
|
||||
class CommandConflict(AdapterError):
|
||||
"""A command or idempotency key conflicts with an existing execution."""
|
||||
|
||||
|
||||
class RuntimeTransport(Protocol):
|
||||
def execute(self, command: OpenClawCommand, timeout_seconds: float) -> OpenClawResult: ...
|
||||
def cancel(self, command_id: str) -> bool: ...
|
||||
def health(self) -> RuntimeHealth: ...
|
||||
|
||||
|
||||
class CallbackSink(Protocol):
|
||||
def on_started(self, result: OpenClawResult) -> None: ...
|
||||
def on_completed(self, result: OpenClawResult) -> None: ...
|
||||
def on_failed(self, result: OpenClawResult) -> None: ...
|
||||
|
||||
|
||||
class InMemoryCallbackSink:
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self.started: list[OpenClawResult] = []
|
||||
self.completed: list[OpenClawResult] = []
|
||||
self.failed: list[OpenClawResult] = []
|
||||
|
||||
def on_started(self, result: OpenClawResult) -> None:
|
||||
with self._lock:
|
||||
self.started.append(result)
|
||||
|
||||
def on_completed(self, result: OpenClawResult) -> None:
|
||||
with self._lock:
|
||||
self.completed.append(result)
|
||||
|
||||
def on_failed(self, result: OpenClawResult) -> None:
|
||||
with self._lock:
|
||||
self.failed.append(result)
|
||||
|
||||
|
||||
SigningMaterialResolver = Callable[[str, str], bytes | str]
|
||||
|
||||
|
||||
class OpenClawAdapter:
|
||||
"""Verify, de-duplicate, execute, cancel and reconcile commands."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
transport: RuntimeTransport,
|
||||
signing_material_resolver: SigningMaterialResolver,
|
||||
*,
|
||||
callback_sink: CallbackSink | None = None,
|
||||
default_timeout_seconds: float = 300.0,
|
||||
) -> None:
|
||||
if default_timeout_seconds <= 0:
|
||||
raise ValueError("default_timeout_seconds must be positive")
|
||||
self.transport = transport
|
||||
self.signing_material_resolver = signing_material_resolver
|
||||
self.callback_sink = callback_sink or InMemoryCallbackSink()
|
||||
self.default_timeout_seconds = float(default_timeout_seconds)
|
||||
self._lock = threading.RLock()
|
||||
self._commands: dict[str, OpenClawCommand] = {}
|
||||
self._results: dict[str, OpenClawResult] = {}
|
||||
self._idempotency: dict[tuple[str, str], tuple[str, str]] = {}
|
||||
|
||||
def _verify(self, command: OpenClawCommand) -> None:
|
||||
try:
|
||||
key_material = self.signing_material_resolver(command.tenant_id, command.key_id)
|
||||
except Exception as exc:
|
||||
raise CommandRejected("signing material unavailable") from exc
|
||||
if not verify_command_signature(command, key_material):
|
||||
raise CommandRejected("invalid or expired command signature")
|
||||
|
||||
@staticmethod
|
||||
def _validate_result(command: OpenClawCommand, result: OpenClawResult) -> None:
|
||||
if (
|
||||
result.command_id != command.command_id
|
||||
or result.tenant_id != command.tenant_id
|
||||
or result.task_id != command.task_id
|
||||
or result.trace_id != command.trace_id
|
||||
):
|
||||
raise AdapterError("runtime returned mismatched command identity")
|
||||
if result.state not in {
|
||||
CommandState.COMPLETED,
|
||||
CommandState.FAILED,
|
||||
CommandState.CANCELLED,
|
||||
CommandState.TIMED_OUT,
|
||||
}:
|
||||
raise AdapterError("runtime returned a non-terminal result")
|
||||
|
||||
def execute(
|
||||
self,
|
||||
command: OpenClawCommand,
|
||||
*,
|
||||
timeout_seconds: float | None = None,
|
||||
) -> OpenClawResult:
|
||||
timeout = self.default_timeout_seconds if timeout_seconds is None else float(timeout_seconds)
|
||||
if timeout <= 0:
|
||||
raise ValueError("timeout_seconds must be positive")
|
||||
self._verify(command)
|
||||
digest = command.signature
|
||||
idem_ref = (command.tenant_id, command.idempotency_key)
|
||||
|
||||
with self._lock:
|
||||
prior_command = self._commands.get(command.command_id)
|
||||
if prior_command is not None:
|
||||
if prior_command.signature != command.signature:
|
||||
raise CommandConflict("command_id reused with different content")
|
||||
return self._results[command.command_id]
|
||||
prior_idem = self._idempotency.get(idem_ref)
|
||||
if prior_idem is not None:
|
||||
prior_id, prior_digest = prior_idem
|
||||
if prior_digest != digest:
|
||||
raise CommandConflict("idempotency key reused with different content")
|
||||
return self._results[prior_id]
|
||||
running = OpenClawResult(
|
||||
command_id=command.command_id,
|
||||
tenant_id=command.tenant_id,
|
||||
task_id=command.task_id,
|
||||
trace_id=command.trace_id,
|
||||
state=CommandState.RUNNING,
|
||||
started_at=utc_now(),
|
||||
)
|
||||
self._commands[command.command_id] = command
|
||||
self._results[command.command_id] = running
|
||||
self._idempotency[idem_ref] = (command.command_id, digest)
|
||||
|
||||
self.callback_sink.on_started(running)
|
||||
try:
|
||||
result = self.transport.execute(command, timeout)
|
||||
self._validate_result(command, result)
|
||||
except TimeoutError as exc:
|
||||
result = replace(
|
||||
running,
|
||||
state=CommandState.TIMED_OUT,
|
||||
error={"code": "runtime_timeout", "message": str(exc) or "runtime timeout"},
|
||||
finished_at=utc_now(),
|
||||
)
|
||||
except Exception as exc:
|
||||
result = replace(
|
||||
running,
|
||||
state=CommandState.FAILED,
|
||||
error={"code": "runtime_error", "message": str(exc)},
|
||||
finished_at=utc_now(),
|
||||
)
|
||||
|
||||
if result.started_at is None:
|
||||
result = replace(result, started_at=running.started_at)
|
||||
if result.finished_at is None:
|
||||
result = replace(result, finished_at=utc_now())
|
||||
with self._lock:
|
||||
self._results[command.command_id] = result
|
||||
if result.state == CommandState.COMPLETED:
|
||||
self.callback_sink.on_completed(result)
|
||||
else:
|
||||
self.callback_sink.on_failed(result)
|
||||
return result
|
||||
|
||||
def status(self, command_id: str, *, tenant_id: str) -> OpenClawResult:
|
||||
with self._lock:
|
||||
command = self._commands.get(command_id)
|
||||
result = self._results.get(command_id)
|
||||
if command is None or result is None or command.tenant_id != tenant_id:
|
||||
raise CommandNotFound("command not found")
|
||||
return result
|
||||
|
||||
def cancel(self, command_id: str, *, tenant_id: str) -> OpenClawResult:
|
||||
with self._lock:
|
||||
command = self._commands.get(command_id)
|
||||
current = self._results.get(command_id)
|
||||
if command is None or current is None or command.tenant_id != tenant_id:
|
||||
raise CommandNotFound("command not found")
|
||||
if current.state in {
|
||||
CommandState.COMPLETED,
|
||||
CommandState.FAILED,
|
||||
CommandState.CANCELLED,
|
||||
CommandState.TIMED_OUT,
|
||||
CommandState.REJECTED,
|
||||
}:
|
||||
return current
|
||||
if not self.transport.cancel(command_id):
|
||||
raise AdapterError("runtime rejected cancellation")
|
||||
cancelled = replace(
|
||||
current,
|
||||
state=CommandState.CANCELLED,
|
||||
error={"code": "cancelled", "message": "cancelled by AIOS"},
|
||||
finished_at=utc_now(),
|
||||
)
|
||||
with self._lock:
|
||||
self._results[command_id] = cancelled
|
||||
self.callback_sink.on_failed(cancelled)
|
||||
return cancelled
|
||||
|
||||
def health(self) -> RuntimeHealth:
|
||||
return self.transport.health()
|
||||
|
||||
def snapshot(self, *, tenant_id: str) -> Mapping[str, OpenClawResult]:
|
||||
with self._lock:
|
||||
return {
|
||||
command_id: result
|
||||
for command_id, result in self._results.items()
|
||||
if self._commands[command_id].tenant_id == tenant_id
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Audit-oriented lifecycle callbacks for OpenClaw execution."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Mapping, Protocol
|
||||
|
||||
from .contract import OpenClawResult, utc_now
|
||||
|
||||
|
||||
class EventPublisher(Protocol):
|
||||
def publish(self, event: Mapping[str, Any]) -> None: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExecutionEvent:
|
||||
event_type: str
|
||||
tenant_id: str
|
||||
task_id: str
|
||||
trace_id: str
|
||||
command_id: str
|
||||
state: str
|
||||
created_at: str = field(default_factory=utc_now)
|
||||
details: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"event_type": self.event_type,
|
||||
"tenant_id": self.tenant_id,
|
||||
"task_id": self.task_id,
|
||||
"trace_id": self.trace_id,
|
||||
"command_id": self.command_id,
|
||||
"state": self.state,
|
||||
"created_at": self.created_at,
|
||||
"details": json.loads(
|
||||
json.dumps(dict(self.details), ensure_ascii=False, default=str)
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class InMemoryEventPublisher:
|
||||
"""Thread-safe publisher used by tests and first integration slice."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self.events: list[dict[str, Any]] = []
|
||||
|
||||
def publish(self, event: Mapping[str, Any]) -> None:
|
||||
with self._lock:
|
||||
self.events.append(
|
||||
json.loads(json.dumps(dict(event), ensure_ascii=False, default=str))
|
||||
)
|
||||
|
||||
|
||||
class AuditCallbackSink:
|
||||
"""Convert runtime callbacks into tenant-scoped audit events."""
|
||||
|
||||
def __init__(self, publisher: EventPublisher) -> None:
|
||||
self.publisher = publisher
|
||||
|
||||
@staticmethod
|
||||
def _event(
|
||||
event_type: str,
|
||||
result: OpenClawResult,
|
||||
) -> ExecutionEvent:
|
||||
details: dict[str, Any] = {
|
||||
"runtime_id": result.runtime_id,
|
||||
"attempts": result.attempts,
|
||||
}
|
||||
if result.error:
|
||||
details["error"] = dict(result.error)
|
||||
if result.output:
|
||||
details["output_present"] = True
|
||||
return ExecutionEvent(
|
||||
event_type=event_type,
|
||||
tenant_id=result.tenant_id,
|
||||
task_id=result.task_id,
|
||||
trace_id=result.trace_id,
|
||||
command_id=result.command_id,
|
||||
state=result.state.value,
|
||||
details=details,
|
||||
)
|
||||
|
||||
def on_started(self, result: OpenClawResult) -> None:
|
||||
self.publisher.publish(self._event("OPENCLAW_STARTED", result).as_dict())
|
||||
|
||||
def on_completed(self, result: OpenClawResult) -> None:
|
||||
self.publisher.publish(self._event("OPENCLAW_COMPLETED", result).as_dict())
|
||||
|
||||
def on_failed(self, result: OpenClawResult) -> None:
|
||||
event_type = {
|
||||
"CANCELLED": "OPENCLAW_CANCELLED",
|
||||
"TIMED_OUT": "OPENCLAW_TIMED_OUT",
|
||||
"REJECTED": "OPENCLAW_REJECTED",
|
||||
}.get(result.state.value, "OPENCLAW_FAILED")
|
||||
self.publisher.publish(self._event(event_type, result).as_dict())
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Versioned AIOS to OpenClaw execution contracts."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import uuid
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Mapping
|
||||
|
||||
CONTRACT_VERSION = "aios.openclaw.command.v1"
|
||||
DEFAULT_CLOCK_SKEW_SECONDS = 300
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _parse_timestamp(value: str) -> datetime:
|
||||
normalized = str(value or "").strip()
|
||||
if not normalized:
|
||||
raise ValueError("timestamp is required")
|
||||
if normalized.endswith("Z"):
|
||||
normalized = normalized[:-1] + "+00:00"
|
||||
parsed = datetime.fromisoformat(normalized)
|
||||
if parsed.tzinfo is None:
|
||||
raise ValueError("timestamp must include a timezone")
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _required(value: str, label: str) -> str:
|
||||
normalized = str(value or "").strip()
|
||||
if not normalized:
|
||||
raise ValueError(f"{label} is required")
|
||||
return normalized
|
||||
|
||||
|
||||
def _safe_mapping(value: Mapping[str, Any] | None) -> dict[str, Any]:
|
||||
return json.loads(json.dumps(dict(value or {}), ensure_ascii=False, sort_keys=True, default=str))
|
||||
|
||||
|
||||
class CommandState(str, Enum):
|
||||
ACCEPTED = "ACCEPTED"
|
||||
RUNNING = "RUNNING"
|
||||
COMPLETED = "COMPLETED"
|
||||
FAILED = "FAILED"
|
||||
CANCELLED = "CANCELLED"
|
||||
TIMED_OUT = "TIMED_OUT"
|
||||
REJECTED = "REJECTED"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OpenClawCommand:
|
||||
command_id: str
|
||||
tenant_id: str
|
||||
task_id: str
|
||||
trace_id: str
|
||||
action: str
|
||||
payload: Mapping[str, Any] = field(default_factory=dict)
|
||||
idempotency_key: str = ""
|
||||
created_at: str = field(default_factory=utc_now)
|
||||
expires_at: str | None = None
|
||||
contract_version: str = CONTRACT_VERSION
|
||||
key_id: str = "default"
|
||||
signature: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for field_name in (
|
||||
"command_id", "tenant_id", "task_id", "trace_id", "action",
|
||||
"idempotency_key", "created_at", "contract_version", "key_id",
|
||||
):
|
||||
object.__setattr__(self, field_name, _required(getattr(self, field_name), field_name))
|
||||
if self.contract_version != CONTRACT_VERSION:
|
||||
raise ValueError(f"unsupported contract_version: {self.contract_version}")
|
||||
object.__setattr__(self, "payload", _safe_mapping(self.payload))
|
||||
_parse_timestamp(self.created_at)
|
||||
if self.expires_at is not None:
|
||||
object.__setattr__(self, "expires_at", _required(self.expires_at, "expires_at"))
|
||||
_parse_timestamp(self.expires_at)
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls,
|
||||
*,
|
||||
tenant_id: str,
|
||||
task_id: str,
|
||||
trace_id: str,
|
||||
action: str,
|
||||
payload: Mapping[str, Any] | None,
|
||||
idempotency_key: str,
|
||||
expires_at: str | None = None,
|
||||
key_id: str = "default",
|
||||
command_id: str | None = None,
|
||||
) -> "OpenClawCommand":
|
||||
return cls(
|
||||
command_id=command_id or f"cmd_{uuid.uuid4().hex}",
|
||||
tenant_id=tenant_id,
|
||||
task_id=task_id,
|
||||
trace_id=trace_id,
|
||||
action=action,
|
||||
payload=_safe_mapping(payload),
|
||||
idempotency_key=idempotency_key,
|
||||
expires_at=expires_at,
|
||||
key_id=key_id,
|
||||
)
|
||||
|
||||
def unsigned_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"command_id": self.command_id,
|
||||
"tenant_id": self.tenant_id,
|
||||
"task_id": self.task_id,
|
||||
"trace_id": self.trace_id,
|
||||
"action": self.action,
|
||||
"payload": _safe_mapping(self.payload),
|
||||
"idempotency_key": self.idempotency_key,
|
||||
"created_at": self.created_at,
|
||||
"expires_at": self.expires_at,
|
||||
"contract_version": self.contract_version,
|
||||
"key_id": self.key_id,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OpenClawResult:
|
||||
command_id: str
|
||||
tenant_id: str
|
||||
task_id: str
|
||||
trace_id: str
|
||||
state: CommandState
|
||||
output: Mapping[str, Any] | None = None
|
||||
error: Mapping[str, Any] | None = None
|
||||
started_at: str | None = None
|
||||
finished_at: str | None = None
|
||||
runtime_id: str | None = None
|
||||
attempts: int = 1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for field_name in ("command_id", "tenant_id", "task_id", "trace_id"):
|
||||
object.__setattr__(self, field_name, _required(getattr(self, field_name), field_name))
|
||||
object.__setattr__(self, "state", CommandState(self.state))
|
||||
if self.output is not None:
|
||||
object.__setattr__(self, "output", _safe_mapping(self.output))
|
||||
if self.error is not None:
|
||||
object.__setattr__(self, "error", _safe_mapping(self.error))
|
||||
if self.started_at is not None:
|
||||
_parse_timestamp(self.started_at)
|
||||
if self.finished_at is not None:
|
||||
_parse_timestamp(self.finished_at)
|
||||
if self.attempts < 1:
|
||||
raise ValueError("attempts must be >= 1")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimeHealth:
|
||||
runtime: str
|
||||
runtime_id: str
|
||||
status: str
|
||||
version: str
|
||||
heartbeat_at: str
|
||||
active_tasks: int = 0
|
||||
workspace_isolated: bool = False
|
||||
credential_ref_configured: bool = False
|
||||
details: Mapping[str, Any] = field(default_factory=dict)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for field_name in ("runtime", "runtime_id", "status", "version", "heartbeat_at"):
|
||||
object.__setattr__(self, field_name, _required(getattr(self, field_name), field_name))
|
||||
_parse_timestamp(self.heartbeat_at)
|
||||
if self.active_tasks < 0:
|
||||
raise ValueError("active_tasks must be >= 0")
|
||||
object.__setattr__(self, "details", _safe_mapping(self.details))
|
||||
|
||||
|
||||
def canonical_command_bytes(command: OpenClawCommand) -> bytes:
|
||||
return json.dumps(
|
||||
command.unsigned_dict(), ensure_ascii=False, sort_keys=True, separators=(",", ":")
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def sign_command(command: OpenClawCommand, key_material: bytes | str) -> OpenClawCommand:
|
||||
key = key_material.encode("utf-8") if isinstance(key_material, str) else bytes(key_material)
|
||||
if not key:
|
||||
raise ValueError("signing material is required")
|
||||
signature = hmac.new(key, canonical_command_bytes(command), hashlib.sha256).hexdigest()
|
||||
return replace(command, signature=signature)
|
||||
|
||||
|
||||
def verify_command_signature(
|
||||
command: OpenClawCommand,
|
||||
key_material: bytes | str,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
clock_skew_seconds: int = DEFAULT_CLOCK_SKEW_SECONDS,
|
||||
) -> bool:
|
||||
if not command.signature:
|
||||
return False
|
||||
key = key_material.encode("utf-8") if isinstance(key_material, str) else bytes(key_material)
|
||||
if not key:
|
||||
return False
|
||||
expected = hmac.new(key, canonical_command_bytes(command), hashlib.sha256).hexdigest()
|
||||
if not hmac.compare_digest(expected, command.signature):
|
||||
return False
|
||||
current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc)
|
||||
created = _parse_timestamp(command.created_at)
|
||||
if created.timestamp() - current.timestamp() > clock_skew_seconds:
|
||||
return False
|
||||
if command.expires_at is not None:
|
||||
expires = _parse_timestamp(command.expires_at)
|
||||
if current.timestamp() - expires.timestamp() > clock_skew_seconds:
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Health policy and self-healing decisions for OpenClaw runtimes."""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Mapping
|
||||
|
||||
from .contract import RuntimeHealth
|
||||
|
||||
|
||||
def _parse(value: str) -> datetime:
|
||||
normalized = value[:-1] + "+00:00" if value.endswith("Z") else value
|
||||
parsed = datetime.fromisoformat(normalized)
|
||||
if parsed.tzinfo is None:
|
||||
raise ValueError("heartbeat timestamp must include timezone")
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HealthPolicy:
|
||||
max_heartbeat_age_seconds: int = 90
|
||||
require_workspace_isolation: bool = True
|
||||
require_credential_ref: bool = True
|
||||
accepted_statuses: frozenset[str] = field(
|
||||
default_factory=lambda: frozenset({"healthy", "ready"})
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.max_heartbeat_age_seconds < 5:
|
||||
raise ValueError("max_heartbeat_age_seconds must be >= 5")
|
||||
if not self.accepted_statuses:
|
||||
raise ValueError("accepted_statuses must not be empty")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HealthAssessment:
|
||||
healthy: bool
|
||||
reasons: tuple[str, ...]
|
||||
recoverable: bool
|
||||
recommended_action: str
|
||||
details: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
class RuntimeHealthMonitor:
|
||||
"""Evaluate runtime health and recommend bounded recovery actions."""
|
||||
|
||||
def __init__(self, policy: HealthPolicy | None = None) -> None:
|
||||
self.policy = policy or HealthPolicy()
|
||||
|
||||
def assess(
|
||||
self,
|
||||
health: RuntimeHealth,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> HealthAssessment:
|
||||
current = (now or datetime.now(timezone.utc)).astimezone(timezone.utc)
|
||||
heartbeat = _parse(health.heartbeat_at)
|
||||
age = max(0.0, current.timestamp() - heartbeat.timestamp())
|
||||
reasons: list[str] = []
|
||||
|
||||
if health.status.lower() not in self.policy.accepted_statuses:
|
||||
reasons.append("runtime_status_unhealthy")
|
||||
if age > self.policy.max_heartbeat_age_seconds:
|
||||
reasons.append("heartbeat_stale")
|
||||
if self.policy.require_workspace_isolation and not health.workspace_isolated:
|
||||
reasons.append("workspace_not_isolated")
|
||||
if self.policy.require_credential_ref and not health.credential_ref_configured:
|
||||
reasons.append("credential_reference_missing")
|
||||
|
||||
if not reasons:
|
||||
return HealthAssessment(
|
||||
healthy=True,
|
||||
reasons=(),
|
||||
recoverable=True,
|
||||
recommended_action="none",
|
||||
details={"heartbeat_age_seconds": age},
|
||||
)
|
||||
|
||||
non_recoverable = {
|
||||
"workspace_not_isolated",
|
||||
"credential_reference_missing",
|
||||
}.intersection(reasons)
|
||||
if non_recoverable:
|
||||
action = "quarantine_runtime"
|
||||
recoverable = False
|
||||
elif "heartbeat_stale" in reasons:
|
||||
action = "restart_runtime"
|
||||
recoverable = True
|
||||
else:
|
||||
action = "drain_and_restart"
|
||||
recoverable = True
|
||||
|
||||
return HealthAssessment(
|
||||
healthy=False,
|
||||
reasons=tuple(reasons),
|
||||
recoverable=recoverable,
|
||||
recommended_action=action,
|
||||
details={
|
||||
"heartbeat_age_seconds": age,
|
||||
"runtime_id": health.runtime_id,
|
||||
"active_tasks": health.active_tasks,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,214 @@
|
||||
"""Runtime transport implementations for the OpenClaw adapter."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any, Callable, Mapping, Protocol
|
||||
|
||||
from .contract import (
|
||||
CommandState,
|
||||
OpenClawCommand,
|
||||
OpenClawResult,
|
||||
RuntimeHealth,
|
||||
utc_now,
|
||||
)
|
||||
|
||||
|
||||
class RuntimeTransport(Protocol):
|
||||
def execute(self, command: OpenClawCommand, timeout_seconds: float) -> OpenClawResult: ...
|
||||
def cancel(self, command_id: str) -> bool: ...
|
||||
def health(self) -> RuntimeHealth: ...
|
||||
|
||||
|
||||
class TransportError(RuntimeError):
|
||||
"""Runtime transport or protocol failure."""
|
||||
|
||||
|
||||
RuntimeHandler = Callable[[OpenClawCommand], Mapping[str, Any] | OpenClawResult]
|
||||
|
||||
|
||||
class InProcessRuntimeTransport:
|
||||
"""Deterministic runtime used by tests and controlled local integrations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
handler: RuntimeHandler,
|
||||
*,
|
||||
runtime_id: str = "openclaw-in-process",
|
||||
version: str = "v1",
|
||||
workspace_isolated: bool = True,
|
||||
credential_ref_configured: bool = True,
|
||||
) -> None:
|
||||
self.handler = handler
|
||||
self.runtime_id = runtime_id
|
||||
self.version = version
|
||||
self.workspace_isolated = workspace_isolated
|
||||
self.credential_ref_configured = credential_ref_configured
|
||||
self._lock = threading.RLock()
|
||||
self._running: set[str] = set()
|
||||
self._cancelled: set[str] = set()
|
||||
|
||||
def execute(self, command: OpenClawCommand, timeout_seconds: float) -> OpenClawResult:
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError("timeout_seconds must be positive")
|
||||
with self._lock:
|
||||
if command.command_id in self._cancelled:
|
||||
return OpenClawResult(
|
||||
command_id=command.command_id,
|
||||
tenant_id=command.tenant_id,
|
||||
task_id=command.task_id,
|
||||
trace_id=command.trace_id,
|
||||
state=CommandState.CANCELLED,
|
||||
error={"code": "cancelled", "message": "cancelled before execution"},
|
||||
finished_at=utc_now(),
|
||||
runtime_id=self.runtime_id,
|
||||
)
|
||||
self._running.add(command.command_id)
|
||||
try:
|
||||
raw = self.handler(command)
|
||||
if isinstance(raw, OpenClawResult):
|
||||
return raw
|
||||
if not isinstance(raw, Mapping):
|
||||
raise TransportError("runtime handler must return a mapping or OpenClawResult")
|
||||
return OpenClawResult(
|
||||
command_id=command.command_id,
|
||||
tenant_id=command.tenant_id,
|
||||
task_id=command.task_id,
|
||||
trace_id=command.trace_id,
|
||||
state=CommandState(str(raw.get("state", "COMPLETED"))),
|
||||
output=raw.get("output"),
|
||||
error=raw.get("error"),
|
||||
started_at=str(raw.get("started_at") or utc_now()),
|
||||
finished_at=str(raw.get("finished_at") or utc_now()),
|
||||
runtime_id=str(raw.get("runtime_id") or self.runtime_id),
|
||||
attempts=int(raw.get("attempts", 1)),
|
||||
)
|
||||
finally:
|
||||
with self._lock:
|
||||
self._running.discard(command.command_id)
|
||||
|
||||
def cancel(self, command_id: str) -> bool:
|
||||
with self._lock:
|
||||
self._cancelled.add(command_id)
|
||||
return command_id in self._running
|
||||
|
||||
def health(self) -> RuntimeHealth:
|
||||
with self._lock:
|
||||
active = len(self._running)
|
||||
return RuntimeHealth(
|
||||
runtime="openclaw",
|
||||
runtime_id=self.runtime_id,
|
||||
status="healthy",
|
||||
version=self.version,
|
||||
heartbeat_at=utc_now(),
|
||||
active_tasks=active,
|
||||
workspace_isolated=self.workspace_isolated,
|
||||
credential_ref_configured=self.credential_ref_configured,
|
||||
details={"transport": "in_process"},
|
||||
)
|
||||
|
||||
|
||||
class HTTPRuntimeTransport:
|
||||
"""Private HTTP transport for an OpenClaw runtime service."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
*,
|
||||
runtime_id: str = "openclaw-http",
|
||||
credential_ref: str | None = None,
|
||||
max_response_bytes: int = 2_097_152,
|
||||
) -> None:
|
||||
normalized = str(base_url).rstrip("/")
|
||||
if not normalized.startswith(("http://", "https://")):
|
||||
raise ValueError("base_url must use http or https")
|
||||
self.base_url = normalized
|
||||
self.runtime_id = runtime_id
|
||||
self.credential_ref = credential_ref
|
||||
self.max_response_bytes = int(max_response_bytes)
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
payload: Mapping[str, Any] | None,
|
||||
timeout_seconds: float,
|
||||
) -> Mapping[str, Any]:
|
||||
body = None
|
||||
headers = {"Accept": "application/json", "User-Agent": "AIOS-OpenClaw/1"}
|
||||
if payload is not None:
|
||||
body = json.dumps(dict(payload), separators=(",", ":")).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
if self.credential_ref:
|
||||
headers["X-AIOS-Credential-Ref"] = self.credential_ref
|
||||
request = urllib.request.Request(
|
||||
f"{self.base_url}{path}",
|
||||
data=body,
|
||||
method=method,
|
||||
headers=headers,
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
|
||||
raw = response.read(self.max_response_bytes + 1)
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise TransportError(f"runtime returned HTTP {exc.code}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise TransportError(f"runtime unavailable: {exc.reason}") from exc
|
||||
if len(raw) > self.max_response_bytes:
|
||||
raise TransportError("runtime response exceeded size limit")
|
||||
decoded = json.loads(raw.decode("utf-8"))
|
||||
if not isinstance(decoded, Mapping):
|
||||
raise TransportError("runtime response must be a JSON object")
|
||||
return decoded
|
||||
|
||||
@staticmethod
|
||||
def _command_payload(command: OpenClawCommand) -> dict[str, Any]:
|
||||
return {**command.unsigned_dict(), "signature": command.signature}
|
||||
|
||||
def execute(self, command: OpenClawCommand, timeout_seconds: float) -> OpenClawResult:
|
||||
value = self._request(
|
||||
"POST",
|
||||
"/v1/execute",
|
||||
payload=self._command_payload(command),
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
return OpenClawResult(
|
||||
command_id=str(value["command_id"]),
|
||||
tenant_id=str(value["tenant_id"]),
|
||||
task_id=str(value["task_id"]),
|
||||
trace_id=str(value["trace_id"]),
|
||||
state=CommandState(str(value["state"])),
|
||||
output=value.get("output"),
|
||||
error=value.get("error"),
|
||||
started_at=value.get("started_at"),
|
||||
finished_at=value.get("finished_at"),
|
||||
runtime_id=str(value.get("runtime_id") or self.runtime_id),
|
||||
attempts=int(value.get("attempts", 1)),
|
||||
)
|
||||
|
||||
def cancel(self, command_id: str) -> bool:
|
||||
value = self._request(
|
||||
"POST",
|
||||
f"/v1/commands/{urllib.parse.quote(command_id, safe='')}/cancel",
|
||||
payload={},
|
||||
timeout_seconds=10.0,
|
||||
)
|
||||
return bool(value.get("accepted"))
|
||||
|
||||
def health(self) -> RuntimeHealth:
|
||||
value = self._request("GET", "/v1/health", payload=None, timeout_seconds=5.0)
|
||||
return RuntimeHealth(
|
||||
runtime=str(value.get("runtime") or "openclaw"),
|
||||
runtime_id=str(value.get("runtime_id") or self.runtime_id),
|
||||
status=str(value.get("status") or "unknown"),
|
||||
version=str(value.get("version") or "unknown"),
|
||||
heartbeat_at=str(value.get("heartbeat_at") or utc_now()),
|
||||
active_tasks=int(value.get("active_tasks", 0)),
|
||||
workspace_isolated=bool(value.get("workspace_isolated", False)),
|
||||
credential_ref_configured=bool(value.get("credential_ref_configured", False)),
|
||||
details=value.get("details") if isinstance(value.get("details"), Mapping) else {},
|
||||
)
|
||||
@@ -0,0 +1,46 @@
|
||||
"""AIOS V2 control plane and execution orchestration."""
|
||||
|
||||
from .control_plane import (
|
||||
AgentRecord,
|
||||
AuditEvent,
|
||||
ControlPlaneError,
|
||||
ControlPlaneService,
|
||||
IdempotencyConflict,
|
||||
InMemoryControlPlaneStore,
|
||||
InvalidTransition,
|
||||
NotFound,
|
||||
PolicyDenied,
|
||||
RequestContext,
|
||||
RevisionConflict,
|
||||
TaskRecord,
|
||||
TaskState,
|
||||
TenantIsolationError,
|
||||
TenantRecord,
|
||||
Unauthorized,
|
||||
)
|
||||
from .execution_tracking import ExecutionEvidence, ExecutionTracker
|
||||
from .orchestrator import DispatchRecord, ExecutionOrchestrator, OrchestrationError
|
||||
|
||||
__all__ = [
|
||||
"AgentRecord",
|
||||
"AuditEvent",
|
||||
"ControlPlaneError",
|
||||
"ControlPlaneService",
|
||||
"DispatchRecord",
|
||||
"ExecutionEvidence",
|
||||
"ExecutionOrchestrator",
|
||||
"ExecutionTracker",
|
||||
"IdempotencyConflict",
|
||||
"InMemoryControlPlaneStore",
|
||||
"InvalidTransition",
|
||||
"NotFound",
|
||||
"OrchestrationError",
|
||||
"PolicyDenied",
|
||||
"RequestContext",
|
||||
"RevisionConflict",
|
||||
"TaskRecord",
|
||||
"TaskState",
|
||||
"TenantIsolationError",
|
||||
"TenantRecord",
|
||||
"Unauthorized",
|
||||
]
|
||||
@@ -0,0 +1,357 @@
|
||||
"""Production-candidate AIOS V2 application entrypoint.
|
||||
|
||||
This process owns the public control-plane API and read-only web console. It
|
||||
loads durable control-plane state, dispatches approved work through runtime
|
||||
adapters, exposes tenant-scoped execution evidence and sanitized GitOps status,
|
||||
and never exposes runtime credentials to the browser.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import urllib.parse
|
||||
from enum import Enum
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from aios_hermes import HermesAdapter, HermesJobState, HermesResult
|
||||
from aios_openclaw import HTTPRuntimeTransport, OpenClawAdapter
|
||||
from aios_openclaw.callback import AuditCallbackSink, EventPublisher
|
||||
|
||||
from .console import CONSOLE_HTML
|
||||
from .control_plane import AuditEvent, RequestContext
|
||||
from .execution_tracking import ExecutionTracker
|
||||
from .gateway import GatewayApplication, error_response
|
||||
from .gitops_status import collect_gitops_status
|
||||
from .orchestrator import ExecutionOrchestrator
|
||||
from .persistence import DurableControlPlaneService, SQLiteSnapshotRepository
|
||||
|
||||
|
||||
def _json_value(value: Any) -> Any:
|
||||
if dataclasses.is_dataclass(value):
|
||||
return {
|
||||
item.name: _json_value(getattr(value, item.name))
|
||||
for item in dataclasses.fields(value)
|
||||
}
|
||||
if isinstance(value, Enum):
|
||||
return value.value
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _json_value(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple, set, frozenset)):
|
||||
return [_json_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
class FileSigningMaterialResolver:
|
||||
"""Resolve runtime signing material from a root-owned file reference."""
|
||||
|
||||
def __init__(self, path: str | Path) -> None:
|
||||
self.path = Path(path).resolve()
|
||||
|
||||
def __call__(self, tenant_id: str, key_id: str) -> bytes:
|
||||
del tenant_id, key_id
|
||||
metadata = self.path.stat()
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
raise RuntimeError("signing material reference is not a regular file")
|
||||
if metadata.st_mode & 0o077:
|
||||
raise RuntimeError("signing material file permissions must be 0600 or stricter")
|
||||
value = self.path.read_bytes().strip()
|
||||
if len(value) < 32:
|
||||
raise RuntimeError("signing material is too short")
|
||||
return value
|
||||
|
||||
|
||||
class ControlPlaneEventPublisher(EventPublisher):
|
||||
"""Append OpenClaw runtime events to the tenant audit stream."""
|
||||
|
||||
def __init__(self, service: DurableControlPlaneService) -> None:
|
||||
self.service = service
|
||||
|
||||
def publish(self, event: Mapping[str, Any]) -> None:
|
||||
tenant_id = str(event.get("tenant_id") or "").strip()
|
||||
trace_id = str(event.get("trace_id") or "").strip()
|
||||
command_id = str(event.get("command_id") or "").strip()
|
||||
event_type = str(event.get("event_type") or "OPENCLAW_EVENT").strip()
|
||||
if not tenant_id or not trace_id or not command_id:
|
||||
raise ValueError("runtime audit event is missing tenant identity")
|
||||
state_value = str(event.get("state") or "unknown").lower()
|
||||
outcome = "allowed" if state_value == "completed" else state_value
|
||||
audit = AuditEvent(
|
||||
event_id=f"runtime_{command_id}_{len(self.service.store.audit_events) + 1}",
|
||||
tenant_id=tenant_id,
|
||||
principal_id="openclaw-runtime",
|
||||
trace_id=trace_id,
|
||||
action=event_type,
|
||||
resource_type="openclaw_command",
|
||||
resource_id=command_id,
|
||||
outcome=outcome,
|
||||
policy_version="runtime-v1",
|
||||
details=dict(event.get("details") or {}),
|
||||
)
|
||||
with self.service.store.lock:
|
||||
self.service.store.audit_events.append(audit)
|
||||
self.service.repository.save(self.service._snapshot())
|
||||
|
||||
|
||||
class CandidateApplication:
|
||||
"""Compose durable control plane, runtimes, evidence and console routes."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
gateway: GatewayApplication,
|
||||
orchestrator: ExecutionOrchestrator,
|
||||
tracker: ExecutionTracker,
|
||||
hermes: HermesAdapter,
|
||||
openclaw: OpenClawAdapter | None,
|
||||
) -> None:
|
||||
self.gateway = gateway
|
||||
self.orchestrator = orchestrator
|
||||
self.tracker = tracker
|
||||
self.hermes = hermes
|
||||
self.openclaw = openclaw
|
||||
|
||||
def context(self, headers: Mapping[str, str]) -> RequestContext:
|
||||
return self.gateway.context_from_headers(headers)
|
||||
|
||||
def handle_json(
|
||||
self,
|
||||
method: str,
|
||||
raw_path: str,
|
||||
headers: Mapping[str, str],
|
||||
body: Any,
|
||||
) -> tuple[int, Mapping[str, Any]]:
|
||||
parsed = urllib.parse.urlparse(raw_path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
method = method.upper()
|
||||
|
||||
if method == "GET" and path == "/api/v2/gitops":
|
||||
context = self.context(headers)
|
||||
return HTTPStatus.OK, {
|
||||
"tenant_id": context.tenant_id,
|
||||
"gitops": collect_gitops_status().as_dict(),
|
||||
}
|
||||
|
||||
if method == "GET" and path == "/api/v2/runtime":
|
||||
context = self.context(headers)
|
||||
runtimes: list[dict[str, Any]] = []
|
||||
if self.openclaw is None:
|
||||
runtimes.append(
|
||||
{
|
||||
"name": "openclaw",
|
||||
"status": "disabled",
|
||||
"details": {"reason": "runtime_not_configured"},
|
||||
}
|
||||
)
|
||||
else:
|
||||
try:
|
||||
health = self.openclaw.health()
|
||||
runtimes.append(
|
||||
{
|
||||
"name": "openclaw",
|
||||
"status": health.status,
|
||||
"details": _json_value(health),
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
runtimes.append(
|
||||
{
|
||||
"name": "openclaw",
|
||||
"status": "unavailable",
|
||||
"details": {"message": str(exc)},
|
||||
}
|
||||
)
|
||||
hermes_health = self.hermes.health()
|
||||
runtimes.append(
|
||||
{
|
||||
"name": "hermes",
|
||||
"status": hermes_health.status,
|
||||
"details": _json_value(hermes_health),
|
||||
}
|
||||
)
|
||||
return HTTPStatus.OK, {
|
||||
"tenant_id": context.tenant_id,
|
||||
"runtimes": runtimes,
|
||||
}
|
||||
|
||||
if method == "GET" and path == "/api/v2/executions":
|
||||
context = self.context(headers)
|
||||
return HTTPStatus.OK, {
|
||||
"executions": _json_value(
|
||||
self.tracker.list(tenant_id=context.tenant_id)
|
||||
)
|
||||
}
|
||||
|
||||
dispatch_match = __import__("re").fullmatch(
|
||||
r"/api/v2/tasks/([^/]+)/dispatch",
|
||||
path,
|
||||
)
|
||||
if method == "POST" and dispatch_match:
|
||||
context = self.context(headers)
|
||||
task_id = urllib.parse.unquote(dispatch_match.group(1))
|
||||
payload = body if isinstance(body, Mapping) else {}
|
||||
timeout = payload.get("timeout_seconds")
|
||||
record = self.orchestrator.dispatch(
|
||||
context,
|
||||
task_id=task_id,
|
||||
timeout_seconds=float(timeout) if timeout is not None else None,
|
||||
)
|
||||
return HTTPStatus.ACCEPTED, {"dispatch": _json_value(record)}
|
||||
|
||||
if method == "POST" and path == "/api/v2/hermes/results/reconcile":
|
||||
context = self.context(headers)
|
||||
if not isinstance(body, Mapping):
|
||||
raise ValueError("JSON body must be an object")
|
||||
result = HermesResult(
|
||||
job_id=str(body["job_id"]),
|
||||
tenant_id=str(body["tenant_id"]),
|
||||
task_id=str(body["task_id"]),
|
||||
trace_id=str(body["trace_id"]),
|
||||
state=HermesJobState(str(body["state"])),
|
||||
worker_id=(
|
||||
str(body["worker_id"])
|
||||
if body.get("worker_id") is not None
|
||||
else None
|
||||
),
|
||||
output=body.get("output"),
|
||||
error=body.get("error"),
|
||||
started_at=body.get("started_at"),
|
||||
finished_at=body.get("finished_at"),
|
||||
)
|
||||
task = self.orchestrator.reconcile_hermes(context, result)
|
||||
return HTTPStatus.OK, {"task": _json_value(task)}
|
||||
|
||||
return self.gateway.handle(method, raw_path, headers, body)
|
||||
|
||||
|
||||
class CandidateRequestHandler(BaseHTTPRequestHandler):
|
||||
server_version = "AIOSV2Candidate/1.0"
|
||||
max_body_bytes = 1_048_576
|
||||
|
||||
@property
|
||||
def application(self) -> CandidateApplication:
|
||||
return self.server.application # type: ignore[attr-defined]
|
||||
|
||||
def _read_body(self) -> Any:
|
||||
length = int(self.headers.get("Content-Length", "0") or "0")
|
||||
if length < 0 or length > self.max_body_bytes:
|
||||
raise ValueError("request body is too large")
|
||||
if length == 0:
|
||||
return {}
|
||||
content_type = self.headers.get("Content-Type", "").split(";", 1)[0].strip()
|
||||
if content_type != "application/json":
|
||||
raise ValueError("Content-Type must be application/json")
|
||||
return json.loads(self.rfile.read(length).decode("utf-8"))
|
||||
|
||||
def _security_headers(self) -> None:
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("X-Frame-Options", "DENY")
|
||||
self.send_header("Referrer-Policy", "no-referrer")
|
||||
self.send_header(
|
||||
"Content-Security-Policy",
|
||||
"default-src 'self'; style-src 'unsafe-inline'; script-src 'unsafe-inline'; connect-src 'self'",
|
||||
)
|
||||
|
||||
def _send_json(self, status_code: int, payload: Mapping[str, Any]) -> None:
|
||||
encoded = json.dumps(
|
||||
_json_value(payload),
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
self.send_response(int(status_code))
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(encoded)))
|
||||
self._security_headers()
|
||||
self.end_headers()
|
||||
self.wfile.write(encoded)
|
||||
|
||||
def _send_console(self) -> None:
|
||||
encoded = CONSOLE_HTML.encode("utf-8")
|
||||
self.send_response(HTTPStatus.OK)
|
||||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(encoded)))
|
||||
self._security_headers()
|
||||
self.end_headers()
|
||||
self.wfile.write(encoded)
|
||||
|
||||
def _dispatch(self, method: str) -> None:
|
||||
path = urllib.parse.urlparse(self.path).path.rstrip("/") or "/"
|
||||
if method == "GET" and path in {"/", "/console"}:
|
||||
self._send_console()
|
||||
return
|
||||
try:
|
||||
body = self._read_body() if method == "POST" else {}
|
||||
status_code, payload = self.application.handle_json(
|
||||
method,
|
||||
self.path,
|
||||
{key: value for key, value in self.headers.items()},
|
||||
body,
|
||||
)
|
||||
except BaseException as exc:
|
||||
status_code, payload = error_response(exc)
|
||||
self._send_json(status_code, payload)
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
self._dispatch("GET")
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
self._dispatch("POST")
|
||||
|
||||
|
||||
class CandidateHTTPServer(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(self, address: tuple[str, int], application: CandidateApplication) -> None:
|
||||
self.application = application
|
||||
super().__init__(address, CandidateRequestHandler)
|
||||
|
||||
|
||||
def build_application() -> CandidateApplication:
|
||||
state_dir = Path(os.getenv("AIOS_V2_STATE_DIR", "/var/lib/aios-v2")).resolve()
|
||||
state_dir.mkdir(parents=True, exist_ok=True)
|
||||
repository = SQLiteSnapshotRepository(state_dir / "control-plane.sqlite3")
|
||||
service = DurableControlPlaneService(repository)
|
||||
tracker = ExecutionTracker(state_dir / "executions")
|
||||
hermes = HermesAdapter()
|
||||
|
||||
openclaw_url = os.getenv("AIOS_OPENCLAW_URL", "").strip()
|
||||
material_file = os.getenv("AIOS_OPENCLAW_SIGNING_MATERIAL_FILE", "").strip()
|
||||
openclaw = None
|
||||
resolver = None
|
||||
if openclaw_url and material_file:
|
||||
resolver = FileSigningMaterialResolver(material_file)
|
||||
transport = HTTPRuntimeTransport(
|
||||
openclaw_url,
|
||||
credential_ref=os.getenv("AIOS_OPENCLAW_CREDENTIAL_REF") or None,
|
||||
)
|
||||
openclaw = OpenClawAdapter(
|
||||
transport,
|
||||
resolver,
|
||||
callback_sink=AuditCallbackSink(ControlPlaneEventPublisher(service)),
|
||||
)
|
||||
|
||||
gateway = GatewayApplication(service)
|
||||
orchestrator = ExecutionOrchestrator(
|
||||
service,
|
||||
openclaw=openclaw,
|
||||
hermes=hermes,
|
||||
openclaw_signing_material_resolver=resolver,
|
||||
)
|
||||
return CandidateApplication(gateway, orchestrator, tracker, hermes, openclaw)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
host = os.getenv("AIOS_V2_HOST", "127.0.0.1")
|
||||
port = int(os.getenv("AIOS_V2_PORT", "18920"))
|
||||
server = CandidateHTTPServer((host, port), build_application())
|
||||
print(f"AIOS V2 candidate listening on {host}:{port}")
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Dependency-free AIOS V2 web console assets.
|
||||
|
||||
Authentication and tenant identity are injected by the trusted reverse proxy.
|
||||
The browser never selects a tenant and never calls OpenClaw or Hermes directly.
|
||||
"""
|
||||
|
||||
CONSOLE_HTML = r'''<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta name="referrer" content="no-referrer">
|
||||
<title>AIOS V2 Console</title>
|
||||
<style>
|
||||
:root{font-family:Inter,ui-sans-serif,system-ui,sans-serif;color:#152033;background:#f4f7fb}
|
||||
*{box-sizing:border-box} body{margin:0} header{background:#111827;color:white;padding:18px 24px;display:flex;justify-content:space-between;align-items:center}
|
||||
header h1{font-size:20px;margin:0} header span{font-size:12px;color:#cbd5e1}
|
||||
main{padding:24px;max-width:1440px;margin:auto}.notice{background:#fff7ed;border:1px solid #fed7aa;padding:12px 14px;border-radius:10px;margin-bottom:18px;font-size:13px}
|
||||
.cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));gap:14px;margin-bottom:20px}.card,.panel{background:white;border:1px solid #e5e7eb;border-radius:12px;box-shadow:0 1px 2px rgba(0,0,0,.04)}
|
||||
.card{padding:16px}.card b{display:block;font-size:28px;margin-top:8px}.card small{color:#64748b}.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(420px,1fr));gap:16px}.panel{padding:16px;overflow:auto}.panel h2{font-size:16px;margin:0 0 12px}
|
||||
table{width:100%;border-collapse:collapse;font-size:12px}th,td{text-align:left;border-bottom:1px solid #edf0f4;padding:8px;vertical-align:top}th{color:#64748b;font-weight:600}.state{font-weight:700}.ok{color:#047857}.bad{color:#b91c1c}.muted{color:#64748b}
|
||||
button{border:0;border-radius:8px;padding:9px 13px;background:#2563eb;color:white;cursor:pointer}button:disabled{opacity:.5}.toolbar{display:flex;gap:8px;align-items:center;margin-bottom:16px}.error{color:#b91c1c;white-space:pre-wrap}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header><h1>AIOS V2 Control Plane</h1><span id="updated">Not loaded</span></header>
|
||||
<main>
|
||||
<div class="notice">Read-only candidate console. Identity and tenant context must be injected by the trusted reverse proxy. The console never calls OpenClaw or Hermes directly.</div>
|
||||
<div class="toolbar"><button id="refresh">Refresh</button><span id="error" class="error"></span></div>
|
||||
<section class="cards">
|
||||
<div class="card"><small>Agents</small><b id="agentCount">—</b></div>
|
||||
<div class="card"><small>Tasks</small><b id="taskCount">—</b></div>
|
||||
<div class="card"><small>Audit events</small><b id="auditCount">—</b></div>
|
||||
<div class="card"><small>Executions</small><b id="executionCount">—</b></div>
|
||||
</section>
|
||||
<section class="grid">
|
||||
<div class="panel"><h2>Runtime health</h2><div id="runtime"></div></div>
|
||||
<div class="panel"><h2>Tasks</h2><div id="tasks"></div></div>
|
||||
<div class="panel"><h2>Agents</h2><div id="agents"></div></div>
|
||||
<div class="panel"><h2>Execution evidence</h2><div id="executions"></div></div>
|
||||
<div class="panel"><h2>Audit</h2><div id="audit"></div></div>
|
||||
</section>
|
||||
</main>
|
||||
<script>
|
||||
const esc=v=>String(v??'').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
|
||||
const table=(cols,rows)=>`<table><thead><tr>${cols.map(c=>`<th>${esc(c[0])}</th>`).join('')}</tr></thead><tbody>${rows.map(r=>`<tr>${cols.map(c=>`<td>${esc(c[1](r))}</td>`).join('')}</tr>`).join('')}</tbody></table>`;
|
||||
async function api(path){const r=await fetch(path,{credentials:'same-origin',headers:{'Accept':'application/json'}});const data=await r.json().catch(()=>({}));if(!r.ok)throw new Error(`${r.status} ${data.message||data.error||path}`);return data}
|
||||
async function refresh(){
|
||||
const btn=document.getElementById('refresh');btn.disabled=true;document.getElementById('error').textContent='';
|
||||
try{
|
||||
const [agents,tasks,audit,runtime,executions]=await Promise.all([
|
||||
api('/api/v2/agents'),api('/api/v2/tasks'),api('/api/v2/audit'),api('/api/v2/runtime'),api('/api/v2/executions')
|
||||
]);
|
||||
document.getElementById('agentCount').textContent=agents.agents.length;
|
||||
document.getElementById('taskCount').textContent=tasks.tasks.length;
|
||||
document.getElementById('auditCount').textContent=audit.events.length;
|
||||
document.getElementById('executionCount').textContent=executions.executions.length;
|
||||
document.getElementById('agents').innerHTML=table([['ID',x=>x.agent_id],['Runtime',x=>x.runtime],['Enabled',x=>x.enabled]],agents.agents);
|
||||
document.getElementById('tasks').innerHTML=table([['Task',x=>x.task_id],['Action',x=>x.action],['State',x=>x.state],['Rev',x=>x.revision]],tasks.tasks);
|
||||
document.getElementById('audit').innerHTML=table([['Time',x=>x.created_at],['Action',x=>x.action],['Outcome',x=>x.outcome],['Resource',x=>x.resource_id]],audit.events.slice(-100).reverse());
|
||||
document.getElementById('executions').innerHTML=table([['Executor',x=>x.executor],['Task',x=>x.task_id],['Status',x=>x.status],['Progress',x=>`${x.progress_completed}/${x.progress_total}`],['Commit',x=>x.last_commit||'']],executions.executions);
|
||||
document.getElementById('runtime').innerHTML=table([['Runtime',x=>x.name],['Status',x=>x.status],['Details',x=>JSON.stringify(x.details)]],runtime.runtimes);
|
||||
document.getElementById('updated').textContent=`Updated ${new Date().toLocaleString()}`;
|
||||
}catch(e){document.getElementById('error').textContent=e.message||String(e)}finally{btn.disabled=false}
|
||||
}
|
||||
document.getElementById('refresh').addEventListener('click',refresh);refresh();setInterval(refresh,30000);
|
||||
</script>
|
||||
</body>
|
||||
</html>'''
|
||||
@@ -0,0 +1,627 @@
|
||||
"""Tenant-isolated AIOS V2 control-plane domain service.
|
||||
|
||||
This module deliberately contains no OpenClaw or Hermes implementation details.
|
||||
It owns authoritative tenant, agent, task-lifecycle, idempotency and audit state.
|
||||
Runtime adapters consume approved tasks through a separate boundary.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import threading
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Callable, Iterable, Mapping
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _new_id(prefix: str) -> str:
|
||||
return f"{prefix}_{uuid.uuid4().hex}"
|
||||
|
||||
|
||||
def _canonical_digest(value: Mapping[str, Any]) -> str:
|
||||
encoded = json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
default=str,
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
class ControlPlaneError(RuntimeError):
|
||||
"""Base error returned by the control-plane domain."""
|
||||
|
||||
|
||||
class Unauthorized(ControlPlaneError):
|
||||
"""The caller is not authenticated or lacks the required role."""
|
||||
|
||||
|
||||
class TenantIsolationError(ControlPlaneError):
|
||||
"""A request attempted to cross a tenant boundary."""
|
||||
|
||||
|
||||
class NotFound(ControlPlaneError):
|
||||
"""A tenant-scoped resource does not exist or is not visible."""
|
||||
|
||||
|
||||
class InvalidTransition(ControlPlaneError):
|
||||
"""A task state transition violates the lifecycle contract."""
|
||||
|
||||
|
||||
class IdempotencyConflict(ControlPlaneError):
|
||||
"""An idempotency key was reused for a different command."""
|
||||
|
||||
|
||||
class RevisionConflict(ControlPlaneError):
|
||||
"""Optimistic task revision does not match current state."""
|
||||
|
||||
|
||||
class PolicyDenied(ControlPlaneError):
|
||||
"""The policy gate denied a control-plane operation."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RequestContext:
|
||||
"""Authenticated request identity propagated across all AIOS layers."""
|
||||
|
||||
tenant_id: str
|
||||
principal_id: str
|
||||
trace_id: str
|
||||
roles: frozenset[str] = field(default_factory=frozenset)
|
||||
session_id: str | None = None
|
||||
policy_version: str = "v1"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name in ("tenant_id", "principal_id", "trace_id"):
|
||||
if not str(getattr(self, name, "")).strip():
|
||||
raise ValueError(f"{name} is required")
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls,
|
||||
*,
|
||||
tenant_id: str,
|
||||
principal_id: str,
|
||||
roles: Iterable[str] = (),
|
||||
trace_id: str | None = None,
|
||||
session_id: str | None = None,
|
||||
policy_version: str = "v1",
|
||||
) -> "RequestContext":
|
||||
return cls(
|
||||
tenant_id=tenant_id.strip(),
|
||||
principal_id=principal_id.strip(),
|
||||
trace_id=trace_id or _new_id("trace"),
|
||||
roles=frozenset(str(role).strip() for role in roles if str(role).strip()),
|
||||
session_id=session_id,
|
||||
policy_version=policy_version,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TenantRecord:
|
||||
tenant_id: str
|
||||
display_name: str
|
||||
status: str = "active"
|
||||
metadata: Mapping[str, Any] = field(default_factory=dict)
|
||||
created_at: str = field(default_factory=_utc_now)
|
||||
updated_at: str = field(default_factory=_utc_now)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AgentRecord:
|
||||
agent_id: str
|
||||
tenant_id: str
|
||||
display_name: str
|
||||
runtime: str
|
||||
capabilities: tuple[str, ...] = field(default_factory=tuple)
|
||||
enabled: bool = True
|
||||
metadata: Mapping[str, Any] = field(default_factory=dict)
|
||||
created_at: str = field(default_factory=_utc_now)
|
||||
updated_at: str = field(default_factory=_utc_now)
|
||||
|
||||
|
||||
class TaskState(str, Enum):
|
||||
CREATED = "CREATED"
|
||||
APPROVED = "APPROVED"
|
||||
RUNNING = "RUNNING"
|
||||
COMPLETED = "COMPLETED"
|
||||
FAILED = "FAILED"
|
||||
CANCELLED = "CANCELLED"
|
||||
|
||||
|
||||
TERMINAL_TASK_STATES = frozenset(
|
||||
{TaskState.COMPLETED, TaskState.FAILED, TaskState.CANCELLED}
|
||||
)
|
||||
|
||||
ALLOWED_TASK_TRANSITIONS: Mapping[TaskState, frozenset[TaskState]] = {
|
||||
TaskState.CREATED: frozenset({TaskState.APPROVED, TaskState.CANCELLED}),
|
||||
TaskState.APPROVED: frozenset({TaskState.RUNNING, TaskState.CANCELLED}),
|
||||
TaskState.RUNNING: frozenset(
|
||||
{TaskState.COMPLETED, TaskState.FAILED, TaskState.CANCELLED}
|
||||
),
|
||||
TaskState.COMPLETED: frozenset(),
|
||||
TaskState.FAILED: frozenset(),
|
||||
TaskState.CANCELLED: frozenset(),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TaskRecord:
|
||||
task_id: str
|
||||
tenant_id: str
|
||||
agent_id: str
|
||||
action: str
|
||||
payload: Mapping[str, Any]
|
||||
state: TaskState = TaskState.CREATED
|
||||
revision: int = 1
|
||||
idempotency_key: str | None = None
|
||||
request_digest: str = ""
|
||||
result: Mapping[str, Any] | None = None
|
||||
error: Mapping[str, Any] | None = None
|
||||
created_by: str = ""
|
||||
approved_by: str | None = None
|
||||
created_at: str = field(default_factory=_utc_now)
|
||||
updated_at: str = field(default_factory=_utc_now)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AuditEvent:
|
||||
event_id: str
|
||||
tenant_id: str
|
||||
principal_id: str
|
||||
trace_id: str
|
||||
action: str
|
||||
resource_type: str
|
||||
resource_id: str
|
||||
outcome: str
|
||||
policy_version: str
|
||||
details: Mapping[str, Any] = field(default_factory=dict)
|
||||
created_at: str = field(default_factory=_utc_now)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PolicyDecision:
|
||||
allowed: bool
|
||||
reason: str
|
||||
policy_version: str
|
||||
|
||||
|
||||
PolicyEvaluator = Callable[[RequestContext, str, Mapping[str, Any]], PolicyDecision]
|
||||
|
||||
|
||||
class InMemoryControlPlaneStore:
|
||||
"""Thread-safe reference store used by tests and the first gateway slice.
|
||||
|
||||
Production backends must preserve the same tenant-scoped interface and add
|
||||
database row-level security, migrations, backup and HA evidence.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.lock = threading.RLock()
|
||||
self.tenants: dict[str, TenantRecord] = {}
|
||||
self.agents: dict[str, AgentRecord] = {}
|
||||
self.tasks: dict[str, TaskRecord] = {}
|
||||
self.idempotency: dict[tuple[str, str], tuple[str, str]] = {}
|
||||
self.audit_events: list[AuditEvent] = []
|
||||
|
||||
|
||||
class ControlPlaneService:
|
||||
"""Authoritative tenant, agent and task lifecycle service."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: InMemoryControlPlaneStore | None = None,
|
||||
*,
|
||||
policy_evaluator: PolicyEvaluator | None = None,
|
||||
) -> None:
|
||||
self.store = store or InMemoryControlPlaneStore()
|
||||
self.policy_evaluator = policy_evaluator or self._default_policy
|
||||
|
||||
@staticmethod
|
||||
def _default_policy(
|
||||
context: RequestContext,
|
||||
action: str,
|
||||
resource: Mapping[str, Any],
|
||||
) -> PolicyDecision:
|
||||
del resource
|
||||
role_requirements = {
|
||||
"tenant.register": {"platform_admin"},
|
||||
"tenant.read": {"tenant_admin", "operator", "auditor", "platform_admin"},
|
||||
"agent.register": {"tenant_admin", "operator", "platform_admin"},
|
||||
"agent.read": {"tenant_admin", "operator", "auditor", "platform_admin"},
|
||||
"task.create": {"tenant_admin", "operator", "platform_admin"},
|
||||
"task.read": {"tenant_admin", "operator", "auditor", "platform_admin"},
|
||||
"task.approve": {"tenant_admin", "approver", "platform_admin"},
|
||||
"task.start": {"runtime", "operator", "platform_admin"},
|
||||
"task.complete": {"runtime", "operator", "platform_admin"},
|
||||
"task.fail": {"runtime", "operator", "platform_admin"},
|
||||
"task.cancel": {"tenant_admin", "operator", "approver", "platform_admin"},
|
||||
"audit.read": {"tenant_admin", "auditor", "platform_admin"},
|
||||
}
|
||||
required = role_requirements.get(action, {"platform_admin"})
|
||||
allowed = bool(context.roles.intersection(required))
|
||||
return PolicyDecision(
|
||||
allowed=allowed,
|
||||
reason="role_allowed" if allowed else "required_role_missing",
|
||||
policy_version=context.policy_version,
|
||||
)
|
||||
|
||||
def _require_context(self, context: RequestContext) -> None:
|
||||
if not context.principal_id or not context.tenant_id or not context.trace_id:
|
||||
raise Unauthorized("authenticated tenant context is required")
|
||||
|
||||
def _authorize(
|
||||
self,
|
||||
context: RequestContext,
|
||||
action: str,
|
||||
resource: Mapping[str, Any],
|
||||
) -> PolicyDecision:
|
||||
self._require_context(context)
|
||||
decision = self.policy_evaluator(context, action, resource)
|
||||
if not decision.allowed:
|
||||
self._audit(
|
||||
context,
|
||||
action=action,
|
||||
resource_type=str(resource.get("resource_type", "policy")),
|
||||
resource_id=str(resource.get("resource_id", "unknown")),
|
||||
outcome="denied",
|
||||
details={"reason": decision.reason},
|
||||
)
|
||||
raise PolicyDenied(f"{action} denied: {decision.reason}")
|
||||
return decision
|
||||
|
||||
@staticmethod
|
||||
def _assert_same_tenant(context: RequestContext, tenant_id: str) -> None:
|
||||
if context.tenant_id != tenant_id and "platform_admin" not in context.roles:
|
||||
raise TenantIsolationError("cross-tenant access denied")
|
||||
|
||||
def _audit(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
action: str,
|
||||
resource_type: str,
|
||||
resource_id: str,
|
||||
outcome: str,
|
||||
details: Mapping[str, Any] | None = None,
|
||||
) -> AuditEvent:
|
||||
event = AuditEvent(
|
||||
event_id=_new_id("audit"),
|
||||
tenant_id=context.tenant_id,
|
||||
principal_id=context.principal_id,
|
||||
trace_id=context.trace_id,
|
||||
action=action,
|
||||
resource_type=resource_type,
|
||||
resource_id=resource_id,
|
||||
outcome=outcome,
|
||||
policy_version=context.policy_version,
|
||||
details=copy.deepcopy(dict(details or {})),
|
||||
)
|
||||
self.store.audit_events.append(event)
|
||||
return event
|
||||
|
||||
def register_tenant(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
tenant_id: str,
|
||||
display_name: str,
|
||||
metadata: Mapping[str, Any] | None = None,
|
||||
) -> TenantRecord:
|
||||
tenant_id = tenant_id.strip()
|
||||
if not tenant_id or not display_name.strip():
|
||||
raise ValueError("tenant_id and display_name are required")
|
||||
self._authorize(
|
||||
context,
|
||||
"tenant.register",
|
||||
{"resource_type": "tenant", "resource_id": tenant_id},
|
||||
)
|
||||
self._assert_same_tenant(context, tenant_id)
|
||||
with self.store.lock:
|
||||
if tenant_id in self.store.tenants:
|
||||
raise ControlPlaneError(f"tenant already exists: {tenant_id}")
|
||||
tenant = TenantRecord(
|
||||
tenant_id=tenant_id,
|
||||
display_name=display_name.strip(),
|
||||
metadata=copy.deepcopy(dict(metadata or {})),
|
||||
)
|
||||
self.store.tenants[tenant_id] = tenant
|
||||
self._audit(
|
||||
context,
|
||||
action="tenant.register",
|
||||
resource_type="tenant",
|
||||
resource_id=tenant_id,
|
||||
outcome="allowed",
|
||||
)
|
||||
return tenant
|
||||
|
||||
def get_tenant(self, context: RequestContext, tenant_id: str | None = None) -> TenantRecord:
|
||||
tenant_id = tenant_id or context.tenant_id
|
||||
self._authorize(
|
||||
context,
|
||||
"tenant.read",
|
||||
{"resource_type": "tenant", "resource_id": tenant_id},
|
||||
)
|
||||
self._assert_same_tenant(context, tenant_id)
|
||||
with self.store.lock:
|
||||
tenant = self.store.tenants.get(tenant_id)
|
||||
if tenant is None:
|
||||
raise NotFound("tenant not found")
|
||||
return tenant
|
||||
|
||||
def register_agent(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
agent_id: str,
|
||||
display_name: str,
|
||||
runtime: str,
|
||||
capabilities: Iterable[str] = (),
|
||||
metadata: Mapping[str, Any] | None = None,
|
||||
) -> AgentRecord:
|
||||
self._authorize(
|
||||
context,
|
||||
"agent.register",
|
||||
{"resource_type": "agent", "resource_id": agent_id},
|
||||
)
|
||||
tenant = self.get_tenant(context)
|
||||
if tenant.status != "active":
|
||||
raise ControlPlaneError("tenant is not active")
|
||||
normalized_capabilities = tuple(
|
||||
sorted({str(item).strip() for item in capabilities if str(item).strip()})
|
||||
)
|
||||
if not agent_id.strip() or not display_name.strip() or not runtime.strip():
|
||||
raise ValueError("agent_id, display_name and runtime are required")
|
||||
with self.store.lock:
|
||||
existing = self.store.agents.get(agent_id)
|
||||
if existing is not None and existing.tenant_id != context.tenant_id:
|
||||
raise TenantIsolationError("agent id belongs to another tenant")
|
||||
if existing is not None:
|
||||
raise ControlPlaneError(f"agent already exists: {agent_id}")
|
||||
agent = AgentRecord(
|
||||
agent_id=agent_id.strip(),
|
||||
tenant_id=context.tenant_id,
|
||||
display_name=display_name.strip(),
|
||||
runtime=runtime.strip(),
|
||||
capabilities=normalized_capabilities,
|
||||
metadata=copy.deepcopy(dict(metadata or {})),
|
||||
)
|
||||
self.store.agents[agent.agent_id] = agent
|
||||
self._audit(
|
||||
context,
|
||||
action="agent.register",
|
||||
resource_type="agent",
|
||||
resource_id=agent.agent_id,
|
||||
outcome="allowed",
|
||||
details={"runtime": agent.runtime},
|
||||
)
|
||||
return agent
|
||||
|
||||
def get_agent(self, context: RequestContext, agent_id: str) -> AgentRecord:
|
||||
self._authorize(
|
||||
context,
|
||||
"agent.read",
|
||||
{"resource_type": "agent", "resource_id": agent_id},
|
||||
)
|
||||
with self.store.lock:
|
||||
agent = self.store.agents.get(agent_id)
|
||||
if agent is None:
|
||||
raise NotFound("agent not found")
|
||||
self._assert_same_tenant(context, agent.tenant_id)
|
||||
return agent
|
||||
|
||||
def list_agents(self, context: RequestContext) -> list[AgentRecord]:
|
||||
self._authorize(
|
||||
context,
|
||||
"agent.read",
|
||||
{"resource_type": "agent_collection", "resource_id": context.tenant_id},
|
||||
)
|
||||
with self.store.lock:
|
||||
return [
|
||||
agent
|
||||
for agent in self.store.agents.values()
|
||||
if agent.tenant_id == context.tenant_id
|
||||
]
|
||||
|
||||
def create_task(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
agent_id: str,
|
||||
action: str,
|
||||
payload: Mapping[str, Any] | None = None,
|
||||
idempotency_key: str | None = None,
|
||||
) -> TaskRecord:
|
||||
self._authorize(
|
||||
context,
|
||||
"task.create",
|
||||
{"resource_type": "task", "resource_id": idempotency_key or "new"},
|
||||
)
|
||||
agent = self.get_agent(context, agent_id)
|
||||
if not agent.enabled:
|
||||
raise ControlPlaneError("agent is disabled")
|
||||
if action not in agent.capabilities and "*" not in agent.capabilities:
|
||||
raise PolicyDenied(f"agent does not advertise capability: {action}")
|
||||
normalized_payload = copy.deepcopy(dict(payload or {}))
|
||||
request_digest = _canonical_digest(
|
||||
{"agent_id": agent_id, "action": action, "payload": normalized_payload}
|
||||
)
|
||||
normalized_key = idempotency_key.strip() if idempotency_key else None
|
||||
with self.store.lock:
|
||||
if normalized_key:
|
||||
idempotency_ref = (context.tenant_id, normalized_key)
|
||||
existing = self.store.idempotency.get(idempotency_ref)
|
||||
if existing is not None:
|
||||
existing_task_id, existing_digest = existing
|
||||
if existing_digest != request_digest:
|
||||
raise IdempotencyConflict(
|
||||
"idempotency key was reused with a different request"
|
||||
)
|
||||
return self.store.tasks[existing_task_id]
|
||||
task = TaskRecord(
|
||||
task_id=_new_id("task"),
|
||||
tenant_id=context.tenant_id,
|
||||
agent_id=agent_id,
|
||||
action=action,
|
||||
payload=normalized_payload,
|
||||
idempotency_key=normalized_key,
|
||||
request_digest=request_digest,
|
||||
created_by=context.principal_id,
|
||||
)
|
||||
self.store.tasks[task.task_id] = task
|
||||
if normalized_key:
|
||||
self.store.idempotency[(context.tenant_id, normalized_key)] = (
|
||||
task.task_id,
|
||||
request_digest,
|
||||
)
|
||||
self._audit(
|
||||
context,
|
||||
action="task.create",
|
||||
resource_type="task",
|
||||
resource_id=task.task_id,
|
||||
outcome="allowed",
|
||||
details={"agent_id": agent_id, "action": action},
|
||||
)
|
||||
return task
|
||||
|
||||
def get_task(self, context: RequestContext, task_id: str) -> TaskRecord:
|
||||
self._authorize(
|
||||
context,
|
||||
"task.read",
|
||||
{"resource_type": "task", "resource_id": task_id},
|
||||
)
|
||||
with self.store.lock:
|
||||
task = self.store.tasks.get(task_id)
|
||||
if task is None:
|
||||
raise NotFound("task not found")
|
||||
self._assert_same_tenant(context, task.tenant_id)
|
||||
return task
|
||||
|
||||
def list_tasks(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
states: Iterable[TaskState | str] | None = None,
|
||||
) -> list[TaskRecord]:
|
||||
self._authorize(
|
||||
context,
|
||||
"task.read",
|
||||
{"resource_type": "task_collection", "resource_id": context.tenant_id},
|
||||
)
|
||||
state_filter = (
|
||||
{TaskState(item) for item in states}
|
||||
if states is not None
|
||||
else None
|
||||
)
|
||||
with self.store.lock:
|
||||
return [
|
||||
task
|
||||
for task in self.store.tasks.values()
|
||||
if task.tenant_id == context.tenant_id
|
||||
and (state_filter is None or task.state in state_filter)
|
||||
]
|
||||
|
||||
def transition_task(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
task_id: str,
|
||||
target_state: TaskState | str,
|
||||
expected_revision: int | None = None,
|
||||
result: Mapping[str, Any] | None = None,
|
||||
error: Mapping[str, Any] | None = None,
|
||||
) -> TaskRecord:
|
||||
target = TaskState(target_state)
|
||||
action_name = {
|
||||
TaskState.APPROVED: "task.approve",
|
||||
TaskState.RUNNING: "task.start",
|
||||
TaskState.COMPLETED: "task.complete",
|
||||
TaskState.FAILED: "task.fail",
|
||||
TaskState.CANCELLED: "task.cancel",
|
||||
}.get(target)
|
||||
if action_name is None:
|
||||
raise InvalidTransition(f"cannot transition to {target.value}")
|
||||
self._authorize(
|
||||
context,
|
||||
action_name,
|
||||
{"resource_type": "task", "resource_id": task_id},
|
||||
)
|
||||
with self.store.lock:
|
||||
current = self.store.tasks.get(task_id)
|
||||
if current is None:
|
||||
raise NotFound("task not found")
|
||||
self._assert_same_tenant(context, current.tenant_id)
|
||||
if expected_revision is not None and current.revision != expected_revision:
|
||||
raise RevisionConflict(
|
||||
f"expected revision {expected_revision}, current {current.revision}"
|
||||
)
|
||||
if target not in ALLOWED_TASK_TRANSITIONS[current.state]:
|
||||
raise InvalidTransition(
|
||||
f"invalid transition {current.state.value} -> {target.value}"
|
||||
)
|
||||
if target == TaskState.COMPLETED and result is None:
|
||||
raise ValueError("completed task requires result")
|
||||
if target == TaskState.FAILED and error is None:
|
||||
raise ValueError("failed task requires error")
|
||||
updated = TaskRecord(
|
||||
task_id=current.task_id,
|
||||
tenant_id=current.tenant_id,
|
||||
agent_id=current.agent_id,
|
||||
action=current.action,
|
||||
payload=current.payload,
|
||||
state=target,
|
||||
revision=current.revision + 1,
|
||||
idempotency_key=current.idempotency_key,
|
||||
request_digest=current.request_digest,
|
||||
result=copy.deepcopy(dict(result)) if result is not None else current.result,
|
||||
error=copy.deepcopy(dict(error)) if error is not None else current.error,
|
||||
created_by=current.created_by,
|
||||
approved_by=(
|
||||
context.principal_id
|
||||
if target == TaskState.APPROVED
|
||||
else current.approved_by
|
||||
),
|
||||
created_at=current.created_at,
|
||||
updated_at=_utc_now(),
|
||||
)
|
||||
self.store.tasks[task_id] = updated
|
||||
self._audit(
|
||||
context,
|
||||
action=action_name,
|
||||
resource_type="task",
|
||||
resource_id=task_id,
|
||||
outcome="allowed",
|
||||
details={
|
||||
"from": current.state.value,
|
||||
"to": target.value,
|
||||
"revision": updated.revision,
|
||||
},
|
||||
)
|
||||
return updated
|
||||
|
||||
def list_audit_events(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
resource_id: str | None = None,
|
||||
) -> list[AuditEvent]:
|
||||
self._authorize(
|
||||
context,
|
||||
"audit.read",
|
||||
{"resource_type": "audit", "resource_id": resource_id or context.tenant_id},
|
||||
)
|
||||
with self.store.lock:
|
||||
return [
|
||||
event
|
||||
for event in self.store.audit_events
|
||||
if event.tenant_id == context.tenant_id
|
||||
and (resource_id is None or event.resource_id == resource_id)
|
||||
]
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Git-backed agent execution evidence for AIOS V2.
|
||||
|
||||
Evidence is tenant-scoped, append-friendly, and contains no runtime credentials.
|
||||
It provides a deterministic source for claim, heartbeat, progress, result and
|
||||
verification state across Codex, OpenClaw and Hermes executors.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from dataclasses import asdict, dataclass, field, replace
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _required(value: str, label: str) -> str:
|
||||
normalized = str(value or "").strip()
|
||||
if not normalized:
|
||||
raise ValueError(f"{label} is required")
|
||||
return normalized
|
||||
|
||||
|
||||
def _safe(value: Mapping[str, Any] | None) -> dict[str, Any]:
|
||||
return json.loads(json.dumps(dict(value or {}), ensure_ascii=False, default=str))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExecutionEvidence:
|
||||
execution_id: str
|
||||
tenant_id: str
|
||||
task_id: str
|
||||
agent_id: str
|
||||
executor: str
|
||||
branch: str
|
||||
status: str = "claimed"
|
||||
progress_completed: int = 0
|
||||
progress_total: int = 1
|
||||
current_step: str = "claim"
|
||||
last_commit: str | None = None
|
||||
test_status: str = "not_run"
|
||||
test_summary: Mapping[str, Any] = field(default_factory=dict)
|
||||
result_summary: Mapping[str, Any] = field(default_factory=dict)
|
||||
claimed_at: str = field(default_factory=utc_now)
|
||||
heartbeat_at: str = field(default_factory=utc_now)
|
||||
completed_at: str | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name in (
|
||||
"execution_id",
|
||||
"tenant_id",
|
||||
"task_id",
|
||||
"agent_id",
|
||||
"executor",
|
||||
"branch",
|
||||
"status",
|
||||
"current_step",
|
||||
"claimed_at",
|
||||
"heartbeat_at",
|
||||
):
|
||||
object.__setattr__(self, name, _required(getattr(self, name), name))
|
||||
if self.progress_completed < 0 or self.progress_total < 1:
|
||||
raise ValueError("invalid progress values")
|
||||
if self.progress_completed > self.progress_total:
|
||||
raise ValueError("progress_completed cannot exceed progress_total")
|
||||
object.__setattr__(self, "test_summary", _safe(self.test_summary))
|
||||
object.__setattr__(self, "result_summary", _safe(self.result_summary))
|
||||
|
||||
|
||||
class ExecutionTracker:
|
||||
"""Persist one JSON evidence record per tenant and execution."""
|
||||
|
||||
def __init__(self, root: str | Path) -> None:
|
||||
self.root = Path(root).resolve()
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
self._lock = threading.RLock()
|
||||
|
||||
@staticmethod
|
||||
def _component(value: str) -> str:
|
||||
normalized = _required(value, "path component")
|
||||
if normalized in {".", ".."} or "/" in normalized or "\\" in normalized:
|
||||
raise ValueError("invalid evidence path component")
|
||||
return normalized
|
||||
|
||||
def _path(self, tenant_id: str, execution_id: str) -> Path:
|
||||
tenant = self._component(tenant_id)
|
||||
execution = self._component(execution_id)
|
||||
return self.root / tenant / f"{execution}.json"
|
||||
|
||||
def claim(self, evidence: ExecutionEvidence) -> ExecutionEvidence:
|
||||
path = self._path(evidence.tenant_id, evidence.execution_id)
|
||||
with self._lock:
|
||||
if path.exists():
|
||||
current = self._read(path)
|
||||
if current.task_id != evidence.task_id or current.agent_id != evidence.agent_id:
|
||||
raise RuntimeError("execution_id already belongs to another task")
|
||||
return current
|
||||
self._write(path, evidence)
|
||||
return evidence
|
||||
|
||||
def heartbeat(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
execution_id: str,
|
||||
status: str | None = None,
|
||||
current_step: str | None = None,
|
||||
progress_completed: int | None = None,
|
||||
progress_total: int | None = None,
|
||||
last_commit: str | None = None,
|
||||
) -> ExecutionEvidence:
|
||||
path = self._path(tenant_id, execution_id)
|
||||
with self._lock:
|
||||
current = self._read(path)
|
||||
updated = replace(
|
||||
current,
|
||||
status=status or current.status,
|
||||
current_step=current_step or current.current_step,
|
||||
progress_completed=(
|
||||
progress_completed
|
||||
if progress_completed is not None
|
||||
else current.progress_completed
|
||||
),
|
||||
progress_total=(
|
||||
progress_total if progress_total is not None else current.progress_total
|
||||
),
|
||||
last_commit=last_commit or current.last_commit,
|
||||
heartbeat_at=utc_now(),
|
||||
)
|
||||
self._write(path, updated)
|
||||
return updated
|
||||
|
||||
def complete(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
execution_id: str,
|
||||
status: str,
|
||||
test_status: str,
|
||||
test_summary: Mapping[str, Any] | None = None,
|
||||
result_summary: Mapping[str, Any] | None = None,
|
||||
last_commit: str | None = None,
|
||||
) -> ExecutionEvidence:
|
||||
path = self._path(tenant_id, execution_id)
|
||||
with self._lock:
|
||||
current = self._read(path)
|
||||
updated = replace(
|
||||
current,
|
||||
status=_required(status, "status"),
|
||||
current_step="complete",
|
||||
progress_completed=current.progress_total,
|
||||
last_commit=last_commit or current.last_commit,
|
||||
test_status=_required(test_status, "test_status"),
|
||||
test_summary=_safe(test_summary),
|
||||
result_summary=_safe(result_summary),
|
||||
heartbeat_at=utc_now(),
|
||||
completed_at=utc_now(),
|
||||
)
|
||||
self._write(path, updated)
|
||||
return updated
|
||||
|
||||
def get(self, *, tenant_id: str, execution_id: str) -> ExecutionEvidence:
|
||||
return self._read(self._path(tenant_id, execution_id))
|
||||
|
||||
def list(self, *, tenant_id: str) -> list[ExecutionEvidence]:
|
||||
tenant_dir = self.root / self._component(tenant_id)
|
||||
if not tenant_dir.exists():
|
||||
return []
|
||||
records = [self._read(path) for path in tenant_dir.glob("*.json")]
|
||||
return sorted(records, key=lambda item: (item.heartbeat_at, item.execution_id), reverse=True)
|
||||
|
||||
@staticmethod
|
||||
def _read(path: Path) -> ExecutionEvidence:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError as exc:
|
||||
raise KeyError("execution evidence not found") from exc
|
||||
if not isinstance(value, Mapping):
|
||||
raise RuntimeError("invalid execution evidence file")
|
||||
return ExecutionEvidence(**dict(value))
|
||||
|
||||
@staticmethod
|
||||
def _write(path: Path, evidence: ExecutionEvidence) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(f".tmp-{os.getpid()}")
|
||||
temporary.write_text(
|
||||
json.dumps(asdict(evidence), ensure_ascii=False, sort_keys=True, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.replace(temporary, path)
|
||||
@@ -0,0 +1,346 @@
|
||||
"""HTTP/JSON gateway for the AIOS V2 control plane.
|
||||
|
||||
The gateway is intentionally runtime-neutral: browsers and external clients
|
||||
call this API, while OpenClaw and Hermes remain behind authenticated adapters.
|
||||
Identity headers are accepted only from a trusted reverse proxy in v1; direct
|
||||
public exposure is prohibited until signed identity middleware is enabled.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import urllib.parse
|
||||
from enum import Enum
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from typing import Any, Mapping
|
||||
|
||||
from .control_plane import (
|
||||
ControlPlaneError,
|
||||
ControlPlaneService,
|
||||
IdempotencyConflict,
|
||||
InvalidTransition,
|
||||
NotFound,
|
||||
PolicyDenied,
|
||||
RequestContext,
|
||||
RevisionConflict,
|
||||
TaskState,
|
||||
TenantIsolationError,
|
||||
Unauthorized,
|
||||
)
|
||||
|
||||
|
||||
TENANT_HEADER = "X-AIOS-Tenant-ID"
|
||||
PRINCIPAL_HEADER = "X-AIOS-Principal-ID"
|
||||
ROLES_HEADER = "X-AIOS-Roles"
|
||||
TRACE_HEADER = "X-AIOS-Trace-ID"
|
||||
SESSION_HEADER = "X-AIOS-Session-ID"
|
||||
POLICY_HEADER = "X-AIOS-Policy-Version"
|
||||
AUTHENTICATED_HEADER = "X-AIOS-Authenticated"
|
||||
|
||||
_RESOURCE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,191}$")
|
||||
|
||||
|
||||
def _json_value(value: Any) -> Any:
|
||||
if dataclasses.is_dataclass(value):
|
||||
return {
|
||||
field.name: _json_value(getattr(value, field.name))
|
||||
for field in dataclasses.fields(value)
|
||||
}
|
||||
if isinstance(value, Enum):
|
||||
return value.value
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _json_value(item) for key, item in value.items()}
|
||||
if isinstance(value, (list, tuple, set, frozenset)):
|
||||
return [_json_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _require_mapping(value: Any, *, label: str = "JSON body") -> Mapping[str, Any]:
|
||||
if not isinstance(value, Mapping):
|
||||
raise ValueError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _require_resource_id(value: Any, *, label: str) -> str:
|
||||
normalized = str(value or "").strip()
|
||||
if not _RESOURCE_ID_RE.fullmatch(normalized):
|
||||
raise ValueError(f"invalid {label}")
|
||||
return normalized
|
||||
|
||||
|
||||
class GatewayApplication:
|
||||
"""Pure request router that can be tested without a listening socket."""
|
||||
|
||||
def __init__(self, service: ControlPlaneService | None = None) -> None:
|
||||
self.service = service or ControlPlaneService()
|
||||
|
||||
@staticmethod
|
||||
def context_from_headers(headers: Mapping[str, str]) -> RequestContext:
|
||||
authenticated = str(headers.get(AUTHENTICATED_HEADER, "")).strip().lower()
|
||||
if authenticated not in {"1", "true", "yes"}:
|
||||
raise Unauthorized("trusted authentication assertion is required")
|
||||
roles = [
|
||||
role.strip()
|
||||
for role in str(headers.get(ROLES_HEADER, "")).split(",")
|
||||
if role.strip()
|
||||
]
|
||||
return RequestContext.create(
|
||||
tenant_id=str(headers.get(TENANT_HEADER, "")),
|
||||
principal_id=str(headers.get(PRINCIPAL_HEADER, "")),
|
||||
trace_id=str(headers.get(TRACE_HEADER, "")).strip() or None,
|
||||
session_id=str(headers.get(SESSION_HEADER, "")).strip() or None,
|
||||
policy_version=str(headers.get(POLICY_HEADER, "v1")).strip() or "v1",
|
||||
roles=roles,
|
||||
)
|
||||
|
||||
def handle(
|
||||
self,
|
||||
method: str,
|
||||
raw_path: str,
|
||||
headers: Mapping[str, str],
|
||||
body: Any = None,
|
||||
) -> tuple[int, Mapping[str, Any]]:
|
||||
parsed = urllib.parse.urlparse(raw_path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
query = urllib.parse.parse_qs(parsed.query, keep_blank_values=False)
|
||||
method = method.upper()
|
||||
|
||||
if method == "GET" and path in {"/healthz", "/api/v2/health"}:
|
||||
return HTTPStatus.OK, {
|
||||
"service": "aios-v2-control-plane",
|
||||
"status": "ok",
|
||||
"store": type(self.service.store).__name__,
|
||||
}
|
||||
|
||||
context = self.context_from_headers(headers)
|
||||
payload = _require_mapping(body or {})
|
||||
|
||||
if method == "POST" and path == "/api/v2/tenants":
|
||||
tenant = self.service.register_tenant(
|
||||
context,
|
||||
tenant_id=_require_resource_id(payload.get("tenant_id"), label="tenant_id"),
|
||||
display_name=str(payload.get("display_name", "")),
|
||||
metadata=_require_mapping(payload.get("metadata", {}), label="metadata"),
|
||||
)
|
||||
return HTTPStatus.CREATED, {"tenant": _json_value(tenant)}
|
||||
|
||||
if method == "GET" and path == "/api/v2/tenants/current":
|
||||
tenant = self.service.get_tenant(context)
|
||||
return HTTPStatus.OK, {"tenant": _json_value(tenant)}
|
||||
|
||||
if method == "POST" and path == "/api/v2/agents":
|
||||
agent = self.service.register_agent(
|
||||
context,
|
||||
agent_id=_require_resource_id(payload.get("agent_id"), label="agent_id"),
|
||||
display_name=str(payload.get("display_name", "")),
|
||||
runtime=str(payload.get("runtime", "")),
|
||||
capabilities=payload.get("capabilities", ()),
|
||||
metadata=_require_mapping(payload.get("metadata", {}), label="metadata"),
|
||||
)
|
||||
return HTTPStatus.CREATED, {"agent": _json_value(agent)}
|
||||
|
||||
if method == "GET" and path == "/api/v2/agents":
|
||||
return HTTPStatus.OK, {
|
||||
"agents": _json_value(self.service.list_agents(context))
|
||||
}
|
||||
|
||||
agent_match = re.fullmatch(r"/api/v2/agents/([^/]+)", path)
|
||||
if method == "GET" and agent_match:
|
||||
agent_id = _require_resource_id(
|
||||
urllib.parse.unquote(agent_match.group(1)), label="agent_id"
|
||||
)
|
||||
return HTTPStatus.OK, {
|
||||
"agent": _json_value(self.service.get_agent(context, agent_id))
|
||||
}
|
||||
|
||||
if method == "POST" and path == "/api/v2/tasks":
|
||||
task = self.service.create_task(
|
||||
context,
|
||||
agent_id=_require_resource_id(payload.get("agent_id"), label="agent_id"),
|
||||
action=str(payload.get("action", "")).strip(),
|
||||
payload=_require_mapping(payload.get("payload", {}), label="payload"),
|
||||
idempotency_key=(
|
||||
str(payload["idempotency_key"]).strip()
|
||||
if payload.get("idempotency_key") is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
return HTTPStatus.CREATED, {"task": _json_value(task)}
|
||||
|
||||
if method == "GET" and path == "/api/v2/tasks":
|
||||
requested_states = query.get("state")
|
||||
tasks = self.service.list_tasks(context, states=requested_states)
|
||||
return HTTPStatus.OK, {"tasks": _json_value(tasks)}
|
||||
|
||||
task_match = re.fullmatch(r"/api/v2/tasks/([^/]+)", path)
|
||||
if method == "GET" and task_match:
|
||||
task_id = _require_resource_id(
|
||||
urllib.parse.unquote(task_match.group(1)), label="task_id"
|
||||
)
|
||||
return HTTPStatus.OK, {
|
||||
"task": _json_value(self.service.get_task(context, task_id))
|
||||
}
|
||||
|
||||
transition_match = re.fullmatch(
|
||||
r"/api/v2/tasks/([^/]+)/(approve|start|complete|fail|cancel)", path
|
||||
)
|
||||
if method == "POST" and transition_match:
|
||||
task_id = _require_resource_id(
|
||||
urllib.parse.unquote(transition_match.group(1)), label="task_id"
|
||||
)
|
||||
transition = transition_match.group(2)
|
||||
target = {
|
||||
"approve": TaskState.APPROVED,
|
||||
"start": TaskState.RUNNING,
|
||||
"complete": TaskState.COMPLETED,
|
||||
"fail": TaskState.FAILED,
|
||||
"cancel": TaskState.CANCELLED,
|
||||
}[transition]
|
||||
expected_revision = payload.get("expected_revision")
|
||||
if expected_revision is not None:
|
||||
expected_revision = int(expected_revision)
|
||||
task = self.service.transition_task(
|
||||
context,
|
||||
task_id=task_id,
|
||||
target_state=target,
|
||||
expected_revision=expected_revision,
|
||||
result=(
|
||||
_require_mapping(payload.get("result"), label="result")
|
||||
if payload.get("result") is not None
|
||||
else None
|
||||
),
|
||||
error=(
|
||||
_require_mapping(payload.get("error"), label="error")
|
||||
if payload.get("error") is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
return HTTPStatus.OK, {"task": _json_value(task)}
|
||||
|
||||
if method == "GET" and path == "/api/v2/audit":
|
||||
resource_id = query.get("resource_id", [None])[0]
|
||||
events = self.service.list_audit_events(
|
||||
context,
|
||||
resource_id=(
|
||||
_require_resource_id(resource_id, label="resource_id")
|
||||
if resource_id
|
||||
else None
|
||||
),
|
||||
)
|
||||
return HTTPStatus.OK, {"events": _json_value(events)}
|
||||
|
||||
return HTTPStatus.NOT_FOUND, {
|
||||
"error": "not_found",
|
||||
"message": "route not found",
|
||||
}
|
||||
|
||||
|
||||
_ERROR_STATUS: tuple[tuple[type[BaseException], HTTPStatus], ...] = (
|
||||
(Unauthorized, HTTPStatus.UNAUTHORIZED),
|
||||
(PolicyDenied, HTTPStatus.FORBIDDEN),
|
||||
(TenantIsolationError, HTTPStatus.NOT_FOUND),
|
||||
(NotFound, HTTPStatus.NOT_FOUND),
|
||||
(IdempotencyConflict, HTTPStatus.CONFLICT),
|
||||
(RevisionConflict, HTTPStatus.CONFLICT),
|
||||
(InvalidTransition, HTTPStatus.CONFLICT),
|
||||
(ValueError, HTTPStatus.BAD_REQUEST),
|
||||
(ControlPlaneError, HTTPStatus.BAD_REQUEST),
|
||||
)
|
||||
|
||||
|
||||
def error_response(exc: BaseException) -> tuple[int, Mapping[str, Any]]:
|
||||
for error_type, status in _ERROR_STATUS:
|
||||
if isinstance(exc, error_type):
|
||||
return status, {
|
||||
"error": error_type.__name__,
|
||||
"message": str(exc),
|
||||
}
|
||||
return HTTPStatus.INTERNAL_SERVER_ERROR, {
|
||||
"error": "InternalServerError",
|
||||
"message": "internal server error",
|
||||
}
|
||||
|
||||
|
||||
class GatewayRequestHandler(BaseHTTPRequestHandler):
|
||||
server_version = "AIOSV2Gateway/1.0"
|
||||
max_body_bytes = 1_048_576
|
||||
|
||||
@property
|
||||
def application(self) -> GatewayApplication:
|
||||
return self.server.application # type: ignore[attr-defined]
|
||||
|
||||
def _read_body(self) -> Any:
|
||||
length = int(self.headers.get("Content-Length", "0") or "0")
|
||||
if length < 0 or length > self.max_body_bytes:
|
||||
raise ValueError("request body is too large")
|
||||
if length == 0:
|
||||
return {}
|
||||
content_type = self.headers.get("Content-Type", "").split(";", 1)[0].strip()
|
||||
if content_type != "application/json":
|
||||
raise ValueError("Content-Type must be application/json")
|
||||
raw = self.rfile.read(length)
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
|
||||
def _send_json(self, status: int, payload: Mapping[str, Any]) -> None:
|
||||
encoded = json.dumps(
|
||||
_json_value(payload), ensure_ascii=False, separators=(",", ":")
|
||||
).encode("utf-8")
|
||||
self.send_response(int(status))
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(encoded)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.end_headers()
|
||||
self.wfile.write(encoded)
|
||||
|
||||
def _dispatch(self, method: str) -> None:
|
||||
try:
|
||||
body = self._read_body() if method in {"POST", "PUT", "PATCH"} else {}
|
||||
status, payload = self.application.handle(
|
||||
method,
|
||||
self.path,
|
||||
{key: value for key, value in self.headers.items()},
|
||||
body,
|
||||
)
|
||||
except BaseException as exc: # handler boundary; sanitized below
|
||||
status, payload = error_response(exc)
|
||||
self._send_json(status, payload)
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
self._dispatch("GET")
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
self._dispatch("POST")
|
||||
|
||||
def log_message(self, format: str, *args: Any) -> None:
|
||||
# Production deployment routes structured access logs through the
|
||||
# reverse proxy; do not log identity headers or request bodies here.
|
||||
super().log_message(format, *args)
|
||||
|
||||
|
||||
class GatewayHTTPServer(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
server_address: tuple[str, int],
|
||||
application: GatewayApplication,
|
||||
) -> None:
|
||||
self.application = application
|
||||
super().__init__(server_address, GatewayRequestHandler)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
host = os.getenv("AIOS_V2_GATEWAY_HOST", "127.0.0.1")
|
||||
port = int(os.getenv("AIOS_V2_GATEWAY_PORT", "18920"))
|
||||
server = GatewayHTTPServer((host, port), GatewayApplication())
|
||||
print(f"AIOS V2 control-plane gateway listening on {host}:{port}")
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Sanitized GitOps runtime status for the AIOS Web control plane."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from dataclasses import dataclass, asdict
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GitOpsStatus:
|
||||
available: bool
|
||||
service_status: str
|
||||
workspace_mode: str
|
||||
workspace_branch: str | None
|
||||
workspace_ref: str | None
|
||||
workspace_dirty: bool | None
|
||||
candidate_branch: str
|
||||
release_channel: str
|
||||
release_version: str
|
||||
sync_status: str
|
||||
details: Mapping[str, Any]
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def _run_git(worktree: Path, *args: str) -> tuple[int, str, str]:
|
||||
process = subprocess.run(
|
||||
["git", "-C", str(worktree), *args],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
return process.returncode, process.stdout.strip(), process.stderr.strip()
|
||||
|
||||
|
||||
def collect_gitops_status() -> GitOpsStatus:
|
||||
"""Return only non-sensitive Git/workspace metadata.
|
||||
|
||||
Remote URLs, environment values, credentials, filesystem contents and
|
||||
command stderr are deliberately excluded from the Web response.
|
||||
"""
|
||||
|
||||
candidate_branch = os.getenv(
|
||||
"AIOS_CANDIDATE_BRANCH",
|
||||
"integration/aios-v2r1-cto-production-candidate-20260720",
|
||||
)
|
||||
release_channel = os.getenv("AIOS_RELEASE_CHANNEL", "candidate")
|
||||
release_version = os.getenv("AIOS_RELEASE_VERSION", "V2R1C0-candidate")
|
||||
sync_status = os.getenv("AIOS_GITOPS_SYNC_STATUS", "unknown")
|
||||
worktree = Path(
|
||||
os.getenv(
|
||||
"AIOS_GITOPS_CONTROLLER_WORKTREE",
|
||||
"/data/aios-gitops/worktrees/controller",
|
||||
)
|
||||
)
|
||||
|
||||
if not worktree.is_dir():
|
||||
return GitOpsStatus(
|
||||
available=False,
|
||||
service_status="unknown",
|
||||
workspace_mode="unavailable",
|
||||
workspace_branch=None,
|
||||
workspace_ref=None,
|
||||
workspace_dirty=None,
|
||||
candidate_branch=candidate_branch,
|
||||
release_channel=release_channel,
|
||||
release_version=release_version,
|
||||
sync_status=sync_status,
|
||||
details={"reason": "controller_worktree_unavailable"},
|
||||
)
|
||||
|
||||
inside_code, inside, _ = _run_git(worktree, "rev-parse", "--is-inside-work-tree")
|
||||
if inside_code != 0 or inside != "true":
|
||||
return GitOpsStatus(
|
||||
available=False,
|
||||
service_status="unknown",
|
||||
workspace_mode="invalid",
|
||||
workspace_branch=None,
|
||||
workspace_ref=None,
|
||||
workspace_dirty=None,
|
||||
candidate_branch=candidate_branch,
|
||||
release_channel=release_channel,
|
||||
release_version=release_version,
|
||||
sync_status=sync_status,
|
||||
details={"reason": "controller_worktree_invalid"},
|
||||
)
|
||||
|
||||
branch_code, branch, _ = _run_git(worktree, "branch", "--show-current")
|
||||
ref_code, ref_value, _ = _run_git(worktree, "rev-parse", "HEAD")
|
||||
dirty_code, dirty_value, _ = _run_git(worktree, "status", "--porcelain=v1")
|
||||
|
||||
service_status = "unknown"
|
||||
try:
|
||||
process = subprocess.run(
|
||||
["systemctl", "is-active", "aios-gitops-model-api"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=3,
|
||||
)
|
||||
service_status = process.stdout.strip() or "inactive"
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
service_status = "unavailable"
|
||||
|
||||
branch_value = branch if branch_code == 0 and branch else None
|
||||
return GitOpsStatus(
|
||||
available=True,
|
||||
service_status=service_status,
|
||||
workspace_mode="branch" if branch_value else "detached",
|
||||
workspace_branch=branch_value,
|
||||
workspace_ref=ref_value if ref_code == 0 else None,
|
||||
workspace_dirty=bool(dirty_value) if dirty_code == 0 else None,
|
||||
candidate_branch=candidate_branch,
|
||||
release_channel=release_channel,
|
||||
release_version=release_version,
|
||||
sync_status=sync_status,
|
||||
details={
|
||||
"controller_clean": dirty_code == 0 and not dirty_value,
|
||||
"controller_detached": branch_code == 0 and not branch,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,264 @@
|
||||
"""AIOS V2 execution orchestrator.
|
||||
|
||||
The orchestrator is the only component allowed to translate an approved AIOS
|
||||
task into an OpenClaw command or Hermes job. OpenClaw and Hermes remain pure
|
||||
execution runtimes and never own tenant policy or task lifecycle.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Mapping
|
||||
|
||||
from aios_hermes import HermesAdapter, HermesJob, HermesResult
|
||||
from aios_openclaw import (
|
||||
CommandState,
|
||||
OpenClawAdapter,
|
||||
OpenClawCommand,
|
||||
OpenClawResult,
|
||||
sign_command,
|
||||
)
|
||||
|
||||
from .control_plane import (
|
||||
AgentRecord,
|
||||
ControlPlaneService,
|
||||
RequestContext,
|
||||
TaskRecord,
|
||||
TaskState,
|
||||
)
|
||||
|
||||
|
||||
class OrchestrationError(RuntimeError):
|
||||
"""AIOS failed to dispatch or reconcile a runtime execution."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DispatchRecord:
|
||||
tenant_id: str
|
||||
task_id: str
|
||||
agent_id: str
|
||||
runtime: str
|
||||
runtime_execution_id: str
|
||||
state: str
|
||||
trace_id: str
|
||||
|
||||
|
||||
class ExecutionOrchestrator:
|
||||
"""Dispatch approved AIOS tasks to runtime adapters and reconcile results."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
control_plane: ControlPlaneService,
|
||||
*,
|
||||
openclaw: OpenClawAdapter | None = None,
|
||||
hermes: HermesAdapter | None = None,
|
||||
openclaw_signing_material_resolver=None,
|
||||
openclaw_key_id: str = "default",
|
||||
default_cell_id: str = "default",
|
||||
default_profile_id: str = "default",
|
||||
) -> None:
|
||||
self.control_plane = control_plane
|
||||
self.openclaw = openclaw
|
||||
self.hermes = hermes
|
||||
self.openclaw_signing_material_resolver = openclaw_signing_material_resolver
|
||||
self.openclaw_key_id = openclaw_key_id
|
||||
self.default_cell_id = default_cell_id
|
||||
self.default_profile_id = default_profile_id
|
||||
self._dispatches: dict[tuple[str, str], DispatchRecord] = {}
|
||||
|
||||
def dispatch(
|
||||
self,
|
||||
context: RequestContext,
|
||||
*,
|
||||
task_id: str,
|
||||
timeout_seconds: float | None = None,
|
||||
) -> DispatchRecord:
|
||||
task = self.control_plane.get_task(context, task_id)
|
||||
if task.state != TaskState.APPROVED:
|
||||
raise OrchestrationError("only APPROVED tasks may be dispatched")
|
||||
agent = self.control_plane.get_agent(context, task.agent_id)
|
||||
existing = self._dispatches.get((task.tenant_id, task.task_id))
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
running = self.control_plane.transition_task(
|
||||
context,
|
||||
task_id=task.task_id,
|
||||
target_state=TaskState.RUNNING,
|
||||
expected_revision=task.revision,
|
||||
)
|
||||
|
||||
runtime = agent.runtime.strip().lower()
|
||||
try:
|
||||
if runtime == "openclaw":
|
||||
record = self._dispatch_openclaw(
|
||||
context,
|
||||
agent,
|
||||
running,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
elif runtime == "hermes":
|
||||
record = self._dispatch_hermes(context, agent, running)
|
||||
else:
|
||||
raise OrchestrationError(f"unsupported runtime: {agent.runtime}")
|
||||
except Exception as exc:
|
||||
current = self.control_plane.get_task(context, task.task_id)
|
||||
if current.state == TaskState.RUNNING:
|
||||
self.control_plane.transition_task(
|
||||
context,
|
||||
task_id=task.task_id,
|
||||
target_state=TaskState.FAILED,
|
||||
expected_revision=current.revision,
|
||||
error={"code": "dispatch_failed", "message": str(exc)},
|
||||
)
|
||||
raise
|
||||
|
||||
self._dispatches[(task.tenant_id, task.task_id)] = record
|
||||
return record
|
||||
|
||||
def _dispatch_openclaw(
|
||||
self,
|
||||
context: RequestContext,
|
||||
agent: AgentRecord,
|
||||
task: TaskRecord,
|
||||
*,
|
||||
timeout_seconds: float | None,
|
||||
) -> DispatchRecord:
|
||||
if self.openclaw is None or self.openclaw_signing_material_resolver is None:
|
||||
raise OrchestrationError("OpenClaw runtime is not configured")
|
||||
key_material = self.openclaw_signing_material_resolver(
|
||||
task.tenant_id,
|
||||
self.openclaw_key_id,
|
||||
)
|
||||
command = OpenClawCommand.create(
|
||||
tenant_id=task.tenant_id,
|
||||
task_id=task.task_id,
|
||||
trace_id=context.trace_id,
|
||||
action=task.action,
|
||||
payload=task.payload,
|
||||
idempotency_key=task.idempotency_key or task.task_id,
|
||||
key_id=self.openclaw_key_id,
|
||||
)
|
||||
command = sign_command(command, key_material)
|
||||
result = self.openclaw.execute(command, timeout_seconds=timeout_seconds)
|
||||
self.reconcile_openclaw(context, result)
|
||||
return DispatchRecord(
|
||||
tenant_id=task.tenant_id,
|
||||
task_id=task.task_id,
|
||||
agent_id=agent.agent_id,
|
||||
runtime="openclaw",
|
||||
runtime_execution_id=command.command_id,
|
||||
state=result.state.value,
|
||||
trace_id=context.trace_id,
|
||||
)
|
||||
|
||||
def _dispatch_hermes(
|
||||
self,
|
||||
context: RequestContext,
|
||||
agent: AgentRecord,
|
||||
task: TaskRecord,
|
||||
) -> DispatchRecord:
|
||||
if self.hermes is None:
|
||||
raise OrchestrationError("Hermes runtime is not configured")
|
||||
metadata = dict(agent.metadata)
|
||||
job = HermesJob.create(
|
||||
tenant_id=task.tenant_id,
|
||||
cell_id=str(metadata.get("cell_id") or self.default_cell_id),
|
||||
profile_id=str(metadata.get("profile_id") or self.default_profile_id),
|
||||
task_id=task.task_id,
|
||||
trace_id=context.trace_id,
|
||||
action=task.action,
|
||||
payload=task.payload,
|
||||
idempotency_key=task.idempotency_key or task.task_id,
|
||||
priority=int(metadata.get("priority", 100)),
|
||||
)
|
||||
queued = self.hermes.enqueue(job)
|
||||
return DispatchRecord(
|
||||
tenant_id=task.tenant_id,
|
||||
task_id=task.task_id,
|
||||
agent_id=agent.agent_id,
|
||||
runtime="hermes",
|
||||
runtime_execution_id=queued.job_id,
|
||||
state=queued.state.value,
|
||||
trace_id=context.trace_id,
|
||||
)
|
||||
|
||||
def reconcile_openclaw(
|
||||
self,
|
||||
context: RequestContext,
|
||||
result: OpenClawResult,
|
||||
) -> TaskRecord:
|
||||
task = self.control_plane.get_task(context, result.task_id)
|
||||
if result.tenant_id != task.tenant_id:
|
||||
raise OrchestrationError("OpenClaw result tenant mismatch")
|
||||
if task.state != TaskState.RUNNING:
|
||||
return task
|
||||
if result.state == CommandState.COMPLETED:
|
||||
return self.control_plane.transition_task(
|
||||
context,
|
||||
task_id=task.task_id,
|
||||
target_state=TaskState.COMPLETED,
|
||||
expected_revision=task.revision,
|
||||
result=result.output or {},
|
||||
)
|
||||
if result.state in {
|
||||
CommandState.FAILED,
|
||||
CommandState.TIMED_OUT,
|
||||
CommandState.REJECTED,
|
||||
}:
|
||||
return self.control_plane.transition_task(
|
||||
context,
|
||||
task_id=task.task_id,
|
||||
target_state=TaskState.FAILED,
|
||||
expected_revision=task.revision,
|
||||
error=result.error or {"code": result.state.value.lower()},
|
||||
)
|
||||
if result.state == CommandState.CANCELLED:
|
||||
return self.control_plane.transition_task(
|
||||
context,
|
||||
task_id=task.task_id,
|
||||
target_state=TaskState.CANCELLED,
|
||||
expected_revision=task.revision,
|
||||
)
|
||||
return task
|
||||
|
||||
def reconcile_hermes(
|
||||
self,
|
||||
context: RequestContext,
|
||||
result: HermesResult,
|
||||
) -> TaskRecord:
|
||||
task = self.control_plane.get_task(context, result.task_id)
|
||||
if result.tenant_id != task.tenant_id:
|
||||
raise OrchestrationError("Hermes result tenant mismatch")
|
||||
if task.state != TaskState.RUNNING:
|
||||
return task
|
||||
if result.state.value == "COMPLETED":
|
||||
return self.control_plane.transition_task(
|
||||
context,
|
||||
task_id=task.task_id,
|
||||
target_state=TaskState.COMPLETED,
|
||||
expected_revision=task.revision,
|
||||
result=result.output or {},
|
||||
)
|
||||
if result.state.value == "FAILED":
|
||||
return self.control_plane.transition_task(
|
||||
context,
|
||||
task_id=task.task_id,
|
||||
target_state=TaskState.FAILED,
|
||||
expected_revision=task.revision,
|
||||
error=result.error or {"code": "hermes_failed"},
|
||||
)
|
||||
if result.state.value == "CANCELLED":
|
||||
return self.control_plane.transition_task(
|
||||
context,
|
||||
task_id=task.task_id,
|
||||
target_state=TaskState.CANCELLED,
|
||||
expected_revision=task.revision,
|
||||
)
|
||||
return task
|
||||
|
||||
def dispatch_snapshot(self, *, tenant_id: str) -> Mapping[str, DispatchRecord]:
|
||||
return {
|
||||
task_id: record
|
||||
for (record_tenant, task_id), record in self._dispatches.items()
|
||||
if record_tenant == tenant_id
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Durable single-process persistence for the AIOS V2 control plane.
|
||||
|
||||
The candidate uses an atomic SQLite snapshot so a gateway restart does not lose
|
||||
tenants, agents, task state, idempotency records, or audit events. The domain
|
||||
service remains storage-agnostic; a later HA backend can replace this module
|
||||
without changing API or runtime-adapter contracts.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from .control_plane import (
|
||||
AgentRecord,
|
||||
AuditEvent,
|
||||
ControlPlaneService,
|
||||
InMemoryControlPlaneStore,
|
||||
TaskRecord,
|
||||
TaskState,
|
||||
TenantRecord,
|
||||
)
|
||||
|
||||
|
||||
class SQLiteSnapshotRepository:
|
||||
"""Store one versioned control-plane snapshot in SQLite."""
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
def __init__(self, path: str | Path) -> None:
|
||||
self.path = Path(path).resolve()
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._lock = threading.RLock()
|
||||
self._initialize()
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
connection = sqlite3.connect(self.path, timeout=30.0)
|
||||
connection.execute("PRAGMA journal_mode=WAL")
|
||||
connection.execute("PRAGMA synchronous=FULL")
|
||||
connection.execute("PRAGMA foreign_keys=ON")
|
||||
return connection
|
||||
|
||||
def _initialize(self) -> None:
|
||||
with self._connect() as connection:
|
||||
connection.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS control_plane_snapshot (
|
||||
singleton_id INTEGER PRIMARY KEY CHECK (singleton_id = 1),
|
||||
schema_version INTEGER NOT NULL,
|
||||
payload_json TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
def load(self) -> dict[str, Any] | None:
|
||||
with self._lock, self._connect() as connection:
|
||||
row = connection.execute(
|
||||
"SELECT schema_version, payload_json FROM control_plane_snapshot WHERE singleton_id = 1"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
schema_version, payload_json = row
|
||||
if int(schema_version) != self.SCHEMA_VERSION:
|
||||
raise RuntimeError(
|
||||
f"unsupported control-plane snapshot version: {schema_version}"
|
||||
)
|
||||
decoded = json.loads(str(payload_json))
|
||||
if not isinstance(decoded, dict):
|
||||
raise RuntimeError("invalid control-plane snapshot")
|
||||
return decoded
|
||||
|
||||
def save(self, payload: dict[str, Any]) -> None:
|
||||
encoded = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
default=str,
|
||||
)
|
||||
with self._lock, self._connect() as connection:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
connection.execute(
|
||||
"""
|
||||
INSERT INTO control_plane_snapshot (
|
||||
singleton_id, schema_version, payload_json, updated_at
|
||||
) VALUES (1, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(singleton_id) DO UPDATE SET
|
||||
schema_version = excluded.schema_version,
|
||||
payload_json = excluded.payload_json,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(self.SCHEMA_VERSION, encoded),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
|
||||
class DurableControlPlaneService(ControlPlaneService):
|
||||
"""Control-plane service that persists after every successful mutation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repository: SQLiteSnapshotRepository,
|
||||
*,
|
||||
policy_evaluator=None,
|
||||
) -> None:
|
||||
self.repository = repository
|
||||
store = InMemoryControlPlaneStore()
|
||||
self._restore(store, repository.load())
|
||||
super().__init__(store, policy_evaluator=policy_evaluator)
|
||||
|
||||
@staticmethod
|
||||
def _restore(
|
||||
store: InMemoryControlPlaneStore,
|
||||
snapshot: dict[str, Any] | None,
|
||||
) -> None:
|
||||
if not snapshot:
|
||||
return
|
||||
for item in snapshot.get("tenants", []):
|
||||
record = TenantRecord(**item)
|
||||
store.tenants[record.tenant_id] = record
|
||||
for item in snapshot.get("agents", []):
|
||||
normalized = dict(item)
|
||||
normalized["capabilities"] = tuple(normalized.get("capabilities", ()))
|
||||
record = AgentRecord(**normalized)
|
||||
store.agents[record.agent_id] = record
|
||||
for item in snapshot.get("tasks", []):
|
||||
normalized = dict(item)
|
||||
normalized["state"] = TaskState(normalized["state"])
|
||||
record = TaskRecord(**normalized)
|
||||
store.tasks[record.task_id] = record
|
||||
store.audit_events.extend(AuditEvent(**item) for item in snapshot.get("audit", []))
|
||||
for item in snapshot.get("idempotency", []):
|
||||
store.idempotency[(item["tenant_id"], item["key"])] = (
|
||||
item["task_id"],
|
||||
item["digest"],
|
||||
)
|
||||
|
||||
def _snapshot(self) -> dict[str, Any]:
|
||||
with self.store.lock:
|
||||
return {
|
||||
"schema_version": self.repository.SCHEMA_VERSION,
|
||||
"tenants": [asdict(item) for item in self.store.tenants.values()],
|
||||
"agents": [asdict(item) for item in self.store.agents.values()],
|
||||
"tasks": [
|
||||
{**asdict(item), "state": item.state.value}
|
||||
for item in self.store.tasks.values()
|
||||
],
|
||||
"audit": [asdict(item) for item in self.store.audit_events],
|
||||
"idempotency": [
|
||||
{
|
||||
"tenant_id": tenant_id,
|
||||
"key": key,
|
||||
"task_id": task_id,
|
||||
"digest": digest,
|
||||
}
|
||||
for (tenant_id, key), (task_id, digest)
|
||||
in self.store.idempotency.items()
|
||||
],
|
||||
}
|
||||
|
||||
def _mutate(self, operation: Callable[[], Any]) -> Any:
|
||||
with self.store.lock:
|
||||
result = operation()
|
||||
self.repository.save(self._snapshot())
|
||||
return result
|
||||
|
||||
def register_tenant(self, *args, **kwargs):
|
||||
return self._mutate(lambda: super(DurableControlPlaneService, self).register_tenant(*args, **kwargs))
|
||||
|
||||
def register_agent(self, *args, **kwargs):
|
||||
return self._mutate(lambda: super(DurableControlPlaneService, self).register_agent(*args, **kwargs))
|
||||
|
||||
def create_task(self, *args, **kwargs):
|
||||
return self._mutate(lambda: super(DurableControlPlaneService, self).create_task(*args, **kwargs))
|
||||
|
||||
def transition_task(self, *args, **kwargs):
|
||||
return self._mutate(lambda: super(DurableControlPlaneService, self).transition_task(*args, **kwargs))
|
||||
@@ -1,141 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove two real central-decision to Gateway-dispatch to Codex lifecycle cycles."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from health_probe_common import (
|
||||
ProbeError, gateway_base, load_config, read_token, request_json, state_version,
|
||||
)
|
||||
|
||||
|
||||
REQUIRED_MARKERS = {"CODEX_AUTO_CLAIM", "TASK_STARTED", "TASK_EXECUTED", "TASK_ENDED", "EVENT_POSTED"}
|
||||
|
||||
|
||||
def marker_counts(path: Path, command_id: str) -> dict[str, int]:
|
||||
counts = {marker: 0 for marker in REQUIRED_MARKERS}
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
marker = record.get("marker")
|
||||
if record.get("command_id") == command_id and marker in counts:
|
||||
counts[marker] += 1
|
||||
return counts
|
||||
|
||||
|
||||
def watcher_status(database: Path, command_id: str) -> str | None:
|
||||
with sqlite3.connect(f"file:{database}?mode=ro", uri=True, timeout=2) as db:
|
||||
row = db.execute("SELECT status FROM commands WHERE command_id=?", (command_id,)).fetchone()
|
||||
return None if row is None else str(row[0])
|
||||
|
||||
|
||||
def wait_for_cycle(config: dict[str, object], token: str, cycle: int) -> tuple[str, str]:
|
||||
endpoint = str(config["gateway_event_url"])
|
||||
project = str(config["project_key"])
|
||||
event_id = f"a0-cycle-{cycle}-" + uuid.uuid4().hex
|
||||
work_unit_id = f"A0-CODEX-CYCLE-{cycle}-" + uuid.uuid4().hex[:16]
|
||||
event = {
|
||||
"schema_version": 1,
|
||||
"project_key": project,
|
||||
"contract_commit": config["contract_commit"],
|
||||
"event_id": event_id,
|
||||
"source_agent": "system",
|
||||
"event_type": "a0_unattended_acceptance_requested",
|
||||
"work_unit_id": work_unit_id,
|
||||
"status": "READY",
|
||||
"state_version": state_version(endpoint, project, token),
|
||||
"implementation_commit": None,
|
||||
"result_commit": None,
|
||||
"blocker": None,
|
||||
"evidence_refs": [],
|
||||
"payload": {
|
||||
"acceptance_contract": {
|
||||
"verdict": "DISPATCH",
|
||||
"work_unit_id": work_unit_id,
|
||||
"track_id": "A0_AUTOMATION",
|
||||
"gate_id": "A0_AUTOMATION_CLOSED",
|
||||
"primary_agent": "codex",
|
||||
"fallback_agent": "openclaw",
|
||||
"clean_base_commit": config["probe_clean_base_commit"],
|
||||
"allowed_paths": [],
|
||||
"required_checks": ["acceptance_probe"],
|
||||
"forbidden_operations": ["production_mutation", "git_mutation"],
|
||||
"instruction": "Run the fixed side-effect-free acceptance probe only.",
|
||||
}
|
||||
},
|
||||
}
|
||||
request_json(endpoint, token, method="POST", payload=event)
|
||||
deadline = time.monotonic() + int(config["cycle_timeout_seconds"])
|
||||
channel = None
|
||||
while time.monotonic() < deadline:
|
||||
_, status = request_json(endpoint + "/" + event_id, token)
|
||||
if status.get("status") == "TRIGGER_FAILED":
|
||||
raise ProbeError("central_trigger_failed")
|
||||
if status.get("status") == "DECIDED" and status.get("decision_applied") is True:
|
||||
channel = status.get("trigger_channel")
|
||||
break
|
||||
time.sleep(float(config["poll_interval_seconds"]))
|
||||
if channel not in {"workspace_agent", "responses_api"}:
|
||||
raise ProbeError("central_decision_timeout")
|
||||
|
||||
database = Path(str(config["codex_database_path"]))
|
||||
audit = Path(str(config["codex_audit_log"]))
|
||||
while time.monotonic() < deadline:
|
||||
if database.exists() and audit.exists() and watcher_status(database, work_unit_id) == "COMPLETED":
|
||||
counts = marker_counts(audit, work_unit_id)
|
||||
if all(counts[marker] == 1 for marker in REQUIRED_MARKERS):
|
||||
for suffix in ("started", "ended"):
|
||||
_, lifecycle = request_json(endpoint + "/" + work_unit_id + ":" + suffix, token)
|
||||
if lifecycle.get("event_id") != work_unit_id + ":" + suffix:
|
||||
raise ProbeError("gateway_lifecycle_missing")
|
||||
return event_id, str(channel)
|
||||
time.sleep(float(config["poll_interval_seconds"]))
|
||||
raise ProbeError("gateway_dispatch_or_codex_lifecycle_timeout")
|
||||
|
||||
|
||||
def project_summaries_since(database: Path, project: str, since: int) -> int:
|
||||
with sqlite3.connect(f"file:{database}?mode=ro", uri=True, timeout=2) as db:
|
||||
row = db.execute(
|
||||
"SELECT COUNT(*) FROM lifecycle WHERE project_key=? AND kind='PROJECT_SUMMARY' AND created_at>=?",
|
||||
(project, since),
|
||||
).fetchone()
|
||||
return 0 if row is None else int(row[0])
|
||||
|
||||
|
||||
def run(config_path: Path) -> int:
|
||||
config = load_config(config_path)
|
||||
endpoint = str(config["gateway_event_url"])
|
||||
gateway_base(endpoint)
|
||||
token = read_token(Path(str(config["gateway_token_file"])))
|
||||
started = int(time.time())
|
||||
results = [wait_for_cycle(config, token, cycle) for cycle in (1, 2)]
|
||||
if len({item[0] for item in results}) != 2:
|
||||
raise ProbeError("duplicate_cycle_event")
|
||||
if project_summaries_since(
|
||||
Path(str(config["gateway_database_path"])), str(config["project_key"]), started
|
||||
) < 2:
|
||||
raise ProbeError("project_summary_not_persisted")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
return run(args.config)
|
||||
except (KeyError, OSError, ProbeError, sqlite3.Error, ValueError):
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,172 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove a real central-model decision through the authenticated Gateway."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
import uuid
|
||||
|
||||
from health_probe_common import (
|
||||
ProbeError, atomic_receipt, gateway_base, load_config, read_token,
|
||||
request_json, state_version,
|
||||
)
|
||||
|
||||
PENDING_PATH = Path("/run/aios-health/central-gpt.pending.json")
|
||||
|
||||
|
||||
def _load_pending(
|
||||
path: Path, contract: str, *, owners: set[int] | None = None
|
||||
) -> tuple[str, int] | None:
|
||||
allowed_owners = {0} if owners is None else owners
|
||||
if not path.exists():
|
||||
return None
|
||||
info = path.lstat()
|
||||
if (
|
||||
not path.is_absolute()
|
||||
or stat.S_ISLNK(info.st_mode)
|
||||
or not stat.S_ISREG(info.st_mode)
|
||||
or stat.S_IMODE(info.st_mode) & ~0o640
|
||||
or (os.name == "posix" and info.st_uid not in allowed_owners)
|
||||
or path.resolve(strict=True) != Path(os.path.abspath(path))
|
||||
):
|
||||
raise ProbeError("unsafe_pending_marker")
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict) or set(value) != {"contract_commit", "event_id", "state_version"}:
|
||||
raise ProbeError("invalid_pending_marker")
|
||||
event_id = value["event_id"]
|
||||
version = value["state_version"]
|
||||
if (
|
||||
value["contract_commit"] != contract
|
||||
or not isinstance(event_id, str)
|
||||
or not event_id.startswith("health-central-")
|
||||
or len(event_id) != 47
|
||||
or not isinstance(version, int)
|
||||
or version < 0
|
||||
):
|
||||
raise ProbeError("invalid_pending_marker")
|
||||
return event_id, version
|
||||
|
||||
|
||||
def _clear_pending(path: Path) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
info = path.lstat()
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode):
|
||||
raise ProbeError("unsafe_pending_marker")
|
||||
path.unlink()
|
||||
|
||||
|
||||
def _safe_reason(exc: BaseException) -> str:
|
||||
reason = str(exc)
|
||||
if reason and len(reason) <= 64 and all(char.islower() or char.isdigit() or char == "_" for char in reason):
|
||||
return reason
|
||||
return "probe_error"
|
||||
|
||||
|
||||
def run(config_path: Path, pending_path: Path = PENDING_PATH) -> int:
|
||||
config = load_config(config_path)
|
||||
endpoint = config["gateway_event_url"]
|
||||
gateway_base(endpoint)
|
||||
token = read_token(Path(config["gateway_token_file"]))
|
||||
project = config["project_key"]
|
||||
contract = config["contract_commit"]
|
||||
pending = _load_pending(pending_path, contract)
|
||||
is_new = pending is None
|
||||
if is_new:
|
||||
event_id = "health-central-" + uuid.uuid4().hex
|
||||
version = state_version(endpoint, project, token)
|
||||
atomic_receipt(
|
||||
pending_path,
|
||||
{"contract_commit": contract, "event_id": event_id, "state_version": version},
|
||||
)
|
||||
else:
|
||||
event_id, version = pending
|
||||
event = {
|
||||
"schema_version": 1,
|
||||
"project_key": project,
|
||||
"contract_commit": contract,
|
||||
"event_id": event_id,
|
||||
"source_agent": "system",
|
||||
"event_type": "central_gpt_health_probe",
|
||||
"work_unit_id": "CENTRAL-GPT-HEALTH",
|
||||
"status": "READY",
|
||||
"state_version": version,
|
||||
"implementation_commit": None,
|
||||
"result_commit": None,
|
||||
"blocker": None,
|
||||
"evidence_refs": [],
|
||||
"payload": {"probe": "side_effect_free", "nonce": event_id},
|
||||
}
|
||||
should_post = is_new
|
||||
if not is_new:
|
||||
try:
|
||||
_, observed = request_json(endpoint + "/" + event_id, token)
|
||||
except ProbeError as exc:
|
||||
if str(exc) != "gateway_http_404":
|
||||
raise
|
||||
should_post = True
|
||||
else:
|
||||
observed = None
|
||||
if should_post:
|
||||
try:
|
||||
status, _ = request_json(endpoint, token, method="POST", payload=event)
|
||||
except ProbeError as exc:
|
||||
if str(exc) == "gateway_fail_closed_409":
|
||||
_clear_pending(pending_path)
|
||||
raise
|
||||
if status not in {200, 202}:
|
||||
raise ProbeError("event_not_accepted")
|
||||
observed = None
|
||||
deadline = time.monotonic() + int(config["central_timeout_seconds"])
|
||||
while time.monotonic() < deadline:
|
||||
_, observed = request_json(endpoint + "/" + event_id, token)
|
||||
if observed.get("status") in {"DECIDED", "TRIGGER_FAILED", "DEAD_LETTER"}:
|
||||
break
|
||||
time.sleep(float(config["poll_interval_seconds"]))
|
||||
if isinstance(observed, dict) and observed.get("status") == "DEAD_LETTER":
|
||||
_clear_pending(pending_path)
|
||||
raise ProbeError("central_event_dead_letter")
|
||||
if not isinstance(observed, dict) or observed.get("status") != "DECIDED" or observed.get("decision_applied") is not True:
|
||||
raise ProbeError("central_decision_not_proven")
|
||||
channel = observed.get("trigger_channel")
|
||||
if channel not in {"workspace_agent", "responses_api", "codex_local"}:
|
||||
raise ProbeError("central_channel_not_proven")
|
||||
_clear_pending(pending_path)
|
||||
now = int(time.time())
|
||||
atomic_receipt(
|
||||
Path(config["central_receipt_path"]),
|
||||
{
|
||||
"subject": "central_gpt",
|
||||
"health": "HEALTHY",
|
||||
"live_probe_succeeded": True,
|
||||
"observed_at_epoch": now,
|
||||
"event_id": event_id,
|
||||
"decision_applied": True,
|
||||
"state_version_after": observed["state_version_after"],
|
||||
"workspace_agent": {"validated_live": channel == "workspace_agent"},
|
||||
"responses_api": {"validated_live": channel == "responses_api"},
|
||||
"codex_local": {"validated_live": channel == "codex_local"},
|
||||
},
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
return run(args.config)
|
||||
except (KeyError, OSError, ProbeError, ValueError) as exc:
|
||||
print(f"CENTRAL_GPT_PROBE=BLOCKED reason={_safe_reason(exc)}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,136 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from services.aios_orchestrator_gateway.codex_local import CodexLocalClient
|
||||
from services.aios_orchestrator_gateway.config import Config
|
||||
from services.aios_orchestrator_gateway.models import Event
|
||||
from services.aios_orchestrator_gateway.securefs import read_secure_json
|
||||
|
||||
|
||||
LOG = logging.getLogger("aios-codex-central-decision")
|
||||
|
||||
|
||||
class CodexCentralWorker:
|
||||
def __init__(self, config: Config, client: CodexLocalClient | None = None):
|
||||
if config.central_channel != "codex_local" or config.codex_queue_dir is None:
|
||||
raise RuntimeError("codex_local_not_selected")
|
||||
self.config = config
|
||||
self.client = client or CodexLocalClient()
|
||||
self.root = config.codex_queue_dir
|
||||
self.pending = self.root / "pending"
|
||||
self.processing = self.root / "processing"
|
||||
self.processed = self.root / "processed"
|
||||
self.dead = self.root / "dead-letter"
|
||||
|
||||
def recover_interrupted(self) -> None:
|
||||
for path in sorted(self.processing.glob("*.json")):
|
||||
target = self.pending / path.name
|
||||
if target.exists():
|
||||
path.unlink()
|
||||
else:
|
||||
path.replace(target)
|
||||
|
||||
def run_one(self) -> bool:
|
||||
candidates = sorted(self.pending.glob("*.json"))
|
||||
if not candidates:
|
||||
return False
|
||||
source = candidates[0]
|
||||
claimed = self.processing / source.name
|
||||
try:
|
||||
source.replace(claimed)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
wrapper = read_secure_json(claimed, expected_uid=None)
|
||||
attempts = wrapper.get("attempts")
|
||||
next_attempt_at = wrapper.get("next_attempt_at")
|
||||
event = Event.from_dict(wrapper.get("event"))
|
||||
if (
|
||||
not isinstance(attempts, int) or attempts < 0
|
||||
or not isinstance(next_attempt_at, (int, float)) or next_attempt_at < 0
|
||||
):
|
||||
self._dead_letter(claimed)
|
||||
return True
|
||||
if next_attempt_at > time.time():
|
||||
claimed.replace(self.pending / claimed.name)
|
||||
return False
|
||||
if event.contract_commit != self.config.contract_commit:
|
||||
self._dead_letter(claimed)
|
||||
return True
|
||||
try:
|
||||
decision = self.client.decide(event)
|
||||
self._submit(decision.to_dict(), event.event_id)
|
||||
except Exception as exc:
|
||||
LOG.warning("codex_decision_failed category=%s", type(exc).__name__)
|
||||
wrapper["attempts"] = attempts + 1
|
||||
wrapper["next_attempt_at"] = int(time.time()) + self.config.retry_base_seconds
|
||||
self._rewrite(claimed, wrapper)
|
||||
if wrapper["attempts"] > self.config.max_retries:
|
||||
self._dead_letter(claimed)
|
||||
else:
|
||||
claimed.replace(self.pending / claimed.name)
|
||||
return True
|
||||
claimed.replace(self.processed / claimed.name)
|
||||
return True
|
||||
|
||||
def _submit(self, decision: dict, event_id: str) -> None:
|
||||
body = json.dumps(decision, separators=(",", ":"), sort_keys=True).encode()
|
||||
url = f"http://{self.config.host}:{self.config.port}/v1/orchestrator/decisions"
|
||||
request = urllib.request.Request(
|
||||
url, data=body, method="POST",
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.config.callback_token()}",
|
||||
"Content-Type": "application/json",
|
||||
"Idempotency-Key": event_id,
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
if response.status not in {200, 202}:
|
||||
raise RuntimeError("gateway_decision_rejected")
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise RuntimeError("gateway_decision_unavailable") from exc
|
||||
|
||||
@staticmethod
|
||||
def _rewrite(path: Path, value: dict) -> None:
|
||||
temp = path.with_name(f".{path.name}.tmp-{os.getpid()}")
|
||||
fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o640)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(value, handle, separators=(",", ":"), sort_keys=True)
|
||||
handle.flush(); os.fsync(handle.fileno())
|
||||
os.replace(temp, path)
|
||||
|
||||
def _dead_letter(self, path: Path) -> None:
|
||||
target = self.dead / path.name
|
||||
if target.exists():
|
||||
path.unlink()
|
||||
else:
|
||||
path.replace(target)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--gateway-config", required=True, type=Path)
|
||||
parser.add_argument("--once", action="store_true")
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s %(message)s")
|
||||
worker = CodexCentralWorker(Config.from_file(args.gateway_config))
|
||||
worker.recover_interrupted()
|
||||
if args.once:
|
||||
worker.run_one()
|
||||
return 0
|
||||
while True:
|
||||
if not worker.run_one():
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,72 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from services.aios_orchestrator_gateway.codex_local import CodexLocalClient
|
||||
from services.aios_orchestrator_gateway.config import Config
|
||||
from services.aios_orchestrator_gateway.models import Event
|
||||
|
||||
|
||||
def _atomic_receipt(path: Path, value: dict) -> None:
|
||||
path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
fd, temp = tempfile.mkstemp(prefix=".codex-health-", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(value, handle, separators=(",", ":"), sort_keys=True)
|
||||
handle.flush(); os.fsync(handle.fileno())
|
||||
os.chmod(temp, 0o600)
|
||||
os.replace(temp, path)
|
||||
finally:
|
||||
if os.path.exists(temp): os.unlink(temp)
|
||||
|
||||
|
||||
def run(config_path: Path, receipt: Path) -> int:
|
||||
config = Config.from_file(config_path)
|
||||
if config.central_channel != "codex_local":
|
||||
raise RuntimeError("codex_local_not_selected")
|
||||
event_id = "codex-health-" + uuid.uuid4().hex
|
||||
event = Event.from_dict({
|
||||
"schema_version": 1, "project_key": config.project_key,
|
||||
"contract_commit": config.contract_commit, "event_id": event_id,
|
||||
"source_agent": "system", "event_type": "codex_local_health_probe",
|
||||
"work_unit_id": "CODEX-LOCAL-HEALTH", "status": "READY",
|
||||
"state_version": 0, "implementation_commit": None,
|
||||
"result_commit": None, "blocker": "health_probe_has_no_release_evidence",
|
||||
"evidence_refs": [], "payload": {"allowed_work_unit_ids": []},
|
||||
})
|
||||
decision = CodexLocalClient().decide(event)
|
||||
if decision.verdict == "DISPATCH" or decision.next_work_units:
|
||||
raise RuntimeError("unsafe_health_decision")
|
||||
_atomic_receipt(receipt, {
|
||||
"subject": "codex_local", "health": "HEALTHY",
|
||||
"codex_login_status": 0, "codex_exec": 0,
|
||||
"decision_schema_valid": True, "observed_at_epoch": int(time.time()),
|
||||
})
|
||||
print("CODEX_CENTRAL_HEALTH=PASS")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--gateway-config", required=True, type=Path)
|
||||
parser.add_argument("--receipt", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
return run(args.gateway_config, args.receipt)
|
||||
except Exception as exc:
|
||||
reason = str(exc)
|
||||
safe = reason if reason.replace("_", "").isalnum() and len(reason) <= 64 else type(exc).__name__
|
||||
print(f"CODEX_CENTRAL_HEALTH=BLOCKED reason={safe}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,113 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Queue one fixed no-side-effect command and prove the watcher lifecycle."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import stat
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from health_probe_common import (
|
||||
ProbeError, atomic_receipt, gateway_base, load_config, read_token,
|
||||
request_json, state_version,
|
||||
)
|
||||
|
||||
|
||||
MARKERS = {"CODEX_AUTO_CLAIM", "TASK_STARTED", "TASK_EXECUTED", "TASK_ENDED", "EVENT_POSTED"}
|
||||
|
||||
|
||||
def _write_command(path: Path, value: dict[str, object]) -> None:
|
||||
directory = path.parent
|
||||
info = directory.lstat()
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) & 0o002:
|
||||
raise ProbeError("unsafe_command_queue")
|
||||
data = (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode()
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o640)
|
||||
try:
|
||||
os.write(fd, data)
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _status(database: Path, command_id: str) -> str | None:
|
||||
with sqlite3.connect(f"file:{database}?mode=ro", uri=True, timeout=2) as db:
|
||||
row = db.execute("SELECT status FROM commands WHERE command_id=?", (command_id,)).fetchone()
|
||||
return None if row is None else str(row[0])
|
||||
|
||||
|
||||
def _markers(path: Path, command_id: str) -> dict[str, int]:
|
||||
result = {name: 0 for name in MARKERS}
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if item.get("command_id") == command_id and item.get("marker") in result:
|
||||
result[item["marker"]] += 1
|
||||
return result
|
||||
|
||||
|
||||
def run(config_path: Path) -> int:
|
||||
config = load_config(config_path)
|
||||
endpoint = config["gateway_event_url"]
|
||||
gateway_base(endpoint)
|
||||
token = read_token(Path(config["gateway_token_file"]))
|
||||
command_id = "health-codex-" + uuid.uuid4().hex
|
||||
version = state_version(endpoint, config["project_key"], token)
|
||||
command = {
|
||||
"schema_version": 1,
|
||||
"command_id": command_id,
|
||||
"project_key": config["project_key"],
|
||||
"work_unit_id": command_id,
|
||||
"contract_commit": config["contract_commit"],
|
||||
"state_version": version,
|
||||
"execution": {"kind": "acceptance_probe", "input": "fixed-side-effect-free-health-probe-v1"},
|
||||
}
|
||||
_write_command(Path(config["codex_command_queue"]) / f"{command_id}.json", command)
|
||||
deadline = time.monotonic() + int(config["codex_timeout_seconds"])
|
||||
database = Path(config["codex_database_path"])
|
||||
audit = Path(config["codex_audit_log"])
|
||||
while time.monotonic() < deadline:
|
||||
if database.exists() and audit.exists() and _status(database, command_id) == "COMPLETED":
|
||||
counts = _markers(audit, command_id)
|
||||
if all(counts[name] == 1 for name in MARKERS):
|
||||
for suffix in ("started", "ended"):
|
||||
_, event = request_json(endpoint + "/" + command_id + ":" + suffix, token)
|
||||
if event.get("event_id") != command_id + ":" + suffix:
|
||||
raise ProbeError("gateway_lifecycle_missing")
|
||||
atomic_receipt(
|
||||
Path(config["codex_receipt_path"]),
|
||||
{
|
||||
"subject": "codex",
|
||||
"health": "HEALTHY",
|
||||
"live_probe_succeeded": True,
|
||||
"observed_at_epoch": int(time.time()),
|
||||
"command_id": command_id,
|
||||
"lifecycle_complete": True,
|
||||
"duplicate_side_effects": False,
|
||||
},
|
||||
)
|
||||
return 0
|
||||
time.sleep(float(config["poll_interval_seconds"]))
|
||||
raise ProbeError("codex_lifecycle_not_proven")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
return run(args.config)
|
||||
except (KeyError, OSError, ProbeError, sqlite3.Error, ValueError):
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,940 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Durable GPT + one-executor quorum and failover controller.
|
||||
|
||||
The controller persists decisions before dispatch and never stores credentials.
|
||||
Health is accepted only from metadata-only, live-probe receipts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, Mapping
|
||||
|
||||
|
||||
ALLOWED_LIFECYCLE_EVENTS = frozenset(
|
||||
{
|
||||
"TASK_DISPATCHED",
|
||||
"TASK_STARTED",
|
||||
"TASK_ENDED",
|
||||
"DECISION_APPLIED",
|
||||
"PROJECT_SUMMARY",
|
||||
"PROJECT_COMPLETED",
|
||||
}
|
||||
)
|
||||
EXECUTORS = ("openclaw", "codex")
|
||||
SECRET_KEYS = frozenset(
|
||||
{
|
||||
"secret",
|
||||
"token",
|
||||
"password",
|
||||
"api_key",
|
||||
"authorization",
|
||||
"credential",
|
||||
"private_key",
|
||||
"environment",
|
||||
"env",
|
||||
"argv",
|
||||
"stdout",
|
||||
"stderr",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class Health(str, Enum):
|
||||
HEALTHY = "HEALTHY"
|
||||
UNHEALTHY = "UNHEALTHY"
|
||||
UNKNOWN = "UNKNOWN"
|
||||
|
||||
|
||||
class ControllerError(RuntimeError):
|
||||
"""Fail-closed controller error."""
|
||||
|
||||
|
||||
class EventConflict(ControllerError):
|
||||
"""An event id was reused with a different payload."""
|
||||
|
||||
|
||||
class StaleStateVersion(ControllerError):
|
||||
"""A dispatch attempted to overwrite a newer task state."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QuorumDecision:
|
||||
allowed: bool
|
||||
priority: str
|
||||
status: str
|
||||
active_executor: str | None
|
||||
standby_executor: str | None
|
||||
repair_required: bool
|
||||
|
||||
|
||||
def canonical_json(value: Any) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
||||
|
||||
|
||||
def digest_json(value: Any) -> str:
|
||||
return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _contains_forbidden_key(value: Any) -> bool:
|
||||
if isinstance(value, Mapping):
|
||||
for key, child in value.items():
|
||||
normalized = str(key).strip().lower().replace("-", "_")
|
||||
if normalized in SECRET_KEYS or _contains_forbidden_key(child):
|
||||
return True
|
||||
elif isinstance(value, list):
|
||||
return any(_contains_forbidden_key(item) for item in value)
|
||||
return False
|
||||
|
||||
|
||||
def validate_metadata_only(value: Mapping[str, Any]) -> None:
|
||||
if _contains_forbidden_key(value):
|
||||
raise ControllerError("secret-bearing or output-bearing fields are forbidden")
|
||||
|
||||
|
||||
def atomic_json_write(path: Path, value: Mapping[str, Any], mode: int = 0o600) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
validate_secure_directory(path.parent)
|
||||
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
|
||||
try:
|
||||
os.fchmod(fd, mode)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
handle.write(canonical_json(value))
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
if os.path.exists(temporary):
|
||||
os.unlink(temporary)
|
||||
|
||||
|
||||
def validate_secure_directory(path: Path, *, expected_uid: int | None = None) -> None:
|
||||
info = path.lstat()
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):
|
||||
raise ControllerError(f"unsafe directory type rejected: {path}")
|
||||
if os.name == "posix" and path.resolve(strict=True) != Path(os.path.abspath(path)):
|
||||
raise ControllerError(f"symlinked directory component rejected: {path}")
|
||||
if os.name == "posix" and info.st_mode & 0o022:
|
||||
raise ControllerError(f"group/world-writable directory rejected: {path}")
|
||||
if expected_uid is not None and info.st_uid != expected_uid:
|
||||
raise ControllerError(f"unexpected owner for directory {path}")
|
||||
|
||||
|
||||
def read_secure_json(path: Path, *, expected_uid: int | None = None) -> dict[str, Any]:
|
||||
info = path.lstat()
|
||||
if stat.S_ISLNK(info.st_mode):
|
||||
raise ControllerError(f"symlink rejected: {path}")
|
||||
if not stat.S_ISREG(info.st_mode):
|
||||
raise ControllerError(f"not a regular file: {path}")
|
||||
if os.name == "posix" and info.st_mode & 0o022:
|
||||
raise ControllerError(f"group/world-writable file rejected: {path}")
|
||||
validate_secure_directory(path.parent, expected_uid=expected_uid)
|
||||
if os.name == "posix" and path.resolve(strict=True) != Path(os.path.abspath(path)):
|
||||
raise ControllerError(f"symlinked file component rejected: {path}")
|
||||
if expected_uid is not None and info.st_uid != expected_uid:
|
||||
raise ControllerError(f"unexpected owner for {path}")
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
value = json.load(handle)
|
||||
if not isinstance(value, dict):
|
||||
raise ControllerError(f"JSON object required: {path}")
|
||||
validate_metadata_only(value)
|
||||
return value
|
||||
|
||||
|
||||
def read_gateway_token(path: Path) -> str:
|
||||
if not path.is_absolute():
|
||||
raise ControllerError("gateway token path must be absolute")
|
||||
try:
|
||||
info = path.lstat()
|
||||
except FileNotFoundError as exc:
|
||||
raise ControllerError("gateway token unavailable") from exc
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode):
|
||||
raise ControllerError("gateway token must be a regular non-symlink file")
|
||||
if stat.S_IMODE(info.st_mode) & ~0o640:
|
||||
raise ControllerError("gateway token mode exceeds 0640")
|
||||
if os.name == "posix" and info.st_uid not in {0, os.geteuid()}:
|
||||
raise ControllerError("gateway token owner is not approved")
|
||||
if os.name == "posix" and path.resolve(strict=True) != Path(os.path.abspath(path)):
|
||||
raise ControllerError("gateway token has a symlinked path component")
|
||||
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
||||
fd = os.open(path, flags)
|
||||
try:
|
||||
opened = os.fstat(fd)
|
||||
if not stat.S_ISREG(opened.st_mode) or opened.st_ino != info.st_ino or opened.st_dev != info.st_dev:
|
||||
raise ControllerError("gateway token changed during validation")
|
||||
data = os.read(fd, 16_385)
|
||||
if os.read(fd, 1):
|
||||
raise ControllerError("gateway token too large")
|
||||
finally:
|
||||
os.close(fd)
|
||||
if not data or len(data) > 16_384 or b"\n" in data or b"\r" in data:
|
||||
raise ControllerError("gateway token format rejected")
|
||||
try:
|
||||
token = data.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ControllerError("gateway token encoding rejected") from exc
|
||||
if not token:
|
||||
raise ControllerError("gateway token unavailable")
|
||||
return token
|
||||
|
||||
|
||||
def validate_health_receipt(
|
||||
receipt: Mapping[str, Any],
|
||||
subject: str,
|
||||
*,
|
||||
now: float | None = None,
|
||||
max_age_seconds: int = 60,
|
||||
) -> Health:
|
||||
validate_metadata_only(receipt)
|
||||
if receipt.get("subject") != subject:
|
||||
return Health.UNKNOWN
|
||||
try:
|
||||
health = Health(str(receipt.get("health", "UNKNOWN")))
|
||||
except ValueError:
|
||||
return Health.UNKNOWN
|
||||
if health is not Health.HEALTHY:
|
||||
return health
|
||||
if receipt.get("live_probe_succeeded") is not True:
|
||||
return Health.UNKNOWN
|
||||
observed_at = receipt.get("observed_at_epoch")
|
||||
if not isinstance(observed_at, (int, float)):
|
||||
return Health.UNKNOWN
|
||||
now = time.time() if now is None else now
|
||||
if observed_at > now + 5 or now - observed_at > max_age_seconds:
|
||||
return Health.UNKNOWN
|
||||
if subject == "central_gpt":
|
||||
workspace = receipt.get("workspace_agent")
|
||||
responses = receipt.get("responses_api")
|
||||
workspace_ok = isinstance(workspace, Mapping) and workspace.get("validated_live") is True
|
||||
responses_ok = isinstance(responses, Mapping) and responses.get("validated_live") is True
|
||||
if not (workspace_ok or responses_ok):
|
||||
return Health.UNKNOWN
|
||||
return Health.HEALTHY
|
||||
|
||||
|
||||
def choose_executor(workload: str) -> str:
|
||||
if workload in {"complex_code", "security_boundary", "architecture", "complex_code_security_architecture"}:
|
||||
return "codex"
|
||||
return "openclaw"
|
||||
|
||||
|
||||
def evaluate_quorum(
|
||||
central_gpt: Health,
|
||||
openclaw: Health,
|
||||
codex: Health,
|
||||
workload: str,
|
||||
) -> QuorumDecision:
|
||||
if central_gpt is not Health.HEALTHY:
|
||||
status = "BLOCKED_CENTRAL_GPT_UNKNOWN" if central_gpt is Health.UNKNOWN else "BLOCKED_CENTRAL_GPT_UNHEALTHY"
|
||||
return QuorumDecision(False, "P0", status, None, None, True)
|
||||
|
||||
healthy = [name for name, health in (("openclaw", openclaw), ("codex", codex)) if health is Health.HEALTHY]
|
||||
if not healthy:
|
||||
has_unknown = Health.UNKNOWN in {openclaw, codex}
|
||||
status = "BLOCKED_EXECUTOR_HEALTH_UNKNOWN" if has_unknown else "BLOCKED_NO_HEALTHY_EXECUTOR"
|
||||
return QuorumDecision(False, "P1", status, None, None, True)
|
||||
|
||||
if len(healthy) == 1:
|
||||
active = healthy[0]
|
||||
standby = "codex" if active == "openclaw" else "openclaw"
|
||||
return QuorumDecision(True, "P3", "EXECUTOR_QUORUM_PASS", active, standby, False)
|
||||
|
||||
active = choose_executor(workload)
|
||||
standby = "codex" if active == "openclaw" else "openclaw"
|
||||
return QuorumDecision(True, "NONE", "EXECUTOR_QUORUM_PASS", active, standby, False)
|
||||
|
||||
|
||||
class QuorumStore:
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
expected_uid = os.geteuid() if os.name == "posix" else None
|
||||
validate_secure_directory(path.parent, expected_uid=expected_uid)
|
||||
if path.exists() or path.is_symlink():
|
||||
info = path.lstat()
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode) or info.st_nlink != 1:
|
||||
raise ControllerError(f"unsafe database path rejected: {path}")
|
||||
if expected_uid is not None and info.st_uid != expected_uid:
|
||||
raise ControllerError(f"unexpected database owner: {path}")
|
||||
self.db = sqlite3.connect(str(path), isolation_level=None, timeout=30)
|
||||
if os.name == "posix":
|
||||
path.chmod(0o600)
|
||||
self.db.row_factory = sqlite3.Row
|
||||
self.db.execute("PRAGMA journal_mode=WAL")
|
||||
self.db.execute("PRAGMA synchronous=FULL")
|
||||
self._initialize()
|
||||
|
||||
def close(self) -> None:
|
||||
self.db.close()
|
||||
|
||||
def _initialize(self) -> None:
|
||||
self.db.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
event_id TEXT PRIMARY KEY,
|
||||
payload_digest TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
created_at REAL NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS tasks (
|
||||
work_unit TEXT PRIMARY KEY,
|
||||
contract_commit TEXT NOT NULL,
|
||||
event_id TEXT NOT NULL,
|
||||
state_version INTEGER NOT NULL,
|
||||
workload TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
active_executor TEXT,
|
||||
standby_executor TEXT,
|
||||
checkpoint TEXT,
|
||||
lease_owner TEXT,
|
||||
lease_until REAL,
|
||||
interrupted_executor TEXT,
|
||||
recovery_priority TEXT NOT NULL,
|
||||
updated_at REAL NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS lifecycle (
|
||||
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
work_unit TEXT NOT NULL,
|
||||
event_type TEXT NOT NULL,
|
||||
executor TEXT,
|
||||
state_version INTEGER NOT NULL,
|
||||
metadata_json TEXT NOT NULL,
|
||||
created_at REAL NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS effects (
|
||||
effect_id TEXT PRIMARY KEY,
|
||||
work_unit TEXT NOT NULL,
|
||||
payload_digest TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
created_at REAL NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS outbox (
|
||||
outbox_id TEXT PRIMARY KEY,
|
||||
destination TEXT NOT NULL,
|
||||
payload_digest TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt REAL NOT NULL,
|
||||
created_at REAL NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS blockers (
|
||||
blocker_key TEXT PRIMARY KEY,
|
||||
occurrences INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
updated_at REAL NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS observed_refs (
|
||||
ref_name TEXT PRIMARY KEY,
|
||||
oid TEXT NOT NULL,
|
||||
updated_at REAL NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS summaries (
|
||||
work_unit TEXT PRIMARY KEY,
|
||||
summary_digest TEXT NOT NULL,
|
||||
summary_json TEXT NOT NULL,
|
||||
created_at REAL NOT NULL
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS lifecycle_once
|
||||
ON lifecycle(work_unit, event_type, COALESCE(executor, ''), state_version);
|
||||
"""
|
||||
)
|
||||
|
||||
def ingest_event(self, event_id: str, payload: Mapping[str, Any], source: str) -> bool:
|
||||
validate_metadata_only(payload)
|
||||
encoded = canonical_json(payload)
|
||||
digest = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
existing = self.db.execute("SELECT payload_digest FROM events WHERE event_id=?", (event_id,)).fetchone()
|
||||
if existing:
|
||||
if existing["payload_digest"] != digest:
|
||||
self.db.execute("UPDATE events SET status='DEAD_LETTER_CONFLICT' WHERE event_id=?", (event_id,))
|
||||
raise EventConflict(f"event id payload conflict: {event_id}")
|
||||
return False
|
||||
self.db.execute(
|
||||
"INSERT INTO events VALUES(?,?,?,?,?,?)",
|
||||
(event_id, digest, encoded, source, "ACCEPTED", time.time()),
|
||||
)
|
||||
return True
|
||||
|
||||
def upsert_task(
|
||||
self,
|
||||
*,
|
||||
work_unit: str,
|
||||
contract_commit: str,
|
||||
event_id: str,
|
||||
state_version: int,
|
||||
workload: str,
|
||||
decision: QuorumDecision,
|
||||
checkpoint: str | None = None,
|
||||
) -> str:
|
||||
now = time.time()
|
||||
self.db.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
current = self.db.execute("SELECT * FROM tasks WHERE work_unit=?", (work_unit,)).fetchone()
|
||||
if current and state_version < current["state_version"]:
|
||||
raise StaleStateVersion(f"stale state version for {work_unit}")
|
||||
interrupted: str | None = None
|
||||
recovery_priority = decision.priority
|
||||
status = "BLOCKED"
|
||||
if decision.allowed:
|
||||
status = "READY"
|
||||
if current and current["active_executor"] and current["active_executor"] != decision.active_executor:
|
||||
interrupted = current["active_executor"]
|
||||
recovery_priority = "P2"
|
||||
status = "INTERRUPTED_AGENT_FAILOVER"
|
||||
if checkpoint is None:
|
||||
checkpoint = current["checkpoint"]
|
||||
if current:
|
||||
if contract_commit != current["contract_commit"] or event_id != current["event_id"]:
|
||||
raise ControllerError("failover must preserve contract_commit and event_id")
|
||||
if (
|
||||
current["state_version"] == state_version
|
||||
and current["active_executor"] == decision.active_executor
|
||||
and current["status"] in {"RUNNING", "COMPLETED"}
|
||||
):
|
||||
self.db.execute("COMMIT")
|
||||
return current["status"]
|
||||
self.db.execute(
|
||||
"""UPDATE tasks SET state_version=?, workload=?, status=?, active_executor=?,
|
||||
standby_executor=?, checkpoint=?, interrupted_executor=?, recovery_priority=?, updated_at=? WHERE work_unit=?""",
|
||||
(
|
||||
state_version,
|
||||
workload,
|
||||
status,
|
||||
decision.active_executor,
|
||||
decision.standby_executor,
|
||||
checkpoint,
|
||||
interrupted,
|
||||
recovery_priority,
|
||||
now,
|
||||
work_unit,
|
||||
),
|
||||
)
|
||||
else:
|
||||
self.db.execute(
|
||||
"INSERT INTO tasks VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(
|
||||
work_unit,
|
||||
contract_commit,
|
||||
event_id,
|
||||
state_version,
|
||||
workload,
|
||||
status,
|
||||
decision.active_executor,
|
||||
decision.standby_executor,
|
||||
checkpoint,
|
||||
None,
|
||||
None,
|
||||
interrupted,
|
||||
recovery_priority,
|
||||
now,
|
||||
),
|
||||
)
|
||||
self.db.execute("COMMIT")
|
||||
return status
|
||||
except Exception:
|
||||
self.db.execute("ROLLBACK")
|
||||
raise
|
||||
|
||||
def claim_lease(self, work_unit: str, worker: str, ttl_seconds: int, now: float | None = None) -> bool:
|
||||
now = time.time() if now is None else now
|
||||
self.db.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
task = self.db.execute(
|
||||
"SELECT active_executor, status, lease_owner, lease_until FROM tasks WHERE work_unit=?",
|
||||
(work_unit,),
|
||||
).fetchone()
|
||||
if not task:
|
||||
raise ControllerError(f"unknown work unit: {work_unit}")
|
||||
if worker != task["active_executor"]:
|
||||
self.db.execute("ROLLBACK")
|
||||
return False
|
||||
if task["status"] in {"BLOCKED", "COMPLETED"}:
|
||||
self.db.execute("ROLLBACK")
|
||||
return False
|
||||
lease_live = task["lease_owner"] and task["lease_until"] and task["lease_until"] > now
|
||||
if lease_live and task["lease_owner"] != worker:
|
||||
self.db.execute("ROLLBACK")
|
||||
return False
|
||||
self.db.execute(
|
||||
"UPDATE tasks SET lease_owner=?, lease_until=?, status='RUNNING', updated_at=? WHERE work_unit=?",
|
||||
(worker, now + ttl_seconds, now, work_unit),
|
||||
)
|
||||
self.db.execute("COMMIT")
|
||||
return True
|
||||
except Exception:
|
||||
if self.db.in_transaction:
|
||||
self.db.execute("ROLLBACK")
|
||||
raise
|
||||
|
||||
def dispatch_task(
|
||||
self,
|
||||
*,
|
||||
work_unit: str,
|
||||
contract_commit: str,
|
||||
event_id: str,
|
||||
state_version: int,
|
||||
workload: str,
|
||||
decision: QuorumDecision,
|
||||
checkpoint: str | None = None,
|
||||
) -> str:
|
||||
status = self.upsert_task(
|
||||
work_unit=work_unit,
|
||||
contract_commit=contract_commit,
|
||||
event_id=event_id,
|
||||
state_version=state_version,
|
||||
workload=workload,
|
||||
decision=decision,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
if decision.allowed and status != "COMPLETED":
|
||||
metadata = {
|
||||
"contract_commit": contract_commit,
|
||||
"event_id": event_id,
|
||||
"status": status,
|
||||
}
|
||||
self.append_lifecycle(work_unit, "TASK_DISPATCHED", decision.active_executor, state_version, metadata)
|
||||
self.enqueue(
|
||||
f"dispatch:{work_unit}:{state_version}:{decision.active_executor}",
|
||||
"gateway",
|
||||
{"event_type": "TASK_DISPATCHED", "work_unit": work_unit, **metadata},
|
||||
)
|
||||
return status
|
||||
|
||||
def start_task(self, work_unit: str, executor: str, state_version: int, lease_ttl_seconds: int) -> bool:
|
||||
if not self.claim_lease(work_unit, executor, lease_ttl_seconds):
|
||||
return False
|
||||
self.append_lifecycle(work_unit, "TASK_STARTED", executor, state_version, {"status": "RUNNING"})
|
||||
self.enqueue(
|
||||
f"started:{work_unit}:{state_version}:{executor}",
|
||||
"gateway",
|
||||
{"event_type": "TASK_STARTED", "work_unit": work_unit, "executor": executor, "state_version": state_version},
|
||||
)
|
||||
return True
|
||||
|
||||
def end_task(
|
||||
self,
|
||||
work_unit: str,
|
||||
executor: str,
|
||||
state_version: int,
|
||||
result: str,
|
||||
summary: Mapping[str, Any],
|
||||
*,
|
||||
project_completed: bool = False,
|
||||
) -> None:
|
||||
validate_metadata_only(summary)
|
||||
self.db.execute("BEGIN IMMEDIATE")
|
||||
try:
|
||||
task = self.db.execute("SELECT active_executor,state_version FROM tasks WHERE work_unit=?", (work_unit,)).fetchone()
|
||||
if not task or task["active_executor"] != executor:
|
||||
raise ControllerError("only the active executor may end a task")
|
||||
if task["state_version"] != state_version:
|
||||
raise StaleStateVersion(f"stale state version for {work_unit}")
|
||||
self.db.execute(
|
||||
"UPDATE tasks SET status='COMPLETED', lease_owner=NULL, lease_until=NULL, updated_at=? WHERE work_unit=?",
|
||||
(time.time(), work_unit),
|
||||
)
|
||||
self.append_lifecycle(work_unit, "TASK_ENDED", executor, state_version, {"result": result})
|
||||
self.append_lifecycle(work_unit, "DECISION_APPLIED", executor, state_version, {"result": result})
|
||||
self.append_lifecycle(work_unit, "PROJECT_SUMMARY", executor, state_version, summary)
|
||||
if project_completed:
|
||||
self.append_lifecycle(work_unit, "PROJECT_COMPLETED", executor, state_version, {"result": result})
|
||||
self.persist_summary(work_unit, summary)
|
||||
outbound = (
|
||||
("TASK_ENDED", {"result": result}),
|
||||
("DECISION_APPLIED", {"result": result}),
|
||||
("PROJECT_SUMMARY", dict(summary)),
|
||||
)
|
||||
for event_type, metadata in outbound:
|
||||
self.enqueue(
|
||||
f"{event_type.lower()}:{work_unit}:{state_version}:{executor}",
|
||||
"gateway",
|
||||
{
|
||||
"event_type": event_type,
|
||||
"work_unit": work_unit,
|
||||
"executor": executor,
|
||||
"state_version": state_version,
|
||||
**metadata,
|
||||
},
|
||||
)
|
||||
if project_completed:
|
||||
self.enqueue(
|
||||
f"project_completed:{work_unit}:{state_version}:{executor}",
|
||||
"gateway",
|
||||
{
|
||||
"event_type": "PROJECT_COMPLETED",
|
||||
"work_unit": work_unit,
|
||||
"executor": executor,
|
||||
"state_version": state_version,
|
||||
"result": result,
|
||||
},
|
||||
)
|
||||
self.db.execute("COMMIT")
|
||||
except Exception:
|
||||
if self.db.in_transaction:
|
||||
self.db.execute("ROLLBACK")
|
||||
raise
|
||||
|
||||
def record_effect(self, effect_id: str, work_unit: str, payload: Mapping[str, Any]) -> bool:
|
||||
validate_metadata_only(payload)
|
||||
digest = digest_json(payload)
|
||||
existing = self.db.execute("SELECT payload_digest FROM effects WHERE effect_id=?", (effect_id,)).fetchone()
|
||||
if existing:
|
||||
if existing["payload_digest"] != digest:
|
||||
raise EventConflict(f"effect id payload conflict: {effect_id}")
|
||||
return False
|
||||
self.db.execute(
|
||||
"INSERT INTO effects VALUES(?,?,?,?,?)",
|
||||
(effect_id, work_unit, digest, "COMPLETED", time.time()),
|
||||
)
|
||||
return True
|
||||
|
||||
def append_lifecycle(
|
||||
self,
|
||||
work_unit: str,
|
||||
event_type: str,
|
||||
executor: str | None,
|
||||
state_version: int,
|
||||
metadata: Mapping[str, Any],
|
||||
) -> None:
|
||||
if event_type not in ALLOWED_LIFECYCLE_EVENTS:
|
||||
raise ControllerError(f"unsupported lifecycle event: {event_type}")
|
||||
validate_metadata_only(metadata)
|
||||
self.db.execute(
|
||||
"INSERT OR IGNORE INTO lifecycle(work_unit,event_type,executor,state_version,metadata_json,created_at) VALUES(?,?,?,?,?,?)",
|
||||
(work_unit, event_type, executor, state_version, canonical_json(metadata), time.time()),
|
||||
)
|
||||
|
||||
def persist_summary(self, work_unit: str, summary: Mapping[str, Any]) -> None:
|
||||
validate_metadata_only(summary)
|
||||
encoded = canonical_json(summary)
|
||||
digest = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
self.db.execute(
|
||||
"INSERT OR REPLACE INTO summaries VALUES(?,?,?,?)",
|
||||
(work_unit, digest, encoded, time.time()),
|
||||
)
|
||||
|
||||
def enqueue(self, outbox_id: str, destination: str, payload: Mapping[str, Any]) -> bool:
|
||||
validate_metadata_only(payload)
|
||||
encoded = canonical_json(payload)
|
||||
digest = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
existing = self.db.execute("SELECT payload_digest FROM outbox WHERE outbox_id=?", (outbox_id,)).fetchone()
|
||||
if existing:
|
||||
if existing["payload_digest"] != digest:
|
||||
raise EventConflict(f"outbox id payload conflict: {outbox_id}")
|
||||
return False
|
||||
self.db.execute(
|
||||
"INSERT INTO outbox VALUES(?,?,?,?,?,?,?,?)",
|
||||
(outbox_id, destination, digest, encoded, "PENDING", 0, time.time(), time.time()),
|
||||
)
|
||||
return True
|
||||
|
||||
def record_blocker(self, blocker_key: str, threshold: int) -> str:
|
||||
now = time.time()
|
||||
row = self.db.execute("SELECT occurrences FROM blockers WHERE blocker_key=?", (blocker_key,)).fetchone()
|
||||
occurrences = (row["occurrences"] if row else 0) + 1
|
||||
status = "DEAD_LETTER" if occurrences > threshold else "RETRY"
|
||||
self.db.execute(
|
||||
"INSERT INTO blockers VALUES(?,?,?,?) ON CONFLICT(blocker_key) DO UPDATE SET occurrences=excluded.occurrences,status=excluded.status,updated_at=excluded.updated_at",
|
||||
(blocker_key, occurrences, status, now),
|
||||
)
|
||||
return status
|
||||
|
||||
def deliver_outbox(self, sender: Callable[[Mapping[str, Any]], None], max_attempts: int, now: float | None = None) -> tuple[int, int]:
|
||||
now = time.time() if now is None else now
|
||||
sent = 0
|
||||
failed = 0
|
||||
rows = self.db.execute(
|
||||
"SELECT * FROM outbox WHERE status='PENDING' AND next_attempt<=? ORDER BY created_at,outbox_id LIMIT 100",
|
||||
(now,),
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
payload = json.loads(row["payload_json"])
|
||||
validate_metadata_only(payload)
|
||||
try:
|
||||
sender(payload)
|
||||
except Exception:
|
||||
attempts = row["attempts"] + 1
|
||||
status = "DEAD_LETTER" if attempts >= max_attempts else "PENDING"
|
||||
delay = min(300, 2 ** min(attempts, 8))
|
||||
self.db.execute(
|
||||
"UPDATE outbox SET attempts=?,status=?,next_attempt=? WHERE outbox_id=?",
|
||||
(attempts, status, now + delay, row["outbox_id"]),
|
||||
)
|
||||
failed += 1
|
||||
else:
|
||||
self.db.execute(
|
||||
"UPDATE outbox SET attempts=attempts+1,status='SENT',next_attempt=? WHERE outbox_id=?",
|
||||
(now, row["outbox_id"]),
|
||||
)
|
||||
sent += 1
|
||||
return sent, failed
|
||||
|
||||
def observe_ref(self, ref_name: str, oid: str) -> bool:
|
||||
row = self.db.execute("SELECT oid FROM observed_refs WHERE ref_name=?", (ref_name,)).fetchone()
|
||||
changed = row is not None and row["oid"] != oid
|
||||
self.db.execute(
|
||||
"INSERT INTO observed_refs VALUES(?,?,?) ON CONFLICT(ref_name) DO UPDATE SET oid=excluded.oid,updated_at=excluded.updated_at",
|
||||
(ref_name, oid, time.time()),
|
||||
)
|
||||
return changed
|
||||
|
||||
|
||||
def spool_after_http_failure(
|
||||
sender: Callable[[Mapping[str, Any]], None],
|
||||
payload: Mapping[str, Any],
|
||||
spool_dir: Path,
|
||||
event_id: str,
|
||||
) -> bool:
|
||||
validate_metadata_only(payload)
|
||||
try:
|
||||
sender(payload)
|
||||
return True
|
||||
except Exception:
|
||||
atomic_json_write(spool_dir / f"{event_id}.json", {"event_id": event_id, "payload": dict(payload)})
|
||||
return False
|
||||
|
||||
|
||||
def poll_file_inbox(store: QuorumStore, inbox_dir: Path) -> int:
|
||||
if not inbox_dir.exists():
|
||||
return 0
|
||||
validate_secure_directory(inbox_dir, expected_uid=os.geteuid() if os.name == "posix" else None)
|
||||
count = 0
|
||||
for path in sorted(inbox_dir.glob("*.json")):
|
||||
if path.is_symlink():
|
||||
raise ControllerError(f"symlink inbox entry rejected: {path}")
|
||||
item = read_secure_json(path, expected_uid=os.geteuid() if os.name == "posix" else None)
|
||||
event_id = item.get("event_id")
|
||||
payload = item.get("payload")
|
||||
if not isinstance(event_id, str) or not isinstance(payload, dict):
|
||||
raise ControllerError(f"invalid inbox envelope: {path}")
|
||||
if store.ingest_event(event_id, payload, "file-inbox"):
|
||||
count += 1
|
||||
path.unlink()
|
||||
return count
|
||||
|
||||
|
||||
def resolve_git_ref(repo: Path, ref_name: str) -> str:
|
||||
if not repo.is_absolute():
|
||||
raise ControllerError("Git repository path must be absolute")
|
||||
info = repo.lstat()
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):
|
||||
raise ControllerError("Git repository must be a non-symlink directory")
|
||||
if os.name == "posix" and repo.resolve(strict=True) != Path(os.path.abspath(repo)):
|
||||
raise ControllerError("Git repository has a symlinked path component")
|
||||
if os.name == "posix" and info.st_mode & 0o002:
|
||||
raise ControllerError("world-writable Git repository rejected")
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"-c",
|
||||
f"safe.directory={repo}",
|
||||
"-C",
|
||||
str(repo),
|
||||
"rev-parse",
|
||||
"--verify",
|
||||
ref_name,
|
||||
],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
|
||||
raise ControllerError("Git ref unavailable") from exc
|
||||
oid = completed.stdout.strip()
|
||||
if len(oid) != 40 or any(character not in "0123456789abcdef" for character in oid.lower()):
|
||||
raise ControllerError("invalid Git object id")
|
||||
return oid
|
||||
|
||||
|
||||
def poll_git_refs(
|
||||
store: QuorumStore,
|
||||
repo: Path,
|
||||
refs: Iterable[str],
|
||||
resolver: Callable[[Path, str], str] = resolve_git_ref,
|
||||
) -> int:
|
||||
changed = 0
|
||||
for ref_name in refs:
|
||||
if not ref_name.startswith("refs/") or ".." in ref_name or ref_name.endswith("/"):
|
||||
raise ControllerError(f"invalid Git ref: {ref_name}")
|
||||
oid = resolver(repo, ref_name)
|
||||
if store.observe_ref(ref_name, oid):
|
||||
event_id = f"gitref:{ref_name}:{oid}"
|
||||
if store.ingest_event(event_id, {"ref": ref_name, "oid": oid}, "git-ref-poll"):
|
||||
changed += 1
|
||||
return changed
|
||||
|
||||
|
||||
def responses_fallback_allowed(
|
||||
workspace_agent_ok: bool,
|
||||
receipt: Mapping[str, Any],
|
||||
*,
|
||||
now: float | None = None,
|
||||
) -> bool:
|
||||
validate_metadata_only(receipt)
|
||||
if workspace_agent_ok:
|
||||
return False
|
||||
responses = receipt.get("responses_api")
|
||||
return (
|
||||
validate_health_receipt(receipt, "central_gpt", now=now) is Health.HEALTHY
|
||||
and isinstance(responses, Mapping)
|
||||
and responses.get("validated_live") is True
|
||||
)
|
||||
|
||||
|
||||
def _validate_gateway_url(url: str) -> None:
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
if (
|
||||
parsed.scheme != "http"
|
||||
or parsed.hostname != "127.0.0.1"
|
||||
or parsed.port != 18920
|
||||
or parsed.path != "/v1/orchestrator/events"
|
||||
or parsed.params
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
raise ControllerError("gateway endpoint must be the fixed loopback endpoint")
|
||||
|
||||
|
||||
def get_gateway_state(url: str, project_key: str, token: str, timeout: float = 5.0) -> int:
|
||||
_validate_gateway_url(url)
|
||||
state_url = url.removesuffix("/events") + "/state/" + urllib.parse.quote(project_key, safe="")
|
||||
request = urllib.request.Request(
|
||||
state_url,
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
payload = json.loads(response.read(262_145))
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
return 0
|
||||
raise ControllerError(f"gateway state rejected HTTP {exc.code}") from exc
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
||||
raise ControllerError("gateway state unavailable") from exc
|
||||
version = payload.get("state_version") if isinstance(payload, Mapping) else None
|
||||
if not isinstance(version, int) or version < 0:
|
||||
raise ControllerError("gateway state version invalid")
|
||||
return version
|
||||
|
||||
|
||||
def post_json(url: str, payload: Mapping[str, Any], token: str, timeout: float = 5.0) -> None:
|
||||
validate_metadata_only(payload)
|
||||
_validate_gateway_url(url)
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=canonical_json(payload).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {token}"},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
if response.status < 200 or response.status >= 300:
|
||||
raise ControllerError(f"gateway rejected HTTP {response.status}")
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise ControllerError(f"gateway rejected HTTP {exc.code}") from exc
|
||||
except (urllib.error.URLError, TimeoutError) as exc:
|
||||
raise ControllerError("gateway unavailable") from exc
|
||||
|
||||
|
||||
def run_once(config_path: Path) -> int:
|
||||
config = read_secure_json(config_path, expected_uid=0 if os.name == "posix" else None)
|
||||
gateway_url = config["gateway_event_url"]
|
||||
token = read_gateway_token(Path(config["gateway_token_file"]))
|
||||
state_version = get_gateway_state(gateway_url, config["project_key"], token)
|
||||
store = QuorumStore(Path(config["database_path"]))
|
||||
try:
|
||||
receipts: dict[str, Health] = {}
|
||||
for subject in ("central_gpt", "openclaw", "codex"):
|
||||
receipt_path = Path(config["health_receipts"][subject])
|
||||
try:
|
||||
receipt = read_secure_json(receipt_path, expected_uid=0 if os.name == "posix" else None)
|
||||
receipts[subject] = validate_health_receipt(
|
||||
receipt,
|
||||
subject,
|
||||
max_age_seconds=int(config.get("max_health_age_seconds", 60)),
|
||||
)
|
||||
except (FileNotFoundError, ControllerError, json.JSONDecodeError):
|
||||
receipts[subject] = Health.UNKNOWN
|
||||
decision = evaluate_quorum(
|
||||
receipts["central_gpt"], receipts["openclaw"], receipts["codex"], config.get("default_workload", "large_repetitive_infrastructure_monitoring")
|
||||
)
|
||||
quorum_event_id = f"quorum-{time.time_ns()}"
|
||||
store.enqueue(
|
||||
quorum_event_id,
|
||||
"gateway",
|
||||
{
|
||||
"schema_version": 1,
|
||||
"project_key": config["project_key"],
|
||||
"contract_commit": config["contract_commit"],
|
||||
"event_id": quorum_event_id,
|
||||
"source_agent": "system",
|
||||
"event_type": "executor_quorum_evaluated",
|
||||
"work_unit_id": "A0-AUTOMATION-QUORUM",
|
||||
"status": "READY" if decision.allowed else "BLOCKED",
|
||||
"state_version": state_version,
|
||||
"implementation_commit": None,
|
||||
"result_commit": None,
|
||||
"blocker": None if decision.allowed else decision.status,
|
||||
"evidence_refs": [],
|
||||
"payload": {
|
||||
"allowed": decision.allowed,
|
||||
"priority": decision.priority,
|
||||
"quorum_status": decision.status,
|
||||
"active_executor": decision.active_executor,
|
||||
"standby_executor": decision.standby_executor,
|
||||
},
|
||||
},
|
||||
)
|
||||
poll_file_inbox(store, Path(config["file_inbox_path"]))
|
||||
git_polling = config.get("git_ref_polling")
|
||||
if isinstance(git_polling, Mapping):
|
||||
poll_git_refs(store, Path(git_polling["repository"]), git_polling["refs"])
|
||||
store.deliver_outbox(
|
||||
lambda payload: post_json(gateway_url, payload, token),
|
||||
max_attempts=int(config["blocker_retry_threshold"]),
|
||||
)
|
||||
return 0 if decision.allowed else 2
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--config", type=Path, required=True)
|
||||
parser.add_argument("--once", action="store_true", required=True)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Iterable[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
return run_once(args.config)
|
||||
except (ControllerError, KeyError, OSError, sqlite3.Error, json.JSONDecodeError) as error:
|
||||
print(f"automation-quorum-controller: fail-closed: {type(error).__name__}", file=os.sys.stderr)
|
||||
return 3
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,146 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Strict metadata-only helpers shared by root health probes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import tempfile
|
||||
from typing import Any
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
class ProbeError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def secure_regular(path: Path, *, max_mode: int, owners: set[int]) -> os.stat_result:
|
||||
if not path.is_absolute():
|
||||
raise ProbeError("path_not_absolute")
|
||||
info = path.lstat()
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode):
|
||||
raise ProbeError("unsafe_file_type")
|
||||
if stat.S_IMODE(info.st_mode) & ~max_mode:
|
||||
raise ProbeError("unsafe_file_mode")
|
||||
if os.name == "posix" and info.st_uid not in owners:
|
||||
raise ProbeError("unsafe_file_owner")
|
||||
if os.name == "posix" and path.resolve(strict=True) != Path(os.path.abspath(path)):
|
||||
raise ProbeError("symlinked_path_component")
|
||||
return info
|
||||
|
||||
|
||||
def load_config(path: Path) -> dict[str, Any]:
|
||||
secure_regular(path, max_mode=0o600, owners={0})
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
if not isinstance(value, dict):
|
||||
raise ProbeError("config_not_object")
|
||||
return value
|
||||
|
||||
|
||||
def read_token(path: Path) -> str:
|
||||
info = secure_regular(path, max_mode=0o640, owners={0, os.geteuid()})
|
||||
if info.st_size <= 0 or info.st_size > 16_384:
|
||||
raise ProbeError("invalid_token_size")
|
||||
fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
|
||||
try:
|
||||
opened = os.fstat(fd)
|
||||
if not stat.S_ISREG(opened.st_mode) or opened.st_ino != info.st_ino or opened.st_dev != info.st_dev:
|
||||
raise ProbeError("token_changed_during_validation")
|
||||
data = os.read(fd, 16_385)
|
||||
if os.read(fd, 1):
|
||||
raise ProbeError("invalid_token_size")
|
||||
finally:
|
||||
os.close(fd)
|
||||
if b"\n" in data or b"\r" in data:
|
||||
raise ProbeError("invalid_token_format")
|
||||
try:
|
||||
return data.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ProbeError("invalid_token_encoding") from exc
|
||||
|
||||
|
||||
def gateway_base(endpoint: str) -> str:
|
||||
parsed = urllib.parse.urlparse(endpoint)
|
||||
if (
|
||||
parsed.scheme != "http"
|
||||
or parsed.hostname != "127.0.0.1"
|
||||
or parsed.port != 18920
|
||||
or parsed.path != "/v1/orchestrator/events"
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
raise ProbeError("invalid_gateway_endpoint")
|
||||
return endpoint.removesuffix("/events")
|
||||
|
||||
|
||||
def request_json(
|
||||
url: str,
|
||||
token: str,
|
||||
*,
|
||||
method: str = "GET",
|
||||
payload: dict[str, Any] | None = None,
|
||||
timeout: float = 10.0,
|
||||
) -> tuple[int, dict[str, Any]]:
|
||||
data = None if payload is None else json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
if data is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
body = json.loads(response.read(262_145))
|
||||
if not isinstance(body, dict):
|
||||
raise ProbeError("gateway_response_not_object")
|
||||
return response.status, body
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in {401, 403, 409, 422}:
|
||||
raise ProbeError(f"gateway_fail_closed_{exc.code}") from exc
|
||||
raise ProbeError(f"gateway_http_{exc.code}") from exc
|
||||
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
|
||||
raise ProbeError("gateway_unavailable") from exc
|
||||
|
||||
|
||||
def state_version(endpoint: str, project_key: str, token: str) -> int:
|
||||
url = gateway_base(endpoint) + "/state/" + urllib.parse.quote(project_key, safe="")
|
||||
try:
|
||||
_, value = request_json(url, token)
|
||||
except ProbeError as exc:
|
||||
if str(exc) == "gateway_http_404":
|
||||
return 0
|
||||
raise
|
||||
version = value.get("state_version")
|
||||
if not isinstance(version, int) or version < 0:
|
||||
raise ProbeError("invalid_state_version")
|
||||
return version
|
||||
|
||||
|
||||
def atomic_receipt(path: Path, value: dict[str, Any]) -> None:
|
||||
if not path.is_absolute() or path.parent != Path("/run/aios-health"):
|
||||
raise ProbeError("invalid_receipt_path")
|
||||
parent = path.parent
|
||||
info = parent.lstat()
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode):
|
||||
raise ProbeError("unsafe_receipt_directory")
|
||||
if os.name == "posix" and (info.st_uid != 0 or stat.S_IMODE(info.st_mode) != 0o750):
|
||||
raise ProbeError("unsafe_receipt_directory_metadata")
|
||||
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=parent)
|
||||
try:
|
||||
# Receipts contain availability metadata only. Give the dedicated
|
||||
# aios-quorum group read access so its unprivileged controller can
|
||||
# consume them, while retaining root ownership and no public access.
|
||||
if os.name == "posix":
|
||||
os.fchown(fd, 0, info.st_gid)
|
||||
os.fchmod(fd, 0o640)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(value, handle, sort_keys=True, separators=(",", ":"))
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
if os.path.exists(temporary):
|
||||
os.unlink(temporary)
|
||||
@@ -1,24 +0,0 @@
|
||||
{
|
||||
"$schema": "../schemas/automation-quorum-controller.schema.json",
|
||||
"project_key": "v2r1-personal-weixin-production",
|
||||
"contract_commit": "cbe33ecab8e9794ba5f318bea43aeaf447a56fbc",
|
||||
"database_path": "/var/lib/aios-automation-quorum/controller.sqlite3",
|
||||
"file_inbox_path": "/var/lib/aios-automation-quorum/inbox",
|
||||
"default_workload": "large_repetitive_infrastructure_monitoring",
|
||||
"health_receipts": {
|
||||
"central_gpt": "/run/aios-health/central-gpt.json",
|
||||
"openclaw": "/run/aios-health/openclaw.json",
|
||||
"codex": "/run/aios-health/codex.json"
|
||||
},
|
||||
"git_ref_polling": {
|
||||
"repository": "/data/aios-gitops/repo.git",
|
||||
"refs": [
|
||||
"refs/heads/task/governance/v2r1-personal-weixin-production-contract-20260717"
|
||||
]
|
||||
},
|
||||
"blocker_retry_threshold": 3,
|
||||
"max_health_age_seconds": 60,
|
||||
"lease_ttl_seconds": 120,
|
||||
"gateway_event_url": "http://127.0.0.1:18920/v1/orchestrator/events",
|
||||
"gateway_token_file": "/etc/aios-orchestrator-gateway/inbound.token"
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
{
|
||||
"active_dir": "/data/aios-gitops/workspace/agents/commands/active",
|
||||
"completed_dir": "/data/aios-gitops/workspace/agents/commands/completed",
|
||||
"dead_letter_dir": "/data/aios-gitops/workspace/agents/commands/dead-letter",
|
||||
"database_path": "/var/lib/aios-codex-watcher/state.sqlite3",
|
||||
"audit_log_path": "/var/log/aios-codex-watcher/audit.jsonl",
|
||||
"gateway_endpoint": "http://127.0.0.1:18920/v1/orchestrator/events",
|
||||
"gateway_token_file": "/etc/aios-orchestrator-gateway/inbound.token",
|
||||
"contract_commit": "cbe33ecab8e9794ba5f318bea43aeaf447a56fbc",
|
||||
"lease_seconds": 60,
|
||||
"retry_seconds": 10,
|
||||
"max_attempts": 5,
|
||||
"poll_seconds": 5,
|
||||
"executor_path": null,
|
||||
"executor_args": [],
|
||||
"executor_timeout_seconds": 1800
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
{
|
||||
"project_key": "v2r1-personal-weixin-production",
|
||||
"contract_commit": "cbe33ecab8e9794ba5f318bea43aeaf447a56fbc",
|
||||
"gateway_event_url": "http://127.0.0.1:18920/v1/orchestrator/events",
|
||||
"gateway_token_file": "/etc/aios-orchestrator-gateway/inbound.token",
|
||||
"gateway_database_path": "/var/lib/aios-orchestrator-gateway/state.sqlite3",
|
||||
"probe_clean_base_commit": "c3984ca2b4388875d2875b4fdfdf95a2984f6ab3",
|
||||
"central_receipt_path": "/run/aios-health/central-gpt.json",
|
||||
"codex_receipt_path": "/run/aios-health/codex.json",
|
||||
"codex_command_queue": "/data/aios-gitops/workspace/agents/commands/active",
|
||||
"codex_database_path": "/var/lib/aios-codex-watcher/state.sqlite3",
|
||||
"codex_audit_log": "/var/log/aios-codex-watcher/audit.jsonl",
|
||||
"central_timeout_seconds": 90,
|
||||
"codex_timeout_seconds": 90,
|
||||
"cycle_timeout_seconds": 180,
|
||||
"poll_interval_seconds": 1
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
{
|
||||
"host": "127.0.0.1",
|
||||
"port": 18920,
|
||||
"database_path": "/var/lib/aios-orchestrator-gateway/state.sqlite3",
|
||||
"event_inbox_dir": "/var/lib/aios-orchestrator-gateway/inbox/events",
|
||||
"decision_inbox_dir": "/var/lib/aios-orchestrator-gateway/inbox/decisions",
|
||||
"dispatch_outbox_dir": "/var/lib/aios-orchestrator-gateway/outbox",
|
||||
"openclaw_sync_dir": "/var/lib/aios-orchestrator-gateway/openclaw-sync",
|
||||
"command_queue_dir": "/var/lib/aios-orchestrator-gateway/commands",
|
||||
"inbound_token_file": "/etc/aios-orchestrator-gateway/inbound.token",
|
||||
"callback_token_file": "/etc/aios-orchestrator-gateway/callback.token",
|
||||
"contract_commit": "cbe33ecab8e9794ba5f318bea43aeaf447a56fbc",
|
||||
"central_channel": "codex_local",
|
||||
"codex_local": {
|
||||
"queue_dir": "/var/lib/aios-codex-brain/queue",
|
||||
"timeout_seconds": 300
|
||||
},
|
||||
"workspace_agent": {
|
||||
"trigger_id": null,
|
||||
"access_token_file": null
|
||||
},
|
||||
"responses_fallback": {
|
||||
"api_key_file": null,
|
||||
"model": null
|
||||
},
|
||||
"polling": {
|
||||
"interval_seconds": 10,
|
||||
"summary_interval_seconds": 300
|
||||
},
|
||||
"loop_controls": {
|
||||
"callback_timeout_seconds": 900,
|
||||
"lease_seconds": 60,
|
||||
"retry_base_seconds": 10,
|
||||
"max_retries": 5,
|
||||
"max_same_blocker_retries": 3,
|
||||
"max_hops_per_gate": 20
|
||||
},
|
||||
"openclaw_sync": {
|
||||
"endpoint": null,
|
||||
"token_file": null
|
||||
},
|
||||
"project_key": "v2r1-personal-weixin-production",
|
||||
"git_poll": {
|
||||
"enabled": true,
|
||||
"bare_repo": "/data/aios-gitops/repo.git",
|
||||
"ref_prefixes": [
|
||||
"refs/heads/main",
|
||||
"refs/heads/task/governance/",
|
||||
"refs/heads/patch/",
|
||||
"refs/heads/result/"
|
||||
],
|
||||
"interval_seconds": 30
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONPATH=/opt/aios/app \
|
||||
AIOS_V2_HOST=0.0.0.0 \
|
||||
AIOS_V2_PORT=18920 \
|
||||
AIOS_V2_STATE_DIR=/var/lib/aios-v2
|
||||
|
||||
RUN groupadd --system --gid 12001 aios \
|
||||
&& useradd --system --uid 12001 --gid aios --home-dir /nonexistent --shell /usr/sbin/nologin aios \
|
||||
&& install -d -o aios -g aios -m 0750 /opt/aios/app /var/lib/aios-v2
|
||||
|
||||
WORKDIR /opt/aios/app
|
||||
COPY --chown=aios:aios active-runtime/app/ /opt/aios/app/
|
||||
|
||||
USER aios:aios
|
||||
EXPOSE 18920
|
||||
|
||||
HEALTHCHECK --interval=15s --timeout=5s --start-period=10s --retries=4 \
|
||||
CMD python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:18920/api/v2/health', timeout=3).read()"
|
||||
|
||||
CMD ["python3", "-m", "aios_v2.app"]
|
||||
@@ -0,0 +1,44 @@
|
||||
services:
|
||||
aios-v2:
|
||||
build:
|
||||
context: ../../..
|
||||
dockerfile: deploy/aios-v2/docker/Dockerfile
|
||||
image: aios-v2:V2R1C0-candidate
|
||||
init: true
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:${AIOS_V2_PORT:-18920}:18920"
|
||||
environment:
|
||||
AIOS_V2_HOST: 0.0.0.0
|
||||
AIOS_V2_PORT: 18920
|
||||
AIOS_V2_STATE_DIR: /var/lib/aios-v2
|
||||
AIOS_OPENCLAW_URL: ${AIOS_OPENCLAW_URL:-}
|
||||
AIOS_OPENCLAW_CREDENTIAL_REF: ${AIOS_OPENCLAW_CREDENTIAL_REF:-}
|
||||
AIOS_OPENCLAW_SIGNING_MATERIAL_FILE: ${AIOS_OPENCLAW_SIGNING_MATERIAL_FILE:-}
|
||||
AIOS_RELEASE_CHANNEL: candidate
|
||||
AIOS_RELEASE_VERSION: V2R1C0-candidate
|
||||
volumes:
|
||||
- aios-v2-state:/var/lib/aios-v2
|
||||
# Mount runtime signing material explicitly when OpenClaw is enabled.
|
||||
# The host file must be mode 0600; never store it in this repository.
|
||||
# - /etc/aios-v2/openclaw-signing-material:/run/secrets/openclaw-signing-material:ro
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,nodev,size=64m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python3
|
||||
- -c
|
||||
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:18920/api/v2/health', timeout=3).read()"
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 4
|
||||
start_period: 10s
|
||||
|
||||
volumes:
|
||||
aios-v2-state:
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
# AIOS V2 candidate runtime configuration.
|
||||
# Copy to /etc/aios-v2/aios-v2.env and keep ownership root:aios, mode 0640.
|
||||
|
||||
AIOS_V2_HOST=127.0.0.1
|
||||
AIOS_V2_PORT=18920
|
||||
AIOS_V2_STATE_DIR=/var/lib/aios-v2
|
||||
|
||||
# Optional OpenClaw runtime. Leave blank to run with OpenClaw disabled.
|
||||
AIOS_OPENCLAW_URL=
|
||||
AIOS_OPENCLAW_CREDENTIAL_REF=
|
||||
|
||||
# File path only. The referenced file must be mode 0600 or stricter and contain
|
||||
# at least 32 bytes. Never place signing material in this env file or Git.
|
||||
AIOS_OPENCLAW_SIGNING_MATERIAL_FILE=
|
||||
|
||||
# Reverse proxy must authenticate users and inject X-AIOS-* identity headers.
|
||||
AIOS_TRUST_PROXY_HEADERS=true
|
||||
|
||||
# Candidate deployment metadata exposed through health tooling.
|
||||
AIOS_RELEASE_CHANNEL=candidate
|
||||
AIOS_RELEASE_VERSION=V2R1C0-candidate
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
BASE_URL="${AIOS_V2_BASE_URL:-http://127.0.0.1:18920}"
|
||||
STATE_DIR="${AIOS_V2_STATE_DIR:-/var/lib/aios-v2}"
|
||||
CONTROLLER_WT="${AIOS_GITOPS_CONTROLLER_WORKTREE:-/data/aios-gitops/worktrees/controller}"
|
||||
SERVICE="${AIOS_V2_SERVICE:-aios-v2}"
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
pass() {
|
||||
echo "PASS: $*"
|
||||
}
|
||||
|
||||
command -v python3 >/dev/null || fail "python3 not installed"
|
||||
|
||||
if command -v systemctl >/dev/null && systemctl list-unit-files "$SERVICE.service" >/dev/null 2>&1; then
|
||||
systemctl is-active --quiet "$SERVICE" || fail "$SERVICE is not active"
|
||||
pass "$SERVICE is active"
|
||||
fi
|
||||
|
||||
python3 - "$BASE_URL" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
base_url = sys.argv[1].rstrip("/")
|
||||
with urllib.request.urlopen(f"{base_url}/api/v2/health", timeout=5) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
if payload.get("status") not in {"ok", "healthy", "ready"}:
|
||||
raise SystemExit(f"unexpected health payload: {payload}")
|
||||
print(f"PASS: HTTP health status={payload.get('status')}")
|
||||
PY
|
||||
|
||||
[[ -d "$STATE_DIR" ]] || fail "state directory missing: $STATE_DIR"
|
||||
[[ -w "$STATE_DIR" ]] || fail "state directory not writable: $STATE_DIR"
|
||||
pass "state directory writable"
|
||||
|
||||
if [[ -e "$STATE_DIR/control-plane.sqlite3" ]]; then
|
||||
python3 - "$STATE_DIR/control-plane.sqlite3" <<'PY'
|
||||
import sqlite3
|
||||
import sys
|
||||
|
||||
path = sys.argv[1]
|
||||
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5)
|
||||
try:
|
||||
row = connection.execute("PRAGMA quick_check").fetchone()
|
||||
finally:
|
||||
connection.close()
|
||||
if not row or row[0] != "ok":
|
||||
raise SystemExit(f"SQLite quick_check failed: {row}")
|
||||
print("PASS: SQLite quick_check=ok")
|
||||
PY
|
||||
else
|
||||
echo "WARN: control-plane.sqlite3 not created yet"
|
||||
fi
|
||||
|
||||
if [[ -d "$CONTROLLER_WT" ]] && git -C "$CONTROLLER_WT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
branch="$(git -C "$CONTROLLER_WT" branch --show-current)"
|
||||
dirty="$(git -C "$CONTROLLER_WT" status --porcelain=v1)"
|
||||
[[ -z "$branch" ]] || fail "GitOps controller attached to branch: $branch"
|
||||
[[ -z "$dirty" ]] || fail "GitOps controller workspace is dirty"
|
||||
pass "GitOps controller detached and clean"
|
||||
fi
|
||||
|
||||
python3 - "$STATE_DIR/executions" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
root = pathlib.Path(sys.argv[1])
|
||||
if not root.exists():
|
||||
print("WARN: execution evidence directory not created yet")
|
||||
raise SystemExit(0)
|
||||
for path in root.glob("*/*.json"):
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
for key in ("execution_id", "tenant_id", "task_id", "executor", "status"):
|
||||
if not value.get(key):
|
||||
raise SystemExit(f"invalid execution evidence {path}: missing {key}")
|
||||
print("PASS: execution evidence files valid")
|
||||
PY
|
||||
|
||||
pass "AIOS V2 candidate healthcheck complete"
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
SERVICE="${AIOS_V2_SERVICE:-aios-v2}"
|
||||
RELEASE_ROOT="${AIOS_V2_RELEASE_ROOT:-/opt/aios/releases}"
|
||||
CURRENT_LINK="${AIOS_V2_CURRENT_LINK:-/opt/aios/current}"
|
||||
STATE_DIR="${AIOS_V2_STATE_DIR:-/var/lib/aios-v2}"
|
||||
BACKUP_ROOT="${AIOS_V2_BACKUP_ROOT:-/var/backups/aios-v2}"
|
||||
TARGET_RELEASE="${1:-}"
|
||||
RESTORE_STATE="${AIOS_V2_RESTORE_STATE:-0}"
|
||||
|
||||
fail() {
|
||||
echo "ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[[ "${EUID}" -eq 0 ]] || fail "rollback must run as root"
|
||||
[[ -n "$TARGET_RELEASE" ]] || fail "usage: $0 <release-name-or-absolute-path>"
|
||||
|
||||
if [[ "$TARGET_RELEASE" = /* ]]; then
|
||||
TARGET_PATH="$(readlink -f "$TARGET_RELEASE")"
|
||||
else
|
||||
[[ "$TARGET_RELEASE" != *"/"* && "$TARGET_RELEASE" != "." && "$TARGET_RELEASE" != ".." ]] \
|
||||
|| fail "invalid release name"
|
||||
TARGET_PATH="$(readlink -f "$RELEASE_ROOT/$TARGET_RELEASE")"
|
||||
fi
|
||||
|
||||
[[ -d "$TARGET_PATH/active-runtime/app" ]] || fail "target release is invalid: $TARGET_PATH"
|
||||
[[ -f "$TARGET_PATH/deploy/aios-v2/healthcheck/check.sh" ]] \
|
||||
|| fail "target release has no healthcheck"
|
||||
|
||||
PREVIOUS_PATH=""
|
||||
if [[ -L "$CURRENT_LINK" ]]; then
|
||||
PREVIOUS_PATH="$(readlink -f "$CURRENT_LINK")"
|
||||
fi
|
||||
|
||||
STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
BACKUP_DIR="$BACKUP_ROOT/rollback-$STAMP"
|
||||
install -d -m 0750 "$BACKUP_DIR"
|
||||
|
||||
if [[ -d "$STATE_DIR" ]]; then
|
||||
tar --xattrs --acls -C "$(dirname "$STATE_DIR")" \
|
||||
-czf "$BACKUP_DIR/state-before-rollback.tar.gz" \
|
||||
"$(basename "$STATE_DIR")"
|
||||
fi
|
||||
|
||||
printf '%s\n' "$PREVIOUS_PATH" > "$BACKUP_DIR/previous-release.txt"
|
||||
printf '%s\n' "$TARGET_PATH" > "$BACKUP_DIR/target-release.txt"
|
||||
|
||||
systemctl stop "$SERVICE"
|
||||
ln -sfn "$TARGET_PATH" "${CURRENT_LINK}.next"
|
||||
mv -Tf "${CURRENT_LINK}.next" "$CURRENT_LINK"
|
||||
systemctl daemon-reload
|
||||
systemctl start "$SERVICE"
|
||||
|
||||
if AIOS_V2_SERVICE="$SERVICE" \
|
||||
AIOS_V2_STATE_DIR="$STATE_DIR" \
|
||||
"$TARGET_PATH/deploy/aios-v2/healthcheck/check.sh"; then
|
||||
echo "ROLLBACK_COMPLETE target=$TARGET_PATH backup=$BACKUP_DIR"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Target release failed healthcheck; restoring previous release." >&2
|
||||
systemctl stop "$SERVICE" || true
|
||||
|
||||
if [[ -n "$PREVIOUS_PATH" && -d "$PREVIOUS_PATH" ]]; then
|
||||
ln -sfn "$PREVIOUS_PATH" "${CURRENT_LINK}.previous"
|
||||
mv -Tf "${CURRENT_LINK}.previous" "$CURRENT_LINK"
|
||||
fi
|
||||
|
||||
if [[ "$RESTORE_STATE" == "1" && -f "$BACKUP_DIR/state-before-rollback.tar.gz" ]]; then
|
||||
rm -rf "$STATE_DIR"
|
||||
tar --xattrs --acls -C "$(dirname "$STATE_DIR")" \
|
||||
-xzf "$BACKUP_DIR/state-before-rollback.tar.gz"
|
||||
fi
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl start "$SERVICE" || true
|
||||
|
||||
if [[ -n "$PREVIOUS_PATH" && -x "$PREVIOUS_PATH/deploy/aios-v2/healthcheck/check.sh" ]]; then
|
||||
AIOS_V2_SERVICE="$SERVICE" \
|
||||
AIOS_V2_STATE_DIR="$STATE_DIR" \
|
||||
"$PREVIOUS_PATH/deploy/aios-v2/healthcheck/check.sh" || true
|
||||
fi
|
||||
|
||||
fail "rollback target failed; previous release restored; evidence=$BACKUP_DIR"
|
||||
@@ -0,0 +1,37 @@
|
||||
[Unit]
|
||||
Description=AIOS V2 Control Plane Candidate
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=aios
|
||||
Group=aios
|
||||
WorkingDirectory=/opt/aios/current/active-runtime/app
|
||||
EnvironmentFile=-/etc/aios-v2/aios-v2.env
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
ExecStart=/usr/bin/python3 -m aios_v2.app
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
TimeoutStartSec=30
|
||||
TimeoutStopSec=30
|
||||
KillSignal=SIGTERM
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
RestrictSUIDSGID=true
|
||||
RestrictRealtime=true
|
||||
LockPersonality=true
|
||||
MemoryDenyWriteExecute=true
|
||||
ReadWritePaths=/var/lib/aios-v2
|
||||
StateDirectory=aios-v2
|
||||
StateDirectoryMode=0750
|
||||
UMask=0027
|
||||
LimitNOFILE=65536
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -13,7 +13,6 @@ Environment=AIOS_GITOPS_TOKEN_FILE=/data/aios-gitops/runtime/secrets/gitops.toke
|
||||
Environment=AIOS_GITOPS_BARE_REPO=/data/aios-gitops/runtime/gitea-bin/data/repositories/aios/aios-v23.git
|
||||
Environment=AIOS_GITOPS_HOST=127.0.0.1
|
||||
Environment=AIOS_GITOPS_PORT=18890
|
||||
EnvironmentFile=-/etc/aios-gitops/orchestrator-callback.env
|
||||
ExecStart=/usr/bin/python3 /data/aios-gitops/workspace/active-runtime/app/aios_chatgpt_actions_api.py
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
# AIOS GPT + one-executor automation quorum
|
||||
|
||||
## Authority and release boundary
|
||||
|
||||
The Gateway is the only authoritative state writer. It must hold the SQLite writer lease before changing task state. A model response, service process, configured credential, or agent presence does not establish health. Release remains paused until all live acceptance rows are backed by logs and exit codes.
|
||||
|
||||
Minimum quorum:
|
||||
|
||||
```text
|
||||
central_gpt=HEALTHY AND (openclaw=HEALTHY OR codex=HEALTHY)
|
||||
```
|
||||
|
||||
`UNKNOWN` never counts as healthy. If one executor is proven healthy, an unhealthy or unknown standby is P3 telemetry only and cannot block the active gate. If no executor is proven healthy, the controller blocks. Central GPT `UNKNOWN` or `UNHEALTHY` always blocks.
|
||||
|
||||
## Live health receipts
|
||||
|
||||
The controller reads three root-owned metadata-only JSON receipts from `/run/aios-health`. Files must be regular files, must not be symlinks, and neither files nor parent directories may be group/world writable. The controller rejects fields named `secret`, `token`, `password`, `api_key`, `authorization`, `credential`, `private_key`, `environment`, `env`, `argv`, `stdout`, or `stderr` at any depth.
|
||||
|
||||
A receipt claiming `HEALTHY` must contain:
|
||||
|
||||
```json
|
||||
{
|
||||
"subject": "codex",
|
||||
"health": "HEALTHY",
|
||||
"live_probe_succeeded": true,
|
||||
"observed_at_epoch": 1784347200
|
||||
}
|
||||
```
|
||||
|
||||
The central GPT receipt additionally requires either `workspace_agent.validated_live=true` or `responses_api.validated_live=true`. Responses fallback is allowed only after a real Responses decision probe succeeded; credential/configuration presence is insufficient.
|
||||
|
||||
The controller and probes use only the fixed loopback endpoint
|
||||
`http://127.0.0.1:18920/v1/orchestrator/events`. The bearer is read from
|
||||
`/etc/aios-orchestrator-gateway/inbound.token`; the file must be absolute,
|
||||
root-owned, regular, non-symlinked, no wider than `0640`, nonempty, and contain
|
||||
no CR/LF. It is never accepted through argv, environment, logs, receipts, or
|
||||
Git. HTTP `401`, `403`, `409`, and `422` are terminal fail-closed results for a
|
||||
probe attempt.
|
||||
|
||||
The authenticated metadata query
|
||||
`GET /v1/orchestrator/events/<event_id>` returns only the event ID, public
|
||||
status, selected central channel, whether the decision was applied, and the
|
||||
resulting state version. It never returns the submitted event, model output,
|
||||
credential, or work-unit instruction.
|
||||
|
||||
## Durable behavior
|
||||
|
||||
SQLite persists ingress event digests, optimistic task state versions, leases, checkpoints, lifecycle events, effect IDs, outbox messages, blocker counts, observed Git refs, and project summaries. Reusing an event/effect/outbox ID with different content fails closed. An expired lease may be reclaimed; a live lease cannot be stolen.
|
||||
|
||||
On active-executor failure, a healthy standby receives the same `contract_commit`, `event_id`, state version, and checkpoint. The former executor is recorded as `INTERRUPTED_AGENT_FAILOVER`. Unique effect IDs prevent replay of completed side effects.
|
||||
|
||||
The Gateway persists these events even when OpenClaw is offline:
|
||||
|
||||
```text
|
||||
TASK_DISPATCHED
|
||||
TASK_STARTED
|
||||
TASK_ENDED
|
||||
DECISION_APPLIED
|
||||
PROJECT_SUMMARY
|
||||
PROJECT_COMPLETED
|
||||
```
|
||||
|
||||
OpenClaw mirroring is optional and uses the durable outbox with bounded retry and dead-letter. HTTP ingress failure spools a metadata-only file for polling; the configured local Git refs are also polled and changes missed by webhooks become idempotent events. Repeated blockers beyond the configured threshold enter `DEAD_LETTER`.
|
||||
|
||||
## Install, verify, and rollback
|
||||
|
||||
Installation is versioned and does not overwrite an existing release:
|
||||
|
||||
```bash
|
||||
sudo scripts/install_automation_quorum_controller.sh "$PWD" 20260718.1
|
||||
sudo systemctl enable --now aios-automation-quorum-controller.timer
|
||||
sudo systemctl enable --now aios-central-gpt-health-probe.timer
|
||||
sudo systemctl enable --now aios-codex-executor-health-probe.timer
|
||||
```
|
||||
|
||||
Run deterministic verification before enablement:
|
||||
|
||||
```bash
|
||||
scripts/verify_automation_quorum_controller.sh "$PWD"
|
||||
```
|
||||
|
||||
The two root probes are fixed, bounded, side-effect-free programs. They write
|
||||
receipts atomically with mode `0600` below the root-owned `0750`
|
||||
`/run/aios-health`. The Codex probe may write only one `acceptance_probe`
|
||||
command into the controlled watcher queue; it cannot accept arbitrary command
|
||||
text from configuration.
|
||||
|
||||
Run the strict two-cycle closure only after both services are installed from an
|
||||
immutable committed release:
|
||||
|
||||
```bash
|
||||
sudo /opt/aios-automation-quorum/current/bin/a0-unattended-loop-probe \
|
||||
--config /etc/aios-automation-quorum/health-probes.json
|
||||
```
|
||||
|
||||
Each cycle requires a live central decision that dispatches the named Codex
|
||||
acceptance work unit through the Gateway. The verifier then requires exactly
|
||||
one claim/start/execute/end/post marker, both lifecycle events in the Gateway,
|
||||
and a persisted `PROJECT_SUMMARY`. Directly queuing a command cannot satisfy
|
||||
this two-cycle closure.
|
||||
|
||||
Rollback accepts only an existing non-symlink directory below `/opt/aios-automation-quorum/releases`. Repeating rollback to the current target is a no-op:
|
||||
|
||||
```bash
|
||||
sudo scripts/rollback_automation_quorum_controller.sh \
|
||||
/opt/aios-automation-quorum/releases/20260718.0
|
||||
```
|
||||
|
||||
## Live acceptance matrix
|
||||
|
||||
Each row requires captured logs, timestamps, exit codes, committed OIDs, Gitleaks output, and where applicable live Gateway DB evidence. Static unit tests are necessary but cannot close the gate.
|
||||
|
||||
1. GPT + OpenClaw healthy, Codex stopped: two unattended cycles.
|
||||
2. GPT + Codex healthy, OpenClaw stopped: two unattended cycles.
|
||||
3. Both executors healthy: OpenClaw gets repetitive infrastructure; Codex gets complex code/security.
|
||||
4. Active stops mid-task: standby continues from checkpoint with no duplicate effect.
|
||||
5. Standby alone stops: active flow continues and no repair gate is created.
|
||||
6. GPT stops: execution blocks even with two healthy executors.
|
||||
7. GPT healthy and both executors stop: `BLOCKED_NO_HEALTHY_EXECUTOR`.
|
||||
8. `UNKNOWN` is never counted as healthy.
|
||||
9. HTTP ingress failure recovers through file polling.
|
||||
10. Missed webhook recovers through Git ref polling.
|
||||
11. Gateway restart restores events, leases, and outbox.
|
||||
12. Repeated event ID does not create a second dispatch; changed payload dead-letters.
|
||||
13. Repeated blocker beyond threshold dead-letters.
|
||||
14. Workspace Agent failure falls back only with a live-validated Responses channel.
|
||||
15. OpenClaw offline: Codex lifecycle and project summary remain complete in Gateway.
|
||||
16. Codex offline: OpenClaw lifecycle and project summary remain complete in Gateway.
|
||||
|
||||
Until all rows pass against the live service, retain:
|
||||
|
||||
```text
|
||||
AUTOMATION_CLOSED=NOT_STARTED
|
||||
BASELINE=V1R4C6
|
||||
RELEASE=paused
|
||||
PRODUCTION_RELEASE=NOT_STARTED
|
||||
```
|
||||
@@ -0,0 +1,243 @@
|
||||
# AIOS V2 用户需求规格
|
||||
|
||||
- 文档版本:`V2R1C0-candidate`
|
||||
- 状态:候选版本需求基线
|
||||
- 适用范围:AIOS Control Plane、Agent Router、OpenClaw Adapter、Hermes Runtime、Web Console、GitOps、审计与部署
|
||||
|
||||
## 1. 背景与目标
|
||||
|
||||
现有系统已经具备 OpenClaw、Hermes、Agent Router、GitOps、局部 WebUI、审计和多租户候选能力。AIOS V2 的目标不是推倒重写,而是将这些分散能力收敛到统一、可审计、可 Web 化、可多租户隔离的控制面。
|
||||
|
||||
AIOS V2 必须成为唯一控制面,负责租户、身份、Agent、策略、任务生命周期、审批、审计、Runtime 目录和执行证据。OpenClaw 与 Hermes 只承担执行职责,不得持有第二套控制面状态。
|
||||
|
||||
## 2. 用户角色
|
||||
|
||||
| 角色 | 主要诉求 |
|
||||
|---|---|
|
||||
| 平台管理员 | 管理租户、运行时、发布、GitOps 状态和全局策略 |
|
||||
| 租户管理员 | 管理本租户 Agent、任务、审批、审计和成员权限 |
|
||||
| 操作员 | 创建任务、发起执行、取消任务、查看运行状态 |
|
||||
| 审批人 | 审批高风险任务,查看策略判定依据 |
|
||||
| 审计员 | 查询本租户完整审计链和执行证据 |
|
||||
| Runtime Worker | 领取已批准任务、心跳、返回执行结果 |
|
||||
| 开发与运维人员 | 通过 GitOps 提交、测试、部署、回滚和修复 Bug |
|
||||
|
||||
## 3. 核心用户需求
|
||||
|
||||
### UR-001 唯一控制面
|
||||
|
||||
所有任务创建、审批、取消、状态变化、策略判定和审计必须由 AIOS Control Plane 统一处理。
|
||||
|
||||
验收:
|
||||
|
||||
- Web Console、API、Codex、OpenClaw 和 Hermes 均不得绕过 AIOS 创建或修改任务生命周期。
|
||||
- OpenClaw/Hermes 的公开接口仅限执行、状态、取消、心跳和结果回传。
|
||||
|
||||
### UR-002 复用现有能力
|
||||
|
||||
系统不得重写已经稳定的 OpenClaw 执行能力、Hermes Runtime、Agent Router 或 GitOps 基础设施。
|
||||
|
||||
验收:
|
||||
|
||||
- 通过 Adapter/Boundary 接入现有能力。
|
||||
- 替换控制逻辑时保留原执行接口和可迁移数据。
|
||||
|
||||
### UR-003 OpenClaw 与 Hermes 解耦
|
||||
|
||||
OpenClaw 与 Hermes 必须从策略、租户、审批和任务生命周期中逐步解耦。
|
||||
|
||||
验收:
|
||||
|
||||
- OpenClaw 仅接收已签名、已批准、带租户上下文的命令。
|
||||
- Hermes 仅接收已批准的租户分区 Job。
|
||||
- 二者均不得直接写 Control Plane 权威状态。
|
||||
|
||||
### UR-004 全面 Web 化
|
||||
|
||||
所有面向用户和运维的能力必须通过 Web Console 或稳定 API 访问。
|
||||
|
||||
Web Console 至少展示:
|
||||
|
||||
- Tenant、Agent、Task、Approval、Runtime、Audit、Execution Evidence、GitOps Status。
|
||||
- 所有写操作由 AIOS API 完成,浏览器不得直接调用 OpenClaw/Hermes。
|
||||
|
||||
### UR-005 多租户隔离
|
||||
|
||||
所有模块必须支持租户隔离,且默认拒绝跨租户访问。
|
||||
|
||||
隔离范围:
|
||||
|
||||
- API 与会话
|
||||
- Tenant/Agent/Task 数据
|
||||
- OpenClaw 命令
|
||||
- Hermes Queue、Cell、Profile、Worker Lease
|
||||
- Memory、Cache、Artifact、SecretRef
|
||||
- Audit、Execution Evidence、日志和指标
|
||||
|
||||
验收:
|
||||
|
||||
- 资源记录的 `tenant_id` 不可为空。
|
||||
- 跨租户访问返回拒绝或“资源不存在”,不得泄露存在性。
|
||||
- 必须有跨租户负向测试。
|
||||
|
||||
### UR-006 权威任务生命周期
|
||||
|
||||
任务必须遵循统一状态机:
|
||||
|
||||
```text
|
||||
CREATED -> APPROVED -> RUNNING -> COMPLETED
|
||||
| | |
|
||||
+----------+----------+-> CANCELLED
|
||||
+-> FAILED
|
||||
```
|
||||
|
||||
验收:
|
||||
|
||||
- 禁止越级变化,例如 `CREATED -> COMPLETED`。
|
||||
- 每次变化包含 revision、principal、trace、时间和审计事件。
|
||||
|
||||
### UR-007 策略与审批
|
||||
|
||||
所有有副作用任务必须经过策略判定;高风险任务必须支持人工审批。
|
||||
|
||||
验收:
|
||||
|
||||
- Policy Decision 包含 allow/deny、reason 和 policy version。
|
||||
- 拒绝事件进入审计。
|
||||
- Runtime 不得自行放宽策略。
|
||||
|
||||
### UR-008 幂等与并发安全
|
||||
|
||||
任务和 Runtime 执行必须支持幂等与乐观并发控制。
|
||||
|
||||
验收:
|
||||
|
||||
- 相同租户与幂等键、相同内容返回同一任务/执行。
|
||||
- 相同幂等键不同内容返回冲突。
|
||||
- revision 或 branch lease 冲突不得静默覆盖。
|
||||
|
||||
### UR-009 可观测与执行证据
|
||||
|
||||
平台必须能够确认 Codex、OpenClaw、Hermes 等执行器是否领取任务、当前进度、最近提交、测试结果和完成状态。
|
||||
|
||||
验收:
|
||||
|
||||
- 存在 claim、heartbeat、progress、result 和 evidence 记录。
|
||||
- Web/API 可按租户查询。
|
||||
- 执行证据不得包含明文密钥。
|
||||
|
||||
### UR-010 GitOps 唯一变更流程
|
||||
|
||||
所有代码、配置、文档和部署资产修改必须进入非 `main` 分支,通过 commit、测试、审查和受控提升。
|
||||
|
||||
验收:
|
||||
|
||||
- Controller worktree 必须 detached 且 clean。
|
||||
- 本地 bare repo 与 Gitea 目标分支 OID 一致。
|
||||
- 不允许直接 push `main`、直接创建生产 Tag 或绕过审批部署。
|
||||
|
||||
### UR-011 持久化与恢复
|
||||
|
||||
租户、Agent、Task、Audit、Idempotency 和 Execution Evidence 必须在服务重启后恢复。
|
||||
|
||||
验收:
|
||||
|
||||
- 候选版支持 SQLite WAL 原子快照。
|
||||
- 有 restart recovery 测试。
|
||||
- 生产版需进一步提供迁移、备份、恢复、HA 和容量证据。
|
||||
|
||||
### UR-012 审计与合规
|
||||
|
||||
所有控制与执行动作必须生成不可混租户的审计事件。
|
||||
|
||||
至少记录:
|
||||
|
||||
- tenant、principal、trace、action、resource、outcome、policy version、timestamp。
|
||||
- OpenClaw/Hermes Runtime 生命周期。
|
||||
- GitOps、部署和回滚动作。
|
||||
|
||||
### UR-013 密钥安全
|
||||
|
||||
Git、日志、Web、任务 payload 和审计记录中不得出现明文 API Key、密码、Token 或私钥。
|
||||
|
||||
验收:
|
||||
|
||||
- Git 只存 `secret_ref`、`credential_ref`、`key_id` 或受保护文件路径。
|
||||
- Secret 文件权限为 `0600` 或更严格。
|
||||
- Secret Scanner 默认拒绝可疑提交。
|
||||
|
||||
### UR-014 部署与回滚
|
||||
|
||||
系统必须提供 systemd、Docker/Compose、健康检查和受保护回滚入口。
|
||||
|
||||
验收:
|
||||
|
||||
- 服务以非 root 权限运行。
|
||||
- 状态目录独立持久化。
|
||||
- 回滚前自动备份状态。
|
||||
- 默认只回滚代码,状态恢复需要显式授权。
|
||||
|
||||
### UR-015 文档与代码同步
|
||||
|
||||
任何影响用户行为、架构边界、API、数据模型、状态机、安全策略、部署或运维方式的变更,必须在同一分支同步更新 `docs/aios-v2/`。
|
||||
|
||||
验收:
|
||||
|
||||
- 架构/功能代码变化但无文档变化时,文档同步门禁失败。
|
||||
- Traceability Matrix 必须指向当前代码和测试。
|
||||
- 新架构决策必须增加或更新 ADR。
|
||||
|
||||
## 4. 非功能需求
|
||||
|
||||
### NFR-001 安全
|
||||
|
||||
- 默认拒绝、最小权限、租户隔离、身份可信链、SecretRef、审计。
|
||||
|
||||
### NFR-002 可靠性
|
||||
|
||||
- 幂等、revision、worker lease、heartbeat、重试边界、原子持久化、健康检查和回滚。
|
||||
|
||||
### NFR-003 可维护性
|
||||
|
||||
- 模块边界稳定,Control Plane 与 Runtime 解耦;代码、文档、测试和部署可追踪。
|
||||
|
||||
### NFR-004 可扩展性
|
||||
|
||||
- Store、Transport、Policy、EventPublisher、Runtime Adapter 均通过接口扩展。
|
||||
|
||||
### NFR-005 可运维性
|
||||
|
||||
- Web/API 状态、systemd/Docker、健康检查、日志、GitOps 状态、候选报告。
|
||||
|
||||
### NFR-006 性能候选指标
|
||||
|
||||
候选版上线前至少验证:
|
||||
|
||||
- API P95、任务调度延迟、队列等待、Worker 心跳、SQLite 锁等待、审计写入时间。
|
||||
- 具体 SLO 在预发布基准测试后固化,禁止无证据承诺。
|
||||
|
||||
## 5. 本候选版本范围
|
||||
|
||||
### 已纳入代码
|
||||
|
||||
- Control Plane、持久化、Gateway、Orchestrator、OpenClaw Adapter、Hermes Boundary、Execution Tracking、Web Console、GitOps Status、部署和回滚。
|
||||
|
||||
### 尚需环境验证
|
||||
|
||||
- 精确测试执行证据。
|
||||
- OpenClaw 真实 Runtime 联调。
|
||||
- 生产数据库/队列替换或容量验证。
|
||||
- 预发布性能、安全、重启和回滚演练。
|
||||
- Owner 对 merge、Tag 和生产部署的批准。
|
||||
|
||||
## 6. 用户验收场景
|
||||
|
||||
1. 平台管理员创建租户并注册 OpenClaw/Hermes Agent。
|
||||
2. 操作员创建任务,审批人批准。
|
||||
3. AIOS Orchestrator 选择 Runtime,并创建签名命令或租户分区 Job。
|
||||
4. Runtime 执行并返回结果。
|
||||
5. AIOS 更新 Task、Audit 和 Execution Evidence。
|
||||
6. 租户用户在 Web Console 查看完整链路。
|
||||
7. 跨租户访问被拒绝。
|
||||
8. 服务重启后状态恢复。
|
||||
9. 候选部署健康检查通过,回滚演练成功。
|
||||
@@ -0,0 +1,461 @@
|
||||
# AIOS V2 目标架构
|
||||
|
||||
- 文档版本:`V2R1C0-candidate`
|
||||
- 架构状态:目标架构已实现为候选代码,待预发布验证
|
||||
- 适用范围:Control Plane、Agent Router、OpenClaw、Hermes、Web Console、GitOps、审计、部署与运维
|
||||
|
||||
## 1. 架构目标
|
||||
|
||||
AIOS V2 的核心目标是把分散在 OpenClaw、Hermes、Agent Router、局部 WebUI 和 GitOps 中的控制能力收敛为一个权威控制面,同时保留现有执行能力,避免推倒重写。
|
||||
|
||||
目标状态:
|
||||
|
||||
```text
|
||||
用户 / Web Console / Public API
|
||||
|
|
||||
v
|
||||
AIOS Control Plane
|
||||
Tenant | Identity | Agent | Task | Policy | Approval | Audit
|
||||
|
|
||||
v
|
||||
Execution Orchestrator / Agent Router
|
||||
/ \
|
||||
v v
|
||||
OpenClaw Adapter Hermes Boundary
|
||||
| |
|
||||
v v
|
||||
OpenClaw Runtime Hermes Workers
|
||||
\ /
|
||||
v v
|
||||
Result Reconciliation + Execution Evidence
|
||||
```
|
||||
|
||||
设计结论:
|
||||
|
||||
- AIOS 是唯一控制面。
|
||||
- Agent Router 是调度平面,不是第二控制面。
|
||||
- OpenClaw 是执行适配与 Runtime。
|
||||
- Hermes 是队列、Worker、Profile 与执行 Runtime。
|
||||
- Web Console 只调用 AIOS API。
|
||||
- GitOps 是代码、配置、文档和发布变更的唯一入口。
|
||||
|
||||
## 2. 从现有架构到目标架构
|
||||
|
||||
### 2.1 现有能力
|
||||
|
||||
现有系统已经具备:
|
||||
|
||||
- OpenClaw 执行与 Fleet 能力。
|
||||
- Hermes Worker、Profile、队列和工作区能力。
|
||||
- Agent Router 路由经验和部分调度接口。
|
||||
- GitOps 分支、提交、审批、回滚和审计能力。
|
||||
- Tenant Context、Tenant Router、Memory Scope、SecretRef、Message Ledger 和审计候选实现。
|
||||
- 局部 WebUI、Gateway 和自动任务领取能力。
|
||||
|
||||
### 2.2 现有问题
|
||||
|
||||
- 控制逻辑分散,可能存在多个任务事实源。
|
||||
- OpenClaw/Hermes 内部仍可能保存租户、策略或任务状态。
|
||||
- 候选能力分散在多个分支,主线缺少权威实现。
|
||||
- WebUI 与 API 入口不统一。
|
||||
- 分支名称、代码、测试和部署证据之间缺少映射。
|
||||
- 多租户隔离多为逻辑约束,尚缺全链路生产证据。
|
||||
|
||||
### 2.3 演进原则
|
||||
|
||||
- 先加边界和适配器,再迁移控制权。
|
||||
- 先复用接口,再替换内部实现。
|
||||
- 先在候选分支验证,再受控提升。
|
||||
- 不允许同时存在两套权威任务生命周期。
|
||||
- 不以“大分支存在”代替验收证据。
|
||||
|
||||
## 3. 逻辑分层
|
||||
|
||||
### 3.1 Web 与 API 层
|
||||
|
||||
职责:
|
||||
|
||||
- 提供 Tenant、Agent、Task、Approval、Runtime、Audit、Execution Evidence 和 GitOps 页面/API。
|
||||
- 依赖可信身份代理注入 `X-AIOS-*` 身份头。
|
||||
- 不允许浏览器直接访问 OpenClaw 或 Hermes。
|
||||
- 不允许客户端自行选择或覆盖 `tenant_id`。
|
||||
|
||||
候选实现:
|
||||
|
||||
- `active-runtime/app/aios_v2/app.py`
|
||||
- `active-runtime/app/aios_v2/gateway.py`
|
||||
- `active-runtime/app/aios_v2/console.py`
|
||||
|
||||
### 3.2 AIOS Control Plane
|
||||
|
||||
权威职责:
|
||||
|
||||
- Tenant Registry
|
||||
- Agent Registry
|
||||
- Task Lifecycle
|
||||
- Policy Decision
|
||||
- Approval
|
||||
- Audit Event
|
||||
- Runtime Inventory
|
||||
- Execution Evidence
|
||||
- Deployment/GitOps Status
|
||||
|
||||
不负责:
|
||||
|
||||
- OpenClaw 内部执行细节。
|
||||
- Hermes Worker 内部执行细节。
|
||||
- 直接持有 Runtime 凭据明文。
|
||||
|
||||
候选实现:
|
||||
|
||||
- `active-runtime/app/aios_v2/control_plane.py`
|
||||
- `active-runtime/app/aios_v2/persistence.py`
|
||||
- `active-runtime/app/aios_v2/execution_tracking.py`
|
||||
- `active-runtime/app/aios_v2/gitops_status.py`
|
||||
|
||||
### 3.3 Execution Orchestrator / Agent Router
|
||||
|
||||
职责:
|
||||
|
||||
- 只接受 `APPROVED` 任务。
|
||||
- 根据 Agent Registry 和 Policy Result 选择 Runtime。
|
||||
- 生成 OpenClawCommand 或 HermesJob。
|
||||
- 传播 tenant、task、trace、idempotency 和 policy version。
|
||||
- 将 Runtime Result 回写 AIOS Task。
|
||||
|
||||
不负责:
|
||||
|
||||
- 创建租户。
|
||||
- 修改策略。
|
||||
- 绕过审批。
|
||||
- 保存第二套任务状态机。
|
||||
|
||||
候选实现:
|
||||
|
||||
- `active-runtime/app/aios_v2/orchestrator.py`
|
||||
|
||||
### 3.4 OpenClaw Adapter
|
||||
|
||||
职责:
|
||||
|
||||
- 验证签名命令。
|
||||
- 执行、查询、取消、健康检查和回调。
|
||||
- 支持幂等、超时和租户隔离。
|
||||
- 把 Runtime 生命周期转换为 AIOS 审计事件。
|
||||
|
||||
不负责:
|
||||
|
||||
- Tenant、Policy、Approval、Task Lifecycle。
|
||||
- 向浏览器暴露公网控制接口。
|
||||
|
||||
候选实现:
|
||||
|
||||
- `active-runtime/app/aios_openclaw/contract.py`
|
||||
- `active-runtime/app/aios_openclaw/adapter.py`
|
||||
- `active-runtime/app/aios_openclaw/transport.py`
|
||||
- `active-runtime/app/aios_openclaw/health.py`
|
||||
- `active-runtime/app/aios_openclaw/callback.py`
|
||||
|
||||
### 3.5 Hermes Runtime Boundary
|
||||
|
||||
职责:
|
||||
|
||||
- Tenant/Cell/Profile 分区队列。
|
||||
- Worker claim、start、heartbeat、complete、fail、cancel。
|
||||
- Lease 超时和自动回队列。
|
||||
- Result 回传和 Runtime Health。
|
||||
|
||||
不负责:
|
||||
|
||||
- Policy、Approval、Tenant Registry、权威 Task Lifecycle。
|
||||
|
||||
候选实现:
|
||||
|
||||
- `active-runtime/app/aios_hermes/contract.py`
|
||||
- `active-runtime/app/aios_hermes/adapter.py`
|
||||
- `active-runtime/app/aios_hermes/worker.py`
|
||||
|
||||
### 3.6 Persistence
|
||||
|
||||
候选实现使用 SQLite WAL 原子快照,持久化:
|
||||
|
||||
- Tenant
|
||||
- Agent
|
||||
- Task
|
||||
- Audit
|
||||
- Idempotency
|
||||
- Execution Evidence 文件
|
||||
|
||||
生产演进:
|
||||
|
||||
- Store 接口保持稳定。
|
||||
- 可替换为 PostgreSQL、对象存储或分布式事件存储。
|
||||
- 生产替换必须提供迁移、备份、恢复、HA、容量和租户隔离证据。
|
||||
|
||||
## 4. 多租户架构
|
||||
|
||||
### 4.1 权威租户上下文
|
||||
|
||||
每次请求和执行至少传播:
|
||||
|
||||
```text
|
||||
tenant_id
|
||||
principal_id
|
||||
session_id
|
||||
trace_id
|
||||
task_id
|
||||
policy_version
|
||||
```
|
||||
|
||||
Runtime 还传播:
|
||||
|
||||
```text
|
||||
command_id / job_id
|
||||
cell_id
|
||||
profile_id
|
||||
idempotency_key
|
||||
worker_id
|
||||
```
|
||||
|
||||
### 4.2 隔离层级
|
||||
|
||||
| 层级 | 隔离要求 |
|
||||
|---|---|
|
||||
| HTTP/API | 可信身份头、租户不可由浏览器覆盖 |
|
||||
| Control Plane | 所有资源带 `tenant_id`,跨租户默认拒绝 |
|
||||
| OpenClaw | 签名命令绑定 tenant/task/trace |
|
||||
| Hermes | tenant/cell/profile 三层分区 |
|
||||
| Storage | 候选为显式租户字段;生产需 RLS 或物理隔离策略 |
|
||||
| Evidence/Audit | 仅按当前租户查询 |
|
||||
| Cache/Memory | key 必须包含 tenant namespace |
|
||||
| Secret | 只保存引用,不保存明文 |
|
||||
| Logs/Metrics | 必须包含 tenant/task/trace 标签,并限制查询范围 |
|
||||
|
||||
### 4.3 失败行为
|
||||
|
||||
- 跨租户资源访问返回拒绝或“未找到”。
|
||||
- 不泄露其他租户资源存在性。
|
||||
- 缺少 tenant context 时 fail closed。
|
||||
- Runtime 返回的 tenant/task/trace 不匹配时拒绝回写。
|
||||
|
||||
## 5. 任务与执行数据流
|
||||
|
||||
### 5.1 任务创建与审批
|
||||
|
||||
```text
|
||||
User/API
|
||||
|
|
||||
v
|
||||
Trusted Identity
|
||||
|
|
||||
v
|
||||
AIOS Gateway
|
||||
|
|
||||
v
|
||||
Policy Gate
|
||||
|
|
||||
v
|
||||
Task CREATED
|
||||
|
|
||||
v
|
||||
Approval
|
||||
|
|
||||
v
|
||||
Task APPROVED
|
||||
```
|
||||
|
||||
### 5.2 OpenClaw 执行
|
||||
|
||||
```text
|
||||
APPROVED Task
|
||||
|
|
||||
v
|
||||
Execution Orchestrator
|
||||
|
|
||||
v
|
||||
Signed OpenClawCommand
|
||||
|
|
||||
v
|
||||
OpenClaw Adapter
|
||||
|
|
||||
v
|
||||
Runtime Result
|
||||
|
|
||||
v
|
||||
Task Reconciliation + Audit + Evidence
|
||||
```
|
||||
|
||||
### 5.3 Hermes 执行
|
||||
|
||||
```text
|
||||
APPROVED Task
|
||||
|
|
||||
v
|
||||
Execution Orchestrator
|
||||
|
|
||||
v
|
||||
Tenant/Cell/Profile HermesJob
|
||||
|
|
||||
v
|
||||
Worker Claim + Lease + Heartbeat
|
||||
|
|
||||
v
|
||||
HermesResult
|
||||
|
|
||||
v
|
||||
Task Reconciliation + Audit + Evidence
|
||||
```
|
||||
|
||||
## 6. 安全架构
|
||||
|
||||
### 6.1 身份边界
|
||||
|
||||
- Public API 位于可信反向代理之后。
|
||||
- 反向代理完成登录、会话、MFA 和身份头注入。
|
||||
- 候选应用拒绝未认证身份断言。
|
||||
- 生产不得允许客户端直接伪造 `X-AIOS-*`。
|
||||
|
||||
### 6.2 Runtime 命令完整性
|
||||
|
||||
OpenClawCommand 使用规范化序列化和 HMAC-SHA256:
|
||||
|
||||
- 签名覆盖 tenant、task、trace、action、payload、idempotency 和时间。
|
||||
- 支持 key ID、到期时间和时钟偏差。
|
||||
- Runtime 密钥通过受保护文件或外部 Secret Provider 解析。
|
||||
|
||||
### 6.3 Secret 边界
|
||||
|
||||
Git 允许:
|
||||
|
||||
```text
|
||||
secret_ref
|
||||
credential_ref
|
||||
key_id
|
||||
受保护文件路径
|
||||
```
|
||||
|
||||
Git 禁止:
|
||||
|
||||
```text
|
||||
API Key
|
||||
密码
|
||||
Token
|
||||
私钥
|
||||
签名材料
|
||||
```
|
||||
|
||||
### 6.4 Web 安全
|
||||
|
||||
候选应用设置:
|
||||
|
||||
- no-store
|
||||
- nosniff
|
||||
- DENY frame
|
||||
- no-referrer
|
||||
- CSP
|
||||
- 请求体大小限制
|
||||
|
||||
生产还需验证:
|
||||
|
||||
- CSRF/CORS
|
||||
- Rate Limit
|
||||
- Session Timeout
|
||||
- CSP Nonce/Hash
|
||||
- WAF 和反向代理访问策略
|
||||
|
||||
## 7. 可靠性架构
|
||||
|
||||
### 7.1 幂等
|
||||
|
||||
- Control Plane:`tenant_id + idempotency_key + request_digest`。
|
||||
- OpenClaw:`tenant_id + idempotency_key + signature`。
|
||||
- Hermes:`tenant_id + idempotency_key + job_digest`。
|
||||
|
||||
### 7.2 并发
|
||||
|
||||
- Task 使用 revision。
|
||||
- Hermes 使用 revision + worker lease。
|
||||
- GitOps 使用 detached controller + branch lease。
|
||||
- 持久化使用 SQLite `BEGIN IMMEDIATE` 和原子快照。
|
||||
|
||||
### 7.3 恢复
|
||||
|
||||
- Controller restart:detached guard。
|
||||
- AIOS restart:SQLite restore。
|
||||
- Worker crash:lease 到期后 Job 回到 QUEUED。
|
||||
- Release failure:健康检查失败自动恢复 previous release。
|
||||
|
||||
## 8. 可观测架构
|
||||
|
||||
统一关联键:
|
||||
|
||||
```text
|
||||
tenant_id
|
||||
principal_id
|
||||
trace_id
|
||||
task_id
|
||||
agent_id
|
||||
command_id / job_id
|
||||
worker_id
|
||||
commit
|
||||
```
|
||||
|
||||
Web Console 展示:
|
||||
|
||||
- Tasks
|
||||
- Agents
|
||||
- Runtime Health
|
||||
- Audit
|
||||
- Execution Evidence
|
||||
- GitOps Status
|
||||
|
||||
生产还需接入:
|
||||
|
||||
- 结构化日志
|
||||
- Metrics
|
||||
- Distributed Tracing
|
||||
- Alerting
|
||||
- SLO Dashboard
|
||||
|
||||
## 9. 部署架构
|
||||
|
||||
支持两种候选部署:
|
||||
|
||||
### systemd
|
||||
|
||||
- 非 root `aios` 用户。
|
||||
- `ProtectSystem=strict`、`NoNewPrivileges` 等安全选项。
|
||||
- 状态目录 `/var/lib/aios-v2`。
|
||||
|
||||
### Docker/Compose
|
||||
|
||||
- 非 root UID/GID。
|
||||
- Read-only root filesystem。
|
||||
- Drop all capabilities。
|
||||
- State volume。
|
||||
- HTTP healthcheck。
|
||||
|
||||
## 10. 架构约束
|
||||
|
||||
以下行为视为架构违规:
|
||||
|
||||
1. OpenClaw/Hermes 直接创建或批准 AIOS Task。
|
||||
2. Web Console 直接调用 Runtime。
|
||||
3. 新模块不携带 `tenant_id`。
|
||||
4. 新增第二套任务状态机或租户事实源。
|
||||
5. 在 Git 中提交明文凭据。
|
||||
6. 绕过 GitOps 修改生产代码/配置。
|
||||
7. 架构或功能变化未同步更新 `docs/aios-v2/`。
|
||||
8. 未通过测试和预发布门禁直接提升候选版本。
|
||||
|
||||
## 11. 目标架构完成判定
|
||||
|
||||
目标架构只有在以下证据齐全时才视为完成:
|
||||
|
||||
- 文档与代码映射完整。
|
||||
- Candidate tests 对精确 commit 执行通过。
|
||||
- OpenClaw/Hermes 实际 Runtime 联调通过。
|
||||
- 多租户负向测试通过。
|
||||
- 重启恢复和回滚演练通过。
|
||||
- GitOps 本地/Gitea OID 一致。
|
||||
- Owner 批准 merge、Tag 和生产部署。
|
||||
@@ -0,0 +1,313 @@
|
||||
# AIOS V2 迁移与交付策略
|
||||
|
||||
- 文档版本:`V2R1C0-candidate`
|
||||
- 交付原则:增量迁移、能力复用、控制权收敛、GitOps 唯一变更入口
|
||||
|
||||
## 1. 迁移目标
|
||||
|
||||
AIOS V2 不通过一次性重写替换现有系统,而是在保留 OpenClaw、Hermes、Agent Router、GitOps 和既有运行能力的前提下,将租户、策略、任务、审批、审计和发布治理收敛到 AIOS Control Plane。
|
||||
|
||||
迁移完成的标志不是“新代码存在”,而是:
|
||||
|
||||
- AIOS 成为唯一权威控制面;
|
||||
- OpenClaw/Hermes 不再保存第二套控制状态;
|
||||
- 所有控制功能 Web/API 化;
|
||||
- 所有模块通过租户隔离、测试和预发布门禁;
|
||||
- 本地 Git、Gitea、测试、部署和回滚证据可追踪。
|
||||
|
||||
## 2. 为什么采用增量迁移
|
||||
|
||||
### 2.1 避免能力回退
|
||||
|
||||
OpenClaw、Hermes 和 Agent Router 已经包含大量运行经验、边界处理和执行能力。整体重写会扩大风险面,并导致业务能力、操作流程和历史数据回退。
|
||||
|
||||
### 2.2 保持可回滚
|
||||
|
||||
每个迁移步骤必须:
|
||||
|
||||
- 独立分支;
|
||||
- 小范围提交;
|
||||
- 独立测试;
|
||||
- 可单独回滚;
|
||||
- 不要求立即关闭旧执行路径。
|
||||
|
||||
### 2.3 先收敛控制权,再优化内部实现
|
||||
|
||||
迁移优先级:
|
||||
|
||||
```text
|
||||
唯一入口
|
||||
-> 权威租户/任务/策略
|
||||
-> Runtime Adapter
|
||||
-> 数据层生产化
|
||||
-> Web 与可观测
|
||||
-> 性能与 HA
|
||||
```
|
||||
|
||||
## 3. 分阶段迁移路线
|
||||
|
||||
### Phase 0:GitOps 与架构事实源
|
||||
|
||||
目标:建立可靠变更链和权威设计基线。
|
||||
|
||||
已完成候选能力:
|
||||
|
||||
- Controller detached guard;
|
||||
- 避免 fetch 到已检出本地分支;
|
||||
- Gitea SSH 镜像;
|
||||
- `docs/aios-v2/` 架构事实源;
|
||||
- 候选分支、部署和回滚资产。
|
||||
|
||||
退出条件:
|
||||
|
||||
- Controller detached 且 clean;
|
||||
- 本地/Gitea OID 一致;
|
||||
- 架构与需求文档纳入版本。
|
||||
|
||||
### Phase 1:Control Plane 权威化
|
||||
|
||||
目标:将 Tenant、Agent、Task、Policy、Approval 和 Audit 收敛到 AIOS。
|
||||
|
||||
候选实现:
|
||||
|
||||
- `aios_v2/control_plane.py`
|
||||
- `aios_v2/persistence.py`
|
||||
- `aios_v2/gateway.py`
|
||||
- `aios_v2/app.py`
|
||||
|
||||
退出条件:
|
||||
|
||||
- 任务状态机统一;
|
||||
- 幂等、revision、审计和跨租户拒绝通过测试;
|
||||
- 服务重启后权威状态恢复。
|
||||
|
||||
### Phase 2:OpenClaw 解耦
|
||||
|
||||
目标:OpenClaw 从控制逻辑转为执行适配器。
|
||||
|
||||
步骤:
|
||||
|
||||
1. 定义签名命令合同;
|
||||
2. 引入 Adapter;
|
||||
3. 将直接调用迁移到 AIOS Orchestrator;
|
||||
4. Runtime 只返回执行结果和健康状态;
|
||||
5. 关闭或网络隔离旧公共控制入口。
|
||||
|
||||
退出条件:
|
||||
|
||||
- 未批准任务无法生成有效命令;
|
||||
- 命令绑定 tenant/task/trace/idempotency;
|
||||
- 旧控制接口冻结并记录替代路径。
|
||||
|
||||
### Phase 3:Hermes 解耦
|
||||
|
||||
目标:Hermes 只承担租户分区队列和 Worker Runtime。
|
||||
|
||||
步骤:
|
||||
|
||||
1. 定义 HermesJob/HermesResult;
|
||||
2. 强制 tenant/cell/profile 分区;
|
||||
3. 引入 claim/lease/heartbeat;
|
||||
4. 将结果统一回写 AIOS;
|
||||
5. 移除 Hermes 内部策略和任务事实源。
|
||||
|
||||
退出条件:
|
||||
|
||||
- Worker 无权创建 AIOS Task;
|
||||
- 租约过期后 Job 可恢复;
|
||||
- 跨租户 claim/read/result 被拒绝。
|
||||
|
||||
### Phase 4:Web 与运维统一
|
||||
|
||||
目标:所有用户与运维功能通过 AIOS Web/API。
|
||||
|
||||
范围:
|
||||
|
||||
- Tenant、Agent、Task、Runtime、Audit、Execution Evidence、GitOps;
|
||||
- systemd/Docker/Compose;
|
||||
- healthcheck、rollback、release report。
|
||||
|
||||
退出条件:
|
||||
|
||||
- 浏览器不直接调用 Runtime;
|
||||
- 生产身份代理和权限模型通过验证;
|
||||
- 部署、重启和回滚演练通过。
|
||||
|
||||
### Phase 5:生产数据层与 HA
|
||||
|
||||
目标:将候选 SQLite/内存运行状态替换或强化为生产级后端。
|
||||
|
||||
要求:
|
||||
|
||||
- 数据迁移;
|
||||
- RLS 或物理隔离;
|
||||
- 备份恢复;
|
||||
- HA 与容量;
|
||||
- 多实例并发;
|
||||
- 灾难恢复目标。
|
||||
|
||||
该阶段不得改变上层合同,避免 Runtime 和 Web 再次耦合存储实现。
|
||||
|
||||
## 4. 分支与提交策略
|
||||
|
||||
### 4.1 分支类型
|
||||
|
||||
| 类型 | 用途 |
|
||||
|---|---|
|
||||
| `task/*` | 可执行任务合同与主体实现 |
|
||||
| `patch/*` | 独立修复或窄范围增强 |
|
||||
| `integration/*` | 多模块候选收敛 |
|
||||
| `result/*` | 自动执行结果与证据 |
|
||||
| `release/*` | 受控发布候选 |
|
||||
|
||||
### 4.2 规则
|
||||
|
||||
- 禁止直接修改 `main`;
|
||||
- 一个提交应可独立审阅和回滚;
|
||||
- 大文件和大范围变更应拆分;
|
||||
- 所有行为变化必须同步文档和测试;
|
||||
- 生产 Tag、部署和回滚需要 Owner 批准。
|
||||
|
||||
## 5. GitOps 交付流程
|
||||
|
||||
```text
|
||||
需求/架构更新
|
||||
|
|
||||
v
|
||||
任务分支 + 文档 + 测试
|
||||
|
|
||||
v
|
||||
代码提交 + Secret Gate
|
||||
|
|
||||
v
|
||||
本地 bare repo 同步
|
||||
|
|
||||
v
|
||||
Gitea 镜像与 OID 校验
|
||||
|
|
||||
v
|
||||
Candidate 集成
|
||||
|
|
||||
v
|
||||
自动测试与预发布
|
||||
|
|
||||
v
|
||||
Owner 审批
|
||||
|
|
||||
v
|
||||
Merge / Tag / Production Deploy
|
||||
```
|
||||
|
||||
### 5.1 同步状态
|
||||
|
||||
GitOps 返回必须区分:
|
||||
|
||||
- local sync;
|
||||
- Gitea sync;
|
||||
- branch OID;
|
||||
- workspace mode;
|
||||
- dirty state。
|
||||
|
||||
仅返回 `common_git_dir synced` 不足以证明 Web 远端可见。
|
||||
|
||||
## 6. 发布门禁
|
||||
|
||||
### Gate A:设计完整性
|
||||
|
||||
- 用户需求、目标架构、模块契约、ADR 和追踪矩阵同步;
|
||||
- 无冲突标记、断链或过期路径。
|
||||
|
||||
### Gate B:代码质量
|
||||
|
||||
- 包可导入;
|
||||
- 静态编译通过;
|
||||
- Secret Scanner 通过;
|
||||
- 无已知越权旁路。
|
||||
|
||||
### Gate C:测试
|
||||
|
||||
- 单元、集成、多租户负向、重启恢复、Runtime 边界测试通过;
|
||||
- 记录 commit、命令、exit code、pass/fail 和 artifact digest。
|
||||
|
||||
### Gate D:预发布
|
||||
|
||||
- systemd 或 Compose 启动成功;
|
||||
- 健康检查通过;
|
||||
- OpenClaw/Hermes 真实联调通过;
|
||||
- 备份、重启和回滚演练通过;
|
||||
- 日志中无敏感信息。
|
||||
|
||||
### Gate E:生产批准
|
||||
|
||||
- Owner 批准 merge、Tag 和生产部署;
|
||||
- 发布窗口、回滚负责人和观察指标明确;
|
||||
- 生产变更通过 GitOps 记录。
|
||||
|
||||
## 7. 回滚策略
|
||||
|
||||
### 7.1 代码回滚
|
||||
|
||||
- 使用 `/opt/aios/current` 原子 symlink 切换;
|
||||
- 回滚前备份状态;
|
||||
- 健康检查失败自动恢复 previous release。
|
||||
|
||||
### 7.2 状态回滚
|
||||
|
||||
默认不自动回滚状态,避免覆盖候选运行期间新增的 Task、Audit 和 Evidence。
|
||||
|
||||
状态恢复只有在:
|
||||
|
||||
- 明确确认 schema 不兼容;
|
||||
- 数据备份可验证;
|
||||
- Owner 显式授权;
|
||||
|
||||
时启用。
|
||||
|
||||
### 7.3 Runtime 回滚
|
||||
|
||||
OpenClaw/Hermes Adapter 采用稳定合同。若候选执行层失败,可恢复旧 Runtime 版本,但不得恢复旧控制权旁路。
|
||||
|
||||
## 8. Bug 修复流程
|
||||
|
||||
生产或预发布 Bug:
|
||||
|
||||
1. 创建 `patch/bugfix/*`;
|
||||
2. 添加可复现测试;
|
||||
3. 修复代码;
|
||||
4. 更新相关需求/架构/契约/ADR;
|
||||
5. 运行候选验收;
|
||||
6. 进入新的 release candidate;
|
||||
7. Owner 批准部署。
|
||||
|
||||
禁止在生产目录直接编辑文件后不回写 Git。
|
||||
|
||||
## 9. 文档同步规则
|
||||
|
||||
以下变化必须更新文档:
|
||||
|
||||
- 用户行为或页面;
|
||||
- API 路径、Header、请求/响应;
|
||||
- 数据模型、状态机、策略;
|
||||
- OpenClaw/Hermes/Router 边界;
|
||||
- 多租户与安全模型;
|
||||
- 部署、健康检查、回滚;
|
||||
- 测试或发布门禁。
|
||||
|
||||
对应关系:
|
||||
|
||||
- 需求变化:更新 `01-USER-REQUIREMENTS.md`;
|
||||
- 架构变化:更新 `02-TARGET-ARCHITECTURE.md` 和 ADR;
|
||||
- 迁移/发布变化:更新本文件;
|
||||
- 合同变化:更新 `04-MODULE-CONTRACTS.md`;
|
||||
- 代码/测试路径变化:更新 `05-TRACEABILITY-MATRIX.md`。
|
||||
|
||||
## 10. 当前候选后续步骤
|
||||
|
||||
1. 完成文档同步门禁和 Acceptance Runner;
|
||||
2. 在精确 Candidate commit 执行测试;
|
||||
3. 推送 Gitea 并校验 OID;
|
||||
4. 部署隔离预发布环境;
|
||||
5. 联调真实 OpenClaw/Hermes;
|
||||
6. 执行重启、性能、安全和回滚演练;
|
||||
7. 生成最终 evidence;
|
||||
8. 请求 Owner 的 merge/Tag/production approval。
|
||||
@@ -0,0 +1,502 @@
|
||||
# AIOS V2 模块契约
|
||||
|
||||
- 文档版本:`V2R1C0-candidate`
|
||||
- 状态:候选实现契约基线
|
||||
- 原则:Control Plane 权威、Runtime 执行、全链路租户隔离、所有状态可审计
|
||||
|
||||
## 1. 通用上下文契约
|
||||
|
||||
所有控制与执行请求必须携带以下上下文:
|
||||
|
||||
| 字段 | 必填 | 说明 |
|
||||
|---|---:|---|
|
||||
| `tenant_id` | 是 | 权威租户标识 |
|
||||
| `principal_id` | 控制请求是 | 已认证主体 |
|
||||
| `trace_id` | 是 | 全链路追踪 |
|
||||
| `task_id` | 执行阶段是 | AIOS 权威任务 |
|
||||
| `session_id` | Web/交互请求建议 | 用户会话 |
|
||||
| `policy_version` | 控制请求是 | 策略版本 |
|
||||
| `idempotency_key` | 创建/执行是 | 幂等键 |
|
||||
|
||||
规则:
|
||||
|
||||
- 缺少必填字段时 fail closed。
|
||||
- 客户端不得覆盖由可信代理或 AIOS 生成的 `tenant_id`。
|
||||
- Runtime 返回的 tenant/task/trace 必须与请求完全匹配。
|
||||
- 所有持久化键、缓存键和队列分区必须包含租户命名空间。
|
||||
|
||||
## 2. Tenant Contract
|
||||
|
||||
### 2.1 TenantRecord
|
||||
|
||||
```text
|
||||
tenant_id
|
||||
display_name
|
||||
status
|
||||
metadata
|
||||
created_at
|
||||
updated_at
|
||||
```
|
||||
|
||||
约束:
|
||||
|
||||
- `tenant_id` 不可为空且不可变。
|
||||
- `status` 至少支持 `active` 和受控停用状态。
|
||||
- Tenant 只由 AIOS Control Plane 创建和维护。
|
||||
- Runtime 不得创建租户。
|
||||
|
||||
实现:`active-runtime/app/aios_v2/control_plane.py`
|
||||
|
||||
## 3. Agent Contract
|
||||
|
||||
### 3.1 AgentRecord
|
||||
|
||||
```text
|
||||
agent_id
|
||||
tenant_id
|
||||
display_name
|
||||
runtime
|
||||
capabilities
|
||||
enabled
|
||||
metadata
|
||||
created_at
|
||||
updated_at
|
||||
```
|
||||
|
||||
约束:
|
||||
|
||||
- Agent 属于唯一租户。
|
||||
- `runtime` 由 Registry 决定,可取 `openclaw`、`hermes` 或后续注册 Runtime。
|
||||
- `capabilities` 是允许调度的动作集合。
|
||||
- `metadata` 可存 `cell_id`、`profile_id`、priority 等非敏感引用。
|
||||
- 不得在 Agent metadata 中保存明文凭据。
|
||||
|
||||
## 4. Task Contract
|
||||
|
||||
### 4.1 TaskRecord
|
||||
|
||||
```text
|
||||
task_id
|
||||
tenant_id
|
||||
agent_id
|
||||
action
|
||||
payload
|
||||
state
|
||||
revision
|
||||
idempotency_key
|
||||
request_digest
|
||||
result
|
||||
error
|
||||
created_by
|
||||
approved_by
|
||||
created_at
|
||||
updated_at
|
||||
```
|
||||
|
||||
### 4.2 状态机
|
||||
|
||||
```text
|
||||
CREATED -> APPROVED -> RUNNING -> COMPLETED
|
||||
| | |
|
||||
+----------+----------+-> CANCELLED
|
||||
+-> FAILED
|
||||
```
|
||||
|
||||
允许转换:
|
||||
|
||||
| 当前 | 允许目标 |
|
||||
|---|---|
|
||||
| CREATED | APPROVED, CANCELLED |
|
||||
| APPROVED | RUNNING, CANCELLED |
|
||||
| RUNNING | COMPLETED, FAILED, CANCELLED |
|
||||
| COMPLETED | 无 |
|
||||
| FAILED | 无 |
|
||||
| CANCELLED | 无 |
|
||||
|
||||
约束:
|
||||
|
||||
- 每次转换必须校验 `revision`。
|
||||
- COMPLETED 必须带 `result`。
|
||||
- FAILED 必须带 `error`。
|
||||
- 每次转换生成 AuditEvent。
|
||||
- Runtime 不得直接写 Task Store,只能通过 Orchestrator reconcile。
|
||||
|
||||
## 5. Policy Contract
|
||||
|
||||
### 5.1 PolicyDecision
|
||||
|
||||
```text
|
||||
allowed
|
||||
reason
|
||||
policy_version
|
||||
```
|
||||
|
||||
约束:
|
||||
|
||||
- Policy 默认拒绝。
|
||||
- Runtime 不得覆盖 PolicyDecision。
|
||||
- 拒绝必须审计。
|
||||
- 高风险动作必须经过 Approval。
|
||||
|
||||
## 6. Audit Contract
|
||||
|
||||
### 6.1 AuditEvent
|
||||
|
||||
```text
|
||||
event_id
|
||||
tenant_id
|
||||
principal_id
|
||||
trace_id
|
||||
action
|
||||
resource_type
|
||||
resource_id
|
||||
outcome
|
||||
policy_version
|
||||
details
|
||||
created_at
|
||||
```
|
||||
|
||||
约束:
|
||||
|
||||
- 按租户查询。
|
||||
- `details` 不得包含明文凭据。
|
||||
- OpenClaw/Hermes 生命周期事件必须映射到统一 AuditEvent。
|
||||
- 生产版需增加不可变外部固化和完整性验证。
|
||||
|
||||
## 7. OpenClaw Contract
|
||||
|
||||
### 7.1 OpenClawCommand
|
||||
|
||||
```text
|
||||
contract_version
|
||||
command_id
|
||||
tenant_id
|
||||
task_id
|
||||
trace_id
|
||||
action
|
||||
payload
|
||||
idempotency_key
|
||||
created_at
|
||||
expires_at
|
||||
key_id
|
||||
signature
|
||||
```
|
||||
|
||||
签名覆盖:
|
||||
|
||||
- command_id
|
||||
- tenant_id
|
||||
- task_id
|
||||
- trace_id
|
||||
- action
|
||||
- payload
|
||||
- idempotency_key
|
||||
- created_at / expires_at
|
||||
- contract_version / key_id
|
||||
|
||||
约束:
|
||||
|
||||
- 使用规范化 JSON 和 HMAC-SHA256。
|
||||
- 只接受已批准任务生成的命令。
|
||||
- 过期、未来时间超差、签名错误命令必须拒绝。
|
||||
- `key_id` 只引用运行时材料,不包含材料本身。
|
||||
|
||||
### 7.2 OpenClawResult
|
||||
|
||||
```text
|
||||
command_id
|
||||
tenant_id
|
||||
task_id
|
||||
trace_id
|
||||
state
|
||||
output
|
||||
error
|
||||
started_at
|
||||
finished_at
|
||||
runtime_id
|
||||
attempts
|
||||
```
|
||||
|
||||
状态:
|
||||
|
||||
```text
|
||||
ACCEPTED
|
||||
RUNNING
|
||||
COMPLETED
|
||||
FAILED
|
||||
CANCELLED
|
||||
TIMED_OUT
|
||||
REJECTED
|
||||
```
|
||||
|
||||
约束:
|
||||
|
||||
- Adapter 对外只接受终态结果用于 reconcile。
|
||||
- 身份字段不匹配时拒绝回写。
|
||||
- 结果通过 Orchestrator 更新 AIOS Task。
|
||||
|
||||
### 7.3 RuntimeHealth
|
||||
|
||||
```text
|
||||
runtime
|
||||
runtime_id
|
||||
status
|
||||
version
|
||||
heartbeat_at
|
||||
active_tasks
|
||||
workspace_isolated
|
||||
credential_ref_configured
|
||||
details
|
||||
```
|
||||
|
||||
健康策略:
|
||||
|
||||
- heartbeat 不得超时。
|
||||
- workspace isolation 与 credential reference 必须符合策略。
|
||||
- 不安全 Runtime 应 quarantine,不得继续接收新任务。
|
||||
|
||||
实现:`active-runtime/app/aios_openclaw/`
|
||||
|
||||
## 8. Hermes Contract
|
||||
|
||||
### 8.1 HermesJob
|
||||
|
||||
```text
|
||||
job_id
|
||||
tenant_id
|
||||
cell_id
|
||||
profile_id
|
||||
task_id
|
||||
trace_id
|
||||
action
|
||||
payload
|
||||
idempotency_key
|
||||
priority
|
||||
state
|
||||
revision
|
||||
assigned_worker_id
|
||||
lease_expires_at
|
||||
created_at
|
||||
updated_at
|
||||
```
|
||||
|
||||
状态机:
|
||||
|
||||
```text
|
||||
QUEUED -> CLAIMED -> RUNNING -> COMPLETED
|
||||
| | |
|
||||
+---------+----------+-> CANCELLED
|
||||
+-> FAILED
|
||||
```
|
||||
|
||||
约束:
|
||||
|
||||
- `tenant_id + cell_id + profile_id` 形成执行分区。
|
||||
- claim 只能领取同分区 Job。
|
||||
- Worker ID 不得跨分区复用。
|
||||
- Lease 到期 Job 回到 QUEUED。
|
||||
- revision 冲突不得覆盖。
|
||||
|
||||
### 8.2 HermesResult
|
||||
|
||||
```text
|
||||
job_id
|
||||
tenant_id
|
||||
task_id
|
||||
trace_id
|
||||
state
|
||||
worker_id
|
||||
output
|
||||
error
|
||||
started_at
|
||||
finished_at
|
||||
```
|
||||
|
||||
约束:
|
||||
|
||||
- COMPLETED 必须有 output。
|
||||
- FAILED 必须有 error。
|
||||
- 只允许 AIOS Orchestrator reconcile。
|
||||
|
||||
### 8.3 HermesHealth
|
||||
|
||||
```text
|
||||
runtime_id
|
||||
status
|
||||
version
|
||||
heartbeat_at
|
||||
active_workers
|
||||
active_jobs
|
||||
queued_jobs
|
||||
tenant_partitions
|
||||
profile_isolation_enforced
|
||||
details
|
||||
```
|
||||
|
||||
实现:`active-runtime/app/aios_hermes/`
|
||||
|
||||
## 9. Execution Evidence Contract
|
||||
|
||||
### 9.1 ExecutionEvidence
|
||||
|
||||
```text
|
||||
execution_id
|
||||
tenant_id
|
||||
task_id
|
||||
agent_id
|
||||
executor
|
||||
branch
|
||||
status
|
||||
progress_completed
|
||||
progress_total
|
||||
current_step
|
||||
last_commit
|
||||
test_status
|
||||
test_summary
|
||||
result_summary
|
||||
claimed_at
|
||||
heartbeat_at
|
||||
completed_at
|
||||
```
|
||||
|
||||
状态建议:
|
||||
|
||||
```text
|
||||
claimed
|
||||
running
|
||||
blocked
|
||||
complete
|
||||
failed
|
||||
cancelled
|
||||
```
|
||||
|
||||
约束:
|
||||
|
||||
- 按租户存储和查询。
|
||||
- 每个阻塞必须记录原因、责任方和解除条件。
|
||||
- 每次结果必须记录 branch、commit、测试命令和状态。
|
||||
- 不得包含 Secret 或完整敏感 payload。
|
||||
|
||||
## 10. GitOps Status Contract
|
||||
|
||||
```text
|
||||
available
|
||||
service_status
|
||||
workspace_mode
|
||||
workspace_branch
|
||||
workspace_ref
|
||||
workspace_dirty
|
||||
candidate_branch
|
||||
release_channel
|
||||
release_version
|
||||
sync_status
|
||||
details
|
||||
```
|
||||
|
||||
约束:
|
||||
|
||||
- Web 响应不得包含 remote URL、凭据或 stderr。
|
||||
- Controller 必须 `detached + clean`。
|
||||
- `sync_status` 必须区分 local 和 Gitea 镜像状态。
|
||||
|
||||
## 11. Web/API Contract
|
||||
|
||||
### 11.1 身份头
|
||||
|
||||
可信代理注入:
|
||||
|
||||
```text
|
||||
X-AIOS-Authenticated
|
||||
X-AIOS-Tenant-ID
|
||||
X-AIOS-Principal-ID
|
||||
X-AIOS-Roles
|
||||
X-AIOS-Trace-ID
|
||||
X-AIOS-Session-ID
|
||||
X-AIOS-Policy-Version
|
||||
```
|
||||
|
||||
生产约束:
|
||||
|
||||
- 代理必须删除客户端同名头后重新注入。
|
||||
- 应用不得直接暴露公网端口。
|
||||
|
||||
### 11.2 核心 API
|
||||
|
||||
```text
|
||||
GET /api/v2/health
|
||||
POST /api/v2/tenants
|
||||
GET /api/v2/tenants/current
|
||||
POST /api/v2/agents
|
||||
GET /api/v2/agents
|
||||
POST /api/v2/tasks
|
||||
GET /api/v2/tasks
|
||||
GET /api/v2/tasks/{id}
|
||||
POST /api/v2/tasks/{id}/approve
|
||||
POST /api/v2/tasks/{id}/start
|
||||
POST /api/v2/tasks/{id}/complete
|
||||
POST /api/v2/tasks/{id}/fail
|
||||
POST /api/v2/tasks/{id}/cancel
|
||||
POST /api/v2/tasks/{id}/dispatch
|
||||
GET /api/v2/audit
|
||||
GET /api/v2/runtime
|
||||
GET /api/v2/executions
|
||||
GET /api/v2/gitops
|
||||
POST /api/v2/hermes/results/reconcile
|
||||
```
|
||||
|
||||
## 12. Persistence Contract
|
||||
|
||||
候选版:
|
||||
|
||||
- SQLite WAL
|
||||
- `BEGIN IMMEDIATE`
|
||||
- 原子 snapshot
|
||||
- schema version
|
||||
- restart restore
|
||||
|
||||
持久化范围:
|
||||
|
||||
- tenants
|
||||
- agents
|
||||
- tasks
|
||||
- audit
|
||||
- idempotency
|
||||
|
||||
生产替换要求:
|
||||
|
||||
- 不改变 Domain/API Contract。
|
||||
- 提供 migration、backup、restore、HA、RLS/隔离和容量测试。
|
||||
|
||||
## 13. Deployment Contract
|
||||
|
||||
### systemd
|
||||
|
||||
- 非 root `aios` 用户。
|
||||
- 状态目录 `/var/lib/aios-v2`。
|
||||
- 最小权限和文件系统保护。
|
||||
|
||||
### Docker
|
||||
|
||||
- 非 root UID/GID。
|
||||
- read-only root filesystem。
|
||||
- drop all capabilities。
|
||||
- 独立状态 volume。
|
||||
|
||||
### Rollback
|
||||
|
||||
- 回滚前备份状态。
|
||||
- 默认只回滚代码。
|
||||
- 状态恢复需显式授权。
|
||||
- 回滚后必须执行 healthcheck。
|
||||
|
||||
## 14. 契约变更规则
|
||||
|
||||
任何以下变化必须在同一提交或同一分支更新本文档:
|
||||
|
||||
- 字段、状态、Header、API、错误语义;
|
||||
- 租户或安全约束;
|
||||
- OpenClaw/Hermes 边界;
|
||||
- Persistence 或 Deployment Contract;
|
||||
- Evidence/Audit 格式。
|
||||
|
||||
不兼容变化必须新增 ADR 和版本号,不得静默修改。
|
||||
@@ -0,0 +1,113 @@
|
||||
# AIOS V2 需求—架构—代码—测试—部署追踪矩阵
|
||||
|
||||
- 文档版本:`V2R1C0-candidate`
|
||||
- 目的:确保每项用户需求都能追踪到架构设计、代码实现、自动测试、部署资产和验收命令
|
||||
|
||||
## 1. 追踪规则
|
||||
|
||||
每条需求至少具备:
|
||||
|
||||
1. 需求编号;
|
||||
2. 架构章节;
|
||||
3. 实现路径;
|
||||
4. 测试路径;
|
||||
5. 部署/运维资产;
|
||||
6. 验收状态。
|
||||
|
||||
状态定义:
|
||||
|
||||
- `Implemented`:代码已进入 Candidate 分支;
|
||||
- `Test-defined`:测试代码已进入 Candidate 分支;
|
||||
- `Test-verified`:精确 commit 的测试已运行并保存证据;
|
||||
- `Pre-production verified`:预发布部署、健康检查和回滚演练已通过;
|
||||
- `Production released`:Owner 批准并完成 merge/tag/deploy。
|
||||
|
||||
## 2. 核心追踪矩阵
|
||||
|
||||
| 需求 | 设计依据 | 代码实现 | 测试 | 部署/运维 | 当前状态 |
|
||||
|---|---|---|---|---|---|
|
||||
| UR-001 AIOS 唯一控制面 | `02-TARGET-ARCHITECTURE.md` §3.2/§3.3;ADR-001 | `aios_v2/control_plane.py`、`aios_v2/orchestrator.py`、`aios_v2/app.py` | `tests/aios_v2_candidate/test_candidate_flow.py` | `deploy/aios-v2/systemd/aios-v2.service` | Implemented / Test-defined |
|
||||
| UR-002 复用现有能力 | `02-TARGET-ARCHITECTURE.md` §2;`03-MIGRATION-AND-DELIVERY.md` §2 | `aios_openclaw/`、`aios_hermes/` | OpenClaw/Hermes flow tests | Candidate branch + adapter boundaries | Implemented / Test-defined |
|
||||
| UR-003 OpenClaw/Hermes 解耦 | `02-TARGET-ARCHITECTURE.md` §3.4/§3.5;ADR-002 | `aios_openclaw/adapter.py`、`aios_hermes/adapter.py` | runtime boundary tests in candidate flow suite | systemd/Docker runtime configuration | Implemented / Test-defined |
|
||||
| UR-004 全面 Web 化 | `02-TARGET-ARCHITECTURE.md` §3.1/§8 | `aios_v2/console.py`、`aios_v2/gateway.py`、`aios_v2/app.py` | API/console smoke test pending | Docker/systemd + reverse proxy requirement | Implemented / Test pending |
|
||||
| UR-005 多租户隔离 | `02-TARGET-ARCHITECTURE.md` §4;ADR-004 | `control_plane.py`、`orchestrator.py`、`aios_openclaw/*`、`aios_hermes/*`、`execution_tracking.py` | `test_cross_tenant_task_read_is_denied` | tenant-scoped state/evidence directories | Implemented / Test-defined |
|
||||
| UR-006 权威任务生命周期 | `04-MODULE-CONTRACTS.md` §4 | `control_plane.py` | OpenClaw/Hermes full-flow tests | persisted state + audit | Implemented / Test-defined |
|
||||
| UR-007 策略与审批 | `04-MODULE-CONTRACTS.md` §5 | `control_plane.py`、`gateway.py` | role/policy negative tests需补充 | reverse proxy identity + audit | Implemented / Partial test |
|
||||
| UR-008 幂等与并发安全 | `02-TARGET-ARCHITECTURE.md` §7 | `control_plane.py`、`aios_openclaw/adapter.py`、`aios_hermes/adapter.py`、`worker.py` | flow + restart tests;并发专项测试需补充 | GitOps detached guard + SQLite WAL | Implemented / Partial test |
|
||||
| UR-009 执行证据 | `04-MODULE-CONTRACTS.md` §9 | `execution_tracking.py` | `test_execution_tracking_persists_claim_heartbeat_and_result` | `/var/lib/aios-v2/executions` | Implemented / Test-defined |
|
||||
| UR-010 GitOps 唯一变更流程 | `03-MIGRATION-AND-DELIVERY.md` §4/§5;ADR-003 | GitOps Runtime、`gitops_status.py` | healthcheck 的 detached/clean 检查 | `deploy/aios-v2/healthcheck/check.sh` | Implemented / Runtime verified partially |
|
||||
| UR-011 持久化与恢复 | `02-TARGET-ARCHITECTURE.md` §3.6 | `persistence.py` | `test_durable_service_recovers_state_after_restart` | state dir + backup/rollback | Implemented / Test-defined |
|
||||
| UR-012 审计与合规 | `04-MODULE-CONTRACTS.md` §6 | `control_plane.py`、`aios_openclaw/callback.py`、`app.py` | flow test + audit assertions需补充 | Console audit view | Implemented / Partial test |
|
||||
| UR-013 密钥安全 | `02-TARGET-ARCHITECTURE.md` §6;ADR-006 | `FileSigningMaterialResolver`、credential refs、Secret Gate | permission/secret-scan test需补充 | env template + external protected file | Implemented / Partial test |
|
||||
| UR-014 部署与回滚 | `03-MIGRATION-AND-DELIVERY.md` §7 | deployment scripts/assets | acceptance runner + healthcheck | `deploy/aios-v2/` | Implemented / Pre-prod pending |
|
||||
| UR-015 文档与代码同步 | `03-MIGRATION-AND-DELIVERY.md` §9;ADR-005 | `tools/aios_v2_docs_code_check.py` | checker self-test / acceptance runner | CI/Gitea Actions 可调用 | Implemented after checker commit |
|
||||
|
||||
## 3. API 追踪
|
||||
|
||||
| API | 需求 | 实现 | 测试状态 |
|
||||
|---|---|---|---|
|
||||
| `GET /api/v2/health` | UR-004, UR-014 | `gateway.py` | healthcheck 定义完成 |
|
||||
| `POST /api/v2/tenants` | UR-001, UR-005 | `gateway.py` / `control_plane.py` | 需增加 HTTP 集成测试 |
|
||||
| `POST /api/v2/agents` | UR-001, UR-005 | `gateway.py` / `control_plane.py` | Domain flow 已覆盖 |
|
||||
| `POST /api/v2/tasks` | UR-006, UR-008 | `gateway.py` / `control_plane.py` | Domain flow 已覆盖 |
|
||||
| `POST /api/v2/tasks/{id}/approve` | UR-006, UR-007 | `gateway.py` | Domain flow 已覆盖 |
|
||||
| `POST /api/v2/tasks/{id}/dispatch` | UR-001, UR-003 | `app.py` / `orchestrator.py` | OpenClaw/Hermes flow 已覆盖 |
|
||||
| `GET /api/v2/runtime` | UR-004, UR-009 | `app.py` | HTTP smoke test待执行 |
|
||||
| `GET /api/v2/executions` | UR-009 | `app.py` / `execution_tracking.py` | Evidence persistence test已定义 |
|
||||
| `GET /api/v2/gitops` | UR-010 | `app.py` / `gitops_status.py` | Runtime/healthcheck验证待执行 |
|
||||
| `POST /api/v2/hermes/results/reconcile` | UR-003, UR-006 | `app.py` / `orchestrator.py` | Hermes flow已覆盖 Domain 路径 |
|
||||
|
||||
## 4. 运行时契约追踪
|
||||
|
||||
| 契约 | 设计 | 代码 | 测试 |
|
||||
|---|---|---|---|
|
||||
| OpenClawCommand | `04-MODULE-CONTRACTS.md` §7.1 | `aios_openclaw/contract.py` | OpenClaw full-flow;签名负向测试需补充 |
|
||||
| OpenClawResult | §7.2 | `aios_openclaw/contract.py` | OpenClaw full-flow |
|
||||
| RuntimeHealth | §7.3 | `aios_openclaw/contract.py`、`health.py` | health policy测试需补充 |
|
||||
| HermesJob | §8.1 | `aios_hermes/contract.py` | Hermes full-flow |
|
||||
| HermesResult | §8.2 | `aios_hermes/contract.py` | Hermes full-flow |
|
||||
| HermesHealth | §8.3 | `aios_hermes/contract.py`、`adapter.py` | healthcheck/worker test待执行 |
|
||||
| ExecutionEvidence | §9 | `execution_tracking.py` | evidence persistence test |
|
||||
| GitOpsStatus | §10 | `gitops_status.py` | healthcheck detached/clean检查 |
|
||||
|
||||
## 5. 部署追踪
|
||||
|
||||
| 能力 | 文件 | 验收命令 |
|
||||
|---|---|---|
|
||||
| systemd | `deploy/aios-v2/systemd/aios-v2.service` | `systemctl status aios-v2` |
|
||||
| 环境模板 | `deploy/aios-v2/env/aios-v2.env.example` | `systemd-analyze verify` + 文件权限检查 |
|
||||
| Docker | `deploy/aios-v2/docker/Dockerfile` | `docker build` |
|
||||
| Compose | `deploy/aios-v2/docker/compose.yaml` | `docker compose config` / `up` |
|
||||
| 健康检查 | `deploy/aios-v2/healthcheck/check.sh` | 直接执行 |
|
||||
| 回滚 | `deploy/aios-v2/rollback/rollback.sh` | 预发布回滚演练 |
|
||||
| 全量验收 | `deploy/aios-v2/acceptance/run-all.sh` | 直接执行 |
|
||||
|
||||
## 6. 文档覆盖追踪
|
||||
|
||||
| 文档 | 覆盖内容 | 代码/变更触发条件 |
|
||||
|---|---|---|
|
||||
| `README.md` | 事实源索引、状态语言、实现地图 | 新模块/目录/版本 |
|
||||
| `01-USER-REQUIREMENTS.md` | 用户需求、NFR、验收场景 | 用户行为、功能范围、验收变化 |
|
||||
| `02-TARGET-ARCHITECTURE.md` | 分层、边界、数据流、安全、租户 | 架构边界、数据流、运行时变化 |
|
||||
| `03-MIGRATION-AND-DELIVERY.md` | 迁移、GitOps、Gate、回滚 | 发布流程、迁移阶段、门禁变化 |
|
||||
| `04-MODULE-CONTRACTS.md` | 字段、API、状态机、错误语义 | Contract/API/模型变化 |
|
||||
| `05-TRACEABILITY-MATRIX.md` | 需求到代码测试部署映射 | 代码、测试、部署路径变化 |
|
||||
| `adr/*` | 关键不可逆或跨模块决策 | 新架构决策或旧决策替换 |
|
||||
|
||||
## 7. 当前未完成验证项
|
||||
|
||||
以下项目代码或测试定义已存在,但尚不能标记为 `Test-verified` 或 `Pre-production verified`:
|
||||
|
||||
1. 精确 Candidate commit 的测试执行结果;
|
||||
2. HTTP API 与 Web Console 冒烟测试;
|
||||
3. 真实 OpenClaw Runtime 签名、取消、超时和健康联调;
|
||||
4. Hermes 长任务心跳、Worker 崩溃和 Lease 恢复演练;
|
||||
5. 多实例 SQLite/并发容量边界;
|
||||
6. systemd、Docker、Compose、restart 和 rollback 演练;
|
||||
7. local/Gitea Candidate OID 一致证明;
|
||||
8. 生产身份代理、防伪 Header、CSRF/CORS/Rate Limit 验证;
|
||||
9. 文档—代码同步检查器在 CI/Acceptance 中执行通过。
|
||||
|
||||
## 8. 更新要求
|
||||
|
||||
任何代码、API、部署或测试路径变化,必须同步更新本矩阵。验收脚本会检查本文档引用的关键路径存在,并检查架构/功能代码变化是否同时包含 `docs/aios-v2/` 变化。
|
||||
@@ -0,0 +1,98 @@
|
||||
# AIOS V2 Architecture and Delivery Index
|
||||
|
||||
- Version line: `V2R1C0-candidate`
|
||||
- Candidate branch: `integration/aios-v2r1-cto-production-candidate-20260720`
|
||||
- Architecture owner: AIOS V2 CTO
|
||||
- Status: implementation-complete candidate; runtime and production verification pending
|
||||
|
||||
## 1. Purpose
|
||||
|
||||
This directory is the single source of truth for AIOS V2 requirements, target architecture, module boundaries, migration strategy, release gates and architectural decisions.
|
||||
|
||||
The design is based on the following non-negotiable principles:
|
||||
|
||||
1. Do not rewrite the existing platform.
|
||||
2. Reuse OpenClaw, Hermes and Agent Router capabilities.
|
||||
3. Decouple OpenClaw and Hermes from control-plane ownership.
|
||||
4. AIOS is the only control plane.
|
||||
5. All user-facing and operational capabilities are Web/API accessible.
|
||||
6. Every module enforces multi-tenant isolation.
|
||||
7. Every change flows through Git branches, commits, tests, review and controlled promotion.
|
||||
|
||||
## 2. Document map
|
||||
|
||||
| Document | Purpose |
|
||||
|---|---|
|
||||
| [01-USER-REQUIREMENTS.md](01-USER-REQUIREMENTS.md) | Product goals, user requirements, non-functional requirements and acceptance criteria |
|
||||
| [02-TARGET-ARCHITECTURE.md](02-TARGET-ARCHITECTURE.md) | Target logical architecture, control boundaries, data flow, security and tenancy model |
|
||||
| [03-MIGRATION-AND-DELIVERY.md](03-MIGRATION-AND-DELIVERY.md) | Incremental migration plan, reuse strategy, GitOps workflow and release gates |
|
||||
| [04-MODULE-CONTRACTS.md](04-MODULE-CONTRACTS.md) | Tenant context, task lifecycle, OpenClaw, Hermes, audit and execution-evidence contracts |
|
||||
| [05-TRACEABILITY-MATRIX.md](05-TRACEABILITY-MATRIX.md) | Requirement-to-design-to-code-to-test mapping |
|
||||
| [adr/README.md](adr/README.md) | Architecture decision records |
|
||||
|
||||
## 3. Current implementation map
|
||||
|
||||
| Architecture capability | Candidate implementation |
|
||||
|---|---|
|
||||
| Tenant, agent, task, policy and audit control plane | `active-runtime/app/aios_v2/control_plane.py` |
|
||||
| Durable control-plane state | `active-runtime/app/aios_v2/persistence.py` |
|
||||
| Web/API gateway | `active-runtime/app/aios_v2/gateway.py` |
|
||||
| Unified application entrypoint | `active-runtime/app/aios_v2/app.py` |
|
||||
| Runtime dispatch and reconciliation | `active-runtime/app/aios_v2/orchestrator.py` |
|
||||
| Agent execution evidence | `active-runtime/app/aios_v2/execution_tracking.py` |
|
||||
| GitOps runtime visibility | `active-runtime/app/aios_v2/gitops_status.py` |
|
||||
| Web Console | `active-runtime/app/aios_v2/console.py` |
|
||||
| OpenClaw execution boundary | `active-runtime/app/aios_openclaw/` |
|
||||
| Hermes runtime boundary | `active-runtime/app/aios_hermes/` |
|
||||
| Candidate tests | `tests/aios_v2_candidate/` |
|
||||
| Deployment, health and rollback | `deploy/aios-v2/` |
|
||||
|
||||
## 4. Status vocabulary
|
||||
|
||||
The project separates code availability from release readiness:
|
||||
|
||||
- **Designed**: documented boundary and acceptance criteria exist.
|
||||
- **Implemented**: code is committed on the candidate branch.
|
||||
- **Test-defined**: automated tests exist in Git.
|
||||
- **Test-verified**: tests were executed for the exact commit and evidence was recorded.
|
||||
- **Pre-production verified**: deployed in an isolated environment with health, restart and rollback evidence.
|
||||
- **Production released**: owner-approved merge/tag/deployment completed.
|
||||
|
||||
A branch or file existing does not, by itself, mean a capability is production-ready.
|
||||
|
||||
## 5. Design summary
|
||||
|
||||
```text
|
||||
Browser / Web Console / Public API
|
||||
|
|
||||
v
|
||||
AIOS Control Plane
|
||||
Tenant | Agent | Task | Policy | Audit
|
||||
|
|
||||
v
|
||||
Execution Orchestrator / Router
|
||||
/ \
|
||||
v v
|
||||
OpenClaw Adapter Hermes Boundary
|
||||
| |
|
||||
v v
|
||||
OpenClaw Runtime Hermes Workers
|
||||
\
|
||||
v
|
||||
Result Reconciliation + Evidence
|
||||
```
|
||||
|
||||
The control plane owns authoritative identity, policy, approval, task lifecycle and audit state. OpenClaw and Hermes receive only tenant-scoped, approved execution work and return runtime results.
|
||||
|
||||
## 6. Release rule
|
||||
|
||||
No candidate may be promoted until:
|
||||
|
||||
- requirement-to-code traceability is complete;
|
||||
- multi-tenant negative tests pass;
|
||||
- controller workspace is detached and clean;
|
||||
- local Git and Gitea branch OIDs match;
|
||||
- persistence restart recovery passes;
|
||||
- healthcheck and rollback pass in pre-production;
|
||||
- no plaintext secret enters Git;
|
||||
- owner approval is recorded for merge, tag and production deployment.
|
||||
@@ -0,0 +1,39 @@
|
||||
# ADR-001: AIOS 作为唯一控制面
|
||||
|
||||
- Status: Accepted
|
||||
- Date: 2026-07-20
|
||||
|
||||
## Context
|
||||
|
||||
现有系统包含多个执行和调度能力,若每个 Runtime 保存自己的任务、租户和策略状态,会形成多个事实源。
|
||||
|
||||
## Decision
|
||||
|
||||
AIOS Control Plane 是唯一权威控制面,负责:
|
||||
|
||||
- Tenant
|
||||
- Agent
|
||||
- Task
|
||||
- Policy
|
||||
- Approval
|
||||
- Audit
|
||||
- Execution Evidence
|
||||
|
||||
OpenClaw、Hermes、Agent Router 只能执行或路由,不拥有第二套控制状态。
|
||||
|
||||
## Consequences
|
||||
|
||||
Positive:
|
||||
|
||||
- 状态唯一;
|
||||
- 审计完整;
|
||||
- Web/API 统一。
|
||||
|
||||
Negative:
|
||||
|
||||
- Runtime 接入需要 Adapter。
|
||||
- 迁移需要阶段性边界收敛。
|
||||
|
||||
## Compliance
|
||||
|
||||
任何新 Runtime 必须通过 AIOS Orchestrator 接入。
|
||||
@@ -0,0 +1,39 @@
|
||||
# AIOS V2 Architecture Decision Records
|
||||
|
||||
本目录记录影响多个模块、长期演进、兼容性、安全或交付流程的关键架构决策。
|
||||
|
||||
## ADR 列表
|
||||
|
||||
| ADR | 决策 |
|
||||
|---|---|
|
||||
| [ADR-001](ADR-001-single-control-plane.md) | AIOS 作为唯一控制面 |
|
||||
| [ADR-002](ADR-002-openclaw-hermes-decoupling.md) | OpenClaw/Hermes 与控制面解耦 |
|
||||
| [ADR-003](ADR-003-gitops-only-change-flow.md) | 所有变更必须通过 GitOps |
|
||||
| [ADR-004](ADR-004-multi-tenant-isolation.md) | 多租户隔离默认拒绝 |
|
||||
| [ADR-005](ADR-005-document-code-sync-gate.md) | 文档与代码同步门禁 |
|
||||
| [ADR-006](ADR-006-candidate-persistence-strategy.md) | 候选持久化采用可替换 SQLite Snapshot |
|
||||
|
||||
## ADR 状态
|
||||
|
||||
- `Proposed`:提出但未采纳。
|
||||
- `Accepted`:当前有效决策。
|
||||
- `Superseded`:被新 ADR 替代。
|
||||
- `Deprecated`:保留历史但禁止新实现采用。
|
||||
|
||||
## 何时必须新增或更新 ADR
|
||||
|
||||
以下变化必须新增或更新 ADR:
|
||||
|
||||
1. 控制面与 Runtime 权限边界变化;
|
||||
2. 多租户隔离模型变化;
|
||||
3. API、数据模型或状态机出现不兼容变化;
|
||||
4. 持久化、消息、密钥、审计或部署方案发生跨模块变化;
|
||||
5. GitOps、发布、回滚或文档门禁变化;
|
||||
6. 替换 OpenClaw、Hermes、Agent Router 或核心基础设施。
|
||||
|
||||
ADR 变化必须同步更新:
|
||||
|
||||
- `02-TARGET-ARCHITECTURE.md`
|
||||
- `04-MODULE-CONTRACTS.md`
|
||||
- `05-TRACEABILITY-MATRIX.md`
|
||||
- 对应代码、测试和部署资产
|
||||
@@ -1,65 +0,0 @@
|
||||
# A0 callback transport and pending-decision recovery
|
||||
|
||||
This runbook separates code acceptance from live callback acceptance. Passing
|
||||
unit tests does not prove that a cloud Workspace Agent can reach the installed
|
||||
tool endpoint.
|
||||
|
||||
## Transport configuration
|
||||
|
||||
Choose exactly one mode in the owner-managed remote-tool configuration:
|
||||
|
||||
- `remote_https_action`: `endpoint` must be the real public HTTPS Action URL.
|
||||
- `secure_mcp_tunnel`: `endpoint` must exactly match an entry in
|
||||
`approved_mcp_tunnel_endpoints`.
|
||||
|
||||
The configuration stores only `access_secret_ref` and `callback_secret_ref`.
|
||||
It must not contain resolved credentials. Startup rejects HTTP, loopback,
|
||||
unspecified/private literal addresses, URL credentials, query/fragment data,
|
||||
unapproved tunnel endpoints, and known placeholder markers.
|
||||
|
||||
Generate the cloud-facing manifest only after owner configuration exists:
|
||||
|
||||
```bash
|
||||
PYTHONPATH=/opt/aios-orchestrator-gateway/current \
|
||||
python3 /opt/aios-orchestrator-gateway/current/scripts/generate_orchestrator_tool_manifest.py \
|
||||
--transport-config /etc/aios-orchestrator-gateway/remote-tool.json \
|
||||
--output /var/lib/aios-orchestrator-gateway/orchestrator-tool-manifest.json
|
||||
```
|
||||
|
||||
The generated Action manifest exposes only
|
||||
`submit_orchestration_decision`. The MCP manifest lists the same single tool
|
||||
with write-side-effect and idempotency annotations.
|
||||
|
||||
## Recovery
|
||||
|
||||
Plan mode is read-only:
|
||||
|
||||
```bash
|
||||
PYTHONPATH=/opt/aios-orchestrator-gateway/current \
|
||||
python3 /opt/aios-orchestrator-gateway/current/scripts/recover_pending_decisions.py \
|
||||
--config /etc/aios-orchestrator-gateway/config.json --plan
|
||||
```
|
||||
|
||||
Application handles at most one leased record and never resends a model prompt
|
||||
or replays a decision:
|
||||
|
||||
```bash
|
||||
PYTHONPATH=/opt/aios-orchestrator-gateway/current \
|
||||
python3 /opt/aios-orchestrator-gateway/current/scripts/recover_pending_decisions.py \
|
||||
--config /etc/aios-orchestrator-gateway/config.json --apply-one
|
||||
```
|
||||
|
||||
## Required live acceptance
|
||||
|
||||
Do not mark the live Gate PASS until metadata-only evidence proves, in order:
|
||||
|
||||
1. Workspace trigger returned HTTP 202.
|
||||
2. The remote callback tool received one authenticated request.
|
||||
3. The bridge returned HTTP 202 from the loopback 18920 decision endpoint.
|
||||
4. The event reached `DECIDED`, `callback_state=APPLIED`, and
|
||||
`decision_applied=true`.
|
||||
5. Recovery planning and one apply cycle did not create a second outbox side
|
||||
effect for the same event.
|
||||
|
||||
Never include Authorization, token values, resolved SecretRefs, prompts, or raw
|
||||
model output in logs or RESULT files.
|
||||
@@ -0,0 +1,47 @@
|
||||
# TASK-AIOS-CONTROL-PLANE-GATEWAY-V1
|
||||
|
||||
## Owner
|
||||
Codex
|
||||
|
||||
## Goal
|
||||
建立 AIOS V2 唯一控制面 Gateway v1。
|
||||
|
||||
## Architecture Boundary
|
||||
|
||||
AIOS Control Plane:
|
||||
- Tenant Registry
|
||||
- Agent Registry
|
||||
- Task Lifecycle
|
||||
- Policy Gate
|
||||
- Approval
|
||||
- Audit Event
|
||||
|
||||
不允许:
|
||||
- OpenClaw 直接管理任务生命周期
|
||||
- Hermes 直接暴露控制 API
|
||||
- Web UI 绕过 AIOS API
|
||||
|
||||
## Deliverables
|
||||
|
||||
1. Control Plane API skeleton
|
||||
2. Tenant context middleware
|
||||
3. Task state machine
|
||||
4. Agent registration contract
|
||||
5. Audit event interface
|
||||
|
||||
## Acceptance
|
||||
|
||||
- 所有 API 携带 tenant_id
|
||||
- 未授权跨租户访问拒绝
|
||||
- Task create/update/cancel 可审计
|
||||
- GitOps commit 可追踪
|
||||
- pytest + integration test 通过
|
||||
|
||||
## Migration
|
||||
|
||||
复用:
|
||||
- Agent Router
|
||||
- OpenClaw adapter
|
||||
- Hermes runtime
|
||||
|
||||
不重写执行层。
|
||||
@@ -1,89 +0,0 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: AIOS Orchestrator Gateway metadata API
|
||||
version: 0.2.0
|
||||
servers:
|
||||
- url: http://127.0.0.1:18920
|
||||
paths:
|
||||
/v1/orchestrator/events:
|
||||
post:
|
||||
security:
|
||||
- bearerAuth: []
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
responses:
|
||||
'202':
|
||||
description: Event accepted
|
||||
'409':
|
||||
description: Idempotency or state conflict
|
||||
'422':
|
||||
description: Invalid event
|
||||
/v1/orchestrator/events/{event_id}:
|
||||
get:
|
||||
security:
|
||||
- bearerAuth: []
|
||||
parameters:
|
||||
- in: path
|
||||
name: event_id
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$'
|
||||
responses:
|
||||
'200':
|
||||
description: Metadata-only event status
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
required: [event_id, status, trigger_http_status, trigger_error_category, trigger_channel, remote_id, callback_state, callback_deadline, callback_attempts, decision_applied, state_version_after]
|
||||
properties:
|
||||
event_id:
|
||||
type: string
|
||||
status:
|
||||
enum: [RECEIVED, AWAITING_DECISION, DECIDED, TRIGGER_FAILED, DEAD_LETTER]
|
||||
trigger_channel:
|
||||
type: string
|
||||
nullable: true
|
||||
enum: [workspace_agent, responses_api]
|
||||
trigger_http_status:
|
||||
type: integer
|
||||
nullable: true
|
||||
minimum: 100
|
||||
maximum: 599
|
||||
trigger_error_category:
|
||||
type: string
|
||||
nullable: true
|
||||
enum: [UNAUTHORIZED, FORBIDDEN, NOT_FOUND, NOT_RUNNABLE, NETWORK_ERROR]
|
||||
remote_id:
|
||||
type: string
|
||||
nullable: true
|
||||
callback_state:
|
||||
enum: [PENDING, SENT, AWAITING_CALLBACK, CALLBACK_RECEIVED, APPLIED, STALE_SUPERSEDED, DEAD_LETTER]
|
||||
callback_deadline:
|
||||
type: integer
|
||||
nullable: true
|
||||
minimum: 0
|
||||
callback_attempts:
|
||||
type: integer
|
||||
minimum: 0
|
||||
decision_applied:
|
||||
type: boolean
|
||||
state_version_after:
|
||||
type: integer
|
||||
nullable: true
|
||||
minimum: 0
|
||||
'401':
|
||||
description: Missing or invalid bearer
|
||||
'404':
|
||||
description: Event not found
|
||||
components:
|
||||
securitySchemes:
|
||||
bearerAuth:
|
||||
type: http
|
||||
scheme: bearer
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"type": "branch",
|
||||
"branch": "integration/aios-v2r1-cto-production-candidate-20260720",
|
||||
"base": "task/codex/AIOS-V2-CONTROL-PLANE-GATEWAY-V1-20260720",
|
||||
"created_by": "chatgpt-gitops-bot",
|
||||
"created_at": "2026-07-20T01:47:01Z",
|
||||
"direct_main_push": false,
|
||||
"owner_approval_required": true
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"type": "branch",
|
||||
"branch": "task/codex/AIOS-V2-CONTROL-PLANE-GATEWAY-V1-20260720",
|
||||
"base": "main",
|
||||
"created_by": "chatgpt-gitops-bot",
|
||||
"created_at": "2026-07-19T17:06:04Z",
|
||||
"direct_main_push": false,
|
||||
"owner_approval_required": true
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://aios.local/schemas/automation-quorum-controller.schema.json",
|
||||
"title": "AIOS GPT plus one-executor quorum controller configuration",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"project_key",
|
||||
"contract_commit",
|
||||
"database_path",
|
||||
"file_inbox_path",
|
||||
"health_receipts",
|
||||
"blocker_retry_threshold",
|
||||
"max_health_age_seconds",
|
||||
"lease_ttl_seconds",
|
||||
"gateway_event_url",
|
||||
"gateway_token_file"
|
||||
],
|
||||
"properties": {
|
||||
"$schema": {"type": "string"},
|
||||
"project_key": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"},
|
||||
"contract_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
|
||||
"database_path": {"type": "string", "pattern": "^/"},
|
||||
"file_inbox_path": {"type": "string", "pattern": "^/"},
|
||||
"default_workload": {
|
||||
"enum": [
|
||||
"large_repetitive_infrastructure_monitoring",
|
||||
"complex_code_security_architecture"
|
||||
]
|
||||
},
|
||||
"health_receipts": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["central_gpt", "openclaw", "codex"],
|
||||
"properties": {
|
||||
"central_gpt": {"type": "string", "pattern": "^/"},
|
||||
"openclaw": {"type": "string", "pattern": "^/"},
|
||||
"codex": {"type": "string", "pattern": "^/"}
|
||||
}
|
||||
},
|
||||
"git_ref_polling": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["repository", "refs"],
|
||||
"properties": {
|
||||
"repository": {"type": "string", "pattern": "^/"},
|
||||
"refs": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"uniqueItems": true,
|
||||
"items": {"type": "string", "pattern": "^refs/"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"blocker_retry_threshold": {"type": "integer", "minimum": 1, "maximum": 20},
|
||||
"max_health_age_seconds": {"type": "integer", "minimum": 5, "maximum": 300},
|
||||
"lease_ttl_seconds": {"type": "integer", "minimum": 15, "maximum": 3600},
|
||||
"gateway_event_url": {"type": "string", "const": "http://127.0.0.1:18920/v1/orchestrator/events"},
|
||||
"gateway_token_file": {"type": "string", "pattern": "^/"}
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "AIOS Callback Transport",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["mode", "endpoint", "approved_mcp_tunnel_endpoints"],
|
||||
"properties": {
|
||||
"mode": {
|
||||
"enum": ["remote_https_action", "secure_mcp_tunnel"]
|
||||
},
|
||||
"endpoint": {
|
||||
"type": "string",
|
||||
"format": "uri",
|
||||
"pattern": "^https://"
|
||||
},
|
||||
"approved_mcp_tunnel_endpoints": {
|
||||
"type": "array",
|
||||
"uniqueItems": true,
|
||||
"items": {"type": "string", "format": "uri", "pattern": "^https://"}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://aios.local/schemas/codex-command.schema.json",
|
||||
"title": "AIOS Codex command",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["schema_version", "command_id", "project_key", "work_unit_id", "contract_commit", "state_version", "execution"],
|
||||
"properties": {
|
||||
"schema_version": {"const": 1},
|
||||
"command_id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"},
|
||||
"project_key": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"},
|
||||
"work_unit_id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$"},
|
||||
"contract_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
|
||||
"state_version": {"type": "integer", "minimum": 0},
|
||||
"execution": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["kind", "input"],
|
||||
"properties": {
|
||||
"kind": {"enum": ["acceptance_probe", "codex"]},
|
||||
"input": {"type": "string", "maxLength": 65536}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,145 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "AIOS Orchestrator Decision",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"schema_version",
|
||||
"project_key",
|
||||
"contract_commit",
|
||||
"event_id",
|
||||
"expected_state_version",
|
||||
"verdict",
|
||||
"state_updates",
|
||||
"next_work_units",
|
||||
"requires_human_input",
|
||||
"stop_reason"
|
||||
],
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"type": "integer",
|
||||
"const": 1
|
||||
},
|
||||
"project_key": {
|
||||
"type": "string"
|
||||
},
|
||||
"contract_commit": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9a-f]{40}$"
|
||||
},
|
||||
"event_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"expected_state_version": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"verdict": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"BLOCKED",
|
||||
"COMPLETE",
|
||||
"DISPATCH",
|
||||
"NOOP"
|
||||
]
|
||||
},
|
||||
"state_updates": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"key",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"next_work_units": {
|
||||
"type": "array",
|
||||
"maxItems": 32,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"id",
|
||||
"track_id",
|
||||
"gate_id",
|
||||
"primary_agent",
|
||||
"fallback_agent",
|
||||
"clean_base_commit",
|
||||
"allowed_paths",
|
||||
"required_checks",
|
||||
"forbidden_operations",
|
||||
"instruction"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"track_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"gate_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"primary_agent": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"openclaw",
|
||||
"codex"
|
||||
]
|
||||
},
|
||||
"fallback_agent": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"openclaw",
|
||||
"codex"
|
||||
]
|
||||
},
|
||||
"clean_base_commit": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9a-f]{40}$"
|
||||
},
|
||||
"allowed_paths": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required_checks": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"forbidden_operations": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"instruction": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"requires_human_input": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"stop_reason": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "AIOS Orchestrator Event",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"schema_version",
|
||||
"project_key",
|
||||
"contract_commit",
|
||||
"event_id",
|
||||
"source_agent",
|
||||
"event_type",
|
||||
"work_unit_id",
|
||||
"status",
|
||||
"state_version",
|
||||
"implementation_commit",
|
||||
"result_commit",
|
||||
"blocker",
|
||||
"evidence_refs",
|
||||
"payload"
|
||||
],
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"const": 1
|
||||
},
|
||||
"project_key": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 128
|
||||
},
|
||||
"contract_commit": {
|
||||
"type": "string",
|
||||
"pattern": "^[0-9a-f]{40}$"
|
||||
},
|
||||
"event_id": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 128
|
||||
},
|
||||
"source_agent": {
|
||||
"enum": [
|
||||
"openclaw",
|
||||
"codex",
|
||||
"gitea",
|
||||
"system",
|
||||
"workspace-agent"
|
||||
]
|
||||
},
|
||||
"event_type": {
|
||||
"type": "string"
|
||||
},
|
||||
"work_unit_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"status": {
|
||||
"enum": [
|
||||
"READY",
|
||||
"RUNNING",
|
||||
"PASS",
|
||||
"BLOCKED",
|
||||
"FAIL",
|
||||
"TIMEOUT",
|
||||
"NOT_EXECUTED"
|
||||
]
|
||||
},
|
||||
"state_version": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"implementation_commit": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
],
|
||||
"pattern": "^[0-9a-f]{40}$"
|
||||
},
|
||||
"result_commit": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
],
|
||||
"pattern": "^[0-9a-f]{40}$"
|
||||
},
|
||||
"blocker": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
],
|
||||
"maxLength": 512
|
||||
},
|
||||
"evidence_refs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"maxItems": 64
|
||||
},
|
||||
"payload": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Seed exactly one orchestration event when the project has no durable state."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from services.aios_orchestrator_gateway.config import Config
|
||||
from services.aios_orchestrator_gateway.models import Event
|
||||
from services.aios_orchestrator_gateway.securefs import atomic_write_bytes_shared
|
||||
from services.aios_orchestrator_gateway.store import Store
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
config = Config.from_file(args.config)
|
||||
store = Store(config.database_path)
|
||||
if store.project_state(config.project_key) is not None:
|
||||
print("ORCHESTRATION_BOOTSTRAP=NOOP reason=STATE_EXISTS")
|
||||
return 0
|
||||
|
||||
event = Event.from_dict(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"project_key": config.project_key,
|
||||
"contract_commit": config.contract_commit,
|
||||
"event_id": "bootstrap-" + config.contract_commit[:24],
|
||||
"source_agent": "system",
|
||||
"event_type": "orchestration_bootstrap",
|
||||
"work_unit_id": "ORCHESTRATION-BOOTSTRAP",
|
||||
"status": "READY",
|
||||
"state_version": 0,
|
||||
"implementation_commit": None,
|
||||
"result_commit": None,
|
||||
"blocker": None,
|
||||
"evidence_refs": [],
|
||||
"payload": {
|
||||
"reason": "initial_gateway_install",
|
||||
"polling_enabled": True,
|
||||
"openclaw_sync_required": True,
|
||||
},
|
||||
}
|
||||
)
|
||||
path = config.event_inbox_dir / "pending" / f"{event.event_id}.json"
|
||||
atomic_write_bytes_shared(
|
||||
path,
|
||||
(event.canonical_json() + "\n").encode(),
|
||||
mode=0o640,
|
||||
group_writable_directory=True,
|
||||
)
|
||||
print(f"ORCHESTRATION_BOOTSTRAP=PASS event_id={event.event_id}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,56 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
from services.aios_orchestrator_gateway.callback_transport import CallbackTransport
|
||||
|
||||
|
||||
def atomic_write(path: Path, value: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(value, handle, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temporary, 0o600)
|
||||
os.replace(temporary, path)
|
||||
directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temporary)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--transport-config", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
value = json.loads(args.transport_config.read_text(encoding="utf-8"))
|
||||
if "callback_transport" in value:
|
||||
value = value["callback_transport"]
|
||||
transport = CallbackTransport.from_dict(value)
|
||||
manifest = (
|
||||
transport.action_openapi()
|
||||
if transport.mode == "remote_https_action"
|
||||
else {"endpoint": transport.endpoint, "tools": [transport.mcp_tool()]}
|
||||
)
|
||||
atomic_write(args.output, manifest)
|
||||
print(f"TOOL_MANIFEST=PASS mode={transport.mode}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,70 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
[[ ${EUID} -eq 0 ]] || { echo "root required" >&2; exit 1; }
|
||||
ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd -P)
|
||||
CONFIG=${1:-/etc/aios-codex-watcher/config.json}
|
||||
[[ "$CONFIG" = /* ]] || { echo "config path must be absolute" >&2; exit 1; }
|
||||
[[ -f "$CONFIG" && ! -L "$CONFIG" ]] || { echo "pre-provisioned regular config required" >&2; exit 1; }
|
||||
|
||||
for path in \
|
||||
services/aios_codex_watcher/__init__.py \
|
||||
services/aios_codex_watcher/__main__.py \
|
||||
services/aios_codex_watcher/watcher.py \
|
||||
systemd/aios-codex-watcher.service \
|
||||
systemd/aios-codex-watcher.timer \
|
||||
systemd/aios-codex-watcher.sysusers \
|
||||
systemd/aios-codex-watcher.tmpfiles \
|
||||
systemd/aios-codex-central.sysusers \
|
||||
systemd/aios-codex-central.tmpfiles; do
|
||||
[[ -f "$ROOT/$path" && ! -L "$ROOT/$path" ]] || { echo "missing or unsafe: $path" >&2; exit 1; }
|
||||
done
|
||||
|
||||
digest=$(sha256sum \
|
||||
"$ROOT/services/aios_codex_watcher/__init__.py" \
|
||||
"$ROOT/services/aios_codex_watcher/__main__.py" \
|
||||
"$ROOT/services/aios_codex_watcher/watcher.py" | sha256sum | awk '{print $1}')
|
||||
release="/opt/aios-codex-watcher/releases/$digest"
|
||||
install -d -o root -g root -m 0755 "$release/services/aios_codex_watcher" "$release/bin"
|
||||
install -o root -g root -m 0644 "$ROOT/services/aios_codex_watcher/"*.py "$release/services/aios_codex_watcher/"
|
||||
|
||||
launcher="$release/bin/aios-codex-watcher"
|
||||
printf '%s\n' '#!/usr/bin/env bash' 'set -euo pipefail' \
|
||||
"export PYTHONPATH='$release'" \
|
||||
"exec /usr/bin/python3 -m services.aios_codex_watcher \"\$@\"" >"$launcher"
|
||||
chown root:root "$launcher"
|
||||
chmod 0755 "$launcher"
|
||||
|
||||
install -d -o root -g root -m 0755 /opt/aios-codex-watcher /etc/aios-codex-watcher
|
||||
ln -sfn "releases/$digest" /opt/aios-codex-watcher/current.new
|
||||
mv -Tf /opt/aios-codex-watcher/current.new /opt/aios-codex-watcher/current
|
||||
install -o root -g root -m 0644 "$ROOT/systemd/aios-codex-watcher.service" /etc/systemd/system/
|
||||
install -o root -g root -m 0644 "$ROOT/systemd/aios-codex-watcher.timer" /etc/systemd/system/
|
||||
install -o root -g root -m 0644 "$ROOT/systemd/aios-codex-watcher.sysusers" /usr/lib/sysusers.d/
|
||||
install -o root -g root -m 0644 "$ROOT/systemd/aios-codex-watcher.tmpfiles" /usr/lib/tmpfiles.d/
|
||||
install -o root -g root -m 0644 "$ROOT/systemd/aios-codex-central.sysusers" /usr/lib/sysusers.d/
|
||||
install -o root -g root -m 0644 "$ROOT/systemd/aios-codex-central.tmpfiles" /usr/lib/tmpfiles.d/
|
||||
systemd-sysusers /usr/lib/sysusers.d/aios-codex-watcher.sysusers
|
||||
systemd-sysusers /usr/lib/sysusers.d/aios-codex-central.sysusers
|
||||
systemd-tmpfiles --create /usr/lib/tmpfiles.d/aios-codex-watcher.tmpfiles
|
||||
systemd-tmpfiles --create /usr/lib/tmpfiles.d/aios-codex-central.tmpfiles
|
||||
commands=/data/aios-gitops/workspace/agents/commands
|
||||
for directory in "$commands/active" "$commands/completed"; do
|
||||
[[ -d "$directory" && ! -L "$directory" ]] || { echo "missing or unsafe command directory: $directory" >&2; exit 1; }
|
||||
[[ $(stat -c '%U:%G' "$directory") == aios-orchestrator:aios-orchestrator ]] || { echo "unexpected command directory owner: $directory" >&2; exit 1; }
|
||||
mode=$(stat -c '%a' "$directory")
|
||||
(( (8#$mode & 2) == 0 )) || { echo "world-writable command directory: $directory" >&2; exit 1; }
|
||||
done
|
||||
if [[ -e "$commands/dead-letter" || -L "$commands/dead-letter" ]]; then
|
||||
[[ -d "$commands/dead-letter" && ! -L "$commands/dead-letter" ]] || { echo "unsafe dead-letter directory" >&2; exit 1; }
|
||||
else
|
||||
install -d -o aios-orchestrator -g aios-orchestrator -m 0750 "$commands/dead-letter"
|
||||
fi
|
||||
[[ $(stat -c '%U:%G' "$commands/dead-letter") == aios-orchestrator:aios-orchestrator ]] || { echo "unexpected dead-letter owner" >&2; exit 1; }
|
||||
mode=$(stat -c '%a' "$commands/dead-letter")
|
||||
(( (8#$mode & 2) == 0 )) || { echo "world-writable dead-letter directory" >&2; exit 1; }
|
||||
printf '{"release_sha256":"%s","installed_at":"%s"}\n' "$digest" "$(date -u +%FT%TZ)" >"$release/receipt.json"
|
||||
chmod 0644 "$release/receipt.json"
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now aios-codex-watcher.timer
|
||||
printf 'installed_sha256=%s\n' "$digest"
|
||||
@@ -1,357 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 0077
|
||||
|
||||
SOURCE_ROOT="${SOURCE_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
INSTALL_ROOT="${INSTALL_ROOT:-/opt/aios-orchestrator-gateway}"
|
||||
CONFIG_ROOT="${CONFIG_ROOT:-/etc/aios-orchestrator-gateway}"
|
||||
STATE_ROOT="${STATE_ROOT:-/var/lib/aios-orchestrator-gateway}"
|
||||
UNIT_ROOT="${UNIT_ROOT:-/etc/systemd/system}"
|
||||
SYSUSERS_ROOT="${SYSUSERS_ROOT:-/etc/sysusers.d}"
|
||||
TMPFILES_ROOT="${TMPFILES_ROOT:-/etc/tmpfiles.d}"
|
||||
DRY_RUN=0
|
||||
|
||||
if [[ "${1:-}" == "--dry-run" ]]; then
|
||||
DRY_RUN=1
|
||||
fi
|
||||
|
||||
fail() {
|
||||
printf 'AIOS_ORCHESTRATOR_INSTALL=BLOCKED reason=%s\n' "$1" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
[[ "$(id -u)" -eq 0 ]] || fail OWNER_INSTALL_REQUIRED
|
||||
|
||||
for cmd in git python3 systemd-analyze systemctl systemd-sysusers systemd-tmpfiles sha256sum find sort xargs; do
|
||||
command -v "$cmd" >/dev/null 2>&1 || fail "MISSING_COMMAND_${cmd}"
|
||||
done
|
||||
|
||||
if [[ -f "$SOURCE_ROOT/SHA256SUMS" ]]; then
|
||||
(cd "$SOURCE_ROOT" && sha256sum --check SHA256SUMS)
|
||||
fi
|
||||
|
||||
for path in \
|
||||
"$SOURCE_ROOT/services/aios_orchestrator_gateway" \
|
||||
"$SOURCE_ROOT/systemd/aios-orchestrator-gateway.service" \
|
||||
"$SOURCE_ROOT/systemd/aios-orchestrator-gateway-poll.service" \
|
||||
"$SOURCE_ROOT/systemd/aios-orchestrator-gateway-poll.timer" \
|
||||
"$SOURCE_ROOT/systemd/aios-orchestrator-gateway.sysusers" \
|
||||
"$SOURCE_ROOT/systemd/aios-orchestrator-gateway.tmpfiles" \
|
||||
"$SOURCE_ROOT/systemd/aios-orchestrator-remote-tool.service" \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central.sysusers" \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central.tmpfiles" \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central-decision.service" \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central-health.service" \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central-health.timer" \
|
||||
"$SOURCE_ROOT/config/aios-orchestrator-gateway.example.json"; do
|
||||
[[ -e "$path" ]] || fail "MISSING_SOURCE_${path}"
|
||||
done
|
||||
|
||||
PYCACHE="/tmp/aios-orchestrator-install-pycache-$$"
|
||||
rm -rf "$PYCACHE"
|
||||
PYTHONPYCACHEPREFIX="$PYCACHE" python3 -m compileall -q "$SOURCE_ROOT/services" "$SOURCE_ROOT/scripts"
|
||||
PYTHONDONTWRITEBYTECODE=1 PYTHONPATH="$SOURCE_ROOT" \
|
||||
python3 -m unittest -v tests.test_aios_orchestrator_gateway
|
||||
rm -rf "$PYCACHE"
|
||||
for script in "$SOURCE_ROOT"/scripts/*.sh; do bash -n "$script"; done
|
||||
python3 -m json.tool "$SOURCE_ROOT/config/aios-orchestrator-gateway.example.json" >/dev/null
|
||||
systemd-analyze verify \
|
||||
"$SOURCE_ROOT/systemd/aios-orchestrator-gateway.service" \
|
||||
"$SOURCE_ROOT/systemd/aios-orchestrator-gateway-poll.service" \
|
||||
"$SOURCE_ROOT/systemd/aios-orchestrator-gateway-poll.timer"
|
||||
systemd-analyze verify \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central-decision.service" \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central-health.service" \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central-health.timer"
|
||||
|
||||
SOURCE_FILES=(
|
||||
README-AIOS-ORCHESTRATOR-GATEWAY.md
|
||||
docs/runbooks/A0-CALLBACK-TRANSPORT-AND-PENDING-RECOVERY.md
|
||||
config/aios-orchestrator-gateway.example.json
|
||||
schemas/decision.schema.json
|
||||
schemas/event.schema.json
|
||||
scripts/bootstrap_orchestration_loop.py
|
||||
scripts/install_aios_orchestrator_gateway.sh
|
||||
scripts/report_orchestrator_event.py
|
||||
scripts/rollback_aios_orchestrator_gateway.sh
|
||||
scripts/verify_aios_orchestrator_gateway.sh
|
||||
systemd/aios-orchestrator-gateway-poll.service
|
||||
systemd/aios-orchestrator-gateway-poll.timer
|
||||
systemd/aios-orchestrator-gateway.service
|
||||
systemd/aios-orchestrator-gateway.sysusers
|
||||
systemd/aios-orchestrator-gateway.tmpfiles
|
||||
systemd/aios-orchestrator-remote-tool.service
|
||||
systemd/aios-codex-central-decision.service
|
||||
systemd/aios-codex-central-health.service
|
||||
systemd/aios-codex-central-health.timer
|
||||
systemd/aios-codex-central.sysusers
|
||||
systemd/aios-codex-central.tmpfiles
|
||||
schemas/callback-transport.schema.json
|
||||
scripts/generate_orchestrator_tool_manifest.py
|
||||
scripts/recover_pending_decisions.py
|
||||
scripts/install_codex_local_central.sh
|
||||
automation/codex_central_decision.py
|
||||
automation/codex_central_health_probe.py
|
||||
tests/test_aios_orchestrator_gateway.py
|
||||
tests/test_aios_orchestrator_gateway_rollback.sh
|
||||
tests/test_aios_callback_recovery.py
|
||||
workspace-agent/CENTRAL_AGENT_INSTRUCTIONS.md
|
||||
workspace-agent/orchestrator-decision-action.openapi.yaml
|
||||
openapi/aios-orchestrator-gateway.openapi.yaml
|
||||
)
|
||||
while IFS= read -r path; do SOURCE_FILES+=("$path"); done < <(
|
||||
cd "$SOURCE_ROOT"
|
||||
find services/aios_orchestrator_gateway -type f ! -path '*/__pycache__/*' ! -name '*.pyc' -print | sort
|
||||
)
|
||||
for path in "${SOURCE_FILES[@]}"; do
|
||||
[[ -f "$SOURCE_ROOT/$path" && ! -L "$SOURCE_ROOT/$path" ]] || fail "INVALID_SOURCE_FILE_${path}"
|
||||
done
|
||||
|
||||
SOURCE_DIGEST="$(
|
||||
cd "$SOURCE_ROOT"
|
||||
printf '%s\0' "${SOURCE_FILES[@]}" | sort -z | xargs -0 sha256sum \
|
||||
| sha256sum \
|
||||
| awk '{print $1}'
|
||||
)"
|
||||
[[ "$SOURCE_DIGEST" =~ ^[0-9a-f]{64}$ ]] || fail SOURCE_DIGEST_INVALID
|
||||
RELEASE_DIR="$INSTALL_ROOT/releases/$SOURCE_DIGEST"
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
printf 'AIOS_ORCHESTRATOR_INSTALL_DRY_RUN=PASS digest=%s release=%s\n' \
|
||||
"$SOURCE_DIGEST" "$RELEASE_DIR"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
systemd-sysusers "$SOURCE_ROOT/systemd/aios-orchestrator-gateway.sysusers"
|
||||
systemd-sysusers "$SOURCE_ROOT/systemd/aios-codex-central.sysusers"
|
||||
|
||||
install -d -m 0755 "$INSTALL_ROOT/releases"
|
||||
install -d -o root -g aios-orchestrator -m 0750 "$CONFIG_ROOT"
|
||||
install -d -m 0700 "$STATE_ROOT"
|
||||
|
||||
systemd-tmpfiles --create "$SOURCE_ROOT/systemd/aios-orchestrator-gateway.tmpfiles"
|
||||
systemd-tmpfiles --create "$SOURCE_ROOT/systemd/aios-codex-central.tmpfiles"
|
||||
|
||||
if [[ ! -d "$RELEASE_DIR" ]]; then
|
||||
TMP_RELEASE="$INSTALL_ROOT/releases/.tmp-$SOURCE_DIGEST-$$"
|
||||
install -d -m 0755 "$TMP_RELEASE"
|
||||
for path in "${SOURCE_FILES[@]}"; do
|
||||
install -d -m 0755 "$TMP_RELEASE/$(dirname "$path")"
|
||||
cp -a "$SOURCE_ROOT/$path" "$TMP_RELEASE/$path"
|
||||
done
|
||||
printf '%s\n' "$SOURCE_DIGEST" > "$TMP_RELEASE/SOURCE_DIGEST"
|
||||
chmod -R go-w "$TMP_RELEASE"
|
||||
mv "$TMP_RELEASE" "$RELEASE_DIR"
|
||||
fi
|
||||
|
||||
PREVIOUS_TARGET=""
|
||||
if [[ -L "$INSTALL_ROOT/current" ]]; then
|
||||
PREVIOUS_TARGET="$(readlink -f "$INSTALL_ROOT/current")"
|
||||
fi
|
||||
|
||||
install -m 0644 "$SOURCE_ROOT/systemd/aios-orchestrator-gateway.service" \
|
||||
"$UNIT_ROOT/aios-orchestrator-gateway.service"
|
||||
install -m 0644 "$SOURCE_ROOT/systemd/aios-orchestrator-gateway-poll.service" \
|
||||
"$UNIT_ROOT/aios-orchestrator-gateway-poll.service"
|
||||
install -m 0644 "$SOURCE_ROOT/systemd/aios-orchestrator-gateway-poll.timer" \
|
||||
"$UNIT_ROOT/aios-orchestrator-gateway-poll.timer"
|
||||
install -m 0644 "$SOURCE_ROOT/systemd/aios-orchestrator-gateway.sysusers" \
|
||||
"$SYSUSERS_ROOT/aios-orchestrator-gateway.conf"
|
||||
install -m 0644 "$SOURCE_ROOT/systemd/aios-orchestrator-gateway.tmpfiles" \
|
||||
"$TMPFILES_ROOT/aios-orchestrator-gateway.conf"
|
||||
install -m 0644 "$SOURCE_ROOT/systemd/aios-codex-central.sysusers" \
|
||||
"$SYSUSERS_ROOT/aios-codex-central.conf"
|
||||
install -m 0644 "$SOURCE_ROOT/systemd/aios-codex-central.tmpfiles" \
|
||||
"$TMPFILES_ROOT/aios-codex-central.conf"
|
||||
install -m 0644 "$SOURCE_ROOT/systemd/aios-orchestrator-remote-tool.service" \
|
||||
"$UNIT_ROOT/aios-orchestrator-remote-tool.service"
|
||||
|
||||
if [[ ! -f "$CONFIG_ROOT/inbound.token" ]]; then
|
||||
python3 - <<'PY' > "$CONFIG_ROOT/inbound.token"
|
||||
import secrets, sys
|
||||
sys.stdout.write(secrets.token_urlsafe(48))
|
||||
PY
|
||||
fi
|
||||
[[ ! -L "$CONFIG_ROOT/inbound.token" ]] || fail INBOUND_TOKEN_SYMLINK
|
||||
chown root:aios-orchestrator "$CONFIG_ROOT/inbound.token"
|
||||
chmod 0640 "$CONFIG_ROOT/inbound.token"
|
||||
|
||||
if [[ ! -f "$CONFIG_ROOT/callback.token" ]]; then
|
||||
python3 - <<'PY' > "$CONFIG_ROOT/callback.token"
|
||||
import secrets, sys
|
||||
sys.stdout.write(secrets.token_urlsafe(48))
|
||||
PY
|
||||
fi
|
||||
[[ ! -L "$CONFIG_ROOT/callback.token" ]] || fail CALLBACK_TOKEN_SYMLINK
|
||||
chown root:aios-orchestrator "$CONFIG_ROOT/callback.token"
|
||||
chmod 0640 "$CONFIG_ROOT/callback.token"
|
||||
|
||||
# Historical installers appended one LF. Remove only that exact legacy byte in
|
||||
# place; reject every other newline form without ever copying or printing data.
|
||||
python3 - "$CONFIG_ROOT/config.json" "$CONFIG_ROOT/inbound.token" "$CONFIG_ROOT/callback.token" <<'PY'
|
||||
import json, os, pathlib, stat, sys
|
||||
config_path = pathlib.Path(sys.argv[1])
|
||||
names = list(sys.argv[2:])
|
||||
if config_path.is_file() and not config_path.is_symlink():
|
||||
config = json.loads(config_path.read_text())
|
||||
candidates = [
|
||||
config.get("inbound_token_file"),
|
||||
config.get("callback_token_file"),
|
||||
(config.get("workspace_agent") or {}).get("access_token_file"),
|
||||
(config.get("responses_fallback") or {}).get("api_key_file"),
|
||||
(config.get("openclaw_sync") or {}).get("bearer_token_file"),
|
||||
]
|
||||
names.extend(name for name in candidates if isinstance(name, str) and name)
|
||||
for name in dict.fromkeys(names):
|
||||
path = pathlib.Path(name)
|
||||
if not path.is_absolute() or path.is_symlink() or path.resolve() != path:
|
||||
raise SystemExit("invalid token path")
|
||||
fd = os.open(name, os.O_RDWR | getattr(os, "O_NOFOLLOW", 0))
|
||||
try:
|
||||
info = os.fstat(fd)
|
||||
if not stat.S_ISREG(info.st_mode) or info.st_size <= 0 or info.st_size > 16384:
|
||||
raise SystemExit("invalid token metadata")
|
||||
data = os.read(fd, info.st_size)
|
||||
if data.endswith(b"\n") and b"\n" not in data[:-1] and b"\r" not in data:
|
||||
os.ftruncate(fd, info.st_size - 1)
|
||||
os.fsync(fd)
|
||||
elif b"\n" in data or b"\r" in data:
|
||||
raise SystemExit("invalid token format")
|
||||
finally:
|
||||
os.close(fd)
|
||||
PY
|
||||
|
||||
if [[ ! -f "$CONFIG_ROOT/config.json" ]]; then
|
||||
install -m 0600 \
|
||||
"$SOURCE_ROOT/config/aios-orchestrator-gateway.example.json" \
|
||||
"$CONFIG_ROOT/config.json.example"
|
||||
printf 'AIOS_ORCHESTRATOR_INSTALL=BLOCKED reason=CONFIGURATION_REQUIRED digest=%s\n' \
|
||||
"$SOURCE_DIGEST"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
chown root:aios-orchestrator "$CONFIG_ROOT/config.json"
|
||||
chmod 0640 "$CONFIG_ROOT/config.json"
|
||||
|
||||
python3 -m json.tool "$CONFIG_ROOT/config.json" >/dev/null
|
||||
AIOS_ORCHESTRATOR_CONFIG_OWNER_UID=0 \
|
||||
AIOS_ORCHESTRATOR_SECRET_OWNER_UID=0 \
|
||||
PYTHONPATH="$RELEASE_DIR" \
|
||||
python3 - "$CONFIG_ROOT/config.json" <<'PY'
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from services.aios_orchestrator_gateway.config import Config
|
||||
cfg = Config.from_file(Path(sys.argv[1]))
|
||||
cfg.inbound_token()
|
||||
cfg.callback_token()
|
||||
if cfg.workspace_trigger_id:
|
||||
cfg.workspace_token()
|
||||
if cfg.responses_api_key_file:
|
||||
cfg.responses_api_key()
|
||||
if cfg.openclaw_sync_endpoint:
|
||||
cfg.openclaw_sync_token()
|
||||
print("CONFIG_VALIDATE=PASS")
|
||||
PY
|
||||
|
||||
# Commit the release pointer only after every candidate/config/secret check has
|
||||
# succeeded. A failed validation therefore leaves the running release intact.
|
||||
ln -s "$RELEASE_DIR" "$INSTALL_ROOT/.current-new-$$"
|
||||
mv -Tf "$INSTALL_ROOT/.current-new-$$" "$INSTALL_ROOT/current"
|
||||
|
||||
python3 - "$STATE_ROOT/install-receipt.json" "$SOURCE_DIGEST" "$RELEASE_DIR" "$PREVIOUS_TARGET" <<'PY'
|
||||
import json, os, sys, tempfile
|
||||
from pathlib import Path
|
||||
path = Path(sys.argv[1])
|
||||
data = {
|
||||
"schema_version": 1,
|
||||
"source_digest": sys.argv[2],
|
||||
"current_release": sys.argv[3],
|
||||
"previous_release": sys.argv[4] or None,
|
||||
"last_rollback": None,
|
||||
}
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, temp = tempfile.mkstemp(prefix=".install-receipt-", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, "w") as handle:
|
||||
json.dump(data, handle, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temp, 0o600)
|
||||
os.replace(temp, path)
|
||||
directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temp)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
PY
|
||||
chown root:root "$STATE_ROOT/install-receipt.json"
|
||||
chmod 0600 "$STATE_ROOT/install-receipt.json"
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable aios-orchestrator-gateway.service
|
||||
systemctl enable aios-orchestrator-gateway-poll.timer
|
||||
systemctl stop aios-orchestrator-gateway-poll.timer || true
|
||||
systemctl stop aios-orchestrator-gateway-poll.service || true
|
||||
|
||||
rollback_activation() {
|
||||
if [[ -n "$PREVIOUS_TARGET" && -d "$PREVIOUS_TARGET" ]]; then
|
||||
bash "$RELEASE_DIR/scripts/rollback_aios_orchestrator_gateway.sh" || true
|
||||
else
|
||||
systemctl stop aios-orchestrator-gateway.service || true
|
||||
fi
|
||||
fail ACTIVATION_FAILED_ROLLED_BACK
|
||||
}
|
||||
|
||||
systemctl restart aios-orchestrator-gateway.service || rollback_activation
|
||||
python3 - <<'PY' || rollback_activation
|
||||
import json, time, urllib.request
|
||||
for _ in range(30):
|
||||
try:
|
||||
with urllib.request.urlopen("http://127.0.0.1:18920/healthz", timeout=1) as response:
|
||||
body = json.loads(response.read(65536))
|
||||
if response.status == 200 and body.get("status") == "active":
|
||||
raise SystemExit(0)
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
systemctl start aios-orchestrator-gateway-poll.timer || rollback_activation
|
||||
systemctl is-active --quiet aios-orchestrator-gateway.service
|
||||
systemctl is-active --quiet aios-orchestrator-gateway-poll.timer
|
||||
|
||||
AIOS_ORCHESTRATOR_CONFIG_OWNER_UID=0 \
|
||||
AIOS_ORCHESTRATOR_SECRET_OWNER_UID=0 \
|
||||
PYTHONPATH="$INSTALL_ROOT/current" \
|
||||
python3 "$INSTALL_ROOT/current/scripts/bootstrap_orchestration_loop.py" \
|
||||
--config "$CONFIG_ROOT/config.json"
|
||||
systemctl start aios-orchestrator-gateway-poll.service || rollback_activation
|
||||
|
||||
if [[ -f "$CONFIG_ROOT/remote-tool.json" ]]; then
|
||||
[[ ! -L "$CONFIG_ROOT/remote-tool.json" ]] || fail REMOTE_TOOL_CONFIG_SYMLINK
|
||||
chown root:aios-orchestrator "$CONFIG_ROOT/remote-tool.json"
|
||||
chmod 0640 "$CONFIG_ROOT/remote-tool.json"
|
||||
PYTHONPATH="$RELEASE_DIR" python3 - "$CONFIG_ROOT/remote-tool.json" <<'PY'
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from services.aios_orchestrator_gateway.remote_tool_server import RemoteToolServerConfig
|
||||
RemoteToolServerConfig.from_file(Path(sys.argv[1]))
|
||||
print("REMOTE_TOOL_CONFIG_VALIDATE=PASS")
|
||||
PY
|
||||
PYTHONPATH="$RELEASE_DIR" python3 \
|
||||
"$RELEASE_DIR/scripts/generate_orchestrator_tool_manifest.py" \
|
||||
--transport-config "$CONFIG_ROOT/remote-tool.json" \
|
||||
--output "$STATE_ROOT/orchestrator-tool-manifest.json"
|
||||
systemctl enable --now aios-orchestrator-remote-tool.service
|
||||
else
|
||||
printf 'REMOTE_TOOL_CONFIGURATION=BLOCKED reason=REMOTE_CALLBACK_ENDPOINT_CONFIGURATION\n'
|
||||
fi
|
||||
|
||||
printf 'AIOS_ORCHESTRATOR_INSTALL=PASS digest=%s previous=%s\n' \
|
||||
"$SOURCE_DIGEST" "${PREVIOUS_TARGET:-NONE}"
|
||||
@@ -1,112 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ ${EUID} -ne 0 ]]; then
|
||||
echo "root required" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SOURCE_ROOT=${1:?usage: install_automation_quorum_controller.sh SOURCE_ROOT VERSION}
|
||||
VERSION=${2:?usage: install_automation_quorum_controller.sh SOURCE_ROOT VERSION}
|
||||
[[ ${VERSION} =~ ^[0-9A-Za-z._-]+$ ]] || { echo "invalid version" >&2; exit 1; }
|
||||
SOURCE_REAL=$(readlink -f -- "${SOURCE_ROOT}" 2>/dev/null || true)
|
||||
[[ ${SOURCE_ROOT} = /* && -n ${SOURCE_REAL} && ${SOURCE_ROOT} == "${SOURCE_REAL}" && -d ${SOURCE_ROOT} && ! -L ${SOURCE_ROOT} ]] || {
|
||||
echo "unsafe source root" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
SOURCE_FILES=(
|
||||
automation/executor_quorum_controller.py
|
||||
automation/health_probe_common.py
|
||||
automation/central_gpt_health_probe.py
|
||||
automation/codex_executor_health_probe.py
|
||||
automation/a0_unattended_loop_probe.py
|
||||
schemas/automation-quorum-controller.schema.json
|
||||
config/aios-automation-quorum-controller.example.json
|
||||
config/aios-health-probes.example.json
|
||||
systemd/aios-automation-quorum-controller.service
|
||||
systemd/aios-automation-quorum-controller.timer
|
||||
systemd/aios-automation-quorum-controller.sysusers
|
||||
systemd/aios-automation-quorum-controller.tmpfiles
|
||||
systemd/aios-central-gpt-health-probe.service
|
||||
systemd/aios-central-gpt-health-probe.timer
|
||||
systemd/aios-codex-executor-health-probe.service
|
||||
systemd/aios-codex-executor-health-probe.timer
|
||||
)
|
||||
for relative in "${SOURCE_FILES[@]}"; do
|
||||
[[ -f ${SOURCE_ROOT}/${relative} && ! -L ${SOURCE_ROOT}/${relative} ]] || {
|
||||
echo "unsafe source file: ${relative}" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
INSTALL_ROOT=/opt/aios-automation-quorum
|
||||
VERSION_DIR=${INSTALL_ROOT}/releases/${VERSION}
|
||||
[[ ! -e ${VERSION_DIR} ]] || { echo "version already installed" >&2; exit 1; }
|
||||
|
||||
install -d -o root -g root -m 0755 "${VERSION_DIR}/bin" "${VERSION_DIR}/share"
|
||||
install -o root -g root -m 0755 "${SOURCE_ROOT}/automation/executor_quorum_controller.py" \
|
||||
"${VERSION_DIR}/bin/executor-quorum-controller"
|
||||
install -o root -g root -m 0644 "${SOURCE_ROOT}/automation/health_probe_common.py" \
|
||||
"${VERSION_DIR}/bin/health_probe_common.py"
|
||||
install -o root -g root -m 0755 "${SOURCE_ROOT}/automation/central_gpt_health_probe.py" \
|
||||
"${VERSION_DIR}/bin/central-gpt-health-probe"
|
||||
install -o root -g root -m 0755 "${SOURCE_ROOT}/automation/codex_executor_health_probe.py" \
|
||||
"${VERSION_DIR}/bin/codex-executor-health-probe"
|
||||
install -o root -g root -m 0755 "${SOURCE_ROOT}/automation/a0_unattended_loop_probe.py" \
|
||||
"${VERSION_DIR}/bin/a0-unattended-loop-probe"
|
||||
install -o root -g root -m 0644 "${SOURCE_ROOT}/schemas/automation-quorum-controller.schema.json" \
|
||||
"${VERSION_DIR}/share/automation-quorum-controller.schema.json"
|
||||
python3 -m py_compile "${VERSION_DIR}/bin/executor-quorum-controller" \
|
||||
"${VERSION_DIR}/bin/health_probe_common.py" \
|
||||
"${VERSION_DIR}/bin/central-gpt-health-probe" \
|
||||
"${VERSION_DIR}/bin/codex-executor-health-probe"
|
||||
python3 -m py_compile "${VERSION_DIR}/bin/a0-unattended-loop-probe"
|
||||
|
||||
install -o root -g root -m 0644 "${SOURCE_ROOT}/systemd/aios-automation-quorum-controller.service" /etc/systemd/system/
|
||||
install -o root -g root -m 0644 "${SOURCE_ROOT}/systemd/aios-automation-quorum-controller.timer" /etc/systemd/system/
|
||||
install -o root -g root -m 0644 "${SOURCE_ROOT}/systemd/aios-central-gpt-health-probe.service" /etc/systemd/system/
|
||||
install -o root -g root -m 0644 "${SOURCE_ROOT}/systemd/aios-central-gpt-health-probe.timer" /etc/systemd/system/
|
||||
install -o root -g root -m 0644 "${SOURCE_ROOT}/systemd/aios-codex-executor-health-probe.service" /etc/systemd/system/
|
||||
install -o root -g root -m 0644 "${SOURCE_ROOT}/systemd/aios-codex-executor-health-probe.timer" /etc/systemd/system/
|
||||
install -o root -g root -m 0644 "${SOURCE_ROOT}/systemd/aios-automation-quorum-controller.sysusers" /usr/lib/sysusers.d/aios-automation-quorum-controller.conf
|
||||
install -o root -g root -m 0644 "${SOURCE_ROOT}/systemd/aios-automation-quorum-controller.tmpfiles" /usr/lib/tmpfiles.d/aios-automation-quorum-controller.conf
|
||||
|
||||
systemd-sysusers /usr/lib/sysusers.d/aios-automation-quorum-controller.conf
|
||||
systemd-tmpfiles --create /usr/lib/tmpfiles.d/aios-automation-quorum-controller.conf
|
||||
install -d -o root -g aios-quorum -m 0750 /etc/aios-automation-quorum
|
||||
if [[ ! -e /etc/aios-automation-quorum/controller.json ]]; then
|
||||
install -o root -g aios-quorum -m 0640 \
|
||||
"${SOURCE_ROOT}/config/aios-automation-quorum-controller.example.json" \
|
||||
/etc/aios-automation-quorum/controller.json
|
||||
fi
|
||||
if [[ ! -e /etc/aios-automation-quorum/health-probes.json ]]; then
|
||||
install -o root -g root -m 0600 \
|
||||
"${SOURCE_ROOT}/config/aios-health-probes.example.json" \
|
||||
/etc/aios-automation-quorum/health-probes.json
|
||||
fi
|
||||
|
||||
PREVIOUS_TARGET=$(readlink -f "${INSTALL_ROOT}/current" 2>/dev/null || true)
|
||||
RECEIPT=${VERSION_DIR}/install-receipt.json
|
||||
python3 - "${VERSION_DIR}" "${VERSION}" "${PREVIOUS_TARGET}" "${RECEIPT}" <<'PY'
|
||||
import hashlib, json, os, pathlib, sys, tempfile
|
||||
root, version, previous, receipt = map(pathlib.Path, (sys.argv[1], sys.argv[2], sys.argv[3] or ".", sys.argv[4]))
|
||||
files = {}
|
||||
for item in sorted(root.rglob("*")):
|
||||
if item.is_file() and not item.is_symlink():
|
||||
files[str(item.relative_to(root))] = hashlib.sha256(item.read_bytes()).hexdigest()
|
||||
payload = {"version": str(version), "previous_target": "" if str(previous) == "." else str(previous), "files": files}
|
||||
fd, temp = tempfile.mkstemp(dir=str(receipt.parent), prefix=".receipt-")
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, sort_keys=True, separators=(",", ":"))
|
||||
handle.write("\n")
|
||||
os.chmod(temp, 0o600)
|
||||
os.replace(temp, receipt)
|
||||
PY
|
||||
|
||||
ln -sfn "${VERSION_DIR}" "${INSTALL_ROOT}/.current-${VERSION}"
|
||||
mv -Tf "${INSTALL_ROOT}/.current-${VERSION}" "${INSTALL_ROOT}/current"
|
||||
|
||||
systemctl daemon-reload
|
||||
echo "installed=${VERSION_DIR}"
|
||||
echo "previous=${PREVIOUS_TARGET}"
|
||||
@@ -1,84 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 0077
|
||||
|
||||
SOURCE_ROOT="${SOURCE_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||
UNIT_ROOT="${UNIT_ROOT:-/etc/systemd/system}"
|
||||
SYSUSERS_ROOT="${SYSUSERS_ROOT:-/etc/sysusers.d}"
|
||||
TMPFILES_ROOT="${TMPFILES_ROOT:-/etc/tmpfiles.d}"
|
||||
CODEX_BINARY=/usr/local/bin/codex
|
||||
CODEX_HOME=/var/lib/aios-codex-brain/.codex
|
||||
AUTH_JSON="$CODEX_HOME/auth.json"
|
||||
CONTROLLER=/data/aios-gitops/worktrees/controller
|
||||
LOCK=/run/lock/aios-codex-auth.lock
|
||||
|
||||
fail() {
|
||||
printf 'STATUS=%s\nAUTOMATION_CLOSED=NO\n' "$1" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
[[ "$(id -u)" -eq 0 ]] || fail OWNER_INSTALL_REQUIRED
|
||||
for command in python3 runuser systemctl systemd-analyze systemd-sysusers systemd-tmpfiles; do
|
||||
command -v "$command" >/dev/null 2>&1 || fail "MISSING_COMMAND_${command}"
|
||||
done
|
||||
|
||||
for path in \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central-decision.service" \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central-health.service" \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central-health.timer" \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central.sysusers" \
|
||||
"$SOURCE_ROOT/systemd/aios-codex-central.tmpfiles" \
|
||||
"$SOURCE_ROOT/automation/codex_central_decision.py" \
|
||||
"$SOURCE_ROOT/automation/codex_central_health_probe.py"; do
|
||||
[[ -f "$path" && ! -L "$path" ]] || fail INVALID_SOURCE
|
||||
done
|
||||
|
||||
systemd-sysusers "$SOURCE_ROOT/systemd/aios-codex-central.sysusers"
|
||||
install -m 0644 "$SOURCE_ROOT/systemd/aios-codex-central.sysusers" "$SYSUSERS_ROOT/aios-codex-central.conf"
|
||||
install -m 0644 "$SOURCE_ROOT/systemd/aios-codex-central.tmpfiles" "$TMPFILES_ROOT/aios-codex-central.conf"
|
||||
systemd-tmpfiles --create "$SOURCE_ROOT/systemd/aios-codex-central.tmpfiles"
|
||||
|
||||
for unit in aios-codex-central-decision.service aios-codex-central-health.service aios-codex-central-health.timer; do
|
||||
install -m 0644 "$SOURCE_ROOT/systemd/$unit" "$UNIT_ROOT/$unit"
|
||||
done
|
||||
systemd-analyze verify \
|
||||
"$UNIT_ROOT/aios-codex-central-decision.service" \
|
||||
"$UNIT_ROOT/aios-codex-central-health.service" \
|
||||
"$UNIT_ROOT/aios-codex-central-health.timer"
|
||||
|
||||
[[ -f "$CODEX_BINARY" && ! -L "$CODEX_BINARY" && -x "$CODEX_BINARY" ]] || \
|
||||
fail BLOCKED_CODEX_CLI_VERSION_INCOMPATIBLE
|
||||
[[ -d "$CONTROLLER" && ! -L "$CONTROLLER" ]] || fail CONTROLLER_WORKDIR_UNAVAILABLE
|
||||
|
||||
help_output="$(runuser -u aios-codex-brain -- env -i \
|
||||
CODEX_HOME="$CODEX_HOME" HOME=/var/lib/aios-codex-brain \
|
||||
PATH=/usr/local/bin:/usr/bin:/bin LANG=C.UTF-8 LC_ALL=C.UTF-8 \
|
||||
"$CODEX_BINARY" exec --help 2>/dev/null)" || fail BLOCKED_CODEX_CLI_VERSION_INCOMPATIBLE
|
||||
for flag in --json --output-schema --output-last-message --sandbox --ask-for-approval --cd --ephemeral; do
|
||||
grep -F -- "$flag" <<<"$help_output" >/dev/null || fail BLOCKED_CODEX_CLI_VERSION_INCOMPATIBLE
|
||||
done
|
||||
unset help_output
|
||||
|
||||
[[ -e "$AUTH_JSON" ]] || fail BLOCKED_CODEX_AUTHENTICATION_REQUIRED
|
||||
if ! python3 - "$AUTH_JSON" <<'PY'
|
||||
import os,pwd,stat,sys
|
||||
p=sys.argv[1]
|
||||
s=os.lstat(p)
|
||||
if stat.S_ISLNK(s.st_mode) or not stat.S_ISREG(s.st_mode): raise SystemExit("unsafe auth")
|
||||
if s.st_uid != pwd.getpwnam("aios-codex-brain").pw_uid: raise SystemExit("wrong owner")
|
||||
if stat.S_IMODE(s.st_mode) != 0o600: raise SystemExit("wrong mode")
|
||||
PY
|
||||
then
|
||||
fail BLOCKED_CODEX_AUTHENTICATION_REQUIRED
|
||||
fi
|
||||
|
||||
runuser -u aios-codex-brain -- env -i \
|
||||
CODEX_HOME="$CODEX_HOME" HOME=/var/lib/aios-codex-brain \
|
||||
PATH=/usr/local/bin:/usr/bin:/bin LANG=C.UTF-8 LC_ALL=C.UTF-8 \
|
||||
"$CODEX_BINARY" login status >/dev/null 2>&1 || fail BLOCKED_CODEX_AUTHENTICATION_REQUIRED
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now aios-codex-central-decision.service
|
||||
systemctl enable --now aios-codex-central-health.timer
|
||||
systemctl start aios-codex-central-health.service || fail BLOCKED_CODEX_AUTHENTICATION_REQUIRED
|
||||
printf 'CODEX_LOCAL_CENTRAL_INSTALL=PASS\n'
|
||||
@@ -1,38 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from services.aios_orchestrator_gateway.config import Config
|
||||
from services.aios_orchestrator_gateway.recovery import (
|
||||
PendingDecisionRecovery,
|
||||
RecoverySettings,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", type=Path, required=True)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--plan", action="store_true")
|
||||
mode.add_argument("--apply-one", action="store_true")
|
||||
args = parser.parse_args()
|
||||
config = Config.from_file(args.config)
|
||||
recovery = PendingDecisionRecovery(RecoverySettings(
|
||||
database_path=config.database_path,
|
||||
contract_commit=config.contract_commit,
|
||||
lease_seconds=config.lease_seconds,
|
||||
max_retries=config.max_retries,
|
||||
retry_after=config.retry_base_seconds,
|
||||
), plan_only=args.plan)
|
||||
if args.plan:
|
||||
print(json.dumps(recovery.plan(), separators=(",", ":"), sort_keys=True))
|
||||
else:
|
||||
print(json.dumps({"processed_event_id": recovery.apply_one()}, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,130 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Submit an OpenClaw/Codex task lifecycle event over HTTP or a polled inbox."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from services.aios_orchestrator_gateway.models import Event
|
||||
from services.aios_orchestrator_gateway.securefs import atomic_write_bytes_shared, read_secure_json, read_secure_text
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--project-key", required=True)
|
||||
p.add_argument("--contract-commit", required=True)
|
||||
p.add_argument("--event-id", required=True)
|
||||
p.add_argument("--source-agent", required=True, choices=["openclaw", "codex", "gitea", "system"])
|
||||
p.add_argument("--event-type", required=True)
|
||||
p.add_argument("--work-unit-id", required=True)
|
||||
p.add_argument(
|
||||
"--status",
|
||||
required=True,
|
||||
choices=["READY", "RUNNING", "PASS", "BLOCKED", "FAIL", "TIMEOUT", "NOT_EXECUTED"],
|
||||
)
|
||||
p.add_argument("--state-version", type=int, required=True)
|
||||
p.add_argument("--implementation-commit")
|
||||
p.add_argument("--result-commit")
|
||||
p.add_argument("--blocker")
|
||||
p.add_argument("--evidence-ref", action="append", default=[])
|
||||
p.add_argument("--payload-file", type=Path)
|
||||
p.add_argument("--endpoint")
|
||||
p.add_argument("--token-file", type=Path)
|
||||
p.add_argument("--inbox-dir", type=Path)
|
||||
p.add_argument("--timeout-seconds", type=int, default=15)
|
||||
return p
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parser().parse_args()
|
||||
if not args.endpoint and not args.inbox_dir:
|
||||
raise SystemExit("either --endpoint or --inbox-dir is required")
|
||||
if args.endpoint and not args.token_file:
|
||||
raise SystemExit("--token-file is required with --endpoint")
|
||||
payload = {}
|
||||
if args.payload_file:
|
||||
if not args.payload_file.is_absolute():
|
||||
raise SystemExit("--payload-file must be absolute")
|
||||
payload = read_secure_json(
|
||||
args.payload_file, expected_uid=os.getuid(), max_bytes=262_144
|
||||
)
|
||||
if not isinstance(payload, dict):
|
||||
raise SystemExit("payload root must be an object")
|
||||
|
||||
event = Event.from_dict(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"project_key": args.project_key,
|
||||
"contract_commit": args.contract_commit,
|
||||
"event_id": args.event_id,
|
||||
"source_agent": args.source_agent,
|
||||
"event_type": args.event_type,
|
||||
"work_unit_id": args.work_unit_id,
|
||||
"status": args.status,
|
||||
"state_version": args.state_version,
|
||||
"implementation_commit": args.implementation_commit,
|
||||
"result_commit": args.result_commit,
|
||||
"blocker": args.blocker,
|
||||
"evidence_refs": args.evidence_ref,
|
||||
"payload": payload,
|
||||
}
|
||||
)
|
||||
encoded = event.canonical_json().encode()
|
||||
http_error: Exception | None = None
|
||||
|
||||
if args.endpoint:
|
||||
token = read_secure_text(
|
||||
args.token_file,
|
||||
expected_uid=None,
|
||||
max_bytes=64_000,
|
||||
)
|
||||
request = urllib.request.Request(
|
||||
args.endpoint.rstrip("/") + "/v1/orchestrator/events",
|
||||
data=encoded,
|
||||
method="POST",
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
"Idempotency-Key": event.event_id,
|
||||
"User-Agent": "aios-agent-event-reporter/0.2",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=args.timeout_seconds) as response:
|
||||
response.read(65_536)
|
||||
if response.status not in {200, 202}:
|
||||
raise RuntimeError(f"unexpected HTTP {response.status}")
|
||||
print(f"EVENT_SUBMIT=PASS event_id={event.event_id} channel=http")
|
||||
return 0
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError) as exc:
|
||||
http_error = exc
|
||||
|
||||
if args.inbox_dir:
|
||||
pending = args.inbox_dir / "pending"
|
||||
path = pending / f"{event.event_id}.json"
|
||||
atomic_write_bytes_shared(
|
||||
path,
|
||||
(json.dumps(event.to_dict(), indent=2, sort_keys=True) + "\n").encode(),
|
||||
mode=0o640,
|
||||
group_writable_directory=True,
|
||||
)
|
||||
print(f"EVENT_SUBMIT=PASS event_id={event.event_id} channel=file_inbox")
|
||||
return 0
|
||||
|
||||
print(
|
||||
f"EVENT_SUBMIT=FAIL event_id={event.event_id} reason={type(http_error).__name__}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,102 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
umask 0077
|
||||
|
||||
INSTALL_ROOT="${INSTALL_ROOT:-/opt/aios-orchestrator-gateway}"
|
||||
STATE_ROOT="${STATE_ROOT:-/var/lib/aios-orchestrator-gateway}"
|
||||
RECEIPT="$STATE_ROOT/install-receipt.json"
|
||||
|
||||
fail() {
|
||||
printf 'AIOS_ORCHESTRATOR_ROLLBACK=BLOCKED reason=%s\n' "$1" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
[[ "$(id -u)" -eq 0 ]] || fail OWNER_REQUIRED
|
||||
[[ -f "$RECEIPT" ]] || fail RECEIPT_MISSING
|
||||
[[ ! -L "$RECEIPT" ]] || fail RECEIPT_SYMLINK
|
||||
[[ "$(stat -c '%u:%a' "$RECEIPT")" == "0:600" ]] || fail RECEIPT_PERMISSION
|
||||
|
||||
readarray -t VALUES < <(
|
||||
python3 - "$RECEIPT" "$INSTALL_ROOT/releases" "$INSTALL_ROOT/current" <<'PY'
|
||||
import json, os, stat, sys
|
||||
from pathlib import Path
|
||||
receipt = Path(sys.argv[1])
|
||||
releases = Path(sys.argv[2]).resolve()
|
||||
current_link = Path(sys.argv[3])
|
||||
data = json.loads(receipt.read_text())
|
||||
previous = data.get("previous_release")
|
||||
current = data.get("current_release")
|
||||
if not previous or not current:
|
||||
raise SystemExit(2)
|
||||
previous_path = Path(previous).resolve()
|
||||
current_path = Path(current).resolve()
|
||||
if previous_path.parent != releases or current_path.parent != releases:
|
||||
raise SystemExit(2)
|
||||
if not stat.S_ISDIR(os.lstat(previous_path).st_mode) or not stat.S_ISDIR(os.lstat(current_path).st_mode):
|
||||
raise SystemExit(2)
|
||||
for release in (previous_path, current_path):
|
||||
if len(release.name) != 64 or any(ch not in "0123456789abcdef" for ch in release.name):
|
||||
raise SystemExit(2)
|
||||
digest_file = release / "SOURCE_DIGEST"
|
||||
if digest_file.is_symlink() or not digest_file.is_file():
|
||||
raise SystemExit(2)
|
||||
if digest_file.read_text(encoding="utf-8").strip() != release.name:
|
||||
raise SystemExit(2)
|
||||
last = data.get("last_rollback")
|
||||
if isinstance(last, dict) and last.get("to") == current and current_link.is_symlink():
|
||||
if current_link.resolve() == current_path:
|
||||
print("ALREADY_ROLLED_BACK")
|
||||
raise SystemExit(0)
|
||||
print(previous_path)
|
||||
print(current_path)
|
||||
PY
|
||||
) || fail RECEIPT_INVALID
|
||||
|
||||
if [[ "${VALUES[0]:-}" == "ALREADY_ROLLED_BACK" ]]; then
|
||||
echo "AIOS_ORCHESTRATOR_ROLLBACK=PASS result=NOOP_ALREADY_ROLLED_BACK"
|
||||
exit 0
|
||||
fi
|
||||
[[ "${#VALUES[@]}" -eq 2 ]] || fail RECEIPT_INVALID
|
||||
PREVIOUS="${VALUES[0]}"
|
||||
CURRENT="${VALUES[1]}"
|
||||
systemctl stop aios-orchestrator-gateway-poll.timer || true
|
||||
systemctl stop aios-orchestrator-gateway-poll.service || true
|
||||
systemctl stop aios-orchestrator-gateway.service || true
|
||||
ln -s "$PREVIOUS" "$INSTALL_ROOT/.current-rollback-$$"
|
||||
mv -Tf "$INSTALL_ROOT/.current-rollback-$$" "$INSTALL_ROOT/current"
|
||||
systemctl start aios-orchestrator-gateway.service
|
||||
systemctl is-active --quiet aios-orchestrator-gateway.service
|
||||
systemctl start aios-orchestrator-gateway-poll.timer
|
||||
systemctl is-active --quiet aios-orchestrator-gateway-poll.timer
|
||||
|
||||
python3 - "$RECEIPT" "$PREVIOUS" "$CURRENT" <<'PY'
|
||||
import json, os, sys, tempfile
|
||||
from pathlib import Path
|
||||
path = Path(sys.argv[1])
|
||||
data = json.loads(path.read_text())
|
||||
data["current_release"] = sys.argv[2]
|
||||
data["previous_release"] = sys.argv[3]
|
||||
data["last_rollback"] = {"from": sys.argv[3], "to": sys.argv[2]}
|
||||
fd, temp = tempfile.mkstemp(prefix=".rollback-receipt-", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, "w") as handle:
|
||||
json.dump(data, handle, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.chmod(temp, 0o600)
|
||||
os.chown(temp, 0, 0)
|
||||
os.replace(temp, path)
|
||||
directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(directory)
|
||||
finally:
|
||||
os.close(directory)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temp)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
PY
|
||||
|
||||
printf 'AIOS_ORCHESTRATOR_ROLLBACK=PASS current=%s previous=%s\n' "$PREVIOUS" "$CURRENT"
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ ${EUID} -ne 0 ]]; then
|
||||
echo "root required" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TARGET=${1:?usage: rollback_automation_quorum_controller.sh ABSOLUTE_RELEASE_PATH}
|
||||
INSTALL_ROOT=/opt/aios-automation-quorum
|
||||
TARGET_REAL=$(readlink -f -- "${TARGET}" 2>/dev/null || true)
|
||||
[[ -n ${TARGET_REAL} && ${TARGET} == "${TARGET_REAL}" ]] || { echo "non-canonical or missing rollback target" >&2; exit 1; }
|
||||
case ${TARGET_REAL} in
|
||||
"${INSTALL_ROOT}"/releases/*) ;;
|
||||
*) echo "target outside release root" >&2; exit 1 ;;
|
||||
esac
|
||||
[[ -d ${TARGET_REAL} && ! -L ${TARGET} ]] || { echo "invalid rollback target" >&2; exit 1; }
|
||||
|
||||
CURRENT=$(readlink -f "${INSTALL_ROOT}/current" 2>/dev/null || true)
|
||||
if [[ ${CURRENT} == "${TARGET_REAL}" ]]; then
|
||||
echo "already_current=${CURRENT}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ln -sfn "${TARGET_REAL}" "${INSTALL_ROOT}/.rollback-current"
|
||||
mv -Tf "${INSTALL_ROOT}/.rollback-current" "${INSTALL_ROOT}/current"
|
||||
systemctl daemon-reload
|
||||
systemctl try-restart aios-automation-quorum-controller.timer
|
||||
echo "rolled_back_to=$(readlink -f "${INSTALL_ROOT}/current")"
|
||||
@@ -1,117 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dependency-free, deterministic validation for the shipped controller config."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
REQUIRED = {
|
||||
"project_key",
|
||||
"contract_commit",
|
||||
"database_path",
|
||||
"file_inbox_path",
|
||||
"health_receipts",
|
||||
"blocker_retry_threshold",
|
||||
"max_health_age_seconds",
|
||||
"lease_ttl_seconds",
|
||||
"gateway_event_url",
|
||||
"gateway_token_file",
|
||||
}
|
||||
OPTIONAL = {"$schema", "default_workload", "git_ref_polling"}
|
||||
WORKLOADS = {
|
||||
"large_repetitive_infrastructure_monitoring",
|
||||
"complex_code_security_architecture",
|
||||
}
|
||||
|
||||
|
||||
def fail(message: str) -> None:
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def absolute_path(value: object, name: str) -> None:
|
||||
if not isinstance(value, str) or not value.startswith("/") or "\x00" in value:
|
||||
fail(f"{name} must be an absolute POSIX path")
|
||||
|
||||
|
||||
def bounded_integer(value: object, name: str, minimum: int, maximum: int) -> None:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum:
|
||||
fail(f"{name} must be an integer in [{minimum}, {maximum}]")
|
||||
|
||||
|
||||
def validate(config: object, schema: object) -> None:
|
||||
if not isinstance(config, dict) or not isinstance(schema, dict):
|
||||
fail("config and schema must be JSON objects")
|
||||
if schema.get("$schema") != "https://json-schema.org/draft/2020-12/schema":
|
||||
fail("unexpected JSON Schema dialect")
|
||||
if schema.get("additionalProperties") is not False:
|
||||
fail("schema must reject additional properties")
|
||||
if set(schema.get("required", [])) != REQUIRED:
|
||||
fail("schema required fields drifted from validator")
|
||||
keys = set(config)
|
||||
if not REQUIRED <= keys or keys - REQUIRED - OPTIONAL:
|
||||
fail("missing or unexpected top-level config fields")
|
||||
absolute_path(config["database_path"], "database_path")
|
||||
absolute_path(config["file_inbox_path"], "file_inbox_path")
|
||||
absolute_path(config["gateway_token_file"], "gateway_token_file")
|
||||
project_key = config["project_key"]
|
||||
if not isinstance(project_key, str) or not project_key or len(project_key) > 128:
|
||||
fail("project_key is invalid")
|
||||
commit = config["contract_commit"]
|
||||
if not isinstance(commit, str) or len(commit) != 40 or any(c not in "0123456789abcdef" for c in commit):
|
||||
fail("contract_commit must be a full lowercase Git OID")
|
||||
receipts = config["health_receipts"]
|
||||
if not isinstance(receipts, dict) or set(receipts) != {"central_gpt", "openclaw", "codex"}:
|
||||
fail("health_receipts must contain exactly central_gpt, openclaw, and codex")
|
||||
for subject, value in receipts.items():
|
||||
absolute_path(value, f"health_receipts.{subject}")
|
||||
git_polling = config.get("git_ref_polling")
|
||||
if git_polling is not None:
|
||||
if not isinstance(git_polling, dict) or set(git_polling) != {"repository", "refs"}:
|
||||
fail("git_ref_polling must contain exactly repository and refs")
|
||||
absolute_path(git_polling["repository"], "git_ref_polling.repository")
|
||||
refs = git_polling["refs"]
|
||||
if not isinstance(refs, list) or not refs or len(refs) != len(set(refs)):
|
||||
fail("git_ref_polling.refs must be a non-empty unique list")
|
||||
if any(not isinstance(ref, str) or not ref.startswith("refs/") or ".." in ref for ref in refs):
|
||||
fail("git_ref_polling.refs contains an invalid ref")
|
||||
if config.get("default_workload", "large_repetitive_infrastructure_monitoring") not in WORKLOADS:
|
||||
fail("unsupported default_workload")
|
||||
bounded_integer(config["blocker_retry_threshold"], "blocker_retry_threshold", 1, 20)
|
||||
bounded_integer(config["max_health_age_seconds"], "max_health_age_seconds", 5, 300)
|
||||
bounded_integer(config["lease_ttl_seconds"], "lease_ttl_seconds", 15, 3600)
|
||||
url = config["gateway_event_url"]
|
||||
parsed = urlparse(url) if isinstance(url, str) else None
|
||||
if (
|
||||
not parsed
|
||||
or parsed.scheme != "http"
|
||||
or parsed.hostname != "127.0.0.1"
|
||||
or parsed.port != 18920
|
||||
or parsed.path != "/v1/orchestrator/events"
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
fail("gateway_event_url must be the fixed loopback orchestrator endpoint")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 3:
|
||||
print(f"usage: {Path(sys.argv[0]).name} CONFIG SCHEMA", file=sys.stderr)
|
||||
return 2
|
||||
with Path(sys.argv[1]).open("r", encoding="utf-8") as handle:
|
||||
config = json.load(handle)
|
||||
with Path(sys.argv[2]).open("r", encoding="utf-8") as handle:
|
||||
schema = json.load(handle)
|
||||
try:
|
||||
validate(config, schema)
|
||||
except ValueError as error:
|
||||
print(f"schema validation failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,32 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
CONFIG="${1:-/etc/aios-orchestrator-gateway/config.json}"
|
||||
PORT="${PORT:-18920}"
|
||||
|
||||
python3 -m json.tool "$CONFIG" >/dev/null
|
||||
systemd-analyze verify \
|
||||
/etc/systemd/system/aios-orchestrator-gateway.service \
|
||||
/etc/systemd/system/aios-orchestrator-gateway-poll.service \
|
||||
/etc/systemd/system/aios-orchestrator-gateway-poll.timer
|
||||
systemctl is-active --quiet aios-orchestrator-gateway.service
|
||||
systemctl is-active --quiet aios-orchestrator-gateway-poll.timer
|
||||
|
||||
health="$(curl --fail --silent --show-error "http://127.0.0.1:${PORT}/healthz")"
|
||||
python3 -c '
|
||||
import json, sys
|
||||
v=json.load(sys.stdin)
|
||||
assert v["service"]=="aios-orchestrator-gateway"
|
||||
assert v["status"]=="active"
|
||||
assert v["polling"] is True
|
||||
assert len(v["contract_commit"])==40
|
||||
' <<<"$health"
|
||||
|
||||
systemctl start aios-orchestrator-gateway-poll.service
|
||||
if journalctl -u aios-orchestrator-gateway.service -n 200 --no-pager \
|
||||
| grep -Eqi 'Bearer |token=|api[_-]?key|secret='; then
|
||||
echo "AIOS_ORCHESTRATOR_VERIFY=BLOCKED reason=SECRET_PATTERN_IN_LOG" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "AIOS_ORCHESTRATOR_VERIFY=PASS"
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT=${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}
|
||||
|
||||
python3 -m py_compile "${ROOT}/automation/executor_quorum_controller.py"
|
||||
python3 -m py_compile \
|
||||
"${ROOT}/automation/health_probe_common.py" \
|
||||
"${ROOT}/automation/central_gpt_health_probe.py" \
|
||||
"${ROOT}/automation/codex_executor_health_probe.py" \
|
||||
"${ROOT}/automation/a0_unattended_loop_probe.py"
|
||||
python3 -m unittest discover -s "${ROOT}/tests" -p 'test_executor_quorum_controller.py' -v
|
||||
python3 -m unittest discover -s "${ROOT}/tests" -p 'test_aios_health_probes.py' -v
|
||||
python3 -m json.tool "${ROOT}/config/aios-automation-quorum-controller.example.json" >/dev/null
|
||||
python3 -m json.tool "${ROOT}/schemas/automation-quorum-controller.schema.json" >/dev/null
|
||||
python3 "${ROOT}/scripts/validate_automation_quorum_schema.py" \
|
||||
"${ROOT}/config/aios-automation-quorum-controller.example.json" \
|
||||
"${ROOT}/schemas/automation-quorum-controller.schema.json"
|
||||
python3 -m py_compile "${ROOT}/scripts/validate_automation_quorum_schema.py"
|
||||
bash -n "${ROOT}/scripts/install_automation_quorum_controller.sh"
|
||||
bash -n "${ROOT}/scripts/rollback_automation_quorum_controller.sh"
|
||||
bash -n "${ROOT}/scripts/verify_automation_quorum_controller.sh"
|
||||
systemd-analyze verify \
|
||||
"${ROOT}/systemd/aios-automation-quorum-controller.service" \
|
||||
"${ROOT}/systemd/aios-automation-quorum-controller.timer" \
|
||||
"${ROOT}/systemd/aios-central-gpt-health-probe.service" \
|
||||
"${ROOT}/systemd/aios-central-gpt-health-probe.timer" \
|
||||
"${ROOT}/systemd/aios-codex-executor-health-probe.service" \
|
||||
"${ROOT}/systemd/aios-codex-executor-health-probe.timer"
|
||||
@@ -1,652 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Hardened WeChat outbound backend for AIOS.
|
||||
|
||||
Only the dedicated UCC adapter is a supported send backend. OpenClaw channel
|
||||
discovery is retained for diagnostics, but generic OpenClaw chat or agent
|
||||
delivery is never promoted to WeChat delivery.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import selectors
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
OPENCLAW_BIN = os.environ.get("AIOS_OPENCLAW_BIN", "openclaw")
|
||||
WECHAT_ACCOUNT_ID = os.environ.get("AIOS_WECHAT_ACCOUNT_ID", "")
|
||||
WECHAT_CHANNEL = os.environ.get("AIOS_WECHAT_CHANNEL", "")
|
||||
SEND_BACKEND = os.environ.get("AIOS_WECHAT_SEND_BACKEND", "")
|
||||
UCC_ADAPTER_URL = os.environ.get("AIOS_UCC_WECHAT_ADAPTER_URL", "http://127.0.0.1:18800")
|
||||
UCC_ADAPTER_TOKEN = os.environ.get("AIOS_UCC_WECHAT_ADAPTER_TOKEN") or os.environ.get(
|
||||
"AIOS_WECHAT_QR_ADAPTER_TOKEN", ""
|
||||
)
|
||||
UCC_SEND_PATH = os.environ.get("AIOS_UCC_WECHAT_SEND_PATH", "/api/internal/wechat/send")
|
||||
_allowed_origins = os.environ.get("AIOS_UCC_WECHAT_ALLOWED_ORIGINS", "").strip()
|
||||
UCC_ADAPTER_ALLOWED_ORIGINS = tuple(
|
||||
origin.strip() for origin in _allowed_origins.split(",") if origin.strip()
|
||||
) or (
|
||||
"http://127.0.0.1:18800",
|
||||
"http://[::1]:18800",
|
||||
)
|
||||
|
||||
MAX_TARGET_BYTES = 512
|
||||
MAX_CONTENT_BYTES = 32 * 1024
|
||||
MAX_IDENTIFIER_BYTES = 512
|
||||
MAX_METADATA_BYTES = 8 * 1024
|
||||
MAX_METADATA_DEPTH = 32
|
||||
MAX_METADATA_NODES = 1024
|
||||
MAX_RESPONSE_BYTES = 64 * 1024
|
||||
MAX_ENDPOINT_BYTES = 256
|
||||
_DEFAULT_TIMEOUT_SECONDS = 10
|
||||
_MAX_TIMEOUT_SECONDS = 30
|
||||
_LEGACY_SEND_PATH = "/internal/channels/wechat/send"
|
||||
_SAFE_PATH = re.compile(r"^/[A-Za-z0-9._~!$&'()*+,;=:@%/-]+$")
|
||||
_MAX_PATH_DECODE_ROUNDS = 4
|
||||
_OPENCLAW_TIMEOUT_RETURN_CODE = 124
|
||||
_OPENCLAW_OUTPUT_LIMIT_RETURN_CODE = 125
|
||||
_OPENCLAW_SPAWN_ERROR_RETURN_CODE = 126
|
||||
|
||||
|
||||
def _bounded_timeout(value: str) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return _DEFAULT_TIMEOUT_SECONDS
|
||||
return min(max(parsed, 1), _MAX_TIMEOUT_SECONDS)
|
||||
|
||||
|
||||
REQUEST_TIMEOUT_SECONDS = _bounded_timeout(os.environ.get("AIOS_UCC_WECHAT_TIMEOUT_SECONDS", "10"))
|
||||
|
||||
|
||||
class BackendResult:
|
||||
def __init__(self, ok: bool, backend: str, status: str, data: Dict[str, Any]) -> None:
|
||||
self.ok = ok
|
||||
self.backend = backend
|
||||
self.status = status
|
||||
self.data = data
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {"ok": self.ok, "backend": self.backend, "status": self.status, **self.data}
|
||||
|
||||
|
||||
class _BackendProtocolError(Exception):
|
||||
"""Internal marker whose details must never cross the module boundary."""
|
||||
|
||||
|
||||
def _openclaw_base_args() -> List[str]:
|
||||
"""Return the OpenClaw 2026.6.11-compatible command prefix.
|
||||
|
||||
Gateway/auth global flags are not supported by the installed CLI and would
|
||||
also expose credentials through the process argument vector.
|
||||
"""
|
||||
|
||||
return [OPENCLAW_BIN]
|
||||
|
||||
|
||||
def _terminate_process_tree(process: Any) -> None:
|
||||
if os.name == "posix":
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
process.wait(timeout=0.5)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
time.sleep(0.05)
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
process.wait(timeout=0.5)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
return
|
||||
if process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
process.terminate()
|
||||
process.wait(timeout=1)
|
||||
return
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
try:
|
||||
process.kill()
|
||||
process.wait(timeout=1)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
|
||||
|
||||
def _run_openclaw(
|
||||
args: List[str], timeout: int = 15
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
bounded_timeout = min(max(int(timeout), 1), _MAX_TIMEOUT_SECONDS)
|
||||
popen_options: Dict[str, Any] = {
|
||||
"stdout": subprocess.PIPE,
|
||||
"stderr": subprocess.PIPE,
|
||||
}
|
||||
if os.name == "posix":
|
||||
popen_options["start_new_session"] = True
|
||||
elif hasattr(subprocess, "CREATE_NEW_PROCESS_GROUP"):
|
||||
popen_options["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
|
||||
try:
|
||||
process = subprocess.Popen(args, **popen_options)
|
||||
except (OSError, ValueError):
|
||||
return subprocess.CompletedProcess(args, _OPENCLAW_SPAWN_ERROR_RETURN_CODE, "", "")
|
||||
|
||||
selector = selectors.DefaultSelector()
|
||||
streams = {"stdout": process.stdout, "stderr": process.stderr}
|
||||
buffers = {"stdout": bytearray(), "stderr": bytearray()}
|
||||
try:
|
||||
for name, stream in streams.items():
|
||||
if stream is not None:
|
||||
selector.register(stream, selectors.EVENT_READ, name)
|
||||
except (OSError, ValueError):
|
||||
selector.close()
|
||||
_terminate_process_tree(process)
|
||||
for stream in streams.values():
|
||||
if stream is not None:
|
||||
stream.close()
|
||||
return subprocess.CompletedProcess(args, _OPENCLAW_SPAWN_ERROR_RETURN_CODE, "", "")
|
||||
|
||||
deadline = time.monotonic() + bounded_timeout
|
||||
total_bytes = 0
|
||||
return_code: Optional[int] = None
|
||||
try:
|
||||
while selector.get_map():
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return_code = _OPENCLAW_TIMEOUT_RETURN_CODE
|
||||
break
|
||||
events = selector.select(min(remaining, 0.1))
|
||||
for key, _mask in events:
|
||||
allowance = MAX_RESPONSE_BYTES - total_bytes
|
||||
try:
|
||||
chunk = os.read(key.fileobj.fileno(), min(8192, allowance + 1))
|
||||
except OSError:
|
||||
return_code = _OPENCLAW_SPAWN_ERROR_RETURN_CODE
|
||||
break
|
||||
if not chunk:
|
||||
selector.unregister(key.fileobj)
|
||||
key.fileobj.close()
|
||||
continue
|
||||
if len(chunk) > allowance:
|
||||
return_code = _OPENCLAW_OUTPUT_LIMIT_RETURN_CODE
|
||||
break
|
||||
buffers[key.data].extend(chunk)
|
||||
total_bytes += len(chunk)
|
||||
if return_code is not None:
|
||||
break
|
||||
finally:
|
||||
selector.close()
|
||||
|
||||
if return_code is not None:
|
||||
_terminate_process_tree(process)
|
||||
for stream in streams.values():
|
||||
if stream is not None and not stream.closed:
|
||||
stream.close()
|
||||
return subprocess.CompletedProcess(args, return_code, "", "")
|
||||
try:
|
||||
completed_return_code = process.wait(timeout=1)
|
||||
except subprocess.TimeoutExpired:
|
||||
_terminate_process_tree(process)
|
||||
return subprocess.CompletedProcess(args, _OPENCLAW_TIMEOUT_RETURN_CODE, "", "")
|
||||
stdout = buffers["stdout"].decode("utf-8", errors="replace")
|
||||
stderr = buffers["stderr"].decode("utf-8", errors="replace")
|
||||
return subprocess.CompletedProcess(args, completed_return_code, stdout, stderr)
|
||||
|
||||
|
||||
def _json_dict(text: str) -> Dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(text or "{}")
|
||||
except (RecursionError, TypeError, ValueError, json.JSONDecodeError):
|
||||
return {}
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def discover_openclaw_wechat_channel() -> Optional[str]:
|
||||
"""Discover an explicitly typed WeChat/Weixin channel for diagnostics."""
|
||||
|
||||
if WECHAT_CHANNEL:
|
||||
return WECHAT_CHANNEL
|
||||
try:
|
||||
proc = _run_openclaw(_openclaw_base_args() + ["channels", "list", "--json"], timeout=15)
|
||||
except (OSError, subprocess.SubprocessError, ValueError):
|
||||
return None
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
data = _json_dict(proc.stdout[:MAX_RESPONSE_BYTES])
|
||||
candidates: List[str] = []
|
||||
|
||||
def walk(value: Any) -> None:
|
||||
if isinstance(value, dict):
|
||||
name = value.get("name") or value.get("channel") or value.get("id")
|
||||
kind = value.get("type") or value.get("kind")
|
||||
if isinstance(name, str) and isinstance(kind, str):
|
||||
label = f"{name} {kind}".lower()
|
||||
if "wechat" in label or "weixin" in label:
|
||||
candidates.append(name)
|
||||
for child in value.values():
|
||||
walk(child)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
walk(child)
|
||||
|
||||
walk(data)
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
|
||||
def _auth_headers() -> Dict[str, str]:
|
||||
headers = {"Content-Type": "application/json", "Accept": "application/json"}
|
||||
if UCC_ADAPTER_TOKEN:
|
||||
headers["Authorization"] = "Bearer " + UCC_ADAPTER_TOKEN
|
||||
return headers
|
||||
|
||||
|
||||
def _safe_path(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
path = value if value.startswith("/") else "/" + value
|
||||
try:
|
||||
encoded_path = path.encode("utf-8")
|
||||
except UnicodeEncodeError:
|
||||
return None
|
||||
if len(encoded_path) > MAX_ENDPOINT_BYTES:
|
||||
return None
|
||||
if not _SAFE_PATH.fullmatch(path):
|
||||
return None
|
||||
decoded = path
|
||||
for _ in range(_MAX_PATH_DECODE_ROUNDS):
|
||||
try:
|
||||
next_decoded = urllib.parse.unquote_to_bytes(decoded).decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
if next_decoded == decoded:
|
||||
break
|
||||
decoded = next_decoded
|
||||
if "%" in decoded or "\\" in decoded or "\x00" in decoded:
|
||||
return None
|
||||
if not _SAFE_PATH.fullmatch(decoded):
|
||||
return None
|
||||
if any(segment in {".", ".."} for segment in decoded.split("/")):
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def _unique_paths(paths: List[str]) -> List[str]:
|
||||
out: List[str] = []
|
||||
for candidate in paths:
|
||||
path = _safe_path(candidate)
|
||||
if path and path not in out:
|
||||
out.append(path)
|
||||
if len(out) == 2:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _ucc_send_paths() -> List[str]:
|
||||
canonical = _safe_path(UCC_SEND_PATH) or "/api/internal/wechat/send"
|
||||
return _unique_paths([canonical, _LEGACY_SEND_PATH])
|
||||
|
||||
|
||||
def _canonical_origin(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str) or not value or len(value) > 2048:
|
||||
return None
|
||||
if "\\" in value or any(ord(character) < 32 for character in value):
|
||||
return None
|
||||
try:
|
||||
parsed = urllib.parse.urlsplit(value)
|
||||
port = parsed.port
|
||||
except (UnicodeError, ValueError):
|
||||
return None
|
||||
scheme = parsed.scheme.lower()
|
||||
if scheme not in {"http", "https"} or not parsed.hostname:
|
||||
return None
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment or parsed.path not in {"", "/"}:
|
||||
return None
|
||||
try:
|
||||
host = parsed.hostname.rstrip(".").encode("idna").decode("ascii").lower()
|
||||
except UnicodeError:
|
||||
return None
|
||||
if not host:
|
||||
return None
|
||||
if port is None:
|
||||
port = 80 if scheme == "http" else 443
|
||||
rendered_host = f"[{host}]" if ":" in host else host
|
||||
return f"{scheme}://{rendered_host}:{port}"
|
||||
|
||||
|
||||
def _adapter_base_url() -> Optional[str]:
|
||||
origin = _canonical_origin(UCC_ADAPTER_URL)
|
||||
allowed = {
|
||||
canonical
|
||||
for canonical in (_canonical_origin(value) for value in UCC_ADAPTER_ALLOWED_ORIGINS)
|
||||
if canonical
|
||||
}
|
||||
return origin if origin and origin in allowed else None
|
||||
|
||||
|
||||
class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(
|
||||
self,
|
||||
req: Any,
|
||||
fp: Any,
|
||||
code: int,
|
||||
msg: str,
|
||||
headers: Any,
|
||||
newurl: str,
|
||||
) -> None:
|
||||
del req, fp, code, msg, headers, newurl
|
||||
return None
|
||||
|
||||
|
||||
def _open_no_redirect(request: urllib.request.Request, timeout: int) -> Any:
|
||||
opener = urllib.request.build_opener(_NoRedirectHandler())
|
||||
return opener.open(request, timeout=timeout)
|
||||
|
||||
|
||||
def ucc_adapter_alive() -> bool:
|
||||
base_url = _adapter_base_url()
|
||||
if not base_url:
|
||||
return False
|
||||
request = urllib.request.Request(
|
||||
base_url + "/health", headers=_auth_headers(), method="GET"
|
||||
)
|
||||
try:
|
||||
with _open_no_redirect(request, timeout=min(5, REQUEST_TIMEOUT_SECONDS)) as response:
|
||||
return 200 <= int(response.status) < 300
|
||||
except (OSError, ValueError, urllib.error.URLError):
|
||||
return False
|
||||
|
||||
|
||||
def select_backend() -> Dict[str, Any]:
|
||||
explicit = SEND_BACKEND.strip().lower()
|
||||
if explicit:
|
||||
if explicit == "ucc-adapter":
|
||||
return {"backend": "ucc-adapter", "reason": "explicit"}
|
||||
return {"backend": "unavailable", "reason": "unsupported_backend"}
|
||||
if ucc_adapter_alive():
|
||||
return {"backend": "ucc-adapter", "reason": "ucc_adapter_alive"}
|
||||
return {"backend": "unavailable", "reason": "ucc_adapter_unavailable"}
|
||||
|
||||
|
||||
def _post_ucc(path: str, payload: Dict[str, Any]) -> Tuple[int, Dict[str, Any]]:
|
||||
base_url = _adapter_base_url()
|
||||
safe_path = _safe_path(path)
|
||||
if not base_url or not safe_path:
|
||||
raise _BackendProtocolError("invalid adapter endpoint")
|
||||
try:
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise _BackendProtocolError("invalid request payload") from exc
|
||||
request = urllib.request.Request(
|
||||
base_url + safe_path,
|
||||
data=body,
|
||||
headers=_auth_headers(),
|
||||
method="POST",
|
||||
)
|
||||
with _open_no_redirect(request, timeout=REQUEST_TIMEOUT_SECONDS) as response:
|
||||
raw = response.read(MAX_RESPONSE_BYTES + 1)
|
||||
if len(raw) > MAX_RESPONSE_BYTES:
|
||||
raise _BackendProtocolError("adapter response too large")
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise _BackendProtocolError("adapter response encoding invalid") from exc
|
||||
data = _json_dict(text)
|
||||
if not data:
|
||||
raise _BackendProtocolError("adapter response invalid")
|
||||
return int(response.status), data
|
||||
|
||||
|
||||
def _ucc_response_sent(data: Dict[str, Any]) -> bool:
|
||||
return isinstance(data, dict) and data.get("ok") is True and data.get("status") == "sent"
|
||||
|
||||
|
||||
def _ucc_result_is_ok(data: Dict[str, Any], dry_run: bool = False) -> bool:
|
||||
expected_status = "dry_run" if dry_run else "sent"
|
||||
return isinstance(data, dict) and data.get("ok") is True and data.get("status") == expected_status
|
||||
|
||||
|
||||
def _utf8_size(value: str) -> Optional[int]:
|
||||
try:
|
||||
return len(value.encode("utf-8"))
|
||||
except UnicodeEncodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _valid_identifier(value: Any, required: bool = False) -> bool:
|
||||
if not isinstance(value, str):
|
||||
return False
|
||||
if required and not value.strip():
|
||||
return False
|
||||
if not value:
|
||||
return not required
|
||||
if len(value) > MAX_IDENTIFIER_BYTES:
|
||||
return False
|
||||
size = _utf8_size(value)
|
||||
return size is not None and size <= MAX_IDENTIFIER_BYTES and not any(
|
||||
ord(character) < 32 or ord(character) == 127 for character in value
|
||||
)
|
||||
|
||||
|
||||
def _validate_metadata(metadata: Any) -> Optional[str]:
|
||||
if not isinstance(metadata, dict):
|
||||
return "invalid_input"
|
||||
nodes = 0
|
||||
stack: List[Tuple[Any, int]] = [(metadata, 0)]
|
||||
while stack:
|
||||
value, depth = stack.pop()
|
||||
nodes += 1
|
||||
if depth > MAX_METADATA_DEPTH or nodes > MAX_METADATA_NODES:
|
||||
return "input_too_large"
|
||||
if isinstance(value, dict):
|
||||
if len(value) + nodes + len(stack) > MAX_METADATA_NODES:
|
||||
return "input_too_large"
|
||||
for key, child in value.items():
|
||||
if (
|
||||
not isinstance(key, str)
|
||||
or len(key) > MAX_METADATA_BYTES
|
||||
or _utf8_size(key) is None
|
||||
):
|
||||
return "invalid_input"
|
||||
stack.append((child, depth + 1))
|
||||
elif isinstance(value, list):
|
||||
if len(value) + nodes + len(stack) > MAX_METADATA_NODES:
|
||||
return "input_too_large"
|
||||
stack.extend((child, depth + 1) for child in value)
|
||||
elif isinstance(value, str):
|
||||
if len(value) > MAX_METADATA_BYTES:
|
||||
return "input_too_large"
|
||||
if _utf8_size(value) is None:
|
||||
return "invalid_input"
|
||||
elif value is not None and not isinstance(value, (bool, int, float)):
|
||||
return "invalid_input"
|
||||
try:
|
||||
encoded = json.dumps(
|
||||
metadata,
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
except (RecursionError, TypeError, UnicodeEncodeError, ValueError):
|
||||
return "invalid_input"
|
||||
return "input_too_large" if len(encoded) > MAX_METADATA_BYTES else None
|
||||
|
||||
|
||||
def _validate_payload(
|
||||
target: Any,
|
||||
content: Any,
|
||||
account_id: Any,
|
||||
message_id: Any,
|
||||
metadata: Any,
|
||||
dry_run: Any,
|
||||
) -> Optional[str]:
|
||||
if not isinstance(target, str) or not isinstance(content, str):
|
||||
return "invalid_input"
|
||||
if not target.strip() or not content.strip() or not isinstance(dry_run, bool):
|
||||
return "invalid_input"
|
||||
if any(ord(character) < 32 or ord(character) == 127 for character in target):
|
||||
return "invalid_input"
|
||||
if "\x00" in content or not _valid_identifier(account_id) or not _valid_identifier(message_id):
|
||||
return "invalid_input"
|
||||
if len(target) > MAX_TARGET_BYTES or len(content) > MAX_CONTENT_BYTES:
|
||||
return "input_too_large"
|
||||
target_size = _utf8_size(target)
|
||||
content_size = _utf8_size(content)
|
||||
if target_size is None or content_size is None:
|
||||
return "invalid_input"
|
||||
if target_size > MAX_TARGET_BYTES or content_size > MAX_CONTENT_BYTES:
|
||||
return "input_too_large"
|
||||
return _validate_metadata(metadata)
|
||||
|
||||
|
||||
def _safe_provider_message_id(value: Any) -> Optional[str]:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
if len(value) > MAX_IDENTIFIER_BYTES:
|
||||
return None
|
||||
size = _utf8_size(value)
|
||||
if size is None or size > MAX_IDENTIFIER_BYTES:
|
||||
return None
|
||||
if any(ord(character) < 32 or ord(character) == 127 for character in value):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def send_ucc_adapter(
|
||||
target: str,
|
||||
content: str,
|
||||
account_id: str = "",
|
||||
message_id: str = "",
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
dry_run: bool = False,
|
||||
) -> BackendResult:
|
||||
effective_account = account_id or WECHAT_ACCOUNT_ID
|
||||
effective_metadata: Any = {} if metadata is None else metadata
|
||||
error_code = _validate_payload(
|
||||
target,
|
||||
content,
|
||||
effective_account,
|
||||
message_id,
|
||||
effective_metadata,
|
||||
dry_run,
|
||||
)
|
||||
if error_code:
|
||||
return BackendResult(False, "ucc-adapter", "failed", {"code": error_code})
|
||||
|
||||
payload = {
|
||||
"target": target,
|
||||
"content": content,
|
||||
"account_id": effective_account,
|
||||
"message_id": message_id,
|
||||
"metadata": effective_metadata,
|
||||
"dry_run": dry_run,
|
||||
}
|
||||
last_endpoint: Optional[str] = None
|
||||
for path in _ucc_send_paths():
|
||||
last_endpoint = path
|
||||
try:
|
||||
http_status, data = _post_ucc(path, payload)
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in {404, 405}:
|
||||
continue
|
||||
return BackendResult(
|
||||
False,
|
||||
"ucc-adapter",
|
||||
"failed",
|
||||
{"code": "ucc_adapter_http_error", "http_status": int(exc.code), "endpoint": path},
|
||||
)
|
||||
except (OSError, ValueError, urllib.error.URLError, _BackendProtocolError):
|
||||
return BackendResult(
|
||||
False,
|
||||
"ucc-adapter",
|
||||
"failed",
|
||||
{"code": "ucc_adapter_unavailable", "endpoint": path},
|
||||
)
|
||||
except Exception:
|
||||
# Fail closed without returning exception text, type, or chained cause.
|
||||
return BackendResult(
|
||||
False,
|
||||
"ucc-adapter",
|
||||
"failed",
|
||||
{"code": "ucc_adapter_unavailable", "endpoint": path},
|
||||
)
|
||||
|
||||
if not (200 <= http_status < 300):
|
||||
return BackendResult(
|
||||
False,
|
||||
"ucc-adapter",
|
||||
"failed",
|
||||
{"code": "ucc_adapter_http_error", "http_status": http_status, "endpoint": path},
|
||||
)
|
||||
if not _ucc_result_is_ok(data, dry_run):
|
||||
return BackendResult(
|
||||
False,
|
||||
"ucc-adapter",
|
||||
"failed",
|
||||
{"code": "ucc_adapter_rejected", "http_status": http_status, "endpoint": path},
|
||||
)
|
||||
confirmed_status = "dry_run" if dry_run else "sent"
|
||||
result_data: Dict[str, Any] = {
|
||||
"code": confirmed_status,
|
||||
"http_status": http_status,
|
||||
"endpoint": path,
|
||||
}
|
||||
provider_message_id = _safe_provider_message_id(data.get("provider_message_id"))
|
||||
if provider_message_id:
|
||||
result_data["provider_message_id"] = provider_message_id
|
||||
return BackendResult(True, "ucc-adapter", confirmed_status, result_data)
|
||||
|
||||
data: Dict[str, Any] = {"code": "ucc_adapter_unavailable"}
|
||||
if last_endpoint:
|
||||
data["endpoint"] = last_endpoint
|
||||
return BackendResult(False, "ucc-adapter", "failed", data)
|
||||
|
||||
|
||||
def send_openclaw_channel(
|
||||
target: str,
|
||||
content: str,
|
||||
account_id: str = "",
|
||||
message_id: str = "",
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
dry_run: bool = False,
|
||||
) -> BackendResult:
|
||||
del target, content, account_id, message_id, metadata, dry_run
|
||||
return BackendResult(
|
||||
False,
|
||||
"openclaw-channel",
|
||||
"failed",
|
||||
{"code": "unsupported_backend"},
|
||||
)
|
||||
|
||||
|
||||
def send_message(
|
||||
target: str,
|
||||
content: str,
|
||||
account_id: str = "",
|
||||
message_id: str = "",
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
dry_run: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
selected = select_backend()
|
||||
if selected.get("backend") == "ucc-adapter":
|
||||
return send_ucc_adapter(
|
||||
target, content, account_id, message_id, metadata, dry_run
|
||||
).to_dict()
|
||||
code = "unsupported_backend" if selected.get("reason") == "unsupported_backend" else "wechat_outbound_backend_unavailable"
|
||||
return BackendResult(False, "unavailable", "failed", {"code": code}).to_dict()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(json.dumps(select_backend(), ensure_ascii=False, separators=(",", ":")))
|
||||
@@ -1 +0,0 @@
|
||||
"""AIOS services namespace."""
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Reliable, fail-closed Codex command watcher."""
|
||||
|
||||
from .watcher import CommandWatcher, Config, load_config
|
||||
|
||||
__all__ = ["CommandWatcher", "Config", "load_config"]
|
||||
@@ -1,19 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
from .watcher import CommandWatcher, load_config
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Claim and execute Codex work units")
|
||||
parser.add_argument("--config", required=True)
|
||||
parser.add_argument("--once", action="store_true", help="process one scan and exit")
|
||||
args = parser.parse_args()
|
||||
watcher = CommandWatcher(load_config(args.config))
|
||||
return watcher.run_once() if args.once else watcher.run_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,501 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from contextlib import contextmanager
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sqlite3
|
||||
import stat
|
||||
import subprocess
|
||||
import time
|
||||
import threading
|
||||
from typing import Any, Callable
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
|
||||
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
|
||||
TERMINAL = {"COMPLETED", "DEAD_LETTER", "REJECTED"}
|
||||
CODEX_AUTH_LOCK = Path("/run/lock/aios-codex-auth.lock")
|
||||
_CODEX_EXECUTION_LOCK = threading.Lock()
|
||||
|
||||
|
||||
class WatcherError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _codex_auth_lock():
|
||||
with _CODEX_EXECUTION_LOCK:
|
||||
fd = os.open(CODEX_AUTH_LOCK, os.O_RDWR | getattr(os, "O_NOFOLLOW", 0))
|
||||
try:
|
||||
info = os.fstat(fd)
|
||||
if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) & ~0o660:
|
||||
raise WatcherError("unsafe_codex_auth_lock")
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
os.close(fd)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
active_dir: Path
|
||||
completed_dir: Path
|
||||
dead_letter_dir: Path
|
||||
database_path: Path
|
||||
audit_log_path: Path
|
||||
gateway_endpoint: str
|
||||
gateway_token_file: Path
|
||||
contract_commit: str
|
||||
lease_seconds: int
|
||||
retry_seconds: int
|
||||
max_attempts: int
|
||||
poll_seconds: float
|
||||
executor_path: Path | None
|
||||
executor_args: tuple[str, ...]
|
||||
executor_timeout_seconds: int
|
||||
|
||||
|
||||
def _absolute(value: Any, key: str) -> Path:
|
||||
if not isinstance(value, str) or not value or not Path(value).is_absolute():
|
||||
raise WatcherError(f"invalid_{key}")
|
||||
return Path(value)
|
||||
|
||||
|
||||
def _secure_regular(path: Path, *, writable_mask: int = 0o022) -> os.stat_result:
|
||||
info = os.lstat(path)
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode):
|
||||
raise WatcherError(f"unsafe_file:{path}")
|
||||
if stat.S_IMODE(info.st_mode) & writable_mask:
|
||||
raise WatcherError(f"unsafe_mode:{path}")
|
||||
return info
|
||||
|
||||
|
||||
def _read_secure_text(
|
||||
path: Path,
|
||||
*,
|
||||
writable_mask: int = 0o022,
|
||||
max_bytes: int,
|
||||
allowed_owners: set[int] | None = None,
|
||||
) -> str:
|
||||
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
||||
fd = os.open(path, flags)
|
||||
try:
|
||||
info = os.fstat(fd)
|
||||
if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) & writable_mask:
|
||||
raise WatcherError(f"unsafe_file:{path}")
|
||||
if allowed_owners is not None and info.st_uid not in allowed_owners:
|
||||
raise WatcherError(f"owner_mismatch:{path}")
|
||||
if info.st_size > max_bytes:
|
||||
raise WatcherError(f"file_too_large:{path}")
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
while True:
|
||||
chunk = os.read(fd, min(65_536, max_bytes + 1 - total))
|
||||
if not chunk:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
total += len(chunk)
|
||||
if total > max_bytes:
|
||||
raise WatcherError(f"file_too_large:{path}")
|
||||
return b"".join(chunks).decode("utf-8")
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def load_config(path_value: str) -> Config:
|
||||
path = _absolute(path_value, "config_path")
|
||||
raw = json.loads(_read_secure_text(path, max_bytes=131_072, allowed_owners={0, os.geteuid()}))
|
||||
expected = {
|
||||
"active_dir", "completed_dir", "dead_letter_dir", "database_path",
|
||||
"audit_log_path", "gateway_endpoint", "gateway_token_file",
|
||||
"contract_commit", "lease_seconds", "retry_seconds", "max_attempts",
|
||||
"poll_seconds", "executor_path", "executor_args",
|
||||
"executor_timeout_seconds",
|
||||
}
|
||||
if not isinstance(raw, dict) or set(raw) != expected:
|
||||
raise WatcherError("invalid_config_fields")
|
||||
contract = raw["contract_commit"]
|
||||
if not isinstance(contract, str) or not OID_RE.fullmatch(contract):
|
||||
raise WatcherError("invalid_contract_commit")
|
||||
endpoint = raw["gateway_endpoint"]
|
||||
if not isinstance(endpoint, str) or not endpoint.startswith("http://127.0.0.1:") or not endpoint.endswith("/v1/orchestrator/events"):
|
||||
raise WatcherError("gateway_endpoint_must_be_loopback")
|
||||
executor_raw = raw["executor_path"]
|
||||
executor = None if executor_raw is None else _absolute(executor_raw, "executor_path")
|
||||
executor_args = raw["executor_args"]
|
||||
if not isinstance(executor_args, list) or any(not isinstance(x, str) or not x for x in executor_args):
|
||||
raise WatcherError("invalid_executor_args")
|
||||
ints = ("lease_seconds", "retry_seconds", "max_attempts", "executor_timeout_seconds")
|
||||
if any(not isinstance(raw[x], int) or raw[x] <= 0 for x in ints):
|
||||
raise WatcherError("invalid_positive_integer")
|
||||
if not isinstance(raw["poll_seconds"], (int, float)) or raw["poll_seconds"] <= 0:
|
||||
raise WatcherError("invalid_poll_seconds")
|
||||
return Config(
|
||||
active_dir=_absolute(raw["active_dir"], "active_dir"),
|
||||
completed_dir=_absolute(raw["completed_dir"], "completed_dir"),
|
||||
dead_letter_dir=_absolute(raw["dead_letter_dir"], "dead_letter_dir"),
|
||||
database_path=_absolute(raw["database_path"], "database_path"),
|
||||
audit_log_path=_absolute(raw["audit_log_path"], "audit_log_path"),
|
||||
gateway_endpoint=endpoint,
|
||||
gateway_token_file=_absolute(raw["gateway_token_file"], "gateway_token_file"),
|
||||
contract_commit=contract,
|
||||
lease_seconds=raw["lease_seconds"], retry_seconds=raw["retry_seconds"],
|
||||
max_attempts=raw["max_attempts"], poll_seconds=float(raw["poll_seconds"]),
|
||||
executor_path=executor, executor_args=tuple(executor_args),
|
||||
executor_timeout_seconds=raw["executor_timeout_seconds"],
|
||||
)
|
||||
|
||||
|
||||
def canonical_digest(value: dict[str, Any]) -> str:
|
||||
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def validate_command(value: Any, expected_contract: str) -> dict[str, Any]:
|
||||
expected = {
|
||||
"schema_version", "command_id", "project_key", "work_unit_id",
|
||||
"contract_commit", "state_version", "execution",
|
||||
}
|
||||
if not isinstance(value, dict) or set(value) != expected or value.get("schema_version") != 1:
|
||||
raise WatcherError("invalid_command_fields")
|
||||
for key in ("command_id", "project_key", "work_unit_id"):
|
||||
if not isinstance(value[key], str) or not ID_RE.fullmatch(value[key]):
|
||||
raise WatcherError(f"invalid_{key}")
|
||||
if value["contract_commit"] != expected_contract:
|
||||
raise WatcherError("contract_commit_mismatch")
|
||||
if not isinstance(value["state_version"], int) or value["state_version"] < 0:
|
||||
raise WatcherError("invalid_state_version")
|
||||
execution = value["execution"]
|
||||
if not isinstance(execution, dict) or set(execution) != {"kind", "input"}:
|
||||
raise WatcherError("invalid_execution")
|
||||
if execution["kind"] not in {"acceptance_probe", "codex"}:
|
||||
raise WatcherError("invalid_execution_kind")
|
||||
if not isinstance(execution["input"], str) or len(execution["input"].encode()) > 65_536:
|
||||
raise WatcherError("invalid_execution_input")
|
||||
return value
|
||||
|
||||
|
||||
class CommandWatcher:
|
||||
def __init__(
|
||||
self,
|
||||
config: Config,
|
||||
*,
|
||||
clock: Callable[[], float] = time.time,
|
||||
event_poster: Callable[[dict[str, Any]], None] | None = None,
|
||||
executor: Callable[[dict[str, Any]], dict[str, Any]] | None = None,
|
||||
worker_id: str | None = None,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.clock = clock
|
||||
self.worker_id = worker_id or f"{os.uname().nodename}:{os.getpid()}:{uuid.uuid4().hex[:8]}"
|
||||
self._uses_gateway = event_poster is None
|
||||
self.event_poster = event_poster or self._post_event
|
||||
self.executor = executor or self._execute
|
||||
self._prepare()
|
||||
|
||||
def _prepare(self) -> None:
|
||||
for directory in (self.config.active_dir, self.config.completed_dir, self.config.dead_letter_dir):
|
||||
info = os.lstat(directory)
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISDIR(info.st_mode) or stat.S_IMODE(info.st_mode) & 0o002:
|
||||
raise WatcherError(f"unsafe_directory:{directory}")
|
||||
self.config.database_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self.config.audit_log_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
with self._db() as db:
|
||||
db.execute("PRAGMA journal_mode=WAL")
|
||||
db.execute("""
|
||||
CREATE TABLE IF NOT EXISTS commands (
|
||||
command_id TEXT PRIMARY KEY, work_unit_id TEXT NOT NULL UNIQUE,
|
||||
payload_sha256 TEXT NOT NULL,
|
||||
source_path TEXT NOT NULL, status TEXT NOT NULL,
|
||||
phase TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0,
|
||||
lease_owner TEXT, lease_until REAL, next_attempt REAL NOT NULL DEFAULT 0,
|
||||
result_json TEXT, start_event_json TEXT, end_event_json TEXT,
|
||||
last_error TEXT, updated_at REAL NOT NULL
|
||||
)
|
||||
""")
|
||||
|
||||
def _db(self) -> sqlite3.Connection:
|
||||
db = sqlite3.connect(self.config.database_path, timeout=5, isolation_level=None)
|
||||
db.row_factory = sqlite3.Row
|
||||
return db
|
||||
|
||||
def _audit(self, marker: str, command_id: str, **fields: Any) -> None:
|
||||
safe = {k: v for k, v in fields.items() if k not in {"input", "token", "secret"}}
|
||||
record = {"time": int(self.clock()), "marker": marker, "command_id": command_id, **safe}
|
||||
fd = os.open(self.config.audit_log_path, os.O_APPEND | os.O_CREAT | os.O_WRONLY | os.O_NOFOLLOW, 0o600)
|
||||
try:
|
||||
os.write(fd, (json.dumps(record, sort_keys=True, separators=(",", ":")) + "\n").encode())
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
def _read_command(self, path: Path) -> tuple[dict[str, Any], str]:
|
||||
raw = _read_secure_text(path, max_bytes=131_072, allowed_owners={0, os.geteuid()})
|
||||
value = validate_command(json.loads(raw), self.config.contract_commit)
|
||||
return value, canonical_digest(value)
|
||||
|
||||
def _register(self, command: dict[str, Any], digest: str, path: Path) -> str:
|
||||
now = self.clock()
|
||||
with self._db() as db:
|
||||
db.execute("BEGIN IMMEDIATE")
|
||||
row = db.execute("SELECT payload_sha256,status FROM commands WHERE command_id=?", (command["command_id"],)).fetchone()
|
||||
if row:
|
||||
db.execute("COMMIT")
|
||||
if row["payload_sha256"] != digest:
|
||||
raise WatcherError("duplicate_command_payload_conflict")
|
||||
return "duplicate_terminal" if row["status"] in TERMINAL else "existing"
|
||||
prior_work_unit = db.execute(
|
||||
"SELECT command_id FROM commands WHERE work_unit_id=?",
|
||||
(command["work_unit_id"],),
|
||||
).fetchone()
|
||||
if prior_work_unit:
|
||||
db.execute("COMMIT")
|
||||
raise WatcherError("duplicate_work_unit")
|
||||
db.execute(
|
||||
"INSERT INTO commands(command_id,work_unit_id,payload_sha256,source_path,status,phase,updated_at) VALUES(?,?,?,?,?,?,?)",
|
||||
(command["command_id"], command["work_unit_id"], digest, str(path), "PENDING", "START", now),
|
||||
)
|
||||
db.execute("COMMIT")
|
||||
return "new"
|
||||
|
||||
def _claim(self, command_id: str) -> sqlite3.Row | None:
|
||||
now = self.clock()
|
||||
with self._db() as db:
|
||||
db.execute("BEGIN IMMEDIATE")
|
||||
row = db.execute("SELECT * FROM commands WHERE command_id=?", (command_id,)).fetchone()
|
||||
if not row or row["status"] in TERMINAL or row["next_attempt"] > now:
|
||||
db.execute("COMMIT")
|
||||
return None
|
||||
if row["lease_until"] is not None and row["lease_until"] > now and row["lease_owner"] != self.worker_id:
|
||||
db.execute("COMMIT")
|
||||
return None
|
||||
attempts = row["attempts"] + 1
|
||||
db.execute(
|
||||
"UPDATE commands SET status='CLAIMED',attempts=?,lease_owner=?,lease_until=?,updated_at=? WHERE command_id=?",
|
||||
(attempts, self.worker_id, now + self.config.lease_seconds, now, command_id),
|
||||
)
|
||||
row = db.execute("SELECT * FROM commands WHERE command_id=?", (command_id,)).fetchone()
|
||||
db.execute("COMMIT")
|
||||
assert row is not None
|
||||
self._audit("CODEX_AUTO_CLAIM", command_id, attempt=row["attempts"], worker=self.worker_id)
|
||||
return row
|
||||
|
||||
def _update(self, command_id: str, *, status: str, phase: str, result: dict[str, Any] | None = None, error: str | None = None) -> None:
|
||||
with self._db() as db:
|
||||
release = status in TERMINAL
|
||||
db.execute(
|
||||
"UPDATE commands SET status=?,phase=?,result_json=COALESCE(?,result_json),last_error=?,lease_owner=CASE WHEN ? THEN NULL ELSE lease_owner END,lease_until=CASE WHEN ? THEN NULL ELSE lease_until END,updated_at=? WHERE command_id=?",
|
||||
(status, phase, json.dumps(result, sort_keys=True) if result is not None else None, error, release, release, self.clock(), command_id),
|
||||
)
|
||||
|
||||
def _retry_or_dead(self, command_id: str, path: Path, error: Exception) -> None:
|
||||
message = f"{type(error).__name__}:{str(error)[:240]}"
|
||||
with self._db() as db:
|
||||
row = db.execute("SELECT attempts,phase FROM commands WHERE command_id=?", (command_id,)).fetchone()
|
||||
attempts = row["attempts"] if row else self.config.max_attempts
|
||||
if str(error) == "gateway_stale_state_version" and row:
|
||||
column = "start_event_json" if row["phase"] == "START" else "end_event_json"
|
||||
db.execute(f"UPDATE commands SET {column}=NULL WHERE command_id=?", (command_id,))
|
||||
if attempts >= self.config.max_attempts:
|
||||
db.execute("UPDATE commands SET status='DEAD_LETTER',last_error=?,lease_owner=NULL,lease_until=NULL,updated_at=? WHERE command_id=?", (message, self.clock(), command_id))
|
||||
self._move(path, self.config.dead_letter_dir)
|
||||
self._audit("DEAD_LETTER", command_id, error=message)
|
||||
else:
|
||||
db.execute("UPDATE commands SET status='RETRY',last_error=?,next_attempt=?,lease_owner=NULL,lease_until=NULL,updated_at=? WHERE command_id=?", (message, self.clock() + self.config.retry_seconds, self.clock(), command_id))
|
||||
self._audit("RETRY_SCHEDULED", command_id, attempt=attempts, error=message)
|
||||
|
||||
def _event(self, command: dict[str, Any], event_type: str, status: str, suffix: str, result: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": 1, "project_key": command["project_key"],
|
||||
"contract_commit": command["contract_commit"],
|
||||
"event_id": f"{command['command_id']}:{suffix}", "source_agent": "codex",
|
||||
"event_type": event_type, "work_unit_id": command["work_unit_id"],
|
||||
"status": status, "state_version": command["state_version"],
|
||||
"implementation_commit": None, "result_commit": None, "blocker": None,
|
||||
"evidence_refs": [], "payload": {"command_id": command["command_id"], "result": result or {}},
|
||||
}
|
||||
|
||||
def _gateway_token(self) -> str:
|
||||
token = _read_secure_text(
|
||||
self.config.gateway_token_file,
|
||||
writable_mask=0o027,
|
||||
max_bytes=16_384,
|
||||
allowed_owners={0, os.geteuid()},
|
||||
)
|
||||
if not token or "\n" in token or "\r" in token:
|
||||
raise WatcherError("invalid_gateway_token")
|
||||
return token
|
||||
|
||||
def _gateway_state_version(self, project_key: str, fallback: int) -> int:
|
||||
state_url = self.config.gateway_endpoint.removesuffix("/events") + "/state/" + urllib.parse.quote(project_key, safe="")
|
||||
request = urllib.request.Request(
|
||||
state_url,
|
||||
headers={"Authorization": f"Bearer {self._gateway_token()}"},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
payload = json.loads(response.read(262_145))
|
||||
version = payload.get("state_version")
|
||||
if not isinstance(version, int) or version < 0:
|
||||
raise WatcherError("invalid_gateway_state")
|
||||
return version
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
return fallback
|
||||
raise WatcherError(f"gateway_state_http_{exc.code}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise WatcherError("gateway_state_unavailable") from exc
|
||||
|
||||
def _durable_event(self, command: dict[str, Any], column: str, event: dict[str, Any]) -> dict[str, Any]:
|
||||
with self._db() as db:
|
||||
row = db.execute(f"SELECT {column} FROM commands WHERE command_id=?", (command["command_id"],)).fetchone()
|
||||
if row and row[column]:
|
||||
return json.loads(row[column])
|
||||
if self._uses_gateway:
|
||||
event["state_version"] = self._gateway_state_version(command["project_key"], command["state_version"])
|
||||
encoded = json.dumps(event, sort_keys=True, separators=(",", ":"))
|
||||
with self._db() as db:
|
||||
db.execute(f"UPDATE commands SET {column}=? WHERE command_id=? AND {column} IS NULL", (encoded, command["command_id"]))
|
||||
saved = db.execute(f"SELECT {column} FROM commands WHERE command_id=?", (command["command_id"],)).fetchone()
|
||||
if not saved or not saved[column]:
|
||||
raise WatcherError("event_persistence_failed")
|
||||
return json.loads(saved[column])
|
||||
|
||||
def _post_event(self, event: dict[str, Any]) -> None:
|
||||
request = urllib.request.Request(
|
||||
self.config.gateway_endpoint,
|
||||
data=json.dumps(event, separators=(",", ":")).encode(),
|
||||
headers={"Authorization": f"Bearer {self._gateway_token()}", "Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
if response.status not in {200, 201, 202}:
|
||||
raise WatcherError(f"gateway_http_{response.status}")
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read(16_384)
|
||||
try:
|
||||
reason = json.loads(body).get("error")
|
||||
except (json.JSONDecodeError, UnicodeDecodeError, AttributeError):
|
||||
reason = None
|
||||
if exc.code == 409 and reason == "stale_state_version":
|
||||
raise WatcherError("gateway_stale_state_version") from exc
|
||||
raise WatcherError(f"gateway_http_{exc.code}") from exc
|
||||
except urllib.error.URLError as exc:
|
||||
raise WatcherError("gateway_unavailable") from exc
|
||||
|
||||
def _execute(self, command: dict[str, Any]) -> dict[str, Any]:
|
||||
execution = command["execution"]
|
||||
if execution["kind"] == "acceptance_probe":
|
||||
return {"kind": "acceptance_probe", "input_sha256": hashlib.sha256(execution["input"].encode()).hexdigest(), "exit_code": 0}
|
||||
assert self.config.executor_path is not None
|
||||
with _codex_auth_lock():
|
||||
proc = subprocess.run(
|
||||
[str(self.config.executor_path), *self.config.executor_args],
|
||||
input=execution["input"], text=True, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, timeout=self.config.executor_timeout_seconds,
|
||||
check=False, env={"PATH": "/usr/bin:/bin", "LANG": "C.UTF-8"},
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise WatcherError(f"codex_executor_failed:{proc.returncode}")
|
||||
return {"kind": "codex", "exit_code": 0}
|
||||
|
||||
def _preflight_execution(self, command: dict[str, Any]) -> None:
|
||||
if command["execution"]["kind"] == "acceptance_probe":
|
||||
return
|
||||
if self.config.executor_path is None:
|
||||
raise WatcherError("codex_executor_unavailable")
|
||||
info = _secure_regular(self.config.executor_path)
|
||||
if not info.st_mode & stat.S_IXUSR:
|
||||
raise WatcherError("codex_executor_not_executable")
|
||||
|
||||
def _move(self, source: Path, target_dir: Path) -> None:
|
||||
if not source.exists() and not source.is_symlink():
|
||||
return
|
||||
target = target_dir / source.name
|
||||
if target.exists() or target.is_symlink():
|
||||
target = target_dir / f"{source.stem}-{uuid.uuid4().hex[:8]}{source.suffix}"
|
||||
os.replace(source, target)
|
||||
|
||||
def _reject_unregistered(self, path: Path, error: Exception) -> None:
|
||||
command_id = path.stem[:128] if ID_RE.fullmatch(path.stem[:128]) else f"invalid-{uuid.uuid4().hex[:12]}"
|
||||
self._audit("COMMAND_REJECTED", command_id, error=f"{type(error).__name__}:{str(error)[:240]}")
|
||||
self._move(path, self.config.dead_letter_dir)
|
||||
|
||||
def process_path(self, path: Path) -> bool:
|
||||
try:
|
||||
command, digest = self._read_command(path)
|
||||
registration = self._register(command, digest, path)
|
||||
if registration == "duplicate_terminal":
|
||||
self._move(path, self.config.completed_dir)
|
||||
return False
|
||||
row = self._claim(command["command_id"])
|
||||
if row is None:
|
||||
return False
|
||||
phase = row["phase"]
|
||||
result = json.loads(row["result_json"]) if row["result_json"] else None
|
||||
if phase == "START":
|
||||
start_event = self._durable_event(
|
||||
command,
|
||||
"start_event_json",
|
||||
self._event(command, "work_unit_started", "RUNNING", "started"),
|
||||
)
|
||||
self.event_poster(start_event)
|
||||
self._audit("TASK_STARTED", command["command_id"])
|
||||
self._update(command["command_id"], status="PENDING", phase="EXECUTE")
|
||||
phase = "EXECUTE"
|
||||
if phase == "EXECUTE":
|
||||
self._preflight_execution(command)
|
||||
self._update(command["command_id"], status="CLAIMED", phase="EXECUTING")
|
||||
phase = "EXECUTING"
|
||||
result = self.executor(command)
|
||||
self._audit("TASK_EXECUTED", command["command_id"], result=result)
|
||||
self._update(command["command_id"], status="PENDING", phase="END", result=result)
|
||||
phase = "END"
|
||||
elif phase == "EXECUTING":
|
||||
raise WatcherError("ambiguous_execution_not_repeated")
|
||||
if phase == "END":
|
||||
assert result is not None
|
||||
end_event = self._durable_event(
|
||||
command,
|
||||
"end_event_json",
|
||||
self._event(command, "work_unit_ended", "PASS", "ended", result),
|
||||
)
|
||||
self.event_poster(end_event)
|
||||
self._audit("TASK_ENDED", command["command_id"])
|
||||
self._audit("EVENT_POSTED", command["command_id"], event="work_unit_ended")
|
||||
self._update(command["command_id"], status="COMPLETED", phase="DONE", result=result)
|
||||
self._move(path, self.config.completed_dir)
|
||||
return True
|
||||
except Exception as exc:
|
||||
try:
|
||||
if isinstance(exc, WatcherError) and str(exc) in {"duplicate_command_payload_conflict", "duplicate_work_unit"}:
|
||||
self._reject_unregistered(path, exc)
|
||||
elif "command" in locals():
|
||||
self._retry_or_dead(command["command_id"], path, exc)
|
||||
else:
|
||||
self._reject_unregistered(path, exc)
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
def run_once(self) -> int:
|
||||
failures = 0
|
||||
for path in sorted(self.config.active_dir.glob("*.json")):
|
||||
if not self.process_path(path):
|
||||
failures += 1
|
||||
return 1 if failures else 0
|
||||
|
||||
def run_forever(self) -> int:
|
||||
while True:
|
||||
self.run_once()
|
||||
time.sleep(self.config.poll_seconds)
|
||||
@@ -1,3 +0,0 @@
|
||||
"""Reliable central orchestration gateway for AIOS."""
|
||||
|
||||
__version__ = "0.2.0"
|
||||
@@ -1,34 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from .config import Config
|
||||
from .service import Runtime, serve
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="AIOS Orchestrator Gateway")
|
||||
parser.add_argument("--config", type=Path, required=True)
|
||||
parser.add_argument("--log-level", default="INFO")
|
||||
parser.add_argument(
|
||||
"--poll-once",
|
||||
action="store_true",
|
||||
help="Run one inbox/model/outbox/OpenClaw synchronization polling cycle.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, args.log_level.upper(), logging.INFO),
|
||||
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||
)
|
||||
if args.poll_once:
|
||||
runtime = Runtime(Config.from_file(args.config))
|
||||
runtime.poll_once()
|
||||
return 0
|
||||
serve(args.config)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,132 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
from typing import Any, Callable
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from .models import Decision
|
||||
|
||||
|
||||
GATEWAY_DECISION_URL = "http://127.0.0.1:18920/v1/orchestrator/decisions"
|
||||
SECRET_REF_RE = re.compile(r"^secret://[A-Za-z0-9._/-]{1,480}$")
|
||||
|
||||
|
||||
class CallbackBridgeError(RuntimeError):
|
||||
def __init__(self, category: str, status_code: int) -> None:
|
||||
super().__init__(category)
|
||||
self.category = category
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
def _trusted_resolver(path: Path) -> Path:
|
||||
if not path.is_absolute():
|
||||
raise CallbackBridgeError("CALLBACK_CONFIGURATION_UNAVAILABLE", 503)
|
||||
try:
|
||||
info = path.lstat()
|
||||
resolved = path.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise CallbackBridgeError("CALLBACK_CONFIGURATION_UNAVAILABLE", 503) from exc
|
||||
if (
|
||||
resolved != path
|
||||
or stat.S_ISLNK(info.st_mode)
|
||||
or not stat.S_ISREG(info.st_mode)
|
||||
or info.st_uid != 0
|
||||
or stat.S_IMODE(info.st_mode) & 0o022
|
||||
or not stat.S_IMODE(info.st_mode) & 0o111
|
||||
):
|
||||
raise CallbackBridgeError("CALLBACK_CONFIGURATION_UNAVAILABLE", 503)
|
||||
return resolved
|
||||
|
||||
|
||||
def resolve_secret_ref(secret_ref: str, resolver: Path) -> str:
|
||||
if not isinstance(secret_ref, str) or not SECRET_REF_RE.fullmatch(secret_ref):
|
||||
raise CallbackBridgeError("CALLBACK_CONFIGURATION_UNAVAILABLE", 503)
|
||||
executable = _trusted_resolver(resolver)
|
||||
request = json.dumps(
|
||||
{"operation": "resolve", "secret_ref": secret_ref},
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
).encode()
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[str(executable)],
|
||||
input=request,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"},
|
||||
timeout=5,
|
||||
check=False,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
raise CallbackBridgeError("CALLBACK_CREDENTIAL_UNAVAILABLE", 503) from exc
|
||||
token = result.stdout.removesuffix(b"\n")
|
||||
if (
|
||||
result.returncode != 0
|
||||
or not token
|
||||
or len(token) > 16_384
|
||||
or b"\n" in token
|
||||
or b"\r" in token
|
||||
):
|
||||
raise CallbackBridgeError("CALLBACK_CREDENTIAL_UNAVAILABLE", 503)
|
||||
try:
|
||||
return token.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise CallbackBridgeError("CALLBACK_CREDENTIAL_UNAVAILABLE", 503) from exc
|
||||
|
||||
|
||||
def submit_orchestration_decision(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
contract_commit: str,
|
||||
secret_ref: str,
|
||||
resolver: Path,
|
||||
request_open: Callable[..., Any] = urllib.request.urlopen,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
decision = Decision.from_dict(payload)
|
||||
except Exception as exc:
|
||||
raise CallbackBridgeError("INVALID_DECISION", 422) from exc
|
||||
if decision.contract_commit != contract_commit:
|
||||
raise CallbackBridgeError("CONTRACT_MISMATCH", 409)
|
||||
token = resolve_secret_ref(secret_ref, resolver)
|
||||
request = urllib.request.Request(
|
||||
GATEWAY_DECISION_URL,
|
||||
data=json.dumps(
|
||||
decision.to_dict(), separators=(",", ":"), sort_keys=True
|
||||
).encode(),
|
||||
method="POST",
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
"Idempotency-Key": decision.event_id,
|
||||
"User-Agent": "aios-chatgpt-actions/decision-bridge-v1",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with request_open(request, timeout=10) as response:
|
||||
raw = response.read(65_537)
|
||||
if response.status != 202 or len(raw) > 65_536:
|
||||
raise CallbackBridgeError("CALLBACK_HTTP_ERROR", 502)
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 409:
|
||||
raise CallbackBridgeError("STALE_STATE", 409) from exc
|
||||
raise CallbackBridgeError("CALLBACK_HTTP_ERROR", 502) from exc
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise CallbackBridgeError("CALLBACK_NETWORK_ERROR", 502) from exc
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise CallbackBridgeError("CALLBACK_RESPONSE_INVALID", 502) from exc
|
||||
if (
|
||||
not isinstance(value, dict)
|
||||
or value.get("accepted") is not True
|
||||
or not isinstance(value.get("state_version"), int)
|
||||
):
|
||||
raise CallbackBridgeError("CALLBACK_RESPONSE_INVALID", 502)
|
||||
return {"accepted": True, "state_version": value["state_version"]}
|
||||
@@ -1,117 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import ipaddress
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from .errors import ConfigurationError
|
||||
from .models import DECISION_SCHEMA
|
||||
|
||||
|
||||
TRANSPORT_MODES = {"remote_https_action", "secure_mcp_tunnel"}
|
||||
FORBIDDEN_HOSTS = {"localhost", "localhost.localdomain"}
|
||||
FORBIDDEN_MARKERS = {"replace_with_actions_host", "placeholder"}
|
||||
|
||||
|
||||
def _https_endpoint(value: Any, *, public: bool) -> str:
|
||||
if not isinstance(value, str) or not value or any(
|
||||
marker in value.lower() for marker in FORBIDDEN_MARKERS
|
||||
):
|
||||
raise ConfigurationError("invalid_callback_endpoint")
|
||||
parsed = urlparse(value)
|
||||
host = (parsed.hostname or "").lower()
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or not host
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
or host in FORBIDDEN_HOSTS
|
||||
):
|
||||
raise ConfigurationError("invalid_callback_endpoint")
|
||||
try:
|
||||
address = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
address = None
|
||||
if address is not None and (
|
||||
address.is_loopback
|
||||
or address.is_private
|
||||
or address.is_unspecified
|
||||
or address.is_link_local
|
||||
or address.is_multicast
|
||||
or address.is_reserved
|
||||
):
|
||||
raise ConfigurationError("invalid_callback_endpoint")
|
||||
if public and (address is None and ("." not in host or host.endswith((".local", ".internal")))):
|
||||
raise ConfigurationError("callback_endpoint_not_public")
|
||||
return value.rstrip("/")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CallbackTransport:
|
||||
mode: str
|
||||
endpoint: str
|
||||
approved_mcp_tunnel_endpoints: tuple[str, ...]
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: dict[str, Any]) -> "CallbackTransport":
|
||||
expected = {"mode", "endpoint", "approved_mcp_tunnel_endpoints"}
|
||||
if not isinstance(value, dict) or set(value) != expected:
|
||||
raise ConfigurationError("invalid_callback_transport_fields")
|
||||
mode = value.get("mode")
|
||||
if mode not in TRANSPORT_MODES:
|
||||
raise ConfigurationError("invalid_callback_transport_mode")
|
||||
approved = value.get("approved_mcp_tunnel_endpoints")
|
||||
if not isinstance(approved, list) or any(not isinstance(x, str) for x in approved):
|
||||
raise ConfigurationError("invalid_approved_mcp_tunnel_endpoints")
|
||||
normalized_approved = tuple(_https_endpoint(item, public=False) for item in approved)
|
||||
endpoint = _https_endpoint(value.get("endpoint"), public=mode == "remote_https_action")
|
||||
if mode == "secure_mcp_tunnel" and endpoint not in normalized_approved:
|
||||
raise ConfigurationError("unapproved_secure_mcp_tunnel_endpoint")
|
||||
return cls(mode, endpoint, normalized_approved)
|
||||
|
||||
def action_openapi(self) -> dict[str, Any]:
|
||||
if self.mode != "remote_https_action":
|
||||
raise ConfigurationError("action_manifest_for_wrong_transport")
|
||||
return {
|
||||
"openapi": "3.1.0",
|
||||
"info": {"title": "AIOS Orchestrator Decision Callback", "version": "1.1.0"},
|
||||
"servers": [{"url": self.endpoint}],
|
||||
"paths": {
|
||||
"/v1/orchestrator/decisions": {
|
||||
"post": {
|
||||
"operationId": "submit_orchestration_decision",
|
||||
"summary": "Submit one strict orchestration decision",
|
||||
"security": [{"bearerAuth": []}],
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {"application/json": {"schema": DECISION_SCHEMA}},
|
||||
},
|
||||
"responses": {"202": {"description": "Accepted"}},
|
||||
}
|
||||
}
|
||||
},
|
||||
"components": {
|
||||
"securitySchemes": {
|
||||
"bearerAuth": {"type": "http", "scheme": "bearer"}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def mcp_tool(self) -> dict[str, Any]:
|
||||
if self.mode != "secure_mcp_tunnel":
|
||||
raise ConfigurationError("mcp_manifest_for_wrong_transport")
|
||||
return {
|
||||
"name": "submit_orchestration_decision",
|
||||
"description": "Validate and submit one AIOS orchestration decision.",
|
||||
"inputSchema": DECISION_SCHEMA,
|
||||
"annotations": {
|
||||
"title": "Submit orchestration decision",
|
||||
"readOnlyHint": False,
|
||||
"destructiveHint": False,
|
||||
"idempotentHint": True,
|
||||
"openWorldHint": False,
|
||||
},
|
||||
}
|
||||
@@ -1,204 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .errors import TriggerError
|
||||
from .models import DECISION_SCHEMA, Decision, Event
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkspaceTriggerResult:
|
||||
http_status: int
|
||||
remote_id: None = None
|
||||
|
||||
|
||||
WORKSPACE_HTTP_CATEGORIES = {
|
||||
401: "UNAUTHORIZED",
|
||||
403: "FORBIDDEN",
|
||||
404: "NOT_FOUND",
|
||||
409: "NOT_RUNNABLE",
|
||||
}
|
||||
|
||||
|
||||
class WorkspaceAgentClient:
|
||||
"""Asynchronous Workspace Agent trigger.
|
||||
|
||||
The official endpoint returns only 202 and no run ID or retrievable output.
|
||||
Completion therefore arrives through the gateway callback endpoint.
|
||||
"""
|
||||
|
||||
def __init__(self, trigger_id: str, token: str, timeout_seconds: int = 30):
|
||||
self.trigger_id = trigger_id
|
||||
self.token = token
|
||||
self.timeout_seconds = timeout_seconds
|
||||
|
||||
def trigger(self, event: Event) -> WorkspaceTriggerResult:
|
||||
url = f"https://api.chatgpt.com/v1/workspace_agents/{self.trigger_id}/trigger"
|
||||
prompt = (
|
||||
"Process this AIOS orchestration event under the committed contract. "
|
||||
"When complete, call submit_orchestration_decision with strict JSON. "
|
||||
"Do not fabricate tests, Git state, secrets, evidence, or deployment facts.\n\n"
|
||||
+ event.canonical_json()
|
||||
)
|
||||
body = json.dumps(
|
||||
{
|
||||
"conversation_key": event.project_key,
|
||||
"input": prompt,
|
||||
}
|
||||
).encode()
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=body,
|
||||
method="POST",
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.token}",
|
||||
"Content-Type": "application/json",
|
||||
"Idempotency-Key": event.event_id,
|
||||
"User-Agent": "aios-orchestrator-gateway/0.2",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=self.timeout_seconds) as response:
|
||||
if response.status != 202:
|
||||
raise TriggerError(
|
||||
"workspace_agent_unexpected_http_status",
|
||||
http_status=response.status,
|
||||
)
|
||||
return WorkspaceTriggerResult(http_status=202)
|
||||
except urllib.error.HTTPError as exc:
|
||||
category = WORKSPACE_HTTP_CATEGORIES.get(exc.code)
|
||||
raise TriggerError(
|
||||
"workspace_agent_http_error",
|
||||
http_status=exc.code,
|
||||
category=category,
|
||||
) from exc
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise TriggerError(
|
||||
"workspace_agent_network_error",
|
||||
category="NETWORK_ERROR",
|
||||
) from exc
|
||||
|
||||
|
||||
class ResponsesClient:
|
||||
"""Background Responses API client with polling and strict output validation."""
|
||||
|
||||
def __init__(self, api_key: str, model: str, timeout_seconds: int = 60):
|
||||
self.api_key = api_key
|
||||
self.model = model
|
||||
self.timeout_seconds = timeout_seconds
|
||||
|
||||
def start(self, event: Event) -> str:
|
||||
body = {
|
||||
"model": self.model,
|
||||
"background": True,
|
||||
"instructions": (
|
||||
"You are the AIOS central orchestration brain. Apply the exact "
|
||||
"contract_commit in the event. Return only a strict decision. "
|
||||
"Never fabricate evidence, secrets, Git facts, tests, approvals, "
|
||||
"or deployment state. Keep external blockers fail-closed."
|
||||
),
|
||||
"input": event.canonical_json(),
|
||||
"text": {
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"name": "aios_orchestrator_decision",
|
||||
"strict": True,
|
||||
"schema": DECISION_SCHEMA,
|
||||
}
|
||||
},
|
||||
"metadata": {
|
||||
"event_id": event.event_id,
|
||||
"project_key": event.project_key,
|
||||
"contract_commit": event.contract_commit,
|
||||
},
|
||||
}
|
||||
payload = self._request(
|
||||
"POST",
|
||||
"https://api.openai.com/v1/responses",
|
||||
body,
|
||||
idempotency_key=event.event_id,
|
||||
)
|
||||
response_id = payload.get("id")
|
||||
if not isinstance(response_id, str) or not response_id.startswith("resp_"):
|
||||
raise TriggerError("responses_missing_id")
|
||||
return response_id
|
||||
|
||||
def poll(self, response_id: str) -> tuple[str, Decision | None]:
|
||||
payload = self._request(
|
||||
"GET",
|
||||
f"https://api.openai.com/v1/responses/{response_id}",
|
||||
None,
|
||||
)
|
||||
status = payload.get("status")
|
||||
if status in {"queued", "in_progress"}:
|
||||
return str(status), None
|
||||
if status == "completed":
|
||||
text = _extract_output_text(payload)
|
||||
try:
|
||||
return "completed", Decision.from_dict(json.loads(text))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise TriggerError("responses_invalid_json") from exc
|
||||
if status in {"failed", "cancelled", "incomplete"}:
|
||||
raise TriggerError(f"responses_{status}")
|
||||
raise TriggerError("responses_unknown_status")
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
body: dict[str, Any] | None,
|
||||
*,
|
||||
idempotency_key: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
encoded = json.dumps(body).encode() if body is not None else None
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "aios-orchestrator-gateway/0.2",
|
||||
}
|
||||
if idempotency_key:
|
||||
headers["Idempotency-Key"] = idempotency_key
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=encoded,
|
||||
method=method,
|
||||
headers=headers,
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=self.timeout_seconds) as response:
|
||||
raw = response.read(4_194_304)
|
||||
if response.status not in {200, 201}:
|
||||
raise TriggerError(f"responses_http_{response.status}")
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise TriggerError(f"responses_http_{exc.code}") from exc
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
raise TriggerError("responses_request_failed") from exc
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise TriggerError("responses_invalid_payload") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise TriggerError("responses_payload_not_object")
|
||||
return payload
|
||||
|
||||
|
||||
def _extract_output_text(payload: dict[str, Any]) -> str:
|
||||
output = payload.get("output")
|
||||
if not isinstance(output, list):
|
||||
raise TriggerError("responses_missing_output")
|
||||
for item in output:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
content = item.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for part in content:
|
||||
if isinstance(part, dict) and part.get("type") == "output_text":
|
||||
text = part.get("text")
|
||||
if isinstance(text, str):
|
||||
return text
|
||||
raise TriggerError("responses_missing_output_text")
|
||||
@@ -1,258 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pwd
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
from typing import Iterator
|
||||
|
||||
from .errors import ConfigurationError, TriggerError, ValidationError
|
||||
from .models import DECISION_SCHEMA, Decision, Event
|
||||
from .securefs import ensure_shared_directory, read_secure_json
|
||||
|
||||
|
||||
CODEX_BINARY = Path("/usr/local/bin/codex")
|
||||
CODEX_HOME = Path("/var/lib/aios-codex-brain/.codex")
|
||||
AUTH_JSON = CODEX_HOME / "auth.json"
|
||||
CONTROLLER_WORKDIR = Path("/data/aios-gitops/worktrees/controller")
|
||||
AUTH_LOCK = Path("/run/lock/aios-codex-auth.lock")
|
||||
REQUIRED_EXEC_FLAGS = (
|
||||
"--json", "--output-schema", "--output-last-message", "--sandbox",
|
||||
"--ask-for-approval", "--cd", "--ephemeral",
|
||||
)
|
||||
FORBIDDEN_INSTRUCTIONS = (
|
||||
"reset --hard", "force push", "push main", "auth.json", "read secret",
|
||||
"production deploy", "production deployment",
|
||||
)
|
||||
_PROCESS_LOCK = threading.Lock()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CodexLocalSettings:
|
||||
timeout_seconds: int = 300
|
||||
binary: Path = CODEX_BINARY
|
||||
codex_home: Path = CODEX_HOME
|
||||
auth_json: Path = AUTH_JSON
|
||||
controller_workdir: Path = CONTROLLER_WORKDIR
|
||||
lock_path: Path = AUTH_LOCK
|
||||
expected_auth_uid: int | None = None
|
||||
|
||||
|
||||
def _regular_nofollow(path: Path) -> os.stat_result:
|
||||
info = path.lstat()
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode):
|
||||
raise ConfigurationError("unsafe_codex_auth_file")
|
||||
if path.resolve(strict=True) != path:
|
||||
raise ConfigurationError("unsafe_codex_auth_path")
|
||||
return info
|
||||
|
||||
|
||||
def validate_auth_json(path: Path, expected_uid: int | None = None) -> None:
|
||||
if not path.is_absolute() or not path.exists():
|
||||
raise ConfigurationError("codex_authentication_required")
|
||||
info = _regular_nofollow(path)
|
||||
uid = pwd.getpwnam("aios-codex-brain").pw_uid if expected_uid is None else expected_uid
|
||||
if info.st_uid != uid:
|
||||
raise ConfigurationError("codex_auth_wrong_owner")
|
||||
if stat.S_IMODE(info.st_mode) != 0o600:
|
||||
raise ConfigurationError("codex_auth_wrong_mode")
|
||||
|
||||
|
||||
class CodexLocalClient:
|
||||
"""Serialized, fail-closed local Codex decision client.
|
||||
|
||||
Authentication values are never opened by this class. The Codex CLI is the
|
||||
sole consumer of its own cache after metadata validation.
|
||||
"""
|
||||
|
||||
def __init__(self, settings: CodexLocalSettings = CodexLocalSettings()):
|
||||
self.settings = settings
|
||||
|
||||
def _environment(self) -> dict[str, str]:
|
||||
return {
|
||||
"CODEX_HOME": str(self.settings.codex_home),
|
||||
"HOME": str(self.settings.codex_home.parent),
|
||||
"PATH": f"{self.settings.binary.parent}:/usr/bin:/bin",
|
||||
"LANG": "C.UTF-8",
|
||||
"LC_ALL": "C.UTF-8",
|
||||
}
|
||||
|
||||
def _validate_fixed_paths(self) -> None:
|
||||
s = self.settings
|
||||
for path in (s.binary, s.codex_home, s.auth_json, s.controller_workdir, s.lock_path):
|
||||
if not path.is_absolute():
|
||||
raise ConfigurationError("codex_path_not_absolute")
|
||||
if not s.binary.exists() or not os.access(s.binary, os.X_OK):
|
||||
raise ConfigurationError("codex_binary_unavailable")
|
||||
if s.binary.is_symlink() or not s.binary.is_file():
|
||||
raise ConfigurationError("unsafe_codex_binary")
|
||||
if not s.controller_workdir.is_dir() or s.controller_workdir.is_symlink():
|
||||
raise ConfigurationError("unsafe_controller_workdir")
|
||||
validate_auth_json(s.auth_json, s.expected_auth_uid)
|
||||
|
||||
@contextmanager
|
||||
def _auth_lock(self) -> Iterator[None]:
|
||||
with _PROCESS_LOCK:
|
||||
flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0)
|
||||
fd = os.open(self.settings.lock_path, flags, 0o600)
|
||||
try:
|
||||
info = os.fstat(fd)
|
||||
if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) & ~0o660:
|
||||
raise ConfigurationError("unsafe_codex_auth_lock")
|
||||
if info.st_uid != os.geteuid():
|
||||
raise ConfigurationError("unsafe_codex_auth_lock_owner")
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
os.close(fd)
|
||||
|
||||
def _run_quiet(self, argv: list[str], timeout: int) -> subprocess.CompletedProcess[str]:
|
||||
try:
|
||||
return subprocess.run(
|
||||
argv, cwd=self.settings.controller_workdir, env=self._environment(),
|
||||
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True, timeout=timeout, check=False,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
raise ConfigurationError("codex_cli_probe_failed") from exc
|
||||
|
||||
def _probe_cli_unlocked(self) -> None:
|
||||
result = self._run_quiet([str(self.settings.binary), "exec", "--help"], 30)
|
||||
if result.returncode != 0 or any(flag not in result.stdout for flag in REQUIRED_EXEC_FLAGS):
|
||||
raise ConfigurationError("blocked_codex_cli_version_incompatible")
|
||||
|
||||
def _login_status_unlocked(self) -> None:
|
||||
result = self._run_quiet([str(self.settings.binary), "login", "status"], 30)
|
||||
if result.returncode != 0:
|
||||
raise ConfigurationError("codex_authentication_required")
|
||||
|
||||
def health_check(self) -> None:
|
||||
self._validate_fixed_paths()
|
||||
with self._auth_lock():
|
||||
self._probe_cli_unlocked()
|
||||
self._login_status_unlocked()
|
||||
|
||||
def decide(self, event: Event) -> Decision:
|
||||
self._validate_fixed_paths()
|
||||
with self._auth_lock():
|
||||
self._probe_cli_unlocked()
|
||||
self._login_status_unlocked()
|
||||
value = self._exec_unlocked(event)
|
||||
return self._validate_decision(value, event)
|
||||
|
||||
def _exec_unlocked(self, event: Event) -> dict:
|
||||
prompt = (
|
||||
"读取已提交治理合同。读取下列本次事件JSON。只输出一个符合Decision Schema的决策。"
|
||||
"不得修改Git、运行部署、读取秘密或伪造证据。证据不足时输出BLOCKED决策。\n"
|
||||
+ event.canonical_json()
|
||||
)
|
||||
runtime_root = self.settings.codex_home.parent / "runtime"
|
||||
runtime_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
if runtime_root.is_symlink() or runtime_root.resolve(strict=True) != runtime_root:
|
||||
raise ConfigurationError("unsafe_codex_runtime_directory")
|
||||
os.chmod(runtime_root, 0o700)
|
||||
with tempfile.TemporaryDirectory(prefix="decision-", dir=runtime_root) as directory:
|
||||
root = Path(directory)
|
||||
schema_path = root / "decision.schema.json"
|
||||
output_path = root / "decision.json"
|
||||
schema_path.write_text(json.dumps(DECISION_SCHEMA, separators=(",", ":")), encoding="utf-8")
|
||||
schema_path.chmod(0o600)
|
||||
argv = [
|
||||
str(self.settings.binary), "exec", "--json",
|
||||
"--output-schema", str(schema_path),
|
||||
"--output-last-message", str(output_path),
|
||||
"--sandbox", "read-only", "--ask-for-approval", "never",
|
||||
"--cd", str(self.settings.controller_workdir), "--ephemeral", "-",
|
||||
]
|
||||
process = subprocess.Popen(
|
||||
argv, cwd=self.settings.controller_workdir, env=self._environment(),
|
||||
stdin=subprocess.PIPE, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, text=True, start_new_session=True,
|
||||
)
|
||||
try:
|
||||
process.communicate(prompt, timeout=self.settings.timeout_seconds)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
self._terminate_group(process)
|
||||
raise TriggerError("codex_exec_timeout") from exc
|
||||
if process.returncode != 0:
|
||||
raise TriggerError("codex_exec_nonzero")
|
||||
try:
|
||||
if output_path.stat().st_size > 1_048_576:
|
||||
raise TriggerError("codex_output_too_large")
|
||||
value = json.loads(output_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
raise TriggerError("codex_exec_invalid_json") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise TriggerError("codex_exec_invalid_json")
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _terminate_group(process: subprocess.Popen[str]) -> None:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
process.wait(timeout=2)
|
||||
except (ProcessLookupError, subprocess.TimeoutExpired):
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
process.wait(timeout=5)
|
||||
|
||||
@staticmethod
|
||||
def _validate_decision(value: dict, event: Event) -> Decision:
|
||||
decision = Decision.from_dict(value)
|
||||
if decision.contract_commit != event.contract_commit:
|
||||
raise ValidationError("contract_version_mismatch")
|
||||
if decision.event_id != event.event_id or decision.project_key != event.project_key:
|
||||
raise ValidationError("event_identity_mismatch")
|
||||
if decision.expected_state_version != event.state_version:
|
||||
raise ValidationError("state_version_mismatch")
|
||||
allowed = event.payload.get("allowed_work_unit_ids", [])
|
||||
if not isinstance(allowed, list) or any(not isinstance(item, str) for item in allowed):
|
||||
raise ValidationError("invalid_allowed_work_units")
|
||||
if any(unit.id not in allowed for unit in decision.next_work_units):
|
||||
raise ValidationError("work_unit_not_allowed")
|
||||
for unit in decision.next_work_units:
|
||||
instruction = unit.instruction.lower()
|
||||
if any(item in instruction for item in FORBIDDEN_INSTRUCTIONS):
|
||||
raise ValidationError("forbidden_operation")
|
||||
return decision
|
||||
|
||||
|
||||
class CodexLocalQueue:
|
||||
"""Atomic, idempotent event handoff from Gateway to the brain service."""
|
||||
|
||||
def __init__(self, root: Path):
|
||||
self.root = root
|
||||
for name in ("pending", "processing", "processed", "dead-letter"):
|
||||
ensure_shared_directory(root / name, group_writable=True)
|
||||
|
||||
def enqueue(self, event: Event) -> Path:
|
||||
pending = self.root / "pending"
|
||||
path = pending / f"{event.event_id}.json"
|
||||
body = json.dumps(
|
||||
{"attempts": 0, "next_attempt_at": 0, "event": event.to_dict()},
|
||||
separators=(",", ":"), sort_keys=True,
|
||||
)
|
||||
try:
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o640)
|
||||
except FileExistsError:
|
||||
existing = read_secure_json(path, expected_uid=None)
|
||||
if existing.get("event") != event.to_dict():
|
||||
raise TriggerError("codex_queue_idempotency_conflict")
|
||||
return path
|
||||
os.fchmod(fd, 0o640)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
handle.write(body)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
return path
|
||||
@@ -1,237 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
from pathlib import Path
|
||||
import stat
|
||||
from typing import Any
|
||||
|
||||
from .errors import ConfigurationError
|
||||
from .securefs import read_secure_bytes, read_secure_json, verify_secure_file
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
host: str
|
||||
port: int
|
||||
project_key: str
|
||||
database_path: Path
|
||||
event_inbox_dir: Path
|
||||
decision_inbox_dir: Path
|
||||
dispatch_outbox_dir: Path
|
||||
openclaw_sync_dir: Path
|
||||
command_queue_dir: Path
|
||||
inbound_token_file: Path
|
||||
callback_token_file: Path
|
||||
contract_commit: str
|
||||
central_channel: str
|
||||
codex_queue_dir: Path | None
|
||||
codex_timeout_seconds: int
|
||||
workspace_trigger_id: str | None
|
||||
workspace_token_file: Path | None
|
||||
responses_api_key_file: Path | None
|
||||
responses_model: str | None
|
||||
poll_interval_seconds: int
|
||||
summary_interval_seconds: int
|
||||
callback_timeout_seconds: int
|
||||
lease_seconds: int
|
||||
retry_base_seconds: int
|
||||
max_retries: int
|
||||
max_same_blocker_retries: int
|
||||
max_hops_per_gate: int
|
||||
openclaw_sync_endpoint: str | None
|
||||
openclaw_sync_token_file: Path | None
|
||||
git_poll_enabled: bool
|
||||
git_poll_bare_repo: Path | None
|
||||
git_poll_ref_prefixes: tuple[str, ...]
|
||||
git_poll_interval_seconds: int
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: Path) -> "Config":
|
||||
owner_uid = int(os.environ.get("AIOS_ORCHESTRATOR_CONFIG_OWNER_UID", "0"))
|
||||
raw = read_secure_json(path, expected_uid=owner_uid)
|
||||
expected = {
|
||||
"host", "port", "project_key", "database_path", "event_inbox_dir",
|
||||
"decision_inbox_dir", "dispatch_outbox_dir", "openclaw_sync_dir",
|
||||
"command_queue_dir", "inbound_token_file", "callback_token_file",
|
||||
"contract_commit", "workspace_agent", "responses_fallback",
|
||||
"central_channel", "codex_local", "polling", "loop_controls",
|
||||
"openclaw_sync", "git_poll",
|
||||
}
|
||||
if set(raw) != expected:
|
||||
raise ConfigurationError("invalid_config_fields")
|
||||
|
||||
def path_value(key: str) -> Path:
|
||||
value = raw.get(key)
|
||||
if not isinstance(value, str):
|
||||
raise ConfigurationError(f"invalid_{key}")
|
||||
result = Path(value)
|
||||
if not result.is_absolute():
|
||||
raise ConfigurationError(f"path_not_absolute:{key}")
|
||||
return result
|
||||
|
||||
def integer(container: dict[str, Any], key: str, lo: int, hi: int) -> int:
|
||||
value = container.get(key)
|
||||
if not isinstance(value, int) or not lo <= value <= hi:
|
||||
raise ConfigurationError(f"invalid_{key}")
|
||||
return value
|
||||
|
||||
host = raw.get("host")
|
||||
port = raw.get("port")
|
||||
project_key = raw.get("project_key")
|
||||
contract = raw.get("contract_commit")
|
||||
if not isinstance(host, str) or not host:
|
||||
raise ConfigurationError("invalid_host")
|
||||
if not isinstance(project_key, str) or not project_key:
|
||||
raise ConfigurationError("invalid_project_key")
|
||||
if not isinstance(port, int) or not 1 <= port <= 65535:
|
||||
raise ConfigurationError("invalid_port")
|
||||
if not isinstance(contract, str) or len(contract) != 40:
|
||||
raise ConfigurationError("invalid_contract_commit")
|
||||
|
||||
workspace = raw.get("workspace_agent")
|
||||
responses = raw.get("responses_fallback")
|
||||
codex_local = raw.get("codex_local")
|
||||
polling = raw.get("polling")
|
||||
loop = raw.get("loop_controls")
|
||||
sync = raw.get("openclaw_sync")
|
||||
git_poll = raw.get("git_poll")
|
||||
if not all(isinstance(x, dict) for x in (workspace, responses, codex_local, polling, loop, sync, git_poll)):
|
||||
raise ConfigurationError("invalid_nested_config")
|
||||
|
||||
central_channel = raw.get("central_channel")
|
||||
if central_channel not in {"codex_local", "workspace_agent", "responses_api"}:
|
||||
raise ConfigurationError("invalid_central_channel")
|
||||
|
||||
trigger_id = workspace.get("trigger_id")
|
||||
workspace_token = workspace.get("access_token_file")
|
||||
if trigger_id is not None:
|
||||
if not isinstance(trigger_id, str) or not trigger_id.startswith("agtch_"):
|
||||
raise ConfigurationError("invalid_workspace_trigger_id")
|
||||
if not isinstance(workspace_token, str) or not Path(workspace_token).is_absolute():
|
||||
raise ConfigurationError("invalid_workspace_token_file")
|
||||
workspace_token_file = Path(workspace_token)
|
||||
else:
|
||||
workspace_token_file = None
|
||||
|
||||
responses_key = responses.get("api_key_file")
|
||||
responses_model = responses.get("model")
|
||||
if responses_key is not None:
|
||||
if not isinstance(responses_key, str) or not Path(responses_key).is_absolute():
|
||||
raise ConfigurationError("invalid_responses_key_file")
|
||||
if not isinstance(responses_model, str) or not responses_model:
|
||||
raise ConfigurationError("invalid_responses_model")
|
||||
responses_key_file = Path(responses_key)
|
||||
else:
|
||||
responses_key_file = None
|
||||
responses_model = None
|
||||
|
||||
codex_queue = codex_local.get("queue_dir")
|
||||
codex_timeout = integer(codex_local, "timeout_seconds", 30, 3600)
|
||||
if central_channel == "codex_local":
|
||||
if trigger_id is not None or responses_key_file is not None:
|
||||
raise ConfigurationError("parked_channel_must_be_disabled")
|
||||
if not isinstance(codex_queue, str) or not Path(codex_queue).is_absolute():
|
||||
raise ConfigurationError("invalid_codex_queue_dir")
|
||||
codex_queue_dir = Path(codex_queue)
|
||||
else:
|
||||
codex_queue_dir = None
|
||||
if central_channel == "workspace_agent" and trigger_id is None:
|
||||
raise ConfigurationError("workspace_channel_not_configured")
|
||||
if central_channel == "responses_api" and responses_key_file is None:
|
||||
raise ConfigurationError("responses_channel_not_configured")
|
||||
|
||||
endpoint = sync.get("endpoint")
|
||||
token_file = sync.get("token_file")
|
||||
if endpoint is not None and (not isinstance(endpoint, str) or not endpoint.startswith(("http://", "https://"))):
|
||||
raise ConfigurationError("invalid_openclaw_sync_endpoint")
|
||||
if endpoint is not None:
|
||||
if not isinstance(token_file, str) or not Path(token_file).is_absolute():
|
||||
raise ConfigurationError("invalid_openclaw_sync_token_file")
|
||||
sync_token_file = Path(token_file)
|
||||
else:
|
||||
sync_token_file = None
|
||||
|
||||
git_enabled = git_poll.get("enabled")
|
||||
if not isinstance(git_enabled, bool):
|
||||
raise ConfigurationError("invalid_git_poll_enabled")
|
||||
git_bare_repo_raw = git_poll.get("bare_repo")
|
||||
git_prefixes_raw = git_poll.get("ref_prefixes")
|
||||
if git_enabled:
|
||||
if not isinstance(git_bare_repo_raw, str) or not Path(git_bare_repo_raw).is_absolute():
|
||||
raise ConfigurationError("invalid_git_poll_bare_repo")
|
||||
if (
|
||||
not isinstance(git_prefixes_raw, list)
|
||||
or not git_prefixes_raw
|
||||
or any(not isinstance(item, str) or not item.startswith("refs/") for item in git_prefixes_raw)
|
||||
):
|
||||
raise ConfigurationError("invalid_git_poll_ref_prefixes")
|
||||
git_bare_repo = Path(git_bare_repo_raw)
|
||||
git_prefixes = tuple(git_prefixes_raw)
|
||||
else:
|
||||
git_bare_repo = None
|
||||
git_prefixes = ()
|
||||
|
||||
return cls(
|
||||
host=host,
|
||||
port=port,
|
||||
project_key=project_key,
|
||||
database_path=path_value("database_path"),
|
||||
event_inbox_dir=path_value("event_inbox_dir"),
|
||||
decision_inbox_dir=path_value("decision_inbox_dir"),
|
||||
dispatch_outbox_dir=path_value("dispatch_outbox_dir"),
|
||||
openclaw_sync_dir=path_value("openclaw_sync_dir"),
|
||||
command_queue_dir=path_value("command_queue_dir"),
|
||||
inbound_token_file=path_value("inbound_token_file"),
|
||||
callback_token_file=path_value("callback_token_file"),
|
||||
contract_commit=contract,
|
||||
central_channel=central_channel,
|
||||
codex_queue_dir=codex_queue_dir,
|
||||
codex_timeout_seconds=codex_timeout,
|
||||
workspace_trigger_id=trigger_id,
|
||||
workspace_token_file=workspace_token_file,
|
||||
responses_api_key_file=responses_key_file,
|
||||
responses_model=responses_model,
|
||||
poll_interval_seconds=integer(polling, "interval_seconds", 1, 300),
|
||||
summary_interval_seconds=integer(polling, "summary_interval_seconds", 10, 86400),
|
||||
callback_timeout_seconds=integer(loop, "callback_timeout_seconds", 30, 86400),
|
||||
lease_seconds=integer(loop, "lease_seconds", 10, 3600),
|
||||
retry_base_seconds=integer(loop, "retry_base_seconds", 1, 3600),
|
||||
max_retries=integer(loop, "max_retries", 0, 20),
|
||||
max_same_blocker_retries=integer(loop, "max_same_blocker_retries", 0, 20),
|
||||
max_hops_per_gate=integer(loop, "max_hops_per_gate", 1, 100),
|
||||
openclaw_sync_endpoint=endpoint,
|
||||
openclaw_sync_token_file=sync_token_file,
|
||||
git_poll_enabled=git_enabled,
|
||||
git_poll_bare_repo=git_bare_repo,
|
||||
git_poll_ref_prefixes=git_prefixes,
|
||||
git_poll_interval_seconds=integer(git_poll, "interval_seconds", 5, 3600),
|
||||
)
|
||||
|
||||
def _secret(self, path: Path) -> str:
|
||||
owner_uid = int(os.environ.get("AIOS_ORCHESTRATOR_SECRET_OWNER_UID", "0"))
|
||||
info = verify_secure_file(path, expected_uid=owner_uid)
|
||||
if stat.S_IMODE(info.st_mode) & ~0o640:
|
||||
raise ConfigurationError(f"permissive_secret_mode:{path}")
|
||||
data = read_secure_bytes(path, expected_uid=owner_uid, max_bytes=64_000)
|
||||
if not data or b"\n" in data or b"\r" in data:
|
||||
raise ConfigurationError(f"invalid_secret_format:{path}")
|
||||
try:
|
||||
return data.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ConfigurationError(f"secret_not_utf8:{path}") from exc
|
||||
|
||||
def inbound_token(self) -> str:
|
||||
return self._secret(self.inbound_token_file)
|
||||
|
||||
def callback_token(self) -> str:
|
||||
return self._secret(self.callback_token_file)
|
||||
|
||||
def workspace_token(self) -> str | None:
|
||||
return self._secret(self.workspace_token_file) if self.workspace_token_file else None
|
||||
|
||||
def responses_api_key(self) -> str | None:
|
||||
return self._secret(self.responses_api_key_file) if self.responses_api_key_file else None
|
||||
|
||||
def openclaw_sync_token(self) -> str | None:
|
||||
return self._secret(self.openclaw_sync_token_file) if self.openclaw_sync_token_file else None
|
||||
@@ -1,122 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import os
|
||||
import stat
|
||||
from typing import Any
|
||||
|
||||
from .errors import DispatchError
|
||||
from .securefs import (
|
||||
atomic_write_bytes,
|
||||
atomic_write_bytes_shared,
|
||||
ensure_secure_directory,
|
||||
ensure_shared_directory,
|
||||
read_secure_bytes,
|
||||
)
|
||||
|
||||
|
||||
SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
|
||||
|
||||
|
||||
class DispatchSink:
|
||||
"""Reliable file-based dispatch adapter for existing OpenClaw/Codex watchers."""
|
||||
|
||||
def __init__(self, outbox_dir: Path, command_queue_dir: Path, contract_commit: str):
|
||||
self.outbox_dir = Path(outbox_dir)
|
||||
self.command_queue_dir = Path(command_queue_dir)
|
||||
self.contract_commit = contract_commit
|
||||
ensure_secure_directory(self.outbox_dir)
|
||||
ensure_shared_directory(self.command_queue_dir)
|
||||
|
||||
def dispatch(self, record: dict[str, Any]) -> tuple[Path, Path]:
|
||||
work_unit_id = record.get("work_unit_id")
|
||||
event_id = record.get("event_id")
|
||||
agent = record.get("assigned_agent")
|
||||
payload_json = record.get("payload_json")
|
||||
if (
|
||||
not isinstance(work_unit_id, str)
|
||||
or not SAFE_ID.fullmatch(work_unit_id)
|
||||
or not isinstance(event_id, str)
|
||||
or not SAFE_ID.fullmatch(event_id)
|
||||
or agent not in {"openclaw", "codex"}
|
||||
or not isinstance(payload_json, str)
|
||||
):
|
||||
raise DispatchError("invalid_dispatch_record")
|
||||
try:
|
||||
work_unit = json.loads(payload_json)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise DispatchError("invalid_dispatch_payload") from exc
|
||||
|
||||
project_key = record.get("project_key")
|
||||
state_version = record.get("state_version")
|
||||
|
||||
envelope = {
|
||||
"schema_version": 1,
|
||||
"contract_commit": self.contract_commit,
|
||||
"event_id": event_id,
|
||||
"assigned_agent": agent,
|
||||
"work_unit": work_unit,
|
||||
"callback": {
|
||||
"event_endpoint": "/v1/orchestrator/events",
|
||||
"start_event_type": "work_unit_started",
|
||||
"end_event_type": "work_unit_completed",
|
||||
},
|
||||
}
|
||||
outbox_path = self.outbox_dir / "dispatched" / f"{work_unit_id}.json"
|
||||
if agent == "codex":
|
||||
if (
|
||||
not isinstance(project_key, str)
|
||||
or not SAFE_ID.fullmatch(project_key)
|
||||
or not isinstance(state_version, int)
|
||||
or state_version < 0
|
||||
):
|
||||
raise DispatchError("invalid_dispatch_context")
|
||||
required_checks = work_unit.get("required_checks", [])
|
||||
execution_kind = (
|
||||
"acceptance_probe"
|
||||
if isinstance(required_checks, list) and "acceptance_probe" in required_checks
|
||||
else "codex"
|
||||
)
|
||||
queue_value = {
|
||||
"schema_version": 1,
|
||||
"command_id": work_unit_id,
|
||||
"project_key": project_key,
|
||||
"work_unit_id": work_unit_id,
|
||||
"contract_commit": self.contract_commit,
|
||||
"state_version": state_version,
|
||||
"execution": {
|
||||
"kind": execution_kind,
|
||||
"input": work_unit.get("instruction", ""),
|
||||
},
|
||||
}
|
||||
# The Codex watcher polls the configured active directory itself.
|
||||
queue_path = self.command_queue_dir / f"{work_unit_id}.json"
|
||||
else:
|
||||
queue_value = envelope
|
||||
queue_path = self.command_queue_dir / agent / "pending" / f"{work_unit_id}.json"
|
||||
ensure_secure_directory(outbox_path.parent)
|
||||
ensure_shared_directory(queue_path.parent)
|
||||
|
||||
# Idempotent immutable payload: existing identical file is accepted.
|
||||
self._write_idempotent_json(outbox_path, envelope, shared=False)
|
||||
self._write_idempotent_json(queue_path, queue_value, shared=True)
|
||||
return outbox_path, queue_path
|
||||
|
||||
@staticmethod
|
||||
def _write_idempotent_json(path: Path, value: dict[str, Any], *, shared: bool) -> None:
|
||||
payload = (json.dumps(value, indent=2, sort_keys=True) + "\n").encode()
|
||||
try:
|
||||
st = os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
if shared:
|
||||
atomic_write_bytes_shared(path, payload, mode=0o640)
|
||||
else:
|
||||
atomic_write_bytes(path, payload, mode=0o600)
|
||||
return
|
||||
if not stat.S_ISREG(st.st_mode):
|
||||
raise DispatchError(f"dispatch_target_not_regular:{path.name}")
|
||||
existing = read_secure_bytes(path, expected_uid=None, max_bytes=1_048_576)
|
||||
if existing != payload:
|
||||
raise DispatchError(f"dispatch_idempotency_conflict:{path.name}")
|
||||
@@ -1,37 +0,0 @@
|
||||
class OrchestratorError(Exception):
|
||||
"""Base orchestration error."""
|
||||
|
||||
|
||||
class ConfigurationError(OrchestratorError):
|
||||
"""Unsafe or invalid configuration."""
|
||||
|
||||
|
||||
class ValidationError(OrchestratorError):
|
||||
"""Invalid event or decision."""
|
||||
|
||||
|
||||
class AuthenticationError(OrchestratorError):
|
||||
"""Authentication failed."""
|
||||
|
||||
|
||||
class ConflictError(OrchestratorError):
|
||||
"""Idempotency, lease, or optimistic concurrency conflict."""
|
||||
|
||||
|
||||
class TriggerError(OrchestratorError):
|
||||
"""Central model trigger failed."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
*,
|
||||
http_status: int | None = None,
|
||||
category: str | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.http_status = http_status
|
||||
self.category = category
|
||||
|
||||
|
||||
class DispatchError(OrchestratorError):
|
||||
"""Dispatch or synchronization output failed."""
|
||||
@@ -1,346 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from .errors import ValidationError
|
||||
|
||||
|
||||
OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
|
||||
AGENTS = {"openclaw", "codex", "gitea", "system", "workspace-agent"}
|
||||
EVENT_STATUSES = {"READY", "RUNNING", "PASS", "BLOCKED", "FAIL", "TIMEOUT", "NOT_EXECUTED"}
|
||||
VERDICTS = {"DISPATCH", "BLOCKED", "COMPLETE", "NOOP"}
|
||||
EXECUTORS = {"openclaw", "codex"}
|
||||
|
||||
|
||||
def req_str(value: dict[str, Any], key: str, pattern: re.Pattern[str] | None = None) -> str:
|
||||
item = value.get(key)
|
||||
if not isinstance(item, str) or not item:
|
||||
raise ValidationError(f"invalid_{key}")
|
||||
if pattern is not None and not pattern.fullmatch(item):
|
||||
raise ValidationError(f"invalid_{key}")
|
||||
return item
|
||||
|
||||
|
||||
def opt_oid(item: Any, key: str) -> str | None:
|
||||
if item is None:
|
||||
return None
|
||||
if not isinstance(item, str) or not OID_RE.fullmatch(item):
|
||||
raise ValidationError(f"invalid_{key}")
|
||||
return item
|
||||
|
||||
|
||||
def string_list(value: dict[str, Any], key: str, limit: int) -> tuple[str, ...]:
|
||||
raw = value.get(key)
|
||||
if not isinstance(raw, list) or len(raw) > limit:
|
||||
raise ValidationError(f"invalid_{key}")
|
||||
result: list[str] = []
|
||||
for item in raw:
|
||||
if not isinstance(item, str) or not item or len(item) > 1024:
|
||||
raise ValidationError(f"invalid_{key}")
|
||||
result.append(item)
|
||||
return tuple(result)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Event:
|
||||
schema_version: int
|
||||
project_key: str
|
||||
contract_commit: str
|
||||
event_id: str
|
||||
source_agent: str
|
||||
event_type: str
|
||||
work_unit_id: str
|
||||
status: str
|
||||
state_version: int
|
||||
implementation_commit: str | None = None
|
||||
result_commit: str | None = None
|
||||
blocker: str | None = None
|
||||
evidence_refs: tuple[str, ...] = ()
|
||||
payload: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: dict[str, Any]) -> "Event":
|
||||
allowed = {
|
||||
"schema_version", "project_key", "contract_commit", "event_id",
|
||||
"source_agent", "event_type", "work_unit_id", "status",
|
||||
"state_version", "implementation_commit", "result_commit",
|
||||
"blocker", "evidence_refs", "payload",
|
||||
}
|
||||
if not isinstance(value, dict) or set(value) - allowed:
|
||||
raise ValidationError("invalid_event_fields")
|
||||
if value.get("schema_version") != 1:
|
||||
raise ValidationError("unsupported_event_schema")
|
||||
source = req_str(value, "source_agent")
|
||||
status = req_str(value, "status")
|
||||
if source not in AGENTS:
|
||||
raise ValidationError("invalid_source_agent")
|
||||
if status not in EVENT_STATUSES:
|
||||
raise ValidationError("invalid_event_status")
|
||||
version = value.get("state_version")
|
||||
if not isinstance(version, int) or version < 0:
|
||||
raise ValidationError("invalid_state_version")
|
||||
blocker = value.get("blocker")
|
||||
if blocker is not None and (not isinstance(blocker, str) or len(blocker) > 512):
|
||||
raise ValidationError("invalid_blocker")
|
||||
payload = value.get("payload", {})
|
||||
if not isinstance(payload, dict):
|
||||
raise ValidationError("invalid_payload")
|
||||
if len(json.dumps(payload, sort_keys=True).encode()) > 262_144:
|
||||
raise ValidationError("payload_too_large")
|
||||
return cls(
|
||||
schema_version=1,
|
||||
project_key=req_str(value, "project_key", ID_RE),
|
||||
contract_commit=req_str(value, "contract_commit", OID_RE),
|
||||
event_id=req_str(value, "event_id", ID_RE),
|
||||
source_agent=source,
|
||||
event_type=req_str(value, "event_type", ID_RE),
|
||||
work_unit_id=req_str(value, "work_unit_id", ID_RE),
|
||||
status=status,
|
||||
state_version=version,
|
||||
implementation_commit=opt_oid(value.get("implementation_commit"), "implementation_commit"),
|
||||
result_commit=opt_oid(value.get("result_commit"), "result_commit"),
|
||||
blocker=blocker,
|
||||
evidence_refs=string_list(value, "evidence_refs", 64),
|
||||
payload=payload,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"project_key": self.project_key,
|
||||
"contract_commit": self.contract_commit,
|
||||
"event_id": self.event_id,
|
||||
"source_agent": self.source_agent,
|
||||
"event_type": self.event_type,
|
||||
"work_unit_id": self.work_unit_id,
|
||||
"status": self.status,
|
||||
"state_version": self.state_version,
|
||||
"implementation_commit": self.implementation_commit,
|
||||
"result_commit": self.result_commit,
|
||||
"blocker": self.blocker,
|
||||
"evidence_refs": list(self.evidence_refs),
|
||||
"payload": self.payload,
|
||||
}
|
||||
|
||||
def canonical_json(self) -> str:
|
||||
return json.dumps(self.to_dict(), separators=(",", ":"), sort_keys=True)
|
||||
|
||||
def digest(self) -> str:
|
||||
return hashlib.sha256(self.canonical_json().encode()).hexdigest()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorkUnit:
|
||||
id: str
|
||||
track_id: str
|
||||
gate_id: str
|
||||
primary_agent: str
|
||||
fallback_agent: str
|
||||
clean_base_commit: str
|
||||
allowed_paths: tuple[str, ...]
|
||||
required_checks: tuple[str, ...]
|
||||
forbidden_operations: tuple[str, ...]
|
||||
instruction: str
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: dict[str, Any]) -> "WorkUnit":
|
||||
expected = {
|
||||
"id", "track_id", "gate_id", "primary_agent", "fallback_agent",
|
||||
"clean_base_commit", "allowed_paths", "required_checks",
|
||||
"forbidden_operations", "instruction",
|
||||
}
|
||||
if not isinstance(value, dict) or set(value) != expected:
|
||||
raise ValidationError("invalid_work_unit_fields")
|
||||
primary = req_str(value, "primary_agent")
|
||||
fallback = req_str(value, "fallback_agent")
|
||||
if primary not in EXECUTORS or fallback not in EXECUTORS or primary == fallback:
|
||||
raise ValidationError("invalid_agent_assignment")
|
||||
instruction = req_str(value, "instruction")
|
||||
if len(instruction.encode()) > 65_536:
|
||||
raise ValidationError("instruction_too_large")
|
||||
return cls(
|
||||
id=req_str(value, "id", ID_RE),
|
||||
track_id=req_str(value, "track_id", ID_RE),
|
||||
gate_id=req_str(value, "gate_id", ID_RE),
|
||||
primary_agent=primary,
|
||||
fallback_agent=fallback,
|
||||
clean_base_commit=req_str(value, "clean_base_commit", OID_RE),
|
||||
allowed_paths=string_list(value, "allowed_paths", 128),
|
||||
required_checks=string_list(value, "required_checks", 128),
|
||||
forbidden_operations=string_list(value, "forbidden_operations", 64),
|
||||
instruction=instruction,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"id": self.id,
|
||||
"track_id": self.track_id,
|
||||
"gate_id": self.gate_id,
|
||||
"primary_agent": self.primary_agent,
|
||||
"fallback_agent": self.fallback_agent,
|
||||
"clean_base_commit": self.clean_base_commit,
|
||||
"allowed_paths": list(self.allowed_paths),
|
||||
"required_checks": list(self.required_checks),
|
||||
"forbidden_operations": list(self.forbidden_operations),
|
||||
"instruction": self.instruction,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Decision:
|
||||
schema_version: int
|
||||
project_key: str
|
||||
contract_commit: str
|
||||
event_id: str
|
||||
expected_state_version: int
|
||||
verdict: str
|
||||
state_updates: dict[str, str]
|
||||
next_work_units: tuple[WorkUnit, ...]
|
||||
requires_human_input: bool
|
||||
stop_reason: str | None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: dict[str, Any]) -> "Decision":
|
||||
expected = {
|
||||
"schema_version", "project_key", "contract_commit", "event_id",
|
||||
"expected_state_version", "verdict", "state_updates",
|
||||
"next_work_units", "requires_human_input", "stop_reason",
|
||||
}
|
||||
if not isinstance(value, dict) or set(value) != expected:
|
||||
raise ValidationError("invalid_decision_fields")
|
||||
if value.get("schema_version") != 1:
|
||||
raise ValidationError("unsupported_decision_schema")
|
||||
version = value.get("expected_state_version")
|
||||
if not isinstance(version, int) or version < 0:
|
||||
raise ValidationError("invalid_expected_state_version")
|
||||
verdict = req_str(value, "verdict")
|
||||
if verdict not in VERDICTS:
|
||||
raise ValidationError("invalid_verdict")
|
||||
updates = value.get("state_updates")
|
||||
normalized: dict[str, str] = {}
|
||||
if isinstance(updates, dict):
|
||||
update_items = [{"key": key, "value": item} for key, item in updates.items()]
|
||||
elif isinstance(updates, list):
|
||||
update_items = updates
|
||||
else:
|
||||
raise ValidationError("invalid_state_updates")
|
||||
if len(update_items) > 128:
|
||||
raise ValidationError("invalid_state_updates")
|
||||
for update in update_items:
|
||||
if not isinstance(update, dict) or set(update) != {"key", "value"}:
|
||||
raise ValidationError("invalid_state_update")
|
||||
key = update.get("key")
|
||||
item = update.get("value")
|
||||
if not isinstance(key, str) or not ID_RE.fullmatch(key):
|
||||
raise ValidationError("invalid_state_update_key")
|
||||
if not isinstance(item, str) or not item or len(item) > 256:
|
||||
raise ValidationError("invalid_state_update_value")
|
||||
if key in normalized:
|
||||
raise ValidationError("duplicate_state_update_key")
|
||||
normalized[key] = item
|
||||
raw_units = value.get("next_work_units")
|
||||
if not isinstance(raw_units, list) or len(raw_units) > 32:
|
||||
raise ValidationError("invalid_next_work_units")
|
||||
units = tuple(WorkUnit.from_dict(item) for item in raw_units)
|
||||
human = value.get("requires_human_input")
|
||||
if not isinstance(human, bool):
|
||||
raise ValidationError("invalid_requires_human_input")
|
||||
stop = value.get("stop_reason")
|
||||
if stop is not None and (not isinstance(stop, str) or len(stop) > 512):
|
||||
raise ValidationError("invalid_stop_reason")
|
||||
if verdict == "DISPATCH" and not units:
|
||||
raise ValidationError("dispatch_without_work_units")
|
||||
if verdict != "DISPATCH" and units:
|
||||
raise ValidationError("work_units_without_dispatch")
|
||||
return cls(
|
||||
schema_version=1,
|
||||
project_key=req_str(value, "project_key", ID_RE),
|
||||
contract_commit=req_str(value, "contract_commit", OID_RE),
|
||||
event_id=req_str(value, "event_id", ID_RE),
|
||||
expected_state_version=version,
|
||||
verdict=verdict,
|
||||
state_updates=normalized,
|
||||
next_work_units=units,
|
||||
requires_human_input=human,
|
||||
stop_reason=stop,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"project_key": self.project_key,
|
||||
"contract_commit": self.contract_commit,
|
||||
"event_id": self.event_id,
|
||||
"expected_state_version": self.expected_state_version,
|
||||
"verdict": self.verdict,
|
||||
"state_updates": [
|
||||
{"key": key, "value": value}
|
||||
for key, value in sorted(self.state_updates.items())
|
||||
],
|
||||
"next_work_units": [u.to_dict() for u in self.next_work_units],
|
||||
"requires_human_input": self.requires_human_input,
|
||||
"stop_reason": self.stop_reason,
|
||||
}
|
||||
|
||||
|
||||
DECISION_SCHEMA: dict[str, Any] = {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": [
|
||||
"schema_version", "project_key", "contract_commit", "event_id",
|
||||
"expected_state_version", "verdict", "state_updates",
|
||||
"next_work_units", "requires_human_input", "stop_reason",
|
||||
],
|
||||
"properties": {
|
||||
"schema_version": {"type": "integer", "const": 1},
|
||||
"project_key": {"type": "string"},
|
||||
"contract_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
|
||||
"event_id": {"type": "string"},
|
||||
"expected_state_version": {"type": "integer", "minimum": 0},
|
||||
"verdict": {"type": "string", "enum": sorted(VERDICTS)},
|
||||
"state_updates": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": ["key", "value"],
|
||||
"properties": {
|
||||
"key": {"type": "string"},
|
||||
"value": {"type": "string"}
|
||||
}
|
||||
}
|
||||
},
|
||||
"next_work_units": {
|
||||
"type": "array",
|
||||
"maxItems": 32,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": False,
|
||||
"required": [
|
||||
"id", "track_id", "gate_id", "primary_agent",
|
||||
"fallback_agent", "clean_base_commit", "allowed_paths",
|
||||
"required_checks", "forbidden_operations", "instruction",
|
||||
],
|
||||
"properties": {
|
||||
"id": {"type": "string"},
|
||||
"track_id": {"type": "string"},
|
||||
"gate_id": {"type": "string"},
|
||||
"primary_agent": {"type": "string", "enum": ["openclaw", "codex"]},
|
||||
"fallback_agent": {"type": "string", "enum": ["openclaw", "codex"]},
|
||||
"clean_base_commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
|
||||
"allowed_paths": {"type": "array", "items": {"type": "string"}},
|
||||
"required_checks": {"type": "array", "items": {"type": "string"}},
|
||||
"forbidden_operations": {"type": "array", "items": {"type": "string"}},
|
||||
"instruction": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"requires_human_input": {"type": "boolean"},
|
||||
"stop_reason": {"type": ["string", "null"]},
|
||||
},
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import socket
|
||||
|
||||
from .store import Store
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RecoverySettings:
|
||||
database_path: Path
|
||||
contract_commit: str
|
||||
lease_seconds: int
|
||||
max_retries: int
|
||||
retry_after: int
|
||||
|
||||
|
||||
class PendingDecisionRecovery:
|
||||
def __init__(self, settings: RecoverySettings, *, plan_only: bool = False):
|
||||
self.settings = settings
|
||||
self.store = Store(settings.database_path, read_only=plan_only)
|
||||
self.owner = f"recovery-{socket.gethostname()}"
|
||||
|
||||
def plan(self) -> dict[str, int]:
|
||||
return self.store.pending_decision_plan(
|
||||
self.settings.contract_commit, self.settings.max_retries
|
||||
)
|
||||
|
||||
def apply_one(self) -> str | None:
|
||||
# Recovery never calls a model or replays a decision. It only advances
|
||||
# durable callback bookkeeping under a lease.
|
||||
return self.store.recover_one_pending(
|
||||
owner=self.owner,
|
||||
contract_commit=self.settings.contract_commit,
|
||||
lease_seconds=self.settings.lease_seconds,
|
||||
max_retries=self.settings.max_retries,
|
||||
retry_after=self.settings.retry_after,
|
||||
)
|
||||
@@ -1,122 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import stat
|
||||
import time
|
||||
from typing import Any, Callable
|
||||
|
||||
from .callback_transport import CallbackTransport
|
||||
from .errors import AuthenticationError, ValidationError
|
||||
from .models import Decision
|
||||
from .models import ID_RE, OID_RE
|
||||
from .securefs import ensure_secure_directory
|
||||
|
||||
|
||||
AUDIT_FIELDS = (
|
||||
"callback_request_id", "event_id", "contract_commit", "received_at",
|
||||
"schema_valid", "auth_valid", "forward_status", "state_version_after",
|
||||
)
|
||||
|
||||
|
||||
class CallbackAudit:
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
|
||||
def write(self, record: dict[str, Any]) -> None:
|
||||
if set(record) != set(AUDIT_FIELDS):
|
||||
raise ValueError("invalid_callback_audit_fields")
|
||||
ensure_secure_directory(self.path.parent)
|
||||
flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0)
|
||||
fd = os.open(self.path, flags, 0o600)
|
||||
try:
|
||||
info = os.fstat(fd)
|
||||
if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) & 0o077:
|
||||
raise ValueError("unsafe_callback_audit_file")
|
||||
data = (json.dumps(record, separators=(",", ":"), sort_keys=True) + "\n").encode()
|
||||
os.write(fd, data)
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
class MinimalDecisionTool:
|
||||
"""Transport-neutral single-tool boundary; no generic GitOps operations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
transport: CallbackTransport,
|
||||
*,
|
||||
contract_commit: str,
|
||||
access_token: str,
|
||||
forward: Callable[[dict[str, Any]], dict[str, Any]],
|
||||
audit: CallbackAudit,
|
||||
):
|
||||
self.transport = transport
|
||||
self.contract_commit = contract_commit
|
||||
self.access_token = access_token
|
||||
self.forward = forward
|
||||
self.audit = audit
|
||||
|
||||
def tools(self) -> list[dict[str, Any]]:
|
||||
return [self.transport.mcp_tool()] if self.transport.mode == "secure_mcp_tunnel" else []
|
||||
|
||||
def authenticate(self, authorization: str) -> None:
|
||||
if not hmac.compare_digest(authorization, f"Bearer {self.access_token}"):
|
||||
raise AuthenticationError("invalid_bearer")
|
||||
|
||||
def submit(self, payload: dict[str, Any], authorization: str) -> dict[str, Any]:
|
||||
request_id = secrets.token_hex(16)
|
||||
received_at = int(time.time())
|
||||
schema_valid = False
|
||||
auth_valid = False
|
||||
raw_event_id = payload.get("event_id") if isinstance(payload, dict) else None
|
||||
raw_contract = payload.get("contract_commit") if isinstance(payload, dict) else None
|
||||
event_id = raw_event_id if isinstance(raw_event_id, str) and ID_RE.fullmatch(raw_event_id) else None
|
||||
contract = raw_contract if isinstance(raw_contract, str) and OID_RE.fullmatch(raw_contract) else None
|
||||
forward_status = "REJECTED"
|
||||
state_version = None
|
||||
try:
|
||||
self.authenticate(authorization)
|
||||
auth_valid = True
|
||||
decision = Decision.from_dict(payload)
|
||||
schema_valid = True
|
||||
if decision.contract_commit != self.contract_commit:
|
||||
raise ValidationError("contract_version_mismatch")
|
||||
result = self.forward(decision.to_dict())
|
||||
if result.get("accepted") is not True or not isinstance(result.get("state_version"), int):
|
||||
raise ValidationError("invalid_forward_response")
|
||||
forward_status = "202"
|
||||
state_version = result["state_version"]
|
||||
return result
|
||||
finally:
|
||||
self.audit.write({
|
||||
"callback_request_id": request_id,
|
||||
"event_id": event_id,
|
||||
"contract_commit": contract,
|
||||
"received_at": received_at,
|
||||
"schema_valid": schema_valid,
|
||||
"auth_valid": auth_valid,
|
||||
"forward_status": forward_status,
|
||||
"state_version_after": state_version,
|
||||
})
|
||||
|
||||
def handle_mcp(self, request: dict[str, Any], authorization: str) -> dict[str, Any]:
|
||||
self.authenticate(authorization)
|
||||
method = request.get("method")
|
||||
request_id = request.get("id")
|
||||
if method == "tools/list":
|
||||
return {"jsonrpc": "2.0", "id": request_id, "result": {"tools": self.tools()}}
|
||||
if method != "tools/call":
|
||||
raise ValidationError("unsupported_mcp_method")
|
||||
params = request.get("params")
|
||||
if not isinstance(params, dict) or params.get("name") != "submit_orchestration_decision":
|
||||
raise ValidationError("unsupported_mcp_tool")
|
||||
arguments = params.get("arguments")
|
||||
if not isinstance(arguments, dict):
|
||||
raise ValidationError("invalid_mcp_arguments")
|
||||
result = self.submit(arguments, authorization)
|
||||
return {"jsonrpc": "2.0", "id": request_id, "result": {"structuredContent": result}}
|
||||
@@ -1,144 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import argparse
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from .action_bridge import resolve_secret_ref, submit_orchestration_decision
|
||||
from .callback_transport import CallbackTransport
|
||||
from .errors import AuthenticationError, ConfigurationError, OrchestratorError
|
||||
from .remote_tool import CallbackAudit, MinimalDecisionTool
|
||||
from .securefs import read_secure_json
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RemoteToolServerConfig:
|
||||
listen_host: str
|
||||
listen_port: int
|
||||
contract_commit: str
|
||||
transport: CallbackTransport
|
||||
access_secret_ref: str
|
||||
callback_secret_ref: str
|
||||
secret_ref_resolver: Path
|
||||
audit_path: Path
|
||||
manifest_path: Path
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, path: Path) -> "RemoteToolServerConfig":
|
||||
value = read_secure_json(path, expected_uid=None)
|
||||
expected = {
|
||||
"listen_host", "listen_port", "contract_commit", "callback_transport",
|
||||
"access_secret_ref", "callback_secret_ref", "secret_ref_resolver",
|
||||
"audit_path", "manifest_path",
|
||||
}
|
||||
if set(value) != expected:
|
||||
raise ConfigurationError("invalid_remote_tool_config_fields")
|
||||
if value["listen_host"] not in {"127.0.0.1", "::1"}:
|
||||
raise ConfigurationError("remote_tool_must_listen_loopback")
|
||||
if not isinstance(value["listen_port"], int) or not 1 <= value["listen_port"] <= 65535:
|
||||
raise ConfigurationError("invalid_remote_tool_port")
|
||||
contract = value["contract_commit"]
|
||||
if not isinstance(contract, str) or len(contract) != 40:
|
||||
raise ConfigurationError("invalid_contract_commit")
|
||||
paths = [Path(value[name]) for name in ("secret_ref_resolver", "audit_path", "manifest_path")]
|
||||
if any(not path_value.is_absolute() for path_value in paths):
|
||||
raise ConfigurationError("remote_tool_path_not_absolute")
|
||||
for name in ("access_secret_ref", "callback_secret_ref"):
|
||||
if not isinstance(value[name], str) or not value[name].startswith("secret://"):
|
||||
raise ConfigurationError("remote_tool_secret_ref_required")
|
||||
return cls(
|
||||
value["listen_host"], value["listen_port"], contract,
|
||||
CallbackTransport.from_dict(value["callback_transport"]),
|
||||
value["access_secret_ref"], value["callback_secret_ref"], paths[0],
|
||||
paths[1], paths[2],
|
||||
)
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
app: MinimalDecisionTool
|
||||
config: RemoteToolServerConfig
|
||||
|
||||
def log_message(self, _format: str, *_args: object) -> None:
|
||||
return
|
||||
|
||||
def reply(self, status: int, payload: dict[str, Any]) -> None:
|
||||
body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def body(self) -> dict[str, Any]:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
if length <= 0 or length > 1_048_576:
|
||||
raise ValueError("invalid_body_size")
|
||||
value = json.loads(self.rfile.read(length))
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError("invalid_json_root")
|
||||
return value
|
||||
|
||||
def do_GET(self) -> None:
|
||||
path = urlparse(self.path).path
|
||||
if path == "/healthz":
|
||||
self.reply(200, {"status": "active", "mode": self.config.transport.mode})
|
||||
return
|
||||
if path == "/orchestrator-tool-manifest":
|
||||
value = json.loads(self.config.manifest_path.read_text(encoding="utf-8"))
|
||||
self.reply(200, value)
|
||||
return
|
||||
self.reply(404, {"error": "not_found"})
|
||||
|
||||
def do_POST(self) -> None:
|
||||
path = urlparse(self.path).path.rstrip("/")
|
||||
try:
|
||||
if self.config.transport.mode == "remote_https_action" and path == "/v1/orchestrator/decisions":
|
||||
result = self.app.submit(self.body(), self.headers.get("Authorization", ""))
|
||||
elif self.config.transport.mode == "secure_mcp_tunnel" and path == "/mcp":
|
||||
result = self.app.handle_mcp(self.body(), self.headers.get("Authorization", ""))
|
||||
else:
|
||||
self.reply(404, {"error": "not_found"})
|
||||
return
|
||||
self.reply(202, result)
|
||||
except AuthenticationError:
|
||||
self.reply(401, {"error": "UNAUTHORIZED"})
|
||||
except OrchestratorError as exc:
|
||||
self.reply(422, {"error": type(exc).__name__})
|
||||
except Exception:
|
||||
self.reply(500, {"error": "INTERNAL_ERROR"})
|
||||
|
||||
|
||||
def serve(config_path: Path) -> None:
|
||||
config = RemoteToolServerConfig.from_file(config_path)
|
||||
access_token = resolve_secret_ref(config.access_secret_ref, config.secret_ref_resolver)
|
||||
|
||||
def forward(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return submit_orchestration_decision(
|
||||
payload, contract_commit=config.contract_commit,
|
||||
secret_ref=config.callback_secret_ref,
|
||||
resolver=config.secret_ref_resolver,
|
||||
)
|
||||
|
||||
Handler.config = config
|
||||
Handler.app = MinimalDecisionTool(
|
||||
config.transport, contract_commit=config.contract_commit,
|
||||
access_token=access_token, forward=forward, audit=CallbackAudit(config.audit_path),
|
||||
)
|
||||
ThreadingHTTPServer((config.listen_host, config.listen_port), Handler).serve_forever()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("config", type=Path)
|
||||
args = parser.parse_args()
|
||||
serve(args.config)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,228 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import stat
|
||||
from typing import Any
|
||||
|
||||
from .errors import ConfigurationError
|
||||
|
||||
|
||||
_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
|
||||
|
||||
|
||||
def reject_symlink_chain(path: Path) -> None:
|
||||
"""Reject symlinks in every existing component of an absolute path."""
|
||||
path = Path(path)
|
||||
if not path.is_absolute():
|
||||
raise ConfigurationError(f"path_not_absolute:{path}")
|
||||
current = Path(path.anchor)
|
||||
for part in path.parts[1:]:
|
||||
current = current / part
|
||||
try:
|
||||
st = os.lstat(current)
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if stat.S_ISLNK(st.st_mode):
|
||||
raise ConfigurationError(f"symlink_rejected:{current}")
|
||||
|
||||
|
||||
def verify_secure_file(
|
||||
path: Path,
|
||||
*,
|
||||
expected_uid: int | None,
|
||||
allowed_write_bits: int = stat.S_IWUSR,
|
||||
require_executable: bool = False,
|
||||
) -> os.stat_result:
|
||||
path = Path(path)
|
||||
reject_symlink_chain(path)
|
||||
before = os.lstat(path)
|
||||
if not stat.S_ISREG(before.st_mode):
|
||||
raise ConfigurationError(f"not_regular_file:{path}")
|
||||
if expected_uid is not None and before.st_uid != expected_uid:
|
||||
raise ConfigurationError(f"unexpected_owner:{path}")
|
||||
write_bits = before.st_mode & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH)
|
||||
if write_bits & ~allowed_write_bits:
|
||||
raise ConfigurationError(f"permissive_mode:{path}")
|
||||
if require_executable and not (before.st_mode & stat.S_IXUSR):
|
||||
raise ConfigurationError(f"not_executable:{path}")
|
||||
fd = os.open(path, os.O_RDONLY | _NOFOLLOW)
|
||||
try:
|
||||
after = os.fstat(fd)
|
||||
if (before.st_dev, before.st_ino) != (after.st_dev, after.st_ino):
|
||||
raise ConfigurationError(f"inode_changed:{path}")
|
||||
finally:
|
||||
os.close(fd)
|
||||
return before
|
||||
|
||||
|
||||
def read_secure_bytes(
|
||||
path: Path,
|
||||
*,
|
||||
expected_uid: int | None,
|
||||
max_bytes: int = 1_048_576,
|
||||
) -> bytes:
|
||||
path = Path(path)
|
||||
before = verify_secure_file(path, expected_uid=expected_uid)
|
||||
if before.st_size > max_bytes:
|
||||
raise ConfigurationError(f"file_too_large:{path}")
|
||||
fd = os.open(path, os.O_RDONLY | _NOFOLLOW)
|
||||
try:
|
||||
data = bytearray()
|
||||
while True:
|
||||
chunk = os.read(fd, min(65536, max_bytes + 1 - len(data)))
|
||||
if not chunk:
|
||||
break
|
||||
data.extend(chunk)
|
||||
if len(data) > max_bytes:
|
||||
raise ConfigurationError(f"file_too_large:{path}")
|
||||
return bytes(data)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def read_secure_text(path: Path, *, expected_uid: int | None, max_bytes: int = 1_048_576) -> str:
|
||||
try:
|
||||
return read_secure_bytes(path, expected_uid=expected_uid, max_bytes=max_bytes).decode("utf-8").strip()
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ConfigurationError(f"not_utf8:{path}") from exc
|
||||
|
||||
|
||||
def read_secure_json(path: Path, *, expected_uid: int | None, max_bytes: int = 1_048_576) -> dict[str, Any]:
|
||||
try:
|
||||
value = json.loads(read_secure_text(path, expected_uid=expected_uid, max_bytes=max_bytes))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ConfigurationError(f"invalid_json:{path}") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise ConfigurationError(f"json_root_not_object:{path}")
|
||||
return value
|
||||
|
||||
|
||||
def ensure_secure_directory(path: Path, *, mode: int = 0o700) -> None:
|
||||
path = Path(path)
|
||||
if not path.is_absolute():
|
||||
raise ConfigurationError(f"path_not_absolute:{path}")
|
||||
reject_symlink_chain(path.parent)
|
||||
path.mkdir(parents=True, exist_ok=True, mode=mode)
|
||||
reject_symlink_chain(path)
|
||||
st = os.lstat(path)
|
||||
if not stat.S_ISDIR(st.st_mode):
|
||||
raise ConfigurationError(f"not_directory:{path}")
|
||||
if st.st_mode & (stat.S_IWGRP | stat.S_IWOTH):
|
||||
raise ConfigurationError(f"permissive_directory:{path}")
|
||||
|
||||
|
||||
def atomic_write_bytes(path: Path, payload: bytes, *, mode: int = 0o600) -> None:
|
||||
"""Write through a verified directory fd and atomically replace the target."""
|
||||
path = Path(path)
|
||||
ensure_secure_directory(path.parent)
|
||||
parent_before = os.lstat(path.parent)
|
||||
parent_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | _NOFOLLOW)
|
||||
temp_name = f".{path.name}.tmp-{os.getpid()}-{os.urandom(6).hex()}"
|
||||
try:
|
||||
parent_after = os.fstat(parent_fd)
|
||||
if (parent_before.st_dev, parent_before.st_ino) != (
|
||||
parent_after.st_dev,
|
||||
parent_after.st_ino,
|
||||
):
|
||||
raise ConfigurationError(f"directory_changed:{path.parent}")
|
||||
fd = os.open(
|
||||
temp_name,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | _NOFOLLOW,
|
||||
mode,
|
||||
dir_fd=parent_fd,
|
||||
)
|
||||
try:
|
||||
view = memoryview(payload)
|
||||
while view:
|
||||
written = os.write(fd, view)
|
||||
view = view[written:]
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
os.replace(
|
||||
temp_name,
|
||||
path.name,
|
||||
src_dir_fd=parent_fd,
|
||||
dst_dir_fd=parent_fd,
|
||||
)
|
||||
os.fsync(parent_fd)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temp_name, dir_fd=parent_fd)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
os.close(parent_fd)
|
||||
|
||||
|
||||
def atomic_write_json(path: Path, value: Any, *, mode: int = 0o600) -> None:
|
||||
payload = (json.dumps(value, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
||||
atomic_write_bytes(path, payload, mode=mode)
|
||||
|
||||
def ensure_shared_directory(path: Path, *, group_writable: bool = False) -> None:
|
||||
"""Verify/create a setgid queue directory shared with a consumer group."""
|
||||
path = Path(path)
|
||||
if not path.is_absolute():
|
||||
raise ConfigurationError(f"path_not_absolute:{path}")
|
||||
reject_symlink_chain(path.parent)
|
||||
mode = 0o2770 if group_writable else 0o2750
|
||||
try:
|
||||
os.lstat(path)
|
||||
except FileNotFoundError:
|
||||
path.mkdir(parents=True, mode=mode)
|
||||
os.chmod(path, mode)
|
||||
reject_symlink_chain(path)
|
||||
st = os.lstat(path)
|
||||
if not stat.S_ISDIR(st.st_mode):
|
||||
raise ConfigurationError(f"not_directory:{path}")
|
||||
observed = stat.S_IMODE(st.st_mode)
|
||||
required = stat.S_ISGID | stat.S_IRGRP | stat.S_IXGRP
|
||||
if group_writable:
|
||||
required |= stat.S_IWGRP
|
||||
if observed & 0o007 or observed & required != required:
|
||||
raise ConfigurationError(f"invalid_shared_directory_mode:{path}")
|
||||
|
||||
def atomic_write_bytes_shared(
|
||||
path: Path,
|
||||
payload: bytes,
|
||||
*,
|
||||
mode: int = 0o640,
|
||||
group_writable_directory: bool = False,
|
||||
) -> None:
|
||||
"""Atomic write into a setgid/group-readable shared queue directory."""
|
||||
path = Path(path)
|
||||
ensure_shared_directory(path.parent, group_writable=group_writable_directory)
|
||||
parent_before = os.lstat(path.parent)
|
||||
parent_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | _NOFOLLOW)
|
||||
temp_name = f".{path.name}.tmp-{os.getpid()}-{os.urandom(6).hex()}"
|
||||
try:
|
||||
parent_after = os.fstat(parent_fd)
|
||||
if (parent_before.st_dev, parent_before.st_ino) != (
|
||||
parent_after.st_dev,
|
||||
parent_after.st_ino,
|
||||
):
|
||||
raise ConfigurationError(f"directory_changed:{path.parent}")
|
||||
fd = os.open(
|
||||
temp_name,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | _NOFOLLOW,
|
||||
mode,
|
||||
dir_fd=parent_fd,
|
||||
)
|
||||
os.fchmod(fd, mode)
|
||||
try:
|
||||
view = memoryview(payload)
|
||||
while view:
|
||||
written = os.write(fd, view)
|
||||
view = view[written:]
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
os.replace(temp_name, path.name, src_dir_fd=parent_fd, dst_dir_fd=parent_fd)
|
||||
os.fsync(parent_fd)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temp_name, dir_fd=parent_fd)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
os.close(parent_fd)
|
||||
@@ -1,716 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from .clients import ResponsesClient, WorkspaceAgentClient
|
||||
from .codex_local import CodexLocalQueue
|
||||
from .config import Config
|
||||
from .dispatch import DispatchSink
|
||||
from .errors import (
|
||||
AuthenticationError,
|
||||
ConflictError,
|
||||
DispatchError,
|
||||
OrchestratorError,
|
||||
TriggerError,
|
||||
ValidationError,
|
||||
)
|
||||
from .models import Decision, Event, ID_RE
|
||||
from .securefs import ensure_secure_directory, ensure_shared_directory, read_secure_json
|
||||
from .store import Store
|
||||
from .sync import OpenClawSynchronizer, lifecycle_record
|
||||
|
||||
|
||||
LOG = logging.getLogger("aios-orchestrator-gateway")
|
||||
|
||||
|
||||
class Runtime:
|
||||
def __init__(self, config: Config):
|
||||
self.config = config
|
||||
self.store = Store(config.database_path)
|
||||
self.dispatch = DispatchSink(
|
||||
config.dispatch_outbox_dir,
|
||||
config.command_queue_dir,
|
||||
config.contract_commit,
|
||||
)
|
||||
self.workspace = None
|
||||
workspace_token = config.workspace_token() if config.central_channel == "workspace_agent" else None
|
||||
if config.central_channel == "workspace_agent" and config.workspace_trigger_id and workspace_token:
|
||||
self.workspace = WorkspaceAgentClient(
|
||||
config.workspace_trigger_id,
|
||||
workspace_token,
|
||||
)
|
||||
self.responses = None
|
||||
responses_key = config.responses_api_key() if config.central_channel == "responses_api" else None
|
||||
if config.central_channel == "responses_api" and responses_key and config.responses_model:
|
||||
self.responses = ResponsesClient(
|
||||
responses_key,
|
||||
config.responses_model,
|
||||
)
|
||||
self.codex_queue = None
|
||||
if config.central_channel == "codex_local" and config.codex_queue_dir is not None:
|
||||
self.codex_queue = CodexLocalQueue(config.codex_queue_dir)
|
||||
self.synchronizer = OpenClawSynchronizer(
|
||||
config.openclaw_sync_dir,
|
||||
endpoint=config.openclaw_sync_endpoint,
|
||||
bearer_token=config.openclaw_sync_token(),
|
||||
)
|
||||
self.worker_id = f"{socket.gethostname()}-{os.getpid()}-{threading.get_native_id()}"
|
||||
self.stop_event = threading.Event()
|
||||
self.worker = threading.Thread(target=self._loop, daemon=True)
|
||||
self._next_git_poll = 0.0
|
||||
for directory in (
|
||||
config.event_inbox_dir,
|
||||
config.decision_inbox_dir,
|
||||
config.dispatch_outbox_dir,
|
||||
config.openclaw_sync_dir,
|
||||
):
|
||||
ensure_secure_directory(directory)
|
||||
ensure_shared_directory(config.command_queue_dir)
|
||||
|
||||
def start(self) -> None:
|
||||
self.worker.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self.stop_event.set()
|
||||
if self.worker.is_alive():
|
||||
self.worker.join(timeout=15)
|
||||
self.store.close()
|
||||
|
||||
def submit_event(self, payload: dict[str, Any]) -> tuple[bool, dict[str, Any]]:
|
||||
event = Event.from_dict(payload)
|
||||
if event.contract_commit != self.config.contract_commit:
|
||||
raise ConflictError("contract_version_mismatch")
|
||||
created, stored = self.store.register_event(event)
|
||||
if created:
|
||||
self._enqueue_event_lifecycle(event)
|
||||
return created, stored
|
||||
|
||||
def submit_decision(self, payload: dict[str, Any]) -> int:
|
||||
decision = Decision.from_dict(payload)
|
||||
if decision.contract_commit != self.config.contract_commit:
|
||||
raise ConflictError("contract_version_mismatch")
|
||||
try:
|
||||
version = self.store.apply_decision(
|
||||
decision,
|
||||
max_hops_per_gate=self.config.max_hops_per_gate,
|
||||
max_same_blocker_retries=self.config.max_same_blocker_retries,
|
||||
)
|
||||
except ConflictError as exc:
|
||||
if str(exc) == "stale_decision_state_version":
|
||||
self.store.mark_callback_state(decision.event_id, "STALE_SUPERSEDED")
|
||||
raise
|
||||
self._enqueue_decision_lifecycle(decision, version)
|
||||
return version
|
||||
|
||||
def _loop(self) -> None:
|
||||
next_summary = 0.0
|
||||
while not self.stop_event.wait(self.config.poll_interval_seconds):
|
||||
try:
|
||||
self.poll_once()
|
||||
if time.monotonic() >= next_summary:
|
||||
self.enqueue_periodic_summaries()
|
||||
next_summary = time.monotonic() + self.config.summary_interval_seconds
|
||||
except Exception:
|
||||
LOG.exception("poll_cycle_failed")
|
||||
|
||||
def poll_once(self) -> None:
|
||||
"""One deterministic reliability cycle.
|
||||
|
||||
This is deliberately callable from a systemd timer or health supervisor
|
||||
in addition to the internal loop.
|
||||
"""
|
||||
self.ingest_file_inboxes()
|
||||
self.poll_git_refs()
|
||||
self.process_new_events()
|
||||
self.process_workspace_timeouts()
|
||||
self.poll_background_responses()
|
||||
self.dispatch_work_units()
|
||||
self.synchronize_openclaw()
|
||||
|
||||
def ingest_file_inboxes(self) -> None:
|
||||
self._ingest_directory(self.config.event_inbox_dir, kind="event")
|
||||
self._ingest_directory(self.config.decision_inbox_dir, kind="decision")
|
||||
|
||||
def _ingest_directory(self, directory: Path, *, kind: str) -> None:
|
||||
pending = directory / "pending"
|
||||
processed = directory / "processed"
|
||||
rejected = directory / "rejected"
|
||||
ensure_shared_directory(pending, group_writable=True)
|
||||
ensure_secure_directory(processed)
|
||||
ensure_secure_directory(rejected)
|
||||
for path in sorted(pending.glob("*.json"))[:128]:
|
||||
try:
|
||||
payload = read_secure_json(path, expected_uid=None)
|
||||
if kind == "event":
|
||||
self.submit_event(payload)
|
||||
else:
|
||||
self.submit_decision(payload)
|
||||
target = processed / path.name
|
||||
except Exception as exc:
|
||||
LOG.warning("file_inbox_rejected path=%s error=%s", path, type(exc).__name__)
|
||||
target = rejected / path.name
|
||||
if target.exists():
|
||||
# Same file name is an idempotency replay. Keep original audit copy.
|
||||
path.unlink()
|
||||
else:
|
||||
path.replace(target)
|
||||
|
||||
def poll_git_refs(self) -> None:
|
||||
"""Poll authoritative local bare refs so missed webhooks cannot stall the loop."""
|
||||
if not self.config.git_poll_enabled or self.config.git_poll_bare_repo is None:
|
||||
return
|
||||
now = time.monotonic()
|
||||
if now < self._next_git_poll:
|
||||
return
|
||||
self._next_git_poll = now + self.config.git_poll_interval_seconds
|
||||
command = [
|
||||
"git",
|
||||
f"--git-dir={self.config.git_poll_bare_repo}",
|
||||
"for-each-ref",
|
||||
"--format=%(refname) %(objectname)",
|
||||
*self.config.git_poll_ref_prefixes,
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(
|
||||
command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=30,
|
||||
env={"PATH": "/usr/bin:/bin", "LANG": "C", "LC_ALL": "C"},
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
LOG.exception("git_ref_poll_failed")
|
||||
return
|
||||
if result.returncode != 0:
|
||||
LOG.warning("git_ref_poll_rc=%s", result.returncode)
|
||||
return
|
||||
refs: dict[str, str] = {}
|
||||
for line in result.stdout.splitlines():
|
||||
refname, sep, oid = line.partition(" ")
|
||||
if not sep or not refname.startswith("refs/") or len(oid) != 40:
|
||||
continue
|
||||
if any(ch not in "0123456789abcdef" for ch in oid):
|
||||
continue
|
||||
refs[refname] = oid
|
||||
initialized, changes = self.store.observe_refs(refs)
|
||||
if not initialized or not changes:
|
||||
return
|
||||
payload_changes = [
|
||||
{"ref": refname, "old_oid": old_oid, "new_oid": new_oid}
|
||||
for refname, old_oid, new_oid in sorted(changes)
|
||||
]
|
||||
canonical = json.dumps(payload_changes, separators=(",", ":"), sort_keys=True)
|
||||
event_id = "gitpoll-" + hashlib.sha256(canonical.encode()).hexdigest()[:40]
|
||||
state = self.store.project_state(self.config.project_key)
|
||||
state_version = 0 if state is None else int(state["state_version"])
|
||||
event = Event.from_dict(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"project_key": self.config.project_key,
|
||||
"contract_commit": self.config.contract_commit,
|
||||
"event_id": event_id,
|
||||
"source_agent": "gitea",
|
||||
"event_type": "git_refs_changed",
|
||||
"work_unit_id": "GIT-REF-POLL",
|
||||
"status": "READY",
|
||||
"state_version": state_version,
|
||||
"implementation_commit": None,
|
||||
"result_commit": None,
|
||||
"blocker": None,
|
||||
"evidence_refs": [],
|
||||
"payload": {"changes": payload_changes},
|
||||
}
|
||||
)
|
||||
try:
|
||||
self.submit_event(event.to_dict())
|
||||
except ConflictError:
|
||||
LOG.info("git_ref_event_already_registered event_id=%s", event_id)
|
||||
|
||||
def process_new_events(self) -> None:
|
||||
rows = self.store.claim_events(
|
||||
owner=self.worker_id,
|
||||
statuses=("RECEIVED", "TRIGGER_FAILED"),
|
||||
limit=1,
|
||||
lease_seconds=self.config.lease_seconds,
|
||||
)
|
||||
for row in rows:
|
||||
event = Event.from_dict(json.loads(row["event_json"]))
|
||||
attempts = int(row["attempts"])
|
||||
try:
|
||||
if self.codex_queue is not None:
|
||||
self.codex_queue.enqueue(event)
|
||||
self.store.update_event_trigger(
|
||||
event.event_id,
|
||||
status="AWAITING_DECISION",
|
||||
channel="codex_local",
|
||||
remote_id=None,
|
||||
callback_state="AWAITING_CALLBACK",
|
||||
callback_deadline_after=self.config.callback_timeout_seconds,
|
||||
)
|
||||
elif self.workspace is not None:
|
||||
try:
|
||||
self.store.update_event_trigger(
|
||||
event.event_id,
|
||||
status="RECEIVED",
|
||||
channel="workspace_agent",
|
||||
remote_id=None,
|
||||
callback_state="SENT",
|
||||
)
|
||||
accepted = self.workspace.trigger(event)
|
||||
except TriggerError as exc:
|
||||
self._retry_or_dead_letter(
|
||||
event,
|
||||
attempts,
|
||||
str(exc),
|
||||
trigger_channel="workspace_agent",
|
||||
trigger_http_status=exc.http_status,
|
||||
trigger_error_category=exc.category,
|
||||
)
|
||||
else:
|
||||
self.store.update_event_trigger(
|
||||
event.event_id,
|
||||
status="AWAITING_DECISION",
|
||||
channel="workspace_agent",
|
||||
remote_id=accepted.remote_id,
|
||||
trigger_http_status=accepted.http_status,
|
||||
callback_state="AWAITING_CALLBACK",
|
||||
callback_deadline_after=self.config.callback_timeout_seconds,
|
||||
)
|
||||
elif self.responses is not None:
|
||||
response_id = self.responses.start(event)
|
||||
self.store.update_event_trigger(
|
||||
event.event_id,
|
||||
status="POLLING_RESPONSE",
|
||||
channel="responses_api",
|
||||
remote_id=response_id,
|
||||
retry_after=self.config.poll_interval_seconds,
|
||||
)
|
||||
else:
|
||||
raise TriggerError("no_model_channel")
|
||||
except TriggerError as exc:
|
||||
self._retry_or_dead_letter(event, attempts, str(exc))
|
||||
|
||||
def process_workspace_timeouts(self) -> None:
|
||||
cutoff = int(time.time())
|
||||
for row in self.store.claim_workspace_timeouts(
|
||||
owner=self.worker_id,
|
||||
older_than=cutoff,
|
||||
limit=1,
|
||||
lease_seconds=self.config.lease_seconds,
|
||||
):
|
||||
event = Event.from_dict(json.loads(row["event_json"]))
|
||||
if self.responses is None:
|
||||
attempts = int(row["callback_attempts"])
|
||||
if attempts + 1 > self.config.max_retries:
|
||||
self.store.update_event_trigger(
|
||||
event.event_id, status="DEAD_LETTER",
|
||||
channel="workspace_agent", remote_id=None,
|
||||
callback_state="DEAD_LETTER",
|
||||
increment_callback_attempt=True,
|
||||
)
|
||||
else:
|
||||
self.store.update_event_trigger(
|
||||
event.event_id, status="AWAITING_DECISION",
|
||||
channel="workspace_agent", remote_id=None,
|
||||
callback_state="AWAITING_CALLBACK",
|
||||
callback_deadline_after=self._backoff(attempts),
|
||||
increment_callback_attempt=True,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
response_id = self.responses.start(event)
|
||||
self.store.update_event_trigger(
|
||||
event.event_id,
|
||||
status="POLLING_RESPONSE",
|
||||
channel="responses_api",
|
||||
remote_id=response_id,
|
||||
retry_after=self.config.poll_interval_seconds,
|
||||
increment_attempt=True,
|
||||
callback_state="AWAITING_CALLBACK",
|
||||
callback_deadline_after=self.config.callback_timeout_seconds,
|
||||
)
|
||||
except TriggerError as exc:
|
||||
self._retry_or_dead_letter(event, int(row["attempts"]), str(exc))
|
||||
|
||||
def poll_background_responses(self) -> None:
|
||||
if self.responses is None:
|
||||
return
|
||||
for row in self.store.claim_response_polls(
|
||||
owner=self.worker_id,
|
||||
limit=1,
|
||||
lease_seconds=self.config.lease_seconds,
|
||||
):
|
||||
event = Event.from_dict(json.loads(row["event_json"]))
|
||||
response_id = row.get("remote_id")
|
||||
if not isinstance(response_id, str):
|
||||
self._retry_or_dead_letter(event, int(row["attempts"]), "missing_response_id")
|
||||
continue
|
||||
try:
|
||||
status, decision = self.responses.poll(response_id)
|
||||
if status in {"queued", "in_progress"}:
|
||||
attempts = int(row["attempts"])
|
||||
if attempts + 1 > self.config.max_retries:
|
||||
self._retry_or_dead_letter(
|
||||
event, attempts, f"response_{status}_retry_limit"
|
||||
)
|
||||
continue
|
||||
self.store.update_event_trigger(
|
||||
event.event_id,
|
||||
status="POLLING_RESPONSE",
|
||||
channel="responses_api",
|
||||
remote_id=response_id,
|
||||
retry_after=self.config.poll_interval_seconds,
|
||||
increment_attempt=True,
|
||||
)
|
||||
continue
|
||||
if decision is None:
|
||||
raise TriggerError("completed_without_decision")
|
||||
self.submit_decision(decision.to_dict())
|
||||
except OrchestratorError as exc:
|
||||
self._retry_or_dead_letter(event, int(row["attempts"]), str(exc))
|
||||
|
||||
def dispatch_work_units(self) -> None:
|
||||
rows = self.store.claim_outbox(
|
||||
owner=self.worker_id,
|
||||
limit=64,
|
||||
lease_seconds=self.config.lease_seconds,
|
||||
)
|
||||
for row in rows:
|
||||
try:
|
||||
outbox_path, queue_path = self.dispatch.dispatch(row)
|
||||
self.store.mark_outbox(row["work_unit_id"], status="DISPATCHED")
|
||||
payload = {
|
||||
"assigned_agent": row["assigned_agent"],
|
||||
"event_id": row["event_id"],
|
||||
"dispatch_outbox": str(outbox_path),
|
||||
"command_queue": str(queue_path),
|
||||
}
|
||||
record = lifecycle_record(
|
||||
contract_commit=self.config.contract_commit,
|
||||
project_key=self._project_for_event(row["event_id"]),
|
||||
kind="TASK_DISPATCHED",
|
||||
work_unit_id=row["work_unit_id"],
|
||||
payload=payload,
|
||||
)
|
||||
self.store.enqueue_lifecycle(
|
||||
sync_id=record["sync_id"],
|
||||
project_key=record["project_key"],
|
||||
work_unit_id=row["work_unit_id"],
|
||||
kind=record["kind"],
|
||||
payload=record,
|
||||
)
|
||||
except Exception:
|
||||
attempts = int(row["attempts"])
|
||||
status = "DEAD_LETTER" if attempts >= self.config.max_retries else "RETRY"
|
||||
self.store.mark_outbox(
|
||||
row["work_unit_id"],
|
||||
status=status,
|
||||
retry_after=self._backoff(attempts),
|
||||
increment_attempt=True,
|
||||
)
|
||||
LOG.exception("dispatch_failed work_unit=%s", row["work_unit_id"])
|
||||
|
||||
def synchronize_openclaw(self) -> None:
|
||||
rows = self.store.claim_lifecycle(
|
||||
owner=self.worker_id,
|
||||
limit=64,
|
||||
lease_seconds=self.config.lease_seconds,
|
||||
)
|
||||
for row in rows:
|
||||
try:
|
||||
payload = json.loads(row["payload_json"])
|
||||
self.synchronizer.emit(payload)
|
||||
self.store.mark_lifecycle(row["sync_id"], status="SENT")
|
||||
except Exception:
|
||||
attempts = int(row["attempts"])
|
||||
status = "DEAD_LETTER" if attempts >= self.config.max_retries else "RETRY"
|
||||
self.store.mark_lifecycle(
|
||||
row["sync_id"],
|
||||
status=status,
|
||||
retry_after=self._backoff(attempts),
|
||||
increment_attempt=True,
|
||||
)
|
||||
LOG.exception("openclaw_sync_failed sync_id=%s", row["sync_id"])
|
||||
|
||||
def enqueue_periodic_summaries(self) -> None:
|
||||
for state in self.store.all_project_states():
|
||||
project_key = state["project_key"]
|
||||
if not self.store.summary_due(project_key, self.config.summary_interval_seconds):
|
||||
continue
|
||||
summary = self.store.project_summary(project_key)
|
||||
if summary is None:
|
||||
continue
|
||||
digest = hashlib.sha256(
|
||||
json.dumps(summary, separators=(",", ":"), sort_keys=True).encode()
|
||||
).hexdigest()
|
||||
record = lifecycle_record(
|
||||
contract_commit=self.config.contract_commit,
|
||||
project_key=project_key,
|
||||
kind="PROJECT_SUMMARY",
|
||||
work_unit_id=None,
|
||||
payload=summary,
|
||||
)
|
||||
self.store.enqueue_lifecycle(
|
||||
sync_id=record["sync_id"],
|
||||
project_key=project_key,
|
||||
work_unit_id=None,
|
||||
kind=record["kind"],
|
||||
payload=record,
|
||||
)
|
||||
self.store.mark_summary(project_key, digest)
|
||||
|
||||
def _enqueue_event_lifecycle(self, event: Event) -> None:
|
||||
if event.event_type == "work_unit_started":
|
||||
kind = "TASK_STARTED"
|
||||
elif event.event_type in {"work_unit_completed", "work_unit_failed", "work_unit_blocked"}:
|
||||
kind = "TASK_ENDED"
|
||||
else:
|
||||
kind = "EVENT_RECEIVED"
|
||||
record = lifecycle_record(
|
||||
contract_commit=self.config.contract_commit,
|
||||
project_key=event.project_key,
|
||||
kind=kind,
|
||||
work_unit_id=event.work_unit_id,
|
||||
payload=event.to_dict(),
|
||||
)
|
||||
self.store.enqueue_lifecycle(
|
||||
sync_id=record["sync_id"],
|
||||
project_key=event.project_key,
|
||||
work_unit_id=event.work_unit_id,
|
||||
kind=kind,
|
||||
payload=record,
|
||||
)
|
||||
|
||||
def _enqueue_decision_lifecycle(self, decision: Decision, version: int) -> None:
|
||||
kind = "PROJECT_COMPLETED" if decision.verdict == "COMPLETE" else "DECISION_APPLIED"
|
||||
record = lifecycle_record(
|
||||
contract_commit=self.config.contract_commit,
|
||||
project_key=decision.project_key,
|
||||
kind=kind,
|
||||
work_unit_id=None,
|
||||
payload={"decision": decision.to_dict(), "state_version": version},
|
||||
)
|
||||
self.store.enqueue_lifecycle(
|
||||
sync_id=record["sync_id"],
|
||||
project_key=decision.project_key,
|
||||
work_unit_id=None,
|
||||
kind=kind,
|
||||
payload=record,
|
||||
)
|
||||
summary = self.store.project_summary(decision.project_key)
|
||||
if summary:
|
||||
summary_record = lifecycle_record(
|
||||
contract_commit=self.config.contract_commit,
|
||||
project_key=decision.project_key,
|
||||
kind="PROJECT_SUMMARY",
|
||||
work_unit_id=None,
|
||||
payload=summary,
|
||||
)
|
||||
self.store.enqueue_lifecycle(
|
||||
sync_id=summary_record["sync_id"],
|
||||
project_key=decision.project_key,
|
||||
work_unit_id=None,
|
||||
kind=summary_record["kind"],
|
||||
payload=summary_record,
|
||||
)
|
||||
|
||||
def _project_for_event(self, event_id: str) -> str:
|
||||
rows = self.store._conn().execute(
|
||||
"SELECT project_key FROM events WHERE event_id=?",
|
||||
(event_id,),
|
||||
).fetchone()
|
||||
if rows is None:
|
||||
raise ConflictError("event_not_found")
|
||||
return str(rows["project_key"])
|
||||
|
||||
def _retry_or_dead_letter(
|
||||
self,
|
||||
event: Event,
|
||||
attempts: int,
|
||||
reason: str,
|
||||
*,
|
||||
trigger_channel: str | None = None,
|
||||
trigger_http_status: int | None = None,
|
||||
trigger_error_category: str | None = None,
|
||||
) -> None:
|
||||
next_attempt = attempts + 1
|
||||
if next_attempt > self.config.max_retries:
|
||||
self.store.update_event_trigger(
|
||||
event.event_id,
|
||||
status="DEAD_LETTER",
|
||||
channel=trigger_channel,
|
||||
remote_id=None,
|
||||
increment_attempt=True,
|
||||
trigger_http_status=trigger_http_status,
|
||||
trigger_error_category=trigger_error_category,
|
||||
)
|
||||
record = lifecycle_record(
|
||||
contract_commit=self.config.contract_commit,
|
||||
project_key=event.project_key,
|
||||
kind="TASK_ENDED",
|
||||
work_unit_id=event.work_unit_id,
|
||||
payload={
|
||||
"status": "FAIL",
|
||||
"reason": reason,
|
||||
"event_id": event.event_id,
|
||||
"attempts": next_attempt,
|
||||
},
|
||||
)
|
||||
self.store.enqueue_lifecycle(
|
||||
sync_id=record["sync_id"],
|
||||
project_key=event.project_key,
|
||||
work_unit_id=event.work_unit_id,
|
||||
kind=record["kind"],
|
||||
payload=record,
|
||||
)
|
||||
return
|
||||
self.store.update_event_trigger(
|
||||
event.event_id,
|
||||
status="TRIGGER_FAILED",
|
||||
channel=trigger_channel,
|
||||
remote_id=None,
|
||||
retry_after=self._backoff(attempts),
|
||||
increment_attempt=True,
|
||||
trigger_http_status=trigger_http_status,
|
||||
trigger_error_category=trigger_error_category,
|
||||
)
|
||||
|
||||
def _backoff(self, attempts: int) -> int:
|
||||
return min(3600, self.config.retry_base_seconds * (2 ** min(attempts, 10)))
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
runtime: Runtime
|
||||
|
||||
def log_message(self, fmt: str, *args: object) -> None:
|
||||
LOG.info("%s %s", self.client_address[0], fmt % args)
|
||||
|
||||
def _reply(self, status: int, payload: dict[str, Any]) -> None:
|
||||
body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _body(self) -> dict[str, Any]:
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
except ValueError as exc:
|
||||
raise ValidationError("invalid_content_length") from exc
|
||||
if length <= 0 or length > 1_048_576:
|
||||
raise ValidationError("invalid_body_size")
|
||||
try:
|
||||
value = json.loads(self.rfile.read(length))
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValidationError("invalid_json") from exc
|
||||
if not isinstance(value, dict):
|
||||
raise ValidationError("json_root_not_object")
|
||||
return value
|
||||
|
||||
def _authenticate(self, expected: str) -> None:
|
||||
header = self.headers.get("Authorization", "")
|
||||
if not header.startswith("Bearer "):
|
||||
raise AuthenticationError("missing_bearer")
|
||||
if not hmac.compare_digest(header[7:], expected):
|
||||
raise AuthenticationError("invalid_bearer")
|
||||
|
||||
def do_GET(self) -> None:
|
||||
path = urlparse(self.path).path.rstrip("/") or "/"
|
||||
try:
|
||||
if path == "/healthz":
|
||||
self._reply(
|
||||
200,
|
||||
{
|
||||
"service": "aios-orchestrator-gateway",
|
||||
"status": "active",
|
||||
"contract_commit": self.runtime.config.contract_commit,
|
||||
"polling": True,
|
||||
},
|
||||
)
|
||||
return
|
||||
if path.startswith("/v1/orchestrator/state/"):
|
||||
self._authenticate(self.runtime.config.inbound_token())
|
||||
key = path.rsplit("/", 1)[-1]
|
||||
state = self.runtime.store.project_summary(key)
|
||||
self._reply(200 if state else 404, state or {"error": "project_not_found"})
|
||||
return
|
||||
if path.startswith("/v1/orchestrator/events/"):
|
||||
self._authenticate(self.runtime.config.inbound_token())
|
||||
event_id = path.rsplit("/", 1)[-1]
|
||||
if not ID_RE.fullmatch(event_id):
|
||||
raise ValidationError("invalid_event_id")
|
||||
status = self.runtime.store.event_status(event_id)
|
||||
self._reply(200 if status else 404, status or {"error": "event_not_found"})
|
||||
return
|
||||
self._reply(404, {"error": "not_found"})
|
||||
except AuthenticationError as exc:
|
||||
self._reply(401, {"error": str(exc)})
|
||||
except OrchestratorError as exc:
|
||||
self._reply(400, {"error": str(exc)})
|
||||
|
||||
def do_POST(self) -> None:
|
||||
path = urlparse(self.path).path.rstrip("/")
|
||||
try:
|
||||
if path == "/v1/orchestrator/events":
|
||||
self._authenticate(self.runtime.config.inbound_token())
|
||||
created, event = self.runtime.submit_event(self._body())
|
||||
self._reply(202 if created else 200, {"created": created, "event": event})
|
||||
return
|
||||
if path == "/v1/orchestrator/decisions":
|
||||
self._authenticate(self.runtime.config.callback_token())
|
||||
version = self.runtime.submit_decision(self._body())
|
||||
self._reply(202, {"accepted": True, "state_version": version})
|
||||
return
|
||||
if path == "/v1/orchestrator/poll":
|
||||
self._authenticate(self.runtime.config.inbound_token())
|
||||
self.runtime.poll_once()
|
||||
self._reply(200, {"polled": True})
|
||||
return
|
||||
self._reply(404, {"error": "not_found"})
|
||||
except AuthenticationError as exc:
|
||||
self._reply(401, {"error": str(exc)})
|
||||
except ConflictError as exc:
|
||||
self._reply(409, {"error": str(exc)})
|
||||
except ValidationError as exc:
|
||||
self._reply(422, {"error": str(exc)})
|
||||
except OrchestratorError as exc:
|
||||
self._reply(400, {"error": str(exc)})
|
||||
except Exception:
|
||||
LOG.exception("request_failed path=%s", path)
|
||||
self._reply(500, {"error": "internal_error"})
|
||||
|
||||
|
||||
def serve(config_path: Path) -> None:
|
||||
config = Config.from_file(config_path)
|
||||
runtime = Runtime(config)
|
||||
Handler.runtime = runtime
|
||||
server = ThreadingHTTPServer((config.host, config.port), Handler)
|
||||
runtime.start()
|
||||
LOG.info(
|
||||
"gateway_started host=%s port=%s contract=%s polling=%ss",
|
||||
config.host,
|
||||
config.port,
|
||||
config.contract_commit,
|
||||
config.poll_interval_seconds,
|
||||
)
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.5)
|
||||
finally:
|
||||
server.server_close()
|
||||
runtime.stop()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user