764 lines
30 KiB
Python
764 lines
30 KiB
Python
from __future__ import annotations
|
|
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timedelta, timezone
|
|
import hashlib
|
|
from dataclasses import asdict
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import json
|
|
from types import SimpleNamespace
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
from aios_chatgpt_bridge_v2_server import GovernedRuntime
|
|
from aios_continuation_executor import ContinuationPermissions, ContinuationRuntimeBinding
|
|
from aios_continuation_lease import WriterLeaseRequestV2
|
|
from aios_gitops_job_controller import GitOpsJobController, JobError, JobStore
|
|
from aios_governed_task_bootstrap import GovernedTaskBootstrap
|
|
|
|
|
|
TASK_ID = "gov-writer-lease-recovery-test"
|
|
BRANCH = "agent/writer-lease-recovery-test"
|
|
BASE = "a" * 40
|
|
JOB_ID_PREFIX = "writer-lease-runtime-lineage"
|
|
|
|
|
|
class LocalAuthority:
|
|
def __init__(self, head: str | None = None) -> None:
|
|
self.head = head
|
|
|
|
def read_ref(self, _ref: str) -> str | None:
|
|
return self.head
|
|
|
|
def commit_exists(self, _oid: str) -> bool:
|
|
return True
|
|
|
|
def ancestry_check(self, _older: str, _newer: str) -> bool:
|
|
return True
|
|
|
|
|
|
def writer_request(task_id: str = TASK_ID, **overrides: object) -> dict[str, object]:
|
|
request: dict[str, object] = {
|
|
"schema": WriterLeaseRequestV2.schema,
|
|
"task_id": task_id,
|
|
"repository": "aios/aios",
|
|
"branch_mode": "CREATE_NEW",
|
|
"base_oid": BASE,
|
|
"expected_branch_oid": BASE,
|
|
"target_branch": BRANCH,
|
|
"generation": 1,
|
|
"lease_seconds": 120,
|
|
}
|
|
request.update(overrides)
|
|
return request
|
|
|
|
|
|
def make_runtime(tmp_path: Path, task_id: str = TASK_ID, *, authority_head: str | None = None) -> GovernedRuntime:
|
|
runtime = GovernedRuntime.__new__(GovernedRuntime)
|
|
runtime.config = SimpleNamespace(repository="aios/aios", state_root=tmp_path)
|
|
runtime.authority = LocalAuthority(authority_head)
|
|
runtime.bindings = {
|
|
task_id: {
|
|
"task_id": task_id,
|
|
"repository": "aios/aios",
|
|
"base_oid": BASE,
|
|
"expected_branch_oid": BASE,
|
|
"target_branch": BRANCH,
|
|
"branch_mode": "CREATE_NEW",
|
|
"generation": 1,
|
|
}
|
|
}
|
|
runtime.runtimes = {}
|
|
runtime.controllers = {}
|
|
runtime.bootstrap = GovernedTaskBootstrap(
|
|
runtime.authority,
|
|
tmp_path,
|
|
remote_name="test",
|
|
token_path=tmp_path / "token",
|
|
ssh_key_path=tmp_path / "key",
|
|
)
|
|
runtime.bindings = {}
|
|
runtime.create_task(
|
|
{
|
|
"task_id": task_id,
|
|
"repository": "aios/aios",
|
|
"base_oid": BASE,
|
|
"expected_branch_oid": BASE,
|
|
"target_branch": BRANCH,
|
|
"risk": "engineering",
|
|
"description": "governed runtime lineage test",
|
|
"idempotency_key": JOB_ID_PREFIX + "-" + task_id,
|
|
}
|
|
)
|
|
job_root = runtime._job_root(task_id)
|
|
runtime.controllers[task_id] = GitOpsJobController(
|
|
JobStore(job_root, payload_root=job_root / "payloads")
|
|
)
|
|
|
|
def build(
|
|
lease: WriterLeaseRequestV2,
|
|
*,
|
|
job_id: str,
|
|
lease_not_before: datetime | None = None,
|
|
lease_expires_at: datetime | None = None,
|
|
) -> tuple[ContinuationRuntimeBinding, GitOpsJobController, object]:
|
|
controller = runtime.controllers.get(lease.task_id)
|
|
if controller is None:
|
|
root = runtime._job_root(lease.task_id)
|
|
controller = GitOpsJobController(JobStore(root, payload_root=root / "payloads"))
|
|
now = datetime.now(timezone.utc)
|
|
return (
|
|
ContinuationRuntimeBinding(
|
|
request=lease,
|
|
job_id=job_id,
|
|
workspace_path=tmp_path / "worktrees" / lease.task_id,
|
|
lease_not_before=lease_not_before or now - timedelta(seconds=1),
|
|
lease_expires_at=lease_expires_at or now + timedelta(seconds=lease.lease_seconds),
|
|
original_receipt_valid=True,
|
|
original_grant_consumed=True,
|
|
permissions=ContinuationPermissions(),
|
|
),
|
|
controller,
|
|
object(),
|
|
)
|
|
|
|
runtime._build_runtime = build
|
|
return runtime
|
|
|
|
|
|
def create_durable_lease(
|
|
runtime: GovernedRuntime,
|
|
*,
|
|
expires_at: str = "2020-01-01T00:00:00+00:00",
|
|
) -> GitOpsJobController:
|
|
root = runtime._job_root(TASK_ID)
|
|
controller = GitOpsJobController(JobStore(root, payload_root=root / "payloads"))
|
|
receipt = SimpleNamespace(job_id=runtime.bindings[TASK_ID]["job_id"])
|
|
controller.lease_job(
|
|
receipt.job_id,
|
|
lease_id="writer-lease-recovery",
|
|
lineage_anchor_oid=BASE,
|
|
current_head_oid=BASE,
|
|
generation=1,
|
|
lease_expires_at=expires_at,
|
|
expected_revision=0,
|
|
)
|
|
runtime.controllers[TASK_ID] = controller
|
|
return controller
|
|
|
|
|
|
def durable_row(runtime: GovernedRuntime) -> tuple[str, int]:
|
|
database = runtime._job_root(TASK_ID) / "state.sqlite3"
|
|
with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection:
|
|
row = connection.execute(
|
|
"SELECT lease_expires_at, revision FROM branch_leases WHERE lease_id=?",
|
|
("writer-lease-recovery",),
|
|
).fetchone()
|
|
assert row is not None
|
|
return row
|
|
|
|
|
|
def test_first_bootstrap_refreshes_only_runtime_without_durable_lease(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
|
|
result = runtime.bootstrap_writer_lease(writer_request())
|
|
|
|
assert result["bootstrapped"] is True
|
|
assert result["runtime_rehydrated"] is False
|
|
assert result["durable_branch_lease_present"] is False
|
|
assert result["idempotent"] is False
|
|
database = runtime._job_root(TASK_ID) / "state.sqlite3"
|
|
with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection:
|
|
assert connection.execute("SELECT COUNT(*) FROM branch_leases").fetchone()[0] == 0
|
|
|
|
|
|
def test_duplicate_bootstrap_is_idempotent_without_consuming_generation(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
|
|
first = runtime.bootstrap_writer_lease(writer_request())
|
|
second = runtime.bootstrap_writer_lease(writer_request())
|
|
|
|
assert first["bootstrapped"] is True
|
|
assert first["generation"] == 1
|
|
assert second["bootstrapped"] is False
|
|
assert second["idempotent"] is True
|
|
assert second["generation"] == 1
|
|
database = runtime._job_root(TASK_ID) / "state.sqlite3"
|
|
with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection:
|
|
assert connection.execute("SELECT COUNT(*) FROM branch_leases").fetchone()[0] == 0
|
|
|
|
|
|
def test_commit_after_bootstrap_reuses_the_exact_runtime_binding(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
|
|
class RecordingWorkspace:
|
|
|
|
def __init__(self) -> None:
|
|
self.files: dict[str, bytes] = {}
|
|
|
|
def materialize_files(self, files: dict[str, bytes]) -> None:
|
|
self.files = dict(files)
|
|
|
|
class RecordingExecutor:
|
|
|
|
def __init__(self) -> None:
|
|
self.workspace = RecordingWorkspace()
|
|
self.binding = None
|
|
self.commit = None
|
|
|
|
def execute(self, binding: object, commit: object) -> dict[str, object]:
|
|
self.binding = binding
|
|
self.commit = commit
|
|
return {'task_id': binding.request.task_id, 'generation': binding.request.generation, 'files': sorted(self.workspace.files)}
|
|
executor = RecordingExecutor()
|
|
original_build = runtime._build_runtime
|
|
|
|
def build(lease: WriterLeaseRequestV2, *, job_id: str, lease_not_before: datetime | None=None, lease_expires_at: datetime | None=None) -> tuple[ContinuationRuntimeBinding, GitOpsJobController, object]:
|
|
binding, controller, _ = original_build(lease, job_id=job_id, lease_not_before=lease_not_before, lease_expires_at=lease_expires_at)
|
|
return (binding, controller, executor)
|
|
runtime._build_runtime = build
|
|
runtime.bootstrap_writer_lease(writer_request())
|
|
lease_response = runtime.acquire_lease(writer_request())
|
|
binding, controller, executor = runtime.runtimes[TASK_ID, 1]
|
|
assert lease_response['job_id'] == binding.job_id
|
|
assert lease_response['durable_branch_lease_present'] is True
|
|
_original_execute = executor.execute
|
|
|
|
def _execute_with_governed_commit_contract(*args, **kwargs):
|
|
_before_complete = kwargs.pop('before_complete', None)
|
|
_legacy = dict(_original_execute(*args, **kwargs))
|
|
_result = {**_legacy, 'job_id': binding.job_id, 'state': 'COMPLETED', 'side_effect_state': 'REMOTE_VERIFIED', 'receipt': {'authority_oid': 'b' * 40, 'git_remote_oid': 'b' * 40, 'api_branch_oid': 'b' * 40, 'commit_visible': True, 'remote_verified': True, 'force_push': False}}
|
|
if _before_complete is not None:
|
|
_before_complete('b' * 40)
|
|
return _result
|
|
executor.execute = _execute_with_governed_commit_contract
|
|
_bound_job_id = binding.job_id
|
|
result = runtime.commit_files({'task_id': TASK_ID, 'repository': 'aios/aios', 'generation': 1, 'target_branch': BRANCH, 'expected_branch_oid': BASE, 'idempotency_key': 'commit-after-bootstrap-v1', 'path': 'docs/recovery.txt', 'content_base64': 'YQ==', 'message': 'test commit after bootstrap'})
|
|
assert result['task_id'] == TASK_ID
|
|
assert result['generation'] == 1
|
|
assert result['files'] == ['docs/recovery.txt']
|
|
assert executor.binding.request.task_id == TASK_ID
|
|
assert executor.binding.request.generation == 1
|
|
database = runtime._job_root(TASK_ID) / 'state.sqlite3'
|
|
with sqlite3.connect(f'file:{database}?mode=ro', uri=True) as connection:
|
|
assert connection.execute('SELECT COUNT(*) FROM branch_leases').fetchone()[0] == 1
|
|
assert connection.execute('PRAGMA foreign_key_check(branch_leases)').fetchall() == []
|
|
assert result['commit_oid'] == 'b' * 40
|
|
assert result['commit_visible'] is True
|
|
assert result['branch_mode'] == 'CONTINUE_EXISTING'
|
|
assert result['target_branch'] == BRANCH
|
|
assert int(result['generation']) == 1
|
|
with controller.store._connect() as connection:
|
|
_persisted = connection.execute('SELECT current_head_oid,revision FROM branch_leases WHERE lease_id=?', ('lease-' + str(_bound_job_id),)).fetchone()
|
|
assert _persisted is not None
|
|
assert _persisted[0] == 'b' * 40
|
|
assert int(_persisted[1]) == 1
|
|
fresh_runtime = make_runtime(tmp_path, authority_head='b' * 40)
|
|
fresh_response = fresh_runtime.bootstrap_writer_lease(writer_request(branch_mode='CONTINUE_EXISTING', expected_branch_oid='b' * 40))
|
|
fresh_binding, _, _ = fresh_runtime.runtimes[TASK_ID, 1]
|
|
assert fresh_response['job_id'] == str(_bound_job_id)
|
|
assert fresh_response['runtime_rehydrated'] is True
|
|
assert fresh_binding.job_id == str(_bound_job_id)
|
|
assert fresh_binding.request.expected_branch_oid == 'b' * 40
|
|
with fresh_runtime.controllers[TASK_ID].store._connect() as connection:
|
|
_fresh_lease = connection.execute('SELECT job_id,current_head_oid,revision FROM branch_leases WHERE lease_id=?', ('lease-' + str(_bound_job_id),)).fetchone()
|
|
assert tuple(_fresh_lease) == (str(_bound_job_id), 'b' * 40, 1)
|
|
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("field", "value", "error"),
|
|
(
|
|
("repository", "other/repository", "LEASE_BINDING_CONFLICT"),
|
|
("base_oid", "b" * 40, "LEASE_BINDING_CONFLICT"),
|
|
("expected_branch_oid", "b" * 40, "LEASE_BINDING_CONFLICT"),
|
|
("target_branch", "agent/wrong-target", "LEASE_BINDING_CONFLICT"),
|
|
("generation", 2, "GENERATION_BINDING_CONFLICT"),
|
|
),
|
|
)
|
|
def test_bootstrap_rejects_any_nonmatching_binding_field(
|
|
tmp_path: Path, field: str, value: object, error: str
|
|
) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
|
|
with pytest.raises(RuntimeError, match=error):
|
|
runtime.bootstrap_writer_lease(writer_request(**{field: value}))
|
|
|
|
|
|
def test_bootstrap_rejects_unknown_task_ownership(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
|
|
with pytest.raises(RuntimeError, match="TASK_OWNERSHIP_INVALID"):
|
|
runtime.bootstrap_writer_lease(writer_request(task_id="unknown-task"))
|
|
|
|
|
|
def test_renew_missing_durable_lease_is_not_bootstrap(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
|
|
with pytest.raises(RuntimeError, match="LEASE_NOT_FOUND"):
|
|
runtime.renew_writer_lease(writer_request())
|
|
|
|
|
|
def test_renew_is_generation_scoped_before_durable_lookup(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
create_durable_lease(runtime)
|
|
|
|
with pytest.raises(RuntimeError, match="GENERATION_BINDING_CONFLICT"):
|
|
runtime.renew_writer_lease(writer_request(generation=2))
|
|
assert durable_row(runtime) == ("2020-01-01T00:00:00+00:00", 0)
|
|
|
|
|
|
def test_restart_rehydrates_only_from_matching_durable_lease(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
create_durable_lease(runtime)
|
|
runtime.runtimes.clear()
|
|
|
|
result = runtime.renew_writer_lease(writer_request())
|
|
|
|
expires_at, revision = durable_row(runtime)
|
|
assert result["recovered"] is True
|
|
assert result["runtime_rehydrated"] is True
|
|
assert result["durable_branch_lease_present"] is True
|
|
assert revision == 1
|
|
assert datetime.fromisoformat(expires_at) > datetime.now(timezone.utc)
|
|
|
|
|
|
def test_duplicate_renewal_is_idempotent_after_first_durable_recovery(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
create_durable_lease(runtime)
|
|
|
|
first = runtime.renew_writer_lease(writer_request())
|
|
second = runtime.renew_writer_lease(writer_request())
|
|
|
|
assert first["recovered"] is True
|
|
assert first["idempotent"] is False
|
|
assert second["recovered"] is False
|
|
assert second["idempotent"] is True
|
|
assert durable_row(runtime)[1] == 1
|
|
|
|
|
|
def test_update_lease_keeps_expiry_without_optional_argument_and_keeps_cas(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
controller = create_durable_lease(runtime)
|
|
|
|
controller.update_lease(
|
|
"writer-lease-recovery",
|
|
expected_revision=0,
|
|
current_head_oid=BASE,
|
|
generation=1,
|
|
)
|
|
assert durable_row(runtime) == ("2020-01-01T00:00:00+00:00", 1)
|
|
with pytest.raises(JobError, match="LEASE_REVISION_CAS_CONFLICT"):
|
|
controller.update_lease(
|
|
"writer-lease-recovery",
|
|
expected_revision=0,
|
|
current_head_oid=BASE,
|
|
generation=1,
|
|
)
|
|
|
|
|
|
def test_two_runtime_connections_resolve_renewal_cas_race_idempotently(tmp_path: Path) -> None:
|
|
first = make_runtime(tmp_path)
|
|
first_controller = create_durable_lease(first)
|
|
second = make_runtime(tmp_path)
|
|
second_controller = GitOpsJobController(
|
|
JobStore(second._job_root(TASK_ID), payload_root=second._job_root(TASK_ID) / "payloads")
|
|
)
|
|
second.controllers[TASK_ID] = second_controller
|
|
gate = threading.Barrier(2)
|
|
|
|
def gate_update(controller: GitOpsJobController) -> None:
|
|
original = controller.update_lease
|
|
|
|
def update(*args: object, **kwargs: object) -> None:
|
|
gate.wait(timeout=5)
|
|
original(*args, **kwargs)
|
|
|
|
controller.update_lease = update
|
|
|
|
gate_update(first_controller)
|
|
gate_update(second_controller)
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
results = list(pool.map(lambda runtime: runtime.renew_writer_lease(writer_request()), (first, second)))
|
|
|
|
assert sorted((item["recovered"], item["idempotent"]) for item in results) == [
|
|
(False, True),
|
|
(True, False),
|
|
]
|
|
assert durable_row(first)[1] == 1
|
|
|
|
|
|
def test_bootstrap_preserves_exact_governed_job_lineage(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
expected_job_id = runtime.bindings[TASK_ID]["job_id"]
|
|
|
|
response = runtime.bootstrap_writer_lease(writer_request())
|
|
binding, _, _ = runtime.runtimes[(TASK_ID, 1)]
|
|
|
|
assert response["job_id"] == expected_job_id
|
|
assert binding.job_id == expected_job_id
|
|
assert binding.request.job_id == expected_job_id
|
|
|
|
|
|
def test_restart_rehydrate_preserves_exact_governed_job_lineage(tmp_path: Path) -> None:
|
|
first = make_runtime(tmp_path)
|
|
expected_job_id = first.bindings[TASK_ID]["job_id"]
|
|
first.acquire_lease(writer_request())
|
|
|
|
runtime = make_runtime(tmp_path)
|
|
assert runtime is not first
|
|
assert runtime.runtimes == {}
|
|
response = runtime.bootstrap_writer_lease(writer_request())
|
|
binding, _, _ = runtime.runtimes[(TASK_ID, 1)]
|
|
|
|
assert response["job_id"] == expected_job_id
|
|
assert binding.job_id == expected_job_id
|
|
assert binding.request.job_id == expected_job_id
|
|
|
|
|
|
def test_wrong_governed_job_id_is_denied_before_runtime_or_lease_side_effect(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
|
|
with pytest.raises(RuntimeError, match="LEASE_BINDING_CONFLICT"):
|
|
runtime.bootstrap_writer_lease(writer_request(job_id="job-wrong-lineage"))
|
|
|
|
assert runtime.runtimes == {}
|
|
database = runtime._job_root(TASK_ID) / "state.sqlite3"
|
|
with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection:
|
|
assert connection.execute("SELECT COUNT(*) FROM branch_leases").fetchone()[0] == 0
|
|
assert connection.execute("PRAGMA foreign_key_check(branch_leases)").fetchall() == []
|
|
|
|
|
|
def test_acquire_creates_exact_governed_writer_lease_with_fk_and_cas(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
expected_job_id = runtime.bindings[TASK_ID]["job_id"]
|
|
|
|
response = runtime.acquire_lease(writer_request())
|
|
|
|
assert response["job_id"] == expected_job_id
|
|
assert response["durable_branch_lease_present"] is True
|
|
assert response["idempotent"] is False
|
|
database = runtime._job_root(TASK_ID) / "state.sqlite3"
|
|
with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection:
|
|
foreign_keys = connection.execute(
|
|
"PRAGMA foreign_key_list(branch_leases)"
|
|
).fetchall()
|
|
row = connection.execute(
|
|
"SELECT job_id,branch,lineage_anchor_oid,current_head_oid,generation,state,revision "
|
|
"FROM branch_leases"
|
|
).fetchone()
|
|
job = connection.execute(
|
|
"SELECT state,side_effect_state,revision FROM jobs WHERE job_id=?",
|
|
(expected_job_id,),
|
|
).fetchone()
|
|
assert connection.execute("PRAGMA foreign_key_check(branch_leases)").fetchall() == []
|
|
assert any(
|
|
fk[2] == "jobs" and fk[3] == "job_id" and fk[4] == "job_id"
|
|
for fk in foreign_keys
|
|
)
|
|
assert row == (expected_job_id, BRANCH, BASE, BASE, 1, "ACTIVE", 0)
|
|
assert job == ("LEASED", "LEASED", 2)
|
|
|
|
|
|
def test_duplicate_acquire_observes_same_durable_writer_lease(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
expected_job_id = runtime.bindings[TASK_ID]["job_id"]
|
|
|
|
first = runtime.acquire_lease(writer_request())
|
|
second = runtime.acquire_lease(writer_request())
|
|
|
|
assert first["job_id"] == expected_job_id
|
|
assert second["job_id"] == expected_job_id
|
|
assert second["idempotent"] is True
|
|
database = runtime._job_root(TASK_ID) / "state.sqlite3"
|
|
with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection:
|
|
leases = connection.execute(
|
|
"SELECT lease_id,job_id,revision FROM branch_leases"
|
|
).fetchall()
|
|
job = connection.execute(
|
|
"SELECT state,revision FROM jobs WHERE job_id=?",
|
|
(expected_job_id,),
|
|
).fetchone()
|
|
assert leases == [("lease-" + expected_job_id, expected_job_id, 0)]
|
|
assert job == ("LEASED", 2)
|
|
|
|
|
|
def test_wrong_job_acquire_is_denied_before_authorize_or_lease(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
expected_job_id = runtime.bindings[TASK_ID]["job_id"]
|
|
|
|
with pytest.raises(RuntimeError, match="LEASE_BINDING_CONFLICT"):
|
|
runtime.acquire_lease(writer_request(job_id="job-wrong-lineage"))
|
|
|
|
database = runtime._job_root(TASK_ID) / "state.sqlite3"
|
|
with sqlite3.connect(f"file:{database}?mode=ro", uri=True) as connection:
|
|
job = connection.execute(
|
|
"SELECT state,side_effect_state,revision FROM jobs WHERE job_id=?",
|
|
(expected_job_id,),
|
|
).fetchone()
|
|
lease_count = connection.execute("SELECT COUNT(*) FROM branch_leases").fetchone()[0]
|
|
assert job == ("REQUESTED", "NONE", 0)
|
|
assert lease_count == 0
|
|
|
|
|
|
def test_omitted_request_job_id_resolves_the_exact_durable_parent(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
expected_job_id = runtime.bindings[TASK_ID]["job_id"]
|
|
raw = dict(writer_request())
|
|
|
|
response = runtime.acquire_lease(raw)
|
|
binding, _, _ = runtime.runtimes[(TASK_ID, 1)]
|
|
|
|
assert response["job_id"] == expected_job_id
|
|
assert binding.job_id == expected_job_id
|
|
assert binding.request.job_id == expected_job_id
|
|
|
|
|
|
def test_durable_job_expected_head_mismatch_is_denied(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
expected_job_id = runtime.bindings[TASK_ID]["job_id"]
|
|
controller = runtime.controllers[TASK_ID]
|
|
with controller.store._connect() as connection:
|
|
row = connection.execute(
|
|
"SELECT request_json FROM jobs WHERE job_id=?", (expected_job_id,)
|
|
).fetchone()
|
|
request = json.loads(row[0])
|
|
request["expected_branch_oid"] = "c" * 40
|
|
request["binding"]["expected_branch_oid"] = "c" * 40
|
|
connection.execute(
|
|
"UPDATE jobs SET request_json=? WHERE job_id=?",
|
|
(json.dumps(request, sort_keys=True, separators=(",", ":")), expected_job_id),
|
|
)
|
|
connection.commit()
|
|
|
|
with pytest.raises(RuntimeError, match="RUNTIME_GOVERNED_JOB_UNRESOLVED"):
|
|
runtime.bootstrap_writer_lease(writer_request())
|
|
|
|
assert runtime.runtimes == {}
|
|
|
|
|
|
def _commit_ready_runtime(tmp_path: Path):
|
|
runtime = make_runtime(tmp_path)
|
|
runtime.bootstrap_writer_lease(writer_request())
|
|
binding, controller, _ = runtime.runtimes[(TASK_ID, 1)]
|
|
|
|
class Workspace:
|
|
def materialize_files(self, _files):
|
|
return None
|
|
|
|
class Executor:
|
|
def __init__(self):
|
|
self.workspace = Workspace()
|
|
self.calls = 0
|
|
|
|
def execute(self, _binding, _commit, *, before_complete=None):
|
|
self.calls += 1
|
|
result = {
|
|
"job_id": binding.job_id,
|
|
"state": "COMPLETED",
|
|
"side_effect_state": "REMOTE_VERIFIED",
|
|
"receipt": {
|
|
"authority_oid": "b" * 40,
|
|
"git_remote_oid": "b" * 40,
|
|
"api_branch_oid": "b" * 40,
|
|
"commit_visible": True,
|
|
"remote_verified": True,
|
|
"force_push": False,
|
|
},
|
|
}
|
|
if before_complete is not None:
|
|
before_complete("b" * 40)
|
|
return result
|
|
|
|
executor = Executor()
|
|
runtime.runtimes[(TASK_ID, 1)] = (binding, controller, executor)
|
|
runtime.acquire_lease(writer_request())
|
|
return runtime, binding, controller, executor
|
|
|
|
|
|
def _commit_request():
|
|
return {
|
|
"task_id": TASK_ID,
|
|
"repository": "aios/aios",
|
|
"generation": 1,
|
|
"target_branch": BRANCH,
|
|
"expected_branch_oid": BASE,
|
|
"idempotency_key": "commit-cas-test-v1",
|
|
"path": "docs/cas.txt",
|
|
"content_base64": "YQ==",
|
|
"message": "test commit cas",
|
|
}
|
|
|
|
|
|
def test_commit_lease_revision_cas_conflict_is_preserved(tmp_path: Path) -> None:
|
|
runtime, _binding, controller, _executor = _commit_ready_runtime(tmp_path)
|
|
|
|
def conflict(*_args, **_kwargs):
|
|
raise JobError("LEASE_REVISION_CAS_CONFLICT")
|
|
|
|
controller.update_lease = conflict
|
|
with pytest.raises(RuntimeError, match="LEASE_CAS_CONFLICT"):
|
|
runtime.commit_files(_commit_request())
|
|
|
|
|
|
def test_commit_divergent_durable_head_fails_before_executor(tmp_path: Path) -> None:
|
|
runtime, binding, controller, executor = _commit_ready_runtime(tmp_path)
|
|
controller.update_lease(
|
|
"lease-" + binding.job_id,
|
|
expected_revision=0,
|
|
current_head_oid="c" * 40,
|
|
generation=1,
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="LEASE_HEAD_RECONCILIATION_REQUIRED"):
|
|
runtime.commit_files(_commit_request())
|
|
assert executor.calls == 0
|
|
|
|
|
|
def test_remote_pending_never_advances_durable_head(tmp_path: Path) -> None:
|
|
runtime, binding, controller, executor = _commit_ready_runtime(tmp_path)
|
|
|
|
def pending(_binding, _commit, **_kwargs):
|
|
executor.calls += 1
|
|
return {
|
|
"job_id": binding.job_id,
|
|
"state": "RECONCILIATION_REQUIRED",
|
|
"side_effect_state": "REMOTE_PUSH_PENDING",
|
|
"receipt": {
|
|
"authority_oid": "b" * 40,
|
|
"desired_remote_oid": "b" * 40,
|
|
"remote_verified": False,
|
|
},
|
|
}
|
|
|
|
executor.execute = pending
|
|
with pytest.raises(RuntimeError, match="RECONCILIATION_REQUIRED"):
|
|
runtime.commit_files(_commit_request())
|
|
with controller.store._connect() as connection:
|
|
row = connection.execute(
|
|
"SELECT current_head_oid,revision FROM branch_leases WHERE lease_id=?",
|
|
("lease-" + binding.job_id,),
|
|
).fetchone()
|
|
assert tuple(row) == (BASE, 0)
|
|
|
|
|
|
def test_retry_accepts_exact_head_already_advanced_at_remote_verified_boundary(tmp_path: Path) -> None:
|
|
runtime, binding, controller, executor = _commit_ready_runtime(tmp_path)
|
|
commit_oid = "b" * 40
|
|
controller.update_lease(
|
|
"lease-" + binding.job_id,
|
|
expected_revision=0,
|
|
current_head_oid=commit_oid,
|
|
generation=1,
|
|
)
|
|
receipt = {
|
|
"authority_oid": commit_oid,
|
|
"git_remote_oid": commit_oid,
|
|
"api_branch_oid": commit_oid,
|
|
"commit_visible": True,
|
|
"remote_verified": True,
|
|
"force_push": False,
|
|
}
|
|
with controller.store._connect() as connection:
|
|
connection.execute(
|
|
"UPDATE jobs SET state='REMOTE_VERIFIED',side_effect_state='REMOTE_VERIFIED',"
|
|
"receipt_json=? WHERE job_id=?",
|
|
(json.dumps(receipt, sort_keys=True, separators=(",", ":")), binding.job_id),
|
|
)
|
|
connection.commit()
|
|
|
|
result = runtime.commit_files(_commit_request())
|
|
|
|
assert result["commit_oid"] == commit_oid
|
|
assert executor.calls == 1
|
|
with controller.store._connect() as connection:
|
|
row = connection.execute(
|
|
"SELECT current_head_oid,revision FROM branch_leases WHERE lease_id=?",
|
|
("lease-" + binding.job_id,),
|
|
).fetchone()
|
|
assert tuple(row) == (commit_oid, 1)
|
|
|
|
|
|
def test_completed_response_with_invalid_oid_uses_canonical_oid_error(tmp_path: Path) -> None:
|
|
runtime, binding, controller, executor = _commit_ready_runtime(tmp_path)
|
|
|
|
def invalid_oid(_binding, _commit, **_kwargs):
|
|
executor.calls += 1
|
|
return {
|
|
"job_id": binding.job_id,
|
|
"state": "COMPLETED",
|
|
"side_effect_state": "REMOTE_VERIFIED",
|
|
"receipt": {
|
|
"authority_oid": "not-an-oid",
|
|
"commit_visible": True,
|
|
"remote_verified": True,
|
|
},
|
|
}
|
|
|
|
executor.execute = invalid_oid
|
|
with pytest.raises(RuntimeError, match="GOVERNED_COMMIT_OID_INVALID"):
|
|
runtime.commit_files(_commit_request())
|
|
with controller.store._connect() as connection:
|
|
row = connection.execute(
|
|
"SELECT current_head_oid,revision FROM branch_leases WHERE lease_id=?",
|
|
("lease-" + binding.job_id,),
|
|
).fetchone()
|
|
assert tuple(row) == (BASE, 0)
|
|
|
|
|
|
def test_acquire_renews_an_expired_matching_durable_lease_with_revision_cas(tmp_path: Path) -> None:
|
|
runtime = make_runtime(tmp_path)
|
|
create_durable_lease(runtime)
|
|
runtime.runtimes.clear()
|
|
|
|
result = runtime.acquire_lease(writer_request())
|
|
|
|
expires_at, revision = durable_row(runtime)
|
|
assert result["recovered"] is True
|
|
assert result["runtime_rehydrated"] is False
|
|
assert result["durable_branch_lease_present"] is True
|
|
assert revision == 1
|
|
assert datetime.fromisoformat(expires_at) > datetime.now(timezone.utc)
|
|
|
|
|
|
def test_authority_recovers_only_clean_detached_single_attempt_worktree(tmp_path: Path) -> None:
|
|
import subprocess
|
|
from aios_authority_git_executor import AuthorityGitExecutor
|
|
|
|
bare = tmp_path / "authority.git"
|
|
seed = tmp_path / "seed"
|
|
subprocess.run(["git", "init", "--bare", str(bare)], check=True, capture_output=True)
|
|
subprocess.run(["git", "init", str(seed)], check=True, capture_output=True)
|
|
(seed / "base.txt").write_text("base\n", encoding="utf-8")
|
|
subprocess.run(["git", "-C", str(seed), "add", "base.txt"], check=True)
|
|
subprocess.run(
|
|
["git", "-C", str(seed), "-c", "user.name=test", "-c", "user.email=test@example.invalid",
|
|
"commit", "-m", "base"], check=True, capture_output=True,
|
|
)
|
|
base = subprocess.check_output(["git", "-C", str(seed), "rev-parse", "HEAD"], text=True).strip()
|
|
subprocess.run(["git", "-C", str(seed), "push", str(bare), "HEAD:refs/heads/main"], check=True, capture_output=True)
|
|
root = tmp_path / "worktrees"
|
|
authority = AuthorityGitExecutor(bare, root)
|
|
worktree = authority.create_detached_worktree(base, "task")
|
|
(worktree / "attempt.txt").write_text("attempt\n", encoding="utf-8")
|
|
subprocess.run(["git", "-C", str(worktree), "add", "attempt.txt"], check=True)
|
|
subprocess.run(
|
|
["git", "-C", str(worktree), "-c", "user.name=test", "-c", "user.email=test@example.invalid",
|
|
"commit", "-m", "failed attempt"], check=True, capture_output=True,
|
|
)
|
|
stale = subprocess.check_output(["git", "-C", str(worktree), "rev-parse", "HEAD"], text=True).strip()
|
|
|
|
recovered = authority.recover_failed_detached_worktree_exact(
|
|
path=worktree,
|
|
expected_oid=base,
|
|
stale_oid=stale,
|
|
target_ref="refs/heads/agent/recovery-test",
|
|
)
|
|
|
|
assert recovered == stale
|
|
assert subprocess.check_output(["git", "-C", str(worktree), "rev-parse", "HEAD"], text=True).strip() == base
|
|
assert subprocess.run(["git", "-C", str(worktree), "symbolic-ref", "-q", "HEAD"]).returncode == 1
|
|
assert subprocess.check_output(["git", "-C", str(worktree), "status", "--porcelain"], text=True) == ""
|