596 lines
29 KiB
Python
596 lines
29 KiB
Python
"""SQLite-backed durable GitOps Job Controller.
|
|
|
|
The controller owns durable intent, idempotency, lease metadata, state
|
|
transitions, receipts, and payload references. A worker owns Git side effects
|
|
behind the campaign capability and branch-CAS boundaries.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from enum import Enum
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import tempfile
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Callable, Mapping
|
|
|
|
|
|
class JobError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class ServiceUnavailable(JobError):
|
|
pass
|
|
|
|
|
|
class BindingConflict(JobError):
|
|
pass
|
|
|
|
|
|
class JobState(str, Enum):
|
|
REQUESTED = "REQUESTED"
|
|
AUTHORIZED = "AUTHORIZED"
|
|
LEASED = "LEASED"
|
|
WORKTREE_READY = "WORKTREE_READY"
|
|
EXECUTING = "EXECUTING"
|
|
LOCAL_VERIFIED = "LOCAL_VERIFIED"
|
|
REMOTE_SYNCING = "REMOTE_SYNCING"
|
|
REMOTE_VERIFIED = "REMOTE_VERIFIED"
|
|
COMPLETED = "COMPLETED"
|
|
RECOVERABLE_FAILURE = "RECOVERABLE_FAILURE"
|
|
RECONCILING = "RECONCILING"
|
|
RECONCILIATION_REQUIRED = "RECONCILIATION_REQUIRED"
|
|
MANUAL_OWNER_REVIEW_REQUIRED = "MANUAL_OWNER_REVIEW_REQUIRED"
|
|
|
|
|
|
_ORDER = {
|
|
JobState.REQUESTED: 0,
|
|
JobState.AUTHORIZED: 1,
|
|
JobState.LEASED: 2,
|
|
JobState.WORKTREE_READY: 3,
|
|
JobState.EXECUTING: 4,
|
|
JobState.LOCAL_VERIFIED: 5,
|
|
JobState.REMOTE_SYNCING: 6,
|
|
JobState.REMOTE_VERIFIED: 7,
|
|
JobState.COMPLETED: 8,
|
|
}
|
|
_RECOVERABLE = {JobState.RECOVERABLE_FAILURE, JobState.RECONCILIATION_REQUIRED}
|
|
_TERMINAL = {JobState.COMPLETED, JobState.MANUAL_OWNER_REVIEW_REQUIRED}
|
|
|
|
|
|
def utc_now() -> str:
|
|
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def canonical_json(value: Any) -> bytes:
|
|
return (
|
|
json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n"
|
|
).encode("utf-8")
|
|
|
|
|
|
def sha256_json(value: Any) -> str:
|
|
return hashlib.sha256(canonical_json(value)).hexdigest()
|
|
|
|
|
|
class PayloadStore:
|
|
"""Content-addressed filesystem payloads; raw content never enters SQLite."""
|
|
|
|
def __init__(self, root: str | os.PathLike[str]) -> None:
|
|
self.root = Path(root)
|
|
if not self.root.is_absolute() or self.root.is_symlink():
|
|
raise JobError("PAYLOAD_ROOT_INVALID")
|
|
self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
os.chmod(self.root, 0o700)
|
|
|
|
def put(self, content: bytes, expected_digest: str) -> tuple[str, int]:
|
|
digest = hashlib.sha256(content).hexdigest()
|
|
if digest != expected_digest:
|
|
raise JobError("PAYLOAD_DIGEST_MISMATCH")
|
|
if len(expected_digest) != 64 or any(ch not in "0123456789abcdef" for ch in expected_digest):
|
|
raise JobError("PAYLOAD_DIGEST_INVALID")
|
|
target = self.root / expected_digest
|
|
if target.is_symlink():
|
|
raise JobError("PAYLOAD_PATH_SYMLINK")
|
|
if target.is_file():
|
|
if target.stat().st_size != len(content) or hashlib.sha256(target.read_bytes()).hexdigest() != digest:
|
|
raise JobError("PAYLOAD_EXISTING_MISMATCH")
|
|
return f"sha256/{digest}", len(content)
|
|
fd, temporary = tempfile.mkstemp(prefix=".payload-", dir=self.root)
|
|
try:
|
|
os.fchmod(fd, 0o600)
|
|
with os.fdopen(fd, "wb") as stream:
|
|
stream.write(content)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(temporary, target)
|
|
os.chmod(target, 0o600)
|
|
finally:
|
|
if os.path.exists(temporary):
|
|
os.unlink(temporary)
|
|
return f"sha256/{digest}", len(content)
|
|
|
|
def read_verified(self, reference: str, expected_digest: str, expected_size: int) -> bytes:
|
|
if reference != f"sha256/{expected_digest}":
|
|
raise JobError("PAYLOAD_REFERENCE_MISMATCH")
|
|
path = self.root / expected_digest
|
|
if path.is_symlink() or not path.is_file():
|
|
raise JobError("PAYLOAD_MISSING")
|
|
content = path.read_bytes()
|
|
if len(content) != expected_size or hashlib.sha256(content).hexdigest() != expected_digest:
|
|
raise JobError("PAYLOAD_VERIFY_FAILED")
|
|
return content
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class JobReceipt:
|
|
job_id: str
|
|
idempotency_key: str
|
|
state: JobState
|
|
side_effect_state: str
|
|
http_status: int
|
|
revision: int
|
|
receipt: Mapping[str, Any] | None = None
|
|
|
|
def as_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"job_id": self.job_id,
|
|
"idempotency_key": self.idempotency_key,
|
|
"state": self.state.value,
|
|
"side_effect_state": self.side_effect_state,
|
|
"http_status": self.http_status,
|
|
"revision": self.revision,
|
|
"receipt": dict(self.receipt) if self.receipt else None,
|
|
}
|
|
|
|
|
|
class JobStore:
|
|
SCHEMA_VERSION = 1
|
|
|
|
def __init__(self, root: str | os.PathLike[str], *, payload_root: str | os.PathLike[str] | None = None) -> None:
|
|
self.root = Path(root)
|
|
if not self.root.is_absolute() or self.root.is_symlink():
|
|
raise JobError("JOB_ROOT_INVALID")
|
|
self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
os.chmod(self.root, 0o700)
|
|
self.db_path = self.root / "state.sqlite3"
|
|
self.payloads = PayloadStore(payload_root or self.root.parent / "payloads" / "sha256")
|
|
self._initialize()
|
|
|
|
def _connect(self) -> sqlite3.Connection:
|
|
if self.db_path.is_symlink():
|
|
raise ServiceUnavailable("JOB_DB_SYMLINK")
|
|
connection = sqlite3.connect(self.db_path, timeout=30.0)
|
|
connection.row_factory = sqlite3.Row
|
|
connection.execute("PRAGMA journal_mode=WAL")
|
|
connection.execute("PRAGMA synchronous=FULL")
|
|
connection.execute("PRAGMA foreign_keys=ON")
|
|
connection.execute("PRAGMA busy_timeout=30000")
|
|
return connection
|
|
|
|
def _initialize(self) -> None:
|
|
try:
|
|
with self._connect() as connection:
|
|
connection.executescript(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
CREATE TABLE IF NOT EXISTS jobs (
|
|
job_id TEXT PRIMARY KEY, idempotency_key TEXT NOT NULL UNIQUE,
|
|
campaign_id TEXT NOT NULL, repository TEXT NOT NULL, operation TEXT NOT NULL,
|
|
branch TEXT NOT NULL, expected_branch_oid TEXT NOT NULL,
|
|
payload_digest TEXT NOT NULL, payload_ref TEXT, payload_size INTEGER,
|
|
state TEXT NOT NULL, side_effect_state TEXT NOT NULL,
|
|
revision INTEGER NOT NULL, request_json TEXT NOT NULL,
|
|
receipt_json TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS idempotency_keys (
|
|
key TEXT PRIMARY KEY, job_id TEXT NOT NULL UNIQUE,
|
|
binding_json TEXT NOT NULL, created_at TEXT NOT NULL,
|
|
FOREIGN KEY(job_id) REFERENCES jobs(job_id)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS branch_leases (
|
|
lease_id TEXT PRIMARY KEY, job_id TEXT NOT NULL UNIQUE,
|
|
branch TEXT NOT NULL, lineage_anchor_oid TEXT NOT NULL,
|
|
current_head_oid TEXT NOT NULL, generation INTEGER NOT NULL,
|
|
lease_expires_at TEXT NOT NULL, state TEXT NOT NULL,
|
|
revision INTEGER NOT NULL, FOREIGN KEY(job_id) REFERENCES jobs(job_id)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS state_transitions (
|
|
sequence INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT NOT NULL,
|
|
from_state TEXT, to_state TEXT NOT NULL, revision INTEGER NOT NULL,
|
|
side_effect_state TEXT NOT NULL, created_at TEXT NOT NULL,
|
|
FOREIGN KEY(job_id) REFERENCES jobs(job_id)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS receipts (
|
|
receipt_id TEXT PRIMARY KEY, job_id TEXT NOT NULL,
|
|
kind TEXT NOT NULL, digest TEXT NOT NULL, payload_json TEXT NOT NULL,
|
|
created_at TEXT NOT NULL, FOREIGN KEY(job_id) REFERENCES jobs(job_id)
|
|
);
|
|
INSERT OR REPLACE INTO metadata(key, value) VALUES('schema_version', '1');
|
|
"""
|
|
)
|
|
except sqlite3.Error as exc:
|
|
raise ServiceUnavailable("JOB_DB_UNAVAILABLE") from exc
|
|
|
|
def _mirror(self, job: Mapping[str, Any]) -> None:
|
|
directory = self.root / str(job["job_id"])
|
|
directory.mkdir(mode=0o700, exist_ok=True)
|
|
(directory / "receipts").mkdir(mode=0o700, exist_ok=True)
|
|
(directory / "logs-digest").mkdir(mode=0o700, exist_ok=True)
|
|
for name, value in (("request.json", job["request"]), ("state.json", job)):
|
|
target = directory / name
|
|
fd, temporary = tempfile.mkstemp(prefix=".mirror-", dir=directory)
|
|
try:
|
|
os.fchmod(fd, 0o600)
|
|
with os.fdopen(fd, "wb") as stream:
|
|
stream.write(canonical_json(value))
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
os.replace(temporary, target)
|
|
finally:
|
|
if os.path.exists(temporary):
|
|
os.unlink(temporary)
|
|
|
|
def read(self, job_id: str) -> dict[str, Any]:
|
|
try:
|
|
with self._connect() as connection:
|
|
row = connection.execute("SELECT * FROM jobs WHERE job_id=?", (job_id,)).fetchone()
|
|
except sqlite3.Error as exc:
|
|
raise ServiceUnavailable("JOB_DB_UNAVAILABLE") from exc
|
|
if row is None:
|
|
raise JobError("JOB_NOT_FOUND")
|
|
return dict(row)
|
|
|
|
def _receipt(self, row: Mapping[str, Any], *, duplicate: bool) -> JobReceipt:
|
|
state = JobState(row["state"])
|
|
if state == JobState.COMPLETED:
|
|
status, side = 200, "COMPLETED"
|
|
elif state in _RECOVERABLE:
|
|
status = 202
|
|
side = row["side_effect_state"] if row["side_effect_state"] == "REMOTE_PUSH_PENDING" else "RESUME_SAME_JOB"
|
|
elif row["side_effect_state"] == "UNKNOWN":
|
|
status, side = 503, "UNKNOWN"
|
|
else:
|
|
status, side = 202, "SAME_JOB" if duplicate else "QUEUED"
|
|
receipt = json.loads(row["receipt_json"]) if row["receipt_json"] else None
|
|
return JobReceipt(row["job_id"], row["idempotency_key"], state, side, status, int(row["revision"]), receipt)
|
|
|
|
|
|
class GitOpsJobController:
|
|
def __init__(self, store: JobStore) -> None:
|
|
self.store = store
|
|
|
|
@staticmethod
|
|
def _binding(request: Mapping[str, Any]) -> dict[str, str]:
|
|
fields = ("campaign_id", "repository", "operation", "branch", "expected_branch_oid", "payload_digest")
|
|
if any(not isinstance(request.get(field), str) or not request[field].strip() for field in fields):
|
|
raise JobError("JOB_BINDING_INVALID")
|
|
oid = request["expected_branch_oid"]
|
|
digest = request["payload_digest"]
|
|
if len(oid) != 40 or any(ch not in "0123456789abcdef" for ch in oid):
|
|
raise JobError("EXPECTED_BRANCH_OID_INVALID")
|
|
if len(digest) != 64 or any(ch not in "0123456789abcdef" for ch in digest):
|
|
raise JobError("PAYLOAD_DIGEST_INVALID")
|
|
if request["branch"] in {"main", "master"} or request["branch"].startswith(("refs/", "tags/")):
|
|
raise JobError("PROTECTED_BRANCH_REJECTED")
|
|
binding = {field: str(request[field]) for field in fields}
|
|
# Operation-specific, non-secret binding inputs must survive the
|
|
# durable round trip and participate in idempotency. Arbitrary request
|
|
# fields are intentionally discarded rather than persisted.
|
|
optional = (
|
|
"idempotency_key",
|
|
"task_id",
|
|
"new_branch_oid",
|
|
"authorization_recovery_digest",
|
|
"lineage_anchor_oid",
|
|
"base_oid",
|
|
"branch_mode",
|
|
"generation",
|
|
"expected_main_oid",
|
|
"pr_number",
|
|
"target_branch",
|
|
"check_context",
|
|
"check_state",
|
|
"check_description",
|
|
)
|
|
for field in optional:
|
|
value = request.get(field)
|
|
if value is None:
|
|
continue
|
|
if isinstance(value, bool) or not isinstance(value, (str, int)):
|
|
raise JobError("JOB_BINDING_OPTION_INVALID")
|
|
text = str(value).strip()
|
|
if not text or len(text) > 512:
|
|
raise JobError("JOB_BINDING_OPTION_INVALID")
|
|
binding[field] = text
|
|
return binding
|
|
|
|
def submit(self, request: Mapping[str, Any], *, payload: bytes | None = None) -> JobReceipt:
|
|
binding = self._binding(request)
|
|
payload_ref = request.get("payload_ref")
|
|
payload_size = request.get("payload_size")
|
|
if payload is not None:
|
|
payload_ref, payload_size = self.store.payloads.put(payload, binding["payload_digest"])
|
|
if payload_ref is not None and payload_ref != f"sha256/{binding['payload_digest']}":
|
|
raise JobError("PAYLOAD_REFERENCE_MISMATCH")
|
|
if payload_ref is not None and (not isinstance(payload_size, int) or payload_size < 0):
|
|
raise JobError("PAYLOAD_SIZE_INVALID")
|
|
client_idempotency_key = binding.get("idempotency_key")
|
|
if client_idempotency_key is not None:
|
|
if len(client_idempotency_key) > 128 or not all(
|
|
character.isalnum() or character in "._:-" for character in client_idempotency_key
|
|
):
|
|
raise JobError("IDEMPOTENCY_KEY_INVALID")
|
|
key = sha256_json({"client_idempotency_key": client_idempotency_key})
|
|
else:
|
|
key = sha256_json(binding)
|
|
job_id = "job-" + key[:32]
|
|
durable_request = dict(binding)
|
|
if payload_ref is not None:
|
|
durable_request.update({"payload_ref": str(payload_ref), "payload_size": int(payload_size)})
|
|
now = utc_now()
|
|
try:
|
|
with self.store._connect() as connection:
|
|
connection.execute("BEGIN IMMEDIATE")
|
|
existing = connection.execute("SELECT * FROM jobs WHERE idempotency_key=?", (key,)).fetchone()
|
|
if existing is not None:
|
|
if json.loads(existing["request_json"]).get("binding") != binding:
|
|
raise BindingConflict("BINDING_CONFLICT")
|
|
row = dict(existing)
|
|
connection.commit()
|
|
return self.store._receipt(row, duplicate=True)
|
|
connection.execute(
|
|
"INSERT INTO jobs(job_id,idempotency_key,campaign_id,repository,operation,branch,expected_branch_oid,payload_digest,payload_ref,payload_size,state,side_effect_state,revision,request_json,receipt_json,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
(job_id, key, binding["campaign_id"], binding["repository"], binding["operation"], binding["branch"], binding["expected_branch_oid"], binding["payload_digest"], payload_ref, payload_size, JobState.REQUESTED.value, "NONE", 0, json.dumps({"binding": binding, **durable_request}, sort_keys=True), None, now, now),
|
|
)
|
|
connection.execute("INSERT INTO idempotency_keys(key,job_id,binding_json,created_at) VALUES(?,?,?,?)", (key, job_id, json.dumps(binding, sort_keys=True), now))
|
|
connection.execute("INSERT INTO state_transitions(job_id,from_state,to_state,revision,side_effect_state,created_at) VALUES(?,?,?,?,?,?)", (job_id, None, JobState.REQUESTED.value, 0, "NONE", now))
|
|
row = dict(connection.execute("SELECT * FROM jobs WHERE job_id=?", (job_id,)).fetchone())
|
|
connection.commit()
|
|
except BindingConflict:
|
|
raise
|
|
except sqlite3.IntegrityError:
|
|
row = self.store.read(job_id)
|
|
return self.store._receipt(row, duplicate=True)
|
|
except sqlite3.Error as exc:
|
|
raise ServiceUnavailable("JOB_DB_UNAVAILABLE") from exc
|
|
row["request"] = json.loads(row["request_json"])
|
|
self.store._mirror(row)
|
|
return self.store._receipt(row, duplicate=False)
|
|
|
|
def lease_job(
|
|
self,
|
|
job_id: str,
|
|
*,
|
|
lease_id: str,
|
|
lineage_anchor_oid: str,
|
|
current_head_oid: str,
|
|
generation: int,
|
|
lease_expires_at: str,
|
|
expected_revision: int,
|
|
) -> JobReceipt:
|
|
"""Atomically create a branch lease and move its Job to LEASED."""
|
|
if generation < 1 or any(len(oid) != 40 for oid in (lineage_anchor_oid, current_head_oid)):
|
|
raise JobError("LEASE_BINDING_INVALID")
|
|
try:
|
|
with self.store._connect() as connection:
|
|
connection.execute("BEGIN IMMEDIATE")
|
|
row = connection.execute("SELECT * FROM jobs WHERE job_id=?", (job_id,)).fetchone()
|
|
if row is None:
|
|
raise JobError("JOB_NOT_FOUND")
|
|
if int(row["revision"]) != expected_revision:
|
|
raise JobError("REVISION_CAS_CONFLICT")
|
|
if row["branch"] in {"main", "master"}:
|
|
raise JobError("PROTECTED_BRANCH_REJECTED")
|
|
now = utc_now()
|
|
connection.execute("INSERT INTO branch_leases(lease_id,job_id,branch,lineage_anchor_oid,current_head_oid,generation,lease_expires_at,state,revision) VALUES(?,?,?,?,?,?,?,?,?)", (lease_id, job_id, row["branch"], lineage_anchor_oid, current_head_oid, generation, lease_expires_at, "ACTIVE", 0))
|
|
next_revision = expected_revision + 1
|
|
updated = connection.execute("UPDATE jobs SET state=?,side_effect_state=?,revision=?,updated_at=? WHERE job_id=? AND revision=?", (JobState.LEASED.value, "LEASED", next_revision, now, job_id, expected_revision))
|
|
if updated.rowcount != 1:
|
|
raise JobError("REVISION_CAS_CONFLICT")
|
|
connection.execute("INSERT INTO state_transitions(job_id,from_state,to_state,revision,side_effect_state,created_at) VALUES(?,?,?,?,?,?)", (job_id, row["state"], JobState.LEASED.value, next_revision, "LEASED", now))
|
|
result = dict(connection.execute("SELECT * FROM jobs WHERE job_id=?", (job_id,)).fetchone())
|
|
connection.commit()
|
|
except sqlite3.IntegrityError as exc:
|
|
raise JobError("LEASE_CONFLICT") from exc
|
|
except (JobError, BindingConflict):
|
|
raise
|
|
except sqlite3.Error as exc:
|
|
raise ServiceUnavailable("JOB_DB_UNAVAILABLE") from exc
|
|
result["request"] = json.loads(result["request_json"])
|
|
self.store._mirror(result)
|
|
return self.store._receipt(result, duplicate=True)
|
|
|
|
def update_lease(self, lease_id: str, *, expected_revision: int, current_head_oid: str, generation: int, state: str = "ACTIVE") -> None:
|
|
if len(current_head_oid) != 40 or generation < 1:
|
|
raise JobError("LEASE_BINDING_INVALID")
|
|
try:
|
|
with self.store._connect() as connection:
|
|
connection.execute("BEGIN IMMEDIATE")
|
|
updated = connection.execute("UPDATE branch_leases SET current_head_oid=?,generation=?,state=?,revision=revision+1 WHERE lease_id=? AND revision=?", (current_head_oid, generation, state, lease_id, expected_revision))
|
|
if updated.rowcount != 1:
|
|
raise JobError("LEASE_REVISION_CAS_CONFLICT")
|
|
connection.commit()
|
|
except JobError:
|
|
raise
|
|
except sqlite3.Error as exc:
|
|
raise ServiceUnavailable("JOB_DB_UNAVAILABLE") from exc
|
|
|
|
def advance(self, job_id: str, state: JobState, *, side_effect_state: str, receipt: Mapping[str, Any] | None = None, expected_revision: int | None = None) -> JobReceipt:
|
|
return self._advance(
|
|
job_id,
|
|
state,
|
|
side_effect_state=side_effect_state,
|
|
receipt=receipt,
|
|
expected_revision=expected_revision,
|
|
)
|
|
|
|
def _advance(
|
|
self,
|
|
job_id: str,
|
|
state: JobState,
|
|
*,
|
|
side_effect_state: str,
|
|
receipt: Mapping[str, Any] | None = None,
|
|
expected_revision: int | None = None,
|
|
_allow_verified_external_recovery: bool = False,
|
|
) -> JobReceipt:
|
|
try:
|
|
with self.store._connect() as connection:
|
|
connection.execute("BEGIN IMMEDIATE")
|
|
row = connection.execute("SELECT * FROM jobs WHERE job_id=?", (job_id,)).fetchone()
|
|
if row is None:
|
|
raise JobError("JOB_NOT_FOUND")
|
|
previous = JobState(row["state"])
|
|
revision = int(row["revision"])
|
|
if expected_revision is not None and expected_revision != revision:
|
|
raise JobError("REVISION_CAS_CONFLICT")
|
|
recovery_transition = (
|
|
_allow_verified_external_recovery
|
|
and previous == JobState.MANUAL_OWNER_REVIEW_REQUIRED
|
|
and state == JobState.COMPLETED
|
|
)
|
|
if previous in _TERMINAL and state != previous and not recovery_transition:
|
|
raise JobError("JOB_TERMINAL")
|
|
if state not in _RECOVERABLE | {JobState.RECONCILING, JobState.MANUAL_OWNER_REVIEW_REQUIRED}:
|
|
if previous in _RECOVERABLE:
|
|
raise JobError("RECONCILE_REQUIRED")
|
|
if previous != JobState.RECONCILING and _ORDER.get(state, -1) < _ORDER.get(previous, -1):
|
|
raise JobError("JOB_STATE_REGRESSION")
|
|
next_revision = revision + 1
|
|
receipt_json = json.dumps(dict(receipt), sort_keys=True) if receipt is not None else row["receipt_json"]
|
|
now = utc_now()
|
|
updated = connection.execute("UPDATE jobs SET state=?,side_effect_state=?,revision=?,receipt_json=?,updated_at=? WHERE job_id=? AND revision=?", (state.value, side_effect_state, next_revision, receipt_json, now, job_id, revision))
|
|
if updated.rowcount != 1:
|
|
raise JobError("REVISION_CAS_CONFLICT")
|
|
connection.execute("INSERT INTO state_transitions(job_id,from_state,to_state,revision,side_effect_state,created_at) VALUES(?,?,?,?,?,?)", (job_id, previous.value, state.value, next_revision, side_effect_state, now))
|
|
if receipt is not None:
|
|
digest = hashlib.sha256(canonical_json(receipt)).hexdigest()
|
|
connection.execute("INSERT INTO receipts(receipt_id,job_id,kind,digest,payload_json,created_at) VALUES(?,?,?,?,?,?)", (f"receipt-{job_id}-{next_revision}", job_id, state.value, digest, json.dumps(dict(receipt), sort_keys=True), now))
|
|
result = dict(connection.execute("SELECT * FROM jobs WHERE job_id=?", (job_id,)).fetchone())
|
|
connection.commit()
|
|
except (JobError, BindingConflict):
|
|
raise
|
|
except sqlite3.Error as exc:
|
|
raise ServiceUnavailable("JOB_DB_UNAVAILABLE") from exc
|
|
result["request"] = json.loads(result["request_json"])
|
|
self.store._mirror(result)
|
|
return self.store._receipt(result, duplicate=True)
|
|
|
|
@staticmethod
|
|
def _validate_verified_external_merge_proof(
|
|
job_id: str,
|
|
binding: Mapping[str, Any],
|
|
proof: Mapping[str, Any],
|
|
) -> dict[str, Any]:
|
|
if not isinstance(proof, Mapping):
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_PROOF_INVALID")
|
|
|
|
def require_oid(field: str) -> str:
|
|
value = proof.get(field)
|
|
if not isinstance(value, str) or len(value) != 40 or any(ch not in "0123456789abcdef" for ch in value):
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_PROOF_INVALID")
|
|
return value
|
|
|
|
if proof.get("proof_schema") != "aios.job-recovery.v1":
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_PROOF_INVALID")
|
|
if proof.get("job_id") != job_id:
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_PROOF_MISMATCH")
|
|
if proof.get("operation") != "PROTECTED_MERGE" or binding.get("operation") != "PROTECTED_MERGE":
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_OPERATION_REJECTED")
|
|
if proof.get("pr_number") != 53 or str(binding.get("pr_number")) != "53":
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_PROOF_MISMATCH")
|
|
if proof.get("pr_merged") is not True or proof.get("protected_merge") is not True:
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_PROOF_MISMATCH")
|
|
if proof.get("force_push") is not False:
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_PROOF_MISMATCH")
|
|
if proof.get("machine_actor") != "aios-autonomy-agent":
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_PROOF_MISMATCH")
|
|
|
|
expected_head = str(binding.get("expected_branch_oid", ""))
|
|
expected_base = str(binding.get("expected_main_oid", ""))
|
|
head_oid = require_oid("head_oid")
|
|
base_oid = require_oid("base_oid")
|
|
merged_main = require_oid("main_oid_after_protected_merge")
|
|
gitea_merged_main = require_oid("gitea_main_oid_after_protected_merge")
|
|
authority_merged_main = require_oid("authority_main_oid_after_protected_merge")
|
|
current_gitea = require_oid("current_gitea_main_oid")
|
|
current_authority = require_oid("current_authority_main_oid")
|
|
if head_oid != expected_head or base_oid != expected_base:
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_PROOF_MISMATCH")
|
|
if not (merged_main == gitea_merged_main == authority_merged_main == head_oid):
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_PROOF_MISMATCH")
|
|
if current_gitea != current_authority or proof.get("current_main_descends_from_protected_merge") is not True:
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_PROOF_MISMATCH")
|
|
return dict(proof)
|
|
|
|
def complete_verified_external_side_effect(
|
|
self,
|
|
job_id: str,
|
|
proof: Mapping[str, Any],
|
|
*,
|
|
expected_revision: int | None = None,
|
|
) -> JobReceipt:
|
|
"""Close one historical manual job from independently verified proof.
|
|
|
|
This is deliberately narrower than advance: it is only valid for the
|
|
known protected PR53 recovery shape and preserves the old receipt.
|
|
The actual state change still uses the normal revision-CAS path.
|
|
"""
|
|
current = self.store.read(job_id)
|
|
revision = int(current["revision"])
|
|
if expected_revision is not None and expected_revision != revision:
|
|
raise JobError("REVISION_CAS_CONFLICT")
|
|
request = json.loads(current["request_json"])
|
|
binding = request.get("binding", request)
|
|
normalized_proof = self._validate_verified_external_merge_proof(job_id, binding, proof)
|
|
existing_receipt = json.loads(current["receipt_json"]) if current["receipt_json"] else None
|
|
if JobState(current["state"]) == JobState.COMPLETED:
|
|
if (
|
|
existing_receipt
|
|
and existing_receipt.get("recovered_from_historical_manual") is True
|
|
and existing_receipt.get("proof") == normalized_proof
|
|
):
|
|
return self.store._receipt(current, duplicate=True)
|
|
raise JobError("JOB_TERMINAL")
|
|
if JobState(current["state"]) != JobState.MANUAL_OWNER_REVIEW_REQUIRED:
|
|
raise JobError("EXTERNAL_SIDE_EFFECT_RECOVERY_STATE_INVALID")
|
|
recovery_receipt = {
|
|
"recovered_from_historical_manual": True,
|
|
"recovery_kind": "VERIFIED_EXTERNAL_SIDE_EFFECT",
|
|
"historical_receipt": existing_receipt,
|
|
"proof": normalized_proof,
|
|
}
|
|
return self._advance(
|
|
job_id,
|
|
JobState.COMPLETED,
|
|
side_effect_state="REMOTE_VERIFIED",
|
|
receipt=recovery_receipt,
|
|
expected_revision=revision,
|
|
_allow_verified_external_recovery=True,
|
|
)
|
|
|
|
def reconcile(self, job_id: str, observer: Callable[[Mapping[str, Any]], Mapping[str, Any]]) -> JobReceipt:
|
|
current = self.store.read(job_id)
|
|
if JobState(current["state"]) not in _RECOVERABLE:
|
|
return self.store._receipt(current, duplicate=True)
|
|
self.advance(job_id, JobState.RECONCILING, side_effect_state="IN_PROGRESS")
|
|
try:
|
|
observed = dict(observer(current))
|
|
stable = JobState(observed.get("state", JobState.REQUESTED.value))
|
|
if stable in {JobState.RECOVERABLE_FAILURE, JobState.RECONCILING}:
|
|
raise JobError("RECONCILIATION_UNSTABLE")
|
|
return self.advance(job_id, stable, side_effect_state=str(observed.get("side_effect_state", "NONE")), receipt=observed.get("receipt"))
|
|
except Exception as exc:
|
|
self.advance(job_id, JobState.RECONCILIATION_REQUIRED, side_effect_state="UNKNOWN", receipt={"error_code": type(exc).__name__})
|
|
raise JobError("RECONCILIATION_REQUIRED") from exc
|
|
|
|
def resume(self, job_id: str) -> JobReceipt:
|
|
current = self.store.read(job_id)
|
|
if JobState(current["state"]) in _TERMINAL:
|
|
raise JobError("JOB_TERMINAL")
|
|
return self.reconcile(job_id, lambda _current: {"state": JobState.REQUESTED.value, "side_effect_state": "RESUME_SAME_JOB"})
|