fix(gitops): persist recoverable executor failures

active-runtime/app/aios_gitops_job_executor.py

@@ -116,6 +116,19 @@ class DurableJobExecutor:
         return row
 
     def execute(self, job_id: str, *, dry_run: bool = False) -> dict[str, Any]:
+        try:
+            return self._execute_unprotected(job_id, dry_run=dry_run)
+        except Exception as exc:
+            try:
+                row = self.controller.store.read(job_id)
+                state = JobState(row["state"])
+                if state not in {JobState.COMPLETED, JobState.MANUAL_OWNER_REVIEW_REQUIRED, JobState.RECOVERABLE_FAILURE, JobState.RECONCILIATION_REQUIRED}:
+                    self.controller.advance(job_id, JobState.RECOVERABLE_FAILURE, side_effect_state="UNKNOWN", receipt={"error_code": type(exc).__name__})
+            except Exception:
+                pass
+            raise
+
+    def _execute_unprotected(self, job_id: str, *, dry_run: bool = False) -> dict[str, Any]:
         row, request = self._request(job_id)
         operation = request.get("operation")
         if operation == "ADVANCE_EXISTING_CAMPAIGN_BRANCH":
@@ -162,7 +175,7 @@ class DurableJobExecutor:
             row = self.controller.store.read(job_id)
         authority_before = self.authority.read_ref(f"refs/heads/{BRANCH}")
         if authority_before == EXPECTED_BRANCH_OID:
-            authority_oid = self.authority.advance_task_branch_CAS(BRANCH and f"refs/heads/{BRANCH}", EXPECTED_BRANCH_OID, new_oid)
+            authority_oid = self.authority.advance_task_branch_CAS(f"refs/heads/{BRANCH}", EXPECTED_BRANCH_OID, new_oid)
         elif authority_before == new_oid:
             authority_oid = new_oid
         else: