aios/ops/aios-v2/v2r1c2-production-control.sh

#!/usr/bin/env bash
set -Eeuo pipefail

VERSION=V2R1C2
EXPECTED_TARGET=3dd622b5a515f1fa4ef72b9f8ae9ce48ac703f24
TAG_OBJECT=a3bd84ebec9c705f7c9c5c03f7f4ec55f8854e36
REPO=/data/aios-gitops/repo.git
SOURCE=/opt/aios-v2r1c2
TAKEOVER=/data/aios-gitops/worktrees/codex/AIOS-V2R1C2-UNIFIED-PORTAL-TAKEOVER-20260722
CANDIDATE_SERVICE=aios-v2r1c2-portal.service
OLD_SERVICE=aios-orchestrator-gateway.service
NEW_SERVICE=aios-v2r1c2-production.service
CANDIDATE_STATE=/var/lib/aios-v2r1c2-portal
PRODUCTION_STATE=/var/lib/aios-v2r1c2-production
RUN_ID=${AIOS_RELEASE_RUN_ID:-AIOS-V2R1C2-PRODUCTION-20260722T011836Z}
EVIDENCE_DIR=/var/lib/aios-gitops/acceptance/$RUN_ID
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
RELEASE_ROOT=$(cd -- "$SCRIPT_DIR/../.." && pwd)
UNIT_SOURCE=$RELEASE_ROOT/deploy/aios-v2/systemd/aios-v2r1c2-production.service
ENV_SOURCE=$RELEASE_ROOT/deploy/aios-v2/systemd/aios-v2r1c2-production.env.example
LOCK=/run/lock/aios-v2r1c2-production-release.lock

require_root() {
  test "$(id -u)" -eq 0 || { echo 'root required' >&2; exit 77; }
}

health_old() {
  curl -fsS --max-time 10 http://127.0.0.1:18920/healthz
}

health_new() {
  curl -fsS --max-time 10 http://127.0.0.1:18920/api/v2/health
}

health_candidate() {
  curl -fsS --max-time 10 http://127.0.0.1:18921/api/v2/health
}

wait_candidate() {
  for _ in $(seq 1 30); do
    if health_candidate; then
      return 0
    fi
    sleep 1
  done
  return 1
}

preflight() {
  require_root
  test "$(git --git-dir="$REPO" rev-parse refs/tags/$VERSION)" = "$TAG_OBJECT"
  test "$(git --git-dir="$REPO" rev-list -n 1 "$VERSION")" = "$EXPECTED_TARGET"
  test -f "$SOURCE/active-runtime/app/aios_web/release_application.py"
  test -f "$UNIT_SOURCE"
  test -f "$ENV_SOURCE"
  command -v sqlite3 >/dev/null
  systemctl is-active --quiet "$CANDIDATE_SERVICE"
  systemctl is-active --quiet "$OLD_SERVICE"
  health_candidate >/dev/null
  health_old >/dev/null
  (
    cd "$TAKEOVER/artifacts/v2r1c2"
    sha256sum -c checksums.sha256 >/dev/null
  )
  diff -qr --exclude=__pycache__ "$TAKEOVER/active-runtime/app" "$SOURCE/active-runtime/app" >/dev/null
  diff -qr "$TAKEOVER/plans" "$SOURCE/plans" >/dev/null
  echo PREFLIGHT=PASS
}

prepare() {
  preflight
  install -d -m 0750 "$EVIDENCE_DIR" "$EVIDENCE_DIR/backup"
  health_old >"$EVIDENCE_DIR/production-health-before.json"
  health_candidate >"$EVIDENCE_DIR/candidate-health-before.json"
  systemctl cat "$OLD_SERVICE" >"$EVIDENCE_DIR/backup/$OLD_SERVICE.unit"
  cp -a /etc/aios-orchestrator-gateway "$EVIDENCE_DIR/backup/"
  cp -a /etc/systemd/system/aios-orchestrator-gateway.service "$EVIDENCE_DIR/backup/"
  if test -e "$PRODUCTION_STATE"; then
    mv "$PRODUCTION_STATE" "$EVIDENCE_DIR/backup/aios-v2r1c2-production-state.preexisting"
  fi
  systemctl stop "$CANDIDATE_SERVICE"
  trap 'systemctl start "$CANDIDATE_SERVICE" || true' EXIT
  while IFS= read -r -d '' database; do
    sqlite3 "$database" 'PRAGMA wal_checkpoint(TRUNCATE);' >/dev/null
    test "$(sqlite3 "$database" 'PRAGMA quick_check;')" = ok
  done < <(find "$CANDIDATE_STATE" -maxdepth 1 -type f -name '*.sqlite3' -print0)
  install -d -o aios -g aios -m 0750 "$PRODUCTION_STATE"
  cp -a "$CANDIDATE_STATE/." "$PRODUCTION_STATE/"
  chown -R aios:aios "$PRODUCTION_STATE"
  while IFS= read -r -d '' database; do
    test "$(sqlite3 "$database" 'PRAGMA quick_check;')" = ok
  done < <(find "$PRODUCTION_STATE" -maxdepth 1 -type f -name '*.sqlite3' -print0)
  install -o root -g root -m 0644 "$UNIT_SOURCE" /etc/systemd/system/aios-v2r1c2-production.service
  install -d -o root -g aios -m 0750 /etc/aios-v2
  install -o root -g aios -m 0640 "$ENV_SOURCE" /etc/aios-v2/aios-v2r1c2-production.env
  systemctl daemon-reload
  systemctl start "$CANDIDATE_SERVICE"
  wait_candidate >"$EVIDENCE_DIR/candidate-health-after-state-copy.json"
  trap - EXIT
  echo PREPARE=PASS
}

rollback() {
  require_root
  systemctl stop "$NEW_SERVICE" || true
  systemctl disable "$NEW_SERVICE" >/dev/null 2>&1 || true
  systemctl enable "$OLD_SERVICE" >/dev/null 2>&1 || true
  systemctl start "$OLD_SERVICE"
  health_old >"$EVIDENCE_DIR/rollback-health.json"
  echo ROLLBACK=PASS
}

promote() {
  require_root
  test -f /etc/systemd/system/aios-v2r1c2-production.service
  test -f /etc/aios-v2/aios-v2r1c2-production.env
  ! systemctl is-active --quiet "$NEW_SERVICE"
  systemctl is-active --quiet "$CANDIDATE_SERVICE"
  health_candidate >/dev/null
  health_old >"$EVIDENCE_DIR/production-health-switch-input.json"
  trap 'trap - ERR; rollback' ERR
  systemctl stop "$OLD_SERVICE"
  systemctl start "$NEW_SERVICE"
  for _ in $(seq 1 30); do
    if health_new >"$EVIDENCE_DIR/production-health-after-switch.json" 2>/dev/null; then
      echo PROMOTE=PASS
      trap - ERR
      return 0
    fi
    sleep 1
  done
  return 1
}

finalize() {
  require_root
  systemctl is-active --quiet "$NEW_SERVICE"
  health_new >"$EVIDENCE_DIR/production-health-final.json"
  systemctl disable "$OLD_SERVICE" >/dev/null
  systemctl enable "$NEW_SERVICE" >/dev/null
  echo FINALIZE=PASS
}

exec 9>"$LOCK"
flock -n 9 || { echo 'release operation already running' >&2; exit 75; }
case "${1:-}" in
  preflight) preflight ;;
  prepare) prepare ;;
  promote) promote ;;
  rollback) rollback ;;
  finalize) finalize ;;
  *) echo "usage: $0 {preflight|prepare|promote|rollback|finalize}" >&2; exit 64 ;;
esac