451 lines
28 KiB
Python
451 lines
28 KiB
Python
"""Thin authenticated application service over existing TaskPlaneStore and JobStore.
|
|
|
|
This module owns no second Task lifecycle store. Added task runtime columns live
|
|
in the supplied Task authority database; the lineage ledger owns only relations.
|
|
The built-in host adapter is READ_ONLY. Mutating operations require separately
|
|
registered governed adapters with receiver fencing/readback, never arbitrary argv.
|
|
"""
|
|
from __future__ import annotations
|
|
from dataclasses import dataclass
|
|
from contextlib import contextmanager
|
|
from datetime import datetime,timezone
|
|
import base64
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Mapping
|
|
|
|
from task_plane.models import TaskPlan,WorkerRegistration,ProbeEvidence,ProbeOutcome,Budget
|
|
from task_plane.errors import TaskPlaneHTTPError
|
|
from aios_gitops_job_controller import GitOpsJobController,JobStore,JobState,JobError
|
|
from .evidence import ReceiptSigner,canonical,digest
|
|
from .runtime_contract import validate_contract
|
|
from .worker_policy import eligible
|
|
from .authorization import AuthorizationError
|
|
from .host import HostError
|
|
|
|
|
|
class ServiceError(RuntimeError):
|
|
def __init__(self,code,status=409):
|
|
super().__init__(code)
|
|
self.code=code
|
|
self.status=status
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Identity:
|
|
tenant_id: str
|
|
subject_id: str
|
|
role: str
|
|
worker_id: str | None=None
|
|
|
|
|
|
class BaselineService:
|
|
def __init__(self,store,job_root,authorization,contracts,operations,worker_profiles,host,lineage,machine_key,
|
|
*,machine_key_id='baseline-machine',trusted_receipt_keys=None):
|
|
self.store=store
|
|
self.signer=ReceiptSigner(machine_key,key_id=machine_key_id,trusted_public_keys=trusted_receipt_keys)
|
|
self.jobs=GitOpsJobController(JobStore(Path(job_root)),generic_receipt_verifier=self.signer.verify)
|
|
self.authorization=authorization
|
|
self.contracts={}
|
|
for entry in contracts:
|
|
value=validate_contract(entry)
|
|
key=(value['tenant_scope'],value['environment'],value['component_id'])
|
|
if key in self.contracts: raise ServiceError('CONTRACT_DUPLICATE')
|
|
self.contracts[key]=value
|
|
self.operations=operations
|
|
self.worker_profiles=dict(worker_profiles)
|
|
self.host=host
|
|
self.lineage=lineage
|
|
self.storage_blocked=False
|
|
self._execute_lock=threading.Lock()
|
|
with self.store._transaction():
|
|
self.store.conn.execute('''CREATE TABLE IF NOT EXISTS baseline_task_runtime(
|
|
task_id TEXT PRIMARY KEY REFERENCES tasks(task_id),ready_at REAL,
|
|
blocker_code TEXT,job_id TEXT,effect_id TEXT NOT NULL UNIQUE)''')
|
|
self.store.conn.execute('''CREATE TABLE IF NOT EXISTS baseline_worker_policy(
|
|
worker_id TEXT PRIMARY KEY,policy_json TEXT NOT NULL)''')
|
|
self.store.probe_attestation_verifier=self._verify_probe
|
|
|
|
def _user(self,identity):
|
|
if not isinstance(identity,Identity) or identity.role not in {'user','owner'}:
|
|
raise ServiceError('USER_IDENTITY_REQUIRED',403)
|
|
|
|
def _worker(self,identity):
|
|
if not isinstance(identity,Identity) or identity.role!='worker':
|
|
raise ServiceError('WORKER_IDENTITY_REQUIRED',403)
|
|
profile=self.worker_profiles.get(identity.worker_id)
|
|
if not profile or profile.identity!=identity.subject_id or identity.tenant_id not in profile.tenant_scope:
|
|
raise ServiceError('WORKER_SCOPE_DENIED',403)
|
|
return profile
|
|
|
|
def _task(self,identity,task_id):
|
|
task=self.store.get_task(task_id)
|
|
meta=task.task_plan.metadata.get('baseline',{})
|
|
if meta.get('tenant_id')!=identity.tenant_id or meta.get('subject_id')!=identity.subject_id:
|
|
raise ServiceError('TASK_SCOPE_DENIED',403)
|
|
return task
|
|
|
|
def _scope(self,task):
|
|
meta=task.task_plan.metadata['baseline']
|
|
return {**{key:meta[key] for key in ('tenant_id','subject_id','mission_id','operation','component_id','environment')},'task_id':task.task_id}
|
|
|
|
def _principal(self,task):
|
|
meta=task.task_plan.metadata['baseline']
|
|
return self.authorization.issue_principal(meta['grant_id'],ttl_seconds=60,**self._scope(task))
|
|
|
|
def _fresh_write(self):
|
|
if self.storage_blocked: raise ServiceError('STORAGE_RECOVERY_REQUIRED',503)
|
|
try:
|
|
with self.store._transaction():
|
|
self.store.conn.execute('UPDATE baseline_task_runtime SET blocker_code=blocker_code WHERE 0')
|
|
except (sqlite3.Error,OSError) as exc:
|
|
self.storage_blocked=True
|
|
raise ServiceError('TASK_STORAGE_UNAVAILABLE',503) from exc
|
|
|
|
def submit_goal(self,identity,request,*,idempotency_key):
|
|
self._user(identity)
|
|
allowed={'mission_id','grant_id','operation','component_id','environment','inputs','intent'}
|
|
if type(request) is not dict or set(request)!=allowed or not isinstance(idempotency_key,str) or not 1<=len(idempotency_key)<=128:
|
|
raise ServiceError('GOAL_SCHEMA_INVALID',400)
|
|
if any(not isinstance(request[key],str) or not request[key] or len(request[key])>2048 for key in allowed-{'inputs'}):
|
|
raise ServiceError('GOAL_SCHEMA_INVALID',400)
|
|
if request['inputs']!={}: raise ServiceError('OPERATION_INPUT_SCHEMA_INVALID',400)
|
|
if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}',request['mission_id']): raise ServiceError('MISSION_ID_INVALID',400)
|
|
operation=self.operations.get(request['operation'])
|
|
if operation.handler_ref!='host.service_status' or operation.effect_class!='READ_ONLY' or operation.required_permissions!=(operation.operation_id,):
|
|
raise ServiceError('REGISTERED_HANDLER_UNAVAILABLE',503)
|
|
contract=self.contracts.get((identity.tenant_id,request['environment'],request['component_id']))
|
|
if not contract or contract['lifecycle']!='active' or request['operation'] not in contract['operations']:
|
|
raise ServiceError('CONTRACT_OPERATION_UNAVAILABLE',403)
|
|
host_key=identity.tenant_id+':'+request['environment']+':'+request['component_id']
|
|
if host_key not in self.host.bindings: raise ServiceError('HOST_BINDING_UNAVAILABLE',503)
|
|
key=digest({'tenant':identity.tenant_id,'subject':identity.subject_id,'key':idempotency_key})
|
|
task_id='task-'+key[:40]
|
|
effect_id='effect-'+digest({'tenant':identity.tenant_id,'task':task_id,'request':request})[:40]
|
|
meta={**request,'tenant_id':identity.tenant_id,'subject_id':identity.subject_id,'effect_id':effect_id,
|
|
'operation_version':operation.version,'contract_revision':contract['contract_revision']}
|
|
plan=TaskPlan.from_dict({'task_id':task_id,'task_type':'host.diagnostic','intent':request['intent'],
|
|
'issuer':{'id':identity.subject_id,'tenant_id':identity.tenant_id,'subject_id':identity.subject_id},
|
|
'required_capabilities':[request['operation']],'risk':'low','approval_policy':'governed-grant',
|
|
'correlation_id':request['mission_id'],'metadata':{'baseline':meta},
|
|
'budgets':{'attempts':operation.retry['max_attempts'],'time_seconds':30*operation.retry['max_attempts']},'evidence_requirements':['diagnostic']})
|
|
# Validate business authorization before creating an executable task.
|
|
self.authorization.issue_principal(request['grant_id'],ttl_seconds=60,tenant_id=identity.tenant_id,
|
|
subject_id=identity.subject_id,task_id=task_id,**{key:request[key] for key in ('mission_id','operation','component_id','environment')})
|
|
self._fresh_write()
|
|
try:
|
|
with self.store._transaction():
|
|
task=self.store.create_task_with_event(plan,idempotency_key=key)
|
|
self.store.conn.execute('INSERT OR IGNORE INTO baseline_task_runtime(task_id,ready_at,effect_id) VALUES(?,?,?)',
|
|
(task_id,time.time(),effect_id))
|
|
self.lineage.link_task(identity.tenant_id,identity.subject_id,request['mission_id'],request['grant_id'],task_id)
|
|
return self.get_task(identity,task_id)
|
|
except TaskPlaneHTTPError as exc:
|
|
raise ServiceError('TASK_CONFLICT') from exc
|
|
except (sqlite3.Error,OSError) as exc:
|
|
self.storage_blocked=True
|
|
raise ServiceError('TASK_STORAGE_UNAVAILABLE',503) from exc
|
|
|
|
def get_task(self,identity,task_id):
|
|
self._user(identity)
|
|
task=self._task(identity,task_id)
|
|
with self.store._transaction():
|
|
row=self.store.conn.execute('SELECT * FROM baseline_task_runtime WHERE task_id=?',(task_id,)).fetchone()
|
|
result=task.to_dict()
|
|
if row:
|
|
result.update({'ready_at':row['ready_at'],'blocker_code':row['blocker_code'],'effect_id':row['effect_id'],'job_id':row['job_id']})
|
|
if row['job_id']:
|
|
job=self.jobs.store.read(row['job_id'])
|
|
result['job_state']=job['state']
|
|
result['side_effect_state']=job['side_effect_state']
|
|
envelope=json.loads(job['receipt_json']) if job.get('receipt_json') else None
|
|
if self._receipt_valid(task,envelope): result['receipt']=envelope['payload']
|
|
result['remaining_budget']=self.store.get_task_budget(task_id).to_dict()
|
|
return result
|
|
|
|
def explain(self,identity,mission_id):
|
|
self._user(identity)
|
|
result=self.lineage.explain(identity.tenant_id,identity.subject_id,mission_id)
|
|
result['tasks_runtime']=[self.get_task(identity,t.task_id) for t in self.store.list_tasks()
|
|
if t.task_plan.metadata.get('baseline',{}).get('mission_id')==mission_id and
|
|
t.task_plan.issuer.get('tenant_id')==identity.tenant_id and t.task_plan.issuer.get('subject_id')==identity.subject_id]
|
|
return result
|
|
|
|
def register_worker(self,identity):
|
|
profile=self._worker(identity)
|
|
try:
|
|
existing=self.store.get_worker(profile.worker_id)
|
|
except TaskPlaneHTTPError:
|
|
existing=None
|
|
now=time.time()
|
|
worker=WorkerRegistration.from_dict({'worker_id':profile.worker_id,'worker_type':'baseline-operation-worker',
|
|
'identity':profile.identity,'capabilities':list(profile.capabilities),'trust_zone':'governed-host',
|
|
'max_concurrency':profile.max_concurrency,'heartbeat_interval_seconds':profile.heartbeat_seconds,
|
|
'supported_protocols':['aios-baseline-worker-v1'],'last_heartbeat':now})
|
|
self.store.register_worker(worker,actor_identity=profile.identity,is_admin=False,
|
|
expected_revision=existing.registration_revision if existing else None)
|
|
with self.store._transaction():
|
|
self.store.conn.execute('INSERT INTO baseline_worker_policy VALUES(?,?) ON CONFLICT(worker_id) DO UPDATE SET policy_json=excluded.policy_json',
|
|
(profile.worker_id,canonical(profile.__dict__).decode()))
|
|
return {'worker_id':profile.worker_id,'registered':True,'fallback_executor':profile.fallback_executor}
|
|
|
|
def heartbeat(self,identity):
|
|
profile=self._worker(identity)
|
|
worker=self.store.get_worker(profile.worker_id)
|
|
self.store.heartbeat_worker(profile.worker_id,actor_identity=profile.identity,expected_revision=worker.registration_revision)
|
|
return {'worker_id':profile.worker_id,'alive':True}
|
|
|
|
def claim_next(self,identity):
|
|
profile=self._worker(identity)
|
|
worker=self.store.get_worker(profile.worker_id)
|
|
if worker.disabled: raise ServiceError('WORKER_DISABLED',403)
|
|
self._fresh_write()
|
|
for task in sorted(self.store.list_tasks(),key=lambda t:(t.created_at,t.task_id)):
|
|
meta=task.task_plan.metadata.get('baseline',{})
|
|
if not meta or task.status not in {'CREATED','QUEUED','FAILED_RECOVERABLE'}: continue
|
|
with self.store._transaction():
|
|
row=self.store.conn.execute('SELECT ready_at FROM baseline_task_runtime WHERE task_id=?',(task.task_id,)).fetchone()
|
|
if not row: continue
|
|
now=time.time()
|
|
task_scope={**meta,'ready_at':row['ready_at']}
|
|
if not eligible(profile,task_scope,now=now,last_heartbeat=worker.last_heartbeat): continue
|
|
# Priority is configuration on ordinary eligible workers; no special main actor.
|
|
candidates=[]
|
|
for registered in self.store.list_workers():
|
|
peer=self.worker_profiles.get(registered.worker_id)
|
|
if not peer or registered.disabled or not eligible(peer,task_scope,now=now,last_heartbeat=registered.last_heartbeat): continue
|
|
with self.store._transaction():
|
|
active=self.store.conn.execute("SELECT count(*) FROM claims WHERE worker_id=? AND status='ACTIVE' AND lease_until>?",(peer.worker_id,now)).fetchone()[0]
|
|
if active<peer.max_concurrency: candidates.append(peer)
|
|
if not candidates or sorted(candidates,key=lambda p:(-p.priority,p.worker_id))[0].worker_id!=profile.worker_id: continue
|
|
try:
|
|
self._principal(task)
|
|
with self.store._transaction():
|
|
self.store.consume_budget(task.task_id,Budget(attempts=1,time_seconds=30))
|
|
claim=self.store.claim_task(task.task_id,profile.worker_id,profile.identity,lease_seconds=30,
|
|
expected_revision=task.revision,operation_digest=task.task_plan.operation_digest())
|
|
except TaskPlaneHTTPError:
|
|
budget=self.store.get_task_budget(task.task_id)
|
|
if budget.attempts==0 or budget.time_seconds==0:
|
|
with self.store._transaction():
|
|
self.store.conn.execute('UPDATE baseline_task_runtime SET blocker_code=? WHERE task_id=?',('TASK_BUDGET_EXHAUSTED',task.task_id))
|
|
continue
|
|
except AuthorizationError:
|
|
with self.store._transaction():
|
|
self.store.conn.execute('UPDATE baseline_task_runtime SET blocker_code=? WHERE task_id=?',('AUTHORIZATION_REQUIRED',task.task_id))
|
|
continue
|
|
return {'task_id':task.task_id,'claim_id':claim.claim_id,'generation':claim.attempt,
|
|
'lease_until':claim.lease_until,'operation':meta['operation']}
|
|
return None
|
|
|
|
def renew(self,identity,task_id,claim_id):
|
|
profile=self._worker(identity)
|
|
claim=self.store.get_claim(claim_id)
|
|
if claim.task_id!=task_id or claim.worker_id!=profile.worker_id or claim.worker_identity!=profile.identity:
|
|
raise ServiceError('CLAIM_IDENTITY_MISMATCH',403)
|
|
task=self.store.get_task(task_id)
|
|
meta=task.task_plan.metadata.get('baseline',{})
|
|
if meta.get('tenant_id') not in profile.tenant_scope or meta.get('subject_id') not in profile.subject_scope or task.status=='CANCELLED':
|
|
raise ServiceError('CLAIM_SCOPE_OR_STATE_DENIED',403)
|
|
self._principal(task)
|
|
self.store.heartbeat_claim(task_id,claim_id,30)
|
|
return {'task_id':task_id,'claim_id':claim_id,'renewed':True}
|
|
|
|
def _advance(self,job_id,state,**kwargs):
|
|
row=self.jobs.store.read(job_id)
|
|
return self.jobs.advance(job_id,state,expected_revision=int(row['revision']),**kwargs)
|
|
|
|
@contextmanager
|
|
def _execution_slot(self,deadline):
|
|
remaining=deadline-time.time()
|
|
if remaining<=0 or not self._execute_lock.acquire(timeout=remaining):
|
|
raise ServiceError('EXECUTION_DEADLINE_EXPIRED',408)
|
|
try:
|
|
if time.time()>=deadline: raise ServiceError('EXECUTION_DEADLINE_EXPIRED',408)
|
|
yield
|
|
finally: self._execute_lock.release()
|
|
|
|
def execute_claim(self,identity,task_id,claim_id,*,deadline=None):
|
|
profile=self._worker(identity)
|
|
deadline=time.time()+30 if deadline is None else deadline
|
|
with self._execution_slot(deadline):
|
|
self._fresh_write()
|
|
task=self.store.get_task(task_id)
|
|
meta=task.task_plan.metadata.get('baseline',{})
|
|
claim=self.store.get_claim(claim_id)
|
|
if claim.task_id!=task_id or claim.worker_id!=profile.worker_id or claim.worker_identity!=profile.identity:
|
|
raise ServiceError('CLAIM_IDENTITY_MISMATCH',403)
|
|
if meta.get('tenant_id') not in profile.tenant_scope or meta.get('subject_id') not in profile.subject_scope:
|
|
raise ServiceError('WORKER_SCOPE_DENIED',403)
|
|
user=Identity(meta['tenant_id'],meta['subject_id'],'user')
|
|
if task.status=='COMPLETED': return self.get_task(user,task_id)
|
|
if claim.status!='ACTIVE' or claim.lease_until<=time.time() or task.status=='CANCELLED':
|
|
raise ServiceError('CLAIM_EXPIRED_OR_CANCELLED')
|
|
principal=self._principal(task)
|
|
self.authorization.authorize(principal,**self._scope(task))
|
|
operation=self.operations.get(meta['operation'])
|
|
if operation.handler_ref!='host.service_status' or operation.effect_class!='READ_ONLY':
|
|
raise ServiceError('HANDLER_NOT_REGISTERED',503)
|
|
self.authorization.assert_effect_bounds(principal,operation,**self._scope(task))
|
|
request={'schema':'aios.operation-job.v1','tenant_id':meta['tenant_id'],'subject_id':meta['subject_id'],
|
|
'mission_id':meta['mission_id'],'grant_id':meta['grant_id'],'task_id':task_id,'effect_id':meta['effect_id'],
|
|
'operation':meta['operation'],'operation_version':str(operation.version),'resource_kind':'systemd-service',
|
|
'resource_id':digest({'tenant':meta['tenant_id'],'environment':meta['environment'],'component':meta['component_id']}),'input_digest':digest(meta['inputs'])}
|
|
try:
|
|
job=self.jobs.submit(request)
|
|
with self.store._transaction():
|
|
self.store.conn.execute('UPDATE baseline_task_runtime SET job_id=? WHERE task_id=?',(job.job_id,task_id))
|
|
self.lineage.link_task(meta['tenant_id'],meta['subject_id'],meta['mission_id'],meta['grant_id'],task_id)
|
|
self.lineage.link_effect(meta['tenant_id'],meta['subject_id'],meta['mission_id'],task_id,meta['effect_id'],job.job_id)
|
|
row=self.jobs.store.read(job.job_id)
|
|
if row['state']=='COMPLETED':
|
|
self._close_task(task,claim,json.loads(row['receipt_json']))
|
|
return self.get_task(user,task_id)
|
|
if row['state'] not in {'REQUESTED','AUTHORIZED'}:
|
|
raise ServiceError('JOB_RECONCILIATION_REQUIRED')
|
|
self.jobs.assert_effect_admission(exclude_job_id=job.job_id)
|
|
self.authorization.reserve_effect(meta['grant_id'],meta['effect_id'],principal,**self._scope(task))
|
|
if row['state']=='REQUESTED': self._advance(job.job_id,JobState.AUTHORIZED,side_effect_state='AUTHORIZED')
|
|
row=self.jobs.store.read(job.job_id)
|
|
self.jobs.lease_resource_job(job.job_id,lease_id=claim_id,resource_kind=request['resource_kind'],
|
|
resource_id=request['resource_id'],generation=None,
|
|
lease_expires_at=datetime.fromtimestamp(min(claim.lease_until,time.time()+30,deadline),timezone.utc).isoformat(),
|
|
expected_revision=int(row['revision']))
|
|
effect_lease=self.jobs.get_execution_lease(job.job_id)
|
|
fence_generation=int(effect_lease['generation'])
|
|
self._advance(job.job_id,JobState.EXECUTING,side_effect_state='IN_PROGRESS')
|
|
# Last authorization/claim check directly before the registered host boundary.
|
|
self.authorization.authorize(principal,**self._scope(task))
|
|
if self.store.get_task(task_id).status=='CANCELLED' or self.store.get_claim(claim_id).lease_until<=time.time():
|
|
raise ServiceError('CLAIM_EXPIRED_OR_CANCELLED')
|
|
self.jobs.assert_execution_lease(job.job_id,generation=fence_generation,lease_id=claim_id)
|
|
if time.time()>=deadline: raise ServiceError('EXECUTION_DEADLINE_EXPIRED',408)
|
|
observation=self.host.status(meta['component_id'],meta['environment'],tenant_id=meta['tenant_id'],deadline=deadline)
|
|
if time.time()>=deadline or self.store.get_claim(claim_id).lease_until<=time.time():
|
|
raise ServiceError('EXECUTION_DEADLINE_EXPIRED',408)
|
|
envelope=self.signer.sign({'task_id':task_id,'job_id':job.job_id,'effect_id':meta['effect_id'],
|
|
'tenant_id':meta['tenant_id'],'subject_id':meta['subject_id'],'mission_id':meta['mission_id'],
|
|
'grant_id':meta['grant_id'],'claim_id':claim_id,'generation':fence_generation,
|
|
'operation':meta['operation'],'operation_version':str(operation.version),
|
|
'input_digest':request['input_digest'],'resource_kind':request['resource_kind'],
|
|
'resource_id':request['resource_id'],'observation':observation,'verified':True})
|
|
self._advance(job.job_id,JobState.LOCAL_VERIFIED,side_effect_state='READBACK_VERIFIED',receipt=envelope)
|
|
self._advance(job.job_id,JobState.COMPLETED,side_effect_state='READBACK_VERIFIED',receipt=envelope)
|
|
self._close_task(task,claim,envelope)
|
|
return self.get_task(user,task_id)
|
|
except (sqlite3.Error,OSError) as exc:
|
|
self.storage_blocked=True
|
|
raise ServiceError('STORAGE_RECOVERY_REQUIRED',503) from exc
|
|
except HostError as exc:
|
|
try:
|
|
self._advance(job.job_id,JobState.RECONCILIATION_REQUIRED,side_effect_state='UNKNOWN')
|
|
with self.store._transaction():
|
|
self.store.conn.execute('UPDATE baseline_task_runtime SET blocker_code=? WHERE task_id=?',(str(exc),task_id))
|
|
except (sqlite3.Error,OSError,JobError): self.storage_blocked=True
|
|
raise ServiceError(str(exc),503) from exc
|
|
except JobError as exc:
|
|
if 'UNAVAILABLE' in str(exc) or 'STORAGE' in str(exc): self.storage_blocked=True
|
|
raise ServiceError(str(exc),503 if self.storage_blocked else 409) from exc
|
|
|
|
def _receipt_valid(self,task,envelope):
|
|
meta=task.task_plan.metadata['baseline']
|
|
if not self.signer.verify(envelope): return False
|
|
payload=envelope['payload']
|
|
with self.store._transaction():
|
|
runtime=self.store.conn.execute('SELECT job_id FROM baseline_task_runtime WHERE task_id=?',(task.task_id,)).fetchone()
|
|
if not runtime or not runtime['job_id']: return False
|
|
expected={**{k:meta[k] for k in ('effect_id','tenant_id','subject_id','mission_id','grant_id','operation')},
|
|
'task_id':task.task_id,'job_id':runtime['job_id'],'operation_version':str(meta['operation_version']),
|
|
'input_digest':digest(meta['inputs']),'resource_kind':'systemd-service',
|
|
'resource_id':digest({'tenant':meta['tenant_id'],'environment':meta['environment'],'component':meta['component_id']})}
|
|
if any(payload.get(k)!=v for k,v in expected.items()) or payload.get('verified') is not True: return False
|
|
try:
|
|
lease=self.jobs.get_execution_lease(runtime['job_id'])
|
|
source_claim=self.store.get_claim(payload.get('claim_id'))
|
|
if payload.get('claim_id')!=lease['lease_id'] or payload.get('generation')!=lease['generation']: return False
|
|
if source_claim.task_id!=task.task_id or source_claim.operation_digest!=task.task_plan.operation_digest(): return False
|
|
except (JobError,TaskPlaneHTTPError,TypeError): return False
|
|
observation=payload.get('observation')
|
|
if type(observation) is not dict: return False
|
|
for k in ('operation','tenant_id','component_id','environment'):
|
|
if observation.get(k)!=meta[k]: return False
|
|
binding=meta['tenant_id']+':'+meta['environment']+':'+meta['component_id']
|
|
if observation.get('unit')!=self.host.bindings.get(binding) or observation.get('source')!='systemd.show': return False
|
|
return True
|
|
|
|
def _close_task(self,task,claim,envelope):
|
|
if not self._receipt_valid(task,envelope): raise ServiceError('RESULT_ATTESTATION_INVALID',403)
|
|
current=self.store.get_task(task.task_id)
|
|
if current.status=='CANCELLED': raise ServiceError('TASK_CANCELLED')
|
|
if current.status=='CLAIMED': self.store.update_task_status(task.task_id,current.revision,'RUNNING')
|
|
self.store.complete_claim(task.task_id,claim.claim_id,'COMPLETED',envelope,
|
|
evidence_entries=[('diagnostic',envelope,claim.worker_identity)])
|
|
|
|
def cancel_task(self,identity,task_id):
|
|
self._user(identity)
|
|
task=self._task(identity,task_id)
|
|
self.store.cancel_task_with_event(task_id,task.revision,'cancelled by authenticated subject')
|
|
return self.get_task(identity,task_id)
|
|
|
|
def _verify_probe(self,probe):
|
|
try:
|
|
self.signer.public_key.verify(base64.b64decode(probe.signature,validate=True),canonical(probe.digest_payload()))
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
def recover(self):
|
|
"""Reconcile expired baseline claims from durable signed receipts; never fake a result."""
|
|
with self._execute_lock:
|
|
return self._recover_locked()
|
|
|
|
def _recover_locked(self):
|
|
recovered=[]
|
|
with self.store._transaction():
|
|
rows=self.store.conn.execute("SELECT claim_id FROM claims WHERE status='ACTIVE' AND lease_until<?",(time.time(),)).fetchall()
|
|
for row in rows:
|
|
claim=self.store.get_claim(row['claim_id'])
|
|
task=self.store.get_task(claim.task_id)
|
|
meta=task.task_plan.metadata.get('baseline')
|
|
if not meta: continue
|
|
with self.store._transaction():
|
|
rt=self.store.conn.execute('SELECT job_id FROM baseline_task_runtime WHERE task_id=?',(task.task_id,)).fetchone()
|
|
outcome=ProbeOutcome.NOT_APPLIED
|
|
ref='read-only-no-persistent-effect'
|
|
if rt and rt['job_id']:
|
|
job=self.jobs.store.read(rt['job_id'])
|
|
env=json.loads(job['receipt_json']) if job.get('receipt_json') else None
|
|
if job['state']=='COMPLETED':
|
|
if self._receipt_valid(task,env):
|
|
outcome=ProbeOutcome.APPLIED_VERIFIED
|
|
ref=digest(env)
|
|
else: outcome=ProbeOutcome.INDETERMINATE
|
|
elif self.operations.get(meta['operation']).effect_class!='READ_ONLY':
|
|
outcome=ProbeOutcome.INDETERMINATE
|
|
now=time.time()
|
|
probe=ProbeEvidence.create(task_id=task.task_id,claim_id=claim.claim_id,worker_id=claim.worker_id,
|
|
worker_identity=claim.worker_identity,operation_digest=claim.operation_digest,outcome=outcome,
|
|
observed_at=now,expires_at=now+30,probe_executor_identity='baseline-reconciler',attestation_ref=ref)
|
|
from dataclasses import replace
|
|
probe=replace(probe,signature=base64.b64encode(self.signer.private_key.sign(canonical(probe.digest_payload()))).decode())
|
|
self.store.recover_claim_with_probe_and_event(task.task_id,claim.claim_id,probe)
|
|
if outcome==ProbeOutcome.NOT_APPLIED and rt and rt['job_id']:
|
|
job=self.jobs.store.read(rt['job_id'])
|
|
if job['state']!='COMPLETED':
|
|
self._advance(rt['job_id'],JobState.RECONCILIATION_REQUIRED,side_effect_state='UNKNOWN')
|
|
self.jobs.reconcile(rt['job_id'],lambda _:{'state':'REQUESTED','side_effect_state':'READ_ONLY_SAFE_TO_RETRY'})
|
|
recovered.append({'task_id':task.task_id,'outcome':outcome.value})
|
|
# A successful writable transaction permits recovery processing, not blind effect replay.
|
|
with self.store._transaction():
|
|
self.store.conn.execute('UPDATE baseline_task_runtime SET blocker_code=blocker_code WHERE 0')
|
|
try:
|
|
self.jobs.assert_effect_admission()
|
|
self.storage_blocked=False
|
|
except JobError:
|
|
self.storage_blocked=True
|
|
return recovered
|