628 lines
22 KiB
Python
628 lines
22 KiB
Python
"""Tenant-isolated AIOS V2 control-plane domain service.
|
|
|
|
This module deliberately contains no OpenClaw or Hermes implementation details.
|
|
It owns authoritative tenant, agent, task-lifecycle, idempotency and audit state.
|
|
Runtime adapters consume approved tasks through a separate boundary.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import threading
|
|
import uuid
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from enum import Enum
|
|
from typing import Any, Callable, Iterable, Mapping
|
|
|
|
|
|
def _utc_now() -> str:
|
|
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def _new_id(prefix: str) -> str:
|
|
return f"{prefix}_{uuid.uuid4().hex}"
|
|
|
|
|
|
def _canonical_digest(value: Mapping[str, Any]) -> str:
|
|
encoded = json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
default=str,
|
|
).encode("utf-8")
|
|
return hashlib.sha256(encoded).hexdigest()
|
|
|
|
|
|
class ControlPlaneError(RuntimeError):
|
|
"""Base error returned by the control-plane domain."""
|
|
|
|
|
|
class Unauthorized(ControlPlaneError):
|
|
"""The caller is not authenticated or lacks the required role."""
|
|
|
|
|
|
class TenantIsolationError(ControlPlaneError):
|
|
"""A request attempted to cross a tenant boundary."""
|
|
|
|
|
|
class NotFound(ControlPlaneError):
|
|
"""A tenant-scoped resource does not exist or is not visible."""
|
|
|
|
|
|
class InvalidTransition(ControlPlaneError):
|
|
"""A task state transition violates the lifecycle contract."""
|
|
|
|
|
|
class IdempotencyConflict(ControlPlaneError):
|
|
"""An idempotency key was reused for a different command."""
|
|
|
|
|
|
class RevisionConflict(ControlPlaneError):
|
|
"""Optimistic task revision does not match current state."""
|
|
|
|
|
|
class PolicyDenied(ControlPlaneError):
|
|
"""The policy gate denied a control-plane operation."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RequestContext:
|
|
"""Authenticated request identity propagated across all AIOS layers."""
|
|
|
|
tenant_id: str
|
|
principal_id: str
|
|
trace_id: str
|
|
roles: frozenset[str] = field(default_factory=frozenset)
|
|
session_id: str | None = None
|
|
policy_version: str = "v1"
|
|
|
|
def __post_init__(self) -> None:
|
|
for name in ("tenant_id", "principal_id", "trace_id"):
|
|
if not str(getattr(self, name, "")).strip():
|
|
raise ValueError(f"{name} is required")
|
|
|
|
@classmethod
|
|
def create(
|
|
cls,
|
|
*,
|
|
tenant_id: str,
|
|
principal_id: str,
|
|
roles: Iterable[str] = (),
|
|
trace_id: str | None = None,
|
|
session_id: str | None = None,
|
|
policy_version: str = "v1",
|
|
) -> "RequestContext":
|
|
return cls(
|
|
tenant_id=tenant_id.strip(),
|
|
principal_id=principal_id.strip(),
|
|
trace_id=trace_id or _new_id("trace"),
|
|
roles=frozenset(str(role).strip() for role in roles if str(role).strip()),
|
|
session_id=session_id,
|
|
policy_version=policy_version,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TenantRecord:
|
|
tenant_id: str
|
|
display_name: str
|
|
status: str = "active"
|
|
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
created_at: str = field(default_factory=_utc_now)
|
|
updated_at: str = field(default_factory=_utc_now)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AgentRecord:
|
|
agent_id: str
|
|
tenant_id: str
|
|
display_name: str
|
|
runtime: str
|
|
capabilities: tuple[str, ...] = field(default_factory=tuple)
|
|
enabled: bool = True
|
|
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
created_at: str = field(default_factory=_utc_now)
|
|
updated_at: str = field(default_factory=_utc_now)
|
|
|
|
|
|
class TaskState(str, Enum):
|
|
CREATED = "CREATED"
|
|
APPROVED = "APPROVED"
|
|
RUNNING = "RUNNING"
|
|
COMPLETED = "COMPLETED"
|
|
FAILED = "FAILED"
|
|
CANCELLED = "CANCELLED"
|
|
|
|
|
|
TERMINAL_TASK_STATES = frozenset(
|
|
{TaskState.COMPLETED, TaskState.FAILED, TaskState.CANCELLED}
|
|
)
|
|
|
|
ALLOWED_TASK_TRANSITIONS: Mapping[TaskState, frozenset[TaskState]] = {
|
|
TaskState.CREATED: frozenset({TaskState.APPROVED, TaskState.CANCELLED}),
|
|
TaskState.APPROVED: frozenset({TaskState.RUNNING, TaskState.CANCELLED}),
|
|
TaskState.RUNNING: frozenset(
|
|
{TaskState.COMPLETED, TaskState.FAILED, TaskState.CANCELLED}
|
|
),
|
|
TaskState.COMPLETED: frozenset(),
|
|
TaskState.FAILED: frozenset(),
|
|
TaskState.CANCELLED: frozenset(),
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TaskRecord:
|
|
task_id: str
|
|
tenant_id: str
|
|
agent_id: str
|
|
action: str
|
|
payload: Mapping[str, Any]
|
|
state: TaskState = TaskState.CREATED
|
|
revision: int = 1
|
|
idempotency_key: str | None = None
|
|
request_digest: str = ""
|
|
result: Mapping[str, Any] | None = None
|
|
error: Mapping[str, Any] | None = None
|
|
created_by: str = ""
|
|
approved_by: str | None = None
|
|
created_at: str = field(default_factory=_utc_now)
|
|
updated_at: str = field(default_factory=_utc_now)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AuditEvent:
|
|
event_id: str
|
|
tenant_id: str
|
|
principal_id: str
|
|
trace_id: str
|
|
action: str
|
|
resource_type: str
|
|
resource_id: str
|
|
outcome: str
|
|
policy_version: str
|
|
details: Mapping[str, Any] = field(default_factory=dict)
|
|
created_at: str = field(default_factory=_utc_now)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PolicyDecision:
|
|
allowed: bool
|
|
reason: str
|
|
policy_version: str
|
|
|
|
|
|
PolicyEvaluator = Callable[[RequestContext, str, Mapping[str, Any]], PolicyDecision]
|
|
|
|
|
|
class InMemoryControlPlaneStore:
|
|
"""Thread-safe reference store used by tests and the first gateway slice.
|
|
|
|
Production backends must preserve the same tenant-scoped interface and add
|
|
database row-level security, migrations, backup and HA evidence.
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
self.lock = threading.RLock()
|
|
self.tenants: dict[str, TenantRecord] = {}
|
|
self.agents: dict[str, AgentRecord] = {}
|
|
self.tasks: dict[str, TaskRecord] = {}
|
|
self.idempotency: dict[tuple[str, str], tuple[str, str]] = {}
|
|
self.audit_events: list[AuditEvent] = []
|
|
|
|
|
|
class ControlPlaneService:
|
|
"""Authoritative tenant, agent and task lifecycle service."""
|
|
|
|
def __init__(
|
|
self,
|
|
store: InMemoryControlPlaneStore | None = None,
|
|
*,
|
|
policy_evaluator: PolicyEvaluator | None = None,
|
|
) -> None:
|
|
self.store = store or InMemoryControlPlaneStore()
|
|
self.policy_evaluator = policy_evaluator or self._default_policy
|
|
|
|
@staticmethod
|
|
def _default_policy(
|
|
context: RequestContext,
|
|
action: str,
|
|
resource: Mapping[str, Any],
|
|
) -> PolicyDecision:
|
|
del resource
|
|
role_requirements = {
|
|
"tenant.register": {"platform_admin"},
|
|
"tenant.read": {"tenant_admin", "operator", "auditor", "platform_admin"},
|
|
"agent.register": {"tenant_admin", "operator", "platform_admin"},
|
|
"agent.read": {"tenant_admin", "operator", "auditor", "platform_admin"},
|
|
"task.create": {"tenant_admin", "operator", "platform_admin"},
|
|
"task.read": {"tenant_admin", "operator", "auditor", "platform_admin"},
|
|
"task.approve": {"tenant_admin", "approver", "platform_admin"},
|
|
"task.start": {"runtime", "operator", "platform_admin"},
|
|
"task.complete": {"runtime", "operator", "platform_admin"},
|
|
"task.fail": {"runtime", "operator", "platform_admin"},
|
|
"task.cancel": {"tenant_admin", "operator", "approver", "platform_admin"},
|
|
"audit.read": {"tenant_admin", "auditor", "platform_admin"},
|
|
}
|
|
required = role_requirements.get(action, {"platform_admin"})
|
|
allowed = bool(context.roles.intersection(required))
|
|
return PolicyDecision(
|
|
allowed=allowed,
|
|
reason="role_allowed" if allowed else "required_role_missing",
|
|
policy_version=context.policy_version,
|
|
)
|
|
|
|
def _require_context(self, context: RequestContext) -> None:
|
|
if not context.principal_id or not context.tenant_id or not context.trace_id:
|
|
raise Unauthorized("authenticated tenant context is required")
|
|
|
|
def _authorize(
|
|
self,
|
|
context: RequestContext,
|
|
action: str,
|
|
resource: Mapping[str, Any],
|
|
) -> PolicyDecision:
|
|
self._require_context(context)
|
|
decision = self.policy_evaluator(context, action, resource)
|
|
if not decision.allowed:
|
|
self._audit(
|
|
context,
|
|
action=action,
|
|
resource_type=str(resource.get("resource_type", "policy")),
|
|
resource_id=str(resource.get("resource_id", "unknown")),
|
|
outcome="denied",
|
|
details={"reason": decision.reason},
|
|
)
|
|
raise PolicyDenied(f"{action} denied: {decision.reason}")
|
|
return decision
|
|
|
|
@staticmethod
|
|
def _assert_same_tenant(context: RequestContext, tenant_id: str) -> None:
|
|
if context.tenant_id != tenant_id and "platform_admin" not in context.roles:
|
|
raise TenantIsolationError("cross-tenant access denied")
|
|
|
|
def _audit(
|
|
self,
|
|
context: RequestContext,
|
|
*,
|
|
action: str,
|
|
resource_type: str,
|
|
resource_id: str,
|
|
outcome: str,
|
|
details: Mapping[str, Any] | None = None,
|
|
) -> AuditEvent:
|
|
event = AuditEvent(
|
|
event_id=_new_id("audit"),
|
|
tenant_id=context.tenant_id,
|
|
principal_id=context.principal_id,
|
|
trace_id=context.trace_id,
|
|
action=action,
|
|
resource_type=resource_type,
|
|
resource_id=resource_id,
|
|
outcome=outcome,
|
|
policy_version=context.policy_version,
|
|
details=copy.deepcopy(dict(details or {})),
|
|
)
|
|
self.store.audit_events.append(event)
|
|
return event
|
|
|
|
def register_tenant(
|
|
self,
|
|
context: RequestContext,
|
|
*,
|
|
tenant_id: str,
|
|
display_name: str,
|
|
metadata: Mapping[str, Any] | None = None,
|
|
) -> TenantRecord:
|
|
tenant_id = tenant_id.strip()
|
|
if not tenant_id or not display_name.strip():
|
|
raise ValueError("tenant_id and display_name are required")
|
|
self._authorize(
|
|
context,
|
|
"tenant.register",
|
|
{"resource_type": "tenant", "resource_id": tenant_id},
|
|
)
|
|
self._assert_same_tenant(context, tenant_id)
|
|
with self.store.lock:
|
|
if tenant_id in self.store.tenants:
|
|
raise ControlPlaneError(f"tenant already exists: {tenant_id}")
|
|
tenant = TenantRecord(
|
|
tenant_id=tenant_id,
|
|
display_name=display_name.strip(),
|
|
metadata=copy.deepcopy(dict(metadata or {})),
|
|
)
|
|
self.store.tenants[tenant_id] = tenant
|
|
self._audit(
|
|
context,
|
|
action="tenant.register",
|
|
resource_type="tenant",
|
|
resource_id=tenant_id,
|
|
outcome="allowed",
|
|
)
|
|
return tenant
|
|
|
|
def get_tenant(self, context: RequestContext, tenant_id: str | None = None) -> TenantRecord:
|
|
tenant_id = tenant_id or context.tenant_id
|
|
self._authorize(
|
|
context,
|
|
"tenant.read",
|
|
{"resource_type": "tenant", "resource_id": tenant_id},
|
|
)
|
|
self._assert_same_tenant(context, tenant_id)
|
|
with self.store.lock:
|
|
tenant = self.store.tenants.get(tenant_id)
|
|
if tenant is None:
|
|
raise NotFound("tenant not found")
|
|
return tenant
|
|
|
|
def register_agent(
|
|
self,
|
|
context: RequestContext,
|
|
*,
|
|
agent_id: str,
|
|
display_name: str,
|
|
runtime: str,
|
|
capabilities: Iterable[str] = (),
|
|
metadata: Mapping[str, Any] | None = None,
|
|
) -> AgentRecord:
|
|
self._authorize(
|
|
context,
|
|
"agent.register",
|
|
{"resource_type": "agent", "resource_id": agent_id},
|
|
)
|
|
tenant = self.get_tenant(context)
|
|
if tenant.status != "active":
|
|
raise ControlPlaneError("tenant is not active")
|
|
normalized_capabilities = tuple(
|
|
sorted({str(item).strip() for item in capabilities if str(item).strip()})
|
|
)
|
|
if not agent_id.strip() or not display_name.strip() or not runtime.strip():
|
|
raise ValueError("agent_id, display_name and runtime are required")
|
|
with self.store.lock:
|
|
existing = self.store.agents.get(agent_id)
|
|
if existing is not None and existing.tenant_id != context.tenant_id:
|
|
raise TenantIsolationError("agent id belongs to another tenant")
|
|
if existing is not None:
|
|
raise ControlPlaneError(f"agent already exists: {agent_id}")
|
|
agent = AgentRecord(
|
|
agent_id=agent_id.strip(),
|
|
tenant_id=context.tenant_id,
|
|
display_name=display_name.strip(),
|
|
runtime=runtime.strip(),
|
|
capabilities=normalized_capabilities,
|
|
metadata=copy.deepcopy(dict(metadata or {})),
|
|
)
|
|
self.store.agents[agent.agent_id] = agent
|
|
self._audit(
|
|
context,
|
|
action="agent.register",
|
|
resource_type="agent",
|
|
resource_id=agent.agent_id,
|
|
outcome="allowed",
|
|
details={"runtime": agent.runtime},
|
|
)
|
|
return agent
|
|
|
|
def get_agent(self, context: RequestContext, agent_id: str) -> AgentRecord:
|
|
self._authorize(
|
|
context,
|
|
"agent.read",
|
|
{"resource_type": "agent", "resource_id": agent_id},
|
|
)
|
|
with self.store.lock:
|
|
agent = self.store.agents.get(agent_id)
|
|
if agent is None:
|
|
raise NotFound("agent not found")
|
|
self._assert_same_tenant(context, agent.tenant_id)
|
|
return agent
|
|
|
|
def list_agents(self, context: RequestContext) -> list[AgentRecord]:
|
|
self._authorize(
|
|
context,
|
|
"agent.read",
|
|
{"resource_type": "agent_collection", "resource_id": context.tenant_id},
|
|
)
|
|
with self.store.lock:
|
|
return [
|
|
agent
|
|
for agent in self.store.agents.values()
|
|
if agent.tenant_id == context.tenant_id
|
|
]
|
|
|
|
def create_task(
|
|
self,
|
|
context: RequestContext,
|
|
*,
|
|
agent_id: str,
|
|
action: str,
|
|
payload: Mapping[str, Any] | None = None,
|
|
idempotency_key: str | None = None,
|
|
) -> TaskRecord:
|
|
self._authorize(
|
|
context,
|
|
"task.create",
|
|
{"resource_type": "task", "resource_id": idempotency_key or "new"},
|
|
)
|
|
agent = self.get_agent(context, agent_id)
|
|
if not agent.enabled:
|
|
raise ControlPlaneError("agent is disabled")
|
|
if action not in agent.capabilities and "*" not in agent.capabilities:
|
|
raise PolicyDenied(f"agent does not advertise capability: {action}")
|
|
normalized_payload = copy.deepcopy(dict(payload or {}))
|
|
request_digest = _canonical_digest(
|
|
{"agent_id": agent_id, "action": action, "payload": normalized_payload}
|
|
)
|
|
normalized_key = idempotency_key.strip() if idempotency_key else None
|
|
with self.store.lock:
|
|
if normalized_key:
|
|
idempotency_ref = (context.tenant_id, normalized_key)
|
|
existing = self.store.idempotency.get(idempotency_ref)
|
|
if existing is not None:
|
|
existing_task_id, existing_digest = existing
|
|
if existing_digest != request_digest:
|
|
raise IdempotencyConflict(
|
|
"idempotency key was reused with a different request"
|
|
)
|
|
return self.store.tasks[existing_task_id]
|
|
task = TaskRecord(
|
|
task_id=_new_id("task"),
|
|
tenant_id=context.tenant_id,
|
|
agent_id=agent_id,
|
|
action=action,
|
|
payload=normalized_payload,
|
|
idempotency_key=normalized_key,
|
|
request_digest=request_digest,
|
|
created_by=context.principal_id,
|
|
)
|
|
self.store.tasks[task.task_id] = task
|
|
if normalized_key:
|
|
self.store.idempotency[(context.tenant_id, normalized_key)] = (
|
|
task.task_id,
|
|
request_digest,
|
|
)
|
|
self._audit(
|
|
context,
|
|
action="task.create",
|
|
resource_type="task",
|
|
resource_id=task.task_id,
|
|
outcome="allowed",
|
|
details={"agent_id": agent_id, "action": action},
|
|
)
|
|
return task
|
|
|
|
def get_task(self, context: RequestContext, task_id: str) -> TaskRecord:
|
|
self._authorize(
|
|
context,
|
|
"task.read",
|
|
{"resource_type": "task", "resource_id": task_id},
|
|
)
|
|
with self.store.lock:
|
|
task = self.store.tasks.get(task_id)
|
|
if task is None:
|
|
raise NotFound("task not found")
|
|
self._assert_same_tenant(context, task.tenant_id)
|
|
return task
|
|
|
|
def list_tasks(
|
|
self,
|
|
context: RequestContext,
|
|
*,
|
|
states: Iterable[TaskState | str] | None = None,
|
|
) -> list[TaskRecord]:
|
|
self._authorize(
|
|
context,
|
|
"task.read",
|
|
{"resource_type": "task_collection", "resource_id": context.tenant_id},
|
|
)
|
|
state_filter = (
|
|
{TaskState(item) for item in states}
|
|
if states is not None
|
|
else None
|
|
)
|
|
with self.store.lock:
|
|
return [
|
|
task
|
|
for task in self.store.tasks.values()
|
|
if task.tenant_id == context.tenant_id
|
|
and (state_filter is None or task.state in state_filter)
|
|
]
|
|
|
|
def transition_task(
|
|
self,
|
|
context: RequestContext,
|
|
*,
|
|
task_id: str,
|
|
target_state: TaskState | str,
|
|
expected_revision: int | None = None,
|
|
result: Mapping[str, Any] | None = None,
|
|
error: Mapping[str, Any] | None = None,
|
|
) -> TaskRecord:
|
|
target = TaskState(target_state)
|
|
action_name = {
|
|
TaskState.APPROVED: "task.approve",
|
|
TaskState.RUNNING: "task.start",
|
|
TaskState.COMPLETED: "task.complete",
|
|
TaskState.FAILED: "task.fail",
|
|
TaskState.CANCELLED: "task.cancel",
|
|
}.get(target)
|
|
if action_name is None:
|
|
raise InvalidTransition(f"cannot transition to {target.value}")
|
|
self._authorize(
|
|
context,
|
|
action_name,
|
|
{"resource_type": "task", "resource_id": task_id},
|
|
)
|
|
with self.store.lock:
|
|
current = self.store.tasks.get(task_id)
|
|
if current is None:
|
|
raise NotFound("task not found")
|
|
self._assert_same_tenant(context, current.tenant_id)
|
|
if expected_revision is not None and current.revision != expected_revision:
|
|
raise RevisionConflict(
|
|
f"expected revision {expected_revision}, current {current.revision}"
|
|
)
|
|
if target not in ALLOWED_TASK_TRANSITIONS[current.state]:
|
|
raise InvalidTransition(
|
|
f"invalid transition {current.state.value} -> {target.value}"
|
|
)
|
|
if target == TaskState.COMPLETED and result is None:
|
|
raise ValueError("completed task requires result")
|
|
if target == TaskState.FAILED and error is None:
|
|
raise ValueError("failed task requires error")
|
|
updated = TaskRecord(
|
|
task_id=current.task_id,
|
|
tenant_id=current.tenant_id,
|
|
agent_id=current.agent_id,
|
|
action=current.action,
|
|
payload=current.payload,
|
|
state=target,
|
|
revision=current.revision + 1,
|
|
idempotency_key=current.idempotency_key,
|
|
request_digest=current.request_digest,
|
|
result=copy.deepcopy(dict(result)) if result is not None else current.result,
|
|
error=copy.deepcopy(dict(error)) if error is not None else current.error,
|
|
created_by=current.created_by,
|
|
approved_by=(
|
|
context.principal_id
|
|
if target == TaskState.APPROVED
|
|
else current.approved_by
|
|
),
|
|
created_at=current.created_at,
|
|
updated_at=_utc_now(),
|
|
)
|
|
self.store.tasks[task_id] = updated
|
|
self._audit(
|
|
context,
|
|
action=action_name,
|
|
resource_type="task",
|
|
resource_id=task_id,
|
|
outcome="allowed",
|
|
details={
|
|
"from": current.state.value,
|
|
"to": target.value,
|
|
"revision": updated.revision,
|
|
},
|
|
)
|
|
return updated
|
|
|
|
def list_audit_events(
|
|
self,
|
|
context: RequestContext,
|
|
*,
|
|
resource_id: str | None = None,
|
|
) -> list[AuditEvent]:
|
|
self._authorize(
|
|
context,
|
|
"audit.read",
|
|
{"resource_type": "audit", "resource_id": resource_id or context.tenant_id},
|
|
)
|
|
with self.store.lock:
|
|
return [
|
|
event
|
|
for event in self.store.audit_events
|
|
if event.tenant_id == context.tenant_id
|
|
and (resource_id is None or event.resource_id == resource_id)
|
|
]
|