233 lines
7.5 KiB
Python
233 lines
7.5 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
APP=ROOT/"active-runtime/app"
|
|
sys.path.insert(0,str(APP))
|
|
|
|
import aios_online_agent_worker as worker
|
|
from aios_external_agent_runtime import (
|
|
ExternalAgentExchange,
|
|
ExternalAgentRuntimeError,
|
|
validate_request,
|
|
)
|
|
|
|
|
|
def envelope():
|
|
value=object.__new__(worker.ExecutionEnvelope)
|
|
object.__setattr__(value,"task_id","task-attempt-identity")
|
|
object.__setattr__(value,"job_id","job-attempt-identity")
|
|
object.__setattr__(value,"digest","d"*64)
|
|
return value
|
|
|
|
|
|
def test_external_request_id_is_stable_within_attempt_and_changes_across_attempts():
|
|
env=envelope()
|
|
a1=worker.external_agent_request_id(env,"hermes",6)
|
|
a1_replay=worker.external_agent_request_id(env,"hermes",6)
|
|
a2=worker.external_agent_request_id(env,"hermes",7)
|
|
assert a1==a1_replay
|
|
assert a1!=a2
|
|
assert a1.startswith("external-hermes-")
|
|
assert a2.startswith("external-hermes-")
|
|
|
|
|
|
def test_external_request_id_rejects_invalid_attempt():
|
|
env=envelope()
|
|
with pytest.raises(worker.WorkerError,match="TASK_ATTEMPT_INVALID"):
|
|
worker.external_agent_request_id(env,"hermes",0)
|
|
|
|
|
|
def test_external_request_schema_binds_attempt():
|
|
value=validate_request({
|
|
"schema":"aios.external-agent-runtime.request.v1",
|
|
"request_id":"external-hermes-0123456789abcdef",
|
|
"runtime":"hermes",
|
|
"task_id":"task-attempt-identity",
|
|
"attempt":7,
|
|
"execution_digest":"d"*64,
|
|
"payload":{
|
|
"goal":"Return ok.",
|
|
"instructions":"",
|
|
"success_criteria":[],
|
|
"constraints":{},
|
|
},
|
|
})
|
|
assert value["attempt"]==7
|
|
|
|
|
|
def test_external_request_schema_rejects_missing_attempt():
|
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_ATTEMPT_INVALID"):
|
|
validate_request({
|
|
"schema":"aios.external-agent-runtime.request.v1",
|
|
"request_id":"external-hermes-0123456789abcdef",
|
|
"runtime":"hermes",
|
|
"task_id":"task-attempt-identity",
|
|
"execution_digest":"d"*64,
|
|
"payload":{
|
|
"goal":"Return ok.",
|
|
"instructions":"",
|
|
"success_criteria":[],
|
|
"constraints":{},
|
|
},
|
|
})
|
|
|
|
|
|
def _write_bound_result(path, request):
|
|
import hashlib
|
|
from aios_external_agent_runtime import canonical, sha256
|
|
material={
|
|
"schema":"aios.external-agent-runtime.result.v1",
|
|
"request_id":request["request_id"],
|
|
"request_sha256":sha256(canonical(request)),
|
|
"runtime":request["runtime"],
|
|
"task_id":request["task_id"],
|
|
"attempt":request["attempt"],
|
|
"execution_digest":request["execution_digest"],
|
|
"exit_code":0,
|
|
"duration_ms":1,
|
|
"output_text":"ok",
|
|
"output_sha256":hashlib.sha256(b"ok").hexdigest(),
|
|
"stderr_sha256":hashlib.sha256(b"").hexdigest(),
|
|
"credential_material_exposed":False,
|
|
"control_plane_access_granted":False,
|
|
}
|
|
material["result_sha256"]=sha256(canonical(material))
|
|
path.write_text(json.dumps(material,sort_keys=True),encoding="utf-8")
|
|
|
|
|
|
def test_exchange_result_binding_rejects_wrong_attempt(tmp_path):
|
|
exchange=ExternalAgentExchange(tmp_path)
|
|
request={
|
|
"schema":"aios.external-agent-runtime.request.v1",
|
|
"request_id":"external-hermes-0123456789abcdef",
|
|
"runtime":"hermes",
|
|
"task_id":"task-attempt-identity",
|
|
"attempt":7,
|
|
"execution_digest":"d"*64,
|
|
"payload":{
|
|
"goal":"Return ok.",
|
|
"instructions":"",
|
|
"success_criteria":[],
|
|
"constraints":{},
|
|
},
|
|
}
|
|
request=validate_request(request)
|
|
result_path=tmp_path/"result.json"
|
|
_write_bound_result(result_path,request)
|
|
value=json.loads(result_path.read_text())
|
|
value["attempt"]=6
|
|
# Recompute digest so the binding check, not the digest check, is what fails.
|
|
from aios_external_agent_runtime import canonical,sha256
|
|
material=dict(value); material.pop("result_sha256")
|
|
value["result_sha256"]=sha256(canonical(material))
|
|
result_path.write_text(json.dumps(value),encoding="utf-8")
|
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_RESULT_BINDING_INVALID"):
|
|
exchange._validated_result(
|
|
result_path,
|
|
request,
|
|
sha256(canonical(request)),
|
|
)
|
|
|
|
|
|
def test_model_governance_request_id_is_attempt_scoped():
|
|
env=envelope()
|
|
attempt6=worker.external_agent_request_id(env,"hermes",6)+":model"
|
|
attempt7=worker.external_agent_request_id(env,"hermes",7)+":model"
|
|
assert attempt6!=attempt7
|
|
|
|
|
|
def test_failed_previous_attempt_does_not_poison_next_model_governance_reservation(tmp_path):
|
|
from aios_model_governance import (
|
|
ModelGovernanceStore,
|
|
ModelRecord,
|
|
ProviderRecord,
|
|
RouteRequest,
|
|
ScopePolicy,
|
|
)
|
|
|
|
store=ModelGovernanceStore(tmp_path/"model-governance.sqlite3",now_fn=lambda:1_800_000_000.0)
|
|
store.register_provider(
|
|
ProviderRecord(
|
|
"oneapi",
|
|
"runtime-profile:hermes",
|
|
max_failures=5,
|
|
cooldown_seconds=60,
|
|
)
|
|
)
|
|
store.register_model(
|
|
ModelRecord(
|
|
"deepseek-v4-flash",
|
|
"oneapi",
|
|
"v1",
|
|
("agent.runtime.hermes","text"),
|
|
input_cost_micros_per_1k=1,
|
|
output_cost_micros_per_1k=1,
|
|
max_input_tokens=10000,
|
|
max_output_tokens=1000,
|
|
evaluation_sha256="a"*64,
|
|
)
|
|
)
|
|
store.set_scope_policy(
|
|
ScopePolicy(
|
|
"tenant",
|
|
"model-governance-live",
|
|
"staging",
|
|
allowed_models=("deepseek-v4-flash",),
|
|
required_capabilities=("text",),
|
|
daily_token_budget=100000,
|
|
daily_cost_budget_micros=100000,
|
|
fallback_limit=1,
|
|
)
|
|
)
|
|
|
|
def route(request_id):
|
|
return store.route(
|
|
RouteRequest(
|
|
request_id=request_id,
|
|
task_id="r42-model-governance-hermes-live-20260818",
|
|
trace_id="trace-hermes-model-governance-live",
|
|
scope_type="tenant",
|
|
tenant_id="model-governance-live",
|
|
project_id="staging",
|
|
required_capabilities=("agent.runtime.hermes",),
|
|
estimated_input_tokens=100,
|
|
max_output_tokens=50,
|
|
)
|
|
)
|
|
|
|
attempt6=route("external-hermes-attempt6:model")
|
|
store.record_failure(attempt6.request_id,"provider_process_failed")
|
|
row6=store.conn.execute(
|
|
"select state from model_governance_reservations where request_id=?",
|
|
(attempt6.request_id,),
|
|
).fetchone()
|
|
assert row6["state"]=="FAILED"
|
|
|
|
attempt7=route("external-hermes-attempt7:model")
|
|
assert attempt7.request_id!=attempt6.request_id
|
|
row7=store.conn.execute(
|
|
"select state from model_governance_reservations where request_id=?",
|
|
(attempt7.request_id,),
|
|
).fetchone()
|
|
assert row7["state"]=="RESERVED"
|
|
|
|
settled=store.record_success(
|
|
attempt7.request_id,
|
|
input_tokens=90,
|
|
output_tokens=40,
|
|
usage_source="runtime",
|
|
)
|
|
assert settled["state"]=="SETTLED"
|
|
assert store.conn.execute(
|
|
"select state from model_governance_reservations where request_id=?",
|
|
(attempt6.request_id,),
|
|
).fetchone()["state"]=="FAILED"
|
|
assert store.verify_audit_chain() is True
|
|
store.close()
|