419 lines
15 KiB
Python
419 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
APP=ROOT/"active-runtime/app"
|
|
sys.path.insert(0,str(APP))
|
|
|
|
from aios_external_agent_runtime import (
|
|
ExternalAgentRuntimeError,
|
|
RuntimeProfile,
|
|
execute_request,
|
|
)
|
|
from aios_online_agent_execution import (
|
|
ExecutionProjectionError,
|
|
GovernedExecutionBinding,
|
|
project_online_task,
|
|
validate_execution_envelope,
|
|
)
|
|
import aios_online_agent_worker as worker
|
|
|
|
|
|
BASE="a"*40
|
|
|
|
|
|
def binding(task_id="task-hermes-runtime"):
|
|
return GovernedExecutionBinding.from_mapping({
|
|
"task_id":task_id,
|
|
"job_id":"job-"+task_id,
|
|
"repository":"aios/aios",
|
|
"base_oid":BASE,
|
|
"expected_branch_oid":BASE,
|
|
"target_branch":"agent/"+task_id,
|
|
"branch_mode":"CREATE_NEW",
|
|
"generation":1,
|
|
"idempotency_key":task_id+"-binding-v1",
|
|
})
|
|
|
|
|
|
def task_request(task_id, runtime):
|
|
return {
|
|
"task_id":task_id,
|
|
"goal":"Summarize the requested operation.",
|
|
"instructions":"Return a concise result.",
|
|
"success_criteria":[{"id":"done","description":"A result is returned."}],
|
|
"constraints":{"network":"runtime-managed"},
|
|
"required_capabilities":[f"agent.runtime.{runtime}"],
|
|
"max_attempts":3,
|
|
}
|
|
|
|
|
|
def projected(task_id="task-hermes-runtime", runtime="hermes"):
|
|
return project_online_task(
|
|
binding(task_id),
|
|
task_request(task_id,runtime),
|
|
{
|
|
"action":"agent_runtime",
|
|
"adapter":runtime,
|
|
"capability_scope":[f"agent.runtime.{runtime}"],
|
|
},
|
|
)
|
|
|
|
|
|
def clear_secret_env(monkeypatch):
|
|
for key in list(os.environ):
|
|
upper=key.upper()
|
|
if any(marker in upper for marker in (
|
|
"TOKEN","SECRET","PASSWORD","CREDENTIAL","AUTHORIZATION","PRIVATE_KEY"
|
|
)):
|
|
monkeypatch.delenv(key,raising=False)
|
|
monkeypatch.setenv("CREDENTIALS_DIRECTORY","/definitely-not-present-aios-test")
|
|
|
|
|
|
def make_helper(tmp_path):
|
|
helper=tmp_path/"runtime_helper.py"
|
|
helper.write_text(
|
|
"import json,sys\n"
|
|
"value=json.load(sys.stdin)\n"
|
|
"print(json.dumps({'answer':'ok','goal':value['goal']},sort_keys=True))\n",
|
|
encoding="utf-8",
|
|
)
|
|
return helper
|
|
|
|
|
|
def make_profile(tmp_path,runtime,helper):
|
|
root=tmp_path/"profiles";root.mkdir(exist_ok=True)
|
|
(root/f"{runtime}.json").write_text(
|
|
json.dumps({
|
|
"schema":"aios.external-agent-runtime.profile.v1",
|
|
"runtime":runtime,
|
|
"argv":["/usr/bin/python3",str(helper)],
|
|
"input_mode":"stdin_json",
|
|
"output_mode":"stdout_json",
|
|
"timeout_seconds":30,
|
|
"max_output_bytes":65536,
|
|
}),
|
|
encoding="utf-8",
|
|
)
|
|
return root
|
|
|
|
|
|
def make_text_request(tmp_path, runtime, payload=None):
|
|
request={
|
|
"schema":"aios.external-agent-runtime.request.v1",
|
|
"request_id":f"external-{runtime}-0123456789abcdef",
|
|
"runtime":runtime,
|
|
"task_id":f"task-{runtime}-runtime",
|
|
"execution_digest":"d"*64,
|
|
"payload":payload or {
|
|
"goal":"Return ok.",
|
|
"instructions":"No side effects.",
|
|
"success_criteria":[],
|
|
"constraints":{},
|
|
},
|
|
}
|
|
request_path=tmp_path/"request.json"
|
|
result_path=tmp_path/"result.json"
|
|
request_path.write_text(json.dumps(request),encoding="utf-8")
|
|
return request_path, result_path
|
|
|
|
|
|
def test_agent_runtime_projection_is_read_only_and_task_bound():
|
|
value=projected()
|
|
execution=value["metadata"]["execution"]
|
|
assert execution["action"]=="agent_runtime"
|
|
assert execution["adapter"]=="hermes"
|
|
assert execution["runtime_input"]["goal"]==value["goal"]
|
|
assert execution["runtime_input"]["instructions"]==value["instructions"]
|
|
assert execution["governance"]["writer_lease"] is False
|
|
assert execution["change_set"]==[]
|
|
assert execution["verification"]==[]
|
|
validate_execution_envelope(
|
|
execution,
|
|
binding=binding(),
|
|
worker_capabilities=["agent.runtime.hermes"],
|
|
)
|
|
|
|
|
|
def test_agent_runtime_projection_requires_runtime_capability():
|
|
with pytest.raises(ExecutionProjectionError,match="AGENT_RUNTIME_CAPABILITY_REQUIRED"):
|
|
project_online_task(
|
|
binding(),
|
|
task_request("task-hermes-runtime","hermes"),
|
|
{
|
|
"action":"agent_runtime",
|
|
"adapter":"hermes",
|
|
"capability_scope":["git.read"],
|
|
},
|
|
)
|
|
|
|
|
|
def test_agent_runtime_projection_rejects_mutation():
|
|
with pytest.raises(ExecutionProjectionError,match="AGENT_RUNTIME_MUTATION_DENY"):
|
|
project_online_task(
|
|
binding(),
|
|
task_request("task-hermes-runtime","hermes"),
|
|
{
|
|
"action":"agent_runtime",
|
|
"adapter":"hermes",
|
|
"capability_scope":["agent.runtime.hermes"],
|
|
"change_set":[{"path":"x.py","operation":"write_file","content":"x=1\n"}],
|
|
},
|
|
)
|
|
|
|
|
|
def test_agent_runtime_projection_rejects_secret_constraint_keys():
|
|
request=task_request("task-hermes-runtime","hermes")
|
|
request["constraints"]={"api_token":"do-not-project"}
|
|
with pytest.raises(ExecutionProjectionError,match="AGENT_RUNTIME_SECRET_FIELD_DENY"):
|
|
project_online_task(
|
|
binding(),
|
|
request,
|
|
{
|
|
"action":"agent_runtime",
|
|
"adapter":"hermes",
|
|
"capability_scope":["agent.runtime.hermes"],
|
|
},
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("runtime",["hermes","openclaw"])
|
|
def test_external_executor_runs_root_owned_profile_without_shell(tmp_path,monkeypatch,runtime):
|
|
clear_secret_env(monkeypatch)
|
|
helper=make_helper(tmp_path)
|
|
profile_root=make_profile(tmp_path,runtime,helper)
|
|
request={
|
|
"schema":"aios.external-agent-runtime.request.v1",
|
|
"request_id":f"external-{runtime}-0123456789abcdef",
|
|
"runtime":runtime,
|
|
"task_id":f"task-{runtime}-runtime",
|
|
"execution_digest":"d"*64,
|
|
"payload":{
|
|
"goal":"Return ok.",
|
|
"instructions":"No side effects.",
|
|
"success_criteria":[],
|
|
"constraints":{},
|
|
},
|
|
}
|
|
request_path=tmp_path/"request.json"
|
|
result_path=tmp_path/"result.json"
|
|
request_path.write_text(json.dumps(request),encoding="utf-8")
|
|
execute_request(request_path,result_path,profile_root=profile_root)
|
|
result=json.loads(result_path.read_text())
|
|
assert result["runtime"]==runtime
|
|
assert result["exit_code"]==0
|
|
assert result["output_json"]["answer"]=="ok"
|
|
assert result["credential_material_exposed"] is False
|
|
assert result["control_plane_access_granted"] is False
|
|
|
|
|
|
def test_runtime_profile_rejects_shell_mode(tmp_path):
|
|
root=tmp_path/"profiles";root.mkdir()
|
|
(root/"hermes.json").write_text(json.dumps({
|
|
"schema":"aios.external-agent-runtime.profile.v1",
|
|
"runtime":"hermes",
|
|
"argv":["/usr/bin/python3","-c","print('x')"],
|
|
"input_mode":"stdin_text",
|
|
"output_mode":"stdout_text",
|
|
}))
|
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_SHELL_MODE_DENY"):
|
|
RuntimeProfile.load("hermes",root)
|
|
|
|
|
|
def test_external_executor_projects_only_selected_provider_credential(tmp_path,monkeypatch):
|
|
clear_secret_env(monkeypatch)
|
|
helper=tmp_path/"credential_helper.py"
|
|
helper.write_text(
|
|
"import os\n"
|
|
"value=os.environ.get('OPENAI_API_KEY','')\n"
|
|
"print('AIOS_PROVIDER_PROJECTED_OK' if value == 'test-provider-key' else 'AIOS_PROVIDER_PROJECTED_FAIL')\n",
|
|
encoding="utf-8",
|
|
)
|
|
profile_root=tmp_path/"profiles";profile_root.mkdir()
|
|
(profile_root/"hermes.json").write_text(json.dumps({
|
|
"schema":"aios.external-agent-runtime.profile.v1",
|
|
"runtime":"hermes",
|
|
"argv":["/usr/bin/python3",str(helper)],
|
|
"input_mode":"stdin_text",
|
|
"output_mode":"stdout_text",
|
|
"environment":{"RUNTIME_PROFILE_MARKER":"staging"},
|
|
"credential_env":{
|
|
"OPENAI_API_KEY":{
|
|
"credential":"provider.env",
|
|
"format":"dotenv",
|
|
"key":"ONEAPI_REGULAR_TOKEN",
|
|
}
|
|
},
|
|
}),encoding="utf-8")
|
|
credential_dir=tmp_path/"credentials";credential_dir.mkdir()
|
|
(credential_dir/"provider.env").write_text(
|
|
"ONEAPI_REGULAR_TOKEN=test-provider-key\nOTHER_PROVIDER_TOKEN=not-projected\n",
|
|
encoding="utf-8",
|
|
)
|
|
monkeypatch.setenv("CREDENTIALS_DIRECTORY",str(credential_dir))
|
|
request_path,result_path=make_text_request(tmp_path,"hermes")
|
|
execute_request(request_path,result_path,profile_root=profile_root)
|
|
result=json.loads(result_path.read_text())
|
|
assert "AIOS_PROVIDER_PROJECTED_OK" in result["output_text"]
|
|
assert "test-provider-key" not in result["output_text"]
|
|
assert result["credential_material_exposed"] is False
|
|
|
|
|
|
def test_runtime_profile_rejects_credential_path_traversal(tmp_path):
|
|
root=tmp_path/"profiles";root.mkdir()
|
|
(root/"hermes.json").write_text(json.dumps({
|
|
"schema":"aios.external-agent-runtime.profile.v1",
|
|
"runtime":"hermes",
|
|
"argv":["/usr/bin/python3","-V"],
|
|
"input_mode":"stdin_text",
|
|
"output_mode":"stdout_text",
|
|
"credential_env":{
|
|
"OPENAI_API_KEY":{
|
|
"credential":"../provider.env",
|
|
"format":"dotenv",
|
|
"key":"ONEAPI_REGULAR_TOKEN",
|
|
}
|
|
},
|
|
}),encoding="utf-8")
|
|
with pytest.raises(ExternalAgentRuntimeError,match="EXTERNAL_AGENT_CREDENTIAL_NAME_INVALID"):
|
|
RuntimeProfile.load("hermes",root)
|
|
|
|
|
|
def test_runtime_profile_prompt_placeholder_is_root_owned(tmp_path,monkeypatch):
|
|
clear_secret_env(monkeypatch)
|
|
helper=tmp_path/"prompt_helper.py"
|
|
helper.write_text(
|
|
"import sys\n"
|
|
"print('AIOS_PROMPT_PLACEHOLDER_OK' if 'Goal:' in sys.argv[1] else 'AIOS_PROMPT_PLACEHOLDER_FAIL')\n",
|
|
encoding="utf-8",
|
|
)
|
|
profile_root=tmp_path/"profiles";profile_root.mkdir()
|
|
(profile_root/"hermes.json").write_text(json.dumps({
|
|
"schema":"aios.external-agent-runtime.profile.v1",
|
|
"runtime":"hermes",
|
|
"argv":["/usr/bin/python3",str(helper),"{prompt}"],
|
|
"input_mode":"stdin_text",
|
|
"output_mode":"stdout_text",
|
|
}),encoding="utf-8")
|
|
request_path,result_path=make_text_request(tmp_path,"hermes")
|
|
execute_request(request_path,result_path,profile_root=profile_root)
|
|
result=json.loads(result_path.read_text())
|
|
assert "AIOS_PROMPT_PLACEHOLDER_OK" in result["output_text"]
|
|
|
|
|
|
@pytest.mark.parametrize("runtime,capability",[
|
|
("hermes","agent.runtime.hermes"),
|
|
("openclaw","agent.runtime.openclaw"),
|
|
])
|
|
def test_worker_external_adapter_builds_readonly_result(monkeypatch,tmp_path,runtime,capability):
|
|
mapped=projected(f"task-{runtime}-runtime",runtime)
|
|
envelope=worker.ExecutionEnvelope.from_task(
|
|
mapped,
|
|
worker_capabilities=(capability,),
|
|
)
|
|
class FakeExchange:
|
|
def __init__(self,*args,**kwargs): pass
|
|
def execute(self,**kwargs):
|
|
return {
|
|
"schema":"aios.external-agent-runtime.result.v1",
|
|
"request_id":kwargs["request_id"],
|
|
"request_sha256":"e"*64,
|
|
"runtime":runtime,
|
|
"task_id":kwargs["task_id"],
|
|
"execution_digest":kwargs["execution_digest"],
|
|
"exit_code":0,
|
|
"duration_ms":10,
|
|
"output_text":"done",
|
|
"output_sha256":"f"*64,
|
|
"stderr_sha256":"0"*64,
|
|
"credential_material_exposed":False,
|
|
"control_plane_access_granted":False,
|
|
"result_sha256":"1"*64,
|
|
}
|
|
monkeypatch.setattr(worker,"ExternalAgentExchange",FakeExchange)
|
|
monkeypatch.setattr(worker,"audit",lambda *a,**k:None)
|
|
executor=worker.HermesExecutor() if runtime=="hermes" else worker.OpenClawExecutor()
|
|
execution=executor.execute(envelope,tmp_path)
|
|
result=worker.build_agent_runtime_result(envelope.task_id,envelope,execution)
|
|
assert result["status"]=="SUCCEEDED"
|
|
assert result["classification"]=="EXTERNAL_AGENT_RUNTIME_SUCCEEDED"
|
|
assert result["execution_mode"]=="agent_runtime"
|
|
assert result["agent_runtime"]==runtime
|
|
assert result["writer_lease"]=="NONE"
|
|
assert result["changed_files"]==[]
|
|
assert result["control_plane_access_granted_to_runtime"] is False
|
|
|
|
|
|
def test_worker_adapter_filter_defaults_exclude_external_runtimes():
|
|
assert "hermes" not in worker.WORKER_ALLOWED_ADAPTERS
|
|
assert "openclaw" not in worker.WORKER_ALLOWED_ADAPTERS
|
|
|
|
|
|
def test_external_executor_systemd_has_no_control_credentials_or_repository_access():
|
|
unit=(ROOT/"deploy/systemd/aios-external-agent-executor.service").read_text()
|
|
assert "User=aios-engineering-executor" in unit
|
|
assert "LoadCredential=aios-external-agent-oneapi-env:/data/agent/secrets/oneapi-model-ready.env" in unit
|
|
assert "LoadCredentialEncrypted" not in unit
|
|
assert "InaccessiblePaths=/run/credentials" not in unit
|
|
assert "InaccessiblePaths=/etc/aios-gitops" in unit
|
|
assert "InaccessiblePaths=/data/aios-gitops" in unit
|
|
assert "InaccessiblePaths=/data/agent/secrets" in unit
|
|
assert "IPAddressDeny=127.0.0.0/8" in unit
|
|
assert "IPAddressDeny=::1/128" in unit
|
|
assert "IPAddressAllow=172.18.0.2/32" in unit
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"unit_name,capability,adapter,state_root",
|
|
[
|
|
(
|
|
"aios-hermes-online-agent-worker.service",
|
|
"agent.runtime.hermes",
|
|
"hermes",
|
|
"/var/lib/aios-online-agent-hermes",
|
|
),
|
|
(
|
|
"aios-openclaw-online-agent-worker.service",
|
|
"agent.runtime.openclaw",
|
|
"openclaw",
|
|
"/var/lib/aios-online-agent-openclaw",
|
|
),
|
|
],
|
|
)
|
|
def test_dedicated_worker_units_are_capability_and_adapter_scoped(
|
|
unit_name,capability,adapter,state_root
|
|
):
|
|
unit=(ROOT/"deploy/systemd"/unit_name).read_text()
|
|
assert f"Environment=AIOS_WORKER_CAPABILITIES={capability}" in unit
|
|
assert f"Environment=AIOS_WORKER_ALLOWED_ADAPTERS={adapter}" in unit
|
|
assert f"Environment=AIOS_WORKER_STATE_ROOT={state_root}" in unit
|
|
assert "AIOS_GOVERNED_WRITE_API_URL=http://127.0.0.1:18892" in unit
|
|
assert "LoadCredentialEncrypted=agent_token:" in unit
|
|
|
|
|
|
def test_example_profiles_are_non_authoritative_examples_only():
|
|
for runtime in ("hermes","openclaw"):
|
|
value=json.loads(
|
|
(ROOT/"config/external-agent-runtime.d"/f"{runtime}.example.json").read_text()
|
|
)
|
|
assert value["schema"]=="aios.external-agent-runtime.profile.v1"
|
|
assert value["runtime"]==runtime
|
|
assert value["input_mode"]=="stdin_text"
|
|
assert value["output_mode"]=="stdout_text"
|
|
|
|
|
|
def test_staging_profiles_are_provider_only_and_root_profile_bound():
|
|
hermes=json.loads((ROOT/"config/external-agent-runtime.d/hermes.profile.json").read_text())
|
|
openclaw=json.loads((ROOT/"config/external-agent-runtime.d/openclaw.profile.json").read_text())
|
|
for runtime,value in (("hermes",hermes),("openclaw",openclaw)):
|
|
assert value["runtime"]==runtime
|
|
assert set(value["credential_env"])=={"OPENAI_API_KEY"}
|
|
assert value["credential_env"]["OPENAI_API_KEY"]["credential"]=="aios-external-agent-oneapi-env"
|
|
assert "OPENAI_API_KEY" not in value.get("environment",{})
|