385 lines
15 KiB
Python
385 lines
15 KiB
Python
"""Bounded read-only repository interface map for governed Online Agent analysis.
|
|
|
|
The compact map intentionally returns symbols and short source excerpts rather than a
|
|
source archive. It is designed for control-plane consumers that cannot safely ingest a
|
|
multi-megabyte repository snapshot.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
from typing import Any, Iterable, Mapping, Sequence
|
|
|
|
|
|
SCHEMA = "aios.engineering.interface-map.v1"
|
|
MAX_RESULT_BYTES = 120 * 1024
|
|
TARGET_RESULT_BYTES = 108 * 1024
|
|
MAX_MODULES = 40
|
|
MAX_SYMBOLS = 80
|
|
MAX_TESTS = 40
|
|
MAX_EXCERPTS = 25
|
|
MAX_EXCERPT_CHARS = 1200
|
|
MAX_FILE_BYTES = 2 * 1024 * 1024
|
|
|
|
ROOTS = (
|
|
"active-runtime/app",
|
|
"runtime/agent-os",
|
|
"tests",
|
|
"db/migrations",
|
|
"migrations",
|
|
"deploy/systemd",
|
|
"docs/task-plane",
|
|
"docs/superpowers/specs",
|
|
)
|
|
SUFFIXES = {".py", ".sql", ".service", ".md", ".json", ".yaml", ".yml"}
|
|
|
|
DEFAULT_TERMS = {
|
|
"agent", "worker", "fleet", "scheduler", "scheduling", "task", "claim", "lease",
|
|
"heartbeat", "capacity", "concurrency", "priority", "fairness", "queue", "retry",
|
|
"backoff", "circuit", "health", "liveness", "scope", "tenant", "project", "trace",
|
|
"message", "checkpoint", "recovery", "idempotency", "dispatch", "assignment",
|
|
}
|
|
CONTRACT_TERMS: dict[str, tuple[str, ...]] = {
|
|
"FLEET_AGENT_REGISTRATION": ("agent", "register", "worker", "capability", "version"),
|
|
"FLEET_CAPACITY_SCHEDULING": ("capacity", "slot", "concurrency", "scheduler", "queue"),
|
|
"FLEET_FAIRNESS": ("fair", "priority", "queue", "tenant", "project", "starvation"),
|
|
"FLEET_FAULT_ISOLATION": ("failure", "health", "liveness", "timeout", "circuit", "poison"),
|
|
"FLEET_WORKER_REASSIGNMENT": ("reassign", "assignment", "claim", "lease", "retry", "worker"),
|
|
"FLEET_SCOPE_ISOLATION": ("tenant", "project", "scope", "conversation", "task", "run"),
|
|
"FLEET_IDEMPOTENT_DISPATCH": ("idempot", "dispatch", "claim", "lease", "result", "attempt"),
|
|
"FLEET_FAILURE_RECOVERY": ("checkpoint", "resume", "recover", "retry", "cleanup", "receipt"),
|
|
}
|
|
SENSITIVE_RE = re.compile(
|
|
r"(?i)(authorization\s*:\s*bearer\s+\S+|"
|
|
r"(?:password|passwd|secret|token|api[_-]?key)\s*[:=]\s*['\"]?[^'\"\s,;]+)"
|
|
)
|
|
SQL_TABLE_RE = re.compile(
|
|
r"(?is)\bcreate\s+table\s+(?:if\s+not\s+exists\s+)?[\"`[]?([A-Za-z_][A-Za-z0-9_.-]*)"
|
|
)
|
|
|
|
|
|
class CompactInterfaceMapError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def _sha(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def _bounded_excerpt(text: str, start_line: int, end_line: int) -> str:
|
|
lines = text.splitlines()
|
|
start = max(0, start_line - 2)
|
|
end = min(len(lines), max(end_line + 1, start + 1))
|
|
excerpt = "\n".join(lines[start:end])
|
|
excerpt = SENSITIVE_RE.sub("<redacted-sensitive-source-pattern>", excerpt)
|
|
if len(excerpt) > MAX_EXCERPT_CHARS:
|
|
excerpt = excerpt[: MAX_EXCERPT_CHARS - 1] + "…"
|
|
return excerpt
|
|
|
|
|
|
def _signal_terms(signals: Sequence[str]) -> set[str]:
|
|
terms = set(DEFAULT_TERMS)
|
|
for signal in signals:
|
|
if not isinstance(signal, str):
|
|
continue
|
|
terms.update(
|
|
token.lower()
|
|
for token in re.findall(r"[A-Za-z][A-Za-z0-9_.-]{2,63}", signal)
|
|
)
|
|
return set(sorted(terms)[:96])
|
|
|
|
|
|
def _candidate_files(root: Path, terms: set[str]) -> list[dict[str, Any]]:
|
|
root = root.resolve()
|
|
candidates: list[dict[str, Any]] = []
|
|
for relative_root in ROOTS:
|
|
directory = (root / relative_root).resolve(strict=False)
|
|
if directory == root or root not in directory.parents:
|
|
raise CompactInterfaceMapError("INTERFACE_MAP_SCOPE_INVALID")
|
|
if not directory.is_dir() or directory.is_symlink():
|
|
continue
|
|
for path in sorted(directory.rglob("*")):
|
|
if path.is_symlink() or not path.is_file() or path.suffix.lower() not in SUFFIXES:
|
|
continue
|
|
resolved = path.resolve(strict=True)
|
|
if root not in resolved.parents:
|
|
raise CompactInterfaceMapError("INTERFACE_MAP_PATH_ESCAPE")
|
|
data = path.read_bytes()
|
|
if len(data) > MAX_FILE_BYTES:
|
|
continue
|
|
relative = path.relative_to(root).as_posix()
|
|
text = data.decode("utf-8", "ignore")
|
|
searchable = (relative + "\n" + text[:384000]).lower()
|
|
matched = sorted(term for term in terms if term in searchable)
|
|
if not matched:
|
|
continue
|
|
path_bonus = sum(3 for term in terms if term in relative.lower())
|
|
test_bonus = 3 if relative.startswith("tests/") else 0
|
|
score = min(250, len(matched) + path_bonus + test_bonus)
|
|
candidates.append(
|
|
{
|
|
"path": relative,
|
|
"sha256": _sha(data),
|
|
"bytes": len(data),
|
|
"score": score,
|
|
"text": text,
|
|
"suffix": path.suffix.lower(),
|
|
}
|
|
)
|
|
candidates.sort(key=lambda item: (-item["score"], item["path"]))
|
|
return candidates
|
|
|
|
|
|
def _python_symbols(item: Mapping[str, Any], terms: set[str]) -> list[dict[str, Any]]:
|
|
try:
|
|
tree = ast.parse(str(item["text"]), filename=str(item["path"]))
|
|
except SyntaxError:
|
|
return []
|
|
result: list[dict[str, Any]] = []
|
|
|
|
class Visitor(ast.NodeVisitor):
|
|
def __init__(self) -> None:
|
|
self.stack: list[str] = []
|
|
|
|
def _record(self, node: ast.AST, kind: str, name: str) -> None:
|
|
qualified = ".".join([*self.stack, name]) if self.stack else name
|
|
haystack = f"{item['path']} {qualified}".lower()
|
|
score = sum(3 if term in haystack else 0 for term in terms)
|
|
if name.startswith("test_"):
|
|
score += 12
|
|
result.append(
|
|
{
|
|
"path": item["path"],
|
|
"symbol": qualified,
|
|
"kind": kind,
|
|
"line": int(getattr(node, "lineno", 1)),
|
|
"end_line": int(getattr(node, "end_lineno", getattr(node, "lineno", 1))),
|
|
"score": score + int(item["score"]),
|
|
}
|
|
)
|
|
|
|
def visit_ClassDef(self, node: ast.ClassDef) -> Any:
|
|
self._record(node, "class", node.name)
|
|
self.stack.append(node.name)
|
|
self.generic_visit(node)
|
|
self.stack.pop()
|
|
|
|
def visit_FunctionDef(self, node: ast.FunctionDef) -> Any:
|
|
self._record(node, "function", node.name)
|
|
self.stack.append(node.name)
|
|
self.generic_visit(node)
|
|
self.stack.pop()
|
|
|
|
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> Any:
|
|
self._record(node, "async_function", node.name)
|
|
self.stack.append(node.name)
|
|
self.generic_visit(node)
|
|
self.stack.pop()
|
|
|
|
Visitor().visit(tree)
|
|
return result
|
|
|
|
|
|
def _surface_ref(symbol: Mapping[str, Any]) -> str:
|
|
return f"{symbol['path']}::{symbol['symbol']}"
|
|
|
|
|
|
def _contract_map(
|
|
symbols: Sequence[Mapping[str, Any]],
|
|
modules: Sequence[Mapping[str, Any]],
|
|
) -> dict[str, Any]:
|
|
result: dict[str, Any] = {}
|
|
module_paths = [str(item["path"]) for item in modules]
|
|
for contract, terms in CONTRACT_TERMS.items():
|
|
scored: list[tuple[int, str]] = []
|
|
for symbol in symbols:
|
|
haystack = _surface_ref(symbol).lower()
|
|
score = sum(1 for term in terms if term in haystack)
|
|
if score:
|
|
scored.append((score, _surface_ref(symbol)))
|
|
scored.sort(key=lambda item: (-item[0], item[1]))
|
|
surfaces = [value for _, value in scored[:6]]
|
|
if not surfaces:
|
|
for path in module_paths:
|
|
score = sum(1 for term in terms if term in path.lower())
|
|
if score:
|
|
surfaces.append(path)
|
|
if len(surfaces) == 6:
|
|
break
|
|
result[contract] = {
|
|
"status": "PARTIAL" if surfaces else "GAP",
|
|
"surfaces": surfaces,
|
|
}
|
|
return result
|
|
|
|
|
|
def _gap_evidence(symbols: Sequence[Mapping[str, Any]]) -> tuple[list[str], list[str], list[str]]:
|
|
haystack = "\n".join(_surface_ref(item).lower() for item in symbols)
|
|
|
|
invariants = [
|
|
"Online Task claim/heartbeat remains the authoritative worker ownership boundary.",
|
|
"Governed mutation remains separated from read-only analysis and requires WriterLease.",
|
|
"P1 attempt-aware result idempotency and durable checkpoint recovery are reusable fleet primitives.",
|
|
"Scope/trace/message lineage must remain attached when work is scheduled or reassigned.",
|
|
]
|
|
failure_windows: list[str] = []
|
|
if "scheduler" not in haystack:
|
|
failure_windows.append("No first-class durable fleet scheduler symbol detected; worker-local polling may still be the scheduling mechanism.")
|
|
if "capacity" not in haystack and "slot" not in haystack:
|
|
failure_windows.append("No explicit durable worker-capacity/slot primitive detected.")
|
|
if "fair" not in haystack and "starvation" not in haystack:
|
|
failure_windows.append("No explicit fairness/starvation policy symbol detected.")
|
|
if "circuit" not in haystack:
|
|
failure_windows.append("No explicit circuit-breaker primitive detected for repeated worker/task failures.")
|
|
if "reassign" not in haystack:
|
|
failure_windows.append("No explicit worker-reassignment primitive detected beyond lease expiry/retry semantics.")
|
|
|
|
reuse_points: list[str] = []
|
|
desired = (
|
|
("checkpoint", "P1 durable checkpoint/resume"),
|
|
("message", "P0-B MessageEnvelope/handoff"),
|
|
("trace", "P0-C trace correlation"),
|
|
("scope", "P0-A scope isolation"),
|
|
("lease", "claim/WriterLease ownership"),
|
|
("heartbeat", "worker liveness"),
|
|
("idempot", "retry-safe dispatch/result"),
|
|
)
|
|
for token, label in desired:
|
|
match = next((item for item in symbols if token in _surface_ref(item).lower()), None)
|
|
if match is not None:
|
|
reuse_points.append(f"{label}: {_surface_ref(match)}")
|
|
return invariants, failure_windows, reuse_points
|
|
|
|
|
|
def _trim_to_budget(value: dict[str, Any]) -> dict[str, Any]:
|
|
def size() -> int:
|
|
return len(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8"))
|
|
|
|
while size() > TARGET_RESULT_BYTES and value["bounded_excerpts"]:
|
|
value["bounded_excerpts"].pop()
|
|
while size() > TARGET_RESULT_BYTES and len(value["implementation_map"]["functions"]) > 20:
|
|
value["implementation_map"]["functions"].pop()
|
|
while size() > TARGET_RESULT_BYTES and len(value["implementation_map"]["classes"]) > 15:
|
|
value["implementation_map"]["classes"].pop()
|
|
while size() > TARGET_RESULT_BYTES and len(value["implementation_map"]["tests"]) > 15:
|
|
value["implementation_map"]["tests"].pop()
|
|
while size() > TARGET_RESULT_BYTES and len(value["implementation_map"]["modules"]) > 20:
|
|
value["implementation_map"]["modules"].pop()
|
|
final_size = size()
|
|
if final_size > MAX_RESULT_BYTES:
|
|
raise CompactInterfaceMapError("INTERFACE_MAP_RESULT_TOO_LARGE")
|
|
value["result_bytes"] = final_size
|
|
return value
|
|
|
|
|
|
def build_compact_interface_map(
|
|
worktree: Path,
|
|
source_signals: Sequence[str],
|
|
*,
|
|
source_commit: str,
|
|
) -> dict[str, Any]:
|
|
root = Path(worktree)
|
|
if not root.is_dir() or root.is_symlink():
|
|
raise CompactInterfaceMapError("INTERFACE_MAP_WORKTREE_INVALID")
|
|
if re.fullmatch(r"[0-9a-f]{40}", source_commit) is None:
|
|
raise CompactInterfaceMapError("INTERFACE_MAP_SOURCE_COMMIT_INVALID")
|
|
|
|
terms = _signal_terms(source_signals)
|
|
candidates = _candidate_files(root, terms)
|
|
if not candidates:
|
|
raise CompactInterfaceMapError("INTERFACE_MAP_EMPTY")
|
|
|
|
module_candidates = candidates[:MAX_MODULES]
|
|
modules = [
|
|
{
|
|
"path": item["path"],
|
|
"sha256": item["sha256"],
|
|
"bytes": item["bytes"],
|
|
"score": item["score"],
|
|
}
|
|
for item in module_candidates
|
|
]
|
|
|
|
symbols: list[dict[str, Any]] = []
|
|
tables: list[dict[str, Any]] = []
|
|
systemd_units: list[dict[str, Any]] = []
|
|
for item in candidates:
|
|
if item["suffix"] == ".py":
|
|
symbols.extend(_python_symbols(item, terms))
|
|
elif item["suffix"] == ".sql":
|
|
for table in SQL_TABLE_RE.findall(str(item["text"])):
|
|
tables.append({"path": item["path"], "table": table, "sha256": item["sha256"]})
|
|
elif item["suffix"] == ".service":
|
|
systemd_units.append({"path": item["path"], "unit": Path(str(item["path"])).name, "sha256": item["sha256"]})
|
|
|
|
symbols.sort(key=lambda item: (-int(item["score"]), str(item["path"]), str(item["symbol"])))
|
|
all_symbols = list(symbols)
|
|
test_symbols = [
|
|
item for item in all_symbols
|
|
if str(item["path"]).startswith("tests/") or str(item["symbol"]).split(".")[-1].startswith("test_")
|
|
][: min(MAX_TESTS, 24)]
|
|
non_test_symbols = [
|
|
item for item in all_symbols
|
|
if item not in test_symbols
|
|
][: max(0, MAX_SYMBOLS - len(test_symbols))]
|
|
symbols = [*test_symbols, *non_test_symbols]
|
|
symbols.sort(key=lambda item: (-int(item["score"]), str(item["path"]), str(item["symbol"])))
|
|
classes = [
|
|
{k: item[k] for k in ("path", "symbol", "line", "end_line")}
|
|
for item in symbols
|
|
if item["kind"] == "class"
|
|
]
|
|
functions = [
|
|
{k: item[k] for k in ("path", "symbol", "line", "end_line")}
|
|
for item in symbols
|
|
if item["kind"] in {"function", "async_function"} and not str(item["symbol"]).split(".")[-1].startswith("test_")
|
|
]
|
|
tests = [
|
|
{k: item[k] for k in ("path", "symbol", "line", "end_line")}
|
|
for item in symbols
|
|
if str(item["path"]).startswith("tests/") or str(item["symbol"]).split(".")[-1].startswith("test_")
|
|
][:MAX_TESTS]
|
|
|
|
excerpts: list[dict[str, Any]] = []
|
|
by_path = {str(item["path"]): item for item in candidates}
|
|
for symbol in symbols:
|
|
if len(excerpts) == MAX_EXCERPTS:
|
|
break
|
|
item = by_path.get(str(symbol["path"]))
|
|
if item is None:
|
|
continue
|
|
excerpt = _bounded_excerpt(str(item["text"]), int(symbol["line"]), int(symbol["end_line"]))
|
|
if not excerpt.strip():
|
|
continue
|
|
excerpts.append(
|
|
{
|
|
"path": symbol["path"],
|
|
"symbol": symbol["symbol"],
|
|
"sha256": item["sha256"],
|
|
"excerpt": excerpt,
|
|
}
|
|
)
|
|
|
|
invariants, failure_windows, reuse_points = _gap_evidence(all_symbols)
|
|
result = {
|
|
"schema": SCHEMA,
|
|
"source_commit": source_commit,
|
|
"implementation_map": {
|
|
"modules": modules,
|
|
"classes": classes[:MAX_SYMBOLS],
|
|
"functions": functions[:MAX_SYMBOLS],
|
|
"tables": tables[:40],
|
|
"systemd_units": systemd_units[:20],
|
|
"tests": tests,
|
|
"current_invariants": invariants,
|
|
"failure_windows": failure_windows,
|
|
"reuse_points": reuse_points,
|
|
},
|
|
"bounded_excerpts": excerpts,
|
|
"contracts": _contract_map(all_symbols, modules),
|
|
}
|
|
return _trim_to_budget(result)
|