968 lines
36 KiB
Python
968 lines
36 KiB
Python
"""Bounded, read-only discovery over a verified immutable Git repository.
|
|
|
|
The module is deliberately independent from Task, lease, credential, worktree,
|
|
and write code. Every public operation verifies the exact repository identity,
|
|
resolves a floating ref once to a full commit OID, and then reads only that
|
|
immutable snapshot.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
from collections.abc import Iterable, Iterator, Mapping
|
|
import fnmatch
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import time
|
|
|
|
from authoritative_git import AuthoritativeGitError, verify_repository
|
|
|
|
|
|
_OID_RE = re.compile(r"^[0-9a-f]{40}$")
|
|
_REF_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,239}$")
|
|
_IDENTITY_RE = re.compile(
|
|
r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}/[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
|
|
)
|
|
_READ_ONLY_COMMANDS = frozenset({"cat-file", "diff", "ls-tree", "rev-parse"})
|
|
_MAX_COMMAND_OUTPUT = 1_048_576
|
|
_MAX_RECORD_BYTES = 1_048_576
|
|
_MAX_LIMIT = 1_000
|
|
_MAX_DEPTH = 32
|
|
_MAX_BYTES = 1_048_576
|
|
_MAX_GLOBS = 16
|
|
_MAX_QUERY_LENGTH = 512
|
|
_MAX_CURSOR_LENGTH = 4_096
|
|
_GIT_TIMEOUT_SECONDS = 8.0
|
|
|
|
|
|
class RepositoryDiscoveryError(RuntimeError):
|
|
"""A fail-closed repository discovery request error."""
|
|
|
|
def __init__(self, code: str, *, status_code: int | None = None) -> None:
|
|
self.code = str(code)
|
|
self.status_code = (
|
|
status_code if status_code is not None else self._status_for(self.code)
|
|
)
|
|
super().__init__(self.code)
|
|
|
|
@staticmethod
|
|
def _status_for(code: str) -> int:
|
|
if "OUTPUT_LIMIT" in code:
|
|
return 413
|
|
if code.endswith("_NOT_FOUND"):
|
|
return 404
|
|
if code in {"REPOSITORY_NOT_ALLOWED", "REPOSITORY_IDENTITY_MISMATCH"}:
|
|
return 403
|
|
if code in {
|
|
"REPOSITORY_DISCOVERY_UNAVAILABLE",
|
|
"REPOSITORY_DISCOVERY_GIT_FAILED",
|
|
"REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID",
|
|
"REPOSITORY_DISCOVERY_CURSOR_SECRET_REQUIRED",
|
|
}:
|
|
return 503
|
|
return 400
|
|
|
|
|
|
class RepositoryDiscoveryNotFound(RepositoryDiscoveryError):
|
|
"""A validated immutable object or path was not found."""
|
|
|
|
|
|
class RepositoryDiscoveryOutputTooLarge(RepositoryDiscoveryError):
|
|
"""A fixed read-only Git query exceeded its collection boundary."""
|
|
|
|
|
|
def _error(code: str) -> RepositoryDiscoveryError:
|
|
return RepositoryDiscoveryError(code)
|
|
|
|
|
|
def _request_mapping(request: Mapping[str, object]) -> Mapping[str, object]:
|
|
if not isinstance(request, Mapping):
|
|
raise _error("REPOSITORY_DISCOVERY_REQUEST_INVALID")
|
|
return request
|
|
|
|
|
|
def _required_request_value(request: Mapping[str, object], field: str) -> object:
|
|
value = request.get(field)
|
|
if value is None:
|
|
raise _error("REPOSITORY_DISCOVERY_REQUEST_INVALID")
|
|
return value
|
|
|
|
|
|
def _canonical(value: Mapping[str, object]) -> bytes:
|
|
return json.dumps(
|
|
value, sort_keys=True, separators=(",", ":"), ensure_ascii=True
|
|
).encode("ascii")
|
|
|
|
|
|
def _positive_int(value: object, field: str, maximum: int) -> int:
|
|
if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= maximum:
|
|
raise _error(f"REPOSITORY_DISCOVERY_INVALID_{field.upper()}")
|
|
return value
|
|
|
|
|
|
def _safe_relative_path(
|
|
value: object,
|
|
field: str,
|
|
*,
|
|
allow_empty: bool,
|
|
allow_glob: bool = False,
|
|
) -> str:
|
|
if not isinstance(value, str) or value != value.strip() or "\x00" in value:
|
|
raise _error("REPOSITORY_DISCOVERY_PATH_INVALID")
|
|
if "\\" in value:
|
|
raise _error("REPOSITORY_DISCOVERY_PATH_INVALID")
|
|
if value in ("", "."):
|
|
if allow_empty:
|
|
return ""
|
|
raise _error("REPOSITORY_DISCOVERY_PATH_INVALID")
|
|
if value.startswith(("/", "-")) or "//" in value or ":" in value:
|
|
raise _error("REPOSITORY_DISCOVERY_PATH_INVALID")
|
|
if not allow_glob and any(character in value for character in "*?["):
|
|
raise _error("REPOSITORY_DISCOVERY_PATH_INVALID")
|
|
parts = value.split("/")
|
|
if any(part in ("", ".", "..") for part in parts):
|
|
raise _error("REPOSITORY_DISCOVERY_PATH_INVALID")
|
|
if len(value) > 512:
|
|
raise _error("REPOSITORY_DISCOVERY_PATH_INVALID")
|
|
return value
|
|
|
|
|
|
def _safe_glob(value: object) -> str:
|
|
return _safe_relative_path(value, "path_glob", allow_empty=False, allow_glob=True)
|
|
|
|
|
|
def _safe_ref(value: str) -> bool:
|
|
return bool(
|
|
_REF_RE.fullmatch(value)
|
|
and ".." not in value
|
|
and "//" not in value
|
|
and "@{" not in value
|
|
and not value.endswith(("/", ".", ".lock"))
|
|
and not any(part in ("", ".", "..") for part in value.split("/"))
|
|
)
|
|
|
|
|
|
def _parse_tree_record(record: bytes) -> dict[str, str] | None:
|
|
try:
|
|
header, raw_path = record.split(b"\t", 1)
|
|
mode, kind, oid = header.decode("ascii").split(" ", 2)
|
|
path = raw_path.decode("utf-8")
|
|
except (UnicodeDecodeError, ValueError) as exc:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID") from exc
|
|
if kind not in {"blob", "tree", "commit"} or not _OID_RE.fullmatch(oid.lower()):
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID")
|
|
try:
|
|
_safe_relative_path(path, "tree_path", allow_empty=False)
|
|
except RepositoryDiscoveryError:
|
|
# Git permits filenames outside the public POSIX-relative contract.
|
|
# They are not exposed as usable discovery paths.
|
|
return None
|
|
return {"mode": mode, "type": kind, "object_oid": oid.lower(), "path": path}
|
|
|
|
|
|
def _decode_output_path(raw: bytes) -> str:
|
|
try:
|
|
value = raw.decode("utf-8")
|
|
except UnicodeDecodeError as exc:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID") from exc
|
|
return _safe_relative_path(value, "git_output_path", allow_empty=False)
|
|
|
|
|
|
class RepositoryDiscovery:
|
|
"""Repository-scoped immutable Git discovery with signed cursors."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
allowed_repositories: Iterable[str] | None = None,
|
|
git_dir: str | Path | None = None,
|
|
cursor_secret: str | bytes | None = None,
|
|
) -> None:
|
|
configured = allowed_repositories
|
|
if configured is None:
|
|
configured = (os.getenv("AIOS_GITOPS_REPOSITORY_IDENTITY", ""),)
|
|
identities = tuple(configured)
|
|
if not identities or any(
|
|
not isinstance(item, str) or not _IDENTITY_RE.fullmatch(item)
|
|
for item in identities
|
|
):
|
|
raise _error("REPOSITORY_DISCOVERY_ALLOWLIST_INVALID")
|
|
self._allowed_repositories = frozenset(identities)
|
|
self._git_dir = git_dir
|
|
secret = (
|
|
cursor_secret
|
|
if cursor_secret is not None
|
|
else os.getenv("AIOS_REPOSITORY_DISCOVERY_CURSOR_SECRET", "")
|
|
)
|
|
if isinstance(secret, str):
|
|
secret = secret.encode("utf-8")
|
|
if not isinstance(secret, bytes) or len(secret) < 16:
|
|
raise _error("REPOSITORY_DISCOVERY_CURSOR_SECRET_REQUIRED")
|
|
self._cursor_secret = secret
|
|
|
|
def _repository_path(self, repository: object) -> Path:
|
|
if not isinstance(repository, str) or repository not in self._allowed_repositories:
|
|
raise _error("REPOSITORY_NOT_ALLOWED")
|
|
try:
|
|
path, actual_identity = verify_repository(
|
|
git_dir=self._git_dir,
|
|
repository_identity=repository,
|
|
)
|
|
except AuthoritativeGitError as exc:
|
|
code = (
|
|
"REPOSITORY_IDENTITY_MISMATCH"
|
|
if "REPOSITORY_IDENTITY_MISMATCH" in str(exc)
|
|
else "REPOSITORY_DISCOVERY_UNAVAILABLE"
|
|
)
|
|
raise _error(code) from exc
|
|
if actual_identity != repository:
|
|
raise _error("REPOSITORY_IDENTITY_MISMATCH")
|
|
return path
|
|
|
|
@staticmethod
|
|
def _git_env() -> dict[str, str]:
|
|
return {
|
|
**os.environ,
|
|
"GIT_OPTIONAL_LOCKS": "0",
|
|
"GIT_CONFIG_NOSYSTEM": "1",
|
|
"GIT_TERMINAL_PROMPT": "0",
|
|
}
|
|
|
|
@staticmethod
|
|
def _run(
|
|
path: Path,
|
|
args: list[str],
|
|
*,
|
|
max_output: int = _MAX_COMMAND_OUTPUT,
|
|
) -> bytes:
|
|
if not args or args[0] not in _READ_ONLY_COMMANDS:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_COMMAND_REJECTED")
|
|
output_limit = min(max_output, _MAX_COMMAND_OUTPUT)
|
|
process: subprocess.Popen[bytes] | None = None
|
|
try:
|
|
process = subprocess.Popen(
|
|
["git", "--git-dir", str(path), *args],
|
|
shell=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
env=RepositoryDiscovery._git_env(),
|
|
)
|
|
assert process.stdout is not None
|
|
output = process.stdout.read(output_limit + 1)
|
|
if len(output) > output_limit:
|
|
process.kill()
|
|
process.wait(timeout=1)
|
|
raise RepositoryDiscoveryOutputTooLarge(
|
|
"REPOSITORY_DISCOVERY_OUTPUT_LIMIT"
|
|
)
|
|
returncode = process.wait(timeout=_GIT_TIMEOUT_SECONDS)
|
|
except RepositoryDiscoveryError:
|
|
raise
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
if process is not None:
|
|
try:
|
|
process.kill()
|
|
process.wait(timeout=1)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
pass
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_FAILED") from exc
|
|
if returncode != 0:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_FAILED")
|
|
return output
|
|
|
|
@staticmethod
|
|
def _read_blob_prefix(path: Path, oid: str, maximum: int) -> tuple[bytes, bool]:
|
|
"""Read at most maximum bytes plus one sentinel byte."""
|
|
process: subprocess.Popen[bytes] | None = None
|
|
try:
|
|
process = subprocess.Popen(
|
|
["git", "--git-dir", str(path), "cat-file", "blob", oid],
|
|
shell=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
env=RepositoryDiscovery._git_env(),
|
|
)
|
|
assert process.stdout is not None
|
|
content = process.stdout.read(maximum + 1)
|
|
truncated = len(content) > maximum
|
|
if truncated:
|
|
process.kill()
|
|
process.wait(timeout=1)
|
|
return content[:maximum], True
|
|
if process.wait(timeout=_GIT_TIMEOUT_SECONDS) != 0:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_FAILED")
|
|
return content, False
|
|
except RepositoryDiscoveryError:
|
|
raise
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
if process is not None:
|
|
try:
|
|
process.kill()
|
|
process.wait(timeout=1)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
pass
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_FAILED") from exc
|
|
|
|
def _iter_nul_records(self, path: Path, args: list[str]) -> Iterator[bytes]:
|
|
if not args or args[0] not in _READ_ONLY_COMMANDS:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_COMMAND_REJECTED")
|
|
process: subprocess.Popen[bytes] | None = None
|
|
buffer = b""
|
|
started = time.monotonic()
|
|
try:
|
|
process = subprocess.Popen(
|
|
["git", "--git-dir", str(path), *args],
|
|
shell=False,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL,
|
|
env=self._git_env(),
|
|
)
|
|
assert process.stdout is not None
|
|
while True:
|
|
if time.monotonic() - started > _GIT_TIMEOUT_SECONDS:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_FAILED")
|
|
chunk = process.stdout.read(64 * 1024)
|
|
if not chunk:
|
|
break
|
|
buffer += chunk
|
|
if len(buffer) > _MAX_RECORD_BYTES and b"\0" not in buffer:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID")
|
|
while b"\0" in buffer:
|
|
record, buffer = buffer.split(b"\0", 1)
|
|
if len(record) > _MAX_RECORD_BYTES:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID")
|
|
yield record
|
|
if buffer:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID")
|
|
if process.wait(
|
|
timeout=max(0.1, _GIT_TIMEOUT_SECONDS - (time.monotonic() - started))
|
|
) != 0:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_FAILED")
|
|
except RepositoryDiscoveryError:
|
|
raise
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_FAILED") from exc
|
|
finally:
|
|
if process is not None and process.poll() is None:
|
|
try:
|
|
process.kill()
|
|
process.wait(timeout=1)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
pass
|
|
|
|
def _resolve_at(self, path: Path, revision: object, field: str) -> str:
|
|
if (
|
|
not isinstance(revision, str)
|
|
or revision != revision.strip()
|
|
or not revision
|
|
or revision.startswith("-")
|
|
):
|
|
raise _error("REPOSITORY_DISCOVERY_INVALID_REVISION")
|
|
if _OID_RE.fullmatch(revision.lower()):
|
|
candidates = [revision.lower()]
|
|
elif revision == "HEAD":
|
|
candidates = ["HEAD"]
|
|
elif _safe_ref(revision) and not any(
|
|
token in revision for token in ("^", "~", ":", "\\")
|
|
):
|
|
if revision.startswith("refs/"):
|
|
if not revision.startswith(("refs/heads/", "refs/tags/")):
|
|
raise _error("REPOSITORY_DISCOVERY_INVALID_REVISION")
|
|
candidates = [revision]
|
|
else:
|
|
candidates = [f"refs/heads/{revision}", f"refs/tags/{revision}"]
|
|
else:
|
|
raise _error("REPOSITORY_DISCOVERY_INVALID_REVISION")
|
|
resolved: list[str] = []
|
|
for candidate in candidates:
|
|
try:
|
|
output = self._run(
|
|
path,
|
|
["rev-parse", "--verify", f"{candidate}^{{commit}}"],
|
|
max_output=256,
|
|
)
|
|
except RepositoryDiscoveryError:
|
|
continue
|
|
oid = output.decode("ascii", errors="ignore").strip().lower()
|
|
if _OID_RE.fullmatch(oid):
|
|
resolved.append(oid)
|
|
if not resolved:
|
|
raise RepositoryDiscoveryNotFound(
|
|
f"REPOSITORY_DISCOVERY_{field.upper()}_NOT_FOUND"
|
|
)
|
|
if len(set(resolved)) != 1:
|
|
raise _error("REPOSITORY_DISCOVERY_AMBIGUOUS_REVISION")
|
|
return resolved[0]
|
|
|
|
def resolve_revision(self, repository: str, revision: str) -> str:
|
|
return self._resolve_at(self._repository_path(repository), revision, "revision")
|
|
|
|
def _cursor_offset(
|
|
self,
|
|
cursor: str | None,
|
|
*,
|
|
operation: str,
|
|
repository: str,
|
|
resolved_oid: str,
|
|
query: Mapping[str, object],
|
|
) -> int:
|
|
if cursor is None:
|
|
return 0
|
|
if (
|
|
not isinstance(cursor, str)
|
|
or not cursor
|
|
or len(cursor) > _MAX_CURSOR_LENGTH
|
|
or "." not in cursor
|
|
):
|
|
raise _error("REPOSITORY_DISCOVERY_CURSOR_INVALID")
|
|
encoded, supplied_signature = cursor.split(".", 1)
|
|
try:
|
|
padded = encoded + "=" * (-len(encoded) % 4)
|
|
payload = json.loads(
|
|
base64.urlsafe_b64decode(padded.encode("ascii")).decode("ascii")
|
|
)
|
|
except (UnicodeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise _error("REPOSITORY_DISCOVERY_CURSOR_INVALID") from exc
|
|
expected_signature = hmac.new(
|
|
self._cursor_secret, encoded.encode("ascii"), hashlib.sha256
|
|
).hexdigest()
|
|
if not hmac.compare_digest(supplied_signature, expected_signature):
|
|
raise _error("REPOSITORY_DISCOVERY_CURSOR_INVALID")
|
|
expected = {
|
|
"operation": operation,
|
|
"repository": repository,
|
|
"resolved_oid": resolved_oid,
|
|
"query": dict(query),
|
|
}
|
|
if (
|
|
not isinstance(payload, dict)
|
|
or payload.get("v") != 1
|
|
or any(payload.get(key) != value for key, value in expected.items())
|
|
):
|
|
raise _error("REPOSITORY_DISCOVERY_CURSOR_INVALID")
|
|
offset = payload.get("offset")
|
|
if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
|
|
raise _error("REPOSITORY_DISCOVERY_CURSOR_INVALID")
|
|
return offset
|
|
|
|
def _next_cursor(
|
|
self,
|
|
*,
|
|
operation: str,
|
|
repository: str,
|
|
resolved_oid: str,
|
|
query: Mapping[str, object],
|
|
offset: int,
|
|
) -> str:
|
|
payload = {
|
|
"v": 1,
|
|
"operation": operation,
|
|
"repository": repository,
|
|
"resolved_oid": resolved_oid,
|
|
"query": dict(query),
|
|
"offset": offset,
|
|
}
|
|
encoded = base64.urlsafe_b64encode(_canonical(payload)).decode("ascii").rstrip("=")
|
|
signature = hmac.new(
|
|
self._cursor_secret, encoded.encode("ascii"), hashlib.sha256
|
|
).hexdigest()
|
|
return f"{encoded}.{signature}"
|
|
|
|
def _iter_tree_entries(
|
|
self,
|
|
path: Path,
|
|
oid: str,
|
|
prefix: str,
|
|
*,
|
|
include_trees: bool,
|
|
) -> Iterator[dict[str, str]]:
|
|
args = ["ls-tree", "-z", "-r", "--full-tree"]
|
|
if include_trees:
|
|
args.append("-t")
|
|
args.append(oid)
|
|
if prefix:
|
|
args.extend(["--", prefix])
|
|
for record in self._iter_nul_records(path, args):
|
|
entry = _parse_tree_record(record)
|
|
if entry is None:
|
|
continue
|
|
if not prefix or entry["path"] == prefix or entry["path"].startswith(prefix + "/"):
|
|
yield entry
|
|
|
|
def list_tree(
|
|
self,
|
|
*,
|
|
repository: str,
|
|
revision: str,
|
|
prefix: str = "",
|
|
depth: int = 8,
|
|
limit: int = 100,
|
|
cursor: str | None = None,
|
|
) -> dict[str, object]:
|
|
prefix = _safe_relative_path(prefix, "prefix", allow_empty=True)
|
|
depth = _positive_int(depth, "depth", _MAX_DEPTH)
|
|
limit = _positive_int(limit, "limit", _MAX_LIMIT)
|
|
path = self._repository_path(repository)
|
|
resolved_oid = self._resolve_at(path, revision, "revision")
|
|
query = {"prefix": prefix, "depth": depth, "limit": limit}
|
|
offset = self._cursor_offset(
|
|
cursor,
|
|
operation="list_tree",
|
|
repository=repository,
|
|
resolved_oid=resolved_oid,
|
|
query=query,
|
|
)
|
|
page: list[dict[str, str]] = []
|
|
matched = 0
|
|
for entry in self._iter_tree_entries(path, resolved_oid, prefix, include_trees=True):
|
|
relative = entry["path"][len(prefix):].lstrip("/") if prefix else entry["path"]
|
|
if relative.count("/") + 1 > depth:
|
|
continue
|
|
if matched >= offset and len(page) < limit:
|
|
page.append(entry)
|
|
matched += 1
|
|
has_more = offset + len(page) < matched
|
|
return {
|
|
"repository": repository,
|
|
"resolved_oid": resolved_oid,
|
|
"entries": page,
|
|
"truncated": has_more,
|
|
"next_cursor": (
|
|
self._next_cursor(
|
|
operation="list_tree",
|
|
repository=repository,
|
|
resolved_oid=resolved_oid,
|
|
query=query,
|
|
offset=offset + len(page),
|
|
)
|
|
if has_more
|
|
else None
|
|
),
|
|
}
|
|
|
|
def list_repository_tree(self, request: Mapping[str, object]) -> dict[str, object]:
|
|
request = _request_mapping(request)
|
|
return self.list_tree(
|
|
repository=_required_request_value(request, "repository"), # type: ignore[arg-type]
|
|
revision=request.get("revision", request.get("ref", "HEAD")), # type: ignore[arg-type]
|
|
prefix=request.get("prefix", ""), # type: ignore[arg-type]
|
|
depth=request.get("depth", 8), # type: ignore[arg-type]
|
|
limit=request.get("limit", 100), # type: ignore[arg-type]
|
|
cursor=request.get("cursor"), # type: ignore[arg-type]
|
|
)
|
|
|
|
def _blob_size(self, path: Path, oid: str) -> int:
|
|
output = self._run(path, ["cat-file", "-s", oid], max_output=128)
|
|
try:
|
|
size = int(output.decode("ascii").strip())
|
|
except (UnicodeDecodeError, ValueError) as exc:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID") from exc
|
|
if size < 0:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID")
|
|
return size
|
|
|
|
def search_source(
|
|
self,
|
|
*,
|
|
repository: str,
|
|
revision: str,
|
|
query: str,
|
|
path_globs: Iterable[str] | None = None,
|
|
max_matches: int = 100,
|
|
max_bytes: int = 262_144,
|
|
cursor: str | None = None,
|
|
) -> dict[str, object]:
|
|
if (
|
|
not isinstance(query, str)
|
|
or not query
|
|
or "\x00" in query
|
|
or len(query) > _MAX_QUERY_LENGTH
|
|
):
|
|
raise _error("REPOSITORY_DISCOVERY_QUERY_INVALID")
|
|
max_matches = _positive_int(max_matches, "max_matches", _MAX_LIMIT)
|
|
max_bytes = _positive_int(max_bytes, "max_bytes", _MAX_BYTES)
|
|
if path_globs is None:
|
|
globs = ("*",)
|
|
else:
|
|
if isinstance(path_globs, (str, bytes)) or not isinstance(path_globs, Iterable):
|
|
raise _error("REPOSITORY_DISCOVERY_PATH_INVALID")
|
|
globs = tuple(_safe_glob(item) for item in path_globs)
|
|
if not globs or len(globs) > _MAX_GLOBS:
|
|
raise _error("REPOSITORY_DISCOVERY_PATH_INVALID")
|
|
path = self._repository_path(repository)
|
|
resolved_oid = self._resolve_at(path, revision, "revision")
|
|
query_binding = {
|
|
"query": query,
|
|
"path_globs": list(globs),
|
|
"max_matches": max_matches,
|
|
"max_bytes": max_bytes,
|
|
}
|
|
offset = self._cursor_offset(
|
|
cursor,
|
|
operation="search_source",
|
|
repository=repository,
|
|
resolved_oid=resolved_oid,
|
|
query=query_binding,
|
|
)
|
|
scanned_bytes = 0
|
|
skipped_for_bound = False
|
|
binary_blobs = 0
|
|
matched = 0
|
|
matches: list[dict[str, object]] = []
|
|
has_more = False
|
|
for entry in self._iter_tree_entries(path, resolved_oid, "", include_trees=False):
|
|
if entry["type"] != "blob" or not any(
|
|
fnmatch.fnmatchcase(entry["path"], pattern) for pattern in globs
|
|
):
|
|
continue
|
|
blob_size = self._blob_size(path, entry["object_oid"])
|
|
remaining = max_bytes - scanned_bytes
|
|
if blob_size > remaining:
|
|
skipped_for_bound = True
|
|
continue
|
|
raw, clipped = self._read_blob_prefix(path, entry["object_oid"], blob_size)
|
|
if clipped:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_FAILED")
|
|
scanned_bytes += len(raw)
|
|
if b"\0" in raw:
|
|
binary_blobs += 1
|
|
continue
|
|
text = raw.decode("utf-8", errors="replace")
|
|
for number, line in enumerate(text.splitlines(), start=1):
|
|
if query not in line:
|
|
continue
|
|
if matched < offset:
|
|
matched += 1
|
|
continue
|
|
if len(matches) >= max_matches:
|
|
has_more = True
|
|
break
|
|
matches.append(
|
|
{
|
|
"path": entry["path"],
|
|
"line_start": number,
|
|
"line_end": number,
|
|
"blob_oid": entry["object_oid"],
|
|
"snippet": line[:512],
|
|
}
|
|
)
|
|
matched += 1
|
|
if has_more:
|
|
break
|
|
return {
|
|
"repository": repository,
|
|
"resolved_oid": resolved_oid,
|
|
"matches": matches,
|
|
"scanned_bytes": scanned_bytes,
|
|
"max_bytes": max_bytes,
|
|
"binary_blobs_skipped": binary_blobs,
|
|
"truncated": bool(has_more or skipped_for_bound),
|
|
"next_cursor": (
|
|
self._next_cursor(
|
|
operation="search_source",
|
|
repository=repository,
|
|
resolved_oid=resolved_oid,
|
|
query=query_binding,
|
|
offset=offset + len(matches),
|
|
)
|
|
if has_more
|
|
else None
|
|
),
|
|
}
|
|
|
|
def search_repository_source(self, request: Mapping[str, object]) -> dict[str, object]:
|
|
request = _request_mapping(request)
|
|
path_globs = request.get("path_globs")
|
|
if path_globs is not None and (
|
|
isinstance(path_globs, (str, bytes)) or not isinstance(path_globs, Iterable)
|
|
):
|
|
raise _error("REPOSITORY_DISCOVERY_PATH_INVALID")
|
|
return self.search_source(
|
|
repository=_required_request_value(request, "repository"), # type: ignore[arg-type]
|
|
revision=request.get("revision", request.get("ref", "HEAD")), # type: ignore[arg-type]
|
|
query=_required_request_value(request, "query"), # type: ignore[arg-type]
|
|
path_globs=path_globs, # type: ignore[arg-type]
|
|
max_matches=request.get("max_matches", 100), # type: ignore[arg-type]
|
|
max_bytes=request.get("max_bytes", 262_144), # type: ignore[arg-type]
|
|
cursor=request.get("cursor"), # type: ignore[arg-type]
|
|
)
|
|
|
|
def _iter_diff_name_entries(
|
|
self,
|
|
path: Path,
|
|
from_oid: str,
|
|
to_oid: str,
|
|
prefix: str,
|
|
) -> Iterator[dict[str, object]]:
|
|
suffix = ["--", prefix] if prefix else []
|
|
records = self._iter_nul_records(
|
|
path,
|
|
[
|
|
"diff",
|
|
"--no-ext-diff",
|
|
"--no-textconv",
|
|
"--find-renames=50%",
|
|
"--name-status",
|
|
"-z",
|
|
from_oid,
|
|
to_oid,
|
|
*suffix,
|
|
],
|
|
)
|
|
iterator = iter(records)
|
|
while True:
|
|
try:
|
|
status_raw = next(iterator)
|
|
except StopIteration:
|
|
return
|
|
try:
|
|
status = status_raw.decode("ascii")
|
|
except UnicodeDecodeError as exc:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID") from exc
|
|
if not status or status[0] not in {"A", "C", "D", "M", "R", "T", "U", "X"}:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID")
|
|
try:
|
|
if status[0] in {"R", "C"}:
|
|
old_path = _decode_output_path(next(iterator))
|
|
new_path = _decode_output_path(next(iterator))
|
|
yield {
|
|
"status": status,
|
|
"path": new_path,
|
|
"old_path": old_path,
|
|
}
|
|
else:
|
|
yield {
|
|
"status": status,
|
|
"path": _decode_output_path(next(iterator)),
|
|
"old_path": None,
|
|
}
|
|
except StopIteration as exc:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID") from exc
|
|
|
|
@staticmethod
|
|
def _parse_numstat_count(value: bytes) -> int | None:
|
|
if value == b"-":
|
|
return None
|
|
try:
|
|
parsed = int(value.decode("ascii"), 10)
|
|
except (UnicodeDecodeError, ValueError) as exc:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID") from exc
|
|
if parsed < 0:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID")
|
|
return parsed
|
|
|
|
def _diff_stats(
|
|
self,
|
|
path: Path,
|
|
from_oid: str,
|
|
to_oid: str,
|
|
prefix: str,
|
|
wanted_paths: set[str],
|
|
) -> dict[str, tuple[int | None, int | None]]:
|
|
suffix = ["--", prefix] if prefix else []
|
|
records = self._iter_nul_records(
|
|
path,
|
|
[
|
|
"diff",
|
|
"--no-ext-diff",
|
|
"--no-textconv",
|
|
"--find-renames=50%",
|
|
"--numstat",
|
|
"-z",
|
|
from_oid,
|
|
to_oid,
|
|
*suffix,
|
|
],
|
|
)
|
|
iterator = iter(records)
|
|
stats: dict[str, tuple[int | None, int | None]] = {}
|
|
for row in iterator:
|
|
if not row:
|
|
continue
|
|
fields = row.split(b"\t", 2)
|
|
if len(fields) != 3:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID")
|
|
additions = self._parse_numstat_count(fields[0])
|
|
deletions = self._parse_numstat_count(fields[1])
|
|
if fields[2]:
|
|
stat_path = _decode_output_path(fields[2])
|
|
else:
|
|
try:
|
|
next(iterator) # old path
|
|
stat_path = _decode_output_path(next(iterator))
|
|
except StopIteration as exc:
|
|
raise _error("REPOSITORY_DISCOVERY_GIT_OUTPUT_INVALID") from exc
|
|
if stat_path in wanted_paths:
|
|
stats[stat_path] = (additions, deletions)
|
|
if wanted_paths <= set(stats):
|
|
break
|
|
return stats
|
|
|
|
def diff_index(
|
|
self,
|
|
*,
|
|
repository: str,
|
|
from_revision: str,
|
|
to_revision: str,
|
|
prefix: str = "",
|
|
limit: int = 100,
|
|
cursor: str | None = None,
|
|
) -> dict[str, object]:
|
|
prefix = _safe_relative_path(prefix, "prefix", allow_empty=True)
|
|
limit = _positive_int(limit, "limit", _MAX_LIMIT)
|
|
path = self._repository_path(repository)
|
|
from_oid = self._resolve_at(path, from_revision, "from")
|
|
to_oid = self._resolve_at(path, to_revision, "to")
|
|
query = {
|
|
"from_oid": from_oid,
|
|
"to_oid": to_oid,
|
|
"prefix": prefix,
|
|
"limit": limit,
|
|
}
|
|
offset = self._cursor_offset(
|
|
cursor,
|
|
operation="diff_index",
|
|
repository=repository,
|
|
resolved_oid=to_oid,
|
|
query=query,
|
|
)
|
|
page: list[dict[str, object]] = []
|
|
total = 0
|
|
for entry in self._iter_diff_name_entries(path, from_oid, to_oid, prefix):
|
|
if total >= offset and len(page) < limit:
|
|
page.append(entry)
|
|
total += 1
|
|
wanted_paths = {str(entry["path"]) for entry in page}
|
|
stat_by_path = (
|
|
self._diff_stats(path, from_oid, to_oid, prefix, wanted_paths)
|
|
if page
|
|
else {}
|
|
)
|
|
for entry in page:
|
|
additions, deletions = stat_by_path.get(
|
|
str(entry["path"]), (None, None)
|
|
)
|
|
entry["additions"] = additions
|
|
entry["deletions"] = deletions
|
|
has_more = offset + len(page) < total
|
|
return {
|
|
"repository": repository,
|
|
"resolved_from_oid": from_oid,
|
|
"resolved_to_oid": to_oid,
|
|
"entries": page,
|
|
"truncated": has_more,
|
|
"next_cursor": (
|
|
self._next_cursor(
|
|
operation="diff_index",
|
|
repository=repository,
|
|
resolved_oid=to_oid,
|
|
query=query,
|
|
offset=offset + len(page),
|
|
)
|
|
if has_more
|
|
else None
|
|
),
|
|
}
|
|
|
|
def get_diff_index(self, request: Mapping[str, object]) -> dict[str, object]:
|
|
request = _request_mapping(request)
|
|
from_revision = request.get("from_revision", request.get("from"))
|
|
to_revision = request.get("to_revision", request.get("to"))
|
|
prefix = request.get("prefix", request.get("path_prefix", ""))
|
|
if from_revision is None or to_revision is None:
|
|
raise _error("REPOSITORY_DISCOVERY_REQUEST_INVALID")
|
|
return self.diff_index(
|
|
repository=_required_request_value(request, "repository"), # type: ignore[arg-type]
|
|
from_revision=from_revision, # type: ignore[arg-type]
|
|
to_revision=to_revision, # type: ignore[arg-type]
|
|
prefix=prefix, # type: ignore[arg-type]
|
|
limit=request.get("limit", 100), # type: ignore[arg-type]
|
|
cursor=request.get("cursor"), # type: ignore[arg-type]
|
|
)
|
|
|
|
def read_file(
|
|
self,
|
|
*,
|
|
repository: str,
|
|
revision: str,
|
|
file_path: str,
|
|
max_bytes: int = 262_144,
|
|
) -> dict[str, object]:
|
|
file_path = _safe_relative_path(file_path, "file_path", allow_empty=False)
|
|
max_bytes = _positive_int(max_bytes, "max_bytes", _MAX_BYTES)
|
|
path = self._repository_path(repository)
|
|
resolved_oid = self._resolve_at(path, revision, "revision")
|
|
exact: list[dict[str, str]] = []
|
|
for entry in self._iter_tree_entries(
|
|
path, resolved_oid, file_path, include_trees=False
|
|
):
|
|
if entry["path"] == file_path and entry["type"] == "blob":
|
|
exact.append(entry)
|
|
if len(exact) != 1:
|
|
raise RepositoryDiscoveryNotFound("REPOSITORY_DISCOVERY_FILE_NOT_FOUND")
|
|
blob_oid = exact[0]["object_oid"]
|
|
content, truncated = self._read_blob_prefix(path, blob_oid, max_bytes)
|
|
return {
|
|
"repository": repository,
|
|
"resolved_oid": resolved_oid,
|
|
"blob_oid": blob_oid,
|
|
"path": file_path,
|
|
"content": content.decode("utf-8", errors="replace"),
|
|
"content_base64": base64.b64encode(content).decode("ascii"),
|
|
"content_bytes": len(content),
|
|
"truncated": truncated,
|
|
}
|
|
|
|
def read_repository_file(self, request: Mapping[str, object]) -> dict[str, object]:
|
|
request = _request_mapping(request)
|
|
file_path = request.get("file_path", request.get("path"))
|
|
if file_path is None:
|
|
raise _error("REPOSITORY_DISCOVERY_REQUEST_INVALID")
|
|
return self.read_file(
|
|
repository=_required_request_value(request, "repository"), # type: ignore[arg-type]
|
|
revision=request.get("revision", request.get("ref", "HEAD")), # type: ignore[arg-type]
|
|
file_path=file_path, # type: ignore[arg-type]
|
|
max_bytes=request.get("max_bytes", 262_144), # type: ignore[arg-type]
|
|
)
|
|
|
|
|
|
def create_repository_discovery(
|
|
*,
|
|
allowed_repositories: Iterable[str] | None = None,
|
|
git_dir: str | Path | None = None,
|
|
cursor_secret: str | bytes | None = None,
|
|
) -> RepositoryDiscovery:
|
|
return RepositoryDiscovery(
|
|
allowed_repositories=allowed_repositories,
|
|
git_dir=git_dir,
|
|
cursor_secret=cursor_secret,
|
|
)
|
|
|
|
|
|
def list_repository_tree(
|
|
request: Mapping[str, object], *, discovery: RepositoryDiscovery | None = None
|
|
) -> dict[str, object]:
|
|
return (discovery or create_repository_discovery()).list_repository_tree(request)
|
|
|
|
|
|
def search_repository_source(
|
|
request: Mapping[str, object], *, discovery: RepositoryDiscovery | None = None
|
|
) -> dict[str, object]:
|
|
return (discovery or create_repository_discovery()).search_repository_source(request)
|
|
|
|
|
|
def get_diff_index(
|
|
request: Mapping[str, object], *, discovery: RepositoryDiscovery | None = None
|
|
) -> dict[str, object]:
|
|
return (discovery or create_repository_discovery()).get_diff_index(request)
|
|
|
|
|
|
def read_repository_file(
|
|
request: Mapping[str, object], *, discovery: RepositoryDiscovery | None = None
|
|
) -> dict[str, object]:
|
|
return (discovery or create_repository_discovery()).read_repository_file(request)
|