test: sweep every tool through the MCP server against a seeded Gitea
This commit is contained in:
Latte committed
1 parent
22c8157ca4
commit
303bd9aa21
7 files changed
+1759
-1
No files matched your search
@@ -0,0 +1,928 @@
|
||||
"""Plan for the all-tools sweep: arguments, ordering, expectations and the coverage report.
|
||||
|
||||
The sweep calls every tool (538 with ``gitea_api_request``) once through the MCP server
|
||||
against a seeded Gitea (:mod:`tests.integration.sweep_seed`). This module holds everything
|
||||
that is data or pure logic, so it can be checked without Docker
|
||||
(``tests/test_sweep_plan.py``):
|
||||
|
||||
* :func:`build_arguments` turns a catalog operation plus the seed context into tool
|
||||
arguments: path values from the context, required bodies generated from their schema,
|
||||
and an override table (:data:`OVERRIDES`) for tools that need specific bodies or ids.
|
||||
* :func:`ordered_operations` sorts the calls: read-only first, then creates and updates,
|
||||
then deletes (leaf resources before their parents).
|
||||
* :func:`classify` decides whether a result is acceptable: a 2xx, a status the swagger
|
||||
documents for that operation, or an explicit entry of :data:`ALLOWANCES` with a reason.
|
||||
* :func:`render_report` writes ``docs/integration-coverage.md``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from collections.abc import Callable, Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from aegis_gitea_mcp.catalog import Catalog, Operation
|
||||
from tests.integration.sweep_seed import PNG_1X1, Ctx, b64, ssh_public_key
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
SPEC_PATH = REPO_ROOT / "spec" / "gitea-swagger.v1.json"
|
||||
REPORT_PATH = REPO_ROOT / "docs" / "integration-coverage.md"
|
||||
REPORT_ENV = "AEGIS_IT_WRITE_REPORT"
|
||||
RAW_TOOL = "gitea_api_request"
|
||||
|
||||
#: Tools the generic sweep never calls, with the test that covers them in isolation.
|
||||
EXCLUDED: dict[str, str] = {
|
||||
"gitea_misc_delete_current_token": (
|
||||
"deletes the token the sweep itself runs with; covered by test_sweep_isolated.py "
|
||||
"with a throwaway token"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Allowance:
|
||||
"""A status that is accepted for a tool although the swagger does not document it."""
|
||||
|
||||
statuses: frozenset[int]
|
||||
reason: str
|
||||
|
||||
|
||||
def allow(reason: str, *statuses: int) -> Allowance:
|
||||
"""Build an :class:`Allowance`."""
|
||||
return Allowance(frozenset(statuses), reason)
|
||||
|
||||
|
||||
#: Tools whose result may be a status the swagger does not list, each with its reason.
|
||||
_NO_SIGNING_KEY = allow(
|
||||
"the throwaway instance has no commit signing key; Gitea answers 404 although the "
|
||||
"swagger lists no 404 for this operation",
|
||||
404,
|
||||
)
|
||||
ALLOWANCES: dict[str, Allowance] = {
|
||||
"gitea_misc_get_signing_key": _NO_SIGNING_KEY,
|
||||
"gitea_misc_get_signing_key_ssh": _NO_SIGNING_KEY,
|
||||
"gitea_repo_signing_key": _NO_SIGNING_KEY,
|
||||
"gitea_repo_signing_key_ssh": _NO_SIGNING_KEY,
|
||||
"gitea_admin_add_user_badges": allow(
|
||||
"Gitea answers 500 for an unknown badge slug and the API cannot create badges", 500
|
||||
),
|
||||
"gitea_admin_delete_unadopted_repository": allow(
|
||||
"no unadopted repository exists on the instance; Gitea answers 404 although the swagger "
|
||||
"lists only 204 and 403",
|
||||
404,
|
||||
),
|
||||
"gitea_user_create_token": allow(
|
||||
"Gitea only accepts basic auth here; the server always sends token authentication", 401
|
||||
),
|
||||
"gitea_user_get_tokens": allow(
|
||||
"Gitea only accepts basic auth here; the server always sends token authentication", 401
|
||||
),
|
||||
"gitea_user_delete_access_token": allow(
|
||||
"Gitea only accepts basic auth here; the server always sends token authentication", 401
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Spec helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_spec() -> dict[str, Any]:
|
||||
"""Load the pinned swagger."""
|
||||
data: dict[str, Any] = json.loads(SPEC_PATH.read_text(encoding="utf-8"))
|
||||
return data
|
||||
|
||||
|
||||
def documented_statuses(spec: dict[str, Any], operation: Operation) -> frozenset[int]:
|
||||
"""Status codes the swagger lists in ``responses`` of this operation."""
|
||||
item = spec["paths"][operation.path][operation.method.lower()]
|
||||
return frozenset(int(code) for code in item["responses"] if code.isdigit())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Argument synthesis
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DEFAULT_PATH_VALUES: dict[str, str] = {
|
||||
"owner": "owner",
|
||||
"repo": "repo",
|
||||
"org": "org",
|
||||
"username": "user",
|
||||
"collaborator": "user",
|
||||
"assignee": "user",
|
||||
"target": "user",
|
||||
"user": "user",
|
||||
"template_owner": "owner",
|
||||
"template_repo": "template_repo",
|
||||
"repo_name": "repo",
|
||||
"index": "issue",
|
||||
"sha": "sha",
|
||||
"run": "run",
|
||||
"job_id": "job_id",
|
||||
"pageName": "page_name",
|
||||
"team": "team_name",
|
||||
"task": "cron_task",
|
||||
}
|
||||
|
||||
CONSTANT_PATH_VALUES: dict[str, Any] = {
|
||||
"ref": "main",
|
||||
"branch": "main",
|
||||
"filepath": "README.md",
|
||||
"archive": "main.zip",
|
||||
"basehead": "main...feat",
|
||||
"diffType": "diff",
|
||||
"base": "main",
|
||||
"head": "feat",
|
||||
"type": "generic",
|
||||
"version": "1.0.0",
|
||||
"attempt": 1,
|
||||
"artifact_id": 1,
|
||||
"runner_id": 1,
|
||||
"position": 1,
|
||||
"workflow_id": "sweep.yml",
|
||||
"topic": "extra",
|
||||
"token": "none",
|
||||
}
|
||||
|
||||
# (regex over the path template, context key) for the ``{id}`` segment, first match wins.
|
||||
ID_RULES: list[tuple[str, str]] = [
|
||||
(r"^/repos/\{owner\}/\{repo\}/pulls/comments/\{id\}", "review_comment"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/pulls/\{index\}/comments/\{id\}", "review_comment"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/pulls/\{index\}/reviews/\{id\}", "review"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/issues/comments/\{id\}", "comment"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/issues/\{index\}/comments/\{id\}", "comment"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/issues/\{index\}/labels/\{id\}", "label"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/issues/\{index\}/times/\{id\}", "time_id"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/labels/\{id\}", "label"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/milestones/\{id\}", "milestone"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/hooks/git/\{id\}", "git_hook"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/hooks/\{id\}", "repo_hook"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/keys/\{id\}", "deploy_key"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/releases/\{id\}", "release"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/tag_protections/\{id\}", "tag_protection"),
|
||||
(r"^/repos/\{owner\}/\{repo\}/projects/\{id\}", "repo_project"),
|
||||
(r"^/orgs/\{org\}/hooks/\{id\}", "org_hook"),
|
||||
(r"^/orgs/\{org\}/labels/\{id\}", "org_label"),
|
||||
(r"^/orgs/\{org\}/projects/\{id\}", "org_project"),
|
||||
(r"^/user/hooks/\{id\}", "user_hook"),
|
||||
(r"^/user/keys/\{id\}", "user_key"),
|
||||
(r"^/user/projects/\{id\}", "user_project"),
|
||||
(r"^/user/applications/oauth2/\{id\}", "oauth_app"),
|
||||
(r"^/admin/hooks/\{id\}", "admin_hook"),
|
||||
(r"^/admin/users/\{username\}/keys/\{id\}", "admin_user_key"),
|
||||
(r"^/teams/\{id\}", "team_id"),
|
||||
(r"^/repositories/\{id\}", "repo_id"),
|
||||
(r"^/notifications/threads/\{id\}", "notification"),
|
||||
]
|
||||
|
||||
|
||||
def _path_value(operation: Operation, name: str, c: Ctx) -> Any:
|
||||
"""Value of the path parameter ``name`` of ``operation`` from the context."""
|
||||
path = operation.path
|
||||
if name == "index":
|
||||
return c["pr"] if "/pulls/" in path else c["issue"]
|
||||
if name == "id":
|
||||
for pattern, key in ID_RULES:
|
||||
if re.match(pattern, path):
|
||||
return c[key]
|
||||
return 1
|
||||
if name == "attachment_id":
|
||||
if "/releases/" in path:
|
||||
return c["release_asset"]
|
||||
return c["comment_attachment"] if "/issues/comments/" in path else c["issue_attachment"]
|
||||
if name == "column_id":
|
||||
if path.startswith("/orgs/"):
|
||||
return c["org_column"]
|
||||
return c["user_column"] if path.startswith("/user/") else c["repo_column"]
|
||||
if name == "issue_id":
|
||||
return c["org_issue_id"] if path.startswith("/orgs/") else c["issue1_id"]
|
||||
if name == "secretname":
|
||||
scope = path.split("/")[1]
|
||||
return {"orgs": "ORG_SECRET", "user": c["user_secret"]}.get(scope, "REPO_SECRET")
|
||||
if name == "variablename":
|
||||
scope = path.split("/")[1]
|
||||
return {"orgs": "ORG_VAR", "user": c["user_var"]}.get(scope, "REPO_VAR")
|
||||
if name == "name":
|
||||
if path.startswith("/gitignore/"):
|
||||
return "Go"
|
||||
if path.startswith("/label/"):
|
||||
return "Advanced"
|
||||
if path.startswith("/licenses/"):
|
||||
return "MIT"
|
||||
if path.startswith("/packages/"):
|
||||
return c["pkg"]
|
||||
if "branch_protections" in path:
|
||||
return c["protection"]
|
||||
return "none"
|
||||
if name == "tag":
|
||||
return c["release_tag"] if "/releases/" in path else "v0.0.1"
|
||||
if name in DEFAULT_PATH_VALUES:
|
||||
return c[DEFAULT_PATH_VALUES[name]]
|
||||
if name in CONSTANT_PATH_VALUES:
|
||||
return CONSTANT_PATH_VALUES[name]
|
||||
return 1
|
||||
|
||||
|
||||
def _coerce(value: Any, schema: dict[str, Any]) -> Any:
|
||||
"""Convert ``value`` to the JSON type the argument schema asks for."""
|
||||
kind = schema.get("type")
|
||||
if kind == "string":
|
||||
return str(value)
|
||||
if kind == "integer" and isinstance(value, str) and value.isdigit():
|
||||
return int(value)
|
||||
return value
|
||||
|
||||
|
||||
def sample(schema: dict[str, Any], name: str = "") -> Any:
|
||||
"""A minimal valid value for ``schema`` (required properties only)."""
|
||||
if "enum" in schema:
|
||||
return schema["enum"][0]
|
||||
if "anyOf" in schema:
|
||||
return sample(schema["anyOf"][0], name)
|
||||
kind = schema.get("type")
|
||||
if kind == "object" or "properties" in schema:
|
||||
required = schema.get("required", [])
|
||||
props = schema.get("properties", {})
|
||||
return {key: sample(props[key], key) for key in required if key in props}
|
||||
if kind == "array":
|
||||
return []
|
||||
if kind in ("integer", "number"):
|
||||
return 1
|
||||
if kind == "boolean":
|
||||
return False
|
||||
lowered = name.lower()
|
||||
if "email" in lowered:
|
||||
return f"sweep-{secrets.token_hex(3)}@example.test"
|
||||
if "url" in lowered or "addr" in lowered:
|
||||
return "http://127.0.0.1:9/sweep"
|
||||
if "color" in lowered:
|
||||
return "#336699"
|
||||
if "base64" in lowered or lowered == "content":
|
||||
return b64("sweep")
|
||||
return f"sweep-{secrets.token_hex(3)}"
|
||||
|
||||
|
||||
def fresh(c: Ctx, prefix: str = "sw") -> str:
|
||||
"""A fresh name for one more resource."""
|
||||
return f"{prefix}-{c['sfx']}-{secrets.token_hex(2)}"
|
||||
|
||||
|
||||
def hook_body(c: Ctx) -> dict[str, Any]:
|
||||
"""A webhook body of type gitea."""
|
||||
return {
|
||||
"type": "gitea",
|
||||
"config": {"url": "http://127.0.0.1:9/sweep", "content_type": "json"},
|
||||
"events": ["push"],
|
||||
"active": True,
|
||||
}
|
||||
|
||||
|
||||
def local_url(c: Ctx) -> str:
|
||||
"""Base URL of the admin's repositories as seen from inside the Gitea container."""
|
||||
return f"http://127.0.0.1:3000/{c['admin']}"
|
||||
|
||||
|
||||
def _project_overrides() -> dict[str, Override]:
|
||||
"""Column and issue handling of org, repo and user projects (all three work alike)."""
|
||||
result: dict[str, Override] = {}
|
||||
for scope, tool_scope in (("org", "org"), ("repo", "repo"), ("user", "user_current")):
|
||||
result[f"gitea_{tool_scope}_move_project_columns"] = lambda c, s=scope: {
|
||||
"body": {"column_ids": c[f"{s}_columns"]}
|
||||
}
|
||||
result[f"gitea_{tool_scope}_move_project_issue"] = lambda c, s=scope: {
|
||||
"body": {"column_id": c[f"{s}_column2"]}
|
||||
}
|
||||
result[f"gitea_{tool_scope}_delete_project_column"] = lambda c, s=scope: {
|
||||
"column_id": c[f"{s}_column2"]
|
||||
}
|
||||
result[f"gitea_{tool_scope}_remove_issue_from_project_column"] = lambda c, s=scope: {
|
||||
"column_id": c[f"{s}_column2"]
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def _org_repo(c: Ctx) -> dict[str, Any]:
|
||||
return {"owner": c["org"], "repo": c["org_repo"]}
|
||||
|
||||
|
||||
def _org_repo_team(c: Ctx) -> dict[str, Any]:
|
||||
return {"owner": c["org"], "repo": c["org_repo"], "team": c["team_name"]}
|
||||
|
||||
|
||||
def _team_repo(c: Ctx) -> dict[str, Any]:
|
||||
return {"org": c["org"], "repo": c["org_repo"]}
|
||||
|
||||
|
||||
PATCH_TEXT = """\
|
||||
diff --git a/applied-{name}.txt b/applied-{name}.txt
|
||||
new file mode 100644
|
||||
index 0000000..e69de29
|
||||
--- /dev/null
|
||||
+++ b/applied-{name}.txt
|
||||
@@ -0,0 +1 @@
|
||||
+applied
|
||||
"""
|
||||
|
||||
Override = Callable[[Ctx], dict[str, Any]]
|
||||
|
||||
OVERRIDES: dict[str, Override] = {
|
||||
# ------------------------------------------------------------------ reads
|
||||
"gitea_repo_get_annotated_tag": lambda c: {"sha": c["tag_sha"]},
|
||||
"gitea_repo_get_push_mirror_by_remote_name": lambda c: {"name": c["push_mirror"]},
|
||||
"gitea_repo_delete_push_mirror": lambda c: {"name": c["push_mirror"]},
|
||||
"gitea_repo_add_push_mirror": lambda c: {
|
||||
"body": {
|
||||
"remote_address": f"{local_url(c)}/{c['push_target2']}.git",
|
||||
"remote_username": c["admin"],
|
||||
"remote_password": c["admin_password"],
|
||||
"interval": "8h0m0s",
|
||||
"sync_on_commit": False,
|
||||
}
|
||||
},
|
||||
"gitea_repo_mirror_sync": lambda c: {"repo": c["mirror_repo"]},
|
||||
"gitea_repo_merge_upstream": lambda c: {
|
||||
"owner": c["org"],
|
||||
"repo": c["fork_repo"],
|
||||
"body": {"branch": "main"},
|
||||
},
|
||||
"gitea_repo_migrate": lambda c: {
|
||||
"body": {
|
||||
"clone_addr": f"{local_url(c)}/{c['template_repo']}.git",
|
||||
"repo_name": fresh(c, "mig"),
|
||||
"service": "git",
|
||||
}
|
||||
},
|
||||
"gitea_repo_add_team": lambda c: {**_org_repo(c), "team": c["team2_name"]},
|
||||
"gitea_repo_delete_team": lambda c: {**_org_repo(c), "team": c["team2_name"]},
|
||||
"gitea_repo_update_topics": lambda c: {"body": {"topics": ["sweep", "seed", "extra"]}},
|
||||
"gitea_issue_delete_stopwatch": lambda c: {"sudo": c["user"], "index": c["issue2"]},
|
||||
"gitea_org_block_user": lambda c: {"username": c["user3"]},
|
||||
"gitea_user_block_user": lambda c: {"username": c["user3"]},
|
||||
"gitea_repo_get_file_contents": lambda c: {"query_body": json.dumps({"files": ["README.md"]})},
|
||||
"gitea_repo_get_file_contents_post": lambda c: {"body": {"files": ["README.md"]}},
|
||||
"gitea_repo_list_git_refs": lambda c: {"ref": "heads/main"},
|
||||
"gitea_repo_get_blob": lambda c: {"sha": c["blob_sha"]},
|
||||
"gitea_repo_check_team": _org_repo_team,
|
||||
"gitea_repo_list_teams": _org_repo,
|
||||
"gitea_team_list_team_repo": _team_repo,
|
||||
"gitea_team_add_team_repository": _team_repo,
|
||||
"gitea_team_remove_team_repository": _team_repo,
|
||||
"gitea_user_check_following": lambda c: {"username": c["admin"], "target": c["user"]},
|
||||
"gitea_misc_render_markdown": lambda c: {"body": {"Text": "# sweep", "Mode": "markdown"}},
|
||||
"gitea_misc_render_markup": lambda c: {
|
||||
"body": {"Text": "# sweep", "Mode": "markdown", "Context": ""}
|
||||
},
|
||||
"gitea_misc_render_markdown_raw": lambda c: {"body": "# sweep"},
|
||||
"gitea_notification_read_thread": lambda c: {"id": str(c["notification"])},
|
||||
"gitea_notification_get_thread": lambda c: {"id": str(c["notification"])},
|
||||
"gitea_user_get_tokens": lambda c: {"username": c["admin"]},
|
||||
# ---------------------------------------------------------------- actions
|
||||
"gitea_actions_create_org_variable": lambda c: {
|
||||
"variablename": "SW_NEW",
|
||||
"body": {"value": "1"},
|
||||
},
|
||||
"gitea_actions_create_repo_variable": lambda c: {
|
||||
"variablename": "SW_NEW",
|
||||
"body": {"value": "1"},
|
||||
},
|
||||
"gitea_actions_create_user_variable": lambda c: {
|
||||
"variablename": f"SW_NEW_{c['sfx'].upper()}",
|
||||
"body": {"value": "1"},
|
||||
},
|
||||
"gitea_actions_dispatch_workflow": lambda c: {"body": {"ref": "main"}},
|
||||
"gitea_actions_update_org_secret": lambda c: {"body": {"data": "rotated-org-value"}},
|
||||
"gitea_actions_update_repo_secret": lambda c: {"body": {"data": "rotated-repo-value"}},
|
||||
"gitea_actions_update_user_secret": lambda c: {"body": {"data": "rotated-user-value"}},
|
||||
# ------------------------------------------------------------------ admin
|
||||
"gitea_admin_add_user_badges": lambda c: {"body": {"badge_slugs": ["sweep"]}},
|
||||
"gitea_admin_delete_user_badges": lambda c: {"body": {"badge_slugs": ["sweep"]}},
|
||||
"gitea_admin_create_hook": lambda c: {"body": hook_body(c)},
|
||||
"gitea_admin_create_org": lambda c: {"body": {"username": fresh(c, "sw-aorg")}},
|
||||
"gitea_admin_create_public_key": lambda c: {
|
||||
"body": {"title": fresh(c, "key"), "key": ssh_public_key()}
|
||||
},
|
||||
"gitea_admin_create_repo": lambda c: {"body": {"name": fresh(c, "sw-arepo")}},
|
||||
"gitea_admin_create_user": lambda c: _new_user(c),
|
||||
"gitea_admin_edit_user": lambda c: {
|
||||
"username": c["user2"],
|
||||
"body": {"source_id": 0, "login_name": c["user2"], "full_name": "Sweep Two"},
|
||||
},
|
||||
"gitea_admin_rename_user": lambda c: {
|
||||
"username": c["rename_user"],
|
||||
"body": {"new_username": fresh(c, "sw-renamed")},
|
||||
},
|
||||
"gitea_admin_delete_user": lambda c: {"username": c["del_user"]},
|
||||
# ------------------------------------------------------------------ hooks
|
||||
"gitea_hook_org_create_hook": lambda c: {"body": hook_body(c)},
|
||||
"gitea_hook_repo_create_hook": lambda c: {"body": hook_body(c)},
|
||||
"gitea_hook_user_create_hook": lambda c: {"body": hook_body(c)},
|
||||
"gitea_hook_repo_edit_git_hook": lambda c: {"body": {"content": "#!/bin/sh\nexit 0\n"}},
|
||||
# ----------------------------------------------------------------- issues
|
||||
"gitea_issue_create_issue": lambda c: {"body": {"title": fresh(c, "issue")}},
|
||||
"gitea_issue_create_comment": lambda c: {"body": {"body": "sweep comment"}},
|
||||
"gitea_issue_create_label": lambda c: {"body": {"name": fresh(c, "lbl"), "color": "#00aa00"}},
|
||||
"gitea_issue_create_milestone": lambda c: {"body": {"title": fresh(c, "ms")}},
|
||||
"gitea_issue_add_label": lambda c: {"body": {"labels": [c["label"]]}},
|
||||
"gitea_issue_replace_labels": lambda c: {"body": {"labels": [c["label"]]}},
|
||||
"gitea_issue_add_assignees": lambda c: {"body": {"assignees": [c["user"]]}},
|
||||
"gitea_issue_remove_assignees": lambda c: {"body": {"assignees": [c["admin"]]}},
|
||||
"gitea_issue_add_time": lambda c: {"body": {"time": 30}},
|
||||
"gitea_issue_edit_issue": lambda c: {"body": {"title": "Sweep main edited"}},
|
||||
"gitea_issue_edit_comment": lambda c: {"body": {"body": "edited"}},
|
||||
"gitea_issue_edit_comment_deprecated": lambda c: {"body": {"body": "edited again"}},
|
||||
"gitea_issue_edit_label": lambda c: {"body": {"name": "bug-renamed"}},
|
||||
"gitea_issue_edit_milestone": lambda c: {"body": {"title": "m1-renamed"}},
|
||||
"gitea_issue_edit_issue_attachment": lambda c: {"body": {"name": "renamed.txt"}},
|
||||
"gitea_issue_edit_issue_comment_attachment": lambda c: {"body": {"name": "renamed.txt"}},
|
||||
"gitea_issue_edit_issue_deadline": lambda c: {"body": {"due_date": "2030-01-01T00:00:00Z"}},
|
||||
"gitea_issue_lock_issue": lambda c: {"body": {"lock_reason": "too heated"}},
|
||||
"gitea_issue_move_issue_pin": lambda c: {"index": c["issue3"]},
|
||||
"gitea_issue_unpin_issue": lambda c: {"index": c["issue3"]},
|
||||
"gitea_issue_post_issue_reaction": lambda c: {"body": {"content": "heart"}},
|
||||
"gitea_issue_post_comment_reaction": lambda c: {"body": {"content": "heart"}},
|
||||
"gitea_issue_delete_issue_reaction": lambda c: {"body": {"content": "+1"}},
|
||||
"gitea_issue_delete_comment_reaction": lambda c: {"body": {"content": "+1"}},
|
||||
"gitea_issue_create_issue_blocking": lambda c: {
|
||||
"body": {"index": c["issue5"], "owner": c["owner"], "repo": c["repo"]}
|
||||
},
|
||||
"gitea_issue_remove_issue_blocking": lambda c: {
|
||||
"body": {"index": c["issue3"], "owner": c["owner"], "repo": c["repo"]}
|
||||
},
|
||||
"gitea_issue_create_issue_dependencies": lambda c: {
|
||||
"body": {"index": c["issue4"], "owner": c["owner"], "repo": c["repo"]}
|
||||
},
|
||||
"gitea_issue_remove_issue_dependencies": lambda c: {
|
||||
"body": {"index": c["issue2"], "owner": c["owner"], "repo": c["repo"]}
|
||||
},
|
||||
"gitea_issue_delete_comment_deprecated": lambda c: {"id": c["comment2"]},
|
||||
# ------------------------------------------------------------------- orgs
|
||||
"gitea_org_create": lambda c: {"body": {"username": fresh(c, "sw-neworg")}},
|
||||
"gitea_org_create_label": lambda c: {"body": {"name": fresh(c, "lbl"), "color": "#00aa00"}},
|
||||
"gitea_org_create_org_repo": lambda c: {"body": {"name": fresh(c, "sw-orgrepo")}},
|
||||
"gitea_org_create_org_repo_deprecated": lambda c: {"body": {"name": fresh(c, "sw-orgrepo")}},
|
||||
"gitea_org_create_project": lambda c: {"body": {"title": fresh(c, "proj")}},
|
||||
"gitea_org_create_project_column": lambda c: {
|
||||
"id": c["org_project"],
|
||||
"body": {"title": fresh(c, "col")},
|
||||
},
|
||||
"gitea_org_edit": lambda c: {"body": {"description": "edited by the sweep"}},
|
||||
"gitea_org_rename": lambda c: {
|
||||
"org": c["rename_org"],
|
||||
"body": {"new_name": fresh(c, "sw-renorg")},
|
||||
},
|
||||
"gitea_org_delete": lambda c: {"org": c["del_org2"]},
|
||||
"gitea_org_delete_repos": lambda c: {"org": c["del_org"]},
|
||||
"gitea_org_update_avatar": lambda c: {"body": {"image": PNG_1X1}},
|
||||
"gitea_org_check_user_block": lambda c: {"username": c["user2"]},
|
||||
"gitea_org_unblock_user": lambda c: {"username": c["user2"]},
|
||||
# ---------------------------------------------------------------- package
|
||||
"gitea_package_delete_package": lambda c: {"name": c["pkg2"]},
|
||||
# ------------------------------------------------------------------ pulls
|
||||
"gitea_pull_create_pull_request": lambda c: {
|
||||
"body": {"title": "Sweep PR three", "base": "main", "head": "feat3"}
|
||||
},
|
||||
"gitea_pull_create_pull_review": lambda c: {"body": {"event": "COMMENT", "body": "sweep"}},
|
||||
"gitea_pull_submit_pull_review": lambda c: {
|
||||
"sudo": c["user"],
|
||||
"id": c["pending_review"],
|
||||
"body": {"event": "COMMENT", "body": "submitted"},
|
||||
},
|
||||
"gitea_pull_delete_pull_review": lambda c: {"id": c["review2"]},
|
||||
"gitea_pull_dismiss_pull_review": lambda c: {"body": {"message": "stale", "priors": False}},
|
||||
"gitea_pull_create_pull_review_comment_reply": lambda c: {"body": {"body": "reply"}},
|
||||
"gitea_pull_create_pull_review_requests": lambda c: {"body": {"reviewers": [c["user2"]]}},
|
||||
"gitea_pull_delete_pull_review_requests": lambda c: {"body": {"reviewers": [c["user2"]]}},
|
||||
"gitea_pull_merge_pull_request": lambda c: {"index": c["pr2"], "body": {"do": "merge"}},
|
||||
"gitea_pull_edit_pull_request": lambda c: {"body": {"title": "Sweep PR edited"}},
|
||||
# --------------------------------------------------------------- releases
|
||||
"gitea_release_create_release": lambda c: {
|
||||
"body": {"tag_name": "v3.0.0", "target_commitish": "main"}
|
||||
},
|
||||
"gitea_release_edit_release": lambda c: {"body": {"name": "renamed"}},
|
||||
"gitea_release_edit_release_attachment": lambda c: {"body": {"name": "renamed.bin"}},
|
||||
"gitea_release_delete_release_by_tag": lambda c: {"tag": c["release_tag2"]},
|
||||
# ------------------------------------------------------------------ repos
|
||||
"gitea_repo_create_file": lambda c: {
|
||||
"filepath": f"new-{fresh(c)}.txt",
|
||||
"body": {"content": b64("new"), "message": "sweep new file"},
|
||||
},
|
||||
"gitea_repo_update_file": lambda c: {
|
||||
"filepath": "to-update.txt",
|
||||
"body": {"content": b64("after"), "sha": c["update_file_sha"], "message": "sweep update"},
|
||||
},
|
||||
"gitea_repo_delete_file": lambda c: {
|
||||
"filepath": "to-delete.txt",
|
||||
"body": {"sha": c["delete_file_sha"], "message": "sweep delete"},
|
||||
},
|
||||
"gitea_repo_change_files": lambda c: {
|
||||
"body": {
|
||||
"files": [{"operation": "create", "path": f"cf-{fresh(c)}.txt", "content": b64("c")}],
|
||||
"message": "sweep change files",
|
||||
}
|
||||
},
|
||||
"gitea_repo_apply_diff_patch": lambda c: {
|
||||
"body": {"content": PATCH_TEXT.format(name=fresh(c)), "message": "sweep patch"}
|
||||
},
|
||||
"gitea_repo_create_branch": lambda c: {
|
||||
"body": {"new_branch_name": fresh(c, "br"), "old_branch_name": "main"}
|
||||
},
|
||||
"gitea_repo_create_branch_protection": lambda c: {"body": {"rule_name": "hotfix/*"}},
|
||||
"gitea_repo_create_tag": lambda c: {"body": {"tag_name": "v0.0.2", "target": "main"}},
|
||||
"gitea_repo_create_tag_protection": lambda c: {
|
||||
"body": {"name_pattern": "w*", "whitelist_usernames": [c["admin"]]}
|
||||
},
|
||||
"gitea_repo_create_fork": lambda c: {
|
||||
"body": {"organization": c["del_org"], "name": fresh(c, "fork")}
|
||||
},
|
||||
"gitea_repo_create_key": lambda c: {
|
||||
"body": {"title": fresh(c, "key"), "key": ssh_public_key()}
|
||||
},
|
||||
"gitea_repo_create_deploy_token": lambda c: {"body": {"title": fresh(c, "dt")}},
|
||||
"gitea_repo_create_status": lambda c: {"body": {"state": "success", "context": "ci/two"}},
|
||||
"gitea_repo_create_project": lambda c: {"body": {"title": fresh(c, "proj")}},
|
||||
"gitea_repo_create_project_column": lambda c: {
|
||||
"id": c["repo_project"],
|
||||
"body": {"title": fresh(c, "col")},
|
||||
},
|
||||
"gitea_repo_edit": lambda c: {"body": {"description": "edited by the sweep"}},
|
||||
"gitea_repo_edit_branch_protection": lambda c: {"body": {"required_approvals": 0}},
|
||||
"gitea_repo_edit_tag_protection": lambda c: {
|
||||
"body": {"name_pattern": "v9*", "whitelist_usernames": [c["admin"]]}
|
||||
},
|
||||
"gitea_repo_edit_project": lambda c: {"body": {"title": "renamed project"}},
|
||||
"gitea_repo_edit_project_column": lambda c: {"body": {"title": "renamed column"}},
|
||||
"gitea_repo_generate_repo": lambda c: {
|
||||
"body": {"name": fresh(c, "gen"), "owner": c["admin"], "git_content": True}
|
||||
},
|
||||
"gitea_repo_rename_branch": lambda c: {"branch": "rn", "body": {"name": "rn2"}},
|
||||
"gitea_repo_transfer": lambda c: {"repo": c["xfer_repo"], "body": {"new_owner": c["org"]}},
|
||||
"gitea_repo_update_avatar": lambda c: {"body": {"image": PNG_1X1}},
|
||||
"gitea_repo_update_branch": lambda c: {
|
||||
"branch": "feat3",
|
||||
"body": {"new_commit_id": c["sha"], "force": True},
|
||||
},
|
||||
"gitea_repo_update_branch_protection_priorities": lambda c: {"body": {"ids": []}},
|
||||
"gitea_repo_delete": lambda c: {"repo": c["del_repo"]},
|
||||
"gitea_repo_delete_branch": lambda c: {"branch": "del-branch"},
|
||||
"gitea_repo_delete_collaborator": lambda c: {"collaborator": c["user2"]},
|
||||
# ------------------------------------------------------------------- wiki
|
||||
"gitea_wiki_create_wiki_page": lambda c: {
|
||||
"body": {"title": "Page2", "content_base64": b64("two"), "message": "page two"}
|
||||
},
|
||||
"gitea_wiki_edit_wiki_page": lambda c: {
|
||||
"body": {"title": "Home", "content_base64": b64("# home v2\n"), "message": "edit"}
|
||||
},
|
||||
# ------------------------------------------------------------------ teams
|
||||
"gitea_team_create_team": lambda c: {
|
||||
"body": {"name": fresh(c, "team"), "permission": "read", "units": ["repo.code"]}
|
||||
},
|
||||
"gitea_team_edit_team": lambda c: {
|
||||
"body": {"name": c["team_name"], "description": "edited by the sweep"}
|
||||
},
|
||||
# ------------------------------------------------------------------ users
|
||||
"gitea_user_add_email": lambda c: {"body": {"emails": [f"sw-{fresh(c)}@example.test"]}},
|
||||
"gitea_user_delete_email": lambda c: {"body": {"emails": [c["email"]]}},
|
||||
"gitea_user_check_user_block": lambda c: {"username": c["user2"]},
|
||||
"gitea_user_unblock_user": lambda c: {"username": c["user2"]},
|
||||
"gitea_user_create_oauth2_application": lambda c: {
|
||||
"body": {"name": fresh(c, "app"), "redirect_uris": ["http://127.0.0.1:1/cb"]}
|
||||
},
|
||||
"gitea_user_update_oauth2_application": lambda c: {
|
||||
"body": {"name": fresh(c, "app2"), "redirect_uris": ["http://127.0.0.1:1/cb2"]}
|
||||
},
|
||||
"gitea_user_current_create_repo": lambda c: {"body": {"name": fresh(c, "sw-urepo")}},
|
||||
"gitea_user_current_create_project": lambda c: {"body": {"title": fresh(c, "proj")}},
|
||||
"gitea_user_current_create_project_column": lambda c: {
|
||||
"id": c["user_project"],
|
||||
"body": {"title": fresh(c, "col")},
|
||||
},
|
||||
"gitea_user_current_post_key": lambda c: {
|
||||
"body": {"title": fresh(c, "key"), "key": ssh_public_key()}
|
||||
},
|
||||
"gitea_user_current_post_gpg_key": lambda c: {"body": {"armored_public_key": "not a key"}},
|
||||
"gitea_user_delete_access_token": lambda c: {
|
||||
"username": c["admin"],
|
||||
"token": c["throwaway_token"],
|
||||
},
|
||||
"gitea_user_update_avatar": lambda c: {"body": {"image": PNG_1X1}},
|
||||
"gitea_user_update_settings": lambda c: {"body": {"description": "sweep"}},
|
||||
# -------------------------------------------------------------------- raw
|
||||
RAW_TOOL: lambda c: {"method": "GET", "path": "/version"},
|
||||
}
|
||||
|
||||
|
||||
OVERRIDES.update(_project_overrides())
|
||||
|
||||
|
||||
def _new_user(c: Ctx) -> dict[str, Any]:
|
||||
name = fresh(c, "sw-new")
|
||||
return {
|
||||
"body": {
|
||||
"username": name,
|
||||
"email": f"{name}@example.test",
|
||||
"password": secrets.token_urlsafe(18),
|
||||
"must_change_password": False,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def build_arguments(operation: Operation, c: Ctx) -> dict[str, Any]:
|
||||
"""Synthesize the arguments of one catalog operation from the seed context."""
|
||||
args: dict[str, Any] = {}
|
||||
for parameter in operation.parameters:
|
||||
if parameter.location == "path":
|
||||
value = _path_value(operation, parameter.name, c)
|
||||
args[parameter.arg_name] = _coerce(value, parameter.schema)
|
||||
elif parameter.is_file:
|
||||
args[parameter.arg_name] = {
|
||||
"filename": "sweep.txt",
|
||||
"content_base64": b64("sweep"),
|
||||
"content_type": "text/plain",
|
||||
}
|
||||
elif parameter.location == "body":
|
||||
needs_members = bool(parameter.schema.get("required"))
|
||||
args[parameter.arg_name] = (
|
||||
sample(parameter.schema) if parameter.required or needs_members else {}
|
||||
)
|
||||
elif parameter.required:
|
||||
args[parameter.arg_name] = sample(parameter.schema, parameter.name)
|
||||
override = OVERRIDES.get(operation.tool_name)
|
||||
if override is not None:
|
||||
args.update(override(c))
|
||||
for name, value in list(args.items()):
|
||||
parameter = next((p for p in operation.parameters if p.arg_name == name), None)
|
||||
if parameter is not None and parameter.location == "path":
|
||||
args[name] = _coerce(value, parameter.schema)
|
||||
return args
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ordering and classification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PHASE_BY_METHOD = {"GET": 1, "HEAD": 1, "POST": 2, "PUT": 3, "PATCH": 4, "DELETE": 5}
|
||||
#: Explicit priorities inside a phase (higher runs later; default 50).
|
||||
PRIORITY: dict[str, int] = {
|
||||
"gitea_pull_merge_pull_request": 90,
|
||||
# Moving columns must list every column, so it runs before any column is created.
|
||||
"gitea_org_move_project_columns": 10,
|
||||
"gitea_repo_move_project_columns": 10,
|
||||
"gitea_user_current_move_project_columns": 10,
|
||||
}
|
||||
|
||||
|
||||
def phase_of(operation: Operation) -> int:
|
||||
"""0 for read-only operations, then 2 (POST), 3 (PUT), 4 (PATCH) and 5 (DELETE)."""
|
||||
if operation.read_only:
|
||||
return 0
|
||||
return PHASE_BY_METHOD.get(operation.method, 2)
|
||||
|
||||
|
||||
def sort_key(operation: Operation) -> tuple[int, int, int, str]:
|
||||
"""Sort key: phase, priority, leaf-first depth for deletes, name."""
|
||||
phase = phase_of(operation)
|
||||
depth = -operation.path.count("/") if phase == 5 else 0
|
||||
return (phase, PRIORITY.get(operation.tool_name, 50), depth, operation.tool_name)
|
||||
|
||||
|
||||
def ordered_operations(catalog: Catalog) -> list[Operation]:
|
||||
"""Every operation that the generic sweep calls, in execution order."""
|
||||
operations = [op for op in catalog.operations if op.tool_name not in EXCLUDED]
|
||||
return sorted(operations, key=sort_key)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SweepRecord:
|
||||
"""The outcome of one sweep call."""
|
||||
|
||||
tool: str
|
||||
domain: str
|
||||
method: str
|
||||
path: str
|
||||
status: int | None
|
||||
is_error: bool
|
||||
kind: str | None
|
||||
message: str
|
||||
outcome: str
|
||||
reason: str = ""
|
||||
|
||||
|
||||
def classify(
|
||||
tool: str,
|
||||
status: int | None,
|
||||
is_error: bool,
|
||||
documented: frozenset[int],
|
||||
) -> tuple[str, str]:
|
||||
"""Return ``(outcome, reason)``: success, documented, allowance or unexpected."""
|
||||
if status is not None and 200 <= status < 300 and not is_error:
|
||||
return "success", ""
|
||||
if status is not None and status in documented:
|
||||
return "documented", ""
|
||||
allowance = ALLOWANCES.get(tool)
|
||||
if allowance is not None and status in allowance.statuses:
|
||||
return "allowance", allowance.reason
|
||||
return "unexpected", ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Results and report
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class CallLog:
|
||||
"""Every ``(tool, status, is_error)`` seen during the session, from every test."""
|
||||
|
||||
calls: list[tuple[str, int | None, bool]] = field(default_factory=list)
|
||||
sweep: list[SweepRecord] = field(default_factory=list)
|
||||
|
||||
def add(self, tool: str, status: int | None, is_error: bool) -> None:
|
||||
"""Record one call."""
|
||||
self.calls.append((tool, status, is_error))
|
||||
|
||||
|
||||
LOG = CallLog()
|
||||
|
||||
|
||||
def _bucket(tool: str, log: CallLog) -> str:
|
||||
"""``success`` when any call of the session got a 2xx, else the sweep outcome."""
|
||||
for name, status, is_error in log.calls:
|
||||
if name == tool and status is not None and 200 <= status < 300 and not is_error:
|
||||
return "success"
|
||||
for record in log.sweep:
|
||||
if record.tool == tool:
|
||||
return record.outcome
|
||||
return "not called"
|
||||
|
||||
|
||||
def render_report(catalog: Catalog, log: CallLog, gitea_version: str) -> str:
|
||||
"""Render the coverage report as Markdown (no em dashes, no bold)."""
|
||||
tool_domain = {op.tool_name: op.domain for op in catalog.operations}
|
||||
tool_domain[RAW_TOOL] = "misc"
|
||||
for name in EXCLUDED:
|
||||
tool_domain.setdefault(name, "misc")
|
||||
by_domain: dict[str, list[str]] = {}
|
||||
for tool, domain in sorted(tool_domain.items()):
|
||||
by_domain.setdefault(domain, []).append(tool)
|
||||
sweep = {record.tool: record for record in log.sweep}
|
||||
totals = {"success": 0, "documented": 0, "allowance": 0, "other": 0}
|
||||
sections: list[str] = []
|
||||
rows: list[str] = []
|
||||
for domain in sorted(by_domain):
|
||||
tools = by_domain[domain]
|
||||
buckets = {tool: _bucket(tool, log) for tool in tools}
|
||||
counts = {key: sum(1 for b in buckets.values() if b == key) for key in totals}
|
||||
counts["other"] = (
|
||||
len(tools) - counts["success"] - counts["documented"] - counts["allowance"]
|
||||
)
|
||||
for key in totals:
|
||||
totals[key] += counts[key]
|
||||
rows.append(
|
||||
f"| {domain} | {len(tools)} | {counts['success']} | {counts['documented']} | "
|
||||
f"{counts['allowance']} | {counts['other']} |"
|
||||
)
|
||||
lines = [f"## {domain}", ""]
|
||||
ok = [t for t in tools if buckets[t] == "success"]
|
||||
lines.append(f"Tools with a 2xx ({len(ok)}):")
|
||||
lines.append("")
|
||||
lines.append(", ".join(f"`{t}`" for t in ok) if ok else "none")
|
||||
lines.append("")
|
||||
documented = [t for t in tools if buckets[t] == "documented"]
|
||||
lines.append(f"Tools with a documented non-2xx status ({len(documented)}):")
|
||||
lines.append("")
|
||||
if documented:
|
||||
lines.extend(["| Tool | Status |", "| --- | --- |"])
|
||||
lines.extend(f"| `{t}` | {sweep[t].status} |" for t in documented)
|
||||
else:
|
||||
lines.append("none")
|
||||
lines.append("")
|
||||
allowed = [t for t in tools if buckets[t] == "allowance"]
|
||||
lines.append(f"Allowances ({len(allowed)}):")
|
||||
lines.append("")
|
||||
if allowed:
|
||||
lines.extend(["| Tool | Status | Reason |", "| --- | --- | --- |"])
|
||||
lines.extend(f"| `{t}` | {sweep[t].status} | {sweep[t].reason} |" for t in allowed)
|
||||
else:
|
||||
lines.append("none")
|
||||
lines.append("")
|
||||
other = [t for t in tools if buckets[t] not in ("success", "documented", "allowance")]
|
||||
if other:
|
||||
lines.append(f"Not covered by this run ({len(other)}):")
|
||||
lines.append("")
|
||||
lines.append(", ".join(f"`{t}`" for t in other))
|
||||
lines.append("")
|
||||
sections.append("\n".join(lines))
|
||||
header = [
|
||||
"# Integration coverage",
|
||||
"",
|
||||
"Generated by the live-Gitea integration suite; do not edit by hand. Regenerate with:",
|
||||
"",
|
||||
"```bash",
|
||||
f"{REPORT_ENV}=1 AEGIS_INTEGRATION=1 uv run pytest tests/integration -m integration --no-cov",
|
||||
"```",
|
||||
"",
|
||||
f"Target: Gitea {gitea_version}. Every tool of the server ({len(tool_domain)}) is called "
|
||||
"at least once through the MCP server against a seeded instance (see "
|
||||
"`tests/integration/test_sweep.py`). A call counts as covered when it reaches Gitea and "
|
||||
"answers 2xx, answers a status the swagger documents for that operation, or answers a "
|
||||
"status listed as an explicit allowance below. Scenario tests "
|
||||
"(`tests/integration/test_scenario_*.py`) add real state changes; a tool counts as 2xx "
|
||||
"when any call in the run got a 2xx.",
|
||||
"",
|
||||
"| Domain | Tools | 2xx | Documented non-2xx | Allowances | Not covered |",
|
||||
"| --- | --- | --- | --- | --- | --- |",
|
||||
*rows,
|
||||
f"| total | {len(tool_domain)} | {totals['success']} | {totals['documented']} | "
|
||||
f"{totals['allowance']} | {totals['other']} |",
|
||||
"",
|
||||
]
|
||||
if EXCLUDED:
|
||||
header.append("Excluded from the generic sweep and covered by a dedicated test:")
|
||||
header.append("")
|
||||
header.extend(f"- `{tool}`: {reason}" for tool, reason in sorted(EXCLUDED.items()))
|
||||
header.append("")
|
||||
return "\n".join(header) + "\n" + "\n".join(sections)
|
||||
|
||||
|
||||
def write_report_if_enabled(catalog: Catalog, gitea_version: str) -> bool:
|
||||
"""Write the report when ``AEGIS_IT_WRITE_REPORT=1`` and the sweep ran."""
|
||||
if os.environ.get(REPORT_ENV) != "1" or not LOG.sweep:
|
||||
return False
|
||||
REPORT_PATH.write_text(render_report(catalog, LOG, gitea_version), encoding="utf-8")
|
||||
return True
|
||||
|
||||
|
||||
def failing(records: Iterable[SweepRecord]) -> list[SweepRecord]:
|
||||
"""The records whose outcome is ``unexpected``."""
|
||||
return [record for record in records if record.outcome == "unexpected"]
|
||||
|
||||
|
||||
def summarize(records: Sequence[SweepRecord]) -> dict[str, int]:
|
||||
"""Counts per outcome."""
|
||||
result: dict[str, int] = {}
|
||||
for record in records:
|
||||
result[record.outcome] = result.get(record.outcome, 0) + 1
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The sweep itself
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CallTool = Callable[..., Any]
|
||||
|
||||
|
||||
def _short(text: str, limit: int = 160) -> str:
|
||||
return " ".join(text.split())[:limit]
|
||||
|
||||
|
||||
async def run_sweep(
|
||||
call: CallTool, catalog: Catalog, spec: dict[str, Any], c: Ctx
|
||||
) -> list[SweepRecord]:
|
||||
"""Call every tool once (in plan order) and classify the results.
|
||||
|
||||
``call(tool, **arguments)`` must return an object with ``status``, ``is_error``,
|
||||
``error_kind`` and ``payload`` (a :class:`tests.integration.conftest.ToolResult`).
|
||||
"""
|
||||
records: list[SweepRecord] = []
|
||||
jobs: list[tuple[str, str, str, str, frozenset[int], dict[str, Any]]] = []
|
||||
for operation in ordered_operations(catalog):
|
||||
jobs.append(
|
||||
(
|
||||
operation.tool_name,
|
||||
operation.domain,
|
||||
operation.method,
|
||||
operation.path,
|
||||
documented_statuses(spec, operation),
|
||||
build_arguments(operation, c),
|
||||
)
|
||||
)
|
||||
jobs.append((RAW_TOOL, "misc", "GET", "/version", frozenset({200}), OVERRIDES[RAW_TOOL](c)))
|
||||
for tool, domain, method, path, documented, arguments in jobs:
|
||||
result = await call(tool, **arguments)
|
||||
error = result.payload.get("error") if result.is_error else None
|
||||
message = ""
|
||||
if isinstance(error, dict):
|
||||
message = _short(str(error.get("message", "")))
|
||||
outcome, reason = classify(tool, result.status, result.is_error, documented)
|
||||
records.append(
|
||||
SweepRecord(
|
||||
tool=tool,
|
||||
domain=domain,
|
||||
method=method,
|
||||
path=path,
|
||||
status=result.status,
|
||||
is_error=result.is_error,
|
||||
kind=result.error_kind,
|
||||
message=message,
|
||||
outcome=outcome,
|
||||
reason=reason,
|
||||
)
|
||||
)
|
||||
return records
|
||||
Reference in new issue
Block a user