74 lines
2.6 KiB
Python
74 lines
2.6 KiB
Python
"""The sweep: every tool is called at least once through the MCP server."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from aegis_gitea_mcp.catalog import load_catalog
|
|
from tests.integration.conftest import McpSession, RegularUser
|
|
from tests.integration.harness import GiteaInstance
|
|
from tests.integration.sweep_plan import (
|
|
EXCLUDED,
|
|
LOG,
|
|
RAW_TOOL,
|
|
failing,
|
|
load_spec,
|
|
run_sweep,
|
|
summarize,
|
|
)
|
|
from tests.integration.sweep_seed import Ctx, seed
|
|
|
|
#: At least this share of the tools must get a real 2xx, so a systematic argument problem
|
|
#: cannot hide behind documented 404 responses.
|
|
MIN_SUCCESS_SHARE = 0.55
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def sweep_context(gitea: GiteaInstance, regular_user: RegularUser) -> Ctx:
|
|
"""The seeded resources the sweep's arguments point at."""
|
|
return seed(gitea, regular_user.login, regular_user.token)
|
|
|
|
|
|
async def test_every_tool_reaches_gitea(
|
|
mcp: McpSession, sweep_context: Ctx, gitea: GiteaInstance, regular_user: RegularUser
|
|
) -> None:
|
|
catalog = load_catalog()
|
|
listed = {tool.name for tool in (await mcp.client.list_tools()).tools}
|
|
expected = {op.tool_name for op in catalog.operations} | {RAW_TOOL}
|
|
assert listed == expected
|
|
assert len(expected) == 538
|
|
|
|
records = await run_sweep(mcp.call, catalog, load_spec(), sweep_context)
|
|
LOG.sweep = records
|
|
|
|
called = [record.tool for record in records]
|
|
assert len(called) == len(set(called)), "a tool was called twice"
|
|
assert set(called) == expected - set(EXCLUDED), "tools missing from the sweep"
|
|
|
|
bad = failing(records)
|
|
summary = summarize(records)
|
|
report = "\n".join(
|
|
f" {r.tool}: status={r.status} kind={r.kind} {r.method} {r.path} :: {r.message}"
|
|
for r in bad
|
|
)
|
|
assert not bad, f"{len(bad)} tool(s) with an unexpected result ({summary}):\n{report}"
|
|
success_share = summary.get("success", 0) / len(records)
|
|
assert success_share >= MIN_SUCCESS_SHARE, f"only {success_share:.0%} of tools got a 2xx"
|
|
|
|
# Whatever the 538 calls did, no credential or secret value came back in a result.
|
|
hidden = [
|
|
gitea.admin_token,
|
|
gitea.admin_password,
|
|
regular_user.token,
|
|
regular_user.password,
|
|
"rotated-org-value",
|
|
"rotated-repo-value",
|
|
"rotated-user-value",
|
|
"org-secret-value",
|
|
"repo-secret-value",
|
|
"user-secret-value",
|
|
]
|
|
for value in hidden:
|
|
assert all(value not in text for text in mcp.texts), "a secret leaked into a result"
|
|
assert value not in mcp.audit_text(), "a secret leaked into the audit log"
|