test: pin the sweep outcome per tool in a baseline, seed a GPG key, fail loudly when seeding finds no run

tests/integration/test_sweep.py

@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import os
+
 import pytest
 
 from aegis_gitea_mcp.audit import validate_audit_log_integrity
@@ -9,20 +11,20 @@ from aegis_gitea_mcp.catalog import load_catalog
 from tests.integration.conftest import McpSession, RegularUser
 from tests.integration.harness import GiteaInstance
 from tests.integration.sweep_plan import (
+    BASELINE_ENV,
     EXCLUDED,
     LOG,
     RAW_TOOL,
+    compare_baseline,
     failing,
+    load_baseline,
     load_spec,
     run_sweep,
     summarize,
+    write_baseline,
 )
 from tests.integration.sweep_seed import Ctx, seed
 
-#: At least this share of the tools must get a real 2xx, so a systematic argument problem
-#: cannot hide behind documented 404 responses.
-MIN_SUCCESS_SHARE = 0.55
-
 
 @pytest.fixture(scope="module")
 def sweep_context(gitea: GiteaInstance, regular_user: RegularUser) -> Ctx:
@@ -53,8 +55,23 @@ async def test_every_tool_reaches_gitea(
         for r in bad
     )
     assert not bad, f"{len(bad)} tool(s) with an unexpected result ({summary}):\n{report}"
-    success_share = summary.get("success", 0) / len(records)
-    assert success_share >= MIN_SUCCESS_SHARE, f"only {success_share:.0%} of tools got a 2xx"
+
+    # The pinned outcome per tool: an argument or path regression cannot hide behind a
+    # documented 404, because a 2xx that turns into any other status fails here.
+    if os.environ.get(BASELINE_ENV) == "1":
+        write_baseline(records, load_catalog().spec_version)
+    else:
+        diff = compare_baseline(records, load_baseline())
+        if diff.upgrades:
+            print(
+                f"\n{len(diff.upgrades)} tool(s) improved on the baseline; refresh it with "
+                f"{BASELINE_ENV}=1:\n  " + "\n  ".join(diff.upgrades)
+            )
+        assert not diff.regressions, (
+            f"{len(diff.regressions)} tool(s) differ from sweep_baseline.json "
+            f"(regenerate with {BASELINE_ENV}=1 only when the change is intended):\n  "
+            + "\n  ".join(diff.regressions)
+        )
 
     # Whatever the 538 calls did, no credential or secret value came back in a result.
     hidden = [