test: pin the sweep outcome per tool in a baseline, seed a GPG key, fail loudly when seeding finds no run
This commit is contained in:
1 parent
84b361962e
commit
9e51f72ec9
6 files changed
+761
-14
No files matched your search
@@ -271,3 +271,49 @@ def test_report_is_only_written_when_enabled(
|
||||
]
|
||||
assert sweep_plan.write_report_if_enabled(CATALOG, "28.1.0") is True
|
||||
assert target.read_text(encoding="utf-8").startswith("# Integration coverage")
|
||||
|
||||
|
||||
def _record(tool: str, status: int | None, is_error: bool = False) -> SweepRecord:
|
||||
return SweepRecord(tool, "repo", "GET", "/x", status, is_error, None, "", "success")
|
||||
|
||||
|
||||
def test_outcome_class_is_2xx_or_the_exact_status() -> None:
|
||||
assert sweep_plan.outcome_class(_record("t", 200)) == "2xx"
|
||||
assert sweep_plan.outcome_class(_record("t", 204)) == "2xx"
|
||||
assert sweep_plan.outcome_class(_record("t", 404, True)) == 404
|
||||
assert sweep_plan.outcome_class(_record("t", 200, True)) == 200 # an error flag wins
|
||||
assert sweep_plan.outcome_class(_record("t", None, True)) == "no-status"
|
||||
|
||||
|
||||
def test_baseline_reports_every_regression_at_once_and_notes_upgrades() -> None:
|
||||
baseline: dict[str, Any] = {"a": "2xx", "b": 404, "c": 404, "d": "2xx", "gone": "2xx"}
|
||||
records = [
|
||||
_record("a", 404, True), # 2xx -> 404: regression
|
||||
_record("b", 200), # 404 -> 2xx: upgrade
|
||||
_record("c", 403, True), # 404 -> 403: a different status is a regression
|
||||
_record("d", 200), # unchanged
|
||||
_record("new", 200), # not in the baseline
|
||||
]
|
||||
|
||||
diff = sweep_plan.compare_baseline(records, baseline)
|
||||
|
||||
assert diff.upgrades == ["b: baseline 404, now 2xx"]
|
||||
assert len(diff.regressions) == 4
|
||||
assert diff.regressions[0].startswith("a: baseline 2xx, got 404")
|
||||
assert diff.regressions[1].startswith("c: baseline 404, got 403")
|
||||
assert "new: no baseline entry" in diff.regressions[2]
|
||||
assert diff.regressions[3] == "gone: in the baseline but not called by the sweep"
|
||||
|
||||
|
||||
def test_baseline_round_trips_and_covers_the_whole_sweep(tmp_path: Any) -> None:
|
||||
target = tmp_path / "baseline.json"
|
||||
records = [_record("b", 404, True), _record("a", 200)]
|
||||
sweep_plan.write_baseline(records, "28.1.0", target)
|
||||
assert sweep_plan.load_baseline(target) == {"a": "2xx", "b": 404}
|
||||
assert list(sweep_plan.load_baseline(target)) == ["a", "b"]
|
||||
|
||||
# The committed baseline names exactly the tools the sweep calls.
|
||||
committed = sweep_plan.load_baseline()
|
||||
swept = {op.tool_name for op in ordered_operations(CATALOG)} | {RAW_TOOL}
|
||||
assert set(committed) == swept
|
||||
assert all(value == "2xx" or isinstance(value, int) for value in committed.values())
|
||||
Reference in new issue
Block a user