latte
73d3dc6e37
All checks were successful
Enterprise AI Code Review / ai-review (pull_request) Successful in 16s
Implements help command that shows all available bot commands with examples and categories. Users can now type '@codebot help' to see complete command list without reading docs.
593 lines
19 KiB
Python
593 lines
19 KiB
Python
"""Test Suite for AI Code Review Workflow
|
|
|
|
Tests for verifying prompt formatting, agent logic, and core functionality.
|
|
Run with: pytest tests/ -v
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
|
|
# Add the tools directory to path
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools", "ai-review"))
|
|
|
|
import pytest
|
|
|
|
|
|
class TestPromptFormatting:
|
|
"""Test that all prompts can be formatted without errors."""
|
|
|
|
def get_prompt_path(self, name: str) -> str:
|
|
"""Get the full path to a prompt file."""
|
|
return os.path.join(
|
|
os.path.dirname(__file__),
|
|
"..",
|
|
"tools",
|
|
"ai-review",
|
|
"prompts",
|
|
f"{name}.md",
|
|
)
|
|
|
|
def load_prompt(self, name: str) -> str:
|
|
"""Load a prompt file."""
|
|
path = self.get_prompt_path(name)
|
|
with open(path) as f:
|
|
return f.read()
|
|
|
|
def test_issue_triage_prompt_formatting(self):
|
|
"""Test that issue_triage.md can be formatted with placeholders."""
|
|
prompt = self.load_prompt("issue_triage")
|
|
|
|
# This should NOT raise a KeyError
|
|
formatted = prompt.format(
|
|
title="Test Issue Title",
|
|
body="This is the issue body content",
|
|
author="testuser",
|
|
existing_labels="bug, urgent",
|
|
)
|
|
|
|
assert "Test Issue Title" in formatted
|
|
assert "This is the issue body content" in formatted
|
|
assert "testuser" in formatted
|
|
assert "bug, urgent" in formatted
|
|
# JSON example should still be present (curly braces escaped)
|
|
assert '"type"' in formatted
|
|
assert '"priority"' in formatted
|
|
|
|
def test_issue_response_prompt_formatting(self):
|
|
"""Test that issue_response.md can be formatted with placeholders."""
|
|
prompt = self.load_prompt("issue_response")
|
|
|
|
formatted = prompt.format(
|
|
issue_type="bug",
|
|
priority="high",
|
|
title="Bug Report",
|
|
body="Description of the bug",
|
|
triage_analysis="This is a high priority bug",
|
|
)
|
|
|
|
assert "bug" in formatted
|
|
assert "high" in formatted
|
|
assert "Bug Report" in formatted
|
|
# JSON example should still be present
|
|
assert '"comment"' in formatted
|
|
|
|
def test_base_prompt_no_placeholders(self):
|
|
"""Test that base.md loads correctly (no placeholders needed)."""
|
|
prompt = self.load_prompt("base")
|
|
|
|
# Should contain key elements
|
|
assert "security" in prompt.lower()
|
|
assert "JSON" in prompt
|
|
assert "severity" in prompt.lower()
|
|
|
|
def test_prompts_have_escaped_json(self):
|
|
"""Verify JSON examples use double curly braces."""
|
|
for prompt_name in ["issue_triage", "issue_response"]:
|
|
prompt = self.load_prompt(prompt_name)
|
|
|
|
# Check that format() doesn't fail
|
|
try:
|
|
# Try with minimal placeholders
|
|
if prompt_name == "issue_triage":
|
|
prompt.format(title="t", body="b", author="a", existing_labels="l")
|
|
elif prompt_name == "issue_response":
|
|
prompt.format(
|
|
issue_type="t",
|
|
priority="p",
|
|
title="t",
|
|
body="b",
|
|
triage_analysis="a",
|
|
)
|
|
except KeyError as e:
|
|
pytest.fail(f"Prompt {prompt_name} has unescaped curly braces: {e}")
|
|
|
|
|
|
class TestImports:
|
|
"""Test that all modules can be imported correctly."""
|
|
|
|
def test_import_agents(self):
|
|
"""Test importing agent classes."""
|
|
from agents.base_agent import AgentContext, AgentResult, BaseAgent
|
|
from agents.codebase_agent import CodebaseAgent
|
|
from agents.issue_agent import IssueAgent
|
|
from agents.pr_agent import PRAgent
|
|
|
|
assert BaseAgent is not None
|
|
assert IssueAgent is not None
|
|
assert PRAgent is not None
|
|
assert CodebaseAgent is not None
|
|
|
|
def test_import_clients(self):
|
|
"""Test importing client classes."""
|
|
from clients.gitea_client import GiteaClient
|
|
from clients.llm_client import LLMClient
|
|
|
|
assert GiteaClient is not None
|
|
assert LLMClient is not None
|
|
|
|
def test_import_security(self):
|
|
"""Test importing security scanner."""
|
|
from security.security_scanner import SecurityScanner
|
|
|
|
assert SecurityScanner is not None
|
|
|
|
def test_import_enterprise(self):
|
|
"""Test importing enterprise features."""
|
|
from enterprise.audit_logger import AuditLogger
|
|
from enterprise.metrics import MetricsCollector
|
|
|
|
assert AuditLogger is not None
|
|
assert MetricsCollector is not None
|
|
|
|
def test_import_dispatcher(self):
|
|
"""Test importing dispatcher."""
|
|
from dispatcher import Dispatcher
|
|
|
|
assert Dispatcher is not None
|
|
|
|
|
|
class TestSecurityScanner:
|
|
"""Test security scanner pattern detection."""
|
|
|
|
def test_detects_hardcoded_secret(self):
|
|
"""Test detection of hardcoded secrets."""
|
|
from security.security_scanner import SecurityScanner
|
|
|
|
scanner = SecurityScanner()
|
|
code = """
|
|
API_KEY = "sk-1234567890abcdef"
|
|
"""
|
|
findings = list(scanner.scan_content(code, "test.py"))
|
|
assert len(findings) >= 1
|
|
assert any(f.severity == "HIGH" for f in findings)
|
|
|
|
def test_detects_eval(self):
|
|
"""Test detection of eval usage."""
|
|
from security.security_scanner import SecurityScanner
|
|
|
|
scanner = SecurityScanner()
|
|
code = """
|
|
result = eval(user_input)
|
|
"""
|
|
findings = list(scanner.scan_content(code, "test.py"))
|
|
assert len(findings) >= 1
|
|
assert any("eval" in f.rule_name.lower() for f in findings)
|
|
|
|
def test_no_false_positives_on_clean_code(self):
|
|
"""Test that clean code doesn't trigger false positives."""
|
|
from security.security_scanner import SecurityScanner
|
|
|
|
scanner = SecurityScanner()
|
|
code = """
|
|
def hello():
|
|
print("Hello, world!")
|
|
return 42
|
|
"""
|
|
findings = list(scanner.scan_content(code, "test.py"))
|
|
# Should have no HIGH severity issues for clean code
|
|
high_findings = [f for f in findings if f.severity == "HIGH"]
|
|
assert len(high_findings) == 0
|
|
|
|
|
|
class TestAgentContext:
|
|
"""Test agent context and result dataclasses."""
|
|
|
|
def test_agent_context_creation(self):
|
|
"""Test creating AgentContext."""
|
|
from agents.base_agent import AgentContext
|
|
|
|
context = AgentContext(
|
|
owner="testowner",
|
|
repo="testrepo",
|
|
event_type="issues",
|
|
event_data={"action": "opened"},
|
|
config={},
|
|
)
|
|
|
|
assert context.owner == "testowner"
|
|
assert context.repo == "testrepo"
|
|
assert context.event_type == "issues"
|
|
|
|
def test_agent_result_creation(self):
|
|
"""Test creating AgentResult."""
|
|
from agents.base_agent import AgentResult
|
|
|
|
result = AgentResult(
|
|
success=True,
|
|
message="Test passed",
|
|
data={"key": "value"},
|
|
actions_taken=["action1", "action2"],
|
|
)
|
|
|
|
assert result.success is True
|
|
assert result.message == "Test passed"
|
|
assert len(result.actions_taken) == 2
|
|
|
|
|
|
class TestMetrics:
|
|
"""Test metrics collection."""
|
|
|
|
def test_counter_increment(self):
|
|
"""Test counter metrics."""
|
|
from enterprise.metrics import Counter
|
|
|
|
counter = Counter("test_counter")
|
|
assert counter.value == 0
|
|
counter.inc()
|
|
assert counter.value == 1
|
|
counter.inc(5)
|
|
assert counter.value == 6
|
|
|
|
def test_histogram_observation(self):
|
|
"""Test histogram metrics."""
|
|
from enterprise.metrics import Histogram
|
|
|
|
hist = Histogram("test_histogram")
|
|
hist.observe(0.1)
|
|
hist.observe(0.5)
|
|
hist.observe(1.0)
|
|
|
|
assert hist.count == 3
|
|
assert hist.sum == 1.6
|
|
|
|
def test_metrics_collector_summary(self):
|
|
"""Test metrics collector summary."""
|
|
from enterprise.metrics import MetricsCollector
|
|
|
|
collector = MetricsCollector()
|
|
collector.record_request_start("TestAgent")
|
|
collector.record_request_end("TestAgent", success=True, duration_seconds=0.5)
|
|
|
|
summary = collector.get_summary()
|
|
assert summary["requests"]["total"] == 1
|
|
assert summary["requests"]["success"] == 1
|
|
|
|
|
|
class TestHelpCommand:
|
|
"""Test help command functionality."""
|
|
|
|
def test_help_command_returns_text(self):
|
|
"""Test that help command returns formatted help text."""
|
|
from agents.issue_agent import IssueAgent
|
|
|
|
agent = IssueAgent(
|
|
gitea_client=None,
|
|
llm_client=None,
|
|
config={
|
|
"interaction": {
|
|
"mention_prefix": "@codebot",
|
|
"commands": ["help", "triage", "explain"],
|
|
}
|
|
},
|
|
)
|
|
|
|
help_text = agent._command_help()
|
|
|
|
assert help_text is not None
|
|
assert len(help_text) > 100
|
|
assert "@codebot" in help_text
|
|
assert "help" in help_text.lower()
|
|
assert "triage" in help_text.lower()
|
|
|
|
def test_help_includes_all_sections(self):
|
|
"""Test that help text includes all major sections."""
|
|
from agents.issue_agent import IssueAgent
|
|
|
|
agent = IssueAgent(
|
|
gitea_client=None,
|
|
llm_client=None,
|
|
config={"interaction": {"mention_prefix": "@codebot"}},
|
|
)
|
|
|
|
help_text = agent._command_help()
|
|
|
|
# Check for main sections
|
|
assert "Issue Triage" in help_text
|
|
assert "Interactive Chat" in help_text
|
|
assert "Setup & Utility" in help_text
|
|
assert "Pull Request" in help_text
|
|
assert "Quick Examples" in help_text
|
|
|
|
def test_help_uses_custom_bot_name(self):
|
|
"""Test that help command uses custom bot name from config."""
|
|
from agents.issue_agent import IssueAgent
|
|
|
|
agent = IssueAgent(
|
|
gitea_client=None,
|
|
llm_client=None,
|
|
config={"interaction": {"mention_prefix": "@mybot"}},
|
|
)
|
|
|
|
help_text = agent._command_help()
|
|
|
|
assert "@mybot" in help_text
|
|
assert "@codebot" not in help_text
|
|
|
|
def test_help_includes_examples(self):
|
|
"""Test that help text includes usage examples."""
|
|
from agents.issue_agent import IssueAgent
|
|
|
|
agent = IssueAgent(
|
|
gitea_client=None,
|
|
llm_client=None,
|
|
config={"interaction": {"mention_prefix": "@codebot"}},
|
|
)
|
|
|
|
help_text = agent._command_help()
|
|
|
|
# Check for example commands
|
|
assert "triage" in help_text
|
|
assert "explain" in help_text
|
|
assert "setup-labels" in help_text
|
|
|
|
|
|
class TestLabelSetup:
|
|
"""Test label setup and schema detection."""
|
|
|
|
def test_detect_prefix_slash_schema(self):
|
|
"""Test detection of Kind/Bug style labels."""
|
|
from agents.issue_agent import IssueAgent
|
|
|
|
# Create mock agent with config
|
|
agent = IssueAgent(
|
|
gitea_client=None,
|
|
llm_client=None,
|
|
config={
|
|
"label_patterns": {
|
|
"prefix_slash": r"^(Kind|Type|Category)/(.+)$",
|
|
"prefix_dash": r"^(Priority|Status|Reviewed) - (.+)$",
|
|
"colon": r"^(type|priority|status): (.+)$",
|
|
}
|
|
},
|
|
)
|
|
|
|
labels = [
|
|
{"name": "Kind/Bug", "color": "d73a4a"},
|
|
{"name": "Kind/Feature", "color": "1d76db"},
|
|
{"name": "Kind/Documentation", "color": "0075ca"},
|
|
{"name": "Priority - High", "color": "d73a4a"},
|
|
{"name": "Priority - Low", "color": "28a745"},
|
|
]
|
|
|
|
schema = agent._detect_label_schema(labels)
|
|
|
|
assert schema is not None
|
|
assert schema["pattern"] == "prefix_slash"
|
|
assert "type" in schema["categories"]
|
|
assert "priority" in schema["categories"]
|
|
assert len(schema["categories"]["type"]) == 3
|
|
|
|
def test_detect_prefix_dash_schema(self):
|
|
"""Test detection of Priority - High style labels."""
|
|
from agents.issue_agent import IssueAgent
|
|
|
|
agent = IssueAgent(
|
|
gitea_client=None,
|
|
llm_client=None,
|
|
config={
|
|
"label_patterns": {
|
|
"prefix_slash": r"^(Kind|Type|Category)/(.+)$",
|
|
"prefix_dash": r"^(Priority|Status|Reviewed) - (.+)$",
|
|
"colon": r"^(type|priority|status): (.+)$",
|
|
}
|
|
},
|
|
)
|
|
|
|
labels = [
|
|
{"name": "Priority - Critical", "color": "b60205"},
|
|
{"name": "Priority - High", "color": "d73a4a"},
|
|
{"name": "Status - Blocked", "color": "fef2c0"},
|
|
]
|
|
|
|
schema = agent._detect_label_schema(labels)
|
|
|
|
assert schema is not None
|
|
assert schema["pattern"] == "prefix_dash"
|
|
assert "priority" in schema["categories"]
|
|
assert "status" in schema["categories"]
|
|
|
|
def test_detect_colon_schema(self):
|
|
"""Test detection of type: bug style labels."""
|
|
from agents.issue_agent import IssueAgent
|
|
|
|
agent = IssueAgent(
|
|
gitea_client=None,
|
|
llm_client=None,
|
|
config={
|
|
"label_patterns": {
|
|
"prefix_slash": r"^(Kind|Type|Category)/(.+)$",
|
|
"prefix_dash": r"^(Priority|Status|Reviewed) - (.+)$",
|
|
"colon": r"^(type|priority|status): (.+)$",
|
|
}
|
|
},
|
|
)
|
|
|
|
labels = [
|
|
{"name": "type: bug", "color": "d73a4a"},
|
|
{"name": "type: feature", "color": "1d76db"},
|
|
{"name": "priority: high", "color": "d73a4a"},
|
|
]
|
|
|
|
schema = agent._detect_label_schema(labels)
|
|
|
|
assert schema is not None
|
|
assert schema["pattern"] == "colon"
|
|
assert "type" in schema["categories"]
|
|
assert "priority" in schema["categories"]
|
|
|
|
def test_build_label_mapping(self):
|
|
"""Test building label mapping from existing labels."""
|
|
from agents.issue_agent import IssueAgent
|
|
|
|
agent = IssueAgent(
|
|
gitea_client=None,
|
|
llm_client=None,
|
|
config={
|
|
"labels": {
|
|
"type": {
|
|
"bug": {"name": "type: bug", "aliases": ["Kind/Bug", "bug"]},
|
|
"feature": {
|
|
"name": "type: feature",
|
|
"aliases": ["Kind/Feature", "feature"],
|
|
},
|
|
},
|
|
"priority": {
|
|
"high": {
|
|
"name": "priority: high",
|
|
"aliases": ["Priority - High", "P1"],
|
|
}
|
|
},
|
|
"status": {},
|
|
}
|
|
},
|
|
)
|
|
|
|
existing_labels = [
|
|
{"name": "Kind/Bug", "color": "d73a4a"},
|
|
{"name": "Kind/Feature", "color": "1d76db"},
|
|
{"name": "Priority - High", "color": "d73a4a"},
|
|
]
|
|
|
|
schema = {
|
|
"pattern": "prefix_slash",
|
|
"categories": {
|
|
"type": ["Kind/Bug", "Kind/Feature"],
|
|
"priority": ["Priority - High"],
|
|
},
|
|
}
|
|
|
|
mapping = agent._build_label_mapping(existing_labels, schema)
|
|
|
|
assert "type" in mapping
|
|
assert "bug" in mapping["type"]
|
|
assert mapping["type"]["bug"] == "Kind/Bug"
|
|
assert "feature" in mapping["type"]
|
|
assert mapping["type"]["feature"] == "Kind/Feature"
|
|
assert "priority" in mapping
|
|
assert "high" in mapping["priority"]
|
|
assert mapping["priority"]["high"] == "Priority - High"
|
|
|
|
def test_suggest_label_name_prefix_slash(self):
|
|
"""Test label name suggestion for prefix_slash pattern."""
|
|
from agents.issue_agent import IssueAgent
|
|
|
|
agent = IssueAgent(
|
|
gitea_client=None,
|
|
llm_client=None,
|
|
config={
|
|
"labels": {
|
|
"type": {"bug": {"name": "type: bug", "color": "d73a4a"}},
|
|
"priority": {},
|
|
"status": {
|
|
"ai_approved": {"name": "ai-approved", "color": "28a745"}
|
|
},
|
|
}
|
|
},
|
|
)
|
|
|
|
# Test type category
|
|
suggested = agent._suggest_label_name("type", "bug", "prefix_slash")
|
|
assert suggested == "Kind/Bug"
|
|
|
|
# Test status category
|
|
suggested = agent._suggest_label_name("status", "ai_approved", "prefix_slash")
|
|
assert suggested == "Status/Ai Approved"
|
|
|
|
def test_suggest_label_name_prefix_dash(self):
|
|
"""Test label name suggestion for prefix_dash pattern."""
|
|
from agents.issue_agent import IssueAgent
|
|
|
|
agent = IssueAgent(
|
|
gitea_client=None,
|
|
llm_client=None,
|
|
config={
|
|
"labels": {
|
|
"type": {},
|
|
"priority": {"high": {"name": "priority: high", "color": "d73a4a"}},
|
|
"status": {},
|
|
}
|
|
},
|
|
)
|
|
|
|
suggested = agent._suggest_label_name("priority", "high", "prefix_dash")
|
|
assert suggested == "Priority - High"
|
|
|
|
def test_get_label_config_backwards_compatibility(self):
|
|
"""Test that old string format still works."""
|
|
from agents.issue_agent import IssueAgent
|
|
|
|
# Old config format (strings)
|
|
agent = IssueAgent(
|
|
gitea_client=None,
|
|
llm_client=None,
|
|
config={
|
|
"labels": {
|
|
"type": {
|
|
"bug": "type: bug" # Old format
|
|
},
|
|
"priority": {},
|
|
"status": {},
|
|
}
|
|
},
|
|
)
|
|
|
|
config = agent._get_label_config("type", "bug")
|
|
|
|
assert config["name"] == "type: bug"
|
|
assert config["color"] == "1d76db" # Default color
|
|
assert config["aliases"] == []
|
|
|
|
def test_get_label_config_new_format(self):
|
|
"""Test that new dict format works."""
|
|
from agents.issue_agent import IssueAgent
|
|
|
|
agent = IssueAgent(
|
|
gitea_client=None,
|
|
llm_client=None,
|
|
config={
|
|
"labels": {
|
|
"type": {
|
|
"bug": {
|
|
"name": "type: bug",
|
|
"color": "d73a4a",
|
|
"description": "Something isn't working",
|
|
"aliases": ["Kind/Bug", "bug"],
|
|
}
|
|
},
|
|
"priority": {},
|
|
"status": {},
|
|
}
|
|
},
|
|
)
|
|
|
|
config = agent._get_label_config("type", "bug")
|
|
|
|
assert config["name"] == "type: bug"
|
|
assert config["color"] == "d73a4a"
|
|
assert config["description"] == "Something isn't working"
|
|
assert "Kind/Bug" in config["aliases"]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v"])
|