"""Test Suite for AI Code Review Workflow Tests for verifying prompt formatting, agent logic, and core functionality. Run with: pytest tests/ -v """ import os import sys # Add the tools directory to path sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "tools", "ai-review")) import pytest class TestPromptFormatting: """Test that all prompts can be formatted without errors.""" def get_prompt_path(self, name: str) -> str: """Get the full path to a prompt file.""" return os.path.join( os.path.dirname(__file__), "..", "tools", "ai-review", "prompts", f"{name}.md", ) def load_prompt(self, name: str) -> str: """Load a prompt file.""" path = self.get_prompt_path(name) with open(path) as f: return f.read() def test_issue_triage_prompt_formatting(self): """Test that issue_triage.md can be formatted with placeholders.""" prompt = self.load_prompt("issue_triage") # This should NOT raise a KeyError formatted = prompt.format( title="Test Issue Title", body="This is the issue body content", author="testuser", existing_labels="bug, urgent", ) assert "Test Issue Title" in formatted assert "This is the issue body content" in formatted assert "testuser" in formatted assert "bug, urgent" in formatted # JSON example should still be present (curly braces escaped) assert '"type"' in formatted assert '"priority"' in formatted def test_issue_response_prompt_formatting(self): """Test that issue_response.md can be formatted with placeholders.""" prompt = self.load_prompt("issue_response") formatted = prompt.format( issue_type="bug", priority="high", title="Bug Report", body="Description of the bug", triage_analysis="This is a high priority bug", ) assert "bug" in formatted assert "high" in formatted assert "Bug Report" in formatted # JSON example should still be present assert '"comment"' in formatted def test_base_prompt_no_placeholders(self): """Test that base.md loads correctly (no placeholders needed).""" prompt = self.load_prompt("base") # Should contain key elements assert "security" in prompt.lower() assert "JSON" in prompt assert "severity" in prompt.lower() def test_prompts_have_escaped_json(self): """Verify JSON examples use double curly braces.""" for prompt_name in ["issue_triage", "issue_response"]: prompt = self.load_prompt(prompt_name) # Check that format() doesn't fail try: # Try with minimal placeholders if prompt_name == "issue_triage": prompt.format(title="t", body="b", author="a", existing_labels="l") elif prompt_name == "issue_response": prompt.format( issue_type="t", priority="p", title="t", body="b", triage_analysis="a", ) except KeyError as e: pytest.fail(f"Prompt {prompt_name} has unescaped curly braces: {e}") class TestImports: """Test that all modules can be imported correctly.""" def test_import_agents(self): """Test importing agent classes.""" from agents.base_agent import AgentContext, AgentResult, BaseAgent from agents.codebase_agent import CodebaseAgent from agents.issue_agent import IssueAgent from agents.pr_agent import PRAgent assert BaseAgent is not None assert IssueAgent is not None assert PRAgent is not None assert CodebaseAgent is not None def test_import_clients(self): """Test importing client classes.""" from clients.gitea_client import GiteaClient from clients.llm_client import LLMClient assert GiteaClient is not None assert LLMClient is not None def test_import_security(self): """Test importing security scanner.""" from security.security_scanner import SecurityScanner assert SecurityScanner is not None def test_import_enterprise(self): """Test importing enterprise features.""" from enterprise.audit_logger import AuditLogger from enterprise.metrics import MetricsCollector assert AuditLogger is not None assert MetricsCollector is not None def test_import_dispatcher(self): """Test importing dispatcher.""" from dispatcher import Dispatcher assert Dispatcher is not None class TestSecurityScanner: """Test security scanner pattern detection.""" def test_detects_hardcoded_secret(self): """Test detection of hardcoded secrets.""" from security.security_scanner import SecurityScanner scanner = SecurityScanner() code = """ API_KEY = "sk-1234567890abcdef" """ findings = list(scanner.scan_content(code, "test.py")) assert len(findings) >= 1 assert any(f.severity == "HIGH" for f in findings) def test_detects_eval(self): """Test detection of eval usage.""" from security.security_scanner import SecurityScanner scanner = SecurityScanner() code = """ result = eval(user_input) """ findings = list(scanner.scan_content(code, "test.py")) assert len(findings) >= 1 assert any("eval" in f.rule_name.lower() for f in findings) def test_no_false_positives_on_clean_code(self): """Test that clean code doesn't trigger false positives.""" from security.security_scanner import SecurityScanner scanner = SecurityScanner() code = """ def hello(): print("Hello, world!") return 42 """ findings = list(scanner.scan_content(code, "test.py")) # Should have no HIGH severity issues for clean code high_findings = [f for f in findings if f.severity == "HIGH"] assert len(high_findings) == 0 class TestAgentContext: """Test agent context and result dataclasses.""" def test_agent_context_creation(self): """Test creating AgentContext.""" from agents.base_agent import AgentContext context = AgentContext( owner="testowner", repo="testrepo", event_type="issues", event_data={"action": "opened"}, config={}, ) assert context.owner == "testowner" assert context.repo == "testrepo" assert context.event_type == "issues" def test_agent_result_creation(self): """Test creating AgentResult.""" from agents.base_agent import AgentResult result = AgentResult( success=True, message="Test passed", data={"key": "value"}, actions_taken=["action1", "action2"], ) assert result.success is True assert result.message == "Test passed" assert len(result.actions_taken) == 2 class TestMetrics: """Test metrics collection.""" def test_counter_increment(self): """Test counter metrics.""" from enterprise.metrics import Counter counter = Counter("test_counter") assert counter.value == 0 counter.inc() assert counter.value == 1 counter.inc(5) assert counter.value == 6 def test_histogram_observation(self): """Test histogram metrics.""" from enterprise.metrics import Histogram hist = Histogram("test_histogram") hist.observe(0.1) hist.observe(0.5) hist.observe(1.0) assert hist.count == 3 assert hist.sum == 1.6 def test_metrics_collector_summary(self): """Test metrics collector summary.""" from enterprise.metrics import MetricsCollector collector = MetricsCollector() collector.record_request_start("TestAgent") collector.record_request_end("TestAgent", success=True, duration_seconds=0.5) summary = collector.get_summary() assert summary["requests"]["total"] == 1 assert summary["requests"]["success"] == 1 class TestHelpCommand: """Test help command functionality.""" def test_help_command_returns_text(self): """Test that help command returns formatted help text.""" from agents.issue_agent import IssueAgent agent = IssueAgent( gitea_client=None, llm_client=None, config={ "interaction": { "mention_prefix": "@codebot", "commands": ["help", "triage", "explain"], } }, ) help_text = agent._command_help() assert help_text is not None assert len(help_text) > 100 assert "@codebot" in help_text assert "help" in help_text.lower() assert "triage" in help_text.lower() def test_help_includes_all_sections(self): """Test that help text includes all major sections.""" from agents.issue_agent import IssueAgent agent = IssueAgent( gitea_client=None, llm_client=None, config={"interaction": {"mention_prefix": "@codebot"}}, ) help_text = agent._command_help() # Check for main sections assert "Issue Triage" in help_text assert "Interactive Chat" in help_text assert "Setup & Utility" in help_text assert "Pull Request" in help_text assert "Quick Examples" in help_text def test_help_uses_custom_bot_name(self): """Test that help command uses custom bot name from config.""" from agents.issue_agent import IssueAgent agent = IssueAgent( gitea_client=None, llm_client=None, config={"interaction": {"mention_prefix": "@mybot"}}, ) help_text = agent._command_help() assert "@mybot" in help_text assert "@codebot" not in help_text def test_help_includes_examples(self): """Test that help text includes usage examples.""" from agents.issue_agent import IssueAgent agent = IssueAgent( gitea_client=None, llm_client=None, config={"interaction": {"mention_prefix": "@codebot"}}, ) help_text = agent._command_help() # Check for example commands assert "triage" in help_text assert "explain" in help_text assert "setup-labels" in help_text class TestLabelSetup: """Test label setup and schema detection.""" def test_detect_prefix_slash_schema(self): """Test detection of Kind/Bug style labels.""" from agents.issue_agent import IssueAgent # Create mock agent with config agent = IssueAgent( gitea_client=None, llm_client=None, config={ "label_patterns": { "prefix_slash": r"^(Kind|Type|Category)/(.+)$", "prefix_dash": r"^(Priority|Status|Reviewed) - (.+)$", "colon": r"^(type|priority|status): (.+)$", } }, ) labels = [ {"name": "Kind/Bug", "color": "d73a4a"}, {"name": "Kind/Feature", "color": "1d76db"}, {"name": "Kind/Documentation", "color": "0075ca"}, {"name": "Priority - High", "color": "d73a4a"}, {"name": "Priority - Low", "color": "28a745"}, ] schema = agent._detect_label_schema(labels) assert schema is not None assert schema["pattern"] == "prefix_slash" assert "type" in schema["categories"] assert "priority" in schema["categories"] assert len(schema["categories"]["type"]) == 3 def test_detect_prefix_dash_schema(self): """Test detection of Priority - High style labels.""" from agents.issue_agent import IssueAgent agent = IssueAgent( gitea_client=None, llm_client=None, config={ "label_patterns": { "prefix_slash": r"^(Kind|Type|Category)/(.+)$", "prefix_dash": r"^(Priority|Status|Reviewed) - (.+)$", "colon": r"^(type|priority|status): (.+)$", } }, ) labels = [ {"name": "Priority - Critical", "color": "b60205"}, {"name": "Priority - High", "color": "d73a4a"}, {"name": "Status - Blocked", "color": "fef2c0"}, ] schema = agent._detect_label_schema(labels) assert schema is not None assert schema["pattern"] == "prefix_dash" assert "priority" in schema["categories"] assert "status" in schema["categories"] def test_detect_colon_schema(self): """Test detection of type: bug style labels.""" from agents.issue_agent import IssueAgent agent = IssueAgent( gitea_client=None, llm_client=None, config={ "label_patterns": { "prefix_slash": r"^(Kind|Type|Category)/(.+)$", "prefix_dash": r"^(Priority|Status|Reviewed) - (.+)$", "colon": r"^(type|priority|status): (.+)$", } }, ) labels = [ {"name": "type: bug", "color": "d73a4a"}, {"name": "type: feature", "color": "1d76db"}, {"name": "priority: high", "color": "d73a4a"}, ] schema = agent._detect_label_schema(labels) assert schema is not None assert schema["pattern"] == "colon" assert "type" in schema["categories"] assert "priority" in schema["categories"] def test_build_label_mapping(self): """Test building label mapping from existing labels.""" from agents.issue_agent import IssueAgent agent = IssueAgent( gitea_client=None, llm_client=None, config={ "labels": { "type": { "bug": {"name": "type: bug", "aliases": ["Kind/Bug", "bug"]}, "feature": { "name": "type: feature", "aliases": ["Kind/Feature", "feature"], }, }, "priority": { "high": { "name": "priority: high", "aliases": ["Priority - High", "P1"], } }, "status": {}, } }, ) existing_labels = [ {"name": "Kind/Bug", "color": "d73a4a"}, {"name": "Kind/Feature", "color": "1d76db"}, {"name": "Priority - High", "color": "d73a4a"}, ] schema = { "pattern": "prefix_slash", "categories": { "type": ["Kind/Bug", "Kind/Feature"], "priority": ["Priority - High"], }, } mapping = agent._build_label_mapping(existing_labels, schema) assert "type" in mapping assert "bug" in mapping["type"] assert mapping["type"]["bug"] == "Kind/Bug" assert "feature" in mapping["type"] assert mapping["type"]["feature"] == "Kind/Feature" assert "priority" in mapping assert "high" in mapping["priority"] assert mapping["priority"]["high"] == "Priority - High" def test_suggest_label_name_prefix_slash(self): """Test label name suggestion for prefix_slash pattern.""" from agents.issue_agent import IssueAgent agent = IssueAgent( gitea_client=None, llm_client=None, config={ "labels": { "type": {"bug": {"name": "type: bug", "color": "d73a4a"}}, "priority": {}, "status": { "ai_approved": {"name": "ai-approved", "color": "28a745"} }, } }, ) # Test type category suggested = agent._suggest_label_name("type", "bug", "prefix_slash") assert suggested == "Kind/Bug" # Test status category suggested = agent._suggest_label_name("status", "ai_approved", "prefix_slash") assert suggested == "Status/Ai Approved" def test_suggest_label_name_prefix_dash(self): """Test label name suggestion for prefix_dash pattern.""" from agents.issue_agent import IssueAgent agent = IssueAgent( gitea_client=None, llm_client=None, config={ "labels": { "type": {}, "priority": {"high": {"name": "priority: high", "color": "d73a4a"}}, "status": {}, } }, ) suggested = agent._suggest_label_name("priority", "high", "prefix_dash") assert suggested == "Priority - High" def test_get_label_config_backwards_compatibility(self): """Test that old string format still works.""" from agents.issue_agent import IssueAgent # Old config format (strings) agent = IssueAgent( gitea_client=None, llm_client=None, config={ "labels": { "type": { "bug": "type: bug" # Old format }, "priority": {}, "status": {}, } }, ) config = agent._get_label_config("type", "bug") assert config["name"] == "type: bug" assert config["color"] == "1d76db" # Default color assert config["aliases"] == [] def test_get_label_config_new_format(self): """Test that new dict format works.""" from agents.issue_agent import IssueAgent agent = IssueAgent( gitea_client=None, llm_client=None, config={ "labels": { "type": { "bug": { "name": "type: bug", "color": "d73a4a", "description": "Something isn't working", "aliases": ["Kind/Bug", "bug"], } }, "priority": {}, "status": {}, } }, ) config = agent._get_label_config("type", "bug") assert config["name"] == "type: bug" assert config["color"] == "d73a4a" assert config["description"] == "Something isn't working" assert "Kind/Bug" in config["aliases"] class TestPRSummaryGeneration: """Test PR summary generation functionality.""" def test_pr_summary_prompt_exists(self): """Verify pr_summary.md prompt file exists.""" prompt_path = os.path.join( os.path.dirname(__file__), "..", "tools", "ai-review", "prompts", "pr_summary.md", ) assert os.path.exists(prompt_path), "pr_summary.md prompt file not found" def test_pr_summary_prompt_formatting(self): """Test that pr_summary.md can be loaded without errors.""" prompt_path = os.path.join( os.path.dirname(__file__), "..", "tools", "ai-review", "prompts", "pr_summary.md", ) with open(prompt_path) as f: prompt = f.read() # Check for key elements assert "summary" in prompt.lower() assert "change_type" in prompt.lower() assert "files_affected" in prompt.lower() assert "impact" in prompt.lower() assert "JSON" in prompt # Verify JSON examples use double curly braces (escaped) # Should not raise KeyError when formatted with empty string try: formatted = prompt.format() except KeyError as e: pytest.fail(f"Prompt has unescaped placeholders: {e}") def test_pr_agent_has_summary_marker(self): """Verify PRAgent has PR_SUMMARY_MARKER constant.""" from agents.pr_agent import PRAgent assert hasattr(PRAgent, "PR_SUMMARY_MARKER") assert PRAgent.PR_SUMMARY_MARKER == "" def test_pr_agent_can_handle_summarize_command(self): """Test that PRAgent can handle @codebot summarize in PR comments.""" from agents.pr_agent import PRAgent config = { "agents": { "pr": { "enabled": True, } }, "interaction": { "mention_prefix": "@codebot", }, } agent = PRAgent(config=config) # Test summarize command in PR comment event_data = { "action": "created", "issue": { "number": 123, "pull_request": {}, # Indicates this is a PR }, "comment": { "body": "@codebot summarize this PR please", }, } assert agent.can_handle("issue_comment", event_data) is True def test_pr_agent_can_handle_summarize_case_insensitive(self): """Test that summarize command is case-insensitive.""" from agents.pr_agent import PRAgent config = { "agents": { "pr": { "enabled": True, } }, "interaction": { "mention_prefix": "@codebot", }, } agent = PRAgent(config=config) # Test various casings for body in [ "@codebot SUMMARIZE", "@codebot Summarize", "@codebot SuMmArIzE", ]: event_data = { "action": "created", "issue": { "number": 123, "pull_request": {}, }, "comment": {"body": body}, } assert agent.can_handle("issue_comment", event_data) is True def test_pr_agent_ignores_summarize_on_non_pr(self): """Test that summarize command is ignored on regular issues.""" from agents.pr_agent import PRAgent config = { "agents": { "pr": { "enabled": True, } }, "interaction": { "mention_prefix": "@codebot", }, } agent = PRAgent(config=config) # Regular issue (no pull_request field) event_data = { "action": "created", "issue": { "number": 123, # No pull_request field }, "comment": { "body": "@codebot summarize", }, } assert agent.can_handle("issue_comment", event_data) is False def test_format_pr_summary_structure(self): """Test _format_pr_summary generates correct markdown structure.""" from agents.pr_agent import PRAgent agent = PRAgent(config={}) summary_data = { "summary": "This PR adds a new feature", "change_type": "Feature", "key_changes": { "added": ["New authentication module", "User login endpoint"], "modified": ["Updated config file"], "removed": ["Deprecated legacy auth"], }, "files_affected": [ { "path": "src/auth.py", "description": "New authentication module", "change_type": "added", }, { "path": "config.yml", "description": "Added auth settings", "change_type": "modified", }, ], "impact": { "scope": "medium", "description": "Adds authentication without breaking existing features", }, } result = agent._format_pr_summary(summary_data) # Verify structure assert "## ๐Ÿ“‹ Pull Request Summary" in result assert "This PR adds a new feature" in result assert "**Type:** โœจ Feature" in result assert "## Changes" in result assert "**โœ… Added:**" in result assert "**๐Ÿ“ Modified:**" in result assert "**โŒ Removed:**" in result assert "## Files Affected" in result assert "โž• `src/auth.py`" in result assert "๐Ÿ“ `config.yml`" in result assert "## Impact" in result assert "๐ŸŸก **Scope:** Medium" in result def test_format_pr_summary_change_types(self): """Test that all change types have correct emojis.""" from agents.pr_agent import PRAgent agent = PRAgent(config={}) change_types = { "Feature": "โœจ", "Bugfix": "๐Ÿ›", "Refactor": "โ™ป๏ธ", "Documentation": "๐Ÿ“š", "Testing": "๐Ÿงช", "Mixed": "๐Ÿ”€", } for change_type, expected_emoji in change_types.items(): summary_data = { "summary": "Test", "change_type": change_type, "key_changes": {}, "files_affected": [], "impact": {}, } result = agent._format_pr_summary(summary_data) assert f"**Type:** {expected_emoji} {change_type}" in result def test_config_has_auto_summary_settings(self): """Verify config.yml has auto_summary configuration.""" config_path = os.path.join( os.path.dirname(__file__), "..", "tools", "ai-review", "config.yml" ) import yaml with open(config_path) as f: config = yaml.safe_load(f) assert "agents" in config assert "pr" in config["agents"] assert "auto_summary" in config["agents"]["pr"] assert "enabled" in config["agents"]["pr"]["auto_summary"] assert "post_as_comment" in config["agents"]["pr"]["auto_summary"] if __name__ == "__main__": pytest.main([__file__, "-v"])