File size: 4,492 Bytes
aa3c874 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 |
#!/usr/bin/env python
"""
Test Runner for Roger Intelligence Platform
Runs all test suites with configurable options:
- Unit tests
- Integration tests
- Evaluation tests (LLM-as-Judge)
- Adversarial tests
- End-to-end tests
Usage:
python run_tests.py # Run all tests
python run_tests.py --unit # Run unit tests only
python run_tests.py --eval # Run evaluation tests only
python run_tests.py --adversarial # Run adversarial tests only
python run_tests.py --with-langsmith # Enable LangSmith tracing
"""
import argparse
import subprocess
import sys
import os
from pathlib import Path
from datetime import datetime
PROJECT_ROOT = Path(__file__).parent
TESTS_DIR = PROJECT_ROOT / "tests"
def run_pytest(args: list, verbose: bool = True) -> int:
"""Run pytest with given arguments."""
cmd = ["pytest"] + args
if verbose:
cmd.append("-v")
print(f"\n{'='*60}")
print(f"Running: {' '.join(cmd)}")
print(f"{'='*60}\n")
result = subprocess.run(cmd, cwd=str(PROJECT_ROOT))
return result.returncode
def run_all_tests(with_coverage: bool = False, with_langsmith: bool = False) -> int:
"""Run all test suites."""
args = [str(TESTS_DIR)]
if with_coverage:
args.extend(["--cov=src", "--cov-report=html", "--cov-report=term"])
if with_langsmith:
os.environ["LANGSMITH_TRACING_TESTS"] = "true"
return run_pytest(args)
def run_unit_tests() -> int:
"""Run unit tests only."""
return run_pytest([str(TESTS_DIR / "unit"), "-m", "not slow"])
def run_integration_tests() -> int:
"""Run integration tests."""
return run_pytest([str(TESTS_DIR / "integration"), "-m", "integration"])
def run_evaluation_tests(with_langsmith: bool = True) -> int:
"""Run LLM-as-Judge evaluation tests."""
if with_langsmith:
os.environ["LANGSMITH_TRACING_TESTS"] = "true"
return run_pytest([str(TESTS_DIR / "evaluation"), "-m", "evaluation", "--tb=short"])
def run_adversarial_tests() -> int:
"""Run adversarial/security tests."""
return run_pytest([str(TESTS_DIR / "evaluation" / "adversarial_tests.py"), "-m", "adversarial", "--tb=short"])
def run_e2e_tests() -> int:
"""Run end-to-end tests."""
return run_pytest([str(TESTS_DIR / "e2e"), "-m", "e2e", "--tb=long"])
def run_evaluator_standalone():
"""Run the standalone agent evaluator."""
from tests.evaluation.agent_evaluator import run_evaluation_cli
return run_evaluation_cli()
def main():
parser = argparse.ArgumentParser(description="Roger Intelligence Platform Test Runner")
parser.add_argument("--all", action="store_true", help="Run all tests")
parser.add_argument("--unit", action="store_true", help="Run unit tests only")
parser.add_argument("--integration", action="store_true", help="Run integration tests")
parser.add_argument("--eval", action="store_true", help="Run evaluation tests")
parser.add_argument("--adversarial", action="store_true", help="Run adversarial tests")
parser.add_argument("--e2e", action="store_true", help="Run end-to-end tests")
parser.add_argument("--evaluator", action="store_true", help="Run standalone evaluator")
parser.add_argument("--coverage", action="store_true", help="Generate coverage report")
parser.add_argument("--with-langsmith", action="store_true", help="Enable LangSmith tracing")
args = parser.parse_args()
print("=" * 70)
print("ROGER INTELLIGENCE PLATFORM - TEST RUNNER")
print(f"Started: {datetime.now().isoformat()}")
print("=" * 70)
exit_code = 0
if args.with_langsmith:
os.environ["LANGSMITH_TRACING_TESTS"] = "true"
print("[Config] LangSmith tracing ENABLED for tests")
if args.evaluator:
run_evaluator_standalone()
elif args.unit:
exit_code = run_unit_tests()
elif args.integration:
exit_code = run_integration_tests()
elif args.eval:
exit_code = run_evaluation_tests(args.with_langsmith)
elif args.adversarial:
exit_code = run_adversarial_tests()
elif args.e2e:
exit_code = run_e2e_tests()
else:
# Default: run all tests
exit_code = run_all_tests(args.coverage, args.with_langsmith)
print("\n" + "=" * 70)
print(f"TEST RUN COMPLETE - Exit Code: {exit_code}")
print("=" * 70)
return exit_code
if __name__ == "__main__":
sys.exit(main())
|