Membangun Test Harness untuk LLM Pipeline

import json
from dataclasses import dataclass
from typing import List

@dataclass
class EvalCase:
    input: str
    expected_output: str
    category: str  # "normal" | "edge" | "adversarial"
    checks: list   # list fungsi evaluasi yang harus pass

class LLMHarness:
    def __init__(self, llm, eval_cases: List[EvalCase]):
        self.llm = llm
        self.cases = eval_cases
        self.results = []

    def run(self, n_runs: int = 3) -> dict:
        for case in self.cases:
            case_results = []
            for _ in range(n_runs):  # run beberapa kali untuk ukur variance
                output = self.llm.invoke(case.input)
                case_results.append({
                    "output": output,
                    "check_results": {
                        check.__name__: check(output, case.expected_output)
                        for check in case.checks
                    }
                })
            self.results.append({
                "case": case,
                "runs": case_results,
                "pass_rate": self._pass_rate(case_results)
            })
        return self._summary()

    def _pass_rate(self, runs: list) -> float:
        passed = sum(1 for r in runs if all(r["check_results"].values()))
        return passed / len(runs)

# Contoh check functions
def is_valid_json(output: str, _) -> bool:
    try: json.loads(output); return True
    except: return False

def has_required_fields(output: str, _) -> bool:
    try:
        d = json.loads(output)
        return all(k in d for k in ["category", "priority", "response"])
    except: return False

def correct_priority(output: str, expected: str) -> bool:
    try:
        out = json.loads(output)
        exp = json.loads(expected)
        return out.get("priority") == exp.get("priority")
    except: return False

def compare_prompts(prompt_a: str, prompt_b: str, eval_cases, llm, n_runs=3):
    results_a = LLMHarness(build_chain(llm, prompt_a), eval_cases).run(n_runs)
    results_b = LLMHarness(build_chain(llm, prompt_b), eval_cases).run(n_runs)

    print(f"Prompt A: {results_a['overall_score']:.1%}")
    print(f"Prompt B: {results_b['overall_score']:.1%}")

    # Highlight categories where B is better or worse
    for cat in ["normal", "edge", "adversarial"]:
        delta = results_b["by_category"][cat] - results_a["by_category"][cat]
        icon = "↑" if delta > 0 else ("↓" if delta < 0 else "=")
        print(f"  {cat}: {icon} {delta:+.1%}")

    # PENTING: Jangan deploy B jika ada category yang turun signifikan
    # meski overall score lebih tinggi
    if any(results_b["by_category"][cat] < results_a["by_category"][cat] - 0.05
          for cat in ["adversarial"]):
        print("⚠ WARNING: Prompt B lebih buruk untuk adversarial cases")

name: LLM Prompt Regression Test

on:
  pull_request:
    paths:
      - 'src/prompts/**'  # hanya run saat file prompt berubah

jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
      - run: pip install -r requirements.txt

      - name: Run LLM Harness
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: python -m pytest tests/test_llm_harness.py -v

      - name: Check Acceptance Threshold
        run: |
          python scripts/check_scores.py \
            --min-overall 0.82 \
            --min-adversarial 0.75