"""Measure retrieval separately from answer checks. Results are not an accuracy claim."""
import json
from pathlib import Path
import re
import time
import pdf_rag as rag

CASES = [
    {'question': 'How many paid annual leave days do full-time employees get?', 'pages': [1], 'facts': ['25'], 'abstain': False},
    {'question': 'What is the hotel reimbursement limit per night?', 'pages': [2], 'facts': ['160'], 'abstain': False},
    {'question': 'What is the annual equipment allowance?', 'pages': [3], 'facts': ['300'], 'abstain': False},
    {'question': 'How many hours do I have to report a lost laptop?', 'pages': [4], 'facts': ['1'], 'abstain': False},
    {'question': "What was Northstar Studio's revenue in 2025?", 'pages': [], 'facts': [], 'abstain': True},
]


def retrieval_metrics(passages, expected, k):
    expected = set(expected)
    if not expected:
        return {'hit': None, 'recall': None, 'reciprocal_rank': None}
    # This single-document example uses physical page numbers as relevance labels.
    returned = [p['page'] for p in passages[:k]]
    overlap = expected.intersection(returned)
    first = next((i + 1 for i, page in enumerate(returned) if page in expected), None)
    return {'hit': int(bool(overlap)), 'recall': len(overlap) / len(expected), 'reciprocal_rank': 1 / first if first else 0}


def fact_check(answer, facts):
    # A narrow regression signal. Presence alone does not establish claim support.
    return all(re.search(r'(?<!\w)' + re.escape(fact) + r'(?!\w)', answer) for fact in facts)


def main():
    root = Path(__file__).resolve().parent
    index_path = root / 'eval-index.json'
    rag.build_index(root / 'docs', index_path)
    index = rag.load_index(index_path)
    rows = []
    for case in CASES:
        started = time.perf_counter()
        passages = rag.retrieve(index, case['question'], k=4)
        row = {**case, 'retrieval': {str(k): retrieval_metrics(passages, case['pages'], k) for k in [1, 2, 4]}}
        try:
            result = rag.answer_question(index, case['question'])
            is_abstention = result['answer'] == rag.NO_ANSWER and not result['citations']
            row.update(result)
            row['expected_behavior'] = is_abstention if case['abstain'] else bool(result['citations']) and fact_check(result['answer'], case['facts'])
            row['quote_check'] = 'passed' if result['citations'] else 'no citations'
        except (ValueError, KeyError, OSError) as error:
            row.update(error=str(error), expected_behavior=False, quote_check='error')
        row['seconds'] = round(time.perf_counter() - started, 3)
        row['human_claim_review'] = 'pending'
        rows.append(row)
        print(json.dumps(row), flush=True)
    (root / 'rag-evaluation-results.json').write_text(json.dumps(rows, indent=2))


if __name__ == '__main__':
    main()