Newer
Older
veriq / backend / scoring.py
"""VerIQ — scoring: raw points -> IQ score + percentile.

v1 model: fixed weights per difficulty tier, z-score normalization against
the calibration population stored in calibration.json. Until enough real
completions accumulate, we fall back to a synthetic prior (see PRIOR below)
and the result page must show the "preliminary calibration" label.
"""
import json
import math
import os
from pathlib import Path

BANK_PATH = Path(os.environ.get(
    "VERIQ_BANK_PATH",
    Path(__file__).resolve().parent.parent / "content" / "questions.json"))
CALIB_PATH = Path(os.environ.get(
    "VERIQ_CALIB_PATH",
    Path(__file__).resolve().parent.parent / "content" / "calibration.json"))

# Synthetic prior until real calibration data exists:
PRIOR_MEAN_RAW = 0.55   # expected fraction of max raw score
PRIOR_SIGMA_RAW = 0.18  # spread of raw fraction
MIN_CALIBRATION_N = 500

_bank_cache: dict | None = None


def load_bank() -> dict:
    global _bank_cache
    if _bank_cache is None:
        _bank_cache = json.loads(BANK_PATH.read_text(encoding="utf-8"))
    return _bank_cache


def bank_version() -> str:
    return load_bank()["version"]


def task_by_id(task_id: str) -> dict | None:
    for t in load_bank()["tasks"]:
        if t["id"] == task_id:
            return t
    return None


def max_raw_score(task_ids: list[str]) -> int:
    total = 0
    for tid in task_ids:
        t = task_by_id(tid)
        if t:
            total += t["weight"]
    return total


def raw_score(answers: dict[str, int], task_ids: list[str]) -> int:
    """answers: {task_id: chosen_index}. Missing = wrong."""
    total = 0
    for tid in task_ids:
        t = task_by_id(tid)
        if not t:
            continue
        chosen = answers.get(tid)
        if chosen is not None and chosen == t["answer_index"]:
            total += t["weight"]
    return total


def _calibration() -> dict:
    """Real calibration stats if enough data, else synthetic prior."""
    try:
        cal = json.loads(CALIB_PATH.read_text(encoding="utf-8"))
        if cal.get("n", 0) >= MIN_CALIBRATION_N:
            return cal
    except Exception:
        pass
    return None


def score_test(answers: dict[str, int], task_ids: list[str],
               total_time_s: float, flagged: bool) -> dict:
    """Compute raw score, IQ, percentile, category breakdown."""
    bank = load_bank()
    max_raw = max_raw_score(task_ids)
    raw = raw_score(answers, task_ids)

    cal = _calibration()
    if cal:
        mean, sigma = cal["mean_raw"], cal["sigma_raw"]
        calibrated = True
    else:
        mean = PRIOR_MEAN_RAW * max_raw
        sigma = PRIOR_SIGMA_RAW * max_raw
        calibrated = False

    z = (raw - mean) / sigma if sigma > 0 else 0.0
    iq = round(100 + 15 * max(-4.0, min(4.0, z)))
    # percentile from normal CDF
    percentile = round(0.5 * (1 + math.erf(z / math.sqrt(2))) * 100, 1)

    # category breakdown
    cats: dict[str, dict] = {}
    for tid in task_ids:
        t = task_by_id(tid)
        if not t:
            continue
        c = cats.setdefault(t["category"], {"correct": 0, "total": 0})
        c["total"] += 1
        chosen = answers.get(tid)
        if chosen is not None and chosen == t["answer_index"]:
            c["correct"] += 1

    return {
        "raw": raw,
        "max_raw": max_raw,
        "iq": iq,
        "percentile": percentile,
        "calibrated": calibrated,
        "flagged": flagged,
        "total_time_s": round(total_time_s, 1),
        "categories": cats,
    }


def sanity_check(answers: dict[str, int], task_ids: list[str],
                 total_time_s: float) -> bool:
    """Flag implausible completions: too fast or suspiciously perfect."""
    n = len(task_ids)
    if n == 0:
        return True
    per_task = total_time_s / n
    raw = raw_score(answers, task_ids)
    max_raw = max_raw_score(task_ids)
    # Under 3 s per task with near-perfect score -> bot/cheat pattern
    if per_task < 3.0 and raw >= 0.9 * max_raw:
        return True
    # Under 1.5 s per task at all -> auto-flag regardless
    if per_task < 1.5:
        return True
    return False