"""VerIQ — scoring: raw points -> IQ score + percentile.
v1 model: fixed weights per difficulty tier, z-score normalization against
the calibration population stored in calibration.json. Until enough real
completions accumulate, we fall back to a synthetic prior (see PRIOR below)
and the result page must show the "preliminary calibration" label.
"""
import json
import math
import os
from pathlib import Path
BANK_PATH = Path(os.environ.get(
"VERIQ_BANK_PATH",
Path(__file__).resolve().parent.parent / "content" / "questions.json"))
CALIB_PATH = Path(os.environ.get(
"VERIQ_CALIB_PATH",
Path(__file__).resolve().parent.parent / "content" / "calibration.json"))
# Synthetic prior until real calibration data exists:
PRIOR_MEAN_RAW = 0.55 # expected fraction of max raw score
PRIOR_SIGMA_RAW = 0.18 # spread of raw fraction
MIN_CALIBRATION_N = 500
_bank_cache: dict | None = None
def load_bank() -> dict:
global _bank_cache
if _bank_cache is None:
_bank_cache = json.loads(BANK_PATH.read_text(encoding="utf-8"))
return _bank_cache
def bank_version() -> str:
return load_bank()["version"]
def task_by_id(task_id: str) -> dict | None:
for t in load_bank()["tasks"]:
if t["id"] == task_id:
return t
return None
def max_raw_score(task_ids: list[str]) -> int:
total = 0
for tid in task_ids:
t = task_by_id(tid)
if t:
total += t["weight"]
return total
def raw_score(answers: dict[str, int], task_ids: list[str]) -> int:
"""answers: {task_id: chosen_index}. Missing = wrong."""
total = 0
for tid in task_ids:
t = task_by_id(tid)
if not t:
continue
chosen = answers.get(tid)
if chosen is not None and chosen == t["answer_index"]:
total += t["weight"]
return total
def _calibration() -> dict:
"""Real calibration stats if enough data, else synthetic prior."""
try:
cal = json.loads(CALIB_PATH.read_text(encoding="utf-8"))
if cal.get("n", 0) >= MIN_CALIBRATION_N:
return cal
except Exception:
pass
return None
def score_test(answers: dict[str, int], task_ids: list[str],
total_time_s: float, flagged: bool) -> dict:
"""Compute raw score, IQ, percentile, category breakdown."""
bank = load_bank()
max_raw = max_raw_score(task_ids)
raw = raw_score(answers, task_ids)
cal = _calibration()
if cal:
mean, sigma = cal["mean_raw"], cal["sigma_raw"]
calibrated = True
else:
mean = PRIOR_MEAN_RAW * max_raw
sigma = PRIOR_SIGMA_RAW * max_raw
calibrated = False
z = (raw - mean) / sigma if sigma > 0 else 0.0
iq = round(100 + 15 * max(-4.0, min(4.0, z)))
# percentile from normal CDF
percentile = round(0.5 * (1 + math.erf(z / math.sqrt(2))) * 100, 1)
# category breakdown
cats: dict[str, dict] = {}
for tid in task_ids:
t = task_by_id(tid)
if not t:
continue
c = cats.setdefault(t["category"], {"correct": 0, "total": 0})
c["total"] += 1
chosen = answers.get(tid)
if chosen is not None and chosen == t["answer_index"]:
c["correct"] += 1
return {
"raw": raw,
"max_raw": max_raw,
"iq": iq,
"percentile": percentile,
"calibrated": calibrated,
"flagged": flagged,
"total_time_s": round(total_time_s, 1),
"categories": cats,
}
def sanity_check(answers: dict[str, int], task_ids: list[str],
total_time_s: float) -> bool:
"""Flag implausible completions: too fast or suspiciously perfect."""
n = len(task_ids)
if n == 0:
return True
per_task = total_time_s / n
raw = raw_score(answers, task_ids)
max_raw = max_raw_score(task_ids)
# Under 3 s per task with near-perfect score -> bot/cheat pattern
if per_task < 3.0 and raw >= 0.9 * max_raw:
return True
# Under 1.5 s per task at all -> auto-flag regardless
if per_task < 1.5:
return True
return False