"""Audit a research report JSON payload and publish inspectable claim evidence.""" import argparse, html, json from pathlib import Path from metrics import score_report def audit_payload(payload): if payload.get("schema_version") != 1: raise ValueError("report schema_version1 required") snippets = payload.get("snippets", {}) documents = {d["id"]: d for d in payload.get("documents", [])} for key, s in snippets.items(): doc = documents.get(s["doc_id"]) if ( doc is None or not all( isinstance(s[field], int) and not isinstance(s[field], bool) for field in ("start", "end") ) or not 0 <= s["start"] <= s["end"] <= len(doc["text"]) or doc["text"][s["start"] : s["end"]] != s["text"] ): raise ValueError("changed or missing source span: " + key) text = "\n".join( sentence["text"].rstrip(".!?") + " " + "".join("[" + c + "]" for c in sentence["cites"]) + "." for section in payload["sections"] for sentence in section["sentences"] ) return with_evidence( score_report(text, {key: s["text"] for key, s in snippets.items()}), {key: s["text"] for key, s in snippets.items()}, ) def with_evidence(result, evidence): return { "schema_version": 1, **result, "verdicts": [ { **v, "evidence": [ {"id": key, "text": evidence.get(key, "Missing source")} for key in v["cites"] ], } for v in result["verdicts"] ], } def render(result): rows = "".join( '
" + html.escape(e["text"]) + "
Lexical checks identify review candidates; they do not establish truth.
State: " + html.escape(result["state"]) + ". Claims: " + str(result["claims"]) + ". Unavailable labels: " + html.escape(", ".join(result["unavailable"])) + "
" '| Claim | Expand cited evidence | Check |
|---|