diff --git a/scripts/staffing_simulation.py b/scripts/staffing_simulation.py index ed032b5..d63985f 100644 --- a/scripts/staffing_simulation.py +++ b/scripts/staffing_simulation.py @@ -234,18 +234,15 @@ def ask_staffing_question(question, verify=True): answer = r.get("answer", "") sources = r.get("sources", []) - # Verify: extract any worker names mentioned in the answer + # Verify: check each source worker actually exists in golden data hallucinations = [] if verify: - # Check each source worker exists for s in sources: wid = s.get("doc_id", "").replace("W-", "") if wid.isdigit(): - ok, issues = verify_worker(int(wid), { - "name": "", # just check existence - }) - if not ok: - hallucinations.extend(issues) + result = sql(f"SELECT worker_id FROM ethereal_workers WHERE worker_id = {wid}") + if "error" in result or not result.get("rows"): + hallucinations.append(f"source {s.get('doc_id')} not found in golden data") return answer, sources, hallucinations