auditor: kb_query surfaces scrum-master reviews for files in PR diff
Some checks failed
lakehouse/auditor 2 blocking issues: unimplemented!() macro call in tests/real-world/hard_task_escalation.ts
Some checks failed
lakehouse/auditor 2 blocking issues: unimplemented!() macro call in tests/real-world/hard_task_escalation.ts
Wires the cohesion-plan Phase C link: the scrum-master pipeline writes per-file reviews to data/_kb/scrum_reviews.jsonl on accept; the auditor now reads that same file and emits one kb_query finding per scrum review whose `file` matches a path in the PR's diff. Severity heuristic: attempt 1-3 → info, attempt 4+ → warn. Reaching the cloud specialist (attempt 4+) means the ladder had to escalate, which is meaningful signal reviewers should see. Tree-split fired is also surfaced in the finding summary. audit.ts now passes pr.files.map(f => f.path) into runKbCheck (the old signature dropped it on the floor). Also adds auditor/audit_one.ts — a dry-run CLI for auditing a single PR without posting to Gitea, useful for verifying check behavior without spamming review comments. Verified: after writing scrum_reviews for auditor/audit.ts and mcp-server/observer.ts (both in PR #7), audit_one 7 surfaced both as info findings with preview + accepted_model + tree_split flag. A scrum review for playbook_memory.rs (NOT in PR #7) was correctly filtered out.
This commit is contained in:
parent
89d188074b
commit
dc01ba0a3b
@ -52,7 +52,7 @@ export async function auditPr(pr: PrSnapshot, opts: AuditOptions = {}): Promise<
|
|||||||
runStaticCheck(diff),
|
runStaticCheck(diff),
|
||||||
opts.skip_dynamic ? Promise.resolve(stubFinding("dynamic", "skipped by options")) : runDynamicCheck(),
|
opts.skip_dynamic ? Promise.resolve(stubFinding("dynamic", "skipped by options")) : runDynamicCheck(),
|
||||||
opts.skip_inference ? Promise.resolve(stubFinding("inference", "skipped by options")) : runInferenceCheck(claims, diff),
|
opts.skip_inference ? Promise.resolve(stubFinding("inference", "skipped by options")) : runInferenceCheck(claims, diff),
|
||||||
runKbCheck(claims),
|
runKbCheck(claims, pr.files.map(f => f.path)),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const allFindings: Finding[] = [
|
const allFindings: Finding[] = [
|
||||||
|
|||||||
68
auditor/audit_one.ts
Normal file
68
auditor/audit_one.ts
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
// One-shot dry-run audit of a single PR. Useful for verifying check
|
||||||
|
// behavior (kb_query scrum surfacing, inference prompts, etc.) without
|
||||||
|
// posting to Gitea. Does NOT touch state.json and does NOT post
|
||||||
|
// commit status or PR comments.
|
||||||
|
//
|
||||||
|
// Run: bun run auditor/audit_one.ts <pr-number>
|
||||||
|
|
||||||
|
import { getPrSnapshot } from "./gitea.ts";
|
||||||
|
import { auditPr } from "./audit.ts";
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const prNumRaw = process.argv[2];
|
||||||
|
if (!prNumRaw) {
|
||||||
|
console.error("usage: bun run auditor/audit_one.ts <pr-number>");
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
const prNum = Number(prNumRaw);
|
||||||
|
if (!Number.isFinite(prNum)) {
|
||||||
|
console.error(`invalid PR number: ${prNumRaw}`);
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[audit_one] fetching PR #${prNum}...`);
|
||||||
|
const pr = await getPrSnapshot(prNum);
|
||||||
|
console.log(`[audit_one] PR #${pr.number}: "${pr.title}" (head=${pr.head_sha.slice(0, 12)})`);
|
||||||
|
console.log(`[audit_one] files in diff: ${pr.files.length}`);
|
||||||
|
for (const f of pr.files) console.log(` - ${f.path} (+${f.additions}/-${f.deletions})`);
|
||||||
|
console.log("");
|
||||||
|
|
||||||
|
const verdict = await auditPr(pr, {
|
||||||
|
dry_run: true, // no Gitea posting
|
||||||
|
skip_dynamic: true, // don't run fixture
|
||||||
|
skip_inference: process.env.LH_AUDITOR_SKIP_INFERENCE === "1",
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log("\n═══ VERDICT ═══");
|
||||||
|
console.log(`overall: ${verdict.overall}`);
|
||||||
|
console.log(`one-liner: ${verdict.one_liner}`);
|
||||||
|
console.log(`findings: total=${verdict.metrics.findings_total} block=${verdict.metrics.findings_block} warn=${verdict.metrics.findings_warn} info=${verdict.metrics.findings_info}`);
|
||||||
|
console.log("");
|
||||||
|
|
||||||
|
// Print findings, highlighting kb_query scrum surfacing
|
||||||
|
const byCheck: Record<string, typeof verdict.findings> = {};
|
||||||
|
for (const f of verdict.findings) (byCheck[f.check] ||= []).push(f);
|
||||||
|
|
||||||
|
for (const [check, findings] of Object.entries(byCheck)) {
|
||||||
|
console.log(`── ${check} (${findings.length}) ──`);
|
||||||
|
for (const f of findings) {
|
||||||
|
const tag = f.severity === "block" ? "🛑" : f.severity === "warn" ? "⚠️ " : "ℹ️ ";
|
||||||
|
console.log(` ${tag} [${f.severity}] ${f.summary}`);
|
||||||
|
if (f.summary.includes("scrum-master")) {
|
||||||
|
for (const e of f.evidence) {
|
||||||
|
console.log(` → ${e.slice(0, 200)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const scrumFindings = verdict.findings.filter(f => f.summary.includes("scrum-master"));
|
||||||
|
console.log("");
|
||||||
|
console.log(`═══ SCRUM WIRE CHECK: ${scrumFindings.length} scrum-master findings surfaced by kb_query ═══`);
|
||||||
|
if (scrumFindings.length === 0) {
|
||||||
|
console.log(" (none — either no matching scrum_reviews.jsonl rows, or files didn't match PR diff)");
|
||||||
|
}
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch(e => { console.error("[audit_one] fatal:", e); process.exit(1); });
|
||||||
@ -8,6 +8,7 @@
|
|||||||
// What this check reads (all file-backed, append-only or periodic):
|
// What this check reads (all file-backed, append-only or periodic):
|
||||||
// data/_kb/outcomes.jsonl — per-scenario outcomes (kb.ts)
|
// data/_kb/outcomes.jsonl — per-scenario outcomes (kb.ts)
|
||||||
// data/_kb/error_corrections.jsonl — fail→succeed deltas on same sig
|
// data/_kb/error_corrections.jsonl — fail→succeed deltas on same sig
|
||||||
|
// data/_kb/scrum_reviews.jsonl — scrum-master accepted reviews
|
||||||
// data/_observer/ops.jsonl — observer ring → disk stream
|
// data/_observer/ops.jsonl — observer ring → disk stream
|
||||||
// data/_bot/cycles/*.json — bot cycle results
|
// data/_bot/cycles/*.json — bot cycle results
|
||||||
//
|
//
|
||||||
@ -21,10 +22,11 @@ import type { Claim, Finding } from "../types.ts";
|
|||||||
const KB_DIR = "/home/profit/lakehouse/data/_kb";
|
const KB_DIR = "/home/profit/lakehouse/data/_kb";
|
||||||
const OBSERVER_OPS = "/home/profit/lakehouse/data/_observer/ops.jsonl";
|
const OBSERVER_OPS = "/home/profit/lakehouse/data/_observer/ops.jsonl";
|
||||||
const BOT_CYCLES_DIR = "/home/profit/lakehouse/data/_bot/cycles";
|
const BOT_CYCLES_DIR = "/home/profit/lakehouse/data/_bot/cycles";
|
||||||
|
const SCRUM_REVIEWS_JSONL = "/home/profit/lakehouse/data/_kb/scrum_reviews.jsonl";
|
||||||
const TAIL_LINES = 500;
|
const TAIL_LINES = 500;
|
||||||
const MAX_BOT_CYCLE_FILES = 30;
|
const MAX_BOT_CYCLE_FILES = 30;
|
||||||
|
|
||||||
export async function runKbCheck(claims: Claim[]): Promise<Finding[]> {
|
export async function runKbCheck(claims: Claim[], prFiles: string[] = []): Promise<Finding[]> {
|
||||||
const findings: Finding[] = [];
|
const findings: Finding[] = [];
|
||||||
|
|
||||||
// 1. Recent scenario outcomes: are strong-claim-style phrases showing
|
// 1. Recent scenario outcomes: are strong-claim-style phrases showing
|
||||||
@ -48,6 +50,15 @@ export async function runKbCheck(claims: Claim[]): Promise<Finding[]> {
|
|||||||
const obsFindings = await checkObserverStream();
|
const obsFindings = await checkObserverStream();
|
||||||
findings.push(...obsFindings);
|
findings.push(...obsFindings);
|
||||||
|
|
||||||
|
// 5. Scrum-master reviews — surface prior accepted reviews for any
|
||||||
|
// file in this PR's diff. Cohesion plan Phase C wire: the
|
||||||
|
// auditor gets to "borrow" the scrum-master's deeper per-file
|
||||||
|
// analysis instead of re-doing that work.
|
||||||
|
if (prFiles.length > 0) {
|
||||||
|
const scrumFindings = await checkScrumReviews(prFiles);
|
||||||
|
findings.push(...scrumFindings);
|
||||||
|
}
|
||||||
|
|
||||||
return findings;
|
return findings;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -181,3 +192,54 @@ function observerBySource(ops: any[]): string {
|
|||||||
}
|
}
|
||||||
return Object.entries(c).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}=${v}`).join(", ") || "empty";
|
return Object.entries(c).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}=${v}`).join(", ") || "empty";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Scrum-master reviews — the scrum pipeline writes one row per
|
||||||
|
// accepted per-file review. We match reviews whose `file` matches
|
||||||
|
// any path in the PR's diff, then surface the *preview* + which
|
||||||
|
// model the escalation ladder had to reach. If the scrum-master
|
||||||
|
// needed the 123B specialist or larger to resolve a file, that's
|
||||||
|
// a meaningful signal about the code's complexity — and it's
|
||||||
|
// surfaced to the PR without the auditor having to re-run the
|
||||||
|
// escalation ladder itself.
|
||||||
|
async function checkScrumReviews(prFiles: string[]): Promise<Finding[]> {
|
||||||
|
const rows = await tailJsonl<any>(SCRUM_REVIEWS_JSONL, TAIL_LINES);
|
||||||
|
if (rows.length === 0) return [];
|
||||||
|
|
||||||
|
// Match by exact file OR filename suffix — PR files arrive as
|
||||||
|
// `auditor/audit.ts`-style relative paths; scrum stores the same.
|
||||||
|
const norm = (p: string) => p.replace(/^\/+/, "").replace(/^home\/profit\/lakehouse\//, "");
|
||||||
|
const prSet = new Set(prFiles.map(norm));
|
||||||
|
|
||||||
|
// Keep only the most recent review per file (last-wins).
|
||||||
|
const latestByFile = new Map<string, any>();
|
||||||
|
for (const r of rows) {
|
||||||
|
const f = norm(String(r.file ?? ""));
|
||||||
|
if (!f) continue;
|
||||||
|
if (!prSet.has(f)) continue;
|
||||||
|
latestByFile.set(f, r);
|
||||||
|
}
|
||||||
|
if (latestByFile.size === 0) return [];
|
||||||
|
|
||||||
|
const findings: Finding[] = [];
|
||||||
|
for (const [file, r] of latestByFile) {
|
||||||
|
const model = String(r.accepted_model ?? "?");
|
||||||
|
const attempt = r.accepted_on_attempt ?? "?";
|
||||||
|
const treeSplit = !!r.tree_split_fired;
|
||||||
|
// Heuristic: if the scrum-master had to escalate past attempt 3,
|
||||||
|
// or had to tree-split, that's context the PR reviewer should see.
|
||||||
|
// Severity: info for low-escalation, warn if escalated far up
|
||||||
|
// the ladder (cloud specialist required).
|
||||||
|
const heavyEscalation = Number(attempt) >= 4;
|
||||||
|
const sev: "warn" | "info" = heavyEscalation ? "warn" : "info";
|
||||||
|
findings.push({
|
||||||
|
check: "kb_query",
|
||||||
|
severity: sev,
|
||||||
|
summary: `scrum-master review for \`${file}\` — accepted on attempt ${attempt} by \`${model}\`${treeSplit ? " (tree-split)" : ""}`,
|
||||||
|
evidence: [
|
||||||
|
`reviewed_at: ${r.reviewed_at ?? "?"}`,
|
||||||
|
`preview: ${String(r.suggestions_preview ?? "").slice(0, 300).replace(/\n/g, " ")}`,
|
||||||
|
],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return findings;
|
||||||
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user