lakehouse/auditor/audit.ts
profit dc01ba0a3b
Some checks failed
lakehouse/auditor 2 blocking issues: unimplemented!() macro call in tests/real-world/hard_task_escalation.ts
auditor: kb_query surfaces scrum-master reviews for files in PR diff
Wires the cohesion-plan Phase C link: the scrum-master pipeline writes
per-file reviews to data/_kb/scrum_reviews.jsonl on accept; the
auditor now reads that same file and emits one kb_query finding per
scrum review whose `file` matches a path in the PR's diff.

Severity heuristic: attempt 1-3 → info, attempt 4+ → warn. Reaching
the cloud specialist (attempt 4+) means the ladder had to escalate,
which is meaningful signal reviewers should see. Tree-split fired is
also surfaced in the finding summary.

audit.ts now passes pr.files.map(f => f.path) into runKbCheck (the
old signature dropped it on the floor). Also adds auditor/audit_one.ts
— a dry-run CLI for auditing a single PR without posting to Gitea,
useful for verifying check behavior without spamming review comments.

Verified: after writing scrum_reviews for auditor/audit.ts and
mcp-server/observer.ts (both in PR #7), audit_one 7 surfaced both as
info findings with preview + accepted_model + tree_split flag. A
scrum review for playbook_memory.rs (NOT in PR #7) was correctly
filtered out.
2026-04-22 21:18:21 -05:00

172 lines
6.8 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// Orchestrator — runs all four checks on a PR, assembles a verdict,
// posts to Gitea. This is task #8's integration layer; the poller
// (task #9) calls this once per PR on every fresh head SHA.
//
// Hard-block mechanism: commit status posted with state="failure"
// and context="lakehouse/auditor". If `main` branch protection
// requires that context to pass, merge is physically impossible
// until the auditor re-audits a fixed commit and flips the status
// to "success".
//
// Human-readable reasoning: posted as a PR issue comment (not a
// review — reviews have self-review restrictions on Gitea and the
// auditor currently uses the same PAT as the PR author).
import { readFile, writeFile, mkdir } from "node:fs/promises";
import { join } from "node:path";
import type { PrSnapshot, Verdict, Finding } from "./types.ts";
import { getPrDiff, postCommitStatus, postIssueComment } from "./gitea.ts";
import { parseClaims } from "./claim_parser.ts";
import { assembleVerdict } from "./policy.ts";
import { runStaticCheck } from "./checks/static.ts";
import { runDynamicCheck } from "./checks/dynamic.ts";
import { runInferenceCheck } from "./checks/inference.ts";
import { runKbCheck } from "./checks/kb_query.ts";
const VERDICTS_DIR = "/home/profit/lakehouse/data/_auditor/verdicts";
export interface AuditOptions {
// Skip the cloud inference call (fast path for iteration). Default false.
skip_inference?: boolean;
// Skip the dynamic check (avoid running the hybrid fixture every PR,
// since it hits live services and mutates playbook state). Default false
// on `main`-branch-target PRs, true when auditing feature branches
// where the fixture would pollute state. Caller decides.
skip_dynamic?: boolean;
// Skip Gitea posting — useful for dry-runs / local testing.
// Default false.
dry_run?: boolean;
}
export async function auditPr(pr: PrSnapshot, opts: AuditOptions = {}): Promise<Verdict> {
const t0 = Date.now();
const diff = await getPrDiff(pr.number);
const { claims } = parseClaims(pr);
// Run checks in parallel where they don't share mutable state.
// Static + kb_query + inference are all read-only. Dynamic mutates
// playbook state (nonce-scoped per run, but still live) so if
// skip_dynamic is false we still run it in parallel — the mutation
// is namespaced.
const [staticFindings, dynamicFindings, inferenceFindings, kbFindings] = await Promise.all([
runStaticCheck(diff),
opts.skip_dynamic ? Promise.resolve(stubFinding("dynamic", "skipped by options")) : runDynamicCheck(),
opts.skip_inference ? Promise.resolve(stubFinding("inference", "skipped by options")) : runInferenceCheck(claims, diff),
runKbCheck(claims, pr.files.map(f => f.path)),
]);
const allFindings: Finding[] = [
...staticFindings,
...dynamicFindings,
...inferenceFindings,
...kbFindings,
];
const duration_ms = Date.now() - t0;
const metrics = {
audit_duration_ms: duration_ms,
findings_total: allFindings.length,
findings_block: allFindings.filter(f => f.severity === "block").length,
findings_warn: allFindings.filter(f => f.severity === "warn").length,
findings_info: allFindings.filter(f => f.severity === "info").length,
claims_strong: claims.filter(c => c.strength === "strong").length,
claims_moderate: claims.filter(c => c.strength === "moderate").length,
claims_weak: claims.filter(c => c.strength === "weak").length,
claims_total: claims.length,
diff_bytes: diff.length,
};
const verdict = assembleVerdict(allFindings, metrics, pr.number, pr.head_sha);
await persistVerdict(verdict);
if (!opts.dry_run) {
await postToGitea(verdict);
}
return verdict;
}
async function persistVerdict(v: Verdict): Promise<void> {
await mkdir(VERDICTS_DIR, { recursive: true });
const filename = `${v.pr_number}-${v.head_sha.slice(0, 12)}.json`;
await writeFile(join(VERDICTS_DIR, filename), JSON.stringify(v, null, 2));
}
export async function postToGitea(v: Verdict): Promise<void> {
// 1. Commit status — the hard block signal (if branch protection
// is configured to require lakehouse/auditor on main).
const state = v.overall === "approve" ? "success" : "failure";
await postCommitStatus({
sha: v.head_sha,
state,
context: "lakehouse/auditor",
description: v.one_liner,
target_url: "", // no URL yet; could point to a verdicts dashboard
});
// 2. Issue comment — the reasoning. Gated so we don't spam the PR
// with identical comments on re-audits of the same SHA. Caller
// (poller) ensures we only re-audit fresh SHAs, but a dedup
// marker inside the body keeps it idempotent if re-run.
const body = formatReviewBody(v);
await postIssueComment({ pr_number: v.pr_number, body });
}
function formatReviewBody(v: Verdict): string {
const byCheck: Record<string, Finding[]> = {};
for (const f of v.findings) {
(byCheck[f.check] ||= []).push(f);
}
const verdictEmoji =
v.overall === "approve" ? "✅" :
v.overall === "request_changes" ? "⚠️" :
"🛑";
const lines: string[] = [];
lines.push(`## Auditor verdict: ${verdictEmoji} \`${v.overall}\``);
lines.push("");
lines.push(`**One-liner:** ${v.one_liner}`);
lines.push(`**Head SHA:** \`${v.head_sha.slice(0, 12)}\``);
lines.push(`**Audited at:** ${v.audited_at}`);
lines.push("");
// Per-check sections, only if the check produced findings.
const checkOrder = ["static", "dynamic", "inference", "kb_query"] as const;
for (const check of checkOrder) {
const fs = byCheck[check] ?? [];
if (fs.length === 0) continue;
const bySev = {
block: fs.filter(f => f.severity === "block").length,
warn: fs.filter(f => f.severity === "warn").length,
info: fs.filter(f => f.severity === "info").length,
};
lines.push(`<details><summary><b>${check}</b> — ${fs.length} findings (${bySev.block} block, ${bySev.warn} warn, ${bySev.info} info)</summary>`);
lines.push("");
for (const f of fs) {
const mark = f.severity === "block" ? "🛑" : f.severity === "warn" ? "⚠️" : "";
lines.push(`${mark} **${f.severity}** — ${f.summary}`);
for (const e of f.evidence.slice(0, 3)) {
lines.push(` - \`${e.slice(0, 180).replace(/\n/g, " ")}\``);
}
}
lines.push("");
lines.push("</details>");
lines.push("");
}
lines.push("### Metrics");
lines.push("```json");
lines.push(JSON.stringify(v.metrics, null, 2));
lines.push("```");
lines.push("");
lines.push(`<sub>Lakehouse auditor · SHA ${v.head_sha.slice(0, 8)} · re-audit on new commit flips the status automatically.</sub>`);
return lines.join("\n");
}
function stubFinding(check: "dynamic" | "inference", why: string): Finding[] {
return [{ check, severity: "info", summary: `${check} check skipped — ${why}`, evidence: [why] }];
}