Some checks failed
lakehouse/auditor 2 blocking issues: unimplemented!() macro call in tests/real-world/hard_task_escalation.ts
Wires the cohesion-plan Phase C link: the scrum-master pipeline writes per-file reviews to data/_kb/scrum_reviews.jsonl on accept; the auditor now reads that same file and emits one kb_query finding per scrum review whose `file` matches a path in the PR's diff. Severity heuristic: attempt 1-3 → info, attempt 4+ → warn. Reaching the cloud specialist (attempt 4+) means the ladder had to escalate, which is meaningful signal reviewers should see. Tree-split fired is also surfaced in the finding summary. audit.ts now passes pr.files.map(f => f.path) into runKbCheck (the old signature dropped it on the floor). Also adds auditor/audit_one.ts — a dry-run CLI for auditing a single PR without posting to Gitea, useful for verifying check behavior without spamming review comments. Verified: after writing scrum_reviews for auditor/audit.ts and mcp-server/observer.ts (both in PR #7), audit_one 7 surfaced both as info findings with preview + accepted_model + tree_split flag. A scrum review for playbook_memory.rs (NOT in PR #7) was correctly filtered out.
172 lines
6.8 KiB
TypeScript
172 lines
6.8 KiB
TypeScript
// Orchestrator — runs all four checks on a PR, assembles a verdict,
|
||
// posts to Gitea. This is task #8's integration layer; the poller
|
||
// (task #9) calls this once per PR on every fresh head SHA.
|
||
//
|
||
// Hard-block mechanism: commit status posted with state="failure"
|
||
// and context="lakehouse/auditor". If `main` branch protection
|
||
// requires that context to pass, merge is physically impossible
|
||
// until the auditor re-audits a fixed commit and flips the status
|
||
// to "success".
|
||
//
|
||
// Human-readable reasoning: posted as a PR issue comment (not a
|
||
// review — reviews have self-review restrictions on Gitea and the
|
||
// auditor currently uses the same PAT as the PR author).
|
||
|
||
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
||
import { join } from "node:path";
|
||
import type { PrSnapshot, Verdict, Finding } from "./types.ts";
|
||
import { getPrDiff, postCommitStatus, postIssueComment } from "./gitea.ts";
|
||
import { parseClaims } from "./claim_parser.ts";
|
||
import { assembleVerdict } from "./policy.ts";
|
||
import { runStaticCheck } from "./checks/static.ts";
|
||
import { runDynamicCheck } from "./checks/dynamic.ts";
|
||
import { runInferenceCheck } from "./checks/inference.ts";
|
||
import { runKbCheck } from "./checks/kb_query.ts";
|
||
|
||
const VERDICTS_DIR = "/home/profit/lakehouse/data/_auditor/verdicts";
|
||
|
||
export interface AuditOptions {
|
||
// Skip the cloud inference call (fast path for iteration). Default false.
|
||
skip_inference?: boolean;
|
||
// Skip the dynamic check (avoid running the hybrid fixture every PR,
|
||
// since it hits live services and mutates playbook state). Default false
|
||
// on `main`-branch-target PRs, true when auditing feature branches
|
||
// where the fixture would pollute state. Caller decides.
|
||
skip_dynamic?: boolean;
|
||
// Skip Gitea posting — useful for dry-runs / local testing.
|
||
// Default false.
|
||
dry_run?: boolean;
|
||
}
|
||
|
||
export async function auditPr(pr: PrSnapshot, opts: AuditOptions = {}): Promise<Verdict> {
|
||
const t0 = Date.now();
|
||
const diff = await getPrDiff(pr.number);
|
||
const { claims } = parseClaims(pr);
|
||
|
||
// Run checks in parallel where they don't share mutable state.
|
||
// Static + kb_query + inference are all read-only. Dynamic mutates
|
||
// playbook state (nonce-scoped per run, but still live) so if
|
||
// skip_dynamic is false we still run it in parallel — the mutation
|
||
// is namespaced.
|
||
const [staticFindings, dynamicFindings, inferenceFindings, kbFindings] = await Promise.all([
|
||
runStaticCheck(diff),
|
||
opts.skip_dynamic ? Promise.resolve(stubFinding("dynamic", "skipped by options")) : runDynamicCheck(),
|
||
opts.skip_inference ? Promise.resolve(stubFinding("inference", "skipped by options")) : runInferenceCheck(claims, diff),
|
||
runKbCheck(claims, pr.files.map(f => f.path)),
|
||
]);
|
||
|
||
const allFindings: Finding[] = [
|
||
...staticFindings,
|
||
...dynamicFindings,
|
||
...inferenceFindings,
|
||
...kbFindings,
|
||
];
|
||
|
||
const duration_ms = Date.now() - t0;
|
||
const metrics = {
|
||
audit_duration_ms: duration_ms,
|
||
findings_total: allFindings.length,
|
||
findings_block: allFindings.filter(f => f.severity === "block").length,
|
||
findings_warn: allFindings.filter(f => f.severity === "warn").length,
|
||
findings_info: allFindings.filter(f => f.severity === "info").length,
|
||
claims_strong: claims.filter(c => c.strength === "strong").length,
|
||
claims_moderate: claims.filter(c => c.strength === "moderate").length,
|
||
claims_weak: claims.filter(c => c.strength === "weak").length,
|
||
claims_total: claims.length,
|
||
diff_bytes: diff.length,
|
||
};
|
||
|
||
const verdict = assembleVerdict(allFindings, metrics, pr.number, pr.head_sha);
|
||
|
||
await persistVerdict(verdict);
|
||
|
||
if (!opts.dry_run) {
|
||
await postToGitea(verdict);
|
||
}
|
||
|
||
return verdict;
|
||
}
|
||
|
||
async function persistVerdict(v: Verdict): Promise<void> {
|
||
await mkdir(VERDICTS_DIR, { recursive: true });
|
||
const filename = `${v.pr_number}-${v.head_sha.slice(0, 12)}.json`;
|
||
await writeFile(join(VERDICTS_DIR, filename), JSON.stringify(v, null, 2));
|
||
}
|
||
|
||
export async function postToGitea(v: Verdict): Promise<void> {
|
||
// 1. Commit status — the hard block signal (if branch protection
|
||
// is configured to require lakehouse/auditor on main).
|
||
const state = v.overall === "approve" ? "success" : "failure";
|
||
await postCommitStatus({
|
||
sha: v.head_sha,
|
||
state,
|
||
context: "lakehouse/auditor",
|
||
description: v.one_liner,
|
||
target_url: "", // no URL yet; could point to a verdicts dashboard
|
||
});
|
||
|
||
// 2. Issue comment — the reasoning. Gated so we don't spam the PR
|
||
// with identical comments on re-audits of the same SHA. Caller
|
||
// (poller) ensures we only re-audit fresh SHAs, but a dedup
|
||
// marker inside the body keeps it idempotent if re-run.
|
||
const body = formatReviewBody(v);
|
||
await postIssueComment({ pr_number: v.pr_number, body });
|
||
}
|
||
|
||
function formatReviewBody(v: Verdict): string {
|
||
const byCheck: Record<string, Finding[]> = {};
|
||
for (const f of v.findings) {
|
||
(byCheck[f.check] ||= []).push(f);
|
||
}
|
||
|
||
const verdictEmoji =
|
||
v.overall === "approve" ? "✅" :
|
||
v.overall === "request_changes" ? "⚠️" :
|
||
"🛑";
|
||
|
||
const lines: string[] = [];
|
||
lines.push(`## Auditor verdict: ${verdictEmoji} \`${v.overall}\``);
|
||
lines.push("");
|
||
lines.push(`**One-liner:** ${v.one_liner}`);
|
||
lines.push(`**Head SHA:** \`${v.head_sha.slice(0, 12)}\``);
|
||
lines.push(`**Audited at:** ${v.audited_at}`);
|
||
lines.push("");
|
||
|
||
// Per-check sections, only if the check produced findings.
|
||
const checkOrder = ["static", "dynamic", "inference", "kb_query"] as const;
|
||
for (const check of checkOrder) {
|
||
const fs = byCheck[check] ?? [];
|
||
if (fs.length === 0) continue;
|
||
const bySev = {
|
||
block: fs.filter(f => f.severity === "block").length,
|
||
warn: fs.filter(f => f.severity === "warn").length,
|
||
info: fs.filter(f => f.severity === "info").length,
|
||
};
|
||
lines.push(`<details><summary><b>${check}</b> — ${fs.length} findings (${bySev.block} block, ${bySev.warn} warn, ${bySev.info} info)</summary>`);
|
||
lines.push("");
|
||
for (const f of fs) {
|
||
const mark = f.severity === "block" ? "🛑" : f.severity === "warn" ? "⚠️" : "ℹ️";
|
||
lines.push(`${mark} **${f.severity}** — ${f.summary}`);
|
||
for (const e of f.evidence.slice(0, 3)) {
|
||
lines.push(` - \`${e.slice(0, 180).replace(/\n/g, " ")}\``);
|
||
}
|
||
}
|
||
lines.push("");
|
||
lines.push("</details>");
|
||
lines.push("");
|
||
}
|
||
|
||
lines.push("### Metrics");
|
||
lines.push("```json");
|
||
lines.push(JSON.stringify(v.metrics, null, 2));
|
||
lines.push("```");
|
||
lines.push("");
|
||
lines.push(`<sub>Lakehouse auditor · SHA ${v.head_sha.slice(0, 8)} · re-audit on new commit flips the status automatically.</sub>`);
|
||
|
||
return lines.join("\n");
|
||
}
|
||
|
||
function stubFinding(check: "dynamic" | "inference", why: string): Finding[] {
|
||
return [{ check, severity: "info", summary: `${check} check skipped — ${why}`, evidence: [why] }];
|
||
}
|