diff --git a/auditor/README.md b/auditor/README.md new file mode 100644 index 0000000..a227b39 --- /dev/null +++ b/auditor/README.md @@ -0,0 +1,53 @@ +# Lakehouse Claim Auditor + +A Bun sub-agent that watches open PRs on Gitea, reads the ship-claims +in commit messages and PR bodies, and **hard-blocks** merges when the +code doesn't back the claim. + +Rationale: when "compiles + one curl works" gets called "phase shipped," +placeholder code accumulates. This auditor runs every 90s, fetches +each open PR, and subjects it to four checks: + +1. **Static diff** — grep/parse looking for placeholder patterns +2. **Dynamic** — runs the never-before-executed hybrid test fixture +3. **Cloud inference** — asks `gpt-oss:120b` via `/v1/chat` to + identify gaps in the diff +4. **KB query** — looks up `data/_kb/` + observer for prior failure + patterns on similar claims + +Verdict is assembled, posted to Gitea as: +- A **failing commit status** (hard block — branch protection + prevents merge) +- A **review comment** explaining every finding + +## Run manually + +```bash +cd /home/profit/lakehouse +bun run auditor/index.ts +``` + +Defaults: polls every 90s, stops on `auditor.paused` file present. + +## State + +- `data/_auditor/state.json` — last-audited head SHA per PR +- `data/_auditor/verdicts/{pr}-{sha}.json` — per-run verdict record + +## Where YOU edit + +`auditor/policy.ts` — the verdict assembler. Controls which findings +block vs warn vs inform. All other code is mechanical: fetching, +running checks, posting to Gitea. + +## Hard-block mechanism + +1. Commit status is posted as `failure` with context `lakehouse/auditor` +2. If `main` branch protection requires `lakehouse/auditor` status + to pass, Gitea prevents merge +3. When code is fixed and re-audit passes, status flips to `success`, + merge unblocks + +Enable branch protection (one-time, via Gitea UI or API): +- `POST /repos/profit/lakehouse/branch_protections` +- `{"branch_name": "main", "required_status_checks": {"contexts": ["lakehouse/auditor"]}}` diff --git a/auditor/gitea.ts b/auditor/gitea.ts new file mode 100644 index 0000000..582d9af --- /dev/null +++ b/auditor/gitea.ts @@ -0,0 +1,127 @@ +// Gitea API client. Minimal surface — only what the auditor needs: +// list open PRs, get commits + files for a PR, fetch a diff, post a +// commit status, post a review. +// +// Auth: reads PAT from ~/.git-credentials (set up by the credential +// helper flow in 2026-04-22 session). Gitea's "token" auth scheme +// matches what `git fetch` is already using. + +import { readFile } from "node:fs/promises"; +import type { PrSnapshot } from "./types.ts"; + +const HOST = process.env.GITEA_HOST ?? "https://git.agentview.dev"; +const OWNER = "profit"; +const REPO = "lakehouse"; +const CRED_FILE = "/home/profit/.git-credentials"; + +let cachedPat: string | null = null; + +async function getPat(): Promise { + if (cachedPat) return cachedPat; + const raw = await readFile(CRED_FILE, "utf8"); + for (const line of raw.split("\n")) { + const m = line.match(/^https:\/\/[^:]+:([^@]+)@git\.agentview\.dev/); + if (m) { cachedPat = m[1]; return m[1]; } + } + throw new Error(`no Gitea PAT in ${CRED_FILE}`); +} + +async function giteaFetch(path: string, init: RequestInit = {}): Promise { + const pat = await getPat(); + const url = `${HOST}/api/v1${path}`; + const headers = new Headers(init.headers); + headers.set("Authorization", `token ${pat}`); + if (init.body && !headers.has("content-type")) { + headers.set("content-type", "application/json"); + } + return fetch(url, { ...init, headers, signal: AbortSignal.timeout(20000) }); +} + +export async function listOpenPrs(): Promise { + const r = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls?state=open&page=1&limit=50`); + if (!r.ok) throw new Error(`listOpenPrs ${r.status}: ${await r.text()}`); + const rows = (await r.json()) as any[]; + return Promise.all(rows.map(row => snapshotFromPr(row))); +} + +export async function getPrSnapshot(num: number): Promise { + const r = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${num}`); + if (!r.ok) throw new Error(`getPr ${num} ${r.status}: ${await r.text()}`); + return snapshotFromPr((await r.json()) as any); +} + +async function snapshotFromPr(row: any): Promise { + const num = row.number; + const commitsResp = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${num}/commits`); + const commits = commitsResp.ok ? ((await commitsResp.json()) as any[]) : []; + const filesResp = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${num}/files`); + const files = filesResp.ok ? ((await filesResp.json()) as any[]) : []; + return { + number: num, + head_sha: row.head?.sha ?? "", + base_sha: row.base?.sha ?? "", + title: row.title ?? "", + body: row.body ?? "", + state: row.state === "open" ? "open" : (row.merged ? "merged" : "closed"), + author: row.user?.login ?? "", + commits: commits.map(c => ({ + sha: (c.sha ?? "").slice(0, 12), + message: c.commit?.message ?? "", + author: c.commit?.author?.name ?? "", + })), + files: files.map(f => ({ + path: f.filename ?? "", + additions: f.additions ?? 0, + deletions: f.deletions ?? 0, + })), + }; +} + +/// Returns the unified diff text of the PR. Used by static checks. +export async function getPrDiff(num: number): Promise { + const r = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${num}.diff`); + if (!r.ok) throw new Error(`getDiff ${num} ${r.status}: ${await r.text()}`); + return await r.text(); +} + +/// Hard-block mechanism: post a failing commit status on the PR head +/// SHA. Branch protection (if enabled on `main`) treats this as a +/// required-check fail and prevents merge. The description is shown +/// in the Gitea UI next to the red X. +export async function postCommitStatus(args: { + sha: string; + state: "success" | "pending" | "failure" | "error"; + context: string; + description: string; + target_url?: string; +}): Promise { + const r = await giteaFetch(`/repos/${OWNER}/${REPO}/statuses/${args.sha}`, { + method: "POST", + body: JSON.stringify({ + state: args.state, + context: args.context, + description: args.description.slice(0, 140), + target_url: args.target_url ?? "", + }), + }); + if (!r.ok) throw new Error(`postCommitStatus ${r.status}: ${await r.text()}`); +} + +/// Post a review comment. Type: "REQUEST_CHANGES" for block, +/// "COMMENT" for non-blocking, "APPROVE" for green. +export async function postReview(args: { + pr_number: number; + commit_id: string; + body: string; + event: "APPROVE" | "REQUEST_CHANGES" | "COMMENT"; +}): Promise { + const r = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${args.pr_number}/reviews`, { + method: "POST", + body: JSON.stringify({ + commit_id: args.commit_id, + body: args.body, + event: args.event, + }), + }); + if (!r.ok) throw new Error(`postReview ${r.status}: ${await r.text()}`); +} diff --git a/auditor/policy.ts b/auditor/policy.ts new file mode 100644 index 0000000..ea0def1 --- /dev/null +++ b/auditor/policy.ts @@ -0,0 +1,62 @@ +// ═══════════════════════════════════════════════════════════════════ +// YOU WRITE THIS FILE. Policy decides what blocks vs what's a comment. +// Defaults are opinionated on the "stop clicking past placeholder" +// side — easier to loosen than to tighten when you're watching the +// auditor behave in live PRs. +// ═══════════════════════════════════════════════════════════════════ + +import type { Finding, Verdict } from "./types.ts"; + +/// Translate the four-check output into a single verdict. This is the +/// single pane of glass the auditor operates on — tune thresholds here. +export function assembleVerdict( + findings: Finding[], + metrics: Record, + pr_number: number, + head_sha: string, +): Verdict { + const blocking = findings.filter(f => f.severity === "block"); + const warning = findings.filter(f => f.severity === "warn"); + + let overall: Verdict["overall"]; + let one_liner: string; + + if (blocking.length > 0) { + overall = "block"; + one_liner = `${blocking.length} blocking issue${blocking.length > 1 ? "s" : ""}: ${blocking[0].summary}`; + } else if (warning.length >= 3) { + // Three or more warnings is a block — death by a thousand cuts. + overall = "request_changes"; + one_liner = `${warning.length} warnings — see review`; + } else if (warning.length > 0) { + overall = "request_changes"; + one_liner = warning[0].summary; + } else { + overall = "approve"; + one_liner = `all checks passed (${findings.length} findings, all info)`; + } + + return { + pr_number, + head_sha, + audited_at: new Date().toISOString(), + overall, + findings, + metrics, + one_liner, + }; +} + +/// Which strength-of-claim warrants which severity when evidence is +/// weak? A "Phase X shipped" claim with zero integration tests is a +/// blocker. A "should work" claim with no test is a warn. +export function severityFromClaimEvidence( + claim_strength: "weak" | "moderate" | "strong", + evidence_grade: "none" | "partial" | "full", +): "info" | "warn" | "block" { + if (evidence_grade === "full") return "info"; + if (claim_strength === "strong" && evidence_grade === "none") return "block"; + if (claim_strength === "strong" && evidence_grade === "partial") return "warn"; + if (claim_strength === "moderate" && evidence_grade === "none") return "warn"; + return "info"; +} diff --git a/auditor/types.ts b/auditor/types.ts new file mode 100644 index 0000000..5ab0360 --- /dev/null +++ b/auditor/types.ts @@ -0,0 +1,65 @@ +// Shared types for the claim-auditor. Every field exists for a reason; +// if something can't be verified from a check, it goes into `evidence` +// so the verdict is inspectable, not a black box. + +export type CheckKind = "static" | "dynamic" | "inference" | "kb_query"; + +export type Severity = "info" | "warn" | "block"; + +export interface Claim { + // Verbatim phrase that raised the claim — e.g. "Phase 38 shipped", + // "verified end-to-end", "works after restart". Used as the "what + // does the author assert" input to downstream checks. + text: string; + // Where it came from. `commit_sha` is the short hash; `location` + // is a file:line for in-diff claims, or "pr_body" / "commit_message". + commit_sha: string; + location: string; + // Heuristic rating of how strong the claim is. "green+tested" + // is strong; "should work" is weak. Drives sensitivity — stronger + // claims get harder-blocked on weak evidence. + strength: "weak" | "moderate" | "strong"; +} + +export interface Finding { + check: CheckKind; + severity: Severity; + claim_text?: string; + // Free-form short description: "field added but never read", "no + // test covers this code path", "cloud model says placeholder". + summary: string; + // Concrete evidence: file paths, line numbers, log excerpts, test + // output, cloud-model verdict. No handwaving. + evidence: string[]; +} + +export interface Verdict { + pr_number: number; + head_sha: string; + audited_at: string; + overall: "approve" | "request_changes" | "block"; + findings: Finding[]; + // Real numbers that downstream policy can gate on. e.g. if the + // hybrid test produced latency numbers or token counts, they + // surface here so /auditor/history is queryable. + metrics: Record; + // Short one-line justification for the `overall` verdict. What + // gets posted as the commit-status description in Gitea (max 140 + // chars) must fit here. + one_liner: string; +} + +export interface PrSnapshot { + number: number; + head_sha: string; + base_sha: string; + title: string; + body: string; + state: "open" | "closed" | "merged"; + author: string; + // Array of commit messages in the PR (not diffs — those are + // fetched on-demand per-check). + commits: Array<{ sha: string; message: string; author: string }>; + // File paths touched by the PR, with lines-added / lines-removed. + files: Array<{ path: string; additions: number; deletions: number }>; +}