Auditor scaffold: types + Gitea client + policy stub + README
All-Bun sub-agent that watches open PRs on Gitea, reads ship-claims, and hard-blocks merges when the code doesn't back the claim. First commit of N; this is the skeleton. Dynamic/static/inference/kb checks + poller land in follow-up commits on this same branch. - auditor/types.ts — Claim, Finding, Verdict, PrSnapshot shapes - auditor/gitea.ts — minimal API client (listOpenPrs, getPrDiff, postCommitStatus, postReview). Live-proven: returned 0 open PRs against our repo (which IS the current state — every commit today went to main directly, which is the problem this auditor is meant to prevent) - auditor/policy.ts — stub `assembleVerdict` + severity rules. Intentionally conservative defaults: strong claim + zero evidence = block, not warn. - auditor/README.md — how to run + the hard-block mechanism Workflow discipline change: starting with this branch, no more direct pushes to main. Every change lands as a PR. When this auditor is fully built and running, it'll review its own completion PR — the recursive self-test. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
profit
parent
affab8ac83
commit
f48dd2f20b
53
auditor/README.md
Normal file
53
auditor/README.md
Normal file
@ -0,0 +1,53 @@
|
||||
# Lakehouse Claim Auditor
|
||||
|
||||
A Bun sub-agent that watches open PRs on Gitea, reads the ship-claims
|
||||
in commit messages and PR bodies, and **hard-blocks** merges when the
|
||||
code doesn't back the claim.
|
||||
|
||||
Rationale: when "compiles + one curl works" gets called "phase shipped,"
|
||||
placeholder code accumulates. This auditor runs every 90s, fetches
|
||||
each open PR, and subjects it to four checks:
|
||||
|
||||
1. **Static diff** — grep/parse looking for placeholder patterns
|
||||
2. **Dynamic** — runs the never-before-executed hybrid test fixture
|
||||
3. **Cloud inference** — asks `gpt-oss:120b` via `/v1/chat` to
|
||||
identify gaps in the diff
|
||||
4. **KB query** — looks up `data/_kb/` + observer for prior failure
|
||||
patterns on similar claims
|
||||
|
||||
Verdict is assembled, posted to Gitea as:
|
||||
- A **failing commit status** (hard block — branch protection
|
||||
prevents merge)
|
||||
- A **review comment** explaining every finding
|
||||
|
||||
## Run manually
|
||||
|
||||
```bash
|
||||
cd /home/profit/lakehouse
|
||||
bun run auditor/index.ts
|
||||
```
|
||||
|
||||
Defaults: polls every 90s, stops on `auditor.paused` file present.
|
||||
|
||||
## State
|
||||
|
||||
- `data/_auditor/state.json` — last-audited head SHA per PR
|
||||
- `data/_auditor/verdicts/{pr}-{sha}.json` — per-run verdict record
|
||||
|
||||
## Where YOU edit
|
||||
|
||||
`auditor/policy.ts` — the verdict assembler. Controls which findings
|
||||
block vs warn vs inform. All other code is mechanical: fetching,
|
||||
running checks, posting to Gitea.
|
||||
|
||||
## Hard-block mechanism
|
||||
|
||||
1. Commit status is posted as `failure` with context `lakehouse/auditor`
|
||||
2. If `main` branch protection requires `lakehouse/auditor` status
|
||||
to pass, Gitea prevents merge
|
||||
3. When code is fixed and re-audit passes, status flips to `success`,
|
||||
merge unblocks
|
||||
|
||||
Enable branch protection (one-time, via Gitea UI or API):
|
||||
- `POST /repos/profit/lakehouse/branch_protections`
|
||||
- `{"branch_name": "main", "required_status_checks": {"contexts": ["lakehouse/auditor"]}}`
|
||||
127
auditor/gitea.ts
Normal file
127
auditor/gitea.ts
Normal file
@ -0,0 +1,127 @@
|
||||
// Gitea API client. Minimal surface — only what the auditor needs:
|
||||
// list open PRs, get commits + files for a PR, fetch a diff, post a
|
||||
// commit status, post a review.
|
||||
//
|
||||
// Auth: reads PAT from ~/.git-credentials (set up by the credential
|
||||
// helper flow in 2026-04-22 session). Gitea's "token" auth scheme
|
||||
// matches what `git fetch` is already using.
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
import type { PrSnapshot } from "./types.ts";
|
||||
|
||||
const HOST = process.env.GITEA_HOST ?? "https://git.agentview.dev";
|
||||
const OWNER = "profit";
|
||||
const REPO = "lakehouse";
|
||||
const CRED_FILE = "/home/profit/.git-credentials";
|
||||
|
||||
let cachedPat: string | null = null;
|
||||
|
||||
async function getPat(): Promise<string> {
|
||||
if (cachedPat) return cachedPat;
|
||||
const raw = await readFile(CRED_FILE, "utf8");
|
||||
for (const line of raw.split("\n")) {
|
||||
const m = line.match(/^https:\/\/[^:]+:([^@]+)@git\.agentview\.dev/);
|
||||
if (m) { cachedPat = m[1]; return m[1]; }
|
||||
}
|
||||
throw new Error(`no Gitea PAT in ${CRED_FILE}`);
|
||||
}
|
||||
|
||||
async function giteaFetch(path: string, init: RequestInit = {}): Promise<Response> {
|
||||
const pat = await getPat();
|
||||
const url = `${HOST}/api/v1${path}`;
|
||||
const headers = new Headers(init.headers);
|
||||
headers.set("Authorization", `token ${pat}`);
|
||||
if (init.body && !headers.has("content-type")) {
|
||||
headers.set("content-type", "application/json");
|
||||
}
|
||||
return fetch(url, { ...init, headers, signal: AbortSignal.timeout(20000) });
|
||||
}
|
||||
|
||||
export async function listOpenPrs(): Promise<PrSnapshot[]> {
|
||||
const r = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls?state=open&page=1&limit=50`);
|
||||
if (!r.ok) throw new Error(`listOpenPrs ${r.status}: ${await r.text()}`);
|
||||
const rows = (await r.json()) as any[];
|
||||
return Promise.all(rows.map(row => snapshotFromPr(row)));
|
||||
}
|
||||
|
||||
export async function getPrSnapshot(num: number): Promise<PrSnapshot> {
|
||||
const r = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${num}`);
|
||||
if (!r.ok) throw new Error(`getPr ${num} ${r.status}: ${await r.text()}`);
|
||||
return snapshotFromPr((await r.json()) as any);
|
||||
}
|
||||
|
||||
async function snapshotFromPr(row: any): Promise<PrSnapshot> {
|
||||
const num = row.number;
|
||||
const commitsResp = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${num}/commits`);
|
||||
const commits = commitsResp.ok ? ((await commitsResp.json()) as any[]) : [];
|
||||
const filesResp = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${num}/files`);
|
||||
const files = filesResp.ok ? ((await filesResp.json()) as any[]) : [];
|
||||
return {
|
||||
number: num,
|
||||
head_sha: row.head?.sha ?? "",
|
||||
base_sha: row.base?.sha ?? "",
|
||||
title: row.title ?? "",
|
||||
body: row.body ?? "",
|
||||
state: row.state === "open" ? "open" : (row.merged ? "merged" : "closed"),
|
||||
author: row.user?.login ?? "",
|
||||
commits: commits.map(c => ({
|
||||
sha: (c.sha ?? "").slice(0, 12),
|
||||
message: c.commit?.message ?? "",
|
||||
author: c.commit?.author?.name ?? "",
|
||||
})),
|
||||
files: files.map(f => ({
|
||||
path: f.filename ?? "",
|
||||
additions: f.additions ?? 0,
|
||||
deletions: f.deletions ?? 0,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
/// Returns the unified diff text of the PR. Used by static checks.
|
||||
export async function getPrDiff(num: number): Promise<string> {
|
||||
const r = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${num}.diff`);
|
||||
if (!r.ok) throw new Error(`getDiff ${num} ${r.status}: ${await r.text()}`);
|
||||
return await r.text();
|
||||
}
|
||||
|
||||
/// Hard-block mechanism: post a failing commit status on the PR head
|
||||
/// SHA. Branch protection (if enabled on `main`) treats this as a
|
||||
/// required-check fail and prevents merge. The description is shown
|
||||
/// in the Gitea UI next to the red X.
|
||||
export async function postCommitStatus(args: {
|
||||
sha: string;
|
||||
state: "success" | "pending" | "failure" | "error";
|
||||
context: string;
|
||||
description: string;
|
||||
target_url?: string;
|
||||
}): Promise<void> {
|
||||
const r = await giteaFetch(`/repos/${OWNER}/${REPO}/statuses/${args.sha}`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
state: args.state,
|
||||
context: args.context,
|
||||
description: args.description.slice(0, 140),
|
||||
target_url: args.target_url ?? "",
|
||||
}),
|
||||
});
|
||||
if (!r.ok) throw new Error(`postCommitStatus ${r.status}: ${await r.text()}`);
|
||||
}
|
||||
|
||||
/// Post a review comment. Type: "REQUEST_CHANGES" for block,
|
||||
/// "COMMENT" for non-blocking, "APPROVE" for green.
|
||||
export async function postReview(args: {
|
||||
pr_number: number;
|
||||
commit_id: string;
|
||||
body: string;
|
||||
event: "APPROVE" | "REQUEST_CHANGES" | "COMMENT";
|
||||
}): Promise<void> {
|
||||
const r = await giteaFetch(`/repos/${OWNER}/${REPO}/pulls/${args.pr_number}/reviews`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
commit_id: args.commit_id,
|
||||
body: args.body,
|
||||
event: args.event,
|
||||
}),
|
||||
});
|
||||
if (!r.ok) throw new Error(`postReview ${r.status}: ${await r.text()}`);
|
||||
}
|
||||
62
auditor/policy.ts
Normal file
62
auditor/policy.ts
Normal file
@ -0,0 +1,62 @@
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// YOU WRITE THIS FILE. Policy decides what blocks vs what's a comment.
|
||||
// Defaults are opinionated on the "stop clicking past placeholder"
|
||||
// side — easier to loosen than to tighten when you're watching the
|
||||
// auditor behave in live PRs.
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
import type { Finding, Verdict } from "./types.ts";
|
||||
|
||||
/// Translate the four-check output into a single verdict. This is the
|
||||
/// single pane of glass the auditor operates on — tune thresholds here.
|
||||
export function assembleVerdict(
|
||||
findings: Finding[],
|
||||
metrics: Record<string, number>,
|
||||
pr_number: number,
|
||||
head_sha: string,
|
||||
): Verdict {
|
||||
const blocking = findings.filter(f => f.severity === "block");
|
||||
const warning = findings.filter(f => f.severity === "warn");
|
||||
|
||||
let overall: Verdict["overall"];
|
||||
let one_liner: string;
|
||||
|
||||
if (blocking.length > 0) {
|
||||
overall = "block";
|
||||
one_liner = `${blocking.length} blocking issue${blocking.length > 1 ? "s" : ""}: ${blocking[0].summary}`;
|
||||
} else if (warning.length >= 3) {
|
||||
// Three or more warnings is a block — death by a thousand cuts.
|
||||
overall = "request_changes";
|
||||
one_liner = `${warning.length} warnings — see review`;
|
||||
} else if (warning.length > 0) {
|
||||
overall = "request_changes";
|
||||
one_liner = warning[0].summary;
|
||||
} else {
|
||||
overall = "approve";
|
||||
one_liner = `all checks passed (${findings.length} findings, all info)`;
|
||||
}
|
||||
|
||||
return {
|
||||
pr_number,
|
||||
head_sha,
|
||||
audited_at: new Date().toISOString(),
|
||||
overall,
|
||||
findings,
|
||||
metrics,
|
||||
one_liner,
|
||||
};
|
||||
}
|
||||
|
||||
/// Which strength-of-claim warrants which severity when evidence is
|
||||
/// weak? A "Phase X shipped" claim with zero integration tests is a
|
||||
/// blocker. A "should work" claim with no test is a warn.
|
||||
export function severityFromClaimEvidence(
|
||||
claim_strength: "weak" | "moderate" | "strong",
|
||||
evidence_grade: "none" | "partial" | "full",
|
||||
): "info" | "warn" | "block" {
|
||||
if (evidence_grade === "full") return "info";
|
||||
if (claim_strength === "strong" && evidence_grade === "none") return "block";
|
||||
if (claim_strength === "strong" && evidence_grade === "partial") return "warn";
|
||||
if (claim_strength === "moderate" && evidence_grade === "none") return "warn";
|
||||
return "info";
|
||||
}
|
||||
65
auditor/types.ts
Normal file
65
auditor/types.ts
Normal file
@ -0,0 +1,65 @@
|
||||
// Shared types for the claim-auditor. Every field exists for a reason;
|
||||
// if something can't be verified from a check, it goes into `evidence`
|
||||
// so the verdict is inspectable, not a black box.
|
||||
|
||||
export type CheckKind = "static" | "dynamic" | "inference" | "kb_query";
|
||||
|
||||
export type Severity = "info" | "warn" | "block";
|
||||
|
||||
export interface Claim {
|
||||
// Verbatim phrase that raised the claim — e.g. "Phase 38 shipped",
|
||||
// "verified end-to-end", "works after restart". Used as the "what
|
||||
// does the author assert" input to downstream checks.
|
||||
text: string;
|
||||
// Where it came from. `commit_sha` is the short hash; `location`
|
||||
// is a file:line for in-diff claims, or "pr_body" / "commit_message".
|
||||
commit_sha: string;
|
||||
location: string;
|
||||
// Heuristic rating of how strong the claim is. "green+tested"
|
||||
// is strong; "should work" is weak. Drives sensitivity — stronger
|
||||
// claims get harder-blocked on weak evidence.
|
||||
strength: "weak" | "moderate" | "strong";
|
||||
}
|
||||
|
||||
export interface Finding {
|
||||
check: CheckKind;
|
||||
severity: Severity;
|
||||
claim_text?: string;
|
||||
// Free-form short description: "field added but never read", "no
|
||||
// test covers this code path", "cloud model says placeholder".
|
||||
summary: string;
|
||||
// Concrete evidence: file paths, line numbers, log excerpts, test
|
||||
// output, cloud-model verdict. No handwaving.
|
||||
evidence: string[];
|
||||
}
|
||||
|
||||
export interface Verdict {
|
||||
pr_number: number;
|
||||
head_sha: string;
|
||||
audited_at: string;
|
||||
overall: "approve" | "request_changes" | "block";
|
||||
findings: Finding[];
|
||||
// Real numbers that downstream policy can gate on. e.g. if the
|
||||
// hybrid test produced latency numbers or token counts, they
|
||||
// surface here so /auditor/history is queryable.
|
||||
metrics: Record<string, number>;
|
||||
// Short one-line justification for the `overall` verdict. What
|
||||
// gets posted as the commit-status description in Gitea (max 140
|
||||
// chars) must fit here.
|
||||
one_liner: string;
|
||||
}
|
||||
|
||||
export interface PrSnapshot {
|
||||
number: number;
|
||||
head_sha: string;
|
||||
base_sha: string;
|
||||
title: string;
|
||||
body: string;
|
||||
state: "open" | "closed" | "merged";
|
||||
author: string;
|
||||
// Array of commit messages in the PR (not diffs — those are
|
||||
// fetched on-demand per-check).
|
||||
commits: Array<{ sha: string; message: string; author: string }>;
|
||||
// File paths touched by the PR, with lines-added / lines-removed.
|
||||
files: Array<{ path: string; additions: number; deletions: number }>;
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user