Claude (review-harness setup) f3ee4722a8 Phase A + B (MVP) — local review harness
Implements the MVP cutline from the planning artifact:
- Phase A: skeleton + CLI dispatch + provider interface + stub model doctor
- Phase B: scanner + git probe + 12 static analyzers + reporters + pipeline
- Phase B fixtures: clean-repo, insecure-repo, degraded-repo

12 static analyzers per PROMPT.md "Suggested Static Checks For MVP":
hardcoded_paths, shell_execution, raw_sql_interpolation, broad_cors,
secret_patterns, large_files, todo_comments, missing_tests,
env_file_committed, unsafe_file_io, exposed_mutation_endpoint,
hardcoded_local_ip.

Acceptance gates passing:
- B1 (intake produces accurate counts) ✓
- B2 (insecure fixture fires ≥8 distinct check_ids — actually 11/12) ✓
- B3 (clean fixture produces 0 confirmed findings — no false positives) ✓
- B4 (scrum mode produces all 6 required markdown + JSON reports) ✓
- B5 (receipts.json marks degraded phases honestly) ✓
- F  (self-review on this repo runs without crashing) ✓ — exit 66 (degraded
  because Phase C LLM review is hardcoded skipped)

Phases C (LLM review), D (validation cross-check), E (memory + diff +
rules subcommands) deferred per the cutline. The MVP delivers the
evidence-first path; LLM is purely additive.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-30 00:56:02 -05:00

337 lines
12 KiB
Go

package reporters
import (
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"local-review-harness/internal/analyzers"
)
// WriteScrumTest produces reports/latest/scrum-test.md per
// docs/SCRUM_TEST_TEMPLATE.md. Sections in fixed order so operators
// can grep section headers reliably.
func WriteScrumTest(path string, intake RepoIntake, findings []analyzers.Finding, llmDegraded bool) error {
summary := SummarizeFindings(findings)
var b strings.Builder
fmt.Fprintf(&b, "# Scrum Test — %s\n\n", filepath.Base(intake.RepoPath))
fmt.Fprintf(&b, "**Generated:** %s\n", intake.GeneratedAt)
fmt.Fprintf(&b, "**Branch:** %s · **Commit:** %s\n\n", coalesce(intake.CurrentBranch, "(no git)"), coalesce(intake.LatestCommit, "—"))
// Verdict per SCRUM_TEST_TEMPLATE.md — blunt, no soften.
fmt.Fprintln(&b, "## Verdict")
fmt.Fprintln(&b)
fmt.Fprintln(&b, verdict(summary, llmDegraded))
fmt.Fprintln(&b)
// Evidence
fmt.Fprintln(&b, "## Evidence")
fmt.Fprintln(&b)
fmt.Fprintf(&b, "- repo path: `%s`\n", intake.RepoPath)
fmt.Fprintf(&b, "- file count: %d\n", intake.FileCount)
if len(intake.LanguageBreakdown) > 0 {
fmt.Fprintf(&b, "- languages: %s\n", langSummary(intake.LanguageBreakdown))
}
fmt.Fprintf(&b, "- dependency manifests: %d (%s)\n", len(intake.DependencyManifests), strings.Join(firstN(intake.DependencyManifests, 5), ", "))
fmt.Fprintf(&b, "- test files/dirs: %d\n", len(intake.TestManifests))
if llmDegraded {
fmt.Fprintln(&b, "- LLM review: **skipped** (Phase C not implemented OR provider unavailable; see model-doctor.json)")
}
fmt.Fprintln(&b)
// Confirmed
fmt.Fprintln(&b, "## Confirmed Risks")
fmt.Fprintln(&b)
confirmed := filterByStatus(findings, analyzers.StatusConfirmed)
if len(confirmed) == 0 {
fmt.Fprintln(&b, "_No confirmed risks at static-scan level. (LLM review may surface more.)_")
} else {
writeFindingTable(&b, confirmed)
}
fmt.Fprintln(&b)
// Suspected
fmt.Fprintln(&b, "## Suspected Risks")
fmt.Fprintln(&b)
suspected := filterByStatus(findings, analyzers.StatusSuspected)
if len(suspected) == 0 {
fmt.Fprintln(&b, "_None._")
} else {
fmt.Fprintf(&b, "Each entry is a static-scan regex hit awaiting validation (Phase D / LLM cross-check).\n\n")
writeFindingTable(&b, suspected)
}
fmt.Fprintln(&b)
// Blocked
fmt.Fprintln(&b, "## Blocked Checks")
fmt.Fprintln(&b)
if llmDegraded {
fmt.Fprintln(&b, "- LLM review (Phase 2 in REVIEW_PIPELINE.md). Reason: provider unavailable or stub. Next command: `review-harness model doctor`")
} else {
fmt.Fprintln(&b, "_None._")
}
fmt.Fprintln(&b)
// Sprint backlog (per SCRUM_TEST_TEMPLATE.md fixed shape)
fmt.Fprintln(&b, "## Sprint Backlog")
fmt.Fprintln(&b)
writeSprintBacklog(&b, summary)
fmt.Fprintln(&b)
// Acceptance gates
fmt.Fprintln(&b, "## Acceptance Gates")
fmt.Fprintln(&b)
writeAcceptanceGates(&b, summary)
fmt.Fprintln(&b)
// Next commands
fmt.Fprintln(&b, "## Next Commands")
fmt.Fprintln(&b)
writeNextCommands(&b, summary, llmDegraded, intake.RepoPath)
return os.WriteFile(path, []byte(b.String()), 0o644)
}
// WriteRiskRegister produces reports/latest/risk-register.md.
func WriteRiskRegister(path string, findings []analyzers.Finding) error {
var b strings.Builder
fmt.Fprintln(&b, "# Risk Register")
fmt.Fprintln(&b)
fmt.Fprintln(&b, "Findings ranked by severity. `Suspected` rows haven't been validated yet (Phase D).")
fmt.Fprintln(&b)
if len(findings) == 0 {
fmt.Fprintln(&b, "_No findings._")
return os.WriteFile(path, []byte(b.String()), 0o644)
}
sorted := sortBySeverity(findings)
fmt.Fprintln(&b, "| ID | Severity | Status | File | Line | Title |")
fmt.Fprintln(&b, "|---|---|---|---|---|---|")
for _, f := range sorted {
fmt.Fprintf(&b, "| `%s` | %s | %s | `%s` | %s | %s |\n",
f.ID, f.Severity, f.Status, mdEscape(f.File), coalesce(f.LineHint, "—"), mdEscape(f.Title))
}
return os.WriteFile(path, []byte(b.String()), 0o644)
}
// WriteClaimCoverage produces reports/latest/claim-coverage-table.md.
// Phase B emits the table shape per REPORT_SCHEMA.md but the LLM-side
// claims aren't generated until Phase C.
func WriteClaimCoverage(path string, findings []analyzers.Finding) error {
var b strings.Builder
fmt.Fprintln(&b, "# Claim Coverage Table")
fmt.Fprintln(&b)
fmt.Fprintln(&b, "Each row is a finding paired with whether existing tests cover the affected area.")
fmt.Fprintln(&b, "Phase B emits this shape; LLM-side claim generation lands in Phase C.")
fmt.Fprintln(&b)
fmt.Fprintln(&b, "| Claim | Code Location | Existing Test | Missing Test | Risk |")
fmt.Fprintln(&b, "|---|---|---|---|---|")
if len(findings) == 0 {
fmt.Fprintln(&b, "| _no claims yet_ | — | — | — | — |")
}
for _, f := range findings {
fmt.Fprintf(&b, "| %s | `%s:%s` | _unknown_ | _likely_ | %s |\n",
mdEscape(f.Title), mdEscape(f.File), coalesce(f.LineHint, "?"), f.Severity)
}
return os.WriteFile(path, []byte(b.String()), 0o644)
}
// WriteSprintBacklog produces reports/latest/sprint-backlog.md.
func WriteSprintBacklog(path string, summary FindingsSummary) error {
var b strings.Builder
fmt.Fprintln(&b, "# Sprint Backlog")
fmt.Fprintln(&b)
writeSprintBacklog(&b, summary)
return os.WriteFile(path, []byte(b.String()), 0o644)
}
// WriteAcceptanceGates produces reports/latest/acceptance-gates.md.
func WriteAcceptanceGates(path string, summary FindingsSummary) error {
var b strings.Builder
fmt.Fprintln(&b, "# Acceptance Gates")
fmt.Fprintln(&b)
writeAcceptanceGates(&b, summary)
return os.WriteFile(path, []byte(b.String()), 0o644)
}
// === helpers ===
func verdict(s FindingsSummary, llmDegraded bool) string {
switch {
case s.Critical > 0:
return "**blocked** — critical-severity finding present. See Confirmed Risks; rotate any leaked credentials, then re-run."
case s.High > 0 && s.Confirmed > 0:
return "**prototype-ready** — confirmed high-severity findings need fixes before production deploy."
case s.High > 0:
return "**prototype-ready** — high-severity findings are suspected (not confirmed); validation pass (Phase D) or LLM review (Phase C) needed before promoting verdict."
case s.Total == 0 && !llmDegraded:
return "**production-ready** — static scan + LLM review found no issues. Re-validate after every wave."
case s.Total == 0:
return "**prototype-ready** — static scan clean; LLM review degraded so production status not certified."
default:
return "**demo-only** — only low/medium-severity findings, mostly suspected. Reasonable to demo; production deploy needs the validator pass + missing-tests gap closed."
}
}
func writeSprintBacklog(b *strings.Builder, s FindingsSummary) {
// Per SCRUM_TEST_TEMPLATE.md fixed format.
fmt.Fprintln(b, "**Sprint 0 — Reproducibility Gate**")
fmt.Fprintln(b)
fmt.Fprintln(b, "- Wire `just verify` (or equivalent) to run the static checks before every commit/PR.")
fmt.Fprintln(b, "- Add a CI step that fails on `critical` findings.")
if s.Total > 0 {
fmt.Fprintf(b, "- Triage the %d findings emitted by this run; mark each as accepted / blocking / dismiss-with-reason.\n", s.Total)
}
fmt.Fprintln(b)
fmt.Fprintln(b, "**Sprint 1 — Trust Boundary Gate**")
fmt.Fprintln(b)
if s.High > 0 || s.Critical > 0 {
fmt.Fprintln(b, "- Resolve every `critical` and `high` finding before non-loopback deploy.")
}
fmt.Fprintln(b, "- Confirm auth posture for any mutation endpoint flagged as exposed.")
fmt.Fprintln(b, "- Replace raw SQL interpolation with parameterized queries.")
fmt.Fprintln(b)
fmt.Fprintln(b, "**Sprint 2 — Memory Correctness Gate**")
fmt.Fprintln(b)
fmt.Fprintln(b, "- (Phase E) Wire append-only `.memory/` writes for known-risks + fixed-patterns.")
fmt.Fprintln(b, "- Add a regression test that re-runs the harness and asserts no regression in confirmed-finding count.")
fmt.Fprintln(b)
fmt.Fprintln(b, "**Sprint 3 — Agent Loop Reality Gate**")
fmt.Fprintln(b)
fmt.Fprintln(b, "- (Phase C) Wire local-Ollama LLM review.")
fmt.Fprintln(b, "- (Phase D) Validator pass cross-checks every LLM finding against repo evidence.")
fmt.Fprintln(b)
fmt.Fprintln(b, "**Sprint 4 — Deployment Gate**")
fmt.Fprintln(b)
fmt.Fprintln(b, "- Ship the harness as a single static binary (`go build -o review-harness`).")
fmt.Fprintln(b, "- Document operator runbook (model setup, profile editing, output retention).")
}
func writeAcceptanceGates(b *strings.Builder, s FindingsSummary) {
fmt.Fprintln(b, "Each gate must be testable. Format: command + verifiable post-condition.")
fmt.Fprintln(b)
fmt.Fprintln(b, "1. **Reproducibility:** `review-harness repo .` exits 0; `reports/latest/repo-intake.json` exists with non-zero `file_count`.")
fmt.Fprintln(b, "2. **No false positives on a clean fixture:** `review-harness repo tests/fixtures/clean-repo` produces zero `confirmed` findings.")
fmt.Fprintln(b, "3. **Every documented static check fires on the insecure fixture:** `jq '[.findings[] | .check_id] | unique | length' reports/latest/static-findings.json` ≥ 8.")
fmt.Fprintln(b, "4. **Receipts are honest about degraded phases:** `jq '[.phases[] | select(.status == \"degraded\")]' reports/latest/receipts.json` lists every skipped/stubbed phase.")
if s.Critical > 0 {
fmt.Fprintln(b, "5. **Critical findings block production deploy:** at least one critical finding is currently present; resolve before deploy.")
}
}
func writeNextCommands(b *strings.Builder, s FindingsSummary, llmDegraded bool, repoPath string) {
if s.Critical > 0 {
fmt.Fprintln(b, "1. Open the risk register: `cat reports/latest/risk-register.md`")
fmt.Fprintln(b, "2. Triage every `critical` finding; rotate any leaked credentials immediately.")
}
if llmDegraded {
fmt.Fprintln(b, "- Probe the model provider: `review-harness model doctor`")
}
fmt.Fprintf(b, "- Re-run after fixes: `review-harness repo %s`\n", repoPath)
fmt.Fprintf(b, "- Generate the full Scrum bundle: `review-harness scrum %s`\n", repoPath)
}
func filterByStatus(findings []analyzers.Finding, st analyzers.Status) []analyzers.Finding {
out := []analyzers.Finding{}
for _, f := range findings {
if f.Status == st {
out = append(out, f)
}
}
return sortBySeverity(out)
}
// severityRank used for sorting tables — critical first.
func severityRank(s analyzers.Severity) int {
switch s {
case analyzers.SeverityCritical:
return 0
case analyzers.SeverityHigh:
return 1
case analyzers.SeverityMedium:
return 2
case analyzers.SeverityLow:
return 3
}
return 4
}
func sortBySeverity(findings []analyzers.Finding) []analyzers.Finding {
out := make([]analyzers.Finding, len(findings))
copy(out, findings)
sort.SliceStable(out, func(i, j int) bool {
ri, rj := severityRank(out[i].Severity), severityRank(out[j].Severity)
if ri != rj {
return ri < rj
}
if out[i].File != out[j].File {
return out[i].File < out[j].File
}
return out[i].LineHint < out[j].LineHint
})
return out
}
func writeFindingTable(b *strings.Builder, findings []analyzers.Finding) {
fmt.Fprintln(b, "| Severity | File:Line | Title | Evidence |")
fmt.Fprintln(b, "|---|---|---|---|")
for _, f := range findings {
loc := mdEscape(f.File)
if f.LineHint != "" {
loc = fmt.Sprintf("%s:%s", mdEscape(f.File), f.LineHint)
}
fmt.Fprintf(b, "| %s | `%s` | %s | `%s` |\n",
f.Severity, loc, mdEscape(f.Title), mdEscape(f.Evidence))
}
}
func mdEscape(s string) string {
s = strings.ReplaceAll(s, "|", "\\|")
s = strings.ReplaceAll(s, "\n", " ")
if len(s) > 120 {
s = s[:120] + "…"
}
return s
}
func coalesce(s, fallback string) string {
if s == "" {
return fallback
}
return s
}
func langSummary(m map[string]int) string {
type kv struct {
k string
v int
}
pairs := make([]kv, 0, len(m))
for k, v := range m {
pairs = append(pairs, kv{k, v})
}
sort.Slice(pairs, func(i, j int) bool { return pairs[i].v > pairs[j].v })
if len(pairs) > 5 {
pairs = pairs[:5]
}
parts := make([]string, 0, len(pairs))
for _, p := range pairs {
parts = append(parts, fmt.Sprintf("%s (%d)", p.k, p.v))
}
return strings.Join(parts, ", ")
}
func firstN(s []string, n int) []string {
if len(s) <= n {
return s
}
return s[:n]
}