Implements the MVP cutline from the planning artifact: - Phase A: skeleton + CLI dispatch + provider interface + stub model doctor - Phase B: scanner + git probe + 12 static analyzers + reporters + pipeline - Phase B fixtures: clean-repo, insecure-repo, degraded-repo 12 static analyzers per PROMPT.md "Suggested Static Checks For MVP": hardcoded_paths, shell_execution, raw_sql_interpolation, broad_cors, secret_patterns, large_files, todo_comments, missing_tests, env_file_committed, unsafe_file_io, exposed_mutation_endpoint, hardcoded_local_ip. Acceptance gates passing: - B1 (intake produces accurate counts) ✓ - B2 (insecure fixture fires ≥8 distinct check_ids — actually 11/12) ✓ - B3 (clean fixture produces 0 confirmed findings — no false positives) ✓ - B4 (scrum mode produces all 6 required markdown + JSON reports) ✓ - B5 (receipts.json marks degraded phases honestly) ✓ - F (self-review on this repo runs without crashing) ✓ — exit 66 (degraded because Phase C LLM review is hardcoded skipped) Phases C (LLM review), D (validation cross-check), E (memory + diff + rules subcommands) deferred per the cutline. The MVP delivers the evidence-first path; LLM is purely additive. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
337 lines
12 KiB
Go
337 lines
12 KiB
Go
package reporters
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
|
|
"local-review-harness/internal/analyzers"
|
|
)
|
|
|
|
// WriteScrumTest produces reports/latest/scrum-test.md per
|
|
// docs/SCRUM_TEST_TEMPLATE.md. Sections in fixed order so operators
|
|
// can grep section headers reliably.
|
|
func WriteScrumTest(path string, intake RepoIntake, findings []analyzers.Finding, llmDegraded bool) error {
|
|
summary := SummarizeFindings(findings)
|
|
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "# Scrum Test — %s\n\n", filepath.Base(intake.RepoPath))
|
|
fmt.Fprintf(&b, "**Generated:** %s\n", intake.GeneratedAt)
|
|
fmt.Fprintf(&b, "**Branch:** %s · **Commit:** %s\n\n", coalesce(intake.CurrentBranch, "(no git)"), coalesce(intake.LatestCommit, "—"))
|
|
|
|
// Verdict per SCRUM_TEST_TEMPLATE.md — blunt, no soften.
|
|
fmt.Fprintln(&b, "## Verdict")
|
|
fmt.Fprintln(&b)
|
|
fmt.Fprintln(&b, verdict(summary, llmDegraded))
|
|
fmt.Fprintln(&b)
|
|
|
|
// Evidence
|
|
fmt.Fprintln(&b, "## Evidence")
|
|
fmt.Fprintln(&b)
|
|
fmt.Fprintf(&b, "- repo path: `%s`\n", intake.RepoPath)
|
|
fmt.Fprintf(&b, "- file count: %d\n", intake.FileCount)
|
|
if len(intake.LanguageBreakdown) > 0 {
|
|
fmt.Fprintf(&b, "- languages: %s\n", langSummary(intake.LanguageBreakdown))
|
|
}
|
|
fmt.Fprintf(&b, "- dependency manifests: %d (%s)\n", len(intake.DependencyManifests), strings.Join(firstN(intake.DependencyManifests, 5), ", "))
|
|
fmt.Fprintf(&b, "- test files/dirs: %d\n", len(intake.TestManifests))
|
|
if llmDegraded {
|
|
fmt.Fprintln(&b, "- LLM review: **skipped** (Phase C not implemented OR provider unavailable; see model-doctor.json)")
|
|
}
|
|
fmt.Fprintln(&b)
|
|
|
|
// Confirmed
|
|
fmt.Fprintln(&b, "## Confirmed Risks")
|
|
fmt.Fprintln(&b)
|
|
confirmed := filterByStatus(findings, analyzers.StatusConfirmed)
|
|
if len(confirmed) == 0 {
|
|
fmt.Fprintln(&b, "_No confirmed risks at static-scan level. (LLM review may surface more.)_")
|
|
} else {
|
|
writeFindingTable(&b, confirmed)
|
|
}
|
|
fmt.Fprintln(&b)
|
|
|
|
// Suspected
|
|
fmt.Fprintln(&b, "## Suspected Risks")
|
|
fmt.Fprintln(&b)
|
|
suspected := filterByStatus(findings, analyzers.StatusSuspected)
|
|
if len(suspected) == 0 {
|
|
fmt.Fprintln(&b, "_None._")
|
|
} else {
|
|
fmt.Fprintf(&b, "Each entry is a static-scan regex hit awaiting validation (Phase D / LLM cross-check).\n\n")
|
|
writeFindingTable(&b, suspected)
|
|
}
|
|
fmt.Fprintln(&b)
|
|
|
|
// Blocked
|
|
fmt.Fprintln(&b, "## Blocked Checks")
|
|
fmt.Fprintln(&b)
|
|
if llmDegraded {
|
|
fmt.Fprintln(&b, "- LLM review (Phase 2 in REVIEW_PIPELINE.md). Reason: provider unavailable or stub. Next command: `review-harness model doctor`")
|
|
} else {
|
|
fmt.Fprintln(&b, "_None._")
|
|
}
|
|
fmt.Fprintln(&b)
|
|
|
|
// Sprint backlog (per SCRUM_TEST_TEMPLATE.md fixed shape)
|
|
fmt.Fprintln(&b, "## Sprint Backlog")
|
|
fmt.Fprintln(&b)
|
|
writeSprintBacklog(&b, summary)
|
|
fmt.Fprintln(&b)
|
|
|
|
// Acceptance gates
|
|
fmt.Fprintln(&b, "## Acceptance Gates")
|
|
fmt.Fprintln(&b)
|
|
writeAcceptanceGates(&b, summary)
|
|
fmt.Fprintln(&b)
|
|
|
|
// Next commands
|
|
fmt.Fprintln(&b, "## Next Commands")
|
|
fmt.Fprintln(&b)
|
|
writeNextCommands(&b, summary, llmDegraded, intake.RepoPath)
|
|
|
|
return os.WriteFile(path, []byte(b.String()), 0o644)
|
|
}
|
|
|
|
// WriteRiskRegister produces reports/latest/risk-register.md.
|
|
func WriteRiskRegister(path string, findings []analyzers.Finding) error {
|
|
var b strings.Builder
|
|
fmt.Fprintln(&b, "# Risk Register")
|
|
fmt.Fprintln(&b)
|
|
fmt.Fprintln(&b, "Findings ranked by severity. `Suspected` rows haven't been validated yet (Phase D).")
|
|
fmt.Fprintln(&b)
|
|
if len(findings) == 0 {
|
|
fmt.Fprintln(&b, "_No findings._")
|
|
return os.WriteFile(path, []byte(b.String()), 0o644)
|
|
}
|
|
|
|
sorted := sortBySeverity(findings)
|
|
fmt.Fprintln(&b, "| ID | Severity | Status | File | Line | Title |")
|
|
fmt.Fprintln(&b, "|---|---|---|---|---|---|")
|
|
for _, f := range sorted {
|
|
fmt.Fprintf(&b, "| `%s` | %s | %s | `%s` | %s | %s |\n",
|
|
f.ID, f.Severity, f.Status, mdEscape(f.File), coalesce(f.LineHint, "—"), mdEscape(f.Title))
|
|
}
|
|
return os.WriteFile(path, []byte(b.String()), 0o644)
|
|
}
|
|
|
|
// WriteClaimCoverage produces reports/latest/claim-coverage-table.md.
|
|
// Phase B emits the table shape per REPORT_SCHEMA.md but the LLM-side
|
|
// claims aren't generated until Phase C.
|
|
func WriteClaimCoverage(path string, findings []analyzers.Finding) error {
|
|
var b strings.Builder
|
|
fmt.Fprintln(&b, "# Claim Coverage Table")
|
|
fmt.Fprintln(&b)
|
|
fmt.Fprintln(&b, "Each row is a finding paired with whether existing tests cover the affected area.")
|
|
fmt.Fprintln(&b, "Phase B emits this shape; LLM-side claim generation lands in Phase C.")
|
|
fmt.Fprintln(&b)
|
|
fmt.Fprintln(&b, "| Claim | Code Location | Existing Test | Missing Test | Risk |")
|
|
fmt.Fprintln(&b, "|---|---|---|---|---|")
|
|
if len(findings) == 0 {
|
|
fmt.Fprintln(&b, "| _no claims yet_ | — | — | — | — |")
|
|
}
|
|
for _, f := range findings {
|
|
fmt.Fprintf(&b, "| %s | `%s:%s` | _unknown_ | _likely_ | %s |\n",
|
|
mdEscape(f.Title), mdEscape(f.File), coalesce(f.LineHint, "?"), f.Severity)
|
|
}
|
|
return os.WriteFile(path, []byte(b.String()), 0o644)
|
|
}
|
|
|
|
// WriteSprintBacklog produces reports/latest/sprint-backlog.md.
|
|
func WriteSprintBacklog(path string, summary FindingsSummary) error {
|
|
var b strings.Builder
|
|
fmt.Fprintln(&b, "# Sprint Backlog")
|
|
fmt.Fprintln(&b)
|
|
writeSprintBacklog(&b, summary)
|
|
return os.WriteFile(path, []byte(b.String()), 0o644)
|
|
}
|
|
|
|
// WriteAcceptanceGates produces reports/latest/acceptance-gates.md.
|
|
func WriteAcceptanceGates(path string, summary FindingsSummary) error {
|
|
var b strings.Builder
|
|
fmt.Fprintln(&b, "# Acceptance Gates")
|
|
fmt.Fprintln(&b)
|
|
writeAcceptanceGates(&b, summary)
|
|
return os.WriteFile(path, []byte(b.String()), 0o644)
|
|
}
|
|
|
|
// === helpers ===
|
|
|
|
func verdict(s FindingsSummary, llmDegraded bool) string {
|
|
switch {
|
|
case s.Critical > 0:
|
|
return "**blocked** — critical-severity finding present. See Confirmed Risks; rotate any leaked credentials, then re-run."
|
|
case s.High > 0 && s.Confirmed > 0:
|
|
return "**prototype-ready** — confirmed high-severity findings need fixes before production deploy."
|
|
case s.High > 0:
|
|
return "**prototype-ready** — high-severity findings are suspected (not confirmed); validation pass (Phase D) or LLM review (Phase C) needed before promoting verdict."
|
|
case s.Total == 0 && !llmDegraded:
|
|
return "**production-ready** — static scan + LLM review found no issues. Re-validate after every wave."
|
|
case s.Total == 0:
|
|
return "**prototype-ready** — static scan clean; LLM review degraded so production status not certified."
|
|
default:
|
|
return "**demo-only** — only low/medium-severity findings, mostly suspected. Reasonable to demo; production deploy needs the validator pass + missing-tests gap closed."
|
|
}
|
|
}
|
|
|
|
func writeSprintBacklog(b *strings.Builder, s FindingsSummary) {
|
|
// Per SCRUM_TEST_TEMPLATE.md fixed format.
|
|
fmt.Fprintln(b, "**Sprint 0 — Reproducibility Gate**")
|
|
fmt.Fprintln(b)
|
|
fmt.Fprintln(b, "- Wire `just verify` (or equivalent) to run the static checks before every commit/PR.")
|
|
fmt.Fprintln(b, "- Add a CI step that fails on `critical` findings.")
|
|
if s.Total > 0 {
|
|
fmt.Fprintf(b, "- Triage the %d findings emitted by this run; mark each as accepted / blocking / dismiss-with-reason.\n", s.Total)
|
|
}
|
|
fmt.Fprintln(b)
|
|
|
|
fmt.Fprintln(b, "**Sprint 1 — Trust Boundary Gate**")
|
|
fmt.Fprintln(b)
|
|
if s.High > 0 || s.Critical > 0 {
|
|
fmt.Fprintln(b, "- Resolve every `critical` and `high` finding before non-loopback deploy.")
|
|
}
|
|
fmt.Fprintln(b, "- Confirm auth posture for any mutation endpoint flagged as exposed.")
|
|
fmt.Fprintln(b, "- Replace raw SQL interpolation with parameterized queries.")
|
|
fmt.Fprintln(b)
|
|
|
|
fmt.Fprintln(b, "**Sprint 2 — Memory Correctness Gate**")
|
|
fmt.Fprintln(b)
|
|
fmt.Fprintln(b, "- (Phase E) Wire append-only `.memory/` writes for known-risks + fixed-patterns.")
|
|
fmt.Fprintln(b, "- Add a regression test that re-runs the harness and asserts no regression in confirmed-finding count.")
|
|
fmt.Fprintln(b)
|
|
|
|
fmt.Fprintln(b, "**Sprint 3 — Agent Loop Reality Gate**")
|
|
fmt.Fprintln(b)
|
|
fmt.Fprintln(b, "- (Phase C) Wire local-Ollama LLM review.")
|
|
fmt.Fprintln(b, "- (Phase D) Validator pass cross-checks every LLM finding against repo evidence.")
|
|
fmt.Fprintln(b)
|
|
|
|
fmt.Fprintln(b, "**Sprint 4 — Deployment Gate**")
|
|
fmt.Fprintln(b)
|
|
fmt.Fprintln(b, "- Ship the harness as a single static binary (`go build -o review-harness`).")
|
|
fmt.Fprintln(b, "- Document operator runbook (model setup, profile editing, output retention).")
|
|
}
|
|
|
|
func writeAcceptanceGates(b *strings.Builder, s FindingsSummary) {
|
|
fmt.Fprintln(b, "Each gate must be testable. Format: command + verifiable post-condition.")
|
|
fmt.Fprintln(b)
|
|
fmt.Fprintln(b, "1. **Reproducibility:** `review-harness repo .` exits 0; `reports/latest/repo-intake.json` exists with non-zero `file_count`.")
|
|
fmt.Fprintln(b, "2. **No false positives on a clean fixture:** `review-harness repo tests/fixtures/clean-repo` produces zero `confirmed` findings.")
|
|
fmt.Fprintln(b, "3. **Every documented static check fires on the insecure fixture:** `jq '[.findings[] | .check_id] | unique | length' reports/latest/static-findings.json` ≥ 8.")
|
|
fmt.Fprintln(b, "4. **Receipts are honest about degraded phases:** `jq '[.phases[] | select(.status == \"degraded\")]' reports/latest/receipts.json` lists every skipped/stubbed phase.")
|
|
if s.Critical > 0 {
|
|
fmt.Fprintln(b, "5. **Critical findings block production deploy:** at least one critical finding is currently present; resolve before deploy.")
|
|
}
|
|
}
|
|
|
|
func writeNextCommands(b *strings.Builder, s FindingsSummary, llmDegraded bool, repoPath string) {
|
|
if s.Critical > 0 {
|
|
fmt.Fprintln(b, "1. Open the risk register: `cat reports/latest/risk-register.md`")
|
|
fmt.Fprintln(b, "2. Triage every `critical` finding; rotate any leaked credentials immediately.")
|
|
}
|
|
if llmDegraded {
|
|
fmt.Fprintln(b, "- Probe the model provider: `review-harness model doctor`")
|
|
}
|
|
fmt.Fprintf(b, "- Re-run after fixes: `review-harness repo %s`\n", repoPath)
|
|
fmt.Fprintf(b, "- Generate the full Scrum bundle: `review-harness scrum %s`\n", repoPath)
|
|
}
|
|
|
|
func filterByStatus(findings []analyzers.Finding, st analyzers.Status) []analyzers.Finding {
|
|
out := []analyzers.Finding{}
|
|
for _, f := range findings {
|
|
if f.Status == st {
|
|
out = append(out, f)
|
|
}
|
|
}
|
|
return sortBySeverity(out)
|
|
}
|
|
|
|
// severityRank used for sorting tables — critical first.
|
|
func severityRank(s analyzers.Severity) int {
|
|
switch s {
|
|
case analyzers.SeverityCritical:
|
|
return 0
|
|
case analyzers.SeverityHigh:
|
|
return 1
|
|
case analyzers.SeverityMedium:
|
|
return 2
|
|
case analyzers.SeverityLow:
|
|
return 3
|
|
}
|
|
return 4
|
|
}
|
|
|
|
func sortBySeverity(findings []analyzers.Finding) []analyzers.Finding {
|
|
out := make([]analyzers.Finding, len(findings))
|
|
copy(out, findings)
|
|
sort.SliceStable(out, func(i, j int) bool {
|
|
ri, rj := severityRank(out[i].Severity), severityRank(out[j].Severity)
|
|
if ri != rj {
|
|
return ri < rj
|
|
}
|
|
if out[i].File != out[j].File {
|
|
return out[i].File < out[j].File
|
|
}
|
|
return out[i].LineHint < out[j].LineHint
|
|
})
|
|
return out
|
|
}
|
|
|
|
func writeFindingTable(b *strings.Builder, findings []analyzers.Finding) {
|
|
fmt.Fprintln(b, "| Severity | File:Line | Title | Evidence |")
|
|
fmt.Fprintln(b, "|---|---|---|---|")
|
|
for _, f := range findings {
|
|
loc := mdEscape(f.File)
|
|
if f.LineHint != "" {
|
|
loc = fmt.Sprintf("%s:%s", mdEscape(f.File), f.LineHint)
|
|
}
|
|
fmt.Fprintf(b, "| %s | `%s` | %s | `%s` |\n",
|
|
f.Severity, loc, mdEscape(f.Title), mdEscape(f.Evidence))
|
|
}
|
|
}
|
|
|
|
func mdEscape(s string) string {
|
|
s = strings.ReplaceAll(s, "|", "\\|")
|
|
s = strings.ReplaceAll(s, "\n", " ")
|
|
if len(s) > 120 {
|
|
s = s[:120] + "…"
|
|
}
|
|
return s
|
|
}
|
|
|
|
func coalesce(s, fallback string) string {
|
|
if s == "" {
|
|
return fallback
|
|
}
|
|
return s
|
|
}
|
|
|
|
func langSummary(m map[string]int) string {
|
|
type kv struct {
|
|
k string
|
|
v int
|
|
}
|
|
pairs := make([]kv, 0, len(m))
|
|
for k, v := range m {
|
|
pairs = append(pairs, kv{k, v})
|
|
}
|
|
sort.Slice(pairs, func(i, j int) bool { return pairs[i].v > pairs[j].v })
|
|
if len(pairs) > 5 {
|
|
pairs = pairs[:5]
|
|
}
|
|
parts := make([]string, 0, len(pairs))
|
|
for _, p := range pairs {
|
|
parts = append(parts, fmt.Sprintf("%s (%d)", p.k, p.v))
|
|
}
|
|
return strings.Join(parts, ", ")
|
|
}
|
|
|
|
func firstN(s []string, n int) []string {
|
|
if len(s) <= n {
|
|
return s
|
|
}
|
|
return s[:n]
|
|
}
|