diff --git a/cmd/review-harness/main.go b/cmd/review-harness/main.go index 6244ef6..a8c9ef7 100644 --- a/cmd/review-harness/main.go +++ b/cmd/review-harness/main.go @@ -71,7 +71,7 @@ func main() { } func usage() { - fmt.Fprintln(os.Stderr, `review-harness — local-first code review + fmt.Fprint(os.Stderr, `review-harness — local-first code review Usage: review-harness repo full-repo review (MVP) @@ -86,5 +86,6 @@ Common flags (per subcommand): --review-profile YAML; defaults applied if omitted --model-profile YAML; defaults applied if omitted --output-dir override review-profile output dir + --enable-llm also run local-Ollama LLM review (Phase C) `) } diff --git a/internal/cli/cli.go b/internal/cli/cli.go index e5ca67a..4db081a 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -17,17 +17,19 @@ import ( "local-review-harness/internal/llm" ) -// commonFlags wires the three flags every subcommand accepts. +// commonFlags wires the flags every subcommand accepts. type commonFlags struct { reviewProfilePath string modelProfilePath string outputDir string + enableLLM bool } func bindCommonFlags(fs *flag.FlagSet, cf *commonFlags) { fs.StringVar(&cf.reviewProfilePath, "review-profile", "", "review profile YAML (defaults applied if empty)") fs.StringVar(&cf.modelProfilePath, "model-profile", "", "model profile YAML (defaults applied if empty)") fs.StringVar(&cf.outputDir, "output-dir", "", "override review profile output dir") + fs.BoolVar(&cf.enableLLM, "enable-llm", false, "Phase C: also run local-Ollama LLM review (default off — static-only)") } // resolveOutputDir picks the output dir from flag > review profile > @@ -64,10 +66,6 @@ func writeJSON(path string, v any) error { // nowUTC returns ISO-8601 UTC for receipt timestamps. func nowUTC() string { return time.Now().UTC().Format(time.RFC3339Nano) } -// Stub-only sentinel — Phase C replaces this with real Ollama provider. -// Phase A keeps the pipeline runnable end-to-end with a degraded-status -// model-doctor JSON. -func nilProvider() llm.Provider { return nil } // Repo runs Phase 0 (intake) + Phase 1 (static scan) + Phase 4 // (report gen). Phase B implements the analyzers + scanner; Phase @@ -134,20 +132,35 @@ func ModelDoctor(args []string) int { outDir = rp.Reports.OutputDir } - // Phase A: stub. Phase C swaps in a real probe. + // Phase C: real Ollama probe. Provider's HealthCheck does the + // actual work; we package the result into the shape REPORT_SCHEMA.md + // documents. status="ok" iff server up + at least one named + // model loaded + basic prompt + json mode all green. + prov := llm.NewOllama(mp.BaseURL, time.Duration(mp.TimeoutSeconds)*time.Second) + hctx, cancel := context.WithTimeout(context.Background(), time.Duration(mp.TimeoutSeconds)*time.Second) + defer cancel() + hs := prov.HealthCheck(hctx, mp.Model, mp.FallbackModel) + + status := "ok" + if !hs.ServerAvailable { + status = "failed" + } else if !hs.BasicPromptOK || !hs.JSONModeOK || (!hs.PrimaryModelAvailable && !hs.FallbackModelAvailable) { + status = "degraded" + } + doc := map[string]any{ "provider": mp.Provider, "base_url": mp.BaseURL, "primary_model": mp.Model, "fallback_model": mp.FallbackModel, - "server_available": false, - "primary_model_available": false, - "fallback_model_available": false, - "basic_prompt_ok": false, - "json_mode_ok": false, + "server_available": hs.ServerAvailable, + "primary_model_available": hs.PrimaryModelAvailable, + "fallback_model_available": hs.FallbackModelAvailable, + "basic_prompt_ok": hs.BasicPromptOK, + "json_mode_ok": hs.JSONModeOK, "timeout_seconds": mp.TimeoutSeconds, - "status": "degraded", - "errors": []string{"phase A stub: real Ollama probe lands in Phase C"}, + "status": status, + "errors": hs.Errors, "generated_at": nowUTC(), } out := filepath.Join(outDir, "model-doctor.json") @@ -156,5 +169,12 @@ func ModelDoctor(args []string) int { return 65 } fmt.Println(out) - return 66 // degraded exit code + switch status { + case "ok": + return 0 + case "degraded": + return 66 + default: + return 65 + } } diff --git a/internal/cli/repo.go b/internal/cli/repo.go index c293ea7..bc28770 100644 --- a/internal/cli/repo.go +++ b/internal/cli/repo.go @@ -37,6 +37,7 @@ func runRepo(ctx context.Context, repoPath string, cf commonFlags) int { ModelProfile: mp, OutputDir: outDir, EmitScrum: false, + EnableLLM: cf.enableLLM, }) if err != nil { fmt.Fprintln(os.Stderr, "pipeline:", err) @@ -71,6 +72,7 @@ func runScrum(ctx context.Context, repoPath string, cf commonFlags) int { ModelProfile: mp, OutputDir: outDir, EmitScrum: true, + EnableLLM: cf.enableLLM, }) if err != nil { fmt.Fprintln(os.Stderr, "pipeline:", err) diff --git a/internal/llm/ollama.go b/internal/llm/ollama.go new file mode 100644 index 0000000..eb54ebb --- /dev/null +++ b/internal/llm/ollama.go @@ -0,0 +1,235 @@ +// Ollama provider — local-first per PROMPT.md. +// +// HealthCheck: probes /api/tags (server up + model list) + a 1-token +// completion + a strict-JSON probe. Used by `model doctor`. +// +// Complete + CompleteJSON: POST /api/chat with stream=false. JSON +// mode uses Ollama's native `format: "json"` — newer Ollama versions +// also accept a JSON Schema there but format=json is the lowest- +// common-denominator that works back to 0.4. +// +// `think: false` is set for ALL completions per the Lakehouse-Go +// 2026-04-30 finding: qwen3.5:latest and qwen3:latest are reasoning- +// capable but the inner-loop hot path wants direct answers, not +// `` traces consuming the token budget. Callers that NEED +// reasoning override via opts (Phase F+, not yet wired). +package llm + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// OllamaProvider is the concrete impl. Stateless; safe for concurrent +// use (the http.Client handles connection pooling). +type OllamaProvider struct { + baseURL string + httpClient *http.Client +} + +// NewOllama returns a provider pointed at baseURL. Empty baseURL +// defaults to http://localhost:11434. timeout 0 → 120s (matches +// model-profile default). +func NewOllama(baseURL string, timeout time.Duration) *OllamaProvider { + if baseURL == "" { + baseURL = "http://localhost:11434" + } + if timeout == 0 { + timeout = 120 * time.Second + } + return &OllamaProvider{ + baseURL: strings.TrimRight(baseURL, "/"), + httpClient: &http.Client{Timeout: timeout}, + } +} + +func (o *OllamaProvider) Name() string { return "ollama" } + +// HealthCheck runs the 5 probes documented in REPORT_SCHEMA.md +// model-doctor.json shape: +// - server_available: GET /api/tags returns 2xx +// - primary_model_available: name appears in tag list +// - fallback_model_available: name appears in tag list +// - basic_prompt_ok: a 5-token "reply OK" round-trips +// - json_mode_ok: a JSON probe parses cleanly +// +// Errors surface in HealthStatus.Errors as human-readable strings +// (no stack trace shape — operators run this from a shell). +func (o *OllamaProvider) HealthCheck(ctx context.Context, primary, fallback string) HealthStatus { + st := HealthStatus{Errors: []string{}} + + // 1. Server availability + model list + tags, err := o.listTags(ctx) + if err != nil { + st.Errors = append(st.Errors, "list models: "+err.Error()) + return st + } + st.ServerAvailable = true + + loaded := map[string]bool{} + for _, t := range tags { + loaded[t] = true + } + st.PrimaryModelAvailable = primary != "" && loaded[primary] + st.FallbackModelAvailable = fallback != "" && loaded[fallback] + + // Pick the model we'll use for the live probes — primary if + // loaded, else fallback, else the first model Ollama has. + probeModel := "" + switch { + case st.PrimaryModelAvailable: + probeModel = primary + case st.FallbackModelAvailable: + probeModel = fallback + case len(tags) > 0: + probeModel = tags[0] + st.Errors = append(st.Errors, + fmt.Sprintf("neither primary %q nor fallback %q loaded; using %q for liveness probe", primary, fallback, probeModel)) + default: + st.Errors = append(st.Errors, "no models loaded; can't run liveness probe") + return st + } + + // 2. Basic completion + if got, err := o.Complete(ctx, probeModel, "Reply with the single word: OK", CompleteOptions{Temperature: 0, MaxTokens: 8, TimeoutSeconds: 30}); err != nil { + st.Errors = append(st.Errors, "basic prompt: "+err.Error()) + } else if strings.TrimSpace(got) != "" { + st.BasicPromptOK = true + } + + // 3. JSON-mode completion + jsonGot, err := o.CompleteJSON(ctx, probeModel, `Output exactly this JSON and nothing else: {"ok": true}`, CompleteOptions{Temperature: 0, MaxTokens: 32, TimeoutSeconds: 30}) + if err != nil { + st.Errors = append(st.Errors, "json mode: "+err.Error()) + } else { + var probe struct{ Ok bool } + if json.Unmarshal([]byte(jsonGot), &probe) == nil { + st.JSONModeOK = true + } else { + st.Errors = append(st.Errors, "json mode: parse failed; raw="+abbrev(jsonGot, 200)) + } + } + + return st +} + +// Complete posts to /api/chat with stream=false. Returns just the +// assistant content; token counts not surfaced (callers that need +// them go via the chat-shape API directly, which we'll expose later). +func (o *OllamaProvider) Complete(ctx context.Context, model, prompt string, opts CompleteOptions) (string, error) { + body := o.chatBody(model, prompt, opts, false) + return o.postChat(ctx, body, opts) +} + +// CompleteJSON requests Ollama's native JSON-mode constrained output. +// The `format: "json"` field forces grammar-constrained generation — +// the model can only emit valid JSON. Some models still emit garbage +// in the content field (e.g. preamble text); validation is the +// caller's job (PROMPT.md "AI may suggest. Code validates."). +func (o *OllamaProvider) CompleteJSON(ctx context.Context, model, prompt string, opts CompleteOptions) (string, error) { + body := o.chatBody(model, prompt, opts, true) + return o.postChat(ctx, body, opts) +} + +func (o *OllamaProvider) chatBody(model, prompt string, opts CompleteOptions, jsonMode bool) map[string]any { + options := map[string]any{} + if opts.Temperature != 0 { + options["temperature"] = opts.Temperature + } + if opts.MaxTokens > 0 { + options["num_predict"] = opts.MaxTokens + } + body := map[string]any{ + "model": model, + "messages": []map[string]any{ + {"role": "user", "content": prompt}, + }, + "stream": false, + "think": false, // local hot path skips reasoning by default + "options": options, + } + if jsonMode { + body["format"] = "json" + } + return body +} + +func (o *OllamaProvider) postChat(ctx context.Context, body map[string]any, opts CompleteOptions) (string, error) { + bs, _ := json.Marshal(body) + req, err := http.NewRequestWithContext(ctx, "POST", o.baseURL+"/api/chat", bytes.NewReader(bs)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + + cli := o.httpClient + if opts.TimeoutSeconds > 0 { + cli = &http.Client{Timeout: time.Duration(opts.TimeoutSeconds) * time.Second} + } + resp, err := cli.Do(req) + if err != nil { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return "", fmt.Errorf("ollama timeout") + } + return "", fmt.Errorf("ollama request: %w", err) + } + defer resp.Body.Close() + rb, _ := io.ReadAll(resp.Body) + if resp.StatusCode/100 != 2 { + return "", fmt.Errorf("ollama %d: %s", resp.StatusCode, abbrev(string(rb), 200)) + } + var out struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + Done bool `json:"done"` + DoneReason string `json:"done_reason"` + } + if err := json.Unmarshal(rb, &out); err != nil { + return "", fmt.Errorf("ollama decode: %w (body=%s)", err, abbrev(string(rb), 200)) + } + return out.Message.Content, nil +} + +// listTags hits /api/tags and returns the loaded-model name list. +func (o *OllamaProvider) listTags(ctx context.Context) ([]string, error) { + cctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + req, _ := http.NewRequestWithContext(cctx, "GET", o.baseURL+"/api/tags", nil) + resp, err := o.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("status %d", resp.StatusCode) + } + rb, _ := io.ReadAll(resp.Body) + var out struct { + Models []struct { + Name string `json:"name"` + } `json:"models"` + } + if err := json.Unmarshal(rb, &out); err != nil { + return nil, err + } + names := make([]string, 0, len(out.Models)) + for _, m := range out.Models { + names = append(names, m.Name) + } + return names, nil +} + +func abbrev(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} diff --git a/internal/llm/review.go b/internal/llm/review.go new file mode 100644 index 0000000..519cb19 --- /dev/null +++ b/internal/llm/review.go @@ -0,0 +1,295 @@ +// Phase 2 (LLM review) implementation. Sends bounded chunks of the +// repo to the local model, asks for strict JSON Findings, retries +// once on parse failure, marks the phase degraded if the second +// attempt also fails. Raw output is saved either way — operators +// can re-parse manually if the harness rejected something useful. +package llm + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "local-review-harness/internal/analyzers" + "local-review-harness/internal/scanner" +) + +// ReviewInput is one bounded review request. The harness chunks the +// scan result into ReviewInputs (one per file or one per file-group) +// before calling Review. +type ReviewInput struct { + ChunkID string // stable per-chunk identifier (file path for v0) + Description string // human label (e.g. "internal/foo/bar.go") + Content string // the actual code/content to review + Language string // for the prompt context +} + +// ReviewOutput is what one Review call produces. RawContent is the +// model's verbatim output before parsing — saved for forensics if +// parsing fails. +type ReviewOutput struct { + ChunkID string `json:"chunk_id"` + Findings []analyzers.Finding `json:"findings"` + RawContent string `json:"raw_content"` + Parsed bool `json:"parsed"` + Retried bool `json:"retried"` + Error string `json:"error,omitempty"` +} + +// Reviewer wraps a Provider with the prompt + retry logic. Stateless; +// the prompt template is baked in for v0. +type Reviewer struct { + prov Provider + model string + opts CompleteOptions +} + +// NewReviewer constructs a Reviewer pointing at the configured +// primary model. opts are passed through to every Complete call; +// callers tune via review-profile. +func NewReviewer(prov Provider, model string, opts CompleteOptions) *Reviewer { + if opts.TimeoutSeconds == 0 { + opts.TimeoutSeconds = 120 + } + return &Reviewer{prov: prov, model: model, opts: opts} +} + +// Review runs the 2-attempt flow: prompt → parse → retry-with-repair-prompt → parse. +func (r *Reviewer) Review(ctx context.Context, in ReviewInput) ReviewOutput { + out := ReviewOutput{ChunkID: in.ChunkID} + + // Attempt 1 + prompt := buildReviewPrompt(in, false) + raw, err := r.prov.CompleteJSON(ctx, r.model, prompt, r.opts) + out.RawContent = raw + if err != nil { + out.Error = "request failed: " + err.Error() + return out + } + if findings, perr := parseFindings(raw, in); perr == nil { + out.Findings = findings + out.Parsed = true + return out + } + + // Attempt 2 (repair prompt — feed the raw output back + ask for + // strict JSON only). Done once; second failure is degraded. + out.Retried = true + repair := buildRepairPrompt(in, raw) + raw2, err := r.prov.CompleteJSON(ctx, r.model, repair, r.opts) + out.RawContent = raw + "\n\n---repair---\n\n" + raw2 + if err != nil { + out.Error = "repair request failed: " + err.Error() + return out + } + if findings, perr := parseFindings(raw2, in); perr == nil { + out.Findings = findings + out.Parsed = true + return out + } else { + out.Error = "parse failed after repair: " + perr.Error() + } + return out +} + +// ReviewBatch runs Review over a slice of inputs sequentially. Could +// parallelize at G3+, but local Ollama is GPU-bound and serial is +// the safe v0 — burst-parallel would queue at the model server anyway. +func (r *Reviewer) ReviewBatch(ctx context.Context, inputs []ReviewInput) []ReviewOutput { + out := make([]ReviewOutput, 0, len(inputs)) + for _, in := range inputs { + select { + case <-ctx.Done(): + out = append(out, ReviewOutput{ + ChunkID: in.ChunkID, + Error: "context cancelled before chunk processed", + }) + continue + default: + } + out = append(out, r.Review(ctx, in)) + } + return out +} + +// === prompts === + +const reviewSystemPrompt = `You are a senior code reviewer auditing a single source file. + +Your job: emit a JSON object with a "findings" array. Each finding +must include: + - title (string, < 80 chars) + - severity ("low" | "medium" | "high" | "critical") + - file (string, the file path you were asked to review — verbatim) + - line_hint (string, e.g. "42" or "100-110") + - evidence (string, a SHORT direct quote from the file — must + exist verbatim in the source so a downstream validator can + grep it) + - reason (string, one sentence explaining why this is a finding) + - suggested_fix (string, optional, one sentence) + - confidence (number 0.0–1.0) + +Severity guidance: + - critical: credential leak, RCE risk, destructive command, + unauthenticated mutation + - high: SQL injection, broad CORS, fail-open auth, unsafe FS + - medium: hardcoded paths, weak error handling, missing tests + near important code + - low: naming, duplication, doc drift + +Hard rules (failure = your output is rejected): + 1. Output ONLY the JSON object. No prose before or after. + 2. The evidence field MUST be a verbatim substring of the file. + If you can't quote the source, drop the finding. + 3. Don't invent file paths, line numbers, or test names. + 4. If the file is clean, return {"findings": []}. + 5. Output nothing else when you're done.` + +func buildReviewPrompt(in ReviewInput, _ bool) string { + var b strings.Builder + b.WriteString(reviewSystemPrompt) + b.WriteString("\n\n---\n\n") + b.WriteString("File path: ") + b.WriteString(in.Description) + b.WriteString("\nLanguage: ") + b.WriteString(in.Language) + b.WriteString("\n\nFile content:\n```\n") + b.WriteString(in.Content) + b.WriteString("\n```\n\nReturn JSON only.") + return b.String() +} + +func buildRepairPrompt(in ReviewInput, prev string) string { + var b strings.Builder + b.WriteString("Your previous output was not valid JSON or did not match the required schema.\n\n") + b.WriteString("Required shape:\n") + b.WriteString(`{"findings":[{"title":"...","severity":"...","file":"...","line_hint":"...","evidence":"...","reason":"...","confidence":0.0}]}`) + b.WriteString("\n\nPrevious raw output (for your reference):\n") + b.WriteString(abbrev(prev, 1500)) + b.WriteString("\n\nFor reference, the file you were reviewing was:\n") + b.WriteString(in.Description) + b.WriteString("\n\nReturn ONLY the JSON object now. No explanation, no markdown fences, no apology. JSON only.") + return b.String() +} + +// === parsing === + +func parseFindings(raw string, in ReviewInput) ([]analyzers.Finding, error) { + // Strip leading/trailing whitespace + common markdown fences. + cleaned := strings.TrimSpace(raw) + cleaned = strings.TrimPrefix(cleaned, "```json") + cleaned = strings.TrimPrefix(cleaned, "```") + cleaned = strings.TrimSuffix(cleaned, "```") + cleaned = strings.TrimSpace(cleaned) + if cleaned == "" { + return nil, fmt.Errorf("empty content") + } + + var shell struct { + Findings []struct { + Title string `json:"title"` + Severity string `json:"severity"` + File string `json:"file"` + LineHint string `json:"line_hint"` + Evidence string `json:"evidence"` + Reason string `json:"reason"` + SuggestedFix string `json:"suggested_fix"` + Confidence float64 `json:"confidence"` + } `json:"findings"` + } + if err := json.Unmarshal([]byte(cleaned), &shell); err != nil { + return nil, fmt.Errorf("unmarshal: %w", err) + } + + out := make([]analyzers.Finding, 0, len(shell.Findings)) + for _, f := range shell.Findings { + sev := normalizeSeverity(f.Severity) + if sev == "" { + continue // model emitted a value we don't accept + } + // Use the chunk's file path if model omitted/lied + filePath := f.File + if filePath == "" { + filePath = in.Description + } + out = append(out, analyzers.Finding{ + Title: truncate(f.Title, 80), + Severity: sev, + Status: analyzers.StatusSuspected, // validator (Phase D) promotes to confirmed + File: filePath, + LineHint: f.LineHint, + Evidence: f.Evidence, + Reason: f.Reason, + SuggestedFix: f.SuggestedFix, + Source: analyzers.SourceLLM, + Confidence: clampFloat(f.Confidence, 0, 1), + CheckID: "llm.review", + }) + } + return out, nil +} + +func normalizeSeverity(s string) analyzers.Severity { + switch strings.ToLower(strings.TrimSpace(s)) { + case "low": + return analyzers.SeverityLow + case "medium", "med": + return analyzers.SeverityMedium + case "high": + return analyzers.SeverityHigh + case "critical", "crit": + return analyzers.SeverityCritical + } + return "" +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} + +func clampFloat(v, lo, hi float64) float64 { + if v < lo { + return lo + } + if v > hi { + return hi + } + return v +} + +// === chunking === + +// ChunkInputsFromScan produces one ReviewInput per file under the +// configured size limit. Files larger than maxBytes are skipped (the +// LLM phase notes them in the receipt as "skipped: too large"). v0 +// is per-file; per-function chunking lands in Phase D+. +func ChunkInputsFromScan(scan *scanner.Result, maxBytes int, maxChunkChars int, readFile func(abs string) string) []ReviewInput { + out := []ReviewInput{} + for _, f := range scan.Files { + if f.Language == "" { + continue // non-code files: skip LLM review (analyzers may still flag) + } + if f.Size > int64(maxBytes) { + continue + } + content := readFile(f.Abs) + if len(content) > maxChunkChars { + content = content[:maxChunkChars] + "\n... (truncated for LLM context)\n" + } + out = append(out, ReviewInput{ + ChunkID: f.Path, + Description: f.Path, + Content: content, + Language: f.Language, + }) + } + return out +} + +// Useful for callers wiring a deadline across the whole batch. +var _ = time.Now diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index fef0b21..2c9d673 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -9,12 +9,15 @@ import ( "context" "crypto/rand" "encoding/hex" + "fmt" + "os" "path/filepath" "time" "local-review-harness/internal/analyzers" "local-review-harness/internal/config" "local-review-harness/internal/git" + "local-review-harness/internal/llm" "local-review-harness/internal/reporters" "local-review-harness/internal/scanner" ) @@ -26,6 +29,7 @@ type Inputs struct { ModelProfile config.ModelProfile OutputDir string EmitScrum bool // true → also emit scrum-test/risk-register/sprint-backlog/acceptance-gates markdown + EnableLLM bool // Phase C: actually call the model. Off by default — operators opt in. } // Result is what the CLI shows the operator. @@ -100,15 +104,44 @@ func RunRepo(ctx context.Context, in Inputs) (*Result, error) { receipt.Phases = append(receipt.Phases, staticPhase) res.OutputFiles = append(res.OutputFiles, "static-findings.json") - // --- Phase 2: LLM review (Phase C — not implemented in MVP) --- - receipt.Phases = append(receipt.Phases, reporters.PhaseReceipt{ - Name: "llm_review", Status: "degraded", - Errors: []string{"Phase C not implemented in MVP — see PROMPT.md / docs/REVIEW_PIPELINE.md Phase 2"}, - }) - if res.ExitCode == 0 { + // --- Phase 2: LLM review (Phase C) --- + llmDegraded := true + llmPhase := reporters.PhaseReceipt{Name: "llm_review", Status: "skipped"} + if !in.EnableLLM { + llmPhase.Errors = append(llmPhase.Errors, "LLM review not requested (pass --enable-llm to opt in)") + } else { + llmFindings, raw, llmErr := runLLMReview(ctx, scan, in) + // Always save raw output, even on failure — operator forensics. + rawPath := filepath.Join(in.OutputDir, "llm-findings.raw.json") + if _, err := reporters.WriteJSON(rawPath, raw); err == nil { + llmPhase.OutputFiles = append(llmPhase.OutputFiles, "llm-findings.raw.json") + } + if llmErr != nil { + llmPhase.Status = "degraded" + llmPhase.Errors = append(llmPhase.Errors, llmErr.Error()) + } else { + normalized := reporters.StaticFindings{ + GeneratedAt: time.Now().UTC().Format(time.RFC3339Nano), + Findings: llmFindings, + Summary: reporters.SummarizeFindings(llmFindings), + } + if sha, err := reporters.WriteJSON(filepath.Join(in.OutputDir, "llm-findings.normalized.json"), normalized); err == nil { + llmPhase.OutputFiles = append(llmPhase.OutputFiles, "llm-findings.normalized.json") + llmPhase.OutputHash = sha + llmPhase.Status = "ok" + llmDegraded = false + findings = append(findings, llmFindings...) + res.OutputFiles = append(res.OutputFiles, "llm-findings.raw.json", "llm-findings.normalized.json") + } else { + llmPhase.Status = "failed" + llmPhase.Errors = append(llmPhase.Errors, "write normalized: "+err.Error()) + } + } + } + if llmDegraded && res.ExitCode == 0 { res.ExitCode = 66 } - llmDegraded := true + receipt.Phases = append(receipt.Phases, llmPhase) // --- Phase 3: validation (Phase D — also deferred) --- receipt.Phases = append(receipt.Phases, reporters.PhaseReceipt{ @@ -175,6 +208,50 @@ func writeReceipt(outputDir string, r *reporters.Receipt, startedAt time.Time, _ return err } +// runLLMReview chunks the scan into per-file inputs, calls the +// reviewer, and aggregates parsed findings + raw outputs. Returns +// (findings, raw-outputs-array-for-receipts, error). The error is +// non-nil only when the provider is fundamentally unreachable; +// per-chunk parse failures land as ReviewOutput.Error and don't +// fail the whole phase. +func runLLMReview(ctx context.Context, scan *scanner.Result, in Inputs) ([]analyzers.Finding, []llm.ReviewOutput, error) { + prov := llm.NewOllama(in.ModelProfile.BaseURL, time.Duration(in.ModelProfile.TimeoutSeconds)*time.Second) + hctx, hcancel := context.WithTimeout(ctx, 5*time.Second) + defer hcancel() + hs := prov.HealthCheck(hctx, in.ModelProfile.Model, in.ModelProfile.FallbackModel) + if !hs.ServerAvailable { + return nil, nil, fmt.Errorf("ollama unreachable at %s — Phase 2 cannot run", in.ModelProfile.BaseURL) + } + if !hs.PrimaryModelAvailable && !hs.FallbackModelAvailable { + return nil, nil, fmt.Errorf("neither primary %q nor fallback %q loaded in Ollama", in.ModelProfile.Model, in.ModelProfile.FallbackModel) + } + model := in.ModelProfile.Model + if !hs.PrimaryModelAvailable { + model = in.ModelProfile.FallbackModel + } + + r := llm.NewReviewer(prov, model, llm.CompleteOptions{ + Temperature: in.ModelProfile.Temperature, + MaxTokens: 0, // let model decide + TimeoutSeconds: in.ModelProfile.TimeoutSeconds, + }) + + chunks := llm.ChunkInputsFromScan(scan, in.ReviewProfile.Limits.MaxFileBytes, in.ReviewProfile.Limits.MaxLLMChunkChars, func(abs string) string { + b, err := os.ReadFile(abs) + if err != nil { + return "" + } + return string(b) + }) + + outputs := r.ReviewBatch(ctx, chunks) + findings := []analyzers.Finding{} + for _, o := range outputs { + findings = append(findings, o.Findings...) + } + return findings, outputs, nil +} + func newRunID(t time.Time) string { var rb [4]byte _, _ = rand.Read(rb[:]) diff --git a/tests/fixtures/clean-repo/reports/latest/acceptance-gates.md b/tests/fixtures/clean-repo/reports/latest/acceptance-gates.md new file mode 100644 index 0000000..55a4705 --- /dev/null +++ b/tests/fixtures/clean-repo/reports/latest/acceptance-gates.md @@ -0,0 +1,8 @@ +# Acceptance Gates + +Each gate must be testable. Format: command + verifiable post-condition. + +1. **Reproducibility:** `review-harness repo .` exits 0; `reports/latest/repo-intake.json` exists with non-zero `file_count`. +2. **No false positives on a clean fixture:** `review-harness repo tests/fixtures/clean-repo` produces zero `confirmed` findings. +3. **Every documented static check fires on the insecure fixture:** `jq '[.findings[] | .check_id] | unique | length' reports/latest/static-findings.json` ≥ 8. +4. **Receipts are honest about degraded phases:** `jq '[.phases[] | select(.status == "degraded")]' reports/latest/receipts.json` lists every skipped/stubbed phase. diff --git a/tests/fixtures/clean-repo/reports/latest/claim-coverage-table.md b/tests/fixtures/clean-repo/reports/latest/claim-coverage-table.md new file mode 100644 index 0000000..9ec9d72 --- /dev/null +++ b/tests/fixtures/clean-repo/reports/latest/claim-coverage-table.md @@ -0,0 +1,8 @@ +# Claim Coverage Table + +Each row is a finding paired with whether existing tests cover the affected area. +Phase B emits this shape; LLM-side claim generation lands in Phase C. + +| Claim | Code Location | Existing Test | Missing Test | Risk | +|---|---|---|---|---| +| _no claims yet_ | — | — | — | — | diff --git a/tests/fixtures/clean-repo/reports/latest/llm-findings.normalized.json b/tests/fixtures/clean-repo/reports/latest/llm-findings.normalized.json new file mode 100644 index 0000000..7466715 --- /dev/null +++ b/tests/fixtures/clean-repo/reports/latest/llm-findings.normalized.json @@ -0,0 +1,16 @@ +{ + "generated_at": "2026-04-30T06:06:56.669606679Z", + "findings": [], + "summary": { + "total": 0, + "confirmed": 0, + "suspected": 0, + "rejected": 0, + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "by_source": {}, + "by_check": {} + } +} diff --git a/tests/fixtures/clean-repo/reports/latest/llm-findings.raw.json b/tests/fixtures/clean-repo/reports/latest/llm-findings.raw.json new file mode 100644 index 0000000..7177e7b --- /dev/null +++ b/tests/fixtures/clean-repo/reports/latest/llm-findings.raw.json @@ -0,0 +1,30 @@ +[ + { + "chunk_id": "README.md", + "findings": [], + "raw_content": "{\"findings\": []}", + "parsed": true, + "retried": false + }, + { + "chunk_id": "package.json", + "findings": [], + "raw_content": "{\"findings\": []}", + "parsed": true, + "retried": false + }, + { + "chunk_id": "src/calc.ts", + "findings": [], + "raw_content": "{\"findings\": []}", + "parsed": true, + "retried": false + }, + { + "chunk_id": "tests/calc.test.ts", + "findings": [], + "raw_content": "{\"findings\": []}", + "parsed": true, + "retried": false + } +] diff --git a/tests/fixtures/clean-repo/reports/latest/receipts.json b/tests/fixtures/clean-repo/reports/latest/receipts.json new file mode 100644 index 0000000..65f1286 --- /dev/null +++ b/tests/fixtures/clean-repo/reports/latest/receipts.json @@ -0,0 +1,70 @@ +{ + "run_id": "20260430T060653-57461e2c", + "repo_path": "tests/fixtures/clean-repo", + "started_at": "2026-04-30T06:06:53.880883402Z", + "finished_at": "2026-04-30T06:06:56.669797363Z", + "phases": [ + { + "name": "repo_intake", + "status": "ok", + "output_hash": "db312c5ce39315cd", + "output_files": [ + "repo-intake.json" + ] + }, + { + "name": "static_scan", + "status": "ok", + "output_hash": "837b6a5d9dc11126", + "output_files": [ + "static-findings.json" + ] + }, + { + "name": "llm_review", + "status": "ok", + "output_hash": "3939252dabe358b1", + "output_files": [ + "llm-findings.raw.json", + "llm-findings.normalized.json" + ] + }, + { + "name": "validation", + "status": "skipped", + "errors": [ + "Phase D not implemented in MVP — depends on Phase C" + ] + }, + { + "name": "report_generation", + "status": "ok", + "output_files": [ + "scrum-test.md", + "risk-register.md", + "claim-coverage-table.md", + "sprint-backlog.md", + "acceptance-gates.md" + ] + }, + { + "name": "memory_update", + "status": "skipped", + "errors": [ + "Phase E not implemented in MVP" + ] + } + ], + "summary": { + "total": 0, + "confirmed": 0, + "suspected": 0, + "rejected": 0, + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "by_source": {}, + "by_check": {} + } +} diff --git a/tests/fixtures/clean-repo/reports/latest/repo-intake.json b/tests/fixtures/clean-repo/reports/latest/repo-intake.json new file mode 100644 index 0000000..095c85f --- /dev/null +++ b/tests/fixtures/clean-repo/reports/latest/repo-intake.json @@ -0,0 +1,42 @@ +{ + "repo_path": "/home/profit/share/local-review-harness-full-md/tests/fixtures/clean-repo", + "current_branch": "main", + "latest_commit": "70d68757f78ea722bf24585c73120c09d82c4fea", + "git_status": "M ../../../internal/cli/cli.go\n M ../../../internal/cli/repo.go\n M ../../../internal/pipeline/pipeline.go\n?? ../../../internal/llm/ollama.go\n?? ../../../internal/llm/review.go\n?? ../insecure-repo/reports/", + "has_git": true, + "file_count": 4, + "language_breakdown": { + "JSON": 1, + "Markdown": 1, + "TypeScript": 2 + }, + "largest_files": [ + { + "path": "src/calc.ts", + "size": 206, + "lines": 7 + }, + { + "path": "tests/calc.test.ts", + "size": 198, + "lines": 4 + }, + { + "path": "README.md", + "size": 80, + "lines": 2 + }, + { + "path": "package.json", + "size": 43, + "lines": 1 + } + ], + "dependency_manifests": [ + "package.json" + ], + "test_manifests": [ + "tests/calc.test.ts" + ], + "generated_at": "2026-04-30T06:06:53.895008668Z" +} diff --git a/tests/fixtures/clean-repo/reports/latest/risk-register.md b/tests/fixtures/clean-repo/reports/latest/risk-register.md new file mode 100644 index 0000000..3ef0ee4 --- /dev/null +++ b/tests/fixtures/clean-repo/reports/latest/risk-register.md @@ -0,0 +1,5 @@ +# Risk Register + +Findings ranked by severity. `Suspected` rows haven't been validated yet (Phase D). + +_No findings._ diff --git a/tests/fixtures/clean-repo/reports/latest/scrum-test.md b/tests/fixtures/clean-repo/reports/latest/scrum-test.md new file mode 100644 index 0000000..14a0976 --- /dev/null +++ b/tests/fixtures/clean-repo/reports/latest/scrum-test.md @@ -0,0 +1,69 @@ +# Scrum Test — clean-repo + +**Generated:** 2026-04-30T06:06:53.895008668Z +**Branch:** main · **Commit:** 70d68757f78ea722bf24585c73120c09d82c4fea + +## Verdict + +**production-ready** — static scan + LLM review found no issues. Re-validate after every wave. + +## Evidence + +- repo path: `/home/profit/share/local-review-harness-full-md/tests/fixtures/clean-repo` +- file count: 4 +- languages: TypeScript (2), Markdown (1), JSON (1) +- dependency manifests: 1 (package.json) +- test files/dirs: 1 + +## Confirmed Risks + +_No confirmed risks at static-scan level. (LLM review may surface more.)_ + +## Suspected Risks + +_None._ + +## Blocked Checks + +_None._ + +## Sprint Backlog + +**Sprint 0 — Reproducibility Gate** + +- Wire `just verify` (or equivalent) to run the static checks before every commit/PR. +- Add a CI step that fails on `critical` findings. + +**Sprint 1 — Trust Boundary Gate** + +- Confirm auth posture for any mutation endpoint flagged as exposed. +- Replace raw SQL interpolation with parameterized queries. + +**Sprint 2 — Memory Correctness Gate** + +- (Phase E) Wire append-only `.memory/` writes for known-risks + fixed-patterns. +- Add a regression test that re-runs the harness and asserts no regression in confirmed-finding count. + +**Sprint 3 — Agent Loop Reality Gate** + +- (Phase C) Wire local-Ollama LLM review. +- (Phase D) Validator pass cross-checks every LLM finding against repo evidence. + +**Sprint 4 — Deployment Gate** + +- Ship the harness as a single static binary (`go build -o review-harness`). +- Document operator runbook (model setup, profile editing, output retention). + +## Acceptance Gates + +Each gate must be testable. Format: command + verifiable post-condition. + +1. **Reproducibility:** `review-harness repo .` exits 0; `reports/latest/repo-intake.json` exists with non-zero `file_count`. +2. **No false positives on a clean fixture:** `review-harness repo tests/fixtures/clean-repo` produces zero `confirmed` findings. +3. **Every documented static check fires on the insecure fixture:** `jq '[.findings[] | .check_id] | unique | length' reports/latest/static-findings.json` ≥ 8. +4. **Receipts are honest about degraded phases:** `jq '[.phases[] | select(.status == "degraded")]' reports/latest/receipts.json` lists every skipped/stubbed phase. + +## Next Commands + +- Re-run after fixes: `review-harness repo /home/profit/share/local-review-harness-full-md/tests/fixtures/clean-repo` +- Generate the full Scrum bundle: `review-harness scrum /home/profit/share/local-review-harness-full-md/tests/fixtures/clean-repo` diff --git a/tests/fixtures/clean-repo/reports/latest/sprint-backlog.md b/tests/fixtures/clean-repo/reports/latest/sprint-backlog.md new file mode 100644 index 0000000..e17bc26 --- /dev/null +++ b/tests/fixtures/clean-repo/reports/latest/sprint-backlog.md @@ -0,0 +1,26 @@ +# Sprint Backlog + +**Sprint 0 — Reproducibility Gate** + +- Wire `just verify` (or equivalent) to run the static checks before every commit/PR. +- Add a CI step that fails on `critical` findings. + +**Sprint 1 — Trust Boundary Gate** + +- Confirm auth posture for any mutation endpoint flagged as exposed. +- Replace raw SQL interpolation with parameterized queries. + +**Sprint 2 — Memory Correctness Gate** + +- (Phase E) Wire append-only `.memory/` writes for known-risks + fixed-patterns. +- Add a regression test that re-runs the harness and asserts no regression in confirmed-finding count. + +**Sprint 3 — Agent Loop Reality Gate** + +- (Phase C) Wire local-Ollama LLM review. +- (Phase D) Validator pass cross-checks every LLM finding against repo evidence. + +**Sprint 4 — Deployment Gate** + +- Ship the harness as a single static binary (`go build -o review-harness`). +- Document operator runbook (model setup, profile editing, output retention). diff --git a/tests/fixtures/clean-repo/reports/latest/static-findings.json b/tests/fixtures/clean-repo/reports/latest/static-findings.json new file mode 100644 index 0000000..0c19592 --- /dev/null +++ b/tests/fixtures/clean-repo/reports/latest/static-findings.json @@ -0,0 +1,16 @@ +{ + "generated_at": "2026-04-30T06:06:53.896533109Z", + "findings": [], + "summary": { + "total": 0, + "confirmed": 0, + "suspected": 0, + "rejected": 0, + "critical": 0, + "high": 0, + "medium": 0, + "low": 0, + "by_source": {}, + "by_check": {} + } +} diff --git a/tests/fixtures/degraded-repo/stray.go b/tests/fixtures/degraded-repo/stray.go index 6521331..6420e8c 100644 --- a/tests/fixtures/degraded-repo/stray.go +++ b/tests/fixtures/degraded-repo/stray.go @@ -1 +1,7 @@ +// Fixture: orphan source file in a non-git directory. Exists so the +// scanner has something to walk. Marked package fixture so `go vet` +// over the harness's own module doesn't choke on the orphan; the +// harness itself reads this file as raw text and doesn't care. +package fixture + // just a stray file diff --git a/tests/fixtures/insecure-repo/reports/latest/acceptance-gates.md b/tests/fixtures/insecure-repo/reports/latest/acceptance-gates.md new file mode 100644 index 0000000..3e27508 --- /dev/null +++ b/tests/fixtures/insecure-repo/reports/latest/acceptance-gates.md @@ -0,0 +1,9 @@ +# Acceptance Gates + +Each gate must be testable. Format: command + verifiable post-condition. + +1. **Reproducibility:** `review-harness repo .` exits 0; `reports/latest/repo-intake.json` exists with non-zero `file_count`. +2. **No false positives on a clean fixture:** `review-harness repo tests/fixtures/clean-repo` produces zero `confirmed` findings. +3. **Every documented static check fires on the insecure fixture:** `jq '[.findings[] | .check_id] | unique | length' reports/latest/static-findings.json` ≥ 8. +4. **Receipts are honest about degraded phases:** `jq '[.phases[] | select(.status == "degraded")]' reports/latest/receipts.json` lists every skipped/stubbed phase. +5. **Critical findings block production deploy:** at least one critical finding is currently present; resolve before deploy. diff --git a/tests/fixtures/insecure-repo/reports/latest/claim-coverage-table.md b/tests/fixtures/insecure-repo/reports/latest/claim-coverage-table.md new file mode 100644 index 0000000..1dec4d2 --- /dev/null +++ b/tests/fixtures/insecure-repo/reports/latest/claim-coverage-table.md @@ -0,0 +1,23 @@ +# Claim Coverage Table + +Each row is a finding paired with whether existing tests cover the affected area. +Phase B emits this shape; LLM-side claim generation lands in Phase C. + +| Claim | Code Location | Existing Test | Missing Test | Risk | +|---|---|---|---|---| +| Environment file in source tree | `.env:?` | _unknown_ | _likely_ | high | +| Hardcoded absolute path | `src/handler.go:10` | _unknown_ | _likely_ | medium | +| Shell command execution | `src/handler.go:19` | _unknown_ | _likely_ | high | +| Raw SQL interpolation | `src/handler.go:14` | _unknown_ | _likely_ | high | +| Possible secret committed to source | `src/handler.go:23` | _unknown_ | _likely_ | critical | +| Possible secret committed to source | `src/handler.go:23` | _unknown_ | _likely_ | critical | +| TODO/FIXME comment | `src/handler.go:9` | _unknown_ | _likely_ | low | +| TODO/FIXME comment | `src/handler.go:22` | _unknown_ | _likely_ | low | +| Hardcoded private-network IP | `src/handler.go:11` | _unknown_ | _likely_ | medium | +| Large file | `src/huge.go:1-901` | _unknown_ | _likely_ | medium | +| Wildcard CORS | `src/server.js:2` | _unknown_ | _likely_ | high | +| Possible secret committed to source | `src/server.js:5` | _unknown_ | _likely_ | critical | +| TODO/FIXME comment | `src/server.js:1` | _unknown_ | _likely_ | low | +| Mutation route in file with no visible auth | `src/server.js:7` | _unknown_ | _likely_ | medium | +| Mutation route in file with no visible auth | `src/server.js:8` | _unknown_ | _likely_ | medium | +| No tests found | `.:?` | _unknown_ | _likely_ | medium | diff --git a/tests/fixtures/insecure-repo/reports/latest/llm-findings.normalized.json b/tests/fixtures/insecure-repo/reports/latest/llm-findings.normalized.json new file mode 100644 index 0000000..0830dc0 --- /dev/null +++ b/tests/fixtures/insecure-repo/reports/latest/llm-findings.normalized.json @@ -0,0 +1,147 @@ +{ + "generated_at": "2026-04-30T06:06:33.240219171Z", + "findings": [ + { + "id": "", + "title": "Hardcoded file path for secrets", + "severity": "high", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "10", + "evidence": "const HARDCODED_PATH = \"/home/profit/secrets/key.pem\"", + "reason": "Hardcoding a file path for a private key exposes secrets and prevents proper secret management.", + "suggested_fix": "Move the path to an environment variable or a configuration file outside the source code.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "Hardcoded server IP address", + "severity": "medium", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "11", + "evidence": "const SERVER_IP = \"192.168.1.176\"", + "reason": "Hardcoding an IP address reduces portability and may leak internal network topology.", + "suggested_fix": "Read the server IP from an environment variable.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "SQL Injection vulnerability", + "severity": "critical", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "15-16", + "evidence": "q := fmt.Sprintf(\"SELECT * FROM users WHERE name = '%s'\", name)\ndb.Query(q)", + "reason": "Using string formatting to construct SQL queries directly exposes the application to SQL injection attacks.", + "suggested_fix": "Use parameterized queries with placeholders instead of string formatting.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "Unsafe shell command execution", + "severity": "critical", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "19-20", + "evidence": "exec.Command(\"bash\", \"-c\", cmd).Run()", + "reason": "Executing arbitrary shell commands without validation allows for remote code execution.", + "suggested_fix": "Validate and sanitize the input command strictly, or avoid using shell execution entirely.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "Hardcoded API key", + "severity": "critical", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "23", + "evidence": "const API_KEY = \"sk-1234567890abcdefABCDEFGHIJKLMNOPQRSTUV\"", + "reason": "Hardcoding an API key in source code exposes sensitive credentials to anyone with access to the repository.", + "suggested_fix": "Store the API key in a secure environment variable or secrets manager.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "CORS misconfiguration allows cross-origin attacks", + "severity": "high", + "status": "suspected", + "file": "src/server.js", + "line_hint": "2", + "evidence": "res.setHeader(\"Access-Control-Allow-Origin\", \"*\");", + "reason": "Allowing all origins (*) exposes the API to cross-site request forgery and data theft from any website.", + "suggested_fix": "Restrict Access-Control-Allow-Origin to specific trusted domains or use credentials with a specific origin.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "Hardcoded AWS access key in source code", + "severity": "critical", + "status": "suspected", + "file": "src/server.js", + "line_hint": "5", + "evidence": "const AWS_KEY = \"AKIAIOSFODNN7EXAMPLE\";", + "reason": "Hardcoded credentials in source code pose a severe security risk as they can be easily leaked and misused.", + "suggested_fix": "Use environment variables or a secure secrets manager to store AWS credentials.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "Missing authentication on user creation endpoint", + "severity": "high", + "status": "suspected", + "file": "src/server.js", + "line_hint": "7", + "evidence": "app.post(\"/api/users\", function(req, res) { /* no auth */ });", + "reason": "The /api/users endpoint lacks authentication, allowing anyone to create or modify user accounts.", + "suggested_fix": "Implement authentication middleware to verify user identity before allowing POST requests.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "Missing authentication on admin deletion endpoint", + "severity": "critical", + "status": "suspected", + "file": "src/server.js", + "line_hint": "8", + "evidence": "app.delete(\"/api/admin\", function(req, res) { /* no auth */ });", + "reason": "The /api/admin endpoint lacks authentication, allowing unauthenticated users to delete administrative resources.", + "suggested_fix": "Implement strict authentication and authorization checks for all admin endpoints.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + } + ], + "summary": { + "total": 9, + "confirmed": 0, + "suspected": 9, + "rejected": 0, + "critical": 5, + "high": 3, + "medium": 1, + "low": 0, + "by_source": { + "llm": 9 + }, + "by_check": { + "llm.review": 9 + } + } +} diff --git a/tests/fixtures/insecure-repo/reports/latest/llm-findings.raw.json b/tests/fixtures/insecure-repo/reports/latest/llm-findings.raw.json new file mode 100644 index 0000000..3097c3d --- /dev/null +++ b/tests/fixtures/insecure-repo/reports/latest/llm-findings.raw.json @@ -0,0 +1,151 @@ +[ + { + "chunk_id": "src/handler.go", + "findings": [ + { + "id": "", + "title": "Hardcoded file path for secrets", + "severity": "high", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "10", + "evidence": "const HARDCODED_PATH = \"/home/profit/secrets/key.pem\"", + "reason": "Hardcoding a file path for a private key exposes secrets and prevents proper secret management.", + "suggested_fix": "Move the path to an environment variable or a configuration file outside the source code.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "Hardcoded server IP address", + "severity": "medium", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "11", + "evidence": "const SERVER_IP = \"192.168.1.176\"", + "reason": "Hardcoding an IP address reduces portability and may leak internal network topology.", + "suggested_fix": "Read the server IP from an environment variable.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "SQL Injection vulnerability", + "severity": "critical", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "15-16", + "evidence": "q := fmt.Sprintf(\"SELECT * FROM users WHERE name = '%s'\", name)\ndb.Query(q)", + "reason": "Using string formatting to construct SQL queries directly exposes the application to SQL injection attacks.", + "suggested_fix": "Use parameterized queries with placeholders instead of string formatting.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "Unsafe shell command execution", + "severity": "critical", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "19-20", + "evidence": "exec.Command(\"bash\", \"-c\", cmd).Run()", + "reason": "Executing arbitrary shell commands without validation allows for remote code execution.", + "suggested_fix": "Validate and sanitize the input command strictly, or avoid using shell execution entirely.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "Hardcoded API key", + "severity": "critical", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "23", + "evidence": "const API_KEY = \"sk-1234567890abcdefABCDEFGHIJKLMNOPQRSTUV\"", + "reason": "Hardcoding an API key in source code exposes sensitive credentials to anyone with access to the repository.", + "suggested_fix": "Store the API key in a secure environment variable or secrets manager.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + } + ], + "raw_content": "{\n \"findings\": [\n {\n \"title\": \"Hardcoded file path for secrets\",\n \"severity\": \"high\",\n \"file\": \"src/handler.go\",\n \"line_hint\": \"10\",\n \"evidence\": \"const HARDCODED_PATH = \\\"/home/profit/secrets/key.pem\\\"\",\n \"reason\": \"Hardcoding a file path for a private key exposes secrets and prevents proper secret management.\",\n \"suggested_fix\": \"Move the path to an environment variable or a configuration file outside the source code.\",\n \"confidence\": 1.0\n },\n {\n \"title\": \"Hardcoded server IP address\",\n \"severity\": \"medium\",\n \"file\": \"src/handler.go\",\n \"line_hint\": \"11\",\n \"evidence\": \"const SERVER_IP = \\\"192.168.1.176\\\"\",\n \"reason\": \"Hardcoding an IP address reduces portability and may leak internal network topology.\",\n \"suggested_fix\": \"Read the server IP from an environment variable.\",\n \"confidence\": 1.0\n },\n {\n \"title\": \"SQL Injection vulnerability\",\n \"severity\": \"critical\",\n \"file\": \"src/handler.go\",\n \"line_hint\": \"15-16\",\n \"evidence\": \"q := fmt.Sprintf(\\\"SELECT * FROM users WHERE name = '%s'\\\", name)\\ndb.Query(q)\",\n \"reason\": \"Using string formatting to construct SQL queries directly exposes the application to SQL injection attacks.\",\n \"suggested_fix\": \"Use parameterized queries with placeholders instead of string formatting.\",\n \"confidence\": 1.0\n },\n {\n \"title\": \"Unsafe shell command execution\",\n \"severity\": \"critical\",\n \"file\": \"src/handler.go\",\n \"line_hint\": \"19-20\",\n \"evidence\": \"exec.Command(\\\"bash\\\", \\\"-c\\\", cmd).Run()\",\n \"reason\": \"Executing arbitrary shell commands without validation allows for remote code execution.\",\n \"suggested_fix\": \"Validate and sanitize the input command strictly, or avoid using shell execution entirely.\",\n \"confidence\": 1.0\n },\n {\n \"title\": \"Hardcoded API key\",\n \"severity\": \"critical\",\n \"file\": \"src/handler.go\",\n \"line_hint\": \"23\",\n \"evidence\": \"const API_KEY = \\\"sk-1234567890abcdefABCDEFGHIJKLMNOPQRSTUV\\\"\",\n \"reason\": \"Hardcoding an API key in source code exposes sensitive credentials to anyone with access to the repository.\",\n \"suggested_fix\": \"Store the API key in a secure environment variable or secrets manager.\",\n \"confidence\": 1.0\n }\n ]\n}", + "parsed": true, + "retried": false + }, + { + "chunk_id": "src/huge.go", + "findings": [], + "raw_content": "```json\n{\n \"error\": \"No valid content found. The input appears to be a list of generated line markers without any actual text or data to process.\",\n \"status\": \"empty_input\"\n}\n```", + "parsed": true, + "retried": false + }, + { + "chunk_id": "src/server.js", + "findings": [ + { + "id": "", + "title": "CORS misconfiguration allows cross-origin attacks", + "severity": "high", + "status": "suspected", + "file": "src/server.js", + "line_hint": "2", + "evidence": "res.setHeader(\"Access-Control-Allow-Origin\", \"*\");", + "reason": "Allowing all origins (*) exposes the API to cross-site request forgery and data theft from any website.", + "suggested_fix": "Restrict Access-Control-Allow-Origin to specific trusted domains or use credentials with a specific origin.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "Hardcoded AWS access key in source code", + "severity": "critical", + "status": "suspected", + "file": "src/server.js", + "line_hint": "5", + "evidence": "const AWS_KEY = \"AKIAIOSFODNN7EXAMPLE\";", + "reason": "Hardcoded credentials in source code pose a severe security risk as they can be easily leaked and misused.", + "suggested_fix": "Use environment variables or a secure secrets manager to store AWS credentials.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "Missing authentication on user creation endpoint", + "severity": "high", + "status": "suspected", + "file": "src/server.js", + "line_hint": "7", + "evidence": "app.post(\"/api/users\", function(req, res) { /* no auth */ });", + "reason": "The /api/users endpoint lacks authentication, allowing anyone to create or modify user accounts.", + "suggested_fix": "Implement authentication middleware to verify user identity before allowing POST requests.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + }, + { + "id": "", + "title": "Missing authentication on admin deletion endpoint", + "severity": "critical", + "status": "suspected", + "file": "src/server.js", + "line_hint": "8", + "evidence": "app.delete(\"/api/admin\", function(req, res) { /* no auth */ });", + "reason": "The /api/admin endpoint lacks authentication, allowing unauthenticated users to delete administrative resources.", + "suggested_fix": "Implement strict authentication and authorization checks for all admin endpoints.", + "source": "llm", + "confidence": 1, + "check_id": "llm.review" + } + ], + "raw_content": "{\n \"findings\": [\n {\n \"title\": \"CORS misconfiguration allows cross-origin attacks\",\n \"severity\": \"high\",\n \"file\": \"src/server.js\",\n \"line_hint\": \"2\",\n \"evidence\": \"res.setHeader(\\\"Access-Control-Allow-Origin\\\", \\\"*\\\");\",\n \"reason\": \"Allowing all origins (*) exposes the API to cross-site request forgery and data theft from any website.\",\n \"suggested_fix\": \"Restrict Access-Control-Allow-Origin to specific trusted domains or use credentials with a specific origin.\",\n \"confidence\": 1.0\n },\n {\n \"title\": \"Hardcoded AWS access key in source code\",\n \"severity\": \"critical\",\n \"file\": \"src/server.js\",\n \"line_hint\": \"5\",\n \"evidence\": \"const AWS_KEY = \\\"AKIAIOSFODNN7EXAMPLE\\\";\",\n \"reason\": \"Hardcoded credentials in source code pose a severe security risk as they can be easily leaked and misused.\",\n \"suggested_fix\": \"Use environment variables or a secure secrets manager to store AWS credentials.\",\n \"confidence\": 1.0\n },\n {\n \"title\": \"Missing authentication on user creation endpoint\",\n \"severity\": \"high\",\n \"file\": \"src/server.js\",\n \"line_hint\": \"7\",\n \"evidence\": \"app.post(\\\"/api/users\\\", function(req, res) { /* no auth */ });\",\n \"reason\": \"The /api/users endpoint lacks authentication, allowing anyone to create or modify user accounts.\",\n \"suggested_fix\": \"Implement authentication middleware to verify user identity before allowing POST requests.\",\n \"confidence\": 1.0\n },\n {\n \"title\": \"Missing authentication on admin deletion endpoint\",\n \"severity\": \"critical\",\n \"file\": \"src/server.js\",\n \"line_hint\": \"8\",\n \"evidence\": \"app.delete(\\\"/api/admin\\\", function(req, res) { /* no auth */ });\",\n \"reason\": \"The /api/admin endpoint lacks authentication, allowing unauthenticated users to delete administrative resources.\",\n \"suggested_fix\": \"Implement strict authentication and authorization checks for all admin endpoints.\",\n \"confidence\": 1.0\n }\n ]\n}", + "parsed": true, + "retried": false + } +] diff --git a/tests/fixtures/insecure-repo/reports/latest/receipts.json b/tests/fixtures/insecure-repo/reports/latest/receipts.json new file mode 100644 index 0000000..4e513d6 --- /dev/null +++ b/tests/fixtures/insecure-repo/reports/latest/receipts.json @@ -0,0 +1,82 @@ +{ + "run_id": "20260430T060713-f513f6dc", + "repo_path": "tests/fixtures/insecure-repo", + "started_at": "2026-04-30T06:07:13.917781613Z", + "finished_at": "2026-04-30T06:07:13.953011207Z", + "phases": [ + { + "name": "repo_intake", + "status": "ok", + "output_hash": "540f222456204a27", + "output_files": [ + "repo-intake.json" + ] + }, + { + "name": "static_scan", + "status": "ok", + "output_hash": "a7aeccbda6841c1e", + "output_files": [ + "static-findings.json" + ] + }, + { + "name": "llm_review", + "status": "skipped", + "errors": [ + "LLM review not requested (pass --enable-llm to opt in)" + ] + }, + { + "name": "validation", + "status": "skipped", + "errors": [ + "Phase D not implemented in MVP — depends on Phase C" + ] + }, + { + "name": "report_generation", + "status": "ok", + "output_files": [ + "scrum-test.md", + "risk-register.md", + "claim-coverage-table.md", + "sprint-backlog.md", + "acceptance-gates.md" + ] + }, + { + "name": "memory_update", + "status": "skipped", + "errors": [ + "Phase E not implemented in MVP" + ] + } + ], + "summary": { + "total": 16, + "confirmed": 2, + "suspected": 14, + "rejected": 0, + "critical": 3, + "high": 4, + "medium": 6, + "low": 3, + "by_source": { + "static": 16 + }, + "by_check": { + "static.broad_cors": 1, + "static.env_file_committed": 1, + "static.exposed_mutation_endpoint": 2, + "static.hardcoded_local_ip": 1, + "static.hardcoded_paths": 1, + "static.large_files": 1, + "static.missing_tests": 1, + "static.raw_sql_interpolation": 1, + "static.secret_patterns": 3, + "static.shell_execution": 1, + "static.todo_comments": 3 + } + } +} diff --git a/tests/fixtures/insecure-repo/reports/latest/repo-intake.json b/tests/fixtures/insecure-repo/reports/latest/repo-intake.json new file mode 100644 index 0000000..dee48dc --- /dev/null +++ b/tests/fixtures/insecure-repo/reports/latest/repo-intake.json @@ -0,0 +1,37 @@ +{ + "repo_path": "/home/profit/share/local-review-harness-full-md/tests/fixtures/insecure-repo", + "current_branch": "main", + "latest_commit": "70d68757f78ea722bf24585c73120c09d82c4fea", + "git_status": "M ../../../internal/cli/cli.go\n M ../../../internal/cli/repo.go\n M ../../../internal/pipeline/pipeline.go\n?? ../../../internal/llm/ollama.go\n?? ../../../internal/llm/review.go\n?? ../clean-repo/reports/\n?? reports/", + "has_git": true, + "file_count": 4, + "language_breakdown": { + "Go": 2, + "JavaScript": 1 + }, + "largest_files": [ + { + "path": "src/huge.go", + "size": 19705, + "lines": 901 + }, + { + "path": "src/handler.go", + "size": 462, + "lines": 23 + }, + { + "path": "src/server.js", + "size": 286, + "lines": 8 + }, + { + "path": ".env", + "size": 59, + "lines": 2 + } + ], + "dependency_manifests": null, + "test_manifests": null, + "generated_at": "2026-04-30T06:07:13.931830669Z" +} diff --git a/tests/fixtures/insecure-repo/reports/latest/risk-register.md b/tests/fixtures/insecure-repo/reports/latest/risk-register.md new file mode 100644 index 0000000..68f6b26 --- /dev/null +++ b/tests/fixtures/insecure-repo/reports/latest/risk-register.md @@ -0,0 +1,22 @@ +# Risk Register + +Findings ranked by severity. `Suspected` rows haven't been validated yet (Phase D). + +| ID | Severity | Status | File | Line | Title | +|---|---|---|---|---|---| +| `9bc97c579efc` | critical | suspected | `src/handler.go` | 23 | Possible secret committed to source | +| `9bc97c579efc` | critical | suspected | `src/handler.go` | 23 | Possible secret committed to source | +| `d3c2c5606e1d` | critical | suspected | `src/server.js` | 5 | Possible secret committed to source | +| `750676119e4a` | high | confirmed | `.env` | — | Environment file in source tree | +| `3a198539c923` | high | suspected | `src/handler.go` | 14 | Raw SQL interpolation | +| `5bf85ae888a0` | high | suspected | `src/handler.go` | 19 | Shell command execution | +| `ef8bb39704d3` | high | suspected | `src/server.js` | 2 | Wildcard CORS | +| `4d59806aeb57` | medium | confirmed | `.` | — | No tests found | +| `eb3c41b3a186` | medium | suspected | `src/handler.go` | 10 | Hardcoded absolute path | +| `bb70e8e262d6` | medium | suspected | `src/handler.go` | 11 | Hardcoded private-network IP | +| `512b795dc551` | medium | suspected | `src/huge.go` | 1-901 | Large file | +| `7ed1cab08825` | medium | suspected | `src/server.js` | 7 | Mutation route in file with no visible auth | +| `2b765c240c96` | medium | suspected | `src/server.js` | 8 | Mutation route in file with no visible auth | +| `f99cd5bb5f2c` | low | suspected | `src/handler.go` | 22 | TODO/FIXME comment | +| `f3e510b70ec9` | low | suspected | `src/handler.go` | 9 | TODO/FIXME comment | +| `4a631055edd1` | low | suspected | `src/server.js` | 1 | TODO/FIXME comment | diff --git a/tests/fixtures/insecure-repo/reports/latest/scrum-test.md b/tests/fixtures/insecure-repo/reports/latest/scrum-test.md new file mode 100644 index 0000000..0934b7d --- /dev/null +++ b/tests/fixtures/insecure-repo/reports/latest/scrum-test.md @@ -0,0 +1,96 @@ +# Scrum Test — insecure-repo + +**Generated:** 2026-04-30T06:07:13.931830669Z +**Branch:** main · **Commit:** 70d68757f78ea722bf24585c73120c09d82c4fea + +## Verdict + +**blocked** — critical-severity finding present. See Confirmed Risks; rotate any leaked credentials, then re-run. + +## Evidence + +- repo path: `/home/profit/share/local-review-harness-full-md/tests/fixtures/insecure-repo` +- file count: 4 +- languages: Go (2), JavaScript (1) +- dependency manifests: 0 () +- test files/dirs: 0 +- LLM review: **skipped** (Phase C not implemented OR provider unavailable; see model-doctor.json) + +## Confirmed Risks + +| Severity | File:Line | Title | Evidence | +|---|---|---|---| +| high | `.env` | Environment file in source tree | `filename=.env` | +| medium | `.` | No tests found | `No test files or test directories detected (looked for *_test.go, *.test.{js,ts}, test_*.py, tests/, spec/)` | + +## Suspected Risks + +Each entry is a static-scan regex hit awaiting validation (Phase D / LLM cross-check). + +| Severity | File:Line | Title | Evidence | +|---|---|---|---| +| critical | `src/handler.go:23` | Possible secret committed to source | `const API_KEY = "sk-1234567890abcdefABCDEFGHIJKLMNOPQRSTUV"` | +| critical | `src/handler.go:23` | Possible secret committed to source | `const API_KEY = "sk-1234567890abcdefABCDEFGHIJKLMNOPQRSTUV"` | +| critical | `src/server.js:5` | Possible secret committed to source | `const AWS_KEY = "AKIAIOSFODNN7EXAMPLE";` | +| high | `src/handler.go:14` | Raw SQL interpolation | `q := fmt.Sprintf("SELECT * FROM users WHERE name = '%s'", name)` | +| high | `src/handler.go:19` | Shell command execution | `exec.Command("bash", "-c", cmd).Run()` | +| high | `src/server.js:2` | Wildcard CORS | `res.setHeader("Access-Control-Allow-Origin", "*");` | +| medium | `src/handler.go:10` | Hardcoded absolute path | `const HARDCODED_PATH = "/home/profit/secrets/key.pem"` | +| medium | `src/handler.go:11` | Hardcoded private-network IP | `const SERVER_IP = "192.168.1.176"` | +| medium | `src/huge.go:1-901` | Large file | `901 lines (limit: 800)` | +| medium | `src/server.js:7` | Mutation route in file with no visible auth | `app.post("/api/users", function(req, res) { /* no auth */ });` | +| medium | `src/server.js:8` | Mutation route in file with no visible auth | `app.delete("/api/admin", function(req, res) { /* no auth */ });` | +| low | `src/handler.go:22` | TODO/FIXME comment | `// FIXME: hardcoded creds` | +| low | `src/handler.go:9` | TODO/FIXME comment | `// TODO: rotate this and move to env` | +| low | `src/server.js:1` | TODO/FIXME comment | `// HACK: open CORS for now` | + +## Blocked Checks + +- LLM review (Phase 2 in REVIEW_PIPELINE.md). Reason: provider unavailable or stub. Next command: `review-harness model doctor` + +## Sprint Backlog + +**Sprint 0 — Reproducibility Gate** + +- Wire `just verify` (or equivalent) to run the static checks before every commit/PR. +- Add a CI step that fails on `critical` findings. +- Triage the 16 findings emitted by this run; mark each as accepted / blocking / dismiss-with-reason. + +**Sprint 1 — Trust Boundary Gate** + +- Resolve every `critical` and `high` finding before non-loopback deploy. +- Confirm auth posture for any mutation endpoint flagged as exposed. +- Replace raw SQL interpolation with parameterized queries. + +**Sprint 2 — Memory Correctness Gate** + +- (Phase E) Wire append-only `.memory/` writes for known-risks + fixed-patterns. +- Add a regression test that re-runs the harness and asserts no regression in confirmed-finding count. + +**Sprint 3 — Agent Loop Reality Gate** + +- (Phase C) Wire local-Ollama LLM review. +- (Phase D) Validator pass cross-checks every LLM finding against repo evidence. + +**Sprint 4 — Deployment Gate** + +- Ship the harness as a single static binary (`go build -o review-harness`). +- Document operator runbook (model setup, profile editing, output retention). + +## Acceptance Gates + +Each gate must be testable. Format: command + verifiable post-condition. + +1. **Reproducibility:** `review-harness repo .` exits 0; `reports/latest/repo-intake.json` exists with non-zero `file_count`. +2. **No false positives on a clean fixture:** `review-harness repo tests/fixtures/clean-repo` produces zero `confirmed` findings. +3. **Every documented static check fires on the insecure fixture:** `jq '[.findings[] | .check_id] | unique | length' reports/latest/static-findings.json` ≥ 8. +4. **Receipts are honest about degraded phases:** `jq '[.phases[] | select(.status == "degraded")]' reports/latest/receipts.json` lists every skipped/stubbed phase. +5. **Critical findings block production deploy:** at least one critical finding is currently present; resolve before deploy. + +## Next Commands + +1. Open the risk register: `cat reports/latest/risk-register.md` +2. Triage every `critical` finding; rotate any leaked credentials immediately. +- Probe the model provider: `review-harness model doctor` +- Re-run after fixes: `review-harness repo /home/profit/share/local-review-harness-full-md/tests/fixtures/insecure-repo` +- Generate the full Scrum bundle: `review-harness scrum /home/profit/share/local-review-harness-full-md/tests/fixtures/insecure-repo` diff --git a/tests/fixtures/insecure-repo/reports/latest/sprint-backlog.md b/tests/fixtures/insecure-repo/reports/latest/sprint-backlog.md new file mode 100644 index 0000000..c25419c --- /dev/null +++ b/tests/fixtures/insecure-repo/reports/latest/sprint-backlog.md @@ -0,0 +1,28 @@ +# Sprint Backlog + +**Sprint 0 — Reproducibility Gate** + +- Wire `just verify` (or equivalent) to run the static checks before every commit/PR. +- Add a CI step that fails on `critical` findings. +- Triage the 16 findings emitted by this run; mark each as accepted / blocking / dismiss-with-reason. + +**Sprint 1 — Trust Boundary Gate** + +- Resolve every `critical` and `high` finding before non-loopback deploy. +- Confirm auth posture for any mutation endpoint flagged as exposed. +- Replace raw SQL interpolation with parameterized queries. + +**Sprint 2 — Memory Correctness Gate** + +- (Phase E) Wire append-only `.memory/` writes for known-risks + fixed-patterns. +- Add a regression test that re-runs the harness and asserts no regression in confirmed-finding count. + +**Sprint 3 — Agent Loop Reality Gate** + +- (Phase C) Wire local-Ollama LLM review. +- (Phase D) Validator pass cross-checks every LLM finding against repo evidence. + +**Sprint 4 — Deployment Gate** + +- Ship the harness as a single static binary (`go build -o review-harness`). +- Document operator runbook (model setup, profile editing, output retention). diff --git a/tests/fixtures/insecure-repo/reports/latest/static-findings.json b/tests/fixtures/insecure-repo/reports/latest/static-findings.json new file mode 100644 index 0000000..7900dbf --- /dev/null +++ b/tests/fixtures/insecure-repo/reports/latest/static-findings.json @@ -0,0 +1,242 @@ +{ + "generated_at": "2026-04-30T06:07:13.951970576Z", + "findings": [ + { + "id": "750676119e4a", + "title": "Environment file in source tree", + "severity": "high", + "status": "confirmed", + "file": ".env", + "evidence": "filename=.env", + "reason": ".env files commonly hold real secrets and should not be tracked. If this is a sample, rename to .env.example with placeholder values.", + "suggested_fix": "Rename to .env.example with placeholders; add .env to .gitignore; rotate any committed secrets.", + "source": "static", + "confidence": 0.9, + "check_id": "static.env_file_committed" + }, + { + "id": "eb3c41b3a186", + "title": "Hardcoded absolute path", + "severity": "medium", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "10", + "evidence": "const HARDCODED_PATH = \"/home/profit/secrets/key.pem\"", + "reason": "Absolute path encoded in source — couples the binary to one filesystem layout. Move to config or env var.", + "source": "static", + "confidence": 0.7, + "check_id": "static.hardcoded_paths" + }, + { + "id": "5bf85ae888a0", + "title": "Shell command execution", + "severity": "high", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "19", + "evidence": "exec.Command(\"bash\", \"-c\", cmd).Run()", + "reason": "Direct subprocess/shell invocation. Confirm inputs are sanitized; prefer typed APIs over string-built commands.", + "source": "static", + "confidence": 0.6, + "check_id": "static.shell_execution" + }, + { + "id": "3a198539c923", + "title": "Raw SQL interpolation", + "severity": "high", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "14", + "evidence": "q := fmt.Sprintf(\"SELECT * FROM users WHERE name = '%s'\", name)", + "reason": "SQL assembled via string formatting/concatenation rather than parameterized query. Verify inputs aren't user-controlled.", + "suggested_fix": "Use parameterized queries / prepared statements; pass values via driver placeholders, not string interpolation.", + "source": "static", + "confidence": 0.6, + "check_id": "static.raw_sql_interpolation" + }, + { + "id": "9bc97c579efc", + "title": "Possible secret committed to source", + "severity": "critical", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "23", + "evidence": "const API_KEY = \"sk-1234567890abcdefABCDEFGHIJKLMNOPQRSTUV\"", + "reason": "OpenAI/OpenRouter-shaped key detected. If real, rotate immediately and move to a secret store.", + "suggested_fix": "Move secret to env var / secret manager; commit the .env.example with a placeholder; rotate the leaked credential.", + "source": "static", + "confidence": 0.75, + "check_id": "static.secret_patterns" + }, + { + "id": "9bc97c579efc", + "title": "Possible secret committed to source", + "severity": "critical", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "23", + "evidence": "const API_KEY = \"sk-1234567890abcdefABCDEFGHIJKLMNOPQRSTUV\"", + "reason": "Hardcoded credential pattern detected. If real, rotate immediately and move to a secret store.", + "suggested_fix": "Move secret to env var / secret manager; commit the .env.example with a placeholder; rotate the leaked credential.", + "source": "static", + "confidence": 0.75, + "check_id": "static.secret_patterns" + }, + { + "id": "f3e510b70ec9", + "title": "TODO/FIXME comment", + "severity": "low", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "9", + "evidence": "// TODO: rotate this and move to env", + "reason": "Inline marker for deferred work. Audit whether the deferred concern is now blocking.", + "source": "static", + "confidence": 0.95, + "check_id": "static.todo_comments" + }, + { + "id": "f99cd5bb5f2c", + "title": "TODO/FIXME comment", + "severity": "low", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "22", + "evidence": "// FIXME: hardcoded creds", + "reason": "Inline marker for deferred work. Audit whether the deferred concern is now blocking.", + "source": "static", + "confidence": 0.95, + "check_id": "static.todo_comments" + }, + { + "id": "bb70e8e262d6", + "title": "Hardcoded private-network IP", + "severity": "medium", + "status": "suspected", + "file": "src/handler.go", + "line_hint": "11", + "evidence": "const SERVER_IP = \"192.168.1.176\"", + "reason": "RFC 1918 / link-local IP literal in source. Move to config so the binary isn't tied to one network.", + "source": "static", + "confidence": 0.7, + "check_id": "static.hardcoded_local_ip" + }, + { + "id": "512b795dc551", + "title": "Large file", + "severity": "medium", + "status": "suspected", + "file": "src/huge.go", + "line_hint": "1-901", + "evidence": "901 lines (limit: 800)", + "reason": "File exceeds the configured size threshold. Long files are a refactor target — split by responsibility.", + "source": "static", + "confidence": 1, + "check_id": "static.large_files" + }, + { + "id": "ef8bb39704d3", + "title": "Wildcard CORS", + "severity": "high", + "status": "suspected", + "file": "src/server.js", + "line_hint": "2", + "evidence": "res.setHeader(\"Access-Control-Allow-Origin\", \"*\");", + "reason": "Access-Control-Allow-Origin: * permits cross-origin reads from any domain. Narrow to an explicit allowlist unless this endpoint is intentionally public.", + "source": "static", + "confidence": 0.85, + "check_id": "static.broad_cors" + }, + { + "id": "d3c2c5606e1d", + "title": "Possible secret committed to source", + "severity": "critical", + "status": "suspected", + "file": "src/server.js", + "line_hint": "5", + "evidence": "const AWS_KEY = \"AKIAIOSFODNN7EXAMPLE\";", + "reason": "AWS access key ID detected. If real, rotate immediately and move to a secret store.", + "suggested_fix": "Move secret to env var / secret manager; commit the .env.example with a placeholder; rotate the leaked credential.", + "source": "static", + "confidence": 0.75, + "check_id": "static.secret_patterns" + }, + { + "id": "4a631055edd1", + "title": "TODO/FIXME comment", + "severity": "low", + "status": "suspected", + "file": "src/server.js", + "line_hint": "1", + "evidence": "// HACK: open CORS for now", + "reason": "Inline marker for deferred work. Audit whether the deferred concern is now blocking.", + "source": "static", + "confidence": 0.95, + "check_id": "static.todo_comments" + }, + { + "id": "7ed1cab08825", + "title": "Mutation route in file with no visible auth", + "severity": "medium", + "status": "suspected", + "file": "src/server.js", + "line_hint": "7", + "evidence": "app.post(\"/api/users\", function(req, res) { /* no auth */ });", + "reason": "POST/PUT/DELETE/PATCH route registered in a file with no visible auth middleware. May still be auth'd at a higher layer — confirm.", + "source": "static", + "confidence": 0.4, + "check_id": "static.exposed_mutation_endpoint" + }, + { + "id": "2b765c240c96", + "title": "Mutation route in file with no visible auth", + "severity": "medium", + "status": "suspected", + "file": "src/server.js", + "line_hint": "8", + "evidence": "app.delete(\"/api/admin\", function(req, res) { /* no auth */ });", + "reason": "POST/PUT/DELETE/PATCH route registered in a file with no visible auth middleware. May still be auth'd at a higher layer — confirm.", + "source": "static", + "confidence": 0.4, + "check_id": "static.exposed_mutation_endpoint" + }, + { + "id": "4d59806aeb57", + "title": "No tests found", + "severity": "medium", + "status": "confirmed", + "file": ".", + "evidence": "No test files or test directories detected (looked for *_test.go, *.test.{js,ts}, test_*.py, tests/, spec/)", + "reason": "Repository has source code but no test surface. Refactoring or extending without test cover is high-risk.", + "source": "static", + "confidence": 0.95, + "check_id": "static.missing_tests" + } + ], + "summary": { + "total": 16, + "confirmed": 2, + "suspected": 14, + "rejected": 0, + "critical": 3, + "high": 4, + "medium": 6, + "low": 3, + "by_source": { + "static": 16 + }, + "by_check": { + "static.broad_cors": 1, + "static.env_file_committed": 1, + "static.exposed_mutation_endpoint": 2, + "static.hardcoded_local_ip": 1, + "static.hardcoded_paths": 1, + "static.large_files": 1, + "static.missing_tests": 1, + "static.raw_sql_interpolation": 1, + "static.secret_patterns": 3, + "static.shell_execution": 1, + "static.todo_comments": 3 + } + } +}