golangLAKEHOUSE/internal/distillation/audit_full_test.go

package distillation

import (
	"os"
	"path/filepath"
	"strings"
	"testing"
)

// TestRunAuditFull_EmptyRoot: missing data directories yield
// failures on required checks but doesn't error out the run.
// Operator running on a fresh box sees the report with the
// expected "missing" actuals.
func TestRunAuditFull_EmptyRoot(t *testing.T) {
	tmp := t.TempDir()
	report := RunAuditFull(AuditFullOptions{Root: tmp})
	if len(report.Checks) == 0 {
		t.Fatalf("expected check rows even on empty root, got %d", len(report.Checks))
	}
	// Phase 3's "scored-runs on disk" must fail (required); the
	// failure count rises by at least 1.
	if report.Failed < 1 {
		t.Errorf("expected ≥1 required failure on empty root, got %d", report.Failed)
	}
}

// TestRunAuditFull_FullFixtureFlow seeds a complete data layout
// and verifies all phases produce the expected metrics + a clean
// PASS verdict. Locks the end-to-end orchestration.
func TestRunAuditFull_FullFixtureFlow(t *testing.T) {
	tmp := t.TempDir()
	// scored-runs: one accepted record (passes phase 3 required check)
	scoredDir := filepath.Join(tmp, "data", "scored-runs", "2026", "05", "01")
	if err := os.MkdirAll(scoredDir, 0o755); err != nil {
		t.Fatalf("mkdir scored: %v", err)
	}
	scoredJSONL := `{"category":"accepted","evidence_run_id":"r1","provenance":{"source_file":"data/_kb/scrum_reviews.jsonl","sig_hash":"a1b2c3d4e5f60718293a4b5c6d7e8f900112233445566778899aabbccddeeff0","recorded_at":"2026-05-01T00:00:00Z"}}
{"category":"partially_accepted","evidence_run_id":"r2","provenance":{"source_file":"data/_kb/scrum_reviews.jsonl","sig_hash":"a1b2c3d4e5f60718293a4b5c6d7e8f900112233445566778899aabbccddeeff1","recorded_at":"2026-05-01T00:00:00Z"}}
{"category":"rejected","evidence_run_id":"r3","provenance":{"source_file":"data/_kb/scrum_reviews.jsonl","sig_hash":"a1b2c3d4e5f60718293a4b5c6d7e8f900112233445566778899aabbccddeeff2","recorded_at":"2026-05-01T00:00:00Z"}}
`
	if err := os.WriteFile(filepath.Join(scoredDir, "run.jsonl"), []byte(scoredJSONL), 0o644); err != nil {
		t.Fatalf("write scored: %v", err)
	}

	// SFT export: only legal quality scores, valid sig_hash on every row.
	sftDir := filepath.Join(tmp, "exports", "sft")
	if err := os.MkdirAll(sftDir, 0o755); err != nil {
		t.Fatalf("mkdir sft: %v", err)
	}
	sftJSONL := `{"quality_score":"accepted","provenance":{"sig_hash":"a1b2c3d4e5f60718293a4b5c6d7e8f900112233445566778899aabbccddeeff0"}}
{"quality_score":"partially_accepted","provenance":{"sig_hash":"a1b2c3d4e5f60718293a4b5c6d7e8f900112233445566778899aabbccddeeff1"}}
`
	if err := os.WriteFile(filepath.Join(sftDir, "instruction_response.jsonl"), []byte(sftJSONL), 0o644); err != nil {
		t.Fatalf("write sft: %v", err)
	}

	// RAG: no rejected leaks
	ragDir := filepath.Join(tmp, "exports", "rag")
	if err := os.MkdirAll(ragDir, 0o755); err != nil {
		t.Fatalf("mkdir rag: %v", err)
	}
	ragJSONL := `{"success_score":"accepted","provenance":{"sig_hash":"a1b2c3d4e5f60718293a4b5c6d7e8f900112233445566778899aabbccddeeff0"}}
`
	if err := os.WriteFile(filepath.Join(ragDir, "playbooks.jsonl"), []byte(ragJSONL), 0o644); err != nil {
		t.Fatalf("write rag: %v", err)
	}

	// Preference: distinct chosen vs rejected, no self-pairs
	prefDir := filepath.Join(tmp, "exports", "preference")
	if err := os.MkdirAll(prefDir, 0o755); err != nil {
		t.Fatalf("mkdir pref: %v", err)
	}
	prefJSONL := `{"chosen_run_id":"a","rejected_run_id":"b","chosen":"good","rejected":"bad","provenance":{"sig_hash":"a1b2c3d4e5f60718293a4b5c6d7e8f900112233445566778899aabbccddeeff0"}}
`
	if err := os.WriteFile(filepath.Join(prefDir, "chosen_rejected.jsonl"), []byte(prefJSONL), 0o644); err != nil {
		t.Fatalf("write pref: %v", err)
	}

	report := RunAuditFull(AuditFullOptions{Root: tmp})
	if report.Failed != 0 {
		t.Errorf("clean fixture should have 0 required failures, got %d", report.Failed)
		for _, c := range report.Checks {
			if c.Required && !c.Passed {
				t.Logf("  failed: phase=%d name=%q actual=%q", c.Phase, c.Name, c.Actual)
			}
		}
	}
	// Metrics populated correctly
	if report.Metrics["p3_accepted"] != 1 {
		t.Errorf("p3_accepted: got %d, want 1", report.Metrics["p3_accepted"])
	}
	if report.Metrics["p3_partial"] != 1 {
		t.Errorf("p3_partial: got %d, want 1", report.Metrics["p3_partial"])
	}
	if report.Metrics["p3_rejected"] != 1 {
		t.Errorf("p3_rejected: got %d, want 1", report.Metrics["p3_rejected"])
	}
	if report.Metrics["p4_sft_rows"] != 2 {
		t.Errorf("p4_sft_rows: got %d, want 2", report.Metrics["p4_sft_rows"])
	}
	if report.Metrics["p4_rag_rows"] != 1 {
		t.Errorf("p4_rag_rows: got %d, want 1", report.Metrics["p4_rag_rows"])
	}
	if report.Metrics["p4_pref_pairs"] != 1 {
		t.Errorf("p4_pref_pairs: got %d, want 1", report.Metrics["p4_pref_pairs"])
	}
}

// TestPhase4_SftFirewallCatchesRejected: contamination must never
// leak into SFT export. Test seeds a row with a forbidden
// quality_score and asserts the firewall flags it.
func TestPhase4_SftFirewallCatchesRejected(t *testing.T) {
	tmp := t.TempDir()
	sftDir := filepath.Join(tmp, "exports", "sft")
	if err := os.MkdirAll(sftDir, 0o755); err != nil {
		t.Fatalf("mkdir: %v", err)
	}
	bad := `{"quality_score":"rejected","provenance":{"sig_hash":"a1b2c3d4e5f60718293a4b5c6d7e8f900112233445566778899aabbccddeeff0"}}
`
	if err := os.WriteFile(filepath.Join(sftDir, "instruction_response.jsonl"), []byte(bad), 0o644); err != nil {
		t.Fatalf("write: %v", err)
	}
	report := RunAuditFull(AuditFullOptions{Root: tmp})
	found := false
	for _, c := range report.Checks {
		if c.Phase == 4 && strings.Contains(c.Name, "SFT contamination firewall") {
			if c.Passed {
				t.Errorf("firewall should fail on rejected SFT row, but check passed")
			}
			if c.Actual != "1" {
				t.Errorf("firewall actual: got %q, want '1'", c.Actual)
			}
			found = true
		}
	}
	if !found {
		t.Errorf("firewall check not present in report")
	}
}

// TestPhase4_PreferenceSelfPairCaught: same chosen + rejected run_id
// is structural noise and must be flagged.
func TestPhase4_PreferenceSelfPairCaught(t *testing.T) {
	tmp := t.TempDir()
	prefDir := filepath.Join(tmp, "exports", "preference")
	if err := os.MkdirAll(prefDir, 0o755); err != nil {
		t.Fatalf("mkdir: %v", err)
	}
	bad := `{"chosen_run_id":"X","rejected_run_id":"X","chosen":"a","rejected":"b","provenance":{"sig_hash":"a1b2c3d4e5f60718293a4b5c6d7e8f900112233445566778899aabbccddeeff0"}}
`
	if err := os.WriteFile(filepath.Join(prefDir, "chosen_rejected.jsonl"), []byte(bad), 0o644); err != nil {
		t.Fatalf("write: %v", err)
	}
	report := RunAuditFull(AuditFullOptions{Root: tmp})
	found := false
	for _, c := range report.Checks {
		if c.Phase == 4 && strings.Contains(c.Name, "self-pairs") {
			if c.Passed {
				t.Errorf("self-pair check should fail, but passed")
			}
			found = true
		}
	}
	if !found {
		t.Errorf("self-pair check not present in report")
	}
}

// TestPhase4_ProvenanceRequiresValidSha256: bad sig_hash must be
// flagged. Locks the regex shape — only 64-char lowercase hex.
func TestPhase4_ProvenanceRequiresValidSha256(t *testing.T) {
	tmp := t.TempDir()
	sftDir := filepath.Join(tmp, "exports", "sft")
	if err := os.MkdirAll(sftDir, 0o755); err != nil {
		t.Fatalf("mkdir: %v", err)
	}
	// Three rows: one valid, one wrong-length, one wrong-charset (uppercase).
	bad := `{"quality_score":"accepted","provenance":{"sig_hash":"a1b2c3d4e5f60718293a4b5c6d7e8f900112233445566778899aabbccddeeff0"}}
{"quality_score":"accepted","provenance":{"sig_hash":"too_short"}}
{"quality_score":"accepted","provenance":{"sig_hash":"A1B2C3D4E5F60718293A4B5C6D7E8F900112233445566778899AABBCCDDEEFF0"}}
`
	if err := os.WriteFile(filepath.Join(sftDir, "instruction_response.jsonl"), []byte(bad), 0o644); err != nil {
		t.Fatalf("write: %v", err)
	}
	report := RunAuditFull(AuditFullOptions{Root: tmp})
	for _, c := range report.Checks {
		if c.Phase == 4 && strings.Contains(c.Name, "sig_hash") {
			if c.Actual != "2 missing" {
				t.Errorf("provenance check: got actual=%q, want '2 missing'", c.Actual)
			}
			if c.Passed {
				t.Errorf("provenance check should fail with 2 bad sig_hashes")
			}
		}
	}
}

// TestFormatAuditFullReport_RendersCheckTable: smoke-test the
// Markdown formatter — operators should see the right verdict +
// per-phase rows.
func TestFormatAuditFullReport_RendersCheckTable(t *testing.T) {
	report := PhaseCheckReport{
		GitHEAD: "deadbeef",
		Checks: []PhaseCheck{
			{Phase: 0, Name: "test check", Expected: "x", Actual: "x", Passed: true, Required: true},
			{Phase: 4, Name: "fail check", Expected: "0", Actual: "5", Passed: false, Required: true},
		},
		Metrics: map[string]int64{"p3_accepted": 42, "p4_sft_rows": 17},
		Failed:  1,
		Skipped: 4,
	}
	out := FormatAuditFullReport(report)
	for _, want := range []string{"FAIL", "deadbeef", "test check", "fail check", "p3_accepted", "42", "deferred"} {
		if !strings.Contains(out, want) {
			t.Errorf("expected %q in formatted report:\n%s", want, out)
		}
	}
}