27b1d27605
Some checks failed
lakehouse/auditor 9 blocking issues: todo!() macro call in tests/real-world/scrum_master_pipeline.ts
Phase 0 — docs/recon/local-distillation-recon.md
Inventories the 23 KB JSONL streams + 20 vector corpora + auditor's
kb_index.ts as substrate for the now.md distillation pipeline. Maps
spec modules to existing producers, identifies real gaps, lists 9
schemas to formalize. ZERO implementation in recon — gating doc only.
Phase 1 — auditor/schemas/distillation/
9 schemas + foundation types + 48 tests passing in 502ms:
types.ts shared validators + canonicalSha256
evidence_record.ts EVIDENCE_SCHEMA_VERSION=1, ModelRole enum
scored_run.ts 4 categories pinned, anchor_grounding ∈ [0,1]
receipt.ts git_sha 40-char, sha256 file refs, validation_pass:bool
playbook.ts non-empty source_run_ids + acceptance_criteria
scratchpad_summary.ts validation_status enum, hash sha256
model_ledger.ts success_rate ∈ [0,1], sample_count ≥ 1
rag_sample.ts success_score ∈ {accepted, partially_accepted}
sft_sample.ts quality_score MUST be 'accepted' (no leak)
preference_sample.ts chosen != rejected, source_run_ids must differ
evidence_record.test.ts 10 tests, JSON-fixture round-trip
schemas.test.ts 30 tests, inline fixtures
realdata.test.ts 8 tests, real-JSONL probe
Real-data validation probe (one of the 3 notables from recon):
46 rows across 7 sources, 100% pass. distilled_facts/procedures alive.
Report at data/_kb/realdata_validation_report.md (also written by the
test). Confirms schema fits existing producers without migration.
Phase 2 scaffold — scripts/distillation/transforms.ts
Promoted PROBES from realdata.test.ts into a real TRANSFORMS array
covering 12 source streams (8 Tier 1 validated + 4 Tier 2 from
recon's untested-streams list). Pure functions: no I/O, no model
calls, no clock reads. Caller supplies recorded_at + sig_hash so
materializer is deterministic by construction.
Spec non-negotiables enforced at schema layer (defense in depth):
- provenance{source_file, sig_hash, recorded_at} required everywhere
- schema_version mismatch hard-rejects (forward-compat gate)
- SFT no-leak: validateSftSample REJECTS partially_accepted, rejected,
needs_human_review — three explicit tests
- Every score has WHY (reasons non-empty)
- Every playbook traces to source (source_run_ids non-empty)
- Every preference has WHY (reason non-empty)
- Receipts substantive (git_sha 40-char, sha256 64-char, validation_pass:bool)
Branch carries uncommitted auditor rebuild work (mode.rs + modes.toml
+ inference.ts + static.ts) blocked on upstream Ollama Cloud kimi-k2
500 ISE; held pending recon-driven design decisions.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
69 lines
3.5 KiB
TypeScript
69 lines
3.5 KiB
TypeScript
// Playbook — procedural knowledge extracted from accepted/partially-
|
|
// accepted runs. Different from pathway_memory's bug_fingerprints (which
|
|
// are pattern-detectors) — playbooks describe HOW to handle a task type.
|
|
import {
|
|
ValidationResult, requireString, requireIsoTimestamp, requireProvenance, requireStringArray,
|
|
} from "./types";
|
|
|
|
export const PLAYBOOK_SCHEMA_VERSION = 1;
|
|
|
|
export interface Playbook {
|
|
schema_version: number;
|
|
playbook_id: string;
|
|
task_type: string; // e.g. "scrum_review", "pr_audit", "staffing.fill"
|
|
problem_pattern: string; // when does this playbook apply?
|
|
useful_context: string[]; // what to retrieve before running
|
|
model_routing_path: string[]; // ordered model attempts that worked
|
|
commands_worked: string[];
|
|
commands_failed: string[];
|
|
validation_steps: string[];
|
|
repo_files_touched: string[];
|
|
recovery_strategy: string; // what to do when the path fails
|
|
known_failure_modes: string[];
|
|
escalation_threshold: string; // when to switch to a stronger model
|
|
acceptance_criteria: string[]; // how to know it succeeded
|
|
source_run_ids: string[]; // FK to EvidenceRecord.run_id (provenance — every playbook traces to source)
|
|
created_at: string;
|
|
provenance: { source_file: string; line_offset?: number; sig_hash: string; recorded_at: string };
|
|
}
|
|
|
|
export function validatePlaybook(input: unknown): ValidationResult<Playbook> {
|
|
const errors: string[] = [];
|
|
if (typeof input !== "object" || input === null) return { valid: false, errors: ["expected object"] };
|
|
const r = input as Record<string, unknown>;
|
|
let ok = true;
|
|
|
|
if (r.schema_version !== PLAYBOOK_SCHEMA_VERSION) {
|
|
errors.push(`schema_version: expected ${PLAYBOOK_SCHEMA_VERSION}, got ${JSON.stringify(r.schema_version)}`);
|
|
ok = false;
|
|
}
|
|
ok = requireString(r.playbook_id, "playbook_id", errors) && ok;
|
|
ok = requireString(r.task_type, "task_type", errors) && ok;
|
|
ok = requireString(r.problem_pattern, "problem_pattern", errors) && ok;
|
|
ok = requireString(r.recovery_strategy, "recovery_strategy", errors) && ok;
|
|
ok = requireString(r.escalation_threshold, "escalation_threshold", errors) && ok;
|
|
ok = requireIsoTimestamp(r.created_at, "created_at", errors) && ok;
|
|
ok = requireStringArray(r.useful_context, "useful_context", errors) && ok;
|
|
ok = requireStringArray(r.model_routing_path, "model_routing_path", errors) && ok;
|
|
ok = requireStringArray(r.commands_worked, "commands_worked", errors) && ok;
|
|
ok = requireStringArray(r.commands_failed, "commands_failed", errors) && ok;
|
|
ok = requireStringArray(r.validation_steps, "validation_steps", errors) && ok;
|
|
ok = requireStringArray(r.repo_files_touched, "repo_files_touched", errors) && ok;
|
|
ok = requireStringArray(r.known_failure_modes, "known_failure_modes", errors) && ok;
|
|
ok = requireStringArray(r.acceptance_criteria, "acceptance_criteria", errors) && ok;
|
|
ok = requireStringArray(r.source_run_ids, "source_run_ids", errors) && ok;
|
|
|
|
if (Array.isArray(r.source_run_ids) && r.source_run_ids.length === 0) {
|
|
errors.push("source_run_ids: must be non-empty — every playbook traces to source evidence (spec non-negotiable)");
|
|
ok = false;
|
|
}
|
|
if (Array.isArray(r.acceptance_criteria) && r.acceptance_criteria.length === 0) {
|
|
errors.push("acceptance_criteria: must be non-empty — every playbook needs success criteria (spec non-negotiable)");
|
|
ok = false;
|
|
}
|
|
ok = requireProvenance(r.provenance, "provenance", errors) && ok;
|
|
|
|
if (!ok) return { valid: false, errors };
|
|
return { valid: true, value: r as unknown as Playbook };
|
|
}
|