Architectural snapshot of the lakehouse codebase at the point where the
full matrix-driven agent loop with Mem0 versioning + deletion was
validated end-to-end.
WHAT THIS REPO IS
A clean single-commit snapshot of the lakehouse code. Heavy test data
(.parquet datasets, vector indexes) excluded — see REPLICATION.md for
regen path. Full lakehouse history at git.agentview.dev/profit/lakehouse.
WHAT WAS PROVEN
- Vector retrieval across multi-corpora matrix (chicago_permits + entity
briefs + sec_tickers + distilled procedural + llm_team runs)
- Observer hand-review (cloud + heuristic fallback) gating each candidate
- Local-model agent loop (qwen3.5:latest) with tool use + scratchpad
- Playbook seal on success → next-iter retrieval surfaces it as preamble
- Mem0 versioning + deletion in pathway_memory:
* UPSERT: ADD on new workflow, UPDATE bumps replay_count on identical
* REVISE: chains versions, parent.superseded_at + superseded_by stamped
* RETIRE: marks specific trace retired with reason, excluded from retrieval
* HISTORY: walks chain root→tip, cycle-safe
KEY DIRECTORIES
- crates/vectord/src/pathway_memory.rs — Mem0 ops live here
- crates/vectord/src/playbook_memory.rs — original Mem0 reference
- tests/agent_test/ — local-model agent harness + PRD + session archives
- scripts/dump_raw_corpus.sh — MinIO bucket dump (raw test corpus)
- scripts/vectorize_raw_corpus.ts — corpus → vector indexes
- scripts/analyze_chicago_contracts.ts — real inference pipeline
- scripts/seal_agent_playbook.ts — Mem0 upsert from agent traces
Replication: see REPLICATION.md for Debian 13 clean install + cloud-only
adaptation (no local Ollama).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
87 lines
2.8 KiB
TypeScript
87 lines
2.8 KiB
TypeScript
// Apply a proposal to disk with Mem0-aligned ADD / UPDATE / NOOP
|
|
// semantics. Whole-file writes only — never deletes. All paths
|
|
// validated repo-relative + path-traversal-free.
|
|
//
|
|
// Three outcomes per file:
|
|
// ADD — file didn't exist, new content written (is_new must be true)
|
|
// UPDATE — file existed, content differs from current, overwritten
|
|
// NOOP — file existed, content identical to current, nothing written
|
|
//
|
|
// The NOOP case is important: a cloud proposal that "re-derives" an
|
|
// existing file shouldn't churn the repo or burn test cycles.
|
|
|
|
import { writeFile, readFile, mkdir, access } from "node:fs/promises";
|
|
import { dirname, resolve, relative } from "node:path";
|
|
import type { ApplyOutcome, Proposal } from "./types.ts";
|
|
|
|
const REPO_ROOT = "/home/profit/lakehouse";
|
|
|
|
export async function applyProposal(p: Proposal): Promise<ApplyOutcome> {
|
|
const added: string[] = [];
|
|
const updated: string[] = [];
|
|
const noop: string[] = [];
|
|
const errors: string[] = [];
|
|
|
|
for (const f of p.files) {
|
|
const err = validatePath(f.path);
|
|
if (err) {
|
|
errors.push(`${f.path}: ${err}`);
|
|
continue;
|
|
}
|
|
const abs = resolve(REPO_ROOT, f.path);
|
|
try {
|
|
const exists = await fileExists(abs);
|
|
|
|
if (f.is_new && exists) {
|
|
errors.push(`${f.path}: is_new=true but file exists`);
|
|
continue;
|
|
}
|
|
if (!f.is_new && !exists) {
|
|
// Proposal claims to update a file that doesn't exist.
|
|
// Refuse rather than silently ADD — model is confused about
|
|
// repo state, and we want that surfaced, not papered over.
|
|
errors.push(`${f.path}: is_new=false but file does not exist`);
|
|
continue;
|
|
}
|
|
|
|
if (!f.is_new) {
|
|
// UPDATE candidate — compare to detect NOOP.
|
|
const current = await readFile(abs, "utf8");
|
|
if (current === f.content) {
|
|
noop.push(f.path);
|
|
continue;
|
|
}
|
|
await writeFile(abs, f.content);
|
|
updated.push(f.path);
|
|
} else {
|
|
// ADD — file doesn't exist yet.
|
|
await mkdir(dirname(abs), { recursive: true });
|
|
await writeFile(abs, f.content);
|
|
added.push(f.path);
|
|
}
|
|
} catch (e) {
|
|
errors.push(`${f.path}: ${(e as Error).message}`);
|
|
}
|
|
}
|
|
return { added, updated, noop, errors };
|
|
}
|
|
|
|
function validatePath(p: string): string | null {
|
|
if (!p) return "empty path";
|
|
if (p.startsWith("/")) return "must be repo-relative";
|
|
if (p.startsWith("..") || p.includes("/../")) return "path traversal";
|
|
const abs = resolve(REPO_ROOT, p);
|
|
const rel = relative(REPO_ROOT, abs);
|
|
if (rel.startsWith("..")) return "resolves outside repo";
|
|
return null;
|
|
}
|
|
|
|
async function fileExists(abs: string): Promise<boolean> {
|
|
try {
|
|
await access(abs);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|