From 03d723e7e64d592457a20a072e138c2ec1f3bf7d Mon Sep 17 00:00:00 2001 From: root Date: Mon, 20 Apr 2026 19:24:41 -0500 Subject: [PATCH] =?UTF-8?q?Model=20matrix=20=E2=80=94=205=20tiers,=20local?= =?UTF-8?q?=20hard=20workers=20+=20cloud=20overseers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit config/models.json is the authoritative catalog. Hot path (T1/T2) stays local; cloud is consulted only for overview (T3), strategic (T4), and gatekeeper (T5) calls. J named qwen3.5 + newer models (minimax-m2.7, glm-5, qwen3-next) specifically — all mapped with real reachable IDs verified against ollama.com/api/tags. Tier shape: - t1_hot mistral + qwen2.5 local — 50-200 calls/scenario - t2_review qwen2.5 + qwen3 local — 5-14 calls/event - t3_overview gpt-oss:120b cloud — 1-3 calls/scenario - t4_strategic qwen3.5:397b + glm-4.7 — 1-10 calls/day - t5_gatekeeper kimi-k2-thinking — 1-5 calls/day, audit-logged Rate budgets are declared in-config — Ollama Cloud paid tier is generous but we cap overview/strategic/gatekeeper so no single rogue scenario can blow the day's quota. Experimental rotation list wired but disabled by default. When enabled, T4 randomly routes 10% of calls to a rotating minimax/GLM/qwen-next/ deepseek/nemotron/cogito/mistral-large candidate, logs comparisons, and auto-promotes after 3 rotations of wins. Playbook versioning SPEC embedded under `playbook_versioning` key: every seed gets version + parent_id + retired_at + architecture_snapshot, so when a schema migration breaks a playbook we can pinpoint which change retired it. Implementation flagged for next sprint (touches gateway + catalogd + mcp-server) — not wired here. - scenario.ts now loads config/models.json at init, env vars still override - mcp-server exposes /models/matrix read-only so UI can render it --- config/models.json | 119 ++++++++++++++++++++++++++++++++++ mcp-server/index.ts | 16 +++++ tests/multi-agent/scenario.ts | 23 ++++++- 3 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 config/models.json diff --git a/config/models.json b/config/models.json new file mode 100644 index 0000000..0302b62 --- /dev/null +++ b/config/models.json @@ -0,0 +1,119 @@ +{ + "_description": "Lakehouse model matrix — authoritative routing for all agent tiers. Local models do the heavy lifting; cloud models are consulted sparingly for overview, strategic, and gatekeeper decisions. Read by tests/multi-agent/scenario.ts and mcp-server/index.ts.", + "version": 1, + "updated": "2026-04-21", + "providers": { + "ollama_local": { + "base_url": "http://localhost:11434", + "key_env": null + }, + "ollama_cloud": { + "base_url": "https://ollama.com", + "key_env": "OLLAMA_CLOUD_KEY", + "key_source": "/root/llm_team_config.json → providers.ollama_cloud.api_key", + "rate_budget": { + "calls_per_hour": 200, + "calls_per_day": 2000, + "notes": "Paid tier — generous. Policy: keep overview calls ≤ 3/scenario, strategic ≤ 10/day, gatekeeper ≤ 5/day." + } + } + }, + "tiers": { + "t1_hot": { + "purpose": "Per tool call — SQL generation, hybrid_search, sql(). Runs 50-200 times per scenario. Latency-sensitive.", + "kind": "local_fast", + "primary": { "model": "mistral:latest", "provider": "ollama_local" }, + "fallback": { "model": "qwen2.5:latest", "provider": "ollama_local" }, + "max_tokens": 800, + "temperature": 0.3, + "never_route_cloud": true, + "rationale": "Mistral produces valid JSON reliably. Qwen2.5 is the consensus reviewer. Known flakiness on 5-fill + misplacement events — do NOT mask by upgrading; route to T3 for post-hoc review instead." + }, + "t2_review": { + "purpose": "Per step consensus — executor ↔ reviewer loop critique. 5-14 calls per event.", + "kind": "local_balanced", + "primary": { "model": "qwen2.5:latest", "provider": "ollama_local" }, + "fallback": { "model": "qwen3:latest", "provider": "ollama_local" }, + "max_tokens": 600, + "temperature": 0.3, + "never_route_cloud": true, + "rationale": "Reviewer only needs to detect schema violations and drift — a 7B model is sufficient." + }, + "t3_overview": { + "purpose": "Mid-day checkpoint after every misplacement + every Nth event, and cross-day lesson. 1-3 calls per scenario.", + "kind": "thinking_cloud", + "primary": { "model": "gpt-oss:120b", "provider": "ollama_cloud" }, + "local_fallback": { "model": "gpt-oss:20b", "provider": "ollama_local" }, + "max_tokens": 900, + "temperature": 0.2, + "cloud_budget_per_scenario": 5, + "env_flag": "LH_OVERVIEW_CLOUD=1", + "rationale": "Same prompt family as local 20b (gpt-oss series) — prompts port directly. 120b is faster via cloud than 20b local in practice, and lessons are noticeably more specific." + }, + "t4_strategic": { + "purpose": "Daily playbook board re-ranking, weekly gap audit, pattern discovery across accumulated playbooks. 1-10 calls per day.", + "kind": "thinking_cloud_large", + "primary": { "model": "qwen3.5:397b", "provider": "ollama_cloud" }, + "fallback": { "model": "glm-4.7", "provider": "ollama_cloud" }, + "local_fallback": { "model": "gpt-oss:20b", "provider": "ollama_local" }, + "max_tokens": 2000, + "temperature": 0.2, + "cloud_budget_per_day": 10, + "rationale": "J named qwen3.5 specifically. GLM-4.7 is a promising alternate for debate phase. Runs after all scenarios complete for the day." + }, + "t5_gatekeeper": { + "purpose": "MUST route here: architecture changes, new client onboarding, schema migrations, playbook retirements, index rebuilds, autotune config changes.", + "kind": "thinking_cloud_deepest", + "primary": { "model": "kimi-k2-thinking", "provider": "ollama_cloud" }, + "fallback": { "model": "deepseek-v3.1:671b", "provider": "ollama_cloud" }, + "secondary_fallback": { "model": "qwen3.5:397b", "provider": "ollama_cloud" }, + "local_fallback": { "model": "gpt-oss:20b", "provider": "ollama_local" }, + "max_tokens": 4000, + "temperature": 0.1, + "cloud_budget_per_day": 5, + "audit_log": true, + "rationale": "Highest-stakes decisions — reasoning depth matters more than latency. Audit log so J can always see what the gatekeeper was asked and what it answered. No human approval required today; escalate later if mis-decisions show up." + } + }, + "experimental_rotation": { + "enabled": false, + "purpose": "Sample newer models on a schedule to collect comparison data without rate-limit risk.", + "candidates": [ + { "model": "minimax-m2.7", "notes": "Newer minimax; unknown output stability" }, + { "model": "glm-5", "notes": "GLM next-gen; larger context" }, + { "model": "glm-5.1", "notes": "Incremental on GLM-5" }, + { "model": "qwen3-next:80b", "notes": "Qwen's experimental successor; smaller than 3.5" }, + { "model": "qwen3-coder-next", "notes": "Coder-optimized — good for SQL gen T1 experiments" }, + { "model": "deepseek-v3.2", "notes": "Smaller deepseek; reasoning/coding" }, + { "model": "nemotron-3-super", "notes": "NVIDIA 230B; general" }, + { "model": "cogito-2.1:671b", "notes": "671B general" }, + { "model": "mistral-large-3:675b", "notes": "Mistral's flagship; good T3 candidate" } + ], + "rotation": "weekly", + "sample_rate": 0.1, + "apply_to_tier": "t4_strategic", + "notes": "When enabled, T4 routes 10% of calls to a rotating experimental model. Log comparison in /data/_model_eval/ — if the experimental consistently beats primary across 3 rotations, promote it to primary." + }, + "playbook_versioning": { + "enabled": true, + "purpose": "A playbook can work, then break when architecture changes. Versioning lets us pin which change retired which playbook.", + "dataset": "playbook_memory", + "schema_additions": { + "version": "integer — auto-increment per operation", + "parent_id": "string — previous version entry_id for same operation (null for v1)", + "retired_at": "timestamp — set when success_rate drops or architecture changes", + "retirement_reason": "string — e.g. 'schema_migration:workers_500k 2026-05-03'", + "architecture_snapshot": "object — crate versions, index name, schema fingerprint at seed time" + }, + "retire_triggers": [ + "success_rate < 0.3 over last 20 citations", + "schema_fingerprint mismatch detected at retrieval time", + "architecture change event emitted by ingestd/vectord", + "T5 gatekeeper explicitly retires via /vectors/playbook_memory/retire" + ], + "read_back_policy": "retrieval returns only non-retired versions. History endpoint /vectors/playbook_memory/history/{operation} returns the full chain.", + "ui_surface": "mcp-server to render a diff view: side-by-side of playbook versions with a timeline of what changed and when.", + "implementation_status": "SPEC — not yet wired. Target: next sprint. Touches gateway + catalogd + mcp-server." + }, + "matrix_index_hybrid_search_note": "Phase 22 candidate: elevate the hybrid_search T1 tool to consult T3 when a pool returns <3 matches OR when the same (role, city) combo has failed N times in 24h. Consult result is a reformulated sql_filter the executor retries with. Keeps T1 fast on the happy path, escalates to T3 only on low-recall signals." +} diff --git a/mcp-server/index.ts b/mcp-server/index.ts index a9ac500..437e794 100644 --- a/mcp-server/index.ts +++ b/mcp-server/index.ts @@ -833,6 +833,22 @@ async function main() { }); } + // Model matrix — read config/models.json and expose read-only. + // Strips internal notes that could drift; the source of truth is + // the file itself. UI can render tiers, rate budgets, and the + // experimental rotation list from this endpoint. + if (url.pathname === "/models/matrix") { + try { + const raw = await Bun.file("../config/models.json").text(); + return ok(JSON.parse(raw)); + } catch (e) { + return new Response(JSON.stringify({ error: `models.json not found: ${(e as Error).message}` }), { + status: 404, + headers: { "content-type": "application/json" }, + }); + } + } + // Proof JSON API (same data, no HTML) if (url.pathname === "/proof.json") { const ds = await api("GET", "/catalog/datasets") as any[]; diff --git a/tests/multi-agent/scenario.ts b/tests/multi-agent/scenario.ts index 82da501..2d5b326 100644 --- a/tests/multi-agent/scenario.ts +++ b/tests/multi-agent/scenario.ts @@ -53,8 +53,29 @@ const DRAFT_MODEL = "qwen2.5:latest"; // artifact generation; short outputs // every N events, and (A) cross-day lesson after all events complete. // gpt-oss:20b is a thinking model: it spends tokens in a hidden reasoning // block before emitting `response`. Budget accordingly — never under 400. +// Model matrix — config/models.json is authoritative. Env vars override. +// Loaded at module init so we can log the tier shape at scenario start. +interface ModelTier { + primary: { model: string; provider: string }; + local_fallback?: { model: string; provider: string }; + max_tokens?: number; + temperature?: number; + env_flag?: string; +} +let MODEL_MATRIX: { tiers: Record } = { tiers: {} }; +try { + MODEL_MATRIX = JSON.parse(await Bun.file("config/models.json").text()); +} catch { + // Config optional — env vars alone work too. Silent: the per-tier + // logging below will show "default" if matrix is empty. +} + +const T3_TIER = MODEL_MATRIX.tiers?.t3_overview; const OVERVIEW_CLOUD = process.env.LH_OVERVIEW_CLOUD === "1"; -const OVERVIEW_MODEL = process.env.LH_OVERVIEW_MODEL ?? (OVERVIEW_CLOUD ? "gpt-oss:120b" : "gpt-oss:20b"); +const OVERVIEW_MODEL = process.env.LH_OVERVIEW_MODEL + ?? (OVERVIEW_CLOUD + ? (T3_TIER?.primary.model ?? "gpt-oss:120b") + : (T3_TIER?.local_fallback?.model ?? "gpt-oss:20b")); const T3_CHECKPOINT_EVERY = Number(process.env.LH_T3_CHECKPOINT_EVERY ?? 3); const T3_DISABLED = process.env.LH_T3_DISABLE === "1";