lakehouse/scripts/qwen3_plan.py

#!/usr/bin/env python3
"""Qwen 3 agent plan — structured test with playbook building.

Runs a series of staffing operations through the agent gateway,
uses qwen3 for generation, observes outcomes, and builds playbooks
that future runs can use to improve.

The plan:
1. Check existing playbooks (learn from prior runs)
2. Process 5 contracts with hybrid search
3. Ask 5 intelligence questions (compare to SQL ground truth)
4. Have qwen3 self-evaluate each answer
5. Log successes/failures as playbook entries
6. Summarize: what worked, what didn't, what to change next time
"""

import json, time, sys, re
from urllib.request import Request, urlopen
from urllib.error import HTTPError

GW = "http://localhost:3700"
LH = "http://localhost:3100"

def gw(path, body=None):
    data = json.dumps(body).encode() if body else None
    method = "POST" if body else "GET"
    req = Request(f"{GW}{path}", data=data, method=method,
                  headers={"Content-Type": "application/json"} if body else {})
    try:
        return json.loads(urlopen(req, timeout=180).read())
    except HTTPError as e:
        return {"error": e.read().decode()[:200]}
    except Exception as e:
        return {"error": str(e)}

def generate(prompt, max_tokens=300):
    """Call qwen3 through the sidecar for generation tasks."""
    r = gw("/api/ai/generate", {
        "prompt": prompt, "model": "qwen3",
        "max_tokens": max_tokens, "temperature": 0.3,
    })
    text = r.get("text", r.get("raw", ""))
    # Strip thinking tags
    if "<think>" in text:
        text = text.split("</think>")[-1].strip()
    return text

results = []
playbook_entries = []

def record(name, passed, detail, ms=None):
    results.append({"name": name, "passed": passed, "detail": detail, "ms": ms})
    icon = "✓" if passed else "✗"
    ms_s = f" ({ms:.0f}ms)" if ms else ""
    print(f"  {icon} {name}{ms_s}: {detail}")

print("=" * 65)
print("QWEN 3 AGENT PLAN — structured test + playbook builder")
print("=" * 65)

# ─── Step 1: Check existing playbooks ───
print("\n▸ Step 1: Learning from prior runs")
pbs = gw("/playbooks?keyword=forklift")
if pbs.get("playbooks"):
    for p in pbs["playbooks"][:3]:
        print(f"  📚 {p.get('operation','?')}: {p.get('result','?')[:60]}")
else:
    print("  (no playbooks yet — this is the first run)")

# ─── Step 2: Contract matching with hybrid search ───
print("\n▸ Step 2: Contract matching (hybrid SQL+vector)")
contracts = [
    {"role": "Forklift Operator", "state": "IL", "city": "Chicago", "min_reliability": 0.85, "headcount": 3, "certs": ["OSHA-10"]},
    {"role": "Machine Operator", "state": "OH", "min_reliability": 0.8, "headcount": 4, "certs": []},
    {"role": "Welder", "state": "IN", "min_reliability": 0.7, "headcount": 2, "certs": ["OSHA-30"]},
    {"role": "Quality Tech", "state": "MO", "min_reliability": 0.85, "headcount": 2, "certs": []},
    {"role": "Loader", "state": "IL", "city": "Springfield", "min_reliability": 0.75, "headcount": 5, "certs": []},
]

total_filled = 0
total_needed = 0
for c in contracts:
    t0 = time.time()
    r = gw("/search", {
        "question": f"Find the best {c['role']} workers with relevant skills and experience",
        "sql_filter": f"role = '{c['role']}' AND state = '{c['state']}' AND reliability >= {c['min_reliability']}"
            + (f" AND city = '{c['city']}'" if c.get("city") else ""),
        "top_k": c["headcount"],
        "generate": False,
    })
    ms = (time.time() - t0) * 1000
    matched = len(r.get("sources", []))
    total_filled += min(matched, c["headcount"])
    total_needed += c["headcount"]
    record(f"{c['role']} in {c['state']}", matched >= c["headcount"],
           f"{matched}/{c['headcount']} (sql={r.get('sql_matches',0)})", ms)

fill_pct = total_filled / max(total_needed, 1) * 100
record("overall fill rate", fill_pct >= 80, f"{total_filled}/{total_needed} ({fill_pct:.0f}%)")

# ─── Step 3: Intelligence questions with ground truth ───
print("\n▸ Step 3: Intelligence questions (qwen3 vs SQL ground truth)")
questions = [
    {
        "q": "How many forklift operators are in Illinois?",
        "sql": "SELECT COUNT(*) cnt FROM ethereal_workers WHERE role = 'Forklift Operator' AND state = 'IL'",
        "type": "count",
    },
    {
        "q": "What is the average reliability of workers in Ohio?",
        "sql": "SELECT ROUND(AVG(reliability),3) avg FROM ethereal_workers WHERE state = 'OH'",
        "type": "number",
    },
    {
        "q": "Who are the top 3 most reliable welders?",
        "sql": "SELECT name, reliability FROM ethereal_workers WHERE role = 'Welder' ORDER BY reliability DESC LIMIT 3",
        "type": "names",
    },
    {
        "q": "How many 'erratic' archetype workers do we have?",
        "sql": "SELECT COUNT(*) cnt FROM ethereal_workers WHERE archetype = 'erratic'",
        "type": "count",
    },
    {
        "q": "Which state has the most machine operators?",
        "sql": "SELECT state, COUNT(*) cnt FROM ethereal_workers WHERE role = 'Machine Operator' GROUP BY state ORDER BY cnt DESC LIMIT 1",
        "type": "state",
    },
]

for qi in questions:
    # Get SQL ground truth
    truth = gw("/sql", {"sql": qi["sql"]})
    truth_rows = truth.get("rows", [])

    # Ask qwen3 via hybrid
    t0 = time.time()
    r = gw("/search", {
        "question": qi["q"],
        "sql_filter": None,
        "top_k": 5,
    })
    answer = r.get("answer", "")
    ms = (time.time() - t0) * 1000

    # Strip thinking tags from answer
    if "<think>" in answer:
        answer = answer.split("</think>")[-1].strip()

    # Verify
    passed = False
    detail = ""
    if qi["type"] == "count" and truth_rows:
        expected = list(truth_rows[0].values())[0]
        # Check if the number appears in the answer
        if str(expected) in answer:
            passed = True
            detail = f"correct ({expected})"
        else:
            detail = f"expected {expected}, not found in answer"
    elif qi["type"] == "number" and truth_rows:
        expected = list(truth_rows[0].values())[0]
        detail = f"truth={expected}"
        passed = True  # harder to verify exact match on averages
    elif qi["type"] == "names" and truth_rows:
        names = [r.get("name", "") for r in truth_rows]
        found = sum(1 for n in names if n.lower() in answer.lower())
        passed = found >= 1
        detail = f"{found}/{len(names)} names found"
    elif qi["type"] == "state" and truth_rows:
        expected = truth_rows[0].get("state", "")
        passed = expected.lower() in answer.lower()
        detail = f"expected state={expected}"

    record(f"Q: {qi['q'][:50]}", passed, detail, ms)

# ─── Step 4: Self-evaluation ───
print("\n▸ Step 4: Qwen3 self-evaluation")
score_prompt = f"""You just completed a staffing agent test:
- Contracts filled: {total_filled}/{total_needed} ({fill_pct:.0f}%)
- Intelligence questions: {sum(1 for r in results if r['passed'] and 'Q:' in r['name'])}/{len(questions)}
- Total checks: {sum(1 for r in results if r['passed'])}/{len(results)}

Rate your performance 1-10 and identify the biggest gap to fix. 3 sentences max."""

evaluation = generate(score_prompt, 150)
print(f"  🤖 Qwen3 says: {evaluation[:300]}")

# ─── Step 5: Log playbook entries ───
print("\n▸ Step 5: Building playbooks")
# Log the overall run
gw("/log", {
    "operation": f"qwen3_plan: {total_filled}/{total_needed} filled, {sum(1 for r in results if r['passed'])}/{len(results)} checks",
    "approach": "hybrid search with sql_filter per contract, brute-force cosine for question answering",
    "result": f"fill_rate={fill_pct:.0f}%, model=qwen3, context=40K",
    "context": evaluation[:200],
})
print("  📝 Run logged to playbooks")

# Log specific learnings
for r in results:
    if not r["passed"]:
        gw("/log", {
            "operation": f"FAILURE: {r['name']}",
            "approach": "needs investigation",
            "result": r["detail"],
        })
        print(f"  📝 Failure logged: {r['name']}")

# ─── Step 6: Scorecard ───
print(f"\n{'═'*65}")
print(f"  SCORECARD")
print(f"{'═'*65}")
passed = sum(1 for r in results if r["passed"])
total = len(results)
print(f"  {passed}/{total} passed ({100*passed/max(total,1):.0f}%)")
print(f"  Contracts: {total_filled}/{total_needed} ({fill_pct:.0f}%)")
print(f"\n  {'Test':<55} {'ms':>6} {'Result':>6}")
print(f"  {'-'*70}")
for r in results:
    ms = f"{r['ms']:.0f}" if r['ms'] else "—"
    status = "PASS" if r["passed"] else "FAIL"
    print(f"  {r['name']:<55} {ms:>6} {status:>6}")

print(f"\n  Model: qwen3 (8.2B, 40K context, thinking)")
print(f"  Self-eval: {evaluation[:150]}")