#!/usr/bin/env bash # Relevance smoke — code-relevance filter via matrixd /matrix/relevance. # All assertions go through gateway :3110. # # Validates the headline adjacency-pollution scenario: # Focus: crates/queryd/src/db.go which defines Connector. # Chunk A is about Connector → kept (defined_match). # Chunk B is about catalogd::Registry which db.go imports → outranked # by Chunk A. # Chunk C is unrelated → dropped (no signals fire). # # Plus negative paths: # - Empty chunks → 400 # - Threshold honored when set explicitly set -euo pipefail cd "$(dirname "$0")/.." export PATH="$PATH:/usr/local/go/bin" echo "[relevance-smoke] building matrixd + vectord + gateway..." go build -o bin/ ./cmd/matrixd ./cmd/vectord ./cmd/gateway pkill -f "bin/(matrixd|vectord|gateway)" 2>/dev/null || true sleep 0.3 PIDS=() TMP="$(mktemp -d)" CFG="$TMP/relevance.toml" cleanup() { echo "[relevance-smoke] cleanup" for p in "${PIDS[@]}"; do [ -n "$p" ] && kill "$p" 2>/dev/null || true; done rm -rf "$TMP" } trap cleanup EXIT INT TERM # Custom toml: vectord persistence disabled. /matrix/relevance doesn't # touch vectord at all, but matrixd config requires the URL anyway. cat > "$CFG" </dev/null 2>&1; then return 0; fi sleep 0.05 done return 1 } echo "[relevance-smoke] launching vectord → matrixd → gateway..." ./bin/vectord -config "$CFG" > /tmp/vectord.log 2>&1 & PIDS+=($!) poll_health 3215 || { echo "vectord failed"; tail /tmp/vectord.log; exit 1; } ./bin/matrixd -config "$CFG" > /tmp/matrixd.log 2>&1 & PIDS+=($!) poll_health 3218 || { echo "matrixd failed"; tail /tmp/matrixd.log; exit 1; } ./bin/gateway -config "$CFG" > /tmp/gateway.log 2>&1 & PIDS+=($!) poll_health 3110 || { echo "gateway failed"; tail /tmp/gateway.log; exit 1; } FAILED=0 # ── 1. Adjacency-pollution scenario ────────────────────────────── echo "[relevance-smoke] adjacency-pollution: Connector outranks Registry, junk dropped:" PAYLOAD='{ "focus": { "Path": "crates/queryd/src/db.go", "Content": "pub struct Connector {}\npub fn open_connector() *Connector { return nil }\nuse catalogd::Registry;" }, "chunks": [ {"source":"lakehouse_symbols_v1","doc_id":"symbol:queryd::struct::Connector","text":"Connector wraps the DuckDB handle. open_connector creates one.","score":0.9}, {"source":"lakehouse_symbols_v1","doc_id":"symbol:catalogd::struct::Registry","text":"Registry stores manifests. Used by ingestd.","score":0.85}, {"source":"lakehouse_symbols_v1","doc_id":"symbol:totally_other::Thing","text":"completely unrelated text about something else entirely","score":0.7} ], "threshold": 0.3 }' RESP="$(curl -sS -X POST http://127.0.0.1:3110/v1/matrix/relevance -H 'Content-Type: application/json' -d "$PAYLOAD")" # Connector chunk should be in kept CONNECTOR_KEPT="$(echo "$RESP" | jq -r '[.kept[] | select(.doc_id | contains("Connector"))] | length')" # The unrelated junk chunk should be in dropped JUNK_DROPPED="$(echo "$RESP" | jq -r '[.dropped[] | select(.doc_id | contains("Thing"))] | length')" # Connector should outrank Registry (whichever bucket they end up in) CONN_REL="$(echo "$RESP" | jq -r '[.kept[], .dropped[] | select(.doc_id | contains("Connector"))] | .[0].relevance // -999')" REG_REL="$(echo "$RESP" | jq -r '[.kept[], .dropped[] | select(.doc_id | contains("Registry"))] | .[0].relevance // -999')" TOTAL_IN="$(echo "$RESP" | jq -r '.total_in')" CONN_OUTRANKS_REG="$(awk -v a="$CONN_REL" -v b="$REG_REL" 'BEGIN{print (a>b)?"true":"false"}')" if [ "$CONNECTOR_KEPT" = "1" ] && [ "$JUNK_DROPPED" = "1" ] && [ "$CONN_OUTRANKS_REG" = "true" ] && [ "$TOTAL_IN" = "3" ]; then echo " ✓ Connector kept, junk dropped, Connector ($CONN_REL) > Registry ($REG_REL)" else echo " ✗ kept_connector=$CONNECTOR_KEPT dropped_junk=$JUNK_DROPPED conn=$CONN_REL reg=$REG_REL total=$TOTAL_IN" echo " full: $RESP" FAILED=1 fi # ── 2. Empty chunks → 400 ──────────────────────────────────────── echo "[relevance-smoke] empty chunks → 400:" HTTP="$(curl -sS -o /dev/null -w '%{http_code}' -X POST http://127.0.0.1:3110/v1/matrix/relevance \ -H 'Content-Type: application/json' \ -d '{"focus":{"Path":"x"},"chunks":[]}')" if [ "$HTTP" = "400" ]; then echo " ✓ 400 on empty chunks" else echo " ✗ got $HTTP"; FAILED=1 fi # ── 3. Threshold honored ───────────────────────────────────────── echo "[relevance-smoke] threshold=10 (impossibly high) drops everything:" PAYLOAD2='{ "focus": {"Path": "x.go", "Content": "pub fn known() {}", "DefinedSymbols": ["known"]}, "chunks": [ {"source":"s","doc_id":"d1","text":"known appears here","score":0.9} ], "threshold": 10 }' RESP2="$(curl -sS -X POST http://127.0.0.1:3110/v1/matrix/relevance -H 'Content-Type: application/json' -d "$PAYLOAD2")" KEPT_COUNT="$(echo "$RESP2" | jq -r '.kept | length')" DROP_COUNT="$(echo "$RESP2" | jq -r '.dropped | length')" if [ "$KEPT_COUNT" = "0" ] && [ "$DROP_COUNT" = "1" ]; then echo " ✓ threshold=10 drops everything (0 kept / 1 dropped)" else echo " ✗ kept=$KEPT_COUNT dropped=$DROP_COUNT"; FAILED=1 fi if [ "$FAILED" -eq 0 ]; then echo "[relevance-smoke] Relevance acceptance gate: PASSED" exit 0 else echo "[relevance-smoke] Relevance acceptance gate: FAILED" exit 1 fi