f971e64745
Pre-push hook caught the regression — the smoke hardcoded MODEL = "nomic-embed-text" and the bump to nomic-embed-text-v2-moe in 4da32ad failed the gate. Fix: glob-match the family prefix (nomic-embed-text*). Both v1 and v2-moe are 768d drop-ins; the property the smoke is locking is dim + distinct-vectors, not the exact model variant. Operators swap the variant in lakehouse.toml without needing to touch the smoke. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
172 lines
6.7 KiB
Bash
Executable File
172 lines
6.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# G2 smoke — embedd service. All assertions go through gateway :3110.
|
|
#
|
|
# Validates:
|
|
# - POST /v1/embed with 2 texts → 200, dim=768 (nomic-embed-text),
|
|
# vectors[0] != vectors[1] (different texts → different vectors)
|
|
# - Same text twice → byte-identical vectors (deterministic)
|
|
# - Empty texts → 400
|
|
# - Bad model → 502 from upstream Ollama
|
|
# - End-to-end with vectord: embed text → store → search by text →
|
|
# same text round-trips at distance ≈ 0 (proves embed→vectord
|
|
# pipeline works)
|
|
#
|
|
# Requires Ollama running at :11434 with nomic-embed-text loaded.
|
|
#
|
|
# Usage: ./scripts/g2_smoke.sh
|
|
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
|
|
export PATH="$PATH:/usr/local/go/bin"
|
|
|
|
echo "[g2-smoke] building embedd + vectord + gateway..."
|
|
go build -o bin/ ./cmd/embedd ./cmd/vectord ./cmd/gateway
|
|
|
|
pkill -f "bin/(embedd|vectord|gateway)" 2>/dev/null || true
|
|
sleep 0.3
|
|
|
|
PIDS=()
|
|
TMP="$(mktemp -d)"
|
|
cleanup() {
|
|
echo "[g2-smoke] cleanup"
|
|
for p in "${PIDS[@]}"; do [ -n "$p" ] && kill "$p" 2>/dev/null || true; done
|
|
rm -rf "$TMP"
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
poll_health() {
|
|
local port="$1" deadline=$(($(date +%s) + 5))
|
|
while [ "$(date +%s)" -lt "$deadline" ]; do
|
|
if curl -sS --max-time 1 "http://127.0.0.1:$port/health" >/dev/null 2>&1; then return 0; fi
|
|
sleep 0.05
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Verify Ollama is up before the test even starts — otherwise every
|
|
# embed call would 502 and the smoke would be misleading.
|
|
if ! curl -sS --max-time 3 http://localhost:11434/api/tags >/dev/null 2>&1; then
|
|
echo "[g2-smoke] Ollama not reachable on :11434 — skipping"
|
|
exit 0
|
|
fi
|
|
|
|
echo "[g2-smoke] launching embedd → vectord (no persist) → gateway..."
|
|
./bin/embedd > /tmp/embedd.log 2>&1 &
|
|
PIDS+=($!)
|
|
poll_health 3216 || { echo "embedd failed"; tail /tmp/embedd.log; exit 1; }
|
|
|
|
# vectord with persistence disabled (matches g1_smoke pattern —
|
|
# this smoke doesn't touch storaged).
|
|
./bin/vectord -config scripts/g1_smoke.toml > /tmp/vectord.log 2>&1 &
|
|
PIDS+=($!)
|
|
poll_health 3215 || { echo "vectord failed"; tail /tmp/vectord.log; exit 1; }
|
|
|
|
./bin/gateway > /tmp/gateway.log 2>&1 &
|
|
PIDS+=($!)
|
|
poll_health 3110 || { echo "gateway failed"; tail /tmp/gateway.log; exit 1; }
|
|
|
|
FAILED=0
|
|
|
|
echo "[g2-smoke] /v1/embed — two distinct texts:"
|
|
RESP="$(curl -sS -X POST http://127.0.0.1:3110/v1/embed \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"texts":["forklift operator with OSHA-30","CNC machinist precision parts"]}')"
|
|
DIM="$(echo "$RESP" | jq -r '.dimension')"
|
|
N="$(echo "$RESP" | jq -r '.vectors | length')"
|
|
MODEL="$(echo "$RESP" | jq -r '.model')"
|
|
SAME="$(echo "$RESP" | jq -r '.vectors[0][0] == .vectors[1][0]')"
|
|
# Accept any nomic-embed-text* family member as the default — v1
|
|
# (137M, 768d) and v2-moe (475M MoE, 768d) are both supported drop-ins.
|
|
# The smoke locks the dimension + the distinct-vectors property, NOT
|
|
# the exact model name (operators bump the model in lakehouse.toml
|
|
# without changing this smoke).
|
|
case "$MODEL" in nomic-embed-text*) MODEL_OK=1 ;; *) MODEL_OK=0 ;; esac
|
|
if [ "$DIM" = "768" ] && [ "$N" = "2" ] && [ "$MODEL_OK" = "1" ] && [ "$SAME" = "false" ]; then
|
|
echo " ✓ dim=768, model=$MODEL, 2 distinct vectors"
|
|
else
|
|
echo " ✗ resp: dim=$DIM n=$N model=$MODEL same=$SAME"; FAILED=1
|
|
fi
|
|
|
|
echo "[g2-smoke] determinism — same text twice → byte-identical vector:"
|
|
RESP1="$(curl -sS -X POST http://127.0.0.1:3110/v1/embed \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"texts":["determinism check"]}' | jq -c '.vectors[0]')"
|
|
RESP2="$(curl -sS -X POST http://127.0.0.1:3110/v1/embed \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"texts":["determinism check"]}' | jq -c '.vectors[0]')"
|
|
if [ "$RESP1" = "$RESP2" ]; then
|
|
echo " ✓ identical text → identical vector"
|
|
else
|
|
echo " ✗ deterministic mismatch"; FAILED=1
|
|
fi
|
|
|
|
echo "[g2-smoke] empty texts → 400:"
|
|
HTTP="$(curl -sS -o /dev/null -w '%{http_code}' -X POST http://127.0.0.1:3110/v1/embed \
|
|
-H 'Content-Type: application/json' -d '{"texts":[]}')"
|
|
if [ "$HTTP" = "400" ]; then echo " ✓ empty → 400"; else echo " ✗ empty → $HTTP"; FAILED=1; fi
|
|
|
|
echo "[g2-smoke] bad model → 502:"
|
|
HTTP="$(curl -sS -o /dev/null -w '%{http_code}' -X POST http://127.0.0.1:3110/v1/embed \
|
|
-H 'Content-Type: application/json' -d '{"texts":["x"],"model":"definitely-not-loaded"}')"
|
|
if [ "$HTTP" = "502" ]; then echo " ✓ unknown model → 502"; else echo " ✗ unknown → $HTTP"; FAILED=1; fi
|
|
|
|
echo "[g2-smoke] end-to-end: embed → vectord add → search by embed → recall:"
|
|
NAME="g2_demo"
|
|
# Create index. Default M/EfSearch; cosine distance.
|
|
curl -sS -o /dev/null -X POST http://127.0.0.1:3110/v1/vectors/index \
|
|
-H 'Content-Type: application/json' \
|
|
-d "{\"name\":\"$NAME\",\"dimension\":768,\"distance\":\"cosine\"}"
|
|
|
|
# Embed a few staffing-ish texts and add them.
|
|
TEXTS='["forklift operator with OSHA-30","CNC machinist precision parts","warehouse picker night shift","dental hygienist 3 years experience"]'
|
|
EMBEDS="$(curl -sS -X POST http://127.0.0.1:3110/v1/embed \
|
|
-H 'Content-Type: application/json' \
|
|
-d "{\"texts\":$TEXTS}")"
|
|
# Build the add payload — id-i + vector from embeds[i].
|
|
python3 - "$EMBEDS" <<'EOF' > "$TMP/add.json"
|
|
import json, sys
|
|
embeds = json.loads(sys.argv[1])
|
|
items = [
|
|
{"id": f"w-{i}", "vector": v, "metadata": {"text": t}}
|
|
for i, (v, t) in enumerate(zip(embeds["vectors"], [
|
|
"forklift operator with OSHA-30",
|
|
"CNC machinist precision parts",
|
|
"warehouse picker night shift",
|
|
"dental hygienist 3 years experience",
|
|
]))
|
|
]
|
|
print(json.dumps({"items": items}))
|
|
EOF
|
|
curl -sS -o /dev/null -X POST "http://127.0.0.1:3110/v1/vectors/index/$NAME/add" \
|
|
-H 'Content-Type: application/json' -d @"$TMP/add.json"
|
|
|
|
# Search by embedding the FIRST text again — should retrieve w-0 at dist≈0
|
|
QUERY_VEC="$(curl -sS -X POST http://127.0.0.1:3110/v1/embed \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"texts":["forklift operator with OSHA-30"]}' | jq -c '.vectors[0]')"
|
|
SEARCH="$(curl -sS -X POST "http://127.0.0.1:3110/v1/vectors/index/$NAME/search" \
|
|
-H 'Content-Type: application/json' \
|
|
-d "{\"vector\":$QUERY_VEC,\"k\":3}")"
|
|
TOP_ID="$(echo "$SEARCH" | jq -r '.results[0].id')"
|
|
TOP_DIST="$(echo "$SEARCH" | jq -r '.results[0].distance')"
|
|
DIST_OK="$(python3 -c "import sys; sys.exit(0 if abs($TOP_DIST) < 1e-4 else 1)" && echo y || echo n)"
|
|
if [ "$TOP_ID" = "w-0" ] && [ "$DIST_OK" = "y" ]; then
|
|
echo " ✓ embed → store → search round-trip: w-0 at dist=$TOP_DIST"
|
|
else
|
|
echo " ✗ recall: top=$TOP_ID dist=$TOP_DIST"
|
|
echo " full: $SEARCH"
|
|
FAILED=1
|
|
fi
|
|
|
|
# Clean up the index.
|
|
curl -sS -o /dev/null -X DELETE "http://127.0.0.1:3110/v1/vectors/index/$NAME" || true
|
|
|
|
if [ "$FAILED" -eq 0 ]; then
|
|
echo "[g2-smoke] G2 acceptance gate: PASSED"
|
|
exit 0
|
|
else
|
|
echo "[g2-smoke] G2 acceptance gate: FAILED"
|
|
exit 1
|
|
fi
|