From b8c072cf0bf1b7ed6ccb86bfc0dba5ff41617d74 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 29 Apr 2026 00:50:28 -0500 Subject: [PATCH] =?UTF-8?q?G1:=20vectord=20=E2=80=94=20HNSW=20vector=20sea?= =?UTF-8?q?rch=20via=20coder/hnsw=20=C2=B7=206=20scrum=20fixes=20applied?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First G1+ piece. Standalone vectord service with in-memory HNSW indexes keyed by string IDs and optional opaque JSON metadata. Wraps github.com/coder/hnsw v0.6.1 (pure Go, no cgo). New port :3215 with /v1/vectors/* routed through gateway. API: POST /v1/vectors/index create GET /v1/vectors/index list GET /v1/vectors/index/{name} get info DELETE /v1/vectors/index/{name} POST /v1/vectors/index/{name}/add (batch) POST /v1/vectors/index/{name}/search Acceptance smoke 7/7 PASS — including recall=1 on inserted vector w-042 (cosine distance 5.96e-8, float32 precision noise), 200- vector batch round-trip, dim mismatch → 400, missing index → 404, duplicate create → 409. Two upstream library quirks worked around in the wrapper: 1. coder/hnsw.Add panics with "node not added" on re-adding an existing key (length-invariant fires because internal delete+re-add doesn't change Len). Pre-Delete fixes for n>1. 2. Delete of the LAST node leaves layers[0] non-empty but entryless; next Add SIGSEGVs in Dims(). Workaround: when re-adding to a 1-node graph, recreate the underlying graph fresh via resetGraphLocked(). Cross-lineage scrum on shipped code: - Opus 4.7 (opencode): 0 BLOCK + 4 WARN + 3 INFO - Kimi K2-0905 (openrouter): 2 BLOCK + 2 WARN + 1 INFO - Qwen3-coder (openrouter): "No BLOCKs" (4 tokens) Fixed (4 real + 2 cleanup): O-W1: Lookup returned the raw []float32 from coder/hnsw — caller mutation would corrupt index. Now copies before return. O-W3: NaN/Inf vectors poison HNSW (distance comparisons return false for both < and >, breaking heap invariants). Zero-norm under cosine produces NaN. Now validated at Add time. K-B1: Re-adding with nil metadata silently cleared the existing entry — JSON-omitted "metadata" field deserializes as nil, making upsert non-idempotent. Now nil = "leave alone"; explicit {} or Delete to clear. O-W4: Batch Add with mid-batch failure left items 0..N-1 committed and item N rejected. Now pre-validates all IDs+dims before any Add. O-I1: jsonItoa hand-roll replaced with strconv.Itoa — no measured allocation win. O-I2: distanceFn re-resolved per Search → use stored i.g.Distance. Dismissed (2 false positives): K-B2 "MaxBytesReader applied after full read" — false, applied BEFORE Decode in decodeJSON K-W1 "Search distances under read lock might see invalidated slices from concurrent Add" — false, RWMutex serializes write-lock during Add against read-lock during Search Deferred (3): HTTP server timeouts (consistent G0 punt), Content-Type validation (internal service behind gateway), Lookup dim assertion (in-memory state can't drift). The K-B1 finding is worth pausing on: nil metadata on re-add is the kind of API ergonomics bug only a code-reading reviewer catches — smoke would never detect it because the smoke always sends explicit metadata. Three lines changed in Add; the resulting API matches what callers actually expect. Co-Authored-By: Claude Opus 4.7 (1M context) --- cmd/gateway/main.go | 5 + cmd/vectord/main.go | 267 +++++++++++++++++++++++++++++++ go.mod | 5 + go.sum | 10 ++ internal/shared/config.go | 17 +- internal/vectord/index.go | 277 +++++++++++++++++++++++++++++++++ internal/vectord/index_test.go | 232 +++++++++++++++++++++++++++ internal/vectord/registry.go | 85 ++++++++++ lakehouse.toml | 4 + scripts/g1_smoke.sh | 168 ++++++++++++++++++++ 10 files changed, 1067 insertions(+), 3 deletions(-) create mode 100644 cmd/vectord/main.go create mode 100644 internal/vectord/index.go create mode 100644 internal/vectord/index_test.go create mode 100644 internal/vectord/registry.go create mode 100755 scripts/g1_smoke.sh diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go index 5da2912..4c93840 100644 --- a/cmd/gateway/main.go +++ b/cmd/gateway/main.go @@ -42,6 +42,7 @@ func main() { "catalogd_url": cfg.Gateway.CatalogdURL, "ingestd_url": cfg.Gateway.IngestdURL, "queryd_url": cfg.Gateway.QuerydURL, + "vectord_url": cfg.Gateway.VectordURL, } for k, v := range upstreams { if v == "" { @@ -59,11 +60,13 @@ func main() { catalogdURL := mustParseUpstream("catalogd_url", cfg.Gateway.CatalogdURL) ingestdURL := mustParseUpstream("ingestd_url", cfg.Gateway.IngestdURL) querydURL := mustParseUpstream("queryd_url", cfg.Gateway.QuerydURL) + vectordURL := mustParseUpstream("vectord_url", cfg.Gateway.VectordURL) storagedProxy := gateway.NewProxyHandler(storagedURL) catalogdProxy := gateway.NewProxyHandler(catalogdURL) ingestdProxy := gateway.NewProxyHandler(ingestdURL) querydProxy := gateway.NewProxyHandler(querydURL) + vectordProxy := gateway.NewProxyHandler(vectordURL) if err := shared.Run("gateway", cfg.Gateway.Bind, func(r chi.Router) { // Storage / catalog have multi-segment paths under their @@ -77,6 +80,8 @@ func main() { // will get the backend's 405. r.Handle("/v1/ingest", ingestdProxy) r.Handle("/v1/sql", querydProxy) + // Vector search routes — /v1/vectors/index, /v1/vectors/index/{name}/... + r.Handle("/v1/vectors/*", vectordProxy) }); err != nil { slog.Error("server", "err", err) os.Exit(1) diff --git a/cmd/vectord/main.go b/cmd/vectord/main.go new file mode 100644 index 0000000..605ea71 --- /dev/null +++ b/cmd/vectord/main.go @@ -0,0 +1,267 @@ +// vectord is the vector-search service. In-memory HNSW indexes +// keyed by string IDs with optional opaque JSON metadata. Wraps +// github.com/coder/hnsw (pure Go, no cgo). +// +// G1 scope: in-memory only, single-process. Persistence to storaged +// + rehydrate-on-restart is the next piece. The HTTP surface is +// stable enough to build the staffing co-pilot's "find workers +// like X" path on top right now — the indexes just have to be +// rebuilt after a restart. +package main + +import ( + "encoding/json" + "errors" + "flag" + "log/slog" + "net/http" + "os" + "strconv" + "strings" + + "github.com/go-chi/chi/v5" + + "git.agentview.dev/profit/golangLAKEHOUSE/internal/shared" + "git.agentview.dev/profit/golangLAKEHOUSE/internal/vectord" +) + +const ( + maxRequestBytes = 64 << 20 // 64 MiB cap for batched Add payloads + defaultK = 10 + maxK = 1000 +) + +func main() { + configPath := flag.String("config", "lakehouse.toml", "path to TOML config") + flag.Parse() + + cfg, err := shared.LoadConfig(*configPath) + if err != nil { + slog.Error("config", "err", err) + os.Exit(1) + } + + h := &handlers{reg: vectord.NewRegistry()} + + if err := shared.Run("vectord", cfg.Vectord.Bind, h.register); err != nil { + slog.Error("server", "err", err) + os.Exit(1) + } +} + +type handlers struct { + reg *vectord.Registry +} + +func (h *handlers) register(r chi.Router) { + r.Post("/vectors/index", h.handleCreate) + r.Get("/vectors/index", h.handleList) + r.Get("/vectors/index/{name}", h.handleGetIndex) + r.Delete("/vectors/index/{name}", h.handleDelete) + r.Post("/vectors/index/{name}/add", h.handleAdd) + r.Post("/vectors/index/{name}/search", h.handleSearch) +} + +// createRequest mirrors POST /vectors/index body. +type createRequest struct { + Name string `json:"name"` + Dimension int `json:"dimension"` + M int `json:"m,omitempty"` + EfSearch int `json:"ef_search,omitempty"` + Distance string `json:"distance,omitempty"` +} + +// indexInfo is the GET /vectors/index/{name} response shape. +type indexInfo struct { + Params vectord.IndexParams `json:"params"` + Length int `json:"length"` +} + +// addRequest is the body for POST /vectors/index/{name}/add. Items +// are batched so callers can amortize HTTP overhead — the smoke +// inserts hundreds of vectors per request. +type addRequest struct { + Items []addItem `json:"items"` +} + +type addItem struct { + ID string `json:"id"` + Vector []float32 `json:"vector"` + Metadata json.RawMessage `json:"metadata,omitempty"` +} + +// searchRequest is the body for POST /vectors/index/{name}/search. +type searchRequest struct { + Vector []float32 `json:"vector"` + K int `json:"k,omitempty"` +} + +type searchResponse struct { + Results []vectord.Result `json:"results"` +} + +func (h *handlers) handleCreate(w http.ResponseWriter, r *http.Request) { + var req createRequest + if !decodeJSON(w, r, &req) { + return + } + idx, err := h.reg.Create(vectord.IndexParams{ + Name: req.Name, + Dimension: req.Dimension, + M: req.M, + EfSearch: req.EfSearch, + Distance: req.Distance, + }) + if errors.Is(err, vectord.ErrIndexAlreadyExists) { + http.Error(w, err.Error(), http.StatusConflict) + return + } + if errors.Is(err, vectord.ErrInvalidParams) || errors.Is(err, vectord.ErrUnknownDistance) { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if err != nil { + slog.Error("create index", "name", req.Name, "err", err) + http.Error(w, "internal", http.StatusInternalServerError) + return + } + writeJSON(w, http.StatusCreated, indexInfo{Params: idx.Params(), Length: idx.Len()}) +} + +func (h *handlers) handleList(w http.ResponseWriter, _ *http.Request) { + names := h.reg.Names() + writeJSON(w, http.StatusOK, map[string]any{"names": names, "count": len(names)}) +} + +func (h *handlers) handleGetIndex(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + idx, err := h.reg.Get(name) + if errors.Is(err, vectord.ErrIndexNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + if err != nil { + http.Error(w, "internal", http.StatusInternalServerError) + return + } + writeJSON(w, http.StatusOK, indexInfo{Params: idx.Params(), Length: idx.Len()}) +} + +func (h *handlers) handleDelete(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + if err := h.reg.Delete(name); errors.Is(err, vectord.ErrIndexNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } else if err != nil { + http.Error(w, "internal", http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *handlers) handleAdd(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + idx, err := h.reg.Get(name) + if errors.Is(err, vectord.ErrIndexNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + var req addRequest + if !decodeJSON(w, r, &req) { + return + } + if len(req.Items) == 0 { + http.Error(w, "items must be non-empty", http.StatusBadRequest) + return + } + // Per scrum O-W4 (Opus): pre-validate all items before any Add, + // so a bad item at position N doesn't leave items 0..N-1 already + // committed and item N rejected. Both checks (empty id, dim + // mismatch) are local; running them up-front is O(N) extra work + // that the success path already paid in idx.Add. + dim := idx.Params().Dimension + for j, it := range req.Items { + if it.ID == "" { + http.Error(w, "items["+strconv.Itoa(j)+"]: empty id", http.StatusBadRequest) + return + } + if len(it.Vector) != dim { + http.Error(w, "items["+strconv.Itoa(j)+"]: dim mismatch (index="+strconv.Itoa(dim)+", got="+strconv.Itoa(len(it.Vector))+")", http.StatusBadRequest) + return + } + } + for j, it := range req.Items { + if err := idx.Add(it.ID, it.Vector, it.Metadata); err != nil { + // Vector-validation errors (NaN/Inf, zero-norm under + // cosine) only surface here; pre-validation is intentional + // minimal scope (id + dim only). + if errors.Is(err, vectord.ErrDimensionMismatch) || + strings.Contains(err.Error(), "non-finite") || + strings.Contains(err.Error(), "zero-norm") { + http.Error(w, "items["+strconv.Itoa(j)+"]: "+err.Error(), http.StatusBadRequest) + return + } + slog.Error("add", "name", name, "id", it.ID, "err", err) + http.Error(w, "internal", http.StatusInternalServerError) + return + } + } + writeJSON(w, http.StatusOK, map[string]any{"added": len(req.Items), "length": idx.Len()}) +} + +func (h *handlers) handleSearch(w http.ResponseWriter, r *http.Request) { + name := chi.URLParam(r, "name") + idx, err := h.reg.Get(name) + if errors.Is(err, vectord.ErrIndexNotFound) { + http.Error(w, "not found", http.StatusNotFound) + return + } + var req searchRequest + if !decodeJSON(w, r, &req) { + return + } + k := req.K + if k <= 0 { + k = defaultK + } + if k > maxK { + k = maxK + } + hits, err := idx.Search(req.Vector, k) + if errors.Is(err, vectord.ErrDimensionMismatch) { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if err != nil { + slog.Error("search", "name", name, "err", err) + http.Error(w, "internal", http.StatusInternalServerError) + return + } + writeJSON(w, http.StatusOK, searchResponse{Results: hits}) +} + +// decodeJSON reads + decodes a JSON body with a body-size cap. +// Returns false (and writes the error response) on failure. +func decodeJSON(w http.ResponseWriter, r *http.Request, v any) bool { + defer r.Body.Close() + r.Body = http.MaxBytesReader(w, r.Body, maxRequestBytes) + if err := json.NewDecoder(r.Body).Decode(v); err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) || strings.Contains(err.Error(), "http: request body too large") { + http.Error(w, "body too large", http.StatusRequestEntityTooLarge) + return false + } + http.Error(w, "decode body: "+err.Error(), http.StatusBadRequest) + return false + } + return true +} + +func writeJSON(w http.ResponseWriter, code int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + if err := json.NewEncoder(w).Encode(v); err != nil { + slog.Warn("write json", "err", err) + } +} + diff --git a/go.mod b/go.mod index 0dd7415..c6b3614 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.20 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.42.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chewxy/math32 v1.10.1 // indirect + github.com/coder/hnsw v0.6.1 // indirect github.com/duckdb/duckdb-go-bindings v0.10502.0 // indirect github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10502.0 // indirect github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10502.0 // indirect @@ -42,9 +44,12 @@ require ( github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect + github.com/google/renameio v1.0.1 // indirect github.com/klauspost/compress v1.18.5 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect + github.com/viterin/partial v1.1.0 // indirect + github.com/viterin/vek v0.4.2 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect golang.org/x/net v0.52.0 // indirect diff --git a/go.sum b/go.sum index 8ce088b..39d76ce 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,10 @@ github.com/aws/smithy-go v1.25.0 h1:Sz/XJ64rwuiKtB6j98nDIPyYrV1nVNJ4YU74gttcl5U= github.com/aws/smithy-go v1.25.0/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chewxy/math32 v1.10.1 h1:LFpeY0SLJXeaiej/eIp2L40VYfscTvKh/FSEZ68uMkU= +github.com/chewxy/math32 v1.10.1/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs= +github.com/coder/hnsw v0.6.1 h1:Dv76pjiFkgMYFqnTCOehJXd06irm2PRwcP/jMMPCyO0= +github.com/coder/hnsw v0.6.1/go.mod h1:wvRc/vZNkK50HFcagwnc/ep/u29Mg2uLlPmc8SD7eEQ= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/duckdb/duckdb-go-bindings v0.10502.0 h1:Uhg/dfvPLQv4cH35lMD48hqUcdOh2Z7bcuykjr4qnOA= @@ -76,6 +80,8 @@ github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9U github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU= +github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= @@ -92,6 +98,10 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/viterin/partial v1.1.0 h1:iH1l1xqBlapXsYzADS1dcbizg3iQUKTU1rbwkHv/80E= +github.com/viterin/partial v1.1.0/go.mod h1:oKGAo7/wylWkJTLrWX8n+f4aDPtQMQ6VG4dd2qur5QA= +github.com/viterin/vek v0.4.2 h1:Vyv04UjQT6gcjEFX82AS9ocgNbAJqsHviheIBdPlv5U= +github.com/viterin/vek v0.4.2/go.mod h1:A4JRAe8OvbhdzBL5ofzjBS0J29FyUrf95tQogvtHHUc= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= diff --git a/internal/shared/config.go b/internal/shared/config.go index 6632575..3234825 100644 --- a/internal/shared/config.go +++ b/internal/shared/config.go @@ -24,6 +24,7 @@ type Config struct { Catalogd CatalogConfig `toml:"catalogd"` Ingestd IngestConfig `toml:"ingestd"` Queryd QuerydConfig `toml:"queryd"` + Vectord VectordConfig `toml:"vectord"` S3 S3Config `toml:"s3"` Log LogConfig `toml:"log"` } @@ -46,15 +47,23 @@ type IngestConfig struct { } // GatewayConfig adds the upstream URLs the reverse proxy fronts. -// Each route family (/v1/storage, /v1/catalog, /v1/ingest, /v1/sql) -// has its own upstream so we can scale services independently or -// move them to different boxes without touching gateway code. +// Each route family (/v1/storage, /v1/catalog, /v1/ingest, /v1/sql, +// /v1/vectors) has its own upstream so we can scale services +// independently or move them to different boxes without touching +// gateway code. type GatewayConfig struct { Bind string `toml:"bind"` StoragedURL string `toml:"storaged_url"` CatalogdURL string `toml:"catalogd_url"` IngestdURL string `toml:"ingestd_url"` QuerydURL string `toml:"queryd_url"` + VectordURL string `toml:"vectord_url"` +} + +// VectordConfig adds vectord-specific knobs. Currently just bind; +// G1+ will add persistence-to-storaged URL + index params defaults. +type VectordConfig struct { + Bind string `toml:"bind"` } // QuerydConfig adds queryd-specific knobs. queryd talks DuckDB @@ -110,6 +119,7 @@ func DefaultConfig() Config { CatalogdURL: "http://127.0.0.1:3212", IngestdURL: "http://127.0.0.1:3213", QuerydURL: "http://127.0.0.1:3214", + VectordURL: "http://127.0.0.1:3215", }, Storaged: ServiceConfig{Bind: "127.0.0.1:3211"}, Catalogd: CatalogConfig{Bind: "127.0.0.1:3212", StoragedURL: "http://127.0.0.1:3211"}, @@ -119,6 +129,7 @@ func DefaultConfig() Config { CatalogdURL: "http://127.0.0.1:3212", MaxIngestBytes: 256 << 20, // 256 MiB; bump per deployment via lakehouse.toml }, + Vectord: VectordConfig{Bind: "127.0.0.1:3215"}, Queryd: QuerydConfig{ Bind: "127.0.0.1:3214", CatalogdURL: "http://127.0.0.1:3212", diff --git a/internal/vectord/index.go b/internal/vectord/index.go new file mode 100644 index 0000000..8c34083 --- /dev/null +++ b/internal/vectord/index.go @@ -0,0 +1,277 @@ +// Package vectord owns the vector-search surface — HNSW indexes +// keyed by string IDs with optional opaque JSON metadata. The +// underlying library is github.com/coder/hnsw (pure Go, no cgo). +// +// G1 scope: in-memory only. Persistence to storaged + rehydrate +// across restart is the next piece — keeping it out of this layer +// makes the index API easier to test and keeps the storaged +// dependency optional for downstream tooling. +package vectord + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "sync" + + "github.com/coder/hnsw" +) + +// Distance names accepted by IndexParams.Distance. +const ( + DistanceCosine = "cosine" + DistanceEuclidean = "euclidean" +) + +// Default HNSW parameters — match coder/hnsw's NewGraph defaults +// which are tuned for OpenAI-shaped embeddings (1536-d, but the +// hyperparameters generalize). +const ( + DefaultM = 16 + DefaultEfSearch = 20 +) + +// IndexParams describes one vector index. Once an Index is built, +// these are fixed — changing M / dimension / distance requires a +// rebuild. +type IndexParams struct { + Name string `json:"name"` + Dimension int `json:"dimension"` + M int `json:"m"` + EfSearch int `json:"ef_search"` + Distance string `json:"distance"` +} + +// Result is one search hit. Distance semantics depend on the +// configured distance function — for cosine it's `1 - cos(a,b)` +// where smaller = closer; for euclidean it's the L2 norm of +// `a - b`. Either way, smaller = closer and the result list is +// sorted ascending. +type Result struct { + ID string `json:"id"` + Distance float32 `json:"distance"` + Metadata json.RawMessage `json:"metadata,omitempty"` +} + +// Index wraps a coder/hnsw graph plus a side map of opaque JSON +// metadata per ID. Concurrency: read-heavy via Search (read-lock); +// Add and Delete take the write lock. +type Index struct { + params IndexParams + g *hnsw.Graph[string] + meta map[string]json.RawMessage + mu sync.RWMutex +} + +// Errors surfaced to HTTP handlers. Sentinel-based so the wire +// layer can map to status codes via errors.Is. +var ( + ErrDimensionMismatch = errors.New("vectord: vector dimension mismatch") + ErrUnknownDistance = errors.New("vectord: unknown distance function") + ErrInvalidParams = errors.New("vectord: invalid index params") +) + +// NewIndex builds a fresh index from validated params. +func NewIndex(p IndexParams) (*Index, error) { + if p.Name == "" { + return nil, fmt.Errorf("%w: empty name", ErrInvalidParams) + } + if p.Dimension <= 0 { + return nil, fmt.Errorf("%w: dimension must be > 0 (got %d)", ErrInvalidParams, p.Dimension) + } + if p.M <= 0 { + p.M = DefaultM + } + if p.EfSearch <= 0 { + p.EfSearch = DefaultEfSearch + } + if p.Distance == "" { + p.Distance = DistanceCosine + } + dist, err := distanceFn(p.Distance) + if err != nil { + return nil, err + } + + g := hnsw.NewGraph[string]() + g.M = p.M + g.EfSearch = p.EfSearch + g.Distance = dist + // Ml stays at the library default (0.25); exposing it as a knob + // is a G2 concern when we have real tuning data. + + return &Index{ + params: p, + g: g, + meta: make(map[string]json.RawMessage), + }, nil +} + +// distanceFn maps the string name to the underlying function. +// Easier to unit-test than calling out to coder/hnsw's registry. +func distanceFn(name string) (hnsw.DistanceFunc, error) { + switch name { + case DistanceCosine, "": + return hnsw.CosineDistance, nil + case DistanceEuclidean: + return hnsw.EuclideanDistance, nil + } + return nil, fmt.Errorf("%w: %q (want cosine or euclidean)", ErrUnknownDistance, name) +} + +// Params returns a copy of the immutable index params. +func (i *Index) Params() IndexParams { return i.params } + +// Len returns the number of vectors currently in the index. +func (i *Index) Len() int { + i.mu.RLock() + defer i.mu.RUnlock() + return i.g.Len() +} + +// Add inserts a vector with optional metadata, with replace +// semantics for the vector: if id already exists, the prior +// vector is removed first. Dim must match the index dim or +// ErrDimensionMismatch is returned. +// +// Metadata semantics (post-scrum K-B1): nil meta is "leave +// existing alone" (upsert-style); to clear metadata, pass an +// empty `{}` or Delete+Add. This avoids silent metadata loss +// when the JSON `metadata` field is omitted on re-add. +// +// Validates that all vector components are finite (post-scrum +// O-W3). NaN/Inf in any component poisons HNSW: distance +// comparisons return false for both `<` and `>`, breaking the +// search heap invariants. Zero-norm vectors are also rejected +// under cosine distance — cos(0,x) = NaN. +// +// Note: coder/hnsw's Graph.Add panics on re-adding an existing +// key (internal "node not added" length-invariant check). We +// pre-Delete to make Add idempotent on re-insert. +func (i *Index) Add(id string, vec []float32, meta json.RawMessage) error { + if id == "" { + return errors.New("vectord: empty id") + } + if len(vec) != i.params.Dimension { + return fmt.Errorf("%w: index dim=%d, got=%d", ErrDimensionMismatch, i.params.Dimension, len(vec)) + } + if err := validateVector(vec, i.params.Distance); err != nil { + return err + } + i.mu.Lock() + defer i.mu.Unlock() + // coder/hnsw has two sharp edges on re-add: + // 1. Add of an existing key panics with "node not added" + // (length-invariant fires because internal delete+re-add + // doesn't change Len). Pre-Delete fixes this for n>1. + // 2. Delete of the LAST node leaves layers[0] non-empty but + // entryless; the next Add SIGSEGVs in Dims() because + // entry().Value is nil. We rebuild the graph in that case. + _, exists := i.g.Lookup(id) + if exists { + if i.g.Len() == 1 { + i.resetGraphLocked() + } else { + i.g.Delete(id) + } + } + i.g.Add(hnsw.MakeNode(id, vec)) + if meta != nil { + // Per scrum K-B1 (Kimi): only OVERWRITE on explicit non-nil. + // nil = "leave existing meta alone" (upsert). To clear, the + // caller should send an empty `{}` body or Delete the id. + i.meta[id] = meta + } + return nil +} + +// resetGraphLocked recreates the underlying coder/hnsw Graph with +// the same params. Caller MUST hold i.mu (write-lock). Used to +// dodge the library's "delete the last node, then segfault on +// next Add" bug — see Add for details. Metadata map is preserved +// because the only entry it could affect is the one being +// re-added, which Add overwrites. +func (i *Index) resetGraphLocked() { + g := hnsw.NewGraph[string]() + g.M = i.params.M + g.EfSearch = i.params.EfSearch + g.Distance = i.g.Distance + i.g = g +} + +// validateVector rejects vectors that would poison the HNSW +// graph or produce NaN distances. Per scrum O-W3 (Opus). +func validateVector(vec []float32, distance string) error { + var sumSq float64 + for j, v := range vec { + f := float64(v) + if math.IsNaN(f) || math.IsInf(f, 0) { + return fmt.Errorf("vectord: vec[%d] is non-finite (got %v)", j, v) + } + sumSq += f * f + } + if distance == DistanceCosine && sumSq == 0 { + return errors.New("vectord: zero-norm vector under cosine distance") + } + return nil +} + +// Delete removes id from the index. Returns true if present. +func (i *Index) Delete(id string) bool { + i.mu.Lock() + defer i.mu.Unlock() + delete(i.meta, id) + return i.g.Delete(id) +} + +// Search returns the k nearest neighbors of query, sorted +// ascending by distance. +// +// Note: coder/hnsw's Search returns `[]Node[K]` without distances — +// they're computed internally in the search candidate heap but +// dropped from the public API. We recompute distance from the +// returned vectors. O(k·dim) per search, negligible at typical +// k=10 / dim<2048. +func (i *Index) Search(query []float32, k int) ([]Result, error) { + if len(query) != i.params.Dimension { + return nil, fmt.Errorf("%w: index dim=%d, got=%d", ErrDimensionMismatch, i.params.Dimension, len(query)) + } + if k <= 0 { + return nil, errors.New("vectord: k must be > 0") + } + i.mu.RLock() + defer i.mu.RUnlock() + + // Per scrum O-I2 (Opus): use the stored distance function + // directly rather than re-resolving the string name on every + // search. The graph's Distance is set once at NewIndex. + dist := i.g.Distance + hits := i.g.Search(query, k) + out := make([]Result, len(hits)) + for j, n := range hits { + out[j] = Result{ + ID: n.Key, + Distance: dist(query, n.Value), + Metadata: i.meta[n.Key], + } + } + return out, nil +} + +// Lookup returns the stored vector + metadata for an id. +// +// Per scrum O-W1 (Opus): the vector is COPIED before return. +// coder/hnsw's Lookup hands back the underlying graph slice; +// caller mutation would corrupt the index without locking. +func (i *Index) Lookup(id string) (vec []float32, meta json.RawMessage, ok bool) { + i.mu.RLock() + defer i.mu.RUnlock() + v, found := i.g.Lookup(id) + if !found { + return nil, nil, false + } + out := make([]float32, len(v)) + copy(out, v) + return out, i.meta[id], true +} diff --git a/internal/vectord/index_test.go b/internal/vectord/index_test.go new file mode 100644 index 0000000..4de0be7 --- /dev/null +++ b/internal/vectord/index_test.go @@ -0,0 +1,232 @@ +package vectord + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "sync" + "testing" +) + +func TestNewIndex_DefaultsAndValidation(t *testing.T) { + idx, err := NewIndex(IndexParams{Name: "x", Dimension: 4}) + if err != nil { + t.Fatal(err) + } + p := idx.Params() + if p.M != DefaultM || p.EfSearch != DefaultEfSearch || p.Distance != DistanceCosine { + t.Errorf("defaults not applied: %+v", p) + } + + if _, err := NewIndex(IndexParams{Dimension: 4}); err == nil { + t.Error("expected error on empty name") + } + if _, err := NewIndex(IndexParams{Name: "y", Dimension: 0}); err == nil { + t.Error("expected error on zero dimension") + } + if _, err := NewIndex(IndexParams{Name: "z", Dimension: 4, Distance: "bogus"}); !errors.Is(err, ErrUnknownDistance) { + t.Errorf("expected ErrUnknownDistance, got %v", err) + } +} + +func TestIndex_AddAndSearch_Recall(t *testing.T) { + idx, err := NewIndex(IndexParams{Name: "x", Dimension: 3, Distance: DistanceCosine}) + if err != nil { + t.Fatal(err) + } + target := []float32{1, 0, 0} + if err := idx.Add("alice", target, json.RawMessage(`{"role":"warehouse"}`)); err != nil { + t.Fatal(err) + } + if err := idx.Add("bob", []float32{0, 1, 0}, nil); err != nil { + t.Fatal(err) + } + if err := idx.Add("carol", []float32{0, 0, 1}, nil); err != nil { + t.Fatal(err) + } + if idx.Len() != 3 { + t.Errorf("Len: got %d, want 3", idx.Len()) + } + hits, err := idx.Search(target, 2) + if err != nil { + t.Fatal(err) + } + if len(hits) < 1 { + t.Fatal("no hits") + } + if hits[0].ID != "alice" { + t.Errorf("nearest: got %q, want alice", hits[0].ID) + } + if hits[0].Distance > 0.001 { + t.Errorf("nearest distance: got %v, want ~0", hits[0].Distance) + } + // Cosine distance of identical unit vectors is 0; metadata round-trips. + if string(hits[0].Metadata) != `{"role":"warehouse"}` { + t.Errorf("metadata: got %q", hits[0].Metadata) + } +} + +func TestIndex_DimensionMismatch(t *testing.T) { + idx, _ := NewIndex(IndexParams{Name: "x", Dimension: 4}) + err := idx.Add("a", []float32{1, 2, 3}, nil) + if !errors.Is(err, ErrDimensionMismatch) { + t.Errorf("Add: expected ErrDimensionMismatch, got %v", err) + } + _, err = idx.Search([]float32{1, 2, 3}, 1) + if !errors.Is(err, ErrDimensionMismatch) { + t.Errorf("Search: expected ErrDimensionMismatch, got %v", err) + } +} + +func TestIndex_DeleteAndLookup(t *testing.T) { + idx, _ := NewIndex(IndexParams{Name: "x", Dimension: 2}) + _ = idx.Add("a", []float32{1, 0}, nil) + if !idx.Delete("a") { + t.Error("Delete returned false on existing key") + } + if _, _, ok := idx.Lookup("a"); ok { + t.Error("Lookup found deleted key") + } + if idx.Delete("a") { + t.Error("Delete should return false on missing key") + } +} + +// TestIndex_ConcurrentSearchAdd exercises the RWMutex — many +// concurrent searches alongside a writer adding distinct keys +// shouldn't deadlock, panic, or interleave incorrectly. Each +// writer goroutine gets its own key namespace so we don't +// stress-test the library's re-add path (which has known issues +// under high churn — the wrapper exposes idempotent semantics +// via single-threaded Delete+Add but isn't a fix-everything for +// upstream). +func TestIndex_ConcurrentSearchAdd(t *testing.T) { + idx, _ := NewIndex(IndexParams{Name: "x", Dimension: 4}) + for i := 0; i < 50; i++ { + _ = idx.Add(fmt.Sprintf("seed-%d", i), []float32{float32(i), 0, 0, 0}, nil) + } + var wg sync.WaitGroup + // One writer goroutine, eight readers — realistic ratio for the + // staffing co-pilot use case where ingestion is occasional and + // queries are common. + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 50; j++ { + _ = idx.Add(fmt.Sprintf("hot-%d", j), []float32{float32(j), 1, 0, 0}, nil) + } + }() + for r := 0; r < 8; r++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 50; j++ { + _, _ = idx.Search([]float32{float32(j), 0, 0, 0}, 5) + } + }() + } + wg.Wait() +} + +func TestRegistry_CreateGetDelete(t *testing.T) { + r := NewRegistry() + idx, err := r.Create(IndexParams{Name: "workers", Dimension: 4}) + if err != nil { + t.Fatal(err) + } + if idx.Params().Name != "workers" { + t.Errorf("name: got %q", idx.Params().Name) + } + got, err := r.Get("workers") + if err != nil || got != idx { + t.Errorf("Get returned different / err: %v", err) + } + if _, err := r.Create(IndexParams{Name: "workers", Dimension: 4}); !errors.Is(err, ErrIndexAlreadyExists) { + t.Errorf("dup create: expected ErrIndexAlreadyExists, got %v", err) + } + if err := r.Delete("workers"); err != nil { + t.Fatal(err) + } + if _, err := r.Get("workers"); !errors.Is(err, ErrIndexNotFound) { + t.Errorf("Get after Delete: expected ErrIndexNotFound, got %v", err) + } + if err := r.Delete("workers"); !errors.Is(err, ErrIndexNotFound) { + t.Errorf("idempotent Delete: expected ErrIndexNotFound, got %v", err) + } +} + +func TestAdd_RejectsNonFinite(t *testing.T) { + idx, _ := NewIndex(IndexParams{Name: "x", Dimension: 3, Distance: DistanceEuclidean}) + cases := [][]float32{ + {float32(math.NaN()), 0, 0}, + {float32(math.Inf(1)), 0, 0}, + {0, float32(math.Inf(-1)), 0}, + } + for _, vec := range cases { + if err := idx.Add("a", vec, nil); err == nil { + t.Errorf("expected error for non-finite vec %v", vec) + } + } +} + +func TestAdd_RejectsZeroNormUnderCosine(t *testing.T) { + idx, _ := NewIndex(IndexParams{Name: "x", Dimension: 3, Distance: DistanceCosine}) + if err := idx.Add("a", []float32{0, 0, 0}, nil); err == nil { + t.Error("expected error for zero-norm under cosine") + } + // Same vec is OK under euclidean (origin is a valid point). + idxE, _ := NewIndex(IndexParams{Name: "y", Dimension: 3, Distance: DistanceEuclidean}) + if err := idxE.Add("a", []float32{0, 0, 0}, nil); err != nil { + t.Errorf("zero vec under euclidean should be valid: %v", err) + } +} + +func TestAdd_PreservesMetaOnNilReAdd(t *testing.T) { + // Per scrum K-B1: re-adding with nil meta must NOT clear existing. + idx, _ := NewIndex(IndexParams{Name: "x", Dimension: 2}) + _ = idx.Add("alice", []float32{1, 0}, json.RawMessage(`{"role":"warehouse"}`)) + if err := idx.Add("alice", []float32{0.9, 0.1}, nil); err != nil { + t.Fatal(err) + } + _, meta, ok := idx.Lookup("alice") + if !ok { + t.Fatal("Lookup not found") + } + if string(meta) != `{"role":"warehouse"}` { + t.Errorf("metadata cleared on nil re-add: got %q", meta) + } + // Explicit empty {} replaces. + _ = idx.Add("alice", []float32{1, 0}, json.RawMessage(`{}`)) + _, meta, _ = idx.Lookup("alice") + if string(meta) != `{}` { + t.Errorf("explicit {} should replace: got %q", meta) + } +} + +func TestLookup_ReturnsCopy(t *testing.T) { + // Per scrum O-W1: caller mutation must not corrupt index state. + idx, _ := NewIndex(IndexParams{Name: "x", Dimension: 3}) + orig := []float32{1, 2, 3} + _ = idx.Add("a", orig, nil) + got, _, _ := idx.Lookup("a") + got[0] = 99 // mutate the returned copy + again, _, _ := idx.Lookup("a") + if again[0] != 1 { + t.Errorf("Lookup didn't copy: index now sees %v", again) + } +} + +func TestRegistry_Names_Sorted(t *testing.T) { + r := NewRegistry() + for _, n := range []string{"zoo", "alpha", "midway"} { + _, _ = r.Create(IndexParams{Name: n, Dimension: 4}) + } + got := r.Names() + want := []string{"alpha", "midway", "zoo"} + for i, w := range want { + if got[i] != w { + t.Errorf("Names[%d]: got %q, want %q", i, got[i], w) + } + } +} diff --git a/internal/vectord/registry.go b/internal/vectord/registry.go new file mode 100644 index 0000000..02feacc --- /dev/null +++ b/internal/vectord/registry.go @@ -0,0 +1,85 @@ +// registry.go — multi-index manager. Mirrors the catalogd Registry +// shape: a thread-safe map[name]*Index with Create / Get / Delete. +// Per-index operations (Add, Search) go through each Index's own +// RWMutex so registry-wide locking only fires on lifecycle events. +package vectord + +import ( + "errors" + "fmt" + "sort" + "sync" +) + +// ErrIndexNotFound is returned by Get / Delete when the requested +// name has no registered index. +var ErrIndexNotFound = errors.New("vectord: index not found") + +// ErrIndexAlreadyExists is returned by Create when the name is +// taken. Callers can treat this as a 409 Conflict — paralleling +// catalogd's ADR-020 idempotency contract, but stricter (no +// "same params reuses index" semantics yet). +var ErrIndexAlreadyExists = errors.New("vectord: index already exists") + +// Registry holds the live indexes by name. +type Registry struct { + mu sync.RWMutex + indexes map[string]*Index +} + +func NewRegistry() *Registry { + return &Registry{indexes: make(map[string]*Index)} +} + +// Create builds a new Index from params and registers it under +// params.Name. Returns ErrIndexAlreadyExists if the name is taken. +func (r *Registry) Create(p IndexParams) (*Index, error) { + idx, err := NewIndex(p) + if err != nil { + return nil, err + } + r.mu.Lock() + defer r.mu.Unlock() + if _, exists := r.indexes[p.Name]; exists { + return nil, fmt.Errorf("%w: %q", ErrIndexAlreadyExists, p.Name) + } + r.indexes[p.Name] = idx + return idx, nil +} + +// Get returns the index for name, or ErrIndexNotFound. +func (r *Registry) Get(name string) (*Index, error) { + r.mu.RLock() + defer r.mu.RUnlock() + idx, ok := r.indexes[name] + if !ok { + return nil, fmt.Errorf("%w: %q", ErrIndexNotFound, name) + } + return idx, nil +} + +// Delete removes the index for name. Returns ErrIndexNotFound if +// not present (so callers see explicit no-op vs success on the +// idempotent path). +func (r *Registry) Delete(name string) error { + r.mu.Lock() + defer r.mu.Unlock() + if _, ok := r.indexes[name]; !ok { + return fmt.Errorf("%w: %q", ErrIndexNotFound, name) + } + delete(r.indexes, name) + return nil +} + +// Names returns the registered index names sorted ascending — +// stable enumeration for /v1/vectors GET listings. +func (r *Registry) Names() []string { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]string, 0, len(r.indexes)) + for name := range r.indexes { + out = append(out, name) + } + sort.Strings(out) + return out +} diff --git a/lakehouse.toml b/lakehouse.toml index fa0aea8..99b31e2 100644 --- a/lakehouse.toml +++ b/lakehouse.toml @@ -10,6 +10,7 @@ storaged_url = "http://127.0.0.1:3211" catalogd_url = "http://127.0.0.1:3212" ingestd_url = "http://127.0.0.1:3213" queryd_url = "http://127.0.0.1:3214" +vectord_url = "http://127.0.0.1:3215" [storaged] bind = "127.0.0.1:3211" @@ -27,6 +28,9 @@ catalogd_url = "http://127.0.0.1:3212" # datasets (e.g. workers_500k → 344 MiB CSV needs 512 MiB). max_ingest_bytes = 268435456 +[vectord] +bind = "127.0.0.1:3215" + [queryd] bind = "127.0.0.1:3214" catalogd_url = "http://127.0.0.1:3212" diff --git a/scripts/g1_smoke.sh b/scripts/g1_smoke.sh new file mode 100755 index 0000000..36dacb1 --- /dev/null +++ b/scripts/g1_smoke.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# G1 smoke — vectord (HNSW vector search). All assertions go +# through gateway :3110, none direct to vectord :3215. +# +# Validates: +# - POST /v1/vectors/index → 201, params echoed back +# - POST /v1/vectors/index/{name}/add (batch of 200) → 200, length=200 +# - POST /v1/vectors/index/{name}/search → top-K with the inserted +# vector at distance≈0 (recall=1 for the inserted vector itself) +# - Dim mismatch on Add → 400 +# - Search on missing index → 404 +# - Duplicate Create → 409 +# - DELETE then Search → 404 +# +# Requires vectord + gateway. Other backing services aren't needed. +# +# Usage: ./scripts/g1_smoke.sh + +set -euo pipefail +cd "$(dirname "$0")/.." + +export PATH="$PATH:/usr/local/go/bin" + +echo "[g1-smoke] building vectord + gateway..." +go build -o bin/ ./cmd/vectord ./cmd/gateway + +pkill -f "bin/vectord|bin/gateway" 2>/dev/null || true +sleep 0.3 + +PIDS=() +TMP="$(mktemp -d)" +cleanup() { + echo "[g1-smoke] cleanup" + for p in "${PIDS[@]}"; do [ -n "$p" ] && kill "$p" 2>/dev/null || true; done + rm -rf "$TMP" +} +trap cleanup EXIT INT TERM + +poll_health() { + local port="$1" deadline=$(($(date +%s) + 5)) + while [ "$(date +%s)" -lt "$deadline" ]; do + if curl -sS --max-time 1 "http://127.0.0.1:$port/health" >/dev/null 2>&1; then return 0; fi + sleep 0.05 + done + return 1 +} + +echo "[g1-smoke] launching vectord → gateway..." +./bin/vectord > /tmp/vectord.log 2>&1 & +PIDS+=($!) +poll_health 3215 || { echo "vectord failed"; tail /tmp/vectord.log; exit 1; } +./bin/gateway > /tmp/gateway.log 2>&1 & +PIDS+=($!) +poll_health 3110 || { echo "gateway failed"; tail /tmp/gateway.log; exit 1; } + +FAILED=0 +NAME="g1_workers" +DIM=8 + +echo "[g1-smoke] /v1/vectors/index — create dim=$DIM:" +HTTP="$(curl -sS -o "$TMP/create.out" -w '%{http_code}' -X POST http://127.0.0.1:3110/v1/vectors/index \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"$NAME\",\"dimension\":$DIM,\"distance\":\"cosine\"}")" +if [ "$HTTP" = "201" ]; then + echo " ✓ create → 201" +else + echo " ✗ create → $HTTP body=$(cat "$TMP/create.out")"; FAILED=1 +fi + +echo "[g1-smoke] duplicate create → 409:" +HTTP="$(curl -sS -o /dev/null -w '%{http_code}' -X POST http://127.0.0.1:3110/v1/vectors/index \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"$NAME\",\"dimension\":$DIM}")" +if [ "$HTTP" = "409" ]; then + echo " ✓ duplicate → 409" +else + echo " ✗ duplicate → $HTTP"; FAILED=1 +fi + +echo "[g1-smoke] add batch of 200 vectors:" +# Build a batch payload with 200 items. ID encodes index; vector +# is a deterministic spread so we can assert recall on a known id. +python3 - < "$TMP/batch.json" +import json, math +items = [] +for i in range(200): + # Each vector is unit-norm with one dominant axis cycling through 8 dims. + v = [0.0] * $DIM + v[i % $DIM] = 1.0 + # Add a tiny perturbation per id so vectors are distinct. + v[(i+1) % $DIM] = (i % 17) * 0.01 + items.append({"id": f"w-{i:03d}", "vector": v, "metadata": {"row": i}}) +print(json.dumps({"items": items})) +EOF +ADD_RESP="$(curl -sS -X POST "http://127.0.0.1:3110/v1/vectors/index/$NAME/add" \ + -H 'Content-Type: application/json' \ + -d @"$TMP/batch.json")" +LEN="$(echo "$ADD_RESP" | jq -r '.length')" +ADDED="$(echo "$ADD_RESP" | jq -r '.added')" +if [ "$LEN" = "200" ] && [ "$ADDED" = "200" ]; then + echo " ✓ added=200, length=200" +else + echo " ✗ add → $ADD_RESP"; FAILED=1 +fi + +echo "[g1-smoke] search for inserted vector w-042 → recall:" +# Build the same vector w-042 was inserted with. +python3 - < "$TMP/q.json" +import json +i = 42 +v = [0.0] * $DIM +v[i % $DIM] = 1.0 +v[(i+1) % $DIM] = (i % 17) * 0.01 +print(json.dumps({"vector": v, "k": 3})) +EOF +SEARCH_RESP="$(curl -sS -X POST "http://127.0.0.1:3110/v1/vectors/index/$NAME/search" \ + -H 'Content-Type: application/json' \ + -d @"$TMP/q.json")" +TOP_ID="$(echo "$SEARCH_RESP" | jq -r '.results[0].id')" +TOP_DIST="$(echo "$SEARCH_RESP" | jq -r '.results[0].distance')" +TOP_META="$(echo "$SEARCH_RESP" | jq -c '.results[0].metadata')" +RC="$(echo "$SEARCH_RESP" | jq -r '.results | length')" +# Distance should be ~0 (cosine of identical unit vectors). Allow 1e-4. +DIST_OK="$(python3 -c "import sys; sys.exit(0 if abs($TOP_DIST) < 1e-4 else 1)" && echo y || echo n)" +if [ "$TOP_ID" = "w-042" ] && [ "$DIST_OK" = "y" ] && [ "$RC" = "3" ] && [ "$TOP_META" = '{"row":42}' ]; then + echo " ✓ top hit = w-042 (dist=$TOP_DIST), 3 results, metadata round-tripped" +else + echo " ✗ search → top_id=$TOP_ID dist=$TOP_DIST rc=$RC meta=$TOP_META" + echo " full: $SEARCH_RESP" + FAILED=1 +fi + +echo "[g1-smoke] dim mismatch on add → 400:" +HTTP="$(curl -sS -o /dev/null -w '%{http_code}' -X POST "http://127.0.0.1:3110/v1/vectors/index/$NAME/add" \ + -H 'Content-Type: application/json' \ + -d '{"items":[{"id":"bad","vector":[1,2,3]}]}')" +if [ "$HTTP" = "400" ]; then + echo " ✓ dim mismatch → 400" +else + echo " ✗ dim mismatch → $HTTP"; FAILED=1 +fi + +echo "[g1-smoke] search on missing index → 404:" +HTTP="$(curl -sS -o /dev/null -w '%{http_code}' -X POST "http://127.0.0.1:3110/v1/vectors/index/no_such/search" \ + -H 'Content-Type: application/json' \ + -d "{\"vector\":[1,0,0,0,0,0,0,0],\"k\":1}")" +if [ "$HTTP" = "404" ]; then + echo " ✓ unknown index → 404" +else + echo " ✗ unknown index → $HTTP"; FAILED=1 +fi + +echo "[g1-smoke] DELETE then GET → 404:" +curl -sS -o /dev/null -X DELETE "http://127.0.0.1:3110/v1/vectors/index/$NAME" +HTTP="$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1:3110/v1/vectors/index/$NAME")" +if [ "$HTTP" = "404" ]; then + echo " ✓ post-delete GET → 404" +else + echo " ✗ post-delete → $HTTP"; FAILED=1 +fi + +if [ "$FAILED" -eq 0 ]; then + echo "[g1-smoke] G1 acceptance gate: PASSED" + exit 0 +else + echo "[g1-smoke] G1 acceptance gate: FAILED" + exit 1 +fi