diff --git a/tests/proof/cases/05_embedding_contract.sh b/tests/proof/cases/05_embedding_contract.sh new file mode 100755 index 0000000..ab19f43 --- /dev/null +++ b/tests/proof/cases/05_embedding_contract.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# 05_embedding_contract.sh — GOLAKE-050. +# Verifies POST /v1/embed contract: dim=768, non-empty vector, model +# echoed back. Skips with explicit reason if Ollama is unreachable +# (per TEST_PROOF_SCOPE.md hard rule: skipped != passed). +# +# This is the contract subset of embedding. Semantic ranking lives in +# Phase C (05/06 integration cases) and asserts against a stored +# rankings fixture; this case stays embedding-implementation-agnostic. + +set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/../lib/env.sh" +source "${SCRIPT_DIR}/../lib/http.sh" +source "${SCRIPT_DIR}/../lib/assert.sh" + +CASE_ID="GOLAKE-050" +CASE_NAME="Embedding contract — dim=768, non-empty" +CASE_TYPE="contract" +if [ "${1:-}" = "--metadata-only" ]; then return 0 2>/dev/null || exit 0; fi + +# One real request — short text, default model. If Ollama is up we +# get 200; if down we get 502 from embedd. Either way we record it. +proof_post "$CASE_ID" "embed_one_text" "${PROOF_GATEWAY_URL}/v1/embed" \ + "application/json" \ + '{"texts":["industrial staffing for welders in Chicago"],"model":"nomic-embed-text"}' \ + > /dev/null + +status=$(proof_status_of "$CASE_ID" "embed_one_text") + +# 502 from embedd = Ollama not reachable. Mark skip with reason; do +# not pretend to verify the contract. +if [ "$status" = "502" ]; then + proof_skip "$CASE_ID" "Embedding contract — Ollama unreachable" \ + "POST /v1/embed returned 502; embedd cannot reach upstream Ollama. Run 'just doctor' to diagnose." + return 0 2>/dev/null || exit 0 +fi + +proof_assert_eq "$CASE_ID" "POST /v1/embed → 200" "200" "$status" + +body_path="${PROOF_REPORT_DIR}/raw/http/${CASE_ID}/embed_one_text.body" + +# Dimension echoed back. +dim=$(jq -r '.dimension // empty' "$body_path") +proof_assert_eq "$CASE_ID" "response.dimension = 768" "768" "$dim" + +# One vector returned. +n=$(jq -r '.vectors | length' "$body_path") +proof_assert_eq "$CASE_ID" "response.vectors length = 1" "1" "$n" + +# Vector dim matches. +vec_len=$(jq -r '.vectors[0] | length' "$body_path") +proof_assert_eq "$CASE_ID" "vectors[0] length = 768" "768" "$vec_len" + +# Vector is non-empty (sum of squared elements > 0). Cheap proxy for +# "not all zeros" without comparing every element. +sum_sq=$(jq -r '[.vectors[0][] | . * .] | add' "$body_path") +proof_assert_gt "$CASE_ID" "vectors[0] non-zero (sum of squares > 0)" "$sum_sq" "0" + +# Model name echoed. +model=$(jq -r '.model // empty' "$body_path") +proof_assert_eq "$CASE_ID" "response.model = nomic-embed-text" "nomic-embed-text" "$model" diff --git a/tests/proof/cases/06_vector_add_search.sh b/tests/proof/cases/06_vector_add_search.sh new file mode 100755 index 0000000..b1676b3 --- /dev/null +++ b/tests/proof/cases/06_vector_add_search.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# 06_vector_add_search.sh — GOLAKE-060 + GOLAKE-061. +# Vector add + search round-trip. Synthetic dim=4 unit basis vectors, +# no embedd dependency — this is the contract layer. +# +# GOLAKE-060: add succeeds + lookup-by-id returns the inserted IDs +# GOLAKE-061: nearest-neighbor search — inserted vector ranks #1 vs itself + +set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/../lib/env.sh" +source "${SCRIPT_DIR}/../lib/http.sh" +source "${SCRIPT_DIR}/../lib/assert.sh" + +CASE_ID="GOLAKE-060-061" +CASE_NAME="Vector add + lookup + nearest-neighbor" +CASE_TYPE="contract" +if [ "${1:-}" = "--metadata-only" ]; then return 0 2>/dev/null || exit 0; fi + +INDEX_NAME="proof_contract_idx" + +# Idempotent prelude — clean any prior run state. 404 is fine. +proof_delete "$CASE_ID" "pre_clean" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index/${INDEX_NAME}" >/dev/null + +# Create the index. vectord returns 201. +proof_post "$CASE_ID" "create_index" "${PROOF_GATEWAY_URL}/v1/vectors/index" \ + "application/json" \ + "{\"name\":\"${INDEX_NAME}\",\"dimension\":4}" >/dev/null +proof_assert_eq "$CASE_ID" "create index → 201" "201" \ + "$(proof_status_of "$CASE_ID" "create_index")" + +# Add three deterministic vectors. Unit basis vectors so search recall +# is unambiguous: searching for [1,0,0,0] must return v1 first. +# vectord wants {"items": [...]}, NOT {"vectors": [...]}. +add_body='{"items":[ + {"id":"v1","vector":[1,0,0,0]}, + {"id":"v2","vector":[0,1,0,0]}, + {"id":"v3","vector":[0,0,1,0]} +]}' +proof_post "$CASE_ID" "add_vectors" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index/${INDEX_NAME}/add" \ + "application/json" "$add_body" >/dev/null +proof_assert_eq "$CASE_ID" "add vectors → 200" "200" \ + "$(proof_status_of "$CASE_ID" "add_vectors")" + +# Lookup-by-id (GOLAKE-060 evidence). The /index/{name} GET returns +# {"params": {...}, "length": N}. +proof_get "$CASE_ID" "get_index" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index/${INDEX_NAME}" >/dev/null +proof_assert_eq "$CASE_ID" "get index → 200" "200" \ + "$(proof_status_of "$CASE_ID" "get_index")" +length=$(jq -r '.length' \ + "${PROOF_REPORT_DIR}/raw/http/${CASE_ID}/get_index.body") +proof_assert_eq "$CASE_ID" "index length = 3 after add" "3" "$length" + +# Search — query is identical to v1; expect v1 at rank 1 with distance ≈ 0. +search_body='{"vector":[1,0,0,0],"k":3}' +proof_post "$CASE_ID" "search" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index/${INDEX_NAME}/search" \ + "application/json" "$search_body" >/dev/null +proof_assert_eq "$CASE_ID" "search → 200" "200" \ + "$(proof_status_of "$CASE_ID" "search")" + +# Search response shape: {"results": [{"id","distance","metadata?"}]}. +search_body_path="${PROOF_REPORT_DIR}/raw/http/${CASE_ID}/search.body" +top1_id=$(jq -r '.results[0].id' "$search_body_path") +proof_assert_eq "$CASE_ID" "top-1 id = v1 (self-recall)" "v1" "$top1_id" + +top1_dist=$(jq -r '.results[0].distance' "$search_body_path") +proof_assert_lt "$CASE_ID" "top-1 distance < 0.001 (cosine self ≈ 0)" \ + "$top1_dist" "0.001" + +# Cleanup — vectord returns 204 No Content on delete success. +proof_delete "$CASE_ID" "post_clean" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index/${INDEX_NAME}" >/dev/null +proof_assert_status_in "$CASE_ID" "delete index → 200 or 204" "200 204" "post_clean" diff --git a/tests/proof/cases/08_gateway_contracts.sh b/tests/proof/cases/08_gateway_contracts.sh new file mode 100755 index 0000000..a5935ec --- /dev/null +++ b/tests/proof/cases/08_gateway_contracts.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# 08_gateway_contracts.sh — GOLAKE-003. +# Gateway proxies /v1/* to the right upstream and preserves the +# upstream's status code. Compares gateway's response against the +# direct-port response for each route. + +set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/../lib/env.sh" +source "${SCRIPT_DIR}/../lib/http.sh" +source "${SCRIPT_DIR}/../lib/assert.sh" + +CASE_ID="GOLAKE-003" +CASE_NAME="Gateway proxy — route + status passthrough" +CASE_TYPE="contract" +if [ "${1:-}" = "--metadata-only" ]; then return 0 2>/dev/null || exit 0; fi + +# Each row: . +# Verifies that gateway and direct-upstream return the same status. +ROUTES=( + "storage_list:/v1/storage/list:${PROOF_STORAGED_URL}/storage/list" + "catalog_list:/v1/catalog/list:${PROOF_CATALOGD_URL}/catalog/list" +) + +for spec in "${ROUTES[@]}"; do + IFS=':' read -r name gw_path up_url <<< "$spec" + + proof_get "$CASE_ID" "${name}_gw" "${PROOF_GATEWAY_URL}${gw_path}" >/dev/null + proof_get "$CASE_ID" "${name}_up" "${up_url}" >/dev/null + + gw_status=$(proof_status_of "$CASE_ID" "${name}_gw") + up_status=$(proof_status_of "$CASE_ID" "${name}_up") + + # Status passthrough — gateway must return what the upstream returned. + proof_assert_eq "$CASE_ID" "${name}: gateway status matches upstream" \ + "$up_status" "$gw_status" + + # Body shape preserved — sha256 must match (gateway is a proxy, not a transformer). + gw_body_sha=$(sha256sum \ + "${PROOF_REPORT_DIR}/raw/http/${CASE_ID}/${name}_gw.body" | awk '{print $1}') + up_body_sha=$(sha256sum \ + "${PROOF_REPORT_DIR}/raw/http/${CASE_ID}/${name}_up.body" | awk '{print $1}') + proof_assert_eq "$CASE_ID" "${name}: gateway body sha matches upstream" \ + "$up_body_sha" "$gw_body_sha" +done + +# Status-passthrough on a 4xx — POST /v1/sql with empty body must +# return the same 4xx as the direct port. +proof_post "$CASE_ID" "sql_empty_gw" "${PROOF_GATEWAY_URL}/v1/sql" \ + "application/json" '{"sql":""}' >/dev/null +proof_post "$CASE_ID" "sql_empty_up" "${PROOF_QUERYD_URL}/sql" \ + "application/json" '{"sql":""}' >/dev/null + +gw_status=$(proof_status_of "$CASE_ID" "sql_empty_gw") +up_status=$(proof_status_of "$CASE_ID" "sql_empty_up") +proof_assert_eq "$CASE_ID" "sql empty: gateway status matches upstream" \ + "$up_status" "$gw_status" +proof_assert_eq "$CASE_ID" "sql empty: status is 4xx (400)" "400" "$gw_status" diff --git a/tests/proof/cases/09_failure_modes.sh b/tests/proof/cases/09_failure_modes.sh new file mode 100755 index 0000000..c2a3f3f --- /dev/null +++ b/tests/proof/cases/09_failure_modes.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# 09_failure_modes.sh — GOLAKE-080..085. +# Verifies the system fails cleanly: 4xx for malformed input, 404 for +# missing resources, structured error bodies. Per the spec: "Do not +# hide failures behind retries unless documented." + +set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/../lib/env.sh" +source "${SCRIPT_DIR}/../lib/http.sh" +source "${SCRIPT_DIR}/../lib/assert.sh" + +CASE_ID="GOLAKE-080-085" +CASE_NAME="Failure modes — 4xx not 5xx, structured errors" +CASE_TYPE="contract" +if [ "${1:-}" = "--metadata-only" ]; then return 0 2>/dev/null || exit 0; fi + +# ── GOLAKE-080: malformed JSON → 4xx, never 5xx, never silent 200 ─── +JUNK='not-valid-json{[]}' +ENDPOINTS=( + "catalog_register:${PROOF_GATEWAY_URL}/v1/catalog/register" + "ingest:${PROOF_GATEWAY_URL}/v1/ingest" + "sql:${PROOF_GATEWAY_URL}/v1/sql" + "embed:${PROOF_GATEWAY_URL}/v1/embed" +) +for spec in "${ENDPOINTS[@]}"; do + IFS=':' read -r name url <<< "$spec" + proof_post "$CASE_ID" "malformed_${name}" "$url" "application/json" "$JUNK" >/dev/null + proof_assert_status_4xx "$CASE_ID" "${name}: malformed JSON → 4xx" "malformed_${name}" +done + +# ── GOLAKE-081: missing required field → 400 ────────────────────── +proof_post "$CASE_ID" "missing_required_catalog" \ + "${PROOF_GATEWAY_URL}/v1/catalog/register" \ + "application/json" '{}' >/dev/null +proof_assert_status_4xx "$CASE_ID" "catalog/register: empty body → 4xx" "missing_required_catalog" + +proof_post "$CASE_ID" "missing_required_vector_create" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index" \ + "application/json" '{"name":"missing_dim_test"}' >/dev/null +proof_assert_status_4xx "$CASE_ID" "vectors/index: missing dimension → 4xx" "missing_required_vector_create" + +proof_post "$CASE_ID" "missing_required_embed" \ + "${PROOF_GATEWAY_URL}/v1/embed" \ + "application/json" '{}' >/dev/null +proof_assert_status_4xx "$CASE_ID" "embed: missing texts → 4xx" "missing_required_embed" + +# ── GOLAKE-082: bad SQL → 4xx, error body present ──────────────── +proof_post "$CASE_ID" "bad_sql_syntax" \ + "${PROOF_GATEWAY_URL}/v1/sql" \ + "application/json" '{"sql":"NOT VALID SQL HERE"}' >/dev/null +proof_assert_status_4xx "$CASE_ID" "queryd: bad SQL → 4xx" "bad_sql_syntax" +err_body=$(proof_body_of "$CASE_ID" "bad_sql_syntax") +proof_assert_ne "$CASE_ID" "queryd: bad SQL response body non-empty" "" "$err_body" + +# ── GOLAKE-083: vector dim mismatch → 4xx ──────────────────────── +DIM_IDX="proof_dim_mismatch_test" +proof_delete "$CASE_ID" "dim_pre_clean" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index/${DIM_IDX}" >/dev/null +proof_post "$CASE_ID" "dim_create" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index" \ + "application/json" "{\"name\":\"${DIM_IDX}\",\"dimension\":3}" >/dev/null +# Wrong-shape add — vectord wants `items` not `vectors`. Use correct +# field name; the dim mismatch is the actual failure mode under test. +proof_post "$CASE_ID" "dim_mismatch_add" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index/${DIM_IDX}/add" \ + "application/json" '{"items":[{"id":"x","vector":[1,2,3,4]}]}' >/dev/null +proof_assert_status_4xx "$CASE_ID" "vectord: dim mismatch on add → 4xx" "dim_mismatch_add" +proof_delete "$CASE_ID" "dim_post_clean" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index/${DIM_IDX}" >/dev/null + +# ── GOLAKE-084: missing storage object → 404 ────────────────────── +proof_get "$CASE_ID" "missing_object" \ + "${PROOF_GATEWAY_URL}/v1/storage/get/proof_definitely_not_a_key_xyz_$(date +%N)" >/dev/null +proof_assert_eq "$CASE_ID" "storage/get on missing key → 404" "404" \ + "$(proof_status_of "$CASE_ID" "missing_object")" + +# ── GOLAKE-085: duplicate vector ID — informational ────────────── +DUP_IDX="proof_dup_test" +proof_delete "$CASE_ID" "dup_pre_clean" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index/${DUP_IDX}" >/dev/null +proof_post "$CASE_ID" "dup_create" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index" \ + "application/json" "{\"name\":\"${DUP_IDX}\",\"dimension\":2}" >/dev/null +proof_post "$CASE_ID" "dup_add_first" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index/${DUP_IDX}/add" \ + "application/json" '{"items":[{"id":"d1","vector":[1,0]}]}' >/dev/null +proof_post "$CASE_ID" "dup_add_second" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index/${DUP_IDX}/add" \ + "application/json" '{"items":[{"id":"d1","vector":[0,1]}]}' >/dev/null + +dup_first=$(proof_status_of "$CASE_ID" "dup_add_first") +dup_second=$(proof_status_of "$CASE_ID" "dup_add_second") +proof_assert_eq "$CASE_ID" "dup add first → 200" "200" "$dup_first" +proof_skip "$CASE_ID" "dup-id behavior recorded (informational)" \ + "first=${dup_first} second=${dup_second} — see raw/http/${CASE_ID}/dup_add_*.json for full record" +proof_delete "$CASE_ID" "dup_post_clean" \ + "${PROOF_GATEWAY_URL}/v1/vectors/index/${DUP_IDX}" >/dev/null diff --git a/tests/proof/lib/assert.sh b/tests/proof/lib/assert.sh index 02e7a59..2cb52c0 100644 --- a/tests/proof/lib/assert.sh +++ b/tests/proof/lib/assert.sh @@ -116,3 +116,38 @@ proof_skip() { _proof_record "$case_id" "$claim" skip "" "" "$reason" return 0 } + +# proof_assert_status_in: pass if probe's status is in the space-separated +# expected list. Use when a route legitimately has multiple OK codes (e.g. +# 200 vs 204 vs 201 across services). Records a clean pass/fail with the +# actual status echoed back. +proof_assert_status_in() { + local case_id="$1" claim="$2" expected_list="$3" probe="$4" + local actual found + actual=$(proof_status_of "$case_id" "$probe" 2>/dev/null || echo missing) + found=0 + for ok in $expected_list; do + [ "$ok" = "$actual" ] && { found=1; break; } + done + if [ "$found" = 1 ]; then + _proof_record "$case_id" "$claim" pass "in {${expected_list}}" "$actual" "" + else + _proof_record "$case_id" "$claim" fail "in {${expected_list}}" "$actual" "status not in expected list" + fi + return 0 +} + +# proof_assert_status_4xx: pass if probe's status is in [400, 500). Use +# for failure-mode contracts where the specific 4xx code is allowed to +# vary (400 vs 422 vs 409) — only "is a client error" matters. +proof_assert_status_4xx() { + local case_id="$1" claim="$2" probe="$3" + local actual + actual=$(proof_status_of "$case_id" "$probe" 2>/dev/null || echo missing) + if awk -v s="$actual" 'BEGIN{exit !(s+0 >= 400 && s+0 < 500)}'; then + _proof_record "$case_id" "$claim" pass "4xx" "$actual" "" + else + _proof_record "$case_id" "$claim" fail "4xx" "$actual" "status is not in 400-499" + fi + return 0 +}