7d6636b33e
Closes the 2026-05-02 parity finding: validator_parity probe found
5/6 body shapes diverging because Go emitted {"Kind":"...","Field":"...","Reason":"..."}
while Rust emits the externally-tagged-enum {"Schema":{"field":"...","reason":"..."}}.
A caller parsing the error envelope would break silently in cutover.
## Changes
internal/validator/types.go:
- Custom MarshalJSON emits the Rust shape:
Schema: {"Schema": {"field":"x","reason":"y"}}
Completeness: {"Completeness":{"reason":"y"}}
Consistency: {"Consistency": {"reason":"y"}}
Policy: {"Policy": {"reason":"y"}}
- Custom UnmarshalJSON accepts BOTH the new Rust shape AND the legacy
flat shape (migration safety for any persisted error rows).
- Unknown variants (e.g. a future Rust addition Go hasn't learned)
surface as an Unmarshal error, not a silent default.
internal/validator/types_test.go:
- 4 pinning tests anchor the wire format. Failing them = wire-format
drift; the parity probe is the secondary line of defense.
scripts/validatord_smoke.sh:
- Updated probes to read the new variant-name shape (jq keys[0],
.Schema.field) instead of legacy .Kind/.Field.
## Verification
- internal/validator unit tests: PASS (4 new + all existing).
- cmd/validatord HTTP tests: PASS (UnmarshalJSON falls through to flat
shape so existing tests reading ValidationError still work).
- validatord_smoke.sh: 5/5 PASS through gateway :3110.
- validator parity probe re-run: **6/6 match** (was 1/6).
## Pattern
Per architecture_comparison's "use the dual-implementation as a
measurement instrument" thesis: a parity probe surfaced this gap;
50 LOC of MarshalJSON closed it; 4 pinning tests prevent regression;
the probe is the longitudinal gate. Cutover-friendly direction (Go
matches Rust) chosen because Rust is the existing production
contract.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
157 lines
6.3 KiB
Bash
Executable File
157 lines
6.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# validatord smoke — Phase 43 PRD parity acceptance gate.
|
|
#
|
|
# Validates:
|
|
# - validatord boots, reports /health
|
|
# - POST /v1/validate with kind=playbook returns 200 + Report on
|
|
# well-formed input
|
|
# - POST /v1/validate with kind=playbook returns 422 + ValidationError
|
|
# when fingerprint is missing
|
|
# - POST /v1/validate with kind=fill consults the JSONL roster
|
|
# (phantom candidate → 422 Consistency)
|
|
# - POST /v1/validate with unknown kind returns 400
|
|
# - All assertions go through gateway :3110 (proxy correct)
|
|
#
|
|
# Doesn't exercise /iterate — that needs a live chat backend, covered
|
|
# by cmd/validatord/main_test.go's fakeChatd helper. CI-friendly.
|
|
#
|
|
# Usage: ./scripts/validatord_smoke.sh
|
|
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
|
|
export PATH="$PATH:/usr/local/go/bin"
|
|
|
|
echo "[validatord-smoke] building validatord + gateway..."
|
|
go build -o bin/ ./cmd/validatord ./cmd/gateway
|
|
|
|
pkill -f "bin/(validatord|gateway)$" 2>/dev/null || true
|
|
sleep 0.3
|
|
|
|
PIDS=()
|
|
TMP="$(mktemp -d)"
|
|
ROSTER="$TMP/roster.jsonl"
|
|
CFG="$TMP/validatord.toml"
|
|
|
|
cleanup() {
|
|
echo "[validatord-smoke] cleanup"
|
|
for p in "${PIDS[@]:-}"; do [ -n "${p:-}" ] && kill "$p" 2>/dev/null || true; done
|
|
rm -rf "$TMP"
|
|
}
|
|
trap cleanup EXIT INT TERM
|
|
|
|
# Tiny synthetic roster so /v1/validate fill-kind has something to
|
|
# pass / fail against. Two real candidates + one inactive.
|
|
cat > "$ROSTER" <<EOF
|
|
{"candidate_id":"W-1","name":"Ada","status":"active","city":"Toledo","state":"OH","role":"Welder","blacklisted_clients":[]}
|
|
{"candidate_id":"W-2","name":"Bea","status":"active","city":"Toledo","state":"OH","role":"Welder","blacklisted_clients":["C-EVIL"]}
|
|
{"candidate_id":"W-3","name":"Cleo","status":"inactive","city":"Toledo","state":"OH","role":"Welder","blacklisted_clients":[]}
|
|
EOF
|
|
|
|
cat > "$CFG" <<EOF
|
|
[gateway]
|
|
bind = "127.0.0.1:3110"
|
|
storaged_url = "http://127.0.0.1:3211"
|
|
catalogd_url = "http://127.0.0.1:3212"
|
|
ingestd_url = "http://127.0.0.1:3213"
|
|
queryd_url = "http://127.0.0.1:3214"
|
|
vectord_url = "http://127.0.0.1:3215"
|
|
embedd_url = "http://127.0.0.1:3216"
|
|
pathwayd_url = "http://127.0.0.1:3217"
|
|
matrixd_url = "http://127.0.0.1:3218"
|
|
observerd_url = "http://127.0.0.1:3219"
|
|
chatd_url = "http://127.0.0.1:3220"
|
|
validatord_url = "http://127.0.0.1:3221"
|
|
|
|
[validatord]
|
|
bind = "127.0.0.1:3221"
|
|
chatd_url = "http://127.0.0.1:3220"
|
|
roster_path = "$ROSTER"
|
|
default_max_iterations = 3
|
|
default_max_tokens = 4096
|
|
chat_timeout_secs = 240
|
|
EOF
|
|
|
|
poll_health() {
|
|
local port="$1" deadline=$(($(date +%s) + 5))
|
|
while [ "$(date +%s)" -lt "$deadline" ]; do
|
|
if curl -sS --max-time 1 "http://127.0.0.1:$port/health" >/dev/null 2>&1; then return 0; fi
|
|
sleep 0.05
|
|
done
|
|
return 1
|
|
}
|
|
|
|
echo "[validatord-smoke] launching validatord → gateway..."
|
|
./bin/validatord -config "$CFG" > /tmp/validatord.log 2>&1 & PIDS+=($!)
|
|
poll_health 3221 || { echo "validatord failed"; tail /tmp/validatord.log; exit 1; }
|
|
./bin/gateway -config "$CFG" > /tmp/validatord_gateway.log 2>&1 & PIDS+=($!)
|
|
poll_health 3110 || { echo "gateway failed"; tail /tmp/validatord_gateway.log; exit 1; }
|
|
|
|
# 1. Roster loaded with 3 records — surface via the daemon's startup log.
|
|
if ! grep -q '"records":3' /tmp/validatord.log && ! grep -q 'records=3' /tmp/validatord.log; then
|
|
echo " ✗ expected validatord to log records=3 from roster; got:"
|
|
grep "validatord roster" /tmp/validatord.log || true
|
|
exit 1
|
|
fi
|
|
echo " ✓ validatord roster loaded with 3 records"
|
|
|
|
# 2. /v1/validate playbook happy path → 200
|
|
echo "[validatord-smoke] /v1/validate playbook happy path:"
|
|
RESP="$(curl -sS -X POST http://127.0.0.1:3110/v1/validate \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"kind":"playbook","artifact":{"operation":"fill: Welder x2 in Toledo, OH","endorsed_names":["W-1","W-2"],"target_count":2,"fingerprint":"abc123"}}')"
|
|
if ! echo "$RESP" | jq -e '.elapsed_ms != null and (.findings | type == "array")' >/dev/null; then
|
|
echo " ✗ unexpected response: $RESP"
|
|
exit 1
|
|
fi
|
|
echo " ✓ playbook OK ($RESP)"
|
|
|
|
# 3. /v1/validate playbook schema error → 422 with ValidationError
|
|
echo "[validatord-smoke] /v1/validate playbook missing fingerprint → 422:"
|
|
STATUS="$(curl -sS -o /tmp/playbook_422.json -w '%{http_code}' -X POST http://127.0.0.1:3110/v1/validate \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"kind":"playbook","artifact":{"operation":"fill: X x1 in A, B","endorsed_names":["a"]}}')"
|
|
if [ "$STATUS" != "422" ]; then
|
|
echo " ✗ expected 422; got $STATUS body=$(cat /tmp/playbook_422.json)"
|
|
exit 1
|
|
fi
|
|
# Rust serde-tagged-enum shape (parity with crates/validator):
|
|
# {"Schema":{"field":"fingerprint","reason":"..."}}
|
|
VARIANT="$(jq -r 'keys[0]' /tmp/playbook_422.json)"
|
|
FIELD="$(jq -r '.Schema.field' /tmp/playbook_422.json)"
|
|
if [ "$VARIANT" != "Schema" ] || [ "$FIELD" != "fingerprint" ]; then
|
|
echo " ✗ expected variant=Schema field=fingerprint; got variant=$VARIANT field=$FIELD"
|
|
exit 1
|
|
fi
|
|
echo " ✓ playbook missing fingerprint → 422 Schema/fingerprint"
|
|
|
|
# 4. /v1/validate fill with phantom candidate → 422 Consistency
|
|
echo "[validatord-smoke] /v1/validate fill with phantom candidate → 422:"
|
|
STATUS="$(curl -sS -o /tmp/fill_422.json -w '%{http_code}' -X POST http://127.0.0.1:3110/v1/validate \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"kind":"fill","artifact":{"fills":[{"candidate_id":"W-PHANTOM","name":"Nobody"}]},"context":{"target_count":1,"city":"Toledo","client_id":"C-1"}}')"
|
|
if [ "$STATUS" != "422" ]; then
|
|
echo " ✗ expected 422; got $STATUS body=$(cat /tmp/fill_422.json)"
|
|
exit 1
|
|
fi
|
|
# Rust serde-tagged-enum shape: {"Consistency":{"reason":"..."}}
|
|
VARIANT="$(jq -r 'keys[0]' /tmp/fill_422.json)"
|
|
if [ "$VARIANT" != "Consistency" ]; then
|
|
echo " ✗ expected variant=Consistency; got variant=$VARIANT body=$(cat /tmp/fill_422.json)"
|
|
exit 1
|
|
fi
|
|
echo " ✓ phantom candidate W-PHANTOM → 422 Consistency"
|
|
|
|
# 5. /v1/validate unknown kind → 400
|
|
echo "[validatord-smoke] /v1/validate unknown kind → 400:"
|
|
STATUS="$(curl -sS -o /tmp/unknown_400.txt -w '%{http_code}' -X POST http://127.0.0.1:3110/v1/validate \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"kind":"foo","artifact":{}}')"
|
|
if [ "$STATUS" != "400" ]; then
|
|
echo " ✗ expected 400; got $STATUS body=$(cat /tmp/unknown_400.txt)"
|
|
exit 1
|
|
fi
|
|
echo " ✓ unknown kind → 400"
|
|
|
|
echo "[validatord-smoke] PASS — 5/5 probes through gateway :3110"
|