Production-readiness gauntlet exploiting the dual Rust/Go
implementation as a measurement instrument.
## Phase 1 — Full smoke chain
21/21 PASS in ~60s. Substrate intact across the full service surface.
## Phase 2 — Per-component scrum (token-volume fix)
Prior wave (165KB diff): Kimi 62 tokens out, Qwen 297 → no useful
analysis. This wave splits today's commits into 4 focused bundles
(36-71KB each):
c1 validatord (46KB) → 0 convergent / 11 distinct
c2 vectord substrate (36KB) → 0 convergent / 10 distinct
c3 materializer (71KB) → 0 convergent / 6 distinct (Opus emitted
a BLOCK then self-retracted in same response)
c4 replay (45KB) → 0 convergent / 10 distinct
Reviewer engagement vs prior wave: Kimi went 62 → ~250 tokens out
once bundles dropped below 60KB.
scripts/scrum_review.sh hardening:
* Diff-size guard (warn >60KB, hard-fail >100KB,
SCRUM_FORCE_OVERSIZE=1 override)
* Tightened prompt — file path must appear EXACTLY as in diff
so post-processor can grep WHERE: lines reliably
* Auto-tally step dedupes by (reviewer, location); convergence
counts distinct lineages (closes the prior `opus+opus+opus`
false-convergence bug)
## Phase 3 — Cross-runtime validator parity probe (the headline finding)
scripts/cutover/parity/validator_parity.sh sends 6 identical
/v1/validate cases to Rust :3100 AND Go :4110, compares status+body.
Result: **6/6 status codes match · 5/6 body shapes diverge.**
Rust returns serde-tagged enum: {"Schema":{"field":"x","reason":"y"}}
Go returns flat exported-fields: {"Kind":"schema","Field":"x","Reason":"y"}
Both round-trip inside their own runtime; a caller swapping one for
the other would break parsing silently. Captured as new _open_ row
in docs/ARCHITECTURE_COMPARISON.md decisions tracker.
This is the "use the dual-implementation as a measurement instrument"
return — single-repo scrums can't catch this class of cross-runtime
drift.
## Phase 4 — Production assessment
ship-with-known-gap. Validator wire-format gap is documented, not
regressed. ~50 LOC future fix on Go side (custom MarshalJSON on
ValidationError to match Rust's serde shape).
Persistent stack config (/tmp/lakehouse-persistent.toml) gains
validatord on :3221 + persistent-validatord binary so operators
bringing up the persistent stack get the new daemon automatically.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
124 lines
4.5 KiB
Bash
Executable File
124 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# validator_parity — send identical /v1/validate requests to BOTH the
|
|
# Rust gateway (default :3100) and the Go gateway (default :4110),
|
|
# compare HTTP status + body. Mismatches surface in the OUTPUT report
|
|
# as a [DIFF] row; converging behavior is captured as [MATCH].
|
|
#
|
|
# This exploits the dual-implementation as a measurement instrument:
|
|
# a divergence is a finding the architecture comparison should record.
|
|
#
|
|
# Usage:
|
|
# ./scripts/cutover/parity/validator_parity.sh
|
|
#
|
|
# Env overrides:
|
|
# RUST_GW=http://127.0.0.1:3100 # Rust gateway URL
|
|
# GO_GW=http://127.0.0.1:4110 # Go gateway URL (persistent stack)
|
|
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/../../.."
|
|
|
|
RUST_GW="${RUST_GW:-http://127.0.0.1:3100}"
|
|
GO_GW="${GO_GW:-http://127.0.0.1:4110}"
|
|
OUT_DIR="reports/cutover/gauntlet_2026-05-02/parity"
|
|
mkdir -p "$OUT_DIR"
|
|
OUT="$OUT_DIR/validator_parity.md"
|
|
|
|
# Test cases: pairs of (label, kind, body). Selected to cover every
|
|
# branch of the validator code paths AND failure modes that should
|
|
# hit the same status code on both runtimes.
|
|
declare -a CASES=(
|
|
"playbook_happy|playbook|{\"operation\":\"fill: Welder x2 in Toledo, OH\",\"endorsed_names\":[\"W-1\",\"W-2\"],\"target_count\":2,\"fingerprint\":\"abc123\"}"
|
|
"playbook_missing_fingerprint|playbook|{\"operation\":\"fill: X x1 in A, B\",\"endorsed_names\":[\"a\"]}"
|
|
"playbook_wrong_prefix|playbook|{\"operation\":\"sms_draft: hello\",\"endorsed_names\":[\"a\"],\"fingerprint\":\"x\"}"
|
|
"playbook_empty_endorsed|playbook|{\"operation\":\"fill: X x1 in A, B\",\"endorsed_names\":[],\"fingerprint\":\"x\"}"
|
|
"playbook_overfull|playbook|{\"operation\":\"fill: X x1 in A, B\",\"endorsed_names\":[\"a\",\"b\",\"c\"],\"target_count\":1,\"fingerprint\":\"x\"}"
|
|
"fill_phantom|fill|{\"fills\":[{\"candidate_id\":\"W-PHANTOM-NEVER-EXISTS\",\"name\":\"Nobody\"}]}|{\"target_count\":1,\"city\":\"Toledo\",\"client_id\":\"C-1\"}"
|
|
)
|
|
|
|
probe() {
|
|
local gw="$1" kind="$2" artifact="$3" ctx="$4"
|
|
local body
|
|
if [ -n "$ctx" ]; then
|
|
body=$(jq -nc --argjson art "$artifact" --argjson c "$ctx" --arg k "$kind" '{kind:$k, artifact:$art, context:$c}')
|
|
else
|
|
body=$(jq -nc --argjson art "$artifact" --arg k "$kind" '{kind:$k, artifact:$art}')
|
|
fi
|
|
curl -sS -m 8 -o /tmp/parity_resp.json -w "%{http_code}" \
|
|
-X POST "$gw/v1/validate" \
|
|
-H 'Content-Type: application/json' \
|
|
--data-binary "$body"
|
|
echo
|
|
}
|
|
|
|
normalize() {
|
|
# Strip elapsed_ms (timing) so the body comparison is content-only.
|
|
jq -S 'del(.elapsed_ms)' "$1" 2>/dev/null || cat "$1"
|
|
}
|
|
|
|
{
|
|
echo "# Validator parity probe — Rust :3100 vs Go :4110"
|
|
echo
|
|
echo "**Date:** $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
echo "**Rust gateway:** \`$RUST_GW\` · **Go gateway:** \`$GO_GW\`"
|
|
echo
|
|
echo "Identical \`POST /v1/validate\` request → both runtimes. Match"
|
|
echo "= identical HTTP status + identical body (modulo \`elapsed_ms\`)."
|
|
echo
|
|
echo "| Case | Rust status | Go status | Status match | Body match |"
|
|
echo "|---|---:|---:|:---:|:---:|"
|
|
} > "$OUT"
|
|
|
|
MATCH=0; DIFF=0
|
|
for entry in "${CASES[@]}"; do
|
|
IFS='|' read -r label kind artifact ctx <<<"$entry"
|
|
rust_status=$(probe "$RUST_GW" "$kind" "$artifact" "$ctx" || echo "000")
|
|
cp /tmp/parity_resp.json /tmp/parity_rust.json
|
|
go_status=$(probe "$GO_GW" "$kind" "$artifact" "$ctx" || echo "000")
|
|
cp /tmp/parity_resp.json /tmp/parity_go.json
|
|
|
|
rust_norm=$(normalize /tmp/parity_rust.json)
|
|
go_norm=$(normalize /tmp/parity_go.json)
|
|
|
|
status_match="✓"
|
|
body_match="✓"
|
|
if [ "$rust_status" != "$go_status" ]; then status_match="✗"; fi
|
|
if [ "$rust_norm" != "$go_norm" ]; then body_match="✗"; fi
|
|
if [ "$status_match" = "✓" ] && [ "$body_match" = "✓" ]; then
|
|
MATCH=$((MATCH+1))
|
|
else
|
|
DIFF=$((DIFF+1))
|
|
# Capture the divergence verbatim for the report.
|
|
{
|
|
echo
|
|
echo "<details><summary>DIFF — \`$label\`</summary>"
|
|
echo
|
|
echo "**Rust** (HTTP $rust_status):"
|
|
echo '```json'
|
|
echo "$rust_norm"
|
|
echo '```'
|
|
echo
|
|
echo "**Go** (HTTP $go_status):"
|
|
echo '```json'
|
|
echo "$go_norm"
|
|
echo '```'
|
|
echo
|
|
echo "</details>"
|
|
} >> "$OUT.diffs"
|
|
fi
|
|
echo "| $label | $rust_status | $go_status | $status_match | $body_match |" >> "$OUT"
|
|
done
|
|
|
|
{
|
|
echo
|
|
echo "**Tally:** $MATCH match · $DIFF diff (out of $((MATCH+DIFF)) cases)"
|
|
echo
|
|
if [ -f "$OUT.diffs" ]; then
|
|
echo "## Divergences"
|
|
cat "$OUT.diffs"
|
|
rm -f "$OUT.diffs"
|
|
fi
|
|
} >> "$OUT"
|
|
|
|
echo "[parity] validator: $MATCH match / $DIFF diff (out of $((MATCH+DIFF))) → $OUT"
|
|
[ "$DIFF" -eq 0 ]
|