a81291e38c
Per docs/TEST_PROOF_SCOPE.md, building the claims-verification tier
above the smoke chain. This commit lays the scaffolding and proves
the orchestrator end-to-end with one canary case (00_health).
What landed:
tests/proof/
README.md how to read a report, layout, modes
claims.yaml 24 claims enumerated (GOLAKE-001..100)
run_proof.sh orchestrator with --mode {contract|integration|performance}
and --no-bootstrap / --regenerate-{rankings,baseline}
lib/
env.sh service URLs, report dir, mode, git context
http.sh curl wrappers writing per-probe JSON + body + headers
assert.sh proof_assert_{eq,ne,contains,lt,gt,status,json_eq} +
proof_skip — each emits one JSONL record per call
metrics.sh start/stop timers, value capture, RSS sampling,
percentile compute (for Phase D)
cases/
00_health.sh canary — gateway + 6 services /health → 200,
body identifies service, latency < 500ms (21 assertions)
fixtures/
csv/workers.csv spec's 5-row deterministic CSV
text/docs.txt 4 deterministic vector docs
expected/queries.json expected results for the 5 SQL assertions
Wired into the task runner:
just proof contract # canary only this commit
just proof integration # Phase C
just proof performance # Phase D
.gitignore: /tests/proof/reports/* with !.gitkeep — same pattern as
reports/scrum/_evidence/. Per-run output is a runtime artifact.
Specs landed alongside (J's drops):
docs/TEST_PROOF_SCOPE.md the harness contract this implements
docs/CLAUDE_REFACTOR_GUARDRAILS.md process discipline this harness obeys
Verified end-to-end (cached binaries):
just proof contract wall < 2s, 21 pass / 0 fail / 0 skip
just verify wall 31s, vet + test + 9 smokes still green
Two bugs fixed during canary run, both in run_proof.sh aggregation:
- grep -c exits 1 on zero matches; the `|| echo 0` form concatenated
"0\n0" and broke jq --argjson + integer comparison. Fixed via a
_count helper that captures count-or-zero cleanly.
- per-case table iterated case scripts (filename-based) but cases
write evidence under CASE_ID. Switched to JSONL-file iteration so
multi-case scripts work and the mapping is faithful.
Phase B (contract cases) lands next: 05_embedding, 06_vector_add,
08_gateway_contracts, 09_failure_modes. Each sourcing the same lib
helpers and writing to the same report shape.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
112 lines
3.9 KiB
Bash
112 lines
3.9 KiB
Bash
#!/usr/bin/env bash
|
|
# lib/http.sh — curl wrappers that capture latency, status, body.
|
|
#
|
|
# Each request emits a JSON file under raw/http/<case_id>/<probe>.json
|
|
# describing the round-trip. Cases consume the JSON via assert.sh.
|
|
#
|
|
# Why JSON files instead of bash variables: gives the final report a
|
|
# diffable, replayable record. Future runs can compare on disk without
|
|
# re-executing the case.
|
|
#
|
|
# Functions:
|
|
# proof_get <case_id> <probe_name> <url> [extra-curl-args...]
|
|
# proof_post <case_id> <probe_name> <url> <content-type> <body> [extra-curl-args...]
|
|
# proof_put <case_id> <probe_name> <url> <content-type> <body|@file> [extra-curl-args...]
|
|
# proof_delete<case_id> <probe_name> <url> [extra-curl-args...]
|
|
#
|
|
# Returns 0 always (capture is independent of HTTP outcome).
|
|
# Stores result at: $PROOF_REPORT_DIR/raw/http/<case_id>/<probe>.json
|
|
# Stores body at: $PROOF_REPORT_DIR/raw/http/<case_id>/<probe>.body
|
|
|
|
_proof_http_emit() {
|
|
local case_id="$1" probe="$2" method="$3" url="$4" status="$5" latency_ms="$6" body_path="$7" headers_path="$8"
|
|
local dir="${PROOF_REPORT_DIR}/raw/http/${case_id}"
|
|
mkdir -p "$dir"
|
|
local body_sha=""
|
|
[ -s "$body_path" ] && body_sha="$(sha256sum "$body_path" | awk '{print $1}')"
|
|
cat > "${dir}/${probe}.json" <<JSON
|
|
{
|
|
"case_id": "${case_id}",
|
|
"probe": "${probe}",
|
|
"method": "${method}",
|
|
"url": "${url}",
|
|
"status": ${status},
|
|
"latency_ms": ${latency_ms},
|
|
"body_path": "raw/http/${case_id}/${probe}.body",
|
|
"body_sha256": "${body_sha}",
|
|
"headers_path": "raw/http/${case_id}/${probe}.headers"
|
|
}
|
|
JSON
|
|
}
|
|
|
|
# Internal common runner — populates a temp body+headers file, times
|
|
# the request, emits the per-probe JSON, prints the body to stdout for
|
|
# convenience (cases can capture or discard).
|
|
_proof_http_run() {
|
|
local case_id="$1" probe="$2" method="$3" url="$4"; shift 4
|
|
local dir="${PROOF_REPORT_DIR}/raw/http/${case_id}"
|
|
mkdir -p "$dir"
|
|
local body_path="${dir}/${probe}.body"
|
|
local headers_path="${dir}/${probe}.headers"
|
|
local start_ms end_ms
|
|
start_ms=$(date +%s%3N)
|
|
local status
|
|
status=$(curl -sS -X "$method" -o "$body_path" -D "$headers_path" -w "%{http_code}" "$@" "$url" 2>/dev/null || echo 0)
|
|
end_ms=$(date +%s%3N)
|
|
local latency_ms=$((end_ms - start_ms))
|
|
_proof_http_emit "$case_id" "$probe" "$method" "$url" "$status" "$latency_ms" "$body_path" "$headers_path"
|
|
cat "$body_path"
|
|
}
|
|
|
|
proof_get() {
|
|
local case_id="$1" probe="$2" url="$3"; shift 3
|
|
_proof_http_run "$case_id" "$probe" GET "$url" "$@"
|
|
}
|
|
|
|
proof_post() {
|
|
local case_id="$1" probe="$2" url="$3" content_type="$4" body="$5"; shift 5
|
|
_proof_http_run "$case_id" "$probe" POST "$url" \
|
|
-H "Content-Type: ${content_type}" \
|
|
--data "$body" \
|
|
"$@"
|
|
}
|
|
|
|
# proof_put accepts either an inline body or @-prefixed file path
|
|
# (curl --upload-file semantics for streaming).
|
|
proof_put() {
|
|
local case_id="$1" probe="$2" url="$3" content_type="$4" body="$5"; shift 5
|
|
if [[ "$body" == @* ]]; then
|
|
local file="${body#@}"
|
|
_proof_http_run "$case_id" "$probe" PUT "$url" \
|
|
-H "Content-Type: ${content_type}" \
|
|
--upload-file "$file" \
|
|
"$@"
|
|
else
|
|
_proof_http_run "$case_id" "$probe" PUT "$url" \
|
|
-H "Content-Type: ${content_type}" \
|
|
--data "$body" \
|
|
"$@"
|
|
fi
|
|
}
|
|
|
|
proof_delete() {
|
|
local case_id="$1" probe="$2" url="$3"; shift 3
|
|
_proof_http_run "$case_id" "$probe" DELETE "$url" "$@"
|
|
}
|
|
|
|
# Helper accessors — reads the per-probe JSON.
|
|
proof_status_of() {
|
|
local case_id="$1" probe="$2"
|
|
jq -r '.status' "${PROOF_REPORT_DIR}/raw/http/${case_id}/${probe}.json"
|
|
}
|
|
|
|
proof_body_of() {
|
|
local case_id="$1" probe="$2"
|
|
cat "${PROOF_REPORT_DIR}/raw/http/${case_id}/${probe}.body"
|
|
}
|
|
|
|
proof_latency_of() {
|
|
local case_id="$1" probe="$2"
|
|
jq -r '.latency_ms' "${PROOF_REPORT_DIR}/raw/http/${case_id}/${probe}.json"
|
|
}
|