golangLAKEHOUSE/scripts/d2_smoke.sh

#!/usr/bin/env bash
# D2 smoke — proves the Day 2 acceptance gate end-to-end against the
# live MinIO at :9000 and the dedicated bucket "lakehouse-go-primary".
#
# Validates:
#   - PUT a small file → 200, body roundtrips on GET, appears in LIST
#   - DELETE removes the key, subsequent GET → 404
#   - PUT exceeding 256 MiB → 413 Payload Too Large
#   - 5th concurrent PUT (4-slot semaphore full) → 503 + Retry-After:5
#
# Usage:  ./scripts/d2_smoke.sh

set -euo pipefail
cd "$(dirname "$0")/.."

export PATH="$PATH:/usr/local/go/bin"

echo "[d2-smoke] building storaged..."
go build -o bin/ ./cmd/storaged

# Cleanup any prior storaged process on :3211 before launching.
pkill -f "bin/storaged" 2>/dev/null || true

STORAGED_PID=""
SLOW_PIDS=()
TMP="$(mktemp -d)"
cleanup() {
  echo "[d2-smoke] cleanup"
  if [ -n "$STORAGED_PID" ]; then
    kill "$STORAGED_PID" 2>/dev/null || true
  fi
  if [ ${#SLOW_PIDS[@]} -gt 0 ]; then
    kill "${SLOW_PIDS[@]}" 2>/dev/null || true
  fi
  rm -rf "$TMP"
}
trap cleanup EXIT INT TERM

echo "[d2-smoke] launching storaged..."
./bin/storaged > /tmp/storaged.log 2>&1 &
STORAGED_PID=$!

# Poll /health up to 5s — same discipline as d1_smoke.
deadline=$(($(date +%s) + 5))
while [ "$(date +%s)" -lt "$deadline" ]; do
  if curl -sS --max-time 1 http://127.0.0.1:3211/health >/dev/null 2>&1; then
    break
  fi
  sleep 0.05
done
if ! curl -sS --max-time 1 http://127.0.0.1:3211/health >/dev/null 2>&1; then
  echo "  [d2-smoke] storaged failed to bind within 5s — log:"
  tail -10 /tmp/storaged.log | sed 's/^/    /'
  exit 1
fi

FAILED=0
KEY="d2-smoke/$(date +%s).bin"

echo "[d2-smoke] PUT round-trip:"
printf "hello-d2-smoke" > "$TMP/sample.bin"
HTTP="$(curl -sS -o "$TMP/put.out" -w '%{http_code}' -X PUT --data-binary @"$TMP/sample.bin" "http://127.0.0.1:3211/storage/put/$KEY")"
if [ "$HTTP" = "200" ]; then
  echo "  ✓ PUT $KEY → 200"
else
  echo "  ✗ PUT $KEY → $HTTP (body=$(cat "$TMP/put.out"))"
  FAILED=1
fi

echo "[d2-smoke] GET echoes bytes:"
curl -sS -o "$TMP/get.out" "http://127.0.0.1:3211/storage/get/$KEY"
if cmp -s "$TMP/sample.bin" "$TMP/get.out"; then
  echo "  ✓ GET $KEY → bytes match"
else
  echo "  ✗ GET $KEY → bytes differ"
  FAILED=1
fi

echo "[d2-smoke] LIST includes key:"
LISTED="$(curl -sS "http://127.0.0.1:3211/storage/list?prefix=d2-smoke/" | grep -o "\"$KEY\"" || true)"
if [ -n "$LISTED" ]; then
  echo "  ✓ LIST prefix=d2-smoke/ → contains $KEY"
else
  echo "  ✗ LIST prefix=d2-smoke/ → missing $KEY"
  FAILED=1
fi

echo "[d2-smoke] DELETE then GET → 404:"
curl -sS -o /dev/null -X DELETE "http://127.0.0.1:3211/storage/delete/$KEY"
HTTP="$(curl -sS -o /dev/null -w '%{http_code}' "http://127.0.0.1:3211/storage/get/$KEY")"
if [ "$HTTP" = "404" ]; then
  echo "  ✓ DELETE then GET → 404"
else
  echo "  ✗ DELETE then GET → $HTTP (expected 404)"
  FAILED=1
fi

echo "[d2-smoke] 256 MiB cap → 413:"
dd if=/dev/zero of="$TMP/big.bin" bs=1M count=257 status=none
HTTP="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT --data-binary @"$TMP/big.bin" "http://127.0.0.1:3211/storage/put/d2-smoke/oversize.bin")"
if [ "$HTTP" = "413" ]; then
  echo "  ✓ PUT 257 MiB → 413"
else
  echo "  ✗ PUT 257 MiB → $HTTP (expected 413)"
  FAILED=1
fi

echo "[d2-smoke] semaphore: 5th concurrent PUT → 503 + Retry-After:5"
# Streaming uploads: -T file with --limit-rate makes curl actually
# pace the body chunks as bytes ship — unlike --data-binary @- which
# buffers stdin before opening the connection.
dd if=/dev/zero of="$TMP/slow.bin" bs=1M count=100 status=none

slow_put() {
  local idx="$1"
  local out="$TMP/slow_${idx}.code"
  curl -sS -o /dev/null -w "%{http_code}" -X PUT \
    -T "$TMP/slow.bin" --limit-rate 5M \
    "http://127.0.0.1:3211/storage/put/d2-smoke-cap-${idx}.bin" > "$out" 2>&1 || echo CURL_FAIL > "$out"
}

for i in 1 2 3 4; do
  slow_put "$i" &
  SLOW_PIDS+=($!)
done

# Give the 4 slow uploads ~750ms to actually start streaming + sit
# on the semaphore. At 5 MiB/s for 100 MiB they'll each run ~20s,
# so a brief warmup is plenty.
sleep 0.75

# Capture full headers on the 5th request to read Retry-After.
curl -sS -i -o "$TMP/blocked.out" -w '%{http_code}' -X PUT \
  --data-binary "blocked" \
  "http://127.0.0.1:3211/storage/put/d2-smoke-blocked.bin" > "$TMP/blocked.code" 2>/dev/null || true
HTTP_BLOCKED="$(cat "$TMP/blocked.code")"
RA="$(grep -i '^Retry-After:' "$TMP/blocked.out" | awk '{print $2}' | tr -d '\r' || true)"

if [ "$HTTP_BLOCKED" = "503" ] && [ "$RA" = "5" ]; then
  echo "  ✓ 5th concurrent PUT → 503 + Retry-After: 5"
else
  echo "  ✗ 5th concurrent PUT → code=$HTTP_BLOCKED retry-after=$RA"
  echo "    (slow PUT codes: $(for i in 1 2 3 4; do printf '%s ' "$(cat "$TMP/slow_${i}.code" 2>/dev/null || echo ?)"; done))"
  FAILED=1
fi

# Drain the 4 slow PUTs cleanly. Don't wait on STORAGED_PID — that
# only exits on signal. Wait only on the slow PUT PIDs we spawned.
for pid in "${SLOW_PIDS[@]}"; do
  wait "$pid" 2>/dev/null || true
done
SLOW_PIDS=()

# Cleanup smoke keys regardless of pass/fail.
for i in 1 2 3 4; do
  curl -sS -o /dev/null -X DELETE "http://127.0.0.1:3211/storage/delete/d2-smoke-cap-${i}.bin" || true
done
curl -sS -o /dev/null -X DELETE "http://127.0.0.1:3211/storage/delete/d2-smoke-blocked.bin" || true
curl -sS -o /dev/null -X DELETE "http://127.0.0.1:3211/storage/delete/d2-smoke/oversize.bin" || true

if [ "$FAILED" -eq 0 ]; then
  echo "[d2-smoke] D2 acceptance gate: PASSED"
  exit 0
else
  echo "[d2-smoke] D2 acceptance gate: FAILED"
  exit 1
fi