golangLAKEHOUSE/scripts/workflow_smoke.sh

#!/usr/bin/env bash
# Workflow smoke — Observer-KB workflow runner end-to-end (SPEC §3.8
# first slice). All assertions go through gateway :3110.
#
# Validates:
#   - GET /observer/workflow/modes lists fixture.echo + fixture.upper
#   - POST /observer/workflow/run executes a 3-node DAG with $-ref
#     substitution: shape (uppercase) → weakness → improvement
#   - Each node's execution lands an ObservedOp via the observer
#     ring (visible in /observer/stats with source="workflow")
#   - Aborting case: unknown mode → 400 with helpful error
#   - Skip cascade: node with failed dep gets skipped, independent
#     siblings still run

set -euo pipefail
cd "$(dirname "$0")/.."

export PATH="$PATH:/usr/local/go/bin"

echo "[workflow-smoke] building observerd + gateway..."
go build -o bin/ ./cmd/observerd ./cmd/gateway

pkill -f "bin/(observerd|gateway)" 2>/dev/null || true
sleep 0.3

PIDS=()
TMP="$(mktemp -d)"
CFG="$TMP/workflow.toml"

cleanup() {
  echo "[workflow-smoke] cleanup"
  for p in "${PIDS[@]}"; do [ -n "$p" ] && kill "$p" 2>/dev/null || true; done
  rm -rf "$TMP"
}
trap cleanup EXIT INT TERM

cat > "$CFG" <<EOF
[gateway]
bind = "127.0.0.1:3110"
storaged_url = "http://127.0.0.1:3211"
catalogd_url = "http://127.0.0.1:3212"
ingestd_url  = "http://127.0.0.1:3213"
queryd_url   = "http://127.0.0.1:3214"
vectord_url  = "http://127.0.0.1:3215"
embedd_url   = "http://127.0.0.1:3216"
pathwayd_url = "http://127.0.0.1:3217"
matrixd_url  = "http://127.0.0.1:3218"
observerd_url = "http://127.0.0.1:3219"

[observerd]
bind = "127.0.0.1:3219"
EOF

poll_health() {
  local port="$1" deadline=$(($(date +%s) + 5))
  while [ "$(date +%s)" -lt "$deadline" ]; do
    if curl -sS --max-time 1 "http://127.0.0.1:$port/health" >/dev/null 2>&1; then return 0; fi
    sleep 0.05
  done
  return 1
}

echo "[workflow-smoke] launching observerd → gateway..."
./bin/observerd -config "$CFG" > /tmp/observerd.log 2>&1 &
PIDS+=($!)
poll_health 3219 || { echo "observerd failed"; tail /tmp/observerd.log; exit 1; }

./bin/gateway -config "$CFG" > /tmp/gateway.log 2>&1 &
PIDS+=($!)
poll_health 3110 || { echo "gateway failed"; tail /tmp/gateway.log; exit 1; }

FAILED=0

# ── 1. /observer/workflow/modes lists registered modes ────────────
echo "[workflow-smoke] /observer/workflow/modes lists fixture modes:"
RESP="$(curl -sS http://127.0.0.1:3110/v1/observer/workflow/modes)"
HAS_ECHO="$(echo "$RESP" | jq -r '.modes | index("fixture.echo") != null')"
HAS_UPPER="$(echo "$RESP" | jq -r '.modes | index("fixture.upper") != null')"
if [ "$HAS_ECHO" = "true" ] && [ "$HAS_UPPER" = "true" ]; then
  echo "  ✓ fixture.echo + fixture.upper registered"
else
  echo "  ✗ resp: $RESP"; FAILED=1
fi

# ── 2. 3-node DAG with $-ref substitution ─────────────────────────
echo "[workflow-smoke] 3-node DAG: shape (upper) → weakness → improvement"
WORKFLOW='{
  "workflow": {
    "name": "smoke-chain",
    "description": "DAG ref substitution test",
    "nodes": [
      {"id":"shape", "mode":"fixture.upper", "prompt":"hello world"},
      {"id":"weakness", "mode":"fixture.echo",
       "prompt":"observed shape: $shape.output.upper",
       "depends_on":["shape"]},
      {"id":"improvement", "mode":"fixture.echo",
       "prompt":"based on $weakness.output.prompt do better",
       "depends_on":["weakness"]}
    ]
  }
}'
RUN="$(curl -sS -X POST http://127.0.0.1:3110/v1/observer/workflow/run \
  -H 'Content-Type: application/json' -d "$WORKFLOW")"
STATUS="$(echo "$RUN" | jq -r '.status')"
SHAPE_UPPER="$(echo "$RUN" | jq -r '.nodes[0].output.upper')"
WEAK_PROMPT="$(echo "$RUN" | jq -r '.nodes[1].output.prompt')"
IMP_PROMPT="$(echo "$RUN" | jq -r '.nodes[2].output.prompt')"

if [ "$STATUS" = "succeeded" ] && [ "$SHAPE_UPPER" = "HELLO WORLD" ] \
    && [[ "$WEAK_PROMPT" == *"HELLO WORLD"* ]] \
    && [[ "$IMP_PROMPT" == *"HELLO WORLD"* ]]; then
  echo "  ✓ status=succeeded · shape=HELLO WORLD · refs propagated through 3-node chain"
else
  echo "  ✗ status=$STATUS shape=$SHAPE_UPPER weak=$WEAK_PROMPT imp=$IMP_PROMPT"
  echo "    full: $RUN"
  FAILED=1
fi

# ── 3. Per-node provenance recorded as ObservedOps ────────────────
echo "[workflow-smoke] /observer/stats reflects workflow ops:"
STATS="$(curl -sS http://127.0.0.1:3110/v1/observer/stats)"
WORKFLOW_OPS="$(echo "$STATS" | jq -r '.by_source.workflow // 0')"
TOTAL="$(echo "$STATS" | jq -r '.total')"
if [ "$WORKFLOW_OPS" = "3" ] && [ "$TOTAL" = "3" ]; then
  echo "  ✓ 3 workflow ops recorded (one per node), total=3"
else
  echo "  ✗ workflow=$WORKFLOW_OPS total=$TOTAL"
  echo "    full: $STATS"; FAILED=1
fi

# ── 4. Unknown mode → 400 ─────────────────────────────────────────
echo "[workflow-smoke] unknown mode → 400:"
HTTP="$(curl -sS -o /tmp/wf_bad.json -w '%{http_code}' -X POST \
  http://127.0.0.1:3110/v1/observer/workflow/run \
  -H 'Content-Type: application/json' \
  -d '{"workflow":{"name":"bad","nodes":[{"id":"a","mode":"does.not.exist"}]}}')"
ERR="$(jq -r '.error' < /tmp/wf_bad.json 2>/dev/null)"
if [ "$HTTP" = "400" ] && echo "$ERR" | grep -qi "unknown mode"; then
  echo "  ✓ unknown mode aborts with 400 + helpful error"
else
  echo "  ✗ http=$HTTP err=$ERR"; FAILED=1
fi

if [ "$FAILED" -eq 0 ]; then
  echo "[workflow-smoke] Workflow runner acceptance: PASSED"
  exit 0
else
  echo "[workflow-smoke] Workflow runner acceptance: FAILED"
  exit 1
fi