diff --git a/scripts/cutover/start_go_stack.sh b/scripts/cutover/start_go_stack.sh index d95de99..347f211 100755 --- a/scripts/cutover/start_go_stack.sh +++ b/scripts/cutover/start_go_stack.sh @@ -11,18 +11,29 @@ # teardown + reads from `lakehouse-go-primary` — can run without # tearing down or polluting our long-running state. # -# Two isolation layers: +# Three isolation layers (the third was added 2026-05-01 after the +# first push test exposed a port-collision bug — smoke catalogd +# failed to bind :3212 because persistent catalogd already had it, +# but smoke's poll_health 3212 succeeded responding to the +# persistent daemon, and the smoke happily proceeded talking to +# the persistent stack with the wrong bucket expectations): +# # 1. BINARY NAMES — persistent stack runs via symlinks # bin/persistent- → bin/. Smoke pkill pattern -# `bin/(storaged|...|gateway)` matches `bin/` substrings; +# `bin/(storaged|...|gateway)$` matches `bin/$` substrings; # `bin/persistent-` doesn't match because the slash is # followed by 'p', not the daemon-name first letter. # 2. MINIO BUCKETS — persistent stack uses lakehouse-go-persistent; # smoke harnesses use lakehouse-go-primary. Different buckets # mean rehydrate paths can't see each other's `_vectors/*` -# persistence files. The temp toml at /tmp/lakehouse-persistent.toml -# overrides only [s3].bucket; everything else inherits from -# lakehouse.toml. +# persistence files. +# 3. PORTS — persistent stack uses :4110 + :4211-:4219 (gateway + +# upstreams). Smoke harness uses :3110 + :3211-:3219. Both +# reach for the SAME chatd at :3220 because chatd is +# read-mostly (LLM dispatch, no persistent state to clobber) +# and operators don't want to maintain two LLM provider key +# sets. The temp toml at /tmp/lakehouse-persistent.toml +# overrides bucket + bind ports + upstream URLs (except chatd). # # Logs land in /tmp/gostack-logs/.log (one per daemon). # @@ -70,10 +81,22 @@ if command -v mc >/dev/null 2>&1; then mc mb --ignore-existing "local/$PERSISTENT_BUCKET" >/dev/null 2>&1 || true fi -# sed-replace the bucket line. Anchored to "lakehouse-go-primary" so -# no other accidental "primary" mention gets touched. -sed "s/lakehouse-go-primary/$PERSISTENT_BUCKET/g" lakehouse.toml > "$TEMP_TOML" -echo "[gostack] config: $TEMP_TOML (bucket=$PERSISTENT_BUCKET)" +# sed-replace the bucket line + port range. Anchored to specific +# substrings so accidental matches don't fire. chatd's :3220 stays +# unchanged (read-mostly LLM dispatch, no persistent state). +sed -e "s/lakehouse-go-primary/$PERSISTENT_BUCKET/g" \ + -e 's|127\.0\.0\.1:3110|127.0.0.1:4110|g' \ + -e 's|127\.0\.0\.1:3211|127.0.0.1:4211|g' \ + -e 's|127\.0\.0\.1:3212|127.0.0.1:4212|g' \ + -e 's|127\.0\.0\.1:3213|127.0.0.1:4213|g' \ + -e 's|127\.0\.0\.1:3214|127.0.0.1:4214|g' \ + -e 's|127\.0\.0\.1:3215|127.0.0.1:4215|g' \ + -e 's|127\.0\.0\.1:3216|127.0.0.1:4216|g' \ + -e 's|127\.0\.0\.1:3217|127.0.0.1:4217|g' \ + -e 's|127\.0\.0\.1:3218|127.0.0.1:4218|g' \ + -e 's|127\.0\.0\.1:3219|127.0.0.1:4219|g' \ + lakehouse.toml > "$TEMP_TOML" +echo "[gostack] config: $TEMP_TOML (bucket=$PERSISTENT_BUCKET, ports=4110+4211-4219)" # ── Cleanup any prior persistent daemons ──────────────────────────── # Match by the persistent- prefix so smoke processes are untouched. @@ -100,17 +123,17 @@ start() { return 1 } -echo "[gostack] starting in dependency order" -start storaged 3211 -start catalogd 3212 -start ingestd 3213 -start queryd 3214 -start embedd 3216 -start vectord 3215 -start pathwayd 3217 -start observerd 3219 -start matrixd 3218 -start gateway 3110 +echo "[gostack] starting in dependency order (port range :4xxx)" +start storaged 4211 +start catalogd 4212 +start ingestd 4213 +start queryd 4214 +start embedd 4216 +start vectord 4215 +start pathwayd 4217 +start observerd 4219 +start matrixd 4218 +start gateway 4110 # chatd is started independently — its provider key files come from # /etc/lakehouse/{ollama_cloud,openrouter,opencode,kimi}.env; if @@ -132,11 +155,12 @@ fi echo echo "[gostack] ready · sweep:" -for p in 3110 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220; do +for p in 4110 4211 4212 4213 4214 4215 4216 4217 4218 4219 3220; do curl -sSf -m 1 "http://127.0.0.1:$p/health" 2>/dev/null | head -c 80 echo done echo -echo "[gostack] persistent stack uses bucket: $PERSISTENT_BUCKET" -echo "[gostack] smoke harnesses use bucket: lakehouse-go-primary" +echo "[gostack] persistent stack: ports :4110+:4211-:4219 · bucket=$PERSISTENT_BUCKET" +echo "[gostack] smoke harnesses: ports :3110+:3211-:3219 · bucket=lakehouse-go-primary" +echo "[gostack] shared: chatd at :3220 (read-mostly LLM dispatch)" echo "[gostack] tear down via: pkill -f 'bin/persistent-'"