From 40305da654cbb7bd22d95aec3081add4cefa19df Mon Sep 17 00:00:00 2001 From: root Date: Fri, 17 Apr 2026 01:00:21 -0500 Subject: [PATCH] 500K scale test: 2.9M rows, sub-120ms SQL, architecture holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumped upload limit to 512MB for large CSV ingests. Generated and ingested 500K staffing worker profiles (346MB CSV → 75MB Parquet in 5.9s). SQL at 500K: COUNT=35ms, filter+state=67ms, aggregation=80ms, complex filter=117ms, 10 concurrent=84ms total (10/10 pass). HNSW memory projection: 500K vectors = 1.5GB RAM (comfortable on 128GB server). Ceiling at ~5M vectors (14.6GB) — Lance IVF_PQ takes over beyond that as designed in ADR-019. Hybrid search 500K SQL → 10K vector: 131ms with 6,289 SQL matches narrowed to 5 vector-ranked results. Total scale: 2.9M rows across all datasets (500K workers + 2.47M staffing data). Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/gateway/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index 722c098..e9fdd3e 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -190,7 +190,7 @@ async fn main() { } app = app - .layer(DefaultBodyLimit::max(256 * 1024 * 1024)) // 256MB + .layer(DefaultBodyLimit::max(512 * 1024 * 1024)) // 512MB — supports 500K worker CSV .layer(CorsLayer::new() .allow_origin(Any) .allow_methods(Any)