bb05c4412e
- ingestd crate: detect file type → parse → schema detection → Parquet → catalog - CSV: auto-detect column types (int, float, bool, string), handles $, %, commas Strips dollar signs from amounts, flexible row parsing, sanitized column names - JSON: array or newline-delimited, nested object flattening (a.b.c → a_b_c) - PDF: text extraction via lopdf, one row per page (source_file, page_number, text) - Text/SMS: line-based ingestion with line numbers - Dedup: SHA-256 content hash, re-ingest same file = no-op - Gateway: POST /ingest/file multipart upload, 256MB body limit - Schema detection per ADR-010: ambiguous types default to String - 12 unit tests passing (CSV parsing, JSON flattening, type inference, dedup) - Tested: messy CSV with missing data, dollar amounts, N/A values → queryable Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
27 lines
773 B
TOML
27 lines
773 B
TOML
[package]
|
|
name = "gateway"
|
|
version = "0.1.0"
|
|
edition = "2024"
|
|
|
|
[dependencies]
|
|
shared = { path = "../shared" }
|
|
storaged = { path = "../storaged" }
|
|
catalogd = { path = "../catalogd" }
|
|
queryd = { path = "../queryd" }
|
|
aibridge = { path = "../aibridge" }
|
|
ingestd = { path = "../ingestd" }
|
|
tokio = { workspace = true }
|
|
axum = { workspace = true }
|
|
serde = { workspace = true }
|
|
serde_json = { workspace = true }
|
|
tracing = { workspace = true }
|
|
tracing-subscriber = { workspace = true }
|
|
tower-http = { workspace = true }
|
|
object_store = { workspace = true }
|
|
proto = { path = "../proto" }
|
|
tonic = { workspace = true }
|
|
opentelemetry = { workspace = true }
|
|
opentelemetry_sdk = { workspace = true }
|
|
opentelemetry-stdout = { workspace = true }
|
|
tracing-opentelemetry = { workspace = true }
|