From 19bdfab2271a4fbf6c6669ed5d2ff56c0b779a88 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 27 Mar 2026 05:48:20 -0500 Subject: [PATCH] Phase 2: DataFusion query engine over Parquet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - queryd: SessionContext with custom URL scheme to avoid path doubling with LocalFileSystem - queryd: ListingTable registration from catalog ObjectRefs with schema inference - queryd: POST /query/sql returns JSON {columns, rows, row_count} - queryd→catalogd wiring: reads all datasets, registers as named tables - gateway: wires QueryEngine with shared store + registry - e2e verified: SELECT *, WHERE/ORDER BY, COUNT/AVG all correct Co-Authored-By: Claude Opus 4.6 (1M context) --- Cargo.lock | 1022 +++++++++++++++++++++++++++++++++- Cargo.toml | 1 + crates/gateway/src/main.rs | 5 +- crates/queryd/Cargo.toml | 8 + crates/queryd/src/context.rs | 82 +++ crates/queryd/src/lib.rs | 1 + crates/queryd/src/service.rs | 86 ++- docs/PHASES.md | 12 +- tests/e2e_query.sh | 57 ++ 9 files changed, 1259 insertions(+), 15 deletions(-) create mode 100644 crates/queryd/src/context.rs create mode 100644 tests/e2e_query.sh diff --git a/Cargo.lock b/Cargo.lock index 717f094..d88f471 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,6 +58,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -73,6 +79,27 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "ar_archive_writer" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +dependencies = [ + "object", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "arrow" version = "55.2.0" @@ -119,6 +146,7 @@ dependencies = [ "arrow-data", "arrow-schema", "chrono", + "chrono-tz", "half", "hashbrown 0.15.5", "num", @@ -149,6 +177,7 @@ dependencies = [ "atoi", "base64", "chrono", + "comfy-table", "half", "lexical-core", "num", @@ -193,6 +222,7 @@ dependencies = [ "arrow-data", "arrow-schema", "flatbuffers", + "lz4_flex", ] [[package]] @@ -280,6 +310,23 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "async-compression" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" +dependencies = [ + "bzip2", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "xz2", + "zstd", + "zstd-safe", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -370,12 +417,48 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bitflags" version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "blake3" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -424,6 +507,25 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "catalogd" version = "0.1.0" @@ -479,6 +581,26 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + [[package]] name = "const-random" version = "0.1.18" @@ -499,6 +621,12 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation" version = "0.10.1" @@ -524,6 +652,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crunchy" version = "0.2.4" @@ -561,6 +704,618 @@ dependencies = [ "memchr", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-macros", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "flate2", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "parquet", + "rand 0.8.5", + "regex", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", + "xz2", + "zstd", +] + +[[package]] +name = "datafusion-catalog" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "log", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c" +dependencies = [ + "ahash", + "arrow", + "arrow-ipc", + "base64", + "half", + "hashbrown 0.14.5", + "indexmap", + "libc", + "log", + "object_store", + "parquet", + "paste", + "recursive", + "sqlparser", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5167bb1d2ccbb87c6bc36c295274d7a0519b14afcfdaf401d53cbcaa4ef4968b" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1" +dependencies = [ + "arrow", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "flate2", + "futures", + "glob", + "itertools", + "log", + "object_store", + "parquet", + "rand 0.8.5", + "tempfile", + "tokio", + "tokio-util", + "url", + "xz2", + "zstd", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "serde_json", + "tokio", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "parquet", + "rand 0.8.5", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91f8c2c5788ef32f48ff56c68e5b545527b744822a284373ac79bba1ba47292" + +[[package]] +name = "datafusion-execution" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84" +dependencies = [ + "arrow", + "dashmap", + "datafusion-common", + "datafusion-expr", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.8.5", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap", + "paste", + "recursive", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap", + "itertools", + "paste", +] + +[[package]] +name = "datafusion-functions" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "hex", + "itertools", + "log", + "md-5", + "rand 0.8.5", + "regex", + "sha2", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-macros", + "datafusion-physical-expr-common", + "itertools", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-table" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", + "paste", +] + +[[package]] +name = "datafusion-functions-window" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cefc2d77646e1aadd1d6a9c40088937aedec04e68c5f0465939912e1291f8193" +dependencies = [ + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4aff082c42fa6da99ce0698c85addd5252928c908eb087ca3cfa64ff16b313" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1" +dependencies = [ + "datafusion-expr", + "quote", + "syn", +] + +[[package]] +name = "datafusion-optimizer" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "indexmap", + "itertools", + "log", + "recursive", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools", + "log", + "paste", + "petgraph", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.14.5", + "itertools", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools", + "log", + "recursive", +] + +[[package]] +name = "datafusion-physical-plan" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55" +dependencies = [ + "ahash", + "arrow", + "arrow-ord", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools", + "log", + "parking_lot", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "datafusion-session" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-sql", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-sql" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607" +dependencies = [ + "arrow", + "bigdecimal", + "datafusion-common", + "datafusion-expr", + "indexmap", + "log", + "recursive", + "regex", + "sqlparser", +] + [[package]] name = "digest" version = "0.10.7" @@ -569,6 +1324,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -604,12 +1360,24 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flatbuffers" version = "25.12.19" @@ -626,6 +1394,7 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ + "crc32fast", "miniz_oxide", "zlib-rs", ] @@ -808,6 +1577,12 @@ dependencies = [ "wasip3", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "h2" version = "0.4.13" @@ -839,6 +1614,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -860,6 +1645,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "1.4.0" @@ -1256,6 +2047,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.1" @@ -1292,6 +2089,17 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "matchers" version = "0.2.0" @@ -1433,6 +2241,15 @@ dependencies = [ "libm", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "object_store" version = "0.12.5" @@ -1454,7 +2271,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand", + "rand 0.9.2", "reqwest", "ring", "serde", @@ -1538,6 +2355,7 @@ dependencies = [ "lz4_flex", "num", "num-bigint", + "object_store", "paste", "seq-macro", "simdutf8", @@ -1560,6 +2378,34 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1615,16 +2461,34 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" +dependencies = [ + "ar_archive_writer", + "cc", +] + [[package]] name = "queryd" version = "0.1.0" dependencies = [ + "arrow", "axum", + "bytes", + "catalogd", + "datafusion", + "futures", + "object_store", "serde", "serde_json", "shared", + "storaged", "tokio", "tracing", + "url", ] [[package]] @@ -1666,7 +2530,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand", + "rand 0.9.2", "ring", "rustc-hash", "rustls", @@ -1713,14 +2577,35 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "rand_chacha", - "rand_core", + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] @@ -1730,7 +2615,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", ] [[package]] @@ -1742,6 +2636,26 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1851,6 +2765,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.37" @@ -2098,6 +3025,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + [[package]] name = "slab" version = "0.4.12" @@ -2126,12 +3059,48 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "sqlparser" +version = "0.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" +dependencies = [ + "log", + "recursive", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stacker" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.52.0", + "windows-sys 0.59.0", +] + [[package]] name = "storaged" version = "0.1.0" @@ -2184,6 +3153,19 @@ dependencies = [ "syn", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -2455,6 +3437,18 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -2755,6 +3749,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.60.2" @@ -2996,6 +3999,15 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yoke" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index 48a7320..ec05f64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,3 +27,4 @@ datafusion = "47" bytes = "1" futures = "0.3" sha2 = "0.10" +url = "2" diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs index 36bad00..7206092 100644 --- a/crates/gateway/src/main.rs +++ b/crates/gateway/src/main.rs @@ -19,11 +19,14 @@ async fn main() { tracing::warn!("catalog rebuild failed (empty store?): {e}"); } + // Query engine — DataFusion over catalog-registered Parquet + let engine = queryd::context::QueryEngine::new(registry.clone(), store.clone()); + let app = Router::new() .route("/health", get(health)) .nest("/storage", storaged::service::router(store)) .nest("/catalog", catalogd::service::router(registry)) - .nest("/query", queryd::service::router()) + .nest("/query", queryd::service::router(engine)) .nest("/ai", aibridge::service::router()) .layer(TraceLayer::new_for_http()); diff --git a/crates/queryd/Cargo.toml b/crates/queryd/Cargo.toml index 22bc8a6..e36b36e 100644 --- a/crates/queryd/Cargo.toml +++ b/crates/queryd/Cargo.toml @@ -5,8 +5,16 @@ edition = "2024" [dependencies] shared = { path = "../shared" } +catalogd = { path = "../catalogd" } +storaged = { path = "../storaged" } tokio = { workspace = true } axum = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tracing = { workspace = true } +datafusion = { workspace = true } +object_store = { workspace = true } +arrow = { workspace = true } +bytes = { workspace = true } +futures = { workspace = true } +url = { workspace = true } diff --git a/crates/queryd/src/context.rs b/crates/queryd/src/context.rs new file mode 100644 index 0000000..7e2d0d7 --- /dev/null +++ b/crates/queryd/src/context.rs @@ -0,0 +1,82 @@ +use catalogd::registry::Registry; +use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl}; +use datafusion::datasource::file_format::parquet::ParquetFormat; +use datafusion::prelude::*; +use object_store::ObjectStore; +use std::sync::Arc; +use url::Url; + +const STORE_SCHEME: &str = "lakehouse"; + +/// Query engine that runs DataFusion over catalog-registered Parquet datasets. +#[derive(Clone)] +pub struct QueryEngine { + registry: Registry, + store: Arc, +} + +impl QueryEngine { + pub fn new(registry: Registry, store: Arc) -> Self { + Self { registry, store } + } + + /// Execute a SQL query. Resolves all catalog datasets as tables before execution. + pub async fn query(&self, sql: &str) -> Result, String> { + let ctx = self.build_context().await?; + + let df = ctx.sql(sql).await.map_err(|e| format!("SQL error: {e}"))?; + let batches = df.collect().await.map_err(|e| format!("execution error: {e}"))?; + Ok(batches) + } + + /// Build a SessionContext with all catalog datasets registered as tables. + async fn build_context(&self) -> Result { + let ctx = SessionContext::new(); + + // Register the object store under a custom scheme to avoid path doubling. + // The store already has the root prefix (e.g. ./data), so paths are relative keys. + let base_url = Url::parse(&format!("{STORE_SCHEME}://data/")) + .map_err(|e| format!("invalid store url: {e}"))?; + ctx.runtime_env().register_object_store(&base_url, self.store.clone()); + + // Register each catalog dataset as a table + let datasets = self.registry.list().await; + for dataset in &datasets { + if dataset.objects.is_empty() { + continue; + } + + let opts = ListingOptions::new(Arc::new(ParquetFormat::default())); + + let table_paths: Vec = dataset.objects.iter() + .filter_map(|o| { + let url_str = format!("{STORE_SCHEME}://data/{}", o.key); + ListingTableUrl::parse(&url_str).ok() + }) + .collect(); + + if table_paths.is_empty() { + tracing::warn!("dataset {} has no valid paths, skipping", dataset.name); + continue; + } + + // Infer schema from the first file + let schema = opts.infer_schema(&ctx.state(), &table_paths[0]).await + .map_err(|e| format!("schema inference failed for {}: {e}", dataset.name))?; + + let config = ListingTableConfig::new_with_multi_paths(table_paths) + .with_listing_options(opts) + .with_schema(schema); + + let table = ListingTable::try_new(config) + .map_err(|e| format!("table creation failed for {}: {e}", dataset.name))?; + + ctx.register_table(&dataset.name, Arc::new(table)) + .map_err(|e| format!("table registration failed for {}: {e}", dataset.name))?; + + tracing::debug!("registered table: {}", dataset.name); + } + + Ok(ctx) + } +} diff --git a/crates/queryd/src/lib.rs b/crates/queryd/src/lib.rs index 1f278a4..3690e02 100644 --- a/crates/queryd/src/lib.rs +++ b/crates/queryd/src/lib.rs @@ -1 +1,2 @@ +pub mod context; pub mod service; diff --git a/crates/queryd/src/service.rs b/crates/queryd/src/service.rs index 60c722c..7e83c4b 100644 --- a/crates/queryd/src/service.rs +++ b/crates/queryd/src/service.rs @@ -1,9 +1,89 @@ -use axum::{Router, routing::get}; +use arrow::array::RecordBatch; +use arrow::json::writer::{JsonArray, Writer as JsonWriter}; +use axum::{ + Json, Router, + extract::State, + http::StatusCode, + response::IntoResponse, + routing::{get, post}, +}; +use serde::{Deserialize, Serialize}; -pub fn router() -> Router { - Router::new().route("/health", get(health)) +use crate::context::QueryEngine; + +pub fn router(engine: QueryEngine) -> Router { + Router::new() + .route("/health", get(health)) + .route("/sql", post(execute_query)) + .with_state(engine) } async fn health() -> &'static str { "queryd ok" } + +#[derive(Deserialize)] +struct QueryRequest { + sql: String, +} + +#[derive(Serialize)] +struct QueryResponse { + columns: Vec, + rows: serde_json::Value, + row_count: usize, +} + +#[derive(Serialize)] +struct ColumnInfo { + name: String, + data_type: String, +} + +fn batches_to_json(batches: &[RecordBatch]) -> Result { + let mut buf = Vec::new(); + let mut writer = JsonWriter::<_, JsonArray>::new(&mut buf); + for batch in batches { + writer.write(batch).map_err(|e| format!("JSON write error: {e}"))?; + } + writer.finish().map_err(|e| format!("JSON finish error: {e}"))?; + drop(writer); + serde_json::from_slice(&buf).map_err(|e| format!("JSON parse error: {e}")) +} + +async fn execute_query( + State(engine): State, + Json(req): Json, +) -> impl IntoResponse { + tracing::info!("executing query: {}", req.sql); + + match engine.query(&req.sql).await { + Ok(batches) => { + if batches.is_empty() { + return Ok(Json(QueryResponse { + columns: vec![], + rows: serde_json::Value::Array(vec![]), + row_count: 0, + })); + } + + let schema = batches[0].schema(); + let columns: Vec = schema.fields().iter().map(|f| ColumnInfo { + name: f.name().clone(), + data_type: f.data_type().to_string(), + }).collect(); + + let rows = batches_to_json(&batches) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?; + + let row_count = rows.as_array().map(|a| a.len()).unwrap_or(0); + + Ok(Json(QueryResponse { + columns, + rows, + row_count, + })) + } + Err(e) => Err((StatusCode::BAD_REQUEST, e)), + } +} diff --git a/docs/PHASES.md b/docs/PHASES.md index a39d601..9cc412a 100644 --- a/docs/PHASES.md +++ b/docs/PHASES.md @@ -22,13 +22,13 @@ **Gate: PASSED** — PUT object → register dataset → list → get by name. All via gateway HTTP. ## Phase 2: Query Engine -- [ ] 2.1 — queryd: SessionContext + object_store config -- [ ] 2.2 — queryd: ListingTable from catalog ObjectRefs -- [ ] 2.3 — queryd service: POST /query → Arrow IPC or JSON -- [ ] 2.4 — queryd → catalogd wiring -- [ ] 2.5 — gateway routes /query +- [x] 2.1 — queryd: SessionContext + object_store config (custom scheme to avoid path doubling) +- [x] 2.2 — queryd: ListingTable from catalog ObjectRefs with schema inference +- [x] 2.3 — queryd service: POST /query/sql → JSON (columns + rows + row_count) +- [x] 2.4 — queryd → catalogd wiring (reads dataset list, registers as tables) +- [x] 2.5 — gateway routes /query with QueryEngine state -**Gate:** SQL over Parquet returns correct results via catalog resolution. +**Gate: PASSED** — SELECT *, WHERE/ORDER BY, COUNT/AVG all return correct results via catalog. ## Phase 3: AI Integration - [ ] 3.1 — Python sidecar: FastAPI + Ollama (embed/generate/rerank) diff --git a/tests/e2e_query.sh b/tests/e2e_query.sh new file mode 100644 index 0000000..48d68e9 --- /dev/null +++ b/tests/e2e_query.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# End-to-end test: upload Parquet → register dataset → SQL query +set -e + +BASE="http://localhost:3100" + +echo "=== Generate test Parquet file ===" +python3 -c " +import struct, io + +# Minimal Parquet via pyarrow if available, else skip +try: + import pyarrow as pa + import pyarrow.parquet as pq + table = pa.table({ + 'id': [1, 2, 3, 4, 5], + 'name': ['alice', 'bob', 'carol', 'dave', 'eve'], + 'score': [9.5, 8.2, 7.8, 6.1, 9.9], + }) + pq.write_table(table, '/tmp/test_data.parquet') + print('generated with pyarrow') +except ImportError: + print('pyarrow not available, generating via rust helper') + exit(1) +" + +echo "=== Upload Parquet to storage ===" +curl -s -X PUT "$BASE/storage/objects/datasets/scores.parquet" \ + --data-binary @/tmp/test_data.parquet +echo "" + +echo "=== Register dataset in catalog ===" +SIZE=$(stat -c%s /tmp/test_data.parquet) +curl -s -X POST "$BASE/catalog/datasets" \ + -H "Content-Type: application/json" \ + -d "{\"name\":\"scores\",\"schema_fingerprint\":\"test\",\"objects\":[{\"bucket\":\"data\",\"key\":\"datasets/scores.parquet\",\"size_bytes\":$SIZE}]}" | python3 -m json.tool +echo "" + +echo "=== SQL: SELECT * FROM scores ===" +curl -s -X POST "$BASE/query/sql" \ + -H "Content-Type: application/json" \ + -d '{"sql":"SELECT * FROM scores"}' | python3 -m json.tool +echo "" + +echo "=== SQL: SELECT name, score FROM scores WHERE score > 8.0 ORDER BY score DESC ===" +curl -s -X POST "$BASE/query/sql" \ + -H "Content-Type: application/json" \ + -d '{"sql":"SELECT name, score FROM scores WHERE score > 8.0 ORDER BY score DESC"}' | python3 -m json.tool +echo "" + +echo "=== SQL: SELECT COUNT(*), AVG(score) FROM scores ===" +curl -s -X POST "$BASE/query/sql" \ + -H "Content-Type: application/json" \ + -d '{"sql":"SELECT COUNT(*) as cnt, AVG(score) as avg_score FROM scores"}' | python3 -m json.tool +echo "" + +echo "=== DONE ==="