From a52ca841c6bc46713e4544293f735911d94ba301 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 27 Mar 2026 04:59:05 -0500 Subject: [PATCH] Phase 0: bootstrap Rust workspace - Cargo workspace with 6 crates: shared, storaged, catalogd, queryd, aibridge, gateway - shared: types (DatasetId, ObjectRef, SchemaFingerprint, DatasetManifest) + error enum - gateway: Axum HTTP entrypoint with nested service routers + tracing - All services expose /health stubs - justfile with build/test/run recipes - PRD, phase tracker, and ADR docs Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 4 + Cargo.lock | 1286 ++++++++++++++++++++++++++++++++ Cargo.toml | 22 + crates/aibridge/Cargo.toml | 12 + crates/aibridge/src/lib.rs | 1 + crates/aibridge/src/service.rs | 9 + crates/catalogd/Cargo.toml | 12 + crates/catalogd/src/lib.rs | 1 + crates/catalogd/src/service.rs | 9 + crates/gateway/Cargo.toml | 18 + crates/gateway/src/main.rs | 29 + crates/queryd/Cargo.toml | 12 + crates/queryd/src/lib.rs | 1 + crates/queryd/src/service.rs | 9 + crates/shared/Cargo.toml | 11 + crates/shared/src/errors.rs | 35 + crates/shared/src/lib.rs | 2 + crates/shared/src/types.rs | 42 ++ crates/storaged/Cargo.toml | 12 + crates/storaged/src/lib.rs | 1 + crates/storaged/src/service.rs | 9 + docs/DECISIONS.md | 26 + docs/PHASES.md | 57 ++ docs/PRD.md | 175 +++++ justfile | 28 + 25 files changed, 1823 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 crates/aibridge/Cargo.toml create mode 100644 crates/aibridge/src/lib.rs create mode 100644 crates/aibridge/src/service.rs create mode 100644 crates/catalogd/Cargo.toml create mode 100644 crates/catalogd/src/lib.rs create mode 100644 crates/catalogd/src/service.rs create mode 100644 crates/gateway/Cargo.toml create mode 100644 crates/gateway/src/main.rs create mode 100644 crates/queryd/Cargo.toml create mode 100644 crates/queryd/src/lib.rs create mode 100644 crates/queryd/src/service.rs create mode 100644 crates/shared/Cargo.toml create mode 100644 crates/shared/src/errors.rs create mode 100644 crates/shared/src/lib.rs create mode 100644 crates/shared/src/types.rs create mode 100644 crates/storaged/Cargo.toml create mode 100644 crates/storaged/src/lib.rs create mode 100644 crates/storaged/src/service.rs create mode 100644 docs/DECISIONS.md create mode 100644 docs/PHASES.md create mode 100644 docs/PRD.md create mode 100644 justfile diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c326ff --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/target +*.swp +*.swo +.env diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..8a16d0f --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1286 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "aibridge" +version = "0.1.0" +dependencies = [ + "axum", + "serde", + "serde_json", + "shared", + "tokio", + "tracing", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "catalogd" +version = "0.1.0" +dependencies = [ + "axum", + "serde", + "serde_json", + "shared", + "tokio", + "tracing", +] + +[[package]] +name = "cc" +version = "1.2.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gateway" +version = "0.1.0" +dependencies = [ + "aibridge", + "axum", + "catalogd", + "queryd", + "serde", + "serde_json", + "shared", + "storaged", + "tokio", + "tower-http", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "queryd" +version = "0.1.0" +dependencies = [ + "axum", + "serde", + "serde_json", + "shared", + "tokio", + "tracing", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shared" +version = "0.1.0" +dependencies = [ + "chrono", + "serde", + "serde_json", + "thiserror", + "uuid", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "storaged" +version = "0.1.0" +dependencies = [ + "axum", + "serde", + "serde_json", + "shared", + "tokio", + "tracing", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tokio" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "uuid" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +dependencies = [ + "getrandom", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..bf34b18 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,22 @@ +[workspace] +resolver = "2" +members = [ + "crates/shared", + "crates/storaged", + "crates/catalogd", + "crates/queryd", + "crates/aibridge", + "crates/gateway", +] + +[workspace.dependencies] +tokio = { version = "1", features = ["full"] } +axum = "0.8" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } +thiserror = "2" +uuid = { version = "1", features = ["v4", "serde"] } +chrono = { version = "0.4", features = ["serde"] } +tower-http = { version = "0.6", features = ["cors", "trace"] } diff --git a/crates/aibridge/Cargo.toml b/crates/aibridge/Cargo.toml new file mode 100644 index 0000000..8f57876 --- /dev/null +++ b/crates/aibridge/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "aibridge" +version = "0.1.0" +edition = "2024" + +[dependencies] +shared = { path = "../shared" } +tokio = { workspace = true } +axum = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tracing = { workspace = true } diff --git a/crates/aibridge/src/lib.rs b/crates/aibridge/src/lib.rs new file mode 100644 index 0000000..1f278a4 --- /dev/null +++ b/crates/aibridge/src/lib.rs @@ -0,0 +1 @@ +pub mod service; diff --git a/crates/aibridge/src/service.rs b/crates/aibridge/src/service.rs new file mode 100644 index 0000000..97c7655 --- /dev/null +++ b/crates/aibridge/src/service.rs @@ -0,0 +1,9 @@ +use axum::{Router, routing::get}; + +pub fn router() -> Router { + Router::new().route("/health", get(health)) +} + +async fn health() -> &'static str { + "aibridge ok" +} diff --git a/crates/catalogd/Cargo.toml b/crates/catalogd/Cargo.toml new file mode 100644 index 0000000..aa48a97 --- /dev/null +++ b/crates/catalogd/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "catalogd" +version = "0.1.0" +edition = "2024" + +[dependencies] +shared = { path = "../shared" } +tokio = { workspace = true } +axum = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tracing = { workspace = true } diff --git a/crates/catalogd/src/lib.rs b/crates/catalogd/src/lib.rs new file mode 100644 index 0000000..1f278a4 --- /dev/null +++ b/crates/catalogd/src/lib.rs @@ -0,0 +1 @@ +pub mod service; diff --git a/crates/catalogd/src/service.rs b/crates/catalogd/src/service.rs new file mode 100644 index 0000000..4d8a9db --- /dev/null +++ b/crates/catalogd/src/service.rs @@ -0,0 +1,9 @@ +use axum::{Router, routing::get}; + +pub fn router() -> Router { + Router::new().route("/health", get(health)) +} + +async fn health() -> &'static str { + "catalogd ok" +} diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml new file mode 100644 index 0000000..9a33d07 --- /dev/null +++ b/crates/gateway/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "gateway" +version = "0.1.0" +edition = "2024" + +[dependencies] +shared = { path = "../shared" } +storaged = { path = "../storaged" } +catalogd = { path = "../catalogd" } +queryd = { path = "../queryd" } +aibridge = { path = "../aibridge" } +tokio = { workspace = true } +axum = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +tower-http = { workspace = true } diff --git a/crates/gateway/src/main.rs b/crates/gateway/src/main.rs new file mode 100644 index 0000000..c8b6fa9 --- /dev/null +++ b/crates/gateway/src/main.rs @@ -0,0 +1,29 @@ +use axum::{Router, routing::get}; +use tower_http::trace::TraceLayer; +use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt}; + +#[tokio::main] +async fn main() { + tracing_subscriber::registry() + .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .with(fmt::layer()) + .init(); + + let app = Router::new() + .route("/health", get(health)) + .nest("/storage", storaged::service::router()) + .nest("/catalog", catalogd::service::router()) + .nest("/query", queryd::service::router()) + .nest("/ai", aibridge::service::router()) + .layer(TraceLayer::new_for_http()); + + let addr = "0.0.0.0:3100"; + tracing::info!("gateway listening on {addr}"); + + let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); + axum::serve(listener, app).await.unwrap(); +} + +async fn health() -> &'static str { + "lakehouse ok" +} diff --git a/crates/queryd/Cargo.toml b/crates/queryd/Cargo.toml new file mode 100644 index 0000000..22bc8a6 --- /dev/null +++ b/crates/queryd/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "queryd" +version = "0.1.0" +edition = "2024" + +[dependencies] +shared = { path = "../shared" } +tokio = { workspace = true } +axum = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tracing = { workspace = true } diff --git a/crates/queryd/src/lib.rs b/crates/queryd/src/lib.rs new file mode 100644 index 0000000..1f278a4 --- /dev/null +++ b/crates/queryd/src/lib.rs @@ -0,0 +1 @@ +pub mod service; diff --git a/crates/queryd/src/service.rs b/crates/queryd/src/service.rs new file mode 100644 index 0000000..60c722c --- /dev/null +++ b/crates/queryd/src/service.rs @@ -0,0 +1,9 @@ +use axum::{Router, routing::get}; + +pub fn router() -> Router { + Router::new().route("/health", get(health)) +} + +async fn health() -> &'static str { + "queryd ok" +} diff --git a/crates/shared/Cargo.toml b/crates/shared/Cargo.toml new file mode 100644 index 0000000..02248bd --- /dev/null +++ b/crates/shared/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "shared" +version = "0.1.0" +edition = "2024" + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +uuid = { workspace = true } +chrono = { workspace = true } diff --git a/crates/shared/src/errors.rs b/crates/shared/src/errors.rs new file mode 100644 index 0000000..6b18a4f --- /dev/null +++ b/crates/shared/src/errors.rs @@ -0,0 +1,35 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum LakehouseError { + #[error("dataset not found: {0}")] + DatasetNotFound(String), + + #[error("object not found: {bucket}/{key}")] + ObjectNotFound { bucket: String, key: String }, + + #[error("storage error: {0}")] + Storage(String), + + #[error("catalog error: {0}")] + Catalog(String), + + #[error("query error: {0}")] + Query(String), + + #[error("ai bridge error: {0}")] + AiBridge(String), + + #[error("serialization error: {0}")] + Serialization(String), +} + +impl LakehouseError { + pub fn status_code(&self) -> u16 { + match self { + Self::DatasetNotFound(_) | Self::ObjectNotFound { .. } => 404, + Self::Storage(_) | Self::Catalog(_) | Self::Query(_) | Self::AiBridge(_) => 500, + Self::Serialization(_) => 400, + } + } +} diff --git a/crates/shared/src/lib.rs b/crates/shared/src/lib.rs new file mode 100644 index 0000000..94483f0 --- /dev/null +++ b/crates/shared/src/lib.rs @@ -0,0 +1,2 @@ +pub mod types; +pub mod errors; diff --git a/crates/shared/src/types.rs b/crates/shared/src/types.rs new file mode 100644 index 0000000..3519a41 --- /dev/null +++ b/crates/shared/src/types.rs @@ -0,0 +1,42 @@ +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// Unique identifier for a dataset in the catalog. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct DatasetId(pub Uuid); + +impl DatasetId { + pub fn new() -> Self { + Self(Uuid::new_v4()) + } +} + +impl std::fmt::Display for DatasetId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Pointer to an object in storage. This is what the catalog stores — never raw data. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ObjectRef { + pub bucket: String, + pub key: String, + pub size_bytes: u64, + pub created_at: chrono::DateTime, +} + +/// Schema fingerprint — deterministic hash of an Arrow schema. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct SchemaFingerprint(pub String); + +/// Dataset manifest — the catalog's view of a dataset. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DatasetManifest { + pub id: DatasetId, + pub name: String, + pub schema_fingerprint: SchemaFingerprint, + pub objects: Vec, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, +} diff --git a/crates/storaged/Cargo.toml b/crates/storaged/Cargo.toml new file mode 100644 index 0000000..8b751db --- /dev/null +++ b/crates/storaged/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "storaged" +version = "0.1.0" +edition = "2024" + +[dependencies] +shared = { path = "../shared" } +tokio = { workspace = true } +axum = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tracing = { workspace = true } diff --git a/crates/storaged/src/lib.rs b/crates/storaged/src/lib.rs new file mode 100644 index 0000000..1f278a4 --- /dev/null +++ b/crates/storaged/src/lib.rs @@ -0,0 +1 @@ +pub mod service; diff --git a/crates/storaged/src/service.rs b/crates/storaged/src/service.rs new file mode 100644 index 0000000..c1b9756 --- /dev/null +++ b/crates/storaged/src/service.rs @@ -0,0 +1,9 @@ +use axum::{Router, routing::get}; + +pub fn router() -> Router { + Router::new().route("/health", get(health)) +} + +async fn health() -> &'static str { + "storaged ok" +} diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md new file mode 100644 index 0000000..fff907a --- /dev/null +++ b/docs/DECISIONS.md @@ -0,0 +1,26 @@ +# Architecture Decision Records + +## ADR-001: Object storage as source of truth +**Date:** 2026-03-27 +**Decision:** All data lives in S3-compatible object storage. No traditional database. +**Rationale:** Eliminates DB operational overhead, enables infinite scale at storage tier, forces clean separation of data and metadata. + +## ADR-002: Catalog metadata persistence +**Date:** 2026-03-27 +**Decision:** catalogd persists manifests as Parquet files in object storage. In-memory index rebuilt on startup. +**Rationale:** No external DB dependency. Storage is already the source of truth. Write-ahead pattern ensures consistency. + +## ADR-003: Real models only (no mocks) +**Date:** 2026-03-27 +**Decision:** AI sidecar hits Ollama with real models from Phase 3 onward. No stub/mock endpoints. +**Rationale:** Local Ollama instance available with nomic-embed-text, qwen2.5, mistral, gemma2, llama3.2. Mocks hide integration bugs. + +## ADR-004: Python sidecar as Ollama adapter +**Date:** 2026-03-27 +**Decision:** Python FastAPI sidecar is a thin HTTP adapter over Ollama's API. No model loading in Python. +**Rationale:** Ollama handles model lifecycle, GPU scheduling, caching. Sidecar stays stateless and lightweight — no torch/transformers deps. + +## ADR-005: HTTP-first, gRPC later +**Date:** 2026-03-27 +**Decision:** All inter-service communication uses HTTP through Phase 4. gRPC migration in Phase 5. +**Rationale:** HTTP is simpler to debug, test, and iterate on. gRPC adds protobuf compilation and streaming complexity before APIs stabilize. diff --git a/docs/PHASES.md b/docs/PHASES.md new file mode 100644 index 0000000..4bf0fdb --- /dev/null +++ b/docs/PHASES.md @@ -0,0 +1,57 @@ +# Phase Tracker + +## Phase 0: Bootstrap +- [ ] 0.1 — Cargo workspace with all crate stubs compiling +- [ ] 0.2 — `shared` crate: error types, ObjectRef, DatasetId +- [ ] 0.3 — `gateway` with Axum: GET /health → 200 +- [ ] 0.4 — tracing + tracing-subscriber wired in gateway +- [ ] 0.5 — justfile with build, test, run recipes +- [ ] 0.6 — docs committed to git + +**Gate:** All crates compile. Gateway runs. Logs emit. Docs committed. + +## Phase 1: Storage + Catalog +- [ ] 1.1 — storaged: object_store backend init (LocalFileSystem → S3) +- [ ] 1.2 — storaged: Axum endpoints (PUT/GET/DELETE /objects/{key}) +- [ ] 1.3 — shared/arrow.rs: RecordBatch ↔ Parquet helpers +- [ ] 1.4 — catalogd/registry.rs: in-memory index + manifest persistence +- [ ] 1.5 — catalogd/schema.rs: schema fingerprinting +- [ ] 1.6 — catalogd service: POST/GET /datasets endpoints +- [ ] 1.7 — gateway routes to storaged + catalogd + +**Gate:** Upload Parquet → register → metadata → read back. All via gateway. + +## Phase 2: Query Engine +- [ ] 2.1 — queryd: SessionContext + object_store config +- [ ] 2.2 — queryd: ListingTable from catalog ObjectRefs +- [ ] 2.3 — queryd service: POST /query → Arrow IPC or JSON +- [ ] 2.4 — queryd → catalogd wiring +- [ ] 2.5 — gateway routes /query + +**Gate:** SQL over Parquet returns correct results via catalog resolution. + +## Phase 3: AI Integration +- [ ] 3.1 — Python sidecar: FastAPI + Ollama (embed/generate/rerank) +- [ ] 3.2 — Dockerfile for sidecar +- [ ] 3.3 — aibridge/client.rs: HTTP client to sidecar +- [ ] 3.4 — aibridge service: Axum proxy endpoints +- [ ] 3.5 — Model config via env vars + +**Gate:** Rust → Python → Ollama → real embeddings return. + +## Phase 4: Frontend +- [ ] 4.1 — Dioxus scaffold, WASM build +- [ ] 4.2 — Dataset browser +- [ ] 4.3 — Query editor + results table +- [ ] 4.4 — Error display + loading states + +**Gate:** Browse datasets and query from browser. + +## Phase 5: Hardening +- [ ] 5.1 — Proto definitions +- [ ] 5.2 — Internal gRPC migration +- [ ] 5.3 — OpenTelemetry tracing +- [ ] 5.4 — Auth middleware +- [ ] 5.5 — Config-driven startup + +**Gate:** gRPC internals, traces, auth, restartable from repo + config. diff --git a/docs/PRD.md b/docs/PRD.md new file mode 100644 index 0000000..ab56156 --- /dev/null +++ b/docs/PRD.md @@ -0,0 +1,175 @@ +# PRD: Lakehouse — Rust-First Object Storage System + +**Status:** Active +**Created:** 2026-03-27 +**Owner:** J + +--- + +## Problem + +Traditional data platforms couple storage, compute, and metadata into monolithic databases. This creates vendor lock-in, scaling bottlenecks, and opaque data access. AI workloads bolt onto these systems awkwardly, sharing resources with transactional queries. + +We need a system where: +- Object storage is the source of truth (not a database) +- Metadata, access, and execution are controlled by Rust services +- Queries run directly over object storage via Arrow/Parquet +- AI inference is isolated and swappable +- The entire system is rebuildable from repository + docs alone + +--- + +## Solution + +A modular Rust service mesh over S3-compatible object storage. + +### Locked Stack + +| Layer | Technology | Locked | +|---|---|---| +| Frontend | Dioxus | Yes | +| API | Axum + Tokio | Yes | +| Object Storage Interface | Apache Arrow `object_store` | Yes | +| Storage Backend | RustFS (fallback: SeaweedFS) | Yes | +| Query Engine | DataFusion | Yes | +| Data Format | Parquet + Arrow | Yes | +| RPC (internal) | tonic (gRPC) | Yes | +| AI Runtime | Ollama (local models) | Yes | +| AI Boundary | Python FastAPI sidecar → Ollama HTTP API | Yes | + +No new frameworks. No exceptions. + +--- + +## Architecture + +### Services + +| Service | Responsibility | +|---|---| +| **gateway** | HTTP ingress, routing, auth envelope, middleware | +| **catalogd** | Metadata control plane — dataset registry, schema versions, manifest index | +| **storaged** | Object I/O — read/write/list/delete via `object_store` crate | +| **queryd** | SQL execution — DataFusion over registered Parquet datasets | +| **aibridge** | Rust↔Python boundary — HTTP client to FastAPI sidecar | +| **ui** | Dioxus frontend — dataset browser, query editor, results viewer | +| **shared** | Types, errors, Arrow helpers, protobuf definitions | + +### AI Sidecar + +Python FastAPI process that adapts Ollama's HTTP API into Arrow-compatible formats: +- `POST /embed` → `nomic-embed-text` via Ollama +- `POST /generate` → configurable model (qwen2.5, mistral, gemma2, llama3.2) +- `POST /rerank` → cross-encoder reranking via generate endpoint + +No mocks. No stubs. Real models from day one. Ollama manages model lifecycle, GPU scheduling, caching. Sidecar is stateless passthrough. + +### Data Flow + +``` +Client → gateway → catalogd (metadata lookup) + → storaged (object read/write) + → queryd (SQL execution over Parquet) + → aibridge → sidecar → Ollama (inference) +``` + +### Invariants + +1. Object storage = source of truth for all data +2. catalogd = sole metadata authority (datasets, schemas, manifests) +3. No raw data stored in catalog — only pointers (bucket, key, schema fingerprint) +4. storaged never interprets data — dumb pipe with presigned URLs +5. queryd registers tables via catalog pointers, not by scanning storage +6. aibridge is stateless — Python sidecar is replaceable without touching Rust +7. All services are modular and independently replaceable + +### Dependency Graph + +``` +shared ← storaged ← catalogd ← queryd +shared ← aibridge +gateway → {storaged, catalogd, queryd, aibridge} +ui → gateway (HTTP only, no crate dependency) +``` + +--- + +## Phases + +### Phase 0: Bootstrap +Workspace compiles, gateway serves health check, structured logging works. + +**Gate:** `cargo build` clean, `GET /health` returns 200, logs on stdout, docs committed. + +### Phase 1: Storage + Catalog +Write Parquet to object storage, register in catalog, read back. + +**Gate:** Upload Parquet → register dataset → retrieve metadata → read back. All via gateway HTTP. + +### Phase 2: Query Engine +SQL queries over registered Parquet datasets via DataFusion. + +**Gate:** `SELECT * FROM dataset LIMIT 10` returns correct results. Resolution goes through catalog. + +### Phase 3: AI Integration +Python sidecar with real Ollama models. Embeddings, generation, reranking. + +**Gate:** Rust sends text → Python → Ollama → real embeddings return as Arrow-compatible floats. + +### Phase 4: Frontend +Dioxus UI: dataset browser, query editor, results table. + +**Gate:** User can browse datasets and run queries from browser. + +### Phase 5: Hardening +gRPC internals, OpenTelemetry, auth, config-driven startup. + +**Gate:** Services communicate via gRPC. Traces propagate. Auth enforced. System restartable from repo + config. + +--- + +## Available Local Models + +| Model | Use | +|---|---| +| `nomic-embed-text` | Embeddings (768d) | +| `qwen2.5` | Code generation, structured output | +| `mistral` | General generation | +| `gemma2` | General generation | +| `llama3.2` | General generation | + +Model selection via environment variables. No hardcoded model names in Rust code. + +--- + +## Non-Goals + +- Multi-tenancy +- Streaming ingestion / CDC +- Custom file formats +- Query caching / materialized views +- Wrapping `object_store` with another abstraction +- Cloud deployment (local-first) + +--- + +## Risks + +| Risk | Severity | Mitigation | +|---|---|---| +| RustFS immaturity | High | Start with LocalFileSystem, test against MinIO, RustFS last. SeaweedFS fallback. | +| DataFusion table registration overhead | Medium | Lazy registration + LRU cache of SessionContext instances. | +| Catalog consistency without DB | Medium | Write-ahead: persist manifest before in-memory update. Rebuild from storage on restart. | +| Dioxus WASM gaps | Medium | Phase 4 is last. Fallback to plain HTML if blocked. | +| Schema evolution | Medium | Schema fingerprinting in Phase 1. Validate before query. | + +--- + +## Operating Rules + +1. PRD > architecture > phases > status > git +2. Git is memory, not chat +3. No undocumented changes +4. No silent architecture drift +5. Always work in smallest valid step +6. Always verify before moving on diff --git a/justfile b/justfile new file mode 100644 index 0000000..e30ad8b --- /dev/null +++ b/justfile @@ -0,0 +1,28 @@ +# Lakehouse task runner + +default: + @just --list + +# Build all crates +build: + cargo build --workspace + +# Run all tests +test: + cargo test --workspace + +# Run gateway +run: + cargo run --bin gateway + +# Check without building +check: + cargo check --workspace + +# Format all code +fmt: + cargo fmt --all + +# Lint +clippy: + cargo clippy --workspace -- -D warnings