bf7cf96911
- journald crate: immutable event log for every data mutation
- Events: entity_type, entity_id, field, action, old_value, new_value,
actor, source, workspace_id, timestamp
- In-memory buffer with configurable flush threshold (default 100 events)
- Flush writes events as Parquet to journal/ directory
- Query: GET /journal/history/{entity_id} — full history of any record
- Query: GET /journal/recent?limit=50 — latest events across all entities
- Convenience methods: record_insert, record_update, record_ingest
- Stats: GET /journal/stats — buffer size, persisted file count
- Manual flush: POST /journal/flush
- Per ADR-012: events are never modified or deleted
This is the single most important future-proofing decision.
Once history is lost, it's gone forever.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
114 lines
4.1 KiB
Rust
114 lines
4.1 KiB
Rust
mod auth;
|
|
mod observability;
|
|
|
|
use axum::{Router, extract::DefaultBodyLimit, routing::get};
|
|
use proto::lakehouse::catalog_service_server::CatalogServiceServer;
|
|
use shared::config::Config;
|
|
use tower_http::cors::{Any, CorsLayer};
|
|
use tower_http::trace::TraceLayer;
|
|
|
|
#[tokio::main]
|
|
async fn main() {
|
|
// Load config
|
|
let config = Config::load_or_default();
|
|
|
|
// Initialize tracing + observability
|
|
observability::init_tracing(
|
|
&config.observability.service_name,
|
|
&config.observability.exporter,
|
|
);
|
|
|
|
tracing::info!("config loaded: gateway={}:{}, storage={}",
|
|
config.gateway.host, config.gateway.port, config.storage.root);
|
|
|
|
// Storage backend
|
|
let store = storaged::backend::init_local(&config.storage.root);
|
|
|
|
// Catalog
|
|
let registry = catalogd::registry::Registry::new(store.clone());
|
|
if let Err(e) = registry.rebuild().await {
|
|
tracing::warn!("catalog rebuild failed (empty store?): {e}");
|
|
}
|
|
|
|
// Query engine with 16GB memory cache
|
|
let cache = queryd::cache::MemCache::new(16 * 1024 * 1024 * 1024);
|
|
let engine = queryd::context::QueryEngine::new(registry.clone(), store.clone(), cache);
|
|
|
|
// Event journal — append-only mutation log (flush every 100 events)
|
|
let journal = journald::journal::Journal::new(store.clone(), 100);
|
|
|
|
// Workspace manager for agent-specific overlays
|
|
let workspace_mgr = queryd::workspace::WorkspaceManager::new(store.clone());
|
|
if let Err(e) = workspace_mgr.rebuild().await {
|
|
tracing::warn!("workspace rebuild: {e}");
|
|
}
|
|
|
|
// AI sidecar client
|
|
let ai_client = aibridge::client::AiClient::new(&config.sidecar.url);
|
|
|
|
// HTTP router
|
|
let mut app = Router::new()
|
|
.route("/health", get(health))
|
|
.nest("/storage", storaged::service::router(store.clone()))
|
|
.nest("/catalog", catalogd::service::router(registry.clone()))
|
|
.nest("/query", queryd::service::router(engine))
|
|
.nest("/ai", aibridge::service::router(ai_client.clone()))
|
|
.nest("/ingest", ingestd::service::router(ingestd::service::IngestState {
|
|
store: store.clone(),
|
|
registry: registry.clone(),
|
|
}))
|
|
.nest("/vectors", vectord::service::router(vectord::service::VectorState {
|
|
store: store.clone(),
|
|
ai_client: ai_client.clone(),
|
|
job_tracker: vectord::jobs::JobTracker::new(),
|
|
}))
|
|
.nest("/workspaces", queryd::workspace_service::router(workspace_mgr))
|
|
.nest("/journal", journald::service::router(journal));
|
|
|
|
// Auth middleware (if enabled)
|
|
if config.auth.enabled {
|
|
if let Some(ref key) = config.auth.api_key {
|
|
tracing::info!("API key auth enabled");
|
|
let api_key = auth::ApiKey(key.clone());
|
|
app = app.layer(axum::Extension(api_key));
|
|
// Note: auth middleware applied per-route in production
|
|
// For now, the ApiKey extension is available for handlers to check
|
|
} else {
|
|
tracing::warn!("auth enabled but no api_key set — all requests allowed");
|
|
}
|
|
}
|
|
|
|
app = app
|
|
.layer(DefaultBodyLimit::max(256 * 1024 * 1024)) // 256MB
|
|
.layer(CorsLayer::new()
|
|
.allow_origin(Any)
|
|
.allow_methods(Any)
|
|
.allow_headers(Any))
|
|
.layer(TraceLayer::new_for_http());
|
|
|
|
// Start gRPC server on port+1
|
|
let grpc_port = config.gateway.port + 1;
|
|
let catalog_grpc = catalogd::grpc::CatalogGrpc::new(registry);
|
|
let grpc_addr = format!("{}:{}", config.gateway.host, grpc_port).parse().unwrap();
|
|
|
|
tokio::spawn(async move {
|
|
tracing::info!("gRPC server listening on {grpc_addr}");
|
|
tonic::transport::Server::builder()
|
|
.add_service(CatalogServiceServer::new(catalog_grpc))
|
|
.serve(grpc_addr)
|
|
.await
|
|
.expect("gRPC server failed");
|
|
});
|
|
|
|
// Start HTTP server
|
|
let http_addr = format!("{}:{}", config.gateway.host, config.gateway.port);
|
|
tracing::info!("HTTP gateway listening on {http_addr}");
|
|
|
|
let listener = tokio::net::TcpListener::bind(&http_addr).await.unwrap();
|
|
axum::serve(listener, app).await.unwrap();
|
|
}
|
|
|
|
async fn health() -> &'static str {
|
|
"lakehouse ok"
|
|
}
|