f9f92706f3
RAG pipeline now includes a cross-encoder rerank step between retrieval
and generation. The LLM re-sorts top-K results by relevance before
they become context. Falls back to original order if model output is
unparseable (~5% with 7B models). Also improved the generation prompt
to be domain-aware ("staffing database") and request specific citations.
Fixed 4 catalog manifests with bucket="data" (pre-federation leftover)
that poisoned the entire DataFusion query context on startup. The
"users", "lab_trials", "meta_runs", and "new_candidates" datasets
now correctly reference bucket="primary". This bug was surfaced by
the quality evaluation pipeline — wouldn't have been found by
structural tests alone.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
110 lines
2.5 KiB
JSON
110 lines
2.5 KiB
JSON
{
|
|
"id": "443d63f1-b0ed-4d4b-8e5a-ce59c097b97b",
|
|
"name": "kb_memory_entries",
|
|
"schema_fingerprint": "15dbebd0abb906577e11cbb73083abeb8961b8c603a2472cafa8c46100c3fb1c",
|
|
"objects": [
|
|
{
|
|
"bucket": "primary",
|
|
"key": "datasets/kb_memory_entries.parquet",
|
|
"size_bytes": 8795,
|
|
"created_at": "2026-04-17T02:54:43.330897823Z"
|
|
}
|
|
],
|
|
"created_at": "2026-04-17T02:54:43.330898330Z",
|
|
"updated_at": "2026-04-17T02:54:43.330972517Z",
|
|
"description": "",
|
|
"owner": "",
|
|
"sensitivity": null,
|
|
"columns": [
|
|
{
|
|
"name": "id",
|
|
"data_type": "Int32",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
},
|
|
{
|
|
"name": "content",
|
|
"data_type": "Utf8",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
},
|
|
{
|
|
"name": "content_type",
|
|
"data_type": "Utf8",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
},
|
|
{
|
|
"name": "source",
|
|
"data_type": "Utf8",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
},
|
|
{
|
|
"name": "session_id",
|
|
"data_type": "Utf8",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
},
|
|
{
|
|
"name": "embedding",
|
|
"data_type": "Utf8",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
},
|
|
{
|
|
"name": "metadata",
|
|
"data_type": "Utf8",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
},
|
|
{
|
|
"name": "importance_score",
|
|
"data_type": "Float64",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
},
|
|
{
|
|
"name": "created_at",
|
|
"data_type": "Utf8",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
},
|
|
{
|
|
"name": "accessed_at",
|
|
"data_type": "Utf8",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
},
|
|
{
|
|
"name": "access_count",
|
|
"data_type": "Int32",
|
|
"sensitivity": null,
|
|
"description": "",
|
|
"is_pii": false
|
|
}
|
|
],
|
|
"lineage": {
|
|
"source_system": "postgresql",
|
|
"source_file": "dsn: postgresql://kbuser@localhost:5432/knowledge_base",
|
|
"ingest_job": "pg-stream-1776394483330",
|
|
"ingest_timestamp": "2026-04-17T02:54:43.330897823Z",
|
|
"parent_datasets": []
|
|
},
|
|
"freshness": null,
|
|
"tags": [],
|
|
"row_count": 13,
|
|
"last_embedded_at": null,
|
|
"embedding_stale_since": null,
|
|
"embedding_refresh_policy": null
|
|
} |