c1e411347a
Phase G0 Day 4 ships ingestd: multipart CSV upload, Arrow schema
inference per ADR-010 (default-to-string on ambiguity), single-pass
streaming CSV → Parquet via pqarrow batched writer (Snappy compressed,
8192 rows per batch), PUT to storaged at content-addressed key
datasets/<name>/<fp_hex>.parquet, register manifest with catalogd.
Acceptance smoke 6/6 PASS including idempotent re-ingest (proves
inference is deterministic — same CSV always produces same fingerprint)
and schema-drift → 409 (proves catalogd's gate fires on ingest traffic).
Schema fingerprint is SHA-256 over (name, type) tuples in header order
using ASCII record/unit separators (0x1e/0x1f) so column names with
commas can't collide. Nullability intentionally NOT in the fingerprint
— a column gaining nulls isn't a schema change.
Cross-lineage scrum on shipped code:
- Opus 4.7 (opencode): 4 WARN + 3 INFO (after 2 self-retracted BLOCKs)
- Kimi K2-0905 (openrouter): 1 BLOCK + 2 WARN + 1 INFO
- Qwen3-coder (openrouter): 2 BLOCK + 2 WARN + 2 INFO
Fixed (2, both Opus single-reviewer):
C-DRIFT: PUT-then-register on fixed datasets/<name>/data.parquet
meant a schema-drift ingest overwrote the live parquet BEFORE
catalogd's 409 fired → storaged inconsistent with manifest.
Fix: content-addressed key datasets/<name>/<fp_hex>.parquet.
Drift writes to a different file (orphan in G2 GC scope); the
live data is never corrupted.
C-WCLOSE: pqarrow.NewFileWriter not Closed on error paths leaks
buffered column data + OS resources per failed ingest.
Fix: deferred guarded close with wClosed flag.
Dismissed (5, all false positives):
Qwen BLOCK "csv.Reader needs LazyQuotes=true for multi-line" — false,
Go csv handles RFC 4180 multi-line quoted fields by default
Qwen BLOCK "row[i] OOB" — already bounds-checked at schema.go:73
and csv.go:201
Kimi BLOCK "type assertion panic if pqarrow reorders fields" —
speculative, no real path
Kimi WARN + Qwen WARN×2 "RecordBuilder leak on early error" —
false convergent. Outer defer rb.Release() captures the current
builder; in-loop release runs before reassignment. No leak.
Deferred (6 INFO + accepted-with-rationale on 3 WARN): sample
boundary type mismatch (G0 cap bounds peak), string-match
paranoia on http.MaxBytesError, multipart double-buffer (G2 spool-
to-disk), separator validation, body close ordering, etc.
The D4 scrum produced fewer real findings than D3 (2 vs 6) — both
were architectural hazards smoke wouldn't catch because the smoke's
"schema drift → 409" assertion was passing even in the corrupted-
state world. The 409 fires correctly; what was wrong was the PUT
having already mutated the live parquet before the validation check.
Opus's PUT-then-register read of the order is exactly the kind of
architectural insight the cross-lineage scrum is designed to surface.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
116 lines
3.8 KiB
Go
116 lines
3.8 KiB
Go
package ingestd
|
|
|
|
import (
|
|
"testing"
|
|
)
|
|
|
|
func TestInferSchema_CleanInts(t *testing.T) {
|
|
headers := []string{"id", "count"}
|
|
samples := [][]string{{"1", "100"}, {"2", "200"}, {"3", "300"}}
|
|
got, err := InferSchema(headers, samples)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, c := range got {
|
|
if c.Type != TypeInt64 {
|
|
t.Errorf("%s: got %s, want int64", c.Name, c.Type)
|
|
}
|
|
if c.Nullable {
|
|
t.Errorf("%s should not be nullable", c.Name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestInferSchema_FloatColumns(t *testing.T) {
|
|
headers := []string{"price", "weight"}
|
|
samples := [][]string{{"1.5", "2.0"}, {"100", "3.14"}, {"0.0", "0"}}
|
|
got, _ := InferSchema(headers, samples)
|
|
// "price" has 1.5 + "100" + 0.0 → float64 (one of the values isn't int-parseable in 1.5)
|
|
if got[0].Type != TypeFloat64 {
|
|
t.Errorf("price: got %s, want float64", got[0].Type)
|
|
}
|
|
if got[1].Type != TypeFloat64 {
|
|
t.Errorf("weight: got %s, want float64", got[1].Type)
|
|
}
|
|
}
|
|
|
|
func TestInferSchema_AmbiguousFallsToString(t *testing.T) {
|
|
// ADR-010: a column with "123", "N/A", and "" is a string, not int.
|
|
headers := []string{"salary"}
|
|
samples := [][]string{{"50000"}, {"N/A"}, {"60000"}, {""}}
|
|
got, _ := InferSchema(headers, samples)
|
|
if got[0].Type != TypeString {
|
|
t.Errorf("salary: got %s, want string (ADR-010 fallback)", got[0].Type)
|
|
}
|
|
if !got[0].Nullable {
|
|
t.Errorf("salary: should be nullable (saw empty cell)")
|
|
}
|
|
}
|
|
|
|
func TestInferSchema_BoolLiterals(t *testing.T) {
|
|
headers := []string{"active", "deleted"}
|
|
samples := [][]string{{"true", "false"}, {"True", "False"}, {"TRUE", "FALSE"}}
|
|
got, _ := InferSchema(headers, samples)
|
|
if got[0].Type != TypeBool {
|
|
t.Errorf("active: got %s, want bool", got[0].Type)
|
|
}
|
|
if got[1].Type != TypeBool {
|
|
t.Errorf("deleted: got %s, want bool", got[1].Type)
|
|
}
|
|
}
|
|
|
|
func TestInferSchema_OneZeroIsInt_NotBool(t *testing.T) {
|
|
// Keeps the type system honest — 1/0 columns in CRM data are
|
|
// typically counts (children, certs), not flags.
|
|
headers := []string{"children"}
|
|
samples := [][]string{{"0"}, {"1"}, {"2"}, {"0"}}
|
|
got, _ := InferSchema(headers, samples)
|
|
if got[0].Type != TypeInt64 {
|
|
t.Errorf("children: got %s, want int64 (1/0 is int, not bool)", got[0].Type)
|
|
}
|
|
}
|
|
|
|
func TestInferSchema_EmptyHeader(t *testing.T) {
|
|
if _, err := InferSchema(nil, nil); err == nil {
|
|
t.Error("nil headers should error")
|
|
}
|
|
if _, err := InferSchema([]string{"valid", ""}, nil); err == nil {
|
|
t.Error("empty header name should error")
|
|
}
|
|
}
|
|
|
|
func TestFingerprint_Deterministic(t *testing.T) {
|
|
s1, _ := InferSchema([]string{"id", "name"}, [][]string{{"1", "alice"}})
|
|
s2, _ := InferSchema([]string{"id", "name"}, [][]string{{"1", "alice"}})
|
|
if s1.Fingerprint() != s2.Fingerprint() {
|
|
t.Errorf("fingerprint not deterministic: %s vs %s", s1.Fingerprint(), s2.Fingerprint())
|
|
}
|
|
}
|
|
|
|
func TestFingerprint_FlipsOnTypeChange(t *testing.T) {
|
|
intSchema, _ := InferSchema([]string{"id"}, [][]string{{"1"}, {"2"}})
|
|
strSchema, _ := InferSchema([]string{"id"}, [][]string{{"1"}, {"abc"}})
|
|
if intSchema.Fingerprint() == strSchema.Fingerprint() {
|
|
t.Error("fingerprint should flip when column type changes")
|
|
}
|
|
}
|
|
|
|
func TestFingerprint_StableUnderNullable(t *testing.T) {
|
|
// Adding null cells doesn't flip the fingerprint — it's only
|
|
// about (name, type), not nullability.
|
|
a, _ := InferSchema([]string{"id"}, [][]string{{"1"}, {"2"}})
|
|
b, _ := InferSchema([]string{"id"}, [][]string{{"1"}, {"2"}, {""}})
|
|
if a.Fingerprint() != b.Fingerprint() {
|
|
t.Error("fingerprint shouldn't flip when nullability changes")
|
|
}
|
|
}
|
|
|
|
func TestFingerprint_RespectsColumnOrder(t *testing.T) {
|
|
// Same columns, swapped order → different fingerprint.
|
|
a, _ := InferSchema([]string{"id", "name"}, [][]string{{"1", "x"}})
|
|
b, _ := InferSchema([]string{"name", "id"}, [][]string{{"x", "1"}})
|
|
if a.Fingerprint() == b.Fingerprint() {
|
|
t.Error("fingerprint should be order-sensitive")
|
|
}
|
|
}
|