golangLAKEHOUSE/internal/catalogd/registry_test.go
root 66a704ca3e G0 D3: catalogd Parquet manifests + ADR-020 idempotent register · 6 scrum fixes
Phase G0 Day 3 ships catalogd: Arrow Parquet manifest codec, in-memory
registry with the ADR-020 idempotency contract (same name+fingerprint
reuses dataset_id; different fingerprint → 409 Conflict), HTTP client
to storaged for persistence, and rehydration on startup. Acceptance
smoke 6/6 PASSES end-to-end including rehydrate-across-restart — the
load-bearing test that the catalog/storaged service split actually
preserves state.

dataset_id derivation diverges from Rust: UUIDv5(namespace, name)
instead of v4 surrogate. Same name on any box generates the same
dataset_id; rehydrate after disk loss converges to the same identity
rather than silently re-issuing. Namespace pinned at
a8f3c1d2-4e5b-5a6c-9d8e-7f0a1b2c3d4e — every dataset_id ever issued
depends on these bytes.

Cross-lineage scrum on shipped code:
  - Opus 4.7 (opencode):                       1 BLOCK + 5 WARN + 3 INFO
  - Kimi K2-0905 (openrouter, validated D2):   2 BLOCK + 2 WARN + 1 INFO
  - Qwen3-coder (openrouter):                  2 BLOCK + 2 WARN + 2 INFO

Fixed:
  C1 list-offsets BLOCK (3-way convergent) → ValueOffsets(0) + bounds
  C2 Rehydrate mutex held across I/O → swap-under-brief-lock pattern
  S1 split-brain on persist failure → candidate-then-swap
  S2 brittle string-match for 400 vs 500 → ErrEmptyName/ErrEmptyFingerprint sentinels
  S3 Get/List shallow-copy aliasing → cloneManifest deep copy
  S4 keep-alive socket leak on error paths → drainAndClose helper

Dismissed (false positives, all single-reviewer):
  Kimi BLOCK "Decode crashes on empty Parquet" — already handled
  Kimi INFO "safeKey double-escapes" — wrong, splitting before escape is required
  Qwen INFO "rb.NewRecord() error unchecked" — API returns no error

Deferred to G1+: name validation regex, per-call deadlines, Snappy
compression, list pagination continuation tokens (storaged caps at
10k with sentinel for now).

Build clean, vet clean, all tests pass, smoke 6/6 PASS after every
fix round. arrow-go/v18 + google/uuid added; Go 1.24 → 1.25 forced
by arrow-go's minimum.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-28 23:36:57 -05:00

161 lines
4.4 KiB
Go

package catalogd
import (
"context"
"errors"
"sync"
"testing"
"time"
)
// memStore is an in-memory Store fake for unit tests.
type memStore struct {
mu sync.Mutex
data map[string][]byte
}
func newMemStore() *memStore { return &memStore{data: map[string][]byte{}} }
func (m *memStore) Put(_ context.Context, key string, body []byte) error {
m.mu.Lock()
defer m.mu.Unlock()
cp := make([]byte, len(body))
copy(cp, body)
m.data[key] = cp
return nil
}
func (m *memStore) Get(_ context.Context, key string) ([]byte, error) {
m.mu.Lock()
defer m.mu.Unlock()
b, ok := m.data[key]
if !ok {
return nil, ErrKeyNotFound
}
return b, nil
}
func (m *memStore) List(_ context.Context, prefix string) ([]string, error) {
m.mu.Lock()
defer m.mu.Unlock()
out := []string{}
for k := range m.data {
if len(k) >= len(prefix) && k[:len(prefix)] == prefix {
out = append(out, k)
}
}
return out, nil
}
func mkRegistry(t *testing.T) (*Registry, *memStore) {
t.Helper()
s := newMemStore()
r := NewRegistry(s)
r.now = func() time.Time { return time.Unix(1777435000, 0).UTC() }
return r, s
}
func TestRegister_NewManifest(t *testing.T) {
r, _ := mkRegistry(t)
rc := int64(100)
m, existing, err := r.Register(context.Background(), "workers", "sha256:abc",
[]Object{{Key: "datasets/workers/p1.parquet", Size: 1024}}, &rc)
if err != nil {
t.Fatalf("Register: %v", err)
}
if existing {
t.Error("expected existing=false for new manifest")
}
if m.DatasetID != DatasetIDForName("workers") {
t.Errorf("DatasetID: got %q, want UUIDv5(workers)", m.DatasetID)
}
}
func TestRegister_SameFingerprint_Idempotent(t *testing.T) {
r, _ := mkRegistry(t)
rc := int64(100)
first, _, _ := r.Register(context.Background(), "workers", "sha256:abc",
[]Object{{Key: "p1.parquet", Size: 1024}}, &rc)
// Re-register same name + fingerprint with new objects.
rc2 := int64(200)
second, existing, err := r.Register(context.Background(), "workers", "sha256:abc",
[]Object{{Key: "p1.parquet", Size: 1024}, {Key: "p2.parquet", Size: 2048}}, &rc2)
if err != nil {
t.Fatalf("Register (idempotent): %v", err)
}
if !existing {
t.Error("expected existing=true on idempotent re-register")
}
if second.DatasetID != first.DatasetID {
t.Errorf("DatasetID changed: %q → %q", first.DatasetID, second.DatasetID)
}
if len(second.Objects) != 2 {
t.Errorf("Objects not replaced: got %d, want 2", len(second.Objects))
}
if second.RowCount == nil || *second.RowCount != 200 {
t.Errorf("RowCount not bumped: got %v, want 200", second.RowCount)
}
}
func TestRegister_DifferentFingerprint_Conflict(t *testing.T) {
r, _ := mkRegistry(t)
_, _, _ = r.Register(context.Background(), "workers", "sha256:abc",
[]Object{{Key: "p1.parquet", Size: 1024}}, nil)
_, _, err := r.Register(context.Background(), "workers", "sha256:DIFFERENT",
[]Object{{Key: "p1.parquet", Size: 1024}}, nil)
if !errors.Is(err, ErrFingerprintConflict) {
t.Fatalf("expected ErrFingerprintConflict, got %v", err)
}
}
func TestRehydrate_RecoversManifests(t *testing.T) {
// Build first registry, register 2 manifests.
r1, store := mkRegistry(t)
_, _, _ = r1.Register(context.Background(), "workers", "sha256:a", nil, nil)
_, _, _ = r1.Register(context.Background(), "candidates", "sha256:b", nil, nil)
// Build a second registry against the same store + rehydrate.
r2 := NewRegistry(store)
n, err := r2.Rehydrate(context.Background())
if err != nil {
t.Fatalf("Rehydrate: %v", err)
}
if n != 2 {
t.Errorf("recovered %d, want 2", n)
}
if _, err := r2.Get("workers"); err != nil {
t.Errorf("Get(workers): %v", err)
}
if _, err := r2.Get("candidates"); err != nil {
t.Errorf("Get(candidates): %v", err)
}
}
func TestList_Sorted(t *testing.T) {
r, _ := mkRegistry(t)
_, _, _ = r.Register(context.Background(), "zoo", "fp", nil, nil)
_, _, _ = r.Register(context.Background(), "alpha", "fp", nil, nil)
_, _, _ = r.Register(context.Background(), "midway", "fp", nil, nil)
got := r.List()
want := []string{"alpha", "midway", "zoo"}
for i, m := range got {
if m.Name != want[i] {
t.Errorf("List[%d]: got %q, want %q", i, m.Name, want[i])
}
}
}
func TestRegister_RejectsEmptyInputs(t *testing.T) {
r, _ := mkRegistry(t)
_, _, err := r.Register(context.Background(), "", "fp", nil, nil)
if err == nil {
t.Error("expected error on empty name")
}
_, _, err = r.Register(context.Background(), "x", "", nil, nil)
if err == nil {
t.Error("expected error on empty fingerprint")
}
}