golangLAKEHOUSE/internal/catalogd/manifest.go

// Package catalogd is the metadata authority — manifests for every
// registered dataset. A Manifest is the catalog row Rust calls
// `Manifest`: dataset_id (deterministic from name via UUIDv5),
// schema_fingerprint (caller-supplied schema hash), the object keys
// that physically back the dataset in storaged, plus timestamps and
// optional row_count.
//
// G0 stores one Manifest per Parquet file at
// primary://_catalog/manifests/<name>.parquet. One row per file —
// catalog manifests are written rarely and read on startup, so the
// per-file shape favors atomic register over storage density.
package catalogd

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"time"

	"github.com/apache/arrow-go/v18/arrow"
	"github.com/apache/arrow-go/v18/arrow/array"
	"github.com/apache/arrow-go/v18/arrow/memory"
	"github.com/apache/arrow-go/v18/parquet"
	"github.com/apache/arrow-go/v18/parquet/file"
	"github.com/apache/arrow-go/v18/parquet/pqarrow"
	"github.com/google/uuid"
)

// catalogNamespace is the v5 UUID namespace for dataset_id derivation.
// Same name → same dataset_id across boxes / cold starts. Don't change
// — every dataset_id ever issued depends on this byte sequence.
var catalogNamespace = uuid.MustParse("a8f3c1d2-4e5b-5a6c-9d8e-7f0a1b2c3d4e")

// Object is one storaged key contributing to a dataset.
type Object struct {
	Key  string `json:"key"`
	Size int64  `json:"size"`
}

// Manifest is the catalog row for one dataset.
type Manifest struct {
	DatasetID         string    `json:"dataset_id"`
	Name              string    `json:"name"`
	SchemaFingerprint string    `json:"schema_fingerprint"`
	Objects           []Object  `json:"objects"`
	CreatedAt         time.Time `json:"created_at"`
	UpdatedAt         time.Time `json:"updated_at"`
	RowCount          *int64    `json:"row_count,omitempty"`
}

// DatasetIDForName returns the deterministic UUIDv5 dataset_id for a
// logical dataset name. Idempotent on the same name across boxes.
func DatasetIDForName(name string) string {
	return uuid.NewSHA1(catalogNamespace, []byte(name)).String()
}

// manifestArrowSchema is the Arrow schema for the on-disk Parquet.
// Field order matters — codec builders rely on it.
var manifestArrowSchema = arrow.NewSchema([]arrow.Field{
	{Name: "dataset_id", Type: arrow.BinaryTypes.String},
	{Name: "name", Type: arrow.BinaryTypes.String},
	{Name: "schema_fingerprint", Type: arrow.BinaryTypes.String},
	{Name: "objects", Type: arrow.ListOf(arrow.StructOf(
		arrow.Field{Name: "key", Type: arrow.BinaryTypes.String},
		arrow.Field{Name: "size", Type: arrow.PrimitiveTypes.Int64},
	))},
	{Name: "created_at_unix_ns", Type: arrow.PrimitiveTypes.Int64},
	{Name: "updated_at_unix_ns", Type: arrow.PrimitiveTypes.Int64},
	{Name: "row_count", Type: arrow.PrimitiveTypes.Int64, Nullable: true},
}, nil)

// Encode writes a single Manifest to a Parquet byte slice. Memory
// allocations are bounded — manifests have tens of objects, not
// millions.
func Encode(m *Manifest) ([]byte, error) {
	mem := memory.NewGoAllocator()

	rb := array.NewRecordBuilder(mem, manifestArrowSchema)
	defer rb.Release()

	rb.Field(0).(*array.StringBuilder).Append(m.DatasetID)
	rb.Field(1).(*array.StringBuilder).Append(m.Name)
	rb.Field(2).(*array.StringBuilder).Append(m.SchemaFingerprint)

	listB := rb.Field(3).(*array.ListBuilder)
	listB.Append(true)
	structB := listB.ValueBuilder().(*array.StructBuilder)
	keyB := structB.FieldBuilder(0).(*array.StringBuilder)
	sizeB := structB.FieldBuilder(1).(*array.Int64Builder)
	for _, o := range m.Objects {
		structB.Append(true)
		keyB.Append(o.Key)
		sizeB.Append(o.Size)
	}

	rb.Field(4).(*array.Int64Builder).Append(m.CreatedAt.UnixNano())
	rb.Field(5).(*array.Int64Builder).Append(m.UpdatedAt.UnixNano())
	if m.RowCount != nil {
		rb.Field(6).(*array.Int64Builder).Append(*m.RowCount)
	} else {
		rb.Field(6).(*array.Int64Builder).AppendNull()
	}

	rec := rb.NewRecord()
	defer rec.Release()

	var buf bytes.Buffer
	props := parquet.NewWriterProperties()
	arrowProps := pqarrow.NewArrowWriterProperties()
	w, err := pqarrow.NewFileWriter(manifestArrowSchema, &buf, props, arrowProps)
	if err != nil {
		return nil, fmt.Errorf("pqarrow writer: %w", err)
	}
	if err := w.Write(rec); err != nil {
		return nil, fmt.Errorf("pqarrow write: %w", err)
	}
	if err := w.Close(); err != nil {
		return nil, fmt.Errorf("pqarrow close: %w", err)
	}
	return buf.Bytes(), nil
}

// Decode reads a single-row Parquet manifest back into a Manifest.
func Decode(b []byte) (*Manifest, error) {
	rdr, err := file.NewParquetReader(bytes.NewReader(b))
	if err != nil {
		return nil, fmt.Errorf("parquet reader: %w", err)
	}
	defer rdr.Close()

	pr, err := pqarrow.NewFileReader(rdr, pqarrow.ArrowReadProperties{}, memory.NewGoAllocator())
	if err != nil {
		return nil, fmt.Errorf("pqarrow reader: %w", err)
	}
	tbl, err := pr.ReadTable(context.Background())
	if err != nil {
		return nil, fmt.Errorf("read table: %w", err)
	}
	defer tbl.Release()

	if tbl.NumRows() != 1 {
		return nil, fmt.Errorf("manifest parquet: expected 1 row, got %d", tbl.NumRows())
	}

	rr := array.NewTableReader(tbl, 1)
	defer rr.Release()
	if !rr.Next() {
		return nil, errors.New("manifest parquet: no record batch")
	}
	rec := rr.Record()

	m := &Manifest{
		DatasetID:         rec.Column(0).(*array.String).Value(0),
		Name:              rec.Column(1).(*array.String).Value(0),
		SchemaFingerprint: rec.Column(2).(*array.String).Value(0),
	}

	// Per scrum C1 (3-way convergent): use ValueOffsets which accounts
	// for the array's own offset (non-zero under slicing) and is bounds-
	// safe by API contract. Direct Offsets()[0]/[1] indexing is fragile
	// under multi-row reads and panics on malformed offset buffers.
	listArr := rec.Column(3).(*array.List)
	structArr := listArr.ListValues().(*array.Struct)
	keyArr := structArr.Field(0).(*array.String)
	sizeArr := structArr.Field(1).(*array.Int64)
	start, end := listArr.ValueOffsets(0)
	if start < 0 || end < start || end > int64(structArr.Len()) {
		return nil, fmt.Errorf("manifest: bad list offsets [%d, %d] for struct len %d",
			start, end, structArr.Len())
	}
	for i := start; i < end; i++ {
		m.Objects = append(m.Objects, Object{
			Key:  keyArr.Value(int(i)),
			Size: sizeArr.Value(int(i)),
		})
	}

	m.CreatedAt = time.Unix(0, rec.Column(4).(*array.Int64).Value(0))
	m.UpdatedAt = time.Unix(0, rec.Column(5).(*array.Int64).Value(0))
	rcArr := rec.Column(6).(*array.Int64)
	if rcArr.IsValid(0) {
		v := rcArr.Value(0)
		m.RowCount = &v
	}

	return m, nil
}

// EncodeReader is a small convenience for callers that want an
// io.Reader over the encoded bytes (matches the storaged HTTP PUT
// signature).
func EncodeReader(m *Manifest) (io.Reader, int64, error) {
	b, err := Encode(m)
	if err != nil {
		return nil, 0, err
	}
	return bytes.NewReader(b), int64(len(b)), nil
}