lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus/durable_outbox_wal.go

package eventbus

import (
	"context"
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	outboxWALMagic           = uint32(0x4c4e5731) // LNW1
	outboxWALHeaderSize      = 12
	outboxWALMaxRecordBytes  = 16 << 20
	defaultWALSegmentBytes   = 16 << 20
	defaultWALAppendQueue    = 100_000
	defaultWALCommitBatch    = 256
	defaultWALCommitInterval = time.Millisecond
	defaultWALSegmentAge     = 5 * time.Second
)

var ErrDurableOutboxWALClosed = errors.New("durable outbox wal is closed")

type durableOutboxWALConfig struct {
	Directory      string
	SyncWrites     bool
	SegmentBytes   int64
	SegmentAge     time.Duration
	AppendQueue    int
	CommitBatch    int
	CommitInterval time.Duration
}

type outboxWALRecordRef struct {
	segmentID uint64
	index     int
}

type storedOutboxRecord struct {
	Ref    outboxWALRecordRef
	Record durableRecord
}

type outboxWALRecordStatus uint8

const (
	outboxWALPending outboxWALRecordStatus = iota
	outboxWALClaimed
	outboxWALAcknowledged
)

type outboxWALRecordState struct {
	payloadOffset int64
	payloadLength uint32
	status        outboxWALRecordStatus
}

type outboxWALSegment struct {
	id        uint64
	path      string
	createdAt time.Time
	size      int64
	closed    bool
	deleting  bool
	acked     int
	records   []*outboxWALRecordState
}

type outboxWALAppendRequest struct {
	record  durableRecord
	payload []byte
	frame   []byte
	result  chan outboxWALAppendResult
}

type outboxWALAppendResult struct {
	stored storedOutboxRecord
	err    error
}

type durableOutboxWAL struct {
	dir            string
	syncWrites     bool
	segmentBytes   int64
	segmentAge     time.Duration
	commitBatch    int
	commitInterval time.Duration

	mu          sync.Mutex
	segments    []*outboxWALSegment
	segmentByID map[uint64]*outboxWALSegment
	current     *outboxWALSegment
	currentFile *os.File
	backlog     int
	fatalErr    error

	appendMu sync.RWMutex
	closed   bool
	queue    chan outboxWALAppendRequest
	writerWG sync.WaitGroup
	closeOne sync.Once
}

func newDurableOutboxWAL(cfg durableOutboxWALConfig) (*durableOutboxWAL, error) {
	dir := strings.TrimSpace(cfg.Directory)
	if dir == "" {
		return nil, errors.New("durable outbox wal directory is required")
	}
	if cfg.SegmentBytes <= 0 {
		cfg.SegmentBytes = defaultWALSegmentBytes
	}
	if cfg.SegmentAge <= 0 {
		cfg.SegmentAge = defaultWALSegmentAge
	}
	if cfg.AppendQueue <= 0 {
		cfg.AppendQueue = defaultWALAppendQueue
	}
	if cfg.CommitBatch <= 0 {
		cfg.CommitBatch = defaultWALCommitBatch
	}
	if cfg.CommitInterval <= 0 {
		cfg.CommitInterval = defaultWALCommitInterval
	}
	if err := os.MkdirAll(dir, 0o750); err != nil {
		return nil, fmt.Errorf("create durable outbox wal directory: %w", err)
	}
	w := &durableOutboxWAL{
		dir:            dir,
		syncWrites:     cfg.SyncWrites,
		segmentBytes:   cfg.SegmentBytes,
		segmentAge:     cfg.SegmentAge,
		commitBatch:    cfg.CommitBatch,
		commitInterval: cfg.CommitInterval,
		segmentByID:    map[uint64]*outboxWALSegment{},
		queue:          make(chan outboxWALAppendRequest, cfg.AppendQueue),
	}
	if err := w.loadSegments(); err != nil {
		return nil, err
	}
	if err := w.createCurrentSegment(); err != nil {
		return nil, err
	}
	w.writerWG.Add(1)
	go w.appendLoop()
	return w, nil
}

func (w *durableOutboxWAL) Append(ctx context.Context, record durableRecord) (storedOutboxRecord, error) {
	payload, err := json.Marshal(record)
	if err != nil {
		return storedOutboxRecord{}, fmt.Errorf("marshal durable outbox wal record: %w", err)
	}
	if len(payload) > outboxWALMaxRecordBytes {
		return storedOutboxRecord{}, fmt.Errorf("durable outbox wal record is %d bytes, max %d", len(payload), outboxWALMaxRecordBytes)
	}
	request := outboxWALAppendRequest{
		record:  record,
		payload: payload,
		frame:   encodeOutboxWALFrame(payload),
		result:  make(chan outboxWALAppendResult, 1),
	}
	if ctx == nil {
		ctx = context.Background()
	}
	w.appendMu.RLock()
	if w.closed {
		w.appendMu.RUnlock()
		return storedOutboxRecord{}, ErrDurableOutboxWALClosed
	}
	select {
	case w.queue <- request:
		w.appendMu.RUnlock()
	case <-ctx.Done():
		w.appendMu.RUnlock()
		return storedOutboxRecord{}, ctx.Err()
	}
	// Once admitted to the WAL queue, wait for the durability result even if
	// the caller context is cancelled. Otherwise a committed record could be
	// left claimed with no publisher responsible for it.
	result := <-request.result
	return result.stored, result.err
}

func (w *durableOutboxWAL) ClaimPending(limit int) ([]storedOutboxRecord, error) {
	if limit <= 0 {
		limit = defaultWALCommitBatch
	}
	w.mu.Lock()
	refs := make([]outboxWALRecordRef, 0, limit)
	for _, segment := range w.segments {
		for index, state := range segment.records {
			if state.status != outboxWALPending {
				continue
			}
			state.status = outboxWALClaimed
			refs = append(refs, outboxWALRecordRef{segmentID: segment.id, index: index})
			if len(refs) >= limit {
				break
			}
		}
		if len(refs) >= limit {
			break
		}
	}
	w.mu.Unlock()
	if len(refs) == 0 {
		return nil, nil
	}
	records, err := w.readClaimedRecords(refs)
	if err != nil {
		for _, ref := range refs {
			w.Release(ref)
		}
		return nil, err
	}
	return records, nil
}

func (w *durableOutboxWAL) Release(ref outboxWALRecordRef) {
	w.mu.Lock()
	defer w.mu.Unlock()
	state := w.recordStateLocked(ref)
	if state != nil && state.status == outboxWALClaimed {
		state.status = outboxWALPending
	}
}

func (w *durableOutboxWAL) Ack(ref outboxWALRecordRef) error {
	w.mu.Lock()
	segment := w.segmentByID[ref.segmentID]
	if segment == nil || ref.index < 0 || ref.index >= len(segment.records) {
		w.mu.Unlock()
		return nil
	}
	state := segment.records[ref.index]
	if state.status == outboxWALAcknowledged {
		w.mu.Unlock()
		return nil
	}
	state.status = outboxWALAcknowledged
	segment.acked++
	if w.backlog > 0 {
		w.backlog--
	}
	shouldDelete := segment.closed && segment.acked == len(segment.records) && !segment.deleting
	if shouldDelete {
		segment.deleting = true
	}
	w.mu.Unlock()
	if !shouldDelete {
		return nil
	}
	return w.deleteAcknowledgedSegment(segment)
}

func (w *durableOutboxWAL) Stats() (backlog int, oldest time.Time) {
	w.mu.Lock()
	defer w.mu.Unlock()
	backlog = w.backlog
	if backlog == 0 {
		return backlog, time.Time{}
	}
	for _, segment := range w.segments {
		if segment.acked < len(segment.records) {
			return backlog, segment.createdAt
		}
	}
	return backlog, time.Time{}
}

func (w *durableOutboxWAL) Close() error {
	w.closeOne.Do(func() {
		w.appendMu.Lock()
		w.closed = true
		close(w.queue)
		w.appendMu.Unlock()
		w.writerWG.Wait()
	})
	w.mu.Lock()
	err := w.fatalErr
	w.mu.Unlock()
	return err
}

func (w *durableOutboxWAL) appendLoop() {
	defer w.writerWG.Done()
	maintenanceInterval := minDuration(w.segmentAge/2, time.Second)
	if maintenanceInterval < 10*time.Millisecond {
		maintenanceInterval = 10 * time.Millisecond
	}
	maintenance := time.NewTicker(maintenanceInterval)
	defer maintenance.Stop()
	for {
		select {
		case request, ok := <-w.queue:
			if !ok {
				w.finishWriter()
				return
			}
			batch := w.collectAppendBatch(request)
			w.commitAppendBatch(batch)
		case <-maintenance.C:
			if err := w.rotateIfAged(); err != nil {
				w.setFatal(err)
			}
		}
	}
}

func (w *durableOutboxWAL) collectAppendBatch(first outboxWALAppendRequest) []outboxWALAppendRequest {
	batch := make([]outboxWALAppendRequest, 0, w.commitBatch)
	batch = append(batch, first)
	timer := time.NewTimer(w.commitInterval)
	defer timer.Stop()
	for len(batch) < w.commitBatch {
		select {
		case request, ok := <-w.queue:
			if !ok {
				return batch
			}
			batch = append(batch, request)
		case <-timer.C:
			return batch
		}
	}
	return batch
}

func (w *durableOutboxWAL) commitAppendBatch(batch []outboxWALAppendRequest) {
	if len(batch) == 0 {
		return
	}
	w.mu.Lock()
	fatalErr := w.fatalErr
	w.mu.Unlock()
	if fatalErr != nil {
		w.completeAppendErrors(batch, fatalErr)
		return
	}
	for len(batch) > 0 {
		if err := w.rotateBeforeAppend(len(batch[0].frame)); err != nil {
			w.setFatal(err)
			w.completeAppendErrors(batch, err)
			return
		}
		count := w.batchCountForCurrentSegment(batch)
		if count <= 0 {
			count = 1
		}
		group := batch[:count]
		if err := w.commitAppendGroup(group); err != nil {
			w.setFatal(err)
			w.completeAppendErrors(batch, err)
			return
		}
		batch = batch[count:]
	}
}

func (w *durableOutboxWAL) commitAppendGroup(group []outboxWALAppendRequest) error {
	start := w.current.size
	totalBytes := 0
	for _, request := range group {
		totalBytes += len(request.frame)
	}
	buffer := make([]byte, 0, totalBytes)
	for _, request := range group {
		buffer = append(buffer, request.frame...)
	}
	written, err := w.currentFile.Write(buffer)
	if err != nil || written != len(buffer) {
		if err == nil {
			err = io.ErrShortWrite
		}
		w.rollbackAppend(start)
		return fmt.Errorf("append durable outbox wal: %w", err)
	}
	if w.syncWrites {
		if err := w.currentFile.Sync(); err != nil {
			w.rollbackAppend(start)
			return fmt.Errorf("sync durable outbox wal: %w", err)
		}
	}

	w.mu.Lock()
	segment := w.current
	offset := start
	results := make([]outboxWALAppendResult, 0, len(group))
	for _, request := range group {
		state := &outboxWALRecordState{
			payloadOffset: offset + outboxWALHeaderSize,
			payloadLength: uint32(len(request.payload)),
			status:        outboxWALClaimed,
		}
		index := len(segment.records)
		segment.records = append(segment.records, state)
		w.backlog++
		results = append(results, outboxWALAppendResult{stored: storedOutboxRecord{
			Ref:    outboxWALRecordRef{segmentID: segment.id, index: index},
			Record: request.record,
		}})
		offset += int64(len(request.frame))
	}
	segment.size += int64(len(buffer))
	w.mu.Unlock()
	for index, request := range group {
		request.result <- results[index]
	}
	return nil
}

func (w *durableOutboxWAL) rollbackAppend(size int64) {
	if w.currentFile == nil {
		return
	}
	_ = w.currentFile.Truncate(size)
	if w.syncWrites {
		_ = w.currentFile.Sync()
	}
}

func (w *durableOutboxWAL) rotateBeforeAppend(frameBytes int) error {
	if w.current == nil {
		return w.createCurrentSegment()
	}
	if len(w.current.records) == 0 {
		return nil
	}
	tooLarge := w.current.size+int64(frameBytes) > w.segmentBytes
	tooOld := time.Since(w.current.createdAt) >= w.segmentAge
	if !tooLarge && !tooOld {
		return nil
	}
	return w.rotateCurrentSegment()
}

func (w *durableOutboxWAL) rotateIfAged() error {
	w.mu.Lock()
	current := w.current
	shouldRotate := current != nil && len(current.records) > 0 && time.Since(current.createdAt) >= w.segmentAge
	w.mu.Unlock()
	if !shouldRotate {
		return nil
	}
	return w.rotateCurrentSegment()
}

func (w *durableOutboxWAL) rotateCurrentSegment() error {
	if w.currentFile == nil || w.current == nil {
		return w.createCurrentSegment()
	}
	if w.syncWrites {
		if err := w.currentFile.Sync(); err != nil {
			return fmt.Errorf("sync closing durable outbox wal segment: %w", err)
		}
	}
	if err := w.currentFile.Close(); err != nil {
		return fmt.Errorf("close durable outbox wal segment: %w", err)
	}
	w.mu.Lock()
	old := w.current
	old.closed = true
	w.current = nil
	w.currentFile = nil
	deleteOld := old.acked == len(old.records) && !old.deleting
	if deleteOld {
		old.deleting = true
	}
	w.mu.Unlock()
	if deleteOld {
		if err := w.deleteAcknowledgedSegment(old); err != nil {
			return err
		}
	}
	return w.createCurrentSegment()
}

func (w *durableOutboxWAL) createCurrentSegment() error {
	w.mu.Lock()
	lastID := uint64(0)
	if len(w.segments) > 0 {
		lastID = w.segments[len(w.segments)-1].id
	}
	w.mu.Unlock()
	id := uint64(time.Now().UnixNano())
	if id <= lastID {
		id = lastID + 1
	}
	path := filepath.Join(w.dir, outboxWALFileName(id))
	file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR|os.O_APPEND, 0o640)
	if err != nil {
		return fmt.Errorf("create durable outbox wal segment: %w", err)
	}
	if w.syncWrites {
		if err := syncDirectory(w.dir); err != nil {
			_ = file.Close()
			return fmt.Errorf("sync durable outbox wal directory after create: %w", err)
		}
	}
	segment := &outboxWALSegment{id: id, path: path, createdAt: time.Now()}
	w.mu.Lock()
	w.segments = append(w.segments, segment)
	w.segmentByID[id] = segment
	w.current = segment
	w.currentFile = file
	w.mu.Unlock()
	return nil
}

func (w *durableOutboxWAL) finishWriter() {
	if w.currentFile == nil || w.current == nil {
		return
	}
	var finishErr error
	if w.syncWrites {
		finishErr = w.currentFile.Sync()
	}
	if closeErr := w.currentFile.Close(); finishErr == nil {
		finishErr = closeErr
	}
	w.mu.Lock()
	current := w.current
	current.closed = true
	w.current = nil
	w.currentFile = nil
	deleteCurrent := current.acked == len(current.records) && !current.deleting
	if deleteCurrent {
		current.deleting = true
	}
	w.mu.Unlock()
	if deleteCurrent {
		if err := w.deleteAcknowledgedSegment(current); finishErr == nil {
			finishErr = err
		}
	}
	if finishErr != nil {
		w.setFatal(finishErr)
	}
}

func (w *durableOutboxWAL) batchCountForCurrentSegment(batch []outboxWALAppendRequest) int {
	remaining := w.segmentBytes - w.current.size
	count := 0
	for _, request := range batch {
		if count > 0 && int64(len(request.frame)) > remaining {
			break
		}
		remaining -= int64(len(request.frame))
		count++
		if remaining <= 0 {
			break
		}
	}
	return count
}

func (w *durableOutboxWAL) completeAppendErrors(batch []outboxWALAppendRequest, err error) {
	for _, request := range batch {
		request.result <- outboxWALAppendResult{err: err}
	}
}

func (w *durableOutboxWAL) readClaimedRecords(refs []outboxWALRecordRef) ([]storedOutboxRecord, error) {
	files := map[uint64]*os.File{}
	defer func() {
		for _, file := range files {
			_ = file.Close()
		}
	}()
	records := make([]storedOutboxRecord, 0, len(refs))
	for _, ref := range refs {
		w.mu.Lock()
		segment := w.segmentByID[ref.segmentID]
		state := w.recordStateLocked(ref)
		w.mu.Unlock()
		if segment == nil || state == nil {
			return nil, fmt.Errorf("durable outbox wal record reference not found: segment=%d index=%d", ref.segmentID, ref.index)
		}
		file := files[segment.id]
		if file == nil {
			opened, err := os.Open(segment.path)
			if err != nil {
				return nil, fmt.Errorf("open durable outbox wal segment for replay: %w", err)
			}
			files[segment.id] = opened
			file = opened
		}
		payload := make([]byte, state.payloadLength)
		if _, err := file.ReadAt(payload, state.payloadOffset); err != nil {
			return nil, fmt.Errorf("read durable outbox wal record: %w", err)
		}
		var record durableRecord
		if err := json.Unmarshal(payload, &record); err != nil {
			return nil, fmt.Errorf("decode durable outbox wal record: %w", err)
		}
		records = append(records, storedOutboxRecord{Ref: ref, Record: record})
	}
	return records, nil
}

func (w *durableOutboxWAL) loadSegments() error {
	paths, err := filepath.Glob(filepath.Join(w.dir, "outbox-*.wal"))
	if err != nil {
		return fmt.Errorf("list durable outbox wal segments: %w", err)
	}
	sort.Strings(paths)
	for _, path := range paths {
		segment, err := loadOutboxWALSegment(path, w.syncWrites)
		if err != nil {
			return err
		}
		if len(segment.records) == 0 {
			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
				return fmt.Errorf("remove empty durable outbox wal segment: %w", err)
			}
			if w.syncWrites {
				if err := syncDirectory(w.dir); err != nil {
					return fmt.Errorf("sync durable outbox wal directory after removing empty segment: %w", err)
				}
			}
			continue
		}
		w.segments = append(w.segments, segment)
		w.segmentByID[segment.id] = segment
		w.backlog += len(segment.records)
	}
	return nil
}

func loadOutboxWALSegment(path string, syncWrites bool) (*outboxWALSegment, error) {
	id, err := parseOutboxWALFileName(filepath.Base(path))
	if err != nil {
		return nil, err
	}
	file, err := os.OpenFile(path, os.O_RDWR, 0)
	if err != nil {
		return nil, fmt.Errorf("open durable outbox wal segment: %w", err)
	}
	defer file.Close()
	info, err := file.Stat()
	if err != nil {
		return nil, fmt.Errorf("stat durable outbox wal segment: %w", err)
	}
	segment := &outboxWALSegment{
		id:        id,
		path:      path,
		createdAt: info.ModTime(),
		closed:    true,
	}
	offset := int64(0)
	header := make([]byte, outboxWALHeaderSize)
	for offset < info.Size() {
		n, readErr := file.ReadAt(header, offset)
		if readErr != nil {
			if readErr == io.EOF && n < outboxWALHeaderSize {
				if err := truncateOutboxWALTail(file, offset, syncWrites); err != nil {
					return nil, err
				}
				break
			}
			return nil, fmt.Errorf("read durable outbox wal header at %d: %w", offset, readErr)
		}
		if binary.BigEndian.Uint32(header[0:4]) != outboxWALMagic {
			return nil, fmt.Errorf("durable outbox wal corruption in %s at offset %d: invalid magic", path, offset)
		}
		length := binary.BigEndian.Uint32(header[4:8])
		checksum := binary.BigEndian.Uint32(header[8:12])
		if length == 0 || length > outboxWALMaxRecordBytes {
			return nil, fmt.Errorf("durable outbox wal corruption in %s at offset %d: invalid length %d", path, offset, length)
		}
		frameEnd := offset + outboxWALHeaderSize + int64(length)
		if frameEnd > info.Size() {
			if err := truncateOutboxWALTail(file, offset, syncWrites); err != nil {
				return nil, err
			}
			break
		}
		payload := make([]byte, length)
		if _, err := file.ReadAt(payload, offset+outboxWALHeaderSize); err != nil {
			return nil, fmt.Errorf("read durable outbox wal payload at %d: %w", offset, err)
		}
		if crc32.ChecksumIEEE(payload) != checksum {
			return nil, fmt.Errorf("durable outbox wal corruption in %s at offset %d: checksum mismatch", path, offset)
		}
		segment.records = append(segment.records, &outboxWALRecordState{
			payloadOffset: offset + outboxWALHeaderSize,
			payloadLength: length,
			status:        outboxWALPending,
		})
		offset = frameEnd
	}
	segment.size = offset
	return segment, nil
}

func truncateOutboxWALTail(file *os.File, size int64, syncWrites bool) error {
	if err := file.Truncate(size); err != nil {
		return fmt.Errorf("truncate incomplete durable outbox wal tail: %w", err)
	}
	if syncWrites {
		if err := file.Sync(); err != nil {
			return fmt.Errorf("sync truncated durable outbox wal tail: %w", err)
		}
	}
	return nil
}

func (w *durableOutboxWAL) deleteAcknowledgedSegment(segment *outboxWALSegment) error {
	err := os.Remove(segment.path)
	if os.IsNotExist(err) {
		err = nil
	}
	if err == nil && w.syncWrites {
		err = syncDirectory(w.dir)
	}
	w.mu.Lock()
	defer w.mu.Unlock()
	if err != nil {
		segment.deleting = false
		return fmt.Errorf("delete acknowledged durable outbox wal segment: %w", err)
	}
	delete(w.segmentByID, segment.id)
	for index, candidate := range w.segments {
		if candidate != segment {
			continue
		}
		w.segments = append(w.segments[:index], w.segments[index+1:]...)
		break
	}
	return nil
}

func (w *durableOutboxWAL) recordStateLocked(ref outboxWALRecordRef) *outboxWALRecordState {
	segment := w.segmentByID[ref.segmentID]
	if segment == nil || ref.index < 0 || ref.index >= len(segment.records) {
		return nil
	}
	return segment.records[ref.index]
}

func (w *durableOutboxWAL) setFatal(err error) {
	if err == nil {
		return
	}
	w.mu.Lock()
	if w.fatalErr == nil {
		w.fatalErr = err
	}
	w.mu.Unlock()
}

func encodeOutboxWALFrame(payload []byte) []byte {
	frame := make([]byte, outboxWALHeaderSize+len(payload))
	binary.BigEndian.PutUint32(frame[0:4], outboxWALMagic)
	binary.BigEndian.PutUint32(frame[4:8], uint32(len(payload)))
	binary.BigEndian.PutUint32(frame[8:12], crc32.ChecksumIEEE(payload))
	copy(frame[outboxWALHeaderSize:], payload)
	return frame
}

func outboxWALFileName(id uint64) string {
	return fmt.Sprintf("outbox-%020d.wal", id)
}

func parseOutboxWALFileName(name string) (uint64, error) {
	if !strings.HasPrefix(name, "outbox-") || !strings.HasSuffix(name, ".wal") {
		return 0, fmt.Errorf("invalid durable outbox wal file name %q", name)
	}
	value := strings.TrimSuffix(strings.TrimPrefix(name, "outbox-"), ".wal")
	id, err := strconv.ParseUint(value, 10, 64)
	if err != nil {
		return 0, fmt.Errorf("parse durable outbox wal file name %q: %w", name, err)
	}
	return id, nil
}

func syncDirectory(dir string) error {
	handle, err := os.Open(dir)
	if err != nil {
		return err
	}
	syncErr := handle.Sync()
	closeErr := handle.Close()
	return errors.Join(syncErr, closeErr)
}

func minDuration(left, right time.Duration) time.Duration {
	if left <= 0 {
		return right
	}
	if right <= 0 || left < right {
		return left
	}
	return right
}