feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
804
go/vehicle-gateway/internal/eventbus/durable_outbox_wal.go
Normal file
804
go/vehicle-gateway/internal/eventbus/durable_outbox_wal.go
Normal file
@@ -0,0 +1,804 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
outboxWALMagic = uint32(0x4c4e5731) // LNW1
|
||||
outboxWALHeaderSize = 12
|
||||
outboxWALMaxRecordBytes = 16 << 20
|
||||
defaultWALSegmentBytes = 16 << 20
|
||||
defaultWALAppendQueue = 100_000
|
||||
defaultWALCommitBatch = 256
|
||||
defaultWALCommitInterval = time.Millisecond
|
||||
defaultWALSegmentAge = 5 * time.Second
|
||||
)
|
||||
|
||||
var ErrDurableOutboxWALClosed = errors.New("durable outbox wal is closed")
|
||||
|
||||
type durableOutboxWALConfig struct {
|
||||
Directory string
|
||||
SyncWrites bool
|
||||
SegmentBytes int64
|
||||
SegmentAge time.Duration
|
||||
AppendQueue int
|
||||
CommitBatch int
|
||||
CommitInterval time.Duration
|
||||
}
|
||||
|
||||
type outboxWALRecordRef struct {
|
||||
segmentID uint64
|
||||
index int
|
||||
}
|
||||
|
||||
type storedOutboxRecord struct {
|
||||
Ref outboxWALRecordRef
|
||||
Record durableRecord
|
||||
}
|
||||
|
||||
type outboxWALRecordStatus uint8
|
||||
|
||||
const (
|
||||
outboxWALPending outboxWALRecordStatus = iota
|
||||
outboxWALClaimed
|
||||
outboxWALAcknowledged
|
||||
)
|
||||
|
||||
type outboxWALRecordState struct {
|
||||
payloadOffset int64
|
||||
payloadLength uint32
|
||||
status outboxWALRecordStatus
|
||||
}
|
||||
|
||||
type outboxWALSegment struct {
|
||||
id uint64
|
||||
path string
|
||||
createdAt time.Time
|
||||
size int64
|
||||
closed bool
|
||||
deleting bool
|
||||
acked int
|
||||
records []*outboxWALRecordState
|
||||
}
|
||||
|
||||
type outboxWALAppendRequest struct {
|
||||
record durableRecord
|
||||
payload []byte
|
||||
frame []byte
|
||||
result chan outboxWALAppendResult
|
||||
}
|
||||
|
||||
type outboxWALAppendResult struct {
|
||||
stored storedOutboxRecord
|
||||
err error
|
||||
}
|
||||
|
||||
type durableOutboxWAL struct {
|
||||
dir string
|
||||
syncWrites bool
|
||||
segmentBytes int64
|
||||
segmentAge time.Duration
|
||||
commitBatch int
|
||||
commitInterval time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
segments []*outboxWALSegment
|
||||
segmentByID map[uint64]*outboxWALSegment
|
||||
current *outboxWALSegment
|
||||
currentFile *os.File
|
||||
backlog int
|
||||
fatalErr error
|
||||
|
||||
appendMu sync.RWMutex
|
||||
closed bool
|
||||
queue chan outboxWALAppendRequest
|
||||
writerWG sync.WaitGroup
|
||||
closeOne sync.Once
|
||||
}
|
||||
|
||||
func newDurableOutboxWAL(cfg durableOutboxWALConfig) (*durableOutboxWAL, error) {
|
||||
dir := strings.TrimSpace(cfg.Directory)
|
||||
if dir == "" {
|
||||
return nil, errors.New("durable outbox wal directory is required")
|
||||
}
|
||||
if cfg.SegmentBytes <= 0 {
|
||||
cfg.SegmentBytes = defaultWALSegmentBytes
|
||||
}
|
||||
if cfg.SegmentAge <= 0 {
|
||||
cfg.SegmentAge = defaultWALSegmentAge
|
||||
}
|
||||
if cfg.AppendQueue <= 0 {
|
||||
cfg.AppendQueue = defaultWALAppendQueue
|
||||
}
|
||||
if cfg.CommitBatch <= 0 {
|
||||
cfg.CommitBatch = defaultWALCommitBatch
|
||||
}
|
||||
if cfg.CommitInterval <= 0 {
|
||||
cfg.CommitInterval = defaultWALCommitInterval
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return nil, fmt.Errorf("create durable outbox wal directory: %w", err)
|
||||
}
|
||||
w := &durableOutboxWAL{
|
||||
dir: dir,
|
||||
syncWrites: cfg.SyncWrites,
|
||||
segmentBytes: cfg.SegmentBytes,
|
||||
segmentAge: cfg.SegmentAge,
|
||||
commitBatch: cfg.CommitBatch,
|
||||
commitInterval: cfg.CommitInterval,
|
||||
segmentByID: map[uint64]*outboxWALSegment{},
|
||||
queue: make(chan outboxWALAppendRequest, cfg.AppendQueue),
|
||||
}
|
||||
if err := w.loadSegments(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := w.createCurrentSegment(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.writerWG.Add(1)
|
||||
go w.appendLoop()
|
||||
return w, nil
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) Append(ctx context.Context, record durableRecord) (storedOutboxRecord, error) {
|
||||
payload, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
return storedOutboxRecord{}, fmt.Errorf("marshal durable outbox wal record: %w", err)
|
||||
}
|
||||
if len(payload) > outboxWALMaxRecordBytes {
|
||||
return storedOutboxRecord{}, fmt.Errorf("durable outbox wal record is %d bytes, max %d", len(payload), outboxWALMaxRecordBytes)
|
||||
}
|
||||
request := outboxWALAppendRequest{
|
||||
record: record,
|
||||
payload: payload,
|
||||
frame: encodeOutboxWALFrame(payload),
|
||||
result: make(chan outboxWALAppendResult, 1),
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
w.appendMu.RLock()
|
||||
if w.closed {
|
||||
w.appendMu.RUnlock()
|
||||
return storedOutboxRecord{}, ErrDurableOutboxWALClosed
|
||||
}
|
||||
select {
|
||||
case w.queue <- request:
|
||||
w.appendMu.RUnlock()
|
||||
case <-ctx.Done():
|
||||
w.appendMu.RUnlock()
|
||||
return storedOutboxRecord{}, ctx.Err()
|
||||
}
|
||||
// Once admitted to the WAL queue, wait for the durability result even if
|
||||
// the caller context is cancelled. Otherwise a committed record could be
|
||||
// left claimed with no publisher responsible for it.
|
||||
result := <-request.result
|
||||
return result.stored, result.err
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) ClaimPending(limit int) ([]storedOutboxRecord, error) {
|
||||
if limit <= 0 {
|
||||
limit = defaultWALCommitBatch
|
||||
}
|
||||
w.mu.Lock()
|
||||
refs := make([]outboxWALRecordRef, 0, limit)
|
||||
for _, segment := range w.segments {
|
||||
for index, state := range segment.records {
|
||||
if state.status != outboxWALPending {
|
||||
continue
|
||||
}
|
||||
state.status = outboxWALClaimed
|
||||
refs = append(refs, outboxWALRecordRef{segmentID: segment.id, index: index})
|
||||
if len(refs) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(refs) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
w.mu.Unlock()
|
||||
if len(refs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
records, err := w.readClaimedRecords(refs)
|
||||
if err != nil {
|
||||
for _, ref := range refs {
|
||||
w.Release(ref)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) Release(ref outboxWALRecordRef) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
state := w.recordStateLocked(ref)
|
||||
if state != nil && state.status == outboxWALClaimed {
|
||||
state.status = outboxWALPending
|
||||
}
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) Ack(ref outboxWALRecordRef) error {
|
||||
w.mu.Lock()
|
||||
segment := w.segmentByID[ref.segmentID]
|
||||
if segment == nil || ref.index < 0 || ref.index >= len(segment.records) {
|
||||
w.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
state := segment.records[ref.index]
|
||||
if state.status == outboxWALAcknowledged {
|
||||
w.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
state.status = outboxWALAcknowledged
|
||||
segment.acked++
|
||||
if w.backlog > 0 {
|
||||
w.backlog--
|
||||
}
|
||||
shouldDelete := segment.closed && segment.acked == len(segment.records) && !segment.deleting
|
||||
if shouldDelete {
|
||||
segment.deleting = true
|
||||
}
|
||||
w.mu.Unlock()
|
||||
if !shouldDelete {
|
||||
return nil
|
||||
}
|
||||
return w.deleteAcknowledgedSegment(segment)
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) Stats() (backlog int, oldest time.Time) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
backlog = w.backlog
|
||||
if backlog == 0 {
|
||||
return backlog, time.Time{}
|
||||
}
|
||||
for _, segment := range w.segments {
|
||||
if segment.acked < len(segment.records) {
|
||||
return backlog, segment.createdAt
|
||||
}
|
||||
}
|
||||
return backlog, time.Time{}
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) Close() error {
|
||||
w.closeOne.Do(func() {
|
||||
w.appendMu.Lock()
|
||||
w.closed = true
|
||||
close(w.queue)
|
||||
w.appendMu.Unlock()
|
||||
w.writerWG.Wait()
|
||||
})
|
||||
w.mu.Lock()
|
||||
err := w.fatalErr
|
||||
w.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) appendLoop() {
|
||||
defer w.writerWG.Done()
|
||||
maintenanceInterval := minDuration(w.segmentAge/2, time.Second)
|
||||
if maintenanceInterval < 10*time.Millisecond {
|
||||
maintenanceInterval = 10 * time.Millisecond
|
||||
}
|
||||
maintenance := time.NewTicker(maintenanceInterval)
|
||||
defer maintenance.Stop()
|
||||
for {
|
||||
select {
|
||||
case request, ok := <-w.queue:
|
||||
if !ok {
|
||||
w.finishWriter()
|
||||
return
|
||||
}
|
||||
batch := w.collectAppendBatch(request)
|
||||
w.commitAppendBatch(batch)
|
||||
case <-maintenance.C:
|
||||
if err := w.rotateIfAged(); err != nil {
|
||||
w.setFatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) collectAppendBatch(first outboxWALAppendRequest) []outboxWALAppendRequest {
|
||||
batch := make([]outboxWALAppendRequest, 0, w.commitBatch)
|
||||
batch = append(batch, first)
|
||||
timer := time.NewTimer(w.commitInterval)
|
||||
defer timer.Stop()
|
||||
for len(batch) < w.commitBatch {
|
||||
select {
|
||||
case request, ok := <-w.queue:
|
||||
if !ok {
|
||||
return batch
|
||||
}
|
||||
batch = append(batch, request)
|
||||
case <-timer.C:
|
||||
return batch
|
||||
}
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) commitAppendBatch(batch []outboxWALAppendRequest) {
|
||||
if len(batch) == 0 {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
fatalErr := w.fatalErr
|
||||
w.mu.Unlock()
|
||||
if fatalErr != nil {
|
||||
w.completeAppendErrors(batch, fatalErr)
|
||||
return
|
||||
}
|
||||
for len(batch) > 0 {
|
||||
if err := w.rotateBeforeAppend(len(batch[0].frame)); err != nil {
|
||||
w.setFatal(err)
|
||||
w.completeAppendErrors(batch, err)
|
||||
return
|
||||
}
|
||||
count := w.batchCountForCurrentSegment(batch)
|
||||
if count <= 0 {
|
||||
count = 1
|
||||
}
|
||||
group := batch[:count]
|
||||
if err := w.commitAppendGroup(group); err != nil {
|
||||
w.setFatal(err)
|
||||
w.completeAppendErrors(batch, err)
|
||||
return
|
||||
}
|
||||
batch = batch[count:]
|
||||
}
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) commitAppendGroup(group []outboxWALAppendRequest) error {
|
||||
start := w.current.size
|
||||
totalBytes := 0
|
||||
for _, request := range group {
|
||||
totalBytes += len(request.frame)
|
||||
}
|
||||
buffer := make([]byte, 0, totalBytes)
|
||||
for _, request := range group {
|
||||
buffer = append(buffer, request.frame...)
|
||||
}
|
||||
written, err := w.currentFile.Write(buffer)
|
||||
if err != nil || written != len(buffer) {
|
||||
if err == nil {
|
||||
err = io.ErrShortWrite
|
||||
}
|
||||
w.rollbackAppend(start)
|
||||
return fmt.Errorf("append durable outbox wal: %w", err)
|
||||
}
|
||||
if w.syncWrites {
|
||||
if err := w.currentFile.Sync(); err != nil {
|
||||
w.rollbackAppend(start)
|
||||
return fmt.Errorf("sync durable outbox wal: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
w.mu.Lock()
|
||||
segment := w.current
|
||||
offset := start
|
||||
results := make([]outboxWALAppendResult, 0, len(group))
|
||||
for _, request := range group {
|
||||
state := &outboxWALRecordState{
|
||||
payloadOffset: offset + outboxWALHeaderSize,
|
||||
payloadLength: uint32(len(request.payload)),
|
||||
status: outboxWALClaimed,
|
||||
}
|
||||
index := len(segment.records)
|
||||
segment.records = append(segment.records, state)
|
||||
w.backlog++
|
||||
results = append(results, outboxWALAppendResult{stored: storedOutboxRecord{
|
||||
Ref: outboxWALRecordRef{segmentID: segment.id, index: index},
|
||||
Record: request.record,
|
||||
}})
|
||||
offset += int64(len(request.frame))
|
||||
}
|
||||
segment.size += int64(len(buffer))
|
||||
w.mu.Unlock()
|
||||
for index, request := range group {
|
||||
request.result <- results[index]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) rollbackAppend(size int64) {
|
||||
if w.currentFile == nil {
|
||||
return
|
||||
}
|
||||
_ = w.currentFile.Truncate(size)
|
||||
if w.syncWrites {
|
||||
_ = w.currentFile.Sync()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) rotateBeforeAppend(frameBytes int) error {
|
||||
if w.current == nil {
|
||||
return w.createCurrentSegment()
|
||||
}
|
||||
if len(w.current.records) == 0 {
|
||||
return nil
|
||||
}
|
||||
tooLarge := w.current.size+int64(frameBytes) > w.segmentBytes
|
||||
tooOld := time.Since(w.current.createdAt) >= w.segmentAge
|
||||
if !tooLarge && !tooOld {
|
||||
return nil
|
||||
}
|
||||
return w.rotateCurrentSegment()
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) rotateIfAged() error {
|
||||
w.mu.Lock()
|
||||
current := w.current
|
||||
shouldRotate := current != nil && len(current.records) > 0 && time.Since(current.createdAt) >= w.segmentAge
|
||||
w.mu.Unlock()
|
||||
if !shouldRotate {
|
||||
return nil
|
||||
}
|
||||
return w.rotateCurrentSegment()
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) rotateCurrentSegment() error {
|
||||
if w.currentFile == nil || w.current == nil {
|
||||
return w.createCurrentSegment()
|
||||
}
|
||||
if w.syncWrites {
|
||||
if err := w.currentFile.Sync(); err != nil {
|
||||
return fmt.Errorf("sync closing durable outbox wal segment: %w", err)
|
||||
}
|
||||
}
|
||||
if err := w.currentFile.Close(); err != nil {
|
||||
return fmt.Errorf("close durable outbox wal segment: %w", err)
|
||||
}
|
||||
w.mu.Lock()
|
||||
old := w.current
|
||||
old.closed = true
|
||||
w.current = nil
|
||||
w.currentFile = nil
|
||||
deleteOld := old.acked == len(old.records) && !old.deleting
|
||||
if deleteOld {
|
||||
old.deleting = true
|
||||
}
|
||||
w.mu.Unlock()
|
||||
if deleteOld {
|
||||
if err := w.deleteAcknowledgedSegment(old); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return w.createCurrentSegment()
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) createCurrentSegment() error {
|
||||
w.mu.Lock()
|
||||
lastID := uint64(0)
|
||||
if len(w.segments) > 0 {
|
||||
lastID = w.segments[len(w.segments)-1].id
|
||||
}
|
||||
w.mu.Unlock()
|
||||
id := uint64(time.Now().UnixNano())
|
||||
if id <= lastID {
|
||||
id = lastID + 1
|
||||
}
|
||||
path := filepath.Join(w.dir, outboxWALFileName(id))
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR|os.O_APPEND, 0o640)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create durable outbox wal segment: %w", err)
|
||||
}
|
||||
if w.syncWrites {
|
||||
if err := syncDirectory(w.dir); err != nil {
|
||||
_ = file.Close()
|
||||
return fmt.Errorf("sync durable outbox wal directory after create: %w", err)
|
||||
}
|
||||
}
|
||||
segment := &outboxWALSegment{id: id, path: path, createdAt: time.Now()}
|
||||
w.mu.Lock()
|
||||
w.segments = append(w.segments, segment)
|
||||
w.segmentByID[id] = segment
|
||||
w.current = segment
|
||||
w.currentFile = file
|
||||
w.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) finishWriter() {
|
||||
if w.currentFile == nil || w.current == nil {
|
||||
return
|
||||
}
|
||||
var finishErr error
|
||||
if w.syncWrites {
|
||||
finishErr = w.currentFile.Sync()
|
||||
}
|
||||
if closeErr := w.currentFile.Close(); finishErr == nil {
|
||||
finishErr = closeErr
|
||||
}
|
||||
w.mu.Lock()
|
||||
current := w.current
|
||||
current.closed = true
|
||||
w.current = nil
|
||||
w.currentFile = nil
|
||||
deleteCurrent := current.acked == len(current.records) && !current.deleting
|
||||
if deleteCurrent {
|
||||
current.deleting = true
|
||||
}
|
||||
w.mu.Unlock()
|
||||
if deleteCurrent {
|
||||
if err := w.deleteAcknowledgedSegment(current); finishErr == nil {
|
||||
finishErr = err
|
||||
}
|
||||
}
|
||||
if finishErr != nil {
|
||||
w.setFatal(finishErr)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) batchCountForCurrentSegment(batch []outboxWALAppendRequest) int {
|
||||
remaining := w.segmentBytes - w.current.size
|
||||
count := 0
|
||||
for _, request := range batch {
|
||||
if count > 0 && int64(len(request.frame)) > remaining {
|
||||
break
|
||||
}
|
||||
remaining -= int64(len(request.frame))
|
||||
count++
|
||||
if remaining <= 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) completeAppendErrors(batch []outboxWALAppendRequest, err error) {
|
||||
for _, request := range batch {
|
||||
request.result <- outboxWALAppendResult{err: err}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) readClaimedRecords(refs []outboxWALRecordRef) ([]storedOutboxRecord, error) {
|
||||
files := map[uint64]*os.File{}
|
||||
defer func() {
|
||||
for _, file := range files {
|
||||
_ = file.Close()
|
||||
}
|
||||
}()
|
||||
records := make([]storedOutboxRecord, 0, len(refs))
|
||||
for _, ref := range refs {
|
||||
w.mu.Lock()
|
||||
segment := w.segmentByID[ref.segmentID]
|
||||
state := w.recordStateLocked(ref)
|
||||
w.mu.Unlock()
|
||||
if segment == nil || state == nil {
|
||||
return nil, fmt.Errorf("durable outbox wal record reference not found: segment=%d index=%d", ref.segmentID, ref.index)
|
||||
}
|
||||
file := files[segment.id]
|
||||
if file == nil {
|
||||
opened, err := os.Open(segment.path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open durable outbox wal segment for replay: %w", err)
|
||||
}
|
||||
files[segment.id] = opened
|
||||
file = opened
|
||||
}
|
||||
payload := make([]byte, state.payloadLength)
|
||||
if _, err := file.ReadAt(payload, state.payloadOffset); err != nil {
|
||||
return nil, fmt.Errorf("read durable outbox wal record: %w", err)
|
||||
}
|
||||
var record durableRecord
|
||||
if err := json.Unmarshal(payload, &record); err != nil {
|
||||
return nil, fmt.Errorf("decode durable outbox wal record: %w", err)
|
||||
}
|
||||
records = append(records, storedOutboxRecord{Ref: ref, Record: record})
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) loadSegments() error {
|
||||
paths, err := filepath.Glob(filepath.Join(w.dir, "outbox-*.wal"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("list durable outbox wal segments: %w", err)
|
||||
}
|
||||
sort.Strings(paths)
|
||||
for _, path := range paths {
|
||||
segment, err := loadOutboxWALSegment(path, w.syncWrites)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(segment.records) == 0 {
|
||||
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("remove empty durable outbox wal segment: %w", err)
|
||||
}
|
||||
if w.syncWrites {
|
||||
if err := syncDirectory(w.dir); err != nil {
|
||||
return fmt.Errorf("sync durable outbox wal directory after removing empty segment: %w", err)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
w.segments = append(w.segments, segment)
|
||||
w.segmentByID[segment.id] = segment
|
||||
w.backlog += len(segment.records)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadOutboxWALSegment(path string, syncWrites bool) (*outboxWALSegment, error) {
|
||||
id, err := parseOutboxWALFileName(filepath.Base(path))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file, err := os.OpenFile(path, os.O_RDWR, 0)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open durable outbox wal segment: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat durable outbox wal segment: %w", err)
|
||||
}
|
||||
segment := &outboxWALSegment{
|
||||
id: id,
|
||||
path: path,
|
||||
createdAt: info.ModTime(),
|
||||
closed: true,
|
||||
}
|
||||
offset := int64(0)
|
||||
header := make([]byte, outboxWALHeaderSize)
|
||||
for offset < info.Size() {
|
||||
n, readErr := file.ReadAt(header, offset)
|
||||
if readErr != nil {
|
||||
if readErr == io.EOF && n < outboxWALHeaderSize {
|
||||
if err := truncateOutboxWALTail(file, offset, syncWrites); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
return nil, fmt.Errorf("read durable outbox wal header at %d: %w", offset, readErr)
|
||||
}
|
||||
if binary.BigEndian.Uint32(header[0:4]) != outboxWALMagic {
|
||||
return nil, fmt.Errorf("durable outbox wal corruption in %s at offset %d: invalid magic", path, offset)
|
||||
}
|
||||
length := binary.BigEndian.Uint32(header[4:8])
|
||||
checksum := binary.BigEndian.Uint32(header[8:12])
|
||||
if length == 0 || length > outboxWALMaxRecordBytes {
|
||||
return nil, fmt.Errorf("durable outbox wal corruption in %s at offset %d: invalid length %d", path, offset, length)
|
||||
}
|
||||
frameEnd := offset + outboxWALHeaderSize + int64(length)
|
||||
if frameEnd > info.Size() {
|
||||
if err := truncateOutboxWALTail(file, offset, syncWrites); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
payload := make([]byte, length)
|
||||
if _, err := file.ReadAt(payload, offset+outboxWALHeaderSize); err != nil {
|
||||
return nil, fmt.Errorf("read durable outbox wal payload at %d: %w", offset, err)
|
||||
}
|
||||
if crc32.ChecksumIEEE(payload) != checksum {
|
||||
return nil, fmt.Errorf("durable outbox wal corruption in %s at offset %d: checksum mismatch", path, offset)
|
||||
}
|
||||
segment.records = append(segment.records, &outboxWALRecordState{
|
||||
payloadOffset: offset + outboxWALHeaderSize,
|
||||
payloadLength: length,
|
||||
status: outboxWALPending,
|
||||
})
|
||||
offset = frameEnd
|
||||
}
|
||||
segment.size = offset
|
||||
return segment, nil
|
||||
}
|
||||
|
||||
func truncateOutboxWALTail(file *os.File, size int64, syncWrites bool) error {
|
||||
if err := file.Truncate(size); err != nil {
|
||||
return fmt.Errorf("truncate incomplete durable outbox wal tail: %w", err)
|
||||
}
|
||||
if syncWrites {
|
||||
if err := file.Sync(); err != nil {
|
||||
return fmt.Errorf("sync truncated durable outbox wal tail: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) deleteAcknowledgedSegment(segment *outboxWALSegment) error {
|
||||
err := os.Remove(segment.path)
|
||||
if os.IsNotExist(err) {
|
||||
err = nil
|
||||
}
|
||||
if err == nil && w.syncWrites {
|
||||
err = syncDirectory(w.dir)
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if err != nil {
|
||||
segment.deleting = false
|
||||
return fmt.Errorf("delete acknowledged durable outbox wal segment: %w", err)
|
||||
}
|
||||
delete(w.segmentByID, segment.id)
|
||||
for index, candidate := range w.segments {
|
||||
if candidate != segment {
|
||||
continue
|
||||
}
|
||||
w.segments = append(w.segments[:index], w.segments[index+1:]...)
|
||||
break
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) recordStateLocked(ref outboxWALRecordRef) *outboxWALRecordState {
|
||||
segment := w.segmentByID[ref.segmentID]
|
||||
if segment == nil || ref.index < 0 || ref.index >= len(segment.records) {
|
||||
return nil
|
||||
}
|
||||
return segment.records[ref.index]
|
||||
}
|
||||
|
||||
func (w *durableOutboxWAL) setFatal(err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
if w.fatalErr == nil {
|
||||
w.fatalErr = err
|
||||
}
|
||||
w.mu.Unlock()
|
||||
}
|
||||
|
||||
func encodeOutboxWALFrame(payload []byte) []byte {
|
||||
frame := make([]byte, outboxWALHeaderSize+len(payload))
|
||||
binary.BigEndian.PutUint32(frame[0:4], outboxWALMagic)
|
||||
binary.BigEndian.PutUint32(frame[4:8], uint32(len(payload)))
|
||||
binary.BigEndian.PutUint32(frame[8:12], crc32.ChecksumIEEE(payload))
|
||||
copy(frame[outboxWALHeaderSize:], payload)
|
||||
return frame
|
||||
}
|
||||
|
||||
func outboxWALFileName(id uint64) string {
|
||||
return fmt.Sprintf("outbox-%020d.wal", id)
|
||||
}
|
||||
|
||||
func parseOutboxWALFileName(name string) (uint64, error) {
|
||||
if !strings.HasPrefix(name, "outbox-") || !strings.HasSuffix(name, ".wal") {
|
||||
return 0, fmt.Errorf("invalid durable outbox wal file name %q", name)
|
||||
}
|
||||
value := strings.TrimSuffix(strings.TrimPrefix(name, "outbox-"), ".wal")
|
||||
id, err := strconv.ParseUint(value, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse durable outbox wal file name %q: %w", name, err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func syncDirectory(dir string) error {
|
||||
handle, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
syncErr := handle.Sync()
|
||||
closeErr := handle.Close()
|
||||
return errors.Join(syncErr, closeErr)
|
||||
}
|
||||
|
||||
func minDuration(left, right time.Duration) time.Duration {
|
||||
if left <= 0 {
|
||||
return right
|
||||
}
|
||||
if right <= 0 || left < right {
|
||||
return left
|
||||
}
|
||||
return right
|
||||
}
|
||||
Reference in New Issue
Block a user