shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(go): batch redis fast writer projections

Browse Source
This commit is contained in:
lingniu
2026-07-03 19:45:00 +08:00
parent 81e050ec1c
commit 3a0d9bd840
7 changed files with 173 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
go/vehicle-gateway/cmd/nats-fast-writer/main.go
View File

@@ -194,6 +194,10 @@ type fastUpdater interface {
FastUpdate(context.Context, envelope.FrameEnvelope) error
}
type fastBatchUpdater interface {
FastUpdateBatch(context.Context, []envelope.FrameEnvelope) error
}
type fastMessage struct {
subject string
data []byte
@@ -290,6 +294,25 @@ func processFastBatch(ctx context.Context, registry *metrics.Registry, appender
if err != nil {
return fmt.Errorf("tdengine batch append: %w", err)
}
if batchUpdater, ok := updater.(fastBatchUpdater); ok {
started = time.Now()
err = batchUpdater.FastUpdateBatch(ctx, envelopes)
recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started))
if err != nil {
return fmt.Errorf("redis fast batch update: %w", err)
}
for _, msg := range validMessages {
if msg.ack != nil {
started = time.Now()
err = msg.ack()
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
if err != nil {
return fmt.Errorf("nats ack: %w", err)
}
}
}
return nil
}
for i, env := range envelopes {
msg := validMessages[i]
started = time.Now()

58
go/vehicle-gateway/cmd/nats-fast-writer/main_test.go
View File

@@ -20,7 +20,7 @@ func TestProcessFastMessageWritesTDengineAndRedisBeforeAck(t *testing.T) {
t.Fatal(err)
}
appender := &recordingFastAppender{}
updater := &recordingFastUpdater{}
updater := &recordingFastSingleUpdater{}
ackCount := 0
msg := &fastMessage{data: payload, ack: func() error {
ackCount++
@@ -95,7 +95,7 @@ func TestProcessFastBatchAppendsTDengineBatchBeforeRedisAndAck(t *testing.T) {
t.Fatal(err)
}
appender := &recordingFastAppender{}
updater := &recordingFastUpdater{}
updater := &recordingFastSingleUpdater{}
ackCount := 0
msgs := []*fastMessage{
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }},
@@ -116,6 +116,38 @@ func TestProcessFastBatchAppendsTDengineBatchBeforeRedisAndAck(t *testing.T) {
}
}
func TestProcessFastBatchUsesRedisBatchUpdaterWhenAvailable(t *testing.T) {
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-batch-1"}
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-batch-2"}
firstPayload, err := first.MarshalJSONBytes()
if err != nil {
t.Fatal(err)
}
secondPayload, err := second.MarshalJSONBytes()
if err != nil {
t.Fatal(err)
}
updater := &recordingFastUpdater{}
ackCount := 0
msgs := []*fastMessage{
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }},
{subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }},
}
if err := processFastBatch(context.Background(), nil, &recordingFastAppender{}, updater, msgs); err != nil {
t.Fatalf("processFastBatch() error = %v", err)
}
if updater.batchCount != 1 || updater.batchRows != 2 {
t.Fatalf("FastUpdateBatch count=%d rows=%d, want count=1 rows=2", updater.batchCount, updater.batchRows)
}
if updater.count != 0 {
t.Fatalf("FastUpdate count=%d, want 0 when batch updater is available", updater.count)
}
if ackCount != 2 {
t.Fatalf("ack count=%d, want 2", ackCount)
}
}
func TestProcessFastBatchExposesPendingMetricsDuringAppend(t *testing.T) {
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-6"}
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-7"}
@@ -249,11 +281,29 @@ func (a *recordingFastAppender) AppendAllBatch(_ context.Context, envs []envelop
}
type recordingFastUpdater struct {
count int
err error
count int
batchCount int
batchRows int
err error
}
func (u *recordingFastUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error {
u.count++
return u.err
}
func (u *recordingFastUpdater) FastUpdateBatch(_ context.Context, envs []envelope.FrameEnvelope) error {
u.batchCount++
u.batchRows += len(envs)
return u.err
}
type recordingFastSingleUpdater struct {
count int
err error
}
func (u *recordingFastSingleUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error {
u.count++
return u.err
}