fix(stats): flush throttled daily mileage projections

2026-07-19 16:00:13 +08:00
parent 81532d91b7
commit fa60bbb2c8
6 changed files with 381 additions and 44 deletions
--- a/go/vehicle-gateway/cmd/stat-writer/main.go
+++ b/go/vehicle-gateway/cmd/stat-writer/main.go
@@ -95,6 +95,14 @@ func main() {
 	quarantiner := fileStatMessageQuarantiner{dir: cfg.QuarantineDir}

 	logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","), "workers", cfg.Workers, "project_interval_seconds", cfg.ProjectInterval.Seconds(), "source_touch_interval_seconds", cfg.SourceTouchInterval.Seconds(), "cache_retention_seconds", cfg.CacheRetention.Seconds(), "cache_cleanup_interval_seconds", cfg.CacheCleanupInterval.Seconds(), "baseline_miss_ttl_seconds", cfg.BaselineMissTTL.Seconds(), "baseline_hit_ttl_seconds", cfg.BaselineHitTTL.Seconds(), "cache_max_entries", cfg.CacheMaxEntries, "batch_size", cfg.BatchSize, "batch_wait_ms", cfg.BatchWait, "retry_attempts", cfg.RetryAttempts, "retry_delay_ms", cfg.RetryDelay.Milliseconds(), "quarantine_dir", cfg.QuarantineDir)
+	var background sync.WaitGroup
+	if cfg.ProjectInterval > 0 {
+		background.Add(1)
+		go func() {
+			defer background.Done()
+			runPendingProjectionFlusher(ctx, logger, registry, writer, cfg.ProjectInterval)
+		}()
+	}
 	var workers sync.WaitGroup
 	for workerID := 1; workerID <= cfg.Workers; workerID++ {
 		workers.Add(1)
@@ -104,6 +112,67 @@ func main() {
 		}(workerID)
 	}
 	workers.Wait()
+	background.Wait()
+	if cfg.ProjectInterval > 0 {
+		flushCtx, cancel := context.WithTimeout(context.Background(), kafkaMessageOperationTimeout)
+		flushPendingProjections(flushCtx, logger, registry, writer, time.Time{})
+		cancel()
+	}
+}
+
+type pendingProjectionFlusher interface {
+	FlushPendingProjections(context.Context, time.Time) (stats.ProjectionFlushResult, error)
+}
+
+func runPendingProjectionFlusher(ctx context.Context, logger interface {
+	Error(string, ...any)
+	Warn(string, ...any)
+}, registry *metrics.Registry, flusher pendingProjectionFlusher, interval time.Duration) {
+	if flusher == nil || interval <= 0 {
+		return
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case now := <-ticker.C:
+			operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout)
+			flushPendingProjections(operationCtx, logger, registry, flusher, now.Add(-interval))
+			cancel()
+		}
+	}
+}
+
+func flushPendingProjections(ctx context.Context, logger interface {
+	Error(string, ...any)
+	Warn(string, ...any)
+}, registry *metrics.Registry, flusher pendingProjectionFlusher, readyBefore time.Time) {
+	if flusher == nil {
+		return
+	}
+	result, err := flusher.FlushPendingProjections(ctx, readyBefore)
+	if registry != nil {
+		if result.Attempted > 0 {
+			registry.AddCounter("vehicle_stat_pending_projections_total", metrics.Labels{"status": "attempted"}, float64(result.Attempted))
+		}
+		if result.Written > 0 {
+			registry.AddCounter("vehicle_stat_pending_projections_total", metrics.Labels{"status": "written"}, float64(result.Written))
+			metrics.RecordLastActivity(registry, "vehicle_stat_last_pending_projection_unix_seconds", metrics.Labels{"status": "written"})
+		}
+		if result.Failed > 0 {
+			registry.AddCounter("vehicle_stat_pending_projections_total", metrics.Labels{"status": "error"}, float64(result.Failed))
+		} else if err != nil {
+			registry.IncCounter("vehicle_stat_pending_projections_total", metrics.Labels{"status": "error"})
+		}
+		if err != nil {
+			metrics.RecordLastActivity(registry, "vehicle_stat_last_pending_projection_unix_seconds", metrics.Labels{"status": "error"})
+		}
+	}
+	if err != nil && logger != nil {
+		logger.Warn("pending daily mileage projection flush failed", "attempted", result.Attempted, "written", result.Written, "failed", result.Failed, "error", err)
+	}
 }

 func runStatConsumer(ctx context.Context, logger interface {
@@ -763,6 +832,7 @@ func recordStatCacheMetrics(registry *metrics.Registry, appender statAppender) {
 	setStatCacheGauge(registry, "last_total_mileage", stats.LastTotalMileageEntries, stats.MaxEntries, stats.LastCleanupTotalMileage, stats.TotalMileageEvictions)
 	setStatCacheGauge(registry, "source_seen", stats.LastSourceSeenEntries, stats.MaxEntries, stats.LastCleanupSourceSeen, stats.SourceSeenEvictions)
 	setStatCacheGauge(registry, "projection", stats.LastProjectionEntries, stats.MaxEntries, stats.LastCleanupProjection, stats.ProjectionEvictions)
+	registry.SetGauge("vehicle_stat_pending_projection_entries", nil, float64(stats.PendingProjectionEntries))
 	setStatCacheGauge(registry, "baseline", stats.BaselineEntries, stats.MaxEntries, stats.LastCleanupBaseline, stats.BaselineEvictions)
 	if !stats.LastCleanupAt.IsZero() {
 		registry.SetGauge("vehicle_stat_cache_last_cleanup_unix_seconds", nil, float64(stats.LastCleanupAt.Unix()))
--- a/go/vehicle-gateway/cmd/stat-writer/main_test.go
+++ b/go/vehicle-gateway/cmd/stat-writer/main_test.go
@@ -105,20 +105,21 @@ func TestProcessStatMessageRecordsMetrics(t *testing.T) {
 			ProjectionsWritten:             1,
 			ProjectionsSkippedThrottled:    2,
 		}, cacheStats: stats.CacheStats{
-			LastTotalMileageEntries: 300,
-			LastSourceSeenEntries:   3,
-			LastProjectionEntries:   280,
-			BaselineEntries:         295,
-			MaxEntries:              1000000,
-			LastCleanupAt:           time.Unix(1782918600, 0),
-			LastCleanupTotalMileage: 11,
-			LastCleanupSourceSeen:   1,
-			LastCleanupProjection:   9,
-			LastCleanupBaseline:     10,
-			TotalMileageEvictions:   4,
-			SourceSeenEvictions:     1,
-			ProjectionEvictions:     2,
-			BaselineEvictions:       3,
+			LastTotalMileageEntries:  300,
+			LastSourceSeenEntries:    3,
+			LastProjectionEntries:    280,
+			PendingProjectionEntries: 12,
+			BaselineEntries:          295,
+			MaxEntries:               1000000,
+			LastCleanupAt:            time.Unix(1782918600, 0),
+			LastCleanupTotalMileage:  11,
+			LastCleanupSourceSeen:    1,
+			LastCleanupProjection:    9,
+			LastCleanupBaseline:      10,
+			TotalMileageEvictions:    4,
+			SourceSeenEvictions:      1,
+			ProjectionEvictions:      2,
+			BaselineEvictions:        3,
 		}},
 		&contextCheckingStatCommitter{},
 		kafka.Message{Topic: "vehicle.fields.go.jt808.v1", Partition: 4, Offset: 30, HighWaterMark: 41, Value: payload},
@@ -150,6 +151,7 @@ func TestProcessStatMessageRecordsMetrics(t *testing.T) {
 		`vehicle_stat_cache_entries{cache="last_total_mileage"} 300`,
 		`vehicle_stat_cache_entries{cache="source_seen"} 3`,
 		`vehicle_stat_cache_entries{cache="projection"} 280`,
+		`vehicle_stat_pending_projection_entries 12`,
 		`vehicle_stat_cache_entries{cache="baseline"} 295`,
 		`vehicle_stat_cache_max_entries{cache="last_total_mileage"} 1000000`,
 		`vehicle_stat_cache_max_entries{cache="source_seen"} 1000000`,
@@ -177,6 +179,52 @@ func TestProcessStatMessageRecordsMetrics(t *testing.T) {
 	}
 }

+func TestFlushPendingProjectionsRecordsEventualProjectionMetrics(t *testing.T) {
+	registry := metrics.NewRegistry()
+	flusher := &recordingPendingProjectionFlusher{
+		result: stats.ProjectionFlushResult{Attempted: 3, Written: 3},
+	}
+	readyBefore := time.Date(2026, 7, 19, 15, 4, 45, 0, time.Local)
+
+	flushPendingProjections(context.Background(), discardStatLogger{}, registry, flusher, readyBefore)
+
+	if flusher.calls != 1 || !flusher.readyBefore.Equal(readyBefore) {
+		t.Fatalf("flusher calls=%d readyBefore=%s, want one call at %s", flusher.calls, flusher.readyBefore, readyBefore)
+	}
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_stat_pending_projections_total{status="attempted"} 3`,
+		`vehicle_stat_pending_projections_total{status="written"} 3`,
+		`vehicle_stat_last_pending_projection_unix_seconds{status="written"} `,
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("pending projection metrics missing %s:\n%s", want, text)
+		}
+	}
+}
+
+func TestFlushPendingProjectionsRetainsErrorMetricForRetry(t *testing.T) {
+	registry := metrics.NewRegistry()
+	flusher := &recordingPendingProjectionFlusher{
+		result: stats.ProjectionFlushResult{Attempted: 3, Written: 1, Failed: 2},
+		err:    errors.New("temporary projection failure"),
+	}
+
+	flushPendingProjections(context.Background(), discardStatLogger{}, registry, flusher, time.Time{})
+
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_stat_pending_projections_total{status="attempted"} 3`,
+		`vehicle_stat_pending_projections_total{status="written"} 1`,
+		`vehicle_stat_pending_projections_total{status="error"} 2`,
+		`vehicle_stat_last_pending_projection_unix_seconds{status="error"} `,
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("pending projection error metrics missing %s:\n%s", want, text)
+		}
+	}
+}
+
 func TestRecordStatWriteE2EDurationSkipsMissingReceivedAt(t *testing.T) {
 	registry := metrics.NewRegistry()

@@ -1232,6 +1280,19 @@ type discardStatLogger struct{}
 func (discardStatLogger) Error(string, ...any) {}
 func (discardStatLogger) Warn(string, ...any)  {}

+type recordingPendingProjectionFlusher struct {
+	result      stats.ProjectionFlushResult
+	err         error
+	calls       int
+	readyBefore time.Time
+}
+
+func (f *recordingPendingProjectionFlusher) FlushPendingProjections(_ context.Context, readyBefore time.Time) (stats.ProjectionFlushResult, error) {
+	f.calls++
+	f.readyBefore = readyBefore
+	return f.result, f.err
+}
+
 var errTestStatAppend = errors.New("test stat append failed")

 func committedOffsets(messages []kafka.Message) []int64 {
--- a/go/vehicle-gateway/internal/stats/daily_metric.go
+++ b/go/vehicle-gateway/internal/stats/daily_metric.go
@@ -3,7 +3,9 @@ package stats
 import (
 	"context"
 	"database/sql"
+	"errors"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -44,6 +46,7 @@ type Writer struct {
 	lastTotalMileage       map[string]float64
 	lastSourceSeen         map[string]time.Time
 	lastProjection         map[string]projectionCacheEntry
+	pendingProjection      map[string]pendingProjectionEntry
 	baselineCache          map[string]sourceBaselineCacheEntry
 	mileageKeysByPrefix    map[string]map[string]struct{}
 	projectionKeysByPrefix map[string]map[string]struct{}
@@ -86,21 +89,28 @@ type AppendResult struct {
 	ProjectionsSkippedThrottled      int
 }

+type ProjectionFlushResult struct {
+	Attempted int
+	Written   int
+	Failed    int
+}
+
 type CacheStats struct {
-	LastTotalMileageEntries int
-	LastSourceSeenEntries   int
-	LastProjectionEntries   int
-	BaselineEntries         int
-	MaxEntries              int
-	LastCleanupAt           time.Time
-	LastCleanupTotalMileage int
-	LastCleanupSourceSeen   int
-	LastCleanupProjection   int
-	LastCleanupBaseline     int
-	TotalMileageEvictions   int
-	SourceSeenEvictions     int
-	ProjectionEvictions     int
-	BaselineEvictions       int
+	LastTotalMileageEntries  int
+	LastSourceSeenEntries    int
+	LastProjectionEntries    int
+	PendingProjectionEntries int
+	BaselineEntries          int
+	MaxEntries               int
+	LastCleanupAt            time.Time
+	LastCleanupTotalMileage  int
+	LastCleanupSourceSeen    int
+	LastCleanupProjection    int
+	LastCleanupBaseline      int
+	TotalMileageEvictions    int
+	SourceSeenEvictions      int
+	ProjectionEvictions      int
+	BaselineEvictions        int
 }

 type cacheCleanupStats struct {
@@ -130,6 +140,7 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
 		lastTotalMileage:       map[string]float64{},
 		lastSourceSeen:         map[string]time.Time{},
 		lastProjection:         map[string]projectionCacheEntry{},
+		pendingProjection:      map[string]pendingProjectionEntry{},
 		baselineCache:          map[string]sourceBaselineCacheEntry{},
 		mileageKeysByPrefix:    map[string]map[string]struct{}{},
 		projectionKeysByPrefix: map[string]map[string]struct{}{},
@@ -209,20 +220,21 @@ func (w *Writer) CacheStats() CacheStats {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	return CacheStats{
-		LastTotalMileageEntries: len(w.lastTotalMileage),
-		LastSourceSeenEntries:   len(w.lastSourceSeen),
-		LastProjectionEntries:   len(w.lastProjection),
-		BaselineEntries:         len(w.baselineCache),
-		MaxEntries:              w.maxCacheEntries,
-		LastCleanupAt:           w.lastCacheCleanup,
-		LastCleanupTotalMileage: w.lastCacheCleanupStats.totalMileage,
-		LastCleanupSourceSeen:   w.lastCacheCleanupStats.sourceSeen,
-		LastCleanupProjection:   w.lastCacheCleanupStats.projection,
-		LastCleanupBaseline:     w.lastCacheCleanupStats.baseline,
-		TotalMileageEvictions:   w.cacheEvictions.totalMileage,
-		SourceSeenEvictions:     w.cacheEvictions.sourceSeen,
-		ProjectionEvictions:     w.cacheEvictions.projection,
-		BaselineEvictions:       w.cacheEvictions.baseline,
+		LastTotalMileageEntries:  len(w.lastTotalMileage),
+		LastSourceSeenEntries:    len(w.lastSourceSeen),
+		LastProjectionEntries:    len(w.lastProjection),
+		PendingProjectionEntries: len(w.pendingProjection),
+		BaselineEntries:          len(w.baselineCache),
+		MaxEntries:               w.maxCacheEntries,
+		LastCleanupAt:            w.lastCacheCleanup,
+		LastCleanupTotalMileage:  w.lastCacheCleanupStats.totalMileage,
+		LastCleanupSourceSeen:    w.lastCacheCleanupStats.sourceSeen,
+		LastCleanupProjection:    w.lastCacheCleanupStats.projection,
+		LastCleanupBaseline:      w.lastCacheCleanupStats.baseline,
+		TotalMileageEvictions:    w.cacheEvictions.totalMileage,
+		SourceSeenEvictions:      w.cacheEvictions.sourceSeen,
+		ProjectionEvictions:      w.cacheEvictions.projection,
+		BaselineEvictions:        w.cacheEvictions.baseline,
 	}
 }

@@ -327,6 +339,8 @@ func (w *Writer) AppendWithResult(ctx context.Context, env envelope.FrameEnvelop
 		if projectDaily {
 			w.markProjected(sample)
 			result.ProjectionsWritten++
+		} else {
+			w.markProjectionPending(sample, time.Now())
 		}
 		w.markMileageWritten(sample)
 		result.SamplesWritten++
@@ -474,12 +488,94 @@ func (w *Writer) markProjected(sample MetricSample) {
 		entry.sourceKeys = map[string]struct{}{}
 	}
 	entry.sourceKeys[sample.SourceKey] = struct{}{}
-	entry.projectedAt = sample.EventTime
+	if entry.projectedAt.IsZero() || sample.EventTime.After(entry.projectedAt) {
+		entry.projectedAt = sample.EventTime
+	}
 	w.lastProjection[key] = entry
+	if pending, ok := w.pendingProjection[key]; ok && !pending.sample.EventTime.After(sample.EventTime) {
+		delete(w.pendingProjection, key)
+	}
 	w.addCacheKeyLocked(w.projectionKeysByPrefix, prefix, key)
 	w.enforceCacheLimitsLocked()
 }

+func (w *Writer) markProjectionPending(sample MetricSample, queuedAt time.Time) {
+	key := projectionCacheKey(sample)
+	prefix := projectionCachePrefix(sample)
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	current, ok := w.pendingProjection[key]
+	if !ok || sample.EventTime.After(current.sample.EventTime) ||
+		(sample.EventTime.Equal(current.sample.EventTime) && queuedAt.After(current.queuedAt)) {
+		w.pendingProjection[key] = pendingProjectionEntry{sample: sample, queuedAt: queuedAt}
+	}
+	w.addCacheKeyLocked(w.projectionKeysByPrefix, prefix, key)
+	w.enforceCacheLimitsLocked()
+}
+
+// FlushPendingProjections closes the eventual-consistency gap introduced by
+// projection throttling. Source candidates are persisted immediately, while
+// final projections are flushed after their throttle window even when a
+// vehicle stops reporting before another message can trigger projection.
+func (w *Writer) FlushPendingProjections(ctx context.Context, readyBefore time.Time) (ProjectionFlushResult, error) {
+	type pendingItem struct {
+		key   string
+		entry pendingProjectionEntry
+	}
+
+	w.mu.Lock()
+	items := make([]pendingItem, 0, len(w.pendingProjection))
+	for key, entry := range w.pendingProjection {
+		if readyBefore.IsZero() || !entry.queuedAt.After(readyBefore) {
+			items = append(items, pendingItem{key: key, entry: entry})
+		}
+	}
+	w.mu.Unlock()
+	sort.Slice(items, func(i, j int) bool {
+		return items[i].key < items[j].key
+	})
+
+	var result ProjectionFlushResult
+	var flushErr error
+	for _, item := range items {
+		if err := ctx.Err(); err != nil {
+			return result, errors.Join(flushErr, err)
+		}
+		result.Attempted++
+		if err := w.projectPendingMileage(ctx, item.entry.sample); err != nil {
+			result.Failed++
+			flushErr = errors.Join(flushErr, fmt.Errorf("project pending mileage %s: %w", item.key, err))
+			continue
+		}
+		w.markProjected(item.entry.sample)
+		result.Written++
+	}
+	return result, flushErr
+}
+
+func (w *Writer) projectPendingMileage(ctx context.Context, sample MetricSample) error {
+	project := func(exec Execer) error {
+		if strings.Contains(sample.SourceKey, platformSourceKeyPrefix) {
+			if err := NormalizePlatformSourceMileage(ctx, exec, sample.VIN, sample.StatDate, sample.Protocol); err != nil {
+				return err
+			}
+		}
+		return projectDailyMileageWithExec(ctx, exec, sample.VIN, sample.StatDate, sample.Protocol)
+	}
+	if beginner, ok := w.exec.(txBeginner); ok {
+		tx, err := beginner.BeginTx(ctx, nil)
+		if err != nil {
+			return err
+		}
+		if err := project(tx); err != nil {
+			_ = tx.Rollback()
+			return err
+		}
+		return tx.Commit()
+	}
+	return project(w.exec)
+}
+
 func (w *Writer) seenSameMileage(sample MetricSample) bool {
 	key := mileageCacheKey(sample)
 	w.mu.Lock()
@@ -719,6 +815,7 @@ func (w *Writer) deleteBaselineKeyLocked(key string) {

 func (w *Writer) deleteProjectionKeyLocked(key string) {
 	delete(w.lastProjection, key)
+	delete(w.pendingProjection, key)
 	w.deleteIndexedCacheKeyLocked(w.projectionKeysByPrefix, cacheKeyPrefix(key), key)
 }

@@ -866,6 +963,11 @@ type projectionCacheEntry struct {
 	sourceKeys  map[string]struct{}
 }

+type pendingProjectionEntry struct {
+	sample   MetricSample
+	queuedAt time.Time
+}
+
 type sourceBaselineCacheEntry struct {
 	baseline sourceBaseline
 	found    bool
--- a/go/vehicle-gateway/internal/stats/daily_metric_test.go
+++ b/go/vehicle-gateway/internal/stats/daily_metric_test.go
@@ -797,6 +797,97 @@ func TestWriterAppendThrottlesDailyProjectionPerVehicleProtocolDate(t *testing.T
 	}
 }

+func TestWriterFlushesFinalProjectionWhenVehicleStopsInsideThrottleWindow(t *testing.T) {
+	exec := &recordingExec{}
+	loc := time.FixedZone("Asia/Shanghai", 8*3600)
+	writer := NewWriter(exec, loc)
+	base := managedTestSource(envelope.FrameEnvelope{
+		Protocol:       envelope.ProtocolJT808,
+		VIN:            "LMRKH9AC7R1004120",
+		Phone:          "64341233712",
+		SourceEndpoint: "115.159.85.149:7339",
+		Fields: map[string]any{
+			"jt808.location.total_mileage_km": 4732.6,
+		},
+	})
+
+	first := base
+	first.EventTimeMS = time.Date(2026, 7, 19, 15, 4, 45, 0, loc).UnixMilli()
+	if err := writer.Append(context.Background(), first); err != nil {
+		t.Fatalf("first Append() error = %v", err)
+	}
+	final := base
+	final.EventTimeMS = time.Date(2026, 7, 19, 15, 4, 52, 0, loc).UnixMilli()
+	final.Fields = map[string]any{"jt808.location.total_mileage_km": 4740.7}
+	if err := writer.Append(context.Background(), final); err != nil {
+		t.Fatalf("final Append() error = %v", err)
+	}
+
+	if got := writer.CacheStats().PendingProjectionEntries; got != 1 {
+		t.Fatalf("pending projections = %d, want final throttled sample queued", got)
+	}
+	beforeDue, err := writer.FlushPendingProjections(context.Background(), time.Now().Add(-time.Hour))
+	if err != nil {
+		t.Fatalf("early FlushPendingProjections() error = %v", err)
+	}
+	if beforeDue.Attempted != 0 || beforeDue.Written != 0 {
+		t.Fatalf("early flush result = %+v, want no eligible projection", beforeDue)
+	}
+
+	flushed, err := writer.FlushPendingProjections(context.Background(), time.Time{})
+	if err != nil {
+		t.Fatalf("FlushPendingProjections() error = %v", err)
+	}
+	if flushed.Attempted != 1 || flushed.Written != 1 {
+		t.Fatalf("flush result = %+v, want final projection written", flushed)
+	}
+	if got := countExecQueries(exec.calls, "INSERT INTO vehicle_daily_mileage\n"); got != 2 {
+		t.Fatalf("daily projections = %d, want initial plus eventual final projection", got)
+	}
+	if got := writer.CacheStats().PendingProjectionEntries; got != 0 {
+		t.Fatalf("pending projections after flush = %d, want 0", got)
+	}
+	key := projectionCacheKey(MetricSample{
+		VIN:      "LMRKH9AC7R1004120",
+		Protocol: envelope.ProtocolJT808,
+		StatDate: "2026-07-19",
+	})
+	wantProjectedAt := time.UnixMilli(final.EventTimeMS).In(loc)
+	if got := writer.lastProjection[key].projectedAt; !got.Equal(wantProjectedAt) {
+		t.Fatalf("last projected event = %s, want %s", got, wantProjectedAt)
+	}
+}
+
+func TestWriterPendingProjectionFailureDoesNotBlockOtherVehicles(t *testing.T) {
+	exec := &selectiveFailingExec{failVIN: "FAIL-VIN"}
+	loc := time.FixedZone("Asia/Shanghai", 8*3600)
+	writer := NewWriter(exec, loc)
+	writer.projectionInterval = time.Hour
+
+	for _, vin := range []string{"FAIL-VIN", "OK-VIN"} {
+		sample := MetricSample{
+			VIN:            vin,
+			Protocol:       envelope.ProtocolJT808,
+			StatDate:       "2026-07-19",
+			SourceKey:      "jt808:test",
+			EventTime:      time.Date(2026, 7, 19, 15, 4, 52, 0, loc),
+			TotalMileageKM: 100,
+		}
+		writer.markProjectionPending(sample, time.Now().Add(-time.Hour))
+	}
+
+	result, err := writer.FlushPendingProjections(context.Background(), time.Time{})
+	if err == nil {
+		t.Fatal("FlushPendingProjections() error = nil, want failed vehicle reported")
+	}
+	if result.Attempted != 2 || result.Written != 1 || result.Failed != 1 {
+		t.Fatalf("flush result = %+v, want one failure and one successful projection", result)
+	}
+	if got := writer.CacheStats().PendingProjectionEntries; got != 1 {
+		t.Fatalf("pending projections after partial failure = %d, want only failed vehicle retained", got)
+	}
+}
+
 func TestWriterAppendProjectsImmediatelyForNewSource(t *testing.T) {
 	exec := &recordingExec{}
 	loc := time.FixedZone("Asia/Shanghai", 8*3600)
@@ -2321,6 +2412,17 @@ type recordingExec struct {
 	errs  []error
 }

+type selectiveFailingExec struct {
+	failVIN string
+}
+
+func (e *selectiveFailingExec) ExecContext(_ context.Context, _ string, args ...any) (sql.Result, error) {
+	if len(args) > 0 && fmt.Sprint(args[0]) == e.failVIN {
+		return nil, errors.New("test projection failure")
+	}
+	return nil, nil
+}
+
 type concurrentExec struct {
 	mu    sync.Mutex
 	calls int