feat(platform): harden telemetry pipeline and unify Semi UI workspaces

This commit is contained in:
lingniu
2026-07-18 00:26:36 +08:00
parent 65b4e4f055
commit 159c80b0ae
136 changed files with 21616 additions and 1785 deletions

View File

@@ -6,6 +6,7 @@ RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/gateway ./cmd/gateway \
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/feichi-bridge ./cmd/feichi-bridge \
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/history-writer ./cmd/history-writer \
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/stat-writer ./cmd/stat-writer \
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/realtime-api ./cmd/realtime-api
@@ -18,6 +19,7 @@ RUN apt-get update \
WORKDIR /app
COPY --from=build /out/gateway /app/gateway
COPY --from=build /out/feichi-bridge /app/feichi-bridge
COPY --from=build /out/history-writer /app/history-writer
COPY --from=build /out/stat-writer /app/stat-writer
COPY --from=build /out/realtime-api /app/realtime-api

View File

@@ -0,0 +1,211 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/feichibridge"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/health"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
)
type authSecret struct {
Username string `json:"username"`
Password string `json:"password"`
}
type targetSecret struct {
PlatformID string `json:"platformId"`
Username string `json:"username"`
Password string `json:"password"`
}
type config struct {
BaseURL string
AuthSecretFile string
TargetSecret string
TargetAddress string
StateFile string
HealthAddress string
HTTPTimeout time.Duration
TargetTimeout time.Duration
OCRImage string
OCRTimeout time.Duration
LoginAttempts int
LoginRetryDelay time.Duration
Service feichibridge.ServiceConfig
}
func main() {
logger := observability.NewLogger("feichi-bridge")
cfg, err := loadConfig()
if err != nil {
logger.Error("load configuration failed", "error", err)
os.Exit(1)
}
auth, err := readJSONSecret[authSecret](cfg.AuthSecretFile)
if err != nil {
logger.Error("read Feichi API secret failed", "error", err)
os.Exit(1)
}
credentials, err := readJSONSecret[targetSecret](cfg.TargetSecret)
if err != nil {
logger.Error("read GB/T 32960 target secret failed", "error", err)
os.Exit(1)
}
source, err := feichibridge.NewAuthenticatedAPIClient(
cfg.BaseURL,
feichibridge.LoginCredentials{
Username: auth.Username, Password: auth.Password,
MaxAttempts: cfg.LoginAttempts, RetryDelay: cfg.LoginRetryDelay,
},
feichibridge.DockerCaptchaSolver{Image: cfg.OCRImage, Timeout: cfg.OCRTimeout},
cfg.HTTPTimeout,
)
if err != nil {
logger.Error("build Feichi API client failed", "error", err)
os.Exit(1)
}
state, err := feichibridge.OpenStateStore(cfg.StateFile)
if err != nil {
logger.Error("open bridge state failed", "error", err)
os.Exit(1)
}
target, err := feichibridge.NewTarget(feichibridge.TargetConfig{
Address: cfg.TargetAddress, PlatformID: credentials.PlatformID,
Username: credentials.Username, Password: credentials.Password,
Timeout: cfg.TargetTimeout,
}, state)
if err != nil {
logger.Error("build GB/T 32960 target failed", "error", err)
os.Exit(1)
}
registry := metrics.NewRegistry()
service, err := feichibridge.NewService(cfg.Service, source, target, state, logger, registry)
if err != nil {
logger.Error("build bridge service failed", "error", err)
os.Exit(1)
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
health.Start(ctx, logger, health.NewServer(cfg.HealthAddress, "feichi-bridge", []health.Check{
{Name: "bridge", Check: service.Ready},
}, registry))
logger.Info("Feichi bridge starting",
"base_url", cfg.BaseURL,
"target_address", cfg.TargetAddress,
"poll_interval", cfg.Service.PollInterval,
"backfill_enabled", cfg.Service.BackfillEnabled,
)
if strings.HasPrefix(strings.ToLower(cfg.BaseURL), "http://") {
logger.Warn("Feichi source uses clear-text HTTP; deploy only through a controlled egress path")
}
if err := service.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
logger.Error("Feichi bridge stopped unexpectedly", "error", err)
os.Exit(1)
}
logger.Info("Feichi bridge stopped")
}
func loadConfig() (config, error) {
cfg := config{
BaseURL: strings.TrimSpace(os.Getenv("FEICHI_BASE_URL")),
AuthSecretFile: env("FEICHI_AUTH_SECRET_FILE", strings.TrimSpace(os.Getenv("FEICHI_AUTH_HEADERS_FILE"))),
TargetSecret: strings.TrimSpace(os.Getenv("FEICHI_TARGET_SECRET_FILE")),
TargetAddress: env("FEICHI_TARGET_ADDR", "127.0.0.1:32960"),
StateFile: env("FEICHI_STATE_FILE", "/var/lib/lingniu-feichi-bridge/state.json"),
HealthAddress: env("HEALTH_ADDR", "127.0.0.1:20219"),
HTTPTimeout: seconds("FEICHI_HTTP_TIMEOUT_SECONDS", 15),
TargetTimeout: seconds("FEICHI_TARGET_TIMEOUT_SECONDS", 10),
OCRImage: env("FEICHI_OCR_IMAGE", "lingniu/feichi-captcha-ocr:1.0.0"),
OCRTimeout: seconds("FEICHI_OCR_TIMEOUT_SECONDS", 20),
LoginAttempts: envInt("FEICHI_LOGIN_MAX_ATTEMPTS", 20),
LoginRetryDelay: seconds("FEICHI_LOGIN_RETRY_SECONDS", 1),
Service: feichibridge.ServiceConfig{
PollInterval: seconds("FEICHI_POLL_INTERVAL_SECONDS", 10),
DiscoveryInterval: seconds("FEICHI_DISCOVERY_INTERVAL_SECONDS", 300),
BackfillInterval: seconds("FEICHI_BACKFILL_INTERVAL_SECONDS", 3600),
BackfillLookback: seconds("FEICHI_BACKFILL_LOOKBACK_SECONDS", 3600),
BackfillWindow: seconds("FEICHI_BACKFILL_WINDOW_SECONDS", 1200),
BackfillSafetyLag: seconds("FEICHI_BACKFILL_SAFETY_SECONDS", 30),
SourceStaleAfter: seconds("FEICHI_SOURCE_STALE_SECONDS", 120),
FetchConcurrency: envInt("FEICHI_FETCH_CONCURRENCY", 4),
BackfillEnabled: envBool("FEICHI_BACKFILL_ENABLED", true),
StaleReissueEnabled: envBool("FEICHI_STALE_REISSUE_ENABLED", true),
},
}
var missing []string
if cfg.BaseURL == "" {
missing = append(missing, "FEICHI_BASE_URL")
}
if cfg.AuthSecretFile == "" {
missing = append(missing, "FEICHI_AUTH_SECRET_FILE")
}
if cfg.TargetSecret == "" {
missing = append(missing, "FEICHI_TARGET_SECRET_FILE")
}
if len(missing) > 0 {
return config{}, fmt.Errorf("required configuration missing: %s", strings.Join(missing, ", "))
}
if cfg.Service.BackfillWindow > 24*time.Hour {
return config{}, errors.New("FEICHI_BACKFILL_WINDOW_SECONDS must not exceed 86400")
}
return cfg, nil
}
func readJSONSecret[T any](path string) (T, error) {
var value T
encoded, err := os.ReadFile(path)
if err != nil {
return value, err
}
if err := json.Unmarshal(encoded, &value); err != nil {
return value, fmt.Errorf("decode %s: %w", path, err)
}
return value, nil
}
func env(name, fallback string) string {
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
return value
}
return fallback
}
func envInt(name string, fallback int) int {
value := strings.TrimSpace(os.Getenv(name))
if value == "" {
return fallback
}
parsed, err := strconv.Atoi(value)
if err != nil || parsed <= 0 {
return fallback
}
return parsed
}
func envBool(name string, fallback bool) bool {
value := strings.TrimSpace(os.Getenv(name))
if value == "" {
return fallback
}
parsed, err := strconv.ParseBool(value)
if err != nil {
return fallback
}
return parsed
}
func seconds(name string, fallback int) time.Duration {
return time.Duration(envInt(name, fallback)) * time.Second
}

View File

@@ -8,6 +8,7 @@ import (
"fmt"
"os"
"os/signal"
"path/filepath"
"sort"
"strconv"
"strings"
@@ -54,6 +55,7 @@ func main() {
registry.SetGauge("vehicle_stat_cache_retention_seconds", nil, cfg.CacheRetention.Seconds())
registry.SetGauge("vehicle_stat_cache_cleanup_interval_seconds", nil, cfg.CacheCleanupInterval.Seconds())
registry.SetGauge("vehicle_stat_baseline_miss_ttl_seconds", nil, cfg.BaselineMissTTL.Seconds())
registry.SetGauge("vehicle_stat_baseline_hit_ttl_seconds", nil, cfg.BaselineHitTTL.Seconds())
registry.SetGauge("vehicle_stat_config", metrics.Labels{"setting": "workers"}, float64(cfg.Workers))
health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "vehicle-stat-writer", []health.Check{
{Name: "mysql", Check: db.PingContext},
@@ -65,6 +67,7 @@ func main() {
writer.SetCacheRetention(cfg.CacheRetention)
writer.SetCacheCleanupInterval(cfg.CacheCleanupInterval)
writer.SetBaselineMissTTL(cfg.BaselineMissTTL)
writer.SetBaselineHitTTL(cfg.BaselineHitTTL)
writer.SetMaxCacheEntries(cfg.CacheMaxEntries)
if cfg.EnsureSchema {
if err := writer.EnsureSchema(ctx); err != nil {
@@ -89,14 +92,15 @@ func main() {
delay: cfg.RetryDelay,
registry: registry,
}
quarantiner := fileStatMessageQuarantiner{dir: cfg.QuarantineDir}
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","), "workers", cfg.Workers, "project_interval_seconds", cfg.ProjectInterval.Seconds(), "source_touch_interval_seconds", cfg.SourceTouchInterval.Seconds(), "cache_retention_seconds", cfg.CacheRetention.Seconds(), "cache_cleanup_interval_seconds", cfg.CacheCleanupInterval.Seconds(), "baseline_miss_ttl_seconds", cfg.BaselineMissTTL.Seconds(), "cache_max_entries", cfg.CacheMaxEntries, "batch_size", cfg.BatchSize, "batch_wait_ms", cfg.BatchWait, "retry_attempts", cfg.RetryAttempts, "retry_delay_ms", cfg.RetryDelay.Milliseconds())
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","), "workers", cfg.Workers, "project_interval_seconds", cfg.ProjectInterval.Seconds(), "source_touch_interval_seconds", cfg.SourceTouchInterval.Seconds(), "cache_retention_seconds", cfg.CacheRetention.Seconds(), "cache_cleanup_interval_seconds", cfg.CacheCleanupInterval.Seconds(), "baseline_miss_ttl_seconds", cfg.BaselineMissTTL.Seconds(), "baseline_hit_ttl_seconds", cfg.BaselineHitTTL.Seconds(), "cache_max_entries", cfg.CacheMaxEntries, "batch_size", cfg.BatchSize, "batch_wait_ms", cfg.BatchWait, "retry_attempts", cfg.RetryAttempts, "retry_delay_ms", cfg.RetryDelay.Milliseconds(), "quarantine_dir", cfg.QuarantineDir)
var workers sync.WaitGroup
for workerID := 1; workerID <= cfg.Workers; workerID++ {
workers.Add(1)
go func(id int) {
defer workers.Done()
runStatConsumer(ctx, logger, registry, appender, cfg, id)
runStatConsumer(ctx, logger, registry, appender, quarantiner, cfg, id)
}(workerID)
}
workers.Wait()
@@ -105,7 +109,7 @@ func main() {
func runStatConsumer(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
}, registry *metrics.Registry, appender statAppender, cfg config, workerID int) {
}, registry *metrics.Registry, appender statAppender, quarantiner statMessageQuarantiner, cfg config, workerID int) {
reader := kafka.NewReader(kafka.ReaderConfig{
Brokers: cfg.KafkaBrokers,
GroupID: cfg.KafkaGroup,
@@ -129,7 +133,7 @@ func runStatConsumer(ctx context.Context, logger interface {
continue
}
batch := collectStatBatch(ctx, reader, message, cfg.BatchSize, time.Duration(cfg.BatchWait)*time.Millisecond)
processStatBatchReliablyForWorker(ctx, logger, registry, appender, reader, batch, cfg.RetryDelay, workerLabels)
processStatBatchReliablyForWorker(ctx, logger, registry, appender, reader, quarantiner, batch, cfg.RetryDelay, workerLabels)
}
}
@@ -155,6 +159,10 @@ type kafkaMessageFetcher interface {
FetchMessage(context.Context) (kafka.Message, error)
}
type statMessageQuarantiner interface {
Quarantine(context.Context, kafka.Message, error) error
}
type statBatchItem struct {
message kafka.Message
processed bool
@@ -163,6 +171,8 @@ type statBatchItem struct {
type statBatchOutcome struct {
commitMessages []kafka.Message
retryMessages []kafka.Message
failedMessage *kafka.Message
writeErr error
}
var statWriteDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
@@ -251,7 +261,12 @@ func processStatBatchForWorker(ctx context.Context, logger interface {
addStatMetric(registry, "vehicle_stat_writes_total", message, "error")
logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
committed, commitErr := commitStatProcessedPrefixAfterFailure(messageCtx, logger, registry, committer, items)
outcome := statBatchOutcome{retryMessages: eventbus.MessagesAfterCommittedPrefixes(messages, committed)}
failedMessage := message
outcome := statBatchOutcome{
retryMessages: eventbus.MessagesAfterCommittedPrefixes(messages, committed),
failedMessage: &failedMessage,
writeErr: err,
}
if commitErr != nil {
outcome.commitMessages = committed
}
@@ -279,13 +294,13 @@ func processStatBatchReliably(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) {
processStatBatchReliablyForWorker(ctx, logger, registry, appender, committer, messages, retryDelay, nil)
processStatBatchReliablyForWorker(ctx, logger, registry, appender, committer, nil, messages, retryDelay, nil)
}
func processStatBatchReliablyForWorker(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration, workerLabels metrics.Labels) {
}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, quarantiner statMessageQuarantiner, messages []kafka.Message, retryDelay time.Duration, workerLabels metrics.Labels) {
defer registry.SetGauge("vehicle_stat_retry_pending_messages", workerLabels, 0)
pending := messages
for len(pending) > 0 {
@@ -297,6 +312,19 @@ func processStatBatchReliablyForWorker(ctx context.Context, logger interface {
}
}
pending = outcome.retryMessages
if outcome.writeErr != nil && outcome.failedMessage != nil && isQuarantinableStatError(outcome.writeErr) && quarantiner != nil {
failed := *outcome.failedMessage
if err := quarantiner.Quarantine(ctx, failed, outcome.writeErr); err != nil {
logger.Error("stat message quarantine failed", "topic", failed.Topic, "partition", failed.Partition, "offset", failed.Offset, "error", err)
registry.IncCounter("vehicle_stat_quarantine_total", metrics.Labels{"status": "error", "topic": failed.Topic})
} else {
registry.IncCounter("vehicle_stat_quarantine_total", metrics.Labels{"status": "ok", "topic": failed.Topic})
logger.Warn("quarantined permanent stat write failure", "topic", failed.Topic, "partition", failed.Partition, "offset", failed.Offset, "error", outcome.writeErr)
if retryStatCommit(ctx, logger, registry, committer, []kafka.Message{failed}, retryDelay) {
pending = messagesExceptStatMessage(pending, failed)
}
}
}
registry.SetGauge("vehicle_stat_retry_pending_messages", workerLabels, float64(len(pending)))
if len(pending) == 0 {
return
@@ -308,6 +336,105 @@ func processStatBatchReliablyForWorker(ctx context.Context, logger interface {
}
}
type quarantinedStatMessage struct {
QuarantinedAt string `json:"quarantined_at"`
Topic string `json:"topic"`
Partition int `json:"partition"`
Offset int64 `json:"offset"`
Key []byte `json:"key,omitempty"`
Value []byte `json:"value"`
Error string `json:"error"`
}
type fileStatMessageQuarantiner struct {
dir string
}
func (q fileStatMessageQuarantiner) Quarantine(ctx context.Context, message kafka.Message, cause error) error {
if err := ctx.Err(); err != nil {
return err
}
dir := strings.TrimSpace(q.dir)
if dir == "" {
return errors.New("stat quarantine directory is empty")
}
if err := os.MkdirAll(dir, 0o750); err != nil {
return fmt.Errorf("create quarantine directory: %w", err)
}
record := quarantinedStatMessage{
QuarantinedAt: time.Now().UTC().Format(time.RFC3339Nano),
Topic: message.Topic,
Partition: message.Partition,
Offset: message.Offset,
Key: message.Key,
Value: message.Value,
Error: cause.Error(),
}
payload, err := json.Marshal(record)
if err != nil {
return fmt.Errorf("marshal quarantine record: %w", err)
}
name := fmt.Sprintf("%s-%d-%d.json", sanitizeStatQuarantineName(message.Topic), message.Partition, message.Offset)
temporary, err := os.CreateTemp(dir, "."+name+"-*")
if err != nil {
return fmt.Errorf("create quarantine record: %w", err)
}
temporaryName := temporary.Name()
removeTemporary := true
defer func() {
_ = temporary.Close()
if removeTemporary {
_ = os.Remove(temporaryName)
}
}()
if err := temporary.Chmod(0o600); err != nil {
return fmt.Errorf("secure quarantine record: %w", err)
}
if _, err := temporary.Write(payload); err != nil {
return fmt.Errorf("write quarantine record: %w", err)
}
if err := temporary.Sync(); err != nil {
return fmt.Errorf("sync quarantine record: %w", err)
}
if err := temporary.Close(); err != nil {
return fmt.Errorf("close quarantine record: %w", err)
}
if err := os.Rename(temporaryName, filepath.Join(dir, name)); err != nil {
return fmt.Errorf("publish quarantine record: %w", err)
}
removeTemporary = false
return nil
}
func sanitizeStatQuarantineName(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return "unknown-topic"
}
var out strings.Builder
for _, r := range value {
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' {
out.WriteRune(r)
} else {
out.WriteByte('_')
}
}
return out.String()
}
func messagesExceptStatMessage(messages []kafka.Message, excluded kafka.Message) []kafka.Message {
out := make([]kafka.Message, 0, len(messages))
removed := false
for _, message := range messages {
if !removed && message.Topic == excluded.Topic && message.Partition == excluded.Partition && message.Offset == excluded.Offset {
removed = true
continue
}
out = append(out, message)
}
return out
}
func retryStatCommit(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
@@ -447,7 +574,7 @@ func (a retryStatAppender) AppendWithResult(ctx context.Context, env envelope.Fr
var err error
for attempt := 1; attempt <= attempts; attempt++ {
result, err = appendStatEnvelope(ctx, a.delegate, env)
if err == nil || !isTransientMySQLStatError(err) {
if err == nil || (!isTransientMySQLStatError(err) && !isQuarantinableStatError(err)) {
return result, err
}
if attempt == attempts {
@@ -512,6 +639,32 @@ func isTransientMySQLStatError(err error) bool {
strings.Contains(text, "no route to host")
}
func isQuarantinableStatError(err error) bool {
if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return false
}
text := strings.ToLower(strings.TrimSpace(err.Error()))
// Only deterministic row-data failures may advance past a Kafka message
// after the configured finite retries. Schema, permission and unknown
// application errors remain failure-closed so a systemic outage cannot
// silently quarantine an entire stream.
return strings.Contains(text, "error 1048") ||
strings.Contains(text, "cannot be null") ||
strings.Contains(text, "error 1264") ||
strings.Contains(text, "out of range value") ||
strings.Contains(text, "error 1265") ||
strings.Contains(text, "data truncated") ||
strings.Contains(text, "error 1292") ||
strings.Contains(text, "incorrect datetime value") ||
strings.Contains(text, "error 1366") ||
strings.Contains(text, "incorrect decimal value") ||
strings.Contains(text, "incorrect integer value") ||
strings.Contains(text, "error 1406") ||
strings.Contains(text, "data too long for column") ||
strings.Contains(text, "error 3819") ||
strings.Contains(text, "check constraint")
}
func addStatMetric(registry *metrics.Registry, name string, message kafka.Message, status string) {
if registry == nil {
return
@@ -686,6 +839,7 @@ type config struct {
CacheRetention time.Duration
CacheCleanupInterval time.Duration
BaselineMissTTL time.Duration
BaselineHitTTL time.Duration
CacheMaxEntries int
Workers int
BatchSize int
@@ -694,6 +848,7 @@ type config struct {
RetryDelay time.Duration
NormalizePlatformSourcesOnStart bool
NormalizePlatformSourcesTimeout time.Duration
QuarantineDir string
}
func (c config) Validate() error {
@@ -728,6 +883,7 @@ func loadConfig() config {
CacheRetention: time.Duration(envInt("STATS_CACHE_RETENTION_HOURS", 72)) * time.Hour,
CacheCleanupInterval: time.Duration(envInt("STATS_CACHE_CLEANUP_INTERVAL_SECONDS", 600)) * time.Second,
BaselineMissTTL: time.Duration(envInt("STATS_BASELINE_MISS_TTL_SECONDS", 60)) * time.Second,
BaselineHitTTL: time.Duration(envInt("STATS_BASELINE_HIT_TTL_SECONDS", 300)) * time.Second,
CacheMaxEntries: envInt("STATS_CACHE_MAX_ENTRIES", 1000000),
Workers: envInt("STATS_WORKERS", 3),
BatchSize: envInt("STATS_BATCH_SIZE", 200),
@@ -736,6 +892,7 @@ func loadConfig() config {
RetryDelay: time.Duration(envInt("STATS_RETRY_DELAY_MS", 20)) * time.Millisecond,
NormalizePlatformSourcesOnStart: env("STATS_NORMALIZE_PLATFORM_SOURCES_ON_START", "true") != "false",
NormalizePlatformSourcesTimeout: time.Duration(envInt("STATS_NORMALIZE_PLATFORM_SOURCES_TIMEOUT_SECONDS", 30)) * time.Second,
QuarantineDir: env("STATS_QUARANTINE_DIR", "/var/lib/lingniu-go-native/stat-writer-quarantine"),
}
}

View File

@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -664,6 +666,72 @@ func TestProcessStatBatchReliablyRetriesFailedSuffix(t *testing.T) {
}
}
func TestProcessStatBatchReliablyQuarantinesPermanentFailureAndContinues(t *testing.T) {
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
payload := marshalStatTestFieldsEnvelope(t, env)
appender := &contextCheckingStatAppender{failOnCount: 1, err: errors.New("Error 1048 (23000): Column 'daily_mileage_km' cannot be null")}
committer := &contextCheckingStatCommitter{}
quarantiner := &recordingStatQuarantiner{}
registry := metrics.NewRegistry()
messages := []kafka.Message{
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload},
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 11, Value: payload},
}
processStatBatchReliablyForWorker(
context.Background(),
discardStatLogger{},
registry,
appender,
committer,
quarantiner,
messages,
time.Nanosecond,
nil,
)
if appender.count != 2 {
t.Fatalf("append attempts = %d, want failed message once and following message once", appender.count)
}
if quarantiner.count != 1 || quarantiner.message.Offset != 10 {
t.Fatalf("quarantine = %d/%d, want one message at offset 10", quarantiner.count, quarantiner.message.Offset)
}
if got := committedOffsets(committer.messages); len(got) != 2 || got[0] != 10 || got[1] != 11 {
t.Fatalf("committed offsets = %#v, want [10 11]", got)
}
text := registry.Render()
if !strings.Contains(text, `vehicle_stat_quarantine_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`) {
t.Fatalf("quarantine success metric missing:\n%s", text)
}
}
func TestFileStatMessageQuarantinerPersistsOriginalKafkaMessage(t *testing.T) {
dir := t.TempDir()
message := kafka.Message{
Topic: "vehicle.fields.go.jt808.v1",
Partition: 2,
Offset: 5094052,
Key: []byte("vehicle-key"),
Value: []byte(`{"event_id":"poison:fields"}`),
}
wantErr := errors.New("permanent mysql error")
if err := (fileStatMessageQuarantiner{dir: dir}).Quarantine(context.Background(), message, wantErr); err != nil {
t.Fatalf("Quarantine() error = %v", err)
}
payload, err := os.ReadFile(filepath.Join(dir, "vehicle.fields.go.jt808.v1-2-5094052.json"))
if err != nil {
t.Fatalf("read quarantine record: %v", err)
}
var record quarantinedStatMessage
if err := json.Unmarshal(payload, &record); err != nil {
t.Fatalf("unmarshal quarantine record: %v", err)
}
if record.Topic != message.Topic || record.Partition != message.Partition || record.Offset != message.Offset || string(record.Key) != string(message.Key) || string(record.Value) != string(message.Value) || record.Error != wantErr.Error() {
t.Fatalf("quarantine record = %#v, want original message and error", record)
}
}
func TestProcessStatBatchReliablyRetriesCommitWithoutReappending(t *testing.T) {
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
payload := marshalStatTestFieldsEnvelope(t, env)
@@ -826,6 +894,30 @@ func TestRetryStatAppenderRecordsExhaustedTransientError(t *testing.T) {
}
}
func TestRetryStatAppenderRetriesQuarantinableDataErrorBeforeIsolation(t *testing.T) {
registry := metrics.NewRegistry()
delegate := &contextCheckingStatAppender{err: errors.New("Error 1048 (23000): Column 'daily_mileage_km' cannot be null")}
appender := retryStatAppender{delegate: delegate, attempts: 3, registry: registry}
_, err := appender.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
if err == nil {
t.Fatal("AppendWithResult() error = nil, want exhausted data error")
}
if delegate.count != 3 {
t.Fatalf("append attempts = %d, want configured finite retries", delegate.count)
}
text := registry.Render()
for _, want := range []string{
`vehicle_stat_write_retries_total{operation="single",status="retry"} 2`,
`vehicle_stat_write_retries_total{operation="single",status="exhausted"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("retry metric missing %s:\n%s", want, text)
}
}
}
func TestRetryStatAppenderDoesNotRetryNonTransientError(t *testing.T) {
wantErr := errors.New("validation failed")
delegate := &contextCheckingStatAppender{err: wantErr}
@@ -841,6 +933,34 @@ func TestRetryStatAppenderDoesNotRetryNonTransientError(t *testing.T) {
}
}
func TestIsQuarantinableStatErrorOnlyAcceptsDeterministicRowDataFailures(t *testing.T) {
for _, err := range []error{
errors.New("Error 1048 (23000): Column 'daily_mileage_km' cannot be null"),
errors.New("Error 1264: Out of range value for column 'daily_mileage_km'"),
errors.New("Error 1265: Data truncated for column 'daily_mileage_km'"),
errors.New("Error 1292: Incorrect datetime value"),
errors.New("Error 1366: Incorrect decimal value"),
errors.New("Error 1406: Data too long for column"),
errors.New("Error 3819: Check constraint is violated"),
} {
if !isQuarantinableStatError(err) {
t.Fatalf("isQuarantinableStatError(%q) = false, want true", err.Error())
}
}
for _, err := range []error{
errors.New("Error 1054: Unknown column 'daily_mileage_km'"),
errors.New("Error 1142: INSERT command denied"),
errors.New("validation failed"),
context.Canceled,
context.DeadlineExceeded,
nil,
} {
if isQuarantinableStatError(err) {
t.Fatalf("isQuarantinableStatError(%v) = true, want false", err)
}
}
}
func TestIsTransientMySQLStatError(t *testing.T) {
for _, err := range []error{
errors.New("dial tcp 127.0.0.1:3306: connection refused"),
@@ -903,6 +1023,9 @@ func TestLoadConfigDefaultsToGoFieldsTopics(t *testing.T) {
if cfg.BaselineMissTTL != time.Minute {
t.Fatalf("BaselineMissTTL = %s, want 1m", cfg.BaselineMissTTL)
}
if cfg.BaselineHitTTL != 5*time.Minute {
t.Fatalf("BaselineHitTTL = %s, want 5m", cfg.BaselineHitTTL)
}
if cfg.RetryAttempts != 3 {
t.Fatalf("RetryAttempts = %d, want 3", cfg.RetryAttempts)
}
@@ -965,6 +1088,7 @@ func TestLoadConfigSetsStatWriteIntervals(t *testing.T) {
t.Setenv("STATS_CACHE_RETENTION_HOURS", "48")
t.Setenv("STATS_CACHE_CLEANUP_INTERVAL_SECONDS", "300")
t.Setenv("STATS_BASELINE_MISS_TTL_SECONDS", "45")
t.Setenv("STATS_BASELINE_HIT_TTL_SECONDS", "180")
t.Setenv("STATS_CACHE_MAX_ENTRIES", "12345")
cfg := loadConfig()
@@ -984,6 +1108,9 @@ func TestLoadConfigSetsStatWriteIntervals(t *testing.T) {
if cfg.BaselineMissTTL != 45*time.Second {
t.Fatalf("BaselineMissTTL = %s, want 45s", cfg.BaselineMissTTL)
}
if cfg.BaselineHitTTL != 3*time.Minute {
t.Fatalf("BaselineHitTTL = %s, want 3m", cfg.BaselineHitTTL)
}
if cfg.CacheMaxEntries != 12345 {
t.Fatalf("CacheMaxEntries = %d, want 12345", cfg.CacheMaxEntries)
}
@@ -997,6 +1124,7 @@ func TestLoadConfigReadsStatBatchSettings(t *testing.T) {
t.Setenv("STATS_RETRY_DELAY_MS", "33")
t.Setenv("STATS_NORMALIZE_PLATFORM_SOURCES_ON_START", "false")
t.Setenv("STATS_NORMALIZE_PLATFORM_SOURCES_TIMEOUT_SECONDS", "17")
t.Setenv("STATS_QUARANTINE_DIR", "/tmp/stat-writer-quarantine-test")
cfg := loadConfig()
@@ -1021,6 +1149,9 @@ func TestLoadConfigReadsStatBatchSettings(t *testing.T) {
if cfg.NormalizePlatformSourcesTimeout != 17*time.Second {
t.Fatalf("NormalizePlatformSourcesTimeout = %s, want 17s", cfg.NormalizePlatformSourcesTimeout)
}
if cfg.QuarantineDir != "/tmp/stat-writer-quarantine-test" {
t.Fatalf("QuarantineDir = %q, want env override", cfg.QuarantineDir)
}
}
type contextCheckingStatAppender struct {
@@ -1070,6 +1201,18 @@ type contextCheckingStatCommitter struct {
failOnCount int
}
type recordingStatQuarantiner struct {
count int
message kafka.Message
err error
}
func (q *recordingStatQuarantiner) Quarantine(_ context.Context, message kafka.Message, _ error) error {
q.count++
q.message = message
return q.err
}
func (c *contextCheckingStatCommitter) CommitMessages(ctx context.Context, messages ...kafka.Message) error {
c.ctxErr = ctx.Err()
c.count++

View File

@@ -36,6 +36,7 @@ type config struct {
Debug bool
EventTimeFullScan bool
ProgressEvery int64
BaselineLookback int
Location *time.Location
}
@@ -50,6 +51,7 @@ type rawFrameRow struct {
EventTimeMS int64
ReceivedAtMS int64
ParsedJSON string
RawText string
}
type metricAgg struct {
@@ -188,6 +190,9 @@ func main() {
}
}
fields := fieldsForStats(row.Protocol, row.VIN, text)
if len(fields) == 0 && row.Protocol == envelope.ProtocolYutongMQTT {
fields = fieldsForStats(row.Protocol, row.VIN, row.RawText)
}
if cfg.Debug && scanned <= 5 {
slog.Info("debug raw frame",
"protocol", row.Protocol,
@@ -427,7 +432,7 @@ func buildLastDiffAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB,
return aggregates, nil
}
func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, _ *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
if aggregates == nil {
return 0, fmt.Errorf("aggregates map is nil")
}
@@ -441,11 +446,6 @@ func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB,
continue
}
latestHistory := map[sourceHistoryID]dailySourceLast{}
preWindow, err := queryPreviousLastSourceRows(ctx, tdDB, cfg, protocol, targetDates[0])
if err != nil {
return added, err
}
rememberLatestSourceRows(latestHistory, preWindow)
aggregateRowsByDate := indexAggregateSourceRowsByDate(aggregates, protocol)
for _, date := range targetDates {
current, err := queryRealtimeLocationLastRows(ctx, mysqlDB, cfg, protocol, date)
@@ -656,7 +656,9 @@ func queryDailyLastSourceRows(ctx context.Context, db *sql.DB, cfg config, proto
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
where = append(where, predicate)
}
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, FIRST(event_time), FIRST(parsed_json), LAST(event_time), LAST(parsed_json), COUNT(*)
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
FIRST(event_time), FIRST(parsed_json), FIRST(raw_text),
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
FROM %s.raw_frames
WHERE %s
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
@@ -668,7 +670,7 @@ func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, pr
// LAST aggregates the complete pre-window history once per source so empty
// calendar days do not make a backfill fall back to the current day's first
// sample.
where := backfillBeforePredicates(date)
where := backfillBeforePredicates(cfg, date)
where = append(where,
"parse_status = 'OK'",
"vin IS NOT NULL",
@@ -679,17 +681,31 @@ func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, pr
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
where = append(where, predicate)
}
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, LAST(event_time), LAST(parsed_json), COUNT(*)
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
FROM %s.raw_frames
WHERE %s
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
return querySourceRows(ctx, db, cfg, protocol, sqlText, false)
}
func backfillBeforePredicates(eventDateExclusive string) []string {
return []string{
func backfillBeforePredicates(cfg config, eventDateExclusive string) []string {
lookbackDays := cfg.BaselineLookback
if lookbackDays <= 0 {
lookbackDays = 7
}
eventDateFrom := shiftDate(eventDateExclusive, -lookbackDays)
where := []string{
fmt.Sprintf("event_time >= '%s 00:00:00'", quote(eventDateFrom)),
fmt.Sprintf("event_time < '%s 00:00:00'", quote(eventDateExclusive)),
}
if !cfg.EventTimeFullScan {
where = append([]string{
fmt.Sprintf("ts >= '%s 00:00:00'", quote(previousDate(eventDateFrom))),
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(eventDateExclusive))),
}, where...)
}
return where
}
func queryRealtimeLocationLastRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
@@ -775,27 +791,30 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
var sourceEndpoint string
var firstTS time.Time
var firstParsedJSON string
var firstRawText string
var ts time.Time
var parsedJSON string
var rawText string
var rawSampleCount int64
if includeFirst {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &firstTS, &firstParsedJSON, &ts, &parsedJSON, &rawSampleCount); err != nil {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &firstTS, &firstParsedJSON, &firstRawText, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
return nil, err
}
} else {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &ts, &parsedJSON, &rawSampleCount); err != nil {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
return nil, err
}
firstTS = ts
firstParsedJSON = parsedJSON
firstRawText = rawText
}
latestTotalKM, ok := mileageFromParsed(protocol, vin, parsedJSON, ts, cfg.Location)
latestTotalKM, ok := mileageFromEvidence(protocol, vin, parsedJSON, rawText, ts, cfg.Location)
if !ok {
continue
}
firstTotalKM := latestTotalKM
if includeFirst {
if parsedFirst, ok := mileageFromParsed(protocol, vin, firstParsedJSON, firstTS, cfg.Location); ok {
if parsedFirst, ok := mileageFromEvidence(protocol, vin, firstParsedJSON, firstRawText, firstTS, cfg.Location); ok {
firstTotalKM = parsedFirst
}
}
@@ -852,6 +871,16 @@ func mileageFromParsed(protocol envelope.Protocol, vin string, parsedJSON string
return samples[0].TotalMileageKM, true
}
func mileageFromEvidence(protocol envelope.Protocol, vin string, parsedJSON string, rawText string, eventTime time.Time, loc *time.Location) (float64, bool) {
if totalKM, ok := mileageFromParsed(protocol, vin, parsedJSON, eventTime, loc); ok {
return totalKM, true
}
if protocol != envelope.ProtocolYutongMQTT || strings.TrimSpace(rawText) == "" {
return 0, false
}
return mileageFromParsed(protocol, vin, rawText, eventTime, loc)
}
func normalizedSourceKey(protocol string, phone string, deviceID string, endpoint string) string {
return normalizedSourceKeyForSource(protocol, phone, deviceID, endpoint, "", "")
}
@@ -951,6 +980,7 @@ func loadConfig() (config, error) {
Debug: envBool("BACKFILL_DEBUG", false),
EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false),
ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
BaselineLookback: envInt("BACKFILL_BASELINE_LOOKBACK_DAYS", 7),
Location: loc,
}, nil
}
@@ -1000,7 +1030,7 @@ func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, err
where = append(where, "protocol IN ("+strings.Join(quoted, ",")+")")
}
where = append(where, realtimeMileageFramePredicate())
sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json
sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json, raw_text
FROM %s.raw_frames
WHERE %s
ORDER BY event_time ASC, ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
@@ -1044,6 +1074,9 @@ func mileageBearingFramePredicate(protocol envelope.Protocol) string {
conditions := make([]string, 0, len(tokens))
for _, token := range tokens {
conditions = append(conditions, fmt.Sprintf("parsed_json LIKE '%%%s%%'", quote(token)))
if protocol == envelope.ProtocolYutongMQTT {
conditions = append(conditions, fmt.Sprintf("raw_text LIKE '%%%s%%'", quote(token)))
}
}
return "(" + strings.Join(conditions, " OR ") + ")"
}
@@ -1071,10 +1104,10 @@ func mileageSearchTokens(protocol envelope.Protocol) []string {
}
func scanRawFrame(rows *sql.Rows) (rawFrameRow, error) {
var protocol, vin, phone, deviceID, sourceEndpoint, eventID, parsed string
var protocol, vin, phone, deviceID, sourceEndpoint, eventID, parsed, rawText string
var messageID int64
var eventTime, receivedAt time.Time
if err := rows.Scan(&protocol, &vin, &phone, &deviceID, &sourceEndpoint, &eventID, &messageID, &eventTime, &receivedAt, &parsed); err != nil {
if err := rows.Scan(&protocol, &vin, &phone, &deviceID, &sourceEndpoint, &eventID, &messageID, &eventTime, &receivedAt, &parsed, &rawText); err != nil {
return rawFrameRow{}, err
}
return rawFrameRow{
@@ -1088,6 +1121,7 @@ func scanRawFrame(rows *sql.Rows) (rawFrameRow, error) {
EventTimeMS: eventTime.UnixMilli(),
ReceivedAtMS: receivedAt.UnixMilli(),
ParsedJSON: parsed,
RawText: rawText,
}, nil
}
@@ -1267,6 +1301,14 @@ func previousDate(date string) string {
return parsed.AddDate(0, 0, -1).Format("2006-01-02")
}
func shiftDate(date string, days int) string {
parsed, err := time.Parse("2006-01-02", date)
if err != nil {
return date
}
return parsed.AddDate(0, 0, days).Format("2006-01-02")
}
func dateRangeWithPrevious(from string, to string) ([]string, error) {
dates, err := dateRange(from, to)
if err != nil {

View File

@@ -233,23 +233,23 @@ func TestBuildLastDiffAggregatesCarriesNearestHistoryAcrossEmptyDays(t *testing.
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
currentRows := func() *sqlmock.Rows {
return sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
})
}
previousRows := func() *sqlmock.Rows {
return sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
})
}
mock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(previousRows())
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, int64(4)))
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", int64(4)))
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-10 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows())
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-11 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, int64(6)))
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", int64(6)))
aggregates, err := buildLastDiffAggregates(context.Background(), nil, tdDB, config{
TDengineDatabase: "lingniu_vehicle_ts",
@@ -342,18 +342,10 @@ func TestAddRealtimeLocationFallbackAggregatesUsesHistoricalBaseline(t *testing.
WithArgs("YUTONG_MQTT", "2026-07-12", "2026-07-12").
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}).
AddRow("LMRKH9AC0R1004086", "mqtt://yutong/ytforward/shln/4", currentTS, 11578.0))
tdMock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-12 00:00:00'.*protocol = 'YUTONG_MQTT'.*TOTAL_MILEAGE").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}).AddRow(
"LMRKH9AC0R1004086",
"",
"LMRKH9AC0R1004086",
"mqtt://yutong/ytforward/shln/4",
previousTS,
`{"data":{"TOTAL_MILEAGE":11500000}}`,
int64(12),
))
sourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong")
mysqlMock.ExpectQuery("(?s)SELECT latest_total_mileage_km, latest_event_time.*FROM vehicle_daily_mileage_source").
WithArgs("LMRKH9AC0R1004086", "2026-07-12", "YUTONG_MQTT", sourceKey).
WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}).AddRow(11500.0, previousTS))
aggregates := map[string]*metricAgg{}
added, err := addRealtimeLocationFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
@@ -373,7 +365,7 @@ func TestAddRealtimeLocationFallbackAggregatesUsesHistoricalBaseline(t *testing.
if agg.FirstKM != 11500 || agg.LatestKM != 11578 {
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
}
if agg.SourceKey != stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong") {
if agg.SourceKey != sourceKey {
t.Fatalf("source key = %q", agg.SourceKey)
}
if agg.SourceCode != "yutong" || agg.PlatformName != "宇通" || agg.SourceKind != "PLATFORM" {
@@ -410,10 +402,6 @@ func TestAddRealtimeLocationFallbackReusesAggregateHistoryAcrossEmptyDays(t *tes
dayOneTS := time.Date(2026, 7, 9, 23, 50, 0, 0, loc)
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
tdMock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}))
for _, date := range []string{"2026-07-09", "2026-07-10"} {
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?").
WithArgs("YUTONG_MQTT", date, date).
@@ -483,16 +471,18 @@ func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
lastTS := time.Date(2026, 7, 8, 18, 0, 0, 0, loc)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
}).AddRow(
"LMRKH9AC6R1004108",
"",
"LMRKH9AC6R1004108",
"mqtt://yutong/ytforward/shln/3",
firstTS,
`{"yutong_mqtt.data.total_mileage":"65422000"}`,
`{"yutong_mqtt.data.latitude":"30.0"}`,
`{"data":{"TOTAL_MILEAGE":65422000}}`,
lastTS,
`{"yutong_mqtt.data.total_mileage":"65423000"}`,
`{"yutong_mqtt.data.latitude":"30.1"}`,
`{"data":{"TOTAL_MILEAGE":65423000}}`,
int64(42),
))
@@ -526,14 +516,15 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
lastTS := time.Date(2026, 7, 7, 23, 58, 0, 0, loc)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-08 00:00:00'.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
}).AddRow(
"LMRKH9AC6R1004108",
"",
"LMRKH9AC6R1004108",
"mqtt://yutong/ytforward/shln/3",
lastTS,
`{"yutong_mqtt.data.total_mileage":"65377000"}`,
`{"yutong_mqtt.data.latitude":"30.0"}`,
`{"data":{"TOTAL_MILEAGE":65377000}}`,
int64(31),
))
@@ -556,13 +547,19 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
}
}
func TestBackfillBeforePredicatesSearchesAllEarlierHistory(t *testing.T) {
where := strings.Join(backfillBeforePredicates("2026-07-08"), " AND ")
func TestBackfillBeforePredicatesBoundsHistoryScan(t *testing.T) {
where := strings.Join(backfillBeforePredicates(config{BaselineLookback: 7}, "2026-07-08"), " AND ")
if !strings.Contains(where, "event_time < '2026-07-08 00:00:00'") {
t.Fatalf("pre-window predicate missing exclusive upper bound: %s", where)
}
if strings.Contains(where, "event_time >=") || strings.Contains(where, "ts >=") {
t.Fatalf("pre-window predicate must not stop at the previous day: %s", where)
for _, predicate := range []string{
"event_time >= '2026-07-01 00:00:00'",
"ts >= '2026-06-30 00:00:00'",
"ts < '2026-07-09 00:00:00'",
} {
if !strings.Contains(where, predicate) {
t.Fatalf("pre-window predicate missing %q: %s", predicate, where)
}
}
}
@@ -577,7 +574,7 @@ func TestQueryPreviousLastSourceRowsNormalizesScannedTimestampToBusinessTimezone
utcInstant := time.Date(2026, 7, 12, 15, 59, 59, 0, time.UTC)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
}).AddRow(
"LMRKH9AC7R1004098",
"",
@@ -585,6 +582,7 @@ func TestQueryPreviousLastSourceRowsNormalizesScannedTimestampToBusinessTimezone
"mqtt://yutong/ytforward/shln/4",
utcInstant,
`{"yutong_mqtt.data.total_mileage":"41249000"}`,
"",
int64(1),
))

View File

@@ -0,0 +1,57 @@
package feichibridge
import (
"bytes"
"context"
"errors"
"fmt"
"os/exec"
"regexp"
"strings"
"time"
)
var captchaPattern = regexp.MustCompile(`^[0-9A-Z]{4}$`)
type CaptchaSolver interface {
Solve(context.Context, []byte) (string, error)
}
type DockerCaptchaSolver struct {
Image string
Timeout time.Duration
}
func (s DockerCaptchaSolver) Solve(ctx context.Context, image []byte) (string, error) {
if len(image) == 0 {
return "", errors.New("captcha image is empty")
}
if strings.TrimSpace(s.Image) == "" {
return "", errors.New("captcha OCR image is required")
}
timeout := s.Timeout
if timeout <= 0 {
timeout = 20 * time.Second
}
solveCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
command := exec.CommandContext(
solveCtx,
"docker", "run", "--rm", "--network=none", "-i", strings.TrimSpace(s.Image),
)
command.Stdin = bytes.NewReader(image)
var stderr bytes.Buffer
command.Stderr = &stderr
output, err := command.Output()
if err != nil {
if solveCtx.Err() != nil {
return "", fmt.Errorf("captcha OCR timed out: %w", solveCtx.Err())
}
return "", fmt.Errorf("captcha OCR failed: %w: %s", err, strings.TrimSpace(stderr.String()))
}
code := strings.ToUpper(strings.TrimSpace(string(output)))
if !captchaPattern.MatchString(code) {
return "", fmt.Errorf("captcha OCR returned invalid result %q", code)
}
return code, nil
}

View File

@@ -0,0 +1,211 @@
package feichibridge
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
)
var ErrUnauthorized = errors.New("feichi API session unauthorized")
type APIClient struct {
baseURL *url.URL
headers http.Header
client *http.Client
login *loginSession
}
func NewAPIClient(baseURL string, headers map[string]string, timeout time.Duration) (*APIClient, error) {
parsed, err := url.Parse(strings.TrimRight(strings.TrimSpace(baseURL), "/") + "/")
if err != nil {
return nil, fmt.Errorf("parse FEICHI_BASE_URL: %w", err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return nil, fmt.Errorf("FEICHI_BASE_URL must use http or https")
}
if timeout <= 0 {
timeout = 15 * time.Second
}
out := &APIClient{
baseURL: parsed,
headers: make(http.Header),
client: &http.Client{Timeout: timeout},
}
for name, value := range headers {
if strings.TrimSpace(name) != "" && strings.TrimSpace(value) != "" {
out.headers.Set(name, value)
}
}
return out, nil
}
func NewAuthenticatedAPIClient(baseURL string, credentials LoginCredentials, solver CaptchaSolver, timeout time.Duration) (*APIClient, error) {
client, err := NewAPIClient(baseURL, nil, timeout)
if err != nil {
return nil, err
}
login, err := newLoginSession(credentials, solver)
if err != nil {
return nil, err
}
client.login = login
return client, nil
}
func (c *APIClient) Vehicles(ctx context.Context) ([]Vehicle, error) {
payload := map[string]any{
"conditions": []map[string]string{
{"name": "iccid", "value": ""},
{"name": "powerMode", "value": ""},
},
"sort": []map[string]string{{"name": "updateTime", "order": "desc"}},
"start": 0,
"limit": 100,
}
var envelope collectionEnvelope[Vehicle]
if err := c.doJSON(ctx, http.MethodPost, "api/v1/sys/vehicleRealStatuss", payload, &envelope); err != nil {
return nil, err
}
if !apiCodeOK(envelope.Code) {
return nil, apiCodeError(envelope.Code)
}
return envelope.Data, nil
}
func (c *APIClient) Snapshot(ctx context.Context, vehicleID string) (Snapshot, error) {
path := "api/v1/sys/vehicleRealStatussByVId/" + url.PathEscape(vehicleID) + "/-1"
var envelope objectEnvelope[Snapshot]
if err := c.doJSON(ctx, http.MethodGet, path, nil, &envelope); err != nil {
return Snapshot{}, err
}
if !apiCodeOK(envelope.Code) {
return Snapshot{}, apiCodeError(envelope.Code)
}
return envelope.Data, nil
}
func (c *APIClient) History(ctx context.Context, vin string, begin, end time.Time) ([]Record, error) {
payload := map[string]any{
"conditions": []map[string]string{
{"name": "queryType", "value": "0"},
{"name": "queryContent", "value": vin},
{"name": "beginTime", "value": begin.In(shanghai).Format("2006-01-02 15:04:05")},
{"name": "endTime", "value": end.In(shanghai).Format("2006-01-02 15:04:05")},
{"name": "hisdataType", "value": "0"},
},
"start": 0,
"limit": 100,
}
var envelope collectionEnvelope[historySegment]
if err := c.doJSON(ctx, http.MethodPost, "api/v1/sys/hisdataQuerys", payload, &envelope); err != nil {
return nil, err
}
if !apiCodeOK(envelope.Code) {
return nil, apiCodeError(envelope.Code)
}
var records []Record
for _, segment := range envelope.Data {
records = append(records, segment.Subdata...)
}
return records, nil
}
func (c *APIClient) doJSON(ctx context.Context, method, path string, body any, output any) error {
if c.login != nil {
if err := c.login.ensure(ctx, c); err != nil {
return err
}
}
for attempt := 0; attempt < 2; attempt++ {
_, err := c.rawJSON(ctx, method, path, body, output)
if !errors.Is(err, ErrUnauthorized) || c.login == nil || attempt == 1 {
return err
}
c.login.invalidate()
if err := c.login.ensure(ctx, c); err != nil {
return err
}
}
return ErrUnauthorized
}
func (c *APIClient) rawJSON(ctx context.Context, method, path string, body any, output any) (http.Header, error) {
endpoint, err := c.baseURL.Parse(path)
if err != nil {
return nil, err
}
var reader io.Reader
if body != nil {
encoded, marshalErr := json.Marshal(body)
if marshalErr != nil {
return nil, marshalErr
}
reader = bytes.NewReader(encoded)
}
request, err := http.NewRequestWithContext(ctx, method, endpoint.String(), reader)
if err != nil {
return nil, err
}
request.Header.Set("Accept", "application/json")
if body != nil {
request.Header.Set("Content-Type", "application/json")
}
for name, values := range c.headers {
for _, value := range values {
request.Header.Add(name, value)
}
}
if c.login != nil {
if cookie := c.login.cookieHeader(); cookie != "" {
request.Header.Set("Cookie", cookie)
}
if csrf := c.login.csrfHeader(); csrf != "" {
request.Header.Set("x-api-csrf", csrf)
}
}
response, err := c.client.Do(request)
if err != nil {
return nil, fmt.Errorf("feichi %s %s: %w", method, path, err)
}
defer response.Body.Close()
if response.StatusCode == http.StatusUnauthorized || response.StatusCode == http.StatusForbidden {
return response.Header, ErrUnauthorized
}
if response.StatusCode < 200 || response.StatusCode >= 300 {
snippet, _ := io.ReadAll(io.LimitReader(response.Body, 1024))
return response.Header, fmt.Errorf("feichi %s %s: HTTP %d: %s", method, path, response.StatusCode, strings.TrimSpace(string(snippet)))
}
encoded, err := io.ReadAll(io.LimitReader(response.Body, 16<<20))
if err != nil {
return response.Header, fmt.Errorf("read feichi %s: %w", path, err)
}
var status struct {
Code any `json:"code"`
}
if err := json.Unmarshal(encoded, &status); err != nil {
return response.Header, fmt.Errorf("decode feichi %s: %w", path, err)
}
if fmt.Sprint(status.Code) == "401" || fmt.Sprint(status.Code) == "403" {
return response.Header, ErrUnauthorized
}
if output != nil {
if err := json.Unmarshal(encoded, output); err != nil {
return response.Header, fmt.Errorf("decode feichi %s: %w", path, err)
}
}
return response.Header, nil
}
func apiCodeError(code any) error {
if fmt.Sprint(code) == "401" || fmt.Sprint(code) == "403" {
return ErrUnauthorized
}
return fmt.Errorf("feichi API returned code %v", code)
}

View File

@@ -0,0 +1,154 @@
package feichibridge
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
type fixedCaptchaSolver struct {
code string
calls int
}
func (s *fixedCaptchaSolver) Solve(_ context.Context, image []byte) (string, error) {
s.calls++
if string(image) != "captcha-image" {
return "", fmt.Errorf("unexpected captcha image")
}
return s.code, nil
}
func TestAPIClientUsesSessionHeadersAndExpectedEndpoints(t *testing.T) {
var calls int
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
calls++
if request.Header.Get("Cookie") != "R_SESS=test" || request.Header.Get("x-api-csrf") != "csrf" {
t.Errorf("missing session headers: %#v", request.Header)
}
response.Header().Set("Content-Type", "application/json")
switch {
case request.URL.Path == "/api/v1/sys/vehicleRealStatuss":
_ = json.NewEncoder(response).Encode(map[string]any{
"code": 200,
"data": []map[string]string{{"vehicleId": "id-1", "vin": "LTEST32960VIN0001"}},
})
case strings.Contains(request.URL.Path, "vehicleRealStatussByVId"):
_ = json.NewEncoder(response).Encode(map[string]any{
"code": 200,
"data": map[string]any{"vehicleId": "id-1", "vin": "LTEST32960VIN0001", "dataItems": map[string]string{"2000": "2026-07-17 12:00:00"}},
})
case request.URL.Path == "/api/v1/sys/hisdataQuerys":
_ = json.NewEncoder(response).Encode(map[string]any{
"code": 200,
"data": []map[string]any{{"subdata": []map[string]string{{"2000": "2026-07-17 11:59:50"}}}},
})
default:
http.NotFound(response, request)
}
}))
defer server.Close()
client, err := NewAPIClient(server.URL, map[string]string{"Cookie": "R_SESS=test", "x-api-csrf": "csrf"}, time.Second)
if err != nil {
t.Fatal(err)
}
vehicles, err := client.Vehicles(context.Background())
if err != nil || len(vehicles) != 1 {
t.Fatalf("vehicles = %#v, err = %v", vehicles, err)
}
if _, err := client.Snapshot(context.Background(), "id-1"); err != nil {
t.Fatal(err)
}
records, err := client.History(context.Background(), vehicles[0].VIN, time.Now().Add(-time.Hour), time.Now())
if err != nil || len(records) != 1 {
t.Fatalf("history = %#v, err = %v", records, err)
}
if calls != 3 {
t.Fatalf("calls = %d", calls)
}
}
func TestAPIClientMapsUnauthorizedEnvelope(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
_, _ = response.Write([]byte(`{"code":401,"data":[]}`))
}))
defer server.Close()
client, err := NewAPIClient(server.URL, nil, time.Second)
if err != nil {
t.Fatal(err)
}
if _, err := client.Vehicles(context.Background()); err != ErrUnauthorized {
t.Fatalf("error = %v", err)
}
}
func TestAuthenticatedAPIClientLogsInAndRetriesExpiredSession(t *testing.T) {
var loginCalls int
var vehicleCalls int
solver := &fixedCaptchaSolver{code: "12WN"}
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
response.Header().Set("Content-Type", "application/json")
switch request.URL.Path {
case "/api/v1/first-login":
response.Header().Set("x-api-csrf", "csrf-value")
response.Header().Add("Set-Cookie", "CSRF=csrf-value; Path=/")
_, _ = response.Write([]byte(`{"code":200}`))
case "/api/v1/login/randCode":
if request.Header.Get("x-api-csrf") != "csrf-value" {
t.Errorf("captcha CSRF = %q", request.Header.Get("x-api-csrf"))
}
src := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString([]byte("captcha-image"))
_ = json.NewEncoder(response).Encode(map[string]any{"code": 200, "data": map[string]string{"src": src}})
case "/api/v1/login":
loginCalls++
var payload map[string]string
_ = json.NewDecoder(request.Body).Decode(&payload)
if payload["username"] != "JXQN0003" || payload["validCode"] != "12WN" {
t.Errorf("login payload = %#v", payload)
}
encrypted, err := base64.StdEncoding.DecodeString(payload["password"])
if err != nil || len(encrypted) != 128 || payload["password"] == "JXQN0003-" {
t.Errorf("password was not RSA encrypted: len=%d err=%v", len(encrypted), err)
}
_ = json.NewEncoder(response).Encode(map[string]any{
"code": 200, "data": map[string]string{"token": fmt.Sprintf("token-%d", loginCalls)},
})
case "/api/v1/sys/vehicleRealStatuss":
vehicleCalls++
if vehicleCalls == 1 {
_, _ = response.Write([]byte(`{"code":401,"data":[]}`))
return
}
if request.Header.Get("Cookie") != "CSRF=csrf-value; R_SESS=token-2" {
t.Errorf("session cookie = %q", request.Header.Get("Cookie"))
}
_, _ = response.Write([]byte(`{"code":200,"data":[]}`))
default:
http.NotFound(response, request)
}
}))
defer server.Close()
client, err := NewAuthenticatedAPIClient(
server.URL,
LoginCredentials{Username: "JXQN0003", Password: "JXQN0003-", MaxAttempts: 2},
solver,
time.Second,
)
if err != nil {
t.Fatal(err)
}
if _, err := client.Vehicles(context.Background()); err != nil {
t.Fatal(err)
}
if loginCalls != 2 || solver.calls != 2 || vehicleCalls != 2 {
t.Fatalf("login=%d solver=%d vehicles=%d", loginCalls, solver.calls, vehicleCalls)
}
}

View File

@@ -0,0 +1,481 @@
package feichibridge
import (
"encoding/binary"
"errors"
"fmt"
"math"
"strconv"
"strings"
"time"
)
const (
CommandRealtime = byte(0x02)
CommandReissue = byte(0x03)
CommandLogin = byte(0x05)
)
type Encoder struct{}
func (Encoder) DataFrame(command byte, vin string, at time.Time, record Record) ([]byte, error) {
if command != CommandRealtime && command != CommandReissue {
return nil, fmt.Errorf("unsupported data command 0x%02X", command)
}
body := encodeTime(at)
units := 0
if unit, ok := wholeVehicleUnit(record); ok {
body = append(body, 0x01)
body = append(body, unit...)
units++
}
if unit, ok := motorUnit(record); ok {
body = append(body, 0x02)
body = append(body, unit...)
units++
}
if unit, ok := fuelCellUnit(record); ok {
body = append(body, 0x03)
body = append(body, unit...)
units++
}
if unit, ok := positionUnit(record); ok {
body = append(body, 0x05)
body = append(body, unit...)
units++
}
if unit, ok := extremeUnit(record); ok {
body = append(body, 0x06)
body = append(body, unit...)
units++
}
if unit, ok := alarmUnit(record); ok {
body = append(body, 0x07)
body = append(body, unit...)
units++
}
for _, unit := range voltageUnits(record) {
body = append(body, 0x08)
body = append(body, unit...)
units++
}
for _, unit := range temperatureUnits(record) {
body = append(body, 0x09)
body = append(body, unit...)
units++
}
if units == 0 {
return nil, errors.New("source record has no mappable GB/T 32960 units")
}
return buildFrame('#', command, 0xFE, vin, body)
}
func LoginFrame(platformID, username, password string, serial uint16, now time.Time) ([]byte, error) {
if len(username) > 12 {
return nil, errors.New("GB/T 32960 platform username exceeds 12 bytes")
}
if len(password) > 20 {
return nil, errors.New("GB/T 32960 platform password exceeds 20 bytes")
}
body := encodeTime(now)
body = binary.BigEndian.AppendUint16(body, serial)
body = appendPaddedASCII(body, username, 12)
body = appendPaddedASCII(body, password, 20)
body = append(body, 0x01)
return buildFrame('#', CommandLogin, 0xFE, platformID, body)
}
func buildFrame(start, command, response byte, vin string, body []byte) ([]byte, error) {
if len(vin) != 17 {
return nil, fmt.Errorf("GB/T 32960 identifier must be exactly 17 bytes, got %d", len(vin))
}
if len(body) > math.MaxUint16 {
return nil, errors.New("GB/T 32960 body exceeds 65535 bytes")
}
frame := make([]byte, 24, 25+len(body))
frame[0], frame[1] = start, start
frame[2], frame[3] = command, response
copy(frame[4:21], vin)
frame[21] = 0x01
binary.BigEndian.PutUint16(frame[22:24], uint16(len(body)))
frame = append(frame, body...)
frame = append(frame, bcc(frame[2:]))
return frame, nil
}
func wholeVehicleUnit(record Record) ([]byte, bool) {
if !hasAny(record, "2201", "2202", "3201", "7615") {
return nil, false
}
out := make([]byte, 20)
out[0] = enum(recordValue(record, "3201"), map[string]byte{
"启动": 1, "启动状态": 1, "行驶": 1, "熄火": 2, "熄火状态": 2, "其他": 3,
}, 0xFE)
out[1] = enum(recordValue(record, "2301"), map[string]byte{
"停车充电": 1, "行驶充电": 2, "未充电": 3, "未充电状态": 3, "充电完成": 4,
}, 0xFE)
out[2] = enum(recordValue(record, "2213"), map[string]byte{
"纯电": 1, "纯电动": 1, "混动": 2, "混合动力": 2, "燃油": 3,
}, 0xFE)
putScaledU16(out[3:5], recordValue(record, "2201"), 10, 0)
putScaledU32(out[5:9], recordValue(record, "2202"), 10)
putScaledU16(out[9:11], recordValue(record, "2613"), 10, 0)
putScaledU16(out[11:13], recordValue(record, "2614"), 10, 1000)
out[13] = byteValue(recordValue(record, "7615"))
out[14] = enum(recordValue(record, "2214"), map[string]byte{"工作": 1, "断开": 2}, 0xFE)
out[15] = gearValue(firstField(recordValue(record, "2203")))
putU16(out[16:18], recordValue(record, "2617"))
out[18] = byteValue(recordValue(record, "2208"))
out[19] = byteValue(recordValue(record, "2209"))
return out, true
}
func motorUnit(record Record) ([]byte, bool) {
// The source carries every motor as a key:value composite separated by "|".
// Fields are the GB/T 32960 sequence numbers exposed by the platform metadata.
composite := recordValue(record, "2308")
if composite == "" {
return nil, false
}
groups := parseCompositeGroups(composite)
if len(groups) == 0 || len(groups) > 253 {
return nil, false
}
out := []byte{byte(len(groups))}
for index, group := range groups {
out = append(out, byteValue(firstNonempty(group["2302"], strconv.Itoa(index+1))))
out = append(out, enum(group["2303"], map[string]byte{
"耗电": 1, "发电": 2, "关闭": 3, "准备": 4, "准备状态": 4,
}, 0xFE))
out = append(out, tempByte(group["2304"]))
out = binary.BigEndian.AppendUint16(out, offsetU16(group["2305"], 1, 20000))
out = binary.BigEndian.AppendUint16(out, offsetU16(group["2306"], 10, 2000))
out = append(out, tempByte(group["2309"]))
out = binary.BigEndian.AppendUint16(out, scaledU16(group["2311"], 10, 0))
out = binary.BigEndian.AppendUint16(out, scaledU16(group["2312"], 10, 1000))
}
return out, true
}
func fuelCellUnit(record Record) ([]byte, bool) {
if !hasAny(record, "2110", "2111", "2112", "2117", "2119") {
return nil, false
}
temperatureGroups := parseSeriesGroups(recordValue(record, "2103"))
var temperatures []string
for _, group := range temperatureGroups {
temperatures = append(temperatures, group.values...)
}
if len(temperatures) > math.MaxUint16 {
temperatures = temperatures[:math.MaxUint16]
}
out := binary.BigEndian.AppendUint16(nil, scaledU16(recordValue(record, "2110"), 10, 0))
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2111"), 10, 0))
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2112"), 100, 0))
out = binary.BigEndian.AppendUint16(out, uint16(len(temperatures)))
for _, value := range temperatures {
out = append(out, tempByte(value))
}
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2115"), 10, 40))
out = append(out, byteValue(recordValue(record, "2116")))
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2117"), 1, 0))
out = append(out, byteValue(recordValue(record, "2118")))
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2119"), 10, 0))
out = append(out, byteValue(recordValue(record, "2120")))
out = append(out, enum(recordValue(record, "2121"), map[string]byte{"工作": 1, "断开": 2}, 0xFE))
return out, true
}
func positionUnit(record Record) ([]byte, bool) {
longitude, lonOK := floatValue(recordValue(record, "2502"))
latitude, latOK := floatValue(recordValue(record, "2503"))
if !lonOK || !latOK || longitude < 0 || latitude < 0 {
return nil, false
}
status := byte(0)
text := strings.TrimSpace(recordValue(record, "2501"))
if text != "" && (strings.Contains(text, "无效") || text == "1") {
status = 1
}
out := []byte{status}
out = binary.BigEndian.AppendUint32(out, uint32(math.Round(longitude*1_000_000)))
out = binary.BigEndian.AppendUint32(out, uint32(math.Round(latitude*1_000_000)))
return out, true
}
func extremeUnit(record Record) ([]byte, bool) {
if !hasAny(record, "2601", "2602", "2603", "2604", "2605", "2606", "2607", "2608", "2609", "2610", "2611", "2612") {
return nil, false
}
out := []byte{
byteValue(recordValue(record, "2601")),
byteValue(recordValue(record, "2602")),
}
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2603"), 1000, 0))
out = append(out, byteValue(recordValue(record, "2604")), byteValue(recordValue(record, "2605")))
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2606"), 1000, 0))
out = append(out,
byteValue(recordValue(record, "2607")),
byteValue(recordValue(record, "2608")),
tempByte(recordValue(record, "2609")),
byteValue(recordValue(record, "2610")),
byteValue(recordValue(record, "2611")),
tempByte(recordValue(record, "2612")),
)
return out, true
}
func alarmUnit(record Record) ([]byte, bool) {
levelRaw := recordValue(record, "2900", "2901")
var general uint32
for bit := 0; bit < 32; bit++ {
value := recordValue(record, strconv.Itoa(2901+bit))
if truthy(value) {
general |= 1 << bit
}
}
if strings.TrimSpace(levelRaw) == "" && general == 0 {
return nil, false
}
out := []byte{byteValue(levelRaw)}
out = binary.BigEndian.AppendUint32(out, general)
out = append(out, 0, 0, 0, 0)
return out, true
}
func voltageUnits(record Record) [][]byte {
raw := recordValue(record, "2003", "batteryVoltages")
groups := parseSeriesGroups(raw)
var units [][]byte
for _, group := range groups {
for start := 0; start < len(group.values); start += 255 {
end := start + 255
if end > len(group.values) {
end = len(group.values)
}
out := []byte{1, byte(group.id)}
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2613"), 10, 0))
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2614"), 10, 1000))
out = binary.BigEndian.AppendUint16(out, uint16(len(group.values)))
out = binary.BigEndian.AppendUint16(out, uint16(start+1))
out = append(out, byte(end-start))
for _, value := range group.values[start:end] {
out = binary.BigEndian.AppendUint16(out, scaledU16(value, 1000, 0))
}
units = append(units, out)
}
}
return units
}
func temperatureUnits(record Record) [][]byte {
groups := parseSeriesGroups(recordValue(record, "2103", "batteryTemperatures"))
var units [][]byte
for _, group := range groups {
for start := 0; start < len(group.values); start += math.MaxUint16 {
end := start + math.MaxUint16
if end > len(group.values) {
end = len(group.values)
}
out := []byte{1, byte(group.id)}
out = binary.BigEndian.AppendUint16(out, uint16(end-start))
for _, value := range group.values[start:end] {
out = append(out, tempByte(value))
}
units = append(units, out)
}
}
return units
}
func parseCompositeGroups(raw string) []map[string]string {
var groups []map[string]string
for _, encoded := range strings.Split(raw, "|") {
group := map[string]string{}
for _, field := range strings.Split(encoded, ",") {
key, value, found := strings.Cut(field, ":")
if found {
group[strings.TrimSpace(key)] = strings.TrimSpace(value)
}
}
if len(group) > 0 {
groups = append(groups, group)
}
}
return groups
}
type seriesGroup struct {
id int
values []string
}
func parseSeriesGroups(raw string) []seriesGroup {
raw = strings.TrimSpace(raw)
if raw == "" {
return nil
}
var groups []seriesGroup
for index, part := range strings.FieldsFunc(raw, func(r rune) bool { return r == ';' || r == '|' }) {
group := seriesGroup{id: index + 1}
if before, after, found := strings.Cut(part, ":"); found {
if parsed, err := strconv.Atoi(strings.TrimSpace(before)); err == nil && parsed > 0 && parsed <= 255 {
group.id = parsed
}
part = after
}
for _, value := range strings.FieldsFunc(part, func(r rune) bool {
return r == '_' || r == ',' || r == ' ' || r == '[' || r == ']'
}) {
if strings.TrimSpace(value) != "" {
group.values = append(group.values, strings.TrimSpace(value))
}
}
if len(group.values) > 0 {
groups = append(groups, group)
}
}
return groups
}
func recordValue(record Record, keys ...string) string {
for _, key := range keys {
if value := strings.TrimSpace(record[key]); value != "" && value != "--" && value != "null" {
return value
}
}
return ""
}
func firstField(value string) string {
fields := strings.Fields(value)
if len(fields) == 0 {
return ""
}
return fields[0]
}
func firstNonempty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func hasAny(record Record, keys ...string) bool { return recordValue(record, keys...) != "" }
func floatValue(raw string) (float64, bool) {
raw = strings.TrimSpace(strings.TrimSuffix(raw, "%"))
if raw == "" {
return 0, false
}
value, err := strconv.ParseFloat(raw, 64)
return value, err == nil && !math.IsNaN(value) && !math.IsInf(value, 0)
}
func byteValue(raw string) byte {
value, ok := floatValue(raw)
if !ok || value < 0 || value > 253 {
return 0xFE
}
return byte(math.Round(value))
}
func tempByte(raw string) byte {
value, ok := floatValue(raw)
if !ok || value < -40 || value > 210 {
return 0xFE
}
return byte(math.Round(value + 40))
}
func putU16(out []byte, raw string) { binary.BigEndian.PutUint16(out, scaledU16(raw, 1, 0)) }
func putScaledU16(out []byte, raw string, scale, offset float64) {
binary.BigEndian.PutUint16(out, scaledU16(raw, scale, offset))
}
func putScaledU32(out []byte, raw string, scale float64) {
value, ok := floatValue(raw)
if !ok || value < 0 || value*scale > math.MaxUint32-2 {
binary.BigEndian.PutUint32(out, 0xFFFFFFFE)
return
}
binary.BigEndian.PutUint32(out, uint32(math.Round(value*scale)))
}
func scaledU16(raw string, scale, offset float64) uint16 {
value, ok := floatValue(raw)
encoded := (value + offset) * scale
if !ok || encoded < 0 || encoded > math.MaxUint16-2 {
return 0xFFFE
}
return uint16(math.Round(encoded))
}
func offsetU16(raw string, scale, offset float64) uint16 {
return scaledU16(raw, scale, offset)
}
func enum(raw string, values map[string]byte, missing byte) byte {
raw = strings.TrimSpace(raw)
if raw == "" {
return missing
}
if numeric, err := strconv.ParseUint(raw, 10, 8); err == nil {
return byte(numeric)
}
for label, value := range values {
if raw == label || strings.Contains(raw, label) {
return value
}
}
return missing
}
func gearValue(raw string) byte {
raw = strings.ToUpper(strings.TrimSpace(raw))
switch raw {
case "P", "P档":
return 15
case "R", "R档":
return 13
case "N", "N档":
return 0
case "D", "D档":
return 14
}
raw = strings.TrimSuffix(raw, "档")
raw = strings.TrimPrefix(raw, "D")
return byteValue(raw)
}
func truthy(raw string) bool {
raw = strings.ToLower(strings.TrimSpace(raw))
return raw != "" && raw != "0" && raw != "false" && raw != "无" && raw != "正常"
}
func appendPaddedASCII(out []byte, value string, width int) []byte {
start := len(out)
out = append(out, make([]byte, width)...)
copy(out[start:start+width], value)
return out
}
func encodeTime(value time.Time) []byte {
local := value.In(shanghai)
return []byte{
byte(local.Year() - 2000), byte(local.Month()), byte(local.Day()),
byte(local.Hour()), byte(local.Minute()), byte(local.Second()),
}
}
func bcc(value []byte) byte {
var result byte
for _, current := range value {
result ^= current
}
return result
}

View File

@@ -0,0 +1,93 @@
package feichibridge
import (
"encoding/binary"
"fmt"
"strings"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
)
func TestDataFrameMapsFeichiFields(t *testing.T) {
at := time.Date(2026, 7, 17, 14, 5, 6, 0, shanghai)
record := Record{
"2000": "2026-07-17 14:05:06",
"2201": "42.3", "2202": "12345.6", "2203": "D 0 0",
"2208": "35", "2209": "0", "2213": "纯电",
"2214": "工作", "2301": "未充电状态", "3201": "启动状态",
"2613": "550.2", "2614": "-20.5", "2617": "2200", "7615": "78",
"2501": "有效定位", "2502": "113.2644", "2503": "23.1291",
"2601": "1", "2602": "8", "2603": "3.955",
"2604": "1", "2605": "26", "2606": "3.821",
"2607": "1", "2608": "3", "2609": "48",
"2610": "1", "2611": "9", "2612": "37",
"2003": "1:3.900_3.901_3.902",
"2103": "1:35_36_37",
}
frame, err := (Encoder{}).DataFrame(CommandRealtime, "LTEST32960VIN0001", at, record)
if err != nil {
t.Fatal(err)
}
env, err := gb32960.ParseFrame(frame, at.UnixMilli(), "test")
if err != nil {
t.Fatal(err)
}
if env.MessageID != "0x02" || env.VIN != "LTEST32960VIN0001" {
t.Fatalf("unexpected envelope: %#v", env)
}
if got := env.Fields["speed_kmh"]; got != 42.3 {
t.Fatalf("speed = %#v", got)
}
if got := env.Fields["total_mileage_km"]; got != 12345.6 {
t.Fatalf("mileage = %#v", got)
}
if got := env.Fields["soc_percent"]; got != 78 {
t.Fatalf("soc = %#v", got)
}
if got := env.Fields["longitude"]; got != 113.2644 {
t.Fatalf("longitude = %#v", got)
}
}
func TestDataFrameSplitsMoreThan255CellVoltages(t *testing.T) {
values := make([]string, 288)
for index := range values {
values[index] = fmt.Sprintf("%.3f", 3.5+float64(index)/1000)
}
frame, err := (Encoder{}).DataFrame(CommandReissue, "LTEST32960VIN0002", time.Now(), Record{
"2003": "1:" + strings.Join(values, "_"),
"2613": "530",
"2614": "10",
})
if err != nil {
t.Fatal(err)
}
body := frame[24 : len(frame)-1]
if got := strings.Count(string(body), string([]byte{0x08})); got < 2 {
t.Fatalf("expected at least two voltage units, got %d", got)
}
if binary.BigEndian.Uint16(frame[22:24]) != uint16(len(body)) {
t.Fatal("body length mismatch")
}
if _, err := gb32960.ParseFrame(frame, time.Now().UnixMilli(), "test"); err != nil {
t.Fatal(err)
}
}
func TestLoginFrameLayout(t *testing.T) {
frame, err := LoginFrame("FEICHIBRIDGE00001", "bridge", "secret", 7, time.Date(2026, 7, 17, 1, 2, 3, 0, shanghai))
if err != nil {
t.Fatal(err)
}
if frame[2] != CommandLogin || frame[3] != 0xFE {
t.Fatalf("unexpected login header: %x", frame[:4])
}
if got := binary.BigEndian.Uint16(frame[30:32]); got != 7 {
t.Fatalf("serial = %d", got)
}
if got, want := frame[len(frame)-1], bcc(frame[2:len(frame)-1]); got != want {
t.Fatalf("BCC = %x, want %x", got, want)
}
}

View File

@@ -0,0 +1,209 @@
package feichibridge
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"errors"
"fmt"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
)
const feichiPublicKey = `-----BEGIN PUBLIC KEY-----
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDOwmFtEk1oJxDU0NI4kVO0Jx0X
nt+Abx+JKGUzcRzPwjUkd5z9Ice5rh87CCmj0XjZ5pPac6TtA3f0v5FqiK/kjQY5
XMLti4weJ4dcp1/q1O7PCYxRX8WgetGxwsjxGn+uoZOZkclN1PFS4wRnKEso6+G/
e60QGB29cZoo4jZnZwIDAQAB
-----END PUBLIC KEY-----`
type LoginCredentials struct {
Username string
Password string
MaxAttempts int
RetryDelay time.Duration
}
type loginSession struct {
credentials LoginCredentials
solver CaptchaSolver
mu sync.Mutex
ready atomic.Bool
csrf atomic.Value
token atomic.Value
}
type firstLoginEnvelope struct {
Code any `json:"code"`
}
type captchaPayload struct {
Src string `json:"src"`
}
type loginPayload struct {
Token string `json:"token"`
}
func newLoginSession(credentials LoginCredentials, solver CaptchaSolver) (*loginSession, error) {
credentials.Username = strings.TrimSpace(credentials.Username)
if credentials.Username == "" || credentials.Password == "" {
return nil, errors.New("Feichi username and password are required")
}
if solver == nil {
return nil, errors.New("captcha solver is required")
}
if credentials.MaxAttempts <= 0 {
credentials.MaxAttempts = 20
}
if credentials.RetryDelay <= 0 {
credentials.RetryDelay = time.Second
}
session := &loginSession{credentials: credentials, solver: solver}
session.csrf.Store("")
session.token.Store("")
return session, nil
}
func (s *loginSession) invalidate() {
s.ready.Store(false)
}
func (s *loginSession) cookieHeader() string {
var cookies []string
if csrf := s.csrf.Load().(string); csrf != "" {
cookies = append(cookies, "CSRF="+csrf)
}
if token := s.token.Load().(string); token != "" {
cookies = append(cookies, "R_SESS="+token)
}
return strings.Join(cookies, "; ")
}
func (s *loginSession) csrfHeader() string {
return s.csrf.Load().(string)
}
func (s *loginSession) ensure(ctx context.Context, client *APIClient) error {
if s.ready.Load() {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
if s.ready.Load() {
return nil
}
var lastErr error
for attempt := 1; attempt <= s.credentials.MaxAttempts; attempt++ {
if err := s.loginAttempt(ctx, client); err == nil {
s.ready.Store(true)
return nil
} else {
lastErr = err
}
if attempt < s.credentials.MaxAttempts {
timer := time.NewTimer(s.credentials.RetryDelay)
select {
case <-ctx.Done():
timer.Stop()
return ctx.Err()
case <-timer.C:
}
}
}
return fmt.Errorf("Feichi automatic login failed after %d attempts: %w", s.credentials.MaxAttempts, lastErr)
}
func (s *loginSession) loginAttempt(ctx context.Context, client *APIClient) error {
s.ready.Store(false)
s.token.Store("")
var first firstLoginEnvelope
headers, err := client.rawJSON(ctx, http.MethodGet, "api/v1/first-login", nil, &first)
if err != nil {
return fmt.Errorf("initialize login session: %w", err)
}
if !apiCodeOK(first.Code) {
return apiCodeError(first.Code)
}
csrf := strings.TrimSpace(headers.Get("x-api-csrf"))
if csrf == "" {
for _, rawCookie := range headers.Values("Set-Cookie") {
cookie, parseErr := http.ParseSetCookie(rawCookie)
if parseErr == nil && cookie.Name == "CSRF" {
csrf = cookie.Value
break
}
}
}
if csrf == "" {
return errors.New("first-login response did not provide CSRF")
}
s.csrf.Store(csrf)
var captcha objectEnvelope[captchaPayload]
if _, err := client.rawJSON(ctx, http.MethodGet, "api/v1/login/randCode", nil, &captcha); err != nil {
return fmt.Errorf("request captcha: %w", err)
}
if !apiCodeOK(captcha.Code) {
return apiCodeError(captcha.Code)
}
encodedImage := captcha.Data.Src
if comma := strings.IndexByte(encodedImage, ','); comma >= 0 {
encodedImage = encodedImage[comma+1:]
}
image, err := base64.StdEncoding.DecodeString(encodedImage)
if err != nil {
return fmt.Errorf("decode captcha image: %w", err)
}
validCode, err := s.solver.Solve(ctx, image)
if err != nil {
return err
}
encrypted, err := encryptFeichiPassword(s.credentials.Password)
if err != nil {
return err
}
var login objectEnvelope[loginPayload]
if _, err := client.rawJSON(ctx, http.MethodPost, "api/v1/login", map[string]string{
"username": s.credentials.Username,
"password": encrypted,
"validCode": validCode,
}, &login); err != nil {
return fmt.Errorf("submit login: %w", err)
}
if !apiCodeOK(login.Code) {
return apiCodeError(login.Code)
}
token := strings.TrimSpace(login.Data.Token)
if token == "" {
return errors.New("login succeeded without R_SESS token")
}
s.token.Store(token)
return nil
}
func encryptFeichiPassword(password string) (string, error) {
block, _ := pem.Decode([]byte(feichiPublicKey))
if block == nil {
return "", errors.New("decode Feichi RSA public key")
}
parsed, err := x509.ParsePKIXPublicKey(block.Bytes)
if err != nil {
return "", fmt.Errorf("parse Feichi RSA public key: %w", err)
}
publicKey, ok := parsed.(*rsa.PublicKey)
if !ok {
return "", errors.New("Feichi public key is not RSA")
}
encrypted, err := rsa.EncryptPKCS1v15(rand.Reader, publicKey, []byte(password))
if err != nil {
return "", fmt.Errorf("encrypt Feichi password: %w", err)
}
return base64.StdEncoding.EncodeToString(encrypted), nil
}

View File

@@ -0,0 +1,94 @@
package feichibridge
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
)
var shanghai = time.FixedZone("Asia/Shanghai", 8*60*60)
type Vehicle struct {
VehicleID string `json:"vehicleId"`
VIN string `json:"vin"`
OnlineStatus any `json:"onlineStatus"`
UpdateTime string `json:"updateTime"`
RuleTypeName string `json:"ruleTypeName"`
}
type Snapshot struct {
VehicleID string `json:"vehicleId"`
VIN string `json:"vin"`
DataItems map[string]string `json:"dataItems"`
}
type Record map[string]string
type collectionEnvelope[T any] struct {
Code any `json:"code"`
Data []T `json:"data"`
}
type objectEnvelope[T any] struct {
Code any `json:"code"`
Data T `json:"data"`
}
type historySegment struct {
Subdata []Record `json:"subdata"`
}
func apiCodeOK(code any) bool {
switch value := code.(type) {
case nil:
return true
case float64:
return value == 0 || value == 200
case string:
return value == "" || value == "0" || value == "200"
default:
return false
}
}
func recordTime(record Record) (time.Time, error) {
for _, key := range []string{"2000", "9999", "dataTime", "updateTime", "time"} {
if raw := strings.TrimSpace(record[key]); raw != "" {
return parseSourceTime(raw)
}
}
return time.Time{}, fmt.Errorf("source record has no timestamp")
}
func parseSourceTime(raw string) (time.Time, error) {
raw = strings.TrimSpace(raw)
for _, layout := range []string{
"2006-01-02 15:04:05",
"2006-01-02T15:04:05",
time.RFC3339Nano,
} {
if layout == time.RFC3339Nano {
if parsed, err := time.Parse(layout, raw); err == nil {
return parsed, nil
}
continue
}
if parsed, err := time.ParseInLocation(layout, raw, shanghai); err == nil {
return parsed, nil
}
}
if unixMS, err := strconv.ParseInt(raw, 10, 64); err == nil {
if unixMS < 10_000_000_000 {
return time.Unix(unixMS, 0).In(shanghai), nil
}
return time.UnixMilli(unixMS).In(shanghai), nil
}
return time.Time{}, fmt.Errorf("unsupported source timestamp %q", raw)
}
func canonicalHash(record Record) string {
encoded, _ := json.Marshal(record)
return hashBytes(encoded)
}

View File

@@ -0,0 +1,469 @@
package feichibridge
import (
"context"
"errors"
"fmt"
"log/slog"
"sort"
"strings"
"sync"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
)
type Source interface {
Vehicles(context.Context) ([]Vehicle, error)
Snapshot(context.Context, string) (Snapshot, error)
History(context.Context, string, time.Time, time.Time) ([]Record, error)
}
type FrameTarget interface {
Connect(context.Context) error
Send(context.Context, []byte) error
Close() error
LastACK() time.Time
}
type ServiceConfig struct {
PollInterval time.Duration
DiscoveryInterval time.Duration
BackfillInterval time.Duration
BackfillLookback time.Duration
BackfillWindow time.Duration
BackfillSafetyLag time.Duration
SourceStaleAfter time.Duration
FetchConcurrency int
BackfillEnabled bool
StaleReissueEnabled bool
}
type Service struct {
config ServiceConfig
source Source
target FrameTarget
state *StateStore
encoder Encoder
logger *slog.Logger
metrics *metrics.Registry
mu sync.RWMutex
vehicles []Vehicle
lastSourceSuccess time.Time
lastError error
}
func NewService(config ServiceConfig, source Source, target FrameTarget, state *StateStore, logger *slog.Logger, registry *metrics.Registry) (*Service, error) {
if source == nil || target == nil || state == nil {
return nil, errors.New("source, target, and state are required")
}
if config.PollInterval <= 0 {
config.PollInterval = 10 * time.Second
}
if config.DiscoveryInterval <= 0 {
config.DiscoveryInterval = 5 * time.Minute
}
if config.BackfillInterval <= 0 {
config.BackfillInterval = time.Hour
}
if config.BackfillLookback <= 0 {
config.BackfillLookback = time.Hour
}
if config.BackfillWindow <= 0 {
config.BackfillWindow = 20 * time.Minute
}
if config.BackfillSafetyLag <= 0 {
config.BackfillSafetyLag = 30 * time.Second
}
if config.SourceStaleAfter <= 0 {
config.SourceStaleAfter = 2 * time.Minute
}
if config.FetchConcurrency <= 0 {
config.FetchConcurrency = 4
}
if logger == nil {
logger = slog.Default()
}
return &Service{
config: config, source: source, target: target, state: state,
logger: logger, metrics: registry,
}, nil
}
func (s *Service) Run(ctx context.Context) error {
if err := s.target.Connect(ctx); err != nil {
s.setError(err)
return fmt.Errorf("connect GB/T 32960 target: %w", err)
}
if err := s.discover(ctx); err != nil {
s.setError(err)
return fmt.Errorf("initial vehicle discovery: %w", err)
}
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
s.discoveryLoop(ctx)
}()
go func() {
defer wg.Done()
s.realtimeLoop(ctx)
}()
if s.config.BackfillEnabled {
wg.Add(1)
go func() {
defer wg.Done()
s.backfillLoop(ctx)
}()
}
<-ctx.Done()
_ = s.target.Close()
wg.Wait()
return nil
}
func (s *Service) Ready(context.Context) error {
s.mu.RLock()
lastSourceSuccess := s.lastSourceSuccess
lastErr := s.lastError
s.mu.RUnlock()
maxSourceAge := max(3*s.config.PollInterval, time.Minute)
if lastSourceSuccess.IsZero() || time.Since(lastSourceSuccess) > maxSourceAge {
if lastErr != nil {
return fmt.Errorf("source unavailable: %w", lastErr)
}
return errors.New("source has not completed a successful request")
}
lastACK := s.target.LastACK()
if lastACK.IsZero() || time.Since(lastACK) > max(3*s.config.DiscoveryInterval, 10*time.Minute) {
return errors.New("GB/T 32960 target has no recent ACK")
}
return nil
}
func (s *Service) discoveryLoop(ctx context.Context) {
ticker := time.NewTicker(s.config.DiscoveryInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := s.discover(ctx); err != nil {
s.recordFailure("discover", err)
}
}
}
}
func (s *Service) realtimeLoop(ctx context.Context) {
s.pollRealtime(ctx)
ticker := time.NewTicker(s.config.PollInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s.pollRealtime(ctx)
}
}
}
func (s *Service) backfillLoop(ctx context.Context) {
s.runBackfill(ctx)
ticker := time.NewTicker(s.config.BackfillInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s.runBackfill(ctx)
}
}
}
func (s *Service) discover(ctx context.Context) error {
start := time.Now()
vehicles, err := s.source.Vehicles(ctx)
s.observeAPI("vehicles", start, err)
if err != nil {
return err
}
filtered := make([]Vehicle, 0, len(vehicles))
for _, vehicle := range vehicles {
vehicle.VIN = strings.TrimSpace(vehicle.VIN)
vehicle.VehicleID = strings.TrimSpace(vehicle.VehicleID)
if len(vehicle.VIN) != 17 || vehicle.VehicleID == "" {
continue
}
if vehicle.RuleTypeName != "" && !strings.Contains(strings.ToUpper(vehicle.RuleTypeName), "32960") {
continue
}
filtered = append(filtered, vehicle)
}
sort.Slice(filtered, func(i, j int) bool { return filtered[i].VIN < filtered[j].VIN })
s.mu.Lock()
s.vehicles = filtered
s.lastSourceSuccess = time.Now()
s.lastError = nil
s.mu.Unlock()
if s.metrics != nil {
s.metrics.SetGauge("vehicle_feichi_bridge_vehicles", nil, float64(len(filtered)))
}
s.logger.Info("feichi vehicles discovered", "count", len(filtered))
return nil
}
type fetchedSnapshot struct {
vehicle Vehicle
record Record
at time.Time
hash string
err error
duration time.Duration
}
func (s *Service) pollRealtime(ctx context.Context) {
vehicles := s.vehicleSnapshot()
jobs := make(chan Vehicle)
results := make(chan fetchedSnapshot, len(vehicles))
var wg sync.WaitGroup
for worker := 0; worker < min(s.config.FetchConcurrency, len(vehicles)); worker++ {
wg.Add(1)
go func() {
defer wg.Done()
for vehicle := range jobs {
start := time.Now()
snapshot, err := s.source.Snapshot(ctx, vehicle.VehicleID)
result := fetchedSnapshot{vehicle: vehicle, duration: time.Since(start), err: err}
if err == nil {
result.record = Record(snapshot.DataItems)
result.at, result.err = recordTime(result.record)
result.hash = canonicalHash(result.record)
}
results <- result
}
}()
}
for _, vehicle := range vehicles {
jobs <- vehicle
}
close(jobs)
wg.Wait()
close(results)
var snapshots []fetchedSnapshot
for result := range results {
s.observeAPIWithDuration("snapshot", result.duration, result.err)
if result.err != nil {
s.recordFailure("snapshot", fmt.Errorf("VIN %s: %w", result.vehicle.VIN, result.err))
continue
}
snapshots = append(snapshots, result)
s.markSourceSuccess()
}
sort.Slice(snapshots, func(i, j int) bool {
if snapshots[i].at.Equal(snapshots[j].at) {
return snapshots[i].vehicle.VIN < snapshots[j].vehicle.VIN
}
return snapshots[i].at.Before(snapshots[j].at)
})
for _, snapshot := range snapshots {
if time.Since(snapshot.at) > s.config.SourceStaleAfter {
if s.config.StaleReissueEnabled {
s.reissueStaleSnapshot(ctx, snapshot)
}
continue
}
current := s.state.Vehicle(snapshot.vehicle.VIN)
if snapshot.at.Before(current.LastRealtimeTime) ||
(snapshot.at.Equal(current.LastRealtimeTime) && snapshot.hash == current.LastRealtimeHash) {
continue
}
frame, err := s.encoder.DataFrame(CommandRealtime, snapshot.vehicle.VIN, snapshot.at, snapshot.record)
if err != nil {
s.recordFailure("encode", fmt.Errorf("VIN %s: %w", snapshot.vehicle.VIN, err))
continue
}
if err := s.target.Send(ctx, frame); err != nil {
s.recordFrame(CommandRealtime, "error", snapshot.vehicle.VIN, snapshot.at)
s.recordFailure("send", fmt.Errorf("VIN %s: %w", snapshot.vehicle.VIN, err))
continue
}
if err := s.state.CommitRealtime(snapshot.vehicle.VIN, snapshot.at, snapshot.hash); err != nil {
s.recordFailure("state", err)
continue
}
s.recordFrame(CommandRealtime, "acked", snapshot.vehicle.VIN, snapshot.at)
}
}
func (s *Service) reissueStaleSnapshot(ctx context.Context, snapshot fetchedSnapshot) {
current := s.state.Vehicle(snapshot.vehicle.VIN)
if snapshot.at.Before(current.LastSnapshotReissueTime) ||
(snapshot.at.Equal(current.LastSnapshotReissueTime) && snapshot.hash == current.LastSnapshotReissueHash) {
return
}
frame, err := s.encoder.DataFrame(CommandReissue, snapshot.vehicle.VIN, snapshot.at, snapshot.record)
if err != nil {
s.recordFailure("encode_stale_reissue", fmt.Errorf("VIN %s: %w", snapshot.vehicle.VIN, err))
return
}
if err := s.target.Send(ctx, frame); err != nil {
s.recordFrame(CommandReissue, "error", snapshot.vehicle.VIN, snapshot.at)
s.recordFailure("send_stale_reissue", fmt.Errorf("VIN %s: %w", snapshot.vehicle.VIN, err))
return
}
if err := s.state.CommitSnapshotReissue(snapshot.vehicle.VIN, snapshot.at, snapshot.hash); err != nil {
s.recordFailure("state", err)
return
}
s.recordFrame(CommandReissue, "acked", snapshot.vehicle.VIN, snapshot.at)
}
func (s *Service) runBackfill(ctx context.Context) {
for _, vehicle := range s.vehicleSnapshot() {
if ctx.Err() != nil {
return
}
if err := s.backfillVehicle(ctx, vehicle); err != nil {
s.recordFailure("backfill", fmt.Errorf("VIN %s: %w", vehicle.VIN, err))
}
}
}
func (s *Service) backfillVehicle(ctx context.Context, vehicle Vehicle) error {
end := time.Now().Add(-s.config.BackfillSafetyLag)
cursor := s.state.Vehicle(vehicle.VIN).BackfillCursor
if cursor.IsZero() {
cursor = end.Add(-s.config.BackfillLookback)
}
for cursor.Before(end) {
windowEnd := cursor.Add(s.config.BackfillWindow)
if windowEnd.After(end) {
windowEnd = end
}
start := time.Now()
records, err := s.source.History(ctx, vehicle.VIN, cursor, windowEnd)
s.observeAPI("history", start, err)
if err != nil {
return err
}
sort.Slice(records, func(i, j int) bool {
left, _ := recordTime(records[i])
right, _ := recordTime(records[j])
return left.Before(right)
})
committed := cursor
for _, record := range records {
at, err := recordTime(record)
if err != nil || !at.After(cursor) || at.After(windowEnd) {
continue
}
frame, err := s.encoder.DataFrame(CommandReissue, vehicle.VIN, at, record)
if err != nil {
return err
}
if err := s.target.Send(ctx, frame); err != nil {
s.recordFrame(CommandReissue, "error", vehicle.VIN, at)
return err
}
if err := s.state.CommitBackfill(vehicle.VIN, at); err != nil {
return err
}
s.recordFrame(CommandReissue, "acked", vehicle.VIN, at)
committed = at
}
if !committed.After(cursor) || committed.Before(windowEnd) {
if err := s.state.AdvanceBackfillCursor(vehicle.VIN, windowEnd); err != nil {
return err
}
}
cursor = windowEnd
s.markSourceSuccess()
}
return nil
}
func (s *Service) vehicleSnapshot() []Vehicle {
s.mu.RLock()
defer s.mu.RUnlock()
return append([]Vehicle(nil), s.vehicles...)
}
func (s *Service) markSourceSuccess() {
s.mu.Lock()
s.lastSourceSuccess = time.Now()
s.lastError = nil
s.mu.Unlock()
if s.metrics != nil {
metrics.RecordLastActivity(s.metrics, "vehicle_feichi_bridge_source_last_success_unix_seconds", nil)
}
}
func (s *Service) setError(err error) {
s.mu.Lock()
s.lastError = err
s.mu.Unlock()
}
func (s *Service) recordFailure(operation string, err error) {
if err == nil {
return
}
s.setError(err)
if s.metrics != nil {
s.metrics.IncCounter("vehicle_feichi_bridge_errors_total", metrics.Labels{"operation": operation})
}
s.logger.Error("feichi bridge operation failed", "operation", operation, "error", err)
}
func (s *Service) observeAPI(operation string, start time.Time, err error) {
s.observeAPIWithDuration(operation, time.Since(start), err)
}
func (s *Service) observeAPIWithDuration(operation string, duration time.Duration, err error) {
if s.metrics == nil {
return
}
status := "ok"
if err != nil {
status = "error"
}
s.metrics.IncCounter("vehicle_feichi_bridge_api_requests_total", metrics.Labels{"operation": operation, "status": status})
s.metrics.ObserveHistogram(
"vehicle_feichi_bridge_api_request_duration_seconds",
metrics.Labels{"operation": operation},
[]float64{0.1, 0.25, 0.5, 1, 2, 5},
duration.Seconds(),
)
}
func (s *Service) recordFrame(command byte, status, vin string, sourceTime time.Time) {
if s.metrics != nil {
s.metrics.IncCounter("vehicle_feichi_bridge_frames_total", metrics.Labels{
"command": fmt.Sprintf("0x%02X", command),
"status": status,
})
}
if status == "acked" {
if s.metrics != nil {
metrics.RecordLastActivity(s.metrics, "vehicle_feichi_bridge_target_last_ack_unix_seconds", nil)
s.metrics.SetGauge(
"vehicle_feichi_bridge_vehicle_last_ack_unix_seconds",
metrics.Labels{"vin": vin, "command": fmt.Sprintf("0x%02X", command)},
float64(time.Now().Unix()),
)
}
s.logger.Info(
"GB/T 32960 vehicle frame acknowledged",
"vin", vin,
"command", fmt.Sprintf("0x%02X", command),
"source_time", sourceTime,
)
}
}

View File

@@ -0,0 +1,123 @@
package feichibridge
import (
"context"
"errors"
"log/slog"
"path/filepath"
"sync"
"testing"
"time"
)
type fakeSource struct {
vehicles []Vehicle
record Record
}
func (f *fakeSource) Vehicles(context.Context) ([]Vehicle, error) {
return append([]Vehicle(nil), f.vehicles...), nil
}
func (f *fakeSource) Snapshot(context.Context, string) (Snapshot, error) {
return Snapshot{DataItems: map[string]string(f.record)}, nil
}
func (f *fakeSource) History(context.Context, string, time.Time, time.Time) ([]Record, error) {
return nil, nil
}
type fakeTarget struct {
mu sync.Mutex
frames [][]byte
sendErr error
lastACK time.Time
}
func (f *fakeTarget) Connect(context.Context) error { return nil }
func (f *fakeTarget) Close() error { return nil }
func (f *fakeTarget) LastACK() time.Time { return f.lastACK }
func (f *fakeTarget) Send(_ context.Context, frame []byte) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.sendErr != nil {
return f.sendErr
}
f.frames = append(f.frames, append([]byte(nil), frame...))
f.lastACK = time.Now()
return nil
}
func TestRealtimeCommitsOnlyAfterACKAndDeduplicates(t *testing.T) {
now := time.Now().In(shanghai).Truncate(time.Second)
source := &fakeSource{
vehicles: []Vehicle{{
VehicleID: "id-1", VIN: "LTEST32960VIN0001", RuleTypeName: "GB_T32960",
}},
record: Record{"2000": now.Format("2006-01-02 15:04:05"), "2201": "10"},
}
target := &fakeTarget{}
store, err := OpenStateStore(filepath.Join(t.TempDir(), "state.json"))
if err != nil {
t.Fatal(err)
}
service, err := NewService(ServiceConfig{
SourceStaleAfter: time.Minute,
FetchConcurrency: 1,
}, source, target, store, slog.Default(), nil)
if err != nil {
t.Fatal(err)
}
if err := service.discover(context.Background()); err != nil {
t.Fatal(err)
}
service.pollRealtime(context.Background())
service.pollRealtime(context.Background())
if len(target.frames) != 1 {
t.Fatalf("frames = %d, want 1", len(target.frames))
}
if got := store.Vehicle("LTEST32960VIN0001"); !got.LastRealtimeTime.Equal(now) {
t.Fatalf("committed state = %#v", got)
}
source.record = Record{"2000": now.Add(time.Second).Format("2006-01-02 15:04:05"), "2201": "11"}
target.sendErr = errors.New("target down")
service.pollRealtime(context.Background())
if got := store.Vehicle("LTEST32960VIN0001"); !got.LastRealtimeTime.Equal(now) {
t.Fatalf("cursor advanced without ACK: %#v", got)
}
}
func TestStaleSnapshotIsReissuedOnlyOnce(t *testing.T) {
at := time.Now().In(shanghai).Add(-24 * time.Hour).Truncate(time.Second)
vin := "LTEST32960VIN0002"
source := &fakeSource{
vehicles: []Vehicle{{VehicleID: "id-2", VIN: vin, RuleTypeName: "GB_T32960"}},
record: Record{"2000": at.Format("2006-01-02 15:04:05"), "2201": "0"},
}
target := &fakeTarget{}
store, err := OpenStateStore(filepath.Join(t.TempDir(), "state.json"))
if err != nil {
t.Fatal(err)
}
service, err := NewService(ServiceConfig{
SourceStaleAfter: time.Minute,
FetchConcurrency: 1,
StaleReissueEnabled: true,
}, source, target, store, slog.Default(), nil)
if err != nil {
t.Fatal(err)
}
if err := service.discover(context.Background()); err != nil {
t.Fatal(err)
}
service.pollRealtime(context.Background())
service.pollRealtime(context.Background())
if len(target.frames) != 1 || target.frames[0][2] != CommandReissue {
t.Fatalf("frames = %d command = %#v", len(target.frames), target.frames)
}
state := store.Vehicle(vin)
if !state.LastSnapshotReissueTime.Equal(at) || state.LastSnapshotReissueACKAt.IsZero() {
t.Fatalf("reissue state = %#v", state)
}
}

View File

@@ -0,0 +1,160 @@
package feichibridge
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"time"
)
type VehicleState struct {
LastRealtimeTime time.Time `json:"last_realtime_time,omitempty"`
LastRealtimeHash string `json:"last_realtime_hash,omitempty"`
LastRealtimeACKAt time.Time `json:"last_realtime_ack_at,omitempty"`
LastSnapshotReissueTime time.Time `json:"last_snapshot_reissue_time,omitempty"`
LastSnapshotReissueHash string `json:"last_snapshot_reissue_hash,omitempty"`
LastSnapshotReissueACKAt time.Time `json:"last_snapshot_reissue_ack_at,omitempty"`
BackfillCursor time.Time `json:"backfill_cursor,omitempty"`
LastBackfillACKAt time.Time `json:"last_backfill_ack_at,omitempty"`
}
func (s *StateStore) CommitSnapshotReissue(vin string, at time.Time, hash string) error {
s.mu.Lock()
defer s.mu.Unlock()
current := s.state.Vehicles[vin]
current.LastSnapshotReissueTime = at
current.LastSnapshotReissueHash = hash
current.LastSnapshotReissueACKAt = time.Now()
s.state.Vehicles[vin] = current
return s.saveLocked()
}
type persistentState struct {
Version int `json:"version"`
PlatformSerial uint16 `json:"platform_serial"`
Vehicles map[string]VehicleState `json:"vehicles"`
}
type StateStore struct {
mu sync.Mutex
path string
state persistentState
}
func OpenStateStore(path string) (*StateStore, error) {
store := &StateStore{
path: path,
state: persistentState{
Version: 1,
Vehicles: map[string]VehicleState{},
},
}
encoded, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
return store, nil
}
if err != nil {
return nil, fmt.Errorf("read bridge state: %w", err)
}
if err := json.Unmarshal(encoded, &store.state); err != nil {
return nil, fmt.Errorf("decode bridge state: %w", err)
}
if store.state.Vehicles == nil {
store.state.Vehicles = map[string]VehicleState{}
}
return store, nil
}
func (s *StateStore) Vehicle(vin string) VehicleState {
s.mu.Lock()
defer s.mu.Unlock()
return s.state.Vehicles[vin]
}
func (s *StateStore) CommitRealtime(vin string, at time.Time, hash string) error {
s.mu.Lock()
defer s.mu.Unlock()
current := s.state.Vehicles[vin]
current.LastRealtimeTime = at
current.LastRealtimeHash = hash
current.LastRealtimeACKAt = time.Now()
s.state.Vehicles[vin] = current
return s.saveLocked()
}
func (s *StateStore) CommitBackfill(vin string, at time.Time) error {
s.mu.Lock()
defer s.mu.Unlock()
current := s.state.Vehicles[vin]
current.BackfillCursor = at
current.LastBackfillACKAt = time.Now()
s.state.Vehicles[vin] = current
return s.saveLocked()
}
func (s *StateStore) AdvanceBackfillCursor(vin string, at time.Time) error {
s.mu.Lock()
defer s.mu.Unlock()
current := s.state.Vehicles[vin]
current.BackfillCursor = at
s.state.Vehicles[vin] = current
return s.saveLocked()
}
func (s *StateStore) NextPlatformSerial() (uint16, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.state.PlatformSerial++
if s.state.PlatformSerial == 0 {
s.state.PlatformSerial = 1
}
if err := s.saveLocked(); err != nil {
return 0, err
}
return s.state.PlatformSerial, nil
}
func (s *StateStore) saveLocked() error {
if err := os.MkdirAll(filepath.Dir(s.path), 0o750); err != nil {
return fmt.Errorf("create bridge state directory: %w", err)
}
encoded, err := json.MarshalIndent(s.state, "", " ")
if err != nil {
return err
}
temp, err := os.CreateTemp(filepath.Dir(s.path), ".state-*.json")
if err != nil {
return err
}
tempName := temp.Name()
defer os.Remove(tempName)
if err := temp.Chmod(0o600); err != nil {
temp.Close()
return err
}
if _, err := temp.Write(encoded); err != nil {
temp.Close()
return err
}
if err := temp.Sync(); err != nil {
temp.Close()
return err
}
if err := temp.Close(); err != nil {
return err
}
if err := os.Rename(tempName, s.path); err != nil {
return fmt.Errorf("replace bridge state: %w", err)
}
return nil
}
func hashBytes(value []byte) string {
sum := sha256.Sum256(value)
return hex.EncodeToString(sum[:])
}

View File

@@ -0,0 +1,38 @@
package feichibridge
import (
"os"
"path/filepath"
"testing"
"time"
)
func TestStateStorePersistsAcknowledgedCursor(t *testing.T) {
path := filepath.Join(t.TempDir(), "nested", "state.json")
store, err := OpenStateStore(path)
if err != nil {
t.Fatal(err)
}
at := time.Date(2026, 7, 17, 12, 0, 0, 0, time.UTC)
if err := store.CommitRealtime("LTEST32960VIN0001", at, "hash"); err != nil {
t.Fatal(err)
}
if err := store.CommitBackfill("LTEST32960VIN0001", at.Add(-time.Minute)); err != nil {
t.Fatal(err)
}
reopened, err := OpenStateStore(path)
if err != nil {
t.Fatal(err)
}
got := reopened.Vehicle("LTEST32960VIN0001")
if !got.LastRealtimeTime.Equal(at) || got.LastRealtimeHash != "hash" {
t.Fatalf("state = %#v", got)
}
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0o600 {
t.Fatalf("state mode = %o", info.Mode().Perm())
}
}

View File

@@ -0,0 +1,193 @@
package feichibridge
import (
"context"
"encoding/binary"
"errors"
"fmt"
"io"
"net"
"strings"
"sync"
"time"
)
type TargetConfig struct {
Address string
PlatformID string
Username string
Password string
Timeout time.Duration
}
type Target struct {
mu sync.Mutex
config TargetConfig
state *StateStore
dialer net.Dialer
conn net.Conn
lastACK time.Time
}
func NewTarget(config TargetConfig, state *StateStore) (*Target, error) {
if strings.TrimSpace(config.Address) == "" {
return nil, errors.New("GB/T 32960 target address is required")
}
if len(config.PlatformID) != 17 {
return nil, fmt.Errorf("target platform ID must be exactly 17 bytes")
}
if state == nil {
return nil, errors.New("state store is required")
}
if config.Timeout <= 0 {
config.Timeout = 10 * time.Second
}
return &Target{
config: config,
state: state,
dialer: net.Dialer{Timeout: config.Timeout, KeepAlive: 30 * time.Second},
}, nil
}
func (t *Target) Send(ctx context.Context, frame []byte) error {
t.mu.Lock()
defer t.mu.Unlock()
var lastErr error
for attempt := 0; attempt < 2; attempt++ {
if err := t.ensureConnected(ctx); err != nil {
lastErr = err
t.closeLocked()
continue
}
if err := t.sendAndACK(ctx, frame); err != nil {
lastErr = err
t.closeLocked()
continue
}
t.lastACK = time.Now()
return nil
}
return fmt.Errorf("send GB/T 32960 frame after reconnect: %w", lastErr)
}
func (t *Target) Connect(ctx context.Context) error {
t.mu.Lock()
defer t.mu.Unlock()
if err := t.ensureConnected(ctx); err != nil {
t.closeLocked()
return err
}
return nil
}
func (t *Target) Close() error {
t.mu.Lock()
defer t.mu.Unlock()
if t.conn == nil {
return nil
}
err := t.conn.Close()
t.conn = nil
return err
}
func (t *Target) LastACK() time.Time {
t.mu.Lock()
defer t.mu.Unlock()
return t.lastACK
}
func (t *Target) ensureConnected(ctx context.Context) error {
if t.conn != nil {
return nil
}
conn, err := t.dialer.DialContext(ctx, "tcp", t.config.Address)
if err != nil {
return fmt.Errorf("dial target %s: %w", t.config.Address, err)
}
t.conn = conn
serial, err := t.state.NextPlatformSerial()
if err != nil {
return fmt.Errorf("allocate platform login serial: %w", err)
}
login, err := LoginFrame(t.config.PlatformID, t.config.Username, t.config.Password, serial, time.Now())
if err != nil {
return err
}
if err := t.sendAndACK(ctx, login); err != nil {
return fmt.Errorf("GB/T 32960 platform login: %w", err)
}
t.lastACK = time.Now()
return nil
}
func (t *Target) sendAndACK(ctx context.Context, frame []byte) error {
if t.conn == nil {
return errors.New("target connection is closed")
}
deadline := time.Now().Add(t.config.Timeout)
if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) {
deadline = contextDeadline
}
if err := t.conn.SetDeadline(deadline); err != nil {
return err
}
if err := writeFull(t.conn, frame); err != nil {
return fmt.Errorf("write target frame: %w", err)
}
response, err := readFrame(t.conn)
if err != nil {
return fmt.Errorf("read target ACK: %w", err)
}
if response[2] != frame[2] {
return fmt.Errorf("target ACK command mismatch: got 0x%02X want 0x%02X", response[2], frame[2])
}
if response[3] != 0x01 {
return fmt.Errorf("target rejected command 0x%02X with response 0x%02X", response[2], response[3])
}
if string(response[4:21]) != string(frame[4:21]) {
return errors.New("target ACK identifier mismatch")
}
return nil
}
func (t *Target) closeLocked() {
if t.conn != nil {
_ = t.conn.Close()
t.conn = nil
}
}
func readFrame(reader io.Reader) ([]byte, error) {
header := make([]byte, 24)
if _, err := io.ReadFull(reader, header); err != nil {
return nil, err
}
if (header[0] != '#' || header[1] != '#') && (header[0] != '$' || header[1] != '$') {
return nil, fmt.Errorf("bad GB/T 32960 ACK start %q", header[:2])
}
bodyLength := int(binary.BigEndian.Uint16(header[22:24]))
tail := make([]byte, bodyLength+1)
if _, err := io.ReadFull(reader, tail); err != nil {
return nil, err
}
frame := append(header, tail...)
if got, want := bcc(frame[2:len(frame)-1]), frame[len(frame)-1]; got != want {
return nil, fmt.Errorf("bad GB/T 32960 ACK BCC: got 0x%02X want 0x%02X", got, want)
}
return frame, nil
}
func writeFull(writer io.Writer, value []byte) error {
for len(value) > 0 {
written, err := writer.Write(value)
if err != nil {
return err
}
if written == 0 {
return io.ErrShortWrite
}
value = value[written:]
}
return nil
}

View File

@@ -0,0 +1,83 @@
package feichibridge
import (
"context"
"net"
"path/filepath"
"sync"
"testing"
"time"
)
func TestTargetLogsInAndWaitsForACK(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
var commands []byte
var mu sync.Mutex
done := make(chan error, 1)
go func() {
conn, acceptErr := listener.Accept()
if acceptErr != nil {
done <- acceptErr
return
}
defer conn.Close()
for count := 0; count < 2; count++ {
frame, readErr := readFrame(conn)
if readErr != nil {
done <- readErr
return
}
mu.Lock()
commands = append(commands, frame[2])
mu.Unlock()
body := []byte(nil)
if len(frame) >= 31 {
body = append(body, frame[24:30]...)
}
ack, buildErr := buildFrame('#', frame[2], 0x01, string(frame[4:21]), body)
if buildErr != nil {
done <- buildErr
return
}
if writeErr := writeFull(conn, ack); writeErr != nil {
done <- writeErr
return
}
}
done <- nil
}()
store, err := OpenStateStore(filepath.Join(t.TempDir(), "state.json"))
if err != nil {
t.Fatal(err)
}
target, err := NewTarget(TargetConfig{
Address: listener.Addr().String(), PlatformID: "FEICHIBRIDGE00001",
Username: "bridge", Password: "secret", Timeout: time.Second,
}, store)
if err != nil {
t.Fatal(err)
}
frame, err := (Encoder{}).DataFrame(CommandRealtime, "LTEST32960VIN0001", time.Now(), Record{"2201": "1"})
if err != nil {
t.Fatal(err)
}
if err := target.Send(context.Background(), frame); err != nil {
t.Fatal(err)
}
if err := <-done; err != nil {
t.Fatal(err)
}
mu.Lock()
defer mu.Unlock()
if len(commands) != 2 || commands[0] != CommandLogin || commands[1] != CommandRealtime {
t.Fatalf("commands = %x", commands)
}
if target.LastACK().IsZero() {
t.Fatal("last ACK was not recorded")
}
}

View File

@@ -151,16 +151,23 @@ func parseDataBody(version string, body []byte, fields map[string]any) (time.Tim
fields[envelope.FieldSOCPercent] = unit["soc_percent"]
cursor += size
case 0x05:
if len(body[cursor:]) < 9 {
size := 9
if version == "V2025" {
size = 10
}
if len(body[cursor:]) < size {
units = append(units, map[string]any{"type": "0x05", "error": "truncated"})
return eventTime, units
}
unit := parsePositionData(body[cursor : cursor+9])
unit := parsePositionData(version, body[cursor:cursor+size])
units = append(units, map[string]any{"type": "0x05", "name": "position", "value": unit})
fields["position_status"] = unit["position_status"]
if coordinateSystem, ok := unit["coordinate_system"]; ok {
fields["coordinate_system"] = coordinateSystem
}
fields[envelope.FieldLongitude] = unit["longitude"]
fields[envelope.FieldLatitude] = unit["latitude"]
cursor += 9
cursor += size
case 0x02:
if len(body[cursor:]) < 1 {
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
@@ -364,21 +371,27 @@ func parsePlatformLogin(body []byte) map[string]any {
}
func parseVehicleData(data []byte) map[string]any {
return map[string]any{
out := map[string]any{
"vehicle_status": int(data[0]),
"charge_status": int(data[1]),
"running_mode": int(data[2]),
"speed_kmh": scaledU16(data[3:5], 10),
"total_mileage_km": scaledU32(data[5:9], 10),
"total_voltage_v": scaledU16(data[9:11], 10),
"total_current_a": scaledU16WithOffset(data[11:13], 10, -1000),
"soc_percent": int(data[13]),
"speed_kmh": nullableScaledU16(data[3:5], 10),
"total_mileage_km": nullableScaledU32(data[5:9], 10),
"total_voltage_v": nullableScaledU16(data[9:11], 10),
"total_current_a": nullableScaledU16WithOffset(data[11:13], 10, -1000),
"soc_percent": nullableByte(data[13]),
"dc_dc_status": int(data[14]),
"gear": int(data[15]),
"insulation_kohm": binary.BigEndian.Uint16(data[16:18]),
"accelerator_pct": int(data[18]),
"brake_pct": int(data[19]),
"insulation_kohm": nullableU16(data[16:18]),
}
// GB/T 32960.3-2025 removes the 2016 accelerator and brake bytes from
// the 0x01 vehicle unit. Keep them only when the frame actually carries
// the 20-byte 2016 payload.
if len(data) >= 20 {
out["accelerator_pct"] = nullableByte(data[18])
out["brake_pct"] = nullableByte(data[19])
}
return out
}
func parseDriveMotorData(data []byte) map[string]any {
@@ -389,12 +402,12 @@ func parseDriveMotorData(data []byte) map[string]any {
motors = append(motors, map[string]any{
"serial_no": int(data[cursor]),
"state": int(data[cursor+1]),
"controller_temperature_c": int(data[cursor+2]) - 40,
"speed_rpm": int(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) - 20000,
"torque_nm": scaledIntWithOffset(int64(binary.BigEndian.Uint16(data[cursor+5:cursor+7])), 10, -20000),
"motor_temperature_c": int(data[cursor+7]) - 40,
"controller_voltage_v": scaledU16(data[cursor+8:cursor+10], 10),
"controller_current_a": scaledU16WithOffset(data[cursor+10:cursor+12], 10, -1000),
"controller_temperature_c": nullableTempOffset40(data[cursor+2]),
"speed_rpm": nullableU16WithOffset(data[cursor+3:cursor+5], -20000),
"torque_nm": nullableScaledU16WithOffset(data[cursor+5:cursor+7], 10, -20000),
"motor_temperature_c": nullableTempOffset40(data[cursor+7]),
"controller_voltage_v": nullableScaledU16(data[cursor+8:cursor+10], 10),
"controller_current_a": nullableScaledU16WithOffset(data[cursor+10:cursor+12], 10, -1000),
})
cursor += 12
}
@@ -406,34 +419,45 @@ func parseDriveMotorData(data []byte) map[string]any {
func parseFuelCellData(data []byte) map[string]any {
probeCount := int(binary.BigEndian.Uint16(data[6:8]))
probes := make([]int, 0, probeCount)
probes := make([]any, 0, probeCount)
cursor := 8
for i := 0; i < probeCount; i++ {
probes = append(probes, int(data[cursor])-40)
probes = append(probes, nullableTempOffset40(data[cursor]))
cursor++
}
out := map[string]any{
"fuel_cell_voltage_v": scaledU16(data[0:2], 10),
"fuel_cell_current_a": scaledU16(data[2:4], 10),
"hydrogen_consumption_kg_per_100km": scaledU16(data[4:6], 100),
"fuel_cell_voltage_v": nullableScaledU16(data[0:2], 10),
"fuel_cell_current_a": nullableScaledU16(data[2:4], 10),
"hydrogen_consumption_kg_per_100km": nullableScaledU16(data[4:6], 100),
"temperature_probe_count": probeCount,
"temperature_probe_values_c": probes,
"max_hydrogen_temperature_c": scaledU16WithOffset(data[cursor:cursor+2], 10, -40),
"max_hydrogen_temperature_c": nullableScaledU16WithOffset(data[cursor:cursor+2], 10, -40),
"max_hydrogen_temperature_probe_id": int(data[cursor+2]),
"max_hydrogen_concentration_fraction": scaledU16(data[cursor+3:cursor+5], 1_000_000),
"max_hydrogen_concentration_probe_id": int(data[cursor+5]),
"max_hydrogen_pressure_mpa": scaledU16(data[cursor+6:cursor+8], 10),
"max_hydrogen_pressure_mpa": nullableScaledU16(data[cursor+6:cursor+8], 10),
"max_hydrogen_pressure_probe_id": int(data[cursor+8]),
"dc_dc_status": int(data[cursor+9]),
}
if concentration := nullableU16(data[cursor+3 : cursor+5]); concentration != nil {
raw := concentration.(int)
out["max_hydrogen_concentration_fraction"] = float64(raw) / 1_000_000
out["max_hydrogen_concentration_ppm"] = raw
out["max_hydrogen_concentration_percent"] = float64(raw) / 10_000
} else {
out["max_hydrogen_concentration_fraction"] = nil
out["max_hydrogen_concentration_ppm"] = nil
out["max_hydrogen_concentration_percent"] = nil
}
return out
}
func parseEngineData(data []byte) map[string]any {
return map[string]any{
"engine_status": int(data[0]),
"crank_speed_rpm": binary.BigEndian.Uint16(data[1:3]),
"fuel_rate": binary.BigEndian.Uint16(data[3:5]),
"crank_speed_rpm": nullableU16(data[1:3]),
// The supplied authoritative reference does not define this field's
// business unit or scale, so preserve the decoded protocol integer.
"fuel_rate": nullableU16(data[3:5]),
}
}
@@ -632,11 +656,13 @@ func parseGdFCStackData(data []byte) map[string]any {
"frame_cell_count": frameCellCount,
}
cursor += 22
frameVoltages := make([]float64, 0, frameCellCount)
maxCell := 0.0
minCell := 0.0
seen := false
for c := 0; c < frameCellCount; c++ {
voltage := scaledU16(data[cursor:cursor+2], 1000)
frameVoltages = append(frameVoltages, voltage)
if !seen || voltage > maxCell {
maxCell = voltage
}
@@ -650,6 +676,7 @@ func parseGdFCStackData(data []byte) map[string]any {
summary["frame_max_cell_voltage_v"] = maxCell
summary["frame_min_cell_voltage_v"] = minCell
}
summary["frame_cell_voltages_v"] = frameVoltages
summaries = append(summaries, summary)
}
return map[string]any{
@@ -760,6 +787,13 @@ func nullableU16WithOffset(data []byte, offset int) any {
return int(value) + offset
}
func nullableByte(value byte) any {
if value == 0xfe || value == 0xff {
return nil
}
return int(value)
}
func scaledU16(data []byte, divisor int64) float64 {
return scaledInt(int64(binary.BigEndian.Uint16(data)), divisor)
}
@@ -791,6 +825,14 @@ func nullableScaledU16(data []byte, divisor int64) any {
return scaledInt(int64(value), divisor)
}
func nullableScaledU32(data []byte, divisor int64) any {
value := binary.BigEndian.Uint32(data)
if value == 0xfffffffe || value == 0xffffffff {
return nil
}
return scaledInt(int64(value), divisor)
}
func nullableScaledU16WithOffset(data []byte, divisor int64, offset int64) any {
value := binary.BigEndian.Uint16(data)
if value == 0xfffe || value == 0xffff {
@@ -806,12 +848,16 @@ func nullableTempOffset40(value byte) any {
return int(value) - 40
}
func parsePositionData(data []byte) map[string]any {
return map[string]any{
"position_status": int(data[0]),
"longitude": float64(binary.BigEndian.Uint32(data[1:5])) / 1_000_000,
"latitude": float64(binary.BigEndian.Uint32(data[5:9])) / 1_000_000,
func parsePositionData(version string, data []byte) map[string]any {
offset := 1
out := map[string]any{"position_status": int(data[0])}
if version == "V2025" {
out["coordinate_system"] = int(data[1])
offset = 2
}
out["longitude"] = float64(binary.BigEndian.Uint32(data[offset:offset+4])) / 1_000_000
out["latitude"] = float64(binary.BigEndian.Uint32(data[offset+4:offset+8])) / 1_000_000
return out
}
func indexStart(data []byte) int {

View File

@@ -222,6 +222,13 @@ func TestParseFrameKeepsParsingRealFuelCellReportUntilUnknownExtension(t *testin
if units[2]["name"] != "fuel_cell" {
t.Fatalf("unexpected fuel cell unit: %#v", units[2])
}
fuelCell, ok := units[2]["value"].(map[string]any)
if !ok {
t.Fatalf("fuel cell payload missing: %#v", units[2])
}
if _, ok := fuelCell["max_hydrogen_concentration_ppm"].(int); !ok {
t.Fatalf("hydrogen concentration ppm missing: %#v", fuelCell["max_hydrogen_concentration_ppm"])
}
if units[9]["name"] != "gd_fc_stack" {
t.Fatalf("unexpected gd stack unit: %#v", units[9])
}
@@ -237,6 +244,69 @@ func TestParseFrameKeepsParsingRealFuelCellReportUntilUnknownExtension(t *testin
}
}
func TestParseFuelCellDataExposesHydrogenConcentrationInReferencePercent(t *testing.T) {
data := make([]byte, 18)
data[11] = 0x01
data[12] = 0xf4
fuelCell := parseFuelCellData(data)
if concentration, ok := fuelCell["max_hydrogen_concentration_ppm"].(int); !ok || concentration != 500 {
t.Fatalf("hydrogen concentration ppm = %#v, want int(500)", fuelCell["max_hydrogen_concentration_ppm"])
}
if fraction, ok := fuelCell["max_hydrogen_concentration_fraction"].(float64); !ok || math.Abs(fraction-0.0005) > 0.0000001 {
t.Fatalf("hydrogen concentration fraction = %#v, want 0.0005", fuelCell["max_hydrogen_concentration_fraction"])
}
if percent, ok := fuelCell["max_hydrogen_concentration_percent"].(float64); !ok || math.Abs(percent-0.05) > 0.0000001 {
t.Fatalf("hydrogen concentration percent = %#v, want 0.05", fuelCell["max_hydrogen_concentration_percent"])
}
}
func TestParseFuelCellDataTreatsProtocolInvalidMarkersAsMissing(t *testing.T) {
data := make([]byte, 18)
for index := range data {
data[index] = 0xff
}
data[6], data[7] = 0, 0
fuelCell := parseFuelCellData(data)
for _, key := range []string{"fuel_cell_voltage_v", "fuel_cell_current_a", "hydrogen_consumption_kg_per_100km", "max_hydrogen_temperature_c", "max_hydrogen_concentration_percent", "max_hydrogen_pressure_mpa"} {
if fuelCell[key] != nil {
t.Fatalf("%s should be nil for protocol invalid marker: %#v", key, fuelCell[key])
}
}
}
func TestParseV2025VehicleAndPositionUsesCoordinateSystemByte(t *testing.T) {
body := []byte{0x1a, 0x07, 0x11, 0x0a, 0x00, 0x00, 0x01}
body = append(body,
0x01, 0x03, 0x01,
0x00, 0x64,
0x00, 0x00, 0x03, 0xe8,
0x13, 0x88,
0x27, 0x10,
80, 0x01, 0x00,
0x03, 0xe8)
body = append(body,
0x05,
0x00, 0x02,
0x07, 0x36, 0x50, 0x40,
0x01, 0xd2, 0x4f, 0x00)
fields := map[string]any{}
_, units := parseDataBody("V2025", body, fields)
if len(units) != 2 {
t.Fatalf("units = %#v", units)
}
vehicle := units[0]["value"].(map[string]any)
if _, exists := vehicle["accelerator_pct"]; exists {
t.Fatalf("2025 vehicle unit must not invent removed pedal fields: %#v", vehicle)
}
position := units[1]["value"].(map[string]any)
if position["coordinate_system"] != 2 {
t.Fatalf("coordinate system = %#v", position["coordinate_system"])
}
if longitude := position["longitude"].(float64); math.Abs(longitude-121) > 0.000001 {
t.Fatalf("longitude = %v", longitude)
}
}
func TestScaledDecimalFieldsDoNotLeakBinaryFloatTails(t *testing.T) {
unit := parseDriveMotorData([]byte{
0x01,

View File

@@ -6,6 +6,8 @@ import (
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/yutongmqtt"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry"
)
func TestRealtimeKVFieldsFromGB32960ParsedDomains(t *testing.T) {
@@ -157,6 +159,37 @@ func TestBuildFieldsEnvelopeCopiesSourceMetadata(t *testing.T) {
}
}
func TestBuildFieldsEnvelopeKeepsYutongJSONNumberMileage(t *testing.T) {
env, err := yutongmqtt.ParseMessage(
"yutong",
"/ytforward/shln/1",
[]byte(`{"device":"LMRKH9AC2R1004106","time":"2026-07-17 11:50:13.021","data":{"LATITUDE":30.466027,"LONGITUDE":103.96096,"METER_SPEED":0,"TOTAL_MILEAGE":77081000}}`),
1784260213051,
)
if err != nil {
t.Fatalf("ParseMessage() error = %v", err)
}
if !EnsureParsedFields(&env) {
t.Fatal("EnsureParsedFields() should compute ingress fields")
}
fieldsEnv, ok := BuildFieldsEnvelope(env)
if !ok {
t.Fatal("BuildFieldsEnvelope() should emit Yutong fields")
}
for _, field := range []string{
"yutong_mqtt.data.total_mileage",
"yutong_mqtt.root.data.total_mileage",
} {
if got := fieldsEnv.Fields[field]; got != "77081000" {
t.Fatalf("%s = %#v, want 77081000", field, got)
}
}
mileageKM, found := telemetry.TotalMileageKM(fieldsEnv.Protocol, fieldsEnv.Fields)
if !found || mileageKM != 77081 {
t.Fatalf("TotalMileageKM() = %.3f, %v; want 77081, true", mileageKM, found)
}
}
func TestBuildFieldsEnvelopeDropsNonPositiveTotalMileage(t *testing.T) {
fieldsEnv, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,

View File

@@ -825,12 +825,27 @@ func positiveNumber(value any) bool {
return typed > 0
case int:
return typed > 0
case int8:
return typed > 0
case int16:
return typed > 0
case int32:
return typed > 0
case int64:
return typed > 0
case uint:
return typed > 0
case uint8:
return typed > 0
case uint16:
return typed > 0
case uint32:
return typed > 0
case uint64:
return typed > 0
case json.Number:
parsed, err := typed.Float64()
return err == nil && parsed > 0
case string:
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
return err == nil && parsed > 0

View File

@@ -18,6 +18,7 @@ const (
defaultCacheRetention = 72 * time.Hour
defaultCacheCleanupInterval = 10 * time.Minute
defaultBaselineMissTTL = time.Minute
defaultBaselineHitTTL = 5 * time.Minute
defaultMaxCacheEntries = 1000000
)
@@ -34,6 +35,7 @@ type Writer struct {
cacheRetention time.Duration
cacheCleanupInterval time.Duration
baselineMissTTL time.Duration
baselineHitTTL time.Duration
maxCacheEntries int
lastCacheCleanup time.Time
lastCacheCleanupStats cacheCleanupStats
@@ -122,6 +124,7 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
cacheRetention: defaultCacheRetention,
cacheCleanupInterval: defaultCacheCleanupInterval,
baselineMissTTL: defaultBaselineMissTTL,
baselineHitTTL: defaultBaselineHitTTL,
maxCacheEntries: defaultMaxCacheEntries,
lastTotalMileage: map[string]float64{},
lastSourceSeen: map[string]time.Time{},
@@ -182,6 +185,15 @@ func (w *Writer) SetBaselineMissTTL(ttl time.Duration) {
w.baselineMissTTL = ttl
}
func (w *Writer) SetBaselineHitTTL(ttl time.Duration) {
w.mu.Lock()
defer w.mu.Unlock()
if ttl < 0 {
ttl = 0
}
w.baselineHitTTL = ttl
}
func (w *Writer) SetMaxCacheEntries(maxEntries int) {
w.mu.Lock()
defer w.mu.Unlock()
@@ -517,8 +529,12 @@ func (w *Writer) previousBaseline(ctx context.Context, candidate SourceMileageSa
now := time.Now()
w.mu.Lock()
if cached, ok := w.baselineCache[cacheKey]; ok {
missExpired := !cached.found && (w.baselineMissTTL == 0 || cached.cachedAt.IsZero() || now.Sub(cached.cachedAt) >= w.baselineMissTTL)
if !missExpired {
ttl := w.baselineMissTTL
if cached.found {
ttl = w.baselineHitTTL
}
expired := ttl == 0 || cached.cachedAt.IsZero() || now.Sub(cached.cachedAt) >= ttl
if !expired {
w.mu.Unlock()
return cached.baseline, cached.found, nil
}
@@ -570,6 +586,16 @@ func (w *Writer) cacheBaseline(candidate SourceMileageSample, cacheKey string, e
return
}
w.mu.Lock()
if previous, ok := w.baselineCache[cacheKey]; ok &&
previous.found && entry.found &&
previous.baseline.LatestTotalKM == entry.baseline.LatestTotalKM &&
previous.baseline.LatestEventTime.Equal(entry.baseline.LatestEventTime) &&
!previous.cachedAt.IsZero() {
// markBaselineWritten runs for every accepted sample. Preserve the
// original cache age when the durable day-boundary baseline has not
// changed, otherwise an active vehicle would keep stale history forever.
entry.cachedAt = previous.cachedAt
}
if entry.cachedAt.IsZero() {
entry.cachedAt = time.Now()
}

View File

@@ -1091,6 +1091,91 @@ func TestWriterRetriesExpiredMissingPreviousDayBaseline(t *testing.T) {
}
}
func TestWriterRefreshesExpiredFoundPreviousDayBaseline(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
writer := NewWriter(db, loc)
writer.SetBaselineHitTTL(time.Minute)
candidate := SourceMileageSample{
VIN: "LMRKH9AC6R1004111",
StatDate: "2026-07-17",
Protocol: envelope.ProtocolYutongMQTT,
SourceKey: "YUTONG_MQTT:LMRKH9AC6R1004111@PLATFORM:yutong",
}
cacheKey := mileageCacheKey(MetricSample{
VIN: candidate.VIN,
Protocol: candidate.Protocol,
StatDate: candidate.StatDate,
SourceKey: candidate.SourceKey,
})
staleTime := time.Date(2026, 7, 12, 4, 12, 20, 0, loc)
writer.baselineCache[cacheKey] = sourceBaselineCacheEntry{
baseline: sourceBaseline{LatestTotalKM: 90778, LatestEventTime: staleTime},
found: true,
cachedAt: time.Now().Add(-2 * time.Minute),
}
refreshedTime := time.Date(2026, 7, 16, 23, 59, 59, 0, loc)
mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`).
WithArgs(candidate.VIN, candidate.StatDate, "YUTONG_MQTT", candidate.SourceKey).
WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}).
AddRow(91302.0, refreshedTime))
baseline, found, err := writer.previousBaseline(context.Background(), candidate)
if err != nil || !found {
t.Fatalf("previousBaseline() found=%v error=%v, want refreshed baseline", found, err)
}
if baseline.LatestTotalKM != 91302 || !baseline.LatestEventTime.Equal(refreshedTime) {
t.Fatalf("baseline = %+v, want nearest refreshed baseline", baseline)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestWriterMarkBaselineKeepsCacheAgeForUnchangedBoundary(t *testing.T) {
db, _, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
writer := NewWriter(db, loc)
candidate := SourceMileageSample{
VIN: "LMRKH9AC6R1004111",
StatDate: "2026-07-17",
Protocol: envelope.ProtocolYutongMQTT,
SourceKey: "YUTONG_MQTT:LMRKH9AC6R1004111@PLATFORM:yutong",
FirstTotalKM: 91302,
FirstEventTime: time.Date(2026, 7, 16, 23, 59, 59, 0, loc),
}
sample := MetricSample{
VIN: candidate.VIN,
Protocol: candidate.Protocol,
StatDate: candidate.StatDate,
SourceKey: candidate.SourceKey,
}
cacheKey := mileageCacheKey(sample)
cachedAt := time.Now().Add(-4 * time.Minute)
writer.baselineCache[cacheKey] = sourceBaselineCacheEntry{
baseline: sourceBaseline{LatestTotalKM: candidate.FirstTotalKM, LatestEventTime: candidate.FirstEventTime},
found: true,
cachedAt: cachedAt,
}
writer.markBaselineWritten(sample, candidate)
if got := writer.baselineCache[cacheKey].cachedAt; !got.Equal(cachedAt) {
t.Fatalf("cachedAt = %s, want unchanged %s", got, cachedAt)
}
}
func TestWriterAppendUsesPreviousSourceBaselineForRealtimeCandidate(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {

View File

@@ -337,10 +337,30 @@ func projectDailyMileageWithExec(ctx context.Context, exec Execer, vin string, s
return err
}
const upsertSourceStatDateStartSQL = `CAST(CONCAT(VALUES(stat_date), ' 00:00:00') AS DATETIME)`
const upsertSourcePreferIncomingFirstSQL = `(
first_event_time IS NULL
OR (
first_event_time < ` + upsertSourceStatDateStartSQL + `
AND VALUES(first_event_time) < ` + upsertSourceStatDateStartSQL + `
AND VALUES(first_event_time) >= first_event_time
)
OR (
first_event_time >= ` + upsertSourceStatDateStartSQL + `
AND VALUES(first_event_time) < ` + upsertSourceStatDateStartSQL + `
)
OR (
first_event_time >= ` + upsertSourceStatDateStartSQL + `
AND VALUES(first_event_time) >= ` + upsertSourceStatDateStartSQL + `
AND VALUES(first_event_time) <= first_event_time
)
)`
const upsertSourceMergedFirstTotalSQL = `CASE
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km)
WHEN first_event_time IS NULL OR VALUES(first_event_time) <= first_event_time
WHEN ` + upsertSourcePreferIncomingFirstSQL + `
THEN VALUES(first_total_mileage_km)
ELSE first_total_mileage_km
END`
@@ -354,7 +374,7 @@ const upsertSourceMergedLatestTotalSQL = `CASE
END`
const upsertSourceMergedFirstEventSQL = `CASE
WHEN first_event_time IS NULL OR VALUES(first_event_time) <= first_event_time
WHEN ` + upsertSourcePreferIncomingFirstSQL + `
THEN VALUES(first_event_time)
ELSE first_event_time
END`
@@ -490,6 +510,8 @@ WHERE s.vin = ?
AND s.stat_date = ?
AND s.protocol = ?
AND s.quality_status IN ('` + QualityOK + `', '` + QualityNoPreviousBaseline + `')
AND s.first_total_mileage_km IS NOT NULL
AND s.latest_total_mileage_km IS NOT NULL
AND COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code) IS NOT NULL
AND COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')) IS NOT NULL
AND s.source_key <> CONCAT(s.protocol, ':', COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')), '` + platformSourceKeyPrefix + `', COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code))
@@ -558,6 +580,8 @@ WHERE s.vin = ?
AND s.stat_date = ?
AND s.protocol = ?
AND s.quality_status IN ('` + QualityOK + `', '` + QualityNoPreviousBaseline + `')
AND s.first_total_mileage_km IS NOT NULL
AND s.latest_total_mileage_km IS NOT NULL
AND COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code) IS NOT NULL
AND COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')) IS NOT NULL
AND s.source_key <> CONCAT(s.protocol, ':', COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')), '` + platformSourceKeyPrefix + `', COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code))

View File

@@ -187,7 +187,9 @@ func TestUpsertSourceMileageTruncatesSubsecondEventTimesAtDayBoundary(t *testing
func TestUpsertSourceMileageUsesEventTimeBoundaries(t *testing.T) {
for _, want := range []string{
"first_total_mileage_km = CASE",
"VALUES(first_event_time) >= first_event_time",
"VALUES(first_event_time) <= first_event_time",
"VALUES(first_event_time) < CAST(CONCAT(VALUES(stat_date), ' 00:00:00') AS DATETIME)",
"latest_total_mileage_km = CASE",
"VALUES(latest_event_time) >= latest_event_time",
"daily_mileage_km = CASE",
@@ -334,6 +336,22 @@ func TestNormalizePlatformSourceMileageMergesLegacyIPKeys(t *testing.T) {
}
}
func TestNormalizePlatformSourceMileageExcludesIncompleteLegacyTotals(t *testing.T) {
for name, query := range map[string]string{
"insert": normalizePlatformSourceMileageInsertSQL,
"delete": normalizePlatformSourceMileageDeleteSQL,
} {
for _, predicate := range []string{
"s.first_total_mileage_km IS NOT NULL",
"s.latest_total_mileage_km IS NOT NULL",
} {
if !strings.Contains(query, predicate) {
t.Fatalf("%s query missing incomplete-total guard %q:\n%s", name, predicate, query)
}
}
}
}
func TestNormalizePlatformSourceMileageForDateNormalizesAndProjectsLegacyVINs(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
if err != nil {

View File

@@ -5,6 +5,7 @@ HOST=""
REMOTE_ROOT="/opt/lingniu-go-native"
DAYS_BACK=1
WINDOW_DAYS=3
BASELINE_LOOKBACK_DAYS=7
PROTOCOLS="GB32960,JT808,YUTONG_MQTT"
ON_CALENDAR="*-*-* 01:30:00"
DRY_RUN="false"
@@ -19,6 +20,9 @@ Options:
--remote-root DIR Remote install root. Defaults to /opt/lingniu-go-native.
--days-back N Target end date relative to local day. Defaults to 1, meaning yesterday.
--window-days N Number of days to recalculate ending at days-back. Defaults to 3.
--baseline-lookback-days N
Maximum days searched before the target window for a mileage baseline.
Defaults to 7.
--protocols LIST Comma-separated protocols. Defaults to GB32960,JT808,YUTONG_MQTT.
--on-calendar SPEC systemd OnCalendar value. Defaults to "*-*-* 01:30:00".
--dry-run Install timer in dry-run mode.
@@ -44,6 +48,10 @@ while [[ $# -gt 0 ]]; do
WINDOW_DAYS="${2:-}"
shift 2
;;
--baseline-lookback-days)
BASELINE_LOOKBACK_DAYS="${2:-}"
shift 2
;;
--protocols)
PROTOCOLS="${2:-}"
shift 2
@@ -86,11 +94,17 @@ case "$WINDOW_DAYS" in
exit 2
;;
esac
case "$BASELINE_LOOKBACK_DAYS" in
''|*[!0-9]*|0)
echo "--baseline-lookback-days must be a positive integer" >&2
exit 2
;;
esac
remote_cmd=""
printf -v remote_cmd \
"REMOTE_ROOT=%q DAYS_BACK=%q WINDOW_DAYS=%q PROTOCOLS=%q ON_CALENDAR=%q DRY_RUN=%q bash -s" \
"$REMOTE_ROOT" "$DAYS_BACK" "$WINDOW_DAYS" "$PROTOCOLS" "$ON_CALENDAR" "$DRY_RUN"
"REMOTE_ROOT=%q DAYS_BACK=%q WINDOW_DAYS=%q BASELINE_LOOKBACK_DAYS=%q PROTOCOLS=%q ON_CALENDAR=%q DRY_RUN=%q bash -s" \
"$REMOTE_ROOT" "$DAYS_BACK" "$WINDOW_DAYS" "$BASELINE_LOOKBACK_DAYS" "$PROTOCOLS" "$ON_CALENDAR" "$DRY_RUN"
ssh "$HOST" "$remote_cmd" <<'REMOTE'
set -euo pipefail
@@ -110,6 +124,7 @@ Environment=BACKFILL_METHOD=last_diff
Environment=BACKFILL_DRY_RUN=${DRY_RUN}
Environment=BACKFILL_DAYS_BACK=${DAYS_BACK}
Environment=BACKFILL_WINDOW_DAYS=${WINDOW_DAYS}
Environment=BACKFILL_BASELINE_LOOKBACK_DAYS=${BASELINE_LOOKBACK_DAYS}
Environment=BACKFILL_PROTOCOLS=${PROTOCOLS}
ExecStart=${REMOTE_ROOT}/current/stats-backfill
TimeoutStartSec=45min