feat(platform): harden telemetry pipeline and unify Semi UI workspaces
This commit is contained in:
@@ -6,6 +6,7 @@ RUN go mod download
|
||||
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/gateway ./cmd/gateway \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/feichi-bridge ./cmd/feichi-bridge \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/history-writer ./cmd/history-writer \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/stat-writer ./cmd/stat-writer \
|
||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /out/realtime-api ./cmd/realtime-api
|
||||
@@ -18,6 +19,7 @@ RUN apt-get update \
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/gateway /app/gateway
|
||||
COPY --from=build /out/feichi-bridge /app/feichi-bridge
|
||||
COPY --from=build /out/history-writer /app/history-writer
|
||||
COPY --from=build /out/stat-writer /app/stat-writer
|
||||
COPY --from=build /out/realtime-api /app/realtime-api
|
||||
|
||||
211
go/vehicle-gateway/cmd/feichi-bridge/main.go
Normal file
211
go/vehicle-gateway/cmd/feichi-bridge/main.go
Normal file
@@ -0,0 +1,211 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/feichibridge"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/health"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
)
|
||||
|
||||
type authSecret struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
type targetSecret struct {
|
||||
PlatformID string `json:"platformId"`
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
type config struct {
|
||||
BaseURL string
|
||||
AuthSecretFile string
|
||||
TargetSecret string
|
||||
TargetAddress string
|
||||
StateFile string
|
||||
HealthAddress string
|
||||
HTTPTimeout time.Duration
|
||||
TargetTimeout time.Duration
|
||||
OCRImage string
|
||||
OCRTimeout time.Duration
|
||||
LoginAttempts int
|
||||
LoginRetryDelay time.Duration
|
||||
Service feichibridge.ServiceConfig
|
||||
}
|
||||
|
||||
func main() {
|
||||
logger := observability.NewLogger("feichi-bridge")
|
||||
cfg, err := loadConfig()
|
||||
if err != nil {
|
||||
logger.Error("load configuration failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
auth, err := readJSONSecret[authSecret](cfg.AuthSecretFile)
|
||||
if err != nil {
|
||||
logger.Error("read Feichi API secret failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
credentials, err := readJSONSecret[targetSecret](cfg.TargetSecret)
|
||||
if err != nil {
|
||||
logger.Error("read GB/T 32960 target secret failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
source, err := feichibridge.NewAuthenticatedAPIClient(
|
||||
cfg.BaseURL,
|
||||
feichibridge.LoginCredentials{
|
||||
Username: auth.Username, Password: auth.Password,
|
||||
MaxAttempts: cfg.LoginAttempts, RetryDelay: cfg.LoginRetryDelay,
|
||||
},
|
||||
feichibridge.DockerCaptchaSolver{Image: cfg.OCRImage, Timeout: cfg.OCRTimeout},
|
||||
cfg.HTTPTimeout,
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error("build Feichi API client failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
state, err := feichibridge.OpenStateStore(cfg.StateFile)
|
||||
if err != nil {
|
||||
logger.Error("open bridge state failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
target, err := feichibridge.NewTarget(feichibridge.TargetConfig{
|
||||
Address: cfg.TargetAddress, PlatformID: credentials.PlatformID,
|
||||
Username: credentials.Username, Password: credentials.Password,
|
||||
Timeout: cfg.TargetTimeout,
|
||||
}, state)
|
||||
if err != nil {
|
||||
logger.Error("build GB/T 32960 target failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
service, err := feichibridge.NewService(cfg.Service, source, target, state, logger, registry)
|
||||
if err != nil {
|
||||
logger.Error("build bridge service failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
health.Start(ctx, logger, health.NewServer(cfg.HealthAddress, "feichi-bridge", []health.Check{
|
||||
{Name: "bridge", Check: service.Ready},
|
||||
}, registry))
|
||||
logger.Info("Feichi bridge starting",
|
||||
"base_url", cfg.BaseURL,
|
||||
"target_address", cfg.TargetAddress,
|
||||
"poll_interval", cfg.Service.PollInterval,
|
||||
"backfill_enabled", cfg.Service.BackfillEnabled,
|
||||
)
|
||||
if strings.HasPrefix(strings.ToLower(cfg.BaseURL), "http://") {
|
||||
logger.Warn("Feichi source uses clear-text HTTP; deploy only through a controlled egress path")
|
||||
}
|
||||
if err := service.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
|
||||
logger.Error("Feichi bridge stopped unexpectedly", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
logger.Info("Feichi bridge stopped")
|
||||
}
|
||||
|
||||
func loadConfig() (config, error) {
|
||||
cfg := config{
|
||||
BaseURL: strings.TrimSpace(os.Getenv("FEICHI_BASE_URL")),
|
||||
AuthSecretFile: env("FEICHI_AUTH_SECRET_FILE", strings.TrimSpace(os.Getenv("FEICHI_AUTH_HEADERS_FILE"))),
|
||||
TargetSecret: strings.TrimSpace(os.Getenv("FEICHI_TARGET_SECRET_FILE")),
|
||||
TargetAddress: env("FEICHI_TARGET_ADDR", "127.0.0.1:32960"),
|
||||
StateFile: env("FEICHI_STATE_FILE", "/var/lib/lingniu-feichi-bridge/state.json"),
|
||||
HealthAddress: env("HEALTH_ADDR", "127.0.0.1:20219"),
|
||||
HTTPTimeout: seconds("FEICHI_HTTP_TIMEOUT_SECONDS", 15),
|
||||
TargetTimeout: seconds("FEICHI_TARGET_TIMEOUT_SECONDS", 10),
|
||||
OCRImage: env("FEICHI_OCR_IMAGE", "lingniu/feichi-captcha-ocr:1.0.0"),
|
||||
OCRTimeout: seconds("FEICHI_OCR_TIMEOUT_SECONDS", 20),
|
||||
LoginAttempts: envInt("FEICHI_LOGIN_MAX_ATTEMPTS", 20),
|
||||
LoginRetryDelay: seconds("FEICHI_LOGIN_RETRY_SECONDS", 1),
|
||||
Service: feichibridge.ServiceConfig{
|
||||
PollInterval: seconds("FEICHI_POLL_INTERVAL_SECONDS", 10),
|
||||
DiscoveryInterval: seconds("FEICHI_DISCOVERY_INTERVAL_SECONDS", 300),
|
||||
BackfillInterval: seconds("FEICHI_BACKFILL_INTERVAL_SECONDS", 3600),
|
||||
BackfillLookback: seconds("FEICHI_BACKFILL_LOOKBACK_SECONDS", 3600),
|
||||
BackfillWindow: seconds("FEICHI_BACKFILL_WINDOW_SECONDS", 1200),
|
||||
BackfillSafetyLag: seconds("FEICHI_BACKFILL_SAFETY_SECONDS", 30),
|
||||
SourceStaleAfter: seconds("FEICHI_SOURCE_STALE_SECONDS", 120),
|
||||
FetchConcurrency: envInt("FEICHI_FETCH_CONCURRENCY", 4),
|
||||
BackfillEnabled: envBool("FEICHI_BACKFILL_ENABLED", true),
|
||||
StaleReissueEnabled: envBool("FEICHI_STALE_REISSUE_ENABLED", true),
|
||||
},
|
||||
}
|
||||
var missing []string
|
||||
if cfg.BaseURL == "" {
|
||||
missing = append(missing, "FEICHI_BASE_URL")
|
||||
}
|
||||
if cfg.AuthSecretFile == "" {
|
||||
missing = append(missing, "FEICHI_AUTH_SECRET_FILE")
|
||||
}
|
||||
if cfg.TargetSecret == "" {
|
||||
missing = append(missing, "FEICHI_TARGET_SECRET_FILE")
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return config{}, fmt.Errorf("required configuration missing: %s", strings.Join(missing, ", "))
|
||||
}
|
||||
if cfg.Service.BackfillWindow > 24*time.Hour {
|
||||
return config{}, errors.New("FEICHI_BACKFILL_WINDOW_SECONDS must not exceed 86400")
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func readJSONSecret[T any](path string) (T, error) {
|
||||
var value T
|
||||
encoded, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return value, err
|
||||
}
|
||||
if err := json.Unmarshal(encoded, &value); err != nil {
|
||||
return value, fmt.Errorf("decode %s: %w", path, err)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func env(name, fallback string) string {
|
||||
if value := strings.TrimSpace(os.Getenv(name)); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envInt(name string, fallback int) int {
|
||||
value := strings.TrimSpace(os.Getenv(name))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func envBool(name string, fallback bool) bool {
|
||||
value := strings.TrimSpace(os.Getenv(name))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func seconds(name string, fallback int) time.Duration {
|
||||
return time.Duration(envInt(name, fallback)) * time.Second
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -54,6 +55,7 @@ func main() {
|
||||
registry.SetGauge("vehicle_stat_cache_retention_seconds", nil, cfg.CacheRetention.Seconds())
|
||||
registry.SetGauge("vehicle_stat_cache_cleanup_interval_seconds", nil, cfg.CacheCleanupInterval.Seconds())
|
||||
registry.SetGauge("vehicle_stat_baseline_miss_ttl_seconds", nil, cfg.BaselineMissTTL.Seconds())
|
||||
registry.SetGauge("vehicle_stat_baseline_hit_ttl_seconds", nil, cfg.BaselineHitTTL.Seconds())
|
||||
registry.SetGauge("vehicle_stat_config", metrics.Labels{"setting": "workers"}, float64(cfg.Workers))
|
||||
health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "vehicle-stat-writer", []health.Check{
|
||||
{Name: "mysql", Check: db.PingContext},
|
||||
@@ -65,6 +67,7 @@ func main() {
|
||||
writer.SetCacheRetention(cfg.CacheRetention)
|
||||
writer.SetCacheCleanupInterval(cfg.CacheCleanupInterval)
|
||||
writer.SetBaselineMissTTL(cfg.BaselineMissTTL)
|
||||
writer.SetBaselineHitTTL(cfg.BaselineHitTTL)
|
||||
writer.SetMaxCacheEntries(cfg.CacheMaxEntries)
|
||||
if cfg.EnsureSchema {
|
||||
if err := writer.EnsureSchema(ctx); err != nil {
|
||||
@@ -89,14 +92,15 @@ func main() {
|
||||
delay: cfg.RetryDelay,
|
||||
registry: registry,
|
||||
}
|
||||
quarantiner := fileStatMessageQuarantiner{dir: cfg.QuarantineDir}
|
||||
|
||||
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","), "workers", cfg.Workers, "project_interval_seconds", cfg.ProjectInterval.Seconds(), "source_touch_interval_seconds", cfg.SourceTouchInterval.Seconds(), "cache_retention_seconds", cfg.CacheRetention.Seconds(), "cache_cleanup_interval_seconds", cfg.CacheCleanupInterval.Seconds(), "baseline_miss_ttl_seconds", cfg.BaselineMissTTL.Seconds(), "cache_max_entries", cfg.CacheMaxEntries, "batch_size", cfg.BatchSize, "batch_wait_ms", cfg.BatchWait, "retry_attempts", cfg.RetryAttempts, "retry_delay_ms", cfg.RetryDelay.Milliseconds())
|
||||
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","), "workers", cfg.Workers, "project_interval_seconds", cfg.ProjectInterval.Seconds(), "source_touch_interval_seconds", cfg.SourceTouchInterval.Seconds(), "cache_retention_seconds", cfg.CacheRetention.Seconds(), "cache_cleanup_interval_seconds", cfg.CacheCleanupInterval.Seconds(), "baseline_miss_ttl_seconds", cfg.BaselineMissTTL.Seconds(), "baseline_hit_ttl_seconds", cfg.BaselineHitTTL.Seconds(), "cache_max_entries", cfg.CacheMaxEntries, "batch_size", cfg.BatchSize, "batch_wait_ms", cfg.BatchWait, "retry_attempts", cfg.RetryAttempts, "retry_delay_ms", cfg.RetryDelay.Milliseconds(), "quarantine_dir", cfg.QuarantineDir)
|
||||
var workers sync.WaitGroup
|
||||
for workerID := 1; workerID <= cfg.Workers; workerID++ {
|
||||
workers.Add(1)
|
||||
go func(id int) {
|
||||
defer workers.Done()
|
||||
runStatConsumer(ctx, logger, registry, appender, cfg, id)
|
||||
runStatConsumer(ctx, logger, registry, appender, quarantiner, cfg, id)
|
||||
}(workerID)
|
||||
}
|
||||
workers.Wait()
|
||||
@@ -105,7 +109,7 @@ func main() {
|
||||
func runStatConsumer(ctx context.Context, logger interface {
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, appender statAppender, cfg config, workerID int) {
|
||||
}, registry *metrics.Registry, appender statAppender, quarantiner statMessageQuarantiner, cfg config, workerID int) {
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: cfg.KafkaBrokers,
|
||||
GroupID: cfg.KafkaGroup,
|
||||
@@ -129,7 +133,7 @@ func runStatConsumer(ctx context.Context, logger interface {
|
||||
continue
|
||||
}
|
||||
batch := collectStatBatch(ctx, reader, message, cfg.BatchSize, time.Duration(cfg.BatchWait)*time.Millisecond)
|
||||
processStatBatchReliablyForWorker(ctx, logger, registry, appender, reader, batch, cfg.RetryDelay, workerLabels)
|
||||
processStatBatchReliablyForWorker(ctx, logger, registry, appender, reader, quarantiner, batch, cfg.RetryDelay, workerLabels)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,6 +159,10 @@ type kafkaMessageFetcher interface {
|
||||
FetchMessage(context.Context) (kafka.Message, error)
|
||||
}
|
||||
|
||||
type statMessageQuarantiner interface {
|
||||
Quarantine(context.Context, kafka.Message, error) error
|
||||
}
|
||||
|
||||
type statBatchItem struct {
|
||||
message kafka.Message
|
||||
processed bool
|
||||
@@ -163,6 +171,8 @@ type statBatchItem struct {
|
||||
type statBatchOutcome struct {
|
||||
commitMessages []kafka.Message
|
||||
retryMessages []kafka.Message
|
||||
failedMessage *kafka.Message
|
||||
writeErr error
|
||||
}
|
||||
|
||||
var statWriteDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
|
||||
@@ -251,7 +261,12 @@ func processStatBatchForWorker(ctx context.Context, logger interface {
|
||||
addStatMetric(registry, "vehicle_stat_writes_total", message, "error")
|
||||
logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||
committed, commitErr := commitStatProcessedPrefixAfterFailure(messageCtx, logger, registry, committer, items)
|
||||
outcome := statBatchOutcome{retryMessages: eventbus.MessagesAfterCommittedPrefixes(messages, committed)}
|
||||
failedMessage := message
|
||||
outcome := statBatchOutcome{
|
||||
retryMessages: eventbus.MessagesAfterCommittedPrefixes(messages, committed),
|
||||
failedMessage: &failedMessage,
|
||||
writeErr: err,
|
||||
}
|
||||
if commitErr != nil {
|
||||
outcome.commitMessages = committed
|
||||
}
|
||||
@@ -279,13 +294,13 @@ func processStatBatchReliably(ctx context.Context, logger interface {
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) {
|
||||
processStatBatchReliablyForWorker(ctx, logger, registry, appender, committer, messages, retryDelay, nil)
|
||||
processStatBatchReliablyForWorker(ctx, logger, registry, appender, committer, nil, messages, retryDelay, nil)
|
||||
}
|
||||
|
||||
func processStatBatchReliablyForWorker(ctx context.Context, logger interface {
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration, workerLabels metrics.Labels) {
|
||||
}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, quarantiner statMessageQuarantiner, messages []kafka.Message, retryDelay time.Duration, workerLabels metrics.Labels) {
|
||||
defer registry.SetGauge("vehicle_stat_retry_pending_messages", workerLabels, 0)
|
||||
pending := messages
|
||||
for len(pending) > 0 {
|
||||
@@ -297,6 +312,19 @@ func processStatBatchReliablyForWorker(ctx context.Context, logger interface {
|
||||
}
|
||||
}
|
||||
pending = outcome.retryMessages
|
||||
if outcome.writeErr != nil && outcome.failedMessage != nil && isQuarantinableStatError(outcome.writeErr) && quarantiner != nil {
|
||||
failed := *outcome.failedMessage
|
||||
if err := quarantiner.Quarantine(ctx, failed, outcome.writeErr); err != nil {
|
||||
logger.Error("stat message quarantine failed", "topic", failed.Topic, "partition", failed.Partition, "offset", failed.Offset, "error", err)
|
||||
registry.IncCounter("vehicle_stat_quarantine_total", metrics.Labels{"status": "error", "topic": failed.Topic})
|
||||
} else {
|
||||
registry.IncCounter("vehicle_stat_quarantine_total", metrics.Labels{"status": "ok", "topic": failed.Topic})
|
||||
logger.Warn("quarantined permanent stat write failure", "topic", failed.Topic, "partition", failed.Partition, "offset", failed.Offset, "error", outcome.writeErr)
|
||||
if retryStatCommit(ctx, logger, registry, committer, []kafka.Message{failed}, retryDelay) {
|
||||
pending = messagesExceptStatMessage(pending, failed)
|
||||
}
|
||||
}
|
||||
}
|
||||
registry.SetGauge("vehicle_stat_retry_pending_messages", workerLabels, float64(len(pending)))
|
||||
if len(pending) == 0 {
|
||||
return
|
||||
@@ -308,6 +336,105 @@ func processStatBatchReliablyForWorker(ctx context.Context, logger interface {
|
||||
}
|
||||
}
|
||||
|
||||
type quarantinedStatMessage struct {
|
||||
QuarantinedAt string `json:"quarantined_at"`
|
||||
Topic string `json:"topic"`
|
||||
Partition int `json:"partition"`
|
||||
Offset int64 `json:"offset"`
|
||||
Key []byte `json:"key,omitempty"`
|
||||
Value []byte `json:"value"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
type fileStatMessageQuarantiner struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
func (q fileStatMessageQuarantiner) Quarantine(ctx context.Context, message kafka.Message, cause error) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
dir := strings.TrimSpace(q.dir)
|
||||
if dir == "" {
|
||||
return errors.New("stat quarantine directory is empty")
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return fmt.Errorf("create quarantine directory: %w", err)
|
||||
}
|
||||
record := quarantinedStatMessage{
|
||||
QuarantinedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Topic: message.Topic,
|
||||
Partition: message.Partition,
|
||||
Offset: message.Offset,
|
||||
Key: message.Key,
|
||||
Value: message.Value,
|
||||
Error: cause.Error(),
|
||||
}
|
||||
payload, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal quarantine record: %w", err)
|
||||
}
|
||||
name := fmt.Sprintf("%s-%d-%d.json", sanitizeStatQuarantineName(message.Topic), message.Partition, message.Offset)
|
||||
temporary, err := os.CreateTemp(dir, "."+name+"-*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create quarantine record: %w", err)
|
||||
}
|
||||
temporaryName := temporary.Name()
|
||||
removeTemporary := true
|
||||
defer func() {
|
||||
_ = temporary.Close()
|
||||
if removeTemporary {
|
||||
_ = os.Remove(temporaryName)
|
||||
}
|
||||
}()
|
||||
if err := temporary.Chmod(0o600); err != nil {
|
||||
return fmt.Errorf("secure quarantine record: %w", err)
|
||||
}
|
||||
if _, err := temporary.Write(payload); err != nil {
|
||||
return fmt.Errorf("write quarantine record: %w", err)
|
||||
}
|
||||
if err := temporary.Sync(); err != nil {
|
||||
return fmt.Errorf("sync quarantine record: %w", err)
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return fmt.Errorf("close quarantine record: %w", err)
|
||||
}
|
||||
if err := os.Rename(temporaryName, filepath.Join(dir, name)); err != nil {
|
||||
return fmt.Errorf("publish quarantine record: %w", err)
|
||||
}
|
||||
removeTemporary = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func sanitizeStatQuarantineName(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return "unknown-topic"
|
||||
}
|
||||
var out strings.Builder
|
||||
for _, r := range value {
|
||||
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' {
|
||||
out.WriteRune(r)
|
||||
} else {
|
||||
out.WriteByte('_')
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func messagesExceptStatMessage(messages []kafka.Message, excluded kafka.Message) []kafka.Message {
|
||||
out := make([]kafka.Message, 0, len(messages))
|
||||
removed := false
|
||||
for _, message := range messages {
|
||||
if !removed && message.Topic == excluded.Topic && message.Partition == excluded.Partition && message.Offset == excluded.Offset {
|
||||
removed = true
|
||||
continue
|
||||
}
|
||||
out = append(out, message)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func retryStatCommit(ctx context.Context, logger interface {
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
@@ -447,7 +574,7 @@ func (a retryStatAppender) AppendWithResult(ctx context.Context, env envelope.Fr
|
||||
var err error
|
||||
for attempt := 1; attempt <= attempts; attempt++ {
|
||||
result, err = appendStatEnvelope(ctx, a.delegate, env)
|
||||
if err == nil || !isTransientMySQLStatError(err) {
|
||||
if err == nil || (!isTransientMySQLStatError(err) && !isQuarantinableStatError(err)) {
|
||||
return result, err
|
||||
}
|
||||
if attempt == attempts {
|
||||
@@ -512,6 +639,32 @@ func isTransientMySQLStatError(err error) bool {
|
||||
strings.Contains(text, "no route to host")
|
||||
}
|
||||
|
||||
func isQuarantinableStatError(err error) bool {
|
||||
if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
text := strings.ToLower(strings.TrimSpace(err.Error()))
|
||||
// Only deterministic row-data failures may advance past a Kafka message
|
||||
// after the configured finite retries. Schema, permission and unknown
|
||||
// application errors remain failure-closed so a systemic outage cannot
|
||||
// silently quarantine an entire stream.
|
||||
return strings.Contains(text, "error 1048") ||
|
||||
strings.Contains(text, "cannot be null") ||
|
||||
strings.Contains(text, "error 1264") ||
|
||||
strings.Contains(text, "out of range value") ||
|
||||
strings.Contains(text, "error 1265") ||
|
||||
strings.Contains(text, "data truncated") ||
|
||||
strings.Contains(text, "error 1292") ||
|
||||
strings.Contains(text, "incorrect datetime value") ||
|
||||
strings.Contains(text, "error 1366") ||
|
||||
strings.Contains(text, "incorrect decimal value") ||
|
||||
strings.Contains(text, "incorrect integer value") ||
|
||||
strings.Contains(text, "error 1406") ||
|
||||
strings.Contains(text, "data too long for column") ||
|
||||
strings.Contains(text, "error 3819") ||
|
||||
strings.Contains(text, "check constraint")
|
||||
}
|
||||
|
||||
func addStatMetric(registry *metrics.Registry, name string, message kafka.Message, status string) {
|
||||
if registry == nil {
|
||||
return
|
||||
@@ -686,6 +839,7 @@ type config struct {
|
||||
CacheRetention time.Duration
|
||||
CacheCleanupInterval time.Duration
|
||||
BaselineMissTTL time.Duration
|
||||
BaselineHitTTL time.Duration
|
||||
CacheMaxEntries int
|
||||
Workers int
|
||||
BatchSize int
|
||||
@@ -694,6 +848,7 @@ type config struct {
|
||||
RetryDelay time.Duration
|
||||
NormalizePlatformSourcesOnStart bool
|
||||
NormalizePlatformSourcesTimeout time.Duration
|
||||
QuarantineDir string
|
||||
}
|
||||
|
||||
func (c config) Validate() error {
|
||||
@@ -728,6 +883,7 @@ func loadConfig() config {
|
||||
CacheRetention: time.Duration(envInt("STATS_CACHE_RETENTION_HOURS", 72)) * time.Hour,
|
||||
CacheCleanupInterval: time.Duration(envInt("STATS_CACHE_CLEANUP_INTERVAL_SECONDS", 600)) * time.Second,
|
||||
BaselineMissTTL: time.Duration(envInt("STATS_BASELINE_MISS_TTL_SECONDS", 60)) * time.Second,
|
||||
BaselineHitTTL: time.Duration(envInt("STATS_BASELINE_HIT_TTL_SECONDS", 300)) * time.Second,
|
||||
CacheMaxEntries: envInt("STATS_CACHE_MAX_ENTRIES", 1000000),
|
||||
Workers: envInt("STATS_WORKERS", 3),
|
||||
BatchSize: envInt("STATS_BATCH_SIZE", 200),
|
||||
@@ -736,6 +892,7 @@ func loadConfig() config {
|
||||
RetryDelay: time.Duration(envInt("STATS_RETRY_DELAY_MS", 20)) * time.Millisecond,
|
||||
NormalizePlatformSourcesOnStart: env("STATS_NORMALIZE_PLATFORM_SOURCES_ON_START", "true") != "false",
|
||||
NormalizePlatformSourcesTimeout: time.Duration(envInt("STATS_NORMALIZE_PLATFORM_SOURCES_TIMEOUT_SECONDS", 30)) * time.Second,
|
||||
QuarantineDir: env("STATS_QUARANTINE_DIR", "/var/lib/lingniu-go-native/stat-writer-quarantine"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -664,6 +666,72 @@ func TestProcessStatBatchReliablyRetriesFailedSuffix(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessStatBatchReliablyQuarantinesPermanentFailureAndContinues(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
|
||||
payload := marshalStatTestFieldsEnvelope(t, env)
|
||||
appender := &contextCheckingStatAppender{failOnCount: 1, err: errors.New("Error 1048 (23000): Column 'daily_mileage_km' cannot be null")}
|
||||
committer := &contextCheckingStatCommitter{}
|
||||
quarantiner := &recordingStatQuarantiner{}
|
||||
registry := metrics.NewRegistry()
|
||||
messages := []kafka.Message{
|
||||
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload},
|
||||
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 11, Value: payload},
|
||||
}
|
||||
|
||||
processStatBatchReliablyForWorker(
|
||||
context.Background(),
|
||||
discardStatLogger{},
|
||||
registry,
|
||||
appender,
|
||||
committer,
|
||||
quarantiner,
|
||||
messages,
|
||||
time.Nanosecond,
|
||||
nil,
|
||||
)
|
||||
|
||||
if appender.count != 2 {
|
||||
t.Fatalf("append attempts = %d, want failed message once and following message once", appender.count)
|
||||
}
|
||||
if quarantiner.count != 1 || quarantiner.message.Offset != 10 {
|
||||
t.Fatalf("quarantine = %d/%d, want one message at offset 10", quarantiner.count, quarantiner.message.Offset)
|
||||
}
|
||||
if got := committedOffsets(committer.messages); len(got) != 2 || got[0] != 10 || got[1] != 11 {
|
||||
t.Fatalf("committed offsets = %#v, want [10 11]", got)
|
||||
}
|
||||
text := registry.Render()
|
||||
if !strings.Contains(text, `vehicle_stat_quarantine_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`) {
|
||||
t.Fatalf("quarantine success metric missing:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileStatMessageQuarantinerPersistsOriginalKafkaMessage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
message := kafka.Message{
|
||||
Topic: "vehicle.fields.go.jt808.v1",
|
||||
Partition: 2,
|
||||
Offset: 5094052,
|
||||
Key: []byte("vehicle-key"),
|
||||
Value: []byte(`{"event_id":"poison:fields"}`),
|
||||
}
|
||||
wantErr := errors.New("permanent mysql error")
|
||||
|
||||
if err := (fileStatMessageQuarantiner{dir: dir}).Quarantine(context.Background(), message, wantErr); err != nil {
|
||||
t.Fatalf("Quarantine() error = %v", err)
|
||||
}
|
||||
payload, err := os.ReadFile(filepath.Join(dir, "vehicle.fields.go.jt808.v1-2-5094052.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("read quarantine record: %v", err)
|
||||
}
|
||||
var record quarantinedStatMessage
|
||||
if err := json.Unmarshal(payload, &record); err != nil {
|
||||
t.Fatalf("unmarshal quarantine record: %v", err)
|
||||
}
|
||||
if record.Topic != message.Topic || record.Partition != message.Partition || record.Offset != message.Offset || string(record.Key) != string(message.Key) || string(record.Value) != string(message.Value) || record.Error != wantErr.Error() {
|
||||
t.Fatalf("quarantine record = %#v, want original message and error", record)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessStatBatchReliablyRetriesCommitWithoutReappending(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
|
||||
payload := marshalStatTestFieldsEnvelope(t, env)
|
||||
@@ -826,6 +894,30 @@ func TestRetryStatAppenderRecordsExhaustedTransientError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryStatAppenderRetriesQuarantinableDataErrorBeforeIsolation(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
delegate := &contextCheckingStatAppender{err: errors.New("Error 1048 (23000): Column 'daily_mileage_km' cannot be null")}
|
||||
appender := retryStatAppender{delegate: delegate, attempts: 3, registry: registry}
|
||||
|
||||
_, err := appender.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("AppendWithResult() error = nil, want exhausted data error")
|
||||
}
|
||||
if delegate.count != 3 {
|
||||
t.Fatalf("append attempts = %d, want configured finite retries", delegate.count)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_stat_write_retries_total{operation="single",status="retry"} 2`,
|
||||
`vehicle_stat_write_retries_total{operation="single",status="exhausted"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("retry metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryStatAppenderDoesNotRetryNonTransientError(t *testing.T) {
|
||||
wantErr := errors.New("validation failed")
|
||||
delegate := &contextCheckingStatAppender{err: wantErr}
|
||||
@@ -841,6 +933,34 @@ func TestRetryStatAppenderDoesNotRetryNonTransientError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsQuarantinableStatErrorOnlyAcceptsDeterministicRowDataFailures(t *testing.T) {
|
||||
for _, err := range []error{
|
||||
errors.New("Error 1048 (23000): Column 'daily_mileage_km' cannot be null"),
|
||||
errors.New("Error 1264: Out of range value for column 'daily_mileage_km'"),
|
||||
errors.New("Error 1265: Data truncated for column 'daily_mileage_km'"),
|
||||
errors.New("Error 1292: Incorrect datetime value"),
|
||||
errors.New("Error 1366: Incorrect decimal value"),
|
||||
errors.New("Error 1406: Data too long for column"),
|
||||
errors.New("Error 3819: Check constraint is violated"),
|
||||
} {
|
||||
if !isQuarantinableStatError(err) {
|
||||
t.Fatalf("isQuarantinableStatError(%q) = false, want true", err.Error())
|
||||
}
|
||||
}
|
||||
for _, err := range []error{
|
||||
errors.New("Error 1054: Unknown column 'daily_mileage_km'"),
|
||||
errors.New("Error 1142: INSERT command denied"),
|
||||
errors.New("validation failed"),
|
||||
context.Canceled,
|
||||
context.DeadlineExceeded,
|
||||
nil,
|
||||
} {
|
||||
if isQuarantinableStatError(err) {
|
||||
t.Fatalf("isQuarantinableStatError(%v) = true, want false", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsTransientMySQLStatError(t *testing.T) {
|
||||
for _, err := range []error{
|
||||
errors.New("dial tcp 127.0.0.1:3306: connection refused"),
|
||||
@@ -903,6 +1023,9 @@ func TestLoadConfigDefaultsToGoFieldsTopics(t *testing.T) {
|
||||
if cfg.BaselineMissTTL != time.Minute {
|
||||
t.Fatalf("BaselineMissTTL = %s, want 1m", cfg.BaselineMissTTL)
|
||||
}
|
||||
if cfg.BaselineHitTTL != 5*time.Minute {
|
||||
t.Fatalf("BaselineHitTTL = %s, want 5m", cfg.BaselineHitTTL)
|
||||
}
|
||||
if cfg.RetryAttempts != 3 {
|
||||
t.Fatalf("RetryAttempts = %d, want 3", cfg.RetryAttempts)
|
||||
}
|
||||
@@ -965,6 +1088,7 @@ func TestLoadConfigSetsStatWriteIntervals(t *testing.T) {
|
||||
t.Setenv("STATS_CACHE_RETENTION_HOURS", "48")
|
||||
t.Setenv("STATS_CACHE_CLEANUP_INTERVAL_SECONDS", "300")
|
||||
t.Setenv("STATS_BASELINE_MISS_TTL_SECONDS", "45")
|
||||
t.Setenv("STATS_BASELINE_HIT_TTL_SECONDS", "180")
|
||||
t.Setenv("STATS_CACHE_MAX_ENTRIES", "12345")
|
||||
|
||||
cfg := loadConfig()
|
||||
@@ -984,6 +1108,9 @@ func TestLoadConfigSetsStatWriteIntervals(t *testing.T) {
|
||||
if cfg.BaselineMissTTL != 45*time.Second {
|
||||
t.Fatalf("BaselineMissTTL = %s, want 45s", cfg.BaselineMissTTL)
|
||||
}
|
||||
if cfg.BaselineHitTTL != 3*time.Minute {
|
||||
t.Fatalf("BaselineHitTTL = %s, want 3m", cfg.BaselineHitTTL)
|
||||
}
|
||||
if cfg.CacheMaxEntries != 12345 {
|
||||
t.Fatalf("CacheMaxEntries = %d, want 12345", cfg.CacheMaxEntries)
|
||||
}
|
||||
@@ -997,6 +1124,7 @@ func TestLoadConfigReadsStatBatchSettings(t *testing.T) {
|
||||
t.Setenv("STATS_RETRY_DELAY_MS", "33")
|
||||
t.Setenv("STATS_NORMALIZE_PLATFORM_SOURCES_ON_START", "false")
|
||||
t.Setenv("STATS_NORMALIZE_PLATFORM_SOURCES_TIMEOUT_SECONDS", "17")
|
||||
t.Setenv("STATS_QUARANTINE_DIR", "/tmp/stat-writer-quarantine-test")
|
||||
|
||||
cfg := loadConfig()
|
||||
|
||||
@@ -1021,6 +1149,9 @@ func TestLoadConfigReadsStatBatchSettings(t *testing.T) {
|
||||
if cfg.NormalizePlatformSourcesTimeout != 17*time.Second {
|
||||
t.Fatalf("NormalizePlatformSourcesTimeout = %s, want 17s", cfg.NormalizePlatformSourcesTimeout)
|
||||
}
|
||||
if cfg.QuarantineDir != "/tmp/stat-writer-quarantine-test" {
|
||||
t.Fatalf("QuarantineDir = %q, want env override", cfg.QuarantineDir)
|
||||
}
|
||||
}
|
||||
|
||||
type contextCheckingStatAppender struct {
|
||||
@@ -1070,6 +1201,18 @@ type contextCheckingStatCommitter struct {
|
||||
failOnCount int
|
||||
}
|
||||
|
||||
type recordingStatQuarantiner struct {
|
||||
count int
|
||||
message kafka.Message
|
||||
err error
|
||||
}
|
||||
|
||||
func (q *recordingStatQuarantiner) Quarantine(_ context.Context, message kafka.Message, _ error) error {
|
||||
q.count++
|
||||
q.message = message
|
||||
return q.err
|
||||
}
|
||||
|
||||
func (c *contextCheckingStatCommitter) CommitMessages(ctx context.Context, messages ...kafka.Message) error {
|
||||
c.ctxErr = ctx.Err()
|
||||
c.count++
|
||||
|
||||
@@ -36,6 +36,7 @@ type config struct {
|
||||
Debug bool
|
||||
EventTimeFullScan bool
|
||||
ProgressEvery int64
|
||||
BaselineLookback int
|
||||
Location *time.Location
|
||||
}
|
||||
|
||||
@@ -50,6 +51,7 @@ type rawFrameRow struct {
|
||||
EventTimeMS int64
|
||||
ReceivedAtMS int64
|
||||
ParsedJSON string
|
||||
RawText string
|
||||
}
|
||||
|
||||
type metricAgg struct {
|
||||
@@ -188,6 +190,9 @@ func main() {
|
||||
}
|
||||
}
|
||||
fields := fieldsForStats(row.Protocol, row.VIN, text)
|
||||
if len(fields) == 0 && row.Protocol == envelope.ProtocolYutongMQTT {
|
||||
fields = fieldsForStats(row.Protocol, row.VIN, row.RawText)
|
||||
}
|
||||
if cfg.Debug && scanned <= 5 {
|
||||
slog.Info("debug raw frame",
|
||||
"protocol", row.Protocol,
|
||||
@@ -427,7 +432,7 @@ func buildLastDiffAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB,
|
||||
return aggregates, nil
|
||||
}
|
||||
|
||||
func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
|
||||
func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, _ *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
|
||||
if aggregates == nil {
|
||||
return 0, fmt.Errorf("aggregates map is nil")
|
||||
}
|
||||
@@ -441,11 +446,6 @@ func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB,
|
||||
continue
|
||||
}
|
||||
latestHistory := map[sourceHistoryID]dailySourceLast{}
|
||||
preWindow, err := queryPreviousLastSourceRows(ctx, tdDB, cfg, protocol, targetDates[0])
|
||||
if err != nil {
|
||||
return added, err
|
||||
}
|
||||
rememberLatestSourceRows(latestHistory, preWindow)
|
||||
aggregateRowsByDate := indexAggregateSourceRowsByDate(aggregates, protocol)
|
||||
for _, date := range targetDates {
|
||||
current, err := queryRealtimeLocationLastRows(ctx, mysqlDB, cfg, protocol, date)
|
||||
@@ -656,7 +656,9 @@ func queryDailyLastSourceRows(ctx context.Context, db *sql.DB, cfg config, proto
|
||||
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
|
||||
where = append(where, predicate)
|
||||
}
|
||||
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, FIRST(event_time), FIRST(parsed_json), LAST(event_time), LAST(parsed_json), COUNT(*)
|
||||
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
|
||||
FIRST(event_time), FIRST(parsed_json), FIRST(raw_text),
|
||||
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
|
||||
FROM %s.raw_frames
|
||||
WHERE %s
|
||||
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
@@ -668,7 +670,7 @@ func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, pr
|
||||
// LAST aggregates the complete pre-window history once per source so empty
|
||||
// calendar days do not make a backfill fall back to the current day's first
|
||||
// sample.
|
||||
where := backfillBeforePredicates(date)
|
||||
where := backfillBeforePredicates(cfg, date)
|
||||
where = append(where,
|
||||
"parse_status = 'OK'",
|
||||
"vin IS NOT NULL",
|
||||
@@ -679,17 +681,31 @@ func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, pr
|
||||
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
|
||||
where = append(where, predicate)
|
||||
}
|
||||
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, LAST(event_time), LAST(parsed_json), COUNT(*)
|
||||
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
|
||||
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
|
||||
FROM %s.raw_frames
|
||||
WHERE %s
|
||||
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
return querySourceRows(ctx, db, cfg, protocol, sqlText, false)
|
||||
}
|
||||
|
||||
func backfillBeforePredicates(eventDateExclusive string) []string {
|
||||
return []string{
|
||||
func backfillBeforePredicates(cfg config, eventDateExclusive string) []string {
|
||||
lookbackDays := cfg.BaselineLookback
|
||||
if lookbackDays <= 0 {
|
||||
lookbackDays = 7
|
||||
}
|
||||
eventDateFrom := shiftDate(eventDateExclusive, -lookbackDays)
|
||||
where := []string{
|
||||
fmt.Sprintf("event_time >= '%s 00:00:00'", quote(eventDateFrom)),
|
||||
fmt.Sprintf("event_time < '%s 00:00:00'", quote(eventDateExclusive)),
|
||||
}
|
||||
if !cfg.EventTimeFullScan {
|
||||
where = append([]string{
|
||||
fmt.Sprintf("ts >= '%s 00:00:00'", quote(previousDate(eventDateFrom))),
|
||||
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(eventDateExclusive))),
|
||||
}, where...)
|
||||
}
|
||||
return where
|
||||
}
|
||||
|
||||
func queryRealtimeLocationLastRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
|
||||
@@ -775,27 +791,30 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
|
||||
var sourceEndpoint string
|
||||
var firstTS time.Time
|
||||
var firstParsedJSON string
|
||||
var firstRawText string
|
||||
var ts time.Time
|
||||
var parsedJSON string
|
||||
var rawText string
|
||||
var rawSampleCount int64
|
||||
if includeFirst {
|
||||
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &firstTS, &firstParsedJSON, &ts, &parsedJSON, &rawSampleCount); err != nil {
|
||||
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &firstTS, &firstParsedJSON, &firstRawText, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &ts, &parsedJSON, &rawSampleCount); err != nil {
|
||||
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
firstTS = ts
|
||||
firstParsedJSON = parsedJSON
|
||||
firstRawText = rawText
|
||||
}
|
||||
latestTotalKM, ok := mileageFromParsed(protocol, vin, parsedJSON, ts, cfg.Location)
|
||||
latestTotalKM, ok := mileageFromEvidence(protocol, vin, parsedJSON, rawText, ts, cfg.Location)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
firstTotalKM := latestTotalKM
|
||||
if includeFirst {
|
||||
if parsedFirst, ok := mileageFromParsed(protocol, vin, firstParsedJSON, firstTS, cfg.Location); ok {
|
||||
if parsedFirst, ok := mileageFromEvidence(protocol, vin, firstParsedJSON, firstRawText, firstTS, cfg.Location); ok {
|
||||
firstTotalKM = parsedFirst
|
||||
}
|
||||
}
|
||||
@@ -852,6 +871,16 @@ func mileageFromParsed(protocol envelope.Protocol, vin string, parsedJSON string
|
||||
return samples[0].TotalMileageKM, true
|
||||
}
|
||||
|
||||
func mileageFromEvidence(protocol envelope.Protocol, vin string, parsedJSON string, rawText string, eventTime time.Time, loc *time.Location) (float64, bool) {
|
||||
if totalKM, ok := mileageFromParsed(protocol, vin, parsedJSON, eventTime, loc); ok {
|
||||
return totalKM, true
|
||||
}
|
||||
if protocol != envelope.ProtocolYutongMQTT || strings.TrimSpace(rawText) == "" {
|
||||
return 0, false
|
||||
}
|
||||
return mileageFromParsed(protocol, vin, rawText, eventTime, loc)
|
||||
}
|
||||
|
||||
func normalizedSourceKey(protocol string, phone string, deviceID string, endpoint string) string {
|
||||
return normalizedSourceKeyForSource(protocol, phone, deviceID, endpoint, "", "")
|
||||
}
|
||||
@@ -951,6 +980,7 @@ func loadConfig() (config, error) {
|
||||
Debug: envBool("BACKFILL_DEBUG", false),
|
||||
EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false),
|
||||
ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
|
||||
BaselineLookback: envInt("BACKFILL_BASELINE_LOOKBACK_DAYS", 7),
|
||||
Location: loc,
|
||||
}, nil
|
||||
}
|
||||
@@ -1000,7 +1030,7 @@ func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, err
|
||||
where = append(where, "protocol IN ("+strings.Join(quoted, ",")+")")
|
||||
}
|
||||
where = append(where, realtimeMileageFramePredicate())
|
||||
sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json
|
||||
sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json, raw_text
|
||||
FROM %s.raw_frames
|
||||
WHERE %s
|
||||
ORDER BY event_time ASC, ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
|
||||
@@ -1044,6 +1074,9 @@ func mileageBearingFramePredicate(protocol envelope.Protocol) string {
|
||||
conditions := make([]string, 0, len(tokens))
|
||||
for _, token := range tokens {
|
||||
conditions = append(conditions, fmt.Sprintf("parsed_json LIKE '%%%s%%'", quote(token)))
|
||||
if protocol == envelope.ProtocolYutongMQTT {
|
||||
conditions = append(conditions, fmt.Sprintf("raw_text LIKE '%%%s%%'", quote(token)))
|
||||
}
|
||||
}
|
||||
return "(" + strings.Join(conditions, " OR ") + ")"
|
||||
}
|
||||
@@ -1071,10 +1104,10 @@ func mileageSearchTokens(protocol envelope.Protocol) []string {
|
||||
}
|
||||
|
||||
func scanRawFrame(rows *sql.Rows) (rawFrameRow, error) {
|
||||
var protocol, vin, phone, deviceID, sourceEndpoint, eventID, parsed string
|
||||
var protocol, vin, phone, deviceID, sourceEndpoint, eventID, parsed, rawText string
|
||||
var messageID int64
|
||||
var eventTime, receivedAt time.Time
|
||||
if err := rows.Scan(&protocol, &vin, &phone, &deviceID, &sourceEndpoint, &eventID, &messageID, &eventTime, &receivedAt, &parsed); err != nil {
|
||||
if err := rows.Scan(&protocol, &vin, &phone, &deviceID, &sourceEndpoint, &eventID, &messageID, &eventTime, &receivedAt, &parsed, &rawText); err != nil {
|
||||
return rawFrameRow{}, err
|
||||
}
|
||||
return rawFrameRow{
|
||||
@@ -1088,6 +1121,7 @@ func scanRawFrame(rows *sql.Rows) (rawFrameRow, error) {
|
||||
EventTimeMS: eventTime.UnixMilli(),
|
||||
ReceivedAtMS: receivedAt.UnixMilli(),
|
||||
ParsedJSON: parsed,
|
||||
RawText: rawText,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -1267,6 +1301,14 @@ func previousDate(date string) string {
|
||||
return parsed.AddDate(0, 0, -1).Format("2006-01-02")
|
||||
}
|
||||
|
||||
func shiftDate(date string, days int) string {
|
||||
parsed, err := time.Parse("2006-01-02", date)
|
||||
if err != nil {
|
||||
return date
|
||||
}
|
||||
return parsed.AddDate(0, 0, days).Format("2006-01-02")
|
||||
}
|
||||
|
||||
func dateRangeWithPrevious(from string, to string) ([]string, error) {
|
||||
dates, err := dateRange(from, to)
|
||||
if err != nil {
|
||||
|
||||
@@ -233,23 +233,23 @@ func TestBuildLastDiffAggregatesCarriesNearestHistoryAcrossEmptyDays(t *testing.
|
||||
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
|
||||
currentRows := func() *sqlmock.Rows {
|
||||
return sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
|
||||
})
|
||||
}
|
||||
previousRows := func() *sqlmock.Rows {
|
||||
return sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
|
||||
})
|
||||
}
|
||||
|
||||
mock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(previousRows())
|
||||
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, int64(4)))
|
||||
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", int64(4)))
|
||||
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-10 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(currentRows())
|
||||
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-11 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, int64(6)))
|
||||
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", int64(6)))
|
||||
|
||||
aggregates, err := buildLastDiffAggregates(context.Background(), nil, tdDB, config{
|
||||
TDengineDatabase: "lingniu_vehicle_ts",
|
||||
@@ -342,18 +342,10 @@ func TestAddRealtimeLocationFallbackAggregatesUsesHistoricalBaseline(t *testing.
|
||||
WithArgs("YUTONG_MQTT", "2026-07-12", "2026-07-12").
|
||||
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}).
|
||||
AddRow("LMRKH9AC0R1004086", "mqtt://yutong/ytforward/shln/4", currentTS, 11578.0))
|
||||
tdMock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-12 00:00:00'.*protocol = 'YUTONG_MQTT'.*TOTAL_MILEAGE").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
}).AddRow(
|
||||
"LMRKH9AC0R1004086",
|
||||
"",
|
||||
"LMRKH9AC0R1004086",
|
||||
"mqtt://yutong/ytforward/shln/4",
|
||||
previousTS,
|
||||
`{"data":{"TOTAL_MILEAGE":11500000}}`,
|
||||
int64(12),
|
||||
))
|
||||
sourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong")
|
||||
mysqlMock.ExpectQuery("(?s)SELECT latest_total_mileage_km, latest_event_time.*FROM vehicle_daily_mileage_source").
|
||||
WithArgs("LMRKH9AC0R1004086", "2026-07-12", "YUTONG_MQTT", sourceKey).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}).AddRow(11500.0, previousTS))
|
||||
|
||||
aggregates := map[string]*metricAgg{}
|
||||
added, err := addRealtimeLocationFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
|
||||
@@ -373,7 +365,7 @@ func TestAddRealtimeLocationFallbackAggregatesUsesHistoricalBaseline(t *testing.
|
||||
if agg.FirstKM != 11500 || agg.LatestKM != 11578 {
|
||||
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
|
||||
}
|
||||
if agg.SourceKey != stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong") {
|
||||
if agg.SourceKey != sourceKey {
|
||||
t.Fatalf("source key = %q", agg.SourceKey)
|
||||
}
|
||||
if agg.SourceCode != "yutong" || agg.PlatformName != "宇通" || agg.SourceKind != "PLATFORM" {
|
||||
@@ -410,10 +402,6 @@ func TestAddRealtimeLocationFallbackReusesAggregateHistoryAcrossEmptyDays(t *tes
|
||||
dayOneTS := time.Date(2026, 7, 9, 23, 50, 0, 0, loc)
|
||||
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
|
||||
|
||||
tdMock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
}))
|
||||
for _, date := range []string{"2026-07-09", "2026-07-10"} {
|
||||
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?").
|
||||
WithArgs("YUTONG_MQTT", date, date).
|
||||
@@ -483,16 +471,18 @@ func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
|
||||
lastTS := time.Date(2026, 7, 8, 18, 0, 0, 0, loc)
|
||||
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
|
||||
}).AddRow(
|
||||
"LMRKH9AC6R1004108",
|
||||
"",
|
||||
"LMRKH9AC6R1004108",
|
||||
"mqtt://yutong/ytforward/shln/3",
|
||||
firstTS,
|
||||
`{"yutong_mqtt.data.total_mileage":"65422000"}`,
|
||||
`{"yutong_mqtt.data.latitude":"30.0"}`,
|
||||
`{"data":{"TOTAL_MILEAGE":65422000}}`,
|
||||
lastTS,
|
||||
`{"yutong_mqtt.data.total_mileage":"65423000"}`,
|
||||
`{"yutong_mqtt.data.latitude":"30.1"}`,
|
||||
`{"data":{"TOTAL_MILEAGE":65423000}}`,
|
||||
int64(42),
|
||||
))
|
||||
|
||||
@@ -526,14 +516,15 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
|
||||
lastTS := time.Date(2026, 7, 7, 23, 58, 0, 0, loc)
|
||||
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-08 00:00:00'.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
|
||||
}).AddRow(
|
||||
"LMRKH9AC6R1004108",
|
||||
"",
|
||||
"LMRKH9AC6R1004108",
|
||||
"mqtt://yutong/ytforward/shln/3",
|
||||
lastTS,
|
||||
`{"yutong_mqtt.data.total_mileage":"65377000"}`,
|
||||
`{"yutong_mqtt.data.latitude":"30.0"}`,
|
||||
`{"data":{"TOTAL_MILEAGE":65377000}}`,
|
||||
int64(31),
|
||||
))
|
||||
|
||||
@@ -556,13 +547,19 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackfillBeforePredicatesSearchesAllEarlierHistory(t *testing.T) {
|
||||
where := strings.Join(backfillBeforePredicates("2026-07-08"), " AND ")
|
||||
func TestBackfillBeforePredicatesBoundsHistoryScan(t *testing.T) {
|
||||
where := strings.Join(backfillBeforePredicates(config{BaselineLookback: 7}, "2026-07-08"), " AND ")
|
||||
if !strings.Contains(where, "event_time < '2026-07-08 00:00:00'") {
|
||||
t.Fatalf("pre-window predicate missing exclusive upper bound: %s", where)
|
||||
}
|
||||
if strings.Contains(where, "event_time >=") || strings.Contains(where, "ts >=") {
|
||||
t.Fatalf("pre-window predicate must not stop at the previous day: %s", where)
|
||||
for _, predicate := range []string{
|
||||
"event_time >= '2026-07-01 00:00:00'",
|
||||
"ts >= '2026-06-30 00:00:00'",
|
||||
"ts < '2026-07-09 00:00:00'",
|
||||
} {
|
||||
if !strings.Contains(where, predicate) {
|
||||
t.Fatalf("pre-window predicate missing %q: %s", predicate, where)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -577,7 +574,7 @@ func TestQueryPreviousLastSourceRowsNormalizesScannedTimestampToBusinessTimezone
|
||||
utcInstant := time.Date(2026, 7, 12, 15, 59, 59, 0, time.UTC)
|
||||
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'").
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
|
||||
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
|
||||
}).AddRow(
|
||||
"LMRKH9AC7R1004098",
|
||||
"",
|
||||
@@ -585,6 +582,7 @@ func TestQueryPreviousLastSourceRowsNormalizesScannedTimestampToBusinessTimezone
|
||||
"mqtt://yutong/ytforward/shln/4",
|
||||
utcInstant,
|
||||
`{"yutong_mqtt.data.total_mileage":"41249000"}`,
|
||||
"",
|
||||
int64(1),
|
||||
))
|
||||
|
||||
|
||||
57
go/vehicle-gateway/internal/feichibridge/captcha.go
Normal file
57
go/vehicle-gateway/internal/feichibridge/captcha.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var captchaPattern = regexp.MustCompile(`^[0-9A-Z]{4}$`)
|
||||
|
||||
type CaptchaSolver interface {
|
||||
Solve(context.Context, []byte) (string, error)
|
||||
}
|
||||
|
||||
type DockerCaptchaSolver struct {
|
||||
Image string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func (s DockerCaptchaSolver) Solve(ctx context.Context, image []byte) (string, error) {
|
||||
if len(image) == 0 {
|
||||
return "", errors.New("captcha image is empty")
|
||||
}
|
||||
if strings.TrimSpace(s.Image) == "" {
|
||||
return "", errors.New("captcha OCR image is required")
|
||||
}
|
||||
timeout := s.Timeout
|
||||
if timeout <= 0 {
|
||||
timeout = 20 * time.Second
|
||||
}
|
||||
solveCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
command := exec.CommandContext(
|
||||
solveCtx,
|
||||
"docker", "run", "--rm", "--network=none", "-i", strings.TrimSpace(s.Image),
|
||||
)
|
||||
command.Stdin = bytes.NewReader(image)
|
||||
var stderr bytes.Buffer
|
||||
command.Stderr = &stderr
|
||||
output, err := command.Output()
|
||||
if err != nil {
|
||||
if solveCtx.Err() != nil {
|
||||
return "", fmt.Errorf("captcha OCR timed out: %w", solveCtx.Err())
|
||||
}
|
||||
return "", fmt.Errorf("captcha OCR failed: %w: %s", err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
code := strings.ToUpper(strings.TrimSpace(string(output)))
|
||||
if !captchaPattern.MatchString(code) {
|
||||
return "", fmt.Errorf("captcha OCR returned invalid result %q", code)
|
||||
}
|
||||
return code, nil
|
||||
}
|
||||
211
go/vehicle-gateway/internal/feichibridge/client.go
Normal file
211
go/vehicle-gateway/internal/feichibridge/client.go
Normal file
@@ -0,0 +1,211 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var ErrUnauthorized = errors.New("feichi API session unauthorized")
|
||||
|
||||
type APIClient struct {
|
||||
baseURL *url.URL
|
||||
headers http.Header
|
||||
client *http.Client
|
||||
login *loginSession
|
||||
}
|
||||
|
||||
func NewAPIClient(baseURL string, headers map[string]string, timeout time.Duration) (*APIClient, error) {
|
||||
parsed, err := url.Parse(strings.TrimRight(strings.TrimSpace(baseURL), "/") + "/")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse FEICHI_BASE_URL: %w", err)
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return nil, fmt.Errorf("FEICHI_BASE_URL must use http or https")
|
||||
}
|
||||
if timeout <= 0 {
|
||||
timeout = 15 * time.Second
|
||||
}
|
||||
out := &APIClient{
|
||||
baseURL: parsed,
|
||||
headers: make(http.Header),
|
||||
client: &http.Client{Timeout: timeout},
|
||||
}
|
||||
for name, value := range headers {
|
||||
if strings.TrimSpace(name) != "" && strings.TrimSpace(value) != "" {
|
||||
out.headers.Set(name, value)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func NewAuthenticatedAPIClient(baseURL string, credentials LoginCredentials, solver CaptchaSolver, timeout time.Duration) (*APIClient, error) {
|
||||
client, err := NewAPIClient(baseURL, nil, timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
login, err := newLoginSession(credentials, solver)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client.login = login
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (c *APIClient) Vehicles(ctx context.Context) ([]Vehicle, error) {
|
||||
payload := map[string]any{
|
||||
"conditions": []map[string]string{
|
||||
{"name": "iccid", "value": ""},
|
||||
{"name": "powerMode", "value": ""},
|
||||
},
|
||||
"sort": []map[string]string{{"name": "updateTime", "order": "desc"}},
|
||||
"start": 0,
|
||||
"limit": 100,
|
||||
}
|
||||
var envelope collectionEnvelope[Vehicle]
|
||||
if err := c.doJSON(ctx, http.MethodPost, "api/v1/sys/vehicleRealStatuss", payload, &envelope); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !apiCodeOK(envelope.Code) {
|
||||
return nil, apiCodeError(envelope.Code)
|
||||
}
|
||||
return envelope.Data, nil
|
||||
}
|
||||
|
||||
func (c *APIClient) Snapshot(ctx context.Context, vehicleID string) (Snapshot, error) {
|
||||
path := "api/v1/sys/vehicleRealStatussByVId/" + url.PathEscape(vehicleID) + "/-1"
|
||||
var envelope objectEnvelope[Snapshot]
|
||||
if err := c.doJSON(ctx, http.MethodGet, path, nil, &envelope); err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
if !apiCodeOK(envelope.Code) {
|
||||
return Snapshot{}, apiCodeError(envelope.Code)
|
||||
}
|
||||
return envelope.Data, nil
|
||||
}
|
||||
|
||||
func (c *APIClient) History(ctx context.Context, vin string, begin, end time.Time) ([]Record, error) {
|
||||
payload := map[string]any{
|
||||
"conditions": []map[string]string{
|
||||
{"name": "queryType", "value": "0"},
|
||||
{"name": "queryContent", "value": vin},
|
||||
{"name": "beginTime", "value": begin.In(shanghai).Format("2006-01-02 15:04:05")},
|
||||
{"name": "endTime", "value": end.In(shanghai).Format("2006-01-02 15:04:05")},
|
||||
{"name": "hisdataType", "value": "0"},
|
||||
},
|
||||
"start": 0,
|
||||
"limit": 100,
|
||||
}
|
||||
var envelope collectionEnvelope[historySegment]
|
||||
if err := c.doJSON(ctx, http.MethodPost, "api/v1/sys/hisdataQuerys", payload, &envelope); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !apiCodeOK(envelope.Code) {
|
||||
return nil, apiCodeError(envelope.Code)
|
||||
}
|
||||
var records []Record
|
||||
for _, segment := range envelope.Data {
|
||||
records = append(records, segment.Subdata...)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func (c *APIClient) doJSON(ctx context.Context, method, path string, body any, output any) error {
|
||||
if c.login != nil {
|
||||
if err := c.login.ensure(ctx, c); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
_, err := c.rawJSON(ctx, method, path, body, output)
|
||||
if !errors.Is(err, ErrUnauthorized) || c.login == nil || attempt == 1 {
|
||||
return err
|
||||
}
|
||||
c.login.invalidate()
|
||||
if err := c.login.ensure(ctx, c); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return ErrUnauthorized
|
||||
}
|
||||
|
||||
func (c *APIClient) rawJSON(ctx context.Context, method, path string, body any, output any) (http.Header, error) {
|
||||
endpoint, err := c.baseURL.Parse(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var reader io.Reader
|
||||
if body != nil {
|
||||
encoded, marshalErr := json.Marshal(body)
|
||||
if marshalErr != nil {
|
||||
return nil, marshalErr
|
||||
}
|
||||
reader = bytes.NewReader(encoded)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, method, endpoint.String(), reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
request.Header.Set("Accept", "application/json")
|
||||
if body != nil {
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
for name, values := range c.headers {
|
||||
for _, value := range values {
|
||||
request.Header.Add(name, value)
|
||||
}
|
||||
}
|
||||
if c.login != nil {
|
||||
if cookie := c.login.cookieHeader(); cookie != "" {
|
||||
request.Header.Set("Cookie", cookie)
|
||||
}
|
||||
if csrf := c.login.csrfHeader(); csrf != "" {
|
||||
request.Header.Set("x-api-csrf", csrf)
|
||||
}
|
||||
}
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("feichi %s %s: %w", method, path, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode == http.StatusUnauthorized || response.StatusCode == http.StatusForbidden {
|
||||
return response.Header, ErrUnauthorized
|
||||
}
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
snippet, _ := io.ReadAll(io.LimitReader(response.Body, 1024))
|
||||
return response.Header, fmt.Errorf("feichi %s %s: HTTP %d: %s", method, path, response.StatusCode, strings.TrimSpace(string(snippet)))
|
||||
}
|
||||
encoded, err := io.ReadAll(io.LimitReader(response.Body, 16<<20))
|
||||
if err != nil {
|
||||
return response.Header, fmt.Errorf("read feichi %s: %w", path, err)
|
||||
}
|
||||
var status struct {
|
||||
Code any `json:"code"`
|
||||
}
|
||||
if err := json.Unmarshal(encoded, &status); err != nil {
|
||||
return response.Header, fmt.Errorf("decode feichi %s: %w", path, err)
|
||||
}
|
||||
if fmt.Sprint(status.Code) == "401" || fmt.Sprint(status.Code) == "403" {
|
||||
return response.Header, ErrUnauthorized
|
||||
}
|
||||
if output != nil {
|
||||
if err := json.Unmarshal(encoded, output); err != nil {
|
||||
return response.Header, fmt.Errorf("decode feichi %s: %w", path, err)
|
||||
}
|
||||
}
|
||||
return response.Header, nil
|
||||
}
|
||||
|
||||
func apiCodeError(code any) error {
|
||||
if fmt.Sprint(code) == "401" || fmt.Sprint(code) == "403" {
|
||||
return ErrUnauthorized
|
||||
}
|
||||
return fmt.Errorf("feichi API returned code %v", code)
|
||||
}
|
||||
154
go/vehicle-gateway/internal/feichibridge/client_test.go
Normal file
154
go/vehicle-gateway/internal/feichibridge/client_test.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fixedCaptchaSolver struct {
|
||||
code string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *fixedCaptchaSolver) Solve(_ context.Context, image []byte) (string, error) {
|
||||
s.calls++
|
||||
if string(image) != "captcha-image" {
|
||||
return "", fmt.Errorf("unexpected captcha image")
|
||||
}
|
||||
return s.code, nil
|
||||
}
|
||||
|
||||
func TestAPIClientUsesSessionHeadersAndExpectedEndpoints(t *testing.T) {
|
||||
var calls int
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
calls++
|
||||
if request.Header.Get("Cookie") != "R_SESS=test" || request.Header.Get("x-api-csrf") != "csrf" {
|
||||
t.Errorf("missing session headers: %#v", request.Header)
|
||||
}
|
||||
response.Header().Set("Content-Type", "application/json")
|
||||
switch {
|
||||
case request.URL.Path == "/api/v1/sys/vehicleRealStatuss":
|
||||
_ = json.NewEncoder(response).Encode(map[string]any{
|
||||
"code": 200,
|
||||
"data": []map[string]string{{"vehicleId": "id-1", "vin": "LTEST32960VIN0001"}},
|
||||
})
|
||||
case strings.Contains(request.URL.Path, "vehicleRealStatussByVId"):
|
||||
_ = json.NewEncoder(response).Encode(map[string]any{
|
||||
"code": 200,
|
||||
"data": map[string]any{"vehicleId": "id-1", "vin": "LTEST32960VIN0001", "dataItems": map[string]string{"2000": "2026-07-17 12:00:00"}},
|
||||
})
|
||||
case request.URL.Path == "/api/v1/sys/hisdataQuerys":
|
||||
_ = json.NewEncoder(response).Encode(map[string]any{
|
||||
"code": 200,
|
||||
"data": []map[string]any{{"subdata": []map[string]string{{"2000": "2026-07-17 11:59:50"}}}},
|
||||
})
|
||||
default:
|
||||
http.NotFound(response, request)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewAPIClient(server.URL, map[string]string{"Cookie": "R_SESS=test", "x-api-csrf": "csrf"}, time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vehicles, err := client.Vehicles(context.Background())
|
||||
if err != nil || len(vehicles) != 1 {
|
||||
t.Fatalf("vehicles = %#v, err = %v", vehicles, err)
|
||||
}
|
||||
if _, err := client.Snapshot(context.Background(), "id-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records, err := client.History(context.Background(), vehicles[0].VIN, time.Now().Add(-time.Hour), time.Now())
|
||||
if err != nil || len(records) != 1 {
|
||||
t.Fatalf("history = %#v, err = %v", records, err)
|
||||
}
|
||||
if calls != 3 {
|
||||
t.Fatalf("calls = %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIClientMapsUnauthorizedEnvelope(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = response.Write([]byte(`{"code":401,"data":[]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
client, err := NewAPIClient(server.URL, nil, time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := client.Vehicles(context.Background()); err != ErrUnauthorized {
|
||||
t.Fatalf("error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthenticatedAPIClientLogsInAndRetriesExpiredSession(t *testing.T) {
|
||||
var loginCalls int
|
||||
var vehicleCalls int
|
||||
solver := &fixedCaptchaSolver{code: "12WN"}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
|
||||
response.Header().Set("Content-Type", "application/json")
|
||||
switch request.URL.Path {
|
||||
case "/api/v1/first-login":
|
||||
response.Header().Set("x-api-csrf", "csrf-value")
|
||||
response.Header().Add("Set-Cookie", "CSRF=csrf-value; Path=/")
|
||||
_, _ = response.Write([]byte(`{"code":200}`))
|
||||
case "/api/v1/login/randCode":
|
||||
if request.Header.Get("x-api-csrf") != "csrf-value" {
|
||||
t.Errorf("captcha CSRF = %q", request.Header.Get("x-api-csrf"))
|
||||
}
|
||||
src := "data:image/jpeg;base64," + base64.StdEncoding.EncodeToString([]byte("captcha-image"))
|
||||
_ = json.NewEncoder(response).Encode(map[string]any{"code": 200, "data": map[string]string{"src": src}})
|
||||
case "/api/v1/login":
|
||||
loginCalls++
|
||||
var payload map[string]string
|
||||
_ = json.NewDecoder(request.Body).Decode(&payload)
|
||||
if payload["username"] != "JXQN0003" || payload["validCode"] != "12WN" {
|
||||
t.Errorf("login payload = %#v", payload)
|
||||
}
|
||||
encrypted, err := base64.StdEncoding.DecodeString(payload["password"])
|
||||
if err != nil || len(encrypted) != 128 || payload["password"] == "JXQN0003-" {
|
||||
t.Errorf("password was not RSA encrypted: len=%d err=%v", len(encrypted), err)
|
||||
}
|
||||
_ = json.NewEncoder(response).Encode(map[string]any{
|
||||
"code": 200, "data": map[string]string{"token": fmt.Sprintf("token-%d", loginCalls)},
|
||||
})
|
||||
case "/api/v1/sys/vehicleRealStatuss":
|
||||
vehicleCalls++
|
||||
if vehicleCalls == 1 {
|
||||
_, _ = response.Write([]byte(`{"code":401,"data":[]}`))
|
||||
return
|
||||
}
|
||||
if request.Header.Get("Cookie") != "CSRF=csrf-value; R_SESS=token-2" {
|
||||
t.Errorf("session cookie = %q", request.Header.Get("Cookie"))
|
||||
}
|
||||
_, _ = response.Write([]byte(`{"code":200,"data":[]}`))
|
||||
default:
|
||||
http.NotFound(response, request)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewAuthenticatedAPIClient(
|
||||
server.URL,
|
||||
LoginCredentials{Username: "JXQN0003", Password: "JXQN0003-", MaxAttempts: 2},
|
||||
solver,
|
||||
time.Second,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := client.Vehicles(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if loginCalls != 2 || solver.calls != 2 || vehicleCalls != 2 {
|
||||
t.Fatalf("login=%d solver=%d vehicles=%d", loginCalls, solver.calls, vehicleCalls)
|
||||
}
|
||||
}
|
||||
481
go/vehicle-gateway/internal/feichibridge/encoder.go
Normal file
481
go/vehicle-gateway/internal/feichibridge/encoder.go
Normal file
@@ -0,0 +1,481 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
CommandRealtime = byte(0x02)
|
||||
CommandReissue = byte(0x03)
|
||||
CommandLogin = byte(0x05)
|
||||
)
|
||||
|
||||
type Encoder struct{}
|
||||
|
||||
func (Encoder) DataFrame(command byte, vin string, at time.Time, record Record) ([]byte, error) {
|
||||
if command != CommandRealtime && command != CommandReissue {
|
||||
return nil, fmt.Errorf("unsupported data command 0x%02X", command)
|
||||
}
|
||||
body := encodeTime(at)
|
||||
units := 0
|
||||
if unit, ok := wholeVehicleUnit(record); ok {
|
||||
body = append(body, 0x01)
|
||||
body = append(body, unit...)
|
||||
units++
|
||||
}
|
||||
if unit, ok := motorUnit(record); ok {
|
||||
body = append(body, 0x02)
|
||||
body = append(body, unit...)
|
||||
units++
|
||||
}
|
||||
if unit, ok := fuelCellUnit(record); ok {
|
||||
body = append(body, 0x03)
|
||||
body = append(body, unit...)
|
||||
units++
|
||||
}
|
||||
if unit, ok := positionUnit(record); ok {
|
||||
body = append(body, 0x05)
|
||||
body = append(body, unit...)
|
||||
units++
|
||||
}
|
||||
if unit, ok := extremeUnit(record); ok {
|
||||
body = append(body, 0x06)
|
||||
body = append(body, unit...)
|
||||
units++
|
||||
}
|
||||
if unit, ok := alarmUnit(record); ok {
|
||||
body = append(body, 0x07)
|
||||
body = append(body, unit...)
|
||||
units++
|
||||
}
|
||||
for _, unit := range voltageUnits(record) {
|
||||
body = append(body, 0x08)
|
||||
body = append(body, unit...)
|
||||
units++
|
||||
}
|
||||
for _, unit := range temperatureUnits(record) {
|
||||
body = append(body, 0x09)
|
||||
body = append(body, unit...)
|
||||
units++
|
||||
}
|
||||
if units == 0 {
|
||||
return nil, errors.New("source record has no mappable GB/T 32960 units")
|
||||
}
|
||||
return buildFrame('#', command, 0xFE, vin, body)
|
||||
}
|
||||
|
||||
func LoginFrame(platformID, username, password string, serial uint16, now time.Time) ([]byte, error) {
|
||||
if len(username) > 12 {
|
||||
return nil, errors.New("GB/T 32960 platform username exceeds 12 bytes")
|
||||
}
|
||||
if len(password) > 20 {
|
||||
return nil, errors.New("GB/T 32960 platform password exceeds 20 bytes")
|
||||
}
|
||||
body := encodeTime(now)
|
||||
body = binary.BigEndian.AppendUint16(body, serial)
|
||||
body = appendPaddedASCII(body, username, 12)
|
||||
body = appendPaddedASCII(body, password, 20)
|
||||
body = append(body, 0x01)
|
||||
return buildFrame('#', CommandLogin, 0xFE, platformID, body)
|
||||
}
|
||||
|
||||
func buildFrame(start, command, response byte, vin string, body []byte) ([]byte, error) {
|
||||
if len(vin) != 17 {
|
||||
return nil, fmt.Errorf("GB/T 32960 identifier must be exactly 17 bytes, got %d", len(vin))
|
||||
}
|
||||
if len(body) > math.MaxUint16 {
|
||||
return nil, errors.New("GB/T 32960 body exceeds 65535 bytes")
|
||||
}
|
||||
frame := make([]byte, 24, 25+len(body))
|
||||
frame[0], frame[1] = start, start
|
||||
frame[2], frame[3] = command, response
|
||||
copy(frame[4:21], vin)
|
||||
frame[21] = 0x01
|
||||
binary.BigEndian.PutUint16(frame[22:24], uint16(len(body)))
|
||||
frame = append(frame, body...)
|
||||
frame = append(frame, bcc(frame[2:]))
|
||||
return frame, nil
|
||||
}
|
||||
|
||||
func wholeVehicleUnit(record Record) ([]byte, bool) {
|
||||
if !hasAny(record, "2201", "2202", "3201", "7615") {
|
||||
return nil, false
|
||||
}
|
||||
out := make([]byte, 20)
|
||||
out[0] = enum(recordValue(record, "3201"), map[string]byte{
|
||||
"启动": 1, "启动状态": 1, "行驶": 1, "熄火": 2, "熄火状态": 2, "其他": 3,
|
||||
}, 0xFE)
|
||||
out[1] = enum(recordValue(record, "2301"), map[string]byte{
|
||||
"停车充电": 1, "行驶充电": 2, "未充电": 3, "未充电状态": 3, "充电完成": 4,
|
||||
}, 0xFE)
|
||||
out[2] = enum(recordValue(record, "2213"), map[string]byte{
|
||||
"纯电": 1, "纯电动": 1, "混动": 2, "混合动力": 2, "燃油": 3,
|
||||
}, 0xFE)
|
||||
putScaledU16(out[3:5], recordValue(record, "2201"), 10, 0)
|
||||
putScaledU32(out[5:9], recordValue(record, "2202"), 10)
|
||||
putScaledU16(out[9:11], recordValue(record, "2613"), 10, 0)
|
||||
putScaledU16(out[11:13], recordValue(record, "2614"), 10, 1000)
|
||||
out[13] = byteValue(recordValue(record, "7615"))
|
||||
out[14] = enum(recordValue(record, "2214"), map[string]byte{"工作": 1, "断开": 2}, 0xFE)
|
||||
out[15] = gearValue(firstField(recordValue(record, "2203")))
|
||||
putU16(out[16:18], recordValue(record, "2617"))
|
||||
out[18] = byteValue(recordValue(record, "2208"))
|
||||
out[19] = byteValue(recordValue(record, "2209"))
|
||||
return out, true
|
||||
}
|
||||
|
||||
func motorUnit(record Record) ([]byte, bool) {
|
||||
// The source carries every motor as a key:value composite separated by "|".
|
||||
// Fields are the GB/T 32960 sequence numbers exposed by the platform metadata.
|
||||
composite := recordValue(record, "2308")
|
||||
if composite == "" {
|
||||
return nil, false
|
||||
}
|
||||
groups := parseCompositeGroups(composite)
|
||||
if len(groups) == 0 || len(groups) > 253 {
|
||||
return nil, false
|
||||
}
|
||||
out := []byte{byte(len(groups))}
|
||||
for index, group := range groups {
|
||||
out = append(out, byteValue(firstNonempty(group["2302"], strconv.Itoa(index+1))))
|
||||
out = append(out, enum(group["2303"], map[string]byte{
|
||||
"耗电": 1, "发电": 2, "关闭": 3, "准备": 4, "准备状态": 4,
|
||||
}, 0xFE))
|
||||
out = append(out, tempByte(group["2304"]))
|
||||
out = binary.BigEndian.AppendUint16(out, offsetU16(group["2305"], 1, 20000))
|
||||
out = binary.BigEndian.AppendUint16(out, offsetU16(group["2306"], 10, 2000))
|
||||
out = append(out, tempByte(group["2309"]))
|
||||
out = binary.BigEndian.AppendUint16(out, scaledU16(group["2311"], 10, 0))
|
||||
out = binary.BigEndian.AppendUint16(out, scaledU16(group["2312"], 10, 1000))
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
func fuelCellUnit(record Record) ([]byte, bool) {
|
||||
if !hasAny(record, "2110", "2111", "2112", "2117", "2119") {
|
||||
return nil, false
|
||||
}
|
||||
temperatureGroups := parseSeriesGroups(recordValue(record, "2103"))
|
||||
var temperatures []string
|
||||
for _, group := range temperatureGroups {
|
||||
temperatures = append(temperatures, group.values...)
|
||||
}
|
||||
if len(temperatures) > math.MaxUint16 {
|
||||
temperatures = temperatures[:math.MaxUint16]
|
||||
}
|
||||
out := binary.BigEndian.AppendUint16(nil, scaledU16(recordValue(record, "2110"), 10, 0))
|
||||
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2111"), 10, 0))
|
||||
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2112"), 100, 0))
|
||||
out = binary.BigEndian.AppendUint16(out, uint16(len(temperatures)))
|
||||
for _, value := range temperatures {
|
||||
out = append(out, tempByte(value))
|
||||
}
|
||||
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2115"), 10, 40))
|
||||
out = append(out, byteValue(recordValue(record, "2116")))
|
||||
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2117"), 1, 0))
|
||||
out = append(out, byteValue(recordValue(record, "2118")))
|
||||
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2119"), 10, 0))
|
||||
out = append(out, byteValue(recordValue(record, "2120")))
|
||||
out = append(out, enum(recordValue(record, "2121"), map[string]byte{"工作": 1, "断开": 2}, 0xFE))
|
||||
return out, true
|
||||
}
|
||||
|
||||
func positionUnit(record Record) ([]byte, bool) {
|
||||
longitude, lonOK := floatValue(recordValue(record, "2502"))
|
||||
latitude, latOK := floatValue(recordValue(record, "2503"))
|
||||
if !lonOK || !latOK || longitude < 0 || latitude < 0 {
|
||||
return nil, false
|
||||
}
|
||||
status := byte(0)
|
||||
text := strings.TrimSpace(recordValue(record, "2501"))
|
||||
if text != "" && (strings.Contains(text, "无效") || text == "1") {
|
||||
status = 1
|
||||
}
|
||||
out := []byte{status}
|
||||
out = binary.BigEndian.AppendUint32(out, uint32(math.Round(longitude*1_000_000)))
|
||||
out = binary.BigEndian.AppendUint32(out, uint32(math.Round(latitude*1_000_000)))
|
||||
return out, true
|
||||
}
|
||||
|
||||
func extremeUnit(record Record) ([]byte, bool) {
|
||||
if !hasAny(record, "2601", "2602", "2603", "2604", "2605", "2606", "2607", "2608", "2609", "2610", "2611", "2612") {
|
||||
return nil, false
|
||||
}
|
||||
out := []byte{
|
||||
byteValue(recordValue(record, "2601")),
|
||||
byteValue(recordValue(record, "2602")),
|
||||
}
|
||||
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2603"), 1000, 0))
|
||||
out = append(out, byteValue(recordValue(record, "2604")), byteValue(recordValue(record, "2605")))
|
||||
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2606"), 1000, 0))
|
||||
out = append(out,
|
||||
byteValue(recordValue(record, "2607")),
|
||||
byteValue(recordValue(record, "2608")),
|
||||
tempByte(recordValue(record, "2609")),
|
||||
byteValue(recordValue(record, "2610")),
|
||||
byteValue(recordValue(record, "2611")),
|
||||
tempByte(recordValue(record, "2612")),
|
||||
)
|
||||
return out, true
|
||||
}
|
||||
|
||||
func alarmUnit(record Record) ([]byte, bool) {
|
||||
levelRaw := recordValue(record, "2900", "2901")
|
||||
var general uint32
|
||||
for bit := 0; bit < 32; bit++ {
|
||||
value := recordValue(record, strconv.Itoa(2901+bit))
|
||||
if truthy(value) {
|
||||
general |= 1 << bit
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(levelRaw) == "" && general == 0 {
|
||||
return nil, false
|
||||
}
|
||||
out := []byte{byteValue(levelRaw)}
|
||||
out = binary.BigEndian.AppendUint32(out, general)
|
||||
out = append(out, 0, 0, 0, 0)
|
||||
return out, true
|
||||
}
|
||||
|
||||
func voltageUnits(record Record) [][]byte {
|
||||
raw := recordValue(record, "2003", "batteryVoltages")
|
||||
groups := parseSeriesGroups(raw)
|
||||
var units [][]byte
|
||||
for _, group := range groups {
|
||||
for start := 0; start < len(group.values); start += 255 {
|
||||
end := start + 255
|
||||
if end > len(group.values) {
|
||||
end = len(group.values)
|
||||
}
|
||||
out := []byte{1, byte(group.id)}
|
||||
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2613"), 10, 0))
|
||||
out = binary.BigEndian.AppendUint16(out, scaledU16(recordValue(record, "2614"), 10, 1000))
|
||||
out = binary.BigEndian.AppendUint16(out, uint16(len(group.values)))
|
||||
out = binary.BigEndian.AppendUint16(out, uint16(start+1))
|
||||
out = append(out, byte(end-start))
|
||||
for _, value := range group.values[start:end] {
|
||||
out = binary.BigEndian.AppendUint16(out, scaledU16(value, 1000, 0))
|
||||
}
|
||||
units = append(units, out)
|
||||
}
|
||||
}
|
||||
return units
|
||||
}
|
||||
|
||||
func temperatureUnits(record Record) [][]byte {
|
||||
groups := parseSeriesGroups(recordValue(record, "2103", "batteryTemperatures"))
|
||||
var units [][]byte
|
||||
for _, group := range groups {
|
||||
for start := 0; start < len(group.values); start += math.MaxUint16 {
|
||||
end := start + math.MaxUint16
|
||||
if end > len(group.values) {
|
||||
end = len(group.values)
|
||||
}
|
||||
out := []byte{1, byte(group.id)}
|
||||
out = binary.BigEndian.AppendUint16(out, uint16(end-start))
|
||||
for _, value := range group.values[start:end] {
|
||||
out = append(out, tempByte(value))
|
||||
}
|
||||
units = append(units, out)
|
||||
}
|
||||
}
|
||||
return units
|
||||
}
|
||||
|
||||
func parseCompositeGroups(raw string) []map[string]string {
|
||||
var groups []map[string]string
|
||||
for _, encoded := range strings.Split(raw, "|") {
|
||||
group := map[string]string{}
|
||||
for _, field := range strings.Split(encoded, ",") {
|
||||
key, value, found := strings.Cut(field, ":")
|
||||
if found {
|
||||
group[strings.TrimSpace(key)] = strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
if len(group) > 0 {
|
||||
groups = append(groups, group)
|
||||
}
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
type seriesGroup struct {
|
||||
id int
|
||||
values []string
|
||||
}
|
||||
|
||||
func parseSeriesGroups(raw string) []seriesGroup {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
var groups []seriesGroup
|
||||
for index, part := range strings.FieldsFunc(raw, func(r rune) bool { return r == ';' || r == '|' }) {
|
||||
group := seriesGroup{id: index + 1}
|
||||
if before, after, found := strings.Cut(part, ":"); found {
|
||||
if parsed, err := strconv.Atoi(strings.TrimSpace(before)); err == nil && parsed > 0 && parsed <= 255 {
|
||||
group.id = parsed
|
||||
}
|
||||
part = after
|
||||
}
|
||||
for _, value := range strings.FieldsFunc(part, func(r rune) bool {
|
||||
return r == '_' || r == ',' || r == ' ' || r == '[' || r == ']'
|
||||
}) {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
group.values = append(group.values, strings.TrimSpace(value))
|
||||
}
|
||||
}
|
||||
if len(group.values) > 0 {
|
||||
groups = append(groups, group)
|
||||
}
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func recordValue(record Record, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if value := strings.TrimSpace(record[key]); value != "" && value != "--" && value != "null" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstField(value string) string {
|
||||
fields := strings.Fields(value)
|
||||
if len(fields) == 0 {
|
||||
return ""
|
||||
}
|
||||
return fields[0]
|
||||
}
|
||||
|
||||
func firstNonempty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func hasAny(record Record, keys ...string) bool { return recordValue(record, keys...) != "" }
|
||||
|
||||
func floatValue(raw string) (float64, bool) {
|
||||
raw = strings.TrimSpace(strings.TrimSuffix(raw, "%"))
|
||||
if raw == "" {
|
||||
return 0, false
|
||||
}
|
||||
value, err := strconv.ParseFloat(raw, 64)
|
||||
return value, err == nil && !math.IsNaN(value) && !math.IsInf(value, 0)
|
||||
}
|
||||
|
||||
func byteValue(raw string) byte {
|
||||
value, ok := floatValue(raw)
|
||||
if !ok || value < 0 || value > 253 {
|
||||
return 0xFE
|
||||
}
|
||||
return byte(math.Round(value))
|
||||
}
|
||||
|
||||
func tempByte(raw string) byte {
|
||||
value, ok := floatValue(raw)
|
||||
if !ok || value < -40 || value > 210 {
|
||||
return 0xFE
|
||||
}
|
||||
return byte(math.Round(value + 40))
|
||||
}
|
||||
|
||||
func putU16(out []byte, raw string) { binary.BigEndian.PutUint16(out, scaledU16(raw, 1, 0)) }
|
||||
|
||||
func putScaledU16(out []byte, raw string, scale, offset float64) {
|
||||
binary.BigEndian.PutUint16(out, scaledU16(raw, scale, offset))
|
||||
}
|
||||
|
||||
func putScaledU32(out []byte, raw string, scale float64) {
|
||||
value, ok := floatValue(raw)
|
||||
if !ok || value < 0 || value*scale > math.MaxUint32-2 {
|
||||
binary.BigEndian.PutUint32(out, 0xFFFFFFFE)
|
||||
return
|
||||
}
|
||||
binary.BigEndian.PutUint32(out, uint32(math.Round(value*scale)))
|
||||
}
|
||||
|
||||
func scaledU16(raw string, scale, offset float64) uint16 {
|
||||
value, ok := floatValue(raw)
|
||||
encoded := (value + offset) * scale
|
||||
if !ok || encoded < 0 || encoded > math.MaxUint16-2 {
|
||||
return 0xFFFE
|
||||
}
|
||||
return uint16(math.Round(encoded))
|
||||
}
|
||||
|
||||
func offsetU16(raw string, scale, offset float64) uint16 {
|
||||
return scaledU16(raw, scale, offset)
|
||||
}
|
||||
|
||||
func enum(raw string, values map[string]byte, missing byte) byte {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return missing
|
||||
}
|
||||
if numeric, err := strconv.ParseUint(raw, 10, 8); err == nil {
|
||||
return byte(numeric)
|
||||
}
|
||||
for label, value := range values {
|
||||
if raw == label || strings.Contains(raw, label) {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
func gearValue(raw string) byte {
|
||||
raw = strings.ToUpper(strings.TrimSpace(raw))
|
||||
switch raw {
|
||||
case "P", "P档":
|
||||
return 15
|
||||
case "R", "R档":
|
||||
return 13
|
||||
case "N", "N档":
|
||||
return 0
|
||||
case "D", "D档":
|
||||
return 14
|
||||
}
|
||||
raw = strings.TrimSuffix(raw, "档")
|
||||
raw = strings.TrimPrefix(raw, "D")
|
||||
return byteValue(raw)
|
||||
}
|
||||
|
||||
func truthy(raw string) bool {
|
||||
raw = strings.ToLower(strings.TrimSpace(raw))
|
||||
return raw != "" && raw != "0" && raw != "false" && raw != "无" && raw != "正常"
|
||||
}
|
||||
|
||||
func appendPaddedASCII(out []byte, value string, width int) []byte {
|
||||
start := len(out)
|
||||
out = append(out, make([]byte, width)...)
|
||||
copy(out[start:start+width], value)
|
||||
return out
|
||||
}
|
||||
|
||||
func encodeTime(value time.Time) []byte {
|
||||
local := value.In(shanghai)
|
||||
return []byte{
|
||||
byte(local.Year() - 2000), byte(local.Month()), byte(local.Day()),
|
||||
byte(local.Hour()), byte(local.Minute()), byte(local.Second()),
|
||||
}
|
||||
}
|
||||
|
||||
func bcc(value []byte) byte {
|
||||
var result byte
|
||||
for _, current := range value {
|
||||
result ^= current
|
||||
}
|
||||
return result
|
||||
}
|
||||
93
go/vehicle-gateway/internal/feichibridge/encoder_test.go
Normal file
93
go/vehicle-gateway/internal/feichibridge/encoder_test.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
|
||||
)
|
||||
|
||||
func TestDataFrameMapsFeichiFields(t *testing.T) {
|
||||
at := time.Date(2026, 7, 17, 14, 5, 6, 0, shanghai)
|
||||
record := Record{
|
||||
"2000": "2026-07-17 14:05:06",
|
||||
"2201": "42.3", "2202": "12345.6", "2203": "D 0 0",
|
||||
"2208": "35", "2209": "0", "2213": "纯电",
|
||||
"2214": "工作", "2301": "未充电状态", "3201": "启动状态",
|
||||
"2613": "550.2", "2614": "-20.5", "2617": "2200", "7615": "78",
|
||||
"2501": "有效定位", "2502": "113.2644", "2503": "23.1291",
|
||||
"2601": "1", "2602": "8", "2603": "3.955",
|
||||
"2604": "1", "2605": "26", "2606": "3.821",
|
||||
"2607": "1", "2608": "3", "2609": "48",
|
||||
"2610": "1", "2611": "9", "2612": "37",
|
||||
"2003": "1:3.900_3.901_3.902",
|
||||
"2103": "1:35_36_37",
|
||||
}
|
||||
frame, err := (Encoder{}).DataFrame(CommandRealtime, "LTEST32960VIN0001", at, record)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env, err := gb32960.ParseFrame(frame, at.UnixMilli(), "test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if env.MessageID != "0x02" || env.VIN != "LTEST32960VIN0001" {
|
||||
t.Fatalf("unexpected envelope: %#v", env)
|
||||
}
|
||||
if got := env.Fields["speed_kmh"]; got != 42.3 {
|
||||
t.Fatalf("speed = %#v", got)
|
||||
}
|
||||
if got := env.Fields["total_mileage_km"]; got != 12345.6 {
|
||||
t.Fatalf("mileage = %#v", got)
|
||||
}
|
||||
if got := env.Fields["soc_percent"]; got != 78 {
|
||||
t.Fatalf("soc = %#v", got)
|
||||
}
|
||||
if got := env.Fields["longitude"]; got != 113.2644 {
|
||||
t.Fatalf("longitude = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDataFrameSplitsMoreThan255CellVoltages(t *testing.T) {
|
||||
values := make([]string, 288)
|
||||
for index := range values {
|
||||
values[index] = fmt.Sprintf("%.3f", 3.5+float64(index)/1000)
|
||||
}
|
||||
frame, err := (Encoder{}).DataFrame(CommandReissue, "LTEST32960VIN0002", time.Now(), Record{
|
||||
"2003": "1:" + strings.Join(values, "_"),
|
||||
"2613": "530",
|
||||
"2614": "10",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := frame[24 : len(frame)-1]
|
||||
if got := strings.Count(string(body), string([]byte{0x08})); got < 2 {
|
||||
t.Fatalf("expected at least two voltage units, got %d", got)
|
||||
}
|
||||
if binary.BigEndian.Uint16(frame[22:24]) != uint16(len(body)) {
|
||||
t.Fatal("body length mismatch")
|
||||
}
|
||||
if _, err := gb32960.ParseFrame(frame, time.Now().UnixMilli(), "test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoginFrameLayout(t *testing.T) {
|
||||
frame, err := LoginFrame("FEICHIBRIDGE00001", "bridge", "secret", 7, time.Date(2026, 7, 17, 1, 2, 3, 0, shanghai))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if frame[2] != CommandLogin || frame[3] != 0xFE {
|
||||
t.Fatalf("unexpected login header: %x", frame[:4])
|
||||
}
|
||||
if got := binary.BigEndian.Uint16(frame[30:32]); got != 7 {
|
||||
t.Fatalf("serial = %d", got)
|
||||
}
|
||||
if got, want := frame[len(frame)-1], bcc(frame[2:len(frame)-1]); got != want {
|
||||
t.Fatalf("BCC = %x, want %x", got, want)
|
||||
}
|
||||
}
|
||||
209
go/vehicle-gateway/internal/feichibridge/login.go
Normal file
209
go/vehicle-gateway/internal/feichibridge/login.go
Normal file
@@ -0,0 +1,209 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const feichiPublicKey = `-----BEGIN PUBLIC KEY-----
|
||||
MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDOwmFtEk1oJxDU0NI4kVO0Jx0X
|
||||
nt+Abx+JKGUzcRzPwjUkd5z9Ice5rh87CCmj0XjZ5pPac6TtA3f0v5FqiK/kjQY5
|
||||
XMLti4weJ4dcp1/q1O7PCYxRX8WgetGxwsjxGn+uoZOZkclN1PFS4wRnKEso6+G/
|
||||
e60QGB29cZoo4jZnZwIDAQAB
|
||||
-----END PUBLIC KEY-----`
|
||||
|
||||
type LoginCredentials struct {
|
||||
Username string
|
||||
Password string
|
||||
MaxAttempts int
|
||||
RetryDelay time.Duration
|
||||
}
|
||||
|
||||
type loginSession struct {
|
||||
credentials LoginCredentials
|
||||
solver CaptchaSolver
|
||||
mu sync.Mutex
|
||||
ready atomic.Bool
|
||||
csrf atomic.Value
|
||||
token atomic.Value
|
||||
}
|
||||
|
||||
type firstLoginEnvelope struct {
|
||||
Code any `json:"code"`
|
||||
}
|
||||
|
||||
type captchaPayload struct {
|
||||
Src string `json:"src"`
|
||||
}
|
||||
|
||||
type loginPayload struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
func newLoginSession(credentials LoginCredentials, solver CaptchaSolver) (*loginSession, error) {
|
||||
credentials.Username = strings.TrimSpace(credentials.Username)
|
||||
if credentials.Username == "" || credentials.Password == "" {
|
||||
return nil, errors.New("Feichi username and password are required")
|
||||
}
|
||||
if solver == nil {
|
||||
return nil, errors.New("captcha solver is required")
|
||||
}
|
||||
if credentials.MaxAttempts <= 0 {
|
||||
credentials.MaxAttempts = 20
|
||||
}
|
||||
if credentials.RetryDelay <= 0 {
|
||||
credentials.RetryDelay = time.Second
|
||||
}
|
||||
session := &loginSession{credentials: credentials, solver: solver}
|
||||
session.csrf.Store("")
|
||||
session.token.Store("")
|
||||
return session, nil
|
||||
}
|
||||
|
||||
func (s *loginSession) invalidate() {
|
||||
s.ready.Store(false)
|
||||
}
|
||||
|
||||
func (s *loginSession) cookieHeader() string {
|
||||
var cookies []string
|
||||
if csrf := s.csrf.Load().(string); csrf != "" {
|
||||
cookies = append(cookies, "CSRF="+csrf)
|
||||
}
|
||||
if token := s.token.Load().(string); token != "" {
|
||||
cookies = append(cookies, "R_SESS="+token)
|
||||
}
|
||||
return strings.Join(cookies, "; ")
|
||||
}
|
||||
|
||||
func (s *loginSession) csrfHeader() string {
|
||||
return s.csrf.Load().(string)
|
||||
}
|
||||
|
||||
func (s *loginSession) ensure(ctx context.Context, client *APIClient) error {
|
||||
if s.ready.Load() {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.ready.Load() {
|
||||
return nil
|
||||
}
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= s.credentials.MaxAttempts; attempt++ {
|
||||
if err := s.loginAttempt(ctx, client); err == nil {
|
||||
s.ready.Store(true)
|
||||
return nil
|
||||
} else {
|
||||
lastErr = err
|
||||
}
|
||||
if attempt < s.credentials.MaxAttempts {
|
||||
timer := time.NewTimer(s.credentials.RetryDelay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("Feichi automatic login failed after %d attempts: %w", s.credentials.MaxAttempts, lastErr)
|
||||
}
|
||||
|
||||
func (s *loginSession) loginAttempt(ctx context.Context, client *APIClient) error {
|
||||
s.ready.Store(false)
|
||||
s.token.Store("")
|
||||
var first firstLoginEnvelope
|
||||
headers, err := client.rawJSON(ctx, http.MethodGet, "api/v1/first-login", nil, &first)
|
||||
if err != nil {
|
||||
return fmt.Errorf("initialize login session: %w", err)
|
||||
}
|
||||
if !apiCodeOK(first.Code) {
|
||||
return apiCodeError(first.Code)
|
||||
}
|
||||
csrf := strings.TrimSpace(headers.Get("x-api-csrf"))
|
||||
if csrf == "" {
|
||||
for _, rawCookie := range headers.Values("Set-Cookie") {
|
||||
cookie, parseErr := http.ParseSetCookie(rawCookie)
|
||||
if parseErr == nil && cookie.Name == "CSRF" {
|
||||
csrf = cookie.Value
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if csrf == "" {
|
||||
return errors.New("first-login response did not provide CSRF")
|
||||
}
|
||||
s.csrf.Store(csrf)
|
||||
|
||||
var captcha objectEnvelope[captchaPayload]
|
||||
if _, err := client.rawJSON(ctx, http.MethodGet, "api/v1/login/randCode", nil, &captcha); err != nil {
|
||||
return fmt.Errorf("request captcha: %w", err)
|
||||
}
|
||||
if !apiCodeOK(captcha.Code) {
|
||||
return apiCodeError(captcha.Code)
|
||||
}
|
||||
encodedImage := captcha.Data.Src
|
||||
if comma := strings.IndexByte(encodedImage, ','); comma >= 0 {
|
||||
encodedImage = encodedImage[comma+1:]
|
||||
}
|
||||
image, err := base64.StdEncoding.DecodeString(encodedImage)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decode captcha image: %w", err)
|
||||
}
|
||||
validCode, err := s.solver.Solve(ctx, image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
encrypted, err := encryptFeichiPassword(s.credentials.Password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var login objectEnvelope[loginPayload]
|
||||
if _, err := client.rawJSON(ctx, http.MethodPost, "api/v1/login", map[string]string{
|
||||
"username": s.credentials.Username,
|
||||
"password": encrypted,
|
||||
"validCode": validCode,
|
||||
}, &login); err != nil {
|
||||
return fmt.Errorf("submit login: %w", err)
|
||||
}
|
||||
if !apiCodeOK(login.Code) {
|
||||
return apiCodeError(login.Code)
|
||||
}
|
||||
token := strings.TrimSpace(login.Data.Token)
|
||||
if token == "" {
|
||||
return errors.New("login succeeded without R_SESS token")
|
||||
}
|
||||
s.token.Store(token)
|
||||
return nil
|
||||
}
|
||||
|
||||
func encryptFeichiPassword(password string) (string, error) {
|
||||
block, _ := pem.Decode([]byte(feichiPublicKey))
|
||||
if block == nil {
|
||||
return "", errors.New("decode Feichi RSA public key")
|
||||
}
|
||||
parsed, err := x509.ParsePKIXPublicKey(block.Bytes)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse Feichi RSA public key: %w", err)
|
||||
}
|
||||
publicKey, ok := parsed.(*rsa.PublicKey)
|
||||
if !ok {
|
||||
return "", errors.New("Feichi public key is not RSA")
|
||||
}
|
||||
encrypted, err := rsa.EncryptPKCS1v15(rand.Reader, publicKey, []byte(password))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("encrypt Feichi password: %w", err)
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(encrypted), nil
|
||||
}
|
||||
94
go/vehicle-gateway/internal/feichibridge/model.go
Normal file
94
go/vehicle-gateway/internal/feichibridge/model.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var shanghai = time.FixedZone("Asia/Shanghai", 8*60*60)
|
||||
|
||||
type Vehicle struct {
|
||||
VehicleID string `json:"vehicleId"`
|
||||
VIN string `json:"vin"`
|
||||
OnlineStatus any `json:"onlineStatus"`
|
||||
UpdateTime string `json:"updateTime"`
|
||||
RuleTypeName string `json:"ruleTypeName"`
|
||||
}
|
||||
|
||||
type Snapshot struct {
|
||||
VehicleID string `json:"vehicleId"`
|
||||
VIN string `json:"vin"`
|
||||
DataItems map[string]string `json:"dataItems"`
|
||||
}
|
||||
|
||||
type Record map[string]string
|
||||
|
||||
type collectionEnvelope[T any] struct {
|
||||
Code any `json:"code"`
|
||||
Data []T `json:"data"`
|
||||
}
|
||||
|
||||
type objectEnvelope[T any] struct {
|
||||
Code any `json:"code"`
|
||||
Data T `json:"data"`
|
||||
}
|
||||
|
||||
type historySegment struct {
|
||||
Subdata []Record `json:"subdata"`
|
||||
}
|
||||
|
||||
func apiCodeOK(code any) bool {
|
||||
switch value := code.(type) {
|
||||
case nil:
|
||||
return true
|
||||
case float64:
|
||||
return value == 0 || value == 200
|
||||
case string:
|
||||
return value == "" || value == "0" || value == "200"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func recordTime(record Record) (time.Time, error) {
|
||||
for _, key := range []string{"2000", "9999", "dataTime", "updateTime", "time"} {
|
||||
if raw := strings.TrimSpace(record[key]); raw != "" {
|
||||
return parseSourceTime(raw)
|
||||
}
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("source record has no timestamp")
|
||||
}
|
||||
|
||||
func parseSourceTime(raw string) (time.Time, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
for _, layout := range []string{
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02T15:04:05",
|
||||
time.RFC3339Nano,
|
||||
} {
|
||||
if layout == time.RFC3339Nano {
|
||||
if parsed, err := time.Parse(layout, raw); err == nil {
|
||||
return parsed, nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
if parsed, err := time.ParseInLocation(layout, raw, shanghai); err == nil {
|
||||
return parsed, nil
|
||||
}
|
||||
}
|
||||
if unixMS, err := strconv.ParseInt(raw, 10, 64); err == nil {
|
||||
if unixMS < 10_000_000_000 {
|
||||
return time.Unix(unixMS, 0).In(shanghai), nil
|
||||
}
|
||||
return time.UnixMilli(unixMS).In(shanghai), nil
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("unsupported source timestamp %q", raw)
|
||||
}
|
||||
|
||||
func canonicalHash(record Record) string {
|
||||
encoded, _ := json.Marshal(record)
|
||||
return hashBytes(encoded)
|
||||
}
|
||||
469
go/vehicle-gateway/internal/feichibridge/service.go
Normal file
469
go/vehicle-gateway/internal/feichibridge/service.go
Normal file
@@ -0,0 +1,469 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
||||
)
|
||||
|
||||
type Source interface {
|
||||
Vehicles(context.Context) ([]Vehicle, error)
|
||||
Snapshot(context.Context, string) (Snapshot, error)
|
||||
History(context.Context, string, time.Time, time.Time) ([]Record, error)
|
||||
}
|
||||
|
||||
type FrameTarget interface {
|
||||
Connect(context.Context) error
|
||||
Send(context.Context, []byte) error
|
||||
Close() error
|
||||
LastACK() time.Time
|
||||
}
|
||||
|
||||
type ServiceConfig struct {
|
||||
PollInterval time.Duration
|
||||
DiscoveryInterval time.Duration
|
||||
BackfillInterval time.Duration
|
||||
BackfillLookback time.Duration
|
||||
BackfillWindow time.Duration
|
||||
BackfillSafetyLag time.Duration
|
||||
SourceStaleAfter time.Duration
|
||||
FetchConcurrency int
|
||||
BackfillEnabled bool
|
||||
StaleReissueEnabled bool
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
config ServiceConfig
|
||||
source Source
|
||||
target FrameTarget
|
||||
state *StateStore
|
||||
encoder Encoder
|
||||
logger *slog.Logger
|
||||
metrics *metrics.Registry
|
||||
|
||||
mu sync.RWMutex
|
||||
vehicles []Vehicle
|
||||
lastSourceSuccess time.Time
|
||||
lastError error
|
||||
}
|
||||
|
||||
func NewService(config ServiceConfig, source Source, target FrameTarget, state *StateStore, logger *slog.Logger, registry *metrics.Registry) (*Service, error) {
|
||||
if source == nil || target == nil || state == nil {
|
||||
return nil, errors.New("source, target, and state are required")
|
||||
}
|
||||
if config.PollInterval <= 0 {
|
||||
config.PollInterval = 10 * time.Second
|
||||
}
|
||||
if config.DiscoveryInterval <= 0 {
|
||||
config.DiscoveryInterval = 5 * time.Minute
|
||||
}
|
||||
if config.BackfillInterval <= 0 {
|
||||
config.BackfillInterval = time.Hour
|
||||
}
|
||||
if config.BackfillLookback <= 0 {
|
||||
config.BackfillLookback = time.Hour
|
||||
}
|
||||
if config.BackfillWindow <= 0 {
|
||||
config.BackfillWindow = 20 * time.Minute
|
||||
}
|
||||
if config.BackfillSafetyLag <= 0 {
|
||||
config.BackfillSafetyLag = 30 * time.Second
|
||||
}
|
||||
if config.SourceStaleAfter <= 0 {
|
||||
config.SourceStaleAfter = 2 * time.Minute
|
||||
}
|
||||
if config.FetchConcurrency <= 0 {
|
||||
config.FetchConcurrency = 4
|
||||
}
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Service{
|
||||
config: config, source: source, target: target, state: state,
|
||||
logger: logger, metrics: registry,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) Run(ctx context.Context) error {
|
||||
if err := s.target.Connect(ctx); err != nil {
|
||||
s.setError(err)
|
||||
return fmt.Errorf("connect GB/T 32960 target: %w", err)
|
||||
}
|
||||
if err := s.discover(ctx); err != nil {
|
||||
s.setError(err)
|
||||
return fmt.Errorf("initial vehicle discovery: %w", err)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
s.discoveryLoop(ctx)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
s.realtimeLoop(ctx)
|
||||
}()
|
||||
if s.config.BackfillEnabled {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
s.backfillLoop(ctx)
|
||||
}()
|
||||
}
|
||||
<-ctx.Done()
|
||||
_ = s.target.Close()
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) Ready(context.Context) error {
|
||||
s.mu.RLock()
|
||||
lastSourceSuccess := s.lastSourceSuccess
|
||||
lastErr := s.lastError
|
||||
s.mu.RUnlock()
|
||||
maxSourceAge := max(3*s.config.PollInterval, time.Minute)
|
||||
if lastSourceSuccess.IsZero() || time.Since(lastSourceSuccess) > maxSourceAge {
|
||||
if lastErr != nil {
|
||||
return fmt.Errorf("source unavailable: %w", lastErr)
|
||||
}
|
||||
return errors.New("source has not completed a successful request")
|
||||
}
|
||||
lastACK := s.target.LastACK()
|
||||
if lastACK.IsZero() || time.Since(lastACK) > max(3*s.config.DiscoveryInterval, 10*time.Minute) {
|
||||
return errors.New("GB/T 32960 target has no recent ACK")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) discoveryLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(s.config.DiscoveryInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := s.discover(ctx); err != nil {
|
||||
s.recordFailure("discover", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) realtimeLoop(ctx context.Context) {
|
||||
s.pollRealtime(ctx)
|
||||
ticker := time.NewTicker(s.config.PollInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.pollRealtime(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) backfillLoop(ctx context.Context) {
|
||||
s.runBackfill(ctx)
|
||||
ticker := time.NewTicker(s.config.BackfillInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.runBackfill(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) discover(ctx context.Context) error {
|
||||
start := time.Now()
|
||||
vehicles, err := s.source.Vehicles(ctx)
|
||||
s.observeAPI("vehicles", start, err)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
filtered := make([]Vehicle, 0, len(vehicles))
|
||||
for _, vehicle := range vehicles {
|
||||
vehicle.VIN = strings.TrimSpace(vehicle.VIN)
|
||||
vehicle.VehicleID = strings.TrimSpace(vehicle.VehicleID)
|
||||
if len(vehicle.VIN) != 17 || vehicle.VehicleID == "" {
|
||||
continue
|
||||
}
|
||||
if vehicle.RuleTypeName != "" && !strings.Contains(strings.ToUpper(vehicle.RuleTypeName), "32960") {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, vehicle)
|
||||
}
|
||||
sort.Slice(filtered, func(i, j int) bool { return filtered[i].VIN < filtered[j].VIN })
|
||||
s.mu.Lock()
|
||||
s.vehicles = filtered
|
||||
s.lastSourceSuccess = time.Now()
|
||||
s.lastError = nil
|
||||
s.mu.Unlock()
|
||||
if s.metrics != nil {
|
||||
s.metrics.SetGauge("vehicle_feichi_bridge_vehicles", nil, float64(len(filtered)))
|
||||
}
|
||||
s.logger.Info("feichi vehicles discovered", "count", len(filtered))
|
||||
return nil
|
||||
}
|
||||
|
||||
type fetchedSnapshot struct {
|
||||
vehicle Vehicle
|
||||
record Record
|
||||
at time.Time
|
||||
hash string
|
||||
err error
|
||||
duration time.Duration
|
||||
}
|
||||
|
||||
func (s *Service) pollRealtime(ctx context.Context) {
|
||||
vehicles := s.vehicleSnapshot()
|
||||
jobs := make(chan Vehicle)
|
||||
results := make(chan fetchedSnapshot, len(vehicles))
|
||||
var wg sync.WaitGroup
|
||||
for worker := 0; worker < min(s.config.FetchConcurrency, len(vehicles)); worker++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for vehicle := range jobs {
|
||||
start := time.Now()
|
||||
snapshot, err := s.source.Snapshot(ctx, vehicle.VehicleID)
|
||||
result := fetchedSnapshot{vehicle: vehicle, duration: time.Since(start), err: err}
|
||||
if err == nil {
|
||||
result.record = Record(snapshot.DataItems)
|
||||
result.at, result.err = recordTime(result.record)
|
||||
result.hash = canonicalHash(result.record)
|
||||
}
|
||||
results <- result
|
||||
}
|
||||
}()
|
||||
}
|
||||
for _, vehicle := range vehicles {
|
||||
jobs <- vehicle
|
||||
}
|
||||
close(jobs)
|
||||
wg.Wait()
|
||||
close(results)
|
||||
|
||||
var snapshots []fetchedSnapshot
|
||||
for result := range results {
|
||||
s.observeAPIWithDuration("snapshot", result.duration, result.err)
|
||||
if result.err != nil {
|
||||
s.recordFailure("snapshot", fmt.Errorf("VIN %s: %w", result.vehicle.VIN, result.err))
|
||||
continue
|
||||
}
|
||||
snapshots = append(snapshots, result)
|
||||
s.markSourceSuccess()
|
||||
}
|
||||
sort.Slice(snapshots, func(i, j int) bool {
|
||||
if snapshots[i].at.Equal(snapshots[j].at) {
|
||||
return snapshots[i].vehicle.VIN < snapshots[j].vehicle.VIN
|
||||
}
|
||||
return snapshots[i].at.Before(snapshots[j].at)
|
||||
})
|
||||
for _, snapshot := range snapshots {
|
||||
if time.Since(snapshot.at) > s.config.SourceStaleAfter {
|
||||
if s.config.StaleReissueEnabled {
|
||||
s.reissueStaleSnapshot(ctx, snapshot)
|
||||
}
|
||||
continue
|
||||
}
|
||||
current := s.state.Vehicle(snapshot.vehicle.VIN)
|
||||
if snapshot.at.Before(current.LastRealtimeTime) ||
|
||||
(snapshot.at.Equal(current.LastRealtimeTime) && snapshot.hash == current.LastRealtimeHash) {
|
||||
continue
|
||||
}
|
||||
frame, err := s.encoder.DataFrame(CommandRealtime, snapshot.vehicle.VIN, snapshot.at, snapshot.record)
|
||||
if err != nil {
|
||||
s.recordFailure("encode", fmt.Errorf("VIN %s: %w", snapshot.vehicle.VIN, err))
|
||||
continue
|
||||
}
|
||||
if err := s.target.Send(ctx, frame); err != nil {
|
||||
s.recordFrame(CommandRealtime, "error", snapshot.vehicle.VIN, snapshot.at)
|
||||
s.recordFailure("send", fmt.Errorf("VIN %s: %w", snapshot.vehicle.VIN, err))
|
||||
continue
|
||||
}
|
||||
if err := s.state.CommitRealtime(snapshot.vehicle.VIN, snapshot.at, snapshot.hash); err != nil {
|
||||
s.recordFailure("state", err)
|
||||
continue
|
||||
}
|
||||
s.recordFrame(CommandRealtime, "acked", snapshot.vehicle.VIN, snapshot.at)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) reissueStaleSnapshot(ctx context.Context, snapshot fetchedSnapshot) {
|
||||
current := s.state.Vehicle(snapshot.vehicle.VIN)
|
||||
if snapshot.at.Before(current.LastSnapshotReissueTime) ||
|
||||
(snapshot.at.Equal(current.LastSnapshotReissueTime) && snapshot.hash == current.LastSnapshotReissueHash) {
|
||||
return
|
||||
}
|
||||
frame, err := s.encoder.DataFrame(CommandReissue, snapshot.vehicle.VIN, snapshot.at, snapshot.record)
|
||||
if err != nil {
|
||||
s.recordFailure("encode_stale_reissue", fmt.Errorf("VIN %s: %w", snapshot.vehicle.VIN, err))
|
||||
return
|
||||
}
|
||||
if err := s.target.Send(ctx, frame); err != nil {
|
||||
s.recordFrame(CommandReissue, "error", snapshot.vehicle.VIN, snapshot.at)
|
||||
s.recordFailure("send_stale_reissue", fmt.Errorf("VIN %s: %w", snapshot.vehicle.VIN, err))
|
||||
return
|
||||
}
|
||||
if err := s.state.CommitSnapshotReissue(snapshot.vehicle.VIN, snapshot.at, snapshot.hash); err != nil {
|
||||
s.recordFailure("state", err)
|
||||
return
|
||||
}
|
||||
s.recordFrame(CommandReissue, "acked", snapshot.vehicle.VIN, snapshot.at)
|
||||
}
|
||||
|
||||
func (s *Service) runBackfill(ctx context.Context) {
|
||||
for _, vehicle := range s.vehicleSnapshot() {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if err := s.backfillVehicle(ctx, vehicle); err != nil {
|
||||
s.recordFailure("backfill", fmt.Errorf("VIN %s: %w", vehicle.VIN, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) backfillVehicle(ctx context.Context, vehicle Vehicle) error {
|
||||
end := time.Now().Add(-s.config.BackfillSafetyLag)
|
||||
cursor := s.state.Vehicle(vehicle.VIN).BackfillCursor
|
||||
if cursor.IsZero() {
|
||||
cursor = end.Add(-s.config.BackfillLookback)
|
||||
}
|
||||
for cursor.Before(end) {
|
||||
windowEnd := cursor.Add(s.config.BackfillWindow)
|
||||
if windowEnd.After(end) {
|
||||
windowEnd = end
|
||||
}
|
||||
start := time.Now()
|
||||
records, err := s.source.History(ctx, vehicle.VIN, cursor, windowEnd)
|
||||
s.observeAPI("history", start, err)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Slice(records, func(i, j int) bool {
|
||||
left, _ := recordTime(records[i])
|
||||
right, _ := recordTime(records[j])
|
||||
return left.Before(right)
|
||||
})
|
||||
committed := cursor
|
||||
for _, record := range records {
|
||||
at, err := recordTime(record)
|
||||
if err != nil || !at.After(cursor) || at.After(windowEnd) {
|
||||
continue
|
||||
}
|
||||
frame, err := s.encoder.DataFrame(CommandReissue, vehicle.VIN, at, record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.target.Send(ctx, frame); err != nil {
|
||||
s.recordFrame(CommandReissue, "error", vehicle.VIN, at)
|
||||
return err
|
||||
}
|
||||
if err := s.state.CommitBackfill(vehicle.VIN, at); err != nil {
|
||||
return err
|
||||
}
|
||||
s.recordFrame(CommandReissue, "acked", vehicle.VIN, at)
|
||||
committed = at
|
||||
}
|
||||
if !committed.After(cursor) || committed.Before(windowEnd) {
|
||||
if err := s.state.AdvanceBackfillCursor(vehicle.VIN, windowEnd); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
cursor = windowEnd
|
||||
s.markSourceSuccess()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) vehicleSnapshot() []Vehicle {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return append([]Vehicle(nil), s.vehicles...)
|
||||
}
|
||||
|
||||
func (s *Service) markSourceSuccess() {
|
||||
s.mu.Lock()
|
||||
s.lastSourceSuccess = time.Now()
|
||||
s.lastError = nil
|
||||
s.mu.Unlock()
|
||||
if s.metrics != nil {
|
||||
metrics.RecordLastActivity(s.metrics, "vehicle_feichi_bridge_source_last_success_unix_seconds", nil)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) setError(err error) {
|
||||
s.mu.Lock()
|
||||
s.lastError = err
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
func (s *Service) recordFailure(operation string, err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
s.setError(err)
|
||||
if s.metrics != nil {
|
||||
s.metrics.IncCounter("vehicle_feichi_bridge_errors_total", metrics.Labels{"operation": operation})
|
||||
}
|
||||
s.logger.Error("feichi bridge operation failed", "operation", operation, "error", err)
|
||||
}
|
||||
|
||||
func (s *Service) observeAPI(operation string, start time.Time, err error) {
|
||||
s.observeAPIWithDuration(operation, time.Since(start), err)
|
||||
}
|
||||
|
||||
func (s *Service) observeAPIWithDuration(operation string, duration time.Duration, err error) {
|
||||
if s.metrics == nil {
|
||||
return
|
||||
}
|
||||
status := "ok"
|
||||
if err != nil {
|
||||
status = "error"
|
||||
}
|
||||
s.metrics.IncCounter("vehicle_feichi_bridge_api_requests_total", metrics.Labels{"operation": operation, "status": status})
|
||||
s.metrics.ObserveHistogram(
|
||||
"vehicle_feichi_bridge_api_request_duration_seconds",
|
||||
metrics.Labels{"operation": operation},
|
||||
[]float64{0.1, 0.25, 0.5, 1, 2, 5},
|
||||
duration.Seconds(),
|
||||
)
|
||||
}
|
||||
|
||||
func (s *Service) recordFrame(command byte, status, vin string, sourceTime time.Time) {
|
||||
if s.metrics != nil {
|
||||
s.metrics.IncCounter("vehicle_feichi_bridge_frames_total", metrics.Labels{
|
||||
"command": fmt.Sprintf("0x%02X", command),
|
||||
"status": status,
|
||||
})
|
||||
}
|
||||
if status == "acked" {
|
||||
if s.metrics != nil {
|
||||
metrics.RecordLastActivity(s.metrics, "vehicle_feichi_bridge_target_last_ack_unix_seconds", nil)
|
||||
s.metrics.SetGauge(
|
||||
"vehicle_feichi_bridge_vehicle_last_ack_unix_seconds",
|
||||
metrics.Labels{"vin": vin, "command": fmt.Sprintf("0x%02X", command)},
|
||||
float64(time.Now().Unix()),
|
||||
)
|
||||
}
|
||||
s.logger.Info(
|
||||
"GB/T 32960 vehicle frame acknowledged",
|
||||
"vin", vin,
|
||||
"command", fmt.Sprintf("0x%02X", command),
|
||||
"source_time", sourceTime,
|
||||
)
|
||||
}
|
||||
}
|
||||
123
go/vehicle-gateway/internal/feichibridge/service_test.go
Normal file
123
go/vehicle-gateway/internal/feichibridge/service_test.go
Normal file
@@ -0,0 +1,123 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeSource struct {
|
||||
vehicles []Vehicle
|
||||
record Record
|
||||
}
|
||||
|
||||
func (f *fakeSource) Vehicles(context.Context) ([]Vehicle, error) {
|
||||
return append([]Vehicle(nil), f.vehicles...), nil
|
||||
}
|
||||
|
||||
func (f *fakeSource) Snapshot(context.Context, string) (Snapshot, error) {
|
||||
return Snapshot{DataItems: map[string]string(f.record)}, nil
|
||||
}
|
||||
|
||||
func (f *fakeSource) History(context.Context, string, time.Time, time.Time) ([]Record, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type fakeTarget struct {
|
||||
mu sync.Mutex
|
||||
frames [][]byte
|
||||
sendErr error
|
||||
lastACK time.Time
|
||||
}
|
||||
|
||||
func (f *fakeTarget) Connect(context.Context) error { return nil }
|
||||
func (f *fakeTarget) Close() error { return nil }
|
||||
func (f *fakeTarget) LastACK() time.Time { return f.lastACK }
|
||||
func (f *fakeTarget) Send(_ context.Context, frame []byte) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.sendErr != nil {
|
||||
return f.sendErr
|
||||
}
|
||||
f.frames = append(f.frames, append([]byte(nil), frame...))
|
||||
f.lastACK = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestRealtimeCommitsOnlyAfterACKAndDeduplicates(t *testing.T) {
|
||||
now := time.Now().In(shanghai).Truncate(time.Second)
|
||||
source := &fakeSource{
|
||||
vehicles: []Vehicle{{
|
||||
VehicleID: "id-1", VIN: "LTEST32960VIN0001", RuleTypeName: "GB_T32960",
|
||||
}},
|
||||
record: Record{"2000": now.Format("2006-01-02 15:04:05"), "2201": "10"},
|
||||
}
|
||||
target := &fakeTarget{}
|
||||
store, err := OpenStateStore(filepath.Join(t.TempDir(), "state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service, err := NewService(ServiceConfig{
|
||||
SourceStaleAfter: time.Minute,
|
||||
FetchConcurrency: 1,
|
||||
}, source, target, store, slog.Default(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := service.discover(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service.pollRealtime(context.Background())
|
||||
service.pollRealtime(context.Background())
|
||||
if len(target.frames) != 1 {
|
||||
t.Fatalf("frames = %d, want 1", len(target.frames))
|
||||
}
|
||||
if got := store.Vehicle("LTEST32960VIN0001"); !got.LastRealtimeTime.Equal(now) {
|
||||
t.Fatalf("committed state = %#v", got)
|
||||
}
|
||||
|
||||
source.record = Record{"2000": now.Add(time.Second).Format("2006-01-02 15:04:05"), "2201": "11"}
|
||||
target.sendErr = errors.New("target down")
|
||||
service.pollRealtime(context.Background())
|
||||
if got := store.Vehicle("LTEST32960VIN0001"); !got.LastRealtimeTime.Equal(now) {
|
||||
t.Fatalf("cursor advanced without ACK: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaleSnapshotIsReissuedOnlyOnce(t *testing.T) {
|
||||
at := time.Now().In(shanghai).Add(-24 * time.Hour).Truncate(time.Second)
|
||||
vin := "LTEST32960VIN0002"
|
||||
source := &fakeSource{
|
||||
vehicles: []Vehicle{{VehicleID: "id-2", VIN: vin, RuleTypeName: "GB_T32960"}},
|
||||
record: Record{"2000": at.Format("2006-01-02 15:04:05"), "2201": "0"},
|
||||
}
|
||||
target := &fakeTarget{}
|
||||
store, err := OpenStateStore(filepath.Join(t.TempDir(), "state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service, err := NewService(ServiceConfig{
|
||||
SourceStaleAfter: time.Minute,
|
||||
FetchConcurrency: 1,
|
||||
StaleReissueEnabled: true,
|
||||
}, source, target, store, slog.Default(), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := service.discover(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
service.pollRealtime(context.Background())
|
||||
service.pollRealtime(context.Background())
|
||||
if len(target.frames) != 1 || target.frames[0][2] != CommandReissue {
|
||||
t.Fatalf("frames = %d command = %#v", len(target.frames), target.frames)
|
||||
}
|
||||
state := store.Vehicle(vin)
|
||||
if !state.LastSnapshotReissueTime.Equal(at) || state.LastSnapshotReissueACKAt.IsZero() {
|
||||
t.Fatalf("reissue state = %#v", state)
|
||||
}
|
||||
}
|
||||
160
go/vehicle-gateway/internal/feichibridge/state.go
Normal file
160
go/vehicle-gateway/internal/feichibridge/state.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type VehicleState struct {
|
||||
LastRealtimeTime time.Time `json:"last_realtime_time,omitempty"`
|
||||
LastRealtimeHash string `json:"last_realtime_hash,omitempty"`
|
||||
LastRealtimeACKAt time.Time `json:"last_realtime_ack_at,omitempty"`
|
||||
LastSnapshotReissueTime time.Time `json:"last_snapshot_reissue_time,omitempty"`
|
||||
LastSnapshotReissueHash string `json:"last_snapshot_reissue_hash,omitempty"`
|
||||
LastSnapshotReissueACKAt time.Time `json:"last_snapshot_reissue_ack_at,omitempty"`
|
||||
BackfillCursor time.Time `json:"backfill_cursor,omitempty"`
|
||||
LastBackfillACKAt time.Time `json:"last_backfill_ack_at,omitempty"`
|
||||
}
|
||||
|
||||
func (s *StateStore) CommitSnapshotReissue(vin string, at time.Time, hash string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
current := s.state.Vehicles[vin]
|
||||
current.LastSnapshotReissueTime = at
|
||||
current.LastSnapshotReissueHash = hash
|
||||
current.LastSnapshotReissueACKAt = time.Now()
|
||||
s.state.Vehicles[vin] = current
|
||||
return s.saveLocked()
|
||||
}
|
||||
|
||||
type persistentState struct {
|
||||
Version int `json:"version"`
|
||||
PlatformSerial uint16 `json:"platform_serial"`
|
||||
Vehicles map[string]VehicleState `json:"vehicles"`
|
||||
}
|
||||
|
||||
type StateStore struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
state persistentState
|
||||
}
|
||||
|
||||
func OpenStateStore(path string) (*StateStore, error) {
|
||||
store := &StateStore{
|
||||
path: path,
|
||||
state: persistentState{
|
||||
Version: 1,
|
||||
Vehicles: map[string]VehicleState{},
|
||||
},
|
||||
}
|
||||
encoded, err := os.ReadFile(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return store, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read bridge state: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(encoded, &store.state); err != nil {
|
||||
return nil, fmt.Errorf("decode bridge state: %w", err)
|
||||
}
|
||||
if store.state.Vehicles == nil {
|
||||
store.state.Vehicles = map[string]VehicleState{}
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func (s *StateStore) Vehicle(vin string) VehicleState {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.state.Vehicles[vin]
|
||||
}
|
||||
|
||||
func (s *StateStore) CommitRealtime(vin string, at time.Time, hash string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
current := s.state.Vehicles[vin]
|
||||
current.LastRealtimeTime = at
|
||||
current.LastRealtimeHash = hash
|
||||
current.LastRealtimeACKAt = time.Now()
|
||||
s.state.Vehicles[vin] = current
|
||||
return s.saveLocked()
|
||||
}
|
||||
|
||||
func (s *StateStore) CommitBackfill(vin string, at time.Time) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
current := s.state.Vehicles[vin]
|
||||
current.BackfillCursor = at
|
||||
current.LastBackfillACKAt = time.Now()
|
||||
s.state.Vehicles[vin] = current
|
||||
return s.saveLocked()
|
||||
}
|
||||
|
||||
func (s *StateStore) AdvanceBackfillCursor(vin string, at time.Time) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
current := s.state.Vehicles[vin]
|
||||
current.BackfillCursor = at
|
||||
s.state.Vehicles[vin] = current
|
||||
return s.saveLocked()
|
||||
}
|
||||
|
||||
func (s *StateStore) NextPlatformSerial() (uint16, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.state.PlatformSerial++
|
||||
if s.state.PlatformSerial == 0 {
|
||||
s.state.PlatformSerial = 1
|
||||
}
|
||||
if err := s.saveLocked(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.state.PlatformSerial, nil
|
||||
}
|
||||
|
||||
func (s *StateStore) saveLocked() error {
|
||||
if err := os.MkdirAll(filepath.Dir(s.path), 0o750); err != nil {
|
||||
return fmt.Errorf("create bridge state directory: %w", err)
|
||||
}
|
||||
encoded, err := json.MarshalIndent(s.state, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temp, err := os.CreateTemp(filepath.Dir(s.path), ".state-*.json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tempName := temp.Name()
|
||||
defer os.Remove(tempName)
|
||||
if err := temp.Chmod(0o600); err != nil {
|
||||
temp.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := temp.Write(encoded); err != nil {
|
||||
temp.Close()
|
||||
return err
|
||||
}
|
||||
if err := temp.Sync(); err != nil {
|
||||
temp.Close()
|
||||
return err
|
||||
}
|
||||
if err := temp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tempName, s.path); err != nil {
|
||||
return fmt.Errorf("replace bridge state: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func hashBytes(value []byte) string {
|
||||
sum := sha256.Sum256(value)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
38
go/vehicle-gateway/internal/feichibridge/state_test.go
Normal file
38
go/vehicle-gateway/internal/feichibridge/state_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStateStorePersistsAcknowledgedCursor(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "nested", "state.json")
|
||||
store, err := OpenStateStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := time.Date(2026, 7, 17, 12, 0, 0, 0, time.UTC)
|
||||
if err := store.CommitRealtime("LTEST32960VIN0001", at, "hash"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.CommitBackfill("LTEST32960VIN0001", at.Add(-time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reopened, err := OpenStateStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := reopened.Vehicle("LTEST32960VIN0001")
|
||||
if !got.LastRealtimeTime.Equal(at) || got.LastRealtimeHash != "hash" {
|
||||
t.Fatalf("state = %#v", got)
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("state mode = %o", info.Mode().Perm())
|
||||
}
|
||||
}
|
||||
193
go/vehicle-gateway/internal/feichibridge/target.go
Normal file
193
go/vehicle-gateway/internal/feichibridge/target.go
Normal file
@@ -0,0 +1,193 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type TargetConfig struct {
|
||||
Address string
|
||||
PlatformID string
|
||||
Username string
|
||||
Password string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type Target struct {
|
||||
mu sync.Mutex
|
||||
config TargetConfig
|
||||
state *StateStore
|
||||
dialer net.Dialer
|
||||
conn net.Conn
|
||||
lastACK time.Time
|
||||
}
|
||||
|
||||
func NewTarget(config TargetConfig, state *StateStore) (*Target, error) {
|
||||
if strings.TrimSpace(config.Address) == "" {
|
||||
return nil, errors.New("GB/T 32960 target address is required")
|
||||
}
|
||||
if len(config.PlatformID) != 17 {
|
||||
return nil, fmt.Errorf("target platform ID must be exactly 17 bytes")
|
||||
}
|
||||
if state == nil {
|
||||
return nil, errors.New("state store is required")
|
||||
}
|
||||
if config.Timeout <= 0 {
|
||||
config.Timeout = 10 * time.Second
|
||||
}
|
||||
return &Target{
|
||||
config: config,
|
||||
state: state,
|
||||
dialer: net.Dialer{Timeout: config.Timeout, KeepAlive: 30 * time.Second},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (t *Target) Send(ctx context.Context, frame []byte) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
if err := t.ensureConnected(ctx); err != nil {
|
||||
lastErr = err
|
||||
t.closeLocked()
|
||||
continue
|
||||
}
|
||||
if err := t.sendAndACK(ctx, frame); err != nil {
|
||||
lastErr = err
|
||||
t.closeLocked()
|
||||
continue
|
||||
}
|
||||
t.lastACK = time.Now()
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("send GB/T 32960 frame after reconnect: %w", lastErr)
|
||||
}
|
||||
|
||||
func (t *Target) Connect(ctx context.Context) error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if err := t.ensureConnected(ctx); err != nil {
|
||||
t.closeLocked()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Target) Close() error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.conn == nil {
|
||||
return nil
|
||||
}
|
||||
err := t.conn.Close()
|
||||
t.conn = nil
|
||||
return err
|
||||
}
|
||||
|
||||
func (t *Target) LastACK() time.Time {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.lastACK
|
||||
}
|
||||
|
||||
func (t *Target) ensureConnected(ctx context.Context) error {
|
||||
if t.conn != nil {
|
||||
return nil
|
||||
}
|
||||
conn, err := t.dialer.DialContext(ctx, "tcp", t.config.Address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dial target %s: %w", t.config.Address, err)
|
||||
}
|
||||
t.conn = conn
|
||||
serial, err := t.state.NextPlatformSerial()
|
||||
if err != nil {
|
||||
return fmt.Errorf("allocate platform login serial: %w", err)
|
||||
}
|
||||
login, err := LoginFrame(t.config.PlatformID, t.config.Username, t.config.Password, serial, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := t.sendAndACK(ctx, login); err != nil {
|
||||
return fmt.Errorf("GB/T 32960 platform login: %w", err)
|
||||
}
|
||||
t.lastACK = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Target) sendAndACK(ctx context.Context, frame []byte) error {
|
||||
if t.conn == nil {
|
||||
return errors.New("target connection is closed")
|
||||
}
|
||||
deadline := time.Now().Add(t.config.Timeout)
|
||||
if contextDeadline, ok := ctx.Deadline(); ok && contextDeadline.Before(deadline) {
|
||||
deadline = contextDeadline
|
||||
}
|
||||
if err := t.conn.SetDeadline(deadline); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeFull(t.conn, frame); err != nil {
|
||||
return fmt.Errorf("write target frame: %w", err)
|
||||
}
|
||||
response, err := readFrame(t.conn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read target ACK: %w", err)
|
||||
}
|
||||
if response[2] != frame[2] {
|
||||
return fmt.Errorf("target ACK command mismatch: got 0x%02X want 0x%02X", response[2], frame[2])
|
||||
}
|
||||
if response[3] != 0x01 {
|
||||
return fmt.Errorf("target rejected command 0x%02X with response 0x%02X", response[2], response[3])
|
||||
}
|
||||
if string(response[4:21]) != string(frame[4:21]) {
|
||||
return errors.New("target ACK identifier mismatch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *Target) closeLocked() {
|
||||
if t.conn != nil {
|
||||
_ = t.conn.Close()
|
||||
t.conn = nil
|
||||
}
|
||||
}
|
||||
|
||||
func readFrame(reader io.Reader) ([]byte, error) {
|
||||
header := make([]byte, 24)
|
||||
if _, err := io.ReadFull(reader, header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if (header[0] != '#' || header[1] != '#') && (header[0] != '$' || header[1] != '$') {
|
||||
return nil, fmt.Errorf("bad GB/T 32960 ACK start %q", header[:2])
|
||||
}
|
||||
bodyLength := int(binary.BigEndian.Uint16(header[22:24]))
|
||||
tail := make([]byte, bodyLength+1)
|
||||
if _, err := io.ReadFull(reader, tail); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
frame := append(header, tail...)
|
||||
if got, want := bcc(frame[2:len(frame)-1]), frame[len(frame)-1]; got != want {
|
||||
return nil, fmt.Errorf("bad GB/T 32960 ACK BCC: got 0x%02X want 0x%02X", got, want)
|
||||
}
|
||||
return frame, nil
|
||||
}
|
||||
|
||||
func writeFull(writer io.Writer, value []byte) error {
|
||||
for len(value) > 0 {
|
||||
written, err := writer.Write(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if written == 0 {
|
||||
return io.ErrShortWrite
|
||||
}
|
||||
value = value[written:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
83
go/vehicle-gateway/internal/feichibridge/target_test.go
Normal file
83
go/vehicle-gateway/internal/feichibridge/target_test.go
Normal file
@@ -0,0 +1,83 @@
|
||||
package feichibridge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTargetLogsInAndWaitsForACK(t *testing.T) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
var commands []byte
|
||||
var mu sync.Mutex
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
conn, acceptErr := listener.Accept()
|
||||
if acceptErr != nil {
|
||||
done <- acceptErr
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
for count := 0; count < 2; count++ {
|
||||
frame, readErr := readFrame(conn)
|
||||
if readErr != nil {
|
||||
done <- readErr
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
commands = append(commands, frame[2])
|
||||
mu.Unlock()
|
||||
body := []byte(nil)
|
||||
if len(frame) >= 31 {
|
||||
body = append(body, frame[24:30]...)
|
||||
}
|
||||
ack, buildErr := buildFrame('#', frame[2], 0x01, string(frame[4:21]), body)
|
||||
if buildErr != nil {
|
||||
done <- buildErr
|
||||
return
|
||||
}
|
||||
if writeErr := writeFull(conn, ack); writeErr != nil {
|
||||
done <- writeErr
|
||||
return
|
||||
}
|
||||
}
|
||||
done <- nil
|
||||
}()
|
||||
|
||||
store, err := OpenStateStore(filepath.Join(t.TempDir(), "state.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
target, err := NewTarget(TargetConfig{
|
||||
Address: listener.Addr().String(), PlatformID: "FEICHIBRIDGE00001",
|
||||
Username: "bridge", Password: "secret", Timeout: time.Second,
|
||||
}, store)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
frame, err := (Encoder{}).DataFrame(CommandRealtime, "LTEST32960VIN0001", time.Now(), Record{"2201": "1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := target.Send(context.Background(), frame); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(commands) != 2 || commands[0] != CommandLogin || commands[1] != CommandRealtime {
|
||||
t.Fatalf("commands = %x", commands)
|
||||
}
|
||||
if target.LastACK().IsZero() {
|
||||
t.Fatal("last ACK was not recorded")
|
||||
}
|
||||
}
|
||||
@@ -151,16 +151,23 @@ func parseDataBody(version string, body []byte, fields map[string]any) (time.Tim
|
||||
fields[envelope.FieldSOCPercent] = unit["soc_percent"]
|
||||
cursor += size
|
||||
case 0x05:
|
||||
if len(body[cursor:]) < 9 {
|
||||
size := 9
|
||||
if version == "V2025" {
|
||||
size = 10
|
||||
}
|
||||
if len(body[cursor:]) < size {
|
||||
units = append(units, map[string]any{"type": "0x05", "error": "truncated"})
|
||||
return eventTime, units
|
||||
}
|
||||
unit := parsePositionData(body[cursor : cursor+9])
|
||||
unit := parsePositionData(version, body[cursor:cursor+size])
|
||||
units = append(units, map[string]any{"type": "0x05", "name": "position", "value": unit})
|
||||
fields["position_status"] = unit["position_status"]
|
||||
if coordinateSystem, ok := unit["coordinate_system"]; ok {
|
||||
fields["coordinate_system"] = coordinateSystem
|
||||
}
|
||||
fields[envelope.FieldLongitude] = unit["longitude"]
|
||||
fields[envelope.FieldLatitude] = unit["latitude"]
|
||||
cursor += 9
|
||||
cursor += size
|
||||
case 0x02:
|
||||
if len(body[cursor:]) < 1 {
|
||||
units = append(units, map[string]any{"type": "0x02", "error": "truncated"})
|
||||
@@ -364,21 +371,27 @@ func parsePlatformLogin(body []byte) map[string]any {
|
||||
}
|
||||
|
||||
func parseVehicleData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
out := map[string]any{
|
||||
"vehicle_status": int(data[0]),
|
||||
"charge_status": int(data[1]),
|
||||
"running_mode": int(data[2]),
|
||||
"speed_kmh": scaledU16(data[3:5], 10),
|
||||
"total_mileage_km": scaledU32(data[5:9], 10),
|
||||
"total_voltage_v": scaledU16(data[9:11], 10),
|
||||
"total_current_a": scaledU16WithOffset(data[11:13], 10, -1000),
|
||||
"soc_percent": int(data[13]),
|
||||
"speed_kmh": nullableScaledU16(data[3:5], 10),
|
||||
"total_mileage_km": nullableScaledU32(data[5:9], 10),
|
||||
"total_voltage_v": nullableScaledU16(data[9:11], 10),
|
||||
"total_current_a": nullableScaledU16WithOffset(data[11:13], 10, -1000),
|
||||
"soc_percent": nullableByte(data[13]),
|
||||
"dc_dc_status": int(data[14]),
|
||||
"gear": int(data[15]),
|
||||
"insulation_kohm": binary.BigEndian.Uint16(data[16:18]),
|
||||
"accelerator_pct": int(data[18]),
|
||||
"brake_pct": int(data[19]),
|
||||
"insulation_kohm": nullableU16(data[16:18]),
|
||||
}
|
||||
// GB/T 32960.3-2025 removes the 2016 accelerator and brake bytes from
|
||||
// the 0x01 vehicle unit. Keep them only when the frame actually carries
|
||||
// the 20-byte 2016 payload.
|
||||
if len(data) >= 20 {
|
||||
out["accelerator_pct"] = nullableByte(data[18])
|
||||
out["brake_pct"] = nullableByte(data[19])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseDriveMotorData(data []byte) map[string]any {
|
||||
@@ -389,12 +402,12 @@ func parseDriveMotorData(data []byte) map[string]any {
|
||||
motors = append(motors, map[string]any{
|
||||
"serial_no": int(data[cursor]),
|
||||
"state": int(data[cursor+1]),
|
||||
"controller_temperature_c": int(data[cursor+2]) - 40,
|
||||
"speed_rpm": int(binary.BigEndian.Uint16(data[cursor+3:cursor+5])) - 20000,
|
||||
"torque_nm": scaledIntWithOffset(int64(binary.BigEndian.Uint16(data[cursor+5:cursor+7])), 10, -20000),
|
||||
"motor_temperature_c": int(data[cursor+7]) - 40,
|
||||
"controller_voltage_v": scaledU16(data[cursor+8:cursor+10], 10),
|
||||
"controller_current_a": scaledU16WithOffset(data[cursor+10:cursor+12], 10, -1000),
|
||||
"controller_temperature_c": nullableTempOffset40(data[cursor+2]),
|
||||
"speed_rpm": nullableU16WithOffset(data[cursor+3:cursor+5], -20000),
|
||||
"torque_nm": nullableScaledU16WithOffset(data[cursor+5:cursor+7], 10, -20000),
|
||||
"motor_temperature_c": nullableTempOffset40(data[cursor+7]),
|
||||
"controller_voltage_v": nullableScaledU16(data[cursor+8:cursor+10], 10),
|
||||
"controller_current_a": nullableScaledU16WithOffset(data[cursor+10:cursor+12], 10, -1000),
|
||||
})
|
||||
cursor += 12
|
||||
}
|
||||
@@ -406,34 +419,45 @@ func parseDriveMotorData(data []byte) map[string]any {
|
||||
|
||||
func parseFuelCellData(data []byte) map[string]any {
|
||||
probeCount := int(binary.BigEndian.Uint16(data[6:8]))
|
||||
probes := make([]int, 0, probeCount)
|
||||
probes := make([]any, 0, probeCount)
|
||||
cursor := 8
|
||||
for i := 0; i < probeCount; i++ {
|
||||
probes = append(probes, int(data[cursor])-40)
|
||||
probes = append(probes, nullableTempOffset40(data[cursor]))
|
||||
cursor++
|
||||
}
|
||||
out := map[string]any{
|
||||
"fuel_cell_voltage_v": scaledU16(data[0:2], 10),
|
||||
"fuel_cell_current_a": scaledU16(data[2:4], 10),
|
||||
"hydrogen_consumption_kg_per_100km": scaledU16(data[4:6], 100),
|
||||
"fuel_cell_voltage_v": nullableScaledU16(data[0:2], 10),
|
||||
"fuel_cell_current_a": nullableScaledU16(data[2:4], 10),
|
||||
"hydrogen_consumption_kg_per_100km": nullableScaledU16(data[4:6], 100),
|
||||
"temperature_probe_count": probeCount,
|
||||
"temperature_probe_values_c": probes,
|
||||
"max_hydrogen_temperature_c": scaledU16WithOffset(data[cursor:cursor+2], 10, -40),
|
||||
"max_hydrogen_temperature_c": nullableScaledU16WithOffset(data[cursor:cursor+2], 10, -40),
|
||||
"max_hydrogen_temperature_probe_id": int(data[cursor+2]),
|
||||
"max_hydrogen_concentration_fraction": scaledU16(data[cursor+3:cursor+5], 1_000_000),
|
||||
"max_hydrogen_concentration_probe_id": int(data[cursor+5]),
|
||||
"max_hydrogen_pressure_mpa": scaledU16(data[cursor+6:cursor+8], 10),
|
||||
"max_hydrogen_pressure_mpa": nullableScaledU16(data[cursor+6:cursor+8], 10),
|
||||
"max_hydrogen_pressure_probe_id": int(data[cursor+8]),
|
||||
"dc_dc_status": int(data[cursor+9]),
|
||||
}
|
||||
if concentration := nullableU16(data[cursor+3 : cursor+5]); concentration != nil {
|
||||
raw := concentration.(int)
|
||||
out["max_hydrogen_concentration_fraction"] = float64(raw) / 1_000_000
|
||||
out["max_hydrogen_concentration_ppm"] = raw
|
||||
out["max_hydrogen_concentration_percent"] = float64(raw) / 10_000
|
||||
} else {
|
||||
out["max_hydrogen_concentration_fraction"] = nil
|
||||
out["max_hydrogen_concentration_ppm"] = nil
|
||||
out["max_hydrogen_concentration_percent"] = nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func parseEngineData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"engine_status": int(data[0]),
|
||||
"crank_speed_rpm": binary.BigEndian.Uint16(data[1:3]),
|
||||
"fuel_rate": binary.BigEndian.Uint16(data[3:5]),
|
||||
"crank_speed_rpm": nullableU16(data[1:3]),
|
||||
// The supplied authoritative reference does not define this field's
|
||||
// business unit or scale, so preserve the decoded protocol integer.
|
||||
"fuel_rate": nullableU16(data[3:5]),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -632,11 +656,13 @@ func parseGdFCStackData(data []byte) map[string]any {
|
||||
"frame_cell_count": frameCellCount,
|
||||
}
|
||||
cursor += 22
|
||||
frameVoltages := make([]float64, 0, frameCellCount)
|
||||
maxCell := 0.0
|
||||
minCell := 0.0
|
||||
seen := false
|
||||
for c := 0; c < frameCellCount; c++ {
|
||||
voltage := scaledU16(data[cursor:cursor+2], 1000)
|
||||
frameVoltages = append(frameVoltages, voltage)
|
||||
if !seen || voltage > maxCell {
|
||||
maxCell = voltage
|
||||
}
|
||||
@@ -650,6 +676,7 @@ func parseGdFCStackData(data []byte) map[string]any {
|
||||
summary["frame_max_cell_voltage_v"] = maxCell
|
||||
summary["frame_min_cell_voltage_v"] = minCell
|
||||
}
|
||||
summary["frame_cell_voltages_v"] = frameVoltages
|
||||
summaries = append(summaries, summary)
|
||||
}
|
||||
return map[string]any{
|
||||
@@ -760,6 +787,13 @@ func nullableU16WithOffset(data []byte, offset int) any {
|
||||
return int(value) + offset
|
||||
}
|
||||
|
||||
func nullableByte(value byte) any {
|
||||
if value == 0xfe || value == 0xff {
|
||||
return nil
|
||||
}
|
||||
return int(value)
|
||||
}
|
||||
|
||||
func scaledU16(data []byte, divisor int64) float64 {
|
||||
return scaledInt(int64(binary.BigEndian.Uint16(data)), divisor)
|
||||
}
|
||||
@@ -791,6 +825,14 @@ func nullableScaledU16(data []byte, divisor int64) any {
|
||||
return scaledInt(int64(value), divisor)
|
||||
}
|
||||
|
||||
func nullableScaledU32(data []byte, divisor int64) any {
|
||||
value := binary.BigEndian.Uint32(data)
|
||||
if value == 0xfffffffe || value == 0xffffffff {
|
||||
return nil
|
||||
}
|
||||
return scaledInt(int64(value), divisor)
|
||||
}
|
||||
|
||||
func nullableScaledU16WithOffset(data []byte, divisor int64, offset int64) any {
|
||||
value := binary.BigEndian.Uint16(data)
|
||||
if value == 0xfffe || value == 0xffff {
|
||||
@@ -806,12 +848,16 @@ func nullableTempOffset40(value byte) any {
|
||||
return int(value) - 40
|
||||
}
|
||||
|
||||
func parsePositionData(data []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"position_status": int(data[0]),
|
||||
"longitude": float64(binary.BigEndian.Uint32(data[1:5])) / 1_000_000,
|
||||
"latitude": float64(binary.BigEndian.Uint32(data[5:9])) / 1_000_000,
|
||||
func parsePositionData(version string, data []byte) map[string]any {
|
||||
offset := 1
|
||||
out := map[string]any{"position_status": int(data[0])}
|
||||
if version == "V2025" {
|
||||
out["coordinate_system"] = int(data[1])
|
||||
offset = 2
|
||||
}
|
||||
out["longitude"] = float64(binary.BigEndian.Uint32(data[offset:offset+4])) / 1_000_000
|
||||
out["latitude"] = float64(binary.BigEndian.Uint32(data[offset+4:offset+8])) / 1_000_000
|
||||
return out
|
||||
}
|
||||
|
||||
func indexStart(data []byte) int {
|
||||
|
||||
@@ -222,6 +222,13 @@ func TestParseFrameKeepsParsingRealFuelCellReportUntilUnknownExtension(t *testin
|
||||
if units[2]["name"] != "fuel_cell" {
|
||||
t.Fatalf("unexpected fuel cell unit: %#v", units[2])
|
||||
}
|
||||
fuelCell, ok := units[2]["value"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("fuel cell payload missing: %#v", units[2])
|
||||
}
|
||||
if _, ok := fuelCell["max_hydrogen_concentration_ppm"].(int); !ok {
|
||||
t.Fatalf("hydrogen concentration ppm missing: %#v", fuelCell["max_hydrogen_concentration_ppm"])
|
||||
}
|
||||
if units[9]["name"] != "gd_fc_stack" {
|
||||
t.Fatalf("unexpected gd stack unit: %#v", units[9])
|
||||
}
|
||||
@@ -237,6 +244,69 @@ func TestParseFrameKeepsParsingRealFuelCellReportUntilUnknownExtension(t *testin
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFuelCellDataExposesHydrogenConcentrationInReferencePercent(t *testing.T) {
|
||||
data := make([]byte, 18)
|
||||
data[11] = 0x01
|
||||
data[12] = 0xf4
|
||||
fuelCell := parseFuelCellData(data)
|
||||
if concentration, ok := fuelCell["max_hydrogen_concentration_ppm"].(int); !ok || concentration != 500 {
|
||||
t.Fatalf("hydrogen concentration ppm = %#v, want int(500)", fuelCell["max_hydrogen_concentration_ppm"])
|
||||
}
|
||||
if fraction, ok := fuelCell["max_hydrogen_concentration_fraction"].(float64); !ok || math.Abs(fraction-0.0005) > 0.0000001 {
|
||||
t.Fatalf("hydrogen concentration fraction = %#v, want 0.0005", fuelCell["max_hydrogen_concentration_fraction"])
|
||||
}
|
||||
if percent, ok := fuelCell["max_hydrogen_concentration_percent"].(float64); !ok || math.Abs(percent-0.05) > 0.0000001 {
|
||||
t.Fatalf("hydrogen concentration percent = %#v, want 0.05", fuelCell["max_hydrogen_concentration_percent"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFuelCellDataTreatsProtocolInvalidMarkersAsMissing(t *testing.T) {
|
||||
data := make([]byte, 18)
|
||||
for index := range data {
|
||||
data[index] = 0xff
|
||||
}
|
||||
data[6], data[7] = 0, 0
|
||||
fuelCell := parseFuelCellData(data)
|
||||
for _, key := range []string{"fuel_cell_voltage_v", "fuel_cell_current_a", "hydrogen_consumption_kg_per_100km", "max_hydrogen_temperature_c", "max_hydrogen_concentration_percent", "max_hydrogen_pressure_mpa"} {
|
||||
if fuelCell[key] != nil {
|
||||
t.Fatalf("%s should be nil for protocol invalid marker: %#v", key, fuelCell[key])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseV2025VehicleAndPositionUsesCoordinateSystemByte(t *testing.T) {
|
||||
body := []byte{0x1a, 0x07, 0x11, 0x0a, 0x00, 0x00, 0x01}
|
||||
body = append(body,
|
||||
0x01, 0x03, 0x01,
|
||||
0x00, 0x64,
|
||||
0x00, 0x00, 0x03, 0xe8,
|
||||
0x13, 0x88,
|
||||
0x27, 0x10,
|
||||
80, 0x01, 0x00,
|
||||
0x03, 0xe8)
|
||||
body = append(body,
|
||||
0x05,
|
||||
0x00, 0x02,
|
||||
0x07, 0x36, 0x50, 0x40,
|
||||
0x01, 0xd2, 0x4f, 0x00)
|
||||
fields := map[string]any{}
|
||||
_, units := parseDataBody("V2025", body, fields)
|
||||
if len(units) != 2 {
|
||||
t.Fatalf("units = %#v", units)
|
||||
}
|
||||
vehicle := units[0]["value"].(map[string]any)
|
||||
if _, exists := vehicle["accelerator_pct"]; exists {
|
||||
t.Fatalf("2025 vehicle unit must not invent removed pedal fields: %#v", vehicle)
|
||||
}
|
||||
position := units[1]["value"].(map[string]any)
|
||||
if position["coordinate_system"] != 2 {
|
||||
t.Fatalf("coordinate system = %#v", position["coordinate_system"])
|
||||
}
|
||||
if longitude := position["longitude"].(float64); math.Abs(longitude-121) > 0.000001 {
|
||||
t.Fatalf("longitude = %v", longitude)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScaledDecimalFieldsDoNotLeakBinaryFloatTails(t *testing.T) {
|
||||
unit := parseDriveMotorData([]byte{
|
||||
0x01,
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/yutongmqtt"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry"
|
||||
)
|
||||
|
||||
func TestRealtimeKVFieldsFromGB32960ParsedDomains(t *testing.T) {
|
||||
@@ -157,6 +159,37 @@ func TestBuildFieldsEnvelopeCopiesSourceMetadata(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFieldsEnvelopeKeepsYutongJSONNumberMileage(t *testing.T) {
|
||||
env, err := yutongmqtt.ParseMessage(
|
||||
"yutong",
|
||||
"/ytforward/shln/1",
|
||||
[]byte(`{"device":"LMRKH9AC2R1004106","time":"2026-07-17 11:50:13.021","data":{"LATITUDE":30.466027,"LONGITUDE":103.96096,"METER_SPEED":0,"TOTAL_MILEAGE":77081000}}`),
|
||||
1784260213051,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseMessage() error = %v", err)
|
||||
}
|
||||
if !EnsureParsedFields(&env) {
|
||||
t.Fatal("EnsureParsedFields() should compute ingress fields")
|
||||
}
|
||||
fieldsEnv, ok := BuildFieldsEnvelope(env)
|
||||
if !ok {
|
||||
t.Fatal("BuildFieldsEnvelope() should emit Yutong fields")
|
||||
}
|
||||
for _, field := range []string{
|
||||
"yutong_mqtt.data.total_mileage",
|
||||
"yutong_mqtt.root.data.total_mileage",
|
||||
} {
|
||||
if got := fieldsEnv.Fields[field]; got != "77081000" {
|
||||
t.Fatalf("%s = %#v, want 77081000", field, got)
|
||||
}
|
||||
}
|
||||
mileageKM, found := telemetry.TotalMileageKM(fieldsEnv.Protocol, fieldsEnv.Fields)
|
||||
if !found || mileageKM != 77081 {
|
||||
t.Fatalf("TotalMileageKM() = %.3f, %v; want 77081, true", mileageKM, found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFieldsEnvelopeDropsNonPositiveTotalMileage(t *testing.T) {
|
||||
fieldsEnv, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
|
||||
@@ -825,12 +825,27 @@ func positiveNumber(value any) bool {
|
||||
return typed > 0
|
||||
case int:
|
||||
return typed > 0
|
||||
case int8:
|
||||
return typed > 0
|
||||
case int16:
|
||||
return typed > 0
|
||||
case int32:
|
||||
return typed > 0
|
||||
case int64:
|
||||
return typed > 0
|
||||
case uint:
|
||||
return typed > 0
|
||||
case uint8:
|
||||
return typed > 0
|
||||
case uint16:
|
||||
return typed > 0
|
||||
case uint32:
|
||||
return typed > 0
|
||||
case uint64:
|
||||
return typed > 0
|
||||
case json.Number:
|
||||
parsed, err := typed.Float64()
|
||||
return err == nil && parsed > 0
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return err == nil && parsed > 0
|
||||
|
||||
@@ -18,6 +18,7 @@ const (
|
||||
defaultCacheRetention = 72 * time.Hour
|
||||
defaultCacheCleanupInterval = 10 * time.Minute
|
||||
defaultBaselineMissTTL = time.Minute
|
||||
defaultBaselineHitTTL = 5 * time.Minute
|
||||
defaultMaxCacheEntries = 1000000
|
||||
)
|
||||
|
||||
@@ -34,6 +35,7 @@ type Writer struct {
|
||||
cacheRetention time.Duration
|
||||
cacheCleanupInterval time.Duration
|
||||
baselineMissTTL time.Duration
|
||||
baselineHitTTL time.Duration
|
||||
maxCacheEntries int
|
||||
lastCacheCleanup time.Time
|
||||
lastCacheCleanupStats cacheCleanupStats
|
||||
@@ -122,6 +124,7 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
|
||||
cacheRetention: defaultCacheRetention,
|
||||
cacheCleanupInterval: defaultCacheCleanupInterval,
|
||||
baselineMissTTL: defaultBaselineMissTTL,
|
||||
baselineHitTTL: defaultBaselineHitTTL,
|
||||
maxCacheEntries: defaultMaxCacheEntries,
|
||||
lastTotalMileage: map[string]float64{},
|
||||
lastSourceSeen: map[string]time.Time{},
|
||||
@@ -182,6 +185,15 @@ func (w *Writer) SetBaselineMissTTL(ttl time.Duration) {
|
||||
w.baselineMissTTL = ttl
|
||||
}
|
||||
|
||||
func (w *Writer) SetBaselineHitTTL(ttl time.Duration) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if ttl < 0 {
|
||||
ttl = 0
|
||||
}
|
||||
w.baselineHitTTL = ttl
|
||||
}
|
||||
|
||||
func (w *Writer) SetMaxCacheEntries(maxEntries int) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
@@ -517,8 +529,12 @@ func (w *Writer) previousBaseline(ctx context.Context, candidate SourceMileageSa
|
||||
now := time.Now()
|
||||
w.mu.Lock()
|
||||
if cached, ok := w.baselineCache[cacheKey]; ok {
|
||||
missExpired := !cached.found && (w.baselineMissTTL == 0 || cached.cachedAt.IsZero() || now.Sub(cached.cachedAt) >= w.baselineMissTTL)
|
||||
if !missExpired {
|
||||
ttl := w.baselineMissTTL
|
||||
if cached.found {
|
||||
ttl = w.baselineHitTTL
|
||||
}
|
||||
expired := ttl == 0 || cached.cachedAt.IsZero() || now.Sub(cached.cachedAt) >= ttl
|
||||
if !expired {
|
||||
w.mu.Unlock()
|
||||
return cached.baseline, cached.found, nil
|
||||
}
|
||||
@@ -570,6 +586,16 @@ func (w *Writer) cacheBaseline(candidate SourceMileageSample, cacheKey string, e
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
if previous, ok := w.baselineCache[cacheKey]; ok &&
|
||||
previous.found && entry.found &&
|
||||
previous.baseline.LatestTotalKM == entry.baseline.LatestTotalKM &&
|
||||
previous.baseline.LatestEventTime.Equal(entry.baseline.LatestEventTime) &&
|
||||
!previous.cachedAt.IsZero() {
|
||||
// markBaselineWritten runs for every accepted sample. Preserve the
|
||||
// original cache age when the durable day-boundary baseline has not
|
||||
// changed, otherwise an active vehicle would keep stale history forever.
|
||||
entry.cachedAt = previous.cachedAt
|
||||
}
|
||||
if entry.cachedAt.IsZero() {
|
||||
entry.cachedAt = time.Now()
|
||||
}
|
||||
|
||||
@@ -1091,6 +1091,91 @@ func TestWriterRetriesExpiredMissingPreviousDayBaseline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterRefreshesExpiredFoundPreviousDayBaseline(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
writer := NewWriter(db, loc)
|
||||
writer.SetBaselineHitTTL(time.Minute)
|
||||
candidate := SourceMileageSample{
|
||||
VIN: "LMRKH9AC6R1004111",
|
||||
StatDate: "2026-07-17",
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
SourceKey: "YUTONG_MQTT:LMRKH9AC6R1004111@PLATFORM:yutong",
|
||||
}
|
||||
cacheKey := mileageCacheKey(MetricSample{
|
||||
VIN: candidate.VIN,
|
||||
Protocol: candidate.Protocol,
|
||||
StatDate: candidate.StatDate,
|
||||
SourceKey: candidate.SourceKey,
|
||||
})
|
||||
staleTime := time.Date(2026, 7, 12, 4, 12, 20, 0, loc)
|
||||
writer.baselineCache[cacheKey] = sourceBaselineCacheEntry{
|
||||
baseline: sourceBaseline{LatestTotalKM: 90778, LatestEventTime: staleTime},
|
||||
found: true,
|
||||
cachedAt: time.Now().Add(-2 * time.Minute),
|
||||
}
|
||||
|
||||
refreshedTime := time.Date(2026, 7, 16, 23, 59, 59, 0, loc)
|
||||
mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`).
|
||||
WithArgs(candidate.VIN, candidate.StatDate, "YUTONG_MQTT", candidate.SourceKey).
|
||||
WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}).
|
||||
AddRow(91302.0, refreshedTime))
|
||||
|
||||
baseline, found, err := writer.previousBaseline(context.Background(), candidate)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("previousBaseline() found=%v error=%v, want refreshed baseline", found, err)
|
||||
}
|
||||
if baseline.LatestTotalKM != 91302 || !baseline.LatestEventTime.Equal(refreshedTime) {
|
||||
t.Fatalf("baseline = %+v, want nearest refreshed baseline", baseline)
|
||||
}
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Fatalf("sql expectations: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterMarkBaselineKeepsCacheAgeForUnchangedBoundary(t *testing.T) {
|
||||
db, _, err := sqlmock.New()
|
||||
if err != nil {
|
||||
t.Fatalf("sqlmock.New() error = %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
loc := time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
writer := NewWriter(db, loc)
|
||||
candidate := SourceMileageSample{
|
||||
VIN: "LMRKH9AC6R1004111",
|
||||
StatDate: "2026-07-17",
|
||||
Protocol: envelope.ProtocolYutongMQTT,
|
||||
SourceKey: "YUTONG_MQTT:LMRKH9AC6R1004111@PLATFORM:yutong",
|
||||
FirstTotalKM: 91302,
|
||||
FirstEventTime: time.Date(2026, 7, 16, 23, 59, 59, 0, loc),
|
||||
}
|
||||
sample := MetricSample{
|
||||
VIN: candidate.VIN,
|
||||
Protocol: candidate.Protocol,
|
||||
StatDate: candidate.StatDate,
|
||||
SourceKey: candidate.SourceKey,
|
||||
}
|
||||
cacheKey := mileageCacheKey(sample)
|
||||
cachedAt := time.Now().Add(-4 * time.Minute)
|
||||
writer.baselineCache[cacheKey] = sourceBaselineCacheEntry{
|
||||
baseline: sourceBaseline{LatestTotalKM: candidate.FirstTotalKM, LatestEventTime: candidate.FirstEventTime},
|
||||
found: true,
|
||||
cachedAt: cachedAt,
|
||||
}
|
||||
|
||||
writer.markBaselineWritten(sample, candidate)
|
||||
|
||||
if got := writer.baselineCache[cacheKey].cachedAt; !got.Equal(cachedAt) {
|
||||
t.Fatalf("cachedAt = %s, want unchanged %s", got, cachedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterAppendUsesPreviousSourceBaselineForRealtimeCandidate(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
if err != nil {
|
||||
|
||||
@@ -337,10 +337,30 @@ func projectDailyMileageWithExec(ctx context.Context, exec Execer, vin string, s
|
||||
return err
|
||||
}
|
||||
|
||||
const upsertSourceStatDateStartSQL = `CAST(CONCAT(VALUES(stat_date), ' 00:00:00') AS DATETIME)`
|
||||
|
||||
const upsertSourcePreferIncomingFirstSQL = `(
|
||||
first_event_time IS NULL
|
||||
OR (
|
||||
first_event_time < ` + upsertSourceStatDateStartSQL + `
|
||||
AND VALUES(first_event_time) < ` + upsertSourceStatDateStartSQL + `
|
||||
AND VALUES(first_event_time) >= first_event_time
|
||||
)
|
||||
OR (
|
||||
first_event_time >= ` + upsertSourceStatDateStartSQL + `
|
||||
AND VALUES(first_event_time) < ` + upsertSourceStatDateStartSQL + `
|
||||
)
|
||||
OR (
|
||||
first_event_time >= ` + upsertSourceStatDateStartSQL + `
|
||||
AND VALUES(first_event_time) >= ` + upsertSourceStatDateStartSQL + `
|
||||
AND VALUES(first_event_time) <= first_event_time
|
||||
)
|
||||
)`
|
||||
|
||||
const upsertSourceMergedFirstTotalSQL = `CASE
|
||||
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
|
||||
THEN VALUES(first_total_mileage_km)
|
||||
WHEN first_event_time IS NULL OR VALUES(first_event_time) <= first_event_time
|
||||
WHEN ` + upsertSourcePreferIncomingFirstSQL + `
|
||||
THEN VALUES(first_total_mileage_km)
|
||||
ELSE first_total_mileage_km
|
||||
END`
|
||||
@@ -354,7 +374,7 @@ const upsertSourceMergedLatestTotalSQL = `CASE
|
||||
END`
|
||||
|
||||
const upsertSourceMergedFirstEventSQL = `CASE
|
||||
WHEN first_event_time IS NULL OR VALUES(first_event_time) <= first_event_time
|
||||
WHEN ` + upsertSourcePreferIncomingFirstSQL + `
|
||||
THEN VALUES(first_event_time)
|
||||
ELSE first_event_time
|
||||
END`
|
||||
@@ -490,6 +510,8 @@ WHERE s.vin = ?
|
||||
AND s.stat_date = ?
|
||||
AND s.protocol = ?
|
||||
AND s.quality_status IN ('` + QualityOK + `', '` + QualityNoPreviousBaseline + `')
|
||||
AND s.first_total_mileage_km IS NOT NULL
|
||||
AND s.latest_total_mileage_km IS NOT NULL
|
||||
AND COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code) IS NOT NULL
|
||||
AND COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')) IS NOT NULL
|
||||
AND s.source_key <> CONCAT(s.protocol, ':', COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')), '` + platformSourceKeyPrefix + `', COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code))
|
||||
@@ -558,6 +580,8 @@ WHERE s.vin = ?
|
||||
AND s.stat_date = ?
|
||||
AND s.protocol = ?
|
||||
AND s.quality_status IN ('` + QualityOK + `', '` + QualityNoPreviousBaseline + `')
|
||||
AND s.first_total_mileage_km IS NOT NULL
|
||||
AND s.latest_total_mileage_km IS NOT NULL
|
||||
AND COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code) IS NOT NULL
|
||||
AND COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')) IS NOT NULL
|
||||
AND s.source_key <> CONCAT(s.protocol, ':', COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')), '` + platformSourceKeyPrefix + `', COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code))
|
||||
|
||||
@@ -187,7 +187,9 @@ func TestUpsertSourceMileageTruncatesSubsecondEventTimesAtDayBoundary(t *testing
|
||||
func TestUpsertSourceMileageUsesEventTimeBoundaries(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"first_total_mileage_km = CASE",
|
||||
"VALUES(first_event_time) >= first_event_time",
|
||||
"VALUES(first_event_time) <= first_event_time",
|
||||
"VALUES(first_event_time) < CAST(CONCAT(VALUES(stat_date), ' 00:00:00') AS DATETIME)",
|
||||
"latest_total_mileage_km = CASE",
|
||||
"VALUES(latest_event_time) >= latest_event_time",
|
||||
"daily_mileage_km = CASE",
|
||||
@@ -334,6 +336,22 @@ func TestNormalizePlatformSourceMileageMergesLegacyIPKeys(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePlatformSourceMileageExcludesIncompleteLegacyTotals(t *testing.T) {
|
||||
for name, query := range map[string]string{
|
||||
"insert": normalizePlatformSourceMileageInsertSQL,
|
||||
"delete": normalizePlatformSourceMileageDeleteSQL,
|
||||
} {
|
||||
for _, predicate := range []string{
|
||||
"s.first_total_mileage_km IS NOT NULL",
|
||||
"s.latest_total_mileage_km IS NOT NULL",
|
||||
} {
|
||||
if !strings.Contains(query, predicate) {
|
||||
t.Fatalf("%s query missing incomplete-total guard %q:\n%s", name, predicate, query)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePlatformSourceMileageForDateNormalizesAndProjectsLegacyVINs(t *testing.T) {
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
|
||||
if err != nil {
|
||||
|
||||
@@ -5,6 +5,7 @@ HOST=""
|
||||
REMOTE_ROOT="/opt/lingniu-go-native"
|
||||
DAYS_BACK=1
|
||||
WINDOW_DAYS=3
|
||||
BASELINE_LOOKBACK_DAYS=7
|
||||
PROTOCOLS="GB32960,JT808,YUTONG_MQTT"
|
||||
ON_CALENDAR="*-*-* 01:30:00"
|
||||
DRY_RUN="false"
|
||||
@@ -19,6 +20,9 @@ Options:
|
||||
--remote-root DIR Remote install root. Defaults to /opt/lingniu-go-native.
|
||||
--days-back N Target end date relative to local day. Defaults to 1, meaning yesterday.
|
||||
--window-days N Number of days to recalculate ending at days-back. Defaults to 3.
|
||||
--baseline-lookback-days N
|
||||
Maximum days searched before the target window for a mileage baseline.
|
||||
Defaults to 7.
|
||||
--protocols LIST Comma-separated protocols. Defaults to GB32960,JT808,YUTONG_MQTT.
|
||||
--on-calendar SPEC systemd OnCalendar value. Defaults to "*-*-* 01:30:00".
|
||||
--dry-run Install timer in dry-run mode.
|
||||
@@ -44,6 +48,10 @@ while [[ $# -gt 0 ]]; do
|
||||
WINDOW_DAYS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--baseline-lookback-days)
|
||||
BASELINE_LOOKBACK_DAYS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--protocols)
|
||||
PROTOCOLS="${2:-}"
|
||||
shift 2
|
||||
@@ -86,11 +94,17 @@ case "$WINDOW_DAYS" in
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
case "$BASELINE_LOOKBACK_DAYS" in
|
||||
''|*[!0-9]*|0)
|
||||
echo "--baseline-lookback-days must be a positive integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
remote_cmd=""
|
||||
printf -v remote_cmd \
|
||||
"REMOTE_ROOT=%q DAYS_BACK=%q WINDOW_DAYS=%q PROTOCOLS=%q ON_CALENDAR=%q DRY_RUN=%q bash -s" \
|
||||
"$REMOTE_ROOT" "$DAYS_BACK" "$WINDOW_DAYS" "$PROTOCOLS" "$ON_CALENDAR" "$DRY_RUN"
|
||||
"REMOTE_ROOT=%q DAYS_BACK=%q WINDOW_DAYS=%q BASELINE_LOOKBACK_DAYS=%q PROTOCOLS=%q ON_CALENDAR=%q DRY_RUN=%q bash -s" \
|
||||
"$REMOTE_ROOT" "$DAYS_BACK" "$WINDOW_DAYS" "$BASELINE_LOOKBACK_DAYS" "$PROTOCOLS" "$ON_CALENDAR" "$DRY_RUN"
|
||||
ssh "$HOST" "$remote_cmd" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
|
||||
@@ -110,6 +124,7 @@ Environment=BACKFILL_METHOD=last_diff
|
||||
Environment=BACKFILL_DRY_RUN=${DRY_RUN}
|
||||
Environment=BACKFILL_DAYS_BACK=${DAYS_BACK}
|
||||
Environment=BACKFILL_WINDOW_DAYS=${WINDOW_DAYS}
|
||||
Environment=BACKFILL_BASELINE_LOOKBACK_DAYS=${BASELINE_LOOKBACK_DAYS}
|
||||
Environment=BACKFILL_PROTOCOLS=${PROTOCOLS}
|
||||
ExecStart=${REMOTE_ROOT}/current/stats-backfill
|
||||
TimeoutStartSec=45min
|
||||
|
||||
Reference in New Issue
Block a user