feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
@@ -0,0 +1,178 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os/signal"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu/vehicle-data-platform/apps/api/internal/config"
|
||||
"lingniu/vehicle-data-platform/apps/api/internal/platform"
|
||||
)
|
||||
|
||||
const alertStreamOperationTimeout = 30 * time.Second
|
||||
|
||||
type alertStreamStore interface {
|
||||
RecordAlertStreamBatch(context.Context, string, []platform.AlertStreamRecord) (platform.AlertStreamBatchResult, error)
|
||||
}
|
||||
|
||||
type kafkaMessageFetcher interface {
|
||||
FetchMessage(context.Context) (kafka.Message, error)
|
||||
}
|
||||
|
||||
type kafkaMessageCommitter interface {
|
||||
CommitMessages(context.Context, ...kafka.Message) error
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg := config.Load()
|
||||
if err := validateAlertStreamConfig(cfg); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
db, err := platform.OpenSQL(ctx, "mysql", cfg.MySQLDSN)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
store := platform.NewProductionStore(db, nil, "").WithAlertStreamConfig(cfg.AlertStreamMode, cfg.AlertStreamKafkaGroup)
|
||||
reader := kafka.NewReader(alertStreamReaderConfig(cfg))
|
||||
defer reader.Close()
|
||||
log.Printf("alert stream evaluator started mode=%s group=%s topics=%s batch_size=%d batch_wait=%s lateness=%s", cfg.AlertStreamMode, cfg.AlertStreamKafkaGroup, strings.Join(cfg.AlertStreamKafkaTopics, ","), cfg.AlertStreamBatchSize, cfg.AlertStreamBatchWait, cfg.AlertStreamLateness)
|
||||
for {
|
||||
message, err := reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
log.Printf("alert stream evaluator stopped")
|
||||
return
|
||||
}
|
||||
log.Printf("alert stream fetch failed error=%v", err)
|
||||
continue
|
||||
}
|
||||
messages := collectAlertStreamBatch(ctx, reader, message, cfg.AlertStreamBatchSize, cfg.AlertStreamBatchWait)
|
||||
started := time.Now()
|
||||
operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), alertStreamOperationTimeout)
|
||||
records := make([]platform.AlertStreamRecord, 0, len(messages))
|
||||
for _, item := range messages {
|
||||
records = append(records, platform.DecodeAlertStreamRecord(item.Topic, item.Partition, item.Offset, item.HighWaterMark, item.Value, cfg.AlertStreamLateness))
|
||||
}
|
||||
invalidCodes := alertStreamInvalidCodes(records)
|
||||
result, recordErr := store.RecordAlertStreamBatch(operationCtx, cfg.AlertStreamKafkaGroup, records)
|
||||
if recordErr == nil {
|
||||
recordErr = reader.CommitMessages(operationCtx, messages...)
|
||||
}
|
||||
cancel()
|
||||
if recordErr != nil {
|
||||
log.Printf("alert stream batch failed fetched=%d duration=%s error=%v", len(messages), time.Since(started), recordErr)
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
log.Printf("alert stream batch completed mode=%s fetched=%d processed=%d valid=%d invalid=%d invalid_codes=%s late=%d replay_skipped=%d partitions=%d rules=%d candidates_advanced=%d duplicate_observations=%d late_observations=%d opened=%d recovered=%d duration=%s", cfg.AlertStreamMode, result.Fetched, result.Processed, result.Valid, result.Invalid, invalidCodes, result.Late, result.ReplaySkipped, result.Partitions, result.RulesEvaluated, result.CandidatesAdvanced, result.DuplicateObservations, result.LateObservations, result.Opened, result.Recovered, time.Since(started))
|
||||
}
|
||||
}
|
||||
|
||||
func alertStreamInvalidCodes(records []platform.AlertStreamRecord) string {
|
||||
counts := map[string]int{}
|
||||
for _, record := range records {
|
||||
if !record.Valid {
|
||||
counts[record.ErrorCode]++
|
||||
}
|
||||
}
|
||||
if len(counts) == 0 {
|
||||
return "none"
|
||||
}
|
||||
keys := make([]string, 0, len(counts))
|
||||
for key := range counts {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
parts = append(parts, fmt.Sprintf("%s:%d", key, counts[key]))
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func alertStreamReaderConfig(cfg config.Config) kafka.ReaderConfig {
|
||||
return kafka.ReaderConfig{
|
||||
Brokers: cfg.AlertStreamKafkaBrokers,
|
||||
GroupID: cfg.AlertStreamKafkaGroup,
|
||||
GroupTopics: cfg.AlertStreamKafkaTopics,
|
||||
MinBytes: 1,
|
||||
MaxBytes: 10e6,
|
||||
StartOffset: kafka.LastOffset,
|
||||
}
|
||||
}
|
||||
|
||||
func validateAlertStreamConfig(cfg config.Config) error {
|
||||
if cfg.AlertStreamMode != "shadow" && cfg.AlertStreamMode != "active" {
|
||||
return fmt.Errorf("ALERT_STREAM_MODE must be shadow or active")
|
||||
}
|
||||
if cfg.MySQLDSN == "" {
|
||||
return fmt.Errorf("MYSQL_DSN is required")
|
||||
}
|
||||
if len(cfg.AlertStreamKafkaBrokers) == 0 || len(cfg.AlertStreamKafkaTopics) == 0 {
|
||||
return fmt.Errorf("ALERT_STREAM_KAFKA_BROKERS and ALERT_STREAM_KAFKA_TOPICS are required")
|
||||
}
|
||||
if strings.TrimSpace(cfg.AlertStreamKafkaGroup) == "" {
|
||||
return fmt.Errorf("ALERT_STREAM_KAFKA_GROUP is required")
|
||||
}
|
||||
if cfg.AlertStreamBatchSize < 1 || cfg.AlertStreamBatchSize > 1000 {
|
||||
return fmt.Errorf("ALERT_STREAM_BATCH_SIZE must be between 1 and 1000")
|
||||
}
|
||||
if cfg.AlertStreamBatchWait < time.Millisecond || cfg.AlertStreamBatchWait > 5*time.Second {
|
||||
return fmt.Errorf("ALERT_STREAM_BATCH_WAIT_MS must be between 1 and 5000")
|
||||
}
|
||||
if cfg.AlertStreamLateness < 0 || cfg.AlertStreamLateness > 24*time.Hour {
|
||||
return fmt.Errorf("ALERT_STREAM_LATENESS_SEC must be between 0 and 86400")
|
||||
}
|
||||
for _, topic := range cfg.AlertStreamKafkaTopics {
|
||||
if _, known := alertStreamTopicProtocol(topic); !known {
|
||||
return fmt.Errorf("unsupported alert stream topic %q", topic)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func alertStreamTopicProtocol(topic string) (string, bool) {
|
||||
switch strings.TrimSpace(topic) {
|
||||
case "vehicle.fields.go.gb32960.v1":
|
||||
return "GB32960", true
|
||||
case "vehicle.fields.go.jt808.v1":
|
||||
return "JT808", true
|
||||
case "vehicle.fields.go.yutong-mqtt.v1":
|
||||
return "YUTONG_MQTT", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func collectAlertStreamBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message {
|
||||
if maxSize <= 1 {
|
||||
return []kafka.Message{first}
|
||||
}
|
||||
messages := []kafka.Message{first}
|
||||
deadline := time.Now().Add(maxWait)
|
||||
for len(messages) < maxSize {
|
||||
remaining := time.Until(deadline)
|
||||
if remaining <= 0 {
|
||||
break
|
||||
}
|
||||
fetchCtx, cancel := context.WithTimeout(ctx, remaining)
|
||||
message, err := fetcher.FetchMessage(fetchCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
messages = append(messages, message)
|
||||
}
|
||||
return messages
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu/vehicle-data-platform/apps/api/internal/config"
|
||||
"lingniu/vehicle-data-platform/apps/api/internal/platform"
|
||||
)
|
||||
|
||||
func validAlertStreamConfig() config.Config {
|
||||
return config.Config{
|
||||
MySQLDSN: "user:pass@tcp(localhost:3306)/db?parseTime=true",
|
||||
AlertStreamMode: "shadow",
|
||||
AlertStreamKafkaBrokers: []string{"kafka-a:9092"},
|
||||
AlertStreamKafkaTopics: []string{"vehicle.fields.go.jt808.v1"},
|
||||
AlertStreamKafkaGroup: "vehicle-alert-stream-v1",
|
||||
AlertStreamBatchSize: 200,
|
||||
AlertStreamBatchWait: 100 * time.Millisecond,
|
||||
AlertStreamLateness: 2 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertStreamConfigAcceptsReleasedModesAndRefusesUnknownTopics(t *testing.T) {
|
||||
cfg := validAlertStreamConfig()
|
||||
if err := validateAlertStreamConfig(cfg); err != nil {
|
||||
t.Fatalf("valid shadow config rejected: %v", err)
|
||||
}
|
||||
cfg.AlertStreamMode = "active"
|
||||
if err := validateAlertStreamConfig(cfg); err != nil {
|
||||
t.Fatalf("released active mode rejected: %v", err)
|
||||
}
|
||||
cfg.AlertStreamMode = "unsafe"
|
||||
if err := validateAlertStreamConfig(cfg); err == nil {
|
||||
t.Fatal("unknown mode must fail closed")
|
||||
}
|
||||
cfg = validAlertStreamConfig()
|
||||
cfg.AlertStreamKafkaTopics = []string{"vehicle.raw.go.jt808.v1"}
|
||||
if err := validateAlertStreamConfig(cfg); err == nil {
|
||||
t.Fatal("raw topic must not be accepted as a fields alert stream")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAlertStreamGroupStartsAtLatestAndThenUsesCommittedOffsets(t *testing.T) {
|
||||
cfg := validAlertStreamConfig()
|
||||
reader := alertStreamReaderConfig(cfg)
|
||||
if reader.StartOffset != kafka.LastOffset {
|
||||
t.Fatalf("new production group would replay the full retained topic: start=%d", reader.StartOffset)
|
||||
}
|
||||
if reader.GroupID != cfg.AlertStreamKafkaGroup || len(reader.GroupTopics) != 1 || reader.GroupTopics[0] != cfg.AlertStreamKafkaTopics[0] {
|
||||
t.Fatalf("reader group contract drifted: %+v", reader)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertStreamInvalidCodesAreAggregatedWithoutPayloads(t *testing.T) {
|
||||
records := []platform.AlertStreamRecord{{ErrorCode: "missing_vin_jt808"}, {Valid: true}, {ErrorCode: "invalid_field_name"}, {ErrorCode: "missing_vin_jt808"}}
|
||||
if got := alertStreamInvalidCodes(records); got != "invalid_field_name:1,missing_vin_jt808:2" {
|
||||
t.Fatalf("invalid code summary=%q", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user