feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
271
vehicle-data-platform/apps/api/internal/platform/alert_stream.go
Normal file
271
vehicle-data-platform/apps/api/internal/platform/alert_stream.go
Normal file
@@ -0,0 +1,271 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const alertStreamMaxFutureSkew = 10 * time.Minute
|
||||
|
||||
type AlertStreamRecord struct {
|
||||
Topic string
|
||||
Partition int
|
||||
Offset int64
|
||||
HighWatermark int64
|
||||
Protocol string
|
||||
VIN string
|
||||
Plate string
|
||||
SourceEventID string
|
||||
EventAt time.Time
|
||||
ReceivedAt time.Time
|
||||
Fields map[string]json.RawMessage
|
||||
Valid bool
|
||||
Late bool
|
||||
ErrorCode string
|
||||
}
|
||||
|
||||
type AlertStreamBatchResult struct {
|
||||
Fetched int
|
||||
Processed int
|
||||
Valid int
|
||||
Invalid int
|
||||
Late int
|
||||
ReplaySkipped int
|
||||
Partitions int
|
||||
RulesEvaluated int
|
||||
CandidatesAdvanced int
|
||||
DuplicateObservations int
|
||||
LateObservations int
|
||||
Opened int
|
||||
Recovered int
|
||||
}
|
||||
|
||||
type alertStreamEnvelope struct {
|
||||
EventID string `json:"event_id"`
|
||||
EventKind string `json:"event_kind"`
|
||||
SourceEventID string `json:"source_event_id"`
|
||||
FieldMapping string `json:"field_mapping"`
|
||||
Protocol string `json:"protocol"`
|
||||
VIN string `json:"vin"`
|
||||
Plate string `json:"plate"`
|
||||
EventTimeMS int64 `json:"event_time_ms"`
|
||||
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||
Fields map[string]json.RawMessage `json:"fields"`
|
||||
}
|
||||
|
||||
func DecodeAlertStreamRecord(topic string, partition int, offset, highWatermark int64, payload []byte, lateness time.Duration) AlertStreamRecord {
|
||||
record := AlertStreamRecord{Topic: strings.TrimSpace(topic), Partition: partition, Offset: offset, HighWatermark: highWatermark}
|
||||
var envelope alertStreamEnvelope
|
||||
if err := json.Unmarshal(payload, &envelope); err != nil {
|
||||
record.ErrorCode = "invalid_json"
|
||||
return record
|
||||
}
|
||||
record.Protocol = strings.TrimSpace(envelope.Protocol)
|
||||
record.VIN = strings.TrimSpace(envelope.VIN)
|
||||
record.Plate = strings.TrimSpace(envelope.Plate)
|
||||
record.SourceEventID = firstNonEmpty(strings.TrimSpace(envelope.SourceEventID), strings.TrimSpace(envelope.EventID))
|
||||
record.Fields = envelope.Fields
|
||||
if expected, known := alertStreamTopicProtocol(record.Topic); known && !strings.EqualFold(expected, record.Protocol) {
|
||||
record.ErrorCode = "protocol_topic_mismatch"
|
||||
return record
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(envelope.EventKind), "FIELDS") {
|
||||
record.ErrorCode = "event_kind_mismatch"
|
||||
return record
|
||||
}
|
||||
if strings.TrimSpace(envelope.FieldMapping) == "" {
|
||||
record.ErrorCode = "missing_field_mapping"
|
||||
return record
|
||||
}
|
||||
if record.VIN == "" {
|
||||
record.ErrorCode = "missing_vin_" + strings.ToLower(record.Protocol)
|
||||
return record
|
||||
}
|
||||
if record.SourceEventID == "" {
|
||||
record.ErrorCode = "missing_source_event_id"
|
||||
return record
|
||||
}
|
||||
if len(envelope.Fields) == 0 {
|
||||
record.ErrorCode = "missing_fields"
|
||||
return record
|
||||
}
|
||||
prefix := map[string]string{"GB32960": "gb32960.", "JT808": "jt808.", "YUTONG_MQTT": "yutong_mqtt."}[strings.ToUpper(record.Protocol)]
|
||||
for field := range envelope.Fields {
|
||||
if prefix != "" && (field != strings.TrimSpace(field) || !strings.HasPrefix(field, prefix) || len(field) <= len(prefix)) {
|
||||
record.ErrorCode = "invalid_field_name"
|
||||
return record
|
||||
}
|
||||
}
|
||||
if envelope.ReceivedAtMS <= 0 {
|
||||
record.ErrorCode = "missing_received_time"
|
||||
return record
|
||||
}
|
||||
receivedAt := time.UnixMilli(envelope.ReceivedAtMS)
|
||||
eventMS := envelope.EventTimeMS
|
||||
if eventMS <= 0 || eventMS > envelope.ReceivedAtMS+alertStreamMaxFutureSkew.Milliseconds() {
|
||||
eventMS = envelope.ReceivedAtMS
|
||||
}
|
||||
record.EventAt = time.UnixMilli(eventMS)
|
||||
record.ReceivedAt = receivedAt
|
||||
if lateness > 0 && receivedAt.Sub(record.EventAt) > lateness {
|
||||
record.Late = true
|
||||
}
|
||||
record.Valid = true
|
||||
return record
|
||||
}
|
||||
|
||||
func alertStreamTopicProtocol(topic string) (string, bool) {
|
||||
switch strings.TrimSpace(topic) {
|
||||
case "vehicle.fields.go.gb32960.v1":
|
||||
return "GB32960", true
|
||||
case "vehicle.fields.go.jt808.v1":
|
||||
return "JT808", true
|
||||
case "vehicle.fields.go.yutong-mqtt.v1":
|
||||
return "YUTONG_MQTT", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
type alertStreamPartition struct {
|
||||
topic string
|
||||
partition int
|
||||
}
|
||||
|
||||
type alertStreamCheckpointUpdate struct {
|
||||
key alertStreamPartition
|
||||
nextOffset, highWatermark int64
|
||||
processed, valid, invalid, late, replay int64
|
||||
lastEventAt, lastReceivedAt any
|
||||
lastSourceEventID string
|
||||
lastInvalidCode string
|
||||
}
|
||||
|
||||
func (s *ProductionStore) RecordAlertStreamBatch(ctx context.Context, consumerGroup string, records []AlertStreamRecord) (AlertStreamBatchResult, error) {
|
||||
result := AlertStreamBatchResult{Fetched: len(records)}
|
||||
consumerGroup = strings.TrimSpace(consumerGroup)
|
||||
if consumerGroup == "" || len(consumerGroup) > 128 {
|
||||
return result, fmt.Errorf("alert stream consumer group is empty or too long")
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
grouped := map[alertStreamPartition][]AlertStreamRecord{}
|
||||
for _, record := range records {
|
||||
if record.Topic == "" || len(record.Topic) > 255 || record.Partition < 0 || record.Offset < 0 {
|
||||
return result, fmt.Errorf("invalid alert stream message position topic=%q partition=%d offset=%d", record.Topic, record.Partition, record.Offset)
|
||||
}
|
||||
key := alertStreamPartition{topic: record.Topic, partition: record.Partition}
|
||||
grouped[key] = append(grouped[key], record)
|
||||
}
|
||||
keys := make([]alertStreamPartition, 0, len(grouped))
|
||||
for key := range grouped {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].topic == keys[j].topic {
|
||||
return keys[i].partition < keys[j].partition
|
||||
}
|
||||
return keys[i].topic < keys[j].topic
|
||||
})
|
||||
tx, err := s.db.BeginTx(ctx, &sql.TxOptions{})
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
updates := make([]alertStreamCheckpointUpdate, 0, len(keys))
|
||||
accepted := make([]AlertStreamRecord, 0, len(records))
|
||||
for _, key := range keys {
|
||||
partitionRecords := grouped[key]
|
||||
sort.Slice(partitionRecords, func(i, j int) bool { return partitionRecords[i].Offset < partitionRecords[j].Offset })
|
||||
var nextOffset int64
|
||||
hasCheckpoint := true
|
||||
if err = tx.QueryRowContext(ctx, `SELECT next_offset FROM vehicle_alert_stream_checkpoint WHERE consumer_group=? AND topic=? AND partition_id=? FOR UPDATE`, consumerGroup, key.topic, key.partition).Scan(&nextOffset); err == sql.ErrNoRows {
|
||||
hasCheckpoint = false
|
||||
nextOffset = -1
|
||||
} else if err != nil {
|
||||
return result, err
|
||||
}
|
||||
update := alertStreamCheckpointUpdate{key: key}
|
||||
maxOffset := nextOffset - 1
|
||||
var lastEventTime time.Time
|
||||
var lastPosition int64
|
||||
hasPosition := false
|
||||
for _, record := range partitionRecords {
|
||||
if record.HighWatermark > update.highWatermark {
|
||||
update.highWatermark = record.HighWatermark
|
||||
}
|
||||
if hasCheckpoint && record.Offset < nextOffset {
|
||||
update.replay++
|
||||
continue
|
||||
}
|
||||
if hasPosition && record.Offset == lastPosition {
|
||||
update.replay++
|
||||
continue
|
||||
}
|
||||
lastPosition = record.Offset
|
||||
hasPosition = true
|
||||
update.processed++
|
||||
if record.Valid {
|
||||
update.valid++
|
||||
if record.Late {
|
||||
update.late++
|
||||
}
|
||||
if lastEventTime.IsZero() || record.EventAt.After(lastEventTime) {
|
||||
lastEventTime = record.EventAt
|
||||
update.lastEventAt = record.EventAt
|
||||
update.lastReceivedAt = record.ReceivedAt
|
||||
update.lastSourceEventID = record.SourceEventID
|
||||
}
|
||||
accepted = append(accepted, record)
|
||||
} else {
|
||||
update.invalid++
|
||||
update.lastInvalidCode = record.ErrorCode
|
||||
}
|
||||
if record.Offset > maxOffset {
|
||||
maxOffset = record.Offset
|
||||
}
|
||||
}
|
||||
update.nextOffset = nextOffset
|
||||
if maxOffset >= 0 && maxOffset+1 > update.nextOffset {
|
||||
update.nextOffset = maxOffset + 1
|
||||
}
|
||||
if update.nextOffset < 0 {
|
||||
update.nextOffset = 0
|
||||
}
|
||||
updates = append(updates, update)
|
||||
result.Processed += int(update.processed)
|
||||
result.Valid += int(update.valid)
|
||||
result.Invalid += int(update.invalid)
|
||||
result.Late += int(update.late)
|
||||
result.ReplaySkipped += int(update.replay)
|
||||
}
|
||||
if strings.EqualFold(s.alertStreamMode, "active") && len(accepted) > 0 {
|
||||
evaluation, evaluationErr := evaluateAlertStreamRecordsTx(ctx, tx, accepted, time.Now())
|
||||
if evaluationErr != nil {
|
||||
return result, evaluationErr
|
||||
}
|
||||
result.RulesEvaluated = evaluation.RulesEvaluated
|
||||
result.CandidatesAdvanced = evaluation.CandidatesAdvanced
|
||||
result.DuplicateObservations = evaluation.DuplicateObservations
|
||||
result.LateObservations = evaluation.LateObservations
|
||||
result.Opened = evaluation.Opened
|
||||
result.Recovered = evaluation.Recovered
|
||||
}
|
||||
for _, update := range updates {
|
||||
_, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_stream_checkpoint(consumer_group,topic,partition_id,next_offset,high_watermark,processed_count,valid_count,invalid_count,late_count,replay_skipped_count,last_event_at,last_received_at,last_source_event_id,last_invalid_code,last_invalid_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,IF(?='',NULL,CURRENT_TIMESTAMP(3))) ON DUPLICATE KEY UPDATE next_offset=GREATEST(next_offset,VALUES(next_offset)),high_watermark=GREATEST(high_watermark,VALUES(high_watermark)),processed_count=processed_count+VALUES(processed_count),valid_count=valid_count+VALUES(valid_count),invalid_count=invalid_count+VALUES(invalid_count),late_count=late_count+VALUES(late_count),replay_skipped_count=replay_skipped_count+VALUES(replay_skipped_count),last_event_at=COALESCE(VALUES(last_event_at),last_event_at),last_received_at=COALESCE(VALUES(last_received_at),last_received_at),last_source_event_id=IF(VALUES(last_source_event_id)='',last_source_event_id,VALUES(last_source_event_id)),last_invalid_code=IF(VALUES(last_invalid_code)='',last_invalid_code,VALUES(last_invalid_code)),last_invalid_at=COALESCE(VALUES(last_invalid_at),last_invalid_at)`, consumerGroup, update.key.topic, update.key.partition, update.nextOffset, update.highWatermark, update.processed, update.valid, update.invalid, update.late, update.replay, update.lastEventAt, update.lastReceivedAt, update.lastSourceEventID, update.lastInvalidCode, update.lastInvalidCode)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
result.Partitions = len(keys)
|
||||
if err = tx.Commit(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
Reference in New Issue
Block a user