shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

331
go/vehicle-gateway/internal/realtime/repository.go
View File

@@ -20,6 +20,32 @@ type Repository struct {
cfg Config
}
type FastUpdateResult struct {
EnvelopesSeen int
EnvelopesUpdated int
EnvelopesSkippedNonRealtime int
EnvelopesSkippedMissingVIN int
EnvelopesSkippedMissingVehicleKey int
EnvelopesSkippedMissingFields int
FieldsSeen int
FieldsWritten int
FieldsSkippedStale int
}
func (r FastUpdateResult) Add(other FastUpdateResult) FastUpdateResult {
return FastUpdateResult{
EnvelopesSeen: r.EnvelopesSeen + other.EnvelopesSeen,
EnvelopesUpdated: r.EnvelopesUpdated + other.EnvelopesUpdated,
EnvelopesSkippedNonRealtime: r.EnvelopesSkippedNonRealtime + other.EnvelopesSkippedNonRealtime,
EnvelopesSkippedMissingVIN: r.EnvelopesSkippedMissingVIN + other.EnvelopesSkippedMissingVIN,
EnvelopesSkippedMissingVehicleKey: r.EnvelopesSkippedMissingVehicleKey + other.EnvelopesSkippedMissingVehicleKey,
EnvelopesSkippedMissingFields: r.EnvelopesSkippedMissingFields + other.EnvelopesSkippedMissingFields,
FieldsSeen: r.FieldsSeen + other.FieldsSeen,
FieldsWritten: r.FieldsWritten + other.FieldsWritten,
FieldsSkippedStale: r.FieldsSkippedStale + other.FieldsSkippedStale,
}
}
func NewRepository(client *redis.Client, cfg Config) *Repository {
if client == nil {
panic("redis client must not be nil")
@@ -28,48 +54,82 @@ func NewRepository(client *redis.Client, cfg Config) *Repository {
}
func (r *Repository) FastUpdate(ctx context.Context, env envelope.FrameEnvelope) error {
_, err := r.FastUpdateWithResult(ctx, env)
return err
}
func (r *Repository) FastUpdateWithResult(ctx context.Context, env envelope.FrameEnvelope) (FastUpdateResult, error) {
result := FastUpdateResult{EnvelopesSeen: 1}
if !envelope.IsRealtimeTelemetryFrame(env) {
return nil
result.EnvelopesSkippedNonRealtime = 1
return result, nil
}
vin := strings.TrimSpace(env.VIN)
if vin == "" {
return nil
result.EnvelopesSkippedMissingVIN = 1
return result, nil
}
vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
return nil
result.EnvelopesSkippedMissingVehicleKey = 1
return result, nil
}
if len(env.ParsedFields) == 0 {
result.EnvelopesSkippedMissingFields = 1
return result, nil
}
return r.setFastProjection(ctx, vehicleKey, vin, env)
}
func (r *Repository) FastUpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error {
_, err := r.FastUpdateBatchWithResult(ctx, envs)
return err
}
func (r *Repository) FastUpdateBatchWithResult(ctx context.Context, envs []envelope.FrameEnvelope) (FastUpdateResult, error) {
if len(envs) == 0 {
return nil
return FastUpdateResult{}, nil
}
pipe := r.client.Pipeline()
queued := 0
var result FastUpdateResult
queued := make([]queuedFastProjection, 0, len(envs))
for _, env := range envs {
result.EnvelopesSeen++
if !envelope.IsRealtimeTelemetryFrame(env) {
result.EnvelopesSkippedNonRealtime++
continue
}
vin := strings.TrimSpace(env.VIN)
if vin == "" {
result.EnvelopesSkippedMissingVIN++
continue
}
vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
result.EnvelopesSkippedMissingVehicleKey++
continue
}
if err := r.queueFastProjection(ctx, pipe, vehicleKey, vin, env); err != nil {
return err
if len(env.ParsedFields) == 0 {
result.EnvelopesSkippedMissingFields++
continue
}
queued++
queuedProjection, err := r.queueFastProjection(ctx, pipe, vehicleKey, vin, env)
if err != nil {
return FastUpdateResult{}, err
}
queued = append(queued, queuedProjection)
}
if queued == 0 {
return nil
if len(queued) == 0 {
return result, nil
}
_, err := pipe.Exec(ctx)
return err
if err != nil {
return FastUpdateResult{}, err
}
for _, item := range queued {
result = result.Add(item.result())
}
return result, nil
}
func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -85,10 +145,7 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err
return nil
}
nowMS := time.Now().UnixMilli()
eventMS := env.EventTimeMS
if eventMS <= 0 {
eventMS = env.ReceivedAtMS
}
eventMS, _ := envelope.NormalizedEventTimeMS(env)
protocolSnapshot := Snapshot{
VehicleKey: vehicleKey,
VIN: vin,
@@ -129,7 +186,7 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err
if err := r.setJSON(ctx, realtimeRawKey(vehicleKey, env.Protocol), protocolSnapshot.Parsed, r.cfg.ttl()); err != nil {
return err
}
if err := r.setKV(ctx, vin, env, protocolSnapshot.Parsed); err != nil {
if err := r.setKV(ctx, vin, env); err != nil {
return err
}
@@ -377,10 +434,7 @@ func (r *Repository) setJSON(ctx context.Context, key string, value any, ttl tim
return r.client.Set(ctx, key, payload, ttl).Err()
}
func (r *Repository) setKV(ctx context.Context, vin string, env envelope.FrameEnvelope, parsed map[string]any) error {
if len(env.ParsedFields) == 0 && len(parsed) > 0 {
env.Parsed = parsed
}
func (r *Repository) setKV(ctx context.Context, vin string, env envelope.FrameEnvelope) error {
values, types := realtimeKVMapsForEnvelope(env)
if len(values) == 0 {
return nil
@@ -394,24 +448,43 @@ func (r *Repository) setKV(ctx context.Context, vin string, env envelope.FrameEn
"source_endpoint": env.SourceEndpoint,
"field_mapping": realtimeFieldMappingVersion,
}
pipe := r.client.Pipeline()
pipe.HSet(ctx, realtimeKVValuesKey(env.Protocol, vin), values)
pipe.HSet(ctx, realtimeKVTypesKey(env.Protocol, vin), types)
pipe.HSet(ctx, realtimeKVMetaKey(env.Protocol, vin), meta)
_, err := pipe.Exec(ctx)
return err
return evalGuardedRealtimeKV(ctx, r.client, env.Protocol, vin, eventTimeOrReceivedMS(env), values, types, meta).Err()
}
func (r *Repository) setFastProjection(ctx context.Context, vehicleKey string, vin string, env envelope.FrameEnvelope) error {
func (r *Repository) setFastProjection(ctx context.Context, vehicleKey string, vin string, env envelope.FrameEnvelope) (FastUpdateResult, error) {
pipe := r.client.Pipeline()
if err := r.queueFastProjection(ctx, pipe, vehicleKey, vin, env); err != nil {
return err
queued, err := r.queueFastProjection(ctx, pipe, vehicleKey, vin, env)
if err != nil {
return FastUpdateResult{}, err
}
_, err := pipe.Exec(ctx)
return err
if _, err := pipe.Exec(ctx); err != nil {
return FastUpdateResult{}, err
}
result := queued.result()
result.EnvelopesSeen = 1
return result, nil
}
func (r *Repository) queueFastProjection(ctx context.Context, pipe redis.Pipeliner, vehicleKey string, vin string, env envelope.FrameEnvelope) error {
type queuedFastProjection struct {
fieldsSeen int
writeCmd *redis.Cmd
}
func (q queuedFastProjection) result() FastUpdateResult {
written := redisCmdInt(q.writeCmd)
skipped := q.fieldsSeen - written
if skipped < 0 {
skipped = 0
}
return FastUpdateResult{
EnvelopesUpdated: 1,
FieldsSeen: q.fieldsSeen,
FieldsWritten: written,
FieldsSkippedStale: skipped,
}
}
func (r *Repository) queueFastProjection(ctx context.Context, pipe redis.Pipeliner, vehicleKey string, vin string, env envelope.FrameEnvelope) (queuedFastProjection, error) {
values, types := realtimeKVMapsForEnvelope(env)
eventTimeMS := eventTimeOrReceivedMS(env)
offlineAfterMS := env.ReceivedAtMS + r.cfg.ttl().Milliseconds()
@@ -428,7 +501,7 @@ func (r *Repository) queueFastProjection(ctx context.Context, pipe redis.Pipelin
}
payload, err := json.Marshal(online)
if err != nil {
return err
return queuedFastProjection{}, err
}
meta := map[string]any{
"event_time_ms": strconv.FormatInt(eventTimeMS, 10),
@@ -449,16 +522,171 @@ func (r *Repository) queueFastProjection(ctx context.Context, pipe redis.Pipelin
"ttl_seconds": strconv.FormatInt(int64(r.cfg.ttl().Seconds()), 10),
"source_endpoint": env.SourceEndpoint,
}
queued := queuedFastProjection{fieldsSeen: len(values)}
if len(values) > 0 {
pipe.HSet(ctx, realtimeKVValuesKey(env.Protocol, vin), values)
pipe.HSet(ctx, realtimeKVTypesKey(env.Protocol, vin), types)
pipe.HSet(ctx, realtimeKVMetaKey(env.Protocol, vin), meta)
queued.writeCmd = evalGuardedRealtimeKV(ctx, pipe, env.Protocol, vin, eventTimeMS, values, types, meta)
}
pipe.Set(ctx, onlineKey(env.Protocol, vin), payload, r.cfg.ttl())
pipe.HSet(ctx, onlineStateKey(env.Protocol, vin), state)
pipe.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: onlineMember(env.Protocol, vin)})
pipe.SAdd(ctx, realtimeIndexKey(env.Protocol), vin)
return nil
if _, err := evalGuardedOnlineStatus(ctx, pipe, online, state, payload, r.cfg.ttl()); err != nil {
return queuedFastProjection{}, err
}
return queued, nil
}
const guardedRealtimeKVScript = `
local incoming = tonumber(ARGV[1]) or 0
local value_count = tonumber(ARGV[2]) or 0
local idx = 3
local written = 0
for i = 1, value_count do
local field = ARGV[idx]
local value = ARGV[idx + 1]
idx = idx + 2
local current = tonumber(redis.call('HGET', KEYS[3], field) or '') or 0
if current <= incoming then
redis.call('HSET', KEYS[1], field, value)
redis.call('HSET', KEYS[3], field, incoming)
written = written + 1
end
end
local type_count = tonumber(ARGV[idx]) or 0
idx = idx + 1
for i = 1, type_count do
local field = ARGV[idx]
local value_type = ARGV[idx + 1]
idx = idx + 2
local current = tonumber(redis.call('HGET', KEYS[3], field) or '') or 0
if current <= incoming then
redis.call('HSET', KEYS[2], field, value_type)
end
end
local meta_count = tonumber(ARGV[idx]) or 0
idx = idx + 1
local current_meta = tonumber(redis.call('HGET', KEYS[4], 'event_time_ms') or '') or 0
if current_meta <= incoming then
for i = 1, meta_count do
redis.call('HSET', KEYS[4], ARGV[idx], ARGV[idx + 1])
idx = idx + 2
end
end
return written
`
const guardedOnlineStatusScript = `
local incoming = tonumber(ARGV[1]) or 0
local ttl_ms = tonumber(ARGV[2]) or 60000
local payload = ARGV[3]
local member = ARGV[4]
local vin = ARGV[5]
local state_count = tonumber(ARGV[6]) or 0
local idx = 7
local current = tonumber(redis.call('HGET', KEYS[2], 'last_seen_ms') or '') or 0
redis.call('SADD', KEYS[4], vin)
if current <= incoming then
redis.call('SET', KEYS[1], payload, 'PX', ttl_ms)
for i = 1, state_count do
redis.call('HSET', KEYS[2], ARGV[idx], ARGV[idx + 1])
idx = idx + 2
end
redis.call('ZADD', KEYS[3], incoming, member)
return 1
end
return 0
`
type redisEvaler interface {
Eval(ctx context.Context, script string, keys []string, args ...any) *redis.Cmd
}
func redisCmdInt(cmd *redis.Cmd) int {
if cmd == nil {
return 0
}
value, err := cmd.Int64()
if err != nil {
return 0
}
return int(value)
}
func evalGuardedRealtimeKV(ctx context.Context, evaler redisEvaler, protocol envelope.Protocol, vin string, eventTimeMS int64, values map[string]any, types map[string]any, meta map[string]any) *redis.Cmd {
keys := []string{
realtimeKVValuesKey(protocol, vin),
realtimeKVTypesKey(protocol, vin),
realtimeKVTimesKey(protocol, vin),
realtimeKVMetaKey(protocol, vin),
}
args := guardedRealtimeKVArgs(eventTimeMS, values, types, meta)
return evaler.Eval(ctx, guardedRealtimeKVScript, keys, args...)
}
func evalGuardedOnlineStatus(ctx context.Context, evaler redisEvaler, online OnlineStatus, state map[string]any, payload []byte, ttl time.Duration) (*redis.Cmd, error) {
protocol := online.Protocol
if protocol == "" && len(online.Protocols) > 0 {
protocol = online.Protocols[0]
}
vin := strings.TrimSpace(online.VIN)
if protocol == "" || vin == "" {
return nil, nil
}
if len(payload) == 0 {
var err error
payload, err = json.Marshal(online)
if err != nil {
return nil, err
}
}
if ttl <= 0 {
ttl = time.Minute
}
keys := []string{
onlineKey(protocol, vin),
onlineStateKey(protocol, vin),
"vehicle:last_seen",
realtimeIndexKey(protocol),
}
args := guardedOnlineStatusArgs(online, protocol, state, payload, ttl)
return evaler.Eval(ctx, guardedOnlineStatusScript, keys, args...), nil
}
func guardedOnlineStatusArgs(online OnlineStatus, protocol envelope.Protocol, state map[string]any, payload []byte, ttl time.Duration) []any {
args := []any{
strconv.FormatInt(online.LastSeenMS, 10),
strconv.FormatInt(ttl.Milliseconds(), 10),
string(payload),
onlineMember(protocol, online.VIN),
strings.TrimSpace(online.VIN),
}
return appendMapPairs(args, state)
}
func guardedRealtimeKVArgs(eventTimeMS int64, values map[string]any, types map[string]any, meta map[string]any) []any {
args := []any{strconv.FormatInt(eventTimeMS, 10)}
args = appendMapPairs(args, values)
args = appendMapPairs(args, types)
args = appendMapPairs(args, meta)
return args
}
func appendMapPairs(args []any, values map[string]any) []any {
keys := make([]string, 0, len(values))
for key := range values {
key = strings.TrimSpace(key)
if key == "" {
continue
}
keys = append(keys, key)
}
sort.Strings(keys)
args = append(args, strconv.Itoa(len(keys)))
for _, key := range keys {
args = append(args, key, fmt.Sprint(values[key]))
}
return args
}
func realtimeKVMapsForEnvelope(env envelope.FrameEnvelope) (map[string]any, map[string]any) {
@@ -469,6 +697,9 @@ func realtimeKVMapsForEnvelope(env envelope.FrameEnvelope) (map[string]any, map[
if strings.TrimSpace(field) == "" {
continue
}
if isRealtimeTotalMileageField(field) && !positiveNumber(value) {
continue
}
stringValue, valueType, ok := stringifyKVValue(value)
if !ok {
continue
@@ -528,7 +759,6 @@ func (r *Repository) setOnlineStatus(ctx context.Context, online OnlineStatus) e
if online.OfflineAfterMS <= 0 {
online.OfflineAfterMS = online.LastSeenMS + r.cfg.ttl().Milliseconds()
}
member := onlineMember(protocol, online.VIN)
state := map[string]any{
"vehicle_key": online.VehicleKey,
"vin": online.VIN,
@@ -544,10 +774,9 @@ func (r *Repository) setOnlineStatus(ctx context.Context, online OnlineStatus) e
if err != nil {
return err
}
pipe.Set(ctx, onlineKey(protocol, online.VIN), payload, r.cfg.ttl())
pipe.HSet(ctx, onlineStateKey(protocol, online.VIN), state)
pipe.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(online.LastSeenMS), Member: member})
pipe.SAdd(ctx, realtimeIndexKey(protocol), online.VIN)
if _, err := evalGuardedOnlineStatus(ctx, pipe, online, state, payload, r.cfg.ttl()); err != nil {
return err
}
_, err = pipe.Exec(ctx)
return err
}
@@ -879,6 +1108,10 @@ func realtimeKVTypesKey(protocol envelope.Protocol, vin string) string {
return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":types"
}
func realtimeKVTimesKey(protocol envelope.Protocol, vin string) string {
return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":times"
}
func realtimeKVMetaKey(protocol envelope.Protocol, vin string) string {
return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":meta"
}
@@ -897,10 +1130,8 @@ func realtimeKVFieldPath(domain string, field string) string {
}
func eventTimeOrReceivedMS(env envelope.FrameEnvelope) int64 {
if env.EventTimeMS > 0 {
return env.EventTimeMS
}
return env.ReceivedAtMS
eventMS, _ := envelope.NormalizedEventTimeMS(env)
return eventMS
}
func onlineKey(protocol envelope.Protocol, vin string) string {