shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(go): simplify redis realtime projections

Browse Source
This commit is contained in:
lingniu
2026-07-03 13:42:15 +08:00
parent a96490618c
commit c359930009
5 changed files with 207 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

190
go/vehicle-gateway/internal/realtime/repository.go
View File

@@ -39,7 +39,7 @@ func (r *Repository) FastUpdate(ctx context.Context, env envelope.FrameEnvelope)
if err := r.setKV(ctx, vin, env, env.Parsed); err != nil {
return err
}
return r.setOnline(ctx, vehicleKey, vin, []envelope.Protocol{env.Protocol}, env.ReceivedAtMS)
return r.setOnline(ctx, vehicleKey, vin, env.Protocol, env.ReceivedAtMS, env.SourceEndpoint)
}
func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -134,6 +134,7 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err
online := OnlineStatus{
VehicleKey: vehicleKey,
VIN: vin,
Protocol: env.Protocol,
Online: true,
LastSeenMS: env.ReceivedAtMS,
Protocols: protocols,
@@ -142,7 +143,7 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err
if err := r.setOnlineStatus(ctx, online); err != nil {
return err
}
return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: vehicleKey}).Err()
return nil
}
func (r *Repository) GetMerged(ctx context.Context, vehicleKey string) (Snapshot, error) {
@@ -164,21 +165,31 @@ func (r *Repository) GetRealtimeRaw(ctx context.Context, vehicleKey string, prot
func (r *Repository) IsOnline(ctx context.Context, vehicleKey string) (OnlineStatus, error) {
vehicleKey = strings.TrimSpace(vehicleKey)
var status OnlineStatus
payload, err := r.client.Get(ctx, onlineKey(vehicleKey)).Bytes()
if err != nil {
if errors.Is(err, redis.Nil) {
return OnlineStatus{VehicleKey: vehicleKey, VIN: vehicleKey, Online: false}, nil
best := OnlineStatus{VehicleKey: vehicleKey, VIN: vehicleKey, Online: false}
for _, protocol := range knownRealtimeProtocols() {
key := onlineKey(protocol, vehicleKey)
payload, err := r.client.Get(ctx, key).Bytes()
if err != nil {
if errors.Is(err, redis.Nil) {
continue
}
return OnlineStatus{}, err
}
var status OnlineStatus
if err := json.Unmarshal(payload, &status); err != nil {
return OnlineStatus{}, err
}
ttl := r.client.TTL(ctx, key).Val()
status.Online = ttl > 0
status.TTLSeconds = int64(ttl.Seconds())
if status.Protocol == "" {
status.Protocol = protocol
}
if status.LastSeenMS >= best.LastSeenMS {
best = status
}
return status, err
}
if err := json.Unmarshal(payload, &status); err != nil {
return status, err
}
ttl := r.client.TTL(ctx, onlineKey(vehicleKey)).Val()
status.Online = ttl > 0
status.TTLSeconds = int64(ttl.Seconds())
return status, nil
return best, nil
}
func (r *Repository) getSnapshot(ctx context.Context, key string) (Snapshot, error) {
@@ -203,49 +214,86 @@ func (r *Repository) setKV(ctx context.Context, vin string, env envelope.FrameEn
if len(rows) == 0 {
return nil
}
grouped := make(map[string]map[string]any)
values := make(map[string]any, len(rows))
types := make(map[string]any, len(rows))
for _, row := range rows {
key := realtimeKVKey(row.Protocol, vin, row.Domain)
values := grouped[key]
if values == nil {
values = map[string]any{
"_event_time_ms": strconv.FormatInt(row.EventTimeMS, 10),
"_received_at_ms": strconv.FormatInt(row.ReceivedAtMS, 10),
"_event_id": row.EventID,
"_protocol": string(row.Protocol),
"_vin": vin,
"_domain": row.Domain,
}
grouped[key] = values
field := realtimeKVFieldPath(row.Domain, row.Field)
if field == "" {
continue
}
values[row.Field] = row.Value
values["_value_type:"+row.Field] = row.ValueType
values[field] = row.Value
types[field] = row.ValueType
}
if len(values) == 0 {
return nil
}
meta := map[string]any{
"event_time_ms": strconv.FormatInt(eventTimeOrReceivedMS(env), 10),
"received_at_ms": strconv.FormatInt(env.ReceivedAtMS, 10),
"event_id": env.StableEventID(),
"protocol": string(env.Protocol),
"vin": vin,
"source_endpoint": env.SourceEndpoint,
}
pipe := r.client.Pipeline()
for key, values := range grouped {
pipe.HSet(ctx, key, values)
}
pipe.HSet(ctx, realtimeKVValuesKey(env.Protocol, vin), values)
pipe.HSet(ctx, realtimeKVTypesKey(env.Protocol, vin), types)
pipe.HSet(ctx, realtimeKVMetaKey(env.Protocol, vin), meta)
_, err := pipe.Exec(ctx)
return err
}
func (r *Repository) setOnline(ctx context.Context, vehicleKey string, vin string, protocols []envelope.Protocol, lastSeenMS int64) error {
func (r *Repository) setOnline(ctx context.Context, vehicleKey string, vin string, protocol envelope.Protocol, lastSeenMS int64, sourceEndpoint string) error {
if protocol == "" {
return nil
}
online := OnlineStatus{
VehicleKey: vehicleKey,
VIN: vin,
Online: true,
LastSeenMS: lastSeenMS,
Protocols: protocols,
TTLSeconds: int64(r.cfg.ttl().Seconds()),
VehicleKey: vehicleKey,
VIN: vin,
Protocol: protocol,
Online: true,
LastSeenMS: lastSeenMS,
OfflineAfterMS: lastSeenMS + r.cfg.ttl().Milliseconds(),
SourceEndpoint: sourceEndpoint,
Protocols: []envelope.Protocol{protocol},
TTLSeconds: int64(r.cfg.ttl().Seconds()),
}
if err := r.setOnlineStatus(ctx, online); err != nil {
return err
}
return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(lastSeenMS), Member: vehicleKey}).Err()
return r.setOnlineStatus(ctx, online)
}
func (r *Repository) setOnlineStatus(ctx context.Context, online OnlineStatus) error {
return r.setJSON(ctx, onlineKey(online.VehicleKey), online, r.cfg.ttl())
protocol := online.Protocol
if protocol == "" && len(online.Protocols) > 0 {
protocol = online.Protocols[0]
}
if protocol == "" {
return nil
}
if online.OfflineAfterMS <= 0 {
online.OfflineAfterMS = online.LastSeenMS + r.cfg.ttl().Milliseconds()
}
member := onlineMember(protocol, online.VIN)
state := map[string]any{
"vehicle_key": online.VehicleKey,
"vin": online.VIN,
"protocol": string(protocol),
"online": strconv.FormatBool(true),
"last_seen_ms": strconv.FormatInt(online.LastSeenMS, 10),
"offline_after_ms": strconv.FormatInt(online.OfflineAfterMS, 10),
"ttl_seconds": strconv.FormatInt(int64(r.cfg.ttl().Seconds()), 10),
"source_endpoint": online.SourceEndpoint,
}
pipe := r.client.Pipeline()
payload, err := json.Marshal(online)
if err != nil {
return err
}
pipe.Set(ctx, onlineKey(protocol, online.VIN), payload, r.cfg.ttl())
pipe.HSet(ctx, onlineStateKey(protocol, online.VIN), state)
pipe.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(online.LastSeenMS), Member: member})
pipe.SAdd(ctx, realtimeIndexKey(protocol), online.VIN)
_, err = pipe.Exec(ctx)
return err
}
func (r *Repository) addProtocol(ctx context.Context, vehicleKey string, protocol envelope.Protocol) ([]envelope.Protocol, error) {
@@ -567,12 +615,56 @@ func realtimeRawKey(vehicleKey string, protocol envelope.Protocol) string {
return "vehicle:realtime-raw:" + string(protocol) + ":" + strings.TrimSpace(vehicleKey)
}
func realtimeKVKey(protocol envelope.Protocol, vin string, domain string) string {
return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":" + strings.TrimSpace(domain)
func realtimeKVValuesKey(protocol envelope.Protocol, vin string) string {
return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":values"
}
func onlineKey(vehicleKey string) string {
return "vehicle:online:" + strings.TrimSpace(vehicleKey)
func realtimeKVTypesKey(protocol envelope.Protocol, vin string) string {
return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":types"
}
func realtimeKVMetaKey(protocol envelope.Protocol, vin string) string {
return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":meta"
}
func realtimeKVFieldPath(domain string, field string) string {
domain = strings.TrimSpace(domain)
field = strings.TrimSpace(field)
switch {
case domain == "":
return field
case field == "":
return domain
default:
return domain + "." + field
}
}
func eventTimeOrReceivedMS(env envelope.FrameEnvelope) int64 {
if env.EventTimeMS > 0 {
return env.EventTimeMS
}
return env.ReceivedAtMS
}
func onlineKey(protocol envelope.Protocol, vin string) string {
return "vehicle:online:" + string(protocol) + ":" + strings.TrimSpace(vin)
}
func onlineStateKey(protocol envelope.Protocol, vin string) string {
return "vehicle:online-state:" + string(protocol) + ":" + strings.TrimSpace(vin)
}
func onlineMember(protocol envelope.Protocol, vin string) string {
return string(protocol) + ":" + strings.TrimSpace(vin)
}
func realtimeIndexKey(protocol envelope.Protocol) string {
return "vehicle:rt-index:" + string(protocol)
}
func knownRealtimeProtocols() []envelope.Protocol {
return []envelope.Protocol{envelope.ProtocolGB32960, envelope.ProtocolJT808, envelope.ProtocolYutongMQTT}
}
func protocolsKey(vehicleKey string) string {