shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(go): add realtime kv projection

Browse Source
This commit is contained in:
lingniu
2026-07-03 11:34:52 +08:00
parent b948a46e64
commit 6fb6262d0d
10 changed files with 653 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
go/vehicle-gateway/internal/realtime/snapshot_writer.go
View File

@@ -111,6 +111,9 @@ func (w *SnapshotWriter) EnsureSchema(ctx context.Context) error {
if _, err := w.exec.ExecContext(ctx, realtimeLocationTableSQL); err != nil {
return err
}
if _, err := w.exec.ExecContext(ctx, realtimeKVTableSQL); err != nil {
return err
}
return nil
}
@@ -129,7 +132,7 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
eventTime := nullableTime(env.EventTimeMS)
receivedAt := nullableTime(env.ReceivedAtMS)
platformName := platformNameFromEnvelope(env)
parsedJSON, err := w.parsedJSONForEnvelope(ctx, env, vin)
parsed, err := w.parsedForEnvelope(ctx, env, vin)
if err != nil {
return err
}
@@ -139,13 +142,16 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
plate,
platformName,
env.SourceEndpoint,
parsedJSON,
marshalParsedJSON(parsed),
eventTime,
receivedAt,
env.StableEventID(),
); err != nil {
return err
}
if err := w.upsertKV(ctx, env, parsed); err != nil {
return err
}
location, ok := realtimeLocationFromEnvelope(env, vin, plate)
if !ok {
return nil
@@ -170,7 +176,7 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
return err
}
func (w *SnapshotWriter) parsedJSONForEnvelope(ctx context.Context, env envelope.FrameEnvelope, vin string) (any, error) {
func (w *SnapshotWriter) parsedForEnvelope(ctx context.Context, env envelope.FrameEnvelope, vin string) (map[string]any, error) {
if len(env.Parsed) == 0 {
return nil, nil
}
@@ -184,7 +190,32 @@ func (w *SnapshotWriter) parsedJSONForEnvelope(ctx context.Context, env envelope
parsed = mergeParsedForProtocol(env.Protocol, existing, env.Parsed)
}
}
return marshalParsedJSON(parsed), nil
return parsed, nil
}
func (w *SnapshotWriter) upsertKV(ctx context.Context, env envelope.FrameEnvelope, parsed map[string]any) error {
rows := realtimeKVFields(env, parsed)
if len(rows) == 0 {
return nil
}
placeholders := make([]string, 0, len(rows))
args := make([]any, 0, len(rows)*9)
for _, row := range rows {
placeholders = append(placeholders, "(?, ?, ?, ?, ?, ?, ?, ?, ?)")
args = append(args,
string(row.Protocol),
row.VIN,
row.Domain,
row.Field,
row.Value,
row.ValueType,
nullableTime(row.EventTimeMS),
nullableTime(row.ReceivedAtMS),
row.EventID,
)
}
_, err := w.exec.ExecContext(ctx, upsertRealtimeKVSQLPrefix+strings.Join(placeholders, ",")+upsertRealtimeKVSQLSuffix, args...)
return err
}
func platformNameFromEnvelope(env envelope.FrameEnvelope) string {
@@ -499,6 +530,37 @@ const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_lo
KEY idx_protocol_updated (protocol, updated_at)
)`
const realtimeKVTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_kv (
protocol VARCHAR(32) NOT NULL,
vin VARCHAR(32) NOT NULL,
domain_name VARCHAR(64) NOT NULL,
field_name VARCHAR(160) NOT NULL,
field_value TEXT NULL,
value_type VARCHAR(16) NOT NULL DEFAULT '',
event_time DATETIME(3) NULL,
received_at DATETIME(3) NULL,
event_id VARCHAR(64) NOT NULL DEFAULT '',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (protocol, vin, domain_name, field_name),
KEY idx_vin_domain (vin, domain_name),
KEY idx_protocol_domain (protocol, domain_name)
)`
const upsertRealtimeKVSQLPrefix = `
INSERT INTO vehicle_realtime_kv
(protocol, vin, domain_name, field_name, field_value, value_type, event_time, received_at, event_id)
VALUES `
const upsertRealtimeKVSQLSuffix = `
ON DUPLICATE KEY UPDATE
field_value = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_kv.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_kv.event_time), VALUES(field_value), field_value),
value_type = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_kv.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_kv.event_time), VALUES(value_type), value_type),
event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_kv.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_kv.event_time), VALUES(event_time), event_time),
received_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_kv.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_kv.event_time), VALUES(received_at), received_at),
event_id = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_kv.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_kv.event_time), VALUES(event_id), event_id),
updated_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_kv.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_kv.event_time), CURRENT_TIMESTAMP, updated_at)
`
const upsertRealtimeLocationSQL = `
INSERT INTO vehicle_realtime_location
(protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km,