feat: add go tdengine history writer
This commit is contained in:
lingniu
300
go/vehicle-gateway/internal/history/writer.go
Normal file
300
go/vehicle-gateway/internal/history/writer.go
Normal file
@@ -0,0 +1,300 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Execer interface {
|
||||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
type Writer struct {
|
||||
exec Execer
|
||||
cache tableCache
|
||||
}
|
||||
|
||||
type tableCache struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]struct{}
|
||||
}
|
||||
|
||||
func NewWriter(exec Execer) *Writer {
|
||||
if exec == nil {
|
||||
panic("history execer must not be nil")
|
||||
}
|
||||
return &Writer{exec: exec, cache: tableCache{seen: map[string]struct{}{}}}
|
||||
}
|
||||
|
||||
func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
|
||||
for _, statement := range SchemaStatements(database) {
|
||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if err := w.AppendRawFrame(ctx, env); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.AppendLocation(ctx, env); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.AppendMileagePoint(ctx, env)
|
||||
}
|
||||
|
||||
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
table := tableName("raw", env)
|
||||
if err := w.ensureChild(ctx, table, "raw_frames", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
|
||||
raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, table), rawValues(env)...)
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
longitude, okLon := floatField(env, envelope.FieldLongitude)
|
||||
latitude, okLat := floatField(env, envelope.FieldLatitude)
|
||||
if !okLon || !okLat {
|
||||
return nil
|
||||
}
|
||||
table := tableName("loc", env)
|
||||
if err := w.ensureChild(ctx, table, "vehicle_locations", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh,
|
||||
direction_deg, alarm_flag, status_flag, total_mileage_km)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, table), locationValues(env, longitude, latitude)...)
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) AppendMileagePoint(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
table := tableName("mil", env)
|
||||
if err := w.ensureChild(ctx, table, "vehicle_mileage_points", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, table), mileageValues(env, totalMileage)...)
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
|
||||
key := stable + "." + table
|
||||
w.cache.mu.Lock()
|
||||
_, ok := w.cache.seen[key]
|
||||
w.cache.mu.Unlock()
|
||||
if ok {
|
||||
return nil
|
||||
}
|
||||
statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s', '%s', '%s', '%s')",
|
||||
table, stable,
|
||||
quote(string(env.Protocol)),
|
||||
quote(env.VehicleKey()),
|
||||
quote(env.VIN),
|
||||
quote(env.Phone),
|
||||
quote(env.DeviceID))
|
||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||
return err
|
||||
}
|
||||
w.cache.mu.Lock()
|
||||
w.cache.seen[key] = struct{}{}
|
||||
w.cache.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func rawValues(env envelope.FrameEnvelope) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
eventTime := millis(env.EventTimeMS)
|
||||
return []any{
|
||||
received,
|
||||
frameID(env),
|
||||
env.StableEventID(),
|
||||
messageIDInt(env.MessageID),
|
||||
eventTime,
|
||||
received,
|
||||
rawSize(env.RawHex),
|
||||
env.RawHex,
|
||||
env.RawText,
|
||||
jsonString(env.Parsed),
|
||||
jsonString(env.Fields),
|
||||
string(env.ParseStatus),
|
||||
env.ParseError,
|
||||
env.SourceEndpoint,
|
||||
}
|
||||
}
|
||||
|
||||
func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
return []any{
|
||||
eventTimeOrReceived(env),
|
||||
env.StableEventID(),
|
||||
frameID(env),
|
||||
received,
|
||||
longitude,
|
||||
latitude,
|
||||
floatFieldOrNil(env, "altitude_m"),
|
||||
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
||||
intFieldOrNil(env, "direction_deg"),
|
||||
intFieldOrNil(env, "alarm_flag"),
|
||||
intFieldOrNil(env, "status_flag"),
|
||||
floatFieldOrNil(env, envelope.FieldTotalMileageKM),
|
||||
}
|
||||
}
|
||||
|
||||
func mileageValues(env envelope.FrameEnvelope, totalMileage float64) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
return []any{
|
||||
eventTimeOrReceived(env),
|
||||
env.StableEventID(),
|
||||
frameID(env),
|
||||
received,
|
||||
totalMileage,
|
||||
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
||||
floatFieldOrNil(env, envelope.FieldLongitude),
|
||||
floatFieldOrNil(env, envelope.FieldLatitude),
|
||||
}
|
||||
}
|
||||
|
||||
func frameID(env envelope.FrameEnvelope) string {
|
||||
return "go_" + env.StableEventID()
|
||||
}
|
||||
|
||||
func tableName(prefix string, env envelope.FrameEnvelope) string {
|
||||
return prefix + "_" + strings.ToLower(string(env.Protocol)) + "_" + hash16(env.VehicleKey())
|
||||
}
|
||||
|
||||
func hash16(value string) string {
|
||||
sum := sha1.Sum([]byte(value))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
func messageIDInt(value string) int64 {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
value = strings.TrimPrefix(value, "0x")
|
||||
parsed, err := strconv.ParseInt(value, 16, 32)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
parsed, _ = strconv.ParseInt(value, 10, 32)
|
||||
return parsed
|
||||
}
|
||||
|
||||
func rawSize(rawHex string) int {
|
||||
rawHex = strings.TrimSpace(rawHex)
|
||||
if len(rawHex)%2 != 0 {
|
||||
return len(rawHex) / 2
|
||||
}
|
||||
return len(rawHex) / 2
|
||||
}
|
||||
|
||||
func jsonString(value any) string {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return "{}"
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
|
||||
if env.Fields == nil {
|
||||
return 0, false
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return 0, false
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case uint16:
|
||||
return float64(typed), true
|
||||
case uint32:
|
||||
return float64(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func floatFieldOrNil(env envelope.FrameEnvelope, key string) any {
|
||||
value, ok := floatField(env, key)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func intFieldOrNil(env envelope.FrameEnvelope, key string) any {
|
||||
if env.Fields == nil {
|
||||
return nil
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return nil
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case int:
|
||||
return typed
|
||||
case int64:
|
||||
return typed
|
||||
case uint16:
|
||||
return int64(typed)
|
||||
case uint32:
|
||||
return int64(typed)
|
||||
case float64:
|
||||
return int64(typed)
|
||||
case string:
|
||||
parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func eventTimeOrReceived(env envelope.FrameEnvelope) time.Time {
|
||||
if env.EventTimeMS > 0 {
|
||||
return millis(env.EventTimeMS)
|
||||
}
|
||||
return millis(env.ReceivedAtMS)
|
||||
}
|
||||
|
||||
func millis(value int64) time.Time {
|
||||
if value <= 0 {
|
||||
return time.UnixMilli(time.Now().UnixMilli()).UTC()
|
||||
}
|
||||
return time.UnixMilli(value).UTC()
|
||||
}
|
||||
|
||||
func quote(value string) string {
|
||||
return strings.ReplaceAll(value, "'", "''")
|
||||
}
|
||||
Reference in New Issue
Block a user