feat: add go tdengine history writer
This commit is contained in:
lingniu
71
go/vehicle-gateway/internal/history/schema.go
Normal file
71
go/vehicle-gateway/internal/history/schema.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package history
|
||||
|
||||
const DefaultDatabase = "lingniu_vehicle_ts"
|
||||
|
||||
func SchemaStatements(database string) []string {
|
||||
if database == "" {
|
||||
database = DefaultDatabase
|
||||
}
|
||||
return []string{
|
||||
"CREATE DATABASE IF NOT EXISTS " + database + " KEEP 7300 DURATION 10 BUFFER 256",
|
||||
"USE " + database,
|
||||
`CREATE STABLE IF NOT EXISTS raw_frames (
|
||||
ts TIMESTAMP,
|
||||
frame_id NCHAR(64),
|
||||
event_id NCHAR(64),
|
||||
message_id INT,
|
||||
event_time TIMESTAMP,
|
||||
received_at TIMESTAMP,
|
||||
raw_size_bytes INT,
|
||||
raw_hex BINARY(16374),
|
||||
raw_text BINARY(16374),
|
||||
parsed_json BINARY(16374),
|
||||
fields_json BINARY(4096),
|
||||
parse_status NCHAR(16),
|
||||
parse_error BINARY(1024),
|
||||
source_endpoint NCHAR(128)
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
)`,
|
||||
`CREATE STABLE IF NOT EXISTS vehicle_locations (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE,
|
||||
altitude_m DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
direction_deg INT,
|
||||
alarm_flag BIGINT,
|
||||
status_flag BIGINT,
|
||||
total_mileage_km DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
)`,
|
||||
`CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
|
||||
ts TIMESTAMP,
|
||||
event_id NCHAR(64),
|
||||
frame_id NCHAR(64),
|
||||
received_at TIMESTAMP,
|
||||
total_mileage_km DOUBLE,
|
||||
speed_kmh DOUBLE,
|
||||
longitude DOUBLE,
|
||||
latitude DOUBLE
|
||||
) TAGS (
|
||||
protocol NCHAR(32),
|
||||
vehicle_key NCHAR(64),
|
||||
vin NCHAR(32),
|
||||
phone NCHAR(32),
|
||||
device_id NCHAR(64)
|
||||
)`,
|
||||
}
|
||||
}
|
||||
300
go/vehicle-gateway/internal/history/writer.go
Normal file
300
go/vehicle-gateway/internal/history/writer.go
Normal file
@@ -0,0 +1,300 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
type Execer interface {
|
||||
ExecContext(context.Context, string, ...any) (sql.Result, error)
|
||||
}
|
||||
|
||||
type Writer struct {
|
||||
exec Execer
|
||||
cache tableCache
|
||||
}
|
||||
|
||||
type tableCache struct {
|
||||
mu sync.Mutex
|
||||
seen map[string]struct{}
|
||||
}
|
||||
|
||||
func NewWriter(exec Execer) *Writer {
|
||||
if exec == nil {
|
||||
panic("history execer must not be nil")
|
||||
}
|
||||
return &Writer{exec: exec, cache: tableCache{seen: map[string]struct{}{}}}
|
||||
}
|
||||
|
||||
func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
|
||||
for _, statement := range SchemaStatements(database) {
|
||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
if err := w.AppendRawFrame(ctx, env); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.AppendLocation(ctx, env); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.AppendMileagePoint(ctx, env)
|
||||
}
|
||||
|
||||
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
table := tableName("raw", env)
|
||||
if err := w.ensureChild(ctx, table, "raw_frames", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
|
||||
raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, table), rawValues(env)...)
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
longitude, okLon := floatField(env, envelope.FieldLongitude)
|
||||
latitude, okLat := floatField(env, envelope.FieldLatitude)
|
||||
if !okLon || !okLat {
|
||||
return nil
|
||||
}
|
||||
table := tableName("loc", env)
|
||||
if err := w.ensureChild(ctx, table, "vehicle_locations", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh,
|
||||
direction_deg, alarm_flag, status_flag, total_mileage_km)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, table), locationValues(env, longitude, latitude)...)
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) AppendMileagePoint(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
table := tableName("mil", env)
|
||||
if err := w.ensureChild(ctx, table, "vehicle_mileage_points", env); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||
(ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, table), mileageValues(env, totalMileage)...)
|
||||
return err
|
||||
}
|
||||
|
||||
func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
|
||||
key := stable + "." + table
|
||||
w.cache.mu.Lock()
|
||||
_, ok := w.cache.seen[key]
|
||||
w.cache.mu.Unlock()
|
||||
if ok {
|
||||
return nil
|
||||
}
|
||||
statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s', '%s', '%s', '%s')",
|
||||
table, stable,
|
||||
quote(string(env.Protocol)),
|
||||
quote(env.VehicleKey()),
|
||||
quote(env.VIN),
|
||||
quote(env.Phone),
|
||||
quote(env.DeviceID))
|
||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||
return err
|
||||
}
|
||||
w.cache.mu.Lock()
|
||||
w.cache.seen[key] = struct{}{}
|
||||
w.cache.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func rawValues(env envelope.FrameEnvelope) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
eventTime := millis(env.EventTimeMS)
|
||||
return []any{
|
||||
received,
|
||||
frameID(env),
|
||||
env.StableEventID(),
|
||||
messageIDInt(env.MessageID),
|
||||
eventTime,
|
||||
received,
|
||||
rawSize(env.RawHex),
|
||||
env.RawHex,
|
||||
env.RawText,
|
||||
jsonString(env.Parsed),
|
||||
jsonString(env.Fields),
|
||||
string(env.ParseStatus),
|
||||
env.ParseError,
|
||||
env.SourceEndpoint,
|
||||
}
|
||||
}
|
||||
|
||||
func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
return []any{
|
||||
eventTimeOrReceived(env),
|
||||
env.StableEventID(),
|
||||
frameID(env),
|
||||
received,
|
||||
longitude,
|
||||
latitude,
|
||||
floatFieldOrNil(env, "altitude_m"),
|
||||
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
||||
intFieldOrNil(env, "direction_deg"),
|
||||
intFieldOrNil(env, "alarm_flag"),
|
||||
intFieldOrNil(env, "status_flag"),
|
||||
floatFieldOrNil(env, envelope.FieldTotalMileageKM),
|
||||
}
|
||||
}
|
||||
|
||||
func mileageValues(env envelope.FrameEnvelope, totalMileage float64) []any {
|
||||
received := millis(env.ReceivedAtMS)
|
||||
return []any{
|
||||
eventTimeOrReceived(env),
|
||||
env.StableEventID(),
|
||||
frameID(env),
|
||||
received,
|
||||
totalMileage,
|
||||
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
||||
floatFieldOrNil(env, envelope.FieldLongitude),
|
||||
floatFieldOrNil(env, envelope.FieldLatitude),
|
||||
}
|
||||
}
|
||||
|
||||
func frameID(env envelope.FrameEnvelope) string {
|
||||
return "go_" + env.StableEventID()
|
||||
}
|
||||
|
||||
func tableName(prefix string, env envelope.FrameEnvelope) string {
|
||||
return prefix + "_" + strings.ToLower(string(env.Protocol)) + "_" + hash16(env.VehicleKey())
|
||||
}
|
||||
|
||||
func hash16(value string) string {
|
||||
sum := sha1.Sum([]byte(value))
|
||||
return hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
func messageIDInt(value string) int64 {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
value = strings.TrimPrefix(value, "0x")
|
||||
parsed, err := strconv.ParseInt(value, 16, 32)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
parsed, _ = strconv.ParseInt(value, 10, 32)
|
||||
return parsed
|
||||
}
|
||||
|
||||
func rawSize(rawHex string) int {
|
||||
rawHex = strings.TrimSpace(rawHex)
|
||||
if len(rawHex)%2 != 0 {
|
||||
return len(rawHex) / 2
|
||||
}
|
||||
return len(rawHex) / 2
|
||||
}
|
||||
|
||||
func jsonString(value any) string {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return "{}"
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
|
||||
if env.Fields == nil {
|
||||
return 0, false
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return 0, false
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case uint16:
|
||||
return float64(typed), true
|
||||
case uint32:
|
||||
return float64(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func floatFieldOrNil(env envelope.FrameEnvelope, key string) any {
|
||||
value, ok := floatField(env, key)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func intFieldOrNil(env envelope.FrameEnvelope, key string) any {
|
||||
if env.Fields == nil {
|
||||
return nil
|
||||
}
|
||||
value, ok := env.Fields[key]
|
||||
if !ok || value == nil {
|
||||
return nil
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case int:
|
||||
return typed
|
||||
case int64:
|
||||
return typed
|
||||
case uint16:
|
||||
return int64(typed)
|
||||
case uint32:
|
||||
return int64(typed)
|
||||
case float64:
|
||||
return int64(typed)
|
||||
case string:
|
||||
parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func eventTimeOrReceived(env envelope.FrameEnvelope) time.Time {
|
||||
if env.EventTimeMS > 0 {
|
||||
return millis(env.EventTimeMS)
|
||||
}
|
||||
return millis(env.ReceivedAtMS)
|
||||
}
|
||||
|
||||
func millis(value int64) time.Time {
|
||||
if value <= 0 {
|
||||
return time.UnixMilli(time.Now().UnixMilli()).UTC()
|
||||
}
|
||||
return time.UnixMilli(value).UTC()
|
||||
}
|
||||
|
||||
func quote(value string) string {
|
||||
return strings.ReplaceAll(value, "'", "''")
|
||||
}
|
||||
118
go/vehicle-gateway/internal/history/writer_test.go
Normal file
118
go/vehicle-gateway/internal/history/writer_test.go
Normal file
@@ -0,0 +1,118 @@
|
||||
package history
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestSchemaStatementsCreateCoreStables(t *testing.T) {
|
||||
statements := strings.Join(SchemaStatements("test_ts"), "\n")
|
||||
for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations", "vehicle_mileage_points"} {
|
||||
if !strings.Contains(statements, want) {
|
||||
t.Fatalf("schema missing %q:\n%s", want, statements)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterAppendsRawLocationAndMileage(t *testing.T) {
|
||||
exec := &recordingExec{}
|
||||
writer := NewWriter(exec)
|
||||
env := sampleEnvelope()
|
||||
|
||||
if err := writer.AppendAll(context.Background(), env); err != nil {
|
||||
t.Fatalf("AppendAll() error = %v", err)
|
||||
}
|
||||
|
||||
if got := countSQL(exec.calls, "USING raw_frames"); got != 1 {
|
||||
t.Fatalf("raw child create count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 {
|
||||
t.Fatalf("location child create count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 1 {
|
||||
t.Fatalf("mileage child create count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
|
||||
t.Fatalf("raw insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
|
||||
t.Fatalf("location insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 1 {
|
||||
t.Fatalf("mileage insert count = %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterSkipsSparseDerivedRows(t *testing.T) {
|
||||
exec := &recordingExec{}
|
||||
writer := NewWriter(exec)
|
||||
env := sampleEnvelope()
|
||||
env.Fields = map[string]any{}
|
||||
|
||||
if err := writer.AppendAll(context.Background(), env); err != nil {
|
||||
t.Fatalf("AppendAll() error = %v", err)
|
||||
}
|
||||
|
||||
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
|
||||
t.Fatalf("raw insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 0 {
|
||||
t.Fatalf("location insert count = %d", got)
|
||||
}
|
||||
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 {
|
||||
t.Fatalf("mileage insert count = %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func sampleEnvelope() envelope.FrameEnvelope {
|
||||
return envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
MessageID: "0x0200",
|
||||
Sequence: 1,
|
||||
VIN: "LNBVIN00000000001",
|
||||
Phone: "013307795425",
|
||||
SourceEndpoint: "127.0.0.1:18080",
|
||||
EventTimeMS: 1782745114000,
|
||||
ReceivedAtMS: 1782745114999,
|
||||
RawHex: "7E0200",
|
||||
Parsed: map[string]any{"message": "location"},
|
||||
Fields: map[string]any{
|
||||
envelope.FieldLongitude: 121.069881,
|
||||
envelope.FieldLatitude: 30.590151,
|
||||
envelope.FieldSpeedKMH: 23.0,
|
||||
envelope.FieldTotalMileageKM: 10241.2,
|
||||
"direction_deg": uint16(79),
|
||||
"alarm_flag": uint32(0),
|
||||
"status_flag": uint32(72),
|
||||
},
|
||||
ParseStatus: envelope.ParseOK,
|
||||
}
|
||||
}
|
||||
|
||||
func countSQL(calls []execCall, pattern string) int {
|
||||
var count int
|
||||
for _, call := range calls {
|
||||
if strings.Contains(call.query, pattern) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
type execCall struct {
|
||||
query string
|
||||
args []any
|
||||
}
|
||||
|
||||
type recordingExec struct {
|
||||
calls []execCall
|
||||
}
|
||||
|
||||
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
|
||||
e.calls = append(e.calls, execCall{query: query, args: args})
|
||||
return nil, nil
|
||||
}
|
||||
Reference in New Issue
Block a user