feat: add go realtime redis api
This commit is contained in:
135
go/vehicle-gateway/cmd/realtime-api/main.go
Normal file
135
go/vehicle-gateway/cmd/realtime-api/main.go
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
"github.com/segmentio/kafka-go"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
logger := observability.NewLogger("vehicle-realtime-api")
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
client := redis.NewClient(&redis.Options{
|
||||||
|
Addr: env("REDIS_ADDR", "127.0.0.1:6379"),
|
||||||
|
Username: env("REDIS_USERNAME", ""),
|
||||||
|
Password: env("REDIS_PASSWORD", ""),
|
||||||
|
DB: envInt("REDIS_DB", 0),
|
||||||
|
})
|
||||||
|
defer client.Close()
|
||||||
|
if err := client.Ping(ctx).Err(); err != nil {
|
||||||
|
logger.Error("redis ping failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
repository := realtime.NewRepository(client, realtime.Config{
|
||||||
|
OnlineTTL: time.Duration(envInt("ONLINE_TTL_SECONDS", 600)) * time.Second,
|
||||||
|
})
|
||||||
|
if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); len(brokers) > 0 {
|
||||||
|
go consumeKafka(ctx, logger, repository, brokers)
|
||||||
|
} else {
|
||||||
|
logger.Warn("KAFKA_BROKERS is empty; realtime api will serve existing redis data only")
|
||||||
|
}
|
||||||
|
|
||||||
|
server := &http.Server{
|
||||||
|
Addr: env("HTTP_ADDR", ":20210"),
|
||||||
|
Handler: realtime.NewHandler(repository),
|
||||||
|
ReadHeaderTimeout: 5 * time.Second,
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_ = server.Shutdown(shutdownCtx)
|
||||||
|
}()
|
||||||
|
|
||||||
|
logger.Info("realtime api started", "addr", server.Addr)
|
||||||
|
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
|
logger.Error("http server failed", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func consumeKafka(ctx context.Context, logger interface {
|
||||||
|
Info(string, ...any)
|
||||||
|
Error(string, ...any)
|
||||||
|
Warn(string, ...any)
|
||||||
|
}, repository *realtime.Repository, brokers []string) {
|
||||||
|
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||||
|
Brokers: brokers,
|
||||||
|
GroupID: env("KAFKA_GROUP", "go-realtime-api"),
|
||||||
|
GroupTopics: splitCSV(env("KAFKA_TOPICS", "vehicle.event.unified.v1")),
|
||||||
|
MinBytes: 1,
|
||||||
|
MaxBytes: 10e6,
|
||||||
|
})
|
||||||
|
defer reader.Close()
|
||||||
|
logger.Info("realtime kafka consumer started", "topics", env("KAFKA_TOPICS", "vehicle.event.unified.v1"))
|
||||||
|
for {
|
||||||
|
message, err := reader.FetchMessage(ctx)
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logger.Error("kafka fetch failed", "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var env envelope.FrameEnvelope
|
||||||
|
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||||
|
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||||
|
_ = reader.CommitMessages(ctx, message)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := repository.Update(ctx, env); err != nil {
|
||||||
|
logger.Error("redis realtime update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := reader.CommitMessages(ctx, message); err != nil {
|
||||||
|
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func env(key string, fallback string) string {
|
||||||
|
value := strings.TrimSpace(os.Getenv(key))
|
||||||
|
if value == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
func envInt(key string, fallback int) int {
|
||||||
|
value := strings.TrimSpace(os.Getenv(key))
|
||||||
|
if value == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
parsed, err := strconv.Atoi(value)
|
||||||
|
if err != nil {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return parsed
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitCSV(value string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, item := range strings.Split(value, ",") {
|
||||||
|
item = strings.TrimSpace(item)
|
||||||
|
if item != "" {
|
||||||
|
out = append(out, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -3,13 +3,17 @@ module lingniu-vehicle-ingest/go/vehicle-gateway
|
|||||||
go 1.26
|
go 1.26
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/alicebob/miniredis/v2 v2.35.0
|
||||||
github.com/go-sql-driver/mysql v1.9.3
|
github.com/go-sql-driver/mysql v1.9.3
|
||||||
|
github.com/redis/go-redis/v9 v9.17.2
|
||||||
github.com/segmentio/kafka-go v0.4.49
|
github.com/segmentio/kafka-go v0.4.49
|
||||||
github.com/taosdata/driver-go/v3 v3.8.1
|
github.com/taosdata/driver-go/v3 v3.8.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
filippo.io/edwards25519 v1.1.0 // indirect
|
filippo.io/edwards25519 v1.1.0 // indirect
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||||
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
github.com/gorilla/websocket v1.5.0 // indirect
|
github.com/gorilla/websocket v1.5.0 // indirect
|
||||||
github.com/json-iterator/go v1.1.12 // indirect
|
github.com/json-iterator/go v1.1.12 // indirect
|
||||||
@@ -17,4 +21,5 @@ require (
|
|||||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
|
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
|
||||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||||
|
github.com/yuin/gopher-lua v1.1.1 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,8 +1,18 @@
|
|||||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||||
|
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
|
||||||
|
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
|
||||||
|
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||||
|
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||||
|
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||||
|
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||||
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||||
@@ -22,6 +32,8 @@ github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0
|
|||||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
||||||
|
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||||
github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
|
github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
|
||||||
github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
|
github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
@@ -41,6 +53,8 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
|||||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||||
|
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||||
|
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||||
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
|
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
|
||||||
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
|
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
|
||||||
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
|
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
|
||||||
|
|||||||
69
go/vehicle-gateway/internal/realtime/http.go
Normal file
69
go/vehicle-gateway/internal/realtime/http.go
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
package realtime
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Handler struct {
|
||||||
|
repository *Repository
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewHandler(repository *Repository) *Handler {
|
||||||
|
if repository == nil {
|
||||||
|
panic("realtime repository must not be nil")
|
||||||
|
}
|
||||||
|
return &Handler{repository: repository}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Method != http.MethodGet {
|
||||||
|
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
path := strings.Trim(strings.TrimPrefix(r.URL.Path, "/api/realtime/vehicles/"), "/")
|
||||||
|
parts := strings.Split(path, "/")
|
||||||
|
if len(parts) == 0 || strings.TrimSpace(parts[0]) == "" {
|
||||||
|
writeError(w, http.StatusNotFound, "vehicle not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
vin := parts[0]
|
||||||
|
switch {
|
||||||
|
case len(parts) == 1:
|
||||||
|
snapshot, err := h.repository.GetMerged(r.Context(), vin)
|
||||||
|
writeResult(w, snapshot, err)
|
||||||
|
case len(parts) == 2 && parts[1] == "online":
|
||||||
|
status, err := h.repository.IsOnline(r.Context(), vin)
|
||||||
|
writeResult(w, status, err)
|
||||||
|
case len(parts) == 3 && parts[1] == "protocols":
|
||||||
|
snapshot, err := h.repository.GetProtocol(r.Context(), vin, envelope.Protocol(strings.ToUpper(parts[2])))
|
||||||
|
writeResult(w, snapshot, err)
|
||||||
|
default:
|
||||||
|
writeError(w, http.StatusNotFound, "route not found")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeResult(w http.ResponseWriter, value any, err error) {
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, redis.Nil) {
|
||||||
|
writeError(w, http.StatusNotFound, "not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeError(w http.ResponseWriter, status int, message string) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(status)
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{"error": message})
|
||||||
|
}
|
||||||
38
go/vehicle-gateway/internal/realtime/model.go
Normal file
38
go/vehicle-gateway/internal/realtime/model.go
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
package realtime
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Snapshot struct {
|
||||||
|
VIN string `json:"vin"`
|
||||||
|
Protocol envelope.Protocol `json:"protocol,omitempty"`
|
||||||
|
EventID string `json:"event_id,omitempty"`
|
||||||
|
EventTimeMS int64 `json:"event_time_ms"`
|
||||||
|
ReceivedAtMS int64 `json:"received_at_ms"`
|
||||||
|
SourceEndpoint string `json:"source_endpoint,omitempty"`
|
||||||
|
Fields map[string]any `json:"fields,omitempty"`
|
||||||
|
FieldTimesMS map[string]int64 `json:"field_times_ms,omitempty"`
|
||||||
|
UpdatedAtMS int64 `json:"updated_at_ms"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type OnlineStatus struct {
|
||||||
|
VIN string `json:"vin"`
|
||||||
|
Online bool `json:"online"`
|
||||||
|
LastSeenMS int64 `json:"last_seen_ms"`
|
||||||
|
Protocols []envelope.Protocol `json:"protocols,omitempty"`
|
||||||
|
TTLSeconds int64 `json:"ttl_seconds"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
OnlineTTL time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c Config) ttl() time.Duration {
|
||||||
|
if c.OnlineTTL <= 0 {
|
||||||
|
return 10 * time.Minute
|
||||||
|
}
|
||||||
|
return c.OnlineTTL
|
||||||
|
}
|
||||||
198
go/vehicle-gateway/internal/realtime/repository.go
Normal file
198
go/vehicle-gateway/internal/realtime/repository.go
Normal file
@@ -0,0 +1,198 @@
|
|||||||
|
package realtime
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Repository struct {
|
||||||
|
client *redis.Client
|
||||||
|
cfg Config
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewRepository(client *redis.Client, cfg Config) *Repository {
|
||||||
|
if client == nil {
|
||||||
|
panic("redis client must not be nil")
|
||||||
|
}
|
||||||
|
return &Repository{client: client, cfg: cfg}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||||
|
vin := strings.TrimSpace(env.VIN)
|
||||||
|
if vin == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
nowMS := time.Now().UnixMilli()
|
||||||
|
eventMS := env.EventTimeMS
|
||||||
|
if eventMS <= 0 {
|
||||||
|
eventMS = env.ReceivedAtMS
|
||||||
|
}
|
||||||
|
protocolSnapshot := Snapshot{
|
||||||
|
VIN: vin,
|
||||||
|
Protocol: env.Protocol,
|
||||||
|
EventID: env.StableEventID(),
|
||||||
|
EventTimeMS: eventMS,
|
||||||
|
ReceivedAtMS: env.ReceivedAtMS,
|
||||||
|
SourceEndpoint: env.SourceEndpoint,
|
||||||
|
Fields: cloneFields(env.Fields),
|
||||||
|
FieldTimesMS: fieldTimes(env.Fields, eventMS),
|
||||||
|
UpdatedAtMS: nowMS,
|
||||||
|
}
|
||||||
|
if err := r.setJSON(ctx, protocolKey(vin, env.Protocol), protocolSnapshot, r.cfg.ttl()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
merged, err := r.GetMerged(ctx, vin)
|
||||||
|
if err != nil && !errors.Is(err, redis.Nil) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if merged.VIN == "" {
|
||||||
|
merged = Snapshot{VIN: vin, Fields: map[string]any{}, FieldTimesMS: map[string]int64{}}
|
||||||
|
}
|
||||||
|
mergeFields(&merged, env.Fields, eventMS)
|
||||||
|
if eventMS >= merged.EventTimeMS {
|
||||||
|
merged.EventTimeMS = eventMS
|
||||||
|
merged.EventID = env.StableEventID()
|
||||||
|
merged.ReceivedAtMS = env.ReceivedAtMS
|
||||||
|
merged.SourceEndpoint = env.SourceEndpoint
|
||||||
|
}
|
||||||
|
merged.UpdatedAtMS = nowMS
|
||||||
|
if err := r.setJSON(ctx, mergedKey(vin), merged, r.cfg.ttl()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
protocols, err := r.addProtocol(ctx, vin, env.Protocol)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
online := OnlineStatus{
|
||||||
|
VIN: vin,
|
||||||
|
Online: true,
|
||||||
|
LastSeenMS: env.ReceivedAtMS,
|
||||||
|
Protocols: protocols,
|
||||||
|
TTLSeconds: int64(r.cfg.ttl().Seconds()),
|
||||||
|
}
|
||||||
|
if err := r.setJSON(ctx, onlineKey(vin), online, r.cfg.ttl()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return r.client.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: vin}).Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) GetMerged(ctx context.Context, vin string) (Snapshot, error) {
|
||||||
|
return r.getSnapshot(ctx, mergedKey(vin))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) GetProtocol(ctx context.Context, vin string, protocol envelope.Protocol) (Snapshot, error) {
|
||||||
|
return r.getSnapshot(ctx, protocolKey(vin, protocol))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) IsOnline(ctx context.Context, vin string) (OnlineStatus, error) {
|
||||||
|
var status OnlineStatus
|
||||||
|
payload, err := r.client.Get(ctx, onlineKey(vin)).Bytes()
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, redis.Nil) {
|
||||||
|
return OnlineStatus{VIN: vin, Online: false}, nil
|
||||||
|
}
|
||||||
|
return status, err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(payload, &status); err != nil {
|
||||||
|
return status, err
|
||||||
|
}
|
||||||
|
ttl := r.client.TTL(ctx, onlineKey(vin)).Val()
|
||||||
|
status.Online = ttl > 0
|
||||||
|
status.TTLSeconds = int64(ttl.Seconds())
|
||||||
|
return status, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) getSnapshot(ctx context.Context, key string) (Snapshot, error) {
|
||||||
|
var snapshot Snapshot
|
||||||
|
payload, err := r.client.Get(ctx, key).Bytes()
|
||||||
|
if err != nil {
|
||||||
|
return snapshot, err
|
||||||
|
}
|
||||||
|
return snapshot, json.Unmarshal(payload, &snapshot)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) setJSON(ctx context.Context, key string, value any, ttl time.Duration) error {
|
||||||
|
payload, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return r.client.Set(ctx, key, payload, ttl).Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Repository) addProtocol(ctx context.Context, vin string, protocol envelope.Protocol) ([]envelope.Protocol, error) {
|
||||||
|
key := protocolsKey(vin)
|
||||||
|
if err := r.client.SAdd(ctx, key, string(protocol)).Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_ = r.client.Expire(ctx, key, r.cfg.ttl()).Err()
|
||||||
|
values, err := r.client.SMembers(ctx, key).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sort.Strings(values)
|
||||||
|
out := make([]envelope.Protocol, 0, len(values))
|
||||||
|
for _, value := range values {
|
||||||
|
out = append(out, envelope.Protocol(value))
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeFields(snapshot *Snapshot, fields map[string]any, eventMS int64) {
|
||||||
|
if snapshot.Fields == nil {
|
||||||
|
snapshot.Fields = map[string]any{}
|
||||||
|
}
|
||||||
|
if snapshot.FieldTimesMS == nil {
|
||||||
|
snapshot.FieldTimesMS = map[string]int64{}
|
||||||
|
}
|
||||||
|
for key, value := range fields {
|
||||||
|
if eventMS >= snapshot.FieldTimesMS[key] {
|
||||||
|
snapshot.Fields[key] = value
|
||||||
|
snapshot.FieldTimesMS[key] = eventMS
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneFields(fields map[string]any) map[string]any {
|
||||||
|
if len(fields) == 0 {
|
||||||
|
return map[string]any{}
|
||||||
|
}
|
||||||
|
out := make(map[string]any, len(fields))
|
||||||
|
for key, value := range fields {
|
||||||
|
out[key] = value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func fieldTimes(fields map[string]any, eventMS int64) map[string]int64 {
|
||||||
|
out := make(map[string]int64, len(fields))
|
||||||
|
for key := range fields {
|
||||||
|
out[key] = eventMS
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergedKey(vin string) string {
|
||||||
|
return "vehicle:latest:" + strings.TrimSpace(vin)
|
||||||
|
}
|
||||||
|
|
||||||
|
func protocolKey(vin string, protocol envelope.Protocol) string {
|
||||||
|
return "vehicle:latest:" + strings.TrimSpace(vin) + ":" + string(protocol)
|
||||||
|
}
|
||||||
|
|
||||||
|
func onlineKey(vin string) string {
|
||||||
|
return "vehicle:online:" + strings.TrimSpace(vin)
|
||||||
|
}
|
||||||
|
|
||||||
|
func protocolsKey(vin string) string {
|
||||||
|
return "vehicle:protocols:" + strings.TrimSpace(vin)
|
||||||
|
}
|
||||||
136
go/vehicle-gateway/internal/realtime/repository_test.go
Normal file
136
go/vehicle-gateway/internal/realtime/repository_test.go
Normal file
@@ -0,0 +1,136 @@
|
|||||||
|
package realtime
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/alicebob/miniredis/v2"
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
|
||||||
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRepositoryUpdatesMergedAndProtocolSnapshots(t *testing.T) {
|
||||||
|
repo, closeFn := newTestRepository(t)
|
||||||
|
defer closeFn()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
VIN: "VIN001",
|
||||||
|
EventTimeMS: 1000,
|
||||||
|
ReceivedAtMS: 1100,
|
||||||
|
Fields: map[string]any{
|
||||||
|
envelope.FieldLongitude: 121.1,
|
||||||
|
envelope.FieldTotalMileageKM: 10.5,
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("Update() error = %v", err)
|
||||||
|
}
|
||||||
|
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolGB32960,
|
||||||
|
VIN: "VIN001",
|
||||||
|
EventTimeMS: 900,
|
||||||
|
ReceivedAtMS: 1200,
|
||||||
|
Fields: map[string]any{
|
||||||
|
envelope.FieldLongitude: 120.0,
|
||||||
|
envelope.FieldLatitude: 30.5,
|
||||||
|
},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("Update() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
merged, err := repo.GetMerged(ctx, "VIN001")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetMerged() error = %v", err)
|
||||||
|
}
|
||||||
|
if merged.Fields[envelope.FieldLongitude] != 121.1 {
|
||||||
|
t.Fatalf("older longitude overwrote newer value: %#v", merged.Fields)
|
||||||
|
}
|
||||||
|
if merged.Fields[envelope.FieldLatitude] != 30.5 {
|
||||||
|
t.Fatalf("new latitude missing: %#v", merged.Fields)
|
||||||
|
}
|
||||||
|
protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetProtocol() error = %v", err)
|
||||||
|
}
|
||||||
|
if protocol.Fields[envelope.FieldTotalMileageKM] != 10.5 {
|
||||||
|
t.Fatalf("protocol snapshot = %#v", protocol)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRepositoryOnlineStatus(t *testing.T) {
|
||||||
|
repo, closeFn := newTestRepository(t)
|
||||||
|
defer closeFn()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
status, err := repo.IsOnline(ctx, "VIN001")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("IsOnline() error = %v", err)
|
||||||
|
}
|
||||||
|
if status.Online {
|
||||||
|
t.Fatal("empty vin should be offline")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := repo.Update(ctx, envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
VIN: "VIN001",
|
||||||
|
EventTimeMS: 1000,
|
||||||
|
ReceivedAtMS: 1100,
|
||||||
|
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("Update() error = %v", err)
|
||||||
|
}
|
||||||
|
status, err = repo.IsOnline(ctx, "VIN001")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("IsOnline() error = %v", err)
|
||||||
|
}
|
||||||
|
if !status.Online || status.LastSeenMS != 1100 {
|
||||||
|
t.Fatalf("online status = %#v", status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerReturnsMergedSnapshot(t *testing.T) {
|
||||||
|
repo, closeFn := newTestRepository(t)
|
||||||
|
defer closeFn()
|
||||||
|
if err := repo.Update(context.Background(), envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
VIN: "VIN001",
|
||||||
|
EventTimeMS: 1000,
|
||||||
|
ReceivedAtMS: 1100,
|
||||||
|
Fields: map[string]any{envelope.FieldSpeedKMH: 12.3},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("Update() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/api/realtime/vehicles/VIN001", nil)
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
NewHandler(repo).ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if !stringsContains(rec.Body.String(), `"vin":"VIN001"`) {
|
||||||
|
t.Fatalf("unexpected body: %s", rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestRepository(t *testing.T) (*Repository, func()) {
|
||||||
|
t.Helper()
|
||||||
|
server, err := miniredis.Run()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("miniredis.Run() error = %v", err)
|
||||||
|
}
|
||||||
|
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||||
|
return NewRepository(client, Config{OnlineTTL: time.Minute}), func() {
|
||||||
|
_ = client.Close()
|
||||||
|
server.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func stringsContains(value string, pattern string) bool {
|
||||||
|
return strings.Contains(value, pattern)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user