feat: build vehicle data platform and production pipeline

This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions

View File

@@ -0,0 +1,176 @@
package app
import (
"crypto/sha256"
"crypto/subtle"
"encoding/json"
"fmt"
"log"
"net/http"
"strings"
"lingniu/vehicle-data-platform/apps/api/internal/config"
"lingniu/vehicle-data-platform/apps/api/internal/httpx"
"lingniu/vehicle-data-platform/apps/api/internal/platform"
)
type configuredPrincipal struct {
Token string `json:"token"`
Name string `json:"name"`
Role string `json:"role"`
}
type tokenPrincipal struct {
hash [sha256.Size]byte
principal platform.Principal
}
type apiAuthenticator struct {
mode string
tokens []tokenPrincipal
}
func withAPIAuth(next http.Handler, cfg config.Config) http.Handler {
authenticator, err := newAPIAuthenticator(cfg)
if err != nil {
log.Printf("platform API authentication misconfigured: %v", err)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
httpx.WriteError(w, http.StatusServiceUnavailable, "AUTH_CONFIG_INVALID", "平台鉴权配置无效", err.Error(), requestTraceID(r))
})
}
return authenticator.middleware(next)
}
func newAPIAuthenticator(cfg config.Config) (*apiAuthenticator, error) {
mode := strings.ToLower(strings.TrimSpace(cfg.AuthMode))
if mode == "" {
mode = "disabled"
}
if mode != "disabled" && mode != "enforce" {
return nil, fmt.Errorf("AUTH_MODE must be disabled or enforce")
}
authenticator := &apiAuthenticator{mode: mode}
configured := []configuredPrincipal{}
if strings.TrimSpace(cfg.AuthTokensJSON) != "" {
if err := json.Unmarshal([]byte(cfg.AuthTokensJSON), &configured); err != nil {
return nil, fmt.Errorf("decode AUTH_TOKENS_JSON: %w", err)
}
}
if token := strings.TrimSpace(cfg.AuthToken); token != "" {
configured = append(configured, configuredPrincipal{Token: token, Name: "platform-admin", Role: "admin"})
}
seen := map[[sha256.Size]byte]bool{}
for _, item := range configured {
item.Token = strings.TrimSpace(item.Token)
item.Name = strings.TrimSpace(item.Name)
item.Role = strings.ToLower(strings.TrimSpace(item.Role))
if len(item.Token) < 16 {
return nil, fmt.Errorf("token for %q must contain at least 16 characters", item.Name)
}
if item.Name == "" || roleRank(item.Role) == 0 {
return nil, fmt.Errorf("token principal requires name and viewer/operator/admin role")
}
hash := sha256.Sum256([]byte(item.Token))
if seen[hash] {
return nil, fmt.Errorf("duplicate authentication token")
}
seen[hash] = true
authenticator.tokens = append(authenticator.tokens, tokenPrincipal{hash: hash, principal: platform.Principal{Name: item.Name, Role: item.Role}})
}
if mode == "enforce" && len(authenticator.tokens) == 0 {
return nil, fmt.Errorf("enforce mode requires AUTH_TOKEN or AUTH_TOKENS_JSON")
}
return authenticator, nil
}
func (a *apiAuthenticator) middleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
principal, ok := a.authenticate(r)
if !ok {
w.Header().Set("WWW-Authenticate", `Bearer realm="lingniu-vehicle-platform"`)
httpx.WriteError(w, http.StatusUnauthorized, "AUTH_REQUIRED", "需要有效的访问令牌", "", requestTraceID(r))
return
}
required := requiredRole(r)
if roleRank(principal.Role) < roleRank(required) {
httpx.WriteError(w, http.StatusForbidden, "PERMISSION_DENIED", "当前角色无权执行该操作", "需要 "+required+" 角色", requestTraceID(r))
return
}
ctx := platform.WithPrincipal(r.Context(), principal)
r = r.WithContext(ctx)
if r.URL.Path == "/api/v2/session" {
httpx.WriteOK(w, requestTraceID(r), struct {
Name string `json:"name"`
Role string `json:"role"`
AuthMode string `json:"authMode"`
}{principal.Name, principal.Role, a.mode})
return
}
next.ServeHTTP(w, r)
})
}
func (a *apiAuthenticator) authenticate(r *http.Request) (platform.Principal, bool) {
if a.mode == "disabled" {
return platform.Principal{Name: "local-developer", Role: "admin"}, true
}
header := strings.TrimSpace(r.Header.Get("Authorization"))
if len(header) < 8 || !strings.EqualFold(header[:7], "Bearer ") {
return platform.Principal{}, false
}
token := strings.TrimSpace(header[7:])
if token == "" {
return platform.Principal{}, false
}
hash := sha256.Sum256([]byte(token))
var match platform.Principal
matched := 0
for _, item := range a.tokens {
equal := subtle.ConstantTimeCompare(hash[:], item.hash[:])
matched |= equal
if equal == 1 {
match = item.principal
}
}
return match, matched == 1
}
func requiredRole(r *http.Request) string {
if (r.Method == http.MethodGet || r.Method == http.MethodHead) && strings.HasPrefix(r.URL.Path, "/api/v2/exports") {
return "operator"
}
if r.Method == http.MethodGet || r.Method == http.MethodHead {
return "viewer"
}
path := r.URL.Path
if r.Method == http.MethodPost {
switch path {
case "/api/vehicle-service/overviews", "/api/history/raw-frames/query", "/api/v2/access/summary", "/api/v2/access/vehicles", "/api/v2/alerts/summary", "/api/v2/alerts/events":
return "viewer"
case "/api/v2/exports", "/api/v2/alerts/notifications/read":
return "operator"
}
if strings.HasPrefix(path, "/api/v2/alerts/events/") && strings.HasSuffix(path, "/actions") {
return "operator"
}
if path == "/api/v2/alerts/rules" {
return "admin"
}
}
if r.Method == http.MethodPut && (path == "/api/v2/access/thresholds" || strings.HasPrefix(path, "/api/v2/alerts/rules/")) {
return "admin"
}
return "admin"
}
func roleRank(role string) int {
switch strings.ToLower(role) {
case "viewer":
return 1
case "operator":
return 2
case "admin":
return 3
}
return 0
}

View File

@@ -0,0 +1,135 @@
package app
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"lingniu/vehicle-data-platform/apps/api/internal/config"
"lingniu/vehicle-data-platform/apps/api/internal/platform"
)
const (
viewerToken = "viewer-token-at-least-16"
operatorToken = "operator-token-at-least-16"
adminToken = "admin-token-at-least-16"
)
func testAuthConfig() config.Config {
return config.Config{
AuthMode: "enforce",
AuthTokensJSON: `[
{"token":"` + viewerToken + `","name":"viewer-a","role":"viewer"},
{"token":"` + operatorToken + `","name":"operator-a","role":"operator"},
{"token":"` + adminToken + `","name":"admin-a","role":"admin"}
]`,
}
}
func authRequest(t *testing.T, cfg config.Config, method, path, token string) *httptest.ResponseRecorder {
t.Helper()
next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
principal, ok := platform.PrincipalFromContext(r.Context())
if !ok {
t.Fatal("authenticated request reached handler without principal")
}
w.Header().Set("X-Principal", principal.Name+":"+principal.Role)
w.WriteHeader(http.StatusNoContent)
})
req := httptest.NewRequest(method, path, nil)
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
rec := httptest.NewRecorder()
withAPIAuth(next, cfg).ServeHTTP(rec, req)
return rec
}
func TestAPIAuthEnforcesTokensAndRoleBoundaries(t *testing.T) {
cfg := testAuthConfig()
missing := authRequest(t, cfg, http.MethodGet, "/api/v2/alerts/rules", "")
if missing.Code != http.StatusUnauthorized || !strings.HasPrefix(missing.Header().Get("WWW-Authenticate"), "Bearer") {
t.Fatalf("missing token status=%d headers=%v body=%s", missing.Code, missing.Header(), missing.Body.String())
}
invalid := authRequest(t, cfg, http.MethodGet, "/api/v2/alerts/rules", "wrong-token-at-least-16")
if invalid.Code != http.StatusUnauthorized {
t.Fatalf("invalid token status=%d body=%s", invalid.Code, invalid.Body.String())
}
viewerQuery := authRequest(t, cfg, http.MethodPost, "/api/v2/alerts/events", viewerToken)
if viewerQuery.Code != http.StatusNoContent || viewerQuery.Header().Get("X-Principal") != "viewer-a:viewer" {
t.Fatalf("viewer query status=%d principal=%s", viewerQuery.Code, viewerQuery.Header().Get("X-Principal"))
}
viewerAction := authRequest(t, cfg, http.MethodPost, "/api/v2/alerts/events/a/actions", viewerToken)
if viewerAction.Code != http.StatusForbidden {
t.Fatalf("viewer mutation should be forbidden, status=%d", viewerAction.Code)
}
viewerExports := authRequest(t, cfg, http.MethodGet, "/api/v2/exports", viewerToken)
if viewerExports.Code != http.StatusForbidden {
t.Fatalf("viewer export listing should be forbidden, status=%d", viewerExports.Code)
}
operatorExports := authRequest(t, cfg, http.MethodGet, "/api/v2/exports/exp_1/download", operatorToken)
if operatorExports.Code != http.StatusNoContent {
t.Fatalf("operator export download status=%d", operatorExports.Code)
}
operatorAction := authRequest(t, cfg, http.MethodPost, "/api/v2/alerts/events/a/actions", operatorToken)
if operatorAction.Code != http.StatusNoContent {
t.Fatalf("operator action status=%d body=%s", operatorAction.Code, operatorAction.Body.String())
}
operatorRule := authRequest(t, cfg, http.MethodPost, "/api/v2/alerts/rules", operatorToken)
if operatorRule.Code != http.StatusForbidden {
t.Fatalf("operator rule mutation should be forbidden, status=%d", operatorRule.Code)
}
operatorProfile := authRequest(t, cfg, http.MethodPut, "/api/v2/vehicles/VIN001/profile", operatorToken)
if operatorProfile.Code != http.StatusForbidden {
t.Fatalf("operator profile mutation should be forbidden, status=%d", operatorProfile.Code)
}
operatorProfileSync := authRequest(t, cfg, http.MethodPost, "/api/v2/vehicle-profiles/sync", operatorToken)
if operatorProfileSync.Code != http.StatusForbidden {
t.Fatalf("operator profile sync should be forbidden, status=%d", operatorProfileSync.Code)
}
adminProfile := authRequest(t, cfg, http.MethodPut, "/api/v2/vehicles/VIN001/profile", adminToken)
if adminProfile.Code != http.StatusNoContent || adminProfile.Header().Get("X-Principal") != "admin-a:admin" {
t.Fatalf("admin profile mutation status=%d principal=%s", adminProfile.Code, adminProfile.Header().Get("X-Principal"))
}
adminProfileSync := authRequest(t, cfg, http.MethodPost, "/api/v2/vehicle-profiles/sync", adminToken)
if adminProfileSync.Code != http.StatusNoContent || adminProfileSync.Header().Get("X-Principal") != "admin-a:admin" {
t.Fatalf("admin profile sync status=%d principal=%s", adminProfileSync.Code, adminProfileSync.Header().Get("X-Principal"))
}
adminThreshold := authRequest(t, cfg, http.MethodPut, "/api/v2/access/thresholds", adminToken)
if adminThreshold.Code != http.StatusNoContent {
t.Fatalf("admin threshold status=%d body=%s", adminThreshold.Code, adminThreshold.Body.String())
}
}
func TestAPIAuthSessionAndDisabledMode(t *testing.T) {
rec := authRequest(t, config.Config{AuthMode: "disabled"}, http.MethodGet, "/api/v2/alerts/rules", "")
if rec.Code != http.StatusNoContent || rec.Header().Get("X-Principal") != "local-developer:admin" {
t.Fatalf("disabled mode status=%d principal=%s", rec.Code, rec.Header().Get("X-Principal"))
}
req := httptest.NewRequest(http.MethodGet, "/api/v2/session", nil)
req.Header.Set("Authorization", "Bearer "+operatorToken)
session := httptest.NewRecorder()
withAPIAuth(http.NotFoundHandler(), testAuthConfig()).ServeHTTP(session, req)
if session.Code != http.StatusOK || !strings.Contains(session.Body.String(), `"name":"operator-a"`) || !strings.Contains(session.Body.String(), `"role":"operator"`) {
t.Fatalf("session status=%d body=%s", session.Code, session.Body.String())
}
}
func TestAPIAuthMisconfigurationFailsClosed(t *testing.T) {
cases := []config.Config{
{AuthMode: "enforce"},
{AuthMode: "unknown"},
{AuthMode: "enforce", AuthTokensJSON: `not-json`},
{AuthMode: "enforce", AuthToken: "short"},
}
for _, cfg := range cases {
rec := authRequest(t, cfg, http.MethodGet, "/api/v2/alerts/rules", "")
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("misconfigured auth should fail closed: cfg=%+v status=%d", cfg, rec.Code)
}
}
}

View File

@@ -23,13 +23,27 @@ import (
)
func NewServer(cfg config.Config) http.Handler {
dataMode := strings.ToLower(strings.TrimSpace(cfg.DataMode))
if dataMode == "" {
dataMode = "mock"
}
var store platform.Store = platform.NewMockStore()
if cfg.MySQLDSN != "" {
var storeErr error
if dataMode != "mock" && dataMode != "production" {
storeErr = fmt.Errorf("DATA_MODE must be mock or production")
}
if dataMode == "production" && strings.TrimSpace(cfg.MySQLDSN) == "" {
storeErr = fmt.Errorf("production data mode requires MYSQL_DSN")
}
if cfg.MySQLDSN != "" && dataMode == "production" {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
db, err := platform.OpenSQL(ctx, "mysql", cfg.MySQLDSN)
if err != nil {
log.Printf("production mysql store disabled: %v", err)
if dataMode == "production" {
storeErr = fmt.Errorf("connect production mysql: %w", err)
}
} else {
var tdengine *sql.DB
if cfg.TDengineDSN != "" {
@@ -49,11 +63,15 @@ func NewServer(cfg config.Config) http.Handler {
productionStore.WithCapacityChecker(platform.NewCapacityCheckCommand(cfg.CapacityCheckBin))
log.Printf("production capacity-check probe enabled")
}
productionStore.WithAlertStreamConfig(cfg.AlertStreamMode, cfg.AlertStreamKafkaGroup)
store = productionStore
storeErr = nil
log.Printf("production mysql store enabled")
}
}
api := platform.NewHandler(platform.NewServiceWithRuntime(store, platform.RuntimeInfo{
var api http.Handler = platform.NewHandler(platform.NewServiceWithRuntime(store, platform.RuntimeInfo{
DataMode: dataMode,
ExportDir: strings.TrimSpace(cfg.ExportDir),
RequestTimeoutMs: int(cfg.RequestTimeout / time.Millisecond),
AMapWebJSConfigured: strings.TrimSpace(cfg.AMapWebJSKey) != "",
AMapAPIConfigured: strings.TrimSpace(cfg.AMapAPIKey) != "",
@@ -61,8 +79,16 @@ func NewServer(cfg config.Config) http.Handler {
AMapSecurityCodeExposed: exposedAMapSecurityCode(cfg) != "",
AMapSecurityServiceHost: strings.TrimSpace(cfg.AMapServiceHost),
PlatformRelease: strings.TrimSpace(cfg.PlatformRelease),
AlertStreamMode: strings.TrimSpace(cfg.AlertStreamMode),
AlertStreamConsumerGroup: strings.TrimSpace(cfg.AlertStreamKafkaGroup),
}))
handler := static.Handler(cfg.StaticDir, api)
if storeErr != nil {
log.Printf("platform data store unavailable: %v", storeErr)
api = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
httpx.WriteError(w, http.StatusServiceUnavailable, "DATA_STORE_UNAVAILABLE", "生产数据源不可用", storeErr.Error(), requestTraceID(r))
})
}
handler := static.Handler(cfg.StaticDir, withAPIAuth(api, cfg))
handler = withAMapReverseGeocodeAPI(handler, cfg, "https://restapi.amap.com", http.DefaultClient)
handler = withAppConfig(handler, cfg)
handler = withAMapSecurityProxy(handler, cfg, defaultAMapProxyUpstreams(), http.DefaultClient)
@@ -244,7 +270,8 @@ func withAMapReverseGeocodeAPI(next http.Handler, cfg config.Config, upstream st
}
query := target.Query()
query.Set("key", apiKey)
query.Set("location", fmt.Sprintf("%.6f,%.6f", longitude, latitude))
mapLongitude, mapLatitude := wgs84ToGCJ02(longitude, latitude)
query.Set("location", fmt.Sprintf("%.6f,%.6f", mapLongitude, mapLatitude))
query.Set("extensions", "base")
query.Set("radius", "1000")
query.Set("output", "JSON")
@@ -321,6 +348,38 @@ func parseReverseGeocodeCoordinate(query url.Values) (float64, float64, error) {
return longitude, latitude, nil
}
func wgs84ToGCJ02(longitude float64, latitude float64) (float64, float64) {
if longitude < 72.004 || longitude > 137.8347 || latitude < 0.8293 || latitude > 55.8271 {
return longitude, latitude
}
const semiMajorAxis = 6378245.0
const eccentricitySquared = 0.006693421622965943
longitudeOffset := transformGCJLongitude(longitude-105, latitude-35)
latitudeOffset := transformGCJLatitude(longitude-105, latitude-35)
radianLatitude := latitude / 180 * math.Pi
magic := 1 - eccentricitySquared*math.Pow(math.Sin(radianLatitude), 2)
squareRootMagic := math.Sqrt(magic)
convertedLatitude := latitude + latitudeOffset*180/((semiMajorAxis*(1-eccentricitySquared))/(magic*squareRootMagic)*math.Pi)
convertedLongitude := longitude + longitudeOffset*180/(semiMajorAxis/squareRootMagic*math.Cos(radianLatitude)*math.Pi)
return convertedLongitude, convertedLatitude
}
func transformGCJLatitude(longitude float64, latitude float64) float64 {
value := -100 + 2*longitude + 3*latitude + 0.2*latitude*latitude + 0.1*longitude*latitude + 0.2*math.Sqrt(math.Abs(longitude))
value += (20*math.Sin(6*longitude*math.Pi) + 20*math.Sin(2*longitude*math.Pi)) * 2 / 3
value += (20*math.Sin(latitude*math.Pi) + 40*math.Sin(latitude/3*math.Pi)) * 2 / 3
value += (160*math.Sin(latitude/12*math.Pi) + 320*math.Sin(latitude*math.Pi/30)) * 2 / 3
return value
}
func transformGCJLongitude(longitude float64, latitude float64) float64 {
value := 300 + longitude + 2*latitude + 0.1*longitude*longitude + 0.1*longitude*latitude + 0.1*math.Sqrt(math.Abs(longitude))
value += (20*math.Sin(6*longitude*math.Pi) + 20*math.Sin(2*longitude*math.Pi)) * 2 / 3
value += (20*math.Sin(longitude*math.Pi) + 40*math.Sin(longitude/3*math.Pi)) * 2 / 3
value += (150*math.Sin(longitude/12*math.Pi) + 300*math.Sin(longitude/30*math.Pi)) * 2 / 3
return value
}
func isCoordinate(value float64, min float64, max float64) bool {
return !math.IsNaN(value) && !math.IsInf(value, 0) && value >= min && value <= max
}

View File

@@ -2,12 +2,14 @@ package app
import (
"encoding/json"
"lingniu/vehicle-data-platform/apps/api/internal/config"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"lingniu/vehicle-data-platform/apps/api/internal/config"
)
func TestWithRequestTimeoutAddsContextDeadline(t *testing.T) {
@@ -31,6 +33,15 @@ func TestWithRequestTimeoutAddsContextDeadline(t *testing.T) {
}
}
func TestProductionDataModeFailsClosedWithoutMySQL(t *testing.T) {
handler := NewServer(config.Config{DataMode: "production", RequestTimeout: time.Second})
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/ops/health", nil))
if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "DATA_STORE_UNAVAILABLE") {
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
}
}
func TestWithRequestTimeoutReturnsEnvelopeWithTraceID(t *testing.T) {
handler := withRequestTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
<-r.Context().Done()
@@ -192,7 +203,8 @@ func TestAMapReverseGeocodeAPIUsesServerSideKey(t *testing.T) {
if rec.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
if gotKey != "server-api-key" || gotLocation != "113.123457,23.765432" {
mapLongitude, mapLatitude := wgs84ToGCJ02(113.1234567, 23.7654321)
if gotKey != "server-api-key" || gotLocation != fmt.Sprintf("%.6f,%.6f", mapLongitude, mapLatitude) {
t.Fatalf("key=%q location=%q", gotKey, gotLocation)
}
var body struct {