perf(go): cache identity binding lookups

This commit is contained in:
lingniu
2026-07-03 08:07:32 +08:00
parent 0b9e803139
commit 400047e08c
5 changed files with 83 additions and 4 deletions

View File

@@ -71,6 +71,7 @@
- `jt808_registration`808 注册、鉴权、最新活跃和 VIN 匹配状态。 - `jt808_registration`808 注册、鉴权、最新活跃和 VIN 匹配状态。
- 状态:`vehicle_identity_binding` 使用 VIN 主键;`jt808_registration` 使用 phone 主键;两张表都不保留代理自增主键和 `created_at` - 状态:`vehicle_identity_binding` 使用 VIN 主键;`jt808_registration` 使用 phone 主键;两张表都不保留代理自增主键和 `created_at`
- 查询:`vehicle_identity_binding` 的 phone/device_id/plate 都是唯一键,接入侧按唯一键直查 VIN不做无意义排序。 - 查询:`vehicle_identity_binding` 的 phone/device_id/plate 都是唯一键,接入侧按唯一键直查 VIN不做无意义排序。
- 写入链路gateway 对 binding 查 VIN 使用短 TTL 内存缓存,包含未命中缓存,避免无 VIN 高频位置帧每条都打 MySQL。
- 生产RDS 已在上线前删除旧 `vehicle_identity_binding_registration``vehicle_identity_bindings`gateway 启动会自动创建最小 schema。 - 生产RDS 已在上线前删除旧 `vehicle_identity_binding_registration``vehicle_identity_bindings`gateway 启动会自动创建最小 schema。
## 字段提升规则 ## 字段提升规则

View File

@@ -146,6 +146,7 @@ func buildIdentityResolver(ctx context.Context, logger *slog.Logger) (identity.R
table := env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding") table := env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding")
resolver := identity.NewMySQLResolverWithOptions(db, table, identity.MySQLResolverOptions{ resolver := identity.NewMySQLResolverWithOptions(db, table, identity.MySQLResolverOptions{
LocationTouchInterval: time.Duration(envInt("JT808_REGISTRATION_LOCATION_TOUCH_INTERVAL_SECONDS", 600)) * time.Second, LocationTouchInterval: time.Duration(envInt("JT808_REGISTRATION_LOCATION_TOUCH_INTERVAL_SECONDS", 600)) * time.Second,
LookupCacheTTL: time.Duration(envInt("IDENTITY_LOOKUP_CACHE_TTL_SECONDS", 600)) * time.Second,
}) })
if envBool("IDENTITY_MYSQL_ENSURE_SCHEMA", true) { if envBool("IDENTITY_MYSQL_ENSURE_SCHEMA", true) {
if err := resolver.EnsureSchema(ctx); err != nil { if err := resolver.EnsureSchema(ctx); err != nil {
@@ -153,7 +154,7 @@ func buildIdentityResolver(ctx context.Context, logger *slog.Logger) (identity.R
return nil, nil, err return nil, nil, err
} }
} }
logger.Info("identity mysql resolver enabled", "table", table) logger.Info("identity mysql resolver enabled", "table", table, "lookup_cache_ttl_seconds", envInt("IDENTITY_LOOKUP_CACHE_TTL_SECONDS", 600))
return resolver, func() { _ = db.Close() }, nil return resolver, func() { _ = db.Close() }, nil
} }

View File

@@ -1,6 +1,8 @@
package main package main
import ( import (
"os"
"strings"
"testing" "testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
@@ -32,6 +34,18 @@ func TestNATSSinkConfigFromEnvUsesExplicitSubjects(t *testing.T) {
} }
} }
func TestGatewayConfiguresIdentityLookupCacheTTL(t *testing.T) {
source, err := os.ReadFile("main.go")
if err != nil {
t.Fatalf("read main.go: %v", err)
}
for _, want := range []string{"LookupCacheTTL", "IDENTITY_LOOKUP_CACHE_TTL_SECONDS"} {
if !strings.Contains(string(source), want) {
t.Fatalf("gateway should expose identity lookup cache ttl, missing %s", want)
}
}
}
func TestNATSSinkConfigFromEnvDefaultsToGoSubjects(t *testing.T) { func TestNATSSinkConfigFromEnvDefaultsToGoSubjects(t *testing.T) {
t.Setenv("NATS_URL", "nats://172.17.111.56:4222") t.Setenv("NATS_URL", "nats://172.17.111.56:4222")

View File

@@ -26,12 +26,22 @@ type MySQLResolver struct {
db *sql.DB db *sql.DB
table string table string
locationTouchInterval time.Duration locationTouchInterval time.Duration
lookupCacheTTL time.Duration
lookupMu sync.Mutex
lookupCache map[string]lookupCacheEntry
touchMu sync.Mutex touchMu sync.Mutex
locationTouches map[string]time.Time locationTouches map[string]time.Time
} }
type MySQLResolverOptions struct { type MySQLResolverOptions struct {
LocationTouchInterval time.Duration LocationTouchInterval time.Duration
LookupCacheTTL time.Duration
}
type lookupCacheEntry struct {
vin string
notFound bool
expiresAt time.Time
} }
func NewMySQLResolver(db *sql.DB, table string) *MySQLResolver { func NewMySQLResolver(db *sql.DB, table string) *MySQLResolver {
@@ -50,10 +60,16 @@ func NewMySQLResolverWithOptions(db *sql.DB, table string, opts MySQLResolverOpt
if interval <= 0 { if interval <= 0 {
interval = 10 * time.Minute interval = 10 * time.Minute
} }
lookupTTL := opts.LookupCacheTTL
if lookupTTL <= 0 {
lookupTTL = 10 * time.Minute
}
return &MySQLResolver{ return &MySQLResolver{
db: db, db: db,
table: table, table: table,
locationTouchInterval: interval, locationTouchInterval: interval,
lookupCacheTTL: lookupTTL,
lookupCache: map[string]lookupCacheEntry{},
locationTouches: map[string]time.Time{}, locationTouches: map[string]time.Time{},
} }
} }
@@ -161,9 +177,32 @@ func (r *MySQLResolver) Resolve(ctx context.Context, env envelope.FrameEnvelope)
} }
func (r *MySQLResolver) lookup(ctx context.Context, column string, value string) (string, error) { func (r *MySQLResolver) lookup(ctx context.Context, column string, value string) (string, error) {
cacheKey := column + "\x00" + value
now := time.Now()
r.lookupMu.Lock()
cached, ok := r.lookupCache[cacheKey]
if ok && now.Before(cached.expiresAt) {
r.lookupMu.Unlock()
if cached.notFound {
return "", sql.ErrNoRows
}
return cached.vin, nil
}
r.lookupMu.Unlock()
query := "SELECT vin FROM " + r.table + " WHERE " + column + " = ? AND vin IS NOT NULL AND vin <> ''" query := "SELECT vin FROM " + r.table + " WHERE " + column + " = ? AND vin IS NOT NULL AND vin <> ''"
var vin string var vin string
err := r.db.QueryRowContext(ctx, query, value).Scan(&vin) err := r.db.QueryRowContext(ctx, query, value).Scan(&vin)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return "", err
}
r.lookupMu.Lock()
r.lookupCache[cacheKey] = lookupCacheEntry{
vin: strings.TrimSpace(vin),
notFound: errors.Is(err, sql.ErrNoRows),
expiresAt: now.Add(r.lookupCacheTTL),
}
r.lookupMu.Unlock()
return vin, err return vin, err
} }

View File

@@ -5,6 +5,7 @@ import (
"database/sql" "database/sql"
"strings" "strings"
"testing" "testing"
"time"
"github.com/DATA-DOG/go-sqlmock" "github.com/DATA-DOG/go-sqlmock"
@@ -188,9 +189,6 @@ func TestMySQLResolverTracksFirstJT808LocationThenThrottles(t *testing.T) {
WillReturnError(sql.ErrNoRows) WillReturnError(sql.ErrNoRows)
mock.ExpectExec("INSERT INTO jt808_registration"). mock.ExpectExec("INSERT INTO jt808_registration").
WillReturnResult(sqlmock.NewResult(1, 1)) WillReturnResult(sqlmock.NewResult(1, 1))
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
WithArgs("13307795425").
WillReturnError(sql.ErrNoRows)
resolver := NewMySQLResolver(db, "vehicle_identity_binding") resolver := NewMySQLResolver(db, "vehicle_identity_binding")
for i := 0; i < 2; i++ { for i := 0; i < 2; i++ {
@@ -210,6 +208,32 @@ func TestMySQLResolverTracksFirstJT808LocationThenThrottles(t *testing.T) {
} }
} }
func TestMySQLResolverCachesIdentityMissesForHighFrequencyFrames(t *testing.T) {
db, mock := newMockDB(t)
defer db.Close()
mock.ExpectQuery("SELECT vin FROM vehicle_identity_binding WHERE phone = \\?").
WithArgs("13307795425").
WillReturnError(sql.ErrNoRows)
resolver := NewMySQLResolverWithOptions(db, "vehicle_identity_binding", MySQLResolverOptions{
LocationTouchInterval: time.Hour,
LookupCacheTTL: time.Hour,
})
for i := 0; i < 2; i++ {
_, err := resolver.Resolve(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "013307795425",
Parsed: map[string]any{},
})
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMySQLResolverEnsuresMinimalIdentitySchema(t *testing.T) { func TestMySQLResolverEnsuresMinimalIdentitySchema(t *testing.T) {
db, mock := newMockDB(t) db, mock := newMockDB(t)
defer db.Close() defer db.Close()