feat(platform): report capacity check health
This commit is contained in:
@@ -38,6 +38,10 @@ func NewServer(cfg config.Config) http.Handler {
|
|||||||
productionStore.WithRedisOnlineKeyCounter(platform.NewRedisOnlineKeyCounter(cfg.RedisAddr, cfg.RedisUsername, cfg.RedisPassword, cfg.RedisDB))
|
productionStore.WithRedisOnlineKeyCounter(platform.NewRedisOnlineKeyCounter(cfg.RedisAddr, cfg.RedisUsername, cfg.RedisPassword, cfg.RedisDB))
|
||||||
log.Printf("production redis health probe enabled")
|
log.Printf("production redis health probe enabled")
|
||||||
}
|
}
|
||||||
|
if cfg.CapacityCheckBin != "" {
|
||||||
|
productionStore.WithCapacityChecker(platform.NewCapacityCheckCommand(cfg.CapacityCheckBin))
|
||||||
|
log.Printf("production capacity-check probe enabled")
|
||||||
|
}
|
||||||
store = productionStore
|
store = productionStore
|
||||||
log.Printf("production mysql store enabled")
|
log.Printf("production mysql store enabled")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
package platform
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type CapacityCheckResult struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
ActiveConnections int64 `json:"active_connections"`
|
||||||
|
KafkaLag float64 `json:"kafka_lag"`
|
||||||
|
Findings []string `json:"findings"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type capacityCheckReport struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Totals struct {
|
||||||
|
ActiveConnections int64 `json:"active_connections"`
|
||||||
|
KafkaLag float64 `json:"kafka_lag"`
|
||||||
|
} `json:"totals"`
|
||||||
|
Findings []string `json:"findings"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type CapacityCheckCommand struct {
|
||||||
|
bin string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewCapacityCheckCommand(bin string) *CapacityCheckCommand {
|
||||||
|
return &CapacityCheckCommand{bin: strings.TrimSpace(bin)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *CapacityCheckCommand) CheckCapacity(ctx context.Context) (CapacityCheckResult, error) {
|
||||||
|
if c.bin == "" {
|
||||||
|
return CapacityCheckResult{}, errors.New("capacity-check bin is empty")
|
||||||
|
}
|
||||||
|
output, err := exec.CommandContext(ctx, c.bin).CombinedOutput()
|
||||||
|
var report capacityCheckReport
|
||||||
|
if jsonErr := json.Unmarshal(output, &report); jsonErr != nil {
|
||||||
|
if err != nil {
|
||||||
|
return CapacityCheckResult{}, errors.New(err.Error() + ": " + strings.TrimSpace(string(output)))
|
||||||
|
}
|
||||||
|
return CapacityCheckResult{}, jsonErr
|
||||||
|
}
|
||||||
|
return CapacityCheckResult{
|
||||||
|
Status: report.Status,
|
||||||
|
ActiveConnections: report.Totals.ActiveConnections,
|
||||||
|
KafkaLag: report.Totals.KafkaLag,
|
||||||
|
Findings: report.Findings,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
@@ -15,12 +15,17 @@ type ProductionStore struct {
|
|||||||
tdengine *sql.DB
|
tdengine *sql.DB
|
||||||
tdDatabase string
|
tdDatabase string
|
||||||
redisOnline redisOnlineKeyCounter
|
redisOnline redisOnlineKeyCounter
|
||||||
|
capacityCheck capacityChecker
|
||||||
}
|
}
|
||||||
|
|
||||||
type redisOnlineKeyCounter interface {
|
type redisOnlineKeyCounter interface {
|
||||||
CountOnlineKeys(context.Context) (int, error)
|
CountOnlineKeys(context.Context) (int, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type capacityChecker interface {
|
||||||
|
CheckCapacity(context.Context) (CapacityCheckResult, error)
|
||||||
|
}
|
||||||
|
|
||||||
func NewProductionStore(db *sql.DB, tdengine *sql.DB, tdengineDatabase string) *ProductionStore {
|
func NewProductionStore(db *sql.DB, tdengine *sql.DB, tdengineDatabase string) *ProductionStore {
|
||||||
if db == nil {
|
if db == nil {
|
||||||
panic("production db must not be nil")
|
panic("production db must not be nil")
|
||||||
@@ -33,6 +38,11 @@ func (s *ProductionStore) WithRedisOnlineKeyCounter(counter redisOnlineKeyCounte
|
|||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *ProductionStore) WithCapacityChecker(checker capacityChecker) *ProductionStore {
|
||||||
|
s.capacityCheck = checker
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
func OpenSQL(ctx context.Context, driver, dsn string) (*sql.DB, error) {
|
func OpenSQL(ctx context.Context, driver, dsn string) (*sql.DB, error) {
|
||||||
db, err := sql.Open(driver, dsn)
|
db, err := sql.Open(driver, dsn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -895,6 +905,7 @@ func (s *ProductionStore) OpsHealth(ctx context.Context) (OpsHealth, error) {
|
|||||||
locationHealth := s.mysqlTableReadHealth(ctx, "vehicle_realtime_location", "读取实时位置表正常")
|
locationHealth := s.mysqlTableReadHealth(ctx, "vehicle_realtime_location", "读取实时位置表正常")
|
||||||
tdengineHealth := s.tdengineRawFrameHealth(ctx)
|
tdengineHealth := s.tdengineRawFrameHealth(ctx)
|
||||||
redisHealth, redisOnlineKeys := s.redisOnlineKeyHealth(ctx)
|
redisHealth, redisOnlineKeys := s.redisOnlineKeyHealth(ctx)
|
||||||
|
capacityHealth, kafkaLag := s.capacityCheckHealth(ctx)
|
||||||
mysqlWritable := mysqlStatus == "ok" && snapshotHealth.Status == "ok" && locationHealth.Status == "ok"
|
mysqlWritable := mysqlStatus == "ok" && snapshotHealth.Status == "ok" && locationHealth.Status == "ok"
|
||||||
return OpsHealth{
|
return OpsHealth{
|
||||||
LinkHealth: []LinkHealth{
|
LinkHealth: []LinkHealth{
|
||||||
@@ -902,15 +913,41 @@ func (s *ProductionStore) OpsHealth(ctx context.Context) (OpsHealth, error) {
|
|||||||
snapshotHealth,
|
snapshotHealth,
|
||||||
locationHealth,
|
locationHealth,
|
||||||
tdengineHealth,
|
tdengineHealth,
|
||||||
{Name: "Kafka lag", Status: "warning", Detail: "平台暂未接入 Kafka consumer lag 监控"},
|
capacityHealth,
|
||||||
redisHealth,
|
redisHealth,
|
||||||
},
|
},
|
||||||
|
KafkaLag: kafkaLag,
|
||||||
RedisOnlineKeys: redisOnlineKeys,
|
RedisOnlineKeys: redisOnlineKeys,
|
||||||
TDengineWritable: tdengineHealth.Status == "ok",
|
TDengineWritable: tdengineHealth.Status == "ok",
|
||||||
MySQLWritable: mysqlWritable,
|
MySQLWritable: mysqlWritable,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *ProductionStore) capacityCheckHealth(ctx context.Context) (LinkHealth, *int) {
|
||||||
|
health := LinkHealth{Name: "Capacity check", Status: "warning", Detail: "平台暂未接入 capacity-check"}
|
||||||
|
if s.capacityCheck == nil {
|
||||||
|
return health, nil
|
||||||
|
}
|
||||||
|
result, err := s.capacityCheck.CheckCapacity(ctx)
|
||||||
|
if err != nil {
|
||||||
|
health.Status = "error"
|
||||||
|
health.Detail = err.Error()
|
||||||
|
return health, nil
|
||||||
|
}
|
||||||
|
kafkaLag := int(result.KafkaLag)
|
||||||
|
detail := "active_connections=" + strconv.Itoa(int(result.ActiveConnections)) + ", kafka_lag=" + strconv.Itoa(kafkaLag)
|
||||||
|
if len(result.Findings) > 0 {
|
||||||
|
detail += ", findings=" + strings.Join(result.Findings, "; ")
|
||||||
|
}
|
||||||
|
if strings.EqualFold(result.Status, "ok") {
|
||||||
|
health.Status = "ok"
|
||||||
|
} else {
|
||||||
|
health.Status = "warning"
|
||||||
|
}
|
||||||
|
health.Detail = detail
|
||||||
|
return health, &kafkaLag
|
||||||
|
}
|
||||||
|
|
||||||
func (s *ProductionStore) redisOnlineKeyHealth(ctx context.Context) (LinkHealth, *int) {
|
func (s *ProductionStore) redisOnlineKeyHealth(ctx context.Context) (LinkHealth, *int) {
|
||||||
health := LinkHealth{Name: "Redis online keys", Status: "warning", Detail: "平台暂未接入 Redis 在线 key 读取"}
|
health := LinkHealth{Name: "Redis online keys", Status: "warning", Detail: "平台暂未接入 Redis 在线 key 读取"}
|
||||||
if s.redisOnline == nil {
|
if s.redisOnline == nil {
|
||||||
|
|||||||
@@ -113,6 +113,15 @@ func (p fakeRedisOnlineProbe) CountOnlineKeys(context.Context) (int, error) {
|
|||||||
return p.count, p.err
|
return p.count, p.err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type fakeCapacityCheckProbe struct {
|
||||||
|
result CapacityCheckResult
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p fakeCapacityCheckProbe) CheckCapacity(context.Context) (CapacityCheckResult, error) {
|
||||||
|
return p.result, p.err
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildVehicleServiceOverviewBatchSQLUsesFuzzyKeywordMatching(t *testing.T) {
|
func TestBuildVehicleServiceOverviewBatchSQLUsesFuzzyKeywordMatching(t *testing.T) {
|
||||||
built := buildVehicleServiceOverviewBatchSQL(VehicleOverviewBatchQuery{
|
built := buildVehicleServiceOverviewBatchSQL(VehicleOverviewBatchQuery{
|
||||||
Keywords: []string{"AG183", "R0LS1426"},
|
Keywords: []string{"AG183", "R0LS1426"},
|
||||||
@@ -195,6 +204,41 @@ func TestOpsHealthUsesRedisOnlineKeyProbe(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpsHealthUsesCapacityCheckProbeForKafkaLag(t *testing.T) {
|
||||||
|
db, err := sql.Open("ops_health_test", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open ops health db: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
store := &ProductionStore{
|
||||||
|
db: db,
|
||||||
|
capacityCheck: fakeCapacityCheckProbe{result: CapacityCheckResult{
|
||||||
|
Status: "degraded",
|
||||||
|
ActiveConnections: 120000,
|
||||||
|
KafkaLag: 42,
|
||||||
|
Findings: []string{"kafka lag 42"},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
health, err := store.OpsHealth(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("OpsHealth returned error: %v", err)
|
||||||
|
}
|
||||||
|
if health.KafkaLag == nil || *health.KafkaLag != 42 {
|
||||||
|
t.Fatalf("OpsHealth should expose capacity-check kafka lag, got %+v", health.KafkaLag)
|
||||||
|
}
|
||||||
|
var capacityHealth *LinkHealth
|
||||||
|
for index := range health.LinkHealth {
|
||||||
|
if health.LinkHealth[index].Name == "Capacity check" {
|
||||||
|
capacityHealth = &health.LinkHealth[index]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if capacityHealth == nil || capacityHealth.Status != "warning" || !strings.Contains(capacityHealth.Detail, "kafka lag 42") {
|
||||||
|
t.Fatalf("OpsHealth should expose capacity-check warning, got %+v in %+v", capacityHealth, health.LinkHealth)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestVehicleServiceOverviewMatchesPartialKeyword(t *testing.T) {
|
func TestVehicleServiceOverviewMatchesPartialKeyword(t *testing.T) {
|
||||||
overview := VehicleServiceOverview{
|
overview := VehicleServiceOverview{
|
||||||
VIN: "LB9A32A24R0LS1426",
|
VIN: "LB9A32A24R0LS1426",
|
||||||
|
|||||||
Reference in New Issue
Block a user