feat(platform): report redis online key health
This commit is contained in:
@@ -33,7 +33,12 @@ func NewServer(cfg config.Config) http.Handler {
|
|||||||
log.Printf("production tdengine store enabled")
|
log.Printf("production tdengine store enabled")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
store = platform.NewProductionStore(db, tdengine, cfg.TDengineDatabase)
|
productionStore := platform.NewProductionStore(db, tdengine, cfg.TDengineDatabase)
|
||||||
|
if cfg.RedisAddr != "" {
|
||||||
|
productionStore.WithRedisOnlineKeyCounter(platform.NewRedisOnlineKeyCounter(cfg.RedisAddr, cfg.RedisUsername, cfg.RedisPassword, cfg.RedisDB))
|
||||||
|
log.Printf("production redis health probe enabled")
|
||||||
|
}
|
||||||
|
store = productionStore
|
||||||
log.Printf("production mysql store enabled")
|
log.Printf("production mysql store enabled")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,11 @@ type ProductionStore struct {
|
|||||||
db *sql.DB
|
db *sql.DB
|
||||||
tdengine *sql.DB
|
tdengine *sql.DB
|
||||||
tdDatabase string
|
tdDatabase string
|
||||||
|
redisOnline redisOnlineKeyCounter
|
||||||
|
}
|
||||||
|
|
||||||
|
type redisOnlineKeyCounter interface {
|
||||||
|
CountOnlineKeys(context.Context) (int, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewProductionStore(db *sql.DB, tdengine *sql.DB, tdengineDatabase string) *ProductionStore {
|
func NewProductionStore(db *sql.DB, tdengine *sql.DB, tdengineDatabase string) *ProductionStore {
|
||||||
@@ -23,6 +28,11 @@ func NewProductionStore(db *sql.DB, tdengine *sql.DB, tdengineDatabase string) *
|
|||||||
return &ProductionStore{db: db, tdengine: tdengine, tdDatabase: tdengineDatabase}
|
return &ProductionStore{db: db, tdengine: tdengine, tdDatabase: tdengineDatabase}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *ProductionStore) WithRedisOnlineKeyCounter(counter redisOnlineKeyCounter) *ProductionStore {
|
||||||
|
s.redisOnline = counter
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
func OpenSQL(ctx context.Context, driver, dsn string) (*sql.DB, error) {
|
func OpenSQL(ctx context.Context, driver, dsn string) (*sql.DB, error) {
|
||||||
db, err := sql.Open(driver, dsn)
|
db, err := sql.Open(driver, dsn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -884,6 +894,7 @@ func (s *ProductionStore) OpsHealth(ctx context.Context) (OpsHealth, error) {
|
|||||||
snapshotHealth := s.mysqlTableReadHealth(ctx, "vehicle_realtime_snapshot", "读取实时快照表正常")
|
snapshotHealth := s.mysqlTableReadHealth(ctx, "vehicle_realtime_snapshot", "读取实时快照表正常")
|
||||||
locationHealth := s.mysqlTableReadHealth(ctx, "vehicle_realtime_location", "读取实时位置表正常")
|
locationHealth := s.mysqlTableReadHealth(ctx, "vehicle_realtime_location", "读取实时位置表正常")
|
||||||
tdengineHealth := s.tdengineRawFrameHealth(ctx)
|
tdengineHealth := s.tdengineRawFrameHealth(ctx)
|
||||||
|
redisHealth, redisOnlineKeys := s.redisOnlineKeyHealth(ctx)
|
||||||
mysqlWritable := mysqlStatus == "ok" && snapshotHealth.Status == "ok" && locationHealth.Status == "ok"
|
mysqlWritable := mysqlStatus == "ok" && snapshotHealth.Status == "ok" && locationHealth.Status == "ok"
|
||||||
return OpsHealth{
|
return OpsHealth{
|
||||||
LinkHealth: []LinkHealth{
|
LinkHealth: []LinkHealth{
|
||||||
@@ -892,13 +903,30 @@ func (s *ProductionStore) OpsHealth(ctx context.Context) (OpsHealth, error) {
|
|||||||
locationHealth,
|
locationHealth,
|
||||||
tdengineHealth,
|
tdengineHealth,
|
||||||
{Name: "Kafka lag", Status: "warning", Detail: "平台暂未接入 Kafka consumer lag 监控"},
|
{Name: "Kafka lag", Status: "warning", Detail: "平台暂未接入 Kafka consumer lag 监控"},
|
||||||
{Name: "Redis online keys", Status: "warning", Detail: "平台暂未接入 Redis 在线 key 读取"},
|
redisHealth,
|
||||||
},
|
},
|
||||||
|
RedisOnlineKeys: redisOnlineKeys,
|
||||||
TDengineWritable: tdengineHealth.Status == "ok",
|
TDengineWritable: tdengineHealth.Status == "ok",
|
||||||
MySQLWritable: mysqlWritable,
|
MySQLWritable: mysqlWritable,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *ProductionStore) redisOnlineKeyHealth(ctx context.Context) (LinkHealth, *int) {
|
||||||
|
health := LinkHealth{Name: "Redis online keys", Status: "warning", Detail: "平台暂未接入 Redis 在线 key 读取"}
|
||||||
|
if s.redisOnline == nil {
|
||||||
|
return health, nil
|
||||||
|
}
|
||||||
|
count, err := s.redisOnline.CountOnlineKeys(ctx)
|
||||||
|
if err != nil {
|
||||||
|
health.Status = "error"
|
||||||
|
health.Detail = err.Error()
|
||||||
|
return health, nil
|
||||||
|
}
|
||||||
|
health.Status = "ok"
|
||||||
|
health.Detail = "Redis 在线 key 读取正常:" + strconv.Itoa(count)
|
||||||
|
return health, &count
|
||||||
|
}
|
||||||
|
|
||||||
func (s *ProductionStore) mysqlTableReadHealth(ctx context.Context, table string, okDetail string) LinkHealth {
|
func (s *ProductionStore) mysqlTableReadHealth(ctx context.Context, table string, okDetail string) LinkHealth {
|
||||||
health := LinkHealth{Name: table, Status: "ok", Detail: okDetail}
|
health := LinkHealth{Name: table, Status: "ok", Detail: okDetail}
|
||||||
query := "SELECT 1 FROM " + table + " LIMIT 1"
|
query := "SELECT 1 FROM " + table + " LIMIT 1"
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
sql.Register("tdengine_missing_table_test", missingTableDriver{})
|
sql.Register("tdengine_missing_table_test", missingTableDriver{})
|
||||||
|
sql.Register("ops_health_test", opsHealthDriver{})
|
||||||
}
|
}
|
||||||
|
|
||||||
type missingTableDriver struct{}
|
type missingTableDriver struct{}
|
||||||
@@ -52,6 +53,66 @@ func (emptyRows) Next(_ []driver.Value) error {
|
|||||||
return io.EOF
|
return io.EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type opsHealthDriver struct{}
|
||||||
|
|
||||||
|
func (opsHealthDriver) Open(_ string) (driver.Conn, error) {
|
||||||
|
return opsHealthConn{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type opsHealthConn struct{}
|
||||||
|
|
||||||
|
func (opsHealthConn) Prepare(_ string) (driver.Stmt, error) {
|
||||||
|
return nil, errors.New("not implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (opsHealthConn) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (opsHealthConn) Begin() (driver.Tx, error) {
|
||||||
|
return nil, errors.New("not implemented")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (opsHealthConn) Ping(_ context.Context) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (opsHealthConn) QueryContext(_ context.Context, _ string, _ []driver.NamedValue) (driver.Rows, error) {
|
||||||
|
return &singleValueRows{columns: []string{"ok"}, values: []driver.Value{1}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type singleValueRows struct {
|
||||||
|
columns []string
|
||||||
|
values []driver.Value
|
||||||
|
read bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r singleValueRows) Columns() []string {
|
||||||
|
return r.columns
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r singleValueRows) Close() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *singleValueRows) Next(dest []driver.Value) error {
|
||||||
|
if r.read {
|
||||||
|
return io.EOF
|
||||||
|
}
|
||||||
|
r.read = true
|
||||||
|
copy(dest, r.values)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeRedisOnlineProbe struct {
|
||||||
|
count int
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p fakeRedisOnlineProbe) CountOnlineKeys(context.Context) (int, error) {
|
||||||
|
return p.count, p.err
|
||||||
|
}
|
||||||
|
|
||||||
func TestBuildVehicleServiceOverviewBatchSQLUsesFuzzyKeywordMatching(t *testing.T) {
|
func TestBuildVehicleServiceOverviewBatchSQLUsesFuzzyKeywordMatching(t *testing.T) {
|
||||||
built := buildVehicleServiceOverviewBatchSQL(VehicleOverviewBatchQuery{
|
built := buildVehicleServiceOverviewBatchSQL(VehicleOverviewBatchQuery{
|
||||||
Keywords: []string{"AG183", "R0LS1426"},
|
Keywords: []string{"AG183", "R0LS1426"},
|
||||||
@@ -107,6 +168,33 @@ func TestRawFramesReturnsEmptyPageWhenTDengineTableIsMissing(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestOpsHealthUsesRedisOnlineKeyProbe(t *testing.T) {
|
||||||
|
db, err := sql.Open("ops_health_test", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open ops health db: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
store := &ProductionStore{db: db, redisOnline: fakeRedisOnlineProbe{count: 73}}
|
||||||
|
|
||||||
|
health, err := store.OpsHealth(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("OpsHealth returned error: %v", err)
|
||||||
|
}
|
||||||
|
if health.RedisOnlineKeys == nil || *health.RedisOnlineKeys != 73 {
|
||||||
|
t.Fatalf("OpsHealth should expose Redis online key count, got %+v", health.RedisOnlineKeys)
|
||||||
|
}
|
||||||
|
var redisHealth *LinkHealth
|
||||||
|
for index := range health.LinkHealth {
|
||||||
|
if health.LinkHealth[index].Name == "Redis online keys" {
|
||||||
|
redisHealth = &health.LinkHealth[index]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if redisHealth == nil || redisHealth.Status != "ok" || !strings.Contains(redisHealth.Detail, "73") {
|
||||||
|
t.Fatalf("OpsHealth should expose Redis online key probe status, got %+v in %+v", redisHealth, health.LinkHealth)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestVehicleServiceOverviewMatchesPartialKeyword(t *testing.T) {
|
func TestVehicleServiceOverviewMatchesPartialKeyword(t *testing.T) {
|
||||||
overview := VehicleServiceOverview{
|
overview := VehicleServiceOverview{
|
||||||
VIN: "LB9A32A24R0LS1426",
|
VIN: "LB9A32A24R0LS1426",
|
||||||
|
|||||||
165
vehicle-data-platform/apps/api/internal/platform/redis_online.go
Normal file
165
vehicle-data-platform/apps/api/internal/platform/redis_online.go
Normal file
@@ -0,0 +1,165 @@
|
|||||||
|
package platform
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type RedisOnlineKeyCounter struct {
|
||||||
|
addr string
|
||||||
|
username string
|
||||||
|
password string
|
||||||
|
db int
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewRedisOnlineKeyCounter(addr, username, password string, db int) *RedisOnlineKeyCounter {
|
||||||
|
return &RedisOnlineKeyCounter{addr: strings.TrimSpace(addr), username: username, password: password, db: db}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *RedisOnlineKeyCounter) CountOnlineKeys(ctx context.Context) (int, error) {
|
||||||
|
if c.addr == "" {
|
||||||
|
return 0, errors.New("redis addr is empty")
|
||||||
|
}
|
||||||
|
dialer := net.Dialer{Timeout: 2 * time.Second}
|
||||||
|
conn, err := dialer.DialContext(ctx, "tcp", c.addr)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
if deadline, ok := ctx.Deadline(); ok {
|
||||||
|
_ = conn.SetDeadline(deadline)
|
||||||
|
} else {
|
||||||
|
_ = conn.SetDeadline(time.Now().Add(3 * time.Second))
|
||||||
|
}
|
||||||
|
reader := bufio.NewReader(conn)
|
||||||
|
if c.password != "" {
|
||||||
|
args := []string{"AUTH"}
|
||||||
|
if c.username != "" {
|
||||||
|
args = append(args, c.username)
|
||||||
|
}
|
||||||
|
args = append(args, c.password)
|
||||||
|
if _, err := redisCommand(conn, reader, args...); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := redisCommand(conn, reader, "PING"); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if c.db >= 0 {
|
||||||
|
if _, err := redisCommand(conn, reader, "SELECT", strconv.Itoa(c.db)); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
total := 0
|
||||||
|
cursor := "0"
|
||||||
|
for {
|
||||||
|
value, err := redisCommand(conn, reader, "SCAN", cursor, "MATCH", "vehicle:online:*", "COUNT", "1000")
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
reply, ok := value.([]any)
|
||||||
|
if !ok || len(reply) != 2 {
|
||||||
|
return 0, fmt.Errorf("unexpected redis scan reply: %#v", value)
|
||||||
|
}
|
||||||
|
nextCursor, ok := reply[0].(string)
|
||||||
|
if !ok {
|
||||||
|
return 0, fmt.Errorf("unexpected redis scan cursor: %#v", reply[0])
|
||||||
|
}
|
||||||
|
keys, ok := reply[1].([]any)
|
||||||
|
if !ok {
|
||||||
|
return 0, fmt.Errorf("unexpected redis scan keys: %#v", reply[1])
|
||||||
|
}
|
||||||
|
total += len(keys)
|
||||||
|
if nextCursor == "0" {
|
||||||
|
return total, nil
|
||||||
|
}
|
||||||
|
cursor = nextCursor
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func redisCommand(conn net.Conn, reader *bufio.Reader, args ...string) (any, error) {
|
||||||
|
if _, err := fmt.Fprintf(conn, "*%d\r\n", len(args)); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, arg := range args {
|
||||||
|
if _, err := fmt.Fprintf(conn, "$%d\r\n%s\r\n", len(arg), arg); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return readRedisReply(reader)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readRedisReply(reader *bufio.Reader) (any, error) {
|
||||||
|
prefix, err := reader.ReadByte()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
switch prefix {
|
||||||
|
case '+':
|
||||||
|
return readRedisLine(reader)
|
||||||
|
case '-':
|
||||||
|
line, err := readRedisLine(reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, errors.New(line)
|
||||||
|
case ':':
|
||||||
|
line, err := readRedisLine(reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return strconv.Atoi(line)
|
||||||
|
case '$':
|
||||||
|
line, err := readRedisLine(reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
size, err := strconv.Atoi(line)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if size < 0 {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
buf := make([]byte, size+2)
|
||||||
|
if _, err := io.ReadFull(reader, buf); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return string(buf[:size]), nil
|
||||||
|
case '*':
|
||||||
|
line, err := readRedisLine(reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
count, err := strconv.Atoi(line)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
values := make([]any, 0, count)
|
||||||
|
for i := 0; i < count; i++ {
|
||||||
|
value, err := readRedisReply(reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
values = append(values, value)
|
||||||
|
}
|
||||||
|
return values, nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unexpected redis reply prefix %q", prefix)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readRedisLine(reader *bufio.Reader) (string, error) {
|
||||||
|
line, err := reader.ReadString('\n')
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r"), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package platform
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRedisOnlineKeyCounterScansVehicleOnlineKeys(t *testing.T) {
|
||||||
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("listen fake redis: %v", err)
|
||||||
|
}
|
||||||
|
defer listener.Close()
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
conn, err := listener.Accept()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
reader := bufio.NewReader(conn)
|
||||||
|
for {
|
||||||
|
command, err := readFakeRedisCommand(reader)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch strings.ToUpper(command[0]) {
|
||||||
|
case "PING":
|
||||||
|
_, _ = conn.Write([]byte("+PONG\r\n"))
|
||||||
|
case "SELECT":
|
||||||
|
_, _ = conn.Write([]byte("+OK\r\n"))
|
||||||
|
case "SCAN":
|
||||||
|
if command[1] == "0" {
|
||||||
|
_, _ = conn.Write(fakeRedisScanReply("7", []string{"vehicle:online:GB32960:VIN001", "vehicle:online:JT808:VIN002"}))
|
||||||
|
} else {
|
||||||
|
_, _ = conn.Write(fakeRedisScanReply("0", []string{"vehicle:online:YUTONG_MQTT:VIN003"}))
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
t.Errorf("unexpected fake redis command: %#v", command)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
counter := NewRedisOnlineKeyCounter(listener.Addr().String(), "", "", 50)
|
||||||
|
count, err := counter.CountOnlineKeys(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CountOnlineKeys returned error: %v", err)
|
||||||
|
}
|
||||||
|
if count != 3 {
|
||||||
|
t.Fatalf("expected 3 online keys, got %d", count)
|
||||||
|
}
|
||||||
|
_ = listener.Close()
|
||||||
|
<-done
|
||||||
|
}
|
||||||
|
|
||||||
|
func fakeRedisScanReply(cursor string, keys []string) []byte {
|
||||||
|
var builder strings.Builder
|
||||||
|
builder.WriteString("*2\r\n")
|
||||||
|
builder.WriteString(fmt.Sprintf("$%d\r\n%s\r\n", len(cursor), cursor))
|
||||||
|
builder.WriteString(fmt.Sprintf("*%d\r\n", len(keys)))
|
||||||
|
for _, key := range keys {
|
||||||
|
builder.WriteString(fmt.Sprintf("$%d\r\n%s\r\n", len(key), key))
|
||||||
|
}
|
||||||
|
return []byte(builder.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
func readFakeRedisCommand(reader *bufio.Reader) ([]string, error) {
|
||||||
|
line, err := reader.ReadString('\n')
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var count int
|
||||||
|
if _, err := fmt.Sscanf(strings.TrimSpace(line), "*%d", &count); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
parts := make([]string, 0, count)
|
||||||
|
for i := 0; i < count; i++ {
|
||||||
|
if _, err := reader.ReadString('\n'); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
value, err := reader.ReadString('\n')
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
parts = append(parts, strings.TrimSuffix(strings.TrimSuffix(value, "\n"), "\r"))
|
||||||
|
}
|
||||||
|
return parts, nil
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user