feat(go): qualify tdengine history writes by database
This commit is contained in:
@@ -440,7 +440,7 @@ FAST_WRITER_TDENGINE_MAX_OPEN_CONNS
|
|||||||
FAST_WRITER_TDENGINE_MAX_IDLE_CONNS
|
FAST_WRITER_TDENGINE_MAX_IDLE_CONNS
|
||||||
```
|
```
|
||||||
|
|
||||||
默认 `max_open=1`、`max_idle=1`。原因是当前 TDengine 写入依赖启动时执行的 `USE <database>`;`database/sql` 连接池新建连接不会继承其他连接上的 `USE` 状态。实测默认跟随 worker 放大到 8 会导致 `[0x200] db is not specified` / `[0x2616] Database not specified`。后续只有在 TDengine DSN 或 writer SQL 改为每条连接都明确选库后,才能提高该连接池。
|
默认 `max_open=1`、`max_idle=1`,生产可以通过 env 小步放大。此前实测默认跟随 worker 放大到 8 会导致 `[0x200] db is not specified` / `[0x2616] Database not specified`,根因是 TDengine `USE <database>` 不会自动作用到 `database/sql` 新建连接。2026-07-03 已将 fast-writer 使用的 history writer 改为生成 `database.table` 形式的 database-qualified SQL,并在 ECS 上将 pool 调到 `2/2` 验证通过:未再出现选库错误,写入 ok 计数持续增长。
|
||||||
|
|
||||||
同时新增 batch pending gauge:
|
同时新增 batch pending gauge:
|
||||||
|
|
||||||
|
|||||||
@@ -136,7 +136,7 @@ go run ./cmd/load-sim \
|
|||||||
| `vehicle_fast_writer_nats_consumer_pending` | 持续增长且 `> 10000` | fast-writer 消费 NATS 的速度跟不上入口写入速度。 |
|
| `vehicle_fast_writer_nats_consumer_pending` | 持续增长且 `> 10000` | fast-writer 消费 NATS 的速度跟不上入口写入速度。 |
|
||||||
| `vehicle_fast_writer_batch_pending_messages` | 持续非 0 或 burst 后不回落 | fast-writer 已拉取 NATS 消息但尚未完成 TDengine/Redis 写入和 ack。 |
|
| `vehicle_fast_writer_batch_pending_messages` | 持续非 0 或 burst 后不回落 | fast-writer 已拉取 NATS 消息但尚未完成 TDengine/Redis 写入和 ack。 |
|
||||||
| `vehicle_fast_writer_batch_pending_envelopes` | 持续非 0 或 burst 后不回落 | fast-writer 当前批次已有有效 envelope 在等待落库或 ack。 |
|
| `vehicle_fast_writer_batch_pending_envelopes` | 持续非 0 或 burst 后不回落 | fast-writer 当前批次已有有效 envelope 在等待落库或 ack。 |
|
||||||
| `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | 某个 stage 的 p99 连续 5 分钟上升 | NATS 快速写链路在 TDengine、Redis 或 NATS ack 某一阶段变慢;当前 TDengine 连接池默认保持 `1`,放大 `FAST_WRITER_TDENGINE_MAX_OPEN_CONNS` 前必须先确认每条连接都能明确选库。 |
|
| `vehicle_fast_writer_stage_duration_ms_histogram_bucket` | 某个 stage 的 p99 连续 5 分钟上升 | NATS 快速写链路在 TDengine、Redis 或 NATS ack 某一阶段变慢;TDengine 阶段可小步调整 `FAST_WRITER_TDENGINE_MAX_OPEN_CONNS`,调整后必须观察是否出现选库错误和 ack-pending 增长。 |
|
||||||
| `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | p99 连续 5 分钟上升 | Redis 实时投影变慢,会直接影响 realtime consumer 追平能力。 |
|
| `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="redis"}` | p99 连续 5 分钟上升 | Redis 实时投影变慢,会直接影响 realtime consumer 追平能力。 |
|
||||||
| `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="mysql"}` | p99 连续 5 分钟上升 | MySQL 当前态/位置投影变慢,需结合 async queue depth 和 dropped 计数判断。 |
|
| `vehicle_realtime_store_update_duration_ms_histogram_bucket{store="mysql"}` | p99 连续 5 分钟上升 | MySQL 当前态/位置投影变慢,需结合 async queue depth 和 dropped 计数判断。 |
|
||||||
| `vehicle_stat_write_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | MySQL 每日里程统计写入变慢,可能导致 stat Kafka lag 增长。 |
|
| `vehicle_stat_write_duration_ms_histogram_bucket` | p99 连续 5 分钟上升 | MySQL 每日里程统计写入变慢,可能导致 stat Kafka lag 增长。 |
|
||||||
|
|||||||
@@ -72,19 +72,13 @@ func main() {
|
|||||||
logger.Error("tdengine ping failed", "error", err)
|
logger.Error("tdengine ping failed", "error", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
historyWriter := history.NewWriter(tdDB)
|
historyWriter := history.NewWriterWithDatabase(tdDB, cfg.TDengineDatabase)
|
||||||
if cfg.TDengineEnsureSchema {
|
if cfg.TDengineEnsureSchema {
|
||||||
if err := historyWriter.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil {
|
if err := historyWriter.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil {
|
||||||
logger.Error("tdengine schema bootstrap failed", "error", err)
|
logger.Error("tdengine schema bootstrap failed", "error", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.TDengineDatabase) != "" {
|
|
||||||
if _, err := tdDB.ExecContext(ctx, "USE "+cfg.TDengineDatabase); err != nil {
|
|
||||||
logger.Error("tdengine use database failed", "database", cfg.TDengineDatabase, "error", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
redisClient := redis.NewClient(&redis.Options{
|
redisClient := redis.NewClient(&redis.Options{
|
||||||
Addr: cfg.RedisAddr,
|
Addr: cfg.RedisAddr,
|
||||||
|
|||||||
@@ -22,8 +22,9 @@ type Execer interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Writer struct {
|
type Writer struct {
|
||||||
exec Execer
|
exec Execer
|
||||||
cache tableCache
|
database string
|
||||||
|
cache tableCache
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -50,6 +51,12 @@ func NewWriter(exec Execer) *Writer {
|
|||||||
return &Writer{exec: exec, cache: tableCache{seen: map[string]struct{}{}}}
|
return &Writer{exec: exec, cache: tableCache{seen: map[string]struct{}{}}}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func NewWriterWithDatabase(exec Execer, database string) *Writer {
|
||||||
|
writer := NewWriter(exec)
|
||||||
|
writer.database = normalizeIdentifier(database)
|
||||||
|
return writer
|
||||||
|
}
|
||||||
|
|
||||||
func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
|
func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
|
||||||
for _, statement := range SchemaStatements(database) {
|
for _, statement := range SchemaStatements(database) {
|
||||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||||
@@ -59,6 +66,14 @@ func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (w *Writer) qualify(table string) string {
|
||||||
|
table = normalizeIdentifier(table)
|
||||||
|
if w.database == "" {
|
||||||
|
return table
|
||||||
|
}
|
||||||
|
return w.database + "." + table
|
||||||
|
}
|
||||||
|
|
||||||
func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
|
func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||||
if err := w.AppendRawFrame(ctx, env); err != nil {
|
if err := w.AppendRawFrame(ctx, env); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -87,7 +102,7 @@ func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope)
|
|||||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||||
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
|
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
|
||||||
raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint)
|
raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint)
|
||||||
VALUES (%s)`, table, joinLiterals(rawValues(env, rawHex, rawText, parsedFields))))
|
VALUES (%s)`, w.qualify(table), joinLiterals(rawValues(env, rawHex, rawText, parsedFields))))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -103,7 +118,7 @@ VALUES (%s)`, table, joinLiterals(rawValues(env, rawHex, rawText, parsedFields))
|
|||||||
for _, chunk := range chunks {
|
for _, chunk := range chunks {
|
||||||
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||||
(ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text)
|
(ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text)
|
||||||
VALUES (%s)`, chunkTable, joinLiterals(chunkValues(env, chunk)))); err != nil {
|
VALUES (%s)`, w.qualify(chunkTable), joinLiterals(chunkValues(env, chunk)))); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -144,7 +159,7 @@ func (w *Writer) AppendRawFrameBatch(ctx context.Context, envelopes []envelope.F
|
|||||||
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||||
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
|
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
|
||||||
raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint)
|
raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint)
|
||||||
VALUES %s`, table, strings.Join(rows, ","))); err != nil {
|
VALUES %s`, w.qualify(table), strings.Join(rows, ","))); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -155,7 +170,7 @@ VALUES %s`, table, strings.Join(rows, ","))); err != nil {
|
|||||||
_ = chunkEnvByTable[table]
|
_ = chunkEnvByTable[table]
|
||||||
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||||
(ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text)
|
(ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text)
|
||||||
VALUES %s`, table, strings.Join(rows, ","))); err != nil {
|
VALUES %s`, w.qualify(table), strings.Join(rows, ","))); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -178,7 +193,7 @@ func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope)
|
|||||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||||
(ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
|
(ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
|
||||||
direction_deg, alarm_flag, status_flag, total_mileage_km)
|
direction_deg, alarm_flag, status_flag, total_mileage_km)
|
||||||
VALUES (%s)`, table, joinLiterals(locationValues(env, longitude, latitude))))
|
VALUES (%s)`, w.qualify(table), joinLiterals(locationValues(env, longitude, latitude))))
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -206,7 +221,7 @@ func (w *Writer) AppendLocationBatch(ctx context.Context, envelopes []envelope.F
|
|||||||
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
||||||
(ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
|
(ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
|
||||||
direction_deg, alarm_flag, status_flag, total_mileage_km)
|
direction_deg, alarm_flag, status_flag, total_mileage_km)
|
||||||
VALUES %s`, table, strings.Join(rows, ","))); err != nil {
|
VALUES %s`, w.qualify(table), strings.Join(rows, ","))); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -222,7 +237,7 @@ func (w *Writer) ensureRawChild(ctx context.Context, table string, stable string
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s', '%s', '%s', '%s')",
|
statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s', '%s', '%s', '%s')",
|
||||||
table, stable,
|
w.qualify(table), w.qualify(stable),
|
||||||
quote(string(env.Protocol)),
|
quote(string(env.Protocol)),
|
||||||
quote(env.VehicleKey()),
|
quote(env.VehicleKey()),
|
||||||
quote(env.VIN),
|
quote(env.VIN),
|
||||||
@@ -232,7 +247,7 @@ func (w *Writer) ensureRawChild(ctx context.Context, table string, stable string
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if phone := normalizePhoneTag(env.Phone); phone != "" {
|
if phone := normalizePhoneTag(env.Phone); phone != "" {
|
||||||
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %s SET TAG phone = '%s'", table, quote(phone))); err != nil {
|
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %s SET TAG phone = '%s'", w.qualify(table), quote(phone))); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -250,8 +265,9 @@ func (w *Writer) ensureLocationChild(ctx context.Context, table string, env enve
|
|||||||
if ok {
|
if ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING vehicle_locations TAGS ('%s', '%s')",
|
statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s')",
|
||||||
table,
|
w.qualify(table),
|
||||||
|
w.qualify("vehicle_locations"),
|
||||||
quote(string(env.Protocol)),
|
quote(string(env.Protocol)),
|
||||||
quote(strings.TrimSpace(env.VIN)))
|
quote(strings.TrimSpace(env.VIN)))
|
||||||
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
|
||||||
@@ -367,6 +383,12 @@ func locationTableName(env envelope.FrameEnvelope) string {
|
|||||||
return "loc_" + strings.ToLower(string(env.Protocol)) + "_" + hash16(strings.TrimSpace(env.VIN))
|
return "loc_" + strings.ToLower(string(env.Protocol)) + "_" + hash16(strings.TrimSpace(env.VIN))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeIdentifier(value string) string {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
value = strings.Trim(value, "`")
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
func normalizePhoneTag(phone string) string {
|
func normalizePhoneTag(phone string) string {
|
||||||
trimmed := strings.TrimLeft(strings.TrimSpace(phone), "0")
|
trimmed := strings.TrimLeft(strings.TrimSpace(phone), "0")
|
||||||
if trimmed == "" {
|
if trimmed == "" {
|
||||||
|
|||||||
@@ -261,6 +261,30 @@ func TestWriterAppendsBatchRowsByChildTable(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestWriterWithDatabaseQualifiesTDengineTables(t *testing.T) {
|
||||||
|
exec := &recordingExec{}
|
||||||
|
writer := NewWriterWithDatabase(exec, "vehicle_ts")
|
||||||
|
env := sampleEnvelope()
|
||||||
|
|
||||||
|
if err := writer.AppendAll(context.Background(), env); err != nil {
|
||||||
|
t.Fatalf("AppendAll() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, want := range []string{
|
||||||
|
"CREATE TABLE IF NOT EXISTS vehicle_ts.raw_",
|
||||||
|
"USING vehicle_ts.raw_frames",
|
||||||
|
"ALTER TABLE vehicle_ts.raw_",
|
||||||
|
"INSERT INTO vehicle_ts.raw_",
|
||||||
|
"CREATE TABLE IF NOT EXISTS vehicle_ts.loc_",
|
||||||
|
"USING vehicle_ts.vehicle_locations",
|
||||||
|
"INSERT INTO vehicle_ts.loc_",
|
||||||
|
} {
|
||||||
|
if !containsSQL(exec.calls, want) {
|
||||||
|
t.Fatalf("qualified sql missing %q, calls=%v", want, exec.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestWriterAppendAllBatchSkipsEmptyBatch(t *testing.T) {
|
func TestWriterAppendAllBatchSkipsEmptyBatch(t *testing.T) {
|
||||||
exec := &recordingExec{}
|
exec := &recordingExec{}
|
||||||
writer := NewWriter(exec)
|
writer := NewWriter(exec)
|
||||||
@@ -338,6 +362,10 @@ func findSQL(calls []execCall, pattern string) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func containsSQL(calls []execCall, pattern string) bool {
|
||||||
|
return findSQL(calls, pattern) != ""
|
||||||
|
}
|
||||||
|
|
||||||
func between(value string, start string, end string) string {
|
func between(value string, start string, end string) string {
|
||||||
startIndex := strings.Index(value, start)
|
startIndex := strings.Index(value, start)
|
||||||
if startIndex < 0 {
|
if startIndex < 0 {
|
||||||
|
|||||||
Reference in New Issue
Block a user