shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden jt808 tdengine ingestion
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
Details

Browse Source
This commit is contained in:
lingniu
2026-06-29 19:53:08 +08:00
parent d1748fcc2f
commit 12de83e37f
17 changed files with 588 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryWriter.java
View File

@@ -3,7 +3,6 @@ package com.lingniu.ingest.tdenginehistory;
import javax.sql.DataSource;
import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
@@ -13,21 +12,19 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public final class TdengineJdbcHistoryWriter implements TdengineHistoryWriter {
private static final Logger log = LoggerFactory.getLogger(TdengineJdbcHistoryWriter.class);
private static final int MAX_LITERAL_ROWS_PER_STATEMENT = 200;
private final DataSource dataSource;
private final TdengineHistorySchema schema;
private final TdengineHistoryStatements statements;
private final Object schemaInitializationMonitor = new Object();
private final Map<String, Instant> lastTimestampByInsertSql = new ConcurrentHashMap<>();
private volatile boolean schemaInitialized;
private volatile boolean literalFallbackLogged;
public TdengineJdbcHistoryWriter(DataSource dataSource, TdengineHistorySchema schema) {
if (dataSource == null) {
@@ -95,87 +92,42 @@ public final class TdengineJdbcHistoryWriter implements TdengineHistoryWriter {
}
}
Set<String> createdChildTables = new LinkedHashSet<>();
Map<String, PreparedStatement> preparedStatements = new LinkedHashMap<>();
Map<String, LiteralBatch> literalBatches = new LinkedHashMap<>();
Map<String, LiteralBatch> literalOnlyBatches = new LinkedHashMap<>();
try {
for (T row : rows) {
if (row == null) {
continue;
}
TdengineBatchStatement batch = mapper.apply(row);
if (createdChildTables.add(batch.createChildTableSql())) {
try (Statement statement = connection.createStatement()) {
statement.execute(batch.createChildTableSql());
}
}
if (hasEmptyString(batch.values())) {
literalOnlyBatches.computeIfAbsent(batch.insertSql(), LiteralBatch::new).add(batch.values());
continue;
}
PreparedStatement prepared = preparedStatements.computeIfAbsent(batch.insertSql(), sql -> {
try {
return connection.prepareStatement(sql);
} catch (SQLException e) {
throw new JdbcRuntimeException(e);
}
});
bind(prepared, batch.values());
prepared.addBatch();
literalBatches.computeIfAbsent(batch.insertSql(), LiteralBatch::new).add(batch.values());
for (T row : rows) {
if (row == null) {
continue;
}
for (Map.Entry<String, PreparedStatement> entry : preparedStatements.entrySet()) {
try {
entry.getValue().executeBatch();
} catch (SQLException e) {
executeLiteralFallback(connection, literalBatches.get(entry.getKey()), e);
TdengineBatchStatement batch = mapper.apply(row);
if (createdChildTables.add(batch.createChildTableSql())) {
try (Statement statement = connection.createStatement()) {
statement.execute(batch.createChildTableSql());
}
}
for (LiteralBatch batch : literalOnlyBatches.values()) {
executeLiteralBatch(connection, batch);
}
} finally {
for (PreparedStatement prepared : preparedStatements.values()) {
prepared.close();
}
literalBatches.computeIfAbsent(batch.insertSql(), LiteralBatch::new)
.add(uniqueTimestampValues(batch.insertSql(), batch.values()));
}
for (LiteralBatch batch : literalBatches.values()) {
executeLiteralBatch(connection, batch);
}
});
}
private static void bind(PreparedStatement prepared, List<Object> values) throws SQLException {
for (int i = 0; i < values.size(); i++) {
Object value = values.get(i);
int parameterIndex = i + 1;
if (value instanceof Instant instant) {
prepared.setTimestamp(parameterIndex, Timestamp.from(instant));
} else {
prepared.setObject(parameterIndex, value);
private List<Object> uniqueTimestampValues(String insertSql, List<Object> values) {
if (values == null || values.isEmpty() || !(values.getFirst() instanceof Instant candidate)) {
return values;
}
Instant uniqueTimestamp = lastTimestampByInsertSql.compute(insertSql, (key, lastTimestamp) -> {
if (lastTimestamp == null || candidate.isAfter(lastTimestamp)) {
return candidate;
}
return lastTimestamp.plusMillis(1);
});
if (candidate.equals(uniqueTimestamp)) {
return values;
}
}
private static boolean hasEmptyString(List<Object> values) {
for (Object value : values) {
if (value instanceof String stringValue && stringValue.isEmpty()) {
return true;
}
}
return false;
}
private void executeLiteralFallback(Connection connection,
LiteralBatch batch,
SQLException preparedFailure) throws SQLException {
if (batch == null || batch.rows().isEmpty()) {
throw preparedFailure;
}
if (!literalFallbackLogged) {
literalFallbackLogged = true;
log.warn("TDengine prepared batch failed; falling back to literal inserts: {}",
preparedFailure.getMessage());
log.debug("TDengine prepared batch failure stacktrace", preparedFailure);
}
executeLiteralBatch(connection, batch);
List<Object> copy = new java.util.ArrayList<>(values);
copy.set(0, uniqueTimestamp);
return copy;
}
private static void executeLiteralBatch(Connection connection, LiteralBatch batch) throws SQLException {