shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: keep jt808 raw history consumer current

Browse Source
This commit is contained in:
lingniu
2026-06-29 15:56:41 +08:00
parent 57ef82f563
commit 98b37e71e9
13 changed files with 274 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryWriter.java
View File

@@ -20,6 +20,7 @@ import org.slf4j.LoggerFactory;
public final class TdengineJdbcHistoryWriter implements TdengineHistoryWriter {
private static final Logger log = LoggerFactory.getLogger(TdengineJdbcHistoryWriter.class);
private static final int MAX_LITERAL_ROWS_PER_STATEMENT = 200;
private final DataSource dataSource;
private final TdengineHistorySchema schema;
@@ -179,26 +180,39 @@ public final class TdengineJdbcHistoryWriter implements TdengineHistoryWriter {
private static void executeLiteralBatch(Connection connection, LiteralBatch batch) throws SQLException {
try (Statement statement = connection.createStatement()) {
for (List<Object> row : batch.rows()) {
statement.execute(toLiteralInsertSql(batch.insertSql(), row));
List<List<Object>> rows = batch.rows();
for (int start = 0; start < rows.size(); start += MAX_LITERAL_ROWS_PER_STATEMENT) {
int end = Math.min(start + MAX_LITERAL_ROWS_PER_STATEMENT, rows.size());
statement.execute(toLiteralInsertSql(batch.insertSql(), rows.subList(start, end)));
}
}
}
private static String toLiteralInsertSql(String insertSql, List<Object> values) {
private static String toLiteralInsertSql(String insertSql, List<List<Object>> rows) {
int valuesIndex = insertSql.lastIndexOf("VALUES");
if (valuesIndex < 0) {
throw new IllegalArgumentException("insert SQL must contain VALUES: " + insertSql);
}
StringBuilder sql = new StringBuilder(insertSql.substring(0, valuesIndex))
.append("VALUES (");
for (int i = 0; i < values.size(); i++) {
if (i > 0) {
sql.append(", ");
}
sql.append(literal(values.get(i)));
if (rows == null || rows.isEmpty()) {
throw new IllegalArgumentException("literal rows must not be empty");
}
return sql.append(')').toString();
StringBuilder sql = new StringBuilder(insertSql.substring(0, valuesIndex))
.append("VALUES ");
for (int rowIndex = 0; rowIndex < rows.size(); rowIndex++) {
if (rowIndex > 0) {
sql.append(' ');
}
List<Object> values = rows.get(rowIndex);
sql.append('(');
for (int i = 0; i < values.size(); i++) {
if (i > 0) {
sql.append(", ");
}
sql.append(literal(values.get(i)));
}
sql.append(')');
}
return sql.toString();
}
private static String literal(Object value) {

19
modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryWriterTest.java
View File

@@ -156,6 +156,25 @@ class TdengineJdbcHistoryWriterTest {
assertThat(jdbc.commits).isEqualTo(1);
}
@Test
void groupsLiteralRowsForSameChildTableIntoOneInsert() throws Exception {
RecordingJdbc jdbc = new RecordingJdbc();
TdengineJdbcHistoryWriter writer = new TdengineJdbcHistoryWriter(jdbc.dataSource(), schema);
writer.appendRawFrames(List.of(
rawFrame("frame-1", Instant.parse("2026-06-29T05:00:01Z")),
rawFrame("frame-2", Instant.parse("2026-06-29T05:00:02Z"))));
List<String> inserts = jdbc.executedSql.stream()
.filter(sql -> sql.startsWith("INSERT INTO raw_jt808_"))
.toList();
assertThat(inserts).hasSize(1);
assertThat(inserts.getFirst())
.contains("frame-1")
.contains("frame-2")
.contains(") (");
}
private static TdengineRawFrameRow rawFrame(String frameId, Instant ts) {
return rawFrame(frameId, ts, "");
}