fix: harden jt808 tdengine ingestion

modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryWriter.java

@@ -3,7 +3,6 @@ package com.lingniu.ingest.tdenginehistory;
 import javax.sql.DataSource;
 import java.io.IOException;
 import java.sql.Connection;
-import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.sql.Timestamp;
@@ -13,21 +12,19 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Function;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 public final class TdengineJdbcHistoryWriter implements TdengineHistoryWriter {
 
-    private static final Logger log = LoggerFactory.getLogger(TdengineJdbcHistoryWriter.class);
     private static final int MAX_LITERAL_ROWS_PER_STATEMENT = 200;
 
     private final DataSource dataSource;
     private final TdengineHistorySchema schema;
     private final TdengineHistoryStatements statements;
     private final Object schemaInitializationMonitor = new Object();
+    private final Map<String, Instant> lastTimestampByInsertSql = new ConcurrentHashMap<>();
     private volatile boolean schemaInitialized;
-    private volatile boolean literalFallbackLogged;
 
     public TdengineJdbcHistoryWriter(DataSource dataSource, TdengineHistorySchema schema) {
         if (dataSource == null) {
@@ -95,87 +92,42 @@ public final class TdengineJdbcHistoryWriter implements TdengineHistoryWriter {
                 }
             }
             Set<String> createdChildTables = new LinkedHashSet<>();
-            Map<String, PreparedStatement> preparedStatements = new LinkedHashMap<>();
             Map<String, LiteralBatch> literalBatches = new LinkedHashMap<>();
-            Map<String, LiteralBatch> literalOnlyBatches = new LinkedHashMap<>();
-            try {
-                for (T row : rows) {
-                    if (row == null) {
-                        continue;
-                    }
-                    TdengineBatchStatement batch = mapper.apply(row);
-                    if (createdChildTables.add(batch.createChildTableSql())) {
-                        try (Statement statement = connection.createStatement()) {
-                            statement.execute(batch.createChildTableSql());
-                        }
-                    }
-                    if (hasEmptyString(batch.values())) {
-                        literalOnlyBatches.computeIfAbsent(batch.insertSql(), LiteralBatch::new).add(batch.values());
-                        continue;
-                    }
-                    PreparedStatement prepared = preparedStatements.computeIfAbsent(batch.insertSql(), sql -> {
-                        try {
-                            return connection.prepareStatement(sql);
-                        } catch (SQLException e) {
-                            throw new JdbcRuntimeException(e);
-                        }
-                    });
-                    bind(prepared, batch.values());
-                    prepared.addBatch();
-                    literalBatches.computeIfAbsent(batch.insertSql(), LiteralBatch::new).add(batch.values());
+            for (T row : rows) {
+                if (row == null) {
+                    continue;
                 }
-                for (Map.Entry<String, PreparedStatement> entry : preparedStatements.entrySet()) {
-                    try {
-                        entry.getValue().executeBatch();
-                    } catch (SQLException e) {
-                        executeLiteralFallback(connection, literalBatches.get(entry.getKey()), e);
+                TdengineBatchStatement batch = mapper.apply(row);
+                if (createdChildTables.add(batch.createChildTableSql())) {
+                    try (Statement statement = connection.createStatement()) {
+                        statement.execute(batch.createChildTableSql());
                     }
                 }
-                for (LiteralBatch batch : literalOnlyBatches.values()) {
-                    executeLiteralBatch(connection, batch);
-                }
-            } finally {
-                for (PreparedStatement prepared : preparedStatements.values()) {
-                    prepared.close();
-                }
+                literalBatches.computeIfAbsent(batch.insertSql(), LiteralBatch::new)
+                        .add(uniqueTimestampValues(batch.insertSql(), batch.values()));
+            }
+            for (LiteralBatch batch : literalBatches.values()) {
+                executeLiteralBatch(connection, batch);
             }
         });
     }
 
-    private static void bind(PreparedStatement prepared, List<Object> values) throws SQLException {
-        for (int i = 0; i < values.size(); i++) {
-            Object value = values.get(i);
-            int parameterIndex = i + 1;
-            if (value instanceof Instant instant) {
-                prepared.setTimestamp(parameterIndex, Timestamp.from(instant));
-            } else {
-                prepared.setObject(parameterIndex, value);
+    private List<Object> uniqueTimestampValues(String insertSql, List<Object> values) {
+        if (values == null || values.isEmpty() || !(values.getFirst() instanceof Instant candidate)) {
+            return values;
+        }
+        Instant uniqueTimestamp = lastTimestampByInsertSql.compute(insertSql, (key, lastTimestamp) -> {
+            if (lastTimestamp == null || candidate.isAfter(lastTimestamp)) {
+                return candidate;
             }
+            return lastTimestamp.plusMillis(1);
+        });
+        if (candidate.equals(uniqueTimestamp)) {
+            return values;
         }
-    }
-
-    private static boolean hasEmptyString(List<Object> values) {
-        for (Object value : values) {
-            if (value instanceof String stringValue && stringValue.isEmpty()) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private void executeLiteralFallback(Connection connection,
-                                        LiteralBatch batch,
-                                        SQLException preparedFailure) throws SQLException {
-        if (batch == null || batch.rows().isEmpty()) {
-            throw preparedFailure;
-        }
-        if (!literalFallbackLogged) {
-            literalFallbackLogged = true;
-            log.warn("TDengine prepared batch failed; falling back to literal inserts: {}",
-                    preparedFailure.getMessage());
-            log.debug("TDengine prepared batch failure stacktrace", preparedFailure);
-        }
-        executeLiteralBatch(connection, batch);
+        List<Object> copy = new java.util.ArrayList<>(values);
+        copy.set(0, uniqueTimestamp);
+        return copy;
     }
 
     private static void executeLiteralBatch(Connection connection, LiteralBatch batch) throws SQLException {

modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryWriterTest.java

@@ -46,13 +46,13 @@ class TdengineJdbcHistoryWriterTest {
         assertThat(jdbc.executedSql)
                 .filteredOn(sql -> sql.contains("USING raw_frames TAGS"))
                 .hasSize(1);
-        assertThat(jdbc.preparedBatches).hasSize(1);
-        RecordingJdbc.PreparedBatch batch = jdbc.preparedBatches.values().iterator().next();
-        assertThat(batch.sql()).contains("INSERT INTO raw_jt808_");
-        assertThat(batch.rows()).hasSize(2);
-        assertThat(batch.rows().getFirst().get(1)).isEqualTo(Timestamp.from(first.ts()));
-        assertThat(batch.rows().getFirst().get(2)).isEqualTo("frame-1");
-        assertThat(batch.rows().get(1).get(2)).isEqualTo("frame-2");
+        assertThat(jdbc.preparedBatches).isEmpty();
+        assertThat(jdbc.executedSql)
+                .anyMatch(sql -> sql.startsWith("INSERT INTO raw_jt808_")
+                        && sql.contains(Long.toString(first.ts().toEpochMilli()))
+                        && sql.contains("frame-1")
+                        && sql.contains("frame-2")
+                        && sql.contains(") ("));
         assertThat(jdbc.commits).isEqualTo(1);
     }
 
@@ -85,9 +85,12 @@ class TdengineJdbcHistoryWriterTest {
         assertThat(jdbc.executedSql).startsWith(schema.bootstrapSql().toArray(String[]::new));
         assertThat(jdbc.executedSql)
                 .anyMatch(sql -> sql.contains("USING vehicle_locations TAGS"));
-        RecordingJdbc.PreparedBatch batch = jdbc.preparedBatches.values().iterator().next();
-        assertThat(batch.sql()).contains("INSERT INTO loc_gb32960_");
-        assertThat(batch.rows().getFirst().get(12)).isNull();
+        assertThat(jdbc.preparedBatches).isEmpty();
+        assertThat(jdbc.executedSql)
+                .anyMatch(sql -> sql.startsWith("INSERT INTO loc_gb32960_")
+                        && sql.contains("fact-1")
+                        && sql.contains("NULL")
+                        && sql.contains("archive://gb32960/frame-1.bin"));
     }
 
     @Test
@@ -103,12 +106,13 @@ class TdengineJdbcHistoryWriterTest {
         assertThat(jdbc.executedSql)
                 .filteredOn(sql -> sql.contains("USING telemetry_fields TAGS"))
                 .hasSize(1);
-        RecordingJdbc.PreparedBatch batch = jdbc.preparedBatches.values().iterator().next();
-        assertThat(batch.sql()).contains("INSERT INTO tf_gb32960_");
-        assertThat(batch.rows()).hasSize(2);
-        assertThat(batch.rows().getFirst().get(2)).isEqualTo("evt-1#0");
-        assertThat(batch.rows().getFirst().get(7)).isEqualTo(123.4);
-        assertThat(batch.rows().get(1).get(6)).isEqualTo("124.5");
+        assertThat(jdbc.preparedBatches).isEmpty();
+        assertThat(jdbc.executedSql)
+                .anyMatch(sql -> sql.startsWith("INSERT INTO tf_gb32960_")
+                        && sql.contains("evt-1#0")
+                        && sql.contains("evt-2#0")
+                        && sql.contains("123.4")
+                        && sql.contains("'124.5'"));
     }
 
     @Test
@@ -126,14 +130,14 @@ class TdengineJdbcHistoryWriterTest {
     }
 
     @Test
-    void fallsBackToLiteralInsertWhenPreparedBatchFails() throws Exception {
+    void writesLiteralInsertWithoutPreparedBatch() throws Exception {
         RecordingJdbc jdbc = new RecordingJdbc();
         jdbc.failPreparedBatches = true;
         TdengineJdbcHistoryWriter writer = new TdengineJdbcHistoryWriter(jdbc.dataSource(), schema);
 
         writer.appendRawFrames(List.of(rawFrame("frame-1", Instant.parse("2026-06-29T05:00:01Z"), "none")));
 
-        assertThat(jdbc.preparedBatches).hasSize(1);
+        assertThat(jdbc.preparedBatches).isEmpty();
         assertThat(jdbc.executedSql)
                 .anyMatch(sql -> sql.startsWith("INSERT INTO raw_jt808_")
                         && sql.contains("parse_error")
@@ -175,6 +179,25 @@ class TdengineJdbcHistoryWriterTest {
                 .contains(") (");
     }
 
+    @Test
+    void bumpsDuplicateTimestampsForSameChildTable() throws Exception {
+        RecordingJdbc jdbc = new RecordingJdbc();
+        TdengineJdbcHistoryWriter writer = new TdengineJdbcHistoryWriter(jdbc.dataSource(), schema);
+        Instant ts = Instant.parse("2026-06-29T05:00:01Z");
+
+        writer.appendRawFrames(List.of(
+                rawFrame("frame-1", ts),
+                rawFrame("frame-2", ts)));
+
+        String insert = jdbc.executedSql.stream()
+                .filter(sql -> sql.startsWith("INSERT INTO raw_jt808_"))
+                .findFirst()
+                .orElseThrow();
+        assertThat(insert)
+                .contains("(" + ts.toEpochMilli() + ", 'frame-1'")
+                .contains("(" + (ts.toEpochMilli() + 1) + ", 'frame-2'");
+    }
+
     private static TdengineRawFrameRow rawFrame(String frameId, Instant ts) {
         return rawFrame(frameId, ts, "");
     }