shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: productionize raw history ingestion

Browse Source
This commit is contained in:
lingniu
2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStore.java
View File

@@ -36,11 +36,13 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
private static final TypeReference<Map<String, String>> STRING_MAP =
new TypeReference<>() {};
private static final String HEADER_VERSION = "event-records-v1";
private static final int VEHICLE_BUCKETS = 64;
private final Path root;
private final Path indexPath;
private final ZoneId partitionZone;
private final ObjectMapper objectMapper;
private final boolean writeParquetEnabled;
private volatile boolean indexInitialized;
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone) {
@@ -48,6 +50,13 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
}
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone, ObjectMapper objectMapper) {
this(root, partitionZone, objectMapper, true);
}
public DuckDbParquetEventFileStore(Path root,
ZoneId partitionZone,
ObjectMapper objectMapper,
boolean writeParquetEnabled) {
if (root == null) {
throw new IllegalArgumentException("root must not be null");
}
@@ -55,6 +64,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
this.indexPath = this.root.resolve("events.duckdb");
this.partitionZone = partitionZone == null ? ZoneId.of("Asia/Shanghai") : partitionZone;
this.objectMapper = objectMapper == null ? new ObjectMapper() : objectMapper;
this.writeParquetEnabled = writeParquetEnabled;
}
@Override
@@ -67,16 +77,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
Partition partition = partition(record);
byPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record);
}
for (Map.Entry<Partition, List<EventFileRecord>> entry : byPartition.entrySet()) {
writePartition(entry.getKey(), entry.getValue());
appendIndex(entry.getKey(), entry.getValue());
if (writeParquetEnabled) {
writePartitions(byPartition);
}
appendIndex(byPartition);
}
@Override
public List<EventFileRecord> query(EventFileQuery query) throws IOException {
// 有 VIN 时直接扫该车分区 parquet，避免先查总索引再和车辆文件 join。
if (query.vin() != null) {
// 有 VIN 且启用 Parquet 副本时，可直接扫分区文件；索引-only 模式走 DuckDB 主索引。
if (query.vin() != null && writeParquetEnabled) {
return queryParquet(query);
}
// 无 VIN 的管理类查询走 DuckDB 索引，代价更高但不影响高频单车查询路径。
@@ -130,6 +140,9 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
if (query.eventTimeTo() != null) {
predicates.add("event_time_ms <= " + query.eventTimeTo().toEpochMilli());
}
if (query.cursorEventTime() != null) {
predicates.add(cursorPredicateSql(query));
}
// 这里的 SQL 只拼接经过 escape 的内部值；外部 rawArchiveUri 查询使用 PreparedStatement。
String where = predicates.isEmpty() ? "" : "WHERE " + String.join(" AND ", predicates) + "\n";
String sql = """
@@ -182,6 +195,9 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
if (query.eventTimeTo() != null) {
where.append(" AND event_time_ms <= ?\n");
}
if (query.cursorEventTime() != null) {
where.append(cursorPredicatePrepared(query.order()));
}
String sql = """
SELECT event_id, protocol, event_type, vin,
event_time_ms, ingest_time_ms,
@@ -209,6 +225,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
if (query.eventTimeTo() != null) {
ps.setLong(index++, query.eventTimeTo().toEpochMilli());
}
if (query.cursorEventTime() != null) {
long eventTimeMs = query.cursorEventTime().toEpochMilli();
long ingestTimeMs = query.cursorIngestTime().toEpochMilli();
ps.setLong(index++, eventTimeMs);
ps.setLong(index++, eventTimeMs);
ps.setLong(index++, ingestTimeMs);
ps.setLong(index++, eventTimeMs);
ps.setLong(index++, ingestTimeMs);
ps.setString(index++, query.cursorEventId());
}
ps.setInt(index, query.limit());
try (ResultSet rs = ps.executeQuery()) {
List<EventFileRecord> result = new ArrayList<>();
@@ -222,16 +248,43 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
}
}
private void writePartition(Partition partition, List<EventFileRecord> records) throws IOException {
Path dir = partition.dir(root);
Files.createDirectories(dir);
Path partFile = dir.resolve("events.parquet");
Path tempFile = dir.resolve("events-" + UUID.randomUUID() + ".tmp.parquet");
private static String cursorPredicateSql(EventFileQuery query) {
long eventTimeMs = query.cursorEventTime().toEpochMilli();
long ingestTimeMs = query.cursorIngestTime().toEpochMilli();
String eventId = escapeSql(query.cursorEventId());
String cmp = query.order() == EventFileQuery.Order.DESC ? "<" : ">";
return """
(
event_time_ms %s %d
OR (event_time_ms = %d AND ingest_time_ms %s %d)
OR (event_time_ms = %d AND ingest_time_ms = %d AND event_id %s '%s')
)
""".formatted(cmp, eventTimeMs, eventTimeMs, cmp, ingestTimeMs,
eventTimeMs, ingestTimeMs, cmp, eventId);
}
private static String cursorPredicatePrepared(EventFileQuery.Order order) {
String cmp = order == EventFileQuery.Order.DESC ? "<" : ">";
return """
AND (
event_time_ms %s ?
OR (event_time_ms = ? AND ingest_time_ms %s ?)
OR (event_time_ms = ? AND ingest_time_ms = ? AND event_id %s ?)
)
""".formatted(cmp, cmp, cmp);
}
private void writePartitions(Map<Partition, List<EventFileRecord>> recordsByPartition) throws IOException {
if (recordsByPartition.isEmpty()) {
return;
}
List<Path> tempFiles = new ArrayList<>();
try (Connection connection = DriverManager.getConnection("jdbc:duckdb:");
Statement statement = connection.createStatement()) {
statement.execute("""
CREATE TEMPORARY TABLE event_records (
partition_id INTEGER,
event_id VARCHAR,
protocol VARCHAR,
event_type VARCHAR,
@@ -246,54 +299,62 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
)
""");
try (PreparedStatement ps = connection.prepareStatement("""
INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""")) {
for (EventFileRecord record : records) {
ps.setString(1, record.eventId());
ps.setString(2, record.protocol().name());
ps.setString(3, record.eventType());
ps.setString(4, record.vin());
ps.setLong(5, record.eventTime().toEpochMilli());
ps.setString(6, record.eventTime().toString());
ps.setLong(7, record.ingestTime().toEpochMilli());
ps.setString(8, record.ingestTime().toString());
ps.setString(9, record.rawArchiveUri());
ps.setString(10, objectMapper.writeValueAsString(record.metadata()));
ps.setString(11, record.payloadJson());
ps.addBatch();
int partitionId = 0;
for (List<EventFileRecord> records : recordsByPartition.values()) {
for (EventFileRecord record : records) {
ps.setInt(1, partitionId);
ps.setString(2, record.eventId());
ps.setString(3, record.protocol().name());
ps.setString(4, record.eventType());
ps.setString(5, record.vin());
ps.setLong(6, record.eventTime().toEpochMilli());
ps.setString(7, record.eventTime().toString());
ps.setLong(8, record.ingestTime().toEpochMilli());
ps.setString(9, record.ingestTime().toString());
ps.setString(10, record.rawArchiveUri());
ps.setString(11, objectMapper.writeValueAsString(record.metadata()));
ps.setString(12, record.payloadJson());
ps.addBatch();
}
partitionId++;
}
ps.executeBatch();
}
if (Files.exists(partFile)) {
// DuckDB 不能原地追加 parquet；先 UNION 旧文件和本批数据，再原子替换分区文件。
int partitionId = 0;
for (Partition partition : recordsByPartition.keySet()) {
Path dir = partition.dir(root);
Files.createDirectories(dir);
String fragmentId = UUID.randomUUID().toString();
Path partFile = dir.resolve("events-" + fragmentId + ".parquet");
Path tempFile = dir.resolve("events-" + fragmentId + ".tmp.parquet");
tempFiles.add(tempFile);
statement.execute("""
CREATE TEMPORARY TABLE combined_records AS
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM read_parquet('%s')
UNION ALL
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM event_records
""".formatted(escapeSql(partFile)));
statement.execute("COPY combined_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
} else {
statement.execute("COPY event_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
COPY (
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM event_records
WHERE partition_id = %d
) TO '%s' (FORMAT parquet)
""".formatted(partitionId, escapeSql(tempFile)));
Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
partitionId++;
}
Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
} catch (SQLException e) {
throw new IOException("write parquet event store failed: " + partFile, e);
throw new IOException("write parquet event store failed", e);
} finally {
Files.deleteIfExists(tempFile);
for (Path tempFile : tempFiles) {
Files.deleteIfExists(tempFile);
}
}
}
private void appendIndex(Partition partition, List<EventFileRecord> records) throws IOException {
private void appendIndex(Map<Partition, List<EventFileRecord>> recordsByPartition) throws IOException {
ensureIndexInitialized();
try (Connection connection = DriverManager.getConnection(indexJdbcUrl())) {
insertIndex(connection, partition, records);
insertIndex(connection, recordsByPartition);
} catch (SQLException e) {
throw new IOException("write duckdb event index failed", e);
}
@@ -364,23 +425,26 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
""".formatted(fileList));
}
private void insertIndex(Connection connection, Partition partition, List<EventFileRecord> records)
private void insertIndex(Connection connection, Map<Partition, List<EventFileRecord>> recordsByPartition)
throws SQLException, IOException {
try (PreparedStatement ps = connection.prepareStatement("""
INSERT OR REPLACE INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""")) {
for (EventFileRecord record : records) {
ps.setString(1, record.eventId());
ps.setString(2, record.protocol().name());
ps.setString(3, record.eventType());
ps.setString(4, record.vin());
ps.setLong(5, record.eventTime().toEpochMilli());
ps.setLong(6, record.ingestTime().toEpochMilli());
ps.setString(7, partition.date().toString());
ps.setString(8, record.rawArchiveUri());
ps.setString(9, objectMapper.writeValueAsString(record.metadata()));
ps.setString(10, record.payloadJson());
ps.addBatch();
for (Map.Entry<Partition, List<EventFileRecord>> entry : recordsByPartition.entrySet()) {
Partition partition = entry.getKey();
for (EventFileRecord record : entry.getValue()) {
ps.setString(1, record.eventId());
ps.setString(2, record.protocol().name());
ps.setString(3, record.eventType());
ps.setString(4, record.vin());
ps.setLong(5, record.eventTime().toEpochMilli());
ps.setLong(6, record.ingestTime().toEpochMilli());
ps.setString(7, partition.date().toString());
ps.setString(8, record.rawArchiveUri());
ps.setString(9, objectMapper.writeValueAsString(record.metadata()));
ps.setString(10, record.payloadJson());
ps.addBatch();
}
}
ps.executeBatch();
}
@@ -390,15 +454,22 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
List<Path> files = new ArrayList<>();
LocalDate date = query.dateFrom();
while (!date.isAfter(query.dateTo())) {
Path dir = query.vin() == null
? partitionDir(query.protocol(), date)
: partitionDir(query.protocol(), date).resolve("vehicle=" + storageName(query.vin()));
// 目录结构把 VIN 放在 date 下，单车多日查询只需要遍历目标日期内的目标车辆目录。
if (Files.isDirectory(dir)) {
try (Stream<Path> stream = Files.walk(dir)) {
stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
.sorted()
.forEach(files::add);
Path dateDir = partitionDir(query.protocol(), date);
List<Path> candidateDirs = query.vin() == null
? List.of(dateDir)
: List.of(
// 新布局：高写入路径按协议/日期写一个批次文件，VIN 由 Parquet/索引字段过滤。
dateDir.resolve("header=" + HEADER_VERSION),
// 兼容已落盘的旧布局。
dateDir.resolve("bucket=" + bucketName(query.vin())),
dateDir.resolve("vehicle=" + storageName(query.vin())));
for (Path dir : candidateDirs) {
if (Files.isDirectory(dir)) {
try (Stream<Path> stream = Files.walk(dir)) {
stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
.sorted()
.forEach(files::add);
}
}
}
date = date.plusDays(1);
@@ -434,8 +505,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
private Partition partition(EventFileRecord record) {
return new Partition(
record.protocol(),
LocalDate.ofInstant(record.eventTime(), partitionZone),
storageName(record.vin()));
LocalDate.ofInstant(record.eventTime(), partitionZone));
}
private Path partitionDir(ProtocolId protocol, LocalDate date) {
@@ -488,11 +558,18 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
return vin.replaceAll("[^A-Za-z0-9._-]", "_");
}
private record Partition(ProtocolId protocol, LocalDate date, String vehicle) {
private static String bucketName(String vin) {
String storageName = storageName(vin);
if ("_unknown".equals(storageName)) {
return "_unknown";
}
return "%02d".formatted(Math.floorMod(storageName.hashCode(), VEHICLE_BUCKETS));
}
private record Partition(ProtocolId protocol, LocalDate date) {
private Path dir(Path root) {
return root.resolve("protocol=" + protocol.name())
.resolve("date=" + date)
.resolve("vehicle=" + vehicle)
.resolve("header=" + HEADER_VERSION);
}
}

23
modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/EventFileQuery.java
View File

@@ -20,7 +20,10 @@ public record EventFileQuery(
Order order,
int limit,
String vin,
String eventType
String eventType,
Instant cursorEventTime,
Instant cursorIngestTime,
String cursorEventId
) {
public enum Order {
ASC,
@@ -45,6 +48,24 @@ public record EventFileQuery(
limit = limit <= 0 ? 100 : limit;
vin = vin == null || vin.isBlank() ? null : vin.trim();
eventType = eventType == null || eventType.isBlank() ? null : eventType.trim();
cursorEventId = cursorEventId == null || cursorEventId.isBlank() ? null : cursorEventId.trim();
if ((cursorEventTime == null || cursorIngestTime == null || cursorEventId == null)
&& !(cursorEventTime == null && cursorIngestTime == null && cursorEventId == null)) {
throw new IllegalArgumentException("cursor eventTime, ingestTime and eventId must be provided together");
}
}
public EventFileQuery(ProtocolId protocol,
LocalDate dateFrom,
LocalDate dateTo,
Instant eventTimeFrom,
Instant eventTimeTo,
Order order,
int limit,
String vin,
String eventType) {
this(protocol, dateFrom, dateTo, eventTimeFrom, eventTimeTo, order, limit,
vin, eventType, null, null, null);
}
public EventFileQuery(ProtocolId protocol,

14
modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/EventFileStoreSink.java
View File

@@ -226,9 +226,9 @@ public final class EventFileStoreSink implements EventSink, AutoCloseable {
return payload;
}
private static Map<String, Object> rawArchivePayload(VehicleEvent.RawArchive raw,
String rawArchiveKey,
String rawArchiveUri) {
private Map<String, Object> rawArchivePayload(VehicleEvent.RawArchive raw,
String rawArchiveKey,
String rawArchiveUri) throws IOException {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("eventId", raw.eventId());
payload.put("vin", raw.vin());
@@ -241,10 +241,18 @@ public final class EventFileStoreSink implements EventSink, AutoCloseable {
payload.put("command", hex(raw.command()));
payload.put("infoType", hex(raw.infoType()));
payload.put("rawSizeBytes", raw.rawBytes() == null ? 0 : raw.rawBytes().length);
putParsedJson(payload, raw.parsedJson());
payload.put("metadata", raw.metadata());
return payload;
}
private void putParsedJson(Map<String, Object> payload, String parsedJson) throws IOException {
if (parsedJson == null || parsedJson.isBlank()) {
return;
}
payload.put("parsed", objectMapper.readTree(parsedJson));
}
private static String hex(int value) {
return "0x" + String.format("%04X", value);
}

3
modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfiguration.java
View File

@@ -44,7 +44,8 @@ public class EventFileStoreAutoConfiguration {
return new DuckDbParquetEventFileStore(
Path.of(properties.getPath()),
ZoneId.of(properties.getZoneId()),
mapper);
mapper,
properties.isWriteParquetEnabled());
}
@Bean

22
modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreProperties.java
View File

@@ -11,10 +11,10 @@ public class EventFileStoreProperties {
private boolean enabled = false;
/**
* Parquet 文件库根路径。
* 历史库根路径。
*
* <p>实际结构为 {@code protocol=GB32960/date=yyyy-MM-dd/vehicle=VIN/header=event-records-v1/events.parquet}，
* 另有 {@code events.duckdb} sidecar index 放在根路径下。
* <p>{@code events.duckdb} 是可查询主索引；开启 Parquet 副本时，还会在该根目录下按
* {@code protocol/date/header} 保存追加文件。
*/
private String path = "./event-store";
@@ -39,6 +39,14 @@ public class EventFileStoreProperties {
*/
private long flushIntervalMillis = 1000;
/**
* 是否同步写 Parquet 副本。
*
* <p>高写入生产链路可以关闭，只保留 DuckDB JSON 主索引和 raw archive 引用；需要离线副本或
* 索引重建能力时再开启。
*/
private boolean writeParquetEnabled = true;
public boolean isEnabled() {
return enabled;
}
@@ -78,4 +86,12 @@ public class EventFileStoreProperties {
public void setFlushIntervalMillis(long flushIntervalMillis) {
this.flushIntervalMillis = flushIntervalMillis;
}
public boolean isWriteParquetEnabled() {
return writeParquetEnabled;
}
public void setWriteParquetEnabled(boolean writeParquetEnabled) {
this.writeParquetEnabled = writeParquetEnabled;
}
}

89
modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStoreTest.java
View File

@@ -1,6 +1,7 @@
package com.lingniu.ingest.eventfilestore;
import com.lingniu.ingest.api.ProtocolId;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -11,6 +12,7 @@ import java.time.LocalDate;
import java.time.ZoneId;
import java.util.List;
import java.util.Map;
import java.util.stream.IntStream;
import static org.assertj.core.api.Assertions.assertThat;
@@ -28,7 +30,7 @@ class DuckDbParquetEventFileStoreTest {
store.appendAll(List.of(first, second));
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
assertThat(parquetFiles(dateDir)).hasSize(2);
assertThat(parquetFiles(dateDir)).hasSize(1);
EventFileQuery ascending = new EventFileQuery(
ProtocolId.GB32960,
@@ -135,16 +137,13 @@ class DuckDbParquetEventFileStoreTest {
}
@Test
void appendsSameVehicleDayHeaderToOneParquetFile() throws Exception {
void appendsSameVehicleDayHeaderAsAppendOnlyParquetFragments() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(record("first", Instant.parse("2026-06-21T16:00:03Z"), "VIN-SAME"));
store.append(record("second", Instant.parse("2026-06-21T16:00:04Z"), "VIN-SAME"));
Path vehicleDayDir = tempDir.resolve("protocol=GB32960")
.resolve("date=2026-06-22")
.resolve("vehicle=VIN-SAME")
.resolve("header=event-records-v1");
assertThat(parquetFiles(vehicleDayDir)).containsExactly(vehicleDayDir.resolve("events.parquet"));
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
assertThat(parquetFiles(dateDir)).hasSize(2);
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
@@ -159,19 +158,54 @@ class DuckDbParquetEventFileStoreTest {
.containsExactly("first", "second");
}
@Test
void writesManyVehiclesIntoBoundedDailyLayoutInsteadOfVehicleDirectories() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
List<EventFileRecord> records = IntStream.range(0, 80)
.mapToObj(index -> rawRecord(
"raw-" + index,
Instant.parse("2026-06-21T16:00:00Z").plusMillis(index),
"VIN-" + index))
.toList();
store.appendAll(records);
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
try (var files = Files.list(dateDir)) {
List<String> partitionDirs = files
.filter(Files::isDirectory)
.map(path -> path.getFileName().toString())
.sorted()
.toList();
assertThat(partitionDirs).noneMatch(name -> name.startsWith("vehicle="));
assertThat(partitionDirs).hasSizeLessThan(records.size());
}
}
@Test
void writesOneDailyParquetFragmentForOneBatchAcrossManyVehicles() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
List<EventFileRecord> records = IntStream.range(0, 80)
.mapToObj(index -> rawRecord(
"daily-raw-" + index,
Instant.parse("2026-06-21T16:00:00Z").plusMillis(index),
"VIN-DAILY-" + index))
.toList();
store.appendAll(records);
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
assertThat(parquetFiles(dateDir)).hasSize(1);
}
@Test
void queriesUseSidecarIndexInsteadOfScanningParquetPartsEveryTime() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(record("indexed", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEXED"));
Path vehicleDayDir = tempDir.resolve("protocol=GB32960")
.resolve("date=2026-06-22")
.resolve("vehicle=VIN-INDEXED")
.resolve("header=event-records-v1");
try (var files = Files.list(vehicleDayDir)) {
for (Path file : files.filter(p -> p.getFileName().toString().endsWith(".parquet")).toList()) {
Files.delete(file);
}
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
for (Path file : parquetFiles(dateDir)) {
Files.delete(file);
}
EventFileQuery query = new EventFileQuery(
@@ -219,6 +253,31 @@ class DuckDbParquetEventFileStoreTest {
assertThat(found.rawArchiveUri()).isEqualTo(record.rawArchiveUri());
}
@Test
void canQueryVinRecordsFromIndexWhenParquetWritingIsDisabled() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(
tempDir,
ZoneId.of("Asia/Shanghai"),
new ObjectMapper(),
false);
EventFileRecord record = rawRecord("index-only", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEX-ONLY");
store.append(record);
assertThat(parquetFiles(tempDir)).isEmpty();
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.DESC,
10,
"VIN-INDEX-ONLY",
"RAW_ARCHIVE");
assertThat(store.query(query))
.extracting(EventFileRecord::eventId)
.containsExactly("index-only");
}
private static List<Path> parquetFiles(Path root) throws Exception {
try (var files = Files.walk(root)) {
return files.filter(p -> p.getFileName().toString().endsWith(".parquet"))

5
modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfigurationTest.java
View File

@@ -20,12 +20,15 @@ class EventFileStoreAutoConfigurationTest {
contextRunner
.withPropertyValues(
"lingniu.ingest.event-file-store.enabled=true",
"lingniu.ingest.event-file-store.path=target/test-event-store")
"lingniu.ingest.event-file-store.path=target/test-event-store",
"lingniu.ingest.event-file-store.write-parquet-enabled=false")
.run(context -> {
assertThat(context).hasSingleBean(EventFileStore.class);
assertThat(context).hasSingleBean(EventFileStoreSink.class);
assertThat(context.getBean(EventFileStoreProperties.class).getPath())
.isEqualTo(Path.of("target/test-event-store").toString());
assertThat(context.getBean(EventFileStoreProperties.class).isWriteParquetEnabled())
.isFalse();
});
}

1
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/EnvelopeMapper.java
View File

@@ -92,6 +92,7 @@ public final class EnvelopeMapper {
.setUri(uri)
.setChecksum(checksum)
.setSizeBytes(size)
.setParsedJson(nullToEmpty(ra.parsedJson()))
.build());
b.setRawFrameFact(com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload.newBuilder()
.setFrameId(frameId)

48
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerRunner.java
View File

@@ -10,17 +10,19 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;
public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCloseable {
private static final Logger log = LoggerFactory.getLogger(KafkaEnvelopeConsumerRunner.class);
private final List<KafkaEnvelopeConsumerWorker> workers;
private final Supplier<List<KafkaEnvelopeConsumerWorker>> workersSupplier;
private final Duration pollTimeout;
private final Duration loopBackoff;
private final boolean autoStartup;
private final AtomicBoolean running = new AtomicBoolean(false);
private final AtomicBoolean closed = new AtomicBoolean(false);
private volatile List<KafkaEnvelopeConsumerWorker> workers;
private ExecutorService executor;
public KafkaEnvelopeConsumerRunner(List<KafkaEnvelopeConsumerWorker> workers,
@@ -30,14 +32,28 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
if (workers == null || workers.isEmpty()) {
throw new IllegalArgumentException("workers must not be empty");
}
this.workersSupplier = () -> List.copyOf(workers);
this.workers = List.copyOf(workers);
this.pollTimeout = pollTimeout == null ? Duration.ofSeconds(1) : pollTimeout;
this.loopBackoff = loopBackoff == null ? Duration.ofSeconds(1) : loopBackoff;
this.autoStartup = autoStartup;
}
public KafkaEnvelopeConsumerRunner(Supplier<List<KafkaEnvelopeConsumerWorker>> workersSupplier,
Duration pollTimeout,
Duration loopBackoff,
boolean autoStartup) {
if (workersSupplier == null) {
throw new IllegalArgumentException("workersSupplier must not be null");
}
this.workersSupplier = workersSupplier;
this.pollTimeout = pollTimeout == null ? Duration.ofSeconds(1) : pollTimeout;
this.loopBackoff = loopBackoff == null ? Duration.ofSeconds(1) : loopBackoff;
this.autoStartup = autoStartup;
}
public List<KafkaEnvelopeConsumerWorker> workers() {
return workers;
return workersOrCreate();
}
@Override
@@ -45,17 +61,35 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
if (!running.compareAndSet(false, true)) {
return;
}
List<KafkaEnvelopeConsumerWorker> activeWorkers = workersOrCreate();
// 每个 worker 一条后台线程，避免某个处理器阻塞时拖慢其他消费组。
executor = Executors.newFixedThreadPool(workers.size(), r -> {
executor = Executors.newFixedThreadPool(activeWorkers.size(), r -> {
Thread thread = new Thread(r, "kafka-envelope-consumer");
thread.setDaemon(true);
return thread;
});
for (KafkaEnvelopeConsumerWorker worker : workers) {
for (KafkaEnvelopeConsumerWorker worker : activeWorkers) {
executor.submit(() -> pollLoop(worker));
}
}
private List<KafkaEnvelopeConsumerWorker> workersOrCreate() {
List<KafkaEnvelopeConsumerWorker> current = workers;
if (current != null) {
return current;
}
synchronized (this) {
if (workers == null) {
List<KafkaEnvelopeConsumerWorker> created = workersSupplier.get();
if (created == null || created.isEmpty()) {
throw new IllegalStateException("no kafka envelope consumer workers created; check consumer bindings");
}
workers = List.copyOf(created);
}
return workers;
}
}
private void pollLoop(KafkaEnvelopeConsumerWorker worker) {
while (running.get()) {
try {
@@ -121,7 +155,11 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
if (!closed.compareAndSet(false, true)) {
return;
}
for (KafkaEnvelopeConsumerWorker worker : workers) {
List<KafkaEnvelopeConsumerWorker> current = workers;
if (current == null) {
return;
}
for (KafkaEnvelopeConsumerWorker worker : current) {
worker.close();
}
}

14
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerWorker.java
View File

@@ -36,26 +36,26 @@ public final class KafkaEnvelopeConsumerWorker implements AutoCloseable {
public int pollOnce(Duration timeout) {
ConsumerRecords<String, byte[]> records = consumer.poll(timeout == null ? Duration.ZERO : timeout);
Map<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> batches = new LinkedHashMap<>();
Map<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> byProcessor = new LinkedHashMap<>();
int processed = 0;
for (ConsumerRecord<String, byte[]> record : records) {
EnvelopeConsumerProcessor processor = processorsByTopic.get(record.topic());
if (processor == null) {
// worker 可能订阅多个 topic；没有显式绑定处理器的 topic 不参与提交语义。
continue;
}
batches.computeIfAbsent(processor, ignored -> new ArrayList<>()).add(new EnvelopeConsumerRecord(
byProcessor.computeIfAbsent(processor, ignored -> new ArrayList<>()).add(new EnvelopeConsumerRecord(
record.topic(),
record.partition(),
record.offset(),
record.key(),
record.value()));
processed++;
}
int processed = 0;
for (Map.Entry<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> entry : batches.entrySet()) {
for (Map.Entry<EnvelopeConsumerProcessor, List<EnvelopeConsumerRecord>> entry : byProcessor.entrySet()) {
// EnvelopeConsumerProcessor 内部会把解析或业务错误转成 DLQ 记录，
// 这里保持 Kafka worker 的职责单一：轮询、批量分发、成功后提交 offset。
entry.getKey().processAll(entry.getValue());
processed += entry.getValue().size();
// 这里保持 Kafka worker 的职责单一：轮询、分发、成功后提交 offset。
entry.getKey().processBatch(entry.getValue());
}
if (processed > 0) {
// commitSync 放在批次末尾，保证同一个 poll 批次内的消息按 Kafka offset 一起确认。

21
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqAutoConfiguration.java
View File

@@ -1,6 +1,5 @@
package com.lingniu.ingest.sink.mq;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import io.github.resilience4j.circuitbreaker.CircuitBreaker;
import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
import org.apache.kafka.clients.producer.KafkaProducer;
@@ -8,15 +7,12 @@ import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.ByteArraySerializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
@@ -100,21 +96,4 @@ public class SinkMqAutoConfiguration {
return new KafkaEnvelopeDeadLetterSink(producer, props.getTopics().getDlq());
}
@Bean
@ConditionalOnMissingBean
@ConditionalOnBean(EnvelopeConsumerProcessor.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(Map<String, EnvelopeConsumerProcessor> processors,
SinkMqProperties props) {
// Consumer runner 根据 processor bean 名和配置 binding 生成 worker；没有 binding 时直接失败，避免静默不消费。
List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(processors, props);
if (workers.isEmpty()) {
throw new IllegalStateException("no kafka envelope consumer workers created; check consumer bindings");
}
return new KafkaEnvelopeConsumerRunner(
workers,
Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()),
Duration.ofMillis(props.getConsumer().getLoopBackoffMillis()),
props.getConsumer().isAutoStartup());
}
}

42
modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqConsumerAutoConfiguration.java Normal file
View File

@@ -0,0 +1,42 @@
package com.lingniu.ingest.sink.mq;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import org.springframework.beans.factory.ListableBeanFactory;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import java.time.Duration;
import java.util.List;
import java.util.Map;
@AutoConfiguration(after = SinkMqAutoConfiguration.class)
@AutoConfigureAfter(name = {
"com.lingniu.ingest.eventhistory.config.EventHistoryAutoConfiguration",
"com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration",
"com.lingniu.ingest.vehiclestat.config.VehicleStatAutoConfiguration"
})
@EnableConfigurationProperties(SinkMqProperties.class)
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq", name = "enabled", havingValue = "true", matchIfMissing = true)
public class SinkMqConsumerAutoConfiguration {
@Bean
@ConditionalOnMissingBean
@ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(ListableBeanFactory beanFactory,
SinkMqProperties props) {
return new KafkaEnvelopeConsumerRunner(
() -> createWorkers(beanFactory, props),
Duration.ofMillis(props.getConsumer().getPollTimeoutMillis()),
Duration.ofMillis(props.getConsumer().getLoopBackoffMillis()),
props.getConsumer().isAutoStartup());
}
private List<KafkaEnvelopeConsumerWorker> createWorkers(ListableBeanFactory beanFactory, SinkMqProperties props) {
Map<String, EnvelopeConsumerProcessor> processors = beanFactory.getBeansOfType(EnvelopeConsumerProcessor.class);
return new KafkaEnvelopeConsumerFactory().createWorkers(processors, props);
}
}

1
modules/sinks/sink-mq/src/main/proto/vehicle_envelope.proto
View File

@@ -45,6 +45,7 @@ message RawArchiveRef {
string uri = 1;
string checksum = 2;
int64 size_bytes = 3;
string parsed_json = 4;
}
message TelemetrySnapshot {

1
modules/sinks/sink-mq/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports
View File

@@ -1 +1,2 @@
com.lingniu.ingest.sink.mq.SinkMqAutoConfiguration
com.lingniu.ingest.sink.mq.SinkMqConsumerAutoConfiguration

37
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerWorkerTest.java
View File

@@ -1,6 +1,7 @@
package com.lingniu.ingest.sink.mq;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerRecord;
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterRecord;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
@@ -14,6 +15,7 @@ import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.atomic.AtomicInteger;
import static org.assertj.core.api.Assertions.assertThat;
@@ -44,4 +46,39 @@ class KafkaEnvelopeConsumerWorkerTest {
assertThat(record.payload()).containsExactly(0x01, 0x02);
});
}
@Test
void pollsKafkaRecordsAndDispatchesBatchWhenProcessorSupportsIt() {
MockConsumer<String, byte[]> consumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST);
TopicPartition partition = new TopicPartition("vehicle.raw", 0);
consumer.assign(List.of(partition));
consumer.updateBeginningOffsets(Map.of(partition, 0L));
consumer.addRecord(new ConsumerRecord<>("vehicle.raw", 0, 12L, "VIN001", new byte[]{0x01}));
consumer.addRecord(new ConsumerRecord<>("vehicle.raw", 0, 13L, "VIN002", new byte[]{0x02}));
AtomicInteger batchCalls = new AtomicInteger();
EnvelopeConsumerProcessor processor = new EnvelopeConsumerProcessor(
"event-history",
new EnvelopeBatchIngestor() {
@Override
public EnvelopeIngestResult tryIngest(byte[] kafkaValue) {
throw new AssertionError("single-record ingest should not be used for a batch-capable processor");
}
@Override
public List<EnvelopeIngestResult> tryIngestAll(List<byte[]> kafkaValues) {
batchCalls.incrementAndGet();
assertThat(kafkaValues).hasSize(2);
return List.of(
EnvelopeIngestResult.stored("event-1", "VIN001"),
EnvelopeIngestResult.invalid("bad envelope"));
}
},
record -> {});
int processed = new KafkaEnvelopeConsumerWorker(
consumer, Map.of("vehicle.raw", processor)).pollOnce(Duration.ZERO);
assertThat(processed).isEqualTo(2);
assertThat(batchCalls).hasValue(1);
}
}

2
modules/sinks/sink-mq/src/test/java/com/lingniu/ingest/sink/mq/SinkMqConsumerAutoConfigurationTest.java
View File

@@ -10,7 +10,7 @@ import static org.assertj.core.api.Assertions.assertThat;
class SinkMqConsumerAutoConfigurationTest {
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withUserConfiguration(SinkMqAutoConfiguration.class)
.withUserConfiguration(SinkMqAutoConfiguration.class, SinkMqConsumerAutoConfiguration.class)
.withBean("vehicleStateEnvelopeConsumerProcessor", EnvelopeConsumerProcessor.class,
() -> new EnvelopeConsumerProcessor(
"vehicle-state",

9
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineEnvelopeRows.java
View File

@@ -43,6 +43,7 @@ public final class TdengineEnvelopeRows {
raw.getParseError(),
raw.getPeer(),
json(metadata),
parsedJson(envelope, metadata),
protocol(envelope),
vehicleKey(envelope, raw.getVehicleKey(), raw.getPhone()),
firstNonBlank(raw.getVin(), envelope.getVin()),
@@ -73,7 +74,6 @@ public final class TdengineEnvelopeRows {
location.getStatusFlag(),
totalMileage(envelope),
rawUri,
json(envelope.getMetadataMap()),
protocol(envelope),
vehicleKey(envelope, envelope.getMetadataOrDefault("vehicle_key", ""),
envelope.getMetadataOrDefault("phone", "")),
@@ -188,6 +188,13 @@ public final class TdengineEnvelopeRows {
return Double.parseDouble(value);
}
private static String parsedJson(VehicleEnvelope envelope, Map<String, String> metadata) {
if (envelope.hasRawArchive() && !envelope.getRawArchive().getParsedJson().isBlank()) {
return envelope.getRawArchive().getParsedJson();
}
return firstNonBlank(metadata.get("parsedJson"), metadata.get("parsed_json"));
}
private static Double valueDouble(TelemetryField field) {
String valueType = field.getValueType();
if (!"DOUBLE".equals(valueType) && !"FLOAT".equals(valueType)) {

4
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryQueries.java
View File

@@ -6,11 +6,11 @@ import java.util.List;
public final class TdengineHistoryQueries {
private static final String RAW_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, "
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json, "
+ "protocol, vehicle_key, vin, phone";
private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, "
+ "metadata_json, protocol, vehicle_key, vin, phone";
+ "protocol, vehicle_key, vin, phone";
private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, field_key, "
+ "value_type, value_text, value_double, value_long, unit, quality, source_path, raw_uri, "
+ "metadata_json, protocol, vehicle_key, vin, phone";

6
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistorySchema.java
View File

@@ -64,7 +64,8 @@ public final class TdengineHistorySchema {
parse_status NCHAR(16),
parse_error NCHAR(512),
peer NCHAR(128),
metadata_json NCHAR(4096)
metadata_json NCHAR(4096),
parsed_json NCHAR(16374)
) TAGS (
protocol NCHAR(16),
vehicle_key NCHAR(128),
@@ -88,8 +89,7 @@ public final class TdengineHistorySchema {
alarm_flag BIGINT,
status_flag BIGINT,
total_mileage_km DOUBLE,
raw_uri NCHAR(512),
metadata_json NCHAR(4096)
raw_uri NCHAR(512)
) TAGS (
protocol NCHAR(16),
vehicle_key NCHAR(128),

12
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatements.java
View File

@@ -6,12 +6,12 @@ import java.util.Arrays;
public final class TdengineHistoryStatements {
private static final String RAW_FRAME_COLUMNS = "ts, frame_id, received_at, message_id, sub_type, event_time, "
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json";
private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
+ "raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json";
private static final String RAW_FRAME_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
private static final String LOCATION_COLUMNS = "ts, fact_id, frame_id, received_at, longitude, latitude, "
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri, metadata_json";
private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
+ "altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, raw_uri";
private static final String LOCATION_PLACEHOLDERS = "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?";
private static final String TELEMETRY_FIELD_COLUMNS = "ts, fact_id, frame_id, received_at, value_type, "
+ "value_text, value_double, value_long, unit, quality, source_path, raw_uri, metadata_json";
@@ -35,7 +35,7 @@ public final class TdengineHistoryStatements {
values(
row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(),
row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(),
row.parseError(), row.peer(), row.metadataJson()
row.parseError(), row.peer(), row.metadataJson(), row.parsedJson()
)
);
}
@@ -49,7 +49,7 @@ public final class TdengineHistoryStatements {
values(
row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(),
row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(),
row.totalMileageKm(), row.rawUri(), row.metadataJson()
row.totalMileageKm(), row.rawUri()
)
);
}

2
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryReader.java
View File

@@ -157,6 +157,7 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
rs.getString("parse_error"),
rs.getString("peer"),
rs.getString("metadata_json"),
rs.getString("parsed_json"),
rs.getString("protocol"),
rs.getString("vehicle_key"),
rs.getString("vin"),
@@ -179,7 +180,6 @@ public final class TdengineJdbcHistoryReader implements TdengineHistoryReader {
rs.getLong("status_flag"),
nullableDouble(rs, "total_mileage_km"),
rs.getString("raw_uri"),
rs.getString("metadata_json"),
rs.getString("protocol"),
rs.getString("vehicle_key"),
rs.getString("vin"),

1
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineLocationRow.java
View File

@@ -16,7 +16,6 @@ public record TdengineLocationRow(
long statusFlag,
Double totalMileageKm,
String rawUri,
String metadataJson,
String protocol,
String vehicleKey,
String vin,

1
modules/sinks/tdengine-history-store/src/main/java/com/lingniu/ingest/tdenginehistory/TdengineRawFrameRow.java
View File

@@ -16,6 +16,7 @@ public record TdengineRawFrameRow(
String parseError,
String peer,
String metadataJson,
String parsedJson,
String protocol,
String vehicleKey,
String vin,

10
modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineHistoryStatementsTest.java
View File

@@ -27,6 +27,7 @@ class TdengineHistoryStatementsTest {
"",
"10.0.0.1:808",
"{\"auth\":\"passed\"}",
"{\"body\":{\"speedKmh\":42.0}}",
"JT808",
"jt808:g7gps",
"VIN123",
@@ -39,13 +40,13 @@ class TdengineHistoryStatementsTest {
.contains(" USING raw_frames TAGS ('JT808', 'jt808:g7gps', 'VIN123', '013800000000')");
assertThat(batch.insertSql())
.startsWith("INSERT INTO raw_jt808_")
.contains("(ts, frame_id, received_at, message_id, sub_type, event_time, raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json)")
.endsWith("VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
.contains("(ts, frame_id, received_at, message_id, sub_type, event_time, raw_uri, checksum, raw_size_bytes, parse_status, parse_error, peer, metadata_json, parsed_json)")
.endsWith("VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
assertThat(batch.values())
.containsExactly(
row.ts(), row.frameId(), row.receivedAt(), row.messageId(), row.subType(),
row.eventTime(), row.rawUri(), row.checksum(), row.rawSizeBytes(), row.parseStatus(),
row.parseError(), row.peer(), row.metadataJson());
row.parseError(), row.peer(), row.metadataJson(), row.parsedJson());
}
@Test
@@ -64,7 +65,6 @@ class TdengineHistoryStatementsTest {
3,
null,
"archive://gb32960/frame-1.bin",
"{}",
"GB32960",
"vin:O'HARE",
"O'HARE",
@@ -79,7 +79,7 @@ class TdengineHistoryStatementsTest {
.containsExactly(
row.ts(), row.factId(), row.frameId(), row.receivedAt(), row.longitude(), row.latitude(),
row.altitudeM(), row.speedKmh(), row.directionDeg(), row.alarmFlag(), row.statusFlag(),
row.totalMileageKm(), row.rawUri(), row.metadataJson());
row.totalMileageKm(), row.rawUri());
}
@Test

2
modules/sinks/tdengine-history-store/src/test/java/com/lingniu/ingest/tdenginehistory/TdengineJdbcHistoryWriterTest.java
View File

@@ -74,7 +74,6 @@ class TdengineJdbcHistoryWriterTest {
3,
null,
"archive://gb32960/frame-1.bin",
"{}",
"GB32960",
"vin:VIN123",
"VIN123",
@@ -217,6 +216,7 @@ class TdengineJdbcHistoryWriterTest {
parseError,
"10.0.0.1:808",
"{}",
"{\"frame\":\"" + frameId + "\"}",
"JT808",
"jt808:g7gps",
"VIN123",