shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: productionize raw history ingestion

Browse Source
This commit is contained in:
lingniu
2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions
Download Patch File Download Diff File Expand all files Collapse all files

219
modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStore.java
View File

@@ -36,11 +36,13 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
private static final TypeReference<Map<String, String>> STRING_MAP =
new TypeReference<>() {};
private static final String HEADER_VERSION = "event-records-v1";
private static final int VEHICLE_BUCKETS = 64;
private final Path root;
private final Path indexPath;
private final ZoneId partitionZone;
private final ObjectMapper objectMapper;
private final boolean writeParquetEnabled;
private volatile boolean indexInitialized;
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone) {
@@ -48,6 +50,13 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
}
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone, ObjectMapper objectMapper) {
this(root, partitionZone, objectMapper, true);
}
public DuckDbParquetEventFileStore(Path root,
ZoneId partitionZone,
ObjectMapper objectMapper,
boolean writeParquetEnabled) {
if (root == null) {
throw new IllegalArgumentException("root must not be null");
}
@@ -55,6 +64,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
this.indexPath = this.root.resolve("events.duckdb");
this.partitionZone = partitionZone == null ? ZoneId.of("Asia/Shanghai") : partitionZone;
this.objectMapper = objectMapper == null ? new ObjectMapper() : objectMapper;
this.writeParquetEnabled = writeParquetEnabled;
}
@Override
@@ -67,16 +77,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
Partition partition = partition(record);
byPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record);
}
for (Map.Entry<Partition, List<EventFileRecord>> entry : byPartition.entrySet()) {
writePartition(entry.getKey(), entry.getValue());
appendIndex(entry.getKey(), entry.getValue());
if (writeParquetEnabled) {
writePartitions(byPartition);
}
appendIndex(byPartition);
}
@Override
public List<EventFileRecord> query(EventFileQuery query) throws IOException {
// 有 VIN 时直接扫该车分区 parquet，避免先查总索引再和车辆文件 join。
if (query.vin() != null) {
// 有 VIN 且启用 Parquet 副本时，可直接扫分区文件；索引-only 模式走 DuckDB 主索引。
if (query.vin() != null && writeParquetEnabled) {
return queryParquet(query);
}
// 无 VIN 的管理类查询走 DuckDB 索引，代价更高但不影响高频单车查询路径。
@@ -130,6 +140,9 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
if (query.eventTimeTo() != null) {
predicates.add("event_time_ms <= " + query.eventTimeTo().toEpochMilli());
}
if (query.cursorEventTime() != null) {
predicates.add(cursorPredicateSql(query));
}
// 这里的 SQL 只拼接经过 escape 的内部值；外部 rawArchiveUri 查询使用 PreparedStatement。
String where = predicates.isEmpty() ? "" : "WHERE " + String.join(" AND ", predicates) + "\n";
String sql = """
@@ -182,6 +195,9 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
if (query.eventTimeTo() != null) {
where.append(" AND event_time_ms <= ?\n");
}
if (query.cursorEventTime() != null) {
where.append(cursorPredicatePrepared(query.order()));
}
String sql = """
SELECT event_id, protocol, event_type, vin,
event_time_ms, ingest_time_ms,
@@ -209,6 +225,16 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
if (query.eventTimeTo() != null) {
ps.setLong(index++, query.eventTimeTo().toEpochMilli());
}
if (query.cursorEventTime() != null) {
long eventTimeMs = query.cursorEventTime().toEpochMilli();
long ingestTimeMs = query.cursorIngestTime().toEpochMilli();
ps.setLong(index++, eventTimeMs);
ps.setLong(index++, eventTimeMs);
ps.setLong(index++, ingestTimeMs);
ps.setLong(index++, eventTimeMs);
ps.setLong(index++, ingestTimeMs);
ps.setString(index++, query.cursorEventId());
}
ps.setInt(index, query.limit());
try (ResultSet rs = ps.executeQuery()) {
List<EventFileRecord> result = new ArrayList<>();
@@ -222,16 +248,43 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
}
}
private void writePartition(Partition partition, List<EventFileRecord> records) throws IOException {
Path dir = partition.dir(root);
Files.createDirectories(dir);
Path partFile = dir.resolve("events.parquet");
Path tempFile = dir.resolve("events-" + UUID.randomUUID() + ".tmp.parquet");
private static String cursorPredicateSql(EventFileQuery query) {
long eventTimeMs = query.cursorEventTime().toEpochMilli();
long ingestTimeMs = query.cursorIngestTime().toEpochMilli();
String eventId = escapeSql(query.cursorEventId());
String cmp = query.order() == EventFileQuery.Order.DESC ? "<" : ">";
return """
(
event_time_ms %s %d
OR (event_time_ms = %d AND ingest_time_ms %s %d)
OR (event_time_ms = %d AND ingest_time_ms = %d AND event_id %s '%s')
)
""".formatted(cmp, eventTimeMs, eventTimeMs, cmp, ingestTimeMs,
eventTimeMs, ingestTimeMs, cmp, eventId);
}
private static String cursorPredicatePrepared(EventFileQuery.Order order) {
String cmp = order == EventFileQuery.Order.DESC ? "<" : ">";
return """
AND (
event_time_ms %s ?
OR (event_time_ms = ? AND ingest_time_ms %s ?)
OR (event_time_ms = ? AND ingest_time_ms = ? AND event_id %s ?)
)
""".formatted(cmp, cmp, cmp);
}
private void writePartitions(Map<Partition, List<EventFileRecord>> recordsByPartition) throws IOException {
if (recordsByPartition.isEmpty()) {
return;
}
List<Path> tempFiles = new ArrayList<>();
try (Connection connection = DriverManager.getConnection("jdbc:duckdb:");
Statement statement = connection.createStatement()) {
statement.execute("""
CREATE TEMPORARY TABLE event_records (
partition_id INTEGER,
event_id VARCHAR,
protocol VARCHAR,
event_type VARCHAR,
@@ -246,54 +299,62 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
)
""");
try (PreparedStatement ps = connection.prepareStatement("""
INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""")) {
for (EventFileRecord record : records) {
ps.setString(1, record.eventId());
ps.setString(2, record.protocol().name());
ps.setString(3, record.eventType());
ps.setString(4, record.vin());
ps.setLong(5, record.eventTime().toEpochMilli());
ps.setString(6, record.eventTime().toString());
ps.setLong(7, record.ingestTime().toEpochMilli());
ps.setString(8, record.ingestTime().toString());
ps.setString(9, record.rawArchiveUri());
ps.setString(10, objectMapper.writeValueAsString(record.metadata()));
ps.setString(11, record.payloadJson());
ps.addBatch();
int partitionId = 0;
for (List<EventFileRecord> records : recordsByPartition.values()) {
for (EventFileRecord record : records) {
ps.setInt(1, partitionId);
ps.setString(2, record.eventId());
ps.setString(3, record.protocol().name());
ps.setString(4, record.eventType());
ps.setString(5, record.vin());
ps.setLong(6, record.eventTime().toEpochMilli());
ps.setString(7, record.eventTime().toString());
ps.setLong(8, record.ingestTime().toEpochMilli());
ps.setString(9, record.ingestTime().toString());
ps.setString(10, record.rawArchiveUri());
ps.setString(11, objectMapper.writeValueAsString(record.metadata()));
ps.setString(12, record.payloadJson());
ps.addBatch();
}
partitionId++;
}
ps.executeBatch();
}
if (Files.exists(partFile)) {
// DuckDB 不能原地追加 parquet；先 UNION 旧文件和本批数据，再原子替换分区文件。
int partitionId = 0;
for (Partition partition : recordsByPartition.keySet()) {
Path dir = partition.dir(root);
Files.createDirectories(dir);
String fragmentId = UUID.randomUUID().toString();
Path partFile = dir.resolve("events-" + fragmentId + ".parquet");
Path tempFile = dir.resolve("events-" + fragmentId + ".tmp.parquet");
tempFiles.add(tempFile);
statement.execute("""
CREATE TEMPORARY TABLE combined_records AS
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM read_parquet('%s')
UNION ALL
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM event_records
""".formatted(escapeSql(partFile)));
statement.execute("COPY combined_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
} else {
statement.execute("COPY event_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
COPY (
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM event_records
WHERE partition_id = %d
) TO '%s' (FORMAT parquet)
""".formatted(partitionId, escapeSql(tempFile)));
Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
partitionId++;
}
Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
} catch (SQLException e) {
throw new IOException("write parquet event store failed: " + partFile, e);
throw new IOException("write parquet event store failed", e);
} finally {
Files.deleteIfExists(tempFile);
for (Path tempFile : tempFiles) {
Files.deleteIfExists(tempFile);
}
}
}
private void appendIndex(Partition partition, List<EventFileRecord> records) throws IOException {
private void appendIndex(Map<Partition, List<EventFileRecord>> recordsByPartition) throws IOException {
ensureIndexInitialized();
try (Connection connection = DriverManager.getConnection(indexJdbcUrl())) {
insertIndex(connection, partition, records);
insertIndex(connection, recordsByPartition);
} catch (SQLException e) {
throw new IOException("write duckdb event index failed", e);
}
@@ -364,23 +425,26 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
""".formatted(fileList));
}
private void insertIndex(Connection connection, Partition partition, List<EventFileRecord> records)
private void insertIndex(Connection connection, Map<Partition, List<EventFileRecord>> recordsByPartition)
throws SQLException, IOException {
try (PreparedStatement ps = connection.prepareStatement("""
INSERT OR REPLACE INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""")) {
for (EventFileRecord record : records) {
ps.setString(1, record.eventId());
ps.setString(2, record.protocol().name());
ps.setString(3, record.eventType());
ps.setString(4, record.vin());
ps.setLong(5, record.eventTime().toEpochMilli());
ps.setLong(6, record.ingestTime().toEpochMilli());
ps.setString(7, partition.date().toString());
ps.setString(8, record.rawArchiveUri());
ps.setString(9, objectMapper.writeValueAsString(record.metadata()));
ps.setString(10, record.payloadJson());
ps.addBatch();
for (Map.Entry<Partition, List<EventFileRecord>> entry : recordsByPartition.entrySet()) {
Partition partition = entry.getKey();
for (EventFileRecord record : entry.getValue()) {
ps.setString(1, record.eventId());
ps.setString(2, record.protocol().name());
ps.setString(3, record.eventType());
ps.setString(4, record.vin());
ps.setLong(5, record.eventTime().toEpochMilli());
ps.setLong(6, record.ingestTime().toEpochMilli());
ps.setString(7, partition.date().toString());
ps.setString(8, record.rawArchiveUri());
ps.setString(9, objectMapper.writeValueAsString(record.metadata()));
ps.setString(10, record.payloadJson());
ps.addBatch();
}
}
ps.executeBatch();
}
@@ -390,15 +454,22 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
List<Path> files = new ArrayList<>();
LocalDate date = query.dateFrom();
while (!date.isAfter(query.dateTo())) {
Path dir = query.vin() == null
? partitionDir(query.protocol(), date)
: partitionDir(query.protocol(), date).resolve("vehicle=" + storageName(query.vin()));
// 目录结构把 VIN 放在 date 下，单车多日查询只需要遍历目标日期内的目标车辆目录。
if (Files.isDirectory(dir)) {
try (Stream<Path> stream = Files.walk(dir)) {
stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
.sorted()
.forEach(files::add);
Path dateDir = partitionDir(query.protocol(), date);
List<Path> candidateDirs = query.vin() == null
? List.of(dateDir)
: List.of(
// 新布局：高写入路径按协议/日期写一个批次文件，VIN 由 Parquet/索引字段过滤。
dateDir.resolve("header=" + HEADER_VERSION),
// 兼容已落盘的旧布局。
dateDir.resolve("bucket=" + bucketName(query.vin())),
dateDir.resolve("vehicle=" + storageName(query.vin())));
for (Path dir : candidateDirs) {
if (Files.isDirectory(dir)) {
try (Stream<Path> stream = Files.walk(dir)) {
stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
.sorted()
.forEach(files::add);
}
}
}
date = date.plusDays(1);
@@ -434,8 +505,7 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
private Partition partition(EventFileRecord record) {
return new Partition(
record.protocol(),
LocalDate.ofInstant(record.eventTime(), partitionZone),
storageName(record.vin()));
LocalDate.ofInstant(record.eventTime(), partitionZone));
}
private Path partitionDir(ProtocolId protocol, LocalDate date) {
@@ -488,11 +558,18 @@ public final class DuckDbParquetEventFileStore implements EventFileStore {
return vin.replaceAll("[^A-Za-z0-9._-]", "_");
}
private record Partition(ProtocolId protocol, LocalDate date, String vehicle) {
private static String bucketName(String vin) {
String storageName = storageName(vin);
if ("_unknown".equals(storageName)) {
return "_unknown";
}
return "%02d".formatted(Math.floorMod(storageName.hashCode(), VEHICLE_BUCKETS));
}
private record Partition(ProtocolId protocol, LocalDate date) {
private Path dir(Path root) {
return root.resolve("protocol=" + protocol.name())
.resolve("date=" + date)
.resolve("vehicle=" + vehicle)
.resolve("header=" + HEADER_VERSION);
}
}

23
modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/EventFileQuery.java
View File

@@ -20,7 +20,10 @@ public record EventFileQuery(
Order order,
int limit,
String vin,
String eventType
String eventType,
Instant cursorEventTime,
Instant cursorIngestTime,
String cursorEventId
) {
public enum Order {
ASC,
@@ -45,6 +48,24 @@ public record EventFileQuery(
limit = limit <= 0 ? 100 : limit;
vin = vin == null || vin.isBlank() ? null : vin.trim();
eventType = eventType == null || eventType.isBlank() ? null : eventType.trim();
cursorEventId = cursorEventId == null || cursorEventId.isBlank() ? null : cursorEventId.trim();
if ((cursorEventTime == null || cursorIngestTime == null || cursorEventId == null)
&& !(cursorEventTime == null && cursorIngestTime == null && cursorEventId == null)) {
throw new IllegalArgumentException("cursor eventTime, ingestTime and eventId must be provided together");
}
}
public EventFileQuery(ProtocolId protocol,
LocalDate dateFrom,
LocalDate dateTo,
Instant eventTimeFrom,
Instant eventTimeTo,
Order order,
int limit,
String vin,
String eventType) {
this(protocol, dateFrom, dateTo, eventTimeFrom, eventTimeTo, order, limit,
vin, eventType, null, null, null);
}
public EventFileQuery(ProtocolId protocol,

14
modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/EventFileStoreSink.java
View File

@@ -226,9 +226,9 @@ public final class EventFileStoreSink implements EventSink, AutoCloseable {
return payload;
}
private static Map<String, Object> rawArchivePayload(VehicleEvent.RawArchive raw,
String rawArchiveKey,
String rawArchiveUri) {
private Map<String, Object> rawArchivePayload(VehicleEvent.RawArchive raw,
String rawArchiveKey,
String rawArchiveUri) throws IOException {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("eventId", raw.eventId());
payload.put("vin", raw.vin());
@@ -241,10 +241,18 @@ public final class EventFileStoreSink implements EventSink, AutoCloseable {
payload.put("command", hex(raw.command()));
payload.put("infoType", hex(raw.infoType()));
payload.put("rawSizeBytes", raw.rawBytes() == null ? 0 : raw.rawBytes().length);
putParsedJson(payload, raw.parsedJson());
payload.put("metadata", raw.metadata());
return payload;
}
private void putParsedJson(Map<String, Object> payload, String parsedJson) throws IOException {
if (parsedJson == null || parsedJson.isBlank()) {
return;
}
payload.put("parsed", objectMapper.readTree(parsedJson));
}
private static String hex(int value) {
return "0x" + String.format("%04X", value);
}

3
modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfiguration.java
View File

@@ -44,7 +44,8 @@ public class EventFileStoreAutoConfiguration {
return new DuckDbParquetEventFileStore(
Path.of(properties.getPath()),
ZoneId.of(properties.getZoneId()),
mapper);
mapper,
properties.isWriteParquetEnabled());
}
@Bean

22
modules/sinks/event-file-store/src/main/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreProperties.java
View File

@@ -11,10 +11,10 @@ public class EventFileStoreProperties {
private boolean enabled = false;
/**
* Parquet 文件库根路径。
* 历史库根路径。
*
* <p>实际结构为 {@code protocol=GB32960/date=yyyy-MM-dd/vehicle=VIN/header=event-records-v1/events.parquet}，
* 另有 {@code events.duckdb} sidecar index 放在根路径下。
* <p>{@code events.duckdb} 是可查询主索引；开启 Parquet 副本时，还会在该根目录下按
* {@code protocol/date/header} 保存追加文件。
*/
private String path = "./event-store";
@@ -39,6 +39,14 @@ public class EventFileStoreProperties {
*/
private long flushIntervalMillis = 1000;
/**
* 是否同步写 Parquet 副本。
*
* <p>高写入生产链路可以关闭，只保留 DuckDB JSON 主索引和 raw archive 引用；需要离线副本或
* 索引重建能力时再开启。
*/
private boolean writeParquetEnabled = true;
public boolean isEnabled() {
return enabled;
}
@@ -78,4 +86,12 @@ public class EventFileStoreProperties {
public void setFlushIntervalMillis(long flushIntervalMillis) {
this.flushIntervalMillis = flushIntervalMillis;
}
public boolean isWriteParquetEnabled() {
return writeParquetEnabled;
}
public void setWriteParquetEnabled(boolean writeParquetEnabled) {
this.writeParquetEnabled = writeParquetEnabled;
}
}

89
modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/DuckDbParquetEventFileStoreTest.java
View File

@@ -1,6 +1,7 @@
package com.lingniu.ingest.eventfilestore;
import com.lingniu.ingest.api.ProtocolId;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -11,6 +12,7 @@ import java.time.LocalDate;
import java.time.ZoneId;
import java.util.List;
import java.util.Map;
import java.util.stream.IntStream;
import static org.assertj.core.api.Assertions.assertThat;
@@ -28,7 +30,7 @@ class DuckDbParquetEventFileStoreTest {
store.appendAll(List.of(first, second));
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
assertThat(parquetFiles(dateDir)).hasSize(2);
assertThat(parquetFiles(dateDir)).hasSize(1);
EventFileQuery ascending = new EventFileQuery(
ProtocolId.GB32960,
@@ -135,16 +137,13 @@ class DuckDbParquetEventFileStoreTest {
}
@Test
void appendsSameVehicleDayHeaderToOneParquetFile() throws Exception {
void appendsSameVehicleDayHeaderAsAppendOnlyParquetFragments() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(record("first", Instant.parse("2026-06-21T16:00:03Z"), "VIN-SAME"));
store.append(record("second", Instant.parse("2026-06-21T16:00:04Z"), "VIN-SAME"));
Path vehicleDayDir = tempDir.resolve("protocol=GB32960")
.resolve("date=2026-06-22")
.resolve("vehicle=VIN-SAME")
.resolve("header=event-records-v1");
assertThat(parquetFiles(vehicleDayDir)).containsExactly(vehicleDayDir.resolve("events.parquet"));
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
assertThat(parquetFiles(dateDir)).hasSize(2);
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
@@ -159,19 +158,54 @@ class DuckDbParquetEventFileStoreTest {
.containsExactly("first", "second");
}
@Test
void writesManyVehiclesIntoBoundedDailyLayoutInsteadOfVehicleDirectories() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
List<EventFileRecord> records = IntStream.range(0, 80)
.mapToObj(index -> rawRecord(
"raw-" + index,
Instant.parse("2026-06-21T16:00:00Z").plusMillis(index),
"VIN-" + index))
.toList();
store.appendAll(records);
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
try (var files = Files.list(dateDir)) {
List<String> partitionDirs = files
.filter(Files::isDirectory)
.map(path -> path.getFileName().toString())
.sorted()
.toList();
assertThat(partitionDirs).noneMatch(name -> name.startsWith("vehicle="));
assertThat(partitionDirs).hasSizeLessThan(records.size());
}
}
@Test
void writesOneDailyParquetFragmentForOneBatchAcrossManyVehicles() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
List<EventFileRecord> records = IntStream.range(0, 80)
.mapToObj(index -> rawRecord(
"daily-raw-" + index,
Instant.parse("2026-06-21T16:00:00Z").plusMillis(index),
"VIN-DAILY-" + index))
.toList();
store.appendAll(records);
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
assertThat(parquetFiles(dateDir)).hasSize(1);
}
@Test
void queriesUseSidecarIndexInsteadOfScanningParquetPartsEveryTime() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(record("indexed", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEXED"));
Path vehicleDayDir = tempDir.resolve("protocol=GB32960")
.resolve("date=2026-06-22")
.resolve("vehicle=VIN-INDEXED")
.resolve("header=event-records-v1");
try (var files = Files.list(vehicleDayDir)) {
for (Path file : files.filter(p -> p.getFileName().toString().endsWith(".parquet")).toList()) {
Files.delete(file);
}
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
for (Path file : parquetFiles(dateDir)) {
Files.delete(file);
}
EventFileQuery query = new EventFileQuery(
@@ -219,6 +253,31 @@ class DuckDbParquetEventFileStoreTest {
assertThat(found.rawArchiveUri()).isEqualTo(record.rawArchiveUri());
}
@Test
void canQueryVinRecordsFromIndexWhenParquetWritingIsDisabled() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(
tempDir,
ZoneId.of("Asia/Shanghai"),
new ObjectMapper(),
false);
EventFileRecord record = rawRecord("index-only", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEX-ONLY");
store.append(record);
assertThat(parquetFiles(tempDir)).isEmpty();
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.DESC,
10,
"VIN-INDEX-ONLY",
"RAW_ARCHIVE");
assertThat(store.query(query))
.extracting(EventFileRecord::eventId)
.containsExactly("index-only");
}
private static List<Path> parquetFiles(Path root) throws Exception {
try (var files = Files.walk(root)) {
return files.filter(p -> p.getFileName().toString().endsWith(".parquet"))

5
modules/sinks/event-file-store/src/test/java/com/lingniu/ingest/eventfilestore/config/EventFileStoreAutoConfigurationTest.java
View File

@@ -20,12 +20,15 @@ class EventFileStoreAutoConfigurationTest {
contextRunner
.withPropertyValues(
"lingniu.ingest.event-file-store.enabled=true",
"lingniu.ingest.event-file-store.path=target/test-event-store")
"lingniu.ingest.event-file-store.path=target/test-event-store",
"lingniu.ingest.event-file-store.write-parquet-enabled=false")
.run(context -> {
assertThat(context).hasSingleBean(EventFileStore.class);
assertThat(context).hasSingleBean(EventFileStoreSink.class);
assertThat(context.getBean(EventFileStoreProperties.class).getPath())
.isEqualTo(Path.of("target/test-event-store").toString());
assertThat(context.getBean(EventFileStoreProperties.class).isWriteParquetEnabled())
.isFalse();
});
}