feat: make gb32960 archive history query production ready

This commit is contained in:
kkfluous
2026-06-23 11:55:44 +08:00
parent b14871ff1c
commit ba68ffe061
462 changed files with 23639 additions and 2341 deletions

View File

@@ -0,0 +1,60 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.lingniu.ingest</groupId>
<artifactId>lingniu-vehicle-ingest</artifactId>
<version>0.1.0-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<artifactId>event-file-store</artifactId>
<name>event-file-store</name>
<description>车辆事件文件型明细库Parquet 存储 + DuckDB 时间顺序查询。</description>
<dependencies>
<dependency>
<groupId>com.lingniu.ingest</groupId>
<artifactId>ingest-api</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.duckdb</groupId>
<artifactId>duckdb_jdbc</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@@ -0,0 +1,485 @@
package com.lingniu.ingest.eventfilestore;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Stream;
public final class DuckDbParquetEventFileStore implements EventFileStore {
private static final TypeReference<Map<String, String>> STRING_MAP =
new TypeReference<>() {};
private static final String HEADER_VERSION = "event-records-v1";
private final Path root;
private final Path indexPath;
private final ZoneId partitionZone;
private final ObjectMapper objectMapper;
private volatile boolean indexInitialized;
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone) {
this(root, partitionZone, new ObjectMapper());
}
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone, ObjectMapper objectMapper) {
if (root == null) {
throw new IllegalArgumentException("root must not be null");
}
this.root = root.toAbsolutePath();
this.indexPath = this.root.resolve("events.duckdb");
this.partitionZone = partitionZone == null ? ZoneId.of("Asia/Shanghai") : partitionZone;
this.objectMapper = objectMapper == null ? new ObjectMapper() : objectMapper;
}
@Override
public synchronized void appendAll(List<EventFileRecord> records) throws IOException {
if (records == null || records.isEmpty()) {
return;
}
Map<Partition, List<EventFileRecord>> byPartition = new LinkedHashMap<>();
for (EventFileRecord record : records) {
Partition partition = partition(record);
byPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record);
}
for (Map.Entry<Partition, List<EventFileRecord>> entry : byPartition.entrySet()) {
writePartition(entry.getKey(), entry.getValue());
appendIndex(entry.getKey(), entry.getValue());
}
}
@Override
public List<EventFileRecord> query(EventFileQuery query) throws IOException {
if (query.vin() != null) {
return queryParquet(query);
}
ensureIndexInitialized();
return queryIndex(query);
}
@Override
public synchronized EventFileRecord findByRawArchiveUri(String rawArchiveUri) throws IOException {
if (rawArchiveUri == null || rawArchiveUri.isBlank()) {
return null;
}
ensureIndexInitialized();
String sql = """
SELECT event_id, protocol, event_type, vin,
event_time_ms, ingest_time_ms,
raw_archive_uri, metadata_json, payload_json
FROM event_records
WHERE raw_archive_uri = ?
ORDER BY event_type = 'RAW_ARCHIVE' DESC, ingest_time_ms DESC, event_id DESC
LIMIT 1
""";
try (Connection connection = DriverManager.getConnection(indexJdbcUrl());
PreparedStatement ps = connection.prepareStatement(sql)) {
ps.setString(1, rawArchiveUri);
try (ResultSet rs = ps.executeQuery()) {
return rs.next() ? record(rs) : null;
}
} catch (SQLException e) {
throw new IOException("query duckdb event index by raw archive uri failed", e);
}
}
private List<EventFileRecord> queryParquet(EventFileQuery query) throws IOException {
List<Path> parquetFiles = parquetFiles(query);
if (parquetFiles.isEmpty()) {
return List.of();
}
String fileList = toDuckDbList(parquetFiles);
String order = query.order() == EventFileQuery.Order.DESC ? "DESC" : "ASC";
List<String> predicates = new ArrayList<>();
if (query.vin() != null) {
predicates.add("vin = '" + escapeSql(query.vin()) + "'");
}
if (query.eventType() != null) {
predicates.add("event_type = '" + escapeSql(query.eventType()) + "'");
}
if (query.eventTimeFrom() != null) {
predicates.add("event_time_ms >= " + query.eventTimeFrom().toEpochMilli());
}
if (query.eventTimeTo() != null) {
predicates.add("event_time_ms <= " + query.eventTimeTo().toEpochMilli());
}
String where = predicates.isEmpty() ? "" : "WHERE " + String.join(" AND ", predicates) + "\n";
String sql = """
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM read_parquet(%s)
%s
ORDER BY event_time_ms %s, ingest_time_ms %s, event_id %s
LIMIT %d
""".formatted(fileList, where, order, order, order, query.limit());
try (Connection connection = DriverManager.getConnection("jdbc:duckdb:");
Statement statement = connection.createStatement();
ResultSet rs = statement.executeQuery(sql)) {
List<EventFileRecord> result = new ArrayList<>();
while (rs.next()) {
result.add(new EventFileRecord(
rs.getString("event_id"),
protocol(rs.getString("protocol")),
rs.getString("event_type"),
rs.getString("vin"),
Instant.ofEpochMilli(rs.getLong("event_time_ms")),
Instant.ofEpochMilli(rs.getLong("ingest_time_ms")),
rs.getString("raw_archive_uri"),
readMetadata(rs.getString("metadata_json")),
rs.getString("payload_json")
));
}
return result;
} catch (SQLException e) {
throw new IOException("query parquet event store failed", e);
}
}
private List<EventFileRecord> queryIndex(EventFileQuery query) throws IOException {
String order = query.order() == EventFileQuery.Order.DESC ? "DESC" : "ASC";
StringBuilder where = new StringBuilder("""
WHERE protocol = ?
AND partition_date BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)
""");
if (query.vin() != null) {
where.append(" AND vin = ?\n");
}
if (query.eventType() != null) {
where.append(" AND event_type = ?\n");
}
if (query.eventTimeFrom() != null) {
where.append(" AND event_time_ms >= ?\n");
}
if (query.eventTimeTo() != null) {
where.append(" AND event_time_ms <= ?\n");
}
String sql = """
SELECT event_id, protocol, event_type, vin,
event_time_ms, ingest_time_ms,
raw_archive_uri, metadata_json, payload_json
FROM event_records
%s
ORDER BY event_time_ms %s, ingest_time_ms %s, event_id %s
LIMIT ?
""".formatted(where, order, order, order);
try (Connection connection = DriverManager.getConnection(indexJdbcUrl());
PreparedStatement ps = connection.prepareStatement(sql)) {
int index = 1;
ps.setString(index++, query.protocol().name());
ps.setString(index++, query.dateFrom().toString());
ps.setString(index++, query.dateTo().toString());
if (query.vin() != null) {
ps.setString(index++, query.vin());
}
if (query.eventType() != null) {
ps.setString(index++, query.eventType());
}
if (query.eventTimeFrom() != null) {
ps.setLong(index++, query.eventTimeFrom().toEpochMilli());
}
if (query.eventTimeTo() != null) {
ps.setLong(index++, query.eventTimeTo().toEpochMilli());
}
ps.setInt(index, query.limit());
try (ResultSet rs = ps.executeQuery()) {
List<EventFileRecord> result = new ArrayList<>();
while (rs.next()) {
result.add(record(rs));
}
return result;
}
} catch (SQLException e) {
throw new IOException("query duckdb event index failed", e);
}
}
private void writePartition(Partition partition, List<EventFileRecord> records) throws IOException {
Path dir = partition.dir(root);
Files.createDirectories(dir);
Path partFile = dir.resolve("events.parquet");
Path tempFile = dir.resolve("events-" + UUID.randomUUID() + ".tmp.parquet");
try (Connection connection = DriverManager.getConnection("jdbc:duckdb:");
Statement statement = connection.createStatement()) {
statement.execute("""
CREATE TEMPORARY TABLE event_records (
event_id VARCHAR,
protocol VARCHAR,
event_type VARCHAR,
vin VARCHAR,
event_time_ms BIGINT,
event_time VARCHAR,
ingest_time_ms BIGINT,
ingest_time VARCHAR,
raw_archive_uri VARCHAR,
metadata_json VARCHAR,
payload_json VARCHAR
)
""");
try (PreparedStatement ps = connection.prepareStatement("""
INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""")) {
for (EventFileRecord record : records) {
ps.setString(1, record.eventId());
ps.setString(2, record.protocol().name());
ps.setString(3, record.eventType());
ps.setString(4, record.vin());
ps.setLong(5, record.eventTime().toEpochMilli());
ps.setString(6, record.eventTime().toString());
ps.setLong(7, record.ingestTime().toEpochMilli());
ps.setString(8, record.ingestTime().toString());
ps.setString(9, record.rawArchiveUri());
ps.setString(10, objectMapper.writeValueAsString(record.metadata()));
ps.setString(11, record.payloadJson());
ps.addBatch();
}
ps.executeBatch();
}
if (Files.exists(partFile)) {
statement.execute("""
CREATE TEMPORARY TABLE combined_records AS
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM read_parquet('%s')
UNION ALL
SELECT event_id, protocol, event_type, vin,
event_time_ms, event_time, ingest_time_ms, ingest_time,
raw_archive_uri, metadata_json, payload_json
FROM event_records
""".formatted(escapeSql(partFile)));
statement.execute("COPY combined_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
} else {
statement.execute("COPY event_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
}
Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
} catch (SQLException e) {
throw new IOException("write parquet event store failed: " + partFile, e);
} finally {
Files.deleteIfExists(tempFile);
}
}
private void appendIndex(Partition partition, List<EventFileRecord> records) throws IOException {
ensureIndexInitialized();
try (Connection connection = DriverManager.getConnection(indexJdbcUrl())) {
insertIndex(connection, partition, records);
} catch (SQLException e) {
throw new IOException("write duckdb event index failed", e);
}
}
private void ensureIndexInitialized() throws IOException {
if (indexInitialized) {
return;
}
try {
Files.createDirectories(root);
} catch (IOException e) {
throw new IOException("create event store root failed: " + root, e);
}
try (Connection connection = DriverManager.getConnection(indexJdbcUrl());
Statement statement = connection.createStatement()) {
createIndexSchema(statement);
if (isIndexEmpty(statement)) {
backfillIndexFromParquet(statement);
}
indexInitialized = true;
} catch (SQLException e) {
throw new IOException("initialize duckdb event index failed", e);
}
}
private void createIndexSchema(Statement statement) throws SQLException {
statement.execute("""
CREATE TABLE IF NOT EXISTS event_records (
event_id VARCHAR PRIMARY KEY,
protocol VARCHAR,
event_type VARCHAR,
vin VARCHAR,
event_time_ms BIGINT,
ingest_time_ms BIGINT,
partition_date DATE,
raw_archive_uri VARCHAR,
metadata_json VARCHAR,
payload_json VARCHAR
)
""");
statement.execute("CREATE INDEX IF NOT EXISTS event_records_protocol_date_time_idx ON event_records(protocol, partition_date, event_time_ms)");
statement.execute("CREATE INDEX IF NOT EXISTS event_records_vin_date_time_idx ON event_records(vin, partition_date, event_time_ms)");
statement.execute("CREATE INDEX IF NOT EXISTS event_records_vin_type_date_time_idx ON event_records(protocol, vin, event_type, partition_date, event_time_ms)");
statement.execute("CREATE INDEX IF NOT EXISTS event_records_raw_archive_uri_idx ON event_records(raw_archive_uri)");
}
private boolean isIndexEmpty(Statement statement) throws SQLException {
try (ResultSet rs = statement.executeQuery("SELECT COUNT(*) FROM event_records")) {
return rs.next() && rs.getLong(1) == 0L;
}
}
private void backfillIndexFromParquet(Statement statement) throws SQLException, IOException {
List<Path> files = allParquetFiles();
if (files.isEmpty()) {
return;
}
String fileList = toDuckDbList(files);
statement.execute("""
INSERT OR IGNORE INTO event_records
SELECT event_id, protocol, event_type, vin,
event_time_ms, ingest_time_ms,
CAST(date AS DATE) AS partition_date,
raw_archive_uri, metadata_json, payload_json
FROM read_parquet(%s, hive_partitioning = true)
""".formatted(fileList));
}
private void insertIndex(Connection connection, Partition partition, List<EventFileRecord> records)
throws SQLException, IOException {
try (PreparedStatement ps = connection.prepareStatement("""
INSERT OR REPLACE INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""")) {
for (EventFileRecord record : records) {
ps.setString(1, record.eventId());
ps.setString(2, record.protocol().name());
ps.setString(3, record.eventType());
ps.setString(4, record.vin());
ps.setLong(5, record.eventTime().toEpochMilli());
ps.setLong(6, record.ingestTime().toEpochMilli());
ps.setString(7, partition.date().toString());
ps.setString(8, record.rawArchiveUri());
ps.setString(9, objectMapper.writeValueAsString(record.metadata()));
ps.setString(10, record.payloadJson());
ps.addBatch();
}
ps.executeBatch();
}
}
private List<Path> parquetFiles(EventFileQuery query) throws IOException {
List<Path> files = new ArrayList<>();
LocalDate date = query.dateFrom();
while (!date.isAfter(query.dateTo())) {
Path dir = query.vin() == null
? partitionDir(query.protocol(), date)
: partitionDir(query.protocol(), date).resolve("vehicle=" + storageName(query.vin()));
if (Files.isDirectory(dir)) {
try (Stream<Path> stream = Files.walk(dir)) {
stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
.sorted()
.forEach(files::add);
}
}
date = date.plusDays(1);
}
return files;
}
private List<Path> allParquetFiles() throws IOException {
if (!Files.isDirectory(root)) {
return List.of();
}
try (Stream<Path> stream = Files.walk(root)) {
return stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
.sorted()
.toList();
}
}
private EventFileRecord record(ResultSet rs) throws SQLException, IOException {
return new EventFileRecord(
rs.getString("event_id"),
protocol(rs.getString("protocol")),
rs.getString("event_type"),
rs.getString("vin"),
Instant.ofEpochMilli(rs.getLong("event_time_ms")),
Instant.ofEpochMilli(rs.getLong("ingest_time_ms")),
rs.getString("raw_archive_uri"),
readMetadata(rs.getString("metadata_json")),
rs.getString("payload_json")
);
}
private Partition partition(EventFileRecord record) {
return new Partition(
record.protocol(),
LocalDate.ofInstant(record.eventTime(), partitionZone),
storageName(record.vin()));
}
private Path partitionDir(ProtocolId protocol, LocalDate date) {
return root.resolve("protocol=" + protocol.name()).resolve("date=" + date);
}
private static ProtocolId protocol(String value) {
if (value == null || value.isBlank()) {
return ProtocolId.UNKNOWN;
}
try {
return ProtocolId.valueOf(value);
} catch (IllegalArgumentException ex) {
return ProtocolId.UNKNOWN;
}
}
private Map<String, String> readMetadata(String json) throws IOException {
if (json == null || json.isBlank()) {
return Map.of();
}
return objectMapper.readValue(json, STRING_MAP);
}
private static String toDuckDbList(List<Path> files) {
return files.stream()
.map(path -> "'" + escapeSql(path) + "'")
.reduce((left, right) -> left + "," + right)
.map(paths -> "[" + paths + "]")
.orElse("[]");
}
private static String escapeSql(Path path) {
return escapeSql(path.toAbsolutePath().toString());
}
private static String escapeSql(String value) {
return value.replace("'", "''");
}
private String indexJdbcUrl() {
return "jdbc:duckdb:" + indexPath;
}
private static String storageName(String vin) {
if (vin == null || vin.isBlank()) {
return "_unknown";
}
return vin.replaceAll("[^A-Za-z0-9._-]", "_");
}
private record Partition(ProtocolId protocol, LocalDate date, String vehicle) {
private Path dir(Path root) {
return root.resolve("protocol=" + protocol.name())
.resolve("date=" + date)
.resolve("vehicle=" + vehicle)
.resolve("header=" + HEADER_VERSION);
}
}
}

View File

@@ -0,0 +1,69 @@
package com.lingniu.ingest.eventfilestore;
import com.lingniu.ingest.api.ProtocolId;
import java.time.Instant;
import java.time.LocalDate;
public record EventFileQuery(
ProtocolId protocol,
LocalDate dateFrom,
LocalDate dateTo,
Instant eventTimeFrom,
Instant eventTimeTo,
Order order,
int limit,
String vin,
String eventType
) {
public enum Order {
ASC,
DESC
}
public EventFileQuery {
if (protocol == null) {
throw new IllegalArgumentException("protocol must not be null");
}
if (dateFrom == null || dateTo == null) {
throw new IllegalArgumentException("date range must not be null");
}
if (dateTo.isBefore(dateFrom)) {
throw new IllegalArgumentException("dateTo must not be before dateFrom");
}
if (eventTimeFrom != null && eventTimeTo != null && eventTimeTo.isBefore(eventTimeFrom)) {
throw new IllegalArgumentException("eventTimeTo must not be before eventTimeFrom");
}
order = order == null ? Order.ASC : order;
limit = limit <= 0 ? 100 : limit;
vin = vin == null || vin.isBlank() ? null : vin.trim();
eventType = eventType == null || eventType.isBlank() ? null : eventType.trim();
}
public EventFileQuery(ProtocolId protocol,
LocalDate dateFrom,
LocalDate dateTo,
Order order,
int limit,
String vin,
String eventType) {
this(protocol, dateFrom, dateTo, null, null, order, limit, vin, eventType);
}
public EventFileQuery(ProtocolId protocol,
LocalDate dateFrom,
LocalDate dateTo,
Order order,
int limit) {
this(protocol, dateFrom, dateTo, order, limit, null, null);
}
public EventFileQuery(ProtocolId protocol,
LocalDate dateFrom,
LocalDate dateTo,
Order order,
int limit,
String vin) {
this(protocol, dateFrom, dateTo, order, limit, vin, null);
}
}

View File

@@ -0,0 +1,44 @@
package com.lingniu.ingest.eventfilestore;
import com.lingniu.ingest.api.ProtocolId;
import java.time.Instant;
import java.util.Map;
/**
* 文件型明细库的一条可查询记录。
*
* <p>{@code payloadJson} 保存统一 telemetry snapshot JSON公共列只保留排序、分区、
* 追溯和通用展示需要的最小字段。
*/
public record EventFileRecord(
String eventId,
ProtocolId protocol,
String eventType,
String vin,
Instant eventTime,
Instant ingestTime,
String rawArchiveUri,
Map<String, String> metadata,
String payloadJson
) {
public EventFileRecord {
if (eventId == null || eventId.isBlank()) {
throw new IllegalArgumentException("eventId must not be blank");
}
if (protocol == null) {
throw new IllegalArgumentException("protocol must not be null");
}
if (eventTime == null) {
throw new IllegalArgumentException("eventTime must not be null");
}
if (ingestTime == null) {
throw new IllegalArgumentException("ingestTime must not be null");
}
eventType = eventType == null ? "" : eventType;
vin = vin == null ? "" : vin;
rawArchiveUri = rawArchiveUri == null ? "" : rawArchiveUri;
metadata = metadata == null ? Map.of() : Map.copyOf(metadata);
payloadJson = payloadJson == null || payloadJson.isBlank() ? "{}" : payloadJson;
}
}

View File

@@ -0,0 +1,19 @@
package com.lingniu.ingest.eventfilestore;
import java.io.IOException;
import java.util.List;
public interface EventFileStore {
default void append(EventFileRecord record) throws IOException {
appendAll(List.of(record));
}
void appendAll(List<EventFileRecord> records) throws IOException;
List<EventFileRecord> query(EventFileQuery query) throws IOException;
default EventFileRecord findByRawArchiveUri(String rawArchiveUri) throws IOException {
return null;
}
}

View File

@@ -0,0 +1,244 @@
package com.lingniu.ingest.eventfilestore;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.event.TelemetrySnapshot;
import com.lingniu.ingest.api.event.VehicleEvent;
import com.lingniu.ingest.api.event.VehicleEventTelemetrySnapshotMapper;
import com.lingniu.ingest.api.sink.EventSink;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* EventBus 出口:把解析后的 {@link VehicleEvent} 明细写入 Parquet 文件库。
*
* <p>{@link VehicleEvent.RawArchive} 只写可查询索引,不写原始 bytes原始 bytes 本体仍由
* sink-archive 独立冷存。
*/
public final class EventFileStoreSink implements EventSink, AutoCloseable {
private static final Logger log = LoggerFactory.getLogger(EventFileStoreSink.class);
private final EventFileStore store;
private final ObjectMapper objectMapper;
private final int batchSize;
private final List<EventFileRecord> buffer = new ArrayList<>();
private final ScheduledExecutorService flusher;
public EventFileStoreSink(EventFileStore store, ObjectMapper objectMapper) {
this(store, objectMapper, 1);
}
public EventFileStoreSink(EventFileStore store, ObjectMapper objectMapper, int batchSize) {
this(store, objectMapper, batchSize, 0);
}
public EventFileStoreSink(EventFileStore store,
ObjectMapper objectMapper,
int batchSize,
long flushIntervalMillis) {
if (store == null) {
throw new IllegalArgumentException("store must not be null");
}
this.store = store;
this.objectMapper = objectMapper == null ? new ObjectMapper() : objectMapper;
this.batchSize = Math.max(1, batchSize);
this.flusher = flushIntervalMillis > 0 && this.batchSize > 1
? Executors.newSingleThreadScheduledExecutor(r -> {
Thread t = Thread.ofVirtual().name("event-file-store-flush-", 0).factory().newThread(r);
t.setDaemon(true);
return t;
})
: null;
if (flusher != null) {
flusher.scheduleWithFixedDelay(this::flushQuietly,
flushIntervalMillis, flushIntervalMillis, TimeUnit.MILLISECONDS);
}
}
@Override
public String name() {
return "event-file-store";
}
@Override
public boolean accepts(VehicleEvent event) {
return event != null
&& !(event instanceof VehicleEvent.Realtime)
&& !(event instanceof VehicleEvent.Location);
}
@Override
public CompletableFuture<Void> publish(VehicleEvent event) {
if (!accepts(event)) {
return CompletableFuture.completedFuture(null);
}
CompletableFuture<Void> future = new CompletableFuture<>();
try {
EventFileRecord record = toRecord(event);
if (batchSize <= 1) {
store.append(record);
} else {
appendBuffered(record);
}
future.complete(null);
} catch (Exception e) {
future.completeExceptionally(e);
}
return future;
}
@Override
public CompletableFuture<Void> publishBatch(List<VehicleEvent> batch) {
if (batch == null || batch.isEmpty()) {
return CompletableFuture.completedFuture(null);
}
CompletableFuture<Void> future = new CompletableFuture<>();
try {
List<EventFileRecord> records = new ArrayList<>();
for (VehicleEvent event : batch) {
if (accepts(event)) {
records.add(toRecord(event));
}
}
if (!records.isEmpty()) {
store.appendAll(records);
}
future.complete(null);
} catch (Exception e) {
future.completeExceptionally(e);
}
return future;
}
@Override
public void close() throws Exception {
if (flusher != null) {
flusher.shutdownNow();
}
flush();
}
private void appendBuffered(EventFileRecord record) throws IOException {
List<EventFileRecord> toFlush = null;
synchronized (buffer) {
buffer.add(record);
if (buffer.size() >= batchSize) {
toFlush = new ArrayList<>(buffer);
buffer.clear();
}
}
if (toFlush != null) {
store.appendAll(toFlush);
}
}
private void flush() throws IOException {
List<EventFileRecord> toFlush;
synchronized (buffer) {
if (buffer.isEmpty()) {
return;
}
toFlush = new ArrayList<>(buffer);
buffer.clear();
}
store.appendAll(toFlush);
}
private void flushQuietly() {
try {
flush();
} catch (Exception e) {
log.warn("event file store scheduled flush failed", e);
}
}
private EventFileRecord toRecord(VehicleEvent event) throws IOException {
if (event instanceof VehicleEvent.RawArchive raw) {
return rawArchiveIndexRecord(raw);
}
TelemetrySnapshot snapshot = VehicleEventTelemetrySnapshotMapper.toSnapshot(event)
.orElseThrow(() -> new IllegalArgumentException("event cannot be converted to telemetry snapshot"));
return new EventFileRecord(
event.eventId(),
event.source(),
snapshot.eventType(),
event.vin(),
event.eventTime(),
event.ingestTime(),
snapshot.rawArchiveUri(),
event.metadata(),
objectMapper.writeValueAsString(snapshotPayload(snapshot))
);
}
private EventFileRecord rawArchiveIndexRecord(VehicleEvent.RawArchive raw) throws IOException {
String rawArchiveKey = RawArchiveKeys.key(raw);
String rawArchiveUri = RawArchiveKeys.logicalUri(rawArchiveKey);
Map<String, String> metadata = new LinkedHashMap<>();
if (raw.metadata() != null) {
metadata.putAll(raw.metadata());
}
metadata.putIfAbsent(RawArchiveKeys.META_EVENT_ID, raw.eventId());
metadata.putIfAbsent(RawArchiveKeys.META_KEY, rawArchiveKey);
metadata.putIfAbsent(RawArchiveKeys.META_URI, rawArchiveUri);
return new EventFileRecord(
raw.eventId(),
raw.source(),
"RAW_ARCHIVE",
raw.vin(),
raw.eventTime(),
raw.ingestTime(),
rawArchiveUri,
metadata,
objectMapper.writeValueAsString(rawArchivePayload(raw, rawArchiveKey, rawArchiveUri))
);
}
private static Map<String, Object> snapshotPayload(TelemetrySnapshot snapshot) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("eventId", snapshot.eventId());
payload.put("vin", snapshot.vin());
payload.put("protocol", snapshot.protocol().name());
payload.put("eventType", snapshot.eventType());
payload.put("eventTime", snapshot.eventTime().toString());
payload.put("ingestTime", snapshot.ingestTime().toString());
payload.put("rawArchiveUri", snapshot.rawArchiveUri());
payload.put("metadata", snapshot.metadata());
payload.put("fields", snapshot.fields());
return payload;
}
private static Map<String, Object> rawArchivePayload(VehicleEvent.RawArchive raw,
String rawArchiveKey,
String rawArchiveUri) {
Map<String, Object> payload = new LinkedHashMap<>();
payload.put("eventId", raw.eventId());
payload.put("vin", raw.vin());
payload.put("protocol", raw.source().name());
payload.put("eventType", "RAW_ARCHIVE");
payload.put("eventTime", raw.eventTime().toString());
payload.put("ingestTime", raw.ingestTime().toString());
payload.put("rawArchiveUri", rawArchiveUri);
payload.put("rawArchiveKey", rawArchiveKey);
payload.put("command", hex(raw.command()));
payload.put("infoType", hex(raw.infoType()));
payload.put("rawSizeBytes", raw.rawBytes() == null ? 0 : raw.rawBytes().length);
payload.put("metadata", raw.metadata());
return payload;
}
private static String hex(int value) {
return "0x" + String.format("%04X", value);
}
}

View File

@@ -0,0 +1,53 @@
package com.lingniu.ingest.eventfilestore.config;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.eventfilestore.EventFileStoreSink;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.context.annotation.Bean;
import java.nio.file.Path;
import java.time.ZoneId;
@AutoConfiguration
@EnableConfigurationProperties(EventFileStoreProperties.class)
@ConditionalOnProperty(
prefix = "lingniu.ingest.event-file-store",
name = "enabled",
havingValue = "true")
public class EventFileStoreAutoConfiguration {
@Bean
@ConditionalOnMissingBean
public EventFileStore eventFileStore(EventFileStoreProperties properties,
ObjectProvider<ObjectMapper> objectMapper) {
ObjectMapper mapper = mapper(objectMapper);
return new DuckDbParquetEventFileStore(
Path.of(properties.getPath()),
ZoneId.of(properties.getZoneId()),
mapper);
}
@Bean
@ConditionalOnMissingBean
public EventFileStoreSink eventFileStoreSink(EventFileStore store,
EventFileStoreProperties properties,
ObjectProvider<ObjectMapper> objectMapper) {
return new EventFileStoreSink(
store,
mapper(objectMapper),
properties.getBatchSize(),
properties.getFlushIntervalMillis());
}
private static ObjectMapper mapper(ObjectProvider<ObjectMapper> provider) {
ObjectMapper base = provider.getIfAvailable(ObjectMapper::new);
return base.copy().registerModule(new JavaTimeModule());
}
}

View File

@@ -0,0 +1,72 @@
package com.lingniu.ingest.eventfilestore.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "lingniu.ingest.event-file-store")
public class EventFileStoreProperties {
/**
* 默认关闭,避免仅引入依赖就开始写明细文件。
*/
private boolean enabled = false;
/**
* Parquet 文件库根路径。
*/
private String path = "./event-store";
/**
* 按事件时间分区时使用的业务时区。
*/
private String zoneId = "Asia/Shanghai";
/**
* Sink 缓冲多少条事件后批量写一个 Parquet part 文件。
*/
private int batchSize = 500;
/**
* 未达到 batchSize 时的最长刷盘间隔。
*/
private long flushIntervalMillis = 1000;
public boolean isEnabled() {
return enabled;
}
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
public String getZoneId() {
return zoneId;
}
public void setZoneId(String zoneId) {
this.zoneId = zoneId;
}
public int getBatchSize() {
return batchSize;
}
public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
}
public long getFlushIntervalMillis() {
return flushIntervalMillis;
}
public void setFlushIntervalMillis(long flushIntervalMillis) {
this.flushIntervalMillis = flushIntervalMillis;
}
}

View File

@@ -0,0 +1 @@
com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration

View File

@@ -0,0 +1,257 @@
package com.lingniu.ingest.eventfilestore;
import com.lingniu.ingest.api.ProtocolId;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.List;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
class DuckDbParquetEventFileStoreTest {
@TempDir
Path tempDir;
@Test
void appendsRecordsToParquetAndQueriesByTimeAscendingAndDescending() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
EventFileRecord first = record("e1", Instant.parse("2026-06-21T16:00:02Z"), "粤A00001");
EventFileRecord second = record("e2", Instant.parse("2026-06-21T16:00:01Z"), "粤A00002");
store.appendAll(List.of(first, second));
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
assertThat(parquetFiles(dateDir)).hasSize(2);
EventFileQuery ascending = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.ASC,
100);
assertThat(store.query(ascending))
.extracting(EventFileRecord::eventId)
.containsExactly("e2", "e1");
EventFileQuery descending = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.DESC,
100);
assertThat(store.query(descending))
.extracting(EventFileRecord::eventId)
.containsExactly("e1", "e2");
}
@Test
void queriesAcrossMultipleDaysInTimeOrder() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(record("d1", Instant.parse("2026-06-20T16:00:00Z"), "粤A00001"));
store.append(record("d2", Instant.parse("2026-06-21T16:00:00Z"), "粤A00001"));
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-21"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.ASC,
100);
assertThat(store.query(query))
.extracting(EventFileRecord::eventId)
.containsExactly("d1", "d2");
}
@Test
void filtersByVinBeforeApplyingLimit() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(record("other-1", Instant.parse("2026-06-21T16:00:03Z"), "VIN-OTHER"));
store.append(record("target-1", Instant.parse("2026-06-21T16:00:02Z"), "VIN-TARGET"));
store.append(record("other-2", Instant.parse("2026-06-21T16:00:01Z"), "VIN-OTHER"));
store.append(record("target-2", Instant.parse("2026-06-21T16:00:00Z"), "VIN-TARGET"));
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.DESC,
1,
"VIN-TARGET");
assertThat(store.query(query))
.extracting(EventFileRecord::eventId)
.containsExactly("target-1");
}
@Test
void filtersByVinAndEventTypeBeforeApplyingLimit() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(rawRecord("raw-other", Instant.parse("2026-06-21T16:00:04Z"), "VIN-OTHER"));
store.append(record("business-target", Instant.parse("2026-06-21T16:00:03Z"), "VIN-TARGET"));
store.append(rawRecord("raw-target", Instant.parse("2026-06-21T16:00:02Z"), "VIN-TARGET"));
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.DESC,
1,
"VIN-TARGET",
"RAW_ARCHIVE");
assertThat(store.query(query))
.extracting(EventFileRecord::eventId)
.containsExactly("raw-target");
}
@Test
void filtersByExactEventTimeRangeBeforeApplyingLimit() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(rawRecord("before", Instant.parse("2026-06-21T16:00:00Z"), "VIN-TIME"));
store.append(rawRecord("inside", Instant.parse("2026-06-21T16:00:02Z"), "VIN-TIME"));
store.append(rawRecord("after", Instant.parse("2026-06-21T16:00:04Z"), "VIN-TIME"));
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
Instant.parse("2026-06-21T16:00:01Z"),
Instant.parse("2026-06-21T16:00:03Z"),
EventFileQuery.Order.ASC,
10,
"VIN-TIME",
"RAW_ARCHIVE");
assertThat(store.query(query))
.extracting(EventFileRecord::eventId)
.containsExactly("inside");
}
@Test
void appendsSameVehicleDayHeaderToOneParquetFile() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(record("first", Instant.parse("2026-06-21T16:00:03Z"), "VIN-SAME"));
store.append(record("second", Instant.parse("2026-06-21T16:00:04Z"), "VIN-SAME"));
Path vehicleDayDir = tempDir.resolve("protocol=GB32960")
.resolve("date=2026-06-22")
.resolve("vehicle=VIN-SAME")
.resolve("header=event-records-v1");
assertThat(parquetFiles(vehicleDayDir)).containsExactly(vehicleDayDir.resolve("events.parquet"));
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.ASC,
10,
"VIN-SAME");
assertThat(store.query(query))
.extracting(EventFileRecord::eventId)
.containsExactly("first", "second");
}
@Test
void queriesUseSidecarIndexInsteadOfScanningParquetPartsEveryTime() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(record("indexed", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEXED"));
Path vehicleDayDir = tempDir.resolve("protocol=GB32960")
.resolve("date=2026-06-22")
.resolve("vehicle=VIN-INDEXED")
.resolve("header=event-records-v1");
try (var files = Files.list(vehicleDayDir)) {
for (Path file : files.filter(p -> p.getFileName().toString().endsWith(".parquet")).toList()) {
Files.delete(file);
}
}
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.DESC,
10);
assertThat(store.query(query))
.extracting(EventFileRecord::eventId)
.containsExactly("indexed");
}
@Test
void vinQueriesReadVehicleDayParquetWithoutGlobalDuckDbIndex() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
store.append(rawRecord("raw-target", Instant.parse("2026-06-21T16:00:03Z"), "VIN-DIRECT"));
Files.deleteIfExists(tempDir.resolve("events.duckdb"));
EventFileQuery query = new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.DESC,
10,
"VIN-DIRECT",
"RAW_ARCHIVE");
assertThat(store.query(query))
.extracting(EventFileRecord::eventId)
.containsExactly("raw-target");
}
@Test
void findsRecordByRawArchiveUriThroughSidecarIndex() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
EventFileRecord record = record("raw-001", Instant.parse("2026-06-21T16:00:03Z"), "VIN-RAW");
store.append(record);
EventFileRecord found = store.findByRawArchiveUri(record.rawArchiveUri());
assertThat(found).isNotNull();
assertThat(found.eventId()).isEqualTo("raw-001");
assertThat(found.rawArchiveUri()).isEqualTo(record.rawArchiveUri());
}
private static List<Path> parquetFiles(Path root) throws Exception {
try (var files = Files.walk(root)) {
return files.filter(p -> p.getFileName().toString().endsWith(".parquet"))
.sorted()
.toList();
}
}
private static EventFileRecord record(String eventId, Instant eventTime, String vin) {
return new EventFileRecord(
eventId,
ProtocolId.GB32960,
"REALTIME",
vin,
eventTime,
eventTime.plusMillis(200),
"file:///archive/" + eventId + ".bin",
Map.of("source", "test"),
"""
{"gb32960":{"vehicle":{"speedKmh":42.1},"fuelCell":{"hydrogenRemainingKg":8.3}}}
""".strip());
}
private static EventFileRecord rawRecord(String eventId, Instant eventTime, String vin) {
return new EventFileRecord(
eventId,
ProtocolId.GB32960,
"RAW_ARCHIVE",
vin,
eventTime,
eventTime.plusMillis(200),
"file:///archive/" + eventId + ".bin",
Map.of("source", "test"),
"{}");
}
}

View File

@@ -0,0 +1,154 @@
package com.lingniu.ingest.eventfilestore;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.event.LocationPayload;
import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.event.RealtimePayload;
import com.lingniu.ingest.api.event.VehicleEvent;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Path;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.Map;
import static org.assertj.core.api.Assertions.assertThat;
class EventFileStoreSinkTest {
@TempDir
Path tempDir;
private final ObjectMapper objectMapper = new ObjectMapper();
@Test
void eventSinkSkipsRealtimeAndLocationEvents() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"), objectMapper);
EventFileStoreSink sink = new EventFileStoreSink(store, objectMapper);
sink.publish(realtimeEvent("evt-1")).join();
sink.publish(locationEvent("loc-1")).join();
var records = store.query(new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.ASC,
10));
assertThat(records).isEmpty();
}
@Test
void batchedSinkFlushesRecordsWhenClosed() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"), objectMapper);
EventFileStoreSink sink = new EventFileStoreSink(store, objectMapper, 10);
sink.publish(rawEvent("raw-batch-1")).join();
assertThat(store.query(new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.ASC,
10))).isEmpty();
sink.close();
assertThat(store.query(new EventFileQuery(
ProtocolId.GB32960,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.ASC,
10)))
.extracting(EventFileRecord::eventId)
.containsExactly("raw-batch-1");
}
@Test
void eventSinkPersistsRawArchiveAsQueryableIndexWithoutEmbeddingRawBytes() throws Exception {
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"), objectMapper);
EventFileStoreSink sink = new EventFileStoreSink(store, objectMapper);
VehicleEvent.RawArchive raw = new VehicleEvent.RawArchive(
"raw-1",
"LTEST000000000808",
ProtocolId.JT808,
Instant.parse("2026-06-21T16:00:03Z"),
Instant.parse("2026-06-21T16:00:04Z"),
"trace-raw-1",
Map.of("phone", "013800138000", "identityResolved", "true"),
0x0200,
0,
new byte[]{0x7e, 0x02, 0x00, 0x7e});
sink.publish(raw).join();
var records = store.query(new EventFileQuery(
ProtocolId.JT808,
LocalDate.parse("2026-06-22"),
LocalDate.parse("2026-06-22"),
EventFileQuery.Order.ASC,
10));
assertThat(records).hasSize(1);
EventFileRecord record = records.getFirst();
String rawArchiveKey = RawArchiveKeys.key(raw);
assertThat(record.eventId()).isEqualTo("raw-1");
assertThat(record.eventType()).isEqualTo("RAW_ARCHIVE");
assertThat(record.rawArchiveUri()).isEqualTo(RawArchiveKeys.logicalUri(rawArchiveKey));
assertThat(record.metadata())
.containsEntry(RawArchiveKeys.META_KEY, rawArchiveKey)
.containsEntry(RawArchiveKeys.META_URI, RawArchiveKeys.logicalUri(rawArchiveKey));
JsonNode payload = objectMapper.readTree(record.payloadJson());
assertThat(payload.get("rawSizeBytes").asLong()).isEqualTo(4);
assertThat(payload.get("command").asText()).isEqualTo("0x0200");
assertThat(payload.toString()).doesNotContain("7e02007e");
}
private static VehicleEvent.Realtime realtimeEvent(String eventId) {
return new VehicleEvent.Realtime(
eventId,
"LTEST000000000001",
ProtocolId.GB32960,
Instant.parse("2026-06-21T16:00:00Z"),
Instant.parse("2026-06-21T16:00:01Z"),
"trace-1",
Map.of("rawArchiveUri", "file:///archive/" + eventId + ".bin"),
new RealtimePayload(
42.1, 1234.5, 87.0, null, null,
null, null, null, 8.3, null, null,
RealtimePayload.VehicleState.STARTED,
null, null, null, null, null,
113.12, 23.45, null, null, null, null
));
}
private static VehicleEvent.Location locationEvent(String eventId) {
return new VehicleEvent.Location(
eventId,
"LTEST000000000001",
ProtocolId.GB32960,
Instant.parse("2026-06-21T16:00:00Z"),
Instant.parse("2026-06-21T16:00:01Z"),
"trace-location",
Map.of("rawArchiveUri", "file:///archive/" + eventId + ".bin"),
new LocationPayload(113.12, 23.45, 0, 42.1, 90, 0, 0));
}
private static VehicleEvent.RawArchive rawEvent(String eventId) {
return new VehicleEvent.RawArchive(
eventId,
"LTEST000000000001",
ProtocolId.GB32960,
Instant.parse("2026-06-21T16:00:00Z"),
Instant.parse("2026-06-21T16:00:01Z"),
"trace-raw",
Map.of(),
0x0002,
0,
new byte[]{0x23, 0x23});
}
}

View File

@@ -0,0 +1,41 @@
package com.lingniu.ingest.eventfilestore.config;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.eventfilestore.EventFileStoreSink;
import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations;
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
import java.nio.file.Path;
import static org.assertj.core.api.Assertions.assertThat;
class EventFileStoreAutoConfigurationTest {
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
.withConfiguration(AutoConfigurations.of(EventFileStoreAutoConfiguration.class));
@Test
void createsStoreAndSinkWhenEnabled() {
contextRunner
.withPropertyValues(
"lingniu.ingest.event-file-store.enabled=true",
"lingniu.ingest.event-file-store.path=target/test-event-store")
.run(context -> {
assertThat(context).hasSingleBean(EventFileStore.class);
assertThat(context).hasSingleBean(EventFileStoreSink.class);
assertThat(context.getBean(EventFileStoreProperties.class).getPath())
.isEqualTo(Path.of("target/test-event-store").toString());
});
}
@Test
void backsOffWhenDisabled() {
contextRunner
.withPropertyValues("lingniu.ingest.event-file-store.enabled=false")
.run(context -> {
assertThat(context).doesNotHaveBean(EventFileStore.class);
assertThat(context).doesNotHaveBean(EventFileStoreSink.class);
});
}
}