feat: make gb32960 archive history query production ready
This commit is contained in:
60
modules/sinks/event-file-store/pom.xml
Normal file
60
modules/sinks/event-file-store/pom.xml
Normal file
@@ -0,0 +1,60 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>lingniu-vehicle-ingest</artifactId>
|
||||
<version>0.1.0-SNAPSHOT</version>
|
||||
<relativePath>../../../pom.xml</relativePath>
|
||||
</parent>
|
||||
<artifactId>event-file-store</artifactId>
|
||||
<name>event-file-store</name>
|
||||
<description>车辆事件文件型明细库:Parquet 存储 + DuckDB 时间顺序查询。</description>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.lingniu.ingest</groupId>
|
||||
<artifactId>ingest-api</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-autoconfigure</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-configuration-processor</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.duckdb</groupId>
|
||||
<artifactId>duckdb_jdbc</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.datatype</groupId>
|
||||
<artifactId>jackson-datatype-jsr310</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
@@ -0,0 +1,485 @@
|
||||
package com.lingniu.ingest.eventfilestore;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lingniu.ingest.api.ProtocolId;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.ZoneId;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public final class DuckDbParquetEventFileStore implements EventFileStore {
|
||||
|
||||
private static final TypeReference<Map<String, String>> STRING_MAP =
|
||||
new TypeReference<>() {};
|
||||
private static final String HEADER_VERSION = "event-records-v1";
|
||||
|
||||
private final Path root;
|
||||
private final Path indexPath;
|
||||
private final ZoneId partitionZone;
|
||||
private final ObjectMapper objectMapper;
|
||||
private volatile boolean indexInitialized;
|
||||
|
||||
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone) {
|
||||
this(root, partitionZone, new ObjectMapper());
|
||||
}
|
||||
|
||||
public DuckDbParquetEventFileStore(Path root, ZoneId partitionZone, ObjectMapper objectMapper) {
|
||||
if (root == null) {
|
||||
throw new IllegalArgumentException("root must not be null");
|
||||
}
|
||||
this.root = root.toAbsolutePath();
|
||||
this.indexPath = this.root.resolve("events.duckdb");
|
||||
this.partitionZone = partitionZone == null ? ZoneId.of("Asia/Shanghai") : partitionZone;
|
||||
this.objectMapper = objectMapper == null ? new ObjectMapper() : objectMapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void appendAll(List<EventFileRecord> records) throws IOException {
|
||||
if (records == null || records.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
Map<Partition, List<EventFileRecord>> byPartition = new LinkedHashMap<>();
|
||||
for (EventFileRecord record : records) {
|
||||
Partition partition = partition(record);
|
||||
byPartition.computeIfAbsent(partition, ignored -> new ArrayList<>()).add(record);
|
||||
}
|
||||
for (Map.Entry<Partition, List<EventFileRecord>> entry : byPartition.entrySet()) {
|
||||
writePartition(entry.getKey(), entry.getValue());
|
||||
appendIndex(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<EventFileRecord> query(EventFileQuery query) throws IOException {
|
||||
if (query.vin() != null) {
|
||||
return queryParquet(query);
|
||||
}
|
||||
ensureIndexInitialized();
|
||||
return queryIndex(query);
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized EventFileRecord findByRawArchiveUri(String rawArchiveUri) throws IOException {
|
||||
if (rawArchiveUri == null || rawArchiveUri.isBlank()) {
|
||||
return null;
|
||||
}
|
||||
ensureIndexInitialized();
|
||||
String sql = """
|
||||
SELECT event_id, protocol, event_type, vin,
|
||||
event_time_ms, ingest_time_ms,
|
||||
raw_archive_uri, metadata_json, payload_json
|
||||
FROM event_records
|
||||
WHERE raw_archive_uri = ?
|
||||
ORDER BY event_type = 'RAW_ARCHIVE' DESC, ingest_time_ms DESC, event_id DESC
|
||||
LIMIT 1
|
||||
""";
|
||||
try (Connection connection = DriverManager.getConnection(indexJdbcUrl());
|
||||
PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
ps.setString(1, rawArchiveUri);
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
return rs.next() ? record(rs) : null;
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
throw new IOException("query duckdb event index by raw archive uri failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
private List<EventFileRecord> queryParquet(EventFileQuery query) throws IOException {
|
||||
List<Path> parquetFiles = parquetFiles(query);
|
||||
if (parquetFiles.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
String fileList = toDuckDbList(parquetFiles);
|
||||
String order = query.order() == EventFileQuery.Order.DESC ? "DESC" : "ASC";
|
||||
List<String> predicates = new ArrayList<>();
|
||||
if (query.vin() != null) {
|
||||
predicates.add("vin = '" + escapeSql(query.vin()) + "'");
|
||||
}
|
||||
if (query.eventType() != null) {
|
||||
predicates.add("event_type = '" + escapeSql(query.eventType()) + "'");
|
||||
}
|
||||
if (query.eventTimeFrom() != null) {
|
||||
predicates.add("event_time_ms >= " + query.eventTimeFrom().toEpochMilli());
|
||||
}
|
||||
if (query.eventTimeTo() != null) {
|
||||
predicates.add("event_time_ms <= " + query.eventTimeTo().toEpochMilli());
|
||||
}
|
||||
String where = predicates.isEmpty() ? "" : "WHERE " + String.join(" AND ", predicates) + "\n";
|
||||
String sql = """
|
||||
SELECT event_id, protocol, event_type, vin,
|
||||
event_time_ms, event_time, ingest_time_ms, ingest_time,
|
||||
raw_archive_uri, metadata_json, payload_json
|
||||
FROM read_parquet(%s)
|
||||
%s
|
||||
ORDER BY event_time_ms %s, ingest_time_ms %s, event_id %s
|
||||
LIMIT %d
|
||||
""".formatted(fileList, where, order, order, order, query.limit());
|
||||
try (Connection connection = DriverManager.getConnection("jdbc:duckdb:");
|
||||
Statement statement = connection.createStatement();
|
||||
ResultSet rs = statement.executeQuery(sql)) {
|
||||
List<EventFileRecord> result = new ArrayList<>();
|
||||
while (rs.next()) {
|
||||
result.add(new EventFileRecord(
|
||||
rs.getString("event_id"),
|
||||
protocol(rs.getString("protocol")),
|
||||
rs.getString("event_type"),
|
||||
rs.getString("vin"),
|
||||
Instant.ofEpochMilli(rs.getLong("event_time_ms")),
|
||||
Instant.ofEpochMilli(rs.getLong("ingest_time_ms")),
|
||||
rs.getString("raw_archive_uri"),
|
||||
readMetadata(rs.getString("metadata_json")),
|
||||
rs.getString("payload_json")
|
||||
));
|
||||
}
|
||||
return result;
|
||||
} catch (SQLException e) {
|
||||
throw new IOException("query parquet event store failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
private List<EventFileRecord> queryIndex(EventFileQuery query) throws IOException {
|
||||
String order = query.order() == EventFileQuery.Order.DESC ? "DESC" : "ASC";
|
||||
StringBuilder where = new StringBuilder("""
|
||||
WHERE protocol = ?
|
||||
AND partition_date BETWEEN CAST(? AS DATE) AND CAST(? AS DATE)
|
||||
""");
|
||||
if (query.vin() != null) {
|
||||
where.append(" AND vin = ?\n");
|
||||
}
|
||||
if (query.eventType() != null) {
|
||||
where.append(" AND event_type = ?\n");
|
||||
}
|
||||
if (query.eventTimeFrom() != null) {
|
||||
where.append(" AND event_time_ms >= ?\n");
|
||||
}
|
||||
if (query.eventTimeTo() != null) {
|
||||
where.append(" AND event_time_ms <= ?\n");
|
||||
}
|
||||
String sql = """
|
||||
SELECT event_id, protocol, event_type, vin,
|
||||
event_time_ms, ingest_time_ms,
|
||||
raw_archive_uri, metadata_json, payload_json
|
||||
FROM event_records
|
||||
%s
|
||||
ORDER BY event_time_ms %s, ingest_time_ms %s, event_id %s
|
||||
LIMIT ?
|
||||
""".formatted(where, order, order, order);
|
||||
try (Connection connection = DriverManager.getConnection(indexJdbcUrl());
|
||||
PreparedStatement ps = connection.prepareStatement(sql)) {
|
||||
int index = 1;
|
||||
ps.setString(index++, query.protocol().name());
|
||||
ps.setString(index++, query.dateFrom().toString());
|
||||
ps.setString(index++, query.dateTo().toString());
|
||||
if (query.vin() != null) {
|
||||
ps.setString(index++, query.vin());
|
||||
}
|
||||
if (query.eventType() != null) {
|
||||
ps.setString(index++, query.eventType());
|
||||
}
|
||||
if (query.eventTimeFrom() != null) {
|
||||
ps.setLong(index++, query.eventTimeFrom().toEpochMilli());
|
||||
}
|
||||
if (query.eventTimeTo() != null) {
|
||||
ps.setLong(index++, query.eventTimeTo().toEpochMilli());
|
||||
}
|
||||
ps.setInt(index, query.limit());
|
||||
try (ResultSet rs = ps.executeQuery()) {
|
||||
List<EventFileRecord> result = new ArrayList<>();
|
||||
while (rs.next()) {
|
||||
result.add(record(rs));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
throw new IOException("query duckdb event index failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void writePartition(Partition partition, List<EventFileRecord> records) throws IOException {
|
||||
Path dir = partition.dir(root);
|
||||
Files.createDirectories(dir);
|
||||
Path partFile = dir.resolve("events.parquet");
|
||||
Path tempFile = dir.resolve("events-" + UUID.randomUUID() + ".tmp.parquet");
|
||||
|
||||
try (Connection connection = DriverManager.getConnection("jdbc:duckdb:");
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute("""
|
||||
CREATE TEMPORARY TABLE event_records (
|
||||
event_id VARCHAR,
|
||||
protocol VARCHAR,
|
||||
event_type VARCHAR,
|
||||
vin VARCHAR,
|
||||
event_time_ms BIGINT,
|
||||
event_time VARCHAR,
|
||||
ingest_time_ms BIGINT,
|
||||
ingest_time VARCHAR,
|
||||
raw_archive_uri VARCHAR,
|
||||
metadata_json VARCHAR,
|
||||
payload_json VARCHAR
|
||||
)
|
||||
""");
|
||||
try (PreparedStatement ps = connection.prepareStatement("""
|
||||
INSERT INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""")) {
|
||||
for (EventFileRecord record : records) {
|
||||
ps.setString(1, record.eventId());
|
||||
ps.setString(2, record.protocol().name());
|
||||
ps.setString(3, record.eventType());
|
||||
ps.setString(4, record.vin());
|
||||
ps.setLong(5, record.eventTime().toEpochMilli());
|
||||
ps.setString(6, record.eventTime().toString());
|
||||
ps.setLong(7, record.ingestTime().toEpochMilli());
|
||||
ps.setString(8, record.ingestTime().toString());
|
||||
ps.setString(9, record.rawArchiveUri());
|
||||
ps.setString(10, objectMapper.writeValueAsString(record.metadata()));
|
||||
ps.setString(11, record.payloadJson());
|
||||
ps.addBatch();
|
||||
}
|
||||
ps.executeBatch();
|
||||
}
|
||||
if (Files.exists(partFile)) {
|
||||
statement.execute("""
|
||||
CREATE TEMPORARY TABLE combined_records AS
|
||||
SELECT event_id, protocol, event_type, vin,
|
||||
event_time_ms, event_time, ingest_time_ms, ingest_time,
|
||||
raw_archive_uri, metadata_json, payload_json
|
||||
FROM read_parquet('%s')
|
||||
UNION ALL
|
||||
SELECT event_id, protocol, event_type, vin,
|
||||
event_time_ms, event_time, ingest_time_ms, ingest_time,
|
||||
raw_archive_uri, metadata_json, payload_json
|
||||
FROM event_records
|
||||
""".formatted(escapeSql(partFile)));
|
||||
statement.execute("COPY combined_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
|
||||
} else {
|
||||
statement.execute("COPY event_records TO '" + escapeSql(tempFile) + "' (FORMAT parquet)");
|
||||
}
|
||||
Files.move(tempFile, partFile, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
|
||||
} catch (SQLException e) {
|
||||
throw new IOException("write parquet event store failed: " + partFile, e);
|
||||
} finally {
|
||||
Files.deleteIfExists(tempFile);
|
||||
}
|
||||
}
|
||||
|
||||
private void appendIndex(Partition partition, List<EventFileRecord> records) throws IOException {
|
||||
ensureIndexInitialized();
|
||||
try (Connection connection = DriverManager.getConnection(indexJdbcUrl())) {
|
||||
insertIndex(connection, partition, records);
|
||||
} catch (SQLException e) {
|
||||
throw new IOException("write duckdb event index failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void ensureIndexInitialized() throws IOException {
|
||||
if (indexInitialized) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
Files.createDirectories(root);
|
||||
} catch (IOException e) {
|
||||
throw new IOException("create event store root failed: " + root, e);
|
||||
}
|
||||
try (Connection connection = DriverManager.getConnection(indexJdbcUrl());
|
||||
Statement statement = connection.createStatement()) {
|
||||
createIndexSchema(statement);
|
||||
if (isIndexEmpty(statement)) {
|
||||
backfillIndexFromParquet(statement);
|
||||
}
|
||||
indexInitialized = true;
|
||||
} catch (SQLException e) {
|
||||
throw new IOException("initialize duckdb event index failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
private void createIndexSchema(Statement statement) throws SQLException {
|
||||
statement.execute("""
|
||||
CREATE TABLE IF NOT EXISTS event_records (
|
||||
event_id VARCHAR PRIMARY KEY,
|
||||
protocol VARCHAR,
|
||||
event_type VARCHAR,
|
||||
vin VARCHAR,
|
||||
event_time_ms BIGINT,
|
||||
ingest_time_ms BIGINT,
|
||||
partition_date DATE,
|
||||
raw_archive_uri VARCHAR,
|
||||
metadata_json VARCHAR,
|
||||
payload_json VARCHAR
|
||||
)
|
||||
""");
|
||||
statement.execute("CREATE INDEX IF NOT EXISTS event_records_protocol_date_time_idx ON event_records(protocol, partition_date, event_time_ms)");
|
||||
statement.execute("CREATE INDEX IF NOT EXISTS event_records_vin_date_time_idx ON event_records(vin, partition_date, event_time_ms)");
|
||||
statement.execute("CREATE INDEX IF NOT EXISTS event_records_vin_type_date_time_idx ON event_records(protocol, vin, event_type, partition_date, event_time_ms)");
|
||||
statement.execute("CREATE INDEX IF NOT EXISTS event_records_raw_archive_uri_idx ON event_records(raw_archive_uri)");
|
||||
}
|
||||
|
||||
private boolean isIndexEmpty(Statement statement) throws SQLException {
|
||||
try (ResultSet rs = statement.executeQuery("SELECT COUNT(*) FROM event_records")) {
|
||||
return rs.next() && rs.getLong(1) == 0L;
|
||||
}
|
||||
}
|
||||
|
||||
private void backfillIndexFromParquet(Statement statement) throws SQLException, IOException {
|
||||
List<Path> files = allParquetFiles();
|
||||
if (files.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String fileList = toDuckDbList(files);
|
||||
statement.execute("""
|
||||
INSERT OR IGNORE INTO event_records
|
||||
SELECT event_id, protocol, event_type, vin,
|
||||
event_time_ms, ingest_time_ms,
|
||||
CAST(date AS DATE) AS partition_date,
|
||||
raw_archive_uri, metadata_json, payload_json
|
||||
FROM read_parquet(%s, hive_partitioning = true)
|
||||
""".formatted(fileList));
|
||||
}
|
||||
|
||||
private void insertIndex(Connection connection, Partition partition, List<EventFileRecord> records)
|
||||
throws SQLException, IOException {
|
||||
try (PreparedStatement ps = connection.prepareStatement("""
|
||||
INSERT OR REPLACE INTO event_records VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""")) {
|
||||
for (EventFileRecord record : records) {
|
||||
ps.setString(1, record.eventId());
|
||||
ps.setString(2, record.protocol().name());
|
||||
ps.setString(3, record.eventType());
|
||||
ps.setString(4, record.vin());
|
||||
ps.setLong(5, record.eventTime().toEpochMilli());
|
||||
ps.setLong(6, record.ingestTime().toEpochMilli());
|
||||
ps.setString(7, partition.date().toString());
|
||||
ps.setString(8, record.rawArchiveUri());
|
||||
ps.setString(9, objectMapper.writeValueAsString(record.metadata()));
|
||||
ps.setString(10, record.payloadJson());
|
||||
ps.addBatch();
|
||||
}
|
||||
ps.executeBatch();
|
||||
}
|
||||
}
|
||||
|
||||
private List<Path> parquetFiles(EventFileQuery query) throws IOException {
|
||||
List<Path> files = new ArrayList<>();
|
||||
LocalDate date = query.dateFrom();
|
||||
while (!date.isAfter(query.dateTo())) {
|
||||
Path dir = query.vin() == null
|
||||
? partitionDir(query.protocol(), date)
|
||||
: partitionDir(query.protocol(), date).resolve("vehicle=" + storageName(query.vin()));
|
||||
if (Files.isDirectory(dir)) {
|
||||
try (Stream<Path> stream = Files.walk(dir)) {
|
||||
stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
|
||||
.sorted()
|
||||
.forEach(files::add);
|
||||
}
|
||||
}
|
||||
date = date.plusDays(1);
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
private List<Path> allParquetFiles() throws IOException {
|
||||
if (!Files.isDirectory(root)) {
|
||||
return List.of();
|
||||
}
|
||||
try (Stream<Path> stream = Files.walk(root)) {
|
||||
return stream.filter(path -> path.getFileName().toString().endsWith(".parquet"))
|
||||
.sorted()
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
|
||||
private EventFileRecord record(ResultSet rs) throws SQLException, IOException {
|
||||
return new EventFileRecord(
|
||||
rs.getString("event_id"),
|
||||
protocol(rs.getString("protocol")),
|
||||
rs.getString("event_type"),
|
||||
rs.getString("vin"),
|
||||
Instant.ofEpochMilli(rs.getLong("event_time_ms")),
|
||||
Instant.ofEpochMilli(rs.getLong("ingest_time_ms")),
|
||||
rs.getString("raw_archive_uri"),
|
||||
readMetadata(rs.getString("metadata_json")),
|
||||
rs.getString("payload_json")
|
||||
);
|
||||
}
|
||||
|
||||
private Partition partition(EventFileRecord record) {
|
||||
return new Partition(
|
||||
record.protocol(),
|
||||
LocalDate.ofInstant(record.eventTime(), partitionZone),
|
||||
storageName(record.vin()));
|
||||
}
|
||||
|
||||
private Path partitionDir(ProtocolId protocol, LocalDate date) {
|
||||
return root.resolve("protocol=" + protocol.name()).resolve("date=" + date);
|
||||
}
|
||||
|
||||
private static ProtocolId protocol(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return ProtocolId.UNKNOWN;
|
||||
}
|
||||
try {
|
||||
return ProtocolId.valueOf(value);
|
||||
} catch (IllegalArgumentException ex) {
|
||||
return ProtocolId.UNKNOWN;
|
||||
}
|
||||
}
|
||||
|
||||
private Map<String, String> readMetadata(String json) throws IOException {
|
||||
if (json == null || json.isBlank()) {
|
||||
return Map.of();
|
||||
}
|
||||
return objectMapper.readValue(json, STRING_MAP);
|
||||
}
|
||||
|
||||
private static String toDuckDbList(List<Path> files) {
|
||||
return files.stream()
|
||||
.map(path -> "'" + escapeSql(path) + "'")
|
||||
.reduce((left, right) -> left + "," + right)
|
||||
.map(paths -> "[" + paths + "]")
|
||||
.orElse("[]");
|
||||
}
|
||||
|
||||
private static String escapeSql(Path path) {
|
||||
return escapeSql(path.toAbsolutePath().toString());
|
||||
}
|
||||
|
||||
private static String escapeSql(String value) {
|
||||
return value.replace("'", "''");
|
||||
}
|
||||
|
||||
private String indexJdbcUrl() {
|
||||
return "jdbc:duckdb:" + indexPath;
|
||||
}
|
||||
|
||||
private static String storageName(String vin) {
|
||||
if (vin == null || vin.isBlank()) {
|
||||
return "_unknown";
|
||||
}
|
||||
return vin.replaceAll("[^A-Za-z0-9._-]", "_");
|
||||
}
|
||||
|
||||
private record Partition(ProtocolId protocol, LocalDate date, String vehicle) {
|
||||
private Path dir(Path root) {
|
||||
return root.resolve("protocol=" + protocol.name())
|
||||
.resolve("date=" + date)
|
||||
.resolve("vehicle=" + vehicle)
|
||||
.resolve("header=" + HEADER_VERSION);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package com.lingniu.ingest.eventfilestore;
|
||||
|
||||
import com.lingniu.ingest.api.ProtocolId;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
|
||||
public record EventFileQuery(
|
||||
ProtocolId protocol,
|
||||
LocalDate dateFrom,
|
||||
LocalDate dateTo,
|
||||
Instant eventTimeFrom,
|
||||
Instant eventTimeTo,
|
||||
Order order,
|
||||
int limit,
|
||||
String vin,
|
||||
String eventType
|
||||
) {
|
||||
public enum Order {
|
||||
ASC,
|
||||
DESC
|
||||
}
|
||||
|
||||
public EventFileQuery {
|
||||
if (protocol == null) {
|
||||
throw new IllegalArgumentException("protocol must not be null");
|
||||
}
|
||||
if (dateFrom == null || dateTo == null) {
|
||||
throw new IllegalArgumentException("date range must not be null");
|
||||
}
|
||||
if (dateTo.isBefore(dateFrom)) {
|
||||
throw new IllegalArgumentException("dateTo must not be before dateFrom");
|
||||
}
|
||||
if (eventTimeFrom != null && eventTimeTo != null && eventTimeTo.isBefore(eventTimeFrom)) {
|
||||
throw new IllegalArgumentException("eventTimeTo must not be before eventTimeFrom");
|
||||
}
|
||||
order = order == null ? Order.ASC : order;
|
||||
limit = limit <= 0 ? 100 : limit;
|
||||
vin = vin == null || vin.isBlank() ? null : vin.trim();
|
||||
eventType = eventType == null || eventType.isBlank() ? null : eventType.trim();
|
||||
}
|
||||
|
||||
public EventFileQuery(ProtocolId protocol,
|
||||
LocalDate dateFrom,
|
||||
LocalDate dateTo,
|
||||
Order order,
|
||||
int limit,
|
||||
String vin,
|
||||
String eventType) {
|
||||
this(protocol, dateFrom, dateTo, null, null, order, limit, vin, eventType);
|
||||
}
|
||||
|
||||
public EventFileQuery(ProtocolId protocol,
|
||||
LocalDate dateFrom,
|
||||
LocalDate dateTo,
|
||||
Order order,
|
||||
int limit) {
|
||||
this(protocol, dateFrom, dateTo, order, limit, null, null);
|
||||
}
|
||||
|
||||
public EventFileQuery(ProtocolId protocol,
|
||||
LocalDate dateFrom,
|
||||
LocalDate dateTo,
|
||||
Order order,
|
||||
int limit,
|
||||
String vin) {
|
||||
this(protocol, dateFrom, dateTo, order, limit, vin, null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.lingniu.ingest.eventfilestore;
|
||||
|
||||
import com.lingniu.ingest.api.ProtocolId;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 文件型明细库的一条可查询记录。
|
||||
*
|
||||
* <p>{@code payloadJson} 保存统一 telemetry snapshot JSON;公共列只保留排序、分区、
|
||||
* 追溯和通用展示需要的最小字段。
|
||||
*/
|
||||
public record EventFileRecord(
|
||||
String eventId,
|
||||
ProtocolId protocol,
|
||||
String eventType,
|
||||
String vin,
|
||||
Instant eventTime,
|
||||
Instant ingestTime,
|
||||
String rawArchiveUri,
|
||||
Map<String, String> metadata,
|
||||
String payloadJson
|
||||
) {
|
||||
public EventFileRecord {
|
||||
if (eventId == null || eventId.isBlank()) {
|
||||
throw new IllegalArgumentException("eventId must not be blank");
|
||||
}
|
||||
if (protocol == null) {
|
||||
throw new IllegalArgumentException("protocol must not be null");
|
||||
}
|
||||
if (eventTime == null) {
|
||||
throw new IllegalArgumentException("eventTime must not be null");
|
||||
}
|
||||
if (ingestTime == null) {
|
||||
throw new IllegalArgumentException("ingestTime must not be null");
|
||||
}
|
||||
eventType = eventType == null ? "" : eventType;
|
||||
vin = vin == null ? "" : vin;
|
||||
rawArchiveUri = rawArchiveUri == null ? "" : rawArchiveUri;
|
||||
metadata = metadata == null ? Map.of() : Map.copyOf(metadata);
|
||||
payloadJson = payloadJson == null || payloadJson.isBlank() ? "{}" : payloadJson;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package com.lingniu.ingest.eventfilestore;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
public interface EventFileStore {
|
||||
|
||||
default void append(EventFileRecord record) throws IOException {
|
||||
appendAll(List.of(record));
|
||||
}
|
||||
|
||||
void appendAll(List<EventFileRecord> records) throws IOException;
|
||||
|
||||
List<EventFileRecord> query(EventFileQuery query) throws IOException;
|
||||
|
||||
default EventFileRecord findByRawArchiveUri(String rawArchiveUri) throws IOException {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package com.lingniu.ingest.eventfilestore;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lingniu.ingest.api.event.RawArchiveKeys;
|
||||
import com.lingniu.ingest.api.event.TelemetrySnapshot;
|
||||
import com.lingniu.ingest.api.event.VehicleEvent;
|
||||
import com.lingniu.ingest.api.event.VehicleEventTelemetrySnapshotMapper;
|
||||
import com.lingniu.ingest.api.sink.EventSink;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* EventBus 出口:把解析后的 {@link VehicleEvent} 明细写入 Parquet 文件库。
|
||||
*
|
||||
* <p>{@link VehicleEvent.RawArchive} 只写可查询索引,不写原始 bytes;原始 bytes 本体仍由
|
||||
* sink-archive 独立冷存。
|
||||
*/
|
||||
public final class EventFileStoreSink implements EventSink, AutoCloseable {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(EventFileStoreSink.class);
|
||||
|
||||
private final EventFileStore store;
|
||||
private final ObjectMapper objectMapper;
|
||||
private final int batchSize;
|
||||
private final List<EventFileRecord> buffer = new ArrayList<>();
|
||||
private final ScheduledExecutorService flusher;
|
||||
|
||||
public EventFileStoreSink(EventFileStore store, ObjectMapper objectMapper) {
|
||||
this(store, objectMapper, 1);
|
||||
}
|
||||
|
||||
public EventFileStoreSink(EventFileStore store, ObjectMapper objectMapper, int batchSize) {
|
||||
this(store, objectMapper, batchSize, 0);
|
||||
}
|
||||
|
||||
public EventFileStoreSink(EventFileStore store,
|
||||
ObjectMapper objectMapper,
|
||||
int batchSize,
|
||||
long flushIntervalMillis) {
|
||||
if (store == null) {
|
||||
throw new IllegalArgumentException("store must not be null");
|
||||
}
|
||||
this.store = store;
|
||||
this.objectMapper = objectMapper == null ? new ObjectMapper() : objectMapper;
|
||||
this.batchSize = Math.max(1, batchSize);
|
||||
this.flusher = flushIntervalMillis > 0 && this.batchSize > 1
|
||||
? Executors.newSingleThreadScheduledExecutor(r -> {
|
||||
Thread t = Thread.ofVirtual().name("event-file-store-flush-", 0).factory().newThread(r);
|
||||
t.setDaemon(true);
|
||||
return t;
|
||||
})
|
||||
: null;
|
||||
if (flusher != null) {
|
||||
flusher.scheduleWithFixedDelay(this::flushQuietly,
|
||||
flushIntervalMillis, flushIntervalMillis, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return "event-file-store";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean accepts(VehicleEvent event) {
|
||||
return event != null
|
||||
&& !(event instanceof VehicleEvent.Realtime)
|
||||
&& !(event instanceof VehicleEvent.Location);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<Void> publish(VehicleEvent event) {
|
||||
if (!accepts(event)) {
|
||||
return CompletableFuture.completedFuture(null);
|
||||
}
|
||||
CompletableFuture<Void> future = new CompletableFuture<>();
|
||||
try {
|
||||
EventFileRecord record = toRecord(event);
|
||||
if (batchSize <= 1) {
|
||||
store.append(record);
|
||||
} else {
|
||||
appendBuffered(record);
|
||||
}
|
||||
future.complete(null);
|
||||
} catch (Exception e) {
|
||||
future.completeExceptionally(e);
|
||||
}
|
||||
return future;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<Void> publishBatch(List<VehicleEvent> batch) {
|
||||
if (batch == null || batch.isEmpty()) {
|
||||
return CompletableFuture.completedFuture(null);
|
||||
}
|
||||
CompletableFuture<Void> future = new CompletableFuture<>();
|
||||
try {
|
||||
List<EventFileRecord> records = new ArrayList<>();
|
||||
for (VehicleEvent event : batch) {
|
||||
if (accepts(event)) {
|
||||
records.add(toRecord(event));
|
||||
}
|
||||
}
|
||||
if (!records.isEmpty()) {
|
||||
store.appendAll(records);
|
||||
}
|
||||
future.complete(null);
|
||||
} catch (Exception e) {
|
||||
future.completeExceptionally(e);
|
||||
}
|
||||
return future;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws Exception {
|
||||
if (flusher != null) {
|
||||
flusher.shutdownNow();
|
||||
}
|
||||
flush();
|
||||
}
|
||||
|
||||
private void appendBuffered(EventFileRecord record) throws IOException {
|
||||
List<EventFileRecord> toFlush = null;
|
||||
synchronized (buffer) {
|
||||
buffer.add(record);
|
||||
if (buffer.size() >= batchSize) {
|
||||
toFlush = new ArrayList<>(buffer);
|
||||
buffer.clear();
|
||||
}
|
||||
}
|
||||
if (toFlush != null) {
|
||||
store.appendAll(toFlush);
|
||||
}
|
||||
}
|
||||
|
||||
private void flush() throws IOException {
|
||||
List<EventFileRecord> toFlush;
|
||||
synchronized (buffer) {
|
||||
if (buffer.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
toFlush = new ArrayList<>(buffer);
|
||||
buffer.clear();
|
||||
}
|
||||
store.appendAll(toFlush);
|
||||
}
|
||||
|
||||
private void flushQuietly() {
|
||||
try {
|
||||
flush();
|
||||
} catch (Exception e) {
|
||||
log.warn("event file store scheduled flush failed", e);
|
||||
}
|
||||
}
|
||||
|
||||
private EventFileRecord toRecord(VehicleEvent event) throws IOException {
|
||||
if (event instanceof VehicleEvent.RawArchive raw) {
|
||||
return rawArchiveIndexRecord(raw);
|
||||
}
|
||||
TelemetrySnapshot snapshot = VehicleEventTelemetrySnapshotMapper.toSnapshot(event)
|
||||
.orElseThrow(() -> new IllegalArgumentException("event cannot be converted to telemetry snapshot"));
|
||||
return new EventFileRecord(
|
||||
event.eventId(),
|
||||
event.source(),
|
||||
snapshot.eventType(),
|
||||
event.vin(),
|
||||
event.eventTime(),
|
||||
event.ingestTime(),
|
||||
snapshot.rawArchiveUri(),
|
||||
event.metadata(),
|
||||
objectMapper.writeValueAsString(snapshotPayload(snapshot))
|
||||
);
|
||||
}
|
||||
|
||||
private EventFileRecord rawArchiveIndexRecord(VehicleEvent.RawArchive raw) throws IOException {
|
||||
String rawArchiveKey = RawArchiveKeys.key(raw);
|
||||
String rawArchiveUri = RawArchiveKeys.logicalUri(rawArchiveKey);
|
||||
Map<String, String> metadata = new LinkedHashMap<>();
|
||||
if (raw.metadata() != null) {
|
||||
metadata.putAll(raw.metadata());
|
||||
}
|
||||
metadata.putIfAbsent(RawArchiveKeys.META_EVENT_ID, raw.eventId());
|
||||
metadata.putIfAbsent(RawArchiveKeys.META_KEY, rawArchiveKey);
|
||||
metadata.putIfAbsent(RawArchiveKeys.META_URI, rawArchiveUri);
|
||||
return new EventFileRecord(
|
||||
raw.eventId(),
|
||||
raw.source(),
|
||||
"RAW_ARCHIVE",
|
||||
raw.vin(),
|
||||
raw.eventTime(),
|
||||
raw.ingestTime(),
|
||||
rawArchiveUri,
|
||||
metadata,
|
||||
objectMapper.writeValueAsString(rawArchivePayload(raw, rawArchiveKey, rawArchiveUri))
|
||||
);
|
||||
}
|
||||
|
||||
private static Map<String, Object> snapshotPayload(TelemetrySnapshot snapshot) {
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put("eventId", snapshot.eventId());
|
||||
payload.put("vin", snapshot.vin());
|
||||
payload.put("protocol", snapshot.protocol().name());
|
||||
payload.put("eventType", snapshot.eventType());
|
||||
payload.put("eventTime", snapshot.eventTime().toString());
|
||||
payload.put("ingestTime", snapshot.ingestTime().toString());
|
||||
payload.put("rawArchiveUri", snapshot.rawArchiveUri());
|
||||
payload.put("metadata", snapshot.metadata());
|
||||
payload.put("fields", snapshot.fields());
|
||||
return payload;
|
||||
}
|
||||
|
||||
private static Map<String, Object> rawArchivePayload(VehicleEvent.RawArchive raw,
|
||||
String rawArchiveKey,
|
||||
String rawArchiveUri) {
|
||||
Map<String, Object> payload = new LinkedHashMap<>();
|
||||
payload.put("eventId", raw.eventId());
|
||||
payload.put("vin", raw.vin());
|
||||
payload.put("protocol", raw.source().name());
|
||||
payload.put("eventType", "RAW_ARCHIVE");
|
||||
payload.put("eventTime", raw.eventTime().toString());
|
||||
payload.put("ingestTime", raw.ingestTime().toString());
|
||||
payload.put("rawArchiveUri", rawArchiveUri);
|
||||
payload.put("rawArchiveKey", rawArchiveKey);
|
||||
payload.put("command", hex(raw.command()));
|
||||
payload.put("infoType", hex(raw.infoType()));
|
||||
payload.put("rawSizeBytes", raw.rawBytes() == null ? 0 : raw.rawBytes().length);
|
||||
payload.put("metadata", raw.metadata());
|
||||
return payload;
|
||||
}
|
||||
|
||||
private static String hex(int value) {
|
||||
return "0x" + String.format("%04X", value);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package com.lingniu.ingest.eventfilestore.config;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
|
||||
import com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore;
|
||||
import com.lingniu.ingest.eventfilestore.EventFileStore;
|
||||
import com.lingniu.ingest.eventfilestore.EventFileStoreSink;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.beans.factory.ObjectProvider;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.time.ZoneId;
|
||||
|
||||
@AutoConfiguration
|
||||
@EnableConfigurationProperties(EventFileStoreProperties.class)
|
||||
@ConditionalOnProperty(
|
||||
prefix = "lingniu.ingest.event-file-store",
|
||||
name = "enabled",
|
||||
havingValue = "true")
|
||||
public class EventFileStoreAutoConfiguration {
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public EventFileStore eventFileStore(EventFileStoreProperties properties,
|
||||
ObjectProvider<ObjectMapper> objectMapper) {
|
||||
ObjectMapper mapper = mapper(objectMapper);
|
||||
return new DuckDbParquetEventFileStore(
|
||||
Path.of(properties.getPath()),
|
||||
ZoneId.of(properties.getZoneId()),
|
||||
mapper);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@ConditionalOnMissingBean
|
||||
public EventFileStoreSink eventFileStoreSink(EventFileStore store,
|
||||
EventFileStoreProperties properties,
|
||||
ObjectProvider<ObjectMapper> objectMapper) {
|
||||
return new EventFileStoreSink(
|
||||
store,
|
||||
mapper(objectMapper),
|
||||
properties.getBatchSize(),
|
||||
properties.getFlushIntervalMillis());
|
||||
}
|
||||
|
||||
private static ObjectMapper mapper(ObjectProvider<ObjectMapper> provider) {
|
||||
ObjectMapper base = provider.getIfAvailable(ObjectMapper::new);
|
||||
return base.copy().registerModule(new JavaTimeModule());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.lingniu.ingest.eventfilestore.config;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@ConfigurationProperties(prefix = "lingniu.ingest.event-file-store")
|
||||
public class EventFileStoreProperties {
|
||||
|
||||
/**
|
||||
* 默认关闭,避免仅引入依赖就开始写明细文件。
|
||||
*/
|
||||
private boolean enabled = false;
|
||||
|
||||
/**
|
||||
* Parquet 文件库根路径。
|
||||
*/
|
||||
private String path = "./event-store";
|
||||
|
||||
/**
|
||||
* 按事件时间分区时使用的业务时区。
|
||||
*/
|
||||
private String zoneId = "Asia/Shanghai";
|
||||
|
||||
/**
|
||||
* Sink 缓冲多少条事件后批量写一个 Parquet part 文件。
|
||||
*/
|
||||
private int batchSize = 500;
|
||||
|
||||
/**
|
||||
* 未达到 batchSize 时的最长刷盘间隔。
|
||||
*/
|
||||
private long flushIntervalMillis = 1000;
|
||||
|
||||
public boolean isEnabled() {
|
||||
return enabled;
|
||||
}
|
||||
|
||||
public void setEnabled(boolean enabled) {
|
||||
this.enabled = enabled;
|
||||
}
|
||||
|
||||
public String getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
public void setPath(String path) {
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
public String getZoneId() {
|
||||
return zoneId;
|
||||
}
|
||||
|
||||
public void setZoneId(String zoneId) {
|
||||
this.zoneId = zoneId;
|
||||
}
|
||||
|
||||
public int getBatchSize() {
|
||||
return batchSize;
|
||||
}
|
||||
|
||||
public void setBatchSize(int batchSize) {
|
||||
this.batchSize = batchSize;
|
||||
}
|
||||
|
||||
public long getFlushIntervalMillis() {
|
||||
return flushIntervalMillis;
|
||||
}
|
||||
|
||||
public void setFlushIntervalMillis(long flushIntervalMillis) {
|
||||
this.flushIntervalMillis = flushIntervalMillis;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration
|
||||
@@ -0,0 +1,257 @@
|
||||
package com.lingniu.ingest.eventfilestore;
|
||||
|
||||
import com.lingniu.ingest.api.ProtocolId;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.ZoneId;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class DuckDbParquetEventFileStoreTest {
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
@Test
|
||||
void appendsRecordsToParquetAndQueriesByTimeAscendingAndDescending() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
EventFileRecord first = record("e1", Instant.parse("2026-06-21T16:00:02Z"), "粤A00001");
|
||||
EventFileRecord second = record("e2", Instant.parse("2026-06-21T16:00:01Z"), "粤A00002");
|
||||
|
||||
store.appendAll(List.of(first, second));
|
||||
|
||||
Path dateDir = tempDir.resolve("protocol=GB32960").resolve("date=2026-06-22");
|
||||
assertThat(parquetFiles(dateDir)).hasSize(2);
|
||||
|
||||
EventFileQuery ascending = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.ASC,
|
||||
100);
|
||||
assertThat(store.query(ascending))
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.containsExactly("e2", "e1");
|
||||
|
||||
EventFileQuery descending = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.DESC,
|
||||
100);
|
||||
assertThat(store.query(descending))
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.containsExactly("e1", "e2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void queriesAcrossMultipleDaysInTimeOrder() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
store.append(record("d1", Instant.parse("2026-06-20T16:00:00Z"), "粤A00001"));
|
||||
store.append(record("d2", Instant.parse("2026-06-21T16:00:00Z"), "粤A00001"));
|
||||
|
||||
EventFileQuery query = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-21"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.ASC,
|
||||
100);
|
||||
|
||||
assertThat(store.query(query))
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.containsExactly("d1", "d2");
|
||||
}
|
||||
|
||||
@Test
|
||||
void filtersByVinBeforeApplyingLimit() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
store.append(record("other-1", Instant.parse("2026-06-21T16:00:03Z"), "VIN-OTHER"));
|
||||
store.append(record("target-1", Instant.parse("2026-06-21T16:00:02Z"), "VIN-TARGET"));
|
||||
store.append(record("other-2", Instant.parse("2026-06-21T16:00:01Z"), "VIN-OTHER"));
|
||||
store.append(record("target-2", Instant.parse("2026-06-21T16:00:00Z"), "VIN-TARGET"));
|
||||
|
||||
EventFileQuery query = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.DESC,
|
||||
1,
|
||||
"VIN-TARGET");
|
||||
|
||||
assertThat(store.query(query))
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.containsExactly("target-1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void filtersByVinAndEventTypeBeforeApplyingLimit() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
store.append(rawRecord("raw-other", Instant.parse("2026-06-21T16:00:04Z"), "VIN-OTHER"));
|
||||
store.append(record("business-target", Instant.parse("2026-06-21T16:00:03Z"), "VIN-TARGET"));
|
||||
store.append(rawRecord("raw-target", Instant.parse("2026-06-21T16:00:02Z"), "VIN-TARGET"));
|
||||
|
||||
EventFileQuery query = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.DESC,
|
||||
1,
|
||||
"VIN-TARGET",
|
||||
"RAW_ARCHIVE");
|
||||
|
||||
assertThat(store.query(query))
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.containsExactly("raw-target");
|
||||
}
|
||||
|
||||
@Test
|
||||
void filtersByExactEventTimeRangeBeforeApplyingLimit() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
store.append(rawRecord("before", Instant.parse("2026-06-21T16:00:00Z"), "VIN-TIME"));
|
||||
store.append(rawRecord("inside", Instant.parse("2026-06-21T16:00:02Z"), "VIN-TIME"));
|
||||
store.append(rawRecord("after", Instant.parse("2026-06-21T16:00:04Z"), "VIN-TIME"));
|
||||
|
||||
EventFileQuery query = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
Instant.parse("2026-06-21T16:00:01Z"),
|
||||
Instant.parse("2026-06-21T16:00:03Z"),
|
||||
EventFileQuery.Order.ASC,
|
||||
10,
|
||||
"VIN-TIME",
|
||||
"RAW_ARCHIVE");
|
||||
|
||||
assertThat(store.query(query))
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.containsExactly("inside");
|
||||
}
|
||||
|
||||
@Test
|
||||
void appendsSameVehicleDayHeaderToOneParquetFile() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
store.append(record("first", Instant.parse("2026-06-21T16:00:03Z"), "VIN-SAME"));
|
||||
store.append(record("second", Instant.parse("2026-06-21T16:00:04Z"), "VIN-SAME"));
|
||||
|
||||
Path vehicleDayDir = tempDir.resolve("protocol=GB32960")
|
||||
.resolve("date=2026-06-22")
|
||||
.resolve("vehicle=VIN-SAME")
|
||||
.resolve("header=event-records-v1");
|
||||
assertThat(parquetFiles(vehicleDayDir)).containsExactly(vehicleDayDir.resolve("events.parquet"));
|
||||
|
||||
EventFileQuery query = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.ASC,
|
||||
10,
|
||||
"VIN-SAME");
|
||||
|
||||
assertThat(store.query(query))
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.containsExactly("first", "second");
|
||||
}
|
||||
|
||||
@Test
|
||||
void queriesUseSidecarIndexInsteadOfScanningParquetPartsEveryTime() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
store.append(record("indexed", Instant.parse("2026-06-21T16:00:03Z"), "VIN-INDEXED"));
|
||||
|
||||
Path vehicleDayDir = tempDir.resolve("protocol=GB32960")
|
||||
.resolve("date=2026-06-22")
|
||||
.resolve("vehicle=VIN-INDEXED")
|
||||
.resolve("header=event-records-v1");
|
||||
try (var files = Files.list(vehicleDayDir)) {
|
||||
for (Path file : files.filter(p -> p.getFileName().toString().endsWith(".parquet")).toList()) {
|
||||
Files.delete(file);
|
||||
}
|
||||
}
|
||||
|
||||
EventFileQuery query = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.DESC,
|
||||
10);
|
||||
|
||||
assertThat(store.query(query))
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.containsExactly("indexed");
|
||||
}
|
||||
|
||||
@Test
|
||||
void vinQueriesReadVehicleDayParquetWithoutGlobalDuckDbIndex() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
store.append(rawRecord("raw-target", Instant.parse("2026-06-21T16:00:03Z"), "VIN-DIRECT"));
|
||||
Files.deleteIfExists(tempDir.resolve("events.duckdb"));
|
||||
|
||||
EventFileQuery query = new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.DESC,
|
||||
10,
|
||||
"VIN-DIRECT",
|
||||
"RAW_ARCHIVE");
|
||||
|
||||
assertThat(store.query(query))
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.containsExactly("raw-target");
|
||||
}
|
||||
|
||||
@Test
|
||||
void findsRecordByRawArchiveUriThroughSidecarIndex() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"));
|
||||
EventFileRecord record = record("raw-001", Instant.parse("2026-06-21T16:00:03Z"), "VIN-RAW");
|
||||
store.append(record);
|
||||
|
||||
EventFileRecord found = store.findByRawArchiveUri(record.rawArchiveUri());
|
||||
|
||||
assertThat(found).isNotNull();
|
||||
assertThat(found.eventId()).isEqualTo("raw-001");
|
||||
assertThat(found.rawArchiveUri()).isEqualTo(record.rawArchiveUri());
|
||||
}
|
||||
|
||||
private static List<Path> parquetFiles(Path root) throws Exception {
|
||||
try (var files = Files.walk(root)) {
|
||||
return files.filter(p -> p.getFileName().toString().endsWith(".parquet"))
|
||||
.sorted()
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
|
||||
private static EventFileRecord record(String eventId, Instant eventTime, String vin) {
|
||||
return new EventFileRecord(
|
||||
eventId,
|
||||
ProtocolId.GB32960,
|
||||
"REALTIME",
|
||||
vin,
|
||||
eventTime,
|
||||
eventTime.plusMillis(200),
|
||||
"file:///archive/" + eventId + ".bin",
|
||||
Map.of("source", "test"),
|
||||
"""
|
||||
{"gb32960":{"vehicle":{"speedKmh":42.1},"fuelCell":{"hydrogenRemainingKg":8.3}}}
|
||||
""".strip());
|
||||
}
|
||||
|
||||
private static EventFileRecord rawRecord(String eventId, Instant eventTime, String vin) {
|
||||
return new EventFileRecord(
|
||||
eventId,
|
||||
ProtocolId.GB32960,
|
||||
"RAW_ARCHIVE",
|
||||
vin,
|
||||
eventTime,
|
||||
eventTime.plusMillis(200),
|
||||
"file:///archive/" + eventId + ".bin",
|
||||
Map.of("source", "test"),
|
||||
"{}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package com.lingniu.ingest.eventfilestore;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.lingniu.ingest.api.ProtocolId;
|
||||
import com.lingniu.ingest.api.event.LocationPayload;
|
||||
import com.lingniu.ingest.api.event.RawArchiveKeys;
|
||||
import com.lingniu.ingest.api.event.RealtimePayload;
|
||||
import com.lingniu.ingest.api.event.VehicleEvent;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.ZoneId;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class EventFileStoreSinkTest {
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@Test
|
||||
void eventSinkSkipsRealtimeAndLocationEvents() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"), objectMapper);
|
||||
EventFileStoreSink sink = new EventFileStoreSink(store, objectMapper);
|
||||
|
||||
sink.publish(realtimeEvent("evt-1")).join();
|
||||
sink.publish(locationEvent("loc-1")).join();
|
||||
|
||||
var records = store.query(new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.ASC,
|
||||
10));
|
||||
|
||||
assertThat(records).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void batchedSinkFlushesRecordsWhenClosed() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"), objectMapper);
|
||||
EventFileStoreSink sink = new EventFileStoreSink(store, objectMapper, 10);
|
||||
|
||||
sink.publish(rawEvent("raw-batch-1")).join();
|
||||
assertThat(store.query(new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.ASC,
|
||||
10))).isEmpty();
|
||||
|
||||
sink.close();
|
||||
|
||||
assertThat(store.query(new EventFileQuery(
|
||||
ProtocolId.GB32960,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.ASC,
|
||||
10)))
|
||||
.extracting(EventFileRecord::eventId)
|
||||
.containsExactly("raw-batch-1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void eventSinkPersistsRawArchiveAsQueryableIndexWithoutEmbeddingRawBytes() throws Exception {
|
||||
EventFileStore store = new DuckDbParquetEventFileStore(tempDir, ZoneId.of("Asia/Shanghai"), objectMapper);
|
||||
EventFileStoreSink sink = new EventFileStoreSink(store, objectMapper);
|
||||
VehicleEvent.RawArchive raw = new VehicleEvent.RawArchive(
|
||||
"raw-1",
|
||||
"LTEST000000000808",
|
||||
ProtocolId.JT808,
|
||||
Instant.parse("2026-06-21T16:00:03Z"),
|
||||
Instant.parse("2026-06-21T16:00:04Z"),
|
||||
"trace-raw-1",
|
||||
Map.of("phone", "013800138000", "identityResolved", "true"),
|
||||
0x0200,
|
||||
0,
|
||||
new byte[]{0x7e, 0x02, 0x00, 0x7e});
|
||||
|
||||
sink.publish(raw).join();
|
||||
|
||||
var records = store.query(new EventFileQuery(
|
||||
ProtocolId.JT808,
|
||||
LocalDate.parse("2026-06-22"),
|
||||
LocalDate.parse("2026-06-22"),
|
||||
EventFileQuery.Order.ASC,
|
||||
10));
|
||||
|
||||
assertThat(records).hasSize(1);
|
||||
EventFileRecord record = records.getFirst();
|
||||
String rawArchiveKey = RawArchiveKeys.key(raw);
|
||||
assertThat(record.eventId()).isEqualTo("raw-1");
|
||||
assertThat(record.eventType()).isEqualTo("RAW_ARCHIVE");
|
||||
assertThat(record.rawArchiveUri()).isEqualTo(RawArchiveKeys.logicalUri(rawArchiveKey));
|
||||
assertThat(record.metadata())
|
||||
.containsEntry(RawArchiveKeys.META_KEY, rawArchiveKey)
|
||||
.containsEntry(RawArchiveKeys.META_URI, RawArchiveKeys.logicalUri(rawArchiveKey));
|
||||
JsonNode payload = objectMapper.readTree(record.payloadJson());
|
||||
assertThat(payload.get("rawSizeBytes").asLong()).isEqualTo(4);
|
||||
assertThat(payload.get("command").asText()).isEqualTo("0x0200");
|
||||
assertThat(payload.toString()).doesNotContain("7e02007e");
|
||||
}
|
||||
|
||||
private static VehicleEvent.Realtime realtimeEvent(String eventId) {
|
||||
return new VehicleEvent.Realtime(
|
||||
eventId,
|
||||
"LTEST000000000001",
|
||||
ProtocolId.GB32960,
|
||||
Instant.parse("2026-06-21T16:00:00Z"),
|
||||
Instant.parse("2026-06-21T16:00:01Z"),
|
||||
"trace-1",
|
||||
Map.of("rawArchiveUri", "file:///archive/" + eventId + ".bin"),
|
||||
new RealtimePayload(
|
||||
42.1, 1234.5, 87.0, null, null,
|
||||
null, null, null, 8.3, null, null,
|
||||
RealtimePayload.VehicleState.STARTED,
|
||||
null, null, null, null, null,
|
||||
113.12, 23.45, null, null, null, null
|
||||
));
|
||||
}
|
||||
|
||||
private static VehicleEvent.Location locationEvent(String eventId) {
|
||||
return new VehicleEvent.Location(
|
||||
eventId,
|
||||
"LTEST000000000001",
|
||||
ProtocolId.GB32960,
|
||||
Instant.parse("2026-06-21T16:00:00Z"),
|
||||
Instant.parse("2026-06-21T16:00:01Z"),
|
||||
"trace-location",
|
||||
Map.of("rawArchiveUri", "file:///archive/" + eventId + ".bin"),
|
||||
new LocationPayload(113.12, 23.45, 0, 42.1, 90, 0, 0));
|
||||
}
|
||||
|
||||
private static VehicleEvent.RawArchive rawEvent(String eventId) {
|
||||
return new VehicleEvent.RawArchive(
|
||||
eventId,
|
||||
"LTEST000000000001",
|
||||
ProtocolId.GB32960,
|
||||
Instant.parse("2026-06-21T16:00:00Z"),
|
||||
Instant.parse("2026-06-21T16:00:01Z"),
|
||||
"trace-raw",
|
||||
Map.of(),
|
||||
0x0002,
|
||||
0,
|
||||
new byte[]{0x23, 0x23});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.lingniu.ingest.eventfilestore.config;
|
||||
|
||||
import com.lingniu.ingest.eventfilestore.EventFileStore;
|
||||
import com.lingniu.ingest.eventfilestore.EventFileStoreSink;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class EventFileStoreAutoConfigurationTest {
|
||||
|
||||
private final ApplicationContextRunner contextRunner = new ApplicationContextRunner()
|
||||
.withConfiguration(AutoConfigurations.of(EventFileStoreAutoConfiguration.class));
|
||||
|
||||
@Test
|
||||
void createsStoreAndSinkWhenEnabled() {
|
||||
contextRunner
|
||||
.withPropertyValues(
|
||||
"lingniu.ingest.event-file-store.enabled=true",
|
||||
"lingniu.ingest.event-file-store.path=target/test-event-store")
|
||||
.run(context -> {
|
||||
assertThat(context).hasSingleBean(EventFileStore.class);
|
||||
assertThat(context).hasSingleBean(EventFileStoreSink.class);
|
||||
assertThat(context.getBean(EventFileStoreProperties.class).getPath())
|
||||
.isEqualTo(Path.of("target/test-event-store").toString());
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void backsOffWhenDisabled() {
|
||||
contextRunner
|
||||
.withPropertyValues("lingniu.ingest.event-file-store.enabled=false")
|
||||
.run(context -> {
|
||||
assertThat(context).doesNotHaveBean(EventFileStore.class);
|
||||
assertThat(context).doesNotHaveBean(EventFileStoreSink.class);
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user