refactor: make event file store optional

This commit is contained in:
lingniu
2026-07-01 07:51:45 +08:00
parent 51b1dd4174
commit 56a6c01f95
32 changed files with 139 additions and 84 deletions

View File

@@ -63,7 +63,8 @@ class Gb32960IngestAppCompositionTest {
assertThat(context).hasSingleBean(ArchiveStore.class);
assertThat(context).hasSingleBean(RawArchiveEventSink.class);
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.EventFileStore");
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore");
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration");
});
}

View File

@@ -62,7 +62,8 @@ class Jt808IngestAppCompositionTest {
assertThat(context).hasSingleBean(ArchiveStore.class);
assertThat(context).hasSingleBean(RawArchiveEventSink.class);
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.EventFileStore");
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore");
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration");
});
}

View File

@@ -58,7 +58,9 @@ class VehicleAnalyticsAppCompositionTest {
assertTypeNotPresent(context, "com.lingniu.ingest.vehiclestate.VehicleStateEnvelopeIngestor");
assertTypeNotPresent(context, "com.lingniu.ingest.vehiclestate.config.VehicleStateAutoConfiguration");
assertTypeNotPresent(context, "com.lingniu.ingest.protocol.gb32960.inbound.Gb32960NettyServer");
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.EventFileStore");
assertTypeNotPresent(context, "com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore");
assertTypeNotPresent(context,
"com.lingniu.ingest.eventfilestore.config.EventFileStoreAutoConfiguration");
});
}

View File

@@ -63,6 +63,22 @@ class MavenModuleProfileTest {
.containsExactly("modules/sinks/raw-archive-store");
}
@Test
void eventFileStoreCompatibilityPathIsOptionalAndOutsideDefaultProductionReactor() throws Exception {
Document pom = rootPom();
Document eventHistoryServicePom = modulePom("modules/services/event-history-service/pom.xml");
assertThat(defaultModules(pom))
.doesNotContain("modules/sinks/event-file-store");
assertThat(profileModules(pom, "optional-event-file-store"))
.containsExactly("modules/sinks/event-file-store");
assertThat(hasDependency(eventHistoryServicePom, "com.lingniu.ingest", "event-file-store"))
.as("event-history-service should depend only on the lightweight history contract")
.isFalse();
}
@Test
void xindaIsLegacyOnlyAndOutsideDefaultProductionReactor() throws Exception {
Document pom = rootPom();
@@ -97,7 +113,7 @@ class MavenModuleProfileTest {
.contains("Netty " + projectProperty(pom, "netty.version"))
.contains("Eclipse Paho " + projectProperty(pom, "paho-mqtt.version"))
.contains("protocol-jsatl12/ 苏标主动安全报警附件optional-attachments profile")
.contains("event-file-store/ 可选兼容索引(旧 Parquet/DuckDB 路径)")
.contains("event-file-store/ 可选兼容索引(旧 Parquet/DuckDB 路径,仅 -Poptional-event-file-store 显式构建")
.contains("vehicle-state-service/ Kafka 全字段事件消费 + Redis 热状态查询optional-latest-state profile")
.contains("vehicle-history-app/ TDengine 历史查询 + RAW JSON")
.contains("vehicle-analytics-app/ JT808 每日里程指标消费")
@@ -133,6 +149,7 @@ class MavenModuleProfileTest {
.contains("JSATL12 仅保留在 `optional-attachments` profile")
.contains("vehicle-state-service 仅保留在 `optional-latest-state` profile")
.contains("raw-archive-store 仅保留在 `optional-raw-archive-store` profile")
.contains("event-file-store 仅保留在 `optional-event-file-store` profile")
.doesNotContain("## ADR-004 信达 Push保留模块但彻底重写\n- **Status**: Accepted")
.doesNotContain("Spring Boot 3.4.x");
}
@@ -225,6 +242,28 @@ class MavenModuleProfileTest {
return false;
}
private static boolean hasDependency(Document pom, String groupId, String artifactId) {
Element dependencies = firstDirectChild(pom.getDocumentElement(), "dependencies");
if (dependencies == null) {
return false;
}
NodeList children = dependencies.getChildNodes();
for (int i = 0; i < children.getLength(); i++) {
if (!(children.item(i) instanceof Element dependency) || !"dependency".equals(dependency.getTagName())) {
continue;
}
Element currentGroupId = firstDirectChild(dependency, "groupId");
Element currentArtifactId = firstDirectChild(dependency, "artifactId");
if (currentGroupId != null
&& currentArtifactId != null
&& groupId.equals(currentGroupId.getTextContent().trim())
&& artifactId.equals(currentArtifactId.getTextContent().trim())) {
return true;
}
}
return false;
}
private static Element firstDirectChild(Element parent, String tagName) {
NodeList children = parent.getChildNodes();
for (int i = 0; i < children.getLength(); i++) {

View File

@@ -3,7 +3,7 @@ package com.lingniu.ingest.historyapp;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterSink;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.eventhistory.EventHistoryEnvelopeIngestor;
import com.lingniu.ingest.eventhistory.Gb32960DecodedFrameService;
import com.lingniu.ingest.eventhistory.Gb32960FrameController;

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.eventfilestore;
package com.lingniu.ingest.api.history;
import com.lingniu.ingest.api.ProtocolId;
@@ -6,10 +6,7 @@ import java.time.Instant;
import java.time.LocalDate;
/**
* 文件历史库查询条件
*
* <p>{@code dateFrom/dateTo} 用于定位分区目录{@code eventTimeFrom/eventTimeTo}
* 用于在分区内做精确时间过滤这样既支持按天快速裁剪也支持接口精确到秒的查询
* Query criteria for the optional file-backed historical event index.
*/
public record EventFileQuery(
ProtocolId protocol,
@@ -43,7 +40,6 @@ public record EventFileQuery(
if (eventTimeFrom != null && eventTimeTo != null && eventTimeTo.isBefore(eventTimeFrom)) {
throw new IllegalArgumentException("eventTimeTo must not be before eventTimeFrom");
}
// 归一化可选字段避免空字符串进入 SQL 谓词或文件分区路径
order = order == null ? Order.ASC : order;
limit = limit <= 0 ? 100 : limit;
vin = vin == null || vin.isBlank() ? null : vin.trim();

View File

@@ -1,4 +1,4 @@
package com.lingniu.ingest.eventfilestore;
package com.lingniu.ingest.api.history;
import com.lingniu.ingest.api.ProtocolId;
@@ -6,13 +6,7 @@ import java.time.Instant;
import java.util.Map;
/**
* 文件型明细库的一条可查询记录
*
* <p>{@code payloadJson} 保存统一 telemetry snapshot JSON公共列只保留排序分区
* 追溯和通用展示需要的最小字段
*
* <p> GB32960 来说{@code rawArchiveUri} 是最关键的追溯列通用查询可以直接展示
* payloadJson专用全字段查询则通过 rawArchiveUri 回读原始包重新解码
* Minimal historical event record shared by optional compatibility stores and history queries.
*/
public record EventFileRecord(
String eventId,
@@ -38,7 +32,6 @@ public record EventFileRecord(
if (ingestTime == null) {
throw new IllegalArgumentException("ingestTime must not be null");
}
// 允许 eventType/vin/rawArchiveUri 为空字符串便于保存平台登录RAW 索引等非车辆事件
eventType = eventType == null ? "" : eventType;
vin = vin == null ? "" : vin;
rawArchiveUri = rawArchiveUri == null ? "" : rawArchiveUri;

View File

@@ -0,0 +1,26 @@
package com.lingniu.ingest.api.history;
import java.io.IOException;
import java.util.List;
/**
* Historical event index contract.
*
* <p>The default production history path uses TDengine directly. This contract remains in core so
* optional compatibility implementations can still plug into query code without pulling their
* storage engines into the default reactor.
*/
public interface EventFileStore {
default void append(EventFileRecord record) throws IOException {
appendAll(List.of(record));
}
void appendAll(List<EventFileRecord> records) throws IOException;
List<EventFileRecord> query(EventFileQuery query) throws IOException;
default EventFileRecord findByRawArchiveUri(String rawArchiveUri) throws IOException {
return null;
}
}

View File

@@ -14,7 +14,7 @@
<dependencies>
<dependency>
<groupId>com.lingniu.ingest</groupId>
<artifactId>event-file-store</artifactId>
<artifactId>ingest-api</artifactId>
</dependency>
<dependency>
<groupId>com.lingniu.ingest</groupId>

View File

@@ -1,9 +1,9 @@
package com.lingniu.ingest.eventhistory;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.eventfilestore.EventFileQuery;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.web.bind.annotation.GetMapping;

View File

@@ -3,8 +3,8 @@ package com.lingniu.ingest.eventhistory;
import com.google.protobuf.InvalidProtocolBufferException;
import com.lingniu.ingest.api.consumer.EnvelopeBatchIngestor;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
import com.lingniu.ingest.tdenginehistory.TdengineEnvelopeRows;
import com.lingniu.ingest.tdenginehistory.TdengineHistoryWriter;

View File

@@ -3,9 +3,9 @@ package com.lingniu.ingest.eventhistory;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.eventfilestore.EventFileQuery;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
import com.lingniu.ingest.protocol.gb32960.model.Gb32960Message;
import com.lingniu.ingest.protocol.gb32960.model.InfoBlock;

View File

@@ -1,6 +1,6 @@
package com.lingniu.ingest.eventhistory;
import com.lingniu.ingest.eventfilestore.EventFileQuery;
import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.tdenginehistory.TdenginePageCursor;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;

View File

@@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.VehicleEnvelope;
import com.google.protobuf.InvalidProtocolBufferException;

View File

@@ -2,7 +2,7 @@ package com.lingniu.ingest.eventhistory.config;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterSink;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.eventhistory.EventHistoryController;
import com.lingniu.ingest.eventhistory.EventHistoryEnvelopeIngestor;
import com.lingniu.ingest.eventhistory.Gb32960DecodedFrameService;

View File

@@ -1,7 +1,7 @@
package com.lingniu.ingest.eventhistory;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
import jakarta.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.Test;
@@ -93,7 +93,7 @@ class ApiExceptionHandlerTest {
}
@Override
public List<EventFileRecord> query(com.lingniu.ingest.eventfilestore.EventFileQuery query) {
public List<EventFileRecord> query(com.lingniu.ingest.api.history.EventFileQuery query) {
return List.of();
}
};

View File

@@ -1,9 +1,9 @@
package com.lingniu.ingest.eventhistory;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.eventfilestore.EventFileQuery;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import org.junit.jupiter.api.Test;
import java.io.IOException;

View File

@@ -6,9 +6,9 @@ import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.consumer.EnvelopeIngestResult;
import com.lingniu.ingest.api.consumer.EnvelopeIngestor;
import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.eventfilestore.EventFileQuery;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.RawFrameFactPayload;
import com.lingniu.ingest.sink.mq.proto.TelemetryField;

View File

@@ -2,9 +2,9 @@ package com.lingniu.ingest.eventhistory;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.codec.BccChecksum;
import com.lingniu.ingest.eventfilestore.EventFileQuery;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960BodyParser;
import com.lingniu.ingest.protocol.gb32960.codec.Gb32960MessageDecoder;
import com.lingniu.ingest.protocol.gb32960.codec.InfoBlockParser;

View File

@@ -1,7 +1,7 @@
package com.lingniu.ingest.eventhistory;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.eventfilestore.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.sink.mq.proto.RawArchiveRef;
import com.lingniu.ingest.sink.mq.proto.TelemetryField;
import com.lingniu.ingest.sink.mq.proto.TelemetrySnapshot;

View File

@@ -2,7 +2,7 @@ package com.lingniu.ingest.eventhistory.config;
import com.lingniu.ingest.api.consumer.EnvelopeConsumerProcessor;
import com.lingniu.ingest.api.consumer.EnvelopeDeadLetterSink;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.eventhistory.EventHistoryController;
import com.lingniu.ingest.eventhistory.EventHistoryEnvelopeIngestor;
import com.lingniu.ingest.eventhistory.Gb32960DecodedFrameService;

View File

@@ -3,6 +3,9 @@ package com.lingniu.ingest.eventfilestore;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import java.io.IOException;
import java.nio.file.Files;

View File

@@ -1,32 +0,0 @@
package com.lingniu.ingest.eventfilestore;
import java.io.IOException;
import java.util.List;
/**
* 历史明细文件库抽象。
*
* <p>写入侧只接受已经标准化的 {@link EventFileRecord};具体实现可以落 Parquet、维护 DuckDB
* sidecar 索引或在测试中用内存实现。32960 专用 snapshot 查询会先用这里按 VIN/日期找到
* rawArchiveUri再回读原始 .bin 解码完整字段。
*/
public interface EventFileStore {
/** 单条追加的便捷方法,最终仍走批量写入路径,保证实现只维护一种落盘语义。 */
default void append(EventFileRecord record) throws IOException {
appendAll(List.of(record));
}
void appendAll(List<EventFileRecord> records) throws IOException;
/** 按协议、日期、VIN、事件类型等条件查询标准化历史记录。 */
List<EventFileRecord> query(EventFileQuery query) throws IOException;
/**
* 按 rawArchiveUri 回查索引记录。默认实现返回 null允许轻量测试实现不维护该索引。
* 生产 DuckDB/Parquet 实现必须覆盖,用于 snapshots 的 sourceFrames 反查。
*/
default EventFileRecord findByRawArchiveUri(String rawArchiveUri) throws IOException {
return null;
}
}

View File

@@ -5,6 +5,8 @@ import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.event.TelemetrySnapshot;
import com.lingniu.ingest.api.event.VehicleEvent;
import com.lingniu.ingest.api.event.VehicleEventTelemetrySnapshotMapper;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.api.sink.EventSink;
import java.io.IOException;

View File

@@ -2,8 +2,8 @@ package com.lingniu.ingest.eventfilestore.config;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.eventfilestore.DuckDbParquetEventFileStore;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.eventfilestore.EventFileStoreSink;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;

View File

@@ -1,7 +1,10 @@
package com.lingniu.ingest.eventfilestore;
import com.lingniu.ingest.api.ProtocolId;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.lingniu.ingest.api.ProtocolId;
import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

View File

@@ -7,6 +7,9 @@ import com.lingniu.ingest.api.event.LocationPayload;
import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.event.RealtimePayload;
import com.lingniu.ingest.api.event.VehicleEvent;
import com.lingniu.ingest.api.history.EventFileQuery;
import com.lingniu.ingest.api.history.EventFileRecord;
import com.lingniu.ingest.api.history.EventFileStore;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

View File

@@ -1,6 +1,6 @@
package com.lingniu.ingest.eventfilestore.config;
import com.lingniu.ingest.eventfilestore.EventFileStore;
import com.lingniu.ingest.api.history.EventFileStore;
import com.lingniu.ingest.eventfilestore.EventFileStoreSink;
import org.junit.jupiter.api.Test;
import org.springframework.boot.autoconfigure.AutoConfigurations;