shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: productionize raw history ingestion

Browse Source
This commit is contained in:
lingniu
2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
modules/core/ingest-core/pom.xml
View File

@@ -24,6 +24,14 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>

36
modules/core/ingest-core/src/main/java/com/lingniu/ingest/core/dispatcher/Dispatcher.java
View File

@@ -1,5 +1,8 @@
package com.lingniu.ingest.core.dispatcher;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
import com.lingniu.ingest.api.event.RawArchiveKeys;
import com.lingniu.ingest.api.event.VehicleEvent;
import com.lingniu.ingest.api.pipeline.IngestContext;
@@ -36,6 +39,9 @@ public final class Dispatcher {
private static final Logger log = LoggerFactory.getLogger(Dispatcher.class);
private static final AtomicLong RAW_ARCHIVE_SEQUENCE = new AtomicLong();
private static final String REQUIRED_DURABLE_SINK = "kafka";
private static final ObjectMapper RAW_ARCHIVE_JSON = JsonMapper.builder()
.findAndAddModules()
.build();
private final HandlerRegistry registry;
private final InterceptorChain interceptors;
@@ -167,7 +173,10 @@ public final class Dispatcher {
byte[] bytes = frame.rawBytes();
if (bytes == null || bytes.length == 0) return RawArchiveLookup.empty();
Map<String, String> meta = frame.sourceMeta() == null ? Map.of() : frame.sourceMeta();
Map<String, String> meta = frame.sourceMeta() == null
? Map.of()
: new LinkedHashMap<>(frame.sourceMeta());
String parsedJson = parsedJson(frame.payload(), meta.remove(RawArchiveKeys.META_PARSED_JSON));
String vin = meta.getOrDefault("vin", "");
Instant ingestTime = frame.receivedAt() != null ? frame.receivedAt() : Instant.now();
String eventId = nextRawArchiveEventId(ingestTime);
@@ -184,11 +193,34 @@ public final class Dispatcher {
archiveMeta,
frame.command(),
frame.infoType(),
bytes);
bytes,
parsedJson);
out.add(raw);
return new RawArchiveLookup(eventId, key, RawArchiveKeys.logicalUri(key));
}
private static String parsedJson(Object payload, String explicitParsedJson) {
if (explicitParsedJson != null && !explicitParsedJson.isBlank()) {
return explicitParsedJson;
}
if (payload == null) {
return "{}";
}
try {
return RAW_ARCHIVE_JSON.writeValueAsString(payload);
} catch (JsonProcessingException ex) {
return "{\"serializationError\":\"" + escapeJson(ex.getMessage()) + "\",\"payloadType\":\""
+ escapeJson(payload.getClass().getName()) + "\"}";
}
}
private static String escapeJson(String value) {
if (value == null) {
return "";
}
return value.replace("\\", "\\\\").replace("\"", "\\\"");
}
private static String nextRawArchiveEventId(Instant ingestTime) {
// 文件名需要可排序且单进程内唯一：毫秒时间放大到微秒量级，再用 AtomicLong 递增兜底。
long base = (ingestTime == null ? Instant.now() : ingestTime).toEpochMilli() * 1000L;

11
modules/core/ingest-core/src/test/java/com/lingniu/ingest/core/dispatcher/DispatcherRawArchiveMetadataTest.java
View File

@@ -42,7 +42,7 @@ class DispatcherRawArchiveMetadataTest {
ProtocolId.JT808,
0x0200,
0,
new Object(),
new SamplePayload("payload-0200", 7),
new byte[]{0x7e, 0x01, 0x7e},
Map.of("vin", "VIN001", "peer", "127.0.0.1:10001"),
Instant.parse("2026-06-22T08:00:00Z")));
@@ -52,6 +52,8 @@ class DispatcherRawArchiveMetadataTest {
String rawArchiveKey = raw.metadata().get("rawArchiveKey");
assertThat(rawArchiveKey).isNotBlank();
assertThat(raw.parsedJson()).contains("\"name\":\"payload-0200\"");
assertThat(raw.metadata()).doesNotContainKey("rawArchiveParsedJson");
assertThat(raw.eventId()).containsOnlyDigits();
assertThat(rawArchiveKey).endsWith("/" + raw.eventId() + ".bin");
assertThat(location.metadata())
@@ -102,7 +104,7 @@ class DispatcherRawArchiveMetadataTest {
private static HandlerRegistry registryWithLocationHandler() throws NoSuchMethodException {
LocationHandler bean = new LocationHandler();
Method method = LocationHandler.class.getDeclaredMethod("onLocation", Object.class);
Method method = LocationHandler.class.getDeclaredMethod("onLocation", SamplePayload.class);
HandlerRegistry registry = new HandlerRegistry();
registry.register(new HandlerDefinition(
ProtocolId.JT808,
@@ -169,7 +171,7 @@ class DispatcherRawArchiveMetadataTest {
@ProtocolHandler(protocol = ProtocolId.JT808)
static final class LocationHandler {
@MessageMapping(command = 0x0200)
VehicleEvent.Location onLocation(Object ignored) {
VehicleEvent.Location onLocation(SamplePayload ignored) {
return new VehicleEvent.Location(
"location-1",
"VIN001",
@@ -182,6 +184,9 @@ class DispatcherRawArchiveMetadataTest {
}
}
record SamplePayload(String name, int value) {
}
@ProtocolHandler(protocol = ProtocolId.JT808)
public static final class AsyncLocationHandler {
@MessageMapping(command = 0x0201)