shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: productionize raw history ingestion

Browse Source
This commit is contained in:
lingniu
2026-06-30 23:21:58 +08:00
parent 3cc7ac9669
commit cbba617801
100 changed files with 2995 additions and 1697 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
modules/protocols/protocol-jt808/src/test/java/com/lingniu/ingest/protocol/jt808/inbound/Jt808ChannelHandlerTest.java
View File

@@ -41,6 +41,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.within;
class Jt808ChannelHandlerTest {
@@ -86,7 +87,7 @@ class Jt808ChannelHandlerTest {
}
@Test
void unresolvedRegisterDoesNotBindUnknownVinToExternalIdentifiers() {
void unresolvedRegisterBindsGeneratedInternalVinToExternalIdentifiers() {
InMemorySessionStore sessions = new InMemorySessionStore();
InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService();
DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(new ImmediateKafkaSink()));
@@ -115,9 +116,9 @@ class Jt808ChannelHandlerTest {
assertThat(identity.resolve(new VehicleIdentityLookup(
ProtocolId.JT808, "", "123456789012", "", "")))
.satisfies(resolved -> {
assertThat(resolved.vin()).isEqualTo("123456789012");
assertThat(resolved.resolved()).isFalse();
assertThat(resolved.source()).isEqualTo(VehicleIdentitySource.FALLBACK_PHONE);
assertThat(resolved.vin()).startsWith("JT808");
assertThat(resolved.resolved()).isTrue();
assertThat(resolved.source()).isEqualTo(VehicleIdentitySource.BOUND_PHONE);
});
batchExecutor.close();
@@ -269,14 +270,143 @@ class Jt808ChannelHandlerTest {
assertThat(archive.metadata())
.containsEntry("vin", "LNVIN000000000808")
.containsEntry("phone", "123456789012")
.containsEntry("messageId", "0x0200")
.containsEntry("messageName", "TERMINAL_LOCATION")
.containsEntry("reportType", "LOCATION")
.containsEntry("body.alarmFlag", "0")
.containsEntry("body.statusFlag", "196608")
.containsEntry("body.status.acc_on", "false")
.containsEntry("body.status.load_status", "EMPTY")
.containsEntry("body.status.door4_open", "true")
.containsEntry("body.status.door5_open", "true")
.containsEntry("body.alarm.overspeed", "false")
.containsEntry("body.longitude", "116.397128")
.containsEntry("body.latitude", "39.916527")
.containsEntry("body.altitudeM", "50")
.containsEntry("body.directionDeg", "90")
.containsEntry("body.deviceTime", "2024-01-01T19:04:05Z")
.containsEntry("body.extra.0x01", "00003039")
.containsEntry("body.extra.mileage_km", "1234.5")
.containsEntry("body.extra.0x30", "58")
.containsEntry("body.extra.network_signal_strength", "88")
.containsEntry("identityResolved", "true")
.containsEntry("identitySource", "BOUND_PHONE");
assertThat(Double.parseDouble(archive.metadata().get("body.speedKmh"))).isCloseTo(52.3, within(0.001));
});
batchExecutor.close();
eventBus.close();
}
@Test
void registerArchiveIncludesParsedBodyFieldsForRawQuery() throws Exception {
RecordingSink sink = new RecordingSink(1);
DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink));
AsyncBatchExecutor batchExecutor = new AsyncBatchExecutor(eventBus::publish);
Dispatcher dispatcher = new Dispatcher(
new HandlerRegistry(),
new InterceptorChain(List.of()),
new HandlerInvoker(),
eventBus,
batchExecutor);
InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService();
identity.bind(new VehicleIdentityBinding(
ProtocolId.JT808, "LNVIN000000000808", "123456789012", "DEV808", "B80808"));
Jt808ChannelHandler handler = new Jt808ChannelHandler(
new Jt808MessageDecoder(new BodyParserRegistry(List.of(new RegisterBodyParser()))),
dispatcher,
new InMemorySessionStore(),
identity,
new Jt808ChannelRegistry(),
new Jt808PendingRequests());
EmbeddedChannel channel = new EmbeddedChannel(handler);
byte[] frame = buildFrame(
Jt808MessageId.TERMINAL_REGISTER,
"123456789012",
3,
buildRegisterBody("DEV808", "B80808"));
channel.writeInbound(frame);
assertThat(sink.await()).isTrue();
assertThat(sink.events).singleElement().satisfies(event -> {
assertThat(event).isInstanceOf(VehicleEvent.RawArchive.class);
VehicleEvent.RawArchive archive = (VehicleEvent.RawArchive) event;
assertThat(archive.metadata())
.containsEntry("messageId", "0x0100")
.containsEntry("messageName", "TERMINAL_REGISTER")
.containsEntry("reportType", "REGISTER")
.containsEntry("body.province", "44")
.containsEntry("body.city", "4401")
.containsEntry("body.maker", "MAKER")
.containsEntry("body.deviceType", "TYPE-A")
.containsEntry("body.deviceId", "DEV808")
.containsEntry("body.plateColor", "1")
.containsEntry("body.plate", "B80808");
});
batchExecutor.close();
eventBus.close();
}
@Test
void unboundRegisterCreatesVinBindingForRegisterAndFollowingRawFrames() throws Exception {
RecordingSink sink = new RecordingSink(2);
DisruptorEventBus eventBus = new DisruptorEventBus(1024, "blocking", List.of(sink));
AsyncBatchExecutor batchExecutor = new AsyncBatchExecutor(eventBus::publish);
Dispatcher dispatcher = new Dispatcher(
new HandlerRegistry(),
new InterceptorChain(List.of()),
new HandlerInvoker(),
eventBus,
batchExecutor);
InMemoryVehicleIdentityService identity = new InMemoryVehicleIdentityService();
Jt808ChannelHandler handler = new Jt808ChannelHandler(
new Jt808MessageDecoder(new BodyParserRegistry(List.of(new RegisterBodyParser(), new LocationBodyParser()))),
dispatcher,
new InMemorySessionStore(),
identity,
new Jt808ChannelRegistry(),
new Jt808PendingRequests());
EmbeddedChannel channel = new EmbeddedChannel(handler);
channel.writeInbound(buildFrame(
Jt808MessageId.TERMINAL_REGISTER,
"13079963289",
3,
buildRegisterBody("9963289", "HUA00113F")));
channel.writeInbound(buildFrame(
Jt808MessageId.TERMINAL_LOCATION,
"13079963289",
4,
buildLocationBody()));
assertThat(sink.await()).isTrue();
VehicleEvent.RawArchive register = (VehicleEvent.RawArchive) sink.events.get(0);
VehicleEvent.RawArchive location = (VehicleEvent.RawArchive) sink.events.get(1);
assertThat(register.metadata())
.containsEntry("reportType", "REGISTER")
.containsEntry("body.deviceId", "9963289")
.containsEntry("body.plate", "HUA00113F")
.containsEntry("identityResolved", "true")
.containsEntry("identitySource", "REGISTERED");
assertThat(register.vin())
.startsWith("JT808")
.hasSize(17);
assertThat(location.metadata())
.containsEntry("reportType", "LOCATION")
.containsEntry("vin", register.vin())
.containsEntry("identityResolved", "true")
.containsEntry("identitySource", "BOUND_PHONE");
assertThat(location.vin()).isEqualTo(register.vin());
assertThat(identity.resolve(new com.lingniu.ingest.identity.VehicleIdentityLookup(
ProtocolId.JT808, "", "13079963289", "", "")).vin())
.isEqualTo(register.vin());
batchExecutor.close();
eventBus.close();
}
@Test
void unboundLocationArchiveUsesUnknownVinInsteadOfPhoneFallback() throws Exception {
RecordingSink sink = new RecordingSink(1);
@@ -309,6 +439,9 @@ class Jt808ChannelHandlerTest {
assertThat(archive.metadata())
.containsEntry("vin", "unknown")
.containsEntry("phone", "123456789012")
.containsEntry("messageId", "0x0200")
.containsEntry("messageName", "TERMINAL_LOCATION")
.containsEntry("reportType", "LOCATION")
.containsEntry("identityResolved", "false")
.containsEntry("identitySource", "FALLBACK_PHONE");
});
@@ -351,13 +484,13 @@ class Jt808ChannelHandlerTest {
VehicleEvent.RawArchive archive = (VehicleEvent.RawArchive) event;
assertThat(archive.command()).isEqualTo(Jt808MessageId.TERMINAL_REGISTER);
assertThat(archive.metadata())
.containsEntry("jt808.register.province", "44")
.containsEntry("jt808.register.city", "4401")
.containsEntry("jt808.register.maker", "MAKER")
.containsEntry("jt808.register.deviceType", "TYPE-A")
.containsEntry("jt808.register.deviceId", "DEV808")
.containsEntry("jt808.register.plateColor", "1")
.containsEntry("jt808.register.plate", "B80808");
.containsEntry("body.province", "44")
.containsEntry("body.city", "4401")
.containsEntry("body.maker", "MAKER")
.containsEntry("body.deviceType", "TYPE-A")
.containsEntry("body.deviceId", "DEV808")
.containsEntry("body.plateColor", "1")
.containsEntry("body.plate", "B80808");
});
batchExecutor.close();
@@ -581,6 +714,12 @@ class Jt808ChannelHandlerTest {
writeU16(os, 90);
byte[] bcd = BcdCodec.encode("240102030405");
os.write(bcd, 0, 6);
os.write(0x01);
os.write(4);
writeU32(os, 12345);
os.write(0x30);
os.write(1);
os.write(0x58);
return os.toByteArray();
}

73
modules/protocols/protocol-jt808/src/test/java/com/lingniu/ingest/protocol/jt808/mapper/Jt808EventMapperTest.java
View File

@@ -38,6 +38,10 @@ class Jt808EventMapperTest {
assertThat(events.get(0).metadata())
.containsEntry("vin", "unknown")
.containsEntry("phone", "123456789012")
.containsEntry("jt808.status.acc_on", "false")
.containsEntry("jt808.status.positioned", "false")
.containsEntry("jt808.status.load_status", "EMPTY")
.containsEntry("jt808.alarm.overspeed", "false")
.containsEntry("identityResolved", "false")
.containsEntry("identitySource", "FALLBACK_PHONE");
}
@@ -58,7 +62,66 @@ class Jt808EventMapperTest {
.extracting(VehicleEvent::metadata)
.satisfies(meta -> assertThat(meta)
.containsEntry("jt808.extra.0x01", "00003039")
.containsEntry("jt808.extra.0x30", "58"));
.containsEntry("jt808.extra.0x30", "58")
.containsEntry("jt808.extra.mileage_km", "1234.5")
.containsEntry("jt808.extra.network_signal_strength", "88"));
}
@Test
void locationAdditionalItemsAreDecodedAsTable27Fields() {
var header = new Jt808Header(
Jt808MessageId.TERMINAL_LOCATION, 64, 0, false,
Jt808Header.ProtocolVersion.V2013, "123456789012", 1, 0, 0);
var body = new Jt808Body.Location(
0, 0, 116.397128, 39.916527, 50, 52.3, 90,
Instant.parse("2024-01-02T03:04:05Z"),
java.util.Map.ofEntries(
java.util.Map.entry(0x01, new byte[]{0x00, 0x00, 0x30, 0x39}),
java.util.Map.entry(0x02, new byte[]{0x00, 0x64}),
java.util.Map.entry(0x03, new byte[]{0x01, (byte) 0xF4}),
java.util.Map.entry(0x04, new byte[]{0x00, 0x02}),
java.util.Map.entry(0x05, tirePressureBytes()),
java.util.Map.entry(0x06, new byte[]{(byte) 0x80, 0x05}),
java.util.Map.entry(0x11, new byte[]{0x01, 0x00, 0x00, 0x00, 0x2A}),
java.util.Map.entry(0x12, new byte[]{0x04, 0x00, 0x00, 0x00, 0x2B, 0x01}),
java.util.Map.entry(0x13, new byte[]{0x00, 0x00, 0x00, 0x2C, 0x00, 0x78, 0x01}),
java.util.Map.entry(0x25, new byte[]{0x00, 0x00, 0x00, 0x05}),
java.util.Map.entry(0x2A, new byte[]{0x00, 0x03}),
java.util.Map.entry(0x2B, new byte[]{0x12, 0x34, 0x56, 0x78}),
java.util.Map.entry(0x30, new byte[]{0x1F}),
java.util.Map.entry(0x31, new byte[]{0x0C})));
VehicleEvent.Location event = (VehicleEvent.Location) mapper.toEvents(new Jt808Message(header, body)).getFirst();
assertThat(event.payload().totalMileageKm()).isEqualTo(1234.5);
assertThat(event.metadata())
.containsEntry("jt808.extra.mileage_km", "1234.5")
.containsEntry("jt808.extra.fuel_l", "10")
.containsEntry("jt808.extra.recorder_speed_kmh", "50")
.containsEntry("jt808.extra.manual_alarm_event_id", "2")
.containsEntry("jt808.extra.tire_pressure_pa_values", "32,33")
.containsEntry("jt808.extra.tire_pressure_invalid_indexes", "2-29")
.containsEntry("jt808.extra.compartment_temp_c", "-5")
.containsEntry("jt808.extra.overspeed.position_type", "1")
.containsEntry("jt808.extra.overspeed.area_route_id", "42")
.containsEntry("jt808.extra.area_route_alarm.position_type", "4")
.containsEntry("jt808.extra.area_route_alarm.area_route_id", "43")
.containsEntry("jt808.extra.area_route_alarm.direction", "1")
.containsEntry("jt808.extra.route_time.segment_id", "44")
.containsEntry("jt808.extra.route_time.duration_seconds", "120")
.containsEntry("jt808.extra.route_time.result", "1")
.containsEntry("jt808.extra.vehicle_signal_raw", "5")
.containsEntry("jt808.extra.vehicle_signal.low_beam", "true")
.containsEntry("jt808.extra.vehicle_signal.high_beam", "false")
.containsEntry("jt808.extra.vehicle_signal.right_turn", "true")
.containsEntry("jt808.extra.io_status_raw", "3")
.containsEntry("jt808.extra.io_status.deep_sleep", "true")
.containsEntry("jt808.extra.io_status.sleep", "true")
.containsEntry("jt808.extra.analog_raw", "305419896")
.containsEntry("jt808.extra.analog_ad0", "22136")
.containsEntry("jt808.extra.analog_ad1", "4660")
.containsEntry("jt808.extra.network_signal_strength", "31")
.containsEntry("jt808.extra.gnss_satellite_count", "12");
}
@Test
@@ -249,4 +312,12 @@ class Jt808EventMapperTest {
.containsEntry("parseError", "true");
});
}
private static byte[] tirePressureBytes() {
byte[] bytes = new byte[30];
java.util.Arrays.fill(bytes, (byte) 0xFF);
bytes[0] = 32;
bytes[1] = 33;
return bytes;
}
}