fix: stabilize jt808 fanout and dedup
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful

This commit is contained in:
lingniu
2026-06-29 17:57:46 +08:00
parent 1d569b7bf3
commit 705b507f88
4 changed files with 100 additions and 9 deletions

View File

@@ -103,8 +103,9 @@ public final class DisruptorEventBus implements AutoCloseable {
if (e == null || !sink.accepts(e)) return;
try {
publishToSinkAndTrack(sink, e);
} finally {
if (endOfBatch) slot.clear();
} catch (Throwable t) {
failed.incrementAndGet();
log.warn("sink {} publish failed", sink.name(), t);
}
};
}

View File

@@ -9,13 +9,16 @@ import org.springframework.core.Ordered;
import java.util.Arrays;
import java.time.Duration;
import java.util.Map;
/**
* 基于 Caffeine 的本地幂等去重。
*
* <p>Key 构造:{@code protocolId + vin + command + seq/fingerprint}。如果上游已经在
* {@link RawFrame#sourceMeta()} 里带了 {@code seq},优先用真实流水号;否则退化为 raw bytes
* fingerprint避免设备重连或 TCP 重发导致同一帧重复进入 DuckDB/Archive。
* <p>Key 构造:{@code protocolId + vehicle identity + command + seq/fingerprint}。vehicle identity
* 优先使用已解析 VINVIN 未解析时使用 phone/deviceId/terminalId/plate避免 JT808 多终端在
* {@code vin=unknown} 时同流水号互相去重。如果上游已经在 {@link RawFrame#sourceMeta()} 里带了
* {@code seq},优先用真实流水号;否则退化为 raw bytes fingerprint避免设备重连或 TCP 重发导致
* 同一帧重复进入 DuckDB/Archive。
*
* <p>当前缓存是进程内的只能保证单实例去重。32960 若做多副本水平扩容,需要把这个位置替换成
* Redis/集中式幂等键,否则不同实例仍可能各自接收一次相同原始包。
@@ -33,13 +36,14 @@ public class DedupInterceptor implements IngestInterceptor, Ordered {
@Override
public boolean before(RawFrame frame, IngestContext ctx) {
String vin = frame.sourceMeta().getOrDefault("vin", "unknown");
String seq = frame.sourceMeta().get("seq");
Map<String, String> meta = frame.sourceMeta() == null ? Map.of() : frame.sourceMeta();
String identity = identityKey(frame, meta);
String seq = meta.get("seq");
if (seq == null || seq.isBlank()) {
// 某些入口没有硬件流水号,使用原始字节哈希作为保底键,至少能挡住同连接内的重复帧。
seq = rawFingerprint(frame);
}
String key = frame.protocolId() + ":" + vin + ":" + frame.command() + ":" + seq;
String key = frame.protocolId() + ":" + identity + ":" + frame.command() + ":" + seq;
if (seen.asMap().putIfAbsent(key, Boolean.TRUE) != null) {
ctx.abort("duplicate:" + key);
return false;
@@ -52,6 +56,20 @@ public class DedupInterceptor implements IngestInterceptor, Ordered {
return 100;
}
private static String identityKey(RawFrame frame, Map<String, String> meta) {
String vin = meta.getOrDefault("vin", "unknown").trim();
if (!vin.isBlank() && !"unknown".equalsIgnoreCase(vin)) {
return "vin:" + vin;
}
for (String key : new String[]{"phone", "deviceId", "terminalId", "plate"}) {
String value = meta.get(key);
if (value != null && !value.isBlank()) {
return key + ":" + value.trim();
}
}
return "raw:" + rawFingerprint(frame);
}
private static String rawFingerprint(RawFrame frame) {
byte[] rawBytes = frame.rawBytes();
if (rawBytes != null && rawBytes.length > 0) {