fix: stabilize jt808 fanout and dedup
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
This commit is contained in:
@@ -103,8 +103,9 @@ public final class DisruptorEventBus implements AutoCloseable {
|
||||
if (e == null || !sink.accepts(e)) return;
|
||||
try {
|
||||
publishToSinkAndTrack(sink, e);
|
||||
} finally {
|
||||
if (endOfBatch) slot.clear();
|
||||
} catch (Throwable t) {
|
||||
failed.incrementAndGet();
|
||||
log.warn("sink {} publish failed", sink.name(), t);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@@ -9,13 +9,16 @@ import org.springframework.core.Ordered;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 基于 Caffeine 的本地幂等去重。
|
||||
*
|
||||
* <p>Key 构造:{@code protocolId + vin + command + seq/fingerprint}。如果上游已经在
|
||||
* {@link RawFrame#sourceMeta()} 里带了 {@code seq},优先用真实流水号;否则退化为 raw bytes
|
||||
* fingerprint,避免设备重连或 TCP 重发导致同一帧重复进入 DuckDB/Archive。
|
||||
* <p>Key 构造:{@code protocolId + vehicle identity + command + seq/fingerprint}。vehicle identity
|
||||
* 优先使用已解析 VIN;VIN 未解析时使用 phone/deviceId/terminalId/plate,避免 JT808 多终端在
|
||||
* {@code vin=unknown} 时同流水号互相去重。如果上游已经在 {@link RawFrame#sourceMeta()} 里带了
|
||||
* {@code seq},优先用真实流水号;否则退化为 raw bytes fingerprint,避免设备重连或 TCP 重发导致
|
||||
* 同一帧重复进入 DuckDB/Archive。
|
||||
*
|
||||
* <p>当前缓存是进程内的,只能保证单实例去重。32960 若做多副本水平扩容,需要把这个位置替换成
|
||||
* Redis/集中式幂等键,否则不同实例仍可能各自接收一次相同原始包。
|
||||
@@ -33,13 +36,14 @@ public class DedupInterceptor implements IngestInterceptor, Ordered {
|
||||
|
||||
@Override
|
||||
public boolean before(RawFrame frame, IngestContext ctx) {
|
||||
String vin = frame.sourceMeta().getOrDefault("vin", "unknown");
|
||||
String seq = frame.sourceMeta().get("seq");
|
||||
Map<String, String> meta = frame.sourceMeta() == null ? Map.of() : frame.sourceMeta();
|
||||
String identity = identityKey(frame, meta);
|
||||
String seq = meta.get("seq");
|
||||
if (seq == null || seq.isBlank()) {
|
||||
// 某些入口没有硬件流水号,使用原始字节哈希作为保底键,至少能挡住同连接内的重复帧。
|
||||
seq = rawFingerprint(frame);
|
||||
}
|
||||
String key = frame.protocolId() + ":" + vin + ":" + frame.command() + ":" + seq;
|
||||
String key = frame.protocolId() + ":" + identity + ":" + frame.command() + ":" + seq;
|
||||
if (seen.asMap().putIfAbsent(key, Boolean.TRUE) != null) {
|
||||
ctx.abort("duplicate:" + key);
|
||||
return false;
|
||||
@@ -52,6 +56,20 @@ public class DedupInterceptor implements IngestInterceptor, Ordered {
|
||||
return 100;
|
||||
}
|
||||
|
||||
private static String identityKey(RawFrame frame, Map<String, String> meta) {
|
||||
String vin = meta.getOrDefault("vin", "unknown").trim();
|
||||
if (!vin.isBlank() && !"unknown".equalsIgnoreCase(vin)) {
|
||||
return "vin:" + vin;
|
||||
}
|
||||
for (String key : new String[]{"phone", "deviceId", "terminalId", "plate"}) {
|
||||
String value = meta.get(key);
|
||||
if (value != null && !value.isBlank()) {
|
||||
return key + ":" + value.trim();
|
||||
}
|
||||
}
|
||||
return "raw:" + rawFingerprint(frame);
|
||||
}
|
||||
|
||||
private static String rawFingerprint(RawFrame frame) {
|
||||
byte[] rawBytes = frame.rawBytes();
|
||||
if (rawBytes != null && rawBytes.length > 0) {
|
||||
|
||||
Reference in New Issue
Block a user