shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ack gb32960 after kafka boundary

Browse Source
This commit is contained in:
lingniu
2026-06-23 17:10:46 +08:00
parent 6b82144f3c
commit 633b3ea9c9
4 changed files with 322 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
modules/core/ingest-core/src/main/java/com/lingniu/ingest/core/concurrency/DisruptorEventBus.java
View File

@@ -14,6 +14,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicLong;
@@ -28,10 +29,12 @@ public final class DisruptorEventBus implements AutoCloseable {
private static final Logger log = LoggerFactory.getLogger(DisruptorEventBus.class);
private final Disruptor<VehicleEventSlot> disruptor;
private final List<EventSink> sinks;
private final AtomicLong published = new AtomicLong();
private final AtomicLong failed = new AtomicLong();
public DisruptorEventBus(int ringBufferSize, String waitStrategyName, List<EventSink> sinks) {
this.sinks = List.copyOf(sinks);
ThreadFactory tf = Thread.ofVirtual().name("ingest-bus-", 0).factory();
this.disruptor = new Disruptor<>(
VehicleEventSlot::new,
@@ -40,14 +43,14 @@ public final class DisruptorEventBus implements AutoCloseable {
ProducerType.MULTI,
waitStrategy(waitStrategyName));
EventHandler<VehicleEventSlot>[] handlers = sinks.stream()
EventHandler<VehicleEventSlot>[] handlers = this.sinks.stream()
.map(this::toHandler)
.toArray(EventHandler[]::new);
// 每个 sink 一个独立 handler，事件按扇出模式同时写 Kafka、event-file-store 等目标。
disruptor.handleEventsWith(handlers);
disruptor.start();
log.info("DisruptorEventBus started ringBuffer={} wait={} sinks={}",
ringBufferSize, waitStrategyName, sinks.size());
ringBufferSize, waitStrategyName, this.sinks.size());
}
public void publish(VehicleEvent event) {
@@ -55,6 +58,18 @@ public final class DisruptorEventBus implements AutoCloseable {
published.incrementAndGet();
}
public CompletableFuture<Void> publishAndAwait(VehicleEvent event) {
published.incrementAndGet();
List<CompletableFuture<Void>> futures = sinks.stream()
.filter(sink -> sink.accepts(event))
.map(sink -> publishToSinkAndTrack(sink, event))
.toList();
if (futures.isEmpty()) {
return CompletableFuture.completedFuture(null);
}
return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new));
}
@Override
public void close() {
disruptor.shutdown();
@@ -66,18 +81,28 @@ public final class DisruptorEventBus implements AutoCloseable {
VehicleEvent e = slot.event;
if (e == null || !sink.accepts(e)) return;
try {
sink.publish(e).exceptionally(ex -> {
// sink 异步失败只计数和打日志，不反向阻塞入站连接；生产侧靠 sink 自身重试/DLQ 保证可观测。
failed.incrementAndGet();
log.warn("sink {} publish failed", sink.name(), ex);
return null;
});
publishToSinkAndTrack(sink, e);
} finally {
if (endOfBatch) slot.clear();
}
};
}
private CompletableFuture<Void> publishToSinkAndTrack(EventSink sink, VehicleEvent event) {
try {
return sink.publish(event).whenComplete((ignored, ex) -> {
if (ex == null) return;
// sink 异步失败只计数和打日志；await 调用方仍会收到异常并决定是否 ACK。
failed.incrementAndGet();
log.warn("sink {} publish failed", sink.name(), ex);
});
} catch (Throwable t) {
failed.incrementAndGet();
log.warn("sink {} publish failed", sink.name(), t);
return CompletableFuture.failedFuture(t);
}
}
private static WaitStrategy waitStrategy(String name) {
return switch (name == null ? "yielding" : name.toLowerCase()) {
case "blocking" -> new BlockingWaitStrategy();

76
modules/core/ingest-core/src/main/java/com/lingniu/ingest/core/dispatcher/Dispatcher.java
View File

@@ -10,11 +10,16 @@ import com.lingniu.ingest.core.pipeline.InterceptorChain;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -52,13 +57,61 @@ public final class Dispatcher {
public void dispatch(RawFrame frame) {
IngestContext ctx = new IngestContext(UUID.randomUUID().toString());
try {
// 在 interceptor 之前发 RawArchive，保证原始字节被无条件落盘（dedup/rate-limit
// 不会过滤它），满足"原始可回放"目标。archive 的写盘靠下游 ArchiveEventSink 消费。
RawArchiveLookup rawArchive = emitRawArchive(frame, ctx);
DispatchPlan plan = mapFrameToEvents(frame, ctx);
for (VehicleEvent e : plan.events()) {
eventBus.publish(e);
}
if (plan.failure() != null) {
throw plan.failure();
}
} catch (Throwable t) {
log.error("dispatch failure traceId={}", ctx.traceId(), t);
interceptors.onError(t, ctx);
}
}
public CompletableFuture<Void> dispatchAndAwait(RawFrame frame) {
IngestContext ctx = new IngestContext(UUID.randomUUID().toString());
try {
DispatchPlan plan = mapFrameToEvents(frame, ctx);
CompletableFuture<?>[] futures = plan.events().stream()
.map(eventBus::publishAndAwait)
.toArray(CompletableFuture[]::new);
CompletableFuture<Void> boundary = CompletableFuture.allOf(futures);
if (plan.failure() == null) {
return boundary;
}
return boundary.handle((ignored, publishFailure) -> {
if (publishFailure != null) {
throw new CompletionException(publishFailure);
}
throw new CompletionException(plan.failure());
});
} catch (Throwable t) {
log.error("dispatch failure traceId={}", ctx.traceId(), t);
interceptors.onError(t, ctx);
return CompletableFuture.failedFuture(t);
}
}
public CompletableFuture<Void> dispatchAndAwait(RawFrame frame, Duration timeout) {
CompletableFuture<Void> future = dispatchAndAwait(frame);
if (timeout == null || timeout.isZero() || timeout.isNegative()) {
return future;
}
return future.orTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS);
}
private DispatchPlan mapFrameToEvents(RawFrame frame, IngestContext ctx) {
List<VehicleEvent> out = new ArrayList<>();
// 在 interceptor 之前发 RawArchive，保证原始字节被无条件落盘（dedup/rate-limit
// 不会过滤它），满足"原始可回放"目标。archive 的写盘靠下游 Sink 消费。
RawArchiveLookup rawArchive = appendRawArchive(frame, ctx, out);
try {
if (!interceptors.before(frame, ctx)) {
log.debug("frame aborted: {}", ctx.abortReason());
return;
return new DispatchPlan(out, null);
}
List<HandlerDefinition> handlers = registry.resolve(
@@ -67,7 +120,7 @@ public final class Dispatcher {
log.debug("no handler for {} cmd=0x{} info=0x{}",
frame.protocolId(), Integer.toHexString(frame.command()),
Integer.toHexString(frame.infoType()));
return;
return new DispatchPlan(out, null);
}
for (HandlerDefinition def : handlers) {
@@ -79,12 +132,12 @@ public final class Dispatcher {
for (VehicleEvent e : events) {
e = enrichWithRawArchive(e, rawArchive);
interceptors.after(e, ctx);
eventBus.publish(e);
out.add(e);
}
}
return new DispatchPlan(out, null);
} catch (Throwable t) {
log.error("dispatch failure traceId={}", ctx.traceId(), t);
interceptors.onError(t, ctx);
return new DispatchPlan(out, t);
}
}
@@ -95,7 +148,7 @@ public final class Dispatcher {
* <p>VIN 取自 sourceMeta 里的 {@code vin} key（由各入站适配器负责填充）；缺失时
* 留空字符串，archive sink 会用 "unknown-vin" 占位保证 key 可解析。
*/
private RawArchiveLookup emitRawArchive(RawFrame frame, IngestContext ctx) {
private RawArchiveLookup appendRawArchive(RawFrame frame, IngestContext ctx, List<VehicleEvent> out) {
byte[] bytes = frame.rawBytes();
if (bytes == null || bytes.length == 0) return RawArchiveLookup.empty();
@@ -117,7 +170,7 @@ public final class Dispatcher {
frame.command(),
frame.infoType(),
bytes);
eventBus.publish(raw);
out.add(raw);
return new RawArchiveLookup(eventId, key, RawArchiveKeys.logicalUri(key));
}
@@ -182,4 +235,7 @@ public final class Dispatcher {
return key == null || key.isBlank();
}
}
private record DispatchPlan(List<VehicleEvent> events, Throwable failure) {
}
}