docs: add detailed 32960 pipeline comments

2026-06-23 13:17:37 +08:00
parent ba68ffe061
commit a096e4ce0e
125 changed files with 493 additions and 14 deletions
--- a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/EnvelopeMapper.java
+++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/EnvelopeMapper.java
@@ -36,6 +36,7 @@ public final class EnvelopeMapper {
                .setIngestTimeMs(event.ingestTime().toEpochMilli())
                .setIngestNodeId(nodeId);
        if (event.metadata() != null) b.putAllMetadata(event.metadata());
+        // telemetrySnapshot 是跨事件类型的统一字段视图，消费者可先读它，必要时再读具体 payload。
        VehicleEventTelemetrySnapshotMapper.toSnapshot(event)
                .map(EnvelopeMapper::buildTelemetrySnapshot)
                .ifPresent(b::setTelemetrySnapshot);
@@ -70,6 +71,8 @@ public final class EnvelopeMapper {
                int size = ra.rawBytes() == null ? 0 : ra.rawBytes().length;
                String key = rawArchiveKey(ra);
                String uri = rawArchiveUri(ra, key);
+                // RAW envelope 只携带引用信息，不把完整原始字节塞进 Kafka，避免 topic 膨胀。
+                // 注意 KafkaEventSink 当前仍过滤 RawArchive；这段用于未来放开 raw 引用转发或消费者测试。
                b.putMetadata(RawArchiveKeys.META_KEY, key);
                b.putMetadata(RawArchiveKeys.META_URI, uri);
                b.putMetadata(RawArchiveKeys.META_EVENT_ID, ra.eventId());
--- a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerFactory.java
+++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerFactory.java
@@ -33,6 +33,8 @@ public final class KafkaEnvelopeConsumerFactory {
        List<KafkaEnvelopeConsumerWorker> workers = new ArrayList<>();
        for (Map.Entry<String, SinkMqProperties.Binding> entry : bindings.entrySet()) {
            String processorBeanName = entry.getKey();
+            // binding 的 key 必须和 Spring Bean 名一致；这样配置只声明 topic/group，
+            // 实际处理逻辑仍由各业务模块自己的 EnvelopeConsumerProcessor 承接。
            EnvelopeConsumerProcessor processor = processors.get(processorBeanName);
            SinkMqProperties.Binding binding = entry.getValue();
            if (processor == null || binding == null || !binding.isEnabled()) {
@@ -56,6 +58,8 @@ public final class KafkaEnvelopeConsumerFactory {
        if (configured != null && !configured.isEmpty()) {
            return configured;
        }
+        // 默认绑定保留老派生模块的消费能力；32960 主链路不依赖这些派生表，
+        // 历史查询以 event-file-store 的 RAW 索引和 archive .bin 为准。
        SinkMqProperties.Topics topics = props.getTopics();
        Map<String, SinkMqProperties.Binding> defaults = new LinkedHashMap<>();
        defaults.put("eventHistoryEnvelopeConsumerProcessor", binding(
@@ -108,6 +112,8 @@ public final class KafkaEnvelopeConsumerFactory {
        p.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
        p.put(ConsumerConfig.GROUP_ID_CONFIG, groupId(binding, processorBeanName));
        p.put(ConsumerConfig.CLIENT_ID_CONFIG, consumer.getClientIdPrefix() + "-" + processorBeanName);
+        // 手动提交 offset：只有本批次至少有一条记录被处理器接收后才 commit，
+        // 避免轮询到空批次或未绑定 topic 时推进消费位点。
        p.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        p.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, consumer.getAutoOffsetReset());
        p.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, consumer.getMaxPollRecords());
--- a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerRunner.java
+++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerRunner.java
@@ -45,6 +45,7 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
        if (!running.compareAndSet(false, true)) {
            return;
        }
+        // 每个 worker 一条后台线程，避免某个处理器阻塞时拖慢其他消费组。
        executor = Executors.newFixedThreadPool(workers.size(), r -> {
            Thread thread = new Thread(r, "kafka-envelope-consumer");
            thread.setDaemon(true);
@@ -60,6 +61,8 @@ public final class KafkaEnvelopeConsumerRunner implements SmartLifecycle, AutoCl
            try {
                worker.pollOnce(pollTimeout);
            } catch (RuntimeException ex) {
+                // Kafka/处理器异常不让 Spring 生命周期退出；退避后继续消费，
+                // 具体坏消息由 EnvelopeConsumerProcessor 写入 DLQ。
                log.warn("Kafka envelope consumer poll failed; the worker will retry after backoff", ex);
                sleepBackoff();
            }
--- a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerWorker.java
+++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeConsumerWorker.java
@@ -37,8 +37,11 @@ public final class KafkaEnvelopeConsumerWorker implements AutoCloseable {
        for (ConsumerRecord<String, byte[]> record : records) {
            EnvelopeConsumerProcessor processor = processorsByTopic.get(record.topic());
            if (processor == null) {
+                // worker 可能订阅多个 topic；没有显式绑定处理器的 topic 不参与提交语义。
                continue;
            }
+            // EnvelopeConsumerProcessor 内部会把解析或业务错误转成 DLQ 记录，
+            // 这里保持 Kafka worker 的职责单一：轮询、分发、成功后提交 offset。
            processor.process(new EnvelopeConsumerRecord(
                    record.topic(),
                    record.partition(),
@@ -48,6 +51,7 @@ public final class KafkaEnvelopeConsumerWorker implements AutoCloseable {
            processed++;
        }
        if (processed > 0) {
+            // commitSync 放在批次末尾，保证同一个 poll 批次内的消息按 Kafka offset 一起确认。
            consumer.commitSync();
        }
        return processed;
--- a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeDeadLetterSink.java
+++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEnvelopeDeadLetterSink.java
@@ -34,6 +34,7 @@ public final class KafkaEnvelopeDeadLetterSink implements EnvelopeDeadLetterSink
            throw new IllegalArgumentException("record must not be null");
        }
        ProducerRecord<String, byte[]> out = new ProducerRecord<>(topic, record.key(), record.payload());
+        // DLQ payload 保持原始 Kafka value；定位信息全部放 header，便于后续重放或人工排查。
        header(out, "dlq-service", record.service());
        header(out, "dlq-source-topic", record.topic());
        header(out, "dlq-source-partition", Integer.toString(record.partition()));
--- a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java
+++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/KafkaEventSink.java
@@ -43,9 +43,9 @@ public final class KafkaEventSink implements EventSink, AutoCloseable {
    }

    /**
-     * 拒绝 {@link VehicleEvent.RawArchive} —— 原始报文体积大、属于冷存域，本期由
-     * {@code ArchiveEventSink} 独占处理。未来若需要将 raw archive URI 回填到 Kafka 的
-     * {@code vehicle.raw.archive} topic，再放开此过滤并在 Envelope 里填 uri/checksum。
+     * 拒绝 {@link VehicleEvent.RawArchive} —— 当前生产 Kafka sink 只投递实时、会话、告警等轻量事件。
+     * {@link EnvelopeMapper} 和 {@link TopicRouter} 已具备 RAW envelope/topic 的映射能力，但这里仍
+     * 显式过滤；如需把 32960 raw 引用转发到 {@code vehicle.raw.archive}，应先移除此过滤并补充测试。
     */
    @Override
    public boolean accepts(VehicleEvent event) {
--- a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqAutoConfiguration.java
+++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqAutoConfiguration.java
@@ -30,6 +30,10 @@ import java.util.Properties;
 *   <li>{@code lingniu.ingest.sink.mq.type=kafka}（默认 kafka）—— 后端选择，预留
 *       rocketmq/pulsar 等扩展。
 * </ol>
+ *
+ * <p>Producer 和 Consumer 是两个独立开关：{@code sink.mq.enabled=true} 只表示可以创建
+ * Kafka producer/sink；是否从 Kafka 拉 envelope 还要单独开启
+ * {@code lingniu.ingest.sink.mq.consumer.enabled=true} 并配置 bindings。
 */
@AutoConfiguration
@EnableConfigurationProperties(SinkMqProperties.class)
@@ -84,6 +88,7 @@ public class SinkMqAutoConfiguration {
                                         TopicRouter router,
                                         SinkMqProperties props,
                                         CircuitBreaker breaker) {
+        // KafkaEventSink 是 EventBus 的生产端出口；它不会启动任何 Kafka 消费线程。
        return new KafkaEventSink(producer, mapper, router, props.getTopics().getDlq(), breaker);
    }

@@ -101,6 +106,7 @@ public class SinkMqAutoConfiguration {
    @ConditionalOnProperty(prefix = "lingniu.ingest.sink.mq.consumer", name = "enabled", havingValue = "true")
    public KafkaEnvelopeConsumerRunner kafkaEnvelopeConsumerRunner(Map<String, EnvelopeConsumerProcessor> processors,
                                                                   SinkMqProperties props) {
+        // Consumer runner 根据 processor bean 名和配置 binding 生成 worker；没有 binding 时直接失败，避免静默不消费。
        List<KafkaEnvelopeConsumerWorker> workers = new KafkaEnvelopeConsumerFactory().createWorkers(processors, props);
        if (workers.isEmpty()) {
            throw new IllegalStateException("no kafka envelope consumer workers created; check consumer bindings");
--- a/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java
+++ b/modules/sinks/sink-mq/src/main/java/com/lingniu/ingest/sink/mq/SinkMqProperties.java
@@ -20,12 +20,14 @@ public class SinkMqProperties {

    /** MQ 后端类型。目前仅支持 kafka，预留 rocketmq/pulsar 等。 */
    private String type = "kafka";
+    /** Kafka bootstrap servers；生产环境应通过环境变量覆盖，不建议使用默认开发地址。 */
    private String bootstrapServers = "114.55.58.251:9092";
    private String compressionType = "zstd";
    private int lingerMs = 20;
    private int batchSize = 65536;
    private String acks = "all";
    private boolean enableIdempotence = true;
+    /** 写入 envelope 的节点标识，进入 Protobuf 字段 ingest_node_id，便于追踪多实例来源。 */
    private String nodeId = "ingest-local";
    private Topics topics = new Topics();
    private Consumer consumer = new Consumer();
@@ -54,12 +56,16 @@ public class SinkMqProperties {
    public void setConsumer(Consumer consumer) { this.consumer = consumer; }

    public static class Topics {
+        /** 实时遥测事件 topic；GB32960 RAW-only 架构下可逐步弱化该 topic。 */
        private String realtime = "vehicle.realtime";
+        /** 位置事件 topic；通常可由 realtime/RAW 派生。 */
        private String location = "vehicle.location";
        private String alarm = "vehicle.alarm";
        private String session = "vehicle.session";
        private String mediaMeta = "vehicle.media.meta";
+        /** RAW 归档引用 topic，payload 应携带 archive URI/size，不建议携带完整 raw bytes。 */
        private String rawArchive = "vehicle.raw.archive";
+        /** producer 熔断或 consumer 处理失败时的死信 topic。 */
        private String dlq = "vehicle.dlq";

        public String getRealtime() { return realtime; }
@@ -79,6 +85,7 @@ public class SinkMqProperties {
    }

    public static class Consumer {
+        /** Kafka 消费总开关。与 producer 总开关分离，默认 false，避免单体服务意外自消费。 */
        private boolean enabled = false;
        private boolean autoStartup = true;
        private String clientIdPrefix = "lingniu-envelope-consumer";
@@ -86,6 +93,12 @@ public class SinkMqProperties {
        private int loopBackoffMillis = 1000;
        private String autoOffsetReset = "earliest";
        private int maxPollRecords = 500;
+        /**
+         * Processor bean name -> Kafka binding。
+         *
+         * <p>示例 key：{@code eventHistoryEnvelopeConsumerProcessor}、
+         * {@code vehicleStateEnvelopeConsumerProcessor}、{@code vehicleStatEnvelopeConsumerProcessor}。
+         */
        private Map<String, Binding> bindings = new LinkedHashMap<>();

        public boolean isEnabled() { return enabled; }
@@ -107,8 +120,11 @@ public class SinkMqProperties {
    }

    public static class Binding {
+        /** 单个 processor binding 开关，用于临时停某个下游消费者而不关整个 consumer runner。 */
        private boolean enabled = true;
+        /** Kafka consumer group id。不同服务要独立消费同一 topic 时必须使用不同 group。 */
        private String groupId;
+        /** 该 processor 订阅的 topic 列表。 */
        private List<String> topics = new ArrayList<>();

        public boolean isEnabled() { return enabled; }