Commit 7cc9b3ed by 仲光辉

feat: 添加基本文档,已经 kafka-clients 原生API 的测试用例编写

parent 6229230e
......@@ -24,6 +24,11 @@
hs_err_pid*
.gradle
build
**/build
**/**/build
.idea
*.iml
......@@ -24,80 +24,75 @@ buildscript {
}
}
description 'dankal-test-kafka'
version '1.0.0.RELEASE'
allprojects {
apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'io.spring.dependency-management'
apply plugin: 'org.springframework.boot'
apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'io.spring.dependency-management'
apply plugin: 'org.springframework.boot'
// jdk 相关配置
sourceCompatibility = '1.8'
targetCompatibility = '1.8'
// jdk 相关配置
sourceCompatibility = '1.8'
targetCompatibility = '1.8'
group 'cn.dankal.test'
version '1.0.0.RELEASE'
group 'cn.dankal.test'
// 类似于 maven Properties
ext {
set('springCloudAlibabaVersion', "2.1.3.RELEASE")
set('springCloudVersion', "Greenwich.SR5")
// 类似于 maven Properties
ext {
set('springCloudAlibabaVersion', "2.1.3.RELEASE")
set('springCloudVersion', "Greenwich.SR5")
set('hutoolCoreVersion', "5.4.6")
set('kafkaClientVersion', '2.7.0')
}
// 依赖仓库配置
repositories {
// GRADLE_USER_HOME : maven 本地仓库地址
mavenLocal()
maven {
url 'https://maven.aliyun.com/repository/public/'
}
maven {
url 'https://maven.aliyun.com/repository/spring/'
}
maven {
url 'https://maven.aliyun.com/repository/gradle-plugin'
set('hutoolCoreVersion', "5.4.6")
set('kafkaClientVersion', '2.7.0')
}
maven {
url 'https://maven.aliyun.com/repository/spring-plugin'
// 依赖仓库配置
repositories {
// GRADLE_USER_HOME : maven 本地仓库地址
mavenLocal()
maven {
url 'https://maven.aliyun.com/repository/public/'
}
maven {
url 'https://maven.aliyun.com/repository/spring/'
}
maven {
url 'https://maven.aliyun.com/repository/gradle-plugin'
}
maven {
url 'https://maven.aliyun.com/repository/spring-plugin'
}
mavenCentral()
}
mavenCentral()
}
dependencies {
// spring dependence
implementation 'org.springframework.boot:spring-boot-starter-web'
implementation 'org.springframework.kafka:spring-kafka'
testImplementation('org.springframework.boot:spring-boot-starter-test') {
exclude group: 'org.junit.vintage', module: 'junit-vintage-engine'
dependencies {
// third party dependence
implementation "cn.hutool:hutool-core:${hutoolCoreVersion}"
compile 'org.projectlombok:lombok'
annotationProcessor 'org.projectlombok:lombok'
testCompileOnly 'org.projectlombok:lombok'
}
// third party dependence
implementation "cn.hutool:hutool-core:${hutoolCoreVersion}"
compile 'org.projectlombok:lombok'
annotationProcessor 'org.projectlombok:lombok'
testCompileOnly 'org.projectlombok:lombok'
}
tasks.withType(JavaCompile) {
options.encoding = "UTF-8"
}
tasks.withType(JavaCompile) {
options.encoding = "UTF-8"
}
// 类似于 maven dependencyManagement
dependencyManagement {
imports {
// spring-cloud
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
// spring-cloud-alibaba
mavenBom "com.alibaba.cloud:spring-cloud-alibaba-dependencies:${springCloudAlibabaVersion}"
dependencyManagement {
imports {
// spring-cloud
mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}"
// spring-cloud-alibaba
mavenBom "com.alibaba.cloud:spring-cloud-alibaba-dependencies:${springCloudAlibabaVersion}"
}
}
}
// 单元测试配置
test {
useJUnitPlatform()
test {
useJUnitPlatform()
}
}
// package 配置
bootJar.enabled = false
bootJar.enabled = true
jar.enabled = true
description 'dankal-test-kafka-protogenesis'
version '1.0.0.RELEASE'
dependencies {
implementation "org.apache.kafka:kafka-clients:${kafkaClientVersion}"
testImplementation 'org.junit.jupiter:junit-jupiter-api'
}
bootJar.enabled = false
jar.enabled = true
\ No newline at end of file
package cn.dankal.test.common;
import org.apache.kafka.common.TopicPartition;
import java.time.format.DateTimeFormatter;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-25
*/
public interface Const {
/**
* format pattern: yyyy-MM-dd HH:mm:ss.SSS
*/
DateTimeFormatter FULL_DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
/**
* TODO 由于只是测试 故就不做执行保存数据操作,此处使用 本地 Map 存储分区与对应偏移量信息
* <pre>
* key: TopicPartition 分区信息
* value: Long 分区偏移量信
* </pre>
*/
ConcurrentHashMap<TopicPartition, Long> PARTITION_OFFSET_INFO_MAP = new ConcurrentHashMap<>();
/**
* Kafka const
*/
interface Kafka {
/**
* kafka topic
*/
interface Topic {
/**
* kafka topic: dankal-test-hello-kafka
*/
String HELLO_TOPIC = "dankal-test-hello-kafka";
}
/**
* kafka consumer group id
*/
interface ConsumerGroupId {
/**
* hello-consumer-group-id
*/
String HELLO_CONSUMER_GROUP_ID = "hello-consumer-group-id";
/**
* test-group-a
*/
String TEST_GROUP_A = "test-group-a";
/**
* test-group-b
*/
String TEST_GROUP_B = "test-group-b";
/**
* test-group-c
*/
String TEST_GROUP_C = "test-group-c";
/**
* test-kafka-rebalance
*/
String TEST_KAFKA_REBALANCE = "test-kafka-rebalance";
}
}
}
package cn.dankal.test.factory;
import cn.hutool.core.util.StrUtil;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* ThreadExecutorFactory
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2020-12-29
*/
public class ThreadExecutorFactory {
/**
* 获取 ThreadPoolExecutor
*
* @param corePoolSize 核心线程数
* @param maximumPoolSize 最大线程数
* @param keepAliveTime 超过核心线程数的线程存活时间
* @param timeUnit 超过核心线程数的线程存活时间单位
* @param threadNamePrefix 创建线程的名称前缀
* @param blockQueueCapacity 线程池阻塞队列的容量
* @return ThreadPoolExecutor
* @see ThreadPoolExecutor
*/
public static ThreadPoolExecutor getThreadPoolExecutor(int corePoolSize,
int maximumPoolSize,
long keepAliveTime,
TimeUnit timeUnit,
String threadNamePrefix,
int blockQueueCapacity) {
// 参数校验
if (corePoolSize <= 0) {
throw new IllegalArgumentException("corePoolSize 必须大于0");
}
if (maximumPoolSize <= 0) {
throw new IllegalArgumentException("maximumPoolSize 必须大于0");
}
if (keepAliveTime < 0) {
throw new IllegalArgumentException("keepAliveTime 必须大于0");
}
if (null == timeUnit) {
throw new IllegalArgumentException("timeUnit 不能为空");
}
if (corePoolSize > maximumPoolSize) {
throw new IllegalArgumentException("corePoolSize 必须小于等于 maximumPoolSize");
}
if (StrUtil.isBlank(threadNamePrefix)) {
throw new IllegalArgumentException("threadNamePrefix 不能为空");
}
if (blockQueueCapacity <= 0) {
throw new IllegalArgumentException("blockQueueCapacity 必须大于0");
}
return new ThreadPoolExecutor(
corePoolSize,
maximumPoolSize,
keepAliveTime,
timeUnit,
new ArrayBlockingQueue<>(blockQueueCapacity),
threadFactory(threadNamePrefix)
);
}
/**
* 初始化 ThreadFactory 并为 创建的 线程 命名
*
* @param namePrefix 线程名称前缀
* @return ThreadFactory
*/
private static ThreadFactory threadFactory(String namePrefix) {
return (runnable -> {
Thread thread = new Thread(runnable);
long id = thread.getId();
thread.setName(namePrefix + "-" + id);
return thread;
});
}
/**
* 获取一个 <code>ThreadPoolExecutor</code>
* <p>
* coreSize: 6
* <br/>
* maximumPoolSize: 10
* <br/>
* KeepAliveTime: 10
* <br/>
* unit: TimeUnit.MILLISECONDS
* <br/>
* blockQueueCapacity: 100
* <br/>
* </p>
*
* @param threadNamePrefix <code>threadNamePrefix</code>
* @return ThreadPoolExecutor
*/
public static ThreadPoolExecutor getThreadPoolExecutor(final String threadNamePrefix) {
return ThreadExecutorFactory.getThreadPoolExecutor(6,
10,
10,
TimeUnit.MILLISECONDS,
threadNamePrefix,
100);
}
}
package cn.dankal.test.concurrent;
import cn.dankal.test.common.Const;
import cn.dankal.test.factory.ThreadExecutorFactory;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.util.Objects;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ThreadPoolExecutor;
/**
* Kafka 测试: 多线程环境的 Kafka Producer
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-19
*/
public class KafkaProducerOnConcurrentTest {
/**
* ThreadPoolExecutor
*/
private static ThreadPoolExecutor threadPoolExecutor;
/**
* KafkaProducer
*/
private static KafkaProducer<String, String> kafkaProducer;
@BeforeAll
public static void before() {
threadPoolExecutor = ThreadExecutorFactory.getThreadPoolExecutor("dankal-test-kafka");
Properties properties = new Properties();
final String bootstrapServers = "192.168.128.3:9092,192.168.128.3:9093,192.168.128.3:9094";
// broker(Kafka服务器) 地址清单
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
// message key 序列化器
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());
// message value 序列化器
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());
// 通过 Properties 配置 实例化 KafkaProducer
kafkaProducer = new KafkaProducer<>(properties);
}
/**
* 测试多线程环境下的 Kafka Producer
*/
@Test
public void test() throws InterruptedException {
final int threadCount = 5;
// 保证所有线程全部完成消息发送
final CountDownLatch countDownLatch = new CountDownLatch(threadCount);
// 保证 所有线程在同一起跑线
final CyclicBarrier cyclicBarrier = new CyclicBarrier(threadCount);
for (int i = 0; i < threadCount; i++) {
final int index = i + 1;
Runnable kafkaProducerRunnable = () -> {
final String key = "dankal-test-" + index;
final String message = "dankal-test-kafka-producer-on-concurrent";
ProducerRecord<String, String> producerRecord = new ProducerRecord<>(
Const.Kafka.Topic.HELLO_TOPIC,
key,
message
);
try {
cyclicBarrier.await();
} catch (Exception e) {
// ignored
}
kafkaProducer.send(
producerRecord,
(recordMetadata, exception) -> {
countDownLatch.countDown();
if (null != exception) {
// 发送异常
exception.printStackTrace();
return;
}
// 发送成功
System.out.printf(
"完成消息发送 %d ==> 主题: %s 分区: %s 偏移量: %d%n",
recordMetadata.timestamp(),
recordMetadata.topic(),
recordMetadata.partition(),
recordMetadata.offset());
});
};
threadPoolExecutor.submit(kafkaProducerRunnable);
}
// 等待所有消息执行完成
countDownLatch.await();
}
@AfterAll
public static void after() {
if (Objects.nonNull(kafkaProducer)) {
kafkaProducer.close();
}
if (null != threadPoolExecutor) {
threadPoolExecutor.shutdown();
}
}
}
package cn.dankal.test.consumercommit;
import cn.dankal.test.common.Const;
import cn.hutool.core.collection.CollectionUtil;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
/**
* 测试Kafka: Kafka Consumer 偏移量提交方式
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-26
*/
public class KafkaConsumerCommitOffsetTest {
/**
* KafkaConsumer
*/
private KafkaConsumer<String, String> kafkaConsumer;
/**
* Properties
*/
private static Properties properties;
@BeforeAll
public static void before() {
properties = new Properties();
final String bootstrapServers = "192.168.128.3:9092,192.168.128.3:9093,192.168.128.3:9094";
// broker(Kafka服务器) 地址清单
properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
// message key 反序列化器
properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
// message value 反序列化器
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
properties.put(ConsumerConfig.GROUP_ID_CONFIG, Const.Kafka.ConsumerGroupId.HELLO_CONSUMER_GROUP_ID);
}
/**
* 测试 Kafka Consumer 自动提交偏移量
*/
@Test
public void testAutoCommitOffset() {
// 通过 Properties 配置 实例化 KafkaConsumer
kafkaConsumer = new KafkaConsumer<>(properties);
// consumer 订阅 主题
kafkaConsumer.subscribe(CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC));
while (true) {
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(500));
for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
// 成功拉取 ConsumerRecords 后,5000毫秒后 Consumer 会自动提交偏移量
System.out.printf(
"%s 消息消息 ==> 主题: %s 分区: %s 偏移量: %s key: %s value: %s %n",
Const.FULL_DATE_TIME_FORMATTER.format(LocalDateTime.now()),
consumerRecord.topic(),
consumerRecord.partition(),
consumerRecord.offset(),
consumerRecord.key(),
consumerRecord.value());
}
}
}
/**
* 测试 Kafka Consumer 同步手动提交
*/
@Test
public void testSyncCommitOffset() {
properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
// 通过 Properties 配置 实例化 KafkaConsumer
kafkaConsumer = new KafkaConsumer<>(properties);
// consumer 订阅 主题
kafkaConsumer.subscribe(CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC));
while (true) {
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(500));
for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
// 成功拉取 ConsumerRecords 后,5000毫秒后 Consumer 会自动提交偏移量
System.out.printf(
"%s 消息消息 ==> 主题: %s 分区: %s 偏移量: %s key: %s value: %s %n",
Const.FULL_DATE_TIME_FORMATTER.format(LocalDateTime.now()),
consumerRecord.topic(),
consumerRecord.partition(),
consumerRecord.offset(),
consumerRecord.key(),
consumerRecord.value());
}
if (CollectionUtil.isNotEmpty(consumerRecords)) {
// 同步手动提交偏移量
kafkaConsumer.commitSync();
System.out.println("after invoke commitSync()");
}
}
}
/**
* 测试 Kafka Consumer 异步手动提交
*/
@Test
public void testAsyncCommitOffset() {
properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
// 通过 Properties 配置 实例化 KafkaConsumer
kafkaConsumer = new KafkaConsumer<>(properties);
// consumer 订阅 主题
kafkaConsumer.subscribe(CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC));
while (true) {
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(500));
for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
// 成功拉取 ConsumerRecords 后,5000毫秒后 Consumer 会自动提交偏移量
System.out.printf(
"%s 消息消息 ==> 主题: %s 分区: %s 偏移量: %s key: %s value: %s %n",
Const.FULL_DATE_TIME_FORMATTER.format(LocalDateTime.now()),
consumerRecord.topic(),
consumerRecord.partition(),
consumerRecord.offset(),
consumerRecord.key(),
consumerRecord.value());
}
if (CollectionUtil.isNotEmpty(consumerRecords)) {
// 异步手动提交偏移量
kafkaConsumer.commitAsync((offsets, exception) -> {
if (null != exception) {
// 提交偏移量发生异常
exception.printStackTrace();
return;
}
// 成功提交偏移量
for (Map.Entry<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataEntry : offsets.entrySet()) {
TopicPartition topicPartition = topicPartitionOffsetAndMetadataEntry.getKey();
OffsetAndMetadata offsetAndMetadata = topicPartitionOffsetAndMetadataEntry.getValue();
System.out.printf(
"成功提交偏移量. 主题: %s 分区:%s 偏移量: %d%n",
topicPartition.topic(),
topicPartition.partition(),
offsetAndMetadata.offset()
);
}
});
System.out.println("after invoke commitASync(OffsetCommitCallback)");
}
}
}
/**
* 测试 Kafka Consumer 混合式提交(commitASync/commitSync)
*/
@Test
public void testKafkaMixtureCommitOffset() {
properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
// 通过 Properties 配置 实例化 KafkaConsumer
kafkaConsumer = new KafkaConsumer<>(properties);
// consumer 订阅 主题
kafkaConsumer.subscribe(CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC));
while (true) {
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(500));
try {
for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
// 成功拉取 ConsumerRecords 后,5000毫秒后 Consumer 会自动提交偏移量
System.out.printf(
"%s 消息消息 ==> 主题: %s 分区: %s 偏移量: %s key: %s value: %s %n",
Const.FULL_DATE_TIME_FORMATTER.format(LocalDateTime.now()),
consumerRecord.topic(),
consumerRecord.partition(),
consumerRecord.offset(),
consumerRecord.key(),
consumerRecord.value());
}
if (CollectionUtil.isNotEmpty(consumerRecords)) {
// 异步手动提交偏移量
kafkaConsumer.commitAsync((offsets, exception) -> {
if (null != exception) {
// 提交偏移量发生异常
exception.printStackTrace();
return;
}
// 成功提交偏移量
for (Map.Entry<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataEntry : offsets.entrySet()) {
TopicPartition topicPartition = topicPartitionOffsetAndMetadataEntry.getKey();
OffsetAndMetadata offsetAndMetadata = topicPartitionOffsetAndMetadataEntry.getValue();
System.out.printf(
"成功提交偏移量. 主题: %s 分区:%s 偏移量: %d%n",
topicPartition.topic(),
topicPartition.partition(),
offsetAndMetadata.offset()
);
}
});
System.out.println("after invoke commitASync(OffsetCommitCallback)");
}
} finally {
if (null != kafkaConsumer && CollectionUtil.isNotEmpty(consumerRecords)) {
kafkaConsumer.commitSync();
System.out.println("finally use commitSync() commit offset");
}
}
}
}
/**
* 测试 kafka Consumer: 特定提交
*/
@Test
public void testKafkaSpecificCommitOffset() {
properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
// 通过 Properties 配置 实例化 KafkaConsumer
kafkaConsumer = new KafkaConsumer<>(properties);
Map<TopicPartition, OffsetAndMetadata> offsetAndMetadataMap = new HashMap<>(1 << 5);
// 提交频率
final int commitFrequency = 2;
// consumer 订阅 主题
kafkaConsumer.subscribe(CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC));
while (true) {
int count = 0;
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(500));
for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
try {
// 成功拉取 ConsumerRecords 后,5000毫秒后 Consumer 会自动提交偏移量
System.out.printf(
"%s 消息消息 ==> 主题: %s 分区: %s 偏移量: %s key: %s value: %s %n",
Const.FULL_DATE_TIME_FORMATTER.format(LocalDateTime.now()),
consumerRecord.topic(),
consumerRecord.partition(),
consumerRecord.offset(),
consumerRecord.key(),
consumerRecord.value());
offsetAndMetadataMap.put(
new TopicPartition(consumerRecord.topic(), consumerRecord.partition()),
new OffsetAndMetadata(consumerRecord.offset() + 1, "no metadata")
);
if (count % commitFrequency == 0) {
kafkaConsumer.commitAsync(offsetAndMetadataMap, (offsets, exception) -> {
if (null != exception) {
// 提交偏移量发生异常
exception.printStackTrace();
return;
}
// 成功提交偏移量
for (Map.Entry<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataEntry : offsets.entrySet()) {
TopicPartition topicPartition = topicPartitionOffsetAndMetadataEntry.getKey();
OffsetAndMetadata offsetAndMetadata = topicPartitionOffsetAndMetadataEntry.getValue();
System.out.printf(
"成功提交偏移量. 主题: %s 分区:%s 偏移量: %d%n",
topicPartition.topic(),
topicPartition.partition(),
offsetAndMetadata.offset()
);
}
});
}
count++;
} finally {
if (null != kafkaConsumer) {
kafkaConsumer.commitSync();
}
}
}
}
}
@AfterEach
public void after() {
if (Objects.nonNull(kafkaConsumer)) {
kafkaConsumer.close();
}
}
}
package cn.dankal.test.consumergroup;
import cn.dankal.test.common.Const;
import cn.dankal.test.factory.ThreadExecutorFactory;
import cn.hutool.core.collection.CollectionUtil;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadPoolExecutor;
/**
* 测试 Kafka: 消费群组
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-26
*/
public class KafkaConsumerGroupTest {
/**
* Properties
*/
private Properties properties;
/**
* ThreadPoolExecutor
*/
private final ThreadPoolExecutor threadPoolExecutor = ThreadExecutorFactory.getThreadPoolExecutor("dankal-test-kafka-consumer-group");
@BeforeEach
public void before() {
this.properties = new Properties();
final String bootstrapServers = "192.168.128.3:9092,192.168.128.3:9093,192.168.128.3:9094";
// broker(Kafka服务器) 地址清单
properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
// message key 反序列化器
properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
// message value 反序列化器
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
}
/**
* 测试Kafka Consumer Group A
*/
@Test
public void testKafkaConsumerGroupA() throws InterruptedException {
properties.put(ConsumerConfig.GROUP_ID_CONFIG, Const.Kafka.ConsumerGroupId.TEST_GROUP_A);
threadPoolExecutor.submit(() -> {
KafkaConsumer<String, String> groupAConsumer01 = new KafkaConsumer<>(properties);
groupAConsumer01.subscribe(CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC));
this.doConsumer("groupAConsumer01", groupAConsumer01);
});
threadPoolExecutor.submit(() -> {
KafkaConsumer<String, String> groupAConsumer02 = new KafkaConsumer<>(properties);
groupAConsumer02.subscribe(CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC));
this.doConsumer("groupAConsumer02", groupAConsumer02);
});
threadPoolExecutor.submit(() -> {
KafkaConsumer<String, String> groupAConsumer03 = new KafkaConsumer<>(properties);
groupAConsumer03.subscribe(CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC));
this.doConsumer("groupAConsumer03", groupAConsumer03);
});
// 不让程序终止运行
new CountDownLatch(1).await();
}
/**
* 测试Kafka Consumer Group B
*/
@Test
public void testKafkaConsumerGroupB() throws InterruptedException {
properties.put(ConsumerConfig.GROUP_ID_CONFIG, Const.Kafka.ConsumerGroupId.TEST_GROUP_B);
threadPoolExecutor.submit(() -> {
KafkaConsumer<String, String> groupBConsumer01 = new KafkaConsumer<>(properties);
groupBConsumer01.subscribe(CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC));
this.doConsumer("groupBConsumer01", groupBConsumer01);
});
threadPoolExecutor.submit(() -> {
KafkaConsumer<String, String> groupBConsumer02 = new KafkaConsumer<>(properties);
groupBConsumer02.subscribe(CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC));
this.doConsumer("groupBConsumer02", groupBConsumer02);
});
new CountDownLatch(1).await();
}
/**
* 测试Kafka Consumer Group C
*/
@Test
public void testKafkaConsumerGroupC() {
properties.put(ConsumerConfig.GROUP_ID_CONFIG, Const.Kafka.ConsumerGroupId.TEST_GROUP_C);
KafkaConsumer<String, String> groupCConsumer = new KafkaConsumer<>(properties);
groupCConsumer.subscribe(CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC));
this.doConsumer("groupCConsumer", groupCConsumer);
}
/**
* 执行 Kafka 消息消费
*
* @param consumerName the consumer name
* @param kafkaConsumer the KafkaConsumer
*/
private void doConsumer(String consumerName, KafkaConsumer<String, String> kafkaConsumer) {
while (true) {
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(500));
for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
System.out.printf(
"%s : %s 消费消息 ==> 主题: %s 分区: %s 偏移量: %s key: %s value: %s %n",
Const.FULL_DATE_TIME_FORMATTER.format(LocalDateTime.now()),
consumerName,
consumerRecord.topic(),
consumerRecord.partition(),
consumerRecord.offset(),
consumerRecord.key(),
consumerRecord.value());
}
}
}
}
package cn.dankal.test.first;
import cn.dankal.test.common.Const;
import cn.hutool.core.collection.CollectionUtil;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Objects;
import java.util.Properties;
/**
* 测试 Kafka clients: 第一个 consumer
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-25
*/
public class KafkaFirstConsumer {
/**
* KafkaConsumer
*/
private static KafkaConsumer<String, String> kafkaConsumer;
@BeforeAll
public static void before() {
Properties properties = new Properties();
final String bootstrapServers = "192.168.128.3:9092,192.168.128.3:9093,192.168.128.3:9094";
// broker(Kafka服务器) 地址清单
properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
// message key 反序列化器
properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
// message value 反序列化器
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
properties.put(ConsumerConfig.GROUP_ID_CONFIG, Const.Kafka.ConsumerGroupId.HELLO_CONSUMER_GROUP_ID);
// 通过 Properties 配置 实例化 KafkaConsumer
kafkaConsumer = new KafkaConsumer<>(properties);
}
/**
* 第一个 Kafka 消费者
*/
@Test
public void testKafkaFirstConsumer() {
// consumer 订阅 主题
kafkaConsumer.subscribe(CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC));
while (true) {
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(500));
for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
System.out.printf(
"%s 消息消息 ==> 主题: %s 分区: %s 偏移量: %s key: %s value: %s %n",
Const.FULL_DATE_TIME_FORMATTER.format(LocalDateTime.now()),
consumerRecord.topic(),
consumerRecord.partition(),
consumerRecord.offset(),
consumerRecord.key(),
consumerRecord.value());
}
}
}
@AfterAll
public static void after() {
if (Objects.nonNull(kafkaConsumer)) {
kafkaConsumer.close();
}
}
}
package cn.dankal.test.first;
import cn.dankal.test.common.Const;
import cn.hutool.core.util.RandomUtil;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.util.Objects;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
/**
* 测试 Kafka clients: 第一个 producer
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-25
*/
public class KafkaFirstProducer {
/**
* KafkaProducer
*/
private static KafkaProducer<String, String> kafkaProducer;
@BeforeAll
public static void before() {
Properties properties = new Properties();
final String bootstrapServers = "192.168.128.3:9092,192.168.128.3:9093,192.168.128.3:9094";
// broker(Kafka服务器) 地址清单
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
// message key 序列化器
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());
// message value 序列化器
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());
// 通过 Properties 配置 实例化 KafkaProducer
kafkaProducer = new KafkaProducer<>(properties);
}
/**
* 第一个 Kafka 生产者
*/
@Test
public void testKafkaFirstProducer() throws ExecutionException, InterruptedException {
final int sendMessageCount = 6;
for (int i = 0; i < sendMessageCount; i++) {
// 消息key (可以为空)
final String key = "dankal-hello-" + RandomUtil.randomInt(10, 5000);
// 消息内容
final String message = RandomUtil.randomString(64);
ProducerRecord<String, String> stringProducerRecord =
new ProducerRecord<String, String>(
Const.Kafka.Topic.HELLO_TOPIC,
key,
message
);
kafkaProducer.send(stringProducerRecord).get();
}
}
@AfterAll
public static void after() {
if (Objects.nonNull(kafkaProducer)) {
kafkaProducer.close();
}
}
}
package cn.dankal.test.independentconsumer;
import cn.dankal.test.common.Const;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Objects;
import java.util.Properties;
import java.util.stream.Collectors;
/**
* 测试 Kafka : Kafka 独立消费者测试
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-27
*/
public class IndependentConsumerTest {
/**
* KafkaConsumer
*/
private static KafkaConsumer<String, String> kafkaConsumer;
@BeforeAll
public static void before() {
Properties properties = new Properties();
final String bootstrapServers = "192.168.128.3:9092,192.168.128.3:9093,192.168.128.3:9094";
// broker(Kafka服务器) 地址清单
properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
// message key 反序列化器
properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
// message value 反序列化器
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
// 通过 Properties 配置 实例化 KafkaConsumer
kafkaConsumer = new KafkaConsumer<>(properties);
}
/**
* Kafka 独立消费者 测试
*/
@Test
public void testKafkaIndependentConsumer() {
// 获取分区信息
List<PartitionInfo> partitionInfoList = kafkaConsumer.partitionsFor(Const.Kafka.Topic.HELLO_TOPIC);
// 通过 PartitionInfoList 封装 topicPartitionList
List<TopicPartition> topicPartitionList = partitionInfoList.stream()
.map(
partitionInfo -> new TopicPartition(partitionInfo.topic(), partitionInfo.partition())
)
.collect(Collectors.toList());
// 给独立消费者分配分区
kafkaConsumer.assign(topicPartitionList);
while (true) {
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(500));
for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
System.out.printf(
"%s 消息消息 ==> 主题: %s 分区: %s 偏移量: %s key: %s value: %s %n",
Const.FULL_DATE_TIME_FORMATTER.format(LocalDateTime.now()),
consumerRecord.topic(),
consumerRecord.partition(),
consumerRecord.offset(),
consumerRecord.key(),
consumerRecord.value());
// TODO 分区效果需要我们自己都过编码实现
}
}
}
@AfterAll
public static void after() {
if (Objects.nonNull(kafkaConsumer)) {
kafkaConsumer.close();
}
}
}
package cn.dankal.test.producersendmode;
import cn.dankal.test.common.Const;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.common.serialization.StringSerializer;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.util.Objects;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
/**
* Kafka 生产者: 生产消息的三种形式
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-26
*/
public class KafkaProducerSendModeTest {
/**
* KafkaProducer
*/
private static KafkaProducer<String, String> kafkaProducer;
@BeforeAll
public static void before() {
Properties properties = new Properties();
final String bootstrapServers = "192.168.128.3:9092,192.168.128.3:9093,192.168.128.3:9094";
// broker(Kafka服务器) 地址清单
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
// message key 序列化器
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());
// message value 序列化器
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());
// 通过 Properties 配置 实例化 KafkaProducer
kafkaProducer = new KafkaProducer<>(properties);
}
/**
* 测试: Kafka生产者发送模式: 发送并忘记
*/
@Test
public void testSendAndForget() {
final String topic = Const.Kafka.Topic.HELLO_TOPIC;
final String key = "dankal-test-send-mode";
final String message = "dankal";
ProducerRecord<String, String> producerRecord =
new ProducerRecord<>(
topic,
key,
message
);
// mode: 发送并忘记
kafkaProducer.send(producerRecord);
}
/**
* 测试: Kafka生产者发送模式: 同步发送
*/
@Test
public void testSyncSendMessage() throws ExecutionException, InterruptedException {
final String topic = Const.Kafka.Topic.HELLO_TOPIC;
final String key = "dankal-test-send-mode";
final String message = "dankal";
ProducerRecord<String, String> producerRecord =
new ProducerRecord<>(
topic,
key,
message
);
// mode: 同步发送
RecordMetadata recordMetadata = kafkaProducer.send(producerRecord)
// 调用 Future#get() 会阻塞当前线程
.get();
System.out.printf(
"完成消息发送 %d ==> 主题: %s 分区: %s 偏移量: %d%n",
recordMetadata.timestamp(),
recordMetadata.topic(),
recordMetadata.partition(),
recordMetadata.offset()
);
}
/**
* 测试: Kafka生产者发送模式: 异步发送
*/
@Test
public void testAsyncSendMessage() {
final String topic = Const.Kafka.Topic.HELLO_TOPIC;
final String key = "dankal-test-send-mode";
final String message = "dankal";
ProducerRecord<String, String> producerRecord =
new ProducerRecord<>(
topic,
key,
message
);
// mode: 异步发送
kafkaProducer.send(
producerRecord,
(recordMetadata, exception) -> {
// 发送消息失败: exception message
if (null != exception) {
exception.printStackTrace();
return;
}
// 发送消息成功
System.out.printf(
"异步发送消息成功 %d ==> 主题: %s 分区: %s 偏移量: %d%n",
recordMetadata.timestamp(),
recordMetadata.topic(),
recordMetadata.partition(),
recordMetadata.offset());
}
);
System.out.println("已经通过异步形式发送 Kafka 消息");
}
@AfterAll
public static void after() {
if (Objects.nonNull(kafkaProducer)) {
kafkaProducer.close();
}
}
}
package cn.dankal.test.rebalance;
import cn.dankal.test.common.Const;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.map.MapUtil;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
/**
* 测试 Kafka Rebalance: ConsumerWorker
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-26
*/
public class ConsumerWorker {
/**
* Map<TopicPartition, OffsetAndMetadata> 当前消费者分区与偏移量信息
*/
private final Map<TopicPartition, OffsetAndMetadata> offsetAndMetadataMap;
/**
* current ConsumerWorker KafkaConsumer instance
*/
private final KafkaConsumer<String, String> kafkaConsumer;
/**
* 是否是临时工作者: 临时工作者在消费五条消息之后将退出当前消费群组
*/
private final boolean isTempWorker;
/**
* 临时工作者的消费消息的次数: 5
*/
private final static int TEMP_WORK_COUNT = 5;
/**
* KafkaConsumer config properties
*/
private static final Properties PROPERTIES;
//TODO maybe 事务类 提供事务支持
//private final Transactional transactional;
static {
PROPERTIES = new Properties();
final String bootstrapServers = "192.168.128.3:9092,192.168.128.3:9093,192.168.128.3:9094";
// broker(Kafka服务器) 地址清单
PROPERTIES.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
// message key 反序列化器
PROPERTIES.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
// message value 反序列化器
PROPERTIES.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
PROPERTIES.put(ConsumerConfig.GROUP_ID_CONFIG, Const.Kafka.ConsumerGroupId.TEST_KAFKA_REBALANCE);
}
public ConsumerWorker(boolean isTempWorker) {
this.offsetAndMetadataMap = new ConcurrentHashMap<>();
this.isTempWorker = isTempWorker;
// 消费者线程不安全,建议一个线程一个对应一个消费者实例
this.kafkaConsumer = new KafkaConsumer<>(PROPERTIES);
// 设置 分区再均衡监听器
this.kafkaConsumer.subscribe(
CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC),
new RebalanceListener(offsetAndMetadataMap, kafkaConsumer)
);
}
/**
* 执行 Kafka message consumer work
*/
public void doWork() {
// 执行计数 ,搭配 isTempWorker 使用
int execCount = 0;
try {
while (true) {
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(500));
for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
try {
// TODO Transaction: transaction begin
// transaction.begin();
// TODO 消费消息 处理业务
long currentMessageOffset = consumerRecord.offset();
System.out.printf(
"%s 消息消息 ==> 主题: %s 分区: %s 偏移量: %s key: %s value: %s %n",
Const.FULL_DATE_TIME_FORMATTER.format(LocalDateTime.now()),
consumerRecord.topic(),
consumerRecord.partition(),
currentMessageOffset,
consumerRecord.key(),
consumerRecord.value());
// TODO 保存分区与偏移量信息
TopicPartition topicPartition = new TopicPartition(
consumerRecord.topic(),
consumerRecord.partition()
);
OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(currentMessageOffset + 1, "no metadata");
offsetAndMetadataMap.put(topicPartition, offsetAndMetadata);
//TODO 保存偏移量与分区信息
if (MapUtil.isNotEmpty(offsetAndMetadataMap)) {
for (TopicPartition partition : offsetAndMetadataMap.keySet()) {
Const.PARTITION_OFFSET_INFO_MAP.put(
partition,
offsetAndMetadataMap.get(partition).offset()
);
}
}
// TODO Transaction: transaction commit
// transaction.commit();
kafkaConsumer.commitAsync();
//TODO 执行计数
execCount++;
// TODO just test kafka Rebalance
if (isTempWorker) {
if (execCount == TEMP_WORK_COUNT) {
// 触发分区再均衡
// 临时工作者工作完成~ 即将推出 当前消费群组
System.out.printf(
"%s 完成临时工作即将推出工作群组. offsetAndMetadataMap: %s%n",
Thread.currentThread().getName(),
offsetAndMetadataMap);
//提交自己已经完成的工作消息偏移量
kafkaConsumer.commitSync();
return;
}
}
} catch (Exception e) {
// TODO Transaction: transaction rollback
// transaction.rollback();
// TODO handler exception
e.printStackTrace();
}
}
}
} finally {
kafkaConsumer.commitSync();
kafkaConsumer.close();
}
}
}
package cn.dankal.test.rebalance;
import cn.dankal.test.factory.ThreadExecutorFactory;
import org.junit.jupiter.api.Test;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* 测试 Kafka 分区再均衡
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-26
*/
public class KafkaRebalanceTest {
/**
* 测试 Kafka 分区再均衡
*/
@Test
public void testKafkaRebalance() throws InterruptedException {
final int workerCount = 2;
ThreadPoolExecutor threadPoolExecutor = ThreadExecutorFactory.getThreadPoolExecutor("dankal-test-kafka-rebalance");
for (int i = 0; i < workerCount; i++) {
ConsumerWorker consumerWorker = new ConsumerWorker(false);
threadPoolExecutor.execute(consumerWorker::doWork);
}
TimeUnit.SECONDS.sleep(30);
//启动一个临时消费者,触发分区再均衡
ConsumerWorker tempConsumerWorker = new ConsumerWorker(true);
threadPoolExecutor.execute(tempConsumerWorker::doWork);
new CountDownLatch(1).await();
}
}
package cn.dankal.test.rebalance;
import cn.dankal.test.common.Const;
import cn.hutool.core.util.RandomUtil;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.util.Objects;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
/**
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-26
*/
public class ProducerWorkerTest {
/**
* KafkaProducer
*/
private static KafkaProducer<String, String> kafkaProducer;
@BeforeAll
public static void before() {
Properties properties = new Properties();
final String bootstrapServers = "192.168.128.3:9092,192.168.128.3:9093,192.168.128.3:9094";
// broker(Kafka服务器) 地址清单
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
// message key 序列化器
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());
// message value 序列化器
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());
// 通过 Properties 配置 实例化 KafkaProducer
kafkaProducer = new KafkaProducer<>(properties);
}
/**
* Kafka 生产者 : 用于 Kafka 分区再均衡的测试
*/
@Test
public void testKafkaFirstProducer() throws ExecutionException, InterruptedException {
for (; ; ) {
// 消息key (可以为空)
final String key = "dankal-hello-" + RandomUtil.randomInt(100, 50000000);
// 消息内容
final String message = RandomUtil.randomString(64);
ProducerRecord<String, String> stringProducerRecord =
new ProducerRecord<String, String>(
Const.Kafka.Topic.HELLO_TOPIC,
key,
message
);
TimeUnit.MILLISECONDS.sleep(300);
kafkaProducer.send(stringProducerRecord).get();
}
}
@AfterAll
public static void after() {
if (Objects.nonNull(kafkaProducer)) {
kafkaProducer.close();
}
}
}
package cn.dankal.test.rebalance;
import cn.dankal.test.common.Const;
import cn.hutool.core.collection.CollectionUtil;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* 测试 Kafka 分区再均衡: RebalanceHandler
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-26
*/
public class RebalanceListener implements ConsumerRebalanceListener {
/**
* Map<TopicPartition, OffsetAndMetadata> 当前消费者分区与偏移量信息
*/
private final Map<TopicPartition, OffsetAndMetadata> offsetAndMetadataMap;
/**
* current ConsumerWorker KafkaConsumer instance
*/
private final KafkaConsumer<String, String> kafkaConsumer;
public RebalanceListener(Map<TopicPartition, OffsetAndMetadata> offsetAndMetadataMap, KafkaConsumer<String, String> kafkaConsumer) {
this.offsetAndMetadataMap = offsetAndMetadataMap;
this.kafkaConsumer = kafkaConsumer;
}
/**
* 再执行分区再平衡之前,回调此方法
* <p>
* Collection<TopicPartition> ==> 当前 Consumer 所有拥有消费权的分区信息
* <p>
* 我们一般在此方法完成分区再平衡之前,分区偏移量的存储
*
* @param partitions Collection<TopicPartition>
*/
@Override
public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
// 分区再均衡之前
System.out.printf(
"%s:onPartitionsRevoked 分区: %s%n",
Thread.currentThread().getName(),
partitions
);
System.out.printf(
"%s 即将开始参与分区再均衡,即将提交工作偏移量.偏移量信息为: %s%n",
Thread.currentThread().getName(),
this.offsetAndMetadataMap
);
System.out.printf("偏移量分区信息:%s%n", Const.PARTITION_OFFSET_INFO_MAP);
// TODO transaction begin
try {
for (TopicPartition partition : partitions) {
//TODO 将偏移量执行事务写入
Const.PARTITION_OFFSET_INFO_MAP.put(
partition,
offsetAndMetadataMap.get(partition).offset());
}
//TODO transaction commit
//可选 将偏移量信息提交到 Kafka 服务器
//因为发生分区再均衡 分区偏移量是我们自己在维护 Const.PARTITION_OFFSET_INFO_MAP
kafkaConsumer.commitSync(offsetAndMetadataMap);
} catch (Exception e) {
// TODO transaction rollback
// TODO handler exception
e.printStackTrace();
}
}
/**
* 完成分区再平衡之后,回调当前方法
* <p>
* Collection<TopicPartition> ==> 当前 Consumer 所有拥有消费权的分区信息
* <p>
* 我们一般在此方法内,通过在方法{@link #onPartitionsRevoked(Collection)}存储的分区偏移量信息,对分区偏移量信息进行矫正
*
* @param partitions Collection<TopicPartition>
*/
@Override
public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
System.out.printf(
"%s:onPartitionsAssigned 分区: %s%n",
Thread.currentThread().getName(),
partitions
);
System.out.printf(
"%s 已完成参与分区再均衡.偏移量信息为: %s%n",
Thread.currentThread().getName(),
Const.PARTITION_OFFSET_INFO_MAP
);
//分区偏移量矫正
for (TopicPartition partition : partitions) {
System.out.printf("%s:topic %n", partition);
// 获取我们自己维护的分区偏移量信息
Long offset = Const.PARTITION_OFFSET_INFO_MAP.get(partition);
if (null == offset) {
continue;
}
// 调整分区偏移量
// 从特定偏移量开始记录分区偏移量
// 保证分区再均衡之后,数据偏移量不会错乱
kafkaConsumer.seek(partition, offset);
}
resetConsumerOffsetAndMetadataMap(partitions);
}
/**
* 重置当前 Kafka Consumer 的 <code>Map<TopicPartition, OffsetAndMetadata></code>
*
* @param partitions Collection<TopicPartition>
*/
private void resetConsumerOffsetAndMetadataMap(Collection<TopicPartition> partitions) {
if (CollectionUtil.isEmpty(partitions)) {
return;
}
List<Integer> ownPartitionIdList = partitions.stream()
.map(TopicPartition::partition)
.collect(Collectors.toList());
System.out.printf("%s before: resetConsumerOffsetAndMetadataMap %s %n", Thread.currentThread().getName(), this.offsetAndMetadataMap);
for (TopicPartition topicPartition : this.offsetAndMetadataMap.keySet()) {
if (!ownPartitionIdList.contains(topicPartition.partition())) {
// 分区再平衡时,当前消费者已经不再拥有当前 partition 的消费权
// 避免出现覆盖,故需要在当前 Consumer 的 offsetAndMetadataMap 移除 当前 partition 的信息
// 出现场景可以参阅项目目录: test-kafka-rebalance-log-error.html
this.offsetAndMetadataMap.remove(topicPartition);
}
}
System.out.printf("%s after: resetConsumerOffsetAndMetadataMap %s %n", Thread.currentThread().getName(), this.offsetAndMetadataMap);
}
}
description 'dankal-test-kafka-spring'
version '1.0.0.RELEASE'
dependencies {
// spring dependence
implementation 'org.springframework.boot:spring-boot-starter-web'
implementation 'org.springframework.kafka:spring-kafka'
testImplementation('org.springframework.boot:spring-boot-starter-test') {
exclude group: 'org.junit.vintage', module: 'junit-vintage-engine'
}
}
// package 配置
bootJar.enabled = true
jar.enabled = true
\ No newline at end of file
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-bin.zip
distributionUrl=https\://res.mercymodest.com/gradle-6.7.1-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
......@@ -11,4 +11,6 @@ pluginManagement {
}
}
rootProject.name = 'dankal-test-kafka'
include 'dankal-test-kafka-protogenesis'
include 'dankal-test-kafka-spring'
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>test-kafka-rebalance-log-error</title>
</head>
<body>
<pre>
SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
SLF4J: Defaulting to no-operation (NOP) logger implementation
SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
dankal-test-kafka-rebalance-12:onPartitionsAssigned 分区: [dankal-test-hello-kafka-0, dankal-test-hello-kafka-1]
dankal-test-kafka-rebalance-13:onPartitionsAssigned 分区: [dankal-test-hello-kafka-2]
dankal-test-kafka-rebalance-12 已完成参与分区再均衡.偏移量信息为: {}
dankal-test-hello-kafka-0:topic
dankal-test-kafka-rebalance-13 已完成参与分区再均衡.偏移量信息为: {}
dankal-test-hello-kafka-1:topic
dankal-test-hello-kafka-2:topic
2021-01-26 17:27:21.271 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 52 key: dankal-hello-10318227 value: oog117yvgya39rntjwfr07tgvu3ng52m6gj1k8u9q417b1zldu48gyq2ttew8c10
2021-01-26 17:27:21.552 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 53 key: dankal-hello-8322630 value: k405ab6daw9jyl0i1pbx849c2ap627litboj16016xlkat7v2u46tk1nzy7l9zy9
2021-01-26 17:27:21.856 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 54 key: dankal-hello-38482217 value: 9l7ysvdquwesadmkc6h5v4eej4cm58qo2xpxpia6vribst9o9ixf6q8067ofe9fw
2021-01-26 17:27:22.158 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 46 key: dankal-hello-22554216 value: mq903ianymul6pz0qkj5fxkf4erbpxkgl6fgjolkuj49xj8wr134g0gp662ap9oo
2021-01-26 17:27:22.464 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 47 key: dankal-hello-34477439 value: csgjr7imjjpuf8p4v9r9fs2sv0jq739f4r18gfmaq1x5b0v19y5xyenm757xtc9w
2021-01-26 17:27:22.763 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 48 key: dankal-hello-37453681 value: cx3zih1bpfydwjlzjvw5b8y08u6mtfiistp3scy68sa7bxj8j1jl48y09jrue4p4
2021-01-26 17:27:23.068 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 49 key: dankal-hello-3108496 value: ntux9g5rhlwstvnr6inskcpitkffhbljenmecgfavfd3ky7i29ot7lej4wlzjel8
2021-01-26 17:27:23.371 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 55 key: dankal-hello-27057183 value: gblelfowh5076yvzgikmu5lhyrchfvewxhqi3z4o2gsgmtdo8lrqtdaqpqbku06b
2021-01-26 17:27:23.675 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 50 key: dankal-hello-29496983 value: 1a6b2fyv5vbgw2zjao6mckhlpviok8xwmmwrqh7i8djbgrfpzsg73tg64peoudki
2021-01-26 17:27:23.976 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 56 key: dankal-hello-4476398 value: e92gnmukwqoyfxwau50nfw3li3vevv8moy4ms3khnqmwt5jx9saouvs2w6r3rjju
2021-01-26 17:27:24.281 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 55 key: dankal-hello-10876367 value: feshzbhp2cn4e8b7dxkbczik12orqfe8ln7autkxabhzkuxlfv2z22jdrtmb3bhx
2021-01-26 17:27:24.585 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 56 key: dankal-hello-49592129 value: sr42r33b5hgjjjnutkfrey20a2522385ulrrj0mln6gnc0msny3qtkzfa710vpgd
2021-01-26 17:27:24.890 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 57 key: dankal-hello-22619573 value: venpt1ez9ledqlw06x0patcy4lr06s2g3mjc9ir8p1hycwv2dr5tzrcw6oise369
2021-01-26 17:27:25.195 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 51 key: dankal-hello-11376993 value: 4nloegbbtug58pm2fowfplqmelw2cbrmt5qz7gdw4w9o0hwz834fc9sfshimv3d8
2021-01-26 17:27:25.498 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 57 key: dankal-hello-32586107 value: q2xkx1asfaugiftrcpqps5tqiuo2zfv6m54p73ftwufu20l5jykli8xw8o8pj9rr
2021-01-26 17:27:25.805 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 58 key: dankal-hello-25778100 value: vr1jgn6pzbyieaun75pgevhx5u67or2ium6wpvvt9blqk9kchq67xcoescs7fo46
2021-01-26 17:27:26.115 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 58 key: dankal-hello-45874660 value: nzgd42lyfoxjseokafx1pnllolr84tgckhv9vjjiiicewbg9akjl3fckxg97f3sl
2021-01-26 17:27:26.419 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 52 key: dankal-hello-12155988 value: dcbl08zimixwuozr1lqxioziwhz2rg4pqnwamurk0nj0143e2sufatx2j9ewhznp
2021-01-26 17:27:26.720 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 59 key: dankal-hello-36138025 value: 356n15vcoy4xg8d99cq6ynyf9zfbrnni2wei3tnaw4aeaeosfahbbnqdxdz4rnba
2021-01-26 17:27:27.026 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 60 key: dankal-hello-29737997 value: bvr37xfz1cozpexm9yripx3b8n5bssmclcd2s246ct3wuzsq7gebwq6atml99n7p
2021-01-26 17:27:27.328 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 59 key: dankal-hello-27292465 value: t0mp0vzvit2tx6i40ju5nnf97xygrq8imzsaat03bo1xbchhcnug74uq304ie31u
2021-01-26 17:27:27.632 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 61 key: dankal-hello-30031594 value: cgvc39ut25g0tda0n9m5s025ub2dua0f50rwq1bjgfqmvkvr0vqc0b5nn6h5rpzp
2021-01-26 17:27:27.945 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 62 key: dankal-hello-30385242 value: bms6us6bg0ynyn2489z90pjw9tdgu77qsoagh6soitzvebuuwg2bva9qkylkpbxz
2021-01-26 17:27:28.248 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 60 key: dankal-hello-30594968 value: jjp1fn1cvgmslge8qx1z5bb0gdhgw35m02xg66xgrrxxgd4gq8nz501n5e17roew
2021-01-26 17:27:28.543 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 53 key: dankal-hello-8522631 value: z83phy28zcr9ka2yjnsmby415x782vgiay1pby7vcp3e9ovryo775lff7kfyk7i4
2021-01-26 17:27:28.846 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 63 key: dankal-hello-1972532 value: n6hwn7kh4ixj5a2svhpoenrlc3f92fejqg06ikbdfr1y1imfmvqdzhsipkljdqqy
2021-01-26 17:27:29.160 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 61 key: dankal-hello-47352724 value: 5vgktx9zejx8zf56rwg06ocucv5k68ucs5y6s8kqlyofvbvnv2vgpiurua8btvoj
2021-01-26 17:27:29.458 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 64 key: dankal-hello-19975998 value: 8htqnwihk9m8xzwrq94xu3pwx4upc44yk298oqu6gc2mzlyut81jkn6eyoy0e3hl
2021-01-26 17:27:29.761 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 65 key: dankal-hello-7742538 value: 3omufh5fe4ayp2hdoakciv15qbta55crkzwtgmcf6c7kz6grptaf5o32ryzxduuc
2021-01-26 17:27:30.064 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 62 key: dankal-hello-15939686 value: 1zjaipd9gwuq1khdrmtx6pi8itjc6oamac524dk5hp2779n2c0kicqnecdg0idkz
2021-01-26 17:27:30.368 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 63 key: dankal-hello-15209176 value: vuedcofw9v0dt9sb8fl8ptsnsg0ivzak8bb5854b0htxwjwll94xg5blskwyjvix
2021-01-26 17:27:30.671 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 66 key: dankal-hello-42680287 value: 90d80d1mjr7odu83uwnonleocd7p7d1mt8x1r1kopk1bz628sh1hi6flh6alwzpe
2021-01-26 17:27:30.975 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 64 key: dankal-hello-14935658 value: 06a5c2bnqf2c9mmhint7jza10y7bm61533b8un9nq0bf8fk35euuzon9q9v9ajqx
2021-01-26 17:27:31.277 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 65 key: dankal-hello-26701662 value: k60hbyzbne07jbb1pr15c0kg1wwmb1l867xb388v1cci27e83f6rbzc3d4jdkujf
2021-01-26 17:27:31.582 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 54 key: dankal-hello-13179562 value: evji7bve763kb5ofli0mt38cfx8kov030vq5dj2rzlirc7w5v2m4vhfjhbc0sj4s
2021-01-26 17:27:31.887 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 66 key: dankal-hello-2371090 value: wk1yd6y9l25jwdoo9w5jzs45lgrxuqi5iwny42zp3nxr5k2zo7fyy0r3dwgakkm1
2021-01-26 17:27:32.189 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 67 key: dankal-hello-39606219 value: r5589kwkn6pgjf57skwpkcti0jl4chjqriezztrvsf1o9ox68vrt9lrbpnjz0ujv
2021-01-26 17:27:32.507 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 68 key: dankal-hello-4069751 value: pwshr0je0kh12roekyx1y9pr6iblmbu21o42j2vm8rgnx1wny6prggf521qxw97e
2021-01-26 17:27:32.810 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 67 key: dankal-hello-7310867 value: wjojrlglz2sl6z84ppjwi01q75y1hpn1gp5njjgb8j2tei9yuy9s0l2gmxhcwphq
2021-01-26 17:27:33.111 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 55 key: dankal-hello-37528001 value: e5xpwzh3unx1jyzgmjc2ylc9a8yaikjf8pecxyoiq1om3sg3sc1i4rp9ctv18egg
2021-01-26 17:27:33.414 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 56 key: dankal-hello-12831786 value: ghlqem6xgakgppg0hejl1ihjxw21k3s4cza8qxnzjza2d59hukd6gcdkto4qdc98
2021-01-26 17:27:33.716 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 69 key: dankal-hello-38485802 value: 6oukveq7anzmml598ugcy9kk8o9osn96ifuqdbp9evro1fnph7l9v7q1gd7tgwsx
2021-01-26 17:27:34.028 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 57 key: dankal-hello-3938806 value: 0mvr3cjcmvpoxaby2nlkh4z848rjlxnbux1rronc0ugvtzqcthsk81pbcx9gwvso
2021-01-26 17:27:34.327 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 70 key: dankal-hello-21853655 value: orldkg9x75enlaqq33fv68rr4oyvygj7jhnhpp9l12e65rcurnj38suyv944yd3z
2021-01-26 17:27:34.631 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 58 key: dankal-hello-22225880 value: ftyke71dp33rxdtygpy0hdbxxm1znulpo5ajhs9qatavmomompgt0j0ehwm17n6v
2021-01-26 17:27:34.935 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 68 key: dankal-hello-16767579 value: 00akrj1j3isxgo9cb9crb1a3pbucicku9y1azla016yyi93k5ph4fnelw3wmarc9
2021-01-26 17:27:35.236 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 59 key: dankal-hello-4215465 value: t6659m68ef35c7ndc21kwrn0mrhwxdj6nb3u5pljf25tx9xnz3u5irnm5fu46nvl
2021-01-26 17:27:35.549 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 71 key: dankal-hello-38749519 value: 26ol8mn9o5nu9zpg960uwacwsnvpuwk1vm2k05q8nekosz346gh0z8gfht4b7qv3
2021-01-26 17:27:35.851 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 69 key: dankal-hello-4386603 value: uzemq22t5k3fi8xsalidgtjirsqkwaq6ip0im2l89nj3wughhjl5og006vvr6v13
2021-01-26 17:27:36.153 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 70 key: dankal-hello-22422213 value: awgbqj8drpipxkbpfrmqdzlc8wx2sc99xclg0ozzp29kc5hwcfb61xubm96c402o
2021-01-26 17:27:36.458 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 72 key: dankal-hello-6896645 value: p04nixwvmjfmoajh2t8zlpyygni1b8azil8965htz358alfym6amoatelqkx58yw
2021-01-26 17:27:36.763 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 73 key: dankal-hello-46451384 value: 9njsja7jh10qkh04l14f2um0rg6jl0aywm43pdg8566p9tm0tvhcfgejhowjnwnc
2021-01-26 17:27:37.082 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 60 key: dankal-hello-6395758 value: 9mis3yli47zeiw28ubgjnh6gav39hb5lqqzs0lkhu9wejyunntclrfnrzzyfw8pr
2021-01-26 17:27:37.375 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 74 key: dankal-hello-43915335 value: fzmwagg90kqn7bh9620xux0roktoeur48vmf2m9ux7w4h7jzhk7um509yjkxgjue
2021-01-26 17:27:37.679 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 61 key: dankal-hello-23526187 value: s0m84qbxyxrtqh9ll0ivtzy3bqihpaeypvptfle8kfpnz12hzr3k2pdno7uf7ajo
2021-01-26 17:27:37.982 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 62 key: dankal-hello-5644954 value: jmalxvso0j0riowc6i9ke6wgbxyozo0xmau0jkwjjccuqyia1ujrxi2n38suv1tp
2021-01-26 17:27:38.293 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 75 key: dankal-hello-21928914 value: w4jxzpaga1x38fwdtmi3qxcjqqnabvi2lcwrugiq9xgwv0dncdhyz0gt0j8mtk82
2021-01-26 17:27:38.592 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 71 key: dankal-hello-4680866 value: wdfs1361qi0p5sb8v4qovep4ng78xxc8ajpec82y9n2uqhhk3hhc45p40knjyyox
2021-01-26 17:27:38.895 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 72 key: dankal-hello-6135831 value: l45ipxy0d50lbd5535ncux09kavpvobg87kukzn57ff4li1fiywv0j78ooh2igmd
2021-01-26 17:27:39.199 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 73 key: dankal-hello-10928105 value: ri9dldme252veya1jl5lxmras1q6wjj0xgkismqrc8cni4yxzfei083lbtcctxdw
2021-01-26 17:27:39.503 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 74 key: dankal-hello-27858174 value: tpi3nm31y4p3t0c3irinh6w0qs289xzw9fozxx42h35p64859h6i7gnd5weu45d6
2021-01-26 17:27:39.805 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 63 key: dankal-hello-27714742 value: nz56tp915eo8s7hjl9nfgvkxx4lm9gr0vs0vqhlfun7o7zc7icy28xl0rd4un18j
2021-01-26 17:27:40.108 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 64 key: dankal-hello-14541464 value: il0sc1fhcn248t8y85sobyidrmtiykb4qotks5ougm7tqm2owpvfydwtelcp0z9p
2021-01-26 17:27:40.410 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 75 key: dankal-hello-48309562 value: 99b71tapm7drzzu3q0iy80h13n56w5ijtnttxafo48f78y0vh8iqtk0xnh11ksko
2021-01-26 17:27:40.714 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 65 key: dankal-hello-2511163 value: vtoo9s4uveiqrtrj5xky43m2o7zi79xtyo9dacyn9x9mxu9ixgk73bygmfd4pipv
2021-01-26 17:27:41.015 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 66 key: dankal-hello-16656702 value: 8p4pzjkemoe7mofteeqt0k6vw9sfk208mdv173tgkxev841xe66rz6kexdhy8hax
2021-01-26 17:27:41.322 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 76 key: dankal-hello-24607226 value: 19kkdjobqcanmtlsy9zwxatd6ao3u04n9d9gfbvg7efda8s8y9wbgv3j5c3vdh2k
2021-01-26 17:27:41.626 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 77 key: dankal-hello-49470592 value: 5atloavny6x0qhuto3ep7czq13qpaortke9eucehba9iakn94ghzl2t6a499c4ei
2021-01-26 17:27:41.938 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 76 key: dankal-hello-48052477 value: o7z9hfmvdemmdt2hkhcbz993cmyhj8mv3630ikrbaartizxs0btdjbv5opy02sz9
2021-01-26 17:27:42.239 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 77 key: dankal-hello-1041188 value: 2pddtdzcd3m28sl57fux1ikltj3dp1ek2ypwa0wywzhzejautu0gbvv6v3t5n59x
2021-01-26 17:27:42.542 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 67 key: dankal-hello-32741368 value: qye7yfpzsr2r34e55sj5c9zr7u2q48qjoa3adpt0esgsn7ilh57nc2hflvzsmyf9
2021-01-26 17:27:42.847 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 68 key: dankal-hello-33609198 value: b9ivprfyvwi2o3a1hvzrn2pu5urqhpg5akwz1iuvkjgo463yze0uie7g2ulolkd8
2021-01-26 17:27:43.152 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 69 key: dankal-hello-30882129 value: nz9ah3lfn4271s2iime8o2hp92bkald9uovgkii09823i2sxmslysvpv1rohy4be
2021-01-26 17:27:43.455 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 78 key: dankal-hello-30849687 value: e3ejf6x3su2eqpat5a5n9j9jmhntojtzptsakut68iz9eoz7ifncgzg95zfohnyl
2021-01-26 17:27:43.758 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 79 key: dankal-hello-23211632 value: ehbgqikfo29si8gbppqawav0ilpwy26spcwj7x0qp1vxcc9hi96kah0r73rp2x55
2021-01-26 17:27:44.060 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 78 key: dankal-hello-33651255 value: cjm88f7v1w6fkgf00eb1pdca1i1p3m6rk2hfxhgnb5k99ex5pbbqc512ytp69j8u
2021-01-26 17:27:44.364 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 70 key: dankal-hello-46369016 value: 6qqmixa7juel9i7nqxxed4r751kcitdqpsb64ixk3ilz2i1kefjxb13iohvno531
2021-01-26 17:27:44.669 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 80 key: dankal-hello-2763316 value: 5tl6tqo0tppihl2odc25vwknrrgyhcl7w5w7panmqff4d7olr9984s62pw5z2tij
2021-01-26 17:27:44.970 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 71 key: dankal-hello-946368 value: gnffzds6ze3qaw09d4uf3035weqq5hcwh6htmjsm4eyoi7mgy6ro0248bh237wg3
2021-01-26 17:27:45.290 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 79 key: dankal-hello-35726972 value: r87ugr72a1l8grcx6wz1jw1d4tyavlqzlzvk6w2i2n1hh10v8ia2hegvxm59ruqo
2021-01-26 17:27:45.585 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 81 key: dankal-hello-16164029 value: mxf53yndo30naj5pc0h0unpzu9olcs02u0pjlq1abo0gb3pcu85suowp7r6l4tyg
2021-01-26 17:27:45.889 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 72 key: dankal-hello-47135584 value: jkxipfsosrktxgahdbvg1gqagzxo58knvjrlbvtfdqgnwhys41mcvt8ff26i8bs3
2021-01-26 17:27:46.191 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 80 key: dankal-hello-35091491 value: cvijckhu4qc56gy99bk5qexfewc4pxlwrj0hk460syobfpysah88ombf0txfvr69
2021-01-26 17:27:46.499 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 81 key: dankal-hello-461387 value: 3pxnbjom6ibye5bvopeekir08hovsyk7jq8u8m3qfqgbpq75hwbazqu5r70npqhw
2021-01-26 17:27:46.814 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 73 key: dankal-hello-20264338 value: 4ie99ygkyr1wn6k1iulzncrbvd5py6scr9j70ger9frxa2ims8rhgy3ftfso4f5m
2021-01-26 17:27:47.111 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 74 key: dankal-hello-20031755 value: 0soy1bkvu9zudsvygopmelgeix0hmg7ejprc0gs4xlmp0f755wpbeg5o1hui52sq
2021-01-26 17:27:47.418 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 75 key: dankal-hello-15146167 value: htr8qlmrbes020i68fms8ro68ptiakyi0au52k6j6o43mllv1xxbm1ngd8rp80rs
dankal-test-kafka-rebalance-13:onPartitionsRevoked 分区: [dankal-test-hello-kafka-2]
dankal-test-kafka-rebalance-13 即将开始参与分区再均衡,即将提交工作偏移量.偏移量信息为: {dankal-test-hello-kafka-2=OffsetAndMetadata{offset=82, leaderEpoch=null, metadata='no metadata'}}
偏移量分区信息:{dankal-test-hello-kafka-2=82, dankal-test-hello-kafka-0=76, dankal-test-hello-kafka-1=82}
dankal-test-kafka-rebalance-12:onPartitionsRevoked 分区: [dankal-test-hello-kafka-0, dankal-test-hello-kafka-1]
dankal-test-kafka-rebalance-12 即将开始参与分区再均衡,即将提交工作偏移量.偏移量信息为: {dankal-test-hello-kafka-1=OffsetAndMetadata{offset=82, leaderEpoch=null, metadata='no metadata'}, dankal-test-hello-kafka-0=OffsetAndMetadata{offset=76, leaderEpoch=null, metadata='no metadata'}}
偏移量分区信息:{dankal-test-hello-kafka-2=82, dankal-test-hello-kafka-0=76, dankal-test-hello-kafka-1=82}
dankal-test-kafka-rebalance-16:onPartitionsAssigned 分区: [dankal-test-hello-kafka-2]
dankal-test-kafka-rebalance-16 已完成参与分区再均衡.偏移量信息为: {dankal-test-hello-kafka-2=82, dankal-test-hello-kafka-0=76, dankal-test-hello-kafka-1=82}
dankal-test-hello-kafka-2:topic
dankal-test-kafka-rebalance-13:onPartitionsAssigned 分区: [dankal-test-hello-kafka-1]
dankal-test-kafka-rebalance-13 已完成参与分区再均衡.偏移量信息为: {dankal-test-hello-kafka-2=82, dankal-test-hello-kafka-0=76, dankal-test-hello-kafka-1=82}
dankal-test-hello-kafka-1:topic
dankal-test-kafka-rebalance-12:onPartitionsAssigned 分区: [dankal-test-hello-kafka-0]
dankal-test-kafka-rebalance-12 已完成参与分区再均衡.偏移量信息为: {dankal-test-hello-kafka-2=82, dankal-test-hello-kafka-0=76, dankal-test-hello-kafka-1=82}
dankal-test-hello-kafka-0:topic
2021-01-26 17:27:47.721 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 82 key: dankal-hello-41787138 value: 703vpifcaoz39mt87g2s3afxbon64pe0hc74pkw6ynh3253asbjr02doq9a0bo0y
2021-01-26 17:27:48.030 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 82 key: dankal-hello-41590252 value: umzpcfn04e329jj3k52ag15e1l74cgs1boi12mcomq01sgxnsmon9w6i5ggi0lzn
2021-01-26 17:27:48.353 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 76 key: dankal-hello-48194826 value: 8ajj1ncivocfapljgxnxn28vtjfe85lstqgumijm5gssxuvu19ese18im9u7bh76
2021-01-26 17:27:48.641 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 77 key: dankal-hello-17663860 value: 86v2ykxt725yf4soxzj76ovu8hdamo9bljbmrawlsy5nb838lwr9a3442x2oh703
2021-01-26 17:27:48.947 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 78 key: dankal-hello-12871667 value: cqtpxsbqigvd4gafrq520g4eocr2fbp9n9egxehp2hk947wo2a0d00jmzmiqg3hv
2021-01-26 17:27:49.249 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 83 key: dankal-hello-39070616 value: 1c9m3rhfpom9k1w9ahaa0sxj32ci1cv32304hh1efgdxps9fe2k5hom2sdgfj59f
2021-01-26 17:27:49.564 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 79 key: dankal-hello-9854944 value: 5fmul7iucsi3txzuu5wq1nukauvzq5iqak6ftsnbren7lsspikrhex13tmy7bfhz
2021-01-26 17:27:49.868 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 83 key: dankal-hello-43174678 value: p37q3g67eufcoicem5qx6ny0yq9tp0y9t7m4xgzhuxtunkgcz87najkyvy1u93rj
2021-01-26 17:27:50.173 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 80 key: dankal-hello-20321737 value: 6kj8053i1svvvvkqz09xxh3gxfxx2tsl6erogrm5m2q23v6dzeh6dp6rujn9gd5q
2021-01-26 17:27:50.472 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 84 key: dankal-hello-12693177 value: 7m5y5msdmtck83n6z3qevf3n5du75h9i0wql5rbj19gbjpy227o5z8txcz57vxoj
2021-01-26 17:27:50.775 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 81 key: dankal-hello-21572357 value: 7dy85y7qz5za61a9gt47rc05l09cgs6r8e8l99l4cgr67ege6ztsvd4ovsvj5ajh
2021-01-26 17:27:51.093 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 85 key: dankal-hello-18496389 value: qy852vmf9716yzp90oan0jrn5lifwncn13414x5p1fr15me50agkbgmfwud2fgeb
2021-01-26 17:27:51.384 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 82 key: dankal-hello-7089942 value: 5la92bhnd9ooptvrxwd53hxrq1a3gcdqr5vgg9ell1le7z9cia0sc1kq4cd7a056
2021-01-26 17:27:51.693 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 86 key: dankal-hello-9243137 value: lhw8j4cu3zxvei83bqp9jerbhoelswkg7yng59niltylha52pklta2aw0hdjzmiw
dankal-test-kafka-rebalance-16 完成临时工作即将推出工作群组. offsetAndMetadataMap: {dankal-test-hello-kafka-2=OffsetAndMetadata{offset=87, leaderEpoch=null, metadata='no metadata'}}
dankal-test-kafka-rebalance-16:onPartitionsRevoked 分区: [dankal-test-hello-kafka-2]
dankal-test-kafka-rebalance-16 即将开始参与分区再均衡,即将提交工作偏移量.偏移量信息为: {dankal-test-hello-kafka-2=OffsetAndMetadata{offset=87, leaderEpoch=null, metadata='no metadata'}}
偏移量分区信息:{dankal-test-hello-kafka-2=87, dankal-test-hello-kafka-0=83, dankal-test-hello-kafka-1=82}
2021-01-26 17:27:51.993 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 84 key: dankal-hello-48571380 value: zx3vocx0pdmo9a02u81mcic3uwujwy7kh75s3qapybx9sqejkew2mtqkcf33gtrc
2021-01-26 17:27:52.296 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 83 key: dankal-hello-27402056 value: eruq4f8j989e4h0avk59liz95bcdjwdo79t55homp3k1nby6hcdoqysjoxvjmpkn
2021-01-26 17:27:52.602 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 84 key: dankal-hello-29285752 value: pi2cb78x8xyunivoqvmgz04g7u8ki1t187j018zwvouvdj5ya4gajkr6z9lu3sw0
2021-01-26 17:27:52.903 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 85 key: dankal-hello-41601753 value: zt305c62c4jat03g4afwyfailtdn6legkuyvbwipee6njahtztow6gl8qm1as4zb
2021-01-26 17:27:53.205 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 85 key: dankal-hello-49439937 value: 41wrycztk34yy7kjx54ry3g95po5b5ey4o2w8fgcu1atzjorqbuepfk3jl06ksj6
2021-01-26 17:27:53.507 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 86 key: dankal-hello-8362552 value: ejiki4e4l7q9ybssmsrw8t8qf4mzeaak71xjjmdc5t1o8u7npmb9ug7gswbees2e
dankal-test-kafka-rebalance-12:onPartitionsRevoked 分区: [dankal-test-hello-kafka-0]
dankal-test-kafka-rebalance-12 即将开始参与分区再均衡,即将提交工作偏移量.偏移量信息为: {dankal-test-hello-kafka-1=OffsetAndMetadata{offset=82, leaderEpoch=null, metadata='no metadata'}, dankal-test-hello-kafka-0=OffsetAndMetadata{offset=87, leaderEpoch=null, metadata='no metadata'}}
偏移量分区信息:{dankal-test-hello-kafka-2=82, dankal-test-hello-kafka-0=87, dankal-test-hello-kafka-1=82}
dankal-test-kafka-rebalance-13:onPartitionsRevoked 分区: [dankal-test-hello-kafka-1]
dankal-test-kafka-rebalance-13 即将开始参与分区再均衡,即将提交工作偏移量.偏移量信息为: {dankal-test-hello-kafka-1=OffsetAndMetadata{offset=86, leaderEpoch=null, metadata='no metadata'}, dankal-test-hello-kafka-2=OffsetAndMetadata{offset=82, leaderEpoch=null, metadata='no metadata'}}
偏移量分区信息:{dankal-test-hello-kafka-2=82, dankal-test-hello-kafka-0=87, dankal-test-hello-kafka-1=82}
dankal-test-kafka-rebalance-13:onPartitionsAssigned 分区: [dankal-test-hello-kafka-2]
dankal-test-kafka-rebalance-13 已完成参与分区再均衡.偏移量信息为: {dankal-test-hello-kafka-2=82, dankal-test-hello-kafka-0=87, dankal-test-hello-kafka-1=86}
dankal-test-hello-kafka-2:topic
dankal-test-kafka-rebalance-12:onPartitionsAssigned 分区: [dankal-test-hello-kafka-0, dankal-test-hello-kafka-1]
dankal-test-kafka-rebalance-12 已完成参与分区再均衡.偏移量信息为: {dankal-test-hello-kafka-2=82, dankal-test-hello-kafka-0=87, dankal-test-hello-kafka-1=86}
dankal-test-hello-kafka-0:topic
dankal-test-hello-kafka-1:topic
2021-01-26 17:27:53.809 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 87 key: dankal-hello-48575984 value: x6qsmqqhandqmg78yl6cbr7419q1b1p354fx4io922mxpehq7ya3gww1f0ycue90
2021-01-26 17:27:53.919 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 82 key: dankal-hello-41787138 value: 703vpifcaoz39mt87g2s3afxbon64pe0hc74pkw6ynh3253asbjr02doq9a0bo0y
2021-01-26 17:27:53.920 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 83 key: dankal-hello-43174678 value: p37q3g67eufcoicem5qx6ny0yq9tp0y9t7m4xgzhuxtunkgcz87najkyvy1u93rj
2021-01-26 17:27:53.921 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 84 key: dankal-hello-12693177 value: 7m5y5msdmtck83n6z3qevf3n5du75h9i0wql5rbj19gbjpy227o5z8txcz57vxoj
2021-01-26 17:27:53.921 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 85 key: dankal-hello-18496389 value: qy852vmf9716yzp90oan0jrn5lifwncn13414x5p1fr15me50agkbgmfwud2fgeb
2021-01-26 17:27:53.921 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 86 key: dankal-hello-9243137 value: lhw8j4cu3zxvei83bqp9jerbhoelswkg7yng59niltylha52pklta2aw0hdjzmiw
2021-01-26 17:27:54.113 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 88 key: dankal-hello-29968446 value: mlyzwwca7aueyaxfkq54genr8ya1b7xfs2ks3vhw0e6rwsv4e4px11ow6xjgrm0q
2021-01-26 17:27:54.416 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 89 key: dankal-hello-10738593 value: hc0bwcdckvvklkedqjx47aj4w0adex0n9kzii8q83uva9yfw7x061fzbvgtmdbbw
2021-01-26 17:27:54.719 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 87 key: dankal-hello-35903890 value: 4gxak6uixilfupykupmsq40a1ii57czjfxyzba4n8j7rfly79wtpnrb0pylymsof
2021-01-26 17:27:55.029 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 88 key: dankal-hello-12792275 value: al9ocilqm1vfk9nb03wwi1qjw1ulwys6m2fqfqf1w8vrheuiahs059uklz2t9084
2021-01-26 17:27:55.341 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 89 key: dankal-hello-12198295 value: m56iompqzjrbp8gky7jsv13biar05kx2pl2t04jjagkvj6snv7gb8gyw1zndgsgr
2021-01-26 17:27:55.644 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 90 key: dankal-hello-33111067 value: g0lazet23e72fqx97w0i3gxo668cinx1bp9lqc4my29rqcythw40lshds763b18y
2021-01-26 17:27:55.949 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 91 key: dankal-hello-47755174 value: er9xfb4jmnm8ajbhad0tb60uuxl642zn3q5l7hh5poz34a8nnxr2da1kgr0glwok
2021-01-26 17:27:56.249 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 90 key: dankal-hello-44701601 value: 90mjo127oad4yyav9swzcp65p4vorfrg4472lmrxw5zcwyw6x9qw1xnklxv4dbr3
2021-01-26 17:27:56.556 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 91 key: dankal-hello-31014648 value: 5lemup27nwa5bgn9z7jjc6oeq3dqrt5buygf18ocyjhck39gl6gnmmvc3rdwt1ri
2021-01-26 17:27:56.867 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 92 key: dankal-hello-49907334 value: 3w9cpp5foqau6klc10cxicrxr8uanoe4vr8c2s8eiihwuzwspk0rlqll328km6zm
2021-01-26 17:27:57.174 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 93 key: dankal-hello-18344663 value: ifyxdg7e03974q33ag3ov35iifarbr1zxr30xy1h97zm2p7d7mu2jjtksk0s61oa
2021-01-26 17:27:57.475 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 86 key: dankal-hello-45532320 value: uz5hmqo3xkpv7kodxxq1zqusje6lsvkcvz9k92k4jgbvhjrle7s6w9snq01jn0a3
2021-01-26 17:27:57.777 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 87 key: dankal-hello-8931051 value: 4q9hum1ronbs4ie5sxelliqxhy9zck5irfgrol07jb6enxz1g2xqp24rimgvhmka
2021-01-26 17:27:58.089 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 94 key: dankal-hello-17945697 value: 6v008o89b2wtpp4abh2x00r5fuehakwfeug3zgcfz3asmwcun2yny2etzi9nwdsp
2021-01-26 17:27:58.393 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 95 key: dankal-hello-22353397 value: ow83ubu7x9mrk1ekahoipzw96sihu9c8nhds9t2ttmgolb35uvvrqnpxzqwxtfvl
2021-01-26 17:27:58.695 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 92 key: dankal-hello-39321977 value: xok2wekq71rkynyogqu6zqjz314bwc9lrqhn7lj0r66snptaozvmwm8ifhrxxnym
2021-01-26 17:27:58.998 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 96 key: dankal-hello-33920135 value: oa3vw7ngt2gwmzuzu0h14v1csnmll54esdb6nviibdtttrtfcqg32p83w0rjc640
2021-01-26 17:27:59.302 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 97 key: dankal-hello-44803813 value: s2f8hpwxuwmx4r17h8g7rv5cqgk2hjkej7234gwubefucj0tquounnv8tcre45mq
2021-01-26 17:27:59.611 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 98 key: dankal-hello-30071441 value: 0bqeq57p69jt0g201tsrozpdoid6hjuebqnrmeq20udrmh5o9bo2c7lik41db6id
2021-01-26 17:27:59.916 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 93 key: dankal-hello-9320574 value: 2gep63hjaza28xoh5w3vqrbk6xps8skgmke3jnsj73mzf0nvqfbuw8hk3acbumes
2021-01-26 17:28:00.216 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 99 key: dankal-hello-19364019 value: fydw41v8rfiebjtk29pjmy0qai2jn06k8cb0r4l956u0l1a8p7bsdbzxhehvl3fb
2021-01-26 17:28:00.520 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 94 key: dankal-hello-36441853 value: 0c9mnnimwqvqlije3ottmr2lz9cqp7tq9ew4fhv4gi1b1vgb98f6d7f4xh9fpw00
2021-01-26 17:28:00.823 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 88 key: dankal-hello-26399814 value: xiuvw7m06z8r1o3vm3qr0x8te192mgqekmfoxj8myfgdvm7spyf82kk2f4l0mctr
2021-01-26 17:28:01.126 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 95 key: dankal-hello-8105741 value: s0snah3r6vue7qj9kqx2ol1ruvncxba1n9e35qnvh3q3vxhta95maq7l08wtc9wm
2021-01-26 17:28:01.430 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 96 key: dankal-hello-7551452 value: pusofo0pd0pl9pwyfkpqe5uqxeujfxpp4dbe3qpqiv3l5osvlvb76bkbemjhlrf5
2021-01-26 17:28:01.732 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 100 key: dankal-hello-46901512 value: 1dl9j4rr747ixb5osmmye98yqzwadl2dblu32323wdmvvqtk193687ije3j4pw4b
2021-01-26 17:28:02.034 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 101 key: dankal-hello-20523246 value: v2n4tam3jn9jt77106blew2sm0g4y2wck7ejhzdgo6sd75asfh1wrgf0qono6fd4
2021-01-26 17:28:02.346 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 102 key: dankal-hello-40438202 value: edy0r3z6t1f0qg7rvlr4cj2h9wdvqnv5f6t3ly8sg97hwhtywogmfqvjdfvd48ro
2021-01-26 17:28:02.642 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 97 key: dankal-hello-44040949 value: p7926stko8xw9iuml69dnrzz5nduuv72n0u9rz5umzmetsb6m0tcs0nqkrsrb5bd
2021-01-26 17:28:02.946 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 89 key: dankal-hello-25482334 value: dalb0id7sd3vbwm7gicinr1cdh55jvgkfs0lu2zcxnkahagophgpf9hmhlq51kme
2021-01-26 17:28:03.248 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 98 key: dankal-hello-2827538 value: ymabq700vef74ubqr6po1edgg645qd0wr0jhu5d6koid1zr94p8qo259li3mnfbk
2021-01-26 17:28:03.551 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 90 key: dankal-hello-238872 value: 21en2tc5bp009trfw5jjwaj49vx3omxn3t6jny7oooqxp1rdytbv8wrqhz27kzrm
2021-01-26 17:28:03.860 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 99 key: dankal-hello-23027653 value: g1emw5glgjrxy7qhze2mdggksawqbkvatfu1n7p2cj062j1ge38tm9yeqf6gcvkl
2021-01-26 17:28:04.159 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 100 key: dankal-hello-15169474 value: o8ae9y6xsm5wx7c4zn8ivjpt6v9l7hnu0nktzbbkzp1xoiwu0p2d1yjxgf34zwzu
2021-01-26 17:28:04.462 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 101 key: dankal-hello-23081836 value: h5zjo82pb7lbmelt0gd5eqy5dmttd16j77cc1fzqj8xiiruemq2ce9hbwahfhbuh
2021-01-26 17:28:04.773 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 91 key: dankal-hello-41541751 value: udfxijzitf1fnv5p4giq4wij81rtg2ajrfgaei9ceew40ppx0pw2og6fr2jy52vm
2021-01-26 17:28:05.068 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 92 key: dankal-hello-36257890 value: 7ixxssxgkdslemcyatli4axoixsxi1kpc0vxwpvh5os9sd8znvevwsaommvv3suv
2021-01-26 17:28:05.370 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 102 key: dankal-hello-45969871 value: 02w3u53mwn5cwsyltw9i6noomlzcamvvcpe9b6021y9wonvk3q2v2cckr6j1ig4j
2021-01-26 17:28:05.676 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 93 key: dankal-hello-5470370 value: 128gna1jhjwn8ym3jgexegvmgo28gild7qm68g3jobx8xvtcwgge52daztew4kds
2021-01-26 17:28:05.977 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 103 key: dankal-hello-6226126 value: yhw2ubrpsbmlyxqr0pgb40ynrrwcwit0zt7cdrjqfsd8mlkibtqn9jo563jrf7nf
2021-01-26 17:28:06.292 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 103 key: dankal-hello-21422986 value: i215m4q9g61ttk5tbxpr8qciijzpd07jvf95ei6t444gjyy7d52dxpw12rgpw9ih
2021-01-26 17:28:06.590 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 104 key: dankal-hello-26681992 value: y0zcibv1ewtg9blikgl8rj184gqr2q6rdcih7ertiojezjtktb83zzhufs55y4vo
2021-01-26 17:28:06.892 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 104 key: dankal-hello-12122125 value: 2isb5ta5egfqne0z9hfx1436w1yi0j93w7pkn4y10z8zibjwymqcnbijd49mggeb
2021-01-26 17:28:07.196 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 105 key: dankal-hello-26360953 value: a40zzmzzwmsg0zkdn3e9kn8cyf3pjo4f7ys3g5qost6620lsn383mpn3gllos0t4
2021-01-26 17:28:07.498 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 94 key: dankal-hello-1686453 value: spa7y2akonpe1we74we5blpmnyln93usjeby5us8ow1d4tjuuc9xw0xs6nr8k8d3
2021-01-26 17:28:07.809 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 105 key: dankal-hello-5861997 value: vx76jvpjfgj1fco44r046l0p1x80j4p1895cvjydqu8focvl1omjtmvbuzl52bz6
2021-01-26 17:28:08.110 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 106 key: dankal-hello-14918959 value: 3gm14hcjmuaw1kbkvtvq6ieaqo27bmi65fyeoy6kot3iahxlbadynmvsb4vlbeen
2021-01-26 17:28:08.416 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 106 key: dankal-hello-21158905 value: po31g6aa8kp7r5vawskz1kdngzveb3innbq1erxo71e6i0h3cwqiwv2n5o51kb9a
2021-01-26 17:28:08.717 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 107 key: dankal-hello-40759875 value: 94094ads3srhml63pahccytjnio45s6oimpi7uf2ajfyhfpk5lv2gwax7gyn09t0
2021-01-26 17:28:09.031 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 95 key: dankal-hello-46224283 value: cd272f54mid3l727apjo726z4e0rsd79ec1bgp3tovw9houdrd3affjt75dbkl7x
2021-01-26 17:28:09.328 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 107 key: dankal-hello-24284154 value: vhq5q7zyvp82ukhio6vefgh3syotgf0ayjdtx0c7o4csxd3ca8eor3q2zbgbrgpm
2021-01-26 17:28:09.630 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 96 key: dankal-hello-4303732 value: pwucwil1s3zs613swxfri1lue02y84sfet531mhj2xkeib0s3bgzh0347n4s32jw
2021-01-26 17:28:09.934 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 97 key: dankal-hello-26773258 value: wv03vg9fwkie68ymh4b39fgmn9upkkipra0tn1d1cyi5j3s8hy079ib57vxaz318
</pre>
</body>
</html>
\ No newline at end of file
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>test-kafka-rebalance-log</title>
</head>
<body>
<pre>
SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
SLF4J: Defaulting to no-operation (NOP) logger implementation
SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
dankal-test-kafka-rebalance-13:onPartitionsAssigned 分区: [dankal-test-hello-kafka-2]
dankal-test-kafka-rebalance-12:onPartitionsAssigned 分区: [dankal-test-hello-kafka-0, dankal-test-hello-kafka-1]
dankal-test-kafka-rebalance-13 已完成参与分区再均衡.偏移量信息为: {}
dankal-test-hello-kafka-2:topic
dankal-test-kafka-rebalance-12 已完成参与分区再均衡.偏移量信息为: {}
dankal-test-hello-kafka-0:topic
dankal-test-hello-kafka-1:topic
before: resetConsumerOffsetAndMetadataMap {}
after: resetConsumerOffsetAndMetadataMap {}
before: resetConsumerOffsetAndMetadataMap {}
after: resetConsumerOffsetAndMetadataMap {}
2021-01-26 18:31:47.545 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 273 key: dankal-hello-41433967 value: zxq7rxs4jfsym60u5zs3phfl4mz3h6mj8ds5q0ezmcs6r6wb6c4jq4gdfj5ws7mj
2021-01-26 18:31:47.831 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 251 key: dankal-hello-46454081 value: gw6pctwvdrlu9yeajjbnrbfkblxevn8ie1jpbt3qtl1u2la45rc8hu7sf9i1yhgl
2021-01-26 18:31:48.134 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 252 key: dankal-hello-32230619 value: ms43j8lfsqf74lw8ea4m2ktrpmuvovrsi624ez54voqf2rsm9pc0r19pdc76hlvr
2021-01-26 18:31:48.446 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 267 key: dankal-hello-1435363 value: hjsilhw121h7tjjydknc1zynpxnplixl93yszoez239mj8rmxecdfmno2mvxkfyk
2021-01-26 18:31:48.743 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 268 key: dankal-hello-9826196 value: pxec4km0kgwhwxlpd5yu6dcx16ywpyf027ib6j3ylmjyutc4a5qa5ldqv8ec7q4j
2021-01-26 18:31:49.052 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 269 key: dankal-hello-49703105 value: f2z7nsjyfguq6iehbzbluiqzvku12g4hwwutcj4snxjloaxq9euhg8d6kbrsq2r3
2021-01-26 18:31:49.363 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 270 key: dankal-hello-48902000 value: pteiz5cs0cm5ip6u675co1g43nyyoua8a4xun0n0nec894ooyh4peushshwfwxx1
2021-01-26 18:31:49.660 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 253 key: dankal-hello-32941397 value: ttn2v5m35kp8na3nug6se4k7jxpo50vixabwe6du5to9ndckryetpwb6ic41by8o
2021-01-26 18:31:49.963 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 271 key: dankal-hello-29457884 value: 44znpczafmtrcnwogdixi2spwhbtx3jdtw8ijtkfv3g8x9r7hyknf2ksrybtaz6z
2021-01-26 18:31:50.289 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 274 key: dankal-hello-31448167 value: bi987pav2jy23zxtgi0va9pm5ud9jfun18fa5axn1aphqyvwf7uvngnh4ose2csn
2021-01-26 18:31:50.605 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 275 key: dankal-hello-32377266 value: mj07tylo2fkv7u8q98udw284xgdmvu54d980v546cxz94bv0rex7ixiyosfigaqi
2021-01-26 18:31:50.912 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 254 key: dankal-hello-31768865 value: fwanhe2be54ywx0dbuvsd5kc8lo14hxoayqbyott9fqly4zr9klwh65vd8x380ot
2021-01-26 18:31:51.210 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 255 key: dankal-hello-17900974 value: crjlkyjslzb899vmref4lsggxt5l2t5rq0jaxtt13yz76tspflloja69a98mz9sx
2021-01-26 18:31:51.514 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 276 key: dankal-hello-33853620 value: mbpvax0l4fqejlhua3ko0pns0w234s8jh7yt43wlnf6lv5my7szyapep0094c1k8
2021-01-26 18:31:51.816 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 277 key: dankal-hello-29102094 value: 4ve5xcmyuod0xs9tksscerkqe3ppy517sabnalr1r0f6e7u6qbrw087w7khcwiag
2021-01-26 18:31:52.133 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 278 key: dankal-hello-38330235 value: ucrdsto4nfa2nm4b11giiwl9fk2kwxfn6ubysfwljr3cm01cnt4srj3ian8g487e
2021-01-26 18:31:52.426 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 279 key: dankal-hello-7014876 value: 2g825udqovcftvxdm15se73kayrfklqgt8hc31fi8uaoms2k6z4qzy38xccsk4q9
2021-01-26 18:31:52.729 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 272 key: dankal-hello-24340677 value: jymqdmk6u7bicbqrlz283utosl7ho34erfbc12cmyauljh2flizqcr7uqv7z3mqt
2021-01-26 18:31:53.032 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 256 key: dankal-hello-48104570 value: 0f2zq62ygsvrxabizphqddc7h91fux9otpd5bn6zj2lngwji74deu90gf3treefv
2021-01-26 18:31:53.334 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 280 key: dankal-hello-34968295 value: zlk9s0hdsxddtmveewqytrkb7nceta2w6ci5el2vi8ha9s8pypjh0do46e1d3zed
2021-01-26 18:31:53.637 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 257 key: dankal-hello-34883436 value: 9loa7ew8p0i8uqul0ybknafzfqktboka4dcf36ozyx3pmg6paxqoqsdqa4wumr63
2021-01-26 18:31:53.940 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 258 key: dankal-hello-14844520 value: yrgljmi3w8opmrv2akxfoq7abnyl4upt8xpc29wbuenmg4hyyxuiwm5va9tq03qc
2021-01-26 18:31:54.242 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 259 key: dankal-hello-45451901 value: 1cq67kaw2getcmtx6eac4w46xd941g7uc0snwt8pophsg5emer3t0taddue579k5
2021-01-26 18:31:54.544 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 281 key: dankal-hello-48344725 value: 8c2xx6eqktupopphd2zz3akqc6as3czl2jwedklo2pw8kkh4qy3yuo20prl5pghn
2021-01-26 18:31:54.850 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 273 key: dankal-hello-20381765 value: at2jz38hv549atvs5pcsimka7wxp7sj0tu9if5hd9l881e180q31vaeufyvuflce
2021-01-26 18:31:55.152 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 274 key: dankal-hello-18041733 value: 4p8ne9oezfyklu1r2y55d9lolk9ylgjd868z670wqvth526i6tym6ez5mr0nh9uh
2021-01-26 18:31:55.455 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 282 key: dankal-hello-14442602 value: k9ztpih7adnnb79kym7dpu9u0kpjc1q91ruwb37xx1mpx3rr48s51gqyndok2jty
2021-01-26 18:31:55.756 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 260 key: dankal-hello-17778947 value: r15mbmsfweu88d56bptqv69mvwed9hrsyqgmpm4ayzjkhid9r6rbmr1udqu45oo3
2021-01-26 18:31:56.062 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 275 key: dankal-hello-49060709 value: k0btlmlosfr9ga4zf9sce1cw5fr2ytdhol5jfncp9w5vco4n3knyqgl996a937b3
2021-01-26 18:31:56.363 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 276 key: dankal-hello-36505187 value: xdy5t2r185z5o7mg3p73zvj9ux6xnhldtfaj2posb5gkqtadrsn24wwmlym7mf39
2021-01-26 18:31:56.665 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 277 key: dankal-hello-39438991 value: f8hwv2pn09xj25v8maf9xr9vy8x1yt4swwe30jffrx0l61sutwezh1qlmonfl9s5
2021-01-26 18:31:56.969 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 278 key: dankal-hello-15402391 value: vovbtub0ux96qri0ccz3vevzvir5l5m11ekq84wwffgjz0s73wpdip87l2pyo68a
2021-01-26 18:31:57.270 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 279 key: dankal-hello-1651779 value: h8t3a6pk2e8m6wtdmsl81vuz86xtr8ns8jd2tpy7hyc9m0ac3nip07u0xsneh77f
2021-01-26 18:31:57.572 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 261 key: dankal-hello-36041393 value: 9p6xw8t4tp2spz8dj6mt4dtjo1z4a4vfzinu3wrf7lkn2qfrg98k4mhxqm0qdld9
2021-01-26 18:31:57.884 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 280 key: dankal-hello-25899419 value: a4qyp67m9f9ozxqx9ama6k9x84hy51ubb647mj2xgjmr6tn0wgop5epigquf2owl
2021-01-26 18:31:58.187 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 262 key: dankal-hello-6480320 value: 7fgke2s8dsuu5rbea6vl2ugymbr0ct98q5ngwrh45no8jrasgyd4qm7o09wn85xm
2021-01-26 18:31:58.489 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 281 key: dankal-hello-19126047 value: h7zxhmqhsbiayuw4kn3vqnk3jdsq2424uepzimr8sekblh056wpg88tqf123hy2q
2021-01-26 18:31:58.792 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 283 key: dankal-hello-29129119 value: tp0rg74agzrrblrth1hr9jdmex0skz5egxoh5qkec86z48wulac9y4m0r1njpmld
2021-01-26 18:31:59.095 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 263 key: dankal-hello-42115112 value: 6t7w0ycq9hxgufjqs7vv84qr9gsxfxt5maa6yv2vkmj3r9yjpcr65kgm67ra1nxs
2021-01-26 18:31:59.398 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 282 key: dankal-hello-15506815 value: khczdjp1ujidwyepudethepe5s1rzgqe589sypl3s7iy4vfpc033t0b6ni4ougeb
2021-01-26 18:31:59.702 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 283 key: dankal-hello-20439235 value: 5onhadvy7xdnqb1dts28kgub1bcgjgmqoxhntf8t5wb3jb4t50kg3ncom7g9urvj
2021-01-26 18:32:00.004 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 284 key: dankal-hello-30124378 value: xybqqdz9ds7gf0ow9s9598gs7hlg9yyuqe5nn9gfv3bspully4hyzzo6qnavl2yj
2021-01-26 18:32:00.310 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 264 key: dankal-hello-42600734 value: o654nfpduwyzum8eb7t8zm8x1yvqi2y3qzmn8cfjnlcab57qzonnsodm619wezze
2021-01-26 18:32:00.613 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 284 key: dankal-hello-41288551 value: rkl7572t1jf69db6ci25oznmusn3vsnmf8wee67qde28m2w2wewudvcc3iezd5lr
2021-01-26 18:32:00.923 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 265 key: dankal-hello-42470392 value: k7x2if7e29clz8gbh5mox4mnx94zuiztvhmktuwaeph8eab4j4hdzz471xijzlym
2021-01-26 18:32:01.223 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 285 key: dankal-hello-11220751 value: o1swju69y3tnexaqf68j83uqhmpkkjm8vgaxdfiffch7581wkctt34mz14ewaboe
2021-01-26 18:32:01.527 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 285 key: dankal-hello-14572232 value: 8sqqrkhcfryvi8maml8xcsbvvw3r3sfyglsognz81ukck9mm5y739ms8fuis578y
2021-01-26 18:32:01.830 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 266 key: dankal-hello-38298206 value: i2k2rdfy5c7o0gdb5cfr8s2ya2h5uqouh0saxb166ivznuwvpvb0jd29tgym27d9
2021-01-26 18:32:02.131 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 267 key: dankal-hello-12923106 value: tn8eztvzecb04bdcecz5ipam66kv59hq7h1s59wvxwnd8vr71cmmzf8giuvkao82
2021-01-26 18:32:02.434 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 268 key: dankal-hello-9025671 value: haakk2c514sxnqawd1m497lmk78asxbws32aahwae4v4ut5udah0npuikiovs2yd
2021-01-26 18:32:02.748 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 269 key: dankal-hello-16948879 value: fxwiyd48o13yrvpehl50oyfowzuds42kyaczdqjd9zntpvx436c6h43vg0nfabkd
2021-01-26 18:32:03.056 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 270 key: dankal-hello-47420821 value: tit41y46nw8wa0711fdlzv3vpxytgyxg3woqmq914lb5z7uyll3rio1wpzwj3iuw
2021-01-26 18:32:03.353 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 271 key: dankal-hello-36188826 value: 515da43j2t5qhl9uzj2zwr1nmu6sqs30fnq88zh8q8iderl3rl6h8aqo1jih672z
2021-01-26 18:32:03.665 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 286 key: dankal-hello-43394527 value: b0p33i38pi6yktv19u6i8ulqben324mkppqg7dovms5aciuzdsmvn5kwtv237j8x
2021-01-26 18:32:03.963 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 286 key: dankal-hello-21688114 value: scomt8f14gixfmu7jt2q6yecaoapnlh0qucxle34oao3bf1saquxjiezka6c955g
2021-01-26 18:32:04.267 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 272 key: dankal-hello-46129828 value: nk7b86yectjz7tvqwk9c4s0lw050lmrjskfowttq6mr3zw9l936ai1equ6dda3fe
2021-01-26 18:32:04.569 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 287 key: dankal-hello-25148822 value: msv2af5r1xsajl6yyyis1xhvgnn1x1vk4vq988alvurg466ttz19cpcf792fw2m6
2021-01-26 18:32:04.872 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 287 key: dankal-hello-40621766 value: vuuojdn2cvhg923u5bwudtxh4y52oepkwhzpowazny7y2rax3uxeiaolqljf9thp
2021-01-26 18:32:05.174 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 273 key: dankal-hello-9099555 value: 35rzm2fcui800wo3ya0id8ikqbzs9wsb539s2fwrhs6fz0quktmxxvc0x7t6lrc8
2021-01-26 18:32:05.477 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 288 key: dankal-hello-25022241 value: tgf4snin7howyeb0kq4ugyi7iprufjwe7lmhh4iqytzjq0bt77mzhn7irmq2q3t2
2021-01-26 18:32:05.780 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 274 key: dankal-hello-16395834 value: 3nb30rqbagiwygux4f4b3cl4ibmrlh2i5q21f1r8pus5jynfpq0drpf1za8kb16s
2021-01-26 18:32:06.090 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 289 key: dankal-hello-37552721 value: s92owy2ayde4zwgar3p3rp0c8v62sewtvlu4j2vd55txqp56a7tlcvrfn6j7mu1r
2021-01-26 18:32:06.385 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 290 key: dankal-hello-8847101 value: fhb9giauxp7nz888ub7mvlf2u3bjs2gnepbz3lpv7p4md7pvnlul7z6fr7irm6l3
2021-01-26 18:32:06.688 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 291 key: dankal-hello-9495732 value: 7h3ka7ihbzwy82yzyhnatfweh2c04z6jadn13wq0g4rjex8o0g6tbi33hcsjnfqa
2021-01-26 18:32:06.990 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 292 key: dankal-hello-22426721 value: 2wzunlt5ftgjwuxvqvdiv2y1pbee9xual2058hh7zaf3umhpju2ds2ux40sengty
2021-01-26 18:32:07.293 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 293 key: dankal-hello-46356585 value: sdopmksh8eyr7ijztmpiua1h2z4f29s8vpg94rdwf01avzo7mqgg57slz9n2qgn6
2021-01-26 18:32:07.596 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 288 key: dankal-hello-36218926 value: nc4y1b09v9019gz7d5k2q3bu3nlmjl91vuz7imzal9f18wpfuztq5my6c5f1yci2
2021-01-26 18:32:07.899 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 289 key: dankal-hello-9664777 value: 7ml59vmypxa5j3fsmyna82nh2ptvhgwv6o5ivxyiaw8ka5paaybk2huukmqv5zw6
2021-01-26 18:32:08.213 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 290 key: dankal-hello-44208937 value: 9rc2zpzmrn4mpxe7sel2ildu7twqv5a7q0q8xa3h09d6mgc9pqlc799no2ou9qhh
2021-01-26 18:32:08.511 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 294 key: dankal-hello-301504 value: sx3bfyavzi503nz84346dt295mrk6z345enrizn29x90hagowmdc2txcefg4grjf
2021-01-26 18:32:08.824 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 275 key: dankal-hello-15783208 value: 3j9n1kc4ns5o0c4pkexgaj3fehprx3uw52awai192yt9rlrk4p1xtftbyiofu4lj
2021-01-26 18:32:09.116 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 276 key: dankal-hello-41230895 value: 6ydm1gl8r7smo2xhpd1nt3osy6urfzl0mde4828gg9kf6vo7nwfxqzaya50tnrm7
2021-01-26 18:32:09.419 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 277 key: dankal-hello-8103865 value: 1ky2xia2wtosdoxuqkd1uwy1mmjmwma7v3kk2uh7l0w9ot85s1cua5jujrh06xzs
2021-01-26 18:32:09.727 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 278 key: dankal-hello-26526077 value: y4fnqzc1fi8ywf9qa6gmyq0dm9a5yn34na8lhrhtz541lqx2lmfx5cyzsos690nz
2021-01-26 18:32:10.028 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 291 key: dankal-hello-20446053 value: ik2q9u56xhf3jyz48niww65ylzz5xsyazkgt8tzo02e2aftxjw9ve6nzmibf70ih
2021-01-26 18:32:10.331 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 279 key: dankal-hello-34053561 value: zklmaohdfl0d8mgqwreqgkbks2bxwo842mf5k6iccjdvfgz957npj41d4kh7gcz7
2021-01-26 18:32:10.634 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 295 key: dankal-hello-20107654 value: tycosimurljkd8gi94825s69k3s8ygai9zvp41ty21r0sat362sudpxv6i6o8z51
2021-01-26 18:32:10.939 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 280 key: dankal-hello-49127713 value: xieutm6k7bkrfijhry1v4z3tk1q75xevaiup08hvepvw9mcnpf3zn6q6uzyebm6d
2021-01-26 18:32:11.239 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 281 key: dankal-hello-6153923 value: tz99cnxqgbkveyuslxz6yx82aonmnjignuhjt162udi3mqb0moernh15i1yvwtrd
2021-01-26 18:32:11.543 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 296 key: dankal-hello-8424548 value: 4bh71bdaasodc2stjd4h5g3t6bi7cux9un3gngb53zwqq55zmwl3av0at47copdl
2021-01-26 18:32:11.846 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 297 key: dankal-hello-28988954 value: qd7ad1oeu7m7rff8lab3kk4vx1etogjgbgyx6dcm9uy649cjz24dr97c74gfls4m
2021-01-26 18:32:12.149 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 292 key: dankal-hello-49571501 value: 1ilmwlznbo952338ymb44h2wl2v850uyt4oa5jncbv1faojsq282ento1iyrjr1x
2021-01-26 18:32:12.459 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 298 key: dankal-hello-27461550 value: 57u1b5dacs3saldjpksiwtwd5j35hjledlqem8w9lh9sa8ncx3owb05rmfwyl4q1
dankal-test-kafka-rebalance-13:onPartitionsRevoked 分区: [dankal-test-hello-kafka-2]
dankal-test-kafka-rebalance-13 即将开始参与分区再均衡,即将提交工作偏移量.偏移量信息为: {dankal-test-hello-kafka-2=OffsetAndMetadata{offset=293, leaderEpoch=null, metadata='no metadata'}}
偏移量分区信息:{dankal-test-hello-kafka-2=293, dankal-test-hello-kafka-0=282, dankal-test-hello-kafka-1=299}
dankal-test-kafka-rebalance-12:onPartitionsRevoked 分区: [dankal-test-hello-kafka-0, dankal-test-hello-kafka-1]
dankal-test-kafka-rebalance-12 即将开始参与分区再均衡,即将提交工作偏移量.偏移量信息为: {dankal-test-hello-kafka-0=OffsetAndMetadata{offset=282, leaderEpoch=null, metadata='no metadata'}, dankal-test-hello-kafka-1=OffsetAndMetadata{offset=299, leaderEpoch=null, metadata='no metadata'}}
偏移量分区信息:{dankal-test-hello-kafka-2=293, dankal-test-hello-kafka-0=282, dankal-test-hello-kafka-1=299}
dankal-test-kafka-rebalance-13:onPartitionsAssigned 分区: [dankal-test-hello-kafka-1]
dankal-test-kafka-rebalance-13 已完成参与分区再均衡.偏移量信息为: {dankal-test-hello-kafka-2=293, dankal-test-hello-kafka-0=282, dankal-test-hello-kafka-1=299}
dankal-test-hello-kafka-1:topic
dankal-test-kafka-rebalance-16:onPartitionsAssigned 分区: [dankal-test-hello-kafka-2]
dankal-test-kafka-rebalance-16 已完成参与分区再均衡.偏移量信息为: {dankal-test-hello-kafka-2=293, dankal-test-hello-kafka-0=282, dankal-test-hello-kafka-1=299}
dankal-test-hello-kafka-2:topic
before: resetConsumerOffsetAndMetadataMap {dankal-test-hello-kafka-2=OffsetAndMetadata{offset=293, leaderEpoch=null, metadata='no metadata'}}
before: resetConsumerOffsetAndMetadataMap {}
after: resetConsumerOffsetAndMetadataMap {}
after: resetConsumerOffsetAndMetadataMap {}
dankal-test-kafka-rebalance-12:onPartitionsAssigned 分区: [dankal-test-hello-kafka-0]
dankal-test-kafka-rebalance-12 已完成参与分区再均衡.偏移量信息为: {dankal-test-hello-kafka-2=293, dankal-test-hello-kafka-0=282, dankal-test-hello-kafka-1=299}
dankal-test-hello-kafka-0:topic
before: resetConsumerOffsetAndMetadataMap {dankal-test-hello-kafka-0=OffsetAndMetadata{offset=282, leaderEpoch=null, metadata='no metadata'}, dankal-test-hello-kafka-1=OffsetAndMetadata{offset=299, leaderEpoch=null, metadata='no metadata'}}
after: resetConsumerOffsetAndMetadataMap {dankal-test-hello-kafka-0=OffsetAndMetadata{offset=282, leaderEpoch=null, metadata='no metadata'}}
2021-01-26 18:32:12.759 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 293 key: dankal-hello-45877408 value: 2zlo21oxmzmkjxrw48ime7i6w8n5b4o3489h6z376ijjqsg01f1r2o7g54z6pyji
2021-01-26 18:32:13.059 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 282 key: dankal-hello-6279759 value: fxq9icqlpvpscbm63ix7dxpmmvasoblju9wk03ttd0y85egl8qziwcn30kt6j4bj
2021-01-26 18:32:13.368 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 294 key: dankal-hello-25751639 value: 135t94udtcieja4vf5z0j5cjp7k7v7l03xzqaqjhjfqpshc0vmy9lq89gupa6t5c
2021-01-26 18:32:13.680 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 295 key: dankal-hello-20589678 value: ynjtp0p2f3v429bk2g7sq7snexyyhwtvzejjf0b09zpn9f6n4otftehqm5csxw6s
2021-01-26 18:32:13.975 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 299 key: dankal-hello-12186325 value: vq9miiq708g340dxogzcn8bc4x2xyluwtlptd3mckxlr0gk2hzd0dy9b6vxj7g9m
2021-01-26 18:32:14.286 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 283 key: dankal-hello-29440083 value: mmzub2xkhhuos9k6pqx58q1ve2gz0wr8ftwwfeudzmcone0sbo38rwg9593gobei
2021-01-26 18:32:14.580 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 300 key: dankal-hello-5023962 value: x8giuqacytldx8fdkfd65rzzxo16plitk9pr8iar4fda8lf7nr9bewue9d43lydr
2021-01-26 18:32:14.883 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 296 key: dankal-hello-34568781 value: 71cst253m0g4m1wa05hv33iynoy6le60b0wzk0etnbek2e5qt7u7rxfz3d5ejoch
2021-01-26 18:32:15.193 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 297 key: dankal-hello-22679 value: entm955var8pefyms77wrtct2178t60qcb6s72f84dwbdfqga4l4hd0l7ckfpfml
dankal-test-kafka-rebalance-16 完成临时工作即将推出工作群组. offsetAndMetadataMap: {dankal-test-hello-kafka-2=OffsetAndMetadata{offset=298, leaderEpoch=null, metadata='no metadata'}}
dankal-test-kafka-rebalance-16:onPartitionsRevoked 分区: [dankal-test-hello-kafka-2]
dankal-test-kafka-rebalance-16 即将开始参与分区再均衡,即将提交工作偏移量.偏移量信息为: {dankal-test-hello-kafka-2=OffsetAndMetadata{offset=298, leaderEpoch=null, metadata='no metadata'}}
偏移量分区信息:{dankal-test-hello-kafka-2=298, dankal-test-hello-kafka-0=284, dankal-test-hello-kafka-1=301}
dankal-test-kafka-rebalance-13:onPartitionsRevoked 分区: [dankal-test-hello-kafka-1]
dankal-test-kafka-rebalance-13 即将开始参与分区再均衡,即将提交工作偏移量.偏移量信息为: {dankal-test-hello-kafka-1=OffsetAndMetadata{offset=301, leaderEpoch=null, metadata='no metadata'}}
偏移量分区信息:{dankal-test-hello-kafka-2=298, dankal-test-hello-kafka-0=284, dankal-test-hello-kafka-1=301}
dankal-test-kafka-rebalance-12:onPartitionsRevoked 分区: [dankal-test-hello-kafka-0]
dankal-test-kafka-rebalance-12 即将开始参与分区再均衡,即将提交工作偏移量.偏移量信息为: {dankal-test-hello-kafka-0=OffsetAndMetadata{offset=284, leaderEpoch=null, metadata='no metadata'}}
偏移量分区信息:{dankal-test-hello-kafka-2=298, dankal-test-hello-kafka-0=284, dankal-test-hello-kafka-1=301}
dankal-test-kafka-rebalance-13:onPartitionsAssigned 分区: [dankal-test-hello-kafka-2]
dankal-test-kafka-rebalance-13 已完成参与分区再均衡.偏移量信息为: {dankal-test-hello-kafka-2=298, dankal-test-hello-kafka-0=284, dankal-test-hello-kafka-1=301}
dankal-test-hello-kafka-2:topic
dankal-test-kafka-rebalance-12:onPartitionsAssigned 分区: [dankal-test-hello-kafka-0, dankal-test-hello-kafka-1]
before: resetConsumerOffsetAndMetadataMap {dankal-test-hello-kafka-1=OffsetAndMetadata{offset=301, leaderEpoch=null, metadata='no metadata'}}
after: resetConsumerOffsetAndMetadataMap {}
dankal-test-kafka-rebalance-12 已完成参与分区再均衡.偏移量信息为: {dankal-test-hello-kafka-2=298, dankal-test-hello-kafka-0=284, dankal-test-hello-kafka-1=301}
dankal-test-hello-kafka-0:topic
dankal-test-hello-kafka-1:topic
before: resetConsumerOffsetAndMetadataMap {dankal-test-hello-kafka-0=OffsetAndMetadata{offset=284, leaderEpoch=null, metadata='no metadata'}}
after: resetConsumerOffsetAndMetadataMap {dankal-test-hello-kafka-0=OffsetAndMetadata{offset=284, leaderEpoch=null, metadata='no metadata'}}
2021-01-26 18:32:15.489 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 284 key: dankal-hello-26004557 value: zxfk1tx90ck9585r51f4em6mw1eb3sibsc3rer0a5cmesdd5j3sybozrtfh1gbc3
2021-01-26 18:32:15.792 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 285 key: dankal-hello-19375637 value: 9agfxql8og95xjobfw1pvwvnxiaskp30i9p3z98mumiwax3cdyd9b7z6te5i79xi
2021-01-26 18:32:16.094 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 286 key: dankal-hello-26939322 value: jqc25q98y0yexm1xt6fxo65vmpdjpt7wrv9ntmwfzin2lmqbihl7f2j873yks4n7
2021-01-26 18:32:16.398 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 301 key: dankal-hello-42003773 value: i0syn5duddz7sc2h8nevo9glm420ykrzfsjsmd0i9xhiq4eg2jsgtweimvaarxoq
2021-01-26 18:32:16.700 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 287 key: dankal-hello-31404226 value: onzkygbxc3tq944rdcxhaej9ghwnq6fcv0m4w1raxwkeiedajcthad85s67q7r7h
2021-01-26 18:32:17.012 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 302 key: dankal-hello-41607277 value: 7svmfhm0ifgfxnkh2vdtkof1rnedab62yrawp4tco50vcy4guajbgn1088f9lwe9
2021-01-26 18:32:17.309 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 303 key: dankal-hello-1738295 value: 8puvt9o57qfh0ntpzwngh0xkqz6mg7ncpn1c0qntmqv883qwfiqxg6dc7436282g
2021-01-26 18:32:17.612 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 304 key: dankal-hello-2360959 value: x7in43tqx6yxa1vh2etclufnyn7toh8aj5mbgvxgyly2t89dm1lkkb7gpb5fsb4s
2021-01-26 18:32:17.915 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 298 key: dankal-hello-6089765 value: mz5759xrvv41897t9letz9gqr9edkaphj0rdyszngby6p2akieq3763tqo9ueq73
2021-01-26 18:32:18.225 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 288 key: dankal-hello-27437634 value: t6znl7eaxzirha7ekqac17xfqf92vcnhuwuyoa1bksa2lhougj3v2xfptyqvul8x
2021-01-26 18:32:18.524 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 299 key: dankal-hello-48459353 value: lrjfwwgx4p8rwmwleoados6j0u6rlq6ua8s85n5kk1i0mxzc123tehfsd6547z2l
2021-01-26 18:32:18.826 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 289 key: dankal-hello-45176467 value: 6699syoxpdaqp7dgb23ggncp5gymabw0odlcabaandi04mbvug2egxrbyt7eei16
2021-01-26 18:32:19.145 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 300 key: dankal-hello-30087635 value: r6gaxxwlmiu4ci3vw6iahligdud0s8jjlus3jqg77fyl8j1jn3ref78bce2hclzb
2021-01-26 18:32:19.447 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 305 key: dankal-hello-38936289 value: gvyfecqfzj030tos38ovusfvdzc5uz1gvv2ku9gjfmbxa7t3o7u9n0sck34u9g0b
2021-01-26 18:32:19.745 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 306 key: dankal-hello-42765128 value: evyox6coeq3z1g3f4s51lnlnkk62y1lwoj608tqguzi0n3qk4vku2x4cy7wea70e
2021-01-26 18:32:20.048 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 307 key: dankal-hello-1634339 value: skci2sshi5dgxsfpzu7sfm88xlp7i35e9tzy4qvmw49mjharxtxtviso56miwzuh
2021-01-26 18:32:20.349 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 301 key: dankal-hello-48219210 value: pndjsy3cwqteea05yxbk6dumhq83eglotf1199w2bnfcuohn08myb2iv86w6ycc8
2021-01-26 18:32:20.655 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 302 key: dankal-hello-24802716 value: emxfvu3l79chugm4ls9bqodt9d9zvytfotldbhwkyfpm94nz9u5hy0cpy9mhl5ub
2021-01-26 18:32:20.958 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 290 key: dankal-hello-10977507 value: lq56dfgdy0oekb6unznk466qu2j19jwa6ycjcg5y3s982fnyx61q70etud8u5d61
2021-01-26 18:32:21.261 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 291 key: dankal-hello-36996822 value: t9h9k7xiwmwo65ied996q7iev37sc7stgw1kruuj5c3mz211zd45ruqtjojun3bg
2021-01-26 18:32:21.562 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 308 key: dankal-hello-11553629 value: e42fdesc9b8abam0y31dm25flqmxi0ju2cghobfnjrhjf6d57yqfrassrn0v1ssj
2021-01-26 18:32:21.865 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 292 key: dankal-hello-12397887 value: z06bw5kgndrai91zjvab112bqmfigwpy6zk0f7m79wjj51vhqgel9ya787gzeqqc
2021-01-26 18:32:22.168 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 293 key: dankal-hello-43434560 value: z1cfgh6z07yg62vxlk1rutp3f6r3x5pypimfmczvw57vidlhxl8yzgmj5scvljor
2021-01-26 18:32:22.498 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 294 key: dankal-hello-46775755 value: koe9hbvahci3ysg505mpr7b4csejyicvqvpy5b01zoi998oqsjw5gs8ov3t9op8p
2021-01-26 18:32:22.774 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 303 key: dankal-hello-31248065 value: mrfrfdnxib3m00p9u6ddhsanpm6o8y52ceepm7zqd3q3b8xilstcdyzkm3x6uzr5
2021-01-26 18:32:23.076 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 304 key: dankal-hello-1322180 value: ro98nxbmorv0jgwg8kb0hjypgsj7nt8v84i9ppolsp5r0a06spstfn3g8z7rmnfh
2021-01-26 18:32:23.378 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 309 key: dankal-hello-42285508 value: bfsjbi3hpi0z2klssgacbs9a908sb28o1cx1uqchyloqmj8scx1v7jk945lg1rpi
2021-01-26 18:32:23.681 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 295 key: dankal-hello-10828945 value: s7djmyp6gqs55ggdn8fd8916tya3n0vtwrxvuly9103kp1a8zuakh1lsc02etsze
2021-01-26 18:32:23.990 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 296 key: dankal-hello-41698077 value: 8jeq1i9b586vshs9wc5x67in5knix7pisvqqjtnv6k0zcl86muu73077wb0l06g4
2021-01-26 18:32:24.287 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 310 key: dankal-hello-18288875 value: sysesa58frso1123wvo43aglrt9t11va353ovmbvv5igebfpytj7q0adkc769bmr
2021-01-26 18:32:24.590 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 305 key: dankal-hello-30552544 value: souvhm4y693fi70yjywagxlzz0659m8xgh563ixpuf63vc81zn5e9k4ccdnnrtwq
2021-01-26 18:32:24.892 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 311 key: dankal-hello-29303590 value: g9g7iuyp5vfr0oc4pkiqq2mhrwqp2oymuepj601e1t822726zjt7e0xitf6o6luo
2021-01-26 18:32:25.196 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 297 key: dankal-hello-4457780 value: gwr5767on4not8e51fxjlo2h02lgya0idmybdbwjt9t5hg3qn3xv1v6ww1td9zft
2021-01-26 18:32:25.497 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 298 key: dankal-hello-19964885 value: 92os8c00xv5odfbbvk2p62f3tl5b0a7tmszgg3j8splt5a55vout4f8qpk93i4v3
2021-01-26 18:32:25.800 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 312 key: dankal-hello-34276477 value: qe8aachme6ex4ww9q0hkxohz7fepuxmbj0xlw3a1xffts950zg9cfbdk0xyrh4ed
2021-01-26 18:32:26.106 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 306 key: dankal-hello-43327271 value: 2fh2x4aspw3gdbt1bctof5hrriirabp3ntdfivlbcs6n7hcgi6fynokqx9tggcop
2021-01-26 18:32:26.407 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 313 key: dankal-hello-48987149 value: n75wvbovjqg8iocp3zkrju6r8toxujhw0o2wzz1yia77jg2pl3mkigvh88qnib6e
2021-01-26 18:32:26.716 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 314 key: dankal-hello-27885018 value: ibisvdwqzmn6t6dtu7kfs7xy487ek107nq7dfpazlem4h5f3u75d140eru0ulhok
2021-01-26 18:32:27.014 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 315 key: dankal-hello-47676063 value: f7295grvjrf8ncy14wbxdnxmm1l9i7ofabkd1v0yqlhdyrh9x9ik62qweodd7mes
2021-01-26 18:32:27.317 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 1 偏移量: 316 key: dankal-hello-9625982 value: 9hnghh6kliwpz8w9s7nrxoxpisif9qy7vf87q3k5wl4yexkbdi1ist70gvnfr33r
2021-01-26 18:32:27.625 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 307 key: dankal-hello-22747184 value: 4fi389o9f610jmh08x8nib8l0psz600wtllniq5fbw35myv4g49il0h22nw8j1tf
2021-01-26 18:32:27.926 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 299 key: dankal-hello-37012774 value: xql90go1n7jsc00pxryp74fj5jtnclash37h4yqxu58awxj313qyb918iyui0u7b
2021-01-26 18:32:28.229 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 308 key: dankal-hello-29158143 value: tpjnkvtz9gt6ag55d8dfvx16yt6zy7cptsjp24t4lw3akxjo4nvlgw2ovhnyf5si
2021-01-26 18:32:28.534 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 309 key: dankal-hello-17342882 value: 8bfw1k7t3f5vh66kpeoz8m5ilfpmu1g34029f6fo7kaz4pkcm0g06r0d6506q228
2021-01-26 18:32:28.844 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 2 偏移量: 310 key: dankal-hello-25272379 value: w2m4ktcb54093vudwf858kyq8pqfxcp40cpbieh2gn5i27osktlk127jxpm30c1r
2021-01-26 18:32:29.151 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 300 key: dankal-hello-43810824 value: p0u1033fmxyjaehchdr1qwwvuqunf3g74fr70y8e30kv4h9lwk7mlzyndeux7ogj
2021-01-26 18:32:29.467 消息消息 ==> 主题: dankal-test-hello-kafka 分区: 0 偏移量: 301 key: dankal-hello-18059301 value: jwasjv5a75ohxwaosibat5sq38ejgm8dxy3szewveu1q7ofb8pndaxw5270v9w71
</pre>
</body>
</html>
\ No newline at end of file
# 蛋壳创意科技-Kafka负载再均衡示例小析
# 蛋壳创意科技-Kafka负载再均衡示例小析
> company: [蛋壳创意科技](https://www.dankal.cn/)
>
> code: []()
>
> log: [test-kafka-rebalance-log](https://res.mercymodest.com/res/test-kafka-rebalance-log.html)
## 测试用例总览
**项目: dankal-test-kafka-protogenesis**
**根目录: test**
**包名: cn.dankal.test.rebalance**
> 消息生产者: ProducerWorkerTest
>
> 消息消费者: ConsumerWorker
>
> 分区再均衡监听器: RebalanceListener
>
> 测试类: KafkaRebalanceTest
## 测试用例详情
### 前景小述
> 理论上,我们需要将分区偏移量信息保存到关系数据库,通过数据库事务控制操作原子性~然,为了简化测试用例,测试用例使用一个 `java.util.concurrent.ConcurrentHashMap`来存储
>
> ![](https://img.mercymodest.com/public/20210127105302.png)
>
> 然后 以 `TODO`的形式,提供 `Transactional` 的参考
==测试用例是对如下图示的具体实现,所以说呢,阅码第一步,读图走起~==
![](https://img.mercymodest.com/public/20210126154516.jpg)
### RebalanceListener: 自定义再均衡监听器
```java
package cn.dankal.test.rebalance;
import cn.dankal.test.common.Const;
import cn.hutool.core.collection.CollectionUtil;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* 测试 Kafka 分区再均衡: RebalanceHandler
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-26
*/
public class RebalanceListener implements ConsumerRebalanceListener {
/**
* Map<TopicPartition, OffsetAndMetadata> 当前消费者分区与偏移量信息
*/
private final Map<TopicPartition, OffsetAndMetadata> offsetAndMetadataMap;
/**
* current ConsumerWorker KafkaConsumer instance
*/
private final KafkaConsumer<String, String> kafkaConsumer;
public RebalanceListener(Map<TopicPartition, OffsetAndMetadata> offsetAndMetadataMap, KafkaConsumer<String, String> kafkaConsumer) {
this.offsetAndMetadataMap = offsetAndMetadataMap;
this.kafkaConsumer = kafkaConsumer;
}
/**
* 再执行分区再平衡之前,回调此方法
* <p>
* Collection<TopicPartition> ==> 当前 Consumer 所有拥有消费权的分区信息
* <p>
* 我们一般在此方法完成分区再平衡之前,分区偏移量的存储
*
* @param partitions Collection<TopicPartition>
*/
@Override
public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
// 分区再均衡之前
System.out.printf(
"%s:onPartitionsRevoked 分区: %s%n",
Thread.currentThread().getName(),
partitions
);
System.out.printf(
"%s 即将开始参与分区再均衡,即将提交工作偏移量.偏移量信息为: %s%n",
Thread.currentThread().getName(),
this.offsetAndMetadataMap
);
System.out.printf("偏移量分区信息:%s%n", Const.PARTITION_OFFSET_INFO_MAP);
// TODO transaction begin
try {
for (TopicPartition partition : partitions) {
//TODO 将偏移量执行事务写入
Const.PARTITION_OFFSET_INFO_MAP.put(
partition,
offsetAndMetadataMap.get(partition).offset());
}
//TODO transaction commit
//可选 将偏移量信息提交到 Kafka 服务器
//因为发生分区再均衡 分区偏移量是我们自己在维护 Const.PARTITION_OFFSET_INFO_MAP
kafkaConsumer.commitSync(offsetAndMetadataMap);
} catch (Exception e) {
// TODO transaction rollback
// TODO handler exception
e.printStackTrace();
}
}
/**
* 完成分区再平衡之后,回调当前方法
* <p>
* Collection<TopicPartition> ==> 当前 Consumer 所有拥有消费权的分区信息
* <p>
* 我们一般在此方法内,通过在方法{@link #onPartitionsRevoked(Collection)}存储的分区偏移量信息,对分区偏移量信息进行矫正
*
* @param partitions Collection<TopicPartition>
*/
@Override
public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
System.out.printf(
"%s:onPartitionsAssigned 分区: %s%n",
Thread.currentThread().getName(),
partitions
);
System.out.printf(
"%s 已完成参与分区再均衡.偏移量信息为: %s%n",
Thread.currentThread().getName(),
Const.PARTITION_OFFSET_INFO_MAP
);
//分区偏移量矫正
for (TopicPartition partition : partitions) {
System.out.printf("%s:topic %n", partition);
// 获取我们自己维护的分区偏移量信息
Long offset = Const.PARTITION_OFFSET_INFO_MAP.get(partition);
if (null == offset) {
continue;
}
// 调整分区偏移量
// 从特定偏移量开始记录分区偏移量
// 保证分区再均衡之后,数据偏移量不会错乱
kafkaConsumer.seek(partition, offset);
}
resetConsumerOffsetAndMetadataMap(partitions);
}
/**
* 重置当前 Kafka Consumer 的 <code>Map<TopicPartition, OffsetAndMetadata></code>
*
* @param partitions Collection<TopicPartition>
*/
private void resetConsumerOffsetAndMetadataMap(Collection<TopicPartition> partitions) {
if (CollectionUtil.isEmpty(partitions)) {
return;
}
List<Integer> ownPartitionIdList = partitions.stream()
.map(TopicPartition::partition)
.collect(Collectors.toList());
System.out.printf("%s before: resetConsumerOffsetAndMetadataMap %s %n", Thread.currentThread().getName(), this.offsetAndMetadataMap);
for (TopicPartition topicPartition : this.offsetAndMetadataMap.keySet()) {
if (!ownPartitionIdList.contains(topicPartition.partition())) {
// 分区再平衡时,当前消费者已经不再拥有当前 partition 的消费权
// 避免出现覆盖,故需要在当前 Consumer 的 offsetAndMetadataMap 移除 当前 partition 的信息
// 出现场景可以参阅项目目录: test-kafka-rebalance-log-error.html
this.offsetAndMetadataMap.remove(topicPartition);
}
}
System.out.printf("%s after: resetConsumerOffsetAndMetadataMap %s %n", Thread.currentThread().getName(), this.offsetAndMetadataMap);
}
}
```
### ConsumerWorker: 再均衡触发源
```java
package cn.dankal.test.rebalance;
import cn.dankal.test.common.Const;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.map.MapUtil;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
/**
* 测试 Kafka Rebalance: ConsumerWorker
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-26
*/
public class ConsumerWorker {
/**
* Map<TopicPartition, OffsetAndMetadata> 当前消费者分区与偏移量信息
*/
private final Map<TopicPartition, OffsetAndMetadata> offsetAndMetadataMap;
/**
* current ConsumerWorker KafkaConsumer instance
*/
private final KafkaConsumer<String, String> kafkaConsumer;
/**
* 是否是临时工作者: 临时工作者在消费五条消息之后将退出当前消费群组
*/
private final boolean isTempWorker;
/**
* 临时工作者的消费消息的次数: 5
*/
private final static int TEMP_WORK_COUNT = 5;
/**
* KafkaConsumer config properties
*/
private static final Properties PROPERTIES;
//TODO maybe 事务类 提供事务支持
//private final Transactional transactional;
static {
PROPERTIES = new Properties();
final String bootstrapServers = "192.168.128.3:9092,192.168.128.3:9093,192.168.128.3:9094";
// broker(Kafka服务器) 地址清单
PROPERTIES.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
// message key 反序列化器
PROPERTIES.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
// message value 反序列化器
PROPERTIES.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
PROPERTIES.put(ConsumerConfig.GROUP_ID_CONFIG, Const.Kafka.ConsumerGroupId.TEST_KAFKA_REBALANCE);
}
public ConsumerWorker(boolean isTempWorker) {
this.offsetAndMetadataMap = new ConcurrentHashMap<>();
this.isTempWorker = isTempWorker;
// 消费者线程不安全,建议一个线程一个对应一个消费者实例
this.kafkaConsumer = new KafkaConsumer<>(PROPERTIES);
// 设置 分区再均衡监听器
this.kafkaConsumer.subscribe(
CollectionUtil.newArrayList(Const.Kafka.Topic.HELLO_TOPIC),
new RebalanceListener(offsetAndMetadataMap, kafkaConsumer)
);
}
/**
* 执行 Kafka message consumer work
*/
public void doWork() {
// 执行计数 ,搭配 isTempWorker 使用
int execCount = 0;
try {
while (true) {
ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(Duration.ofMillis(500));
for (ConsumerRecord<String, String> consumerRecord : consumerRecords) {
try {
// TODO Transaction: transaction begin
// transaction.begin();
// TODO 消费消息 处理业务
long currentMessageOffset = consumerRecord.offset();
System.out.printf(
"%s 消息消息 ==> 主题: %s 分区: %s 偏移量: %s key: %s value: %s %n",
Const.FULL_DATE_TIME_FORMATTER.format(LocalDateTime.now()),
consumerRecord.topic(),
consumerRecord.partition(),
currentMessageOffset,
consumerRecord.key(),
consumerRecord.value());
// TODO 保存分区与偏移量信息
TopicPartition topicPartition = new TopicPartition(
consumerRecord.topic(),
consumerRecord.partition()
);
OffsetAndMetadata offsetAndMetadata = new OffsetAndMetadata(currentMessageOffset + 1, "no metadata");
offsetAndMetadataMap.put(topicPartition, offsetAndMetadata);
//TODO 保存偏移量与分区信息
if (MapUtil.isNotEmpty(offsetAndMetadataMap)) {
for (TopicPartition partition : offsetAndMetadataMap.keySet()) {
Const.PARTITION_OFFSET_INFO_MAP.put(
partition,
offsetAndMetadataMap.get(partition).offset()
);
}
}
// TODO Transaction: transaction commit
// transaction.commit();
kafkaConsumer.commitAsync();
//TODO 执行计数
execCount++;
// TODO just test kafka Rebalance
if (isTempWorker) {
if (execCount == TEMP_WORK_COUNT) {
// 触发分区再均衡
// 临时工作者工作完成~ 即将推出 当前消费群组
System.out.printf(
"%s 完成临时工作即将推出工作群组. offsetAndMetadataMap: %s%n",
Thread.currentThread().getName(),
offsetAndMetadataMap);
//提交自己已经完成的工作消息偏移量
kafkaConsumer.commitSync();
return;
}
}
} catch (Exception e) {
// TODO Transaction: transaction rollback
// transaction.rollback();
// TODO handler exception
e.printStackTrace();
}
}
}
} finally {
kafkaConsumer.commitSync();
kafkaConsumer.close();
}
}
}
```
### ProducerWorkerTest: Kafka 消息提供者
```java
package cn.dankal.test.rebalance;
import cn.dankal.test.common.Const;
import cn.hutool.core.util.RandomUtil;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import java.util.Objects;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
/**
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-26
*/
public class ProducerWorkerTest {
/**
* KafkaProducer
*/
private static KafkaProducer<String, String> kafkaProducer;
@BeforeAll
public static void before() {
Properties properties = new Properties();
final String bootstrapServers = "192.168.128.3:9092,192.168.128.3:9093,192.168.128.3:9094";
// broker(Kafka服务器) 地址清单
properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
// message key 序列化器
properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());
// message value 序列化器
properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getCanonicalName());
// 通过 Properties 配置 实例化 KafkaProducer
kafkaProducer = new KafkaProducer<>(properties);
}
/**
* Kafka 生产者 : 用于 Kafka 分区再均衡的测试
*/
@Test
public void testKafkaFirstProducer() throws ExecutionException, InterruptedException {
for (; ; ) {
// 消息key (可以为空)
final String key = "dankal-hello-" + RandomUtil.randomInt(100, 50000000);
// 消息内容
final String message = RandomUtil.randomString(64);
ProducerRecord<String, String> stringProducerRecord =
new ProducerRecord<String, String>(
Const.Kafka.Topic.HELLO_TOPIC,
key,
message
);
TimeUnit.MILLISECONDS.sleep(300);
kafkaProducer.send(stringProducerRecord).get();
}
}
@AfterAll
public static void after() {
if (Objects.nonNull(kafkaProducer)) {
kafkaProducer.close();
}
}
}
```
### KafkaRebalanceTest: 测试用例启动类
```java
package cn.dankal.test.rebalance;
import cn.dankal.test.factory.ThreadExecutorFactory;
import org.junit.jupiter.api.Test;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* 测试 Kafka 分区再均衡
*
* @author ZGH.MercyModest
* @version V1.0.0
* @create 2021-01-26
*/
public class KafkaRebalanceTest {
/**
* 测试 Kafka 分区再均衡
*/
@Test
public void testKafkaRebalance() throws InterruptedException {
final int workerCount = 2;
ThreadPoolExecutor threadPoolExecutor = ThreadExecutorFactory.getThreadPoolExecutor("dankal-test-kafka-rebalance");
for (int i = 0; i < workerCount; i++) {
ConsumerWorker consumerWorker = new ConsumerWorker(false);
threadPoolExecutor.execute(consumerWorker::doWork);
}
TimeUnit.SECONDS.sleep(30);
//启动一个临时消费者,触发分区再均衡
ConsumerWorker tempConsumerWorker = new ConsumerWorker(true);
threadPoolExecutor.execute(tempConsumerWorker::doWork);
new CountDownLatch(1).await();
}
}
```
## 日志小析
### 日志全文
[test-kafka-rebalance-log](https://res.mercymodest.com/res/test-kafka-rebalance-log.html)
### 小析
#### 刚开始启动
> dankal-test-kafka-rebalance-12: [dankal-test-hello-kafka-0, dankal-test-hello-kafka-1]
> dankal-test-kafka-rebalance-12 已完成参与分区再均衡.偏移量信息为: {}
>
> dankal-test-kafka-rebalance-13: [dankal-test-hello-kafka-2]
> dankal-test-kafka-rebalance-13 已完成参与分区再均衡.偏移量信息为: {}
#### 临时消费者加入 触发分区再平衡
##### onPartitionsRevoked
> dankal-test-kafka-rebalance-12: [dankal-test-hello-kafka-0, dankal-test-hello-kafka-1]
>
> dankal-test-kafka-rebalance-13: [dankal-test-hello-kafka-2]
>
>
>
> {dankal-test-hello-kafka-2=293, dankal-test-hello-kafka-0=282, dankal-test-hello-kafka-1=299}
##### 分区偏移量信息
> 分区0 消费偏移量:281
>
> 分区1 消费偏移量:298
>
> 分区2 消费偏移量:292
##### onPartitionsAssigned
> dankal-test-kafka-rebalance-12: [dankal-test-hello-kafka-0]
>
> dankal-test-kafka-rebalance-13: [dankal-test-hello-kafka-1]
>
> dankal-test-kafka-rebalance-16: [dankal-test-hello-kafka-2]
##### 分区偏移量信息
> 分区0 消费偏移量:282
>
> 分区1 消费偏移量:299
>
> 分区2 消费偏移量:293
#### 临时消费者退出消费,再次触发分区再平衡
##### 临时消费信息
> dankal-test-kafka-rebalance-16: [dankal-test-hello-kafka-2]
>
> 最后消费偏移量: 297 [dankal-test-hello-kafka-2]
##### onPartitionsRevoked
> dankal-test-kafka-rebalance-16: [dankal-test-hello-kafka-2]
>
>
>
> dankal-test-kafka-rebalance-12: [dankal-test-hello-kafka-0]
>
> dankal-test-kafka-rebalance-13: [dankal-test-hello-kafka-1]
>
>
>
> {dankal-test-hello-kafka-2=298, dankal-test-hello-kafka-0=284, dankal-test-hello-kafka-1=301}
##### 分区偏移量信息
> 分区0 消费偏移量:283
>
> 分区1 消费偏移量:300
>
> 分区2 消费偏移量:297
##### onPartitionsAssigned
> dankal-test-kafka-rebalance-12: [dankal-test-hello-kafka-0, dankal-test-hello-kafka-1]
>
> dankal-test-kafka-rebalance-13: [dankal-test-hello-kafka-2]
>
> {dankal-test-hello-kafka-2=298, dankal-test-hello-kafka-0=284, dankal-test-hello-kafka-1=301}
++ "b/\350\233\213\345\243\263\345\210\233\346\204\217\347\247\221\346\212\200-\346\212\200\346\234\257\346\265\213\350\257\225-.md"
This source diff could not be displayed because it is too large. You can view the blob instead.
# 蛋壳创意科技-技术测试-Kafka集群安装示例文档
# 蛋壳创意科技-技术测试-Kafka集群安装示例文档
> company: [蛋壳创意科技](https://www.dankal.cn/)
>
> official website:
>
> - **[Kafka](https://kafka.apache.org/)**
> - **[Zookeeper](https://zookeeper.apache.org/)**
> - **[Logi-KafkaManager](https://github.com/didi/Logi-KafkaManager)**
## 前置环境准备
### Docker 安装
> **[docker-install-linux](https://docs.docker.com/engine/install/)**
>
> **[Docker 安装 | 菜鸟教程](https://www.runoob.com/docker/centos-docker-install.html)**
### Docker Compose 服务编排
> **[docker-compose-install](https://docs.docker.com/compose/install/)**
>
> **[Docker Compose 安装 | 菜鸟教程](https://www.runoob.com/docker/docker-compose.html)**
### 测试用例环境版本
> ##### CentOS 7.9.2009
>
> > LSB Version: :core-4.1-amd64:core-4.1-noarch:cxx-4.1-amd64:cxx-4.1-noarch:desktop-4.1-amd64:desktop-4.1-noarch:languages-4.1-amd64:languages-4.1-noarch:printing-4.1-amd64:printing-4.1-noarch
> > Distributor ID: CentOS
> > Description: CentOS Linux release 7.9.2009 (Core)
> > Release: 7.9.2009
> > Codename: Core
>
> ##### docker
>
> > Docker version 20.10.1, build 831ebea
>
> ##### docker compose
>
> > docker-compose version 1.27.4, build 40524192
## 安装梗概
### kafka集群
> - kafka01
> - port: 9092
> - JMX Port: 9999
>
> - kafka02
> - port: 9093
> - JMX Port: 9988
> - kafka03
> - port: 9094
> - JMX Port: 9977
### Zookeeper 集群
> - zookeeper01
> - id : 1
> - port: 2181
> - zookeeper02
> - id : 2
> - port: 2182
> - zookeeper03
> - id : 3
> - port: 2183
### KafkaManager
> didi 2020 年新开源产品
### 安装方式
- Kafka 和 Zookeepr 集群采用 docker 形式安装
- docker-compose: docker compose 异步编排
- docker-compose.yml version 和 docker 版本关系
详情可参阅官网: [compose-file](https://docs.docker.com/compose/compose-file/)
![](https://img.mercymodest.com/public/20210125114852.png)
- Logi-KafkaManager 使用 jar 的形式运行
## docker network
便于 kafka 容器集群与 zookeeper 容器集群的通信,故在启动 zookeeper 和 kafka 集群之前需要先创建自定义的docker network
### ==执行命令==
> docker network create --driver bridge --subnet 172.128.0.0/25 --gateway 172.128.0.1 zookeeper_network
## Zookeeper 集群安装(docker)
Kafka 的集群管理依赖于 Zookeepr,虽然 Kafka 默认嵌入了 Zookeepr 但是建议使用自定义的 Zookeepr 集群
### docker-compose.yml
```yml
version: '3.7'
services:
zookeeper01:
image: zookeeper
restart: always
privileged: true
container_name: zookeeper01
hostname: zookeeper01
ports:
- 2181:2181
volumes: # 挂载数据
- ./zookeeper01/data:/data
- ./zookeeper01/datalog:/datalog
environment:
ZOO_MY_ID: 1
ZOO_SERVERS: server.1=0.0.0.0:2888:3888;2181 server.2=zookeeper02:2888:3888;2181 server.3=zookeeper03:2888:3888;2181
networks:
default:
ipv4_address: 172.128.0.10
zookeeper02:
image: zookeeper
restart: always
privileged: true
container_name: zookeeper02
hostname: zookeeper02
ports:
- 2182:2181
volumes: # 挂载数据
- ./zookeeper02/data:/data
- ./zookeeper02/datalog:/datalog
environment:
ZOO_MY_ID: 2
ZOO_SERVERS: server.1=zookeeper01:2888:3888;2181 server.2=0.0.0.0:2888:3888;2181 server.3=zookeeper03:2888:3888;2181
networks:
default:
ipv4_address: 172.128.0.11
zookeeper03:
image: zookeeper
restart: always
privileged: true
container_name: zookeeper03
hostname: zookeeper03
ports:
- 2183:2181
volumes: # 挂载数据
- ./zookeeper03/data:/data
- ./zookeeper03/datalog:/datalog
environment:
ZOO_MY_ID: 3
ZOO_SERVERS: server.1=zookeeper01:2888:3888;2181 server.2=zookeeper02:2888:3888;2181 server.3=0.0.0.0:2888:3888;2181
networks:
default:
ipv4_address: 172.128.0.12
networks: # 自定义网络
default:
external:
name: zookeeper_network
```
### 查看当前所在路径
![](https://img.mercymodest.com/public/20210125121417.png)
### 执行命令
==执行此命令 要求我们需要在 docker-compose.yml 所在目录下==
这也是为啥会有`查看当前所在路径`的步骤了~
> docker-compose up -d
![](https://img.mercymodest.com/public/20210125121625.png)tips: 因为我早已完成下载,所以我执行命令是直接启动容器,如果是第一次使用 当前 docker-compose.yml 会先下载我们所需要的容器,然后才会执行 `Createing`(自动执行,我们只需耐心等待即可)
### 集群启动结果确认
> docker ps
![](https://img.mercymodest.com/public/20210125122448.png)
> docker logs zookeeper01
>
> - (zookeeper01|zookeeper02|zookeeper03)
![](https://img.mercymodest.com/public/20210125122141.png)
如果没有出现异常日志,并且出现类似图示所圈日志信息,那么基本可以确认 基于 docker 容器的 zookeeper 已经成功启动~
## Kafka集群安装(docker)
相关路径问题可以参阅 Zookeeper集群安装(docker)#`查看当前所在路径`
### docker-compose.yml
```yml
version: '3.7'
services:
kafka01:
image: wurstmeister/kafka
#restart: always
hostname: kafka01
container_name: kafka01
ports:
- "9092:9092"
- "9999:9999"
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.128.3:9092
KAFKA_ADVERTISED_HOST_NAME: 192.168.128.3
KAFKA_ADVERTISED_PORT: 9092
KAFKA_ZOOKEEPER_CONNECT: zookeeper01:2181,zookeeper02:2181,zookeeper03:2181
JMX_PORT: 9999
volumes:
- ./kafka01/logs:/kafka
external_links:
- zookeeper01
- zookeeper02
- zookeeper03
networks:
default:
ipv4_address: 172.128.0.20
kafka02:
image: wurstmeister/kafka
#restart: always
hostname: kafka02
container_name: kafka02
ports:
- "9093:9092"
- "9988:9988"
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.128.3:9093
KAFKA_ADVERTISED_HOST_NAME: 192.168.128.3
KAFKA_ADVERTISED_PORT: 9093
KAFKA_ZOOKEEPER_CONNECT: zookeeper01:2181,zookeeper02:2181,zookeeper03:2181
JMX_PORT: 9988
volumes:
- ./kafka02/logs:/kafka
external_links:
- zookeeper01
- zookeeper02
- zookeeper03
networks:
default:
ipv4_address: 172.128.0.21
kafka03:
image: wurstmeister/kafka
#restart: always
hostname: kafka03
container_name: kafka03
ports:
- "9094:9092"
- "9977:9977"
environment:
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.128.3:9094
KAFKA_ADVERTISED_HOST_NAME: 192.168.128.3
KAFKA_ADVERTISED_PORT: 9094
KAFKA_ZOOKEEPER_CONNECT: zookeeper01:2181,zookeeper02:2181,zookeeper03:2181
JMX_PORT: 9977
volumes:
- ./kafka03/logs:/kafka
external_links:
- zookeeper01
- zookeeper02
- zookeeper03
networks:
default:
ipv4_address: 172.128.0.22
networks:
default:
external:
name: zookeeper_network
```
### 执行命令
> docker-compose up -d
![](https://img.mercymodest.com/public/20210125122939.png)
### 集群启动结果确认
> docker ps
![](https://img.mercymodest.com/public/20210125123035.png)
> docker logs kafka01
>
> - (kafka01|kafka02|kafka03)
![](https://img.mercymodest.com/public/20210125123209.png)
## Logi-KafkaManager 安装
> jar 包下载 : [kafka-manager-2.0.0.tar.gz](https://res.mercymodest.com/res/kafka-manager-2.0.0.tar.gz)
>
> 官方文档:[install_guide_cn.md](https://github.com/didi/Logi-KafkaManager/blob/master/docs/install_guide/install_guide_cn.md)
### 启动参考
#### 解压 tar 包
> tar -zxvf ./kafka-manager-2.0.0.tar.gz
![](https://img.mercymodest.com/public/20210125123854.png)
#### 创建所需 MySQL 数据库
> 执行 create_mysql_table.sql 创建数据库
![](https://img.mercymodest.com/public/20210125124118.png)
#### 修改 application.yml
![](https://img.mercymodest.com/public/20210125124310.png)
##### etc.
```yml
server:
port: 8182
tomcat:
accept-count: 1000
max-connections: 10000
max-threads: 800
min-spare-threads: 100
spring:
application:
name: kafkamanager
datasource:
kafka-manager:
jdbc-url: jdbc:mysql://rm-wz9z40fs394wti63aio.mysql.rds.aliyuncs.com:3306/dankal_kafka_manager?characterEncoding=UTF-8&serverTimezone=GMT%2B8
username: dankal_master
password: Dankal_system
driver-class-name: com.mysql.jdbc.Driver
main:
allow-bean-definition-overriding: true
profiles:
active: dev
servlet:
multipart:
max-file-size: 60MB
max-request-size: 60MB
logging:
config: classpath:logback-spring.xml
custom:
idc: cn
jmx:
max-conn: 10
account:
ldap:
kcm:
enabled: false
n9e:
base-url: http://127.0.0.1:8080
user-token: 12345678
timeout: 300
account: km
script-file: kcm_script.sh
monitor:
enabled: false
n9e:
nid: 2
user-token: 123456
mon:
base-url: http://127.0.0.1:8032
sink:
base-url: http://127.0.0.1:8006
rdb:
base-url: http://127.0.0.1:80
notify:
kafka:
cluster-id: 95
topic-name: didi-kafka-notify
order:
detail-url: http://127.0.0.1
```
#### 执行命令
> nohup java -jar kafka-manager.jar --spring.config.location=./application.yml &
#### 执行结果确认
> tail -f nohup.out
![](https://img.mercymodest.com/public/20210125124651.png)
#### 浏览使用
##### 浏览器输入
> http://192.168.128.3:8182/
>
> tips: 192.168.128.3 是 运行 `Logi-KafkaManager `所在服务器IP 地址, 8182 是 在启动 `Logi-KafkaManager `使用的 `application.yml`所配置的 服务端口
##### 账号密码
> 用户名:
>
> ​ admin
>
> 密码:
>
> ​ admin
![](https://img.mercymodest.com/public/20210125142328.png)
### 更多的使用
更多的使用可以参阅官方文档
**[user_guide](https://github.com/didi/Logi-KafkaManager/tree/master/docs/user_guide)**
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment