2019独角兽企业重金招聘Python工程师标准>>> hot3.png

SpringBoot 整合 kafka 实现组订阅模式：

工程结构图

fdc40aa4eee6c9d545315fac884036b8a42.jpg

 

 

消息生产者pom.xml配置
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>scms-parent</artifactId>
        <groupId>yj.scms.parent</groupId>
        <version>1.0-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <groupId>yj.scms.kafka</groupId>
    <artifactId>scms-kafka</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.springframework.kafka/spring-kafka -->
        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
            <version>1.3.5.RELEASE</version>
        </dependency>


        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.8.2</version>
        </dependency>
        <!-- fastjson jar的神奇功能  对象转json  json转 对象 -->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.47</version>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>
</project>
消息消费者配置：
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>scms-parent</artifactId>
        <groupId>yj.scms.parent</groupId>
        <version>1.0-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <groupId>yj.scms.kafka</groupId>
    <artifactId>scms-kafka</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.springframework.kafka/spring-kafka -->
        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
            <version>1.3.5.RELEASE</version>
        </dependency>


        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.8.2</version>
        </dependency>
        <!-- fastjson jar的神奇功能  对象转json  json转 对象 -->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.47</version>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>
</project>

 

生产者application.properties配置文件
#设置端口号
server.port=9003

#返回页面、数据中文乱码问题
spring.http.encoding.force=true
spring.http.encoding.charset=UTF-8
spring.http.encoding.enabled=true
server.tomcat.uri-encoding = UTF-8
#解决程序读配置文件乱码问题
spring.message.encoding = UTF-8

#客户端注册进eureka服务列表内
#eureka.client.service-url.defaultZone=http://eureka8002.com:8002/eureka/,http://eureka8003.com:8003/eureka/,http://eureka8004.com:8004/eureka/
eureka.client.service-url.defaultZone=http://127.0.0.1:8001/eureka/
#微服务名称设置在注册中心显示的名称
spring.application.name=scms-kafka-send

################ 显示ip端口号方式1 ##################################
#访问路径可以显示IP地址
eureka.instance.preferIpAddress=true
#微服务名称设置在注册中心显示的名称
eureka.instance.instance-id=${spring.cloud.client.ipAddress}:${server.port}

################ 显示ip端口号方式2 ##################################
#访问路径可以显示IP地址
#eureka.instance.prefer-ip-address=true
#微服务名称设置在注册中心显示的名称
#eureka.instance.instance-id=${spring.cloud.client.ip-address}:${server.port}

#在eureka微服务info显示内容详细信息
info.app.name=user-service
info.company.name=www.baidu.com
info.build.artifactId=$project.artifactId$
info.build.version=$project.version$


##################################kafka配置文件###############################################
#============== kafka ===================
# 指定kafka 代理地址，可以多个
spring.kafka.bootstrap-servers=127.0.0.1:9092

#=============== provider  =======================

spring.kafka.producer.retries=0
# 每次批量发送消息的数量
spring.kafka.producer.batch-size=16384
spring.kafka.producer.buffer-memory=33554432

# 指定消息key和消息体的编解码方式
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer

#=============== consumer  =======================
# 指定默认消费者group id
#spring.kafka.consumer.group-id=test-consumer-group

spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.consumer.enable-auto-commit=true
spring.kafka.consumer.auto-commit-interval=100

# 指定消息key和消息体的编解码方式
spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer

###########################kafak配置结束#######################################################################
消费者application.properties配置文件
#设置端口号
server.port=9002

#返回页面、数据中文乱码问题
spring.http.encoding.force=true
spring.http.encoding.charset=UTF-8
spring.http.encoding.enabled=true
server.tomcat.uri-encoding = UTF-8
#解决程序读配置文件乱码问题
spring.message.encoding = UTF-8

#客户端注册进eureka服务列表内
#eureka.client.service-url.defaultZone=http://eureka8002.com:8002/eureka/,http://eureka8003.com:8003/eureka/,http://eureka8004.com:8004/eureka/
eureka.client.service-url.defaultZone=http://127.0.0.1:8001/eureka/
#微服务名称设置在注册中心显示的名称
spring.application.name=scms-kafka

################ 显示ip端口号方式1 ##################################
#访问路径可以显示IP地址
eureka.instance.preferIpAddress=true
#微服务名称设置在注册中心显示的名称
eureka.instance.instance-id=${spring.cloud.client.ipAddress}:${server.port}

################ 显示ip端口号方式2 ##################################
#访问路径可以显示IP地址
#eureka.instance.prefer-ip-address=true
#微服务名称设置在注册中心显示的名称
#eureka.instance.instance-id=${spring.cloud.client.ip-address}:${server.port}

#在eureka微服务info显示内容详细信息
info.app.name=user-service
info.company.name=www.baidu.com
info.build.artifactId=$project.artifactId$
info.build.version=$project.version$


##################################kafka配置文件###############################################
#============== kafka ===================
# 指定kafka 代理地址，可以多个
spring.kafka.bootstrap-servers=127.0.0.1:9092

#=============== provider  =======================

spring.kafka.producer.retries=0
# 每次批量发送消息的数量
spring.kafka.producer.batch-size=16384
spring.kafka.producer.buffer-memory=33554432

# 指定消息key和消息体的编解码方式
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.apache.kafka.common.serialization.StringSerializer

#=============== consumer  =======================
# 指定默认消费者group id
#spring.kafka.consumer.group-id=test-consumer-group

spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.consumer.enable-auto-commit=true
spring.kafka.consumer.auto-commit-interval=100

# 指定消息key和消息体的编解码方式
spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer

###########################kafak配置结束#######################################################################
 

 

生产者send 的java代码
package yj.scms.kafka.sender;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Component;
import yj.scms.kafka.pojo.Message;

import java.util.Date;
import java.util.UUID;

/**
 * kafka生产者
 */
@Component
public class KafkaSender {
    protected final Logger logger = LoggerFactory.getLogger(this.getClass());

    @Autowired
    private KafkaTemplate<String, String> kafkaTemplate;

    private Gson gson = new GsonBuilder().create();

    //发送消息方法
    public void send() {
        Message message = new Message();
        message.setId(System.currentTimeMillis());
        message.setMsg(UUID.randomUUID().toString());
        message.setSendTime(new Date());
        logger.info("+++++++++++++++++++++  message = {}", gson.toJson(message));
        kafkaTemplate.send("hhh", gson.toJson(message));
    }
}
消费者组1 java代码
package yj.scms.kafka.receiver;


import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;

import java.util.Optional;

/**
 * kafka消费者
 */
@Component
public class KafkaReceiver {

    protected final Logger logger = LoggerFactory.getLogger(this.getClass());

    @KafkaListener(topics = {"hhh"},groupId = "xiaofeng1")
    public void listen(ConsumerRecord<?, ?> record) {

        Optional<?> kafkaMessage = Optional.ofNullable(record.value());

        if (kafkaMessage.isPresent()) {
            Object message = kafkaMessage.get();
            logger.info("-xiaofeng1-- test1：" + record+"，record:" + record+",message:" + message+",---");
        }

    }
}

 

消费者组2java代码
package yj.scms.kafka.receiver;


import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;

import java.util.Optional;

/**
 * kafka消费者
 */
@Component
public class KafkaReceiverTwo {

    protected final Logger logger = LoggerFactory.getLogger(this.getClass());

    @KafkaListener(topics = {"hhh"},groupId = "xiaofeng2")
    public void listen(ConsumerRecord<?, ?> record) {

        Optional<?> kafkaMessage = Optional.ofNullable(record.value());

        if (kafkaMessage.isPresent()) {
            Object message = kafkaMessage.get();
            logger.info("-xiaofeng2-- test2：" + record+"，record:" + record+",message:" + message+",---");
        }

    }
}

 

 

一个生产者发送消息
f214b292af33c2a829bbe9f86e641673aa4.jpg
两个小消费者同时消费
176c0f34cd3ba7717c1bcd2d384d733a286.jpg

 

 

 您觉得本文章对您有帮助，那么可以选择打赏。
打赏多少，您高兴就行，谢谢您的支持！ ~(@^_^@)~

                          微信               支付宝
08cdeab56f6808041ef833491f96a941fbc.jpgd886d6660cd1dd7c002516d1c1f77147adf.jpg

 

转载于:https://my.oschina.net/u/3621360/blog/2249024

# java # 大数据 # 系统架构
Logo
Kafka开源项目指南

Kafka开源项目指南提供详尽教程，助开发者掌握其架构、配置和使用，实现高效数据流管理和实时处理。它高性能、可扩展，适合日志收集和实时数据处理，通过持久化保障数据安全，是企业大数据生态系统的核心。

更多推荐

Kafka入门（一） 概述、部署与API的简单使用

Kafka概述、部署与API的简单使用

avatar Kafka开源项目指南

基于canal和kafka同步，实现binlog同步ElasticSearch

文章目录前言elasticsearch 安装canal安装canal-adapter 安装及配置mysql 安装zk及kafaka安装查看效果注意事项前言中间件版本elasticsearch7.5.2canal1.1.4client-adapter1.1.5-alpha-1zookeeper3.4.13kafka2.6.0mysql5.7.31elasticsearch 安装{"settings"

avatar Kafka开源项目指南

基于 Iceberg 的湖仓一体架构在 B 站的实践

背景在B站，每天都有PB级的数据注入到大数据平台，经过离线或实时的ETL建模后，提供给下游的分析、推荐及预测等场景使用。面对如此大规模的数据，如何高效低成本地满足下游数据的分析需求，一直是我们重点的工作方向。我们之前的数据处理流程基本上是这样的：采集端将客户端埋点、服务端埋点、日志、业务数据库等数据收集到HDFS、Kafka等存储系统中，然后通过Hive、Spark、Fl...

avatar Kafka开源项目指南