docker swarm 搭建kafka集群

创建docker-compose.yml文件

在根目录下创建了kafkacompose.yml文件


version: '3.7'

services:
  zoo1:
    image: zookeeper:latest
    networks:
      - kafka
    hostname: zoo1
    ports: # 端口
      - 2181:2181
    environment:
      ZOO_MY_ID: 1
      ZOO_SERVERS: server.1=0.0.0.0:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - "/gfs-share/zookeeper/zoo1conf:/conf"
      - "/gfs-share/zookeeper/zoo1data:/data"
      - "/gfs-share/zookeeper/zoo1datalog:/datalog"
      - "/gfs-share/zookeeper/zoo1logs:/logs"
    deploy:
      mode: replicated
      replicas: 1
      placement:
         constraints: # 添加条件约束
            - node.labels.role1==zookeeper
  zoo2:
    image: zookeeper:latest
    networks:
      - kafka
    hostname: zoo2
    ports: # 端口
      - 2182:2181
    environment:
      ZOO_MY_ID: 2
      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=0.0.0.0:2888:3888;2181 server.3=zoo3:2888:3888;2181
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - "/gfs-share/zookeeper/zoo2conf:/conf"
      - "/gfs-share/zookeeper/zoo2data:/data"
      - "/gfs-share/zookeeper/zoo2datalog:/datalog"
      - "/gfs-share/zookeeper/zoo2logs:/logs"
    deploy:
      mode: replicated
      replicas: 1
      placement:
         constraints: # 添加条件约束
            - node.labels.role1==zookeeper
  zoo3:
    image: zookeeper:latest
    networks:
      - kafka
    hostname: zoo3
    ports: # 端口
      - 2183:2181
    environment:
      ZOO_MY_ID: 3
      ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=0.0.0.0:2888:3888;2181
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - "/gfs-share/zookeeper/zoo3conf:/conf"
      - "/gfs-share/zookeeper/zoo3data:/data"
      - "/gfs-share/zookeeper/zoo3datalog:/datalog"
      - "/gfs-share/zookeeper/zoo3logs:/logs"
    deploy:
      mode: replicated
      replicas: 1 
      placement:
         constraints: # 添加条件约束
            - node.labels.role1==zookeeper
      
  kafka1:
    image: wurstmeister/kafka:latest
    hostname: kafka1
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - target: 9094
        published: 9094
        protocol: tcp
        mode: host
    environment:
      HOSTNAME_COMMAND: "docker info -f '{{`{{.Swarm.NodeAddr}}`}}'"
      KAFKA_BROKER_ID: 1
      KAFKA_MESSAGE_MAX_BYTES: 5000000
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_LOG_DIRS: /kafka/kafka_log
      KAFKA_LOG_CLEANUP_POLICY: 'delete'
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG_RETENTION_BYTES: 1048576
      KAFKA_LOG_SEGMENT_BYTES: 1048576
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
      KAFKA_LOG_CLEANLE_ENABLE: 'true'
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - /var/run/docker.sock:/var/run/docker.sock
      - "/gfs-share/kafka/kafka1:/kafka"
    networks:
      - kafka
    deploy:
      placement:
         constraints: # 添加条件约束
            - node.labels.role==kafka1
  
  kafka2:
    image: wurstmeister/kafka:latest
    hostname: kafka2
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - target: 9094
        published: 9094
        protocol: tcp
        mode: host
    environment:
      HOSTNAME_COMMAND: "docker info -f '{{`{{.Swarm.NodeAddr}}`}}'"
      KAFKA_BROKER_ID: 2
      KAFKA_MESSAGE_MAX_BYTES: 5000000
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_LOG_DIRS: /kafka/kafka_log
      KAFKA_LOG_CLEANUP_POLICY: 'delete'
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG_RETENTION_BYTES: 1048576
      KAFKA_LOG_SEGMENT_BYTES: 1048576
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
      KAFKA_LOG_CLEANLE_ENABLE: 'true'
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - /var/run/docker.sock:/var/run/docker.sock
      - "/gfs-share/kafka/kafka2:/kafka"
    networks:
      - kafka
    deploy:
      placement:
         constraints: # 添加条件约束
            - node.labels.role==kafka2
            
  kafka3:
    image: wurstmeister/kafka:latest
    hostname: kafka3
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - target: 9094
        published: 9094
        protocol: tcp
        mode: host
    environment:
      HOSTNAME_COMMAND: "docker info -f '{{`{{.Swarm.NodeAddr}}`}}'"
      KAFKA_BROKER_ID: 3
      KAFKA_MESSAGE_MAX_BYTES: 5000000
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_LOG_DIRS: /kafka/kafka_log
      KAFKA_LOG_CLEANUP_POLICY: 'delete'
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG_RETENTION_BYTES: 1048576
      KAFKA_LOG_SEGMENT_BYTES: 1048576
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
      KAFKA_LOG_CLEANLE_ENABLE: 'true'
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - /var/run/docker.sock:/var/run/docker.sock
      - "/gfs-share/kafka/kafka3:/kafka"
    networks:
      - kafka
    deploy:
      placement:
         constraints: # 添加条件约束
            - node.labels.role==kafka3
            
  kafka4:
    image: wurstmeister/kafka:latest
    hostname: kafka4
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - target: 9094
        published: 9094
        protocol: tcp
        mode: host
    environment:
      HOSTNAME_COMMAND: "docker info -f '{{`{{.Swarm.NodeAddr}}`}}'"
      KAFKA_BROKER_ID: 4
      KAFKA_MESSAGE_MAX_BYTES: 5000000
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_LOG_DIRS: /kafka/kafka_log
      KAFKA_LOG_CLEANUP_POLICY: 'delete'
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG_RETENTION_BYTES: 1048576
      KAFKA_LOG_SEGMENT_BYTES: 1048576
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
      KAFKA_LOG_CLEANLE_ENABLE: 'true'
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - /var/run/docker.sock:/var/run/docker.sock
      - "/gfs-share/kafka/kafka4:/kafka"
    networks:
      - kafka
    deploy:
      placement:
         constraints: # 添加条件约束
            - node.labels.role==kafka4
            
  kafka5:
    image: wurstmeister/kafka:latest
    hostname: kafka5
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - target: 9094
        published: 9094
        protocol: tcp
        mode: host
    environment:
      HOSTNAME_COMMAND: "docker info -f '{{`{{.Swarm.NodeAddr}}`}}'"
      KAFKA_BROKER_ID: 5
      KAFKA_MESSAGE_MAX_BYTES: 5000000
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_LOG_DIRS: /kafka/kafka_log
      KAFKA_LOG_CLEANUP_POLICY: 'delete'
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG_RETENTION_BYTES: 1048576
      KAFKA_LOG_SEGMENT_BYTES: 1048576
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
      KAFKA_LOG_CLEANLE_ENABLE: 'true'
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - /var/run/docker.sock:/var/run/docker.sock
      - "/gfs-share/kafka/kafka5:/kafka"
    networks:
      - kafka
    deploy:
      placement:
         constraints: # 添加条件约束
            - node.labels.role==kafka5
            
  kafka6:
    image: wurstmeister/kafka:latest
    hostname: kafka6
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - target: 9094
        published: 9094
        protocol: tcp
        mode: host
    environment:
      HOSTNAME_COMMAND: "docker info -f '{{`{{.Swarm.NodeAddr}}`}}'"
      KAFKA_BROKER_ID: 6
      KAFKA_MESSAGE_MAX_BYTES: 5000000
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_LOG_DIRS: /kafka/kafka_log
      KAFKA_LOG_CLEANUP_POLICY: 'delete'
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG_RETENTION_BYTES: 1048576
      KAFKA_LOG_SEGMENT_BYTES: 1048576
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
      KAFKA_LOG_CLEANLE_ENABLE: 'true'
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - /var/run/docker.sock:/var/run/docker.sock
      - "/gfs-share/kafka/kafka6:/kafka"
    networks:
      - kafka
    deploy:
      placement:
         constraints: # 添加条件约束
            - node.labels.role==kafka6
            
  kafka7:
    image: wurstmeister/kafka:latest
    hostname: kafka7
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - target: 9094
        published: 9094
        protocol: tcp
        mode: host
    environment:
      HOSTNAME_COMMAND: "docker info -f '{{`{{.Swarm.NodeAddr}}`}}'"
      KAFKA_BROKER_ID: 7
      KAFKA_MESSAGE_MAX_BYTES: 5000000
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_LOG_DIRS: /kafka/kafka_log
      KAFKA_LOG_CLEANUP_POLICY: 'delete'
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG_RETENTION_BYTES: 1048576
      KAFKA_LOG_SEGMENT_BYTES: 1048576
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
      KAFKA_LOG_CLEANLE_ENABLE: 'true'
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - /var/run/docker.sock:/var/run/docker.sock
      - "/gfs-share/kafka/kafka7:/kafka"
    networks:
      - kafka
    deploy:
      placement:
         constraints: # 添加条件约束
            - node.labels.role==kafka7
            
  kafka8:
    image: wurstmeister/kafka:latest
    hostname: kafka8
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - target: 9094
        published: 9094
        protocol: tcp
        mode: host
    environment:
      HOSTNAME_COMMAND: "docker info -f '{{`{{.Swarm.NodeAddr}}`}}'"
      KAFKA_BROKER_ID: 8
      KAFKA_MESSAGE_MAX_BYTES: 5000000
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_LOG_DIRS: /kafka/kafka_log
      KAFKA_LOG_CLEANUP_POLICY: 'delete'
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG_RETENTION_BYTES: 1048576
      KAFKA_LOG_SEGMENT_BYTES: 1048576
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
      KAFKA_LOG_CLEANLE_ENABLE: 'true'
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - /var/run/docker.sock:/var/run/docker.sock
      - "/gfs-share/kafka/kafka8:/kafka"
    networks:
      - kafka
    deploy:
      placement:
         constraints: # 添加条件约束
            - node.labels.role==kafka8
            
  kafka9:
    image: wurstmeister/kafka:latest
    hostname: kafka9
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - target: 9094
        published: 9094
        protocol: tcp
        mode: host
    environment:
      HOSTNAME_COMMAND: "docker info -f '{{`{{.Swarm.NodeAddr}}`}}'"
      KAFKA_BROKER_ID: 9
      KAFKA_MESSAGE_MAX_BYTES: 5000000
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_LOG_DIRS: /kafka/kafka_log
      KAFKA_LOG_CLEANUP_POLICY: 'delete'
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG_RETENTION_BYTES: 1048576
      KAFKA_LOG_SEGMENT_BYTES: 1048576
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
      KAFKA_LOG_CLEANLE_ENABLE: 'true'
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - /var/run/docker.sock:/var/run/docker.sock
      - "/gfs-share/kafka/kafka9:/kafka"
    networks:
      - kafka
    deploy:
      placement:
         constraints: # 添加条件约束
            - node.labels.role==kafka9
            
  kafka10:
    image: wurstmeister/kafka:latest
    hostname: kafka10
    depends_on:
      - zoo1
      - zoo2
      - zoo3
    ports:
      - target: 9094
        published: 9094
        protocol: tcp
        mode: host
    environment:
      HOSTNAME_COMMAND: "docker info -f '{{`{{.Swarm.NodeAddr}}`}}'"
      KAFKA_BROKER_ID: 10
      KAFKA_MESSAGE_MAX_BYTES: 5000000
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://_{HOSTNAME_COMMAND}:9094
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
      KAFKA_LOG_DIRS: /kafka/kafka_log
      KAFKA_LOG_CLEANUP_POLICY: 'delete'
      KAFKA_LOG_RETENTION_HOURS: 1
      KAFKA_LOG_RETENTION_BYTES: 1048576
      KAFKA_LOG_SEGMENT_BYTES: 1048576
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
      KAFKA_LOG_CLEANLE_ENABLE: 'true'
    volumes:
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
      - /var/run/docker.sock:/var/run/docker.sock
      - "/gfs-share/kafka/kafka10:/kafka"
    networks:
      - kafka
    deploy:
      placement:
         constraints: # 添加条件约束
            - node.labels.role==kafka10
            
networks:
  kafka:
    external:
      name: publicNetWork

启动集群

docker stack deploy -c kafkacompose.yml kafka
# kafka # docker # 分布式
Logo
Kafka开源项目指南

Kafka开源项目指南提供详尽教程，助开发者掌握其架构、配置和使用，实现高效数据流管理和实时处理。它高性能、可扩展，适合日志收集和实时数据处理，通过持久化保障数据安全，是企业大数据生态系统的核心。

更多推荐

Kafka入门（一） 概述、部署与API的简单使用

Kafka概述、部署与API的简单使用

avatar Kafka开源项目指南

基于canal和kafka同步，实现binlog同步ElasticSearch

文章目录前言elasticsearch 安装canal安装canal-adapter 安装及配置mysql 安装zk及kafaka安装查看效果注意事项前言中间件版本elasticsearch7.5.2canal1.1.4client-adapter1.1.5-alpha-1zookeeper3.4.13kafka2.6.0mysql5.7.31elasticsearch 安装{"settings"

avatar Kafka开源项目指南

基于 Iceberg 的湖仓一体架构在 B 站的实践

背景在B站，每天都有PB级的数据注入到大数据平台，经过离线或实时的ETL建模后，提供给下游的分析、推荐及预测等场景使用。面对如此大规模的数据，如何高效低成本地满足下游数据的分析需求，一直是我们重点的工作方向。我们之前的数据处理流程基本上是这样的：采集端将客户端埋点、服务端埋点、日志、业务数据库等数据收集到HDFS、Kafka等存储系统中，然后通过Hive、Spark、Fl...

avatar Kafka开源项目指南