快速部署kafka集群和可视化界面

快速部署kafka集群和可视化界面

2024-12-28 05:39

3台机器部署kafka集群的分布
10.1.161.111 -> kafka（1个节点）+ zookeeper（一个节点）+ kowl（一个节点）
10.1.161.112 -> kafka（1个节点）+ zookeeper（一个节点）
10.1.161.113 -> kafka（1个节点）+ zookeeper（一个节点）

快速部署kafka集群和可视化界面

curl -fsSL https://get.docker.com | bash

yum -y install docker-ce
sudo systemctl start docker
sudo systemctl enable docker

sudo curl -L https://get.daocloud.io/docker/compose/releases/download/1.29.2/docker-compose-- -o /usr/local/bin/docker-compose

sudo chmod +x /usr/local/bin/docker-compose

docker compose -v
部署方法：
bash（10.1.161.111）

https://hub.docker.com/r/bitnami/kafka

https://hub.docker.com/r/bitnami/zookeeper

version: '2'
services:
zoo:
image: 'bitnami/zookeeper:latest'
restart: unless-stopped
hostname: zoo
container_name: zoo
ports:
- 2181:2181
- 2888:2888
- 3888:3888
volumes:
- https://www.cnblogs.com/zwnfdswww/p/data/zookeeper:/bitnami/zookeeper
environment:
ZOO_SERVER_ID: 1
ZOO_SERVERS: 0.0.0.0:2888:3888,10.1.161.112:2888:3888,10.1.161.113:2888:3888
ALLOW_ANONYMOUS_LOGIN: yes

kafka:
image: 'bitnami/kafka:latest'
restart: unless-stopped
hostname: kafka
container_name: kafka
ports:
- 9092:9092
environment:
KAFKA_ADVERTISED_HOST_NAME: 10.1.161.111
KAFKA_HOST_NAME: 10.1.161.111
KAFKA_ADVERTISED_PORT: 9092
KAFKA_BROKER_ID: 1
KAFKA_CFG_ZOOKEEPER_CONNECT: 10.1.161.111:2181,10.1.161.112:2181,10.1.161.113:2181
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://10.1.161.111:9092
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true
volumes:
- https://www.cnblogs.com/zwnfdswww/p/data/kafka:/bitnami/kafka

docker compose -f 10.1.161.111.yml up -d

bash（10.1.161.112）

https://hub.docker.com/r/bitnami/kafka

https://hub.docker.com/r/bitnami/zookeeper

version: '2'
services:
zoo:
image: 'bitnami/zookeeper:latest'
restart: unless-stopped
hostname: zoo
container_name: zoo
ports:
- 2181:2181
- 2888:2888
- 3888:3888
volumes:
- https://www.cnblogs.com/zwnfdswww/p/data/zookeeper:/bitnami/zookeeper
environment:
ZOO_SERVER_ID: 2
ZOO_SERVERS: 10.1.161.111:2888:3888,0.0.0.0:2888:3888,10.1.161.113:2888:3888
ALLOW_ANONYMOUS_LOGIN: yes

kafka:
image: 'bitnami/kafka:latest'
restart: unless-stopped
hostname: kafka
container_name: kafka
ports:
- 9092:9092
environment:
KAFKA_ADVERTISED_HOST_NAME: 10.1.161.112
KAFKA_HOST_NAME: 10.1.161.112
KAFKA_ADVERTISED_PORT: 9092
KAFKA_BROKER_ID: 2
KAFKA_CFG_ZOOKEEPER_CONNECT: 10.1.161.111:2181,10.1.161.112:2181,10.1.161.113:2181
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://10.1.161.112:9092
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true
volumes:
- https://www.cnblogs.com/zwnfdswww/p/data/kafka:/bitnami/kafka

docker compose -f 10.1.161.112.yml up -d

bash（10.1.161.113）

https://hub.docker.com/r/bitnami/kafka

https://hub.docker.com/r/bitnami/zookeeper

version: '2'
services:
zoo:
image: 'bitnami/zookeeper:latest'
restart: unless-stopped
hostname: zoo
container_name: zoo
ports:
- 2181:2181
- 2888:2888
- 3888:3888
volumes:
- https://www.cnblogs.com/zwnfdswww/p/data/zookeeper:/bitnami/zookeeper
environment:
ZOO_SERVER_ID: 3
ZOO_SERVERS: 10.1.161.111:2888:3888,10.1.161.112:2888:3888,0.0.0.0:2888:3888
ALLOW_ANONYMOUS_LOGIN: yes

kafka:
image: 'bitnami/kafka:latest'
restart: unless-stopped
hostname: kafka
container_name: kafka
ports:
- 9092:9092
environment:
KAFKA_ADVERTISED_HOST_NAME: 10.1.161.113
KAFKA_HOST_NAME: 10.1.161.113
KAFKA_ADVERTISED_PORT: 9092
KAFKA_BROKER_ID: 3
KAFKA_CFG_ZOOKEEPER_CONNECT: 10.1.161.111:2181,10.1.161.112:2181,10.1.161.113:2181
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://10.1.161.113:9092
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092
ALLOW_PLAINTEXT_LISTENER: yes
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true
volumes:
- https://www.cnblogs.com/zwnfdswww/p/data/kafka:/bitnami/kafka

docker compose -f 10.1.161.113.yml up -d

bash（10.1.161.111）
mkdir -p /app/conf
chmod -R 777 /app/conf
放这里面

docker-compose.yml

参考 https://github.com/cloudhut/kowl

version: '2'
services:
kowl:
image: 'quay.io/cloudhut/kowl:master'
restart: unless-stopped
hostname: kowl
container_name: kowl
ports:
- 8085:8085
volumes:
- .:/app/conf
environment:
CONFIG_FILEPATH: /app/conf/kowl.yaml

kowl.yaml

参考 https://github.com/cloudhut/kowl/blob/master/docs/config/kowl.yaml

kafka:
brokers:
- 10.1.161.111:9092
- 10.1.161.112:19092
- 10.1.161.113:19092

server:
listenPort: 8085
basePath: "/kowl/"
readTimeout: 30s
writeTimeout: 30s
idleTimeout: 30s
compressionLevel: 4

cd /app/conf
docker compose up -d

镜像源可能拉不下来，看我前面的文章，里面有解决办法
没权限自动新建文件夹:
yml文件放主目录里
sudo mkdir -p /root/data/zookeeper
sudo chmod -R 777 /root/data/zookeeper

sudo mkdir -p /root/data/zookeeper
sudo chmod -R 777 /root/data/kafka