Kafka和Zookeeper的伪集群安装

小天Code
713 阅读6分钟

最近在学习Kafka，记录一下Zookeeper和Kafka的伪集群安装步骤,以备随时查看。用的是我本地虚拟机（CentOS 7.4）在跑，模拟3Zk，3Kafka的集群方式，下面进入正题

Zookeeper单节点安装

  1. 创建zk目录
[root@localhost local]# pwd
/usr/local
[root@localhost local]# mkdir zookeeper
[root@localhost local]# cd zookeeper/
[root@localhost zookeeper]#
  1. 下载并解压zk

华中科技大学镜像地址 mirrors.hust.edu.cn/apache/zook… ，注意是-bin.tar.gz格式 image.png

[root@localhost zookeeper]# wget http://mirrors.hust.edu.cn/apache/zookeeper/zookeeper-3.5.9/apache-zookeeper-3.5.9-bin.tar.gz
// 下载进度...
[root@localhost zookeeper]# tar -zxvf apache-zookeeper-3.5.9-bin.tar.gz 
// 解压明细...
[root@localhost zookeeper]# ll
总用量 9400
drwxr-xr-x. 6 root root     134 6月   8 11:39 apache-zookeeper-3.5.9-bin
-rw-r--r--. 1 root root 9623007 1月  15 03:46 apache-zookeeper-3.5.9-bin.tar.gz
  1. 环境准备
    • 复制zoo.cfg
    [root@localhost zookeeper]# cd apache-zookeeper-3.5.9-bin
[root@localhost apache-zookeeper-3.5.9-bin]# ll
总用量 32
drwxr-xr-x. 2  502 games   232 1月   7 02:56 bin
drwxr-xr-x. 2  502 games    77 1月   7 02:56 conf
drwxr-xr-x. 5  502 games  4096 1月   7 03:48 docs
drwxr-xr-x. 2 root root   4096 6月   8 11:39 lib
-rw-r--r--. 1  502 games 11358 10月  5 2020 LICENSE.txt
-rw-r--r--. 1  502 games   432 1月   7 00:12 NOTICE.txt
-rw-r--r--. 1  502 games  1560 1月   7 02:56 README.md
-rw-r--r--. 1  502 games  1347 1月   7 02:56 README_packaging.txt
[root@localhost apache-zookeeper-3.5.9-bin]# cd conf/
[root@localhost conf]# ll
总用量 12
-rw-r--r--. 1 502 games  535 10月  5 2020 configuration.xsl
-rw-r--r--. 1 502 games 2712 1月   7 02:56 log4j.properties
-rw-r--r--. 1 502 games  922 1月   7 02:56 zoo_sample.cfg
[root@localhost conf]# cp zoo_sample.cfg zoo.cfg
[root@localhost conf]# ll
总用量 16
-rw-r--r--. 1  502 games  535 10月  5 2020 configuration.xsl
-rw-r--r--. 1  502 games 2712 1月   7 02:56 log4j.properties
-rw-r--r--. 1 root root   922 6月   8 11:47 zoo.cfg
-rw-r--r--. 1  502 games  922 1月   7 02:56 zoo_sample.cfg
    • 编辑zoo.cfg
    -- 指定数据输出目录，保存退出
[root@localhost conf]# vim zoo.cfg
dataDir=/usr/local/zookeeper/apache-zookeeper-3.5.9-bin/data
-- 退回根目录，创建data目录
[root@localhost apache-zookeeper-3.5.9-bin]# mkdir data
[root@localhost apache-zookeeper-3.5.9-bin]# ll
总用量 32
drwxr-xr-x. 2  502 games   232 1月   7 02:56 bin
drwxr-xr-x. 2  502 games    92 6月   8 11:52 conf
drwxr-xr-x. 2 root root      6 6月   8 11:53 data
drwxr-xr-x. 5  502 games  4096 1月   7 03:48 docs
drwxr-xr-x. 2 root root   4096 6月   8 11:39 lib
-rw-r--r--. 1  502 games 11358 10月  5 2020 LICENSE.txt
-rw-r--r--. 1  502 games   432 1月   7 00:12 NOTICE.txt
-rw-r--r--. 1  502 games  1560 1月   7 02:56 README.md
-rw-r--r--. 1  502 games  1347 1月   7 02:56 README_packaging.txt
  2. 启动与查看

    ./zkServer.sh start
    ./zkServer.sh status

    -- 启动
[root@localhost bin]# ./zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/apache-zookeeper-3.5.9-bin/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
-- 查看是否成功
[root@localhost bin]# ./zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/apache-zookeeper-3.5.9-bin/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: standalone

Zookeeper集群安装

  1. 集群环境配置管理
    • 复制三处配置
    [root@localhost conf]# pwd
/usr/local/zookeeper/apache-zookeeper-3.5.9-bin/conf
[root@localhost conf]# cp zoo.cfg zoo1.cfg
[root@localhost conf]# cp zoo.cfg zoo2.cfg
[root@localhost conf]# cp zoo.cfg zoo3.cfg
[root@localhost conf]# ll
总用量 28
-rw-r--r--. 1  502 games  535 10月  5 2020 configuration.xsl
-rw-r--r--. 1  502 games 2712 1月   7 02:56 log4j.properties
-rw-r--r--. 1 root root   960 6月   8 12:00 zoo1.cfg
-rw-r--r--. 1 root root   960 6月   8 12:00 zoo2.cfg
-rw-r--r--. 1 root root   960 6月   8 12:00 zoo3.cfg
-rw-r--r--. 1 root root   960 6月   8 12:00 zoo.cfg
-rw-r--r--. 1  502 games  922 1月   7 02:56 zoo_sample.cfg
    • 编辑三处配置
    [root@localhost conf]# vim zoo1.cfg
-- 改动下边这三处（数据输出目录，端口号，集群环境地址）
dataDir=/usr/local/zookeeper/apache-zookeeper-3.5.9-bin/data/zk1
# the port at which the clients will connect
clientPort=2191
server.1=localhost:2888:3888
server.2=localhost:2889:3889
server.3=localhost:2890:3890

[root@localhost conf]# vim zoo2.cfg
dataDir=/usr/local/zookeeper/apache-zookeeper-3.5.9-bin/data/zk2
# the port at which the clients will connect
clientPort=2192
server.1=localhost:2888:3888
server.2=localhost:2889:3889
server.3=localhost:2890:3890

[root@localhost conf]# vim zoo3.cfg
dataDir=/usr/local/zookeeper/apache-zookeeper-3.5.9-bin/data/zk3
# the port at which the clients will connect
clientPort=2193
server.1=localhost:2888:3888
server.2=localhost:2889:3889
server.3=localhost:2890:3890
  2. 在dataDir目录下为集群环境创建子目录zk_n 
    [root@localhost data]# pwd
/usr/local/zookeeper/apache-zookeeper-3.5.9-bin/data
[root@localhost data]# mkdir zk1
[root@localhost data]# mkdir zk2
[root@localhost data]# mkdir zk3
[root@localhost data]# ll
总用量 4
drwxr-xr-x. 2 root root 24 6月   8 11:56 version-2
drwxr-xr-x. 2 root root  6 6月   8 12:11 zk1
drwxr-xr-x. 2 root root  6 6月   8 12:11 zk2
drwxr-xr-x. 2 root root  6 6月   8 12:11 zk3
-rw-r--r--. 1 root root  5 6月   8 11:55 zookeeper_server.pid
  3. 创建myid文件

    touch ./data/zk_n/myid
    echo '1' > ./data/zk_n/myid

    [root@localhost apache-zookeeper-3.5.9-bin]# touch ./data/zk1/myid
[root@localhost apache-zookeeper-3.5.9-bin]# echo '1' > ./data/zk1/myid

[root@localhost apache-zookeeper-3.5.9-bin]# touch ./data/zk2/myid
[root@localhost apache-zookeeper-3.5.9-bin]# echo '2' > ./data/zk2/myid

[root@localhost apache-zookeeper-3.5.9-bin]# touch ./data/zk3/myid
[root@localhost apache-zookeeper-3.5.9-bin]# echo '3' > ./data/zk3/myid
  4. 启动zk集群环境

    ./bin/zkServer.sh start ./conf/zoo_n.cfg

    [root@localhost apache-zookeeper-3.5.9-bin]# pwd
/usr/local/zookeeper/apache-zookeeper-3.5.9-bin
[root@localhost apache-zookeeper-3.5.9-bin]# ./bin/zkServer.sh start ./conf/zoo1.cfg
ZooKeeper JMX enabled by default
Using config: ./conf/zoo1.cfg
Starting zookeeper ... STARTED
[root@localhost apache-zookeeper-3.5.9-bin]# ./bin/zkServer.sh start ./conf/zoo2.cfg
ZooKeeper JMX enabled by default
Using config: ./conf/zoo2.cfg
Starting zookeeper ... STARTED
[root@localhost apache-zookeeper-3.5.9-bin]# ./bin/zkServer.sh start ./conf/zoo3.cfg
ZooKeeper JMX enabled by default
Using config: ./conf/zoo3.cfg
Starting zookeeper ... STARTED
  5. 验证zk集群状态(是否成功)

    ./bin/zkServer.sh status ./conf/zoo_n.cfg

    [root@localhost apache-zookeeper-3.5.9-bin]# ./bin/zkServer.sh status ./conf/zoo1.cfg
ZooKeeper JMX enabled by default
Using config: ./conf/zoo1.cfg
Client port found: 2191. Client address: localhost. Client SSL: false.
Mode: follower
[root@localhost apache-zookeeper-3.5.9-bin]# ./bin/zkServer.sh status ./conf/zoo2.cfg
ZooKeeper JMX enabled by default
Using config: ./conf/zoo2.cfg
Client port found: 2192. Client address: localhost. Client SSL: false.
Mode: leader
[root@localhost apache-zookeeper-3.5.9-bin]# ./bin/zkServer.sh status ./conf/zoo3.cfg
ZooKeeper JMX enabled by default
Using config: ./conf/zoo3.cfg
Client port found: 2193. Client address: localhost. Client SSL: false.
Mode: follower

Kafka单节点安装

  1. 创建kafka目录 
    [root@localhost local]# pwd
/usr/local
[root@localhost local]# mkdir kafka
mkdir: 无法创建目录"kafka": 文件已存在
[root@localhost local]# cd kafka
[root@localhost kafka]#
  2. 下载并解压缩kafka

    下载地址，kafka.apache.org/downloads

    [root@localhost kafka]# wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.8.0/kafka_2.12-2.8.0.tgz 
// 下载进度...
[root@localhost kafka]# tar -xvf kafka_2.12-2.8.0.tgz 
// 解压明细...
[root@localhost kafka]# ll
总用量 69868
drwxr-xr-x. 7 root root      105 4月  14 22:31 kafka_2.12-2.8.0
-rw-r--r--. 1 root root 71542357 4月  19 12:44 kafka_2.12-2.8.0.tgz
  3. 修改配置信息 
    [root@localhost config]# vim server.properties
log.dirs=/usr/local/kafka/kafka-logs
  4. 启动kafka 
    [root@localhost kafka_2.12-2.8.0]# nohup ./bin/kafka-server-start.sh config/server.properties &

Kafka集群搭建

  1. 集群配置环境管理
    • 复制三处配置
    [root@localhost config]# pwd
/usr/local/kafka/kafka_2.12-2.8.0/config
[root@localhost config]# cp server.properties server1.properties
[root@localhost config]# cp server.properties server2.properties
[root@localhost config]# cp server.properties server3.properties
    • 编辑三处配置信息
    [root@localhost config]# vim server1.properties
broker.id=1
listeners=PLAINTEXT://:8091
log.dirs=/usr/local/kafka/kafka-logs/kafka1
zookeeper.connect=localhost:2191,localhost:2192,localhost:2193

broker.id=2
listeners=PLAINTEXT://:8092
log.dirs=/usr/local/kafka/kafka-logs/kafka2
zookeeper.connect=localhost:2191,localhost:2192,localhost:2193

broker.id=3
listeners=PLAINTEXT://:8093
log.dirs=/usr/local/kafka/kafka-logs/kafka3
zookeeper.connect=localhost:2191,localhost:2192,localhost:2193
  2. 在kafka-logs目录下创建子目录 
    [root@localhost kafka-logs]# pwd
/usr/local/kafka/kafka-logs
[root@localhost kafka-logs]# mkdir kafka1
[root@localhost kafka-logs]# mkdir kafka2
[root@localhost kafka-logs]# mkdir kafka3
  3. 启动kafka集群并查看集群状态 
    [root@localhost kafka_2.12-2.8.0]# pwd
/usr/local/kafka/kafka_2.12-2.8.0
[root@localhost kafka_2.12-2.8.0]# ./bin/kafka-server-start.sh -daemon ./config/server1.properties
[root@localhost kafka_2.12-2.8.0]# ./bin/kafka-server-start.sh -daemon ./config/server2.properties
[root@localhost kafka_2.12-2.8.0]# ./bin/kafka-server-start.sh -daemon ./config/server3.properties
[root@localhost kafka_2.12-2.8.0]# jps | grep Kafka
23872 Kafka
22945 Kafka
24277 Kafka
24701 Kafka
avatar
文章
阅读
粉丝