kafka单机部署和集群化部署

小无敌犇犇
112 阅读4分钟
broker.id0Broker唯一标识
broker.id0Broker唯一标识
listenersPLAINTEXT://192.168.110.121:9092监听信息，PLAINTEXT表示明文传输
log.dirskafka/logskafka数据存放地址，可以填写多个。用","间隔
message.max.bytesmessage.max.bytes单个消息长度限制，单位是字节
num.partitions1默认分区数
log.flush.interval.messagesLong.MaxValue在数据被写入到硬盘和消费者可用前最大累积的消息的数量
log.flush.interval.msLong.MaxValue在数据被写入到硬盘前的最大时间
log.flush.scheduler.interval.msLong.MaxValue检查数据是否要写入到硬盘的时间间隔。
log.retention.hours24控制一个log保留时间，单位：小时
zookeeper.connect192.168.110.121:2181ZooKeeper服务器地址，多台用","间隔

1.1单节点部署

tar -zxf zookeeper-3.4.14.tar.gz -C /opt
tar -zxf jdk-8u201-linux-arm64-vfp-hflt.tar.gz -C /opt
tar -zxf kafka_2.11-2.4.0.tgz -C /opt

ln -s /opt/zookeeper-3.4.14 /usr/local/zookeeper
ln -s /opt/jdk-8u201 /usr/local/jdk
ln -s /opt/kafka_2.11-2.4.0 /usr/local/kafka

vim /etc/profile
export JAVA_HOME=/usr/local/jdk
export PATH=$JAVA_HOME/bin:$PATH

source /etc/profile

cd /opt/zookeeper-3.4.14/conf
mv zoo_sample.cfg zoo.cfg
cp zoo.cfg zoo.cfg.bak

mkdir /usr/local/zookeeper/data
vim zoo.cfg
dataDir=/usr/local/zookeeper/data

cd ../bin
./zkServer.sh start
./zkServer.sh status

mkdir /usr/local/kafka/kafka-logs
cd /usr/local/kafka/config
vim server.properties
#新增
auto.create.topics.enable=true
delete.topic.enable=true
allow.everyone.if.no.acl.found=true

#修改
listeners=PLAINTEXT://hostname:9092
advertised.listeners=PLAINTEXT://hostname:9092
log.dirs=/usr/local/kafka/kafka-logs
#连接
zookeeper.connect=hostename:2181

cd ../bin
#首次启动用以下命令，查看有无报错
./kafka-server-start.sh ../config/server.properties    
#等待一会

#再次启动用以下命令，后台启动
nohup ./kafka-server-start.sh ../config/server.properties 1>/dev/null 2>&1 &

#查看日志有没有报错
cd ../logs
tail -f server.log

2.1集群部署

#默认情况所有主机都操作
tar -zxf zookeeper-3.4.14.tar.gz -C /opt
tar -zxf jdk-8u201-linux-arm64-vfp-hflt.tar.gz -C /opt
tar -zxf kafka_2.11-2.4.0.tgz -C /opt

ln -s /opt/zookeeper-3.4.14 /usr/local/zookeeper
ln -s /opt/jdk1.8.0_201 /usr/local/jdk
ln -s /opt/kafka_2.11-2.4.0 /usr/local/kafka

vim /etc/profile
export JAVA_HOME=/usr/local/jdk
export PATH=$JAVA_HOME/bin:$PATH

source /etc/profile

cd /opt/zookeeper-3.4.14/conf
mv zoo_sample.cfg zoo.cfg
cp zoo.cfg zoo.cfg.bak

mkdir -p /usr/local/zookeeper/data
vim zoo.cfg
dataDir=/usr/local/zookeeper/data
#新增其他节点信息
server.1=192.168.203.52:2888:3888  
server.2=192.168.203.53:2888:3888
server.3=192.168.203.54:2888:3888
#给每台主机设置标识   每台主机不一样
echo "1" > /usr/local/zookeeper/data/myid
echo "2" > /usr/local/zookeeper/data/myid
echo "3" > /usr/local/zookeeper/data/myid

#启动zookeeper 每台都要
cd ../bin
./zkServer.sh start

ps -elf |grep zook #查看是否启动成功


#安装kafka
cd /usr/local/kafka/config
mkdir -p /usr/local/kafka/kafka-logs
vim server.properties
#新增
auto.create.topics.enable=true
delete.topic.enable=true
allow.everyone.if.no.acl.found=true

listeners=PLAINTEXT://hostname:9092
advertised.listeners=PLAINTEXT://hostname:9092
log.dirs=/usr/local/kafka/kafka-logs
zookeeper.connect=hostename:2181,hosstname:2181,hostname:2181
broker.id=0 #每台id要唯一 不可重复

cd ../bin
#首次启动用以下命令，查看有无报错
./kafka-server-start.sh ../config/server.properties    
#等待一会
#[2023-06-02 10:47:54,268] INFO Kafka startTimeMs: 1685674074262 (org.apache.kafka.common.utils.AppInfoParser)
#[2023-06-02 10:47:54,273] INFO [KafkaServer id=1] started (kafka.server.KafkaServer)
启动时出现以上提示说明部署没有问题，kafka正常启动，Ctrl+C，结束即可，结尾会出现如下提示
#[2023-06-02 10:48:57,617] INFO [SocketServer brokerId=1] Shutdown completed (kafka.network.SocketServer)
#[2023-06-02 10:48:57,624] INFO [KafkaServer id=1] shut down completed (kafka.server.KafkaServer)

#再次启动用以下命令，后台启动
nohup ./kafka-server-start.sh ../config/server.properties 1>/dev/null 2>&1 &

#查看日志有没有报错
cd ../logs
tail -f server.log

https://www.cnblogs.com/nbzws/p/15675402.html
#默认情况所有主机都操作
tar -zxf zookeeper-3.4.14.tar.gz -C /opt
tar -zxf jdk-8u201-linux-arm64-vfp-hflt.tar.gz -C /opt
tar -zxf kafka_2.11-2.4.0.tgz -C /opt

ln -s /opt/zookeeper-3.4.14 /usr/local/zookeeper
ln -s /opt/jdk1.8.0_201 /usr/local/jdk
ln -s /opt/kafka_2.11-2.4.0 /usr/local/kafka

vim /etc/profile
export JAVA_HOME=/usr/local/jdk
export PATH=$JAVA_HOME/bin:$PATH

source /etc/profile

cd /opt/zookeeper-3.4.14/conf
mv zoo_sample.cfg zoo.cfg
cp zoo.cfg zoo.cfg.bak

mkdir -p /usr/local/zookeeper/data
vim zoo.cfg
dataDir=/usr/local/zookeeper/data
#新增其他节点信息
server.1=192.168.203.52:2888:3888  
server.2=192.168.203.53:2888:3888
server.3=192.168.203.54:2888:3888
#给每台主机设置标识   每台主机不一样
echo "1" > /usr/local/zookeeper/data/myid
echo "2" > /usr/local/zookeeper/data/myid
echo "3" > /usr/local/zookeeper/data/myid

#启动zookeeper 每台都要
cd ../bin
./zkServer.sh start

ps -elf |grep zook #查看是否启动成功


#安装kafka
cd /usr/local/kafka/config
mkdir -p /usr/local/kafka/kafka-logs
vim server.properties
#新增
auto.create.topics.enable=true
delete.topic.enable=true
allow.everyone.if.no.acl.found=true

listeners=PLAINTEXT://hostname:9092
advertised.listeners=PLAINTEXT://hostname:9092
log.dirs=/usr/local/kafka/kafka-logs
zookeeper.connect=hostename:2181,hosstname:2181,hostname:2181
broker.id=0 #每台id要唯一 不可重复

cd ../bin
#首次启动用以下命令，查看有无报错
./kafka-server-start.sh ../config/server.properties    
#等待一会
#[2023-06-02 10:47:54,268] INFO Kafka startTimeMs: 1685674074262 (org.apache.kafka.common.utils.AppInfoParser)
#[2023-06-02 10:47:54,273] INFO [KafkaServer id=1] started (kafka.server.KafkaServer)
启动时出现以上提示说明部署没有问题，kafka正常启动，Ctrl+C，结束即可，结尾会出现如下提示
#[2023-06-02 10:48:57,617] INFO [SocketServer brokerId=1] Shutdown completed (kafka.network.SocketServer)
#[2023-06-02 10:48:57,624] INFO [KafkaServer id=1] shut down completed (kafka.server.KafkaServer)

#再次启动用以下命令，后台启动
nohup ./kafka-server-start.sh ../config/server.properties 1>/dev/null 2>&1 &

#查看日志有没有报错
cd ../logs
tail -f server.log

2.2测试

创建topic 就类似于一个组
./kafka-topics.sh --create --bootstrap-server master1:9092 --replication-factor 2 --partitions 3 --topic test
查看所有组
./kafka-topics.sh --list --bootstrap-server master1:9092
往这个组里写东西 输入这个命令给你返回一个交互的窗口
./kafka-console-producer.sh --broker-list  master1:9092  --topic test
〉 test 1

#区别的机器上试试能不能获取到刚写的 test 1
./kafka-console-consumer.sh --bootstrap-server master1:9092 --topic test --from-beginning
test 1
./kafka-console-consumer.sh --bootstrap-server master2:9092 --topic test --from-beginning
test 1
./kafka-console-consumer.sh --bootstrap-server node1:9092 --topic test --from-beginning
test 1

2.3编辑 crondns2


kubectl edit cm coredns -n kube-system

kubernetes { …  
}

# 新增

hosts {  
192.168.1.1 master1 fallthrough } kubectl delete pod coredns ```



## 2.5开机自启

vim /etc/rc.local

/opt/zookeeper-3.4.14/bin/zkServer.sh start

nohup /opt/kafka*2.11-2.4.0/bin/kafka-server-start.sh /opt/kafka*2.11-2.4.0/config/server.properties 1>/dev/null 2>&1 &
avatar
文章
阅读
粉丝