kafka3.7.2集群搭建及SASL认证

==阅读前说明==

• 下面用到的confi和bin目录都是相对于kafka目录来进行的
• ./xxx.sh都是相对kafka/bin目录执行的
• 请自行将我的kafka地址替换为你的正确地址包括执行命令时所使用到的地址.

1. 修改config/kraft/server.properties

############################# Server Basics #############################

# The role of this server. Setting this puts us in KRaft mode
#角色
process.roles=broker,controller

# The node id associated with this instance's roles
#id 这个id必须要和controller.quorum.voters的对应上
#比如这个controller.quorum.voters配到了3,就必须有三台分别配置了node.id=1,node.id=2,node.id=3
node.id=1

# The connect string for the controller quorum
controller.quorum.voters=1@192.168.1.11:19093,2@192.168.1.12:19093,3@192.168.1.13:19093

############################# Socket Server Settings #############################

# 这个地方要改成当前节点的IP地址
listeners=SASL_PLAINTEXT://192.168.1.11:19092,CONTROLLER://192.168.1.11:19093
# Name of listener used for communication between brokers.
inter.broker.listener.name=SASL_PLAINTEXT


# 这个地方要改成当前节点的IP地址
advertised.listeners=SASL_PLAINTEXT://192.168.1.11:19092

controller.listener.names=CONTROLLER

# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
#CONTROLLER:SASL_PLAINTEXT需要修改
listener.security.protocol.map=CONTROLLER:SASL_PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
# 设置必须授权才能用
allow.everyone.if.no.acl.found=false
# The number of threads that the server uses for receiving requests from the network and sending responses to the network
num.network.threads=3

# The number of threads that the server uses for processing requests, which may include disk I/O
num.io.threads=8

# The send buffer (SO_SNDBUF) used by the socket server
socket.send.buffer.bytes=102400

# The receive buffer (SO_RCVBUF) used by the socket server
socket.receive.buffer.bytes=102400

# The maximum size of a request that the socket server will accept (protection against OOM)
socket.request.max.bytes=104857600


############################# Log Basics #############################

# A comma separated list of directories under which to store log files
# 这个路径很重要
log.dirs=/data/kafka_3.7.2.sasl/logs


num.partitions=1

num.recovery.threads.per.data.dir=1


offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1

############################# Log Flush Policy #############################


log.retention.hours=168

# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
#log.retention.bytes=1073741824

# The maximum size of a log segment file. When this size is reached a new log segment will be created.
log.segment.bytes=1073741824

# The interval at which log segments are checked to see if they can be deleted according
# to the retention policies
log.retention.check.interval.ms=300000
# 认证方式，用了最简单的PLAIN，缺点是不能动态添加用户
sasl.mechanism.inter.broker.protocol=PLAIN
sasl.enabled.mechanisms=PLAIN
sasl.mechanism=PLAIN
# 禁用了自动创建topic
auto.create.topics.enable = false
# 设置必须授权才能用
allow.everyone.if.no.acl.found=false
# 设置超级管理员
super.users=User:admin
# 这个是3.2.0版本新引入的认证方式，可以参考 https://cwiki.apache.org/confluence/display/KAFKA/KIP-801%3A+Implement+an+Authorizer+that+stores+metadata+in+__cluster_metadata
# 这个脱离了使用zookeeper进行认证
authorizer.class.name=org.apache.kafka.metadata.authorizer.StandardAuthorizer
# 集群间认证时用的认证方式
sasl.mechanism.controller.protocol=PLAIN

1.2 注意事项

下面这三个配置每一个节点都要改,其他的不用改

# 这个每一个节点都要不一样,要注意
node.id=1

############################# Socket Server Settings #############################

# 这个地方要改成当前节点的IP地址
listeners=SASL_PLAINTEXT://192.168.1.11:19092,CONTROLLER://192.168.1.11:19093



# 这个地方要改成当前节点的IP地址
advertised.listeners=SASL_PLAINTEXT://192.168.1.11:19092

2. 在config下新增两个文件

KafkaServer {
org.apache.kafka.common.security.plain.PlainLoginModule required
username="admin"
password="password"
# 这个是必须有的,哪怕是要创建多个账号来做读写账号分离这个也要
# user_是固定前缀, admin是账号名,后面的是密码
# 如果需要多个账号,和权限分离看最下面
user_admin="password";

};

3.kafka_client_jaas.properties 客户端用来登录用的

# 指定 SASL 认证机制和协议
security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN

# 配置用户名和密码（替换为你的实际用户名和密码）
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
    username="admin" \
    password="password";

3. 修改bin/kafka-server-start.sh 在29行左右

# 所有节点都要改
export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G -Djava.security.auth.login.config=/data/kafka_3.7.2.sasl/config/kafka_server_jaas.conf"

4. 初始化数据目录 数据目录为log.dirs的值

bin/kafka-storage.sh random-uuid
# 会输出a1HxK892T6iV3wOHuHm1BA类似的一个随机ID,直接用这个也可以
# 在所有节点都执行一遍下面这句,会创建log.dirs指定的目录并初始化内容
bin/kafka-storage.sh format -t a1HxK892T6iV3wOHuHm1BA -c /data/kafka_3.7.2.sasl/config/kraft/server.properties

5. 依次启动节点

nohup /data/kafka_3.7.2.sasl/bin/kafka-server-start-sasl.sh /data/kafka_3.7.2.sasl/config/kraft/server.properties >/data/kafka_3.7.2.sasl/kfk.out 2>&1 &

6. 校验

# 创建topic
./kafka-topics.sh --bootstrap-server 192.168.1.11:19092 --create --topic test1 --partitions 3 --replication-factor 2 --command-config ../config/kafka_client_jaas.properties

# 查看所有topic
./kafka-topics.sh --bootstrap-server 192.168.1.11:19092 --list --command-config ../config/kafka_client_jaas.properties

# 生产者
./kafka-console-producer.sh broker-list --bootstrap-server 192.168.1.11:19092   --topic test1  --producer.config ../config/kafka_client_jaas.properties

# 消费者
./kafka-console-consumer.sh broker-list --bootstrap-server 192.168.1.11:19092   --topic test1  --consumer.config ../config/kafka_client_jaas.properties

补充

1. 多账号及权限设置

KafkaServer {
    org.apache.kafka.common.security.plain.PlainLoginModule required
    username="admin"
    password="admin的密码"
    user_admin="admin的密码"
    user_a="a的密码" # 读
    user_b="b的密码"; # 写
};

# 给指定主题指定用户设置读写权限
./kafka-acls.sh --bootstrap-server 192.168.1.11:19092 --add --allow-principal User:a --operation Read --topic test1 --command-config ../config/kafka_client_jaas.properties
./kafka-acls.sh --bootstrap-server 192.168.1.11:19092 --add --allow-principal User:b --operation Read --operation Write --topic test1 --command-config ../config/kafka_client_jaas.properties

# 删除指定主题指定用户的读权限
./kafka-acls.sh --bootstrap-server 192.168.1.11:19092 --remove --allow-principal User:a --operation Read --topic test1 --command-config ../config//data/kafka_3.7.2.sasl

# 查看已有的 ACL 配置
./kafka-acls.sh --bootstrap-server 192.168.1.11:19092 --list --command-config ../config/kafka_client_jaas.properties

# 查看特定主题的 ACL 配置
./kafka-acls.sh --bootstrap-server 192.168.1.11:19092 --list --topic test1 --command-config /data/kafka_3.7.2.sasl/config/kafka_client_jaas.properties

# 查看特定用户的 ACL 配置：
./kafka-acls.sh --bootstrap-server 192.168.1.11:19092 --list --principal User:a --command-config /data/kafka_3.7.2.sasl/config/kafka_client_jaas.properties

2. java8生产消费

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;

import java.time.Duration;
import java.util.Collections;
import java.util.Properties;

public class Main {
    public static void main(String[] args) {
        producer();	# 生产要确认主题是否存在
//      consumer();
    }

    private static void producer(){
        KafkaProducer<String, String> producer = createProducer();
        // 发送消息到 Kafka
        producer.send(new ProducerRecord<>("test1", "key", "value"), (metadata, exception) -> {
            if (exception != null) {
                exception.printStackTrace();
            } else {
                System.out.println("Message sent to topic " + metadata.topic() + " with offset " + metadata.offset());
            }
        });
        producer.flush();
        producer.close();
    }

    private static void consumer(){
        KafkaConsumer<String, String> consumer = createConsumer();

        consumer.subscribe(Collections.singletonList("test1"));

        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
            for (ConsumerRecord<String, String> record : records) {
                System.out.println("Consumed message: " + record.value() + " from topic: " + record.topic());
            }
        }
    }


    public static KafkaProducer<String, String> createProducer() {
        Properties props = new Properties();

        // Kafka broker地址
        props.put("bootstrap.servers", "192.168.1.11:19092");

        // 使用SASL认证
        props.put("security.protocol", "SASL_PLAINTEXT"); // 或者 SASL_SSL 如果你启用了SSL
        props.put("sasl.mechanism", "PLAIN"); // 或 SCRAM-SHA-256, SCRAM-SHA-512

        // 指定JAAS配置文件路径
        props.put("sasl.jaas.config",
                "org.apache.kafka.common.security.plain.PlainLoginModule required " +
                        "username=\"admin\" password=\"password\";");

        // 生产者设置
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

        return new KafkaProducer<>(props);
    }

    public static KafkaConsumer<String, String> createConsumer() {
        Properties props = new Properties();

        // Kafka broker地址
        props.put("bootstrap.servers", "192.168.1.11:19092");

        // 使用SASL认证
        props.put("security.protocol", "SASL_PLAINTEXT"); // 或者 SASL_SSL 如果你启用了SSL
        props.put("sasl.mechanism", "PLAIN"); // 或 SCRAM-SHA-256, SCRAM-SHA-512

        // 指定JAAS配置文件路径
        props.put("sasl.jaas.config",
                "org.apache.kafka.common.security.plain.PlainLoginModule required " +
                        "username=\"admin\" password=\"password\";");

        // 消费者设置
        props.put(ConsumerConfig.GROUP_ID_CONFIG, "test-consumer-group");
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());

        return new KafkaConsumer<>(props);
    }
}