elasticsearch、kibana、filebeat、logstash环境搭建

已注销
93 阅读4分钟

虚拟机环境准备

  1. 采用vagrant搭建虚拟机环境，vagrant文件内容如下:
Vagrant.require_version ">= 1.6.0"
boxes = [
    {
        :name => "dawn-100",
        :eth1 => "192.168.10.100",
        :mem => "2048",
        :cpu => "2"
    },
    {
        :name => "dawn-101",
        :eth1 => "192.168.10.101",
        :mem => "2048",
        :cpu => "2" 
    },
    {
        :name => "dawn-102",
        :eth1 => "192.168.10.102",
        :mem => "2048",
        :cpu => "2" 
    }
]


Vagrant.configure("2") do |config|
  config.vm.box = "centos-7-empty"

  boxes.each do |opts|
    config.vm.define opts[:name] do |config|
      config.vm.hostname = opts[:name]
      # config.vm.provider "vmware_fusion" do |v|
      #   v.vmx["memsize"] = opts[:mem]
      #   v.vmx["numvcpus"] = opts[:cpu]
      # end
      config.vm.provider "virtualbox" do |v|
        v.customize ["modifyvm", :id, "--memory", opts[:mem]]
        v.customize ["modifyvm", :id, "--cpus", opts[:cpu]]
        v.name = opts[:name]
      end
      #config.vm.network "forwarded_port", guest: 3306, host: 3306
  	  #config.vm.network "forwarded_port", guest: 8080, host: 8080
      config.vm.network :private_network, ip: opts[:eth1]
    end
    config.vm.synced_folder "../share", "/share", create: true, owner: "root",group: "root"
  	# config.vm.provision "shell", privileged: true, path: "./setup.sh"
  end
end
  1. 修改软件源 mirrors.tuna.tsinghua.edu.cn/help/centos…
# 对于 CentOS 7
sed -e 's|^mirrorlist=|#mirrorlist=|g' \
    -e 's|^#baseurl=http://mirror.centos.org/centos|baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos|g' \
    -i.bak \
    /etc/yum.repos.d/CentOS-*.repo
  1. 修改sshd服务配置
sed -ri 's@^#UseDNS yes@UseDNS no@g' /etc/ssh/sshd_config
grep ^UseDNS /etc/ssh/sshd_config

sed -ri 's@^GSSAPIAuthentication yes@GSSAPIAuthentication no@g' /etc/ssh/sshd_config
grep ^GSSAPIAuthentication /etc/ssh/sshd_config

sed -ri 's/^#PasswordAuthentication yes/PasswordAuthentication yes/g' /etc/ssh/sshd_config
sed -ri 's/^PasswordAuthentication no/#PasswordAuthentication no/g' /etc/ssh/sshd_config
cat /etc/ssh/sshd_config |grep PasswordAuthentication

systemctl restart sshd
  1. 关闭防火墙
systemctl disable --now firewalld && systemctl is-enabled firewalld 
systemctl status firewalld
  1. 禁用selinux
sed -ri 's@(SELINUX=)enforcing@\1disabled@' /etc/selinux/config
grep ^SELINUX= /etc/selinux/config
setenforce 0
getenforce
  1. 配置免密登录
1. 修改主机列表
cat >> /etc/hosts <<'EOF' 
192.168.10.100 dawn100.dawn.com
192.168.10.101 dawn101.dawn.com
192.168.10.102 dawn102.dawn.com
EOF
2. dawn100主机上生成密钥对
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa -q

3. 配置所有集群节点的免密登录
for ((host_id=100;host_id<=102;host_id++));do 
ssh-copy-id dawn${host_id}.dawn.com ;done

4. 安装数据同步工具rsync
yum -y install rsync

5. 编写同步脚本
vim /usr/local/sbin/data_rsync.sh # 将下⾯的内容拷⻉到该⽂件即可

if [ $# -ne 1 ];then
    echo "Usage: $0 /path/to/file(绝对路径)"
    exit
fi

# 判断⽂件是否存在
if [ ! -e $1 ];then
    echo "[ $1 ] dir or file not find!"
    exit
fi

# 获取⽗路径
fullpath=`dirname $1`

# 获取⼦路径
basename=`basename $1`

# 进⼊到⽗路径
cd $fullpath
for ((host_id=100;host_id<=102;host_id++))
do
    # 使得终端输出变为绿⾊
    tput setaf 2
    echo ===== rsyncing dawn${host_id}.dawn.com: $basename =====

    # 使得终端恢复原来的颜⾊
    tput setaf 7
    # 将数据同步到其他两个节点
    rsync -az $basename `whoami`@dawn${host_id}.dawn.com:$fullpath
    if [ $? -eq 0 ];then
        echo "命令执⾏成功!"
    fi
done

6. 给脚本授权
chmod +x /usr/local/sbin/data_rsync.sh
  1. 配置集群时间
1. 安装net-tools网络管理软件包
yum install net-tools -y

2. 安装chrony时钟同步服务
yum -y install ntpdate chrony

3. 修改chrony配置文件
vim /etc/chrony.conf

# 注释官⽅的时间服务器，换成国内的时间服务器即可
server ntp.aliyun.com iburst
server ntp1.aliyun.com iburst
server ntp2.aliyun.com iburst
server ntp3.aliyun.com iburst
server ntp4.aliyun.com iburst
server ntp5.aliyun.com iburst

4. 设置chronyd开机自启
systemctl enable --now chronyd
systemctl restart chronyd
systemctl status chronyd

部署单节点elasticsearch

1. 下载elasticsearch
https://www.elastic.co/downloads

2. 创建如下文件夹,并解压下载的elasticsearch包
/usr/local/elasticsearch
/usr/local/elasticsearch/log
/usr/local/elasticsearch/data
/usr/local/elasticsearch/config/certs

tar -xzvf /share/elasticsearch-8.11.1-linux-x86_64.tar.gz -C /usr/local/elasticsearch

3. 创建es用户,修改/usr/local/elasticsearch文件的所有者
[root@dawn-102 bin]# useradd es
[root@dawn-102 bin]# passwd es
[root@dawn-102 local]# chown -R es:es elasticsearch/

4. 设置虚拟内存
# 在当前会话中修改
sysctl-w vm.max_map_count=262144 
# 修改配置文件并使配置生效
echo "vm.max_map_count=262144" >> /etc/sysctl.conf
sysctl -p

5. 设置文件打开数
cat >>/etc/security/limits.conf<<EOF
* soft nofile 65535
* hard nofile 65535
* soft nproc 65535
* hard nproc 65535
EOF

4. 修改elasticsearch.yml配置文件
egrep -v "^#|^$" /usr/local/elasticsearch/elasticsearch-8.11.1/config/elasticsearch.yml

# 集群名称,默认为elasticsearch，日志前缀也为集群名称
cluster.name: dawn-elastic-cluster
# 节点名称
node.name: elk102
path.data: /usr/local/elasticsearch/data
path.logs: /usr/local/elasticsearch/log
# ES服务监听的IP地址
network.host: 192.168.10.102
# 服务发现的主机列表，对于单点部署⽽⾔，主机列表和"network.host"字段配置相同即可。
discovery.seed_hosts: ["192.168.10.102"]
# 关闭安全认证：
xpack.security.enabled: false
# 单节点启动集群
discovery.type: single-node

5. 后台启动
./elasticsearch -d

部署elasticsearch集群

  1. 配置修改
# 修改dawn100配置
cluster.name: dawn-elastic-cluster
node.name: elk100
path.data: /usr/local/elasticsearch/data
path.logs: /usr/local/elasticsearch/log
network.host: _eth1_
discovery.seed_hosts: ["dawn100.dawn.com","dawn101.dawn.com","dawn102.dawn.com"]
cluster.initial_master_nodes: ["elk100","elk101","elk102"]
xpack.security.enabled: false

# 并同步到其他节点
data_rsync.sh /usr/local/elasticsearch/elasticsearch-8.11.1/config/elasticsearch.yml

# 修改dawn-101节点配置
node.name: elk101
# 修改dawn-102节点配置
node.name: elk102
  1. 启动服务
# 停止所有进程
pkill java
# 删除所有节点的临时数据
rm -rf /usr/local/elasticsearch/{log,data}/*
# 在每台节点上启动服务
./elasticsearch -d
  1. 检查服务是否启动正常
curl dawn100.dawn.com:9200/_cat/nodes?v

[es@dawn-100 bin]$ curl dawn100.dawn.com:9200/_cat/nodes?v
ip             heap.percent ram.percent cpu load_1m load_5m load_15m node.role   master name
192.168.10.102           62          93   1    0.00    0.03     0.10 cdfhilmrstw *      elk102
192.168.10.101           44          93   1    0.05    0.06     0.12 cdfhilmrstw -      elk101
192.168.10.100           44          92   1    0.00    0.04     0.10 cdfhilmrstw -      elk100

安装kibana

  1. 配置修改
# 解压
tar -zxvf /share/kibana-8.11.1-linux-x86_64.tar.gz -C /usr/local/kibana
# 更改所属用户
chown -R es:es /usr/local/kibana

# 修改如下配置
[root@dawn-100 config]# egrep -v "^#|^$" /usr/local/kibana/kibana-8.11.1/config/kibana.yml
server.host: "192.168.10.100"
server.name: "dawn-kibana-server"
elasticsearch.hosts: ["http://dawn100.dawn.com:9200","http://dawn101.dawn.com:9200","http://dawn102.dawn.com:9200"]
i18n.locale: "zh-CN"
  1. 启动验证
nohup ./kibana &

http://192.168.10.100:5601/

安装filebeate

  1. 配置修改
# 解压
tar -xzvf /share/filebeat-8.11.1-linux-x86_64.tar.gz -C /usr/local/filebeat
# 更改所属用户
chown -R es:es filebeat
# 创建软链接
ln -sv /usr/local/filebeat/filebeat-8.11.1-linux-x86_64 /usr/local/bin/
  1. 添加配置
cat > /usr/local/filebeat/config/hello.yml <<'EOF'
# 指定输入的类型
filebeat.inputs:
# 指定输入的类型为"stdin",表示标准输入
- type: stdin
# 指定输出的类型
output.console:
  # 格式化输出
  pretty: true
EOF
  1. 将"组"和"其他人"的写权限去掉
chmod go-w /usr/local/filebeat/config/hello.yml
  1. 运⾏filebeat实例
./filebeat -e -c /usr/local/filebeat/config/hello.yml

运行后在终端上输入任何消息，将会格式后回显到终端

安装Logstash

  1. 配置修改
# 解压
tar -xzvf logstash-8.11.1-linux-x86_64.tar.gz -C /usr/local/logstash

ln -sv /usr/local/logstash/logstash-8.11.1/bin/logstash /usr/local/bin/

# 更改所属用户
chown -R es:es logstash
  1. 测试
logstash -e "input {stdin {}} output {stdout{}}"
  1. 文件语法检查
logstash -rf /usr/local/logstash/conf/stdin-to-stdout.conf
  1. 启动logstash实例
logstash -f /usr/local/logstash/conf/stdin-to-stdout.conf
avatar
文章
阅读
粉丝