ELK日志平台搭建

方式一：CentoOS直接安装方式（版本6.8.16）

1、提前环境准备

• JDK1.8
• 创建ELK的文件夹专用用户（不能使用root用户）

# 创建文件
cd /usr/local/
mkdir elk

# 创建分组和用户
groupadd elk
adduser elk
useradd -g elk elk
passwd elk

# 把elk文件夹归属权给elk用户
chown elk:elk /usr/local/elk/

2、ES安装

• ES下载

# 下载ES
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.16.tar.gz
# 解压
tar -zxvf elasticsearch-6.8.16.tar.gz

• 配置文件编辑 config/elasticsearch.yml

network.host: 0.0.0.0
http.port: 9200

• 启动成功显示如下

[root@instance-g5pyqzzl elk]# curl http://127.0.0.1:9200
{
  "name" : "G4x8BMj",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "XRtAcyUQQry0Z5x9x2f-Ng",
  "version" : {
    "number" : "6.8.16",
    "build_flavor" : "default",
    "build_type" : "tar",
    "build_hash" : "1f62092",
    "build_date" : "2021-05-21T19:27:57.985321Z",
    "build_snapshot" : false,
    "lucene_version" : "7.7.3",
    "minimum_wire_compatibility_version" : "5.6.0",
    "minimum_index_compatibility_version" : "5.0.0"
  },
  "tagline" : "You Know, for Search"
}

• 可能遇到的错误

#  max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
# /etc/sysctl.conf 文件最后添加一行 vm.max_map_count=262144
# 立即生效
/sbin/sysctl -p

• 开启用户名密码验证

# 编辑/config/elasticsearch.yml 加入如下内容
xpack.security.enabled: true
xpack.license.self_generated.type: basic
xpack.security.transport.ssl.enabled: true

# 执行命令初始化密码 按照提示进行输入
./bin/elasticsearch-setup-passwords interactive

# 再次访问就需要密码才能访问了

3、Logstash安装

• Logstash下载

wget https://artifacts.elastic.co/downloads/logstash/logstash-6.8.16.tar.gz
# 解压
tar -zxvf logstash-6.8.16.tar.gz

• 如果ES开启了密码验证需要做如下配置

• logstash.yml配置

# 特别注意，这里的用户名权限有问题，可以修改权限，或者重新创建一个用户，在kibana上
http.host: "192.168.137.100"
# 如果ES开启了密码形式则需要配置
xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.username: logstash_system
xpack.monitoring.elasticsearch.password: elastic
xpack.monitoring.elasticsearch.hosts: ["https://192.168.137.100:9200"]

• logstash.conf配置

input {
  tcp {
    #模式选择为server
    mode => "server"
    #ip和端口根据自己情况填写，端口默认4560,对应下文logback.xml里appender中的destination
    host => "192.168.137.100"
    port => 4560
    #格式json
    codec => json_lines
    #如果开启了SSL则需要SSL验证
    ssl_enable => true
    ssl_verify => true
    ssl_certificate_authorities => ["/usr/local/elk/logstash-6.8.16/config/ssl/ca-dp.pem"]
    ssl_cert => "/usr/local/elk/logstash-6.8.16/config/ssl/server-cert-dp.pem"
    ssl_key => "/usr/local/elk/logstash-6.8.16/config/ssl/server-key-dp.pem"
  }
}
filter {
  #过滤器，根据需要填写
}
output {
  elasticsearch {
    action => "index"
    #这里是es的地址，多个es要写成数组的形式
    hosts  => "192.168.137.100:9200"
    #用于kibana过滤，可以填项目名称
    #index  => "applog"
    index => "springboot-%{[appname]}-%{+YYYY.MM.dd}"
    user => elastic
    password => elastic
  }
}

• SpringBoot logback.xml配置如下

<appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <destination>192.168.137.100:4560</destination>
        <!-- encoder必须配置,有多种可选 -->
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder">
                <customFields>{"appname":"demo"}</customFields>
        </encoder>
        # 如果logstash开启了ssl则需要生成响应证书
        <ssl>
                <keyStore>
                        <location>classpath:/keystore.jks</location>
                        <password>123456</password>
                </keyStore>
                <trustStore>
                        <location>classpath:/truststore.jks</location>
                        <password>123456</password>
                </trustStore>
        </ssl>
</appender>

4、kibana安装

• kibana下载

wget https://artifacts.elastic.co/downloads/kibana/kibana-6.8.16-linux-x86_64.tar.gz
# 解压

• 配置

vim kibana.yml
#修改如下配置
server.host: "192.168.137.100"
server.name: "192.168.137.100"
elasticsearch.hosts: ["http://192.168.137.100:9200"]
# 如果有ES有密码则需要配置密码
elasticsearch.username: "kibana"
elasticsearch.password: "elastic"

5、后续日志查看配置

• 查看有多少索引

• 创建查询集

• 正常使用

• 用户管理

ES curl操作说明

6、logstash证书和logback证书生成

# 一键生成脚本 auto.sh

#!/bin/bash
# 以下是配置信息
# --[BEGIN]------------------------------

CODE="dp"
IP="docker服务器ip"
PASSWORD="证书密码"
COUNTRY="CN"
STATE="BEIJING"
CITY="BEIJING"
ORGANIZATION="公司"
ORGANIZATIONAL_UNIT="Dev"
COMMON_NAME="$IP"
EMAIL="邮箱"

# --[END]--

# Generate CA key
openssl genrsa -aes256 -passout "pass:$PASSWORD" -out "ca-key-$CODE.pem" 4096
# Generate CA
openssl req -new -x509 -days 365 -key "ca-key-$CODE.pem" -sha256 -out "ca-$CODE.pem" -passin "pass:$PASSWORD" -subj "/C=$COUNTRY/ST=$STATE/L=$CITY/O=$ORGANIZATION/OU=$ORGANIZATIONAL_UNIT/CN=$COMMON_NAME/emailAddress=$EMAIL"
# Generate Server key
openssl genrsa -out "server-key-$CODE.pem" 4096

# Generate Server Certs.
openssl req -subj "/CN=$COMMON_NAME" -sha256 -new -key "server-key-$CODE.pem" -out server.csr

echo "subjectAltName = IP:$IP,IP:127.0.0.1" >> extfile.cnf
echo "extendedKeyUsage = serverAuth" >> extfile.cnf

openssl x509 -req -days 365 -sha256 -in server.csr -passin "pass:$PASSWORD" -CA "ca-$CODE.pem" -CAkey "ca-key-$CODE.pem" -CAcreateserial -out "server-cert-$CODE.pem" -extfile extfile.cnf


# Generate Client Certs.
rm -f extfile.cnf

openssl genrsa -out "key-$CODE.pem" 4096
openssl req -subj '/CN=client' -new -key "key-$CODE.pem" -out client.csr
echo extendedKeyUsage = clientAuth >> extfile.cnf
openssl x509 -req -days 365 -sha256 -in client.csr -passin "pass:$PASSWORD" -CA "ca-$CODE.pem" -CAkey "ca-key-$CODE.pem" -CAcreateserial -out "cert-$CODE.pem" -extfile extfile.cnf

rm -vf client.csr server.csr

chmod -v 0400 "ca-key-$CODE.pem" "key-$CODE.pem" "server-key-$CODE.pem"
chmod -v 0444 "ca-$CODE.pem" "server-cert-$CODE.pem" "cert-$CODE.pem"

################下面单独执行
# Generate keystore.jks

# 下载工具包
wget https://biteeniu.github.io/files/jetty-6.1.26.jar

openssl pkcs12 -export -out cert.p12 -in cert-dp.pem -inkey key-dp.pem


java -cp jetty-6.1.26.jar org.mortbay.jetty.security.PKCS12Import cert.p12 keystore.jks


# Generate truststore.jks
keytool -import -file ca-$CODE.pem -keystore truststore.jks

# 查看基本信息
curl --user elastic:elastic http://127.0.0.1:9200/