1.Es集群,准备三台机器
| 名称 | 机器ip |
|---|---|
| es-node1 | 192.168.193.1 |
| es-node2 | 192.168.193.2 |
| es-node | 192.168.193.3 |
| 2.es-node1配置文件: |
cluster.name: elasticsearch-cluster #集群名称
node.name: es-node1 #节点名称
network.bind_host: 0.0.0.0 #可访问节点的地址
network.publish_host: 192.168.193.1
http.port: 9200
transport.tcp.port: 9300 #集群节点通信
http.cors.enabled: true
http.cors.allow-origin: "*"
xpack.security.enabled: true
xpack.security.authc.accept_default_password: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/elastic-certificates.p12
node.master: true #是否主节点
node.data: true #是否数据节点
discovery.zen.ping.unicast.hosts: ["192.168.193.1:9300","192.168.193.2:9300","192.168.193.3:9300"]
discovery.zen.minimum_master_nodes: 2 # 配置当前集群中最少具有 master 资格节点数，对于多于两个节点的集群环境，建议配置大于1
2.es-node2配置文件:
cluster.name: elasticsearch-cluster
node.name: es-node2
network.bind_host: 0.0.0.0
network.publish_host: 192.168.193.2
http.port: 9200
transport.tcp.port: 9300
http.cors.enabled: true
http.cors.allow-origin: "*"
xpack.security.enabled: true
xpack.security.authc.accept_default_password: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/elastic-certificates.p12
node.master: true
node.data: true
discovery.zen.ping.unicast.hosts: ["192.168.193.1:9300","192.168.193.2:9300","192.168.193.3:9300"]
discovery.zen.minimum_master_nodes: 2
3.es-node3配置文件:
cluster.name: elasticsearch-cluster
node.name: es-node3
network.bind_host: 0.0.0.0
network.publish_host: 192.168.193.3
http.port: 9200
transport.tcp.port: 9300
http.cors.enabled: true
http.cors.allow-origin: "*"
xpack.security.enabled: true
xpack.security.authc.accept_default_password: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/elastic-certificates.p12
xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/elastic-certificates.p12
node.master: false
node.data: true
discovery.zen.ping.unicast.hosts: ["192.168.193.1:9300","192.168.193.2:9300","192.168.193.3:9300"]
discovery.zen.minimum_master_nodes: 1
~
4.kibana配置文件
# Kibana由后端服务器提供服务。此设置指定要使用的端口。
server.port: 5601
#
# # Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
# #指定Kibana服务器要绑定的地址。IP地址和主机名都是有效值。
# # The default is 'localhost', which usually means remote machines will not be able to connect.
# #默认是“localhost”，这通常意味着远程机器无法连接。
# # To allow connections from remote users, set this parameter to a non-loopback address.
# #若要允许来自远程用户的连接，请将此参数设置为非环回地址。
server.host: "0.0.0.0"
#
# # Enables you to specify a path to mount Kibana at if you are running behind a proxy.
# #允许您指定mount Kibana的路径，如果您在代理后运行。
# # Use the `server.rewriteBasePath` setting to tell Kibana if it should remove the basePath
# #使用“服务器”。重写basePath的设置，告诉Kibana是否应该移除basePath
# # from requests it receives, and to prevent a deprecation warning at startup.
# #，并防止启动时出现弃用警告。
# # This setting cannot end in a slash.
# #此设置不能以斜杠结束。
# #server.basePath: ""
#
# # Specifies whether Kibana should rewrite requests that are prefixed with
# #指定Kibana是否应该重写带前缀的请求
# # `server.basePath` or require that they are rewritten by your reverse proxy.
# #服务器或要求它们由反向代理重写。
# # This setting was effectively always `false` before Kibana 6.3 and will
# #在Kibana 6.3和will之前，这个设置实际上总是“false”
# # default to `true` starting in Kibana 7.0.
# #Kibana 7.0默认为“true”。
# #server.rewriteBasePath: false
#
# # The maximum payload size in bytes for incoming server requests.
# #传入服务器请求的最大负载大小(以字节为单位)。
# #server.maxPayloadBytes: 1048576
#
# # The Kibana server's name. This is used for display purposes.
# #Kibana服务器的名字。它用于显示目的。
# #server.name: "your-hostname"
#
# # The URLs of the Elasticsearch instances to use for all your queries.
# #用于所有查询的Elasticsearch实例的url。
elasticsearch.hosts: ["http://192.168.193.1:9200","http://192.168.193.2:9200","http://192.168.193.3:9200"]
xpack.monitoring.ui.container.elasticsearch.enabled: true
#
# # When this setting's value is true Kibana uses the hostname specified in the server.host
# #当此设置的值为true时，Kibana使用server.host中指定的主机名
# # setting. When the value of this setting is false, Kibana uses the hostname of the host
# #设置。当该设置的值为false时，Kibana使用主机的主机名
# # that connects to this Kibana instance.
# #连接到这个Kibana实例。
# #elasticsearch.preserveHost: true
#
# # Kibana uses an index in Elasticsearch to store saved searches, visualizations and
# #Kibana在Elasticsearch中使用索引来存储已保存的搜索、可视化和
# # dashboards. Kibana creates a new index if the index doesn't already exist.
# #仪表盘。如果索引不存在，Kibana会创建一个新索引。
# #kibana.index: ".kibana"
#
# # The default application to load.
# #要加载的默认应用程序。
# #kibana.defaultAppId: "home"
#
# # If your Elasticsearch is protected with basic authentication, these settings provide
# #如果您的Elasticsearch被基本身份验证保护，这些设置提供
# # the username and password that the Kibana server uses to perform maintenance on the Kibana
# #Kibana服务器用于对Kibana执行维护的用户名和密码
# # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
# #启动时的索引。Kibana用户仍然需要使用Elasticsearch进行身份验证
# # is proxied through the Kibana server.
# #通过Kibana服务器代理。
elasticsearch.username: "kibana"
elasticsearch.password: "123456"
#
# # Enables SSL and paths to the PEM-format SSL certificate and SSL key files, respectively.
# #分别启用SSL和PEM-format SSL证书和SSL密钥文件的路径。
# # These settings enable SSL for outgoing requests from the Kibana server to the browser.
# ##这些设置为Kibana服务器向浏览器发出的请求启用了SSL。
# #server.ssl.enabled: false
# #server.ssl.certificate: /path/to/your/server.crt
# #server.ssl.key: /path/to/your/server.key
#
# # Optional settings that provide the paths to the PEM-format SSL certificate and key files.
# #可选设置，提供通往PEM-format SSL证书和密钥文件的路径。
# # These files are used to verify the identity of Kibana to Elasticsearch and are required when
# #这些文件用于验证Kibana的身份，以Elasticsearch和需要
# # xpack.security.http.ssl.client_authentication in Elasticsearch is set to required.
# #xpack.security.http.ssl。在Elasticsearch中将client_authentication设置为required。
# #elasticsearch.ssl.certificate: /path/to/your/client.crt
# #elasticsearch.ssl.key: /path/to/your/client.key
#
# # Optional setting that enables you to specify a path to the PEM file for the certificate
# #可选设置，允许您指定证书的PEM文件的路径
# # authority for your Elasticsearch instance.
# #为您的Elasticsearch实例设置#权限。
# #elasticsearch.ssl.certificateAuthorities: [ "/path/to/your/CA.pem" ]
#
# # To disregard the validity of SSL certificates, change this setting's value to 'none'.
# #若要忽略SSL证书的有效性，请将此设置的值更改为“none”。
# #elasticsearch.ssl.verificationMode: full
#
# # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
# #等待Elasticsearch响应ping的时间(毫秒)。默认值
# # the elasticsearch.requestTimeout setting.
# #elasticsearch。requestTimeout设置。
# #elasticsearch.pingTimeout: 1500
#
# # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
# #等待后端或Elasticsearch响应的时间(毫秒)。这个值
# # must be a positive integer.
# #必须是正整数。
# #elasticsearch.requestTimeout: 30000
#
# # List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
# #Kibana客户端头的列表发送到Elasticsearch。发送*no*客户端
# # headers, set this value to [] (an empty list).
# #headers，将此值设置为[](空列表)。
# #elasticsearch.requestHeadersWhitelist: [ authorization ]
#
# # Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten
# #发送到Elasticsearch的标题名称和值。不能覆盖任何自定义标头
# # by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration.
# #由客户端头，而不考虑elasticsearch。requestHeadersWhitelist配置。
# #elasticsearch.customHeaders: {}
#
# # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
# #Elasticsearch等待碎片响应的时间(毫秒)。设置为0为禁用。
# #elasticsearch.shardTimeout: 30000
#
# # Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying.
# #在Kibana启动之前等待Elasticsearch的时间(毫秒)。
# #elasticsearch.startupTimeout: 5000
#
# # Logs queries sent to Elasticsearch. Requires logging.verbose set to true.
# #记录发送到Elasticsearch的查询。需要记录。verbose设置为true
# #elasticsearch.logQueries: false
#
# # Specifies the path where Kibana creates the process ID file.
# #指定Kibana创建进程ID文件的路径
# #pid.file: /var/run/kibana.pid
#
# # Enables you specify a file where Kibana stores log output.
# #允许指定Kibana存储日志输出的文件
# #logging.dest: stdout
#
# # Set the value of this setting to true to suppress all logging output.
# #将此设置的值设置为true以禁止所有日志记录输出。
# #logging.silent: false
#
# # Set the value of this setting to true to suppress all logging output other than error messages.
# #将此设置的值设置为true以禁止除错误消息外的所有日志记录输出。
# #logging.quiet: false
#
# # Set the value of this setting to true to log all events, including system usage information
# #将此设置的值设置为true以记录所有事件，
# # and all requests.
# #包括系统使用信息
# #logging.verbose: false
#
# # Set the interval in milliseconds to sample system and process performance
# #设置以毫秒为单位的时间间隔来采样系统和进程性能
# # metrics. Minimum is 100ms. Defaults to 5000.
# #指标。最低是100 ms。默认为5000。
# #ops.interval: 5000
#
# # Specifies locale to be used for all localizable strings, dates and number formats.
# #指定可用于所有可本地化字符串、日期和数字格式的语言环境。
# # Supported languages are the following: English - en , by default , Chinese - zh-CN .
# #支持的语言如下:English - en，默认情况下，Chinese - chu - cn。
i18n.locale: "zh-CN"
5.logstash.yml配置文件
#配置详情
http.host: "0.0.0.0"
xpack.monitoring.enabled: "true"
xpack.monitoring.elasticsearch.hosts: ["http://192.168.193.1:9200","http://192.168.193.2:9200","http://192.168.193.3:9200"]
xpack.monitoring.elasticsearch.username: "elastic"
xpack.monitoring.elasticsearch.password: "123456"
logstash.conf配置
input {
beats {
host => "0.0.0.0"
port => 5044
}
}
filter {
grok {
match => {
"message" => "%{IPV4:userIP} - - \[%{HTTPDATE:timestamp}\] %{IPV4:ip} %{QUOTEDSTRING:mapping} %{INT:code} %{INT:size} %{BASE10NUM:resTime} %{QUOTEDSTRING:url} %{QUOTEDSTRING:info} - - -"
}
overwrite => ["message"]
remove_field => ["message"]
}
date {
match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
locale => "en"
target => "@timestamp"
timezone => "Asia/Shanghai"
}
}
output {
elasticsearch {
hosts => ["http://192.168.193.1:9200","http://192.168.193.2:9200","http://192.168.193.3:9200"]
index => "access-%{+yyyy.MM.dd}"
user => elastic
password => "123456"
}
}
6.filebeat.yml配置文件
filebeat.inputs:
- type: log
paths:
- /mnt/ldata/data/apisix-package/compose/apisix_log/access.log
tags: ["access"]
fields:
index: 'access'
filetype: access
output.logstash:
hosts: ["/192.168.193.1:5044"]
7.部署docker-compose
version: '3'
services:
elasticsearch:
image: elasticsearch:7.9.2
container_name: elasticsearch
environment:
- "ES_JAVA_OPTS=-Xms32g -Xmx32g"
volumes:
- /root/elk/elasticsearch/plugins:/usr/share/elasticsearch/plugins
- /mnt/ldata/elasticsearch/data:/usr/share/elasticsearch/data
- /root/elk/elasticsearch/conf/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- /root/elk/elasticsearch/cert/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12
ports:
- 9222:9200
- 9300:9300
kibana:
image: kibana:7.9.2
container_name: kibana
links:
- elasticsearch:es
depends_on:
- elasticsearch
environment:
- "elasticsearch.hosts=http://es:9222"
- "i18n.locale=zh-CN"
volumes:
- /root/elk/kibana/conf/kibana.yml:/usr/share/kibana/config/kibana.yml
ports:
- 5601:5601
logstash:
image: logstash:7.9.2
container_name: logstash
links:
- elasticsearch:es
depends_on:
- elasticsearch
volumes:
- /root/elk/logstash/conf/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
- /root/elk/logstash/conf/logstash.yml:/usr/share/logstash/config/logstash.yml
ports:
- 5044:5044
7.es-node2部署docker-compose
version: '3'
services:
elasticsearch:
image: elasticsearch:7.9.2
container_name: elasticsearch
environment:
- "ES_JAVA_OPTS=-Xms32g -Xmx32g"
volumes:
- /root/elk/elasticsearch/plugins:/usr/share/elasticsearch/plugins
- /mnt/ldata/elasticsearch/data:/usr/share/elasticsearch/data
- /root/elk/elasticsearch/conf/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- /root/elk/elasticsearch/cert/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12
ports:
- 9222:9200
- 9300:9300
8.es-node3部署docker-compose
version: '3'
services:
elasticsearch:
image: elasticsearch:7.9.2
container_name: elasticsearch
environment:
- "ES_JAVA_OPTS=-Xms32g -Xmx32g"
volumes:
- /root/elk/elasticsearch/plugins:/usr/share/elasticsearch/plugins
- /mnt/ldata/elasticsearch/data:/usr/share/elasticsearch/data
- /root/elk/elasticsearch/conf/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
- /root/elk/elasticsearch/cert/elastic-certificates.p12:/usr/share/elasticsearch/config/elastic-certificates.p12
ports:
- 9222:9200
- 9300:9300
9.es集群配置x-pack
#查看启动的容器
docker ps
#进入其中一个容器
docker exec -it 容器ID或名称 /bin/bash
#创建证书颁发机构
bin/elasticsearch-certutil ca
#生成证书和私钥
bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12
#退出容器
exit
#从容器中复制证书到本地/home/admin/es/config
docker cp 容器ID:/usr/share/elasticsearch/elastic-certificates.p12 /home/admin/es/config
#停掉原本的es docker stop 容器ID或名称
#移除原本的es docker rm 容器ID或名称
10.filebeat安装部署:
