ELK 相关配置

filebeat

# 从日志文件输入日志
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /usr/local/logs/*.log
  # 定义日志标签
  tags: ["user-log"]
- type: log
  enabled: true
  paths:
    - /var/log/nginx/api/*.log
  tags: ["nginx-api"]
setup.template.settings: 
  # 设置主分片数
  index.number_of_shards: 1 
  # 因为测试环境只有一个es节点，所以将副本分片设置为0，否则集群会报黄 
  index.number_of_replicas: 0 
# 输出到logstash 
output.logstash: 
  # logstash所在服务器的ip和端口 
  hosts: ["localhost:5044"] 
  # 默认配置，不做改动 
processors: 
  - add_host_metadata: 
    when.not.contains.tags: forwarded 
  - add_cloud_metadata: ~ 
  - add_docker_metadata: ~ 
  - add_kubernetes_metadata: ~

logstash

input {
  beats {
    port => 5044
  }
}

filter {
  grok {
    match => { "message" => "\[%{DATA:time}\] %{WORD:log_level} - %{IPORHOST:remote_ip} \"%{WORD:method} %{URIPATHPARAM:url} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:body_sent:bytes} %{BASE10NUM:request_time} \"%{DATA:referrer}\" \"%{DATA:agent}\""}
    remove_field => "message"
  }
}

output {
  if "user-log" in [tags]{
    elasticsearch {
      hosts => ["http://36.140.183.90:9200"]
      index => "[user-log]-%{+YYYY.MM.dd}"
    }
  }
  if "nginx-api" in [tags]{
    elasticsearch {
      hosts => ["http://36.140.183.90:9200"]
      index => "[nginx-api]-%{+YYYY.MM.dd}"
    }
  }
}

nginx 日志格式

log_format api '[$time_zh_ms] $log_level - $remote_addr "$request" '
            '$status $body_bytes_sent $request_time "$http_referer" '
            '"$http_user_agent" "$http_x_forwarded_for"';

elasticsearch

cluster.name: cluster1
cluster.initial_master_nodes: ["node-1"]

node.name: node-1
node.roles: [master,data,remote_cluster_client]

path.data: usr/local/elasticsearch_data
path.logs: usr/local/elasticsearch_logs

network.host: 0.0.0.0
http.cors.enabled: true
http.cors.allow-origin: "*"

nginx 实际打印日志

[2024-01-04 17:01:28.170] ERROR - 58.32.236.249 "POST /api/users/overTime HTTP/1.1" 499 0 5.000 "http://101.37.255.224/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36" "-"

解决filebeat 自动暂停

systemctl start filebeat

[Unit]
Description=Filebeat is a lightweight shipper for metrics.
Documentation=https://www.elastic.co/products/beats/filebeat
Wants=network-online.target
After=network-online.target

[Service]
Environment="LOG_OPTS=-e"
Environment="CONFIG_OPTS=-c /usr/local/filebeat/filebeat.yml"
Environment="PATH_OPTS=-path.home /usr/local/filebeat/bin/filebeat -path.config /usr/local/filebeat/bin -path.data /usr/local/filebeat/data -path.logs /usr/local/filebeat/logs"
ExecStart=/usr/local/filebeat/filebeat $LOG_OPTS $CONFIG_OPTS $PATH_OPTS
Restart=always

[Install]
WantedBy=multi-user.target