一、Logback整合Elasticsearch

• 基础环境信息参考上一节文章配置好

1. 添加logback和Elasticsearch整合日志依赖
   • 源码地址：gitee.com/tianxincoor…

<dependency>
  <groupId>com.internetitem</groupId>
  <artifactId>logback-elasticsearch-appender</artifactId>
  <version>1.6</version>
</dependency>

2. 创建定时日志类，用于定时写入日志，同时使用MDC记录需要核心信息

import java.time.LocalDateTime;
import java.util.Random;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

@EnableScheduling
@Component
public class LogstashLogbackAggregation {

    private static final Logger LOGGER = LoggerFactory.getLogger(LogstashLogbackAggregation.class);
    private static final Random RANDOM = new Random();

    @Scheduled(cron = "0/3 * * * * ?")
    public void logbackMdc() {
        MDC.put("code", UUID.randomUUID().toString());
        MDC.put("createTime", LocalDateTime.now().toString());
        MDC.put("content", "logstash日志采集错误" + LocalDateTime.now());
        LOGGER.info("logstash日志采集，当前时间【{}】", LocalDateTime.now());
        MDC.clear();
        System.out.println(String.format("logstash日志采集，当前时间【%s】", LocalDateTime.now()));
    }
}

3. 添加logback配置文件配置es相关信息

<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="true" scan="true" scanPeriod="1 seconds">

    <contextName>logback</contextName>
    <!--定义参数,后面可以通过${name}使用-->
    <property name="log_pattern" value="%d{HH:mm:ss.SSS} %-5level [%thread] %logger{36} - %msg%n%ex"/>

    <!-- 控制台输出-->
    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
            <level>INFO</level>
        </filter>
        <encoder>
            <pattern>${log_pattern}</pattern>
        </encoder>
    </appender>

    <!-- Elasticsearch&Logback整合配置 -->
    <appender name="LOGSTASH_MDC" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
        <url>http://127.0.0.1:9200/_bulk</url>
        <index>logstash-logs{yyyy-MM}</index>
        <errorLoggerName>es-error</errorLoggerName>
        <connectTimeout>60000</connectTimeout>
        <authentication class="com.internetitem.logback.elasticsearch.config.BasicAuthentication"/>
        <!-- 配置属性 -->
        <properties>
            <!-- 内置参数 -->
            <property>
                <name>host</name>
                <value>${HOSTNAME}</value>
            </property>
            <!-- MDC参数 -->
            <property>
                <name>code</name>
                <value>%X{code}</value>
            </property>
            <property>
                <name>createTime</name>
                <value>%X{createTime}</value>
            </property>
            <property>
                <name>content</name>
                <value>%X{content}</value>
            </property>
            <!-- 内置转换器 -->
            <property>
                <name>logger</name>
                <value>%logger</value>
            </property>
        </properties>
        <!-- 头部信息 -->
        <headers>
            <header>
                <name>Content-Type</name>
                <value>application/json</value>
            </header>
        </headers>
    </appender>

    <!-- logstash日志采集 -->
    <logger name="com.codecoord.springboot.elasticsearch.logstash" level="INFO" additivity="false">
        <appender-ref ref="LOGSTASH_MDC"/>
    </logger>

    <root level="INFO">
        <appender-ref ref="STDOUT"/>
    </root>
</configuration>

4. 整合结果

{
  "_index" : "logstash-logs-2021-06",
  "_type" : "_doc",
  "_id" : "fcumGHoBUgXCDHMjqHTK",
  "_score" : 1.0,
  "_source" : {
    "@timestamp" : "2021-06-17T14:26:53.002+0800",
    "message" : "logstash日志采集，当前时间【2021-06-17T14:26:53.002】",
    "host" : "DESKTOP-MBL60LI",
    "code" : "1baa76f6-815b-4004-8d5a-34243446ec69",
    "createTime" : "2021-06-17T14:26:53.002",
    "content" : "logstash日志采集错误2021-06-17T14:26:53.002",
    "logger" : "com.codecoord.springboot.elasticsearch.logstash.LogstashLogbackAggregation"
  }
}

二、logback-elasticsearch-appender

1. Logback Elasticsearch Appender配置示例

 <appender name="ELASTIC" class="com.internetitem.logback.elasticsearch.ElasticsearchAppender">
   <url>http://yourserver/_bulk</url>
   <index>logs-%date{yyyy-MM-dd}</index>
   <type>tester</type>
   <loggerName>es-logger</loggerName> <!-- optional -->
   <errorLoggerName>es-error-logger</errorLoggerName> <!-- optional -->
   <connectTimeout>30000</connectTimeout> <!-- optional (in ms, default 30000) -->
   <errorsToStderr>false</errorsToStderr> <!-- optional (default false) -->
   <includeCallerData>false</includeCallerData> <!-- optional (default false) -->
   <logsToStderr>false</logsToStderr> <!-- optional (default false) -->
   <maxQueueSize>104857600</maxQueueSize> <!-- optional (default 104857600) -->
   <maxRetries>3</maxRetries> <!-- optional (default 3) -->
   <readTimeout>30000</readTimeout> <!-- optional (in ms, default 30000) -->
   <sleepTime>250</sleepTime> <!-- optional (in ms, default 250) -->
   <rawJsonMessage>false</rawJsonMessage> <!-- optional (default false) -->
   <includeMdc>false</includeMdc> <!-- optional (default false) -->
   <maxMessageSize>100</maxMessageSize> <!-- optional (default -1 -->
   <authentication class="com.internetitem.logback.elasticsearch.config.BasicAuthentication" /> <!-- optional -->
   <properties>
     <property>
       <name>host</name>
       <value>${HOSTNAME}</value>
       <allowEmpty>false</allowEmpty>
     </property>
     <property>
       <name>severity</name>
       <value>%level</value>
     </property>
     <property>
       <name>thread</name>
       <value>%thread</value>
     </property>
     <property>
       <name>stacktrace</name>
       <value>%ex</value>
     </property>
     <property>
       <name>logger</name>
       <value>%logger</value>
     </property>
   </properties>
   <headers>
     <header>
       <name>Content-Type</name>
       <value>application/json</value>
     </header>
   </headers>
</appender>

<root level="info">
  <appender-ref ref="FILELOGGER" />
  <appender-ref ref="ELASTIC" />
</root>

<logger name="es-error-logger" level="INFO" additivity="false">
  <appender-ref ref="FILELOGGER" />
</logger>

<logger name="es-logger" level="INFO" additivity="false">
  <appender name="ES_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <!-- ... -->
    <encoder>
      <pattern>%msg</pattern> <!-- This pattern is important, otherwise it won't be the raw Elasticsearch format anyomre -->
    </encoder>
  </appender>
</logger>

2. 配置说明

• url (required): The URL to your Elasticsearch bulk API endpoint
• index (required): Name if the index to publish to (populated using PatternLayout just like individual properties - see below)
• type (optional): Elasticsearch _type field for records. Although this library does not require type to be populated, Elasticsearch may, unless the configured URL includes the type (i.e. {index}/{type}/_bulk as opposed to /_bulk and /{index}/_bulk). See the Elasticsearch Bulk API documentation for more information
• sleepTime (optional, default 250): Time (in ms) to sleep between attempts at delivering a message
• maxRetries (optional, default 3): Number of times to attempt retrying a message on failure. Note that subsequent log messages reset the retry count to 0. This value is important if your program is about to exit (i.e. it is not producing any more log lines) but is unable to deliver some messages to ES
• connectTimeout (optional, default 30000): Elasticsearch connect timeout (in ms)
• readTimeout (optional, default 30000): Elasticsearch read timeout (in ms)
• includeCallerData (optional, default false): If set to true, save the caller data (identical to the AsyncAppender's includeCallerData)
• errorsToStderr (optional, default false): If set to true, any errors in communicating with Elasticsearch will also be dumped to stderr (normally they are only reported to the internal Logback Status system, in order to prevent a feedback loop)
• logsToStderr (optional, default false): If set to true, dump the raw Elasticsearch messages to stderr
• maxQueueSize (optional, default 104,857,600 = 200MB): Maximum size (in characters) of the send buffer. After this point, logs will be dropped. This should only happen if Elasticsearch is down, but this is a self-protection mechanism to ensure that the logging system doesn't cause the main process to run out of memory. Note that this maximum is approximate; once the maximum is hit, no new logs will be accepted until it shrinks, but any logs already accepted to be processed will still be added to the buffer
• loggerName (optional): If set, raw ES-formatted log data will be sent to this logger
• errorLoggerName (optional): If set, any internal errors or problems will be logged to this logger
• rawJsonMessage (optional, default false): If set to true, the log message is interpreted as pre-formatted raw JSON message.
• includeMdc (optional, default false): If set to true, then all MDC values will be mapped to properties on the JSON payload.
• maxMessageSize (optional, default -1): If set to a number greater than 0, truncate messages larger than this length, then append ".." to denote that the message was truncated
• authentication (optional): Add the ability to send authentication headers (see below)

The fields @timestamp and message are always sent and can not currently be configured. Additional fields can be sent by adding <property> elements to the <properties> set.

• name (required): Key to be used in the log event
• value (required): Text string to be sent. Internally, the value is populated using a Logback PatternLayout, so all Conversion Words can be used (in addition to the standard static variable interpolations like ${HOSTNAME}).
• allowEmpty (optional, default false): Normally, if the value results in a null or empty string, the field will not be sent. If allowEmpty is set to true then the field will be sent regardless
• type (optional, default String): type of the field on the resulting JSON message. Possible values are: String, int, float and boolean.