本地ELK日志系统搭建并使用GO写入日志（二）

在上文中使用docker-elk搭建了本地的elk系统，接下来希望接入开发项目，希望具备以下功能：

1. 不同服务建立不同的索引，按天进行索引归类；

2. 日志的fields设计

接入方式：通过TCP直接写入logstash

logstash配置

cd /app/docker-elk/logstash/pipeline
vim logstash.conf

// 修改为以下内容：
input {
        beats {
                port => 5044
        }

        tcp {
                port => 5000
                codec => json
        }
}
## Add your filters / logstash plugins configuration here
output {
        elasticsearch {
                hosts => "10.30.239.171:9200"
                index => "%{[service]}-%{+YYYY.MM.dd}"
                user => "logstash_internal"
                password => "${LOGSTASH_INTERNAL_PASSWORD}"
        }
}

修改完配置后重启

docker-compose restart

Go写入日志

package middleware

import (
	"encoding/json"
	"fmt"
	"net"
	"time"
)

type ElkLogger struct {
	Logger *LogStash
}

type LogStash struct {
	hostname string
	port     int
	Conn     *net.TCPConn
	TimeOut  int
}

func newLogStash(hostname string, port int, timeout int) *LogStash {
	return &LogStash{
		hostname: hostname,
		port:     port,
		Conn:     nil,
		TimeOut:  timeout,
	}
}

func (l *LogStash) Connect() (*net.TCPConn, error) {
	var conn *net.TCPConn
	service := fmt.Sprintf("%s:%d", l.hostname, l.port)
	addr, err := net.ResolveTCPAddr("tcp", service)
	if err != nil {
		return nil, err
	}
	conn, err = net.DialTCP("tcp", nil, addr)
	if err != nil {
		return nil, err
	}
	if conn != nil {
		l.Conn = conn
		l.Conn.SetKeepAlive(true)
		l.Conn.SetKeepAlivePeriod(time.Duration(5) * time.Second)
		l.setTimeouts()
	}
	return conn, nil
}

func (l *LogStash) setTimeouts() {
	deadline := time.Now().Add(time.Duration(l.TimeOut) * time.Millisecond)
	_ = l.Conn.SetDeadline(deadline)
	_ = l.Conn.SetReadDeadline(deadline)
	_ = l.Conn.SetWriteDeadline(deadline)
}

// Write. message: json
func (l *LogStash) Write(message string) (err error) {
	message = fmt.Sprintf("%s\n", message)
	if l.Conn != nil {
		_, err = l.Conn.Write([]byte(message))
		if err != nil {
			l.Connect()
			return
		} else {
			l.setTimeouts()
			return nil
		}
	}
	return
}

func NewElkLogger(hostname string, port int, timeout int) *ElkLogger {
	return &ElkLogger{Logger: newLogStash(hostname, port, timeout)}
}

var Logger *ElkLogger

type ElkMsg struct {
	Service string
	Level   string
	Msg     string
}

func (log *ElkLogger) Errorf(format string, args ...interface{}) {
	msg := &ElkMsg{
		Service: "ceph-backend",
		Level:   "error",
		Msg:     fmt.Sprintf(format, args...),
	}
	var (
		bytes []byte
		err   error
	)
	if bytes, err = json.Marshal(msg); err != nil {
		fmt.Println("elk msg marshal error")
		return
	}
	log.Logger.Write(string(bytes))
}

func (log *ElkLogger) Infof(format string, args ...interface{}) {
	msg := &ElkMsg{
		Service: "ceph-backend",
		Level:   "info",
		Msg:     fmt.Sprintf(format, args...),
	}
	var (
		bytes []byte
		err   error
	)
	if bytes, err = json.Marshal(msg); err != nil {
		fmt.Println("elk msg marshal error")
		return
	}
	log.Logger.Write(string(bytes))
}

func (log *ElkLogger) Warnf(format string, args ...interface{}) {
	msg := &ElkMsg{
		Service: "ceph-backend",
		Level:   "warn",
		Msg:     fmt.Sprintf(format, args...),
	}
	var (
		bytes []byte
		err   error
	)
	if bytes, err = json.Marshal(msg); err != nil {
		fmt.Println("elk msg marshal error")
		return
	}
	log.Logger.Write(string(bytes))
}

func main(){
        Logger = NewElkLogger("10.30.239.171", 5000, 5)
        Logger.Infof("connect mysql success")
}

验证

1. 建立索引，这里服务名为ceph-backend

2. Discover中进行检索

注意：

权限不足问题

之前通过logstash_internal这个账号建立index报错：

应该是这个账号没有建立新的es indices权限，去Management里添加管理员对应的权限即可