够查看某一段时间内nginx日志访问情况，并且错误状态码超过一定比列就告警
功能实现
package tools
import (
"bufio"
"encoding/json"
"fmt"
"github.com/shopspring/decimal"
"io"
"log"
"os"
"regexp"
"sort"
"strconv"
"time"
)
var (
LOGFILE = "D:\work\perlearn\go_space\total\access.log"
nf NginxFields
errCounter int //错误状态码数量
totalCounter int //总共状态码数量
MapIpNums = make(map[string]int) //map 客户端ip：数量
MapUrlNums = make(map[string]int) //map url：数量
StatusMap = make(map[string]interface{}) // {"total": {"200": "50", "101": "179", "500": "390","404":"290"}}
tmp1=make(map[string]int) //{"200": "50"}
Threshold = 0.01 //百分比阀值，超过1就通知
Top int
TimeLong string //最近多长时间的，以分钟为单位
bt,et int64
regx = "(^127\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$)|(^10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$)|(^172\.1[6-9]{1}[0-9]{0,1}\.[0-9]{1,3}\.[0-9]{1,3}$)|(^172\.2[0-9]{1}[0-9]{0,1}\.[0-9]{1,3}\.[0-9]{1,3}$)|(^172\.3[0-1]{1}[0-9]{0,1}\.[0-9]{1,3}\.[0-9]{1,3}$)|(^192\.168\.[0-9]{1,3}\.[0-9]{1,3}$)"
)
func init() {
rootCmd.PersistentFlags().StringVar(&LOGFILE, "file","D:\work\perlearn\go_space\total\access.log" , "nginx access log file ")
rootCmd.PersistentFlags().IntVar(&Top, "top", 20, "Top值，默认是20")
rootCmd.PersistentFlags().StringVar(&TimeLong, "long", "30", "最近多长时间，默认是30")
}
type NginxFields struct {
Timestamp string `json:"@timestamp"`
RemoteAddr string `json:"remote_addr"`
HTTPXForwardedFor string `json:"http_x_forwarded_for"`
HTTPHost string `json:"http_host"`
ServerPort string `json:"server_port"`
Scheme string `json:"scheme"`
RequestMethod string `json:"request_method"`
RequestURI string `json:"request_uri"`
Status string `json:"status"`
UpstreamAddr string `json:"upstream_addr"`
Authbid string `json:"AUTHBID"`
Apptoken string `json:"APPTOKEN"`
Authuid string `json:"AUTHUID"`
BodyBytesSent string `json:"body_bytes_sent"`
BytesSent string `json:"bytes_sent"`
RequestTime string `json:"request_time"`
RequestBody string `json:"request_body"`
HTTPReferer string `json:"http_referer"`
HTTPUserAgent string `json:"http_user_agent"`
Host string `json:"host"`
}
//readNginxLog 读取文件
func ReadNginxLog() {
currentTime:=time.Now()
ago:=fmt.Sprintf("-%sm",TimeLong)
d,_:=time.ParseDuration(ago)
bt=currentTime.Add(d).Unix()
et=time.Now().Unix()
fmt.Printf("开始时间《%v》结束时间《%v》\n",TimestampTransTime(bt),TimestampTransTime(et))
f, err := os.OpenFile(LOGFILE,os.O_APPEND|os.O_CREATE|os.O_RDONLY,0644)
defer f.Close()
if err != nil{
panic("file open error")
return
}
reader:=bufio.NewReader(f) //默认是4096
for {
//读取文件
line,err := reader.ReadBytes('\n')
if err == io.EOF{
break
}
//json序列化
if err:=json.Unmarshal([]byte(line),&nf);err!=nil{
log.Printf("解析json失败，当前处理行：%s",line)
panic(err)
}
//状态码转换为整型
status,_:=strconv.Atoi(nf.Status)
readTime:=UTCTransLocal(nf.Timestamp)
//默认读取整个文件，不判断时间
if readTime>=bt && readTime <=et{
if status!=200 && status!=101 {
errCounter++
}
//{"total": {"200": "50", "101": "179", "500": "390","404":"290"}}
tmp1[nf.Status]+=1
re := regexp.MustCompile(regx)
match := re.MatchString(nf.RemoteAddr)
if match{
continue
}
MapIpNums[nf.RemoteAddr]+=1
MapUrlNums[nf.RequestURI]+=1
//计算所读取的行数
totalCounter++
}
}
StatusMap["total"]=tmp1
}
//计算的值和阀值相比较,如果大于阀值返回true，否则为false
func computer(errPer,threshold float64) bool {
if errPer >threshold{
return true
}
return false
}
//percent 计算百分比,错误数/总数
func percent(k1, k2 int) (float64, bool) {
decimal.DivisionPrecision=4
if k1==0{
fmt.Print("nginx错误状态码数量是0【无错误状态码】\n")
return float64(0),false
}
res := decimal.NewFromFloat(float64(k1)).Div(decimal.NewFromFloat(float64(k2)))
fmt.Printf("Nginx错误状态码/所有状态码百分比：%v\n",res)
return res.Float64()
}
//nginx UTC格式转换为时间戳
func UTCTransLocal(utcTime string) int64 {
t, _ := time.Parse("2006-01-02T15:04:05+08:00", utcTime)
return t.Unix()-3600*8
}
//时间戳转换为时间
func TimestampTransTime(timestamp int64) string {
timeLayout := "2006-01-02 15:04:05"
// time.Unix的第二个参数传递0或10结果一样，因为都不大于1e9
timeStr := time.Unix(timestamp, 0).Format(timeLayout)
return timeStr
}
//按照map中值降序排，
type impounds struct {
Key string
Val int
}
func SortMapByValue(m map[string]int) []impounds {
var l []impounds
for k,v := range m{
l=append(l, impounds{k,v})
}
sort.Slice(l, func(i, j int) bool {
return l[i].Val > l[j].Val // 降序
})
return l
}
func Start() {
ReadNginxLog()
fmt.Printf("nginx错误状态码数量:%v\n",errCounter)
fmt.Printf("总数量:%v\n",totalCounter)
erbium,_:=percent(errCounter,totalCounter)
flag := computer(erbium, Threshold)
if flag{
_ = fmt.Sprintf("占比超阀值了，开始告警通知\n")
//SendPhone(msg,"18601301393")
}else {
fmt.Printf("错误状态码百分比没超过阀值\n")
}
result:=SortMapByValue(MapIpNums)
res:=SortMapByValue(MapUrlNums)
//判断切片长度，防止溢出,以及切片长度不能超过top 的值，不然也会溢出
if len(result)==0||len(res)==0||len(result)<Top||len(res)<Top{
fmt.Print("top值太大了")
return
}
for _,v:= range result[0:Top]{
fmt.Printf("\033[0;35;40m%-30v \t %-30v\n\033[0m",v.Key,v.Val)
}
fmt.Println("---------------------------------------")
for _,v:= range res[0:Top]{
//fmt.Printf("\033[0;32;40m \t %v \033[0m\033[0;31;40m %v \n \033[0m",v.Key,v.Val)
fmt.Printf("\033[0;32;40m%-50v \t %-70v\n\033[0m",v.Key,v.Val)
}
fmt.Println(StatusMap)
}
//格式化输出
func OutPrintf(m map[string]int){
}
