这是我参与8月更文挑战的第2天，活动详情查看：8月更文挑战
最近发现服务很不稳定，报错很多，我们想要根据日志进行分析报错原因并review 于是根据我们留存在Elasticsearch 上的日志进行了以下的一些操作
发送邮件方法：
from email.mime.text import MIMEText
from smtplib import SMTP
import requests
def send_mail(to, subject, msg, smtpserver, username, password, _subtype='plain'):
msg=MIMEText(msg, _subtype=_subtype, _charset='utf-8')
msg['Subject'], msg['From'], msg['To']=subject, username, ";".join(to)
try:
smtp=SMTP()
smtp.connect(smtpserver)
smtp.login(username, password)
smtp.sendmail(username, to, msg.as_string())
smtp.close()
except Exception as e:
return e
根据通过操作es 过滤日志并发送邮件
def monitor(self, user_list=['']):
try:
es = Elasticsearch([{'host': '127.0.0.1', 'port': '9200'}], http_auth=('admin', '123456'), timeout=3600)
a_dict = {"1": "测试APP", "2": "测试PC", "3": "测试APP1", "4": "测试PC2"}
p_dict = {"sakjldk": "测试", "hasdja": "测试2", "lljl": "测试3"}
l_dict = {"dhjsjkda": "北京机房", "opwq": "aws",
"dkjashdj": "阿里云", "kdhka": "杭州机房",
"llys": "广州机房"}
t_dict = {"ooiui": "【线上】", "kkk": "【aws 线上】",
"kjkjhg": "【预上线】", "iuuiui": "【线上】",
"hhhmgg": "【测试】"}
index = 'logs'
p_url = {"jjii": "xxxx"}
s_url = '/list'
# 获取 错误数据 按照logger_name字段聚合 当天
body = {
"query": {
"bool": {
"must": [ # 根据标识进行获取想要的请求 一般标记为ERROR的请求
{"match": {"level": "ERROR"}},
{"range": {"@timestamp": {"gt": "now-5m"}}}
],
"must_not": [ # 过滤不想看到的错误请求
{"match_phrase": {"message": "RedisSetNxLock Error"}},
]
},
}
}
# 当天 错误数据
queryErrorData = es.search(index=index, body=body, scroll='1m', )
except Exception as e:
# es如果宕机需要报警处理
send_mail(user_list, '日志报警{}服务请求报错'.format("ES"), "ES 链接故障 {}".format(e), 'smtp.exmail.qq.com',
'admin@qq.com', 'jjdasda')
content = ""
title = ""
repeat = []
if queryErrorData['hits']['hits']:
for row in queryErrorData['hits']['hits']:
if row['_source']['logger_name'] not in repeat:
repeat.append(row['_source']['logger_name'])
title += "|" + log_tag_name_dict[row['_source']['logger_name']] + row['_source']['logger_name']
b = re.findall("'me': '(.*?)'", row['_source']['message']) # 根据条件进行过滤
a = re.findall("'app': '(.*?)'", row['_source']['message'])
p = re.findall("'p': '(.*?)'", row['_source']['message'])
body = re.findall("Body((.*?))]-", row['_source']['message'])
if a:
a_content = "应用： " + a[0] + " " + a_dict[a[0]] + "\n"
else:
a_content = ""
if p:
p_content = "客户端： " + p[0] + " " + p_dict[p[0]] + "\n"
else:
p_content = ""
user_content = "触发人信息：{} \n".format(info)
body = "请求信息Body:" + body[0] + "\n\n" if body else ''
if b and a:
url = p_url[row['_source']['logger_name']] + s_url
params = {"aList": [a[0]]}
headers = {"Content-Type": "application/json"}
s_list = requests.post(url=url, json=params, headers=headers).json()
s_dict_list = {str(row['id']): row for row in s_list['data']}
if s_dict_list.get(str(b[0])):
m_id = "ID: " + b[0] + "| 名称: " + s_dict_list.get(str(b[0]))[
'name'] + "\n"
else:
m_id = ''
else:
m_id = ''
# 处理日志时间 一般+8小时
eta_temp = row['_source']['timestamp']
fd = datetime.datetime.strptime(eta_temp, "%Y-%m-%d %H:%M:%S")
eta = (fd + datetime.timedelta(hours=8)).strftime("%Y-%m-%d %H:%M:%S")
eta_date = (fd + datetime.timedelta(hours=8)).strftime("%Y-%m-%d")
timestamp = "请求时间: " + eta + "\n"
content += "\n" * 3 + "-" * 200 + "\n" + "服务名称： " + row['_source']['logger_name'] + " {}".format(
log_name_dict.get(row['_source']['logger_name'],
'')) + "\n" + timestamp + menu_id + user_content + application_content + portal_content + body + "\n\n".join(
row['_source']['message'].split(')-'))
if content:
send_mail(user_list, '日志报警{}服务请求报错'.format(title), content, 'smtp.exmail.qq.com',
'admin@qq.com', 'hjhhj')
return queryErrorData
