实现功能：
- 日志记录
- 按保留目录个数清理备份的目录
- 压缩多久之前的日志文件
- 清理多久之前的压缩文件
#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @Time : 2021/9/13 16:52
# @Author : XuLiLiang
# @Email : xuliliang@epailive.com
# @File : del_log.py
import shutil, os, bz2, datetime, time, logging, fnmatch, subprocess,threading
from stat import ST_CTIME, ST_MTIME, ST_UID, ST_GID
from multiprocessing.dummy import Pool as ThreadPool
from rich.console import Console
console = Console()
# -------------------日志记录-----------------
log_pth="/data/tmplog/info.log"
# 记录器方式记录log
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logHandler = logging.FileHandler(filename=log_pth, mode="a", encoding="utf-8")
logHandler.setLevel(logging.DEBUG)
fm = logging.Formatter(fmt="%(asctime)s - %(name)s - %(levelname)-9s - %(filename)-8s : %(lineno)s line - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
logHandler.setFormatter(fm)
logger.addHandler(logHandler)
# --------------------清理目录---------------------------------
src_dir = "/data/backup/application/backup/"
blNum = 7 # 要保留的目录个数
list_objs = os.listdir(src_dir)
def process_dirs(item):
new_id = os.listdir(src_dir + item)
new_id = sorted(map(int, new_id))
length = len(new_id)
if length > blNum:
rm_list = new_id[0:(length - blNum)]
for x in rm_list:
rm_full_path = f'{src_dir}{item}/{str(x)}'
logger.info(f"开清理目录{rm_full_path}")
try:
shutil.rmtree(rm_full_path)
except Exception as e:
raise e
# -----------------压缩日志------------------------------------
# 压缩5天前的文件
beforeDate = 3
# 判断文件最后修改日期
logPath = "/data/program/tjship"
logTime = 30 # 删除多少天之前压缩文件
# 获取当前时间
today = datetime.datetime.now()
# 计算偏移量,前3天
offset = datetime.timedelta(days=-beforeDate)
# 获取想要的日期的时间,即前3天时间
re_date = (today + offset)
# 前3天时间转换为时间戳
re_date_unix = time.mktime(re_date.timetuple())
# 返回所有路径下所有目录及文件
def _find_dirs(path):
for curDir, dirs, files in os.walk(path):
return dirs, files
# 压缩文件
def _zipfile(file):
logger.info(f"开始压缩文件=========>>>>>>{file}")
# 压缩后的文件名
new_file = file + '.bz2'
file_stat = os.stat(file)
with open(file, 'rb') as r, bz2.open(new_file, 'wb') as w:
shutil.copyfileobj(r, w, length=4096)
os.utime(new_file, (file_stat[ST_CTIME], file_stat[ST_MTIME])) # 修改时间
os.chown(new_file, file_stat[ST_UID], file_stat[ST_GID]) # 修改用户
try:
# 压缩完后删除原文件
if os.path.exists(file):
os.remove(file)
except Exception as e:
raise e
def process_zip_files():
list_dirs, _ = _find_dirs(logPath)
for d in list_dirs:
new_path=f'{logPath}/{d}/log/'
_find_dirs(new_path)
dirs, _ = _find_dirs(new_path)
for r in dirs:
if len(r):
ful_log_path = f'{new_path}{r}/'
_, files = _find_dirs(ful_log_path)
for a in range(0,len(files)):
f_name = ful_log_path + files[a]
if fnmatch.fnmatch(f_name, "*.log") and os.path.getmtime(f_name) <= re_date_unix:
# _zipfile(f_name)
t=threading.Thread(target=_zipfile,args=(f_name,))
t.start()
t.join()
# -----------------删除压缩日志------------------------------------
def delete_zip_files() -> object:
# cmd = '/usr/bin/find %s -type f -name "*.bz2" -mtime +%s' % (logPath, logTime)
cmd = f'/usr/bin/find {logPath} -type f -name "*.bz2" -mtime +{logTime}'
res = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
files_list = res.stdout.readlines()
for log_f in files_list:
f_name = str(log_f.strip(), "utf-8")
try:
logger.info(f"开始删除bzip2 file ======>>>>>>>>>>>>>{f_name}")
os.remove(f_name)
pass
except Exception as e:
raise e
def main():
with ThreadPool() as p:
p.map(process_dirs, list_objs)
process_zip_files()
delete_zip_files()
if __name__ == '__main__':
main()
更多精彩关注公众号“51运维com” 和 个人博客
