python实现磁盘日志清理的示例

 #!/usr/bin/env python # -*- coding: utf-8 -*- import commands import os import time import re import getopt import sys # commands.getstatusoutput 返回两个元素的元组tuple(status, result)，status为int类型，result为string类型 def execute_local_shell_cmd(cmd): status, result = commands.getstatusoutput(cmd) result = result.split("\n") return status, result def send_alert_mail(): pass ''' 获取某一磁盘的空间使用率 ''' def get_disk_used(disk_name): status, result = execute_local_shell_cmd("df | grep %s | awk '{print $5}'" % disk_name) return status, result[0] #print(get_disk_used('/data0')) ''' 判断文件是否在指定时间内修改过 ''' def file_modify_in(file_path,time_interval='1d'): current_time = time.time() # os.path.getmtime 返回最后修改时间。返回从unix纪元开始的跳秒数 if current_time - os.path.getmtime(file_path)  before_seconds_remove \ and candidate_file.find(pattern) != -1 \ and not probable_current_log_file(candidate_file_fullpath): # yield 就是return一个值，并且记住这个返回值的位置，下次迭代就从这个位置后开始 yield candidate_file_fullpath ''' 获取超过大小的日志文件（注意默认不会返回修改时间小于1天的文件） ''' def get_clean_log_list_by_size(target_dir,file_size_limit='10g',pattern="log"): file_size_limit_byte = translate_file_size_to_kb(file_size_limit) for candidate_file in os.listdir(target_dir): candidate_file_fullpath = "%s/%s" %(target_dir,candidate_file) if os.path.isfile(candidate_file_fullpath): # stat返回相关文件的系统状态信息 file_stat = os.stat(candidate_file_fullpath) if candidate_file.find(pattern) != -1 and \ file_stat.st_size >= file_size_limit_byte: yield candidate_file_fullpath # 如果文件在modify_in之内修改过，则不返回 # if not (modify_in and file_modify_in(candidate_file_fullpath, time_interval=modify_in)) and \ #   not probable_current_log_file(candidate_file_fullpath): #    yield candidate_file_fullpath ''' remove文件列表 ''' def remove_file_list(file_list,pattern='log',roll_back=False): for file_item in file_list: if roll_back or probable_current_log_file(file_item,pattern=pattern,modify_in='1d'): print('roll back file %s' % file_item) execute_local_shell_cmd("cat /dev/null > {0}".format(file_item)) else: print('remove file %s' % file_item) # os.remove 删除指定路径文件。如果指定的路径是一个目录，将抛出OSError os.remove(file_item) ''' 清理掉超过日期的日志文件 ''' def remove_files_by_date(target_dir,before_days_remove='7d',pattern='log'): file_list = get_clean_log_list_by_date(target_dir,before_days_remove,pattern) remove_file_list(file_list) ''' 清理掉超过大小的日志文件 ''' def remove_files_by_size(target_dir,file_size_limit='10g',pattern='log'): file_list = get_clean_log_list_by_size(target_dir,file_size_limit,pattern) remove_file_list(file_list) ''' 清空当前的日志文件，使用cat /dev/null > {log_file}方式 ''' def clean_curren_log_file(target_dir,file_size_limit='10g',pattern='log'): for candidate_file in os.listdir(target_dir): candidate_file_fullpath = '%s/%s' % (target_dir,candidate_file) if candidate_file.endswith(pattern) and os.path.isfile(candidate_file_fullpath): file_stat = os.stat(candidate_file_fullpath) if file_stat.st_size >= translate_file_size_to_kb(file_size_limit): remove_file_list([candidate_file_fullpath],roll_back=True) def clean_data_release_disk(disk_name, target_dir, disk_used_limit='80%', before_days_remove='7d', file_size_limit='10g', pattern='log'): disk_used_limit = disk_used_limit.replace('%', '') # 第一步执行按时间的日志清理 print('Step one remove files {0} ago.'.format(before_days_remove)) remove_files_by_date(target_dir, before_days_remove=before_days_remove, pattern=pattern) # 如果磁盘空间还是没有充分释放,则执行按大小的日志清理 current_disk_used = int(get_disk_used(disk_name)[1].replace('%', '')) if current_disk_used > int(disk_used_limit): print("Disk {0}'s current used {1}% great than input used limit {2}%," "so we will remove files bigger than {3}". format(disk_name, current_disk_used, disk_used_limit, file_size_limit)) remove_files_by_size(target_dir, file_size_limit=file_size_limit, pattern=pattern) # 如果磁盘空间开没有释放,清空当前正在写的log文件,并alert current_disk_used = int(get_disk_used(disk_name)[1].replace('%', '')) if current_disk_used > int(disk_used_limit): print("Disk {0}'s current used {1}% great than input used limit {2}%," "so we will roll back current log file". format(disk_name, current_disk_used, disk_used_limit, file_size_limit)) clean_curren_log_file(target_dir, file_size_limit=file_size_limit, pattern=pattern) # 如果还是没有,alert mail if int(get_disk_used(disk_name)[1].replace('%', '')) > int(disk_used_limit): send_alert_mail() def usage(): print('clean.py -d  -r  ' '-f  -p  ' '-t  ') if __name__ == "__main__": target_disk_input = '/data0' target_dir_input = '/data0/hadoop2/logs' disk_used_limit_input = '80%' file_size_limit_input = '10g' pattern_input = 'log' before_days_remove_input = '7d' try: # getopt 命令解析，有短选项和长选项 # getopt 返回两人个参数：一个对应参数选项和value元组，另一个一般为空 opts,args = getopt.getopt(sys.argv[1:], 'hd:r:u:f:p:t:', ['help' 'disk=', 'directory=', 'diskUsedLimit=', 'fileSizeLimit=', 'filePattern=', 'beforeDaysRemove=']) # getopt模块函数异常错误，捕获异常并打印错误 except getopt.GetoptError as err: print err usage() sys.exit(2) if len(opts) <6: usage() sys.exit(2) for opt,arg in opts: if opt '-h': sys.exit() elif ("-d","--disk"): target_disk_input=arg.replace('/','') ("-r","--directory"): target_dir_input=arg ("-u","--diskusedlimit"): disk_used_limit_input=arg ("-f","--filesizelimit"): file_size_limit_input=arg translate_file_size_to_kb(file_size_limit_input) ("-p","filepattern"): pattern_input=arg ("-t","--beforedaysremove"): before_days_remove_input=arg translate_time_interval_to_second(before_days_remove_input) print ("{0} start clean job.target_disk:{1},target_directory:{2},disk_used_limit:{3}," "file_size_limit:{4},pattern:{5},before_days_remove:{6}".format(time.ctime(time.time()), target_disk_input, target_dir_input, disk_used_limit_input, file_size_limit_input, pattern_input, before_days_remove_input)) clean_data_release_disk(target_disk_input, disk_used_limit=disk_used_limit_input, file_size_limit=file_size_limit_input, pattern=pattern_input, before_days_remove=before_days_remove_input)

 #!/usr/bin/env python # -*- coding: utf-8 -*- import os # 遍历目录 def Lisdir(targetdir): list_dirs = os.walk(targetdir) for root,list_dirs,files in list_dirs: for d in list_dirs: yield os.path.join(root,d) def log_dir(targetdir): list_dirs = os.listdir(targetdir) for ph in list_dirs: if os.path.isdir(os.path.join(targetdir,ph)): yield Lisdir(os.path.join(targetdir,ph)) for path in log_dir('/data0/backup_log-bin'): for ppp in path: # 以log-bin结尾 为假 if ppp.endswith('log-bin') is False: os.system("db_script/clean_robo.py -d /data0 -r {0} -u 75% -f 501M -p bin -t 5d".format(ppp))