自动删除目录下重复的文件（可以自定义类型）

前言

最近整理nas群晖的文件，发现影音库里面很多重复的资源，自己就写了一个python脚本自动的处理。

import os import hashlib # 只删除以下列表中的重复文件类型.如果想删除其他类型的文件,自己添加一下就行了 file_type = ['.jpg', '.jpeg', '.png', '.gif', '.psd', '.bmp', '.webp', '.mp4', '.mkv', '.avi', '.mov', 'mpeg', 'mpg', '.rar', '.zip','img'] check_files = [] def remove_reapt_files(): for root, dirs, files in os.walk(r'/Users/yuqianjun/Downloads/'): #这个路径/Users/yuqianjun/Downloads/就是你需要 查重的路径；自己定义就好了 for name in files: print(name) p_type = os.path.splitext(os.path.join(root, name))[1] if p_type in file_type: check_files.append(os.path.join(root, name)) for name in dirs: p_type = os.path.splitext(os.path.join(root, name))[1] if p_type in file_type: check_files.append(os.path.join(root, name)) files_dict = {} r_index = 0 print('Fiels Num:%s' % len(check_files)) for value in check_files: md5_hash = hashlib.md5() try: with open(value, "rb+") as f: for byte_block in iter(lambda: f.read(4096), b""): md5_hash.update(byte_block) file_md5 = md5_hash.hexdigest() print('Check file MD5:%s' % value) if files_dict.get(file_md5) is None: files_dict[file_md5] = value else: d_path = files_dict[file_md5] d_path_stats = os.stat(d_path) file_stats = os.stat(value) d_time = d_path_stats.st_ctime f_time = file_stats.st_ctime if d_time > f_time: os.remove(d_path) files_dict[file_md5] = value print('Delete File:', d_path) r_index += 1 else: os.remove(value) print('Delete File:', value) r_index += 1 except Exception as e: pass print('File does not exist or has been deleted') print('File Count:%s, Repeat Files Num:%s. All deleted!' %( len(check_files),str(r_index))) if __name__ == '__main__': remove_reapt_files()