Python批量转换文件编码格式
Eclipse中看ANSI编码的文件有乱码，所以希望通过python将相关文件转换成utf-8编码。
源：https://www.cnblogs.com/tsbc/p/4450675.html
| ''' 遍历文件夹 如果文件名是.cpp .h 如果原来的编码不是utf-8,将文件编码格式改成utf-8 ''' import os,sys import chardet def convert( filename, out_enc="UTF8" ): try: fp = open(filename,'rb+') content = fp.read() coding = chardet.detect(content)['encoding'] #获取encoding的值[编码格式] if coding != 'utf-8': new_content = content.decode(coding,"ignore").encode(out_enc) fp.seek(0) fp.write(new_content) fp.close() except IOError: print( " error") def explore(dir): #遍历目录 for root, dirs, files in os.walk(dir): for file in files: if '.cpp' in file or '.h' in file: path = os.path.join(root, file) convert(path) fiePath = r'E:\Code' def main(): explore(fiePath) if __name__ == "__main__": main() |
Python解压
https://www.cnblogs.com/Oliva/p/8824040.html 多线程字典破解加密zip
https://www.cnblogs.com/fyqq0403/p/9710420.html 解压加密的zip
https://www.cnblogs.com/flyhigh1860/p/3884842.html 解压zip
Python多线程&多进程
https://www.cnblogs.com/yeayee/p/4952022.html 基础介绍
https://www.cnblogs.com/kellyseeme/p/5525017.html 锁的应用
https://www.cnblogs.com/znicy/p/6234522.html 通过多进程的方式解决了解压缩的性能问题
https://www.cnblogs.com/xybaby/p/6510941.html#undefined python性能优化，介绍了GIL导致多线程的问题
https://www.cnblogs.com/SuKiWX/p/8804974.html python GIL解释
用python解压多个压缩文件（环境中有6000个左右压缩文件）遇到瓶颈，解压过程非常慢。尝试用多线程解压，处理时间不仅没有减少，还增加了。后搜索上述博客后，用多进程解压以缩短处理时间。
| import zipfile import tarfile import gzip import os from time import ctime from multiprocessing import Pool from multiprocessing import cpu_count dayZipsPath = r'.' quarterZipsPath = r'./tmp' zipPassWord = b'password' mrFilePath = r'./data' def unzipDayFile(): for file_name in os.listdir(dayZipsPath): if os.path.splitext(file_name)[1] == '.zip': print( file_name) file_zip = zipfile.ZipFile(file_name, 'r') file_zip.extractall(path = quarterZipsPath, pwd = zipPassWord) file_zip.close() #os.remove(file_name) def untarDayFile(): for file_name in os.listdir(dayZipsPath): if '.tar.gz' in file_name: print( file_name) file_tar = tarfile.open(file_name) file_tar.extractall(path = quarterZipsPath) file_tar.close() #os.remove(file_name) def unzip(zipsList): for file_name in zipsList: if os.path.splitext(file_name)[1] == '.zip': zipFileName = quarterZipsPath +'/'+ file_name file_zip = zipfile.ZipFile(zipFileName, 'r') file_zip.extractall(path = mrFilePath, pwd = zipPassWord) file_zip.close() os.remove(zipFileName) if __name__ == '__main__': print('Begin:%s' % ctime()) #获取CPU核个数 cpuNum = cpu_count() print(cpuNum)
unzipDayFile() untarDayFile()
#多进程解压，大大缩短处理时间 quarterZipsList = list(os.listdir(quarterZipsPath)) zipFileNum = len(quarterZipsList) print("total zip files num:%d" % (zipFileNum)) print("begin unzip:%s" % ctime()) p = Pool() for i in range(cpuNum): beginPos = int(i*zipFileNum/cpuNum) endPos = min(int((i+1)*zipFileNum/cpuNum),zipFileNum) print("proc %d - %d" % (beginPos, endPos)) p.apply_async(unzip,args=(quarterZipsList[beginPos:endPos],)) print("waiting for unzip quarter mr data ...") p.close() p.join() print("end unzip:%s" % ctime()) print( "End:%s" % ctime()) |