python3 文本文件内容去除重复行

环境:python3.8

import shutil

# 文件去除重复行
def remove_duplicates(path):
    lines_seen = set()
    outfile = open(f"{path}.out", 'a+', encoding='utf-8')
    f = open(path, 'r', encoding='utf-8')
    for line in f:
        if line not in lines_seen:
            outfile.write(line)
            lines_seen.add(line)
    outfile.close()
    f.close()

    # 移动文件
    shutil.move(f"{path}.out", path)
发布了58 篇原创文章 · 获赞 604 · 访问量 511万+

猜你喜欢

转载自blog.csdn.net/whatday/article/details/104874321