[python] Data processing of .csv files: extract specific columns from all .csv files under the directory file, deduplicate and store as new .csv files

Raw data:

After processing:

solution:

import pandas as pd
import os
import csv

path = r"D:\xxx\数据"
for dirpath,dirnames,filenames in os.walk(path):
   for filename in filenames:
      # 使用pandas读入
      data = pd.read_csv(os.path.join(dirpath,filename)) #读取文件中所有数据
      x = data[['x','y','z']]#读取x,y,z列
      print(x)
      a=x.drop_duplicates(subset=['x','y','z'],keep='first', inplace=False) #去重
      print(a)
      a.to_csv(r'./userid.csv',sep=",")#储存为新的文件，userid.csv：为文件名