python实现LOAD DATA INFILE
实现代码如下,注释详解
里面的sql语句是通用的
class Infile():
def __init__(self):
self.flag = 1
def do_load_data_infile(self, filepath=None, filename=None):
# 命令传参
try:
csv_filepath = filepath # csv文件路径 其实可以和文件名写在一起,看你传参需求
csv_filename = filename # csv格式的文件
database = "dbname" # mysql数据库名dbname
table_name = csv_filename # mysql表名,假设无表用来建表的名称
#打开csv文件
file = open(csv_filepath + '/' + csv_filename , 'r',encoding='utf-8')
#读取csv文件第一行字段名,创建表
reader = file.readline()
b = reader.split(',')
colum = ''
for a in b:
colum = colum + a + ' varchar(255),'
colum = colum[:-1]
#编写sql,create_sql负责创建表,data_sql负责导入数据
table = 'create table if not exists ' + table_name + ' ' + '(' + colum + ')' + ' DEFAULT CHARSET=utf8'
# print(table_name)
data = " load data local infile '" \
+ csv_filepath + "/" \
+ csv_filename + "'" + \
" replace into table "+ database +"." \
+ table_name + \
" FIELDS TERMINATED BY ','" \
" LINES TERMINATED BY '\\r\\n' " \
" IGNORE 1 LINES; " # " LINES TERMINATED BY '\\r\\n' " \
# 连接到mysql,主要一定要加上local_infile=1参数,否则会报错的
# print('load data local infile ', data) #load data infile语句,从本地上传所以加local
# load data infile 测试代码段
conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='123456', db=database, local_infile=1) #本地测试用
cursor = conn.cursor()
cursor.execute('set names utf8')
cursor.execute('set character_set_connection=utf8')
cursor.execute(table) #执行建表语句
cursor.execute(data) #执行数据上传语句
conn.commit()
cursor.close()
conn.close()
except Exception as e:
print(e.args)
traceback.print_exc()
if __name__ == '__main__':
parent_dir = 'C:/Users/Administrator/Desktop/csvfile' #注意'/'
df = Infile() # 实例化
filelist = os.listdir(parent_dir) #csv文件所在文件夹所有文件列表
# print(filelist)
for i in filelist: # 遍历所有后缀名为csv的文件执行load data infile
print(i)
if i.split('.')[-1] == 'csv':
# df.Thread_csv(i)
df.do_load_data_infile(parent_dir,i)