需求：系统对接了厂家的GPS数据，基于这些GPS数据，过滤出指定区域的数据
从网上找到了一个电子围栏的python脚本，现在需要的是循环取数据判断是否在指定区域，在指定区域就把这部分数据拿出来放到另外一个库表
碰到的其中一个问题是脚本的效率问题，以5W条数据来测试
脚本1：使用cur.fetchone()，逐条读取数据，逐条判断，逐条插入列表，批量入库，批量commit
#!/usr/bin/env python # -*- coding: utf-8 -*- import json import math import MySQLdb import time ISOTIMEFORMAT='%Y-%m-%d %X' print 'start',':',time.strftime(ISOTIMEFORMAT, time.localtime()) lnglatlist = [] data = '[{"name":"广本黄埔工厂","points":[{"lng":113.642124,"lat":23.167372},{"lng":113.636176,"lat":23.175162},{"lng":113.644930,"lat":23.179870},{"lng":113.652108,"lat":23.173823}],"type":0}]' data = json.loads(data) if 'points' in data[0]: for point in data[0]['points']: #print(str(point['lng'])+" "+str(point['lat'])) lnglat = [] lnglat.append(float(str(point['lng']))) lnglat.append(float(str(point['lat']))) lnglatlist.append(lnglat) def windingNumber(point, poly): poly.append(poly[0]) px = point[0] py = point[1] sum = 0 length = len(poly)-1 for index in range(0,length): sx = poly[index][0] sy = poly[index][1] tx = poly[index+1][0] ty = poly[index+1][1] #点与多边形顶点重合或在多边形的边上 if((sx - px) * (px - tx) >= 0 and (sy - py) * (py - ty) >= 0 and (px - sx) * (ty - sy) == (py - sy) * (tx - sx)): return "on" #点与相邻顶点连线的夹角 angle = math.atan2(sy - py, sx - px) - math.atan2(ty - py, tx - px) #确保夹角不超出取值范围（-π 到 π） if(angle >= math.pi): angle = angle - math.pi * 2 elif(angle <= -math.pi): angle = angle + math.pi * 2 sum += angle #计算回转数并判断点和多边形的几何关系 result = 'out' if int(sum / math.pi) == 0 else 'in' return result ################循环取GPS数据########################## conn=MySQLdb.connect(user='root',passwd='XXX',host='XXX',charset="utf8") #连接到mysql cur=conn.cursor() conn.select_db('XXX') cur.execute("select id,sim,alarm,status,latitude,longitude,asl,speed,direction,sample_time,attch_msg,create_time,car_no from gps_msg where sample_time>='2019-04-10 18:00:00' and sample_time<'2019-04-10 20:00:00' limit 50000") ####################第一种方式######################## count=1 scope_gps_list=[] while count<=50000: #这种方式，这个数量不能比SQL的数据量小，不然会报错 gps_data_per=cur.fetchone() # print gps_data_per point=gps_data_per[5].split(",")+gps_data_per[4].split(",") #取出每条数据的经纬度，split()转换成列表 point=map(float,point) #字符串类型转换成浮点型 # print point # print count if count in(10000,20000,30000,40000,50000): print count,':',time.strftime(ISOTIMEFORMAT, time.localtime()) # print windingNumber(point,lnglatlist) if windingNumber(point,lnglatlist)=='in': scope_gps_list.append(gps_data_per) #生成[(1,2,3),(1,2,3)] count=count+1 sqlin="insert into gps_msg_20190411(id,sim,alarm,status,latitude,longitude,asl,speed,direction,sample_time,attch_msg,create_time,car_no) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" cur.executemany(sqlin,scope_gps_list) conn.commit() ####################第一种方式######################## ####################第二种方式######################## # gps_data_all=cur.fetchall() # count=0 # for gps_data_per in gps_data_all: # sqlin="insert into gps_msg_20190411(id,sim,alarm,status,latitude,longitude,asl,speed,direction,sample_time,attch_msg,create_time,car_no) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" # point=gps_data_per[5].split(",")+gps_data_per[4].split(",") #取出每条数据的经纬度，split()转换成列表 # point=map(float,point) #字符串类型转换成浮点型 # if windingNumber(point,lnglatlist)=='in': # cur.execute(sqlin,gps_data_per) # count=count+1 # print count # conn.commit() ####################第二种方式######################## cur.close() #关闭游标 conn.close() #关闭数据链接 ################循环取GPS数据########################## print 'end',':',time.strftime(ISOTIMEFORMAT, time.localtime())
为什么要逐条插入列表？
因为使用cur.fetchone()读取数据后马上去insert into数据，再次cur.fenchone数据，取不到数据
执行效率：30分6秒跑完
![image_thumb[5]](https://img2018.cnblogs.com/blog/1537822/201904/1537822-20190419221922289-982759776.png "image_thumb[5]")
start : 2019-04-19 20:30:00 |
总结：最后的列表插入数据库很快，不到1秒
脚本2:使用cur.fetchall()先存全部数据，再使用for循环，逐条读取，逐条判断，逐条入库，批量commit
对比脚本1，主要是逐条入库，避免插入到列表，确定下是否插入到列表，列表越来越大导致的慢
#!/usr/bin/env python # -*- coding: utf-8 -*- import json import math import MySQLdb import time ISOTIMEFORMAT='%Y-%m-%d %X' print 'start',':',time.strftime(ISOTIMEFORMAT, time.localtime()) lnglatlist = [] data = '[{"name":"工厂","points":[{"lng":113.642124,"lat":23.167372},{"lng":113.636176,"lat":23.175162},{"lng":113.644930,"lat":23.179870},{"lng":113.652108,"lat":23.173823}],"type":0}]' data = json.loads(data) if 'points' in data[0]: for point in data[0]['points']: #print(str(point['lng'])+" "+str(point['lat'])) lnglat = [] lnglat.append(float(str(point['lng']))) lnglat.append(float(str(point['lat']))) lnglatlist.append(lnglat) def windingNumber(point, poly): poly.append(poly[0]) px = point[0] py = point[1] sum = 0 length = len(poly)-1 for index in range(0,length): sx = poly[index][0] sy = poly[index][1] tx = poly[index+1][0] ty = poly[index+1][1] #点与多边形顶点重合或在多边形的边上 if((sx - px) * (px - tx) >= 0 and (sy - py) * (py - ty) >= 0 and (px - sx) * (ty - sy) == (py - sy) * (tx - sx)): return "on" #点与相邻顶点连线的夹角 angle = math.atan2(sy - py, sx - px) - math.atan2(ty - py, tx - px) #确保夹角不超出取值范围（-π 到 π） if(angle >= math.pi): angle = angle - math.pi * 2 elif(angle <= -math.pi): angle = angle + math.pi * 2 sum += angle #计算回转数并判断点和多边形的几何关系 result = 'out' if int(sum / math.pi) == 0 else 'in' return result ################循环取GPS数据########################## conn=MySQLdb.connect(user='root',passwd='XXX',host='XXX',charset="utf8") #连接到mysql cur=conn.cursor() conn.select_db('XXX') cur.execute("select id,sim,alarm,status,latitude,longitude,asl,speed,direction,sample_time,attch_msg,create_time,car_no from gps_msg where sample_time>='2019-04-10 18:00:00' and sample_time<'2019-04-10 20:00:00' limit 50000") ####################第一种方式######################## # count=1 # scope_gps_list=[] # while count<=50000: #这种方式，这个数量不能比SQL的数据量小，不然会报错 # gps_data_per=cur.fetchone() # point=gps_data_per[5].split(",")+gps_data_per[4].split(",") #取出每条数据的经纬度，split()转换成列表 # point=map(float,point) #字符串类型转换成浮点型 # if count in(10000,20000,30000,40000,50000): # print count,':',time.strftime(ISOTIMEFORMAT, time.localtime()) # if windingNumber(point,lnglatlist)=='in': # scope_gps_list.append(gps_data_per) #生成[(1,2,3),(1,2,3)] # count=count+1 # sqlin="insert into gps_msg_20190411(id,sim,alarm,status,latitude,longitude,asl,speed,direction,sample_time,attch_msg,create_time,car_no) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" # cur.executemany(sqlin,scope_gps_list) # conn.commit() ####################第一种方式######################## ####################第二种方式######################## gps_data_all=cur.fetchall() count=0 for gps_data_per in gps_data_all: sqlin="insert into gps_msg_20190411(id,sim,alarm,status,latitude,longitude,asl,speed,direction,sample_time,attch_msg,create_time,car_no) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" point=gps_data_per[5].split(",")+gps_data_per[4].split(",") #取出每条数据的经纬度，split()转换成列表 point=map(float,point) #字符串类型转换成浮点型 if windingNumber(point,lnglatlist)=='in': cur.execute(sqlin,gps_data_per) count=count+1 if count in(10000,20000,30000,40000,50000): print count,':',time.strftime(ISOTIMEFORMAT, time.localtime()) conn.commit() ####################第二种方式######################## cur.close() #关闭游标 conn.close() #关闭数据链接 ################循环取GPS数据########################## print 'end',':',time.strftime(ISOTIMEFORMAT, time.localtime())
执行效率：29分钟22秒
![image_thumb[4]](https://img2018.cnblogs.com/blog/1537822/201904/1537822-20190419221923084-643099493.png "image_thumb[4]")
start : 2019-04-19 21:05:09 |
总结，看来不是插入到列表，导致的速度慢
脚本3:使用cur.fetchall()先存全部数据，再使用for循环，逐条读取，逐条判断，逐条入库，逐条commit
对比脚本2，逐条入库，逐条commit，只是做个简单的对比
#!/usr/bin/env python # -*- coding: utf-8 -*- import json import math import MySQLdb import time ISOTIMEFORMAT='%Y-%m-%d %X' print 'start',':',time.strftime(ISOTIMEFORMAT, time.localtime()) lnglatlist = [] data = '[{"name":"工厂","points":[{"lng":113.642124,"lat":23.167372},{"lng":113.636176,"lat":23.175162},{"lng":113.644930,"lat":23.179870},{"lng":113.652108,"lat":23.173823}],"type":0}]' data = json.loads(data) if 'points' in data[0]: for point in data[0]['points']: #print(str(point['lng'])+" "+str(point['lat'])) lnglat = [] lnglat.append(float(str(point['lng']))) lnglat.append(float(str(point['lat']))) lnglatlist.append(lnglat) def windingNumber(point, poly): poly.append(poly[0]) px = point[0] py = point[1] sum = 0 length = len(poly)-1 for index in range(0,length): sx = poly[index][0] sy = poly[index][1] tx = poly[index+1][0] ty = poly[index+1][1] #点与多边形顶点重合或在多边形的边上 if((sx - px) * (px - tx) >= 0 and (sy - py) * (py - ty) >= 0 and (px - sx) * (ty - sy) == (py - sy) * (tx - sx)): return "on" #点与相邻顶点连线的夹角 angle = math.atan2(sy - py, sx - px) - math.atan2(ty - py, tx - px) #确保夹角不超出取值范围（-π 到 π） if(angle >= math.pi): angle = angle - math.pi * 2 elif(angle <= -math.pi): angle = angle + math.pi * 2 sum += angle #计算回转数并判断点和多边形的几何关系 result = 'out' if int(sum / math.pi) == 0 else 'in' return result ################循环取GPS数据########################## conn=MySQLdb.connect(user='root',passwd='XXX',host='XXX',charset="utf8") #连接到mysql cur=conn.cursor() conn.select_db('XXX') cur.execute("select id,sim,alarm,status,latitude,longitude,asl,speed,direction,sample_time,attch_msg,create_time,car_no from gps_msg where sample_time>='2019-04-10 18:00:00' and sample_time<'2019-04-10 20:00:00' limit 50000") ####################第一种方式######################## # count=1 # scope_gps_list=[] # while count<=50000: #这种方式，这个数量不能比SQL的数据量小，不然会报错 # gps_data_per=cur.fetchone() # point=gps_data_per[5].split(",")+gps_data_per[4].split(",") #取出每条数据的经纬度，split()转换成列表 # point=map(float,point) #字符串类型转换成浮点型 # if count in(10000,20000,30000,40000,50000): # print count,':',time.strftime(ISOTIMEFORMAT, time.localtime()) # if windingNumber(point,lnglatlist)=='in': # scope_gps_list.append(gps_data_per) #生成[(1,2,3),(1,2,3)] # count=count+1 # sqlin="insert into gps_msg_20190411(id,sim,alarm,status,latitude,longitude,asl,speed,direction,sample_time,attch_msg,create_time,car_no) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" # cur.executemany(sqlin,scope_gps_list) # conn.commit() ####################第一种方式######################## ####################第二种方式######################## gps_data_all=cur.fetchall() count=0 for gps_data_per in gps_data_all: sqlin="insert into gps_msg_20190411(id,sim,alarm,status,latitude,longitude,asl,speed,direction,sample_time,attch_msg,create_time,car_no) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" point=gps_data_per[5].split(",")+gps_data_per[4].split(",") #取出每条数据的经纬度，split()转换成列表 point=map(float,point) #字符串类型转换成浮点型 if windingNumber(point,lnglatlist)=='in': cur.execute(sqlin,gps_data_per) conn.commit() count=count+1 if count in(10000,20000,30000,40000,50000): print count,':',time.strftime(ISOTIMEFORMAT, time.localtime()) ####################第二种方式######################## cur.close() #关闭游标 conn.close() #关闭数据链接 ################循环取GPS数据########################## print 'end',':',time.strftime(ISOTIMEFORMAT, time.localtime())
执行效率：
![image_thumb[3]](https://img2018.cnblogs.com/blog/1537822/201904/1537822-20190419221924295-826247779.png "image_thumb[3]")
start : 2019-04-19 21:45:11 |
总结：逐条commit会降低效率