1.commonutils.py
# -*- coding: UTF-8 -*-
import urllib2
#读取配置文件
def readConfigFile(filename, dataDict):
CANDataList = []
VehicleDataList = []
DataPathList = []
try:
configFile = file(filename)
while True:
line = configFile.readline()
if len(line) == 0:
break
if line.startswith("CANData"):
tmpList = line.split("=")[1].split(",")
for i in tmpList:
CANDataList.append(i.strip())
dataDict["CANData"] = CANDataList
if line.startswith("VehicleData"):
tmpList = line.split("=")[1].split(",")
for i in tmpList:
VehicleDataList.append(i.strip())
dataDict["VehicleData"] = VehicleDataList
if line.startswith("DataPath"):
tmpList = line.split("=")[1].split(",")
for i in tmpList:
DataPathList.append(i.strip())
dataDict["DataPath"] = DataPathList
except:
print "Exception: readConfigFile"
else:
print "Read config file successfully"
finally:
configFile.close()
def proxy_connect():
proxy_handler = urllib2.ProxyHandler({"http": "http://proxy.baidu.com:8080/"})
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
proxy_auth_handler = urllib2.ProxyBasicAuthHandler(password_mgr)
proxy_auth_handler.add_password(None, "http://proxy.baidu.com:8080", "w000baidu", "baidu@123,.,")
opener = urllib2.build_opener(proxy_handler, proxy_auth_handler)
urllib2.install_opener(opener)
def createHtmlFile(content):
try:
hFile = file("htmlFile.txt", "w")
hFile.write(content)
except:
print "Exception: htmlFile"
else:
print "htmlFile.txt is ok"
finally:
hFile.close()
def getZipList(filename):
zipList = []
try:
hFile = file(filename)
while True:
line = hFile.readline()
if len(line) == 0:
break
if ".zip" in line:
zipName = line.split('">')[0].split("/")[-1]
zipList.append(zipName)
except:
print "Exception: getZipList"
zipList = []
else:
print "zipList is ok"
finally:
hFile.close()
return zipList
2. conf
# -*- coding: UTF-8 -*-
#填写服务器进行数据分类的CAN，以逗号分隔
#如: CANData = BMS_CHARGE, BmuInner, PowertraintSubnet, SmartSubnet
CANData = BmsCharge, ChargerCAN, BmuInner, BmuInnerCAN, PowertraintSubnet, SmartSubnet, PowertrainSubnet, EnergySubnet, OriginalSubnet
#填写要获取数据的车辆，以逗号分隔，如下
#如: VehicleData = 1st_car, 2nd_car, 3rd_car
VehicleData = 1st_car, 2nd_car, secondmodelx, modelx
#填写服务器存储数据路径
#如: DataPath = /opt/BMS_Vehicle_Data/
DataPath = /home/bms/BMS_Vehicle_Data
3.htmlFile.txt
<HTML><HEAD><LINK HREF="jetty-dir.css" REL="stylesheet" TYPE="text/css"/><TITLE>Directory: /MON/secondmodelx/</TITLE></HEAD><BODY>
<H1>Directory: /MON/secondmodelx/</H1>
<TABLE BORDER=0>
<TR><TD><A HREF="/MON/secondmodelx/../">Parent Directory</A></TD><TD></TD><TD></TD></TR>
</TABLE>
</BODY></HTML>
4.index.html
<!DOCTYPE html>
<!--STATUS OK-->
<html>
<head>
<meta http-equiv=content-type content=text/html;charset=utf-8>
<meta http-equiv=X-UA-Compatible content=IE=Edge>
<meta content=always name=referrer>
<link rel=stylesheet type=text/css href=http://s1.bdstatic.com/r/www/cache/bdorz/baidu.min.css>
<title>百度一下，你就知道</title></head>
<body link=#0000cc>
<div id=wrapper>
<div id=head>
<div class=head_wrapper>
<div class=s_form>
<div class=s_form_wrapper>
<div id=lg><img hidefocus=true src=//www.baidu.com/img/bd_logo1.png width=270 height=129></div>
<form id=form name=f action=//www.baidu.com/s class=fm><input type=hidden name=bdorz_come value=1>
<input type=hidden name=ie value=utf-8> <input type=hidden name=f value=8> <input type=hidden
name=rsv_bp
value=1>
<input type=hidden name=rsv_idx value=1> <input type=hidden name=tn value=baidu><span
class="bg s_ipt_wr"><input id=kw name=wd class=s_ipt value maxlength=255
autocomplete=off autofocus></span><span
class="bg s_btn_wr"><input type=submit id=su value=百度一下 class="bg s_btn"></span></form>
</div>
</div>
<div id=u1><a href=http://news.baidu.com name=tj_trnews class=mnav>新闻</a> <a href=http://www.hao123.com
name=tj_trhao123 class=mnav>hao123</a>
<a href=http://map.baidu.com name=tj_trmap class=mnav>地图</a> <a href=http://v.baidu.com name=tj_trvideo
class=mnav>视频</a> <a
href=http://tieba.baidu.com name=tj_trtieba class=mnav>贴吧</a>
<noscript><a
href=http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1
name=tj_login class=lb>登录</a></noscript>
<script>document.write('<a href="http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u=' + encodeURIComponent(window.location.href + (window.location.search === "" ? "?" : "&") + "bdorz_come=1") + '" name="tj_login" class="lb">登录</a>');</script>
<a href=//www.baidu.com/more/ name=tj_briicon class=bri style="display: block;">更多产品</a></div>
</div>
</div>
<div id=ftCon>
<div id=ftConw><p id=lh><a href=http://home.baidu.com>关于百度</a> <a href=http://ir.baidu.com>About Baidu</a></p>
<p id=cp>©2017 Baidu <a href=http://www.baidu.com/duty/>使用百度前必读</a> <a
href=http://jianyi.baidu.com/ class=cp-feedback>意见反馈</a> 京ICP证030173号 <img
src=//www.baidu.com/img/gs.gif></p></div>
</div>
</div>
</body>
</html>
5.synchronizeData.py
# -*- coding: UTF-8 -*-
#!/usr/bin/python
# Filename : obtainData.py
import urllib2, sys, os, time
import commonUtils
def main(url, pathList, oldZipList, bakPath):
try:
#建立proxy连接
commonUtils.proxy_connect()
# 把url页面源代码写入txt文本，并提取zip包名
f = urllib2.urlopen(url)
content = f.read()
commonUtils.createHtmlFile(content)
zipNameList = commonUtils.getZipList("htmlFile.txt")
zipList = []
#排除已经获取的zip包名
for i in zipNameList:
logName = i[:-4] + ".log"
if logName not in oldZipList:
zipList.append(i)
if len(zipList) == 0:
print "There is no vehicle data to be synchronied."
return
#下载zip包并解压
for zipName in zipList:
try:
zipUrl = url + "/" + zipName
path = None
g = urllib2.urlopen(zipUrl)
data = g.read()
for i in pathList:
if zipName.split("_")[0] in i:
path = i
break
serverPath = path + zipName
with open(serverPath, "wb") as code:
code.write(data)
t = 0
while True:
if os.system("unzip -tq %s" % serverPath) == 0:
if os.system("unzip -q %s -d %s" % (serverPath, path)) == 0:
if os.system("rm %s" % serverPath) == 0:
print "Succeeded in unzip and removing the zip."
else:
print "Failed to upzip %s" % serverPath
break
else:
time.sleep(1)
t += 1
print "time.sleep %ds" % t
if t > 10:
os.system("mv %s %s" % (serverPath, bakPath))
break
except:
if os.path.isfile(serverPath):
print "An exception occurs when getting %s, so delete the zip." % serverPath
os.system("rm -r %s" % serverPath)
except:
print "There is an exception when synchronizing data."
if __name__ == "__main__":
dataDict = {}
sourceUrl = "http://1.2.3.4:8084/"
while True:
time.sleep(5)
# 读取配置文件
commonUtils.readConfigFile("conf", dataDict)
print dataDict
CANDataList = dataDict["CANData"]
VehicleDataList = dataDict["VehicleData"]
DataPath = dataDict["DataPath"][0]
#根据配置文件内容获取数据
for vehicle in VehicleDataList:
bakPath = DataPath + os.sep + vehicle + os.sep + "bak" + os.sep
if not os.path.isdir(bakPath):
os.makedirs(bakPath)
pathList = []
oldZipList = []
targetUrl = sourceUrl + "MON/" + vehicle
for CAN in CANDataList:
path = DataPath + os.sep + vehicle + os.sep + CAN + os.sep
if not os.path.isdir(path):
os.makedirs(path)
pathList.append(path)
# 查找已经获取过的zip包名
oldZipList.extend(os.listdir(path))
#处理已经获取过的zip包名
#oldZipFile = vehicle + "_" + CAN
#oldZipList = []
#if os.path.exists(oldZipFile):
# f = file(oldZipFile)
# oldZipList = cPickle.load(f)
# f.close()
main(targetUrl, pathList, oldZipList, bakPath)
#f = file(oldZipFile, "w")
#cPickle.dump(templist, f)
#f.close()
6.temp.log
# -*- coding: UTF-8 -*-
#!/usr/bin/python
# Filename : obtainData.py
import urllib2, sys, os, time
import commonUtils
def main(url, pathList, oldZipList, bakPath):
try:
#建立proxy连接
commonUtils.proxy_connect()
# 把url页面源代码写入txt文本，并提取zip包名
f = urllib2.urlopen(url)
content = f.read()
commonUtils.createHtmlFile(content)
zipNameList = commonUtils.getZipList("htmlFile.txt")
zipList = []
#排除已经获取的zip包名
for i in zipNameList:
logName = i[:-4] + ".log"
if logName not in oldZipList:
zipList.append(i)
if len(zipList) == 0:
print "There is no vehicle data to be synchronied."
return
#下载zip包并解压
for zipName in zipList:
try:
zipUrl = url + "/" + zipName
path = None
g = urllib2.urlopen(zipUrl)
data = g.read()
for i in pathList:
if zipName.split("_")[0] in i:
path = i
break
serverPath = path + zipName
with open(serverPath, "wb") as code:
code.write(data)
t = 0
while True:
if os.system("unzip -tq %s" % serverPath) == 0:
if os.system("unzip -q %s -d %s" % (serverPath, path)) == 0:
if os.system("rm %s" % serverPath) == 0:
print "Succeeded in unzip and removing the zip."
else:
print "Failed to upzip %s" % serverPath
break
else:
time.sleep(1)
t += 1
print "time.sleep %ds" % t
if t > 10:
os.system("mv %s %s" % (serverPath, bakPath))
break
except:
if os.path.isfile(serverPath):
print "An exception occurs when getting %s, so delete the zip." % serverPath
os.system("rm -r %s" % serverPath)
except:
print "There is an exception when synchronizing data."
if __name__ == "__main__":
dataDict = {}
sourceUrl = "http://1.2.3.4:8084/"
while True:
time.sleep(5)
# 读取配置文件
commonUtils.readConfigFile("conf", dataDict)
print dataDict
CANDataList = dataDict["CANData"]
VehicleDataList = dataDict["VehicleData"]
DataPath = dataDict["DataPath"][0]
#根据配置文件内容获取数据
for vehicle in VehicleDataList:
bakPath = DataPath + os.sep + vehicle + os.sep + "bak" + os.sep
if not os.path.isdir(bakPath):
os.makedirs(bakPath)
pathList = []
oldZipList = []
targetUrl = sourceUrl + "MON/" + vehicle
for CAN in CANDataList:
path = DataPath + os.sep + vehicle + os.sep + CAN + os.sep
if not os.path.isdir(path):
os.makedirs(path)
pathList.append(path)
# 查找已经获取过的zip包名
oldZipList.extend(os.listdir(path))
#处理已经获取过的zip包名
#oldZipFile = vehicle + "_" + CAN
#oldZipList = []
#if os.path.exists(oldZipFile):
# f = file(oldZipFile)
# oldZipList = cPickle.load(f)
# f.close()
main(targetUrl, pathList, oldZipList, bakPath)
#f = file(oldZipFile, "w")
#cPickle.dump(templist, f)
#f.close()
