ElasticSearch 数据导入导出工具

1. '''
2. Export and Import ElasticSearch Data.
3. Simple Example At __main__
4. @author: [email protected]
5. @note: uncheck consistency of data, please do it by self
6. '''
7.  
8. import json
9. import os
10. import sys
11. import time
12. import urllib2
13.  
14. reload(sys)
15. sys.setdefaultencoding('utf-8')# @UndefinedVariable
16.  
17. class exportEsData():
18. size =10000
19. def __init__(self, url,index,type):
20. self.url = url+"/"+index+"/"+type+"/_search"
21. self.index = index
22. self.type = type
23. def exportData(self):
24. print("export data begin...")
25. begin= time.time()
26. try:
27. os.remove(self.index+"_"+self.type+".json")
28. except:
29. os.mknod(self.index+"_"+self.type+".json")
30. msg = urllib2.urlopen(self.url).read()
31. print(msg)
32. obj = json.loads(msg)
33. num = obj["hits"]["total"]
34. start =0
35. end= num/self.size+1
36. while(start<end):
37. msg = urllib2.urlopen(self.url+"?from="+str(start*self.size)+"&size="+str(self.size)).read()
38. self.writeFile(msg)
39. start=start+1
40. print("export data end!!!\n\t total consuming time:"+str(time.time()-begin)+"s")
41. def writeFile(self,msg):
42. obj = json.loads(msg)
43. vals = obj["hits"]["hits"]
44. try:
45. f = open(self.index+"_"+self.type+".json","a")
46. for val in vals:
47. a = json.dumps(val["_source"],ensure_ascii=False)
48. f.write(a+"\n")
49. finally:
50. f.flush()
51. f.close()
52.  
53. class importEsData():
54. def __init__(self,url,index,type):
55. self.url = url+"/"+index+"/"+type
56. self.index = index
57. self.type = type
59. def importData(self):
60. print("import data begin...")
61. begin= time.time()
62. try:
63. f = open(self.index+"_"+self.type+".json","r")
64. for line in f:
65. self.post(line)
66. finally:
67. f.close()
68. print("import data end!!!\n\t total consuming time:"+str(time.time()-begin)+"s")
69. def post(self,data):
70. req = urllib2.Request(self.url,data,{"Content-Type":"application/json; charset=UTF-8"})
71. urllib2.urlopen(req)
72.  
73. if __name__ =='__main__':
74. '''
75. Export Data
76. e.g.
77. URL index type
78. exportEsData("http://10.100.142.60:9200","watchdog","mexception").exportData()
80. export file name: watchdog_mexception.json
81. '''
82. #exportEsData("http://10.100.142.60:9200","watchdog","mexception").exportData()
83. exportEsData("http://10.100.142.60:9200","watchdog","mexception").exportData()
86. '''
87. Import Data
89. *import file name:watchdog_test.json (important)
90. "_" front part represents the elasticsearch index
91. "_" after part represents the elasticsearch type
92. e.g.
93. URL index type
94. mportEsData("http://10.100.142.60:9200","watchdog","test").importData()
95. '''
96. #importEsData("http://10.100.142.60:9200","watchdog","test").importData()
97. importEsData("http://10.100.142.60:9200","watchdog","test").importData()