导入导出elasticsearch索引数据

1.方式一

    

dump插件

    在elasticsearch5.x中

1.1 elasticsearch-dump安装

1) yum install epel-release
2) yum install nodejs
3) yum install npm
4) npm install elasticdump
5) cd node_modules/elasticdump/bin  后便可以执行操作。

1.2 elasticsearch-dump使用

    mapping：

./elasticdump  --input=http://10.10.10.215:8400/~mdap-china-city_ihl  --output=http://10.10.10.138:9200/~mdap-china-city_ihl  --type=mapping

    data:

2.方式二

    代码方式（java 或 python）

java代码：

package com.andong.es.util;

import java.io.File;
import java.io.FileOutputStream;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFCellStyle;
import org.apache.poi.hssf.usermodel.HSSFFont;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequestBuilder;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder;
import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.alibaba.fastjson.JSON;

/**
 * ESUtils
 * 
 * @author andong
 * 
 */
public class ESUtils
{
    
   public final static DateFormat esdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");

    private final static Logger logger = LoggerFactory.getLogger(ESUtils.class);
    
    /** ES host */
    private static String host        = "10.10.10.215";
    /** ES port */
    private static int    port        = 8500;
    /** ES clusterName */
    private static String clusterName = "imdap-215";

    /** ES client */
    private static Client client      = new TransportClient(ImmutableSettings.settingsBuilder()
                                              .put("cluster.name", clusterName).build())
                                              .addTransportAddress(new InetSocketTransportAddress(host, port));

    public static Client getClient()
    {
        return client;
    }

    /**
     * 判断是否存在 类型
     * 
     * @param index
     *            索引
     * @param types
     *            类型
     * @return
     */
    public static boolean isTypeExists(String index, String... types)
    {
        IndicesAdminClient indexClient = client.admin().indices();
        TypesExistsRequestBuilder typeExist = null;
        if (index == null || index.equals(""))
        {
            typeExist = indexClient.prepareTypesExists();
        }
        else
        {
            typeExist = indexClient.prepareTypesExists(index);
        }

        TypesExistsRequestBuilder typeExists = typeExist.setTypes(types);
        TypesExistsResponse response = typeExists.get();
        return response.isExists();
    }

    /**
     * 判断是否存在 索引
     * 
     * @param indicese
     *            索引
     * @return
     */
    public static boolean isIndexExists(String... indicese)
    {
        IndicesExistsRequestBuilder indexExists = client.admin().indices().prepareExists(indicese);
        IndicesExistsResponse response = indexExists.get();
        return response.isExists();
    }

    
    public static void insertOne(String index,String type,Object data){
    	List<Object> datas = new ArrayList<Object>();
    	datas.add(data);
    	ESUtils.insert(index, type, datas);
    }
    
    /**
     * 写ES库
     * 
     * @param index
     *            索引
     * @param type
     *            类型
     * @param datas
     */
    public static void insert(String index, String type, List<Object> datas)
    {

        BulkRequestBuilder bulkRequest = client.prepareBulk();
        for (Object data : datas)
        {
            String dataStr = JSON.toJSONStringWithDateFormat(data, "yyyy-MM-dd'T'HH:mm:ss.SSS");
            bulkRequest.add(client.prepareIndex(index, type).setSource(dataStr));
        }
        BulkResponse bulkResponse = bulkRequest.execute().actionGet();
    }
  
 
    public static Pageable search(String[] indexs,String type, FilterBuilder filterBuilders, QueryBuilder queryBuilders,
            LinkedHashMap<String, SortOrder> orders, int from, int size)
    {
        Pageable page = new Pageable<Object>();
        try
        {
            if (from < 0)
            {
                from = 0;
            }
            if (size < 0)
            {
                size = 10;
            }
            
            SearchRequestBuilder searchRequestBuilder = client.prepareSearch();
            
            if (!StringUtils.isEmpty(type))
            {
            	if(type.split(",").length<=1){
            		searchRequestBuilder.setTypes(type);
            	}else{
            		searchRequestBuilder.setTypes(type.split(","));
            	}
            }
            if (orders != null && !orders.isEmpty())
            {
                for (String key : orders.keySet())
                {
                    searchRequestBuilder.addSort(key, orders.get(key));
                }
            }
            SearchResponse response = searchRequestBuilder.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
                    .setQuery((QueryBuilder) queryBuilders) // Query
                    .setPostFilter((FilterBuilder) filterBuilders) // Filter
                    .setFrom(from).setSize(size).setExplain(false).execute().actionGet();

            SearchHit[] ss = response.getHits().getHits();
            List<Map<String, Object>> datas = new ArrayList<Map<String, Object>>();

            Map<String, Object> data = null;
            for (SearchHit s : ss)
            {
                data = s.getSource();
                datas.add(data);
            }

            page.setTotal(response.getHits().getTotalHits());
            page.setContent(datas);
        }
        catch (SearchPhaseExecutionException e)
        {
            page.setTotal(0L);
        }
        // }
        return page;
    }

    
   
  /*  *//**
     * 解析返回的数据
     * 
     * @param content
     * @return
     * @throws Exception
     *//*
    public static EsResult parseEs(String content) throws Exception
    {
        EsResult esResult = new EsResult();

        Gson gson = new Gson();

        Type type = new TypeToken<Map<String, Object>>()
        {

        }.getType();
         获取查询时长 
        Map<String, Object> map = gson.fromJson(content, type);
        // logger.debug("ES执行查询结果数据"+map.toString());
        long took = ((Double) map.get("took")).longValue();
        esResult.setTook(took);
        logger.debug("ES执行查询时长" + esResult.getTook());
         获取查询总数 
        Map<String, Object> _hits = (Map) map.get("hits");
        esResult.setTotal(((Double) _hits.get("total")).intValue());
         如果总数为0 
        logger.debug("ES执行查询总数" + esResult.getTotal());
        if (esResult.getTotal() == 0)
        {
            return esResult;
        }
         获取查询的结果集并解析成JavaBean 
        esResult.setParaseBefore((List) _hits.get("hits"));
         返回统计结果集Map 这里不处理 具体类处理 
        esResult.setFacets(parseLogFaces((Map) map.get("aggregations")));
        return esResult;
    }
*/
  

    public static void main(String[] args) throws Exception
    {
    	ESUtils esUtils = new ESUtils();
    	boolean typeExists = esUtils.isTypeExists("~mdap-china-city_ihl", "province");
    	if(typeExists) {
    		String[] indexs= {"~mdap-china-city_ihl"};
			FilterBuilder filterBuilders = null;
			QueryBuilder queryBuilders = null;
			LinkedHashMap<String, SortOrder> orders = null;
			Pageable search = esUtils.search(indexs, "province", filterBuilders, queryBuilders, orders, 0, 10000);
			List<Map<String, Object>> content = search.getContent();
			//esUtils.parseEs(content);
			
			
			 // 创建execl 文件
		       HSSFWorkbook workbook = new HSSFWorkbook();
		       HSSFSheet sheet = workbook.createSheet();
		       // 设定默认宽度
		       sheet.setDefaultColumnWidth(18);
		       // 数字double样式
		       HSSFCellStyle contentCellStyle = workbook.createCellStyle();
		       HSSFFont font = workbook.createFont();
		       font.setFontName("宋体");
		       font.setFontHeightInPoints((short) 11);
		       contentCellStyle.setFont(font);
		       // 标题样式
		       HSSFCellStyle headerStyle = (HSSFCellStyle) workbook.createCellStyle();// 创建标题样式
		       headerStyle.setVerticalAlignment(HSSFCellStyle.VERTICAL_CENTER); // 设置垂直居中
		       headerStyle.setAlignment(HSSFCellStyle.ALIGN_CENTER); // 设置水平居中
		       HSSFFont headerFont = (HSSFFont) workbook.createFont(); // 创建字体样式
		       headerFont.setBoldweight(HSSFFont.BOLDWEIGHT_BOLD); // 字体加粗
		       headerFont.setFontName("Times New Roman"); // 设置字体类型
		       headerFont.setFontHeightInPoints((short) 12); // 设置字体大小
		       headerStyle.setFont(headerFont); // 为标题样式设置字体样式
		       HSSFRow  row = null;
		       HSSFCell cell = null;
		      StringBuffer idBuffer = new StringBuffer("");
		      LinkedHashMap<String, String> map = new LinkedHashMap<String, String>();
		      row = sheet.createRow(0);
		      cell = row.createCell(0);  
	          cell.setCellValue("name");
	          cell = row.createCell(1); 
	          cell.setCellValue("IN_TIME");
	          cell = row.createCell(2); 
	          cell.setCellValue("latitude");
	          cell = row.createCell(3); 
	          cell.setCellValue("cid");
	          cell = row.createCell(4); 
	          cell.setCellValue("id");
	          cell = row.createCell(5); 
	          cell.setCellValue("longitude");
	          cell = row.createCell(6); 
	          cell.setCellValue("brief");
	          int index = 0;
		      for(int i=0 ; i< content.size();i++) {
		    	  index++;  
		    	  System.out.println(content.get(i));
		    	  row = sheet.createRow(index);
		    		  
	    		  cell = row.createCell(0);  
		          cell.setCellValue(content.get(i).get("name").toString());
		          cell = row.createCell(1); 
		          cell.setCellValue(content.get(i).get("IN_TIME").toString());
		          cell = row.createCell(2); 
		          cell.setCellValue(content.get(i).get("latitude").toString());
		          cell = row.createCell(3); 
		          cell.setCellValue(content.get(i).get("cid").toString());
		          cell = row.createCell(4); 
		          cell.setCellValue(content.get(i).get("id").toString());
		          cell = row.createCell(5); 
		          cell.setCellValue(content.get(i).get("longitude").toString());
		          cell = row.createCell(6); 
		          cell.setCellValue(content.get(i).get("brief").toString());
	    	  
		          
		          //idBuffer.append(fields.get("cacti_id")+",");
		      }
		      FileOutputStream out = new FileOutputStream(new File("D://province.xls"));
		      workbook.write(out);
		      out.flush();
    	}
    }

   
}

main方法中： 导出为excel格式

扫描二维码关注公众号，回复： 9714395 查看本文章

public static void main(String[] args) throws Exception
	    {
	    	ESUtils esUtils = new ESUtils();
	    	boolean typeExists = esUtils.isTypeExists("~mdap-china-city_ihl", "province");
	    	if(typeExists) {
	    		String[] indexs= {"~mdap-china-city_ihl"};
				FilterBuilder filterBuilders = null;
				QueryBuilder queryBuilders = null;
				LinkedHashMap<String, SortOrder> orders = null;
				Pageable search = esUtils.search(indexs, "province", filterBuilders, queryBuilders, orders, 0, 10000);
				List<Map<String, Object>> content = search.getContent();
				//esUtils.parseEs(content);
				
				
				 // 创建execl 文件
			       HSSFWorkbook workbook = new HSSFWorkbook();
			       HSSFSheet sheet = workbook.createSheet();
			       // 设定默认宽度
			       sheet.setDefaultColumnWidth(18);
			       // 数字double样式
			       HSSFCellStyle contentCellStyle = workbook.createCellStyle();
			       HSSFFont font = workbook.createFont();
			       font.setFontName("宋体");
			       font.setFontHeightInPoints((short) 11);
			       contentCellStyle.setFont(font);
			       // 标题样式
			       HSSFCellStyle headerStyle = (HSSFCellStyle) workbook.createCellStyle();// 创建标题样式
			       headerStyle.setVerticalAlignment(HSSFCellStyle.VERTICAL_CENTER); // 设置垂直居中
			       headerStyle.setAlignment(HSSFCellStyle.ALIGN_CENTER); // 设置水平居中
			       HSSFFont headerFont = (HSSFFont) workbook.createFont(); // 创建字体样式
			       headerFont.setBoldweight(HSSFFont.BOLDWEIGHT_BOLD); // 字体加粗
			       headerFont.setFontName("Times New Roman"); // 设置字体类型
			       headerFont.setFontHeightInPoints((short) 12); // 设置字体大小
			       headerStyle.setFont(headerFont); // 为标题样式设置字体样式
			       HSSFRow  row = null;
			       HSSFCell cell = null;
			      StringBuffer idBuffer = new StringBuffer("");
			      LinkedHashMap<String, String> map = new LinkedHashMap<String, String>();
			      row = sheet.createRow(0);
			      cell = row.createCell(0);  
		          cell.setCellValue("name");
		          cell = row.createCell(1); 
		          cell.setCellValue("IN_TIME");
		          cell = row.createCell(2); 
		          cell.setCellValue("latitude");
		          cell = row.createCell(3); 
		          cell.setCellValue("cid");
		          cell = row.createCell(4); 
		          cell.setCellValue("id");
		          cell = row.createCell(5); 
		          cell.setCellValue("longitude");
		          cell = row.createCell(6); 
		          cell.setCellValue("brief");
		          int index = 0;
			      for(int i=0 ; i< content.size();i++) {
			    	  index++;  
			    	  System.out.println(content.get(i));
			    	  row = sheet.createRow(index);
			    		  
		    		  cell = row.createCell(0);  
			          cell.setCellValue(content.get(i).get("name").toString());
			          cell = row.createCell(1); 
			          cell.setCellValue(content.get(i).get("IN_TIME").toString());
			          cell = row.createCell(2); 
			          cell.setCellValue(content.get(i).get("latitude").toString());
			          cell = row.createCell(3); 
			          cell.setCellValue(content.get(i).get("cid").toString());
			          cell = row.createCell(4); 
			          cell.setCellValue(content.get(i).get("id").toString());
			          cell = row.createCell(5); 
			          cell.setCellValue(content.get(i).get("longitude").toString());
			          cell = row.createCell(6); 
			          cell.setCellValue(content.get(i).get("brief").toString());
		    	  
			          
			          //idBuffer.append(fields.get("cacti_id")+",");
			      }
			      FileOutputStream out = new FileOutputStream(new File("D://province.xls"));
			      workbook.write(out);
			      out.flush();
	    	}
	    }

python代码：

# coding: utf-8
import json
import os
import sys
import time

import urllib3
class importEsData():
    def __init__(self,url,index,type):
        self.url = url+"/"+index+"/"+type
        self.index = index
        self.type = type
         
    def importData(self):
        print("import data begin...")
        begin = time.time()
        try:
            f = open(self.index+"_"+self.type+".json","r")
            for line in f:
                self.post(line)
        finally:
            f.close()
        print("import data end!!!\n\t total consuming time:"+str(time.time()-begin)+"s")
    def post(self,data):
        req = urllib3.request(self.url,data,{"Content-Type":"application/json; charset=UTF-8"})
        urllib3.connection(req)
        
        
if __name__ == '__main__':
     #importEsData("http://10.100.142.60:9200","watchdog","test").importData()
     importEsData("http://10.10.10.215:8400/","~mdap-china-city_ihl","mapping").importData()

# coding: utf-8
import json
import os
import sys
import time
import urllib3


class exportEsData():
    size = 10000
    def __init__(self, url,index,type):
        self.url = url+"/"+index+"/"+type+"/_search"
        self.index = index
        self.type = type
    def exportData(self):
        print("export data begin...")
        begin = time.time()
        try:
            os.remove(self.index+"_"+self.type+".json")
        except:
            pass
        msg = urllib3.request(self.url).read()
        print(msg)
        obj = json.loads(msg)
        num = obj["hits"]["total"]
        start = 0
        end =  num/self.size+1
        while(start<end):
            msg = urllib3.request(self.url+"?from="+str(start*self.size)+"&size="+str(self.size)).read()
            self.writeFile(msg)
            start=start+1
        print("export data end!!!\n\t total consuming time:"+str(time.time()-begin)+"s")
    def writeFile(self,msg):
        obj = json.loads(msg)
        vals = obj["hits"]["hits"]
        try:
            f = open(self.index+"_"+self.type+".json","a")
            for val in vals:
                a = json.dumps(val["_source"],ensure_ascii=False)
                f.write(a+"\n")
        finally:
            f.flush()
            f.close()
            
            
if __name__ == '__main__':
    #exportEsData("http://10.100.142.60:9200","watchdog","mexception").exportData()
    exportEsData("http://10.10.10.215:8500","~mdap-china-city_ihl","district").exportData()