Elasticsearch5.X java API
1. Java API批量导出
Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", "elasticsearch-bigdata").build();
Client client = new TransportClient(settings)
.addTransportAddress(new InetSocketTransportAddress("10.58.71.6", 9300));
SearchResponse response = client.prepareSearch("bigdata").setTypes("student")
.setQuery(QueryBuilders.matchAllQuery()).setSize(10000).setScroll(new TimeValue(6000 00))
.setSearchType(SearchType.SCAN).execute().actionGet();//setSearchType(SearchType.Scan) 告诉ES不需要排序只要结果返回即可 setScroll(new TimeValue(600000)) 设置滚动的时间
String scrollid = response.getScrollId();
try {
//把导出的结果以JSON的格式写到文件里
BufferedWriter out = new BufferedWriter(new FileWriter("es", true));
//每次返回数据10000条。一直循环查询直到所有的数据都查询出来
while (true) {
SearchResponse response2 = client.prepareSearchScroll(scrollid).setScroll(new TimeValue(1000000))
.execute().actionGet();
SearchHits searchHit = response2.getHits();
//再次查询不到数据时跳出循环
if (searchHit.getHits().length == 0) {
break;
}
System.out.println("查询数量 :" + searchHit.getHits().length);
for (int i = 0; i < searchHit.getHits().length; i++) {
String json = searchHit.getHits()[i].getSourceAsString();
out.write(json);
out.write("\r\n");
}
}
System.out.println("查询结束");
out.close();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
2:Java API 批量导入
Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", "elasticsearch-bigdata").build();
Client client = new TransportClient(settings)
.addTransportAddress(new InetSocketTransportAddress("10.58.71.6", 9300));
try {
//读取刚才导出的ES数据
BufferedReader br = new BufferedReader(new FileReader("es"));
String json = null;
int count = 0;
//开启批量插入
BulkRequestBuilder bulkRequest = client.prepareBulk();
while ((json = br.readLine()) != null) {
bulkRequest.add(client.prepareIndex("bigdata", "student").setSource(json));
//每一千条提交一次
if (count% 1000==0) {
bulkRequest.execute().actionGet();
System.out.println("提交了:" + count);
}
count++;
}
bulkRequest.execute().actionGet();
System.out.println("插入完毕");
br.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
3.ES读取Json文件添加索引
import com.esindex.CrmTeacherObject;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.Client;
import org.json.JSONObject;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
/**
* Created by bin.zhang on 2017/4/14.
* 1.逐条处理 json文件,得当相应的索引字段
* 2.调用 JsonUtil.objToJsonData 得当所以字段json字符串
*/
public class DataFactory {
private static final String indexName = "索引名";
private static final String type = "索引类型main";
private static final String HOST = "ES集群IP";
private static final int PORT = ES端口;
//读取json数据文件 list
public static void readJsonFile(){
//List list = new ArrayList();
BufferedReader br;
try{
//创建BufferedReader(FileReader)
//br = new BufferedReader(new FileReader("D:/Work_Space/es-index/src/test.txt"));
String line = null;
while ((line=br.readLine())!= null ){
JSONObject dataJson = new JSONObject(line);
//读一行处理一行 (获得需要的索引字段)
CrmTeacherObject teacherObject = new CrmTeacherObject(
dataJson.get("teacherId")==JSONObject.NULL ? 0:dataJson.getLong("teacherId"),
dataJson.get("realName") == JSONObject.NULL ? "":dataJson.getString("realName"),
dataJson.get("mobile") == JSONObject.NULL ? "":dataJson.get("mobile").toString()
);
//交给JsonUtil处理,成创建索引需要的字符串
String JsonString = JsonUtil.objToJsonData(teacherObject);
//创建索引
ElasticSearchIndex esi = new ElasticSearchIndex();
//得到ES连接 client
Client client = esi.getClient(HOST,PORT);
//创建索引
IndexResponse indexResponse = esi.doCreateIndexResponse( client,indexName ,type ,JsonString);
//System.out.println(indexResponse.getIndex());
//查询索引
//GetResponse getResponse = esi.getIndexResponse(client);
//System.out.println(getResponse.getIndex());
//关闭ES连接
client.close();
}
//关闭 BufferedReader
br.close();
}catch (IOException e){
e.printStackTrace();
}
}
public static void main(String[] args) {
readJsonFile();
}
}
package com.esindex;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
/**
* Created by Administrator on 2017/4/14.
*/
public class ElasticSearchIndex {
//获得client 连接
public Client getClient(String host,int port ){
Client client = null;
Settings settings = ImmutableSettings.settingsBuilder().put("client.transport.ping_timeout", "10s")
.put("client.transport.ignore_cluster_name", true)
.put("node.client", true)
.put("client.transport.sniff", true).build();
client = new TransportClient(settings).addTransportAddress(new InetSocketTransportAddress(host, port));
return client;
}
//关闭client 连接
public void close(Client client) {
if (client != null) {
client.close();
}
}
//创建索引
public IndexResponse doCreateIndexResponse(Client client,String indexName, String type, String json) {
//Client client = getClient(HOST,PORT);
IndexResponse response = client.prepareIndex(indexName, type)
.setSource(json)
.execute()
.actionGet();
return response;
}
//查询索引
public SearchResponse doSerch(Client client,String indexName,String type){
SearchResponse response = client.prepareSearch(indexName).setTypes(type).execute().actionGet();
return response;
}
//删除索引
}
package com.esindex;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import java.io.IOException;
/**
* Created by bin.zhang on 2017/4/14.
* 用于生成新的JSON字符串
*/
public class JsonUtil {
public static String objToJsonData(CrmTeacherObject crmTeacherObject){
String jsonData=null;
try{
XContentBuilder jsonBuilder = XContentFactory.jsonBuilder();
jsonBuilder.startObject()
.field("teacherId",crmTeacherObject.getTeacherId())
.field("realName",crmTeacherObject.getRealName())
.field("mobile",crmTeacherObject.getMobile())
.endObject();
jsonData = jsonBuilder.string();
}catch (IOException e){
e.printStackTrace();
}
return jsonData;
}
}
package com.esindex;
/**
* Created by bin.zhang on 2017/4/14.
*/
public class CrmTeacherObject {
private Long teacherId;
private String realName;
private String mobile;
//初始化赋值
public CrmTeacherObject(Long teacherId,String realName,String mobile){
this.teacherId = teacherId;
this.realName = realName;
this.mobile=mobile;
}
public Long getTeacherId() {
return teacherId;
}
public void setTeacherId(Long teacherId) {
this.teacherId = teacherId;
}
public String getRealName() {
return realName;
}
public void setRealName(String realName) {
this.realName = realName;
}
public String getMobile() {
return mobile;
}
public void setMobile(String mobile) {
this.mobile = mobile;
}
}
Example 1
Project: elasticsearch_my File: SimpleIndexTemplateIT.java View source code | 5votes |
public void testAliasInvalidFilterInvalidJson() throws Exception { //invalid json: put index template fails PutIndexTemplateRequestBuilder putIndexTemplateRequestBuilder = client().admin().indices().preparePutTemplate("template_1") .setPatterns(Collections.singletonList("te*")) .addAlias(new Alias("invalid_alias").filter("abcde")); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> putIndexTemplateRequestBuilder.get()); assertThat(e.getMessage(), equalTo("failed to parse filter for alias [invalid_alias]")); GetIndexTemplatesResponse response = client().admin().indices().prepareGetTemplates("template_1").get(); assertThat(response.getIndexTemplates().size(), equalTo(0)); }
Example 2
Project: elasticsearch_my File: SimpleIndexTemplateIT.java View source code | 5votes |
public void testAliasEmptyName() throws Exception { PutIndexTemplateRequestBuilder putIndexTemplateRequestBuilder = client().admin().indices().preparePutTemplate("template_1") .setPatterns(Collections.singletonList("te*")) .addAlias(new Alias(" ").indexRouting("1,2,3")); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> putIndexTemplateRequestBuilder.get()); assertThat(e.getMessage(), equalTo("alias name is required")); }
Example 3
Project: elasticsearch_my File: SimpleIndexTemplateIT.java View source code | 5votes |
public void testAliasWithMultipleIndexRoutings() throws Exception { PutIndexTemplateRequestBuilder putIndexTemplateRequestBuilder = client().admin().indices().preparePutTemplate("template_1") .setPatterns(Collections.singletonList("te*")) .addAlias(new Alias("alias").indexRouting("1,2,3")); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> putIndexTemplateRequestBuilder.get()); assertThat(e.getMessage(), equalTo("alias [alias] has several index routing values associated with it")); }
Example 4
Project: streams-examples File: ElasticsearchReindexParentIT.java View source code | 5votes |
@BeforeClass public void prepareTest() throws Exception { Config reference = ConfigFactory.load(); File conf_file = new File("target/test-classes/ElasticsearchReindexParentIT.conf"); assert(conf_file.exists()); Config testResourceConfig = ConfigFactory.parseFileAnySyntax(conf_file, ConfigParseOptions.defaults().setAllowMissing(false)); Config typesafe = testResourceConfig.withFallback(reference).resolve(); testConfiguration = new ComponentConfigurator<>(ElasticsearchReindexConfiguration.class).detectConfiguration(typesafe); testClient = ElasticsearchClientManager.getInstance(testConfiguration.getSource()).client(); ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(); ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest).actionGet(); assertThat(clusterHealthResponse.getStatus(), not(ClusterHealthStatus.RED)); IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getSource().getIndexes().get(0)); IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet(); assertTrue(indicesExistsResponse.isExists()); SearchRequestBuilder countRequest = testClient .prepareSearch(testConfiguration.getSource().getIndexes().get(0)) .setTypes(testConfiguration.getSource().getTypes().get(0)); SearchResponse countResponse = countRequest.execute().actionGet(); count = (int)countResponse.getHits().getTotalHits(); PutIndexTemplateRequestBuilder putTemplateRequestBuilder = testClient.admin().indices().preparePutTemplate("mappings"); URL templateURL = ElasticsearchParentChildWriterIT.class.getResource("/ActivityChildObjectParent.json"); ObjectNode template = MAPPER.readValue(templateURL, ObjectNode.class); String templateSource = MAPPER.writeValueAsString(template); putTemplateRequestBuilder.setSource(templateSource); testClient.admin().indices().putTemplate(putTemplateRequestBuilder.request()).actionGet(); assertThat(count, not(0)); }
Example 5
Project: heroic File: Connection.java View source code | 5votes |
public AsyncFuture<Void> configure() { final IndicesAdminClient indices = client.getClient().admin().indices(); log.info("[{}] updating template for {}", templateName, index.template()); final PutIndexTemplateRequestBuilder put = indices.preparePutTemplate(templateName); put.setSettings(type.getSettings()); put.setTemplate(index.template()); for (final Map.Entry<String, Map<String, Object>> mapping : type.getMappings().entrySet()) { put.addMapping(mapping.getKey(), mapping.getValue()); } final ResolvableFuture<Void> future = async.future(); final ListenableActionFuture<PutIndexTemplateResponse> target = put.execute(); target.addListener(new ActionListener<PutIndexTemplateResponse>() { @Override public void onResponse(final PutIndexTemplateResponse response) { if (!response.isAcknowledged()) { future.fail(new Exception("request not acknowledged")); return; } future.resolve(null); } @Override public void onFailure(Exception e) { future.fail(e); } }); future.onCancelled(() -> target.cancel(false)); return future; }
Example 6
Project: foxtrot File: ElasticsearchUtils.java View source code | 5votes |
public static PutIndexTemplateRequest getClusterTemplateMapping(IndicesAdminClient indicesAdminClient) { try { PutIndexTemplateRequestBuilder builder = new PutIndexTemplateRequestBuilder(indicesAdminClient, "generic_template"); builder.setTemplate(String.format("%s-*", ElasticsearchUtils.TABLENAME_PREFIX)); System.out.println(getDocumentMapping().string()); builder.addMapping(DOCUMENT_TYPE_NAME, getDocumentMapping()); return builder.request(); } catch (IOException ex) { logger.error("TEMPLATE_CREATION_FAILED", ex); return null; } }
Example 7
Project: foxtrot File: InitializerCommand.java View source code | 5votes |
@Override protected void run(Bootstrap<FoxtrotServerConfiguration> bootstrap, Namespace namespace, FoxtrotServerConfiguration configuration) throws Exception { ElasticsearchConfig esConfig = configuration.getElasticsearch(); ElasticsearchConnection connection = new ElasticsearchConnection(esConfig); connection.start(); ClusterHealthResponse clusterHealth = new ClusterHealthRequestBuilder(connection.getClient().admin().cluster()) .execute() .get(); int numDataNodes = clusterHealth.getNumberOfDataNodes(); int numReplicas = (numDataNodes < 2) ? 0 : 1; logger.info("# data nodes: {}, Setting replica count to: {}", numDataNodes, numReplicas); createMetaIndex(connection, "consoles", numDataNodes - 1); createMetaIndex(connection, "table-meta", numDataNodes - 1); PutIndexTemplateResponse response = new PutIndexTemplateRequestBuilder(connection.getClient().admin().indices(), "template_foxtrot_mappings") .setTemplate(String.format("%s-*",configuration.getElasticsearch().getTableNamePrefix())) .setSettings( ImmutableSettings.builder() .put("number_of_shards", 10) .put("number_of_replicas", numReplicas) ) .addMapping("document", ElasticsearchUtils.getDocumentMapping()) .execute() .get(); logger.info("Create mapping: {}", response.isAcknowledged()); logger.info("Creating hbase table"); HBaseUtil.createTable(configuration.getHbase(), configuration.getHbase().getTableName()); }
Example 8
Project: Surf File: ElasticSearchTarget.java View source code | 5votes |
@Override public void open(VDSConfiguration ctx) throws Exception { _logger.info("Initializing ElasticSearch target"); _indexName = ctx.optString("index-name", "surf"); _typeName = ctx.optString("type-name", "http"); boolean useTemplate = ctx.optBoolean("use-template", true); if(!_indexName.startsWith("surf") && useTemplate){ _logger.error("The index name *must* start with 'surf' if the default Surf template is used"); _logger.error("Either use an index-name starting with 'surf' or set use-template to false and provide your own template"); throw new IllegalArgumentException("Invalid index name for default template"); } String cluster = ctx.optString("cluster-name", "elasticsearch"); _node = nodeBuilder() .client(true) .clusterName(cluster) .node(); _client = _node.client(); _logger.info("ElasticSearch target initialized"); if(useTemplate){ InputStream stream = getClass().getResourceAsStream("/surf-template.json"); ByteArrayOutputStream bos = new ByteArrayOutputStream(); byte buf[] = new byte[1024]; while(stream.read(buf) > 0){ bos.write(buf); } stream.close(); bos.close(); byte [] template = bos.toByteArray(); PutIndexTemplateRequestBuilder req = _client.admin().indices().preparePutTemplate("surf-template") .setSource(template); PutIndexTemplateResponse resp = req.execute().actionGet(); if(resp.isAcknowledged()){ _logger.info("Template loaded"); } } else{ _logger.info("Default Surf template not loaded - make sure you have configured ElasticSearch correctly"); } }
Example 9
Project: elasticsearch_my File: ESIntegTestCase.java View source code | 4votes |
/** * Creates a randomized index template. This template is used to pass in randomized settings on a * per index basis. Allows to enable/disable the randomization for number of shards and replicas */ public void randomIndexTemplate() throws IOException { // TODO move settings for random directory etc here into the index based randomized settings. if (cluster().size() > 0) { Settings.Builder randomSettingsBuilder = setRandomIndexSettings(random(), Settings.builder()); if (isInternalCluster()) { // this is only used by mock plugins and if the cluster is not internal we just can't set it randomSettingsBuilder.put(INDEX_TEST_SEED_SETTING.getKey(), random().nextLong()); } randomSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, numberOfShards()) .put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas()); // if the test class is annotated with SuppressCodecs("*"), it means don't use lucene's codec randomization // otherwise, use it, it has assertions and so on that can find bugs. SuppressCodecs annotation = getClass().getAnnotation(SuppressCodecs.class); if (annotation != null && annotation.value().length == 1 && "*".equals(annotation.value()[0])) { randomSettingsBuilder.put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)); } else { randomSettingsBuilder.put("index.codec", CodecService.LUCENE_DEFAULT_CODEC); } XContentBuilder mappings = null; if (frequently() && randomDynamicTemplates()) { mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_").endObject().endObject(); } for (String setting : randomSettingsBuilder.internalMap().keySet()) { assertThat("non index. prefix setting set on index template, its a node setting...", setting, startsWith("index.")); } // always default delayed allocation to 0 to make sure we have tests are not delayed randomSettingsBuilder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0); if (randomBoolean()) { randomSettingsBuilder.put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), randomBoolean()); } if (randomBoolean()) { randomSettingsBuilder.put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), randomBoolean()); } if (randomBoolean()) { randomSettingsBuilder.put(IndexModule.INDEX_QUERY_CACHE_TERM_QUERIES_SETTING.getKey(), randomBoolean()); } PutIndexTemplateRequestBuilder putTemplate = client().admin().indices() .preparePutTemplate("random_index_template") .setPatterns(Collections.singletonList("*")) .setOrder(0) .setSettings(randomSettingsBuilder); if (mappings != null) { logger.info("test using _default_ mappings: [{}]", mappings.bytes().utf8ToString()); putTemplate.addMapping("_default_", mappings); } assertAcked(putTemplate.execute().actionGet()); } }
Example 10
Project: elasticsearch_my File: AbstractClient.java View source code | 4votes |
@Override public PutIndexTemplateRequestBuilder preparePutTemplate(String name) { return new PutIndexTemplateRequestBuilder(this, PutIndexTemplateAction.INSTANCE, name); }
Example 11
Project: Elasticsearch File: AbstractClient.java View source code | 4votes |
@Override public PutIndexTemplateRequestBuilder preparePutTemplate(String name) { return new PutIndexTemplateRequestBuilder(this, PutIndexTemplateAction.INSTANCE, name); }
Example 12
Project: elasticsearch-sample-plugin-audit File: ElasticsearchIntegrationTest.java View source code | 4votes |
/** * Creates a randomized index template. This template is used to pass in randomized settings on a * per index basis. Allows to enable/disable the randomization for number of shards and replicas */ private void randomIndexTemplate() throws IOException { // TODO move settings for random directory etc here into the index based randomized settings. if (cluster().size() > 0) { final ImmutableSettings.Builder randomSettingsBuilder = setRandomSettings(getRandom(), ImmutableSettings.builder()).put( SETTING_INDEX_SEED, getRandom().nextLong()); if (randomizeNumberOfShardsAndReplicas()) { randomSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, numberOfShards()).put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas()); } XContentBuilder mappings = null; if (frequently() && randomDynamicTemplates()) { mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_"); if (randomBoolean()) { mappings.startObject(IdFieldMapper.NAME).field("index", randomFrom("not_analyzed", "no")).endObject(); } if (randomBoolean()) { mappings.startObject(TypeFieldMapper.NAME).field("index", randomFrom("no", "not_analyzed")).endObject(); } if (randomBoolean()) { mappings.startObject(TimestampFieldMapper.NAME).field("enabled", randomBoolean()).startObject("fielddata") .field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values")).endObject().endObject(); } if (randomBoolean()) { mappings.startObject(SizeFieldMapper.NAME).field("enabled", randomBoolean()).endObject(); } if (randomBoolean()) { mappings.startObject(AllFieldMapper.NAME).field("auto_boost", true).endObject(); } if (randomBoolean()) { mappings.startObject(SourceFieldMapper.NAME).field("compress", randomBoolean()).endObject(); } if (compatibilityVersion().onOrAfter(Version.V_1_3_0)) { mappings.startObject(FieldNamesFieldMapper.NAME).startObject("fielddata") .field(FieldDataType.FORMAT_KEY, randomFrom("paged_bytes", "fst", "doc_values")).endObject().endObject(); } mappings.startArray("dynamic_templates") .startObject() .startObject("template-strings") .field("match_mapping_type", "string") .startObject("mapping") .startObject("fielddata") .field(FieldDataType.FORMAT_KEY, randomFrom("paged_bytes", "fst")) // unfortunately doc values only work on not_analyzed fields .field(Loading.KEY, randomLoadingValues()).endObject().endObject().endObject().endObject().startObject() .startObject("template-longs").field("match_mapping_type", "long").startObject("mapping").startObject("fielddata") .field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values")) .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER)).endObject().endObject().endObject().endObject() .startObject().startObject("template-doubles").field("match_mapping_type", "double").startObject("mapping") .startObject("fielddata").field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values")) .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER)).endObject().endObject().endObject().endObject() .startObject().startObject("template-geo_points").field("match_mapping_type", "geo_point").startObject("mapping") .startObject("fielddata").field(FieldDataType.FORMAT_KEY, randomFrom("array", "doc_values")) .field(Loading.KEY, randomFrom(Loading.LAZY, Loading.EAGER)).endObject().endObject().endObject().endObject() .endArray(); mappings.endObject().endObject(); } final PutIndexTemplateRequestBuilder putTemplate = client().admin().indices().preparePutTemplate("random_index_template") .setTemplate("*").setOrder(0).setSettings(randomSettingsBuilder); if (mappings != null) { logger.info("test using _default_ mappings: [{}]", mappings.bytesStream().bytes().toUtf8()); putTemplate.addMapping("_default_", mappings); } assertAcked(putTemplate.execute().actionGet()); } }
Example 13
Project: elasticsearch_my File: IndicesAdminClient.java View source code | 2votes |
/** * Puts an index template. * * @param name The name of the template. */ PutIndexTemplateRequestBuilder preparePutTemplate(String name);
Example 14
Project: Elasticsearch File: IndicesAdminClient.java View source code | 2votes |
/** * Puts an index template. * * @param name The name of the template. */ PutIndexTemplateRequestBuilder preparePutTemplate(String name);