Official API method used by Elasticsearch in combination with Java
1. Preliminary preparation
- Create a new Maven project, it is recommended to use Springboot to build the project, and introduce the Elasticsearch dependency in pom.xml:
Note : There is no need to specify the version number here. Springboot will automatically load the corresponding version of Elasticsearch according to its own version. You can choose the version of Springboot according to the relationship diagram below.<dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-high-level-client</artifactId> //<version>7.3.0</version>//如果项目不是用Springboot搭建的，可以自己指定Elasticsearch的版本号 </dependency> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-client</artifactId> //<version>7.3.0</version>//如果项目不是用Springboot搭建的，可以自己指定Elasticsearch的版本号 </dependency> <dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch</artifactId> //<version>7.3.0</version>//如果项目不是用Springboot搭建的，可以自己指定Elasticsearch的版本号 </dependency>Elasticsearch Spring Boot 7.9.3 2.4.x 7.6.2 2.3.x 6.8.12 2.2.x 6.2.2 2.1.x 5.5.0 2.0.x 2.4.0 1.5.x - Create a configuration file application.yml under resources, which configures the connection properties of Elasticsearch:
Then create a new package and create a configuration class to load configuration properties:elasticsearch: ip: 127.0.0.1 port: 9200 //如果你安装的Elasticsearch没有设置用户名和密码的话，以下两个属性则填任意值就可以 username: elastic password: 123456
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; @Component @ConfigurationProperties(prefix = "elasticsearch") public class ElasticSearchConfig { private String ip; private int port; private String username; private String password; public String getIp() { return ip; } public void setIp(String ip) { this.ip = ip; } public int getPort() { return port; } public void setPort(int port) { this.port = port; } public String getUsername() { return username; } public void setUsername(String username) { this.username = username; } public String getPassword() { return password; } public void setPassword(String password) { this.password = password; } } - We use the configuration method to inject into the Spring container, so that there is no need to manually create and close the connection frequently during subsequent use. Create a new package and create the following two classes below:
import com.common.ElasticSearchConfig; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.io.IOException; @Component public class ESClientSpringFactory { public final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); public static int CONNECT_TIMEOUT_MILLIS = 1000; public static int SOCKET_TIMEOUT_MILLIS = 30000; public static int CONNECTION_REQUEST_TIMEOUT_MILLIS = 500; public static int MAX_CONN_PER_ROUTE = 10; public static int MAX_CONN_TOTAL = 30; private static HttpHost[] HTTP_HOST; private RestClientBuilder builder; private RestClient restClient; private RestHighLevelClient restHighLevelClient; private static ESClientSpringFactory esClientSpringFactory = new ESClientSpringFactory(); @Autowired private ElasticSearchConfig elasticSearchConfig; private ESClientSpringFactory() { } public static ESClientSpringFactory build(HttpHost[] httpHost, Integer maxConnectNum, Integer maxConnectPerRoute) { HTTP_HOST = httpHost; MAX_CONN_TOTAL = maxConnectNum; MAX_CONN_PER_ROUTE = maxConnectPerRoute; return esClientSpringFactory; } public static ESClientSpringFactory build(HttpHost[] httpHost, Integer connectTimeOut, Integer socketTimeOut, Integer connectionRequestTime, Integer maxConnectNum, Integer maxConnectPerRoute) { HTTP_HOST = httpHost; CONNECT_TIMEOUT_MILLIS = connectTimeOut; SOCKET_TIMEOUT_MILLIS = socketTimeOut; CONNECTION_REQUEST_TIMEOUT_MILLIS = connectionRequestTime; MAX_CONN_TOTAL = maxConnectNum; MAX_CONN_PER_ROUTE = maxConnectPerRoute; return esClientSpringFactory; } public void init() { builder = RestClient.builder(HTTP_HOST); setConnectTimeOutConfig(); setMutiConnectConfig(); restClient = builder.build(); restHighLevelClient = new RestHighLevelClient(builder); } public void setConnectTimeOutConfig() { builder.setRequestConfigCallback(requestConfigBuilder -> { requestConfigBuilder.setConnectTimeout(CONNECT_TIMEOUT_MILLIS); requestConfigBuilder.setSocketTimeout(SOCKET_TIMEOUT_MILLIS); requestConfigBuilder.setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT_MILLIS); return requestConfigBuilder; }); } public void setMutiConnectConfig() { credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticSearchConfig.getUsername(), elasticSearchConfig.getPassword())); builder.setHttpClientConfigCallback(httpClientBuilder -> { httpClientBuilder.setMaxConnTotal(MAX_CONN_TOTAL); httpClientBuilder.setMaxConnPerRoute(MAX_CONN_PER_ROUTE); httpClientBuilder.disableAuthCaching(); httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); return httpClientBuilder; }); } public RestClient getClient() { return restClient; } public RestHighLevelClient getRhlClient() { return restHighLevelClient; } public void close() { if (restClient != null) { try { restClient.close(); } catch (IOException e) { e.printStackTrace(); } } } }import com.alibaba.fastjson.JSONArray; import com.common.ElasticSearchConfig; import org.apache.http.HttpHost; import org.elasticsearch.client.RestHighLevelClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration public class ESConfig { private static final String HTTP_SCHEME = "http"; @Autowired private ElasticSearchConfig elasticSearchConfig; @Bean public HttpHost[] httpHost() { JSONArray array = new JSONArray(); if (elasticSearchConfig.getIp().indexOf(",") != -1) { for (String ip : elasticSearchConfig.getIp().split(",")) { array.add(new HttpHost(ip, elasticSearchConfig.getPort(), HTTP_SCHEME)); } } else { array.add(new HttpHost(elasticSearchConfig.getIp(), elasticSearchConfig.getPort(), HTTP_SCHEME)); } HttpHost[] hosts = array.toArray(new HttpHost[0]); return hosts; } @Bean(initMethod = "init", destroyMethod = "close") public ESClientSpringFactory getFactory() { return ESClientSpringFactory.build(httpHost(), 300, 100); } @Bean(name = "highLevelClient") public RestHighLevelClient highLevelClient() { return getFactory().getRhlClient(); } }
2. Create a new index
public class CreateIndexes {
@Autowired
private RestHighLevelClient highLevelClient;
public void createDemoIndex(String indexname) {
CreateIndexRequest request = new CreateIndexRequest(indexname);
try {
//判断是否已存在该索引，如果存在则先删除
GetIndexRequest exist = new GetIndexRequest(indexname);
if (highLevelClient.indices().exists(exist, RequestOptions.DEFAULT)) {
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexname);
highLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT);
}
XContentBuilder builder = XContentFactory.jsonBuilder().startObject()
.startObject("properties")
.startObject("demoName")
.field("type", "text")
.field("analyzer", "keyword")
.endObject()
.startObject("demoValue")
.field("type", "text")
.field("analyzer", "keyword")
.endObject()
.startObject("demoNumber")
.field("type", "integer")
.endObject()
.startObject("demoTime")
.field("type", "date")
.endObject()
.endObject()
.endObject();
request.mapping(builder);
highLevelClient.indices().create(request, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
}
}
3. Query documents
public class GetDocument {
@Autowired
private RestHighLevelClient highLevelClient;
public DemoBO getDocument(String indexName) {
DemoBO bo = new DemoBO();
try {
SearchRequest searchRequest = new SearchRequest(indexName);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//searchSourceBuilder.query(QueryBuilders.matchAllQuery()).from(0).size(10).sort("demoTime", SortOrder.DESC);//查询全部文档用此方法
searchSourceBuilder.query(QueryBuilders.matchPhraseQuery(indexName, "abc")).from(0).size(10).sort("demoTime", SortOrder.DESC);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchHits searchHits = searchResponse.getHits();
if (searchHits.getTotalHits().value > 0) {
for (SearchHit hit: searchHits) {
if (hit.getSourceAsMap().get("demoName") != null) {
bo.setDemoName(hit.getSourceAsMap().get("demoName"))；
}
if (hit.getSourceAsMap().get("demoValue") != null) {
bo.setDemoValue(hit.getSourceAsMap().get("demoValue"))；
}
if (hit.getSourceAsMap().get("demoNumber") != null) {
bo.setDemoNumber(hit.getSourceAsMap().get("demoNumber"))；
}
if (hit.getSourceAsMap().get("demoTime") != null) {
bo.setDemoTime(hit.getSourceAsMap().get("demoTime"))；
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
return bo;
}
}
4. New documents
public class PutDocument {
@Autowired
private RestHighLevelClient highLevelClient;
public void putDocument(String indexName, String id) {
try {
IndexRequest indexRequest = new IndexRequest(indexName);
//IndexRequest indexRequest = new IndexRequest(indexName).id(id);//如果你要自己定义文档ID的值，可以这么写，否则Elasticsearch会自动帮你生成一个，不过官方也是不建议让你来建。
indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);//此句功能为立即刷新索引文档，不然你会在页面上发现我明明已经新增一个文档了，却要过个几秒钟再刷新页面才会显示出来。
Map<String, Object> map = new HashMap<>();
map.put("demoName", "abc");
map.put("demoValue", "123");
map.put("demoNumber", 1);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
map.put("demoTime", sdf.format(new Date()));
indexRequest.source(map);
highLevelClient.index(indexRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
}
}
5. Update documents
public class PostDocument {
@Autowired
private RestHighLevelClient highLevelClient;
public void postDocument(String indexName, String id) {
try {
UpdateRequest updateRequest = new UpdateRequest(indexName, id);
updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);//跟上面新增文档那里一样的作用
Map<String, Object> map = new HashMap<>();
map.put("demoName", "def");
map.put("demoValue", "456");
map.put("demoNumber", 2);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
map.put("demoTime", sdf.format(new Date()));
updateRequest.doc(map);
highLevelClient.update(updateRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
}
}
6. Delete documents
public class DeleteDocument {
@Autowired
private RestHighLevelClient highLevelClient;
public void deleteDocument(String indexName, String id) {
try {
DeleteRequest deleteRequest = new DeleteRequest(indexName, id);
deleteRequest .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);//跟上面新增文档那里一样的作用
highLevelClient.delete(deleteRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
}
}
7. Conditional query (Boolean query)
public class BoolGetDocument {
@Autowired
private RestHighLevelClient highLevelClient;
public DemoBO boolGetDocument(String indexName) {
DemoBO bo = new DemoBO();
try {
SearchRequest searchRequest = new SearchRequest(indexName);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery("demoName", "abc"));
boolQueryBuilder.should(QueryBuilders.matchPhraseQuery("demoValue", "123"));
boolQueryBuilder.should(QueryBuilders.rangeQuery("demoNumber").gte(1)).should(QueryBuilders.rangeQuery("demoNumber").lte(5));
searchSourceBuilder.query(boolQueryBuilder).from(0).size(10).sort("demoTime", SortOrder.DESC);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
SearchHits searchHits = searchResponse.getHits();
if (searchHits.getTotalHits().value > 0) {
for (SearchHit hit: searchHits) {
if (hit.getSourceAsMap().get("demoName") != null) {
bo.setDemoName(hit.getSourceAsMap().get("demoName"))；
}
if (hit.getSourceAsMap().get("demoValue") != null) {
bo.setDemoValue(hit.getSourceAsMap().get("demoValue"))；
}
if (hit.getSourceAsMap().get("demoNumber") != null) {
bo.setDemoNumber(hit.getSourceAsMap().get("demoNumber"))；
}
if (hit.getSourceAsMap().get("demoTime") != null) {
bo.setDemoTime(hit.getSourceAsMap().get("demoTime"))；
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
return bo;
}
}
Eight, query all documents
public class GetAllDocuments {
@Autowired
private RestHighLevelClient highLevelClient;
public List<DemoBO> getAllDocuments(String indexName) {
List<DemoBO> list = new ArrayList<>();
try {
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchAllQuery()).size(1000);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
StringBuilder scrollId = new StringBuilder(searchResponse.getScrollId());
SearchHits searchHits = searchResponse.getHits();
while (searchHits.getHits() != null && searchHits.getTotalHits().value > 0) {
for (SearchHit hit: searchHits) {
DemoBO bo = new DemoBO();
if (hit.getSourceAsMap().get("demoName") != null) {
bo.setDemoName(hit.getSourceAsMap().get("demoName"))；
}
if (hit.getSourceAsMap().get("demoValue") != null) {
bo.setDemoValue(hit.getSourceAsMap().get("demoValue"))；
}
if (hit.getSourceAsMap().get("demoNumber") != null) {
bo.setDemoNumber(hit.getSourceAsMap().get("demoNumber"))；
}
if (hit.getSourceAsMap().get("demoTime") != null) {
bo.setDemoTime(hit.getSourceAsMap().get("demoTime"))；
}
list.add(bo);
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId.toString());
scrollRequest.scroll(scroll);
searchResponse = highLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId.replace(0, scrollId.length(), searchResponse.getScrollId());
searchHits = searchResponse.getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId.toString());
highLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
}
9. Aggregation query
public class GetAggregation {
@Autowired
private RestHighLevelClient highLevelClient;
public List<DemoBO> getAggregation (String indexName) {
List<DemoBO> list = new ArrayList<>();
try {
SearchRequest searchRequest = new SearchRequest(indexName);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
AggregationBuilder aggregation = AggregationBuilders.terms("qc").field("demoName.keyword").size(10).subAggregation(AggregationBuilders.topHits("tj").sort("demoName.keyword", SortOrder.DESC).size(1));//"qc"和"tj"是自定义名称，可以自己取，"qc"那部分指的是去重，"tj"那部分指的是统计，相当于SQL查询中的Group by。
searchSourceBuilder.query(QueryBuilders.matchPhraseQuery("demoValue", "123")).aggregation(aggregation).size(0);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
Terms terms = searchResponse.getAggregations().get("qc");
for (Bucket b: terms.getBuckets()) {
DemoBO bo = new DemoBO ();
bo.setDemoName(b.getKeyAsString());
TopHits top = b.getAggregations().get("tj");
for (SearchHit hit: top.getHits()) {
if (hit.getSourceAsMap().get("demoValue") != null) {
bo.setDemoValue(hit.getSourceAsMap().get("demoValue").toString());
} else {
bo.setDemoValue("");
}
}
list.add(bo);
}
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
}
10. Batch import data
The import from Oracle database to Elasticsearch is used here, which can be used as a reference, and the implementation method is the same.
public class OracleToES {
@Autowired
private RestHighLevelClient highLevelClient;
public void writeOracleDataToES(String indexName, String sql) {
BulkProcessor bulkProcessor = getBulkProcessor();
Connection conn = null;
PreparedStatement ps = null;
ResultSet rs = null;
try {
System.out.println("开始导入数据: " + indexName);
Class.forName("oracle.jdbc.driver.OracleDriver");
conn = DriverManager.getConnection("jdbc:oracle:thin:@127.0.0.1:1521:sjk", "demo", "123456");
ps = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
ps.setFetchSize(0);
rs = ps.executeQuery();
ResultSetMetaData colData = rs.getMetaData();
List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> map;
int count = 0;
StringBuilder c = new StringBuilder("0");
Object v;
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
while (rs.next()) {
count++;
map = new HashMap<>(100);
for (int i = 1; i <= colData.getColumnCount(); i++) {
c.replace(0, c.length(), colData.getColumnName(i));
if ("DEMO_TIME".equals(c.toString())) {
//Oracle字段为Date类型时需要单独处理，否则导入会报错
if (rs.getDate(c.toString()) != null) {
v = sdf.format(rs.getDate(c.toString()));
} else {
v = null;
}
} else {
if (rs.getObject(c.toString()) != null) {
v = rs.getObject(c.toString());
} else {
v = null;
}
}
map.put(c.toString().toLowerCase(), v);//Oracle字段默认为大写，需要转换为小写
}
dataList.add(map);
if (count % 100000 == 0) {
//每10万条数据提交一次
System.out.println("已提交数据量: " + count);
for (HashMap<String, Object> hashMap: dataList) {
bulkProcessor.add(new IndexRequest(indexName.toLowerCase()).source(hashMap));
}
map.clear();
dataList.clear();
}
}
for (HashMap<String, Object> hashMap: dataList) {
//提交剩余的数据
bulkProcessor.add(new IndexRequest(indexName.toLowerCase()).source(hashMap));
}
System.out.println("成功提交数据量: " + count);
bulkProcessor.flush();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
rs.close();
ps.close();
conn.close();
boolean terminatedFlag = bulkProcessor.awaitClose(150L, TimeUnit.SECONDS);
System.out.println(terminatedFlag);
} catch (Exception e) {
System.out.println(e.getMessage());
}
}
}
private BulkProcessor getBulkProcessor() {
BulkProcessor bulkProcessor = null;
try {
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
public void beforeBulk(long executionId, BulkRequest request) {
System.out.println("尝试导入数据量: " + request.numberOfActions());
}
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
System.out.println("成功导入数据量: " + request.numberOfActions() + " , id: " + executionId);
}
@Override
public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
System.out.println("导入失败: " + failure + ", executionId: " + executionId);
}
};
BiConsumer<BulkRequest, ActionListener<BulkResponse>> bulkConsumer = (request, bulkListener) -> highLevelClient.bulkAsync(request, RequestOptions.DEFAULT, bulkListener);
BulkProcessor.Builder builder = BulkProcessor.builder(bulkConsumer, listener);
builder.setBulkActions(1000);
builder.setBulkSize(new ByteSizeValue(100L, ByteSizeUnit.MB));
builder.setConcurrentRequests(10);
builder.setFlushInterval(TimeValue.timeValueSeconds(5L));
builder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3));
bulkProcessor = builder.build();
} catch (Exception e) {
e.printStackTrace();
try {
bulkProcessor.awaitClose(100L, TimeUnit.SECONDS);
} catch (Exception e1) {
System.out.println(e1.getMessage());
}
}
return bulkProcessor;
}
}