package com.topwalk.analysis.es.operation.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.apache.log4j.Logger;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.bulk.BulkProcessor.Listener;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.count.CountRequestBuilder;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogram;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.metrics.stats.Stats;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.sort.SortOrder;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.topwalk.analysis.es.configuration.EsAggregation;
import com.topwalk.analysis.es.configuration.EsAggregationParams;
import com.topwalk.analysis.es.configuration.EsQueryParam;
import com.topwalk.analysis.es.configuration.LogicOperator;
import com.topwalk.analysis.es.configuration.Operator;
import com.topwalk.analysis.es.configuration.Pager;
import com.topwalk.analysis.es.factory.EsClient;
import com.topwalk.analysis.es.operation.intf.EsService;
public class EsServiceImpl implements EsService{
private Logger logger = Logger.getLogger(EsServiceImpl.class);
@Override
public boolean indexDataById(Client client, String index, String type, String id,
String data) {
// TODO Auto-generated method stub
IndexResponse response = null;
try {
response = client.prepareIndex(index, type, id).setSource(data).execute().actionGet();
} catch (Exception e) {
e.printStackTrace();
return false;
}
return response.isCreated();
}
@Override
public boolean bulkindexData(Client client, String index, String type,
String data) {
// TODO Auto-generated method stub
BulkRequestBuilder bulkRequest = client.prepareBulk();
JSONArray arry = JSON.parseArray(data);
for(int i = 0; i < arry.size(); i++) {
JSONObject jobj = JSON.parseObject(arry.getString(i));
if(jobj.containsKey("_id")) {
String _id = jobj.getString("_id");
jobj.remove("_id");
bulkRequest.add(client.prepareIndex(index, type, _id).setSource(arry.getString(i)) );
} else {
bulkRequest.add(client.prepareIndex(index, type).setSource(arry.getString(i)) );
}
}
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if(bulkResponse.hasFailures()) {
logger.error("bulkinsert fail message...." + bulkResponse.buildFailureMessage());
return false;
} else {
return true;
}
}
@Override
public boolean bulkupdateData(Client client, String index, String type,
String data, List<EsQueryParam> params) {
// TODO Auto-generated method stub
List<String> types = new ArrayList<String>();
types.add(type);
EsServiceImpl esimpl = new EsServiceImpl();
Pager page = new Pager();
page.setFrom(0);
int count = esimpl.getCountByFields(client, index, types, params);
page.setNum(count);
List<String> ids = esimpl.getIdByFields(client, index, type, params, page);
if(ids.size() == 0 || count == 0) {
return false;
}
Map<String, Object> fieldValue = (Map<String, Object>) JSON.parse(data);
for (String id : ids ) {
esimpl.updateIndexById(client, index, type, id, fieldValue);
}
return true;
}
@Override
public boolean indexData(Client client, String index, String type, String data) {
// TODO Auto-generated method stub
IndexResponse response = null;
try {
response = client.prepareIndex(index, type).setSource(data).execute().actionGet();
} catch (Exception e) {
e.printStackTrace();
return false;
}
return response.isCreated();
}
@Override
public String getIndexById(Client client, String index, String type, String id) {
// TODO Auto-generated method stub
GetResponse response = null ;
String result = null;
try {
response = client.prepareGet(index, type, id).execute().actionGet();
result = response.getSourceAsString();
} catch (Exception e ) {
e.printStackTrace();
}
return result;
}
@Override
public boolean isIndexExists(Client client, String index, String type, String id) {
// TODO Auto-generated method stub
GetResponse response = null ;
boolean result = false;
try {
response = client.prepareGet(index, type, id).execute().actionGet();
result = response.isExists();
} catch (Exception e ) {
e.printStackTrace();
return false;
}
return result;
}
@Override
public boolean delIndexById(Client client, String index, String type, String id) {
// TODO Auto-generated method stub
DeleteResponse response = null ;
boolean result = false;
try {
response = client.prepareDelete(index, type, id).execute().actionGet();
result = response.isFound();
} catch (Exception e ) {
e.printStackTrace();
return result;
}
return result;
}
@Override
public boolean delIndexByFields(Client client, String index, String type,
List<EsQueryParam> params) {
// TODO Auto-generated method stub
BoolQueryBuilder db = QueryBuilders.boolQuery();
try {
DeleteByQueryRequestBuilder dbq = client.prepareDeleteByQuery(index).setTypes(type);
db = getQueryBuilder(params);
dbq.setQuery(db);
dbq.execute().actionGet();
} catch (Exception e ) {
e.printStackTrace();
return true;
}
return true;
}
@Override
public boolean updateIndexById(Client client, String index, String type, String id,
Map<String, Object> fieldValue) {
UpdateResponse response = null;
UpdateRequest updateRequest = new UpdateRequest();
updateRequest.index(index);
if(type != null) {
updateRequest.type(type);
}
updateRequest.id(id);
updateRequest.doc(fieldValue);
updateRequest.retryOnConflict(5);
try {
response = client.update(updateRequest).get();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return false;
} catch (ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return false;
}
return true;
}
@Override
public boolean addFieldToDoc(Client client, String index, String type, String id,
Map<String, Object> fieldValue) {
// TODO Auto-generated method stub
Iterator<String> iter = fieldValue.keySet().iterator();
boolean isExist = false;
while (iter.hasNext()) {
String key = iter.next();
isExist = isFieldExist(client, index, type, id, key);
if(isExist) {
try {
throw new Exception("the key [" + key + "] is exist!");
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
}
UpdateResponse response = null;
UpdateRequest updateRequest = new UpdateRequest();
updateRequest.index(index).type(type).id(id);
updateRequest.doc(fieldValue);
try {
response = client.update(updateRequest).get();
} catch (InterruptedException e) {
e.printStackTrace();
return false;
} catch (ExecutionException e) {
e.printStackTrace();
return false;
}
return true;
}
@Override
public boolean removeieldFromDoc(Client client, String index, String type, String id,
String field) {
boolean isExist = false;
isExist = isFieldExist( client, index, type, id, field);
if(! isExist) {
try {
throw new Exception("the key [" + field + "] is not exist!");
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
try {
client.prepareUpdate(index, type, id).setScript("ctx._source.remove(\"" + field+ "\")", ScriptService.ScriptType.INLINE).execute().actionGet();
} catch (Exception e) {
e.printStackTrace();
return false;
}
return true;
}
public List<String> getIdByFields(Client client, String index, String type,
List<EsQueryParam> params, Pager pager) {
BoolQueryBuilder db = QueryBuilders.boolQuery();
List<String> list= new ArrayList<String>();
SearchRequestBuilder searchRequest = null;
db = getQueryBuilder(params);
if (params != null && params.size() > 0 ) {
searchRequest = client.prepareSearch(index).setQuery(db).setTypes(type);
} else {
searchRequest = client.prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()).setTypes(type);
}
if(pager != null && pager.getFrom() >= 0 && pager.getNum() > 0 ) {
searchRequest.setFrom(pager.getFrom()).setSize(pager.getNum());
} else if(pager != null && pager.getSortField() != null && pager.getSortValue().equals( "ASC")) {
searchRequest.addSort(pager.getSortField(), SortOrder.ASC);
} else if(pager != null && pager.getSortField() != null && pager.getSortValue().equals("desc")) {
searchRequest.addSort(pager.getSortField(), SortOrder.DESC);
}
SearchResponse response = searchRequest.execute().actionGet();
for(SearchHit hit : response.getHits()) {
list.add(hit.getId());
}
return list;
}
@Override
public List<String> getIndexByFields(Client client, String index, List<String> types,
List<EsQueryParam> params, Pager pager) {
BoolQueryBuilder db = QueryBuilders.boolQuery();
List<String> list= new ArrayList<String>();
try {
SearchRequestBuilder searchRequest = null;
if (params != null && params.size() > 0 ) {
db = getQueryBuilder(params);
searchRequest = client.prepareSearch(index).setQuery(db);
} else {
searchRequest = client.prepareSearch(index).setQuery(QueryBuilders.matchAllQuery());
}
if(pager != null && pager.getFrom() >= 0 && pager.getNum() > 0 ) {
searchRequest.setFrom(pager.getFrom()).setSize(pager.getNum());
} else if(pager != null && pager.getSortField() != null && pager.getSortValue().equals("asc")) {
searchRequest.addSort(pager.getSortField(), SortOrder.ASC);
} else if(pager != null && pager.getSortField() != null && pager.getSortValue().equals("desc")) {
searchRequest.addSort(pager.getSortField(), SortOrder.DESC);
}
if(types != null && types.size() > 0) {
String[] a = new String[types.size()];
searchRequest.setTypes(types.toArray(a));
}
SearchResponse response = searchRequest.execute().actionGet();
if(response.getHits().totalHits() > 0) {
for(SearchHit hit : response.getHits()) {
list.add(JSON.toJSONString(hit.getSource()));
}
}
} catch (Exception e) {
e.printStackTrace();
}
return list;
}
@Override
public List<String> scrollSearch(Client client, String index, String type,
List<EsQueryParam> params, Pager pager) {
BoolQueryBuilder db = QueryBuilders.boolQuery();
List<String> list= new ArrayList<String>();
try {
SearchRequestBuilder searchRequest = null;
if (params != null && params.size() > 0 ) {
db = getQueryBuilder(params);
searchRequest = client.prepareSearch(index).setSearchType(SearchType.SCAN).setScroll(new TimeValue(60000)).setQuery(db);
} else {
searchRequest = client.prepareSearch(index).setSearchType(SearchType.SCAN).setScroll(new TimeValue(60000)).setQuery(QueryBuilders.matchAllQuery());
}
if (type != null) {
searchRequest.setTypes(type);
}
if(pager != null) {
searchRequest.setSize(pager.getNum());
}
SearchResponse scrollResp = searchRequest.execute().actionGet();
while (true) {
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet();
if(scrollResp.getHits().getHits().length == 0) {
break;
}
for(SearchHit hit: scrollResp.getHits()) {
list.add(hit.getSourceAsString());
}
}
} catch(Exception e) {
e.printStackTrace();
}
return list;
}
@Override
public boolean reindex(Client client, String index, String type,String newindex, String newtype, List<EsQueryParam> params) {
BoolQueryBuilder db = QueryBuilders.boolQuery();
try {
SearchRequestBuilder searchRequest = null;
if (params != null && params.size() > 0 ) {
db = getQueryBuilder(params);
searchRequest = client.prepareSearch(index).setSearchType(SearchType.SCAN).setScroll(new TimeValue(60000)).setQuery(db);
} else {
searchRequest = client.prepareSearch(index).setSearchType(SearchType.SCAN).setScroll(new TimeValue(60000)).setQuery(QueryBuilders.matchAllQuery());
}
if (type != null) {
searchRequest.setTypes(type);
}
SearchResponse scrollResp = searchRequest.execute().actionGet();
BulkProcessor bulkProcessor = BulkProcessor.builder(client, createBulkProcessorListener())
.setBulkActions(10000)
.setFlushInterval(TimeValue.timeValueSeconds(5)).build();
while (true) {
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet();
if(scrollResp.getHits().getHits().length == 0) {
bulkProcessor.close();
break;
}
for(SearchHit hit: scrollResp.getHits()) {
IndexRequest request = new IndexRequest(newindex, newtype, hit.id());
request.source(hit.getSourceAsString());
bulkProcessor.add(request);
}
bulkProcessor.flush();
}
} catch(Exception e) {
e.printStackTrace();
return false;
}
return true;
}
public Map<String, Object> getMappings(Client client, String index, String type) {
Map<String, Object> result = null;
IndexMetaData imd = null;
try {
ClusterState cs = client.admin()
.cluster()
.prepareState()
.setIndices(index)
.execute()
.actionGet()
.getState();
imd = cs.getMetaData().index(index);
MappingMetaData mdd = imd.mapping(type);
result = mdd.getSourceAsMap();
} catch (IndexMissingException | IOException e) {
}
return result;
}
public Listener createBulkProcessorListener() {
return new BulkProcessor.Listener() {
@Override
public void beforeBulk(long executionId, BulkRequest request) {
// TODO Auto-generated method stub
}
@Override
public void afterBulk(long executionId, BulkRequest request,
Throwable failure) {
// TODO Auto-generated method stub
}
@Override
public void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
// TODO Auto-generated method stub
}
};
}
@Override
public int getCountByFields(Client client, String index, List<String> types,
List<EsQueryParam> params) {
CountResponse response = null;
CountRequestBuilder countBuilder = null;
BoolQueryBuilder db = null;
try {
db = getQueryBuilder(params);
countBuilder = client.prepareCount(index);
if(types != null && types.size() > 0) {
String[] a = new String[types.size()];
countBuilder.setTypes(types.toArray(a));
}
if (params != null && params.size() > 0) {
response = countBuilder.setQuery(db).execute().actionGet();
} else {
response = countBuilder.setQuery(QueryBuilders.matchAllQuery()).execute().actionGet();
}
} catch (Exception e) {
e.printStackTrace();
}
return (int) response.getCount();
}
//check field isexist
public boolean isFieldExist(Client client, String index, String type, String id, String key) {
GetResponse response = null ;
boolean result = false;
try {
response = client.prepareGet(index, type, id).execute().actionGet();
result = JSON.parseObject(response.getSourceAsString()).containsKey(key);
} catch (Exception e ) {
e.printStackTrace();
}
return result;
}
@Override
public int getTermsCountByAgg(Client client, String index,
List<String> types, List<EsQueryParam> params,
List<EsAggregationParams> aggList, Pager page) {
BoolQueryBuilder db = null;
if(params != null) {
db = getQueryBuilder(params);
}
List<Map<String, String>> result = new ArrayList<Map<String, String>>();
AbstractAggregationBuilder parentBuilder = null;
for (int i = aggList.size()-1; i >= 0; i--) {
EsAggregationParams aggParams = aggList.get(i);
if(i == aggList.size()-1) {
if (aggParams.getEsAggregation() == EsAggregation.TERM) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(page.getNum());
}
} else {
if (aggParams.getEsAggregation() == EsAggregation.TERM) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(page.getNum()).subAggregation(parentBuilder);
}
}
}
SearchRequestBuilder requestBuilder = client.prepareSearch(index);
if(types != null && types.size() > 0) {
String[] a = new String[types.size()];
requestBuilder.setTypes(types.toArray(a));
}
requestBuilder.addAggregation(parentBuilder);
if(db != null) {
requestBuilder.setQuery(db);
}
SearchResponse sr = requestBuilder.execute().actionGet();
Terms t = sr.getAggregations().get(aggList.get(0).getAggName());
Iterator< ? extends MultiBucketsAggregation.Bucket > topBucket = null;
topBucket = t.getBuckets().iterator();
result = getMap(topBucket, aggList, 0, result, null);
return result.size();
}
@Override
public List<Map<String, String>> AggregationByFields(Client client, String index,
List<String> types, List<EsQueryParam> params, List<EsAggregationParams> aggList) {
BoolQueryBuilder db = null;
if(params != null && params.size() > 0) {
db = getQueryBuilder(params);
}
List<Map<String, String>> result = new ArrayList<Map<String, String>>();
AbstractAggregationBuilder parentBuilder = null;
for (int i = aggList.size()-1; i >= 0; i--) {
EsAggregationParams aggParams = aggList.get(i);
if(i == aggList.size()-1) {
if (aggParams.getEsAggregation() == EsAggregation.STATS) {
parentBuilder = AggregationBuilders.stats( aggParams.getAggName()) .field(aggParams.getField());
} else if (aggParams.getEsAggregation() == EsAggregation.DATE_HISTOGRAM) {
DateHistogram.Order order = null;
if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) {
order = DateHistogram.Order.KEY_ASC;
} else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) {
order = DateHistogram.Order.KEY_DESC;
}else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) {
order = DateHistogram.Order.COUNT_ASC;
} else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){
order = DateHistogram.Order.COUNT_DESC;
}
parentBuilder = AggregationBuilders.dateHistogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getDateInterval()).order(order);
} else if (aggParams.getEsAggregation() == EsAggregation.TERM) {
Terms.Order order = null;
if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) {
order = Terms.Order.term(true);
} else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) {
order = Terms.Order.term(false);
}else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) {
order = Terms.Order.count(true);
} else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){
order = Terms.Order.count(false);
}
if( aggParams.isKeyOrder() == null && aggParams.isValOrder() == null) {
if (aggParams.getAggSize() == 0) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize());
} else {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize());
}
} else {
if (aggParams.getAggSize() == 0) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).order(order);
} else {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).order(order);
}
}
} else if (aggParams.getEsAggregation() == EsAggregation.TERM_SCRIPTS) {
Terms.Order order = null;
if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) {
order = Terms.Order.term(true);
} else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) {
order = Terms.Order.term(false);
}else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) {
order = Terms.Order.count(true);
} else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){
order = Terms.Order.count(false);
}
if( aggParams.isKeyOrder() == null || aggParams.isValOrder() == null) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField());
} else {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()).order(order);
}
} else if (aggParams.getEsAggregation() == EsAggregation.HISTOGRAM) {
parentBuilder = AggregationBuilders.histogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getHisaggrateInterval());
} else if (aggParams.getEsAggregation() == EsAggregation.SUM) {
parentBuilder = AggregationBuilders.sum(aggParams.getAggName()).field(aggParams.getField());
}
} else {
if (aggParams.getEsAggregation() == EsAggregation.TERM) {
Terms.Order order = null;
if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) {
order = Terms.Order.term(true);
} else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) {
order = Terms.Order.term(false);
}else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) {
order = Terms.Order.count(true);
} else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){
order = Terms.Order.count(false);
}
if( aggParams.isKeyOrder() == null && aggParams.isValOrder() == null) {
if (aggParams.getAggSize() == 0 ) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).subAggregation(parentBuilder);
} else {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).subAggregation(parentBuilder);
}
} else {
if (aggParams.getAggSize() == 0 ) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).order(order).size(aggParams.getAggSize()).subAggregation(parentBuilder);
} else {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).order(order).size(aggParams.getAggSize()).subAggregation(parentBuilder);
}
}
} else if (aggParams.getEsAggregation() == EsAggregation.TERM_SCRIPTS) {
Terms.Order order = null;
if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) {
order = Terms.Order.term(true);
} else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) {
order = Terms.Order.term(false);
}else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) {
order = Terms.Order.count(true);
} else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){
order = Terms.Order.count(false);
}
if( aggParams.isKeyOrder() == null || aggParams.isValOrder() == null) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()).subAggregation(parentBuilder);
} else {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()).order(order).subAggregation(parentBuilder);
}
} else if (aggParams.getEsAggregation() == EsAggregation.DATE_HISTOGRAM) {
DateHistogram.Order order = null;
if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) {
order = DateHistogram.Order.KEY_ASC;
} else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) {
order = DateHistogram.Order.KEY_DESC;
}else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) {
order = DateHistogram.Order.COUNT_ASC;
} else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){
order = DateHistogram.Order.COUNT_DESC;
}
parentBuilder = AggregationBuilders.dateHistogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getDateInterval()).subAggregation(parentBuilder).order(order);
} else if (aggParams.getEsAggregation() == EsAggregation.HISTOGRAM) {
parentBuilder = AggregationBuilders.histogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getHisaggrateInterval()).subAggregation(parentBuilder);
}
}
}
SearchRequestBuilder requestBuilder = client.prepareSearch(index);
if(types != null && types.size() > 0) {
String[] a = new String[types.size()];
requestBuilder.setTypes(types.toArray(a));
}
requestBuilder.addAggregation(parentBuilder);
if(db != null) {
requestBuilder.setQuery(db);
}
logger.debug(requestBuilder.toString());
SearchResponse sr = requestBuilder.execute().actionGet();
Iterator< ? extends MultiBucketsAggregation.Bucket > topBucket = null;
if(aggList.get(0).getEsAggregation() == EsAggregation.DATE_HISTOGRAM) {
DateHistogram agg = sr.getAggregations().get(aggList.get(0).getAggName());
topBucket = agg.getBuckets().iterator();
} else if (aggList.get(0).getEsAggregation() == EsAggregation.TERM ||aggList.get(0).getEsAggregation() == EsAggregation.TERM_SCRIPTS ) {
Terms t = sr.getAggregations().get(aggList.get(0).getAggName());
topBucket = t.getBuckets().iterator();
} else if (aggList.get(0).getEsAggregation() == EsAggregation.HISTOGRAM) {
Histogram agg = sr.getAggregations().get(aggList.get(0).getAggName());
topBucket = agg.getBuckets().iterator();
} else if (aggList.get(0).getEsAggregation() == EsAggregation.SUM) {
Sum agg = sr.getAggregations().get(aggList.get(0).getAggName());
Map<String, String> sumresult = new HashMap<String, String>();
sumresult.put(agg.getName(), String.valueOf(agg.getValue()));
result.add(sumresult);
return result;
} else if(aggList.get(0).getEsAggregation() == EsAggregation.STATS) {
Stats stats = sr.getAggregations().get(aggList.get(0).getAggName());
Map<String, String> statsresult = new HashMap<String, String>();
statsresult.put("count", String.valueOf(stats.getCount()));
statsresult.put("avg", String.valueOf(stats.getAvg()));
statsresult.put("max", String.valueOf(stats.getMax()));
statsresult.put("min", String.valueOf(stats.getMin()));
statsresult.put("sum", String.valueOf(stats.getSum()));
result.add(statsresult);
return result;
}
Map<String, String> map = new HashMap<String, String>();
result = getMap(topBucket, aggList, 0, result, map);
return result;
}
@Override
public SearchResponse getResponseByAggregationField(Client client,
String index, List<String> types, List<EsQueryParam> params,
List<EsAggregationParams> aggList) {
BoolQueryBuilder db = null;
if(params != null && params.size() > 0) {
db = getQueryBuilder(params);
}
List<Map<String, String>> result = new ArrayList<Map<String, String>>();
AbstractAggregationBuilder parentBuilder = null;
for (int i = aggList.size()-1; i >= 0; i--) {
EsAggregationParams aggParams = aggList.get(i);
if(i == aggList.size()-1) {
if (aggParams.getEsAggregation() == EsAggregation.STATS) {
parentBuilder = AggregationBuilders.stats( aggParams.getAggName()) .field(aggParams.getField());
} else if (aggParams.getEsAggregation() == EsAggregation.DATE_HISTOGRAM) {
DateHistogram.Order order = null;
if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) {
order = DateHistogram.Order.KEY_ASC;
} else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) {
order = DateHistogram.Order.KEY_DESC;
}else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) {
order = DateHistogram.Order.COUNT_ASC;
} else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){
order = DateHistogram.Order.COUNT_DESC;
}
parentBuilder = AggregationBuilders.dateHistogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getDateInterval()).order(order);
} else if (aggParams.getEsAggregation() == EsAggregation.TERM) {
Terms.Order order = null;
if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) {
order = Terms.Order.term(true);
} else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) {
order = Terms.Order.term(false);
}else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) {
order = Terms.Order.count(true);
} else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){
order = Terms.Order.count(false);
}
if( aggParams.isKeyOrder() == null && aggParams.isValOrder() == null) {
if (aggParams.getAggSize() == 0) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize());
} else {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize());
}
} else {
if (aggParams.getAggSize() == 0) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).order(order);
} else {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).order(order);
}
}
} else if (aggParams.getEsAggregation() == EsAggregation.TERM_SCRIPTS) {
Terms.Order order = null;
if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) {
order = Terms.Order.term(true);
} else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) {
order = Terms.Order.term(false);
}else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) {
order = Terms.Order.count(true);
} else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){
order = Terms.Order.count(false);
}
if( aggParams.isKeyOrder() == null || aggParams.isValOrder() == null) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField());
} else {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()).order(order);
}
} else if (aggParams.getEsAggregation() == EsAggregation.HISTOGRAM) {
parentBuilder = AggregationBuilders.histogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getHisaggrateInterval());
} else if (aggParams.getEsAggregation() == EsAggregation.SUM) {
parentBuilder = AggregationBuilders.sum(aggParams.getAggName()).field(aggParams.getField());
} else if (aggParams.getEsAggregation() == EsAggregation.SUM_SCRIPTS) {
parentBuilder = AggregationBuilders.sum(aggParams.getAggName()).script(aggParams.getField());
}
} else {
if (aggParams.getEsAggregation() == EsAggregation.TERM) {
Terms.Order order = null;
if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) {
order = Terms.Order.term(true);
} else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) {
order = Terms.Order.term(false);
}else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) {
order = Terms.Order.count(true);
} else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){
order = Terms.Order.count(false);
}
if( aggParams.isKeyOrder() == null && aggParams.isValOrder() == null) {
if (aggParams.getAggSize() == 0 ) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).subAggregation(parentBuilder);
} else {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).size(aggParams.getAggSize()).subAggregation(parentBuilder);
}
} else {
if (aggParams.getAggSize() == 0 ) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).order(order).size(aggParams.getAggSize()).subAggregation(parentBuilder);
} else {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).field(aggParams.getField()).order(order).size(aggParams.getAggSize()).subAggregation(parentBuilder);
}
}
} else if (aggParams.getEsAggregation() == EsAggregation.TERM_SCRIPTS) {
Terms.Order order = null;
if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) {
order = Terms.Order.term(true);
} else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) {
order = Terms.Order.term(false);
}else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) {
order = Terms.Order.count(true);
} else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){
order = Terms.Order.count(false);
}
if( aggParams.isKeyOrder() == null || aggParams.isValOrder() == null) {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()).subAggregation(parentBuilder);
} else {
parentBuilder = AggregationBuilders.terms(aggParams.getAggName()).script(aggParams.getField()).order(order).subAggregation(parentBuilder);
}
} else if (aggParams.getEsAggregation() == EsAggregation.DATE_HISTOGRAM) {
DateHistogram.Order order = null;
if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("asc")) {
order = DateHistogram.Order.KEY_ASC;
} else if( aggParams.isKeyOrder() !=null && aggParams.isKeyOrder().equals("desc") ) {
order = DateHistogram.Order.KEY_DESC;
}else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("asc") ) {
order = DateHistogram.Order.COUNT_ASC;
} else if( aggParams.isValOrder() !=null && aggParams.isValOrder().equals("desc") ){
order = DateHistogram.Order.COUNT_DESC;
}
parentBuilder = AggregationBuilders.dateHistogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getDateInterval()).subAggregation(parentBuilder).order(order);
} else if (aggParams.getEsAggregation() == EsAggregation.HISTOGRAM) {
parentBuilder = AggregationBuilders.histogram(aggParams.getAggName()).field(aggParams.getField()).interval(aggParams.getHisaggrateInterval()).subAggregation(parentBuilder);
}
}
}
SearchRequestBuilder requestBuilder = client.prepareSearch(index);
if(types != null && types.size() > 0) {
String[] a = new String[types.size()];
requestBuilder.setTypes(types.toArray(a));
}
requestBuilder.addAggregation(parentBuilder);
if(db != null) {
requestBuilder.setQuery(db);
}
System.out.println(requestBuilder.toString());
SearchResponse sr = requestBuilder.execute().actionGet();
return sr;
}
public List<Map<String, String>> getMap(Iterator< ? extends MultiBucketsAggregation.Bucket > topBucket , List<EsAggregationParams> aggList, int level, List<Map<String, String>> result, Map<String, String> map) {
if (aggList.size() ==1&& level == 0) {
while(topBucket.hasNext()) {
map = new HashMap<String, String>();
MultiBucketsAggregation.Bucket b3= topBucket.next();
if (aggList.get(0).getEsAggregation() == EsAggregation.TERM_SCRIPTS) {
map.put( b3.getKey(), String.valueOf(b3.getDocCount()) );
} else {
map.put(aggList.get(0).getField(), b3.getKey());
map.put("doc_count", String.valueOf(b3.getDocCount()));
}
result.add(map);
}
return result;
} else {
level ++;
}
if(level < aggList.size() && aggList.size() != 1) {
while(topBucket.hasNext()) {
Map<String, String> map2 = new HashMap<String, String>();
MultiBucketsAggregation.Bucket b2= topBucket.next();
map2.put(aggList.get(level-1).getField(), b2.getKey());
map2.put("doc_count" , String.valueOf(b2.getDocCount()) );
if(aggList.get(level).getEsAggregation() == EsAggregation.TERM || aggList.get(level).getEsAggregation() == EsAggregation.TERM_SCRIPTS) {
Terms t = b2.getAggregations().get(aggList.get(level ).getAggName());
map2.put(aggList.get(level).getField(), b2.getKey());
getMap(t.getBuckets().iterator(), aggList, level, result, map2);
} else if (aggList.get(level).getEsAggregation() == EsAggregation.STATS) {
Stats stats = b2.getAggregations().get(aggList.get(level ).getAggName());
map2.put("count", String.valueOf(stats.getCount()));
map2.put("sum", String.valueOf(stats.getSum()));
map2.put("avg", String.valueOf(stats.getAvg()));
map2.put("max", String.valueOf(stats.getMax()));
map2.put("min", String.valueOf(stats.getMin()));
result.add(map2);
} else if (aggList.get(level).getEsAggregation() == EsAggregation.SUM) {
Sum agg = b2.getAggregations().get(aggList.get(level).getAggName());
map2.put(aggList.get(level ).getField(), String.valueOf(agg.getValue()));
result.add(map2);
} else if (aggList.get(level).getEsAggregation() == EsAggregation.DATE_HISTOGRAM) {
DateHistogram datehis = b2.getAggregations().get(aggList.get(level ).getAggName());
map2.put(aggList.get(level ).getField(), b2.getKey());
getMap( datehis.getBuckets().iterator(), aggList, level, result, map2);
} else if (aggList.get(level).getEsAggregation() == EsAggregation.HISTOGRAM) {
Histogram his = b2.getAggregations().get(aggList.get(level ).getAggName());
map2.put(aggList.get(level ).getField(), b2.getKey());
getMap( his.getBuckets().iterator(), aggList, level, result, map2);
}
}
return result;
} else {
List<Map<String, String>> list = new ArrayList<Map<String, String>>();
//List<String> list = new ArrayList<String>();
while(topBucket.hasNext()) {
Map<String, String> m = new HashMap<String, String>();
MultiBucketsAggregation.Bucket b3= topBucket.next();
m.put(aggList.get(level -1 ).getField(), b3.getKey());
m.put("doc_count", String.valueOf(b3.getDocCount()));
list.add(m);
}
map.put(aggList.get(level -1 ).getField(), JSON.toJSONString(list));
result.add(map);
return result;
}
}
public BoolQueryBuilder getQueryBuilder(List<EsQueryParam> params) {
BoolQueryBuilder db = null;
if (params != null) {
db = QueryBuilders.boolQuery();
for (EsQueryParam param : params) {
if(param.getLogicOperator() == LogicOperator.AND && param.getOperator() == Operator.EQUAL) {
db = db.must(QueryBuilders.matchQuery(param.getField(), param.getValue()));
} else if(param.getLogicOperator() == LogicOperator.AND && param.getOperator() == Operator.LIKE) {
db = db.must(QueryBuilders.wildcardQuery(param.getField(), param.getValue()));
} else if(param.getLogicOperator() == LogicOperator.OR && param.getOperator() == Operator.LIKE) {
db = db.should(QueryBuilders.wildcardQuery(param.getField(), param.getValue()));
} else if(param.getLogicOperator() == LogicOperator.MUST_NOT && param.getOperator() == Operator.LIKE) {
db = db.mustNot(QueryBuilders.wildcardQuery(param.getField(), param.getValue()));
} else if(param.getLogicOperator() == LogicOperator.MUST_NOT && param.getOperator() == Operator.EQUAL) {
db = db.mustNot(QueryBuilders.matchQuery(param.getField(), param.getValue()));
} else if(param.getOperator() == Operator.GT) {
db = db.must(QueryBuilders.rangeQuery(param.getField()).gt(param.getValue()));
} else if(param.getOperator() == Operator.GTE) {
db = db.must(QueryBuilders.rangeQuery(param.getField()).gte(param.getValue()));
} else if(param.getOperator() == Operator.LT) {
db = db.must(QueryBuilders.rangeQuery(param.getField()).lt(param.getValue()));
} else if(param.getOperator() == Operator.LTE) {
db = db.must(QueryBuilders.rangeQuery(param.getField()).lte(param.getValue()));
} else if (param.getLogicOperator() == LogicOperator.OR) {
db = db.should(QueryBuilders.matchQuery(param.getField(), param.getValue()));
} else if (param.getLogicOperator() == LogicOperator.AND ) {
db = db.must(QueryBuilders.matchQuery(param.getField(), param.getValue()));
} else if (param.getOperator() == Operator.EQUAL ) {
db = db.must(QueryBuilders.matchQuery(param.getField(), param.getValue()));
} else if (param.getOperator() == Operator.NOTEQUAL ) {
db = db.mustNot(QueryBuilders.matchQuery(param.getField(), param.getValue()));
} else if(param.getOperator() == Operator.TERMS) {
db = db.must(QueryBuilders.termsQuery(param.getField(), param.getTermsList()).minimumMatch(1));
} else if (param.getLogicOperator() == null &¶m.getOperator() == null ) {
db = db.must(QueryBuilders.matchQuery(param.getField(), param.getValue()));
}
}
}
return db;
}
@Override
public boolean createType(Client client, String index, String type,
String mapping) {
// TODO Auto-generated method stub
IndicesExistsRequest indexrequest = new IndicesExistsRequest(index);
boolean exist = client.admin().indices().exists(indexrequest).actionGet().isExists();
if (! exist) {
client.admin().indices().create(new CreateIndexRequest(index)).actionGet();
}
PutMappingRequest putMappingRequest = new PutMappingRequest(index);
putMappingRequest.type(type);
putMappingRequest.source(mapping);
PutMappingResponse putMappingResponse = client.admin().indices().putMapping(putMappingRequest).actionGet();
return putMappingResponse.isAcknowledged();
}
@Override
public boolean isIndexExistsByFieldVal(Client client, String index,
String type, String field, String fieldVal) {
// TODO Auto-generated method stub
SearchRequestBuilder searchRequest = null ;
boolean result = false;
try {
searchRequest = client.prepareSearch(index).setTypes(type).setQuery(QueryBuilders.matchQuery(field, fieldVal));
SearchResponse response = searchRequest.execute().actionGet();
result = (response.getHits().hits().length > 0) ? true : false;
} catch (Exception e ) {
e.printStackTrace();
return false;
}
return result;
}
@Override
public SearchHits termsQuery(Client client, String index,
String type,String field,List<String> terms) {
SearchRequestBuilder searchRequest = null ;
QueryBuilder qb = QueryBuilders.termsQuery(field, terms).minimumMatch(1);
searchRequest = client.prepareSearch(index).setTypes(type).setQuery(qb).setSize(1000000000);
SearchResponse response = searchRequest.execute().actionGet();
return response.getHits();
//后续优化
}
}
捐助开发者
在兴趣的驱动下,写一个免费的东西，有欣喜，也还有汗水，希望你喜欢我的作品，同时也能支持一下。 当然，有钱捧个钱场（右上角的爱心标志，支持支付宝和PayPal捐助），没钱捧个人场，谢谢各位。
谢谢您的赞助，我会做的更好！