jest client implements Sliced+Scroll parallel query

First of all, I would like to thank the author of Bboss and the owner of the elasticsearch exchange group (a very enthusiastic big guy)


Look directly at the code here:

 
 
public Map<Integer, List<JestResult>> searchSlicedScrolls(MyYangBao yangBao, QueryBuilder queryBuilders, Set<String> includePatterns) {
    final int slicesMax = 5;
    ExecutorService singleThreadPool = UtilsJava.returnSingleThreadPool(slicesMax);
    Map<Integer, List<JestResult>> stringListMap = Collections.synchronizedMap(new HashMap<>(slicesMax));
    // Divide into 5 slices to execute
    for (int id = 0; id < slicesMax; id++) {
        int finalId = id;
        ArrayList<JestResult> jestResults = new ArrayList<>();
        singleThreadPool.submit(() -> {
            SliceBuilder sliceBuilder = new SliceBuilder(finalId, slicesMax);
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
                    .query(queryBuilders)
                    // Set no sorting and return according to the order in which the documents are stored
                    .sort("_doc")
                    .slice(sliceBuilder);
            Search.Builder builder = new Search.Builder(searchSourceBuilder.toString())
                    .addIndex(yangBao.getIndexName())
                    .addType(yangBao.getTypeName())
                    .setParameter(Parameters.SIZE, GlobalConstants.HITS_SIZE)
                    .setParameter(Parameters.SCROLL, "5m")
                    // can mask metadata
                    .setParameter("filter_path", "hits.hits,hits.hits._source,hits.hits._id,_scroll_id");
            for (String includePattern : includePatterns) {
                builder.addSourceIncludePattern(includePattern);
            }
            Search search = builder.build();
            JestResult jestResult = null;
            try {
                jestResult = yangBao.getJestClient().execute(search);
            } catch (IOException e) {
                e.printStackTrace ();
            }
            assert jestResult != null;
            if (jestResult.getJsonObject().getAsJsonObject(GlobalConstants.JSON_HITS).getAsJsonArray(GlobalConstants.JSON_HITS).size() > 0) {
                String   scrollId = jestResult.getJsonObject().get(GlobalConstants.JSON_SCROLL_ID).getAsString();
                jestResults.add(jestResult);
                do {
                    SearchScroll scroll = new SearchScroll.Builder(scrollId, "5m")
                            .setParameter("filter_path", "hits.hits,hits.hits._source,hits.hits._id,_scroll_id")
                            .build();
                    try {
                        jestResult = yangBao.getJestClient().execute(scroll);
                    } catch (IOException e) {
                        e.printStackTrace ();
                    }
                    if (jestResult.getJsonObject().getAsJsonObject(GlobalConstants.JSON_HITS).getAsJsonArray(GlobalConstants.JSON_HITS).size() > 0) {
                        jestResults.add(jestResult);
                    } else {
                        break;
                    }
                } while (true);

            }
        });
        stringListMap.put(finalId, jestResults);
    }
    singleThreadPool.shutdown();
    try {
        singleThreadPool.awaitTermination(100, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        e.printStackTrace ();
    }

    return stringListMap;
}


 
 
 
 

Reference: https://my.oschina.net/bboss/blog/1788729


Guess you like

Origin http://43.154.161.224:23101/article/api/json?id=325688405&siteId=291194637