es 6.x 实现使用scroll 实现查询海量数据分页

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/u011066470/article/details/89850044

#第一步：调用

    public Integer getDailyLeaseBike3(OperateAreasBo operateArea, String type, String type1, Long stime, Long etime, String queryDateField) {
        BoolQueryBuilder boolQueryBuilder = createBoolQueryBuilder(operateArea, stime, etime, queryDateField);
        boolQueryBuilder.must(QueryBuilders.matchQuery("orderStatus", 300));
        SearchQuery searchQuery = new NativeSearchQueryBuilder()
                .withQuery(boolQueryBuilder)
                .withIndices(ES_INDEX_NAME)
                .withTypes(ES_TYPE_NAME)
                //.withPageable(PageRequest.of(0, 10))//从0页开始查，每页10个结果
                .withSourceFilter(new FetchSourceFilter(new String[]{"bikeId", "id"}, null))
                .build();
        Set<Long> st = new HashSet<Long>();
       List<Long> list=queryAllDataByScroll( searchQuery);//全部查询
       System.out.println("old:"+list.size());
      for (long bikeid:list){
        st.add(bikeid);
      }
        System.out.println("old:"+list.size()+"去重=="+st.size());
        return st.size();
    }

#第二步：

     /**
     * scroll游标快照超时时间，单位ms
     */
    private static final long SCROLL_TIMEOUT = 3000;


 /**
     * 用于将Scroll获取到的结果，处理成dto列表，做复杂映射
     */
    private final SearchResultMapper searchResultMapper = new SearchResultMapper() {
        @Override
        public <T> AggregatedPage<T> mapResults(SearchResponse response, Class<T> aClass, Pageable pageable) {
            List<OrderDetailEntityNew> result = new ArrayList<OrderDetailEntityNew>();
            for (SearchHit hit : response.getHits()) {
                if (response.getHits().getHits().length <= 0) {
                    return new AggregatedPageImpl<T>(Collections.EMPTY_LIST, pageable, response.getHits().getTotalHits(), response.getScrollId());
                }
                //可以做更复杂的映射逻辑
                Object id = hit.getSourceAsMap().get("id");
                Object bikeId = hit.getSourceAsMap().get("bikeId");
                OrderDetailEntityNew orderDetailEntityNew = new OrderDetailEntityNew();
                orderDetailEntityNew.setId(Long.parseLong(id + ""));
                orderDetailEntityNew.setBikeId(Long.parseLong(bikeId + ""));
                result.add(orderDetailEntityNew);
            }
            if (result.isEmpty()) {
                return new AggregatedPageImpl<T>(Collections.EMPTY_LIST, pageable, response.getHits().getTotalHits(), response.getScrollId());
            }
            return new AggregatedPageImpl<T>((List<T>) result, pageable, response.getHits().getTotalHits(), response.getScrollId());
        }
    };

    public List<Long> queryAllDataByScroll(SearchQuery searchQuery) {
        List<Long> rtmdList = new ArrayList<Long>();
        ScrolledPage<OrderDetailEntityNew> scroll = (ScrolledPage<OrderDetailEntityNew>) elasticsearchTemplate.startScroll(SCROLL_TIMEOUT, searchQuery, OrderDetailEntityNew.class, searchResultMapper);
        System.out.println("查询总命中数：" + scroll.getTotalElements());
        while (scroll.hasContent()) {
            for (OrderDetailEntityNew dto : scroll.getContent()) {
                //Do your work here
                System.out.println(dto.getBikeId());
                rtmdList.add(dto.getBikeId());
            }
            //取下一页，scrollId在es服务器上可能会发生变化，需要用最新的。发起continueScroll请求会重新刷新快照保留时间
            scroll = (ScrolledPage<OrderDetailEntityNew>) elasticsearchTemplate.continueScroll(scroll.getScrollId(), SCROLL_TIMEOUT, OrderDetailEntityNew.class, searchResultMapper);
        }
        //及时释放es服务器资源
        elasticsearchTemplate.clearScroll(scroll.getScrollId());
                return rtmdList;
    }

结果：

13-1-1015 old:2551====2551====groupy by:2552

参考地址：https://blog.csdn.net/garvey_wong/article/details/86624016