elk client integration with springboot

Get into the habit of writing together! This is the 11th day of my participation in the "Nuggets Daily New Plan · April Update Challenge", click to view the details of the event .

Through the above study, I believe that you have synchronized the logs printed by springboot to elasticsearch, and then you need to query the corresponding data in the system module according to the conditions. Since the blogger's needs here are relatively simple and the usage scenarios are single, the start provided by spring is used as the client. How to integrate is described below.

1. ElasticsearchTemplate query syntax

When using ElasticsearchTemplate, you only need to master the provided API. The following is a summary of the commonly used query APIs for small partners to use.

        // 返回对象
        Result<List<ActionLogVO>> result = new Result();
        NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
        BoolQueryBuilder bool = QueryBuilders.boolQuery();
 
 
        //must为and的语法
        //matchPhraseQuery为将type分词 然后将第二个参数匹配 可以模糊查询使用
        bool.must(QueryBuilders.matchPhraseQuery("Type", "日志"));
 
        //wildcardQuery模糊查询使用  但是字段需要未英文 
        bool.must(QueryBuilders.wildcardQuery("userName", "*admin*"));
 
        
        //通过时间筛选
        List<QueryBuilder> filters = bool.filter();                           
        filters.add(QueryBuilders.rangeQuery("time").gte(sd.parse("2021-01-19 17:28:41"))
        .lte(sd.parse("2021-01-19 17:28:43")));
  
 
        //分页(第一个参数PageNum从第0页开始 第二个参数pageSize)
        builder.withPageable(PageRequest.of(1, 10));
 
        //排序
        builder.withSort(SortBuilders.fieldSort("time").order(SortOrder.ASC));
 
 
        // 构造查询条件
        builder.withQuery(bool);
        NativeSearchQuery query = builder.build();
        Iterable<XXXVO> resultIter = XXXXService.search(query);
复制代码

2. Conditional query

1. Modify the pom file

        <!-- elasticsearch -->
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
            <version>2.3.7.RELEASE</version>
        </dependency>
复制代码

2. Modify application.yml

spring: 
 elasticsearch:
    rest:
      uris: ip:9200
      #上文安装时配置的账号密码
      username: xxxx      
      password: xxxx
  data:
    elasticsearch:
      repositories:
        enabled: true
      client:
        reactive:
#          username: xxx 这种写法是不对的
#          password: xxx 这种写法是不对的
#          endpoints: ip:9200 这种写法是不对的
          use-ssl: false
复制代码

Notice:

The configuration es address should be written in the landlord's location, otherwise it will default to localhost:9200. If es is installed on other servers, the connection will fail.
Unlike the configuration of the old version, the open port number is 9200.

3. New config file

If the following files are not added, there will be java.lang.IllegalStateException: availableProcessors is already set to [4], rejecting [4] exception.

@Configuration
public class ElasticSearchConfig {
    /**
     * 防止netty的bug
     * java.lang.IllegalStateException: availableProcessors is already set to [4], rejecting [4]
     */
    @PostConstruct
    void init() {
        System.setProperty("es.set.netty.runtime.available.processors", "false");
    }
}

复制代码

4. Interface writing

Mainly call the API under ElasticsearchRestTemplate, see the comments below for a detailed explanation.

    @Autowired
    ElasticsearchRestTemplate elasticsearchTemplate;

    @Log(operationName = "日志-查询登录日志")
    @PostMapping("/selectLoginLog")
    public Result<List<LoginLogVO>> selectLoginLog(@RequestBody LoginLogInputVO loginLogInputVO) throws ParseException {
        // 日期格式化
        SimpleDateFormat sd = new SimpleDateFormat(DateFormatEnum.YYYY_MM_DD_HH_MM_SS.getFormat());
        // 返回对象
        Result<List<LoginLogVO>> result = new Result();
        NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
        BoolQueryBuilder bool = QueryBuilders.boolQuery();
        // 用户名不为空
        if (!CommonUtil.isEmpty(loginLogInputVO.getUserName())) {
            bool.must(QueryBuilders.wildcardQuery("userName", "*" + loginLogInputVO.getUserName() + "*"));
        }
        // 时间不为空
        if (!CommonUtil.isEmpty(loginLogInputVO.getBeginTime()) && !CommonUtil.isEmpty(loginLogInputVO.getEndTime())) {
            List<QueryBuilder> filters = bool.filter();
            filters.add(QueryBuilders.rangeQuery("time")
                .gte(sd.parse(loginLogInputVO.getBeginTime() + DateFormatEnum.BEGIN_HH_MM_SS.getFormat()))
                .lte(sd.parse(loginLogInputVO.getEndTime() + DateFormatEnum.END_HH_MM_SS.getFormat())));
        }
        // 分页查询
        if (!CommonUtil.isEmpty(loginLogInputVO.getPageSize()) && !CommonUtil.isEmpty(loginLogInputVO.getPageNum())) {
            // 从第0页开始
            builder.withPageable(PageRequest.of(loginLogInputVO.getPageNum() - 1, loginLogInputVO.getPageSize()));
        }
        builder.withSort(SortBuilders.fieldSort("time").order(SortOrder.ASC));
        // 构造查询条件
        builder.withQuery(bool);
        NativeSearchQuery query = builder.build();
        //查询的索引
        IndexCoordinates indexCoordinates = IndexCoordinates.of("loginlog-*");
        //执行查询
        SearchHits<LoginLogVO> resultIter = elasticsearchTemplate.search(query, LoginLogVO.class, indexCoordinates);
        // 格式化输出
        List<LoginLogVO> resultList = new ArrayList<>();
        List<SearchHit<LoginLogVO>> SearchHitList = resultIter.getSearchHits();
        for (SearchHit<LoginLogVO> loginLogVOSearchHit : SearchHitList) {
            resultList.add(loginLogVOSearchHit.getContent());
        }
        // 分页返回
        if (!CommonUtil.isEmpty(loginLogInputVO.getPageSize()) && !CommonUtil.isEmpty(loginLogInputVO.getPageNum())) {
            result.setTotal(resultIter.getTotalHits());
            result.setPageNum(loginLogInputVO.getPageNum());
            result.setPageSize(loginLogInputVO.getPageSize());
        }
        result.setData(resultList);
        return result;
    }
复制代码

The loginlog-* indexes here refer to all indexes starting with loginlog-.

5. Entity class

1. Elasticsearch client query use

A special entity class is required when executing the elasticsearchTemplate.search method.

@Document(indexName = "loginlog-*", shards = 1, replicas = 0)
public class LoginLogVO {

    @Id
    private String id;

    /**
     * 信息
     */
    @Field(type = FieldType.Keyword, analyzer = "ik_max_word")
    private String message;

    @Field(type = FieldType.Date, store = true, format = DateFormat.date_time)
    private Date time;
}
复制代码

The explanation is as follows:

@Document: Indicates the domain object mapped to the Elasticsearch document, where indexName represents the index name, shards represents the default number of shards, and replicas represents the default number of replicas.
@Id: indicates the id of the document, and the document can be considered as the concept of table row in mysql
@Field: The type of the field in the document, where analyzer represents the tokenizer type, format represents the formatting type, and the type types include Text (character type that will be segmented and indexed), Integer, Long, Date, Float, Double, Boolean Wait.

2. Enter the PO

For a common entity class, pass in query parameters.

public class LoginLogInputVO extends BaseEntity {

    /**
     * 用户账号
     */
    private String userName;

    /**
     * 开始时间
     */
    private String beginTime;

    /**
     * 结束时间
     */
    private String endTime;
}

复制代码

Because the time type agreed by the front and back ends of the landlord is string, it is necessary to convert the type and then return it.

6. Pay attention

elasticsearchTemplate can only query 10,000 items at a time. Therefore, it is not recommended to return all pages. It is recommended to use the following paging method. Please refer to Baidu. insert image description here

3. Export the data within the time interval

1. Interface writing

    @ApiOperation(value = "导出详细日志接口", notes = "导出详细日志接口")
    @PostMapping("/exportDetailLog")
    public void exportDetailLog(@ApiParam(name = "导出详细日志接口输入参数实体", value = "导出详细日志接口输入参数实体",
        required = false) @RequestBody HandleDetailLogVO handleDetailLogVO) throws IOException, ParseException {
        SimpleDateFormat sd = new SimpleDateFormat(DateFormatEnum.YYYY_MM_DD_HH_MM_SS.getFormat());
        String beginTime = handleDetailLogVO.getBeginTime() + DateFormatEnum.BEGIN_HH_MM_SS.getFormat();
        String endTime = handleDetailLogVO.getEndTime() + DateFormatEnum.END_HH_MM_SS.getFormat();
        String path = handleDetailLogVO.getPath() + "/" + handleDetailLogVO.getBeginTime() + "-"
            + handleDetailLogVO.getEndTime() + ".txt";
        // 根据时间查询
        NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
        BoolQueryBuilder bool = QueryBuilders.boolQuery();
        List<QueryBuilder> filters = bool.filter();
        //构建查询条件
        filters.add(QueryBuilders.rangeQuery("time").gte(sd.parse(beginTime)).lte(sd.parse(endTime)));
        builder.withQuery(bool);
        NativeSearchQuery query = builder.build();
        //查询的索引
        IndexCoordinates indexCoordinates = IndexCoordinates.of("loginlog-*");
        // 查询前1w条
        SearchScrollHits<DetailLogVO> scroll =
            elasticsearchTemplate.searchScrollStart(3000, query, DetailLogVO.class, indexCoordinates);
        List<SearchHit<DetailLogVO>> resultList = new ArrayList<>();
        while (scroll.hasSearchHits()) {
            List<SearchHit<DetailLogVO>> searchHitList = scroll.getSearchHits();
            resultList.addAll(searchHitList);
            scroll = elasticsearchTemplate.searchScrollContinue(scroll.getScrollId(), 3000, DetailLogVO.class,
                indexCoordinates);
        }
        // 导出为text流
        Writer out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(path, true), "utf-8"));
        for (int i = 0; i < resultList.size(); i++) {
            out.write(resultList.get(i).getContent().getMessage());
            out.write("\r\n");
        }
        out.flush();
        out.close();
        // 删除索引
        elasticsearchTemplate.delete(query, DetailLogVO.class, indexCoordinates);
    }
复制代码

Note: Because there is a 1w query limit, the query is looped. But the logs everywhere cannot exceed 8g.

2. Entity class

It is mainly the condition that the input parameter carries the query time.

public class HandleDetailLogVO {
 
    /**
     * 结束时间
     */
    @ApiModelProperty(value = "结束时间", name = "结束时间")
    private String endTime;
 
    /**
     * 开始时间
     */
    @ApiModelProperty(value = "开始时间", name = "开始时间")
    private String beginTime;
 
    /**
     * 导出路径
     */
    @ApiModelProperty(value = "导出路径", name = "导出路径")
    private String path;
}
复制代码

4. Delete the data in the time period

1. Interface writing

Also delete the data in the time period through the API. See code comments for details.

    @ApiOperation(value = "清除详细日志接口", notes = "清除详细日志接口")
    @PostMapping("/clearDetailLog")
    public void clearDetailLog(@ApiParam(name = "清除详细日志接口输入参数实体", value = "清除详细日志接口输入参数实体",
        required = false) @RequestBody HandleDetailLogVO handleDetailLogVO) throws IOException, ParseException {
        SimpleDateFormat sd = new SimpleDateFormat(DateFormatEnum.YYYY_MM_DD_HH_MM_SS.getFormat());
        String beginTime = handleDetailLogVO.getBeginTime() + DateFormatEnum.BEGIN_HH_MM_SS.getFormat();
        String endTime = handleDetailLogVO.getEndTime() + DateFormatEnum.END_HH_MM_SS.getFormat();
        // 根据时间查询
        NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
        BoolQueryBuilder bool = QueryBuilders.boolQuery();
        List<QueryBuilder> filters = bool.filter();
        filters.add(QueryBuilders.rangeQuery("time").gte(sd.parse(beginTime)) .lte(sd.parse(endTime)));
        builder.withQuery(bool);
        //构建查询条件
        NativeSearchQuery query = builder.build();
        //指定删除索引的名称
        IndexCoordinates indexCoordinates = IndexCoordinates.of("datalog-*");
        // 调用API删除索引
        elasticsearchTemplate.delete(query, DetailLogVO.class, indexCoordinates);
    }
复制代码

2. Entity class

The input entity that carries the query conditions.

public class HandleDetailLogVO {
 
        /**
         * 结束时间
         */
        @ApiModelProperty(value = "结束时间", name = "结束时间")
        private String endTime;
 
        /**
         * 开始时间
         */
        @ApiModelProperty(value = "开始时间", name = "开始时间")
        private String beginTime;
}
复制代码