Elasticsearch在springcloud项目中同步数据的运用

一，ES的基本概念

1.什么是全文搜索引擎：

我们搜索时按结构化的拼音搜到读音，然后按其指向的页数，便可找到我们的非结构化数据——也即对字的解释。这种先建立索引，再对索引进行搜索的过程就叫全文检索(Full-text Search)。

代表就是lucence。Lucene是根据关健字来搜索的文本搜索工具，只能在某个网站内部搜索文本内容，不能跨网站搜索。

对lucence进行简化可以采用ES，ES是面向文档的（document oriented）的，对文档（注意不是成行成列的数据）进行索引，搜索，排序，过滤。

对比着传统mysql的概念理解

database对应indexes索引，

table对应types类型，

rows对应doucment文档，

column对应field字段。

创建，删除索引，创建映射，增删改查文档。

查文档分为带分词器的query string查询和不带索引的term查询。

二，前期准备

安装Elasticsearch，头信息，ik分词器。

创建索引库xccourse

创建映射

三，logstash将mysql数据库中的内容同步到ES索引库中的

课程系统远程通过feign调用CMS内容管理系统，并在课程系统中完成发布，并页面保存到本地，，同时将信息保存在coursepub中，完成课程发布。

基于logstash同步mysql数据库中coursepub表的信息到es后台，可以在ES后台“数据浏览”看到与coursepub完全一致的内容。

配置文件mysql.config

 1 input {
 2   stdin {
 3   }
 4   jdbc {
 5   jdbc_connection_string => "jdbc:mysql://localhost:3306/xc_course?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC"
 6   # the user we wish to excute our statement as
 7   jdbc_user => "root"
 8   jdbc_password => root
 9   # the path to our downloaded jdbc driver  
10   jdbc_driver_library => "E:/soft/apache-maven-3.3.9/repository/mysql/mysql-connector-java/5.1.36/mysql-connector-java-5.1.36.jar"
11   # the name of the driver class for mysql
12   jdbc_driver_class => "com.mysql.jdbc.Driver"
13   jdbc_paging_enabled => "true"
14   jdbc_page_size => "50000"
15   #要执行的sql文件
16   #statement_filepath => "/conf/course.sql"
17   statement => "select * from course_pub where timestamp > date_add(:sql_last_value,INTERVAL 8 HOUR)"
18   #定时配置
19   schedule => "* * * * *"
20   record_last_run => true
21   last_run_metadata_path => "‪E:/xuecheng/es/logstash-6.2.1/config/logstash_metadata"
22   }
23 }
24 
25 
26 output {
27   elasticsearch {
28   #ES的ip地址和端口
29   hosts => "localhost:9200"
30   #hosts => ["localhost:9200","localhost:9202","localhost:9203"]
31   #ES索引库名称
32   index => "xc_course"
33   document_id => "%{id}"
34   document_type => "doc"
35   template =>"E:/xuecheng/es/logstash-6.2.1/config/xc_course_template.json"
36   template_name =>"xc_course"
37   template_overwrite =>"true"
38   }
39   stdout {
40  #日志输出
41   codec => json_lines
42   }
43 }

21行的logstash_metadata文件

--- 2018-06-30 11:26:00.150000000 Z

启动logstash

数据浏览显示mysql的数据。

四，搜索微服务普通分级搜索和按照关键字查询，以及按照分类等级查询，按照分页查询

配置类

 ElasticsearchConfig

 1 import org.apache.http.HttpHost;
 2 import org.elasticsearch.client.RestClient;
 3 import org.elasticsearch.client.RestHighLevelClient;
 4 import org.springframework.beans.factory.annotation.Value;
 5 import org.springframework.context.annotation.Bean;
 6 import org.springframework.context.annotation.Configuration;
 7 
 8 /**
 9  * @author Administrator
10  * @version 1.0
11  **/
12 @Configuration
13 public class ElasticsearchConfig {
14 
15     @Value("${xuecheng.elasticsearch.hostlist}")
16     private String hostlist;
17 
18     @Bean
19     public RestHighLevelClient restHighLevelClient(){
20         //解析hostlist配置信息
21         String[] split = hostlist.split(",");
22         //创建HttpHost数组，其中存放es主机和端口的配置信息
23         HttpHost[] httpHostArray = new HttpHost[split.length];
24         for(int i=0;i<split.length;i++){
25             String item = split[i];
26             httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
27         }
28         //创建RestHighLevelClient客户端
29         return new RestHighLevelClient(RestClient.builder(httpHostArray));
30     }
31 
32     //项目主要使用RestHighLevelClient，对于低级的客户端暂时不用
33     @Bean
34     public RestClient restClient(){
35         //解析hostlist配置信息
36         String[] split = hostlist.split(",");
37         //创建HttpHost数组，其中存放es主机和端口的配置信息
38         HttpHost[] httpHostArray = new HttpHost[split.length];
39         for(int i=0;i<split.length;i++){
40             String item = split[i];
41             httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
42         }
43         return RestClient.builder(httpHostArray).build();
44     }
45 
46 }

api

 8 @Api(value = "课程搜索",description = "课程搜索",tags = {"课程搜索"})
 9 public interface EsCourseControllerApi {
10 
11     @ApiOperation("课程搜索")
12    QueryResponseResult<CoursePub> searchCourse(int page, int size, CourseSearchParam courseSearchParam);
13 
14 }

controller

 1 @RestController
 2 @RequestMapping("/search/course")
 3 public class EsCourseController implements EsCourseControllerApi {
 4 
 5     @Autowired
 6     private CourseSearchService courseSearchService;
 7 
 8 
 9     @Override
10     @GetMapping("/list/{page}/{size}")
11     public QueryResponseResult<CoursePub> searchCourse(@PathVariable("page") int page, @PathVariable("size")int size, CourseSearchParam courseSearchParam) {
12         return courseSearchService.searchCourse(page,size,courseSearchParam);
13     }
14 }

yml

 1 server:
 2   port: ${port:40100}
 3 spring:
 4   application:
 5     name: xc-search-service
 6 xc:
 7   elasticsearch:
 8     hostlist: ${eshostlist:127.0.0.1:9200} #多个结点中间用逗号分隔
 9     course:
10       index: xc_course
11       type: doc

service

  1 import com.xuecheng.filesystem.framework.domain.course.response.CoursePub;
  2 import com.xuecheng.filesystem.framework.domain.search.CourseSearchParam;
  3 import com.xuecheng.filesystem.framework.model.response.CommonCode;
  4 import com.xuecheng.filesystem.framework.model.response.QueryResponseResult;
  5 import com.xuecheng.filesystem.framework.model.response.QueryResult;
  6 import org.apache.commons.lang3.StringUtils;
  7 import org.elasticsearch.action.search.SearchRequest;
  8 import org.elasticsearch.action.search.SearchResponse;
  9 import org.elasticsearch.client.RestHighLevelClient;
 10 import org.elasticsearch.common.text.Text;
 11 import org.elasticsearch.index.query.BoolQueryBuilder;
 12 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 13 import org.elasticsearch.index.query.QueryBuilders;
 14 import org.elasticsearch.search.SearchHit;
 15 import org.elasticsearch.search.SearchHits;
 16 import org.elasticsearch.search.builder.SearchSourceBuilder;
 17 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 18 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
 19 import org.springframework.beans.factory.annotation.Autowired;
 20 import org.springframework.beans.factory.annotation.Value;
 21 import org.springframework.stereotype.Service;
 22 
 23 import java.io.IOException;
 24 import java.util.ArrayList;
 25 import java.util.List;
 26 import java.util.Map;
 27 
 28 @Service
 29 public class CourseSearchService {
 30 
 31     @Autowired
 32     private RestHighLevelClient restHighLevelClient;
 33 
 34     @Value("${xuecheng.elasticsearch.course.index}")
 35     private String index;
 36 
 37     @Value("${xuecheng.elasticsearch.course.type}")
 38     private String type;
 39 
 40     /**
 41      * 课程搜索
 42      * 关键字查询
 43      * 一级分类，二级分类，难度等级
 44      * 分页查询
 45      * 高亮查询
 46      * @param page
 47      * @param size
 48      * @param courseSearchParam
 49      * @return
 50      */
 51     public QueryResponseResult<CoursePub> searchCourse(int page, int size, CourseSearchParam courseSearchParam) {
 52 
 53         SearchRequest searchRequest = new SearchRequest(index);
 54         searchRequest.types(type);
 55 
 56         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
 57 
 58         //boolQueryBuilder  must  C1  C2 C1 AND C2        or  C1 OR C2
 59         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
 60 
 61         //按照关键字查询
 62         if (StringUtils.isNotEmpty(courseSearchParam.getKeyword())){
 63 
 64             MultiMatchQueryBuilder multiMatchQueryBuilder = QueryBuilders.multiMatchQuery(courseSearchParam.getKeyword(), "name", "description", "teachplan");
 65             //查询权重
 66             multiMatchQueryBuilder.field("name",10);
 67             multiMatchQueryBuilder.minimumShouldMatch("70%");
 68 
 69             boolQueryBuilder.must(multiMatchQueryBuilder);
 70         }
 71 
 72         //过滤查询&等值查询
 73         //一级分类查询
 74         if (StringUtils.isNotEmpty(courseSearchParam.getMt())){
 75             boolQueryBuilder.filter(QueryBuilders.termQuery("mt",courseSearchParam.getMt()));
 76         }
 77 
 78         //二级分类查询
 79         if (StringUtils.isNotEmpty(courseSearchParam.getSt())){
 80             boolQueryBuilder.filter(QueryBuilders.termQuery("st",courseSearchParam.getSt()));
 81         }
 82 
 83         //难度等级查询
 84         if (StringUtils.isNotEmpty(courseSearchParam.getGrade())){
 85             boolQueryBuilder.filter(QueryBuilders.termQuery("grade",courseSearchParam.getGrade()));
 86         }
 87 
 88         //分页查询
 89         if (page <=0){ //当前页
 90             page =1;
 91         }
 92         if (size <= 0 ){
 93             size = 10;
 94         }
 95 
 96         int start = (page-1)*size;
 97         //从哪开始查
 98         sourceBuilder.from(start);
 99         //每页显示多少
100         sourceBuilder.size(size);
101 
102         //高亮查询
103         HighlightBuilder highlightBuilder = new HighlightBuilder();
104         //高亮前缀
105         highlightBuilder.preTags("<font class='eslight'>");
106         //高亮后缀
107         highlightBuilder.postTags("</font>");
108         //高亮域
109         highlightBuilder.fields().add(new HighlightBuilder.Field("name"));
110         sourceBuilder.highlighter(highlightBuilder);
111 
112 
113         sourceBuilder.query(boolQueryBuilder);
114 
115         searchRequest.source(sourceBuilder);
116 
117         SearchResponse searchResponse = null;
118         try {
119             searchResponse = restHighLevelClient.search(searchRequest);
120         } catch (IOException e) {
121             e.printStackTrace();
122         }
123         //获取查询结果
124         SearchHits hits = searchResponse.getHits();
125         SearchHit[] searchHits = hits.getHits();
126 
127         List<CoursePub> coursePubList = new ArrayList<>();
128 
129         for (SearchHit searchHit : searchHits) {
130 
131             Map<String, Object> sourceAsMap = searchHit.getSourceAsMap();
132             CoursePub coursePub = new CoursePub();
133 
134             String id = (String) sourceAsMap.get("id");
135             coursePub.setId(id);
136 
137             //名称
138             String name = (String) sourceAsMap.get("name");
139             //获取高亮
140             Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
141             if (highlightFields != null){
142                 HighlightField highlightField = highlightFields.get("name");
143                 if (highlightField != null){
144                     Text[] fragments = highlightField.fragments();
145                     if (fragments !=null){
146                         StringBuffer stringBuffer = new StringBuffer();
147                         for (Text fragment : fragments) {
148                             stringBuffer.append(fragment);
149                         }
150                         name = stringBuffer.toString();
151                     }
152                 }
153                 coursePub.setName(name);
154             }
155 
156 
157             //图片
158             String pic = (String) sourceAsMap.get("pic");
159             coursePub.setPic(pic);
160 
161             //价格
162             Float price = null;
163             if (sourceAsMap.get("price") != null){
164                 price = Float.parseFloat(String.valueOf(sourceAsMap.get("price")));
165             }
166             coursePub.setPrice(price);
167 
168             //原价
169             Float price_old = null;
170             if (sourceAsMap.get("price_old")!=null){
171                 price_old = Float.parseFloat(String.valueOf(sourceAsMap.get("price_old")));
172             }
173             coursePub.setPrice_old(price_old);
174 
175             coursePubList.add(coursePub);
176 
177         }
178 
179         //数据封装
180         QueryResult queryResult = new QueryResult();
181         queryResult.setTotal(hits.getTotalHits());
182         queryResult.setList(coursePubList);
183         return new QueryResponseResult<CoursePub>(CommonCode.SUCCESS,queryResult);
184     }
185 }

nginx代理转发，用户请求/course/search的nginx将请求转发给nuxt。js服务。nginx在转发时根据每台nuxt服务器的负载情况进行转发，实现负载均衡

最后实现走进‘’‘课程搜索’可以进行关键字查询并对关键字实现高亮显示，并采用二级联动实现分级。