Spring-boot搭建elk环境以及对数据流程完整展示

ELK的数据处理流程：

第一步：产生日志数据

spring-boot项目基本配置：

新建spring-boot项目2.1.14，勾选好下图两个依赖，开始新建项目

导入guava依赖

<dependency>
		<groupId>com.google.guava</groupId>
		<artifactId>guava</artifactId>
		<version>18.0</version>
</dependency>
 <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>1.2.17</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core -->
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-core</artifactId>
            <version>2.9.9</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind -->
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.9.9</version>
        </dependency>
<dependency>
	<groupId>net.logstash.logback</groupId>
	<artifactId>logstash-logback-encoder</artifactId>
	<version>4.11</version>
</dependency>

并修改application.properties为application.yml，添加es的集群配置和dao接口配置

spring:
  data:
    elasticsearch:
      cluster-nodes: niceday-es
      cluster-name: 192.168.56.101:9300
      repositories:
        enabled: true

创建controller包

对线程生产的封装，我以将代码开源到github上，可以去star项目慢慢看

第二步：清洗日志数据格式

第三步：存储日志数据并分析

第四步：显示日志数据

control层

@RestController
public class UserActionCtl {
	@Autowired
	private UserActionServices uas;

	@RequestMapping("/fenzu")
	public List<VUserModelPie> groupAndCount(){
		return uas.aggsSearch();
	}
}

dao层

public interface UserActionDAO extends ElasticsearchRepository<UserAction,String> {
}

model层
browserInfos

public class BrowserInfos {
	private String browserVersion;
	private int custid;
	private Events events;
	private static BrowserInfos browserInfos;


	public static BrowserInfos getBrowserInstance(){
		if (browserInfos == null){
			synchronized (BrowserInfos.class){
				if (browserInfos == null){
					browserInfos = new BrowserInfos();
				}
			}
		}
		return browserInfos;
	}

	private BrowserInfos(){}

	public Events getEvents() {
		return events;
	}

	public void setEvents(Events events) {
		this.events = events;
	}

	public int getCustid() {
		return custid;
	}

	public void setCustid(int custid) {
		this.custid = custid;
	}

	public String getBrowserVersion() {
		return browserVersion;
	}

	public void setBrowserVersion(String browserVersion) {
		this.browserVersion = browserVersion;
	}
}

events

public class Events {
	private String eventCateGory;
	private String position;
	private String time;
	private String pageName; // 哪一个页面
	private String msg;
	private static volatile Events events;


	private Events() {
	}

	public static Events getEventInstance(){
		if (events == null){
			synchronized (Events.class){
				if (events == null){
					events = new Events();
				}
			}
		}
		return events;
	}


	public Events(String eventCateGory, String position, String time, String pageName, String msg) {
		this.eventCateGory = eventCateGory;
		this.position = position;
		this.time = time;
		this.pageName = pageName;
		this.msg = msg;
	}

	public String getEventCateGory() {
		return eventCateGory;
	}

	public void setEventCateGory(String eventCateGory) {
		this.eventCateGory = eventCateGory;
	}

	public String getPosition() {
		return position;
	}

	public void setPosition(String position) {
		this.position = position;
	}

	public String getTime() {
		return time;
	}

	public void setTime(String time) {
		this.time = time;
	}

	public String getPageName() {
		return pageName;
	}

	public void setPageName(String pageName) {
		this.pageName = pageName;
	}

	public String getMsg() {
		return msg;
	}

	public void setMsg(String msg) {
		this.msg = msg;
	}
}

useraction类

@Document(indexName = "users" ,type = "useraction")
public class UserAction implements Serializable {
	@JsonProperty("id")
	private String id;
	@JsonProperty("pageName")
	private String pageName;
	@JsonProperty("time")
	private String time;
	@JsonProperty("position")
	private String position;
	@JsonProperty("msg")
	private String msg;
	@JsonProperty("custid")
	private long custid;
	@JsonProperty("browserVersion")
	private String browserVersion;
	@JsonProperty("eventCateGory")
	private String eventCateGory;

	public String getId() {
		return id;
	}

	public void setId(String id) {
		this.id = id;
	}

	public long getCustid() {
		return custid;
	}

	public void setCustid(long custid) {
		this.custid = custid;
	}


	public String getPageName() {
		return pageName;
	}

	public void setPageName(String pageName) {
		this.pageName = pageName;
	}

	public String getTime() {
		return time;
	}

	public void setTime(String time) {
		this.time = time;
	}

	public String getPosition() {
		return position;
	}

	public void setPosition(String position) {
		this.position = position;
	}

	public String getMsg() {
		return msg;
	}

	public void setMsg(String msg) {
		this.msg = msg;
	}

	public String getBrowserVersion() {
		return browserVersion;
	}

	public void setBrowserVersion(String browserVersion) {
		this.browserVersion = browserVersion;
	}

	public String getEventCateGory() {
		return eventCateGory;
	}

	public void setEventCateGory(String eventCateGory) {
		this.eventCateGory = eventCateGory;
	}
}

vusermodepie类

public class VUserModelPie {
	private String name;
	private Long value;


	public String getName() {
		return name;
	}

	public void setName(String name) {
		this.name = name;
	}

	public Long getValue() {
		return value;
	}

	public void setValue(Long value) {
		this.value = value;
	}
}

service层
UserActionServices

扫描二维码关注公众号，回复： 11370495 查看本文章

@Service
public class UserActionServices {
	@Autowired
	private UserActionDAO uad;

	// 分组查询
	public List<VUserModelPie> aggsSearch() {
		//第一个字段进行分组 聚合name之后查找自己起的叫abc名字的
		TermsAggregationBuilder builder = AggregationBuilders.terms("niceday").field("eventCateGory");
		NativeSearchQueryBuilder nsqb = new NativeSearchQueryBuilder();
		// withSourceFilter通过拿到的数据源进行处理，FetchSourceFilter(想要的数组，不想要的数组)过滤后拿到一个数据源
		//	nsqb.withSourceFilter(new FetchSourceFilter(new String[]{""},null));
		nsqb.addAggregation(builder);
		AggregatedPage ap = (AggregatedPage) uad.search(nsqb.build());
		// 拿到聚合后的表名
		StringTerms abc = (StringTerms) ap.getAggregation("niceday");

		//拿表名的捅字段
		List<StringTerms.Bucket> buckets = abc.getBuckets();

		List<VUserModelPie> vump = new ArrayList<>();
		for (StringTerms.Bucket bucket : buckets) {
			VUserModelPie vp = new VUserModelPie();
			vp.setName(bucket.getKeyAsString());
			vp.setValue((bucket.getDocCount()));
			vump.add(vp);
		}
		return vump;

	}

	public Map<String,List<TempBIObject>> findCount(){
		Script script = new Script("if(doc['time'].values.length == 6) if(doc['time'].values[5].length() == 4) doc['time'].values[5] + doc['time'].values[2] + '#' + doc['eventCateGory'].values");

		TermsAggregationBuilder group = AggregationBuilders.terms("abc").script(script).size(1000);
		NativeSearchQueryBuilder nqb = new NativeSearchQueryBuilder();
		nqb.addAggregation(group);
		AggregatedPage<Action> acts = (AggregatedPage<Action>)actionDAO.search(nqb.build());
		StringTerms terms = (StringTerms)acts.getAggregation("abc");
		List<StringTerms.Bucket> buckets = terms.getBuckets();
		Map<String, List<TempBIObject>> change = change(buckets);
		for (String s : change.keySet()) {
			System.out.println(s + ":" );
		}
		return change;

	}


	private Map<String,List<TempBIObject>> change(List<StringTerms.Bucket> bck){
		Map<String, List<TempBIObject>> groups = new HashMap<>();
		// 将所有数据封装成对象中
		for (StringTerms.Bucket bucket : bck) {
			String[] split = bucket.getKeyAsString().split("#");
			// 通过事件的名称到分组集合中获取事件组集合
			List<TempBIObject> tbo = null;
			//填充map的链表
			if(groups.containsKey(split[1])){
				tbo = groups.get(split[1]);
			}else {
				tbo = new ArrayList<>();
				groups.put(split[1],tbo);+
			}
			// 塞成排序对象，将对象塞到链表里
			TempBIObject tempBIObject = new TempBIObject();
			tempBIObject.setTime(Long.parseLong(split[0]));
			tempBIObject.setEvent(split[1]);
			tempBIObject.setClickNum(bucket.getDocCount());
			tbo.add(tempBIObject);
		}

		// 对分组数组中的对象进行排序
		for (List<TempBIObject> val : groups.values() ){
			// 给全局临时变量赋值
			temp = val;
			// 调用排序完成日期排序
			quick(0,temp.size() - 1);
		}
		return groups;
	}

	private void quick(int low,int high){
		int lo = low, hi = high;
		if (lo >= hi)return;

		boolean flag = false;
		while(lo < hi){
			// hi < lo 时
			if (temp.get(lo).compareTo(temp.get(hi)) > 0){
				TempBIObject tmTbo = temp.get(lo);
				temp.set(lo,temp.get(hi));
				temp.set(hi,tmTbo);

				flag = !flag;
			}

			if (flag)lo++;else hi--;
		}
		lo--;hi++;
		quick(low,lo);
		quick(hi,high);
	}
}

logFactory包：
logFactory

public interface LogFactory{
	UserLog createUserLogThread();
	ServerLog createServerLogThread();
}

logProduct

public class LogProduct implements LogFactory {

	@Override
	public UserLog createUserLogThread() {
		return UserLog.getInstance("action");
	}

	@Override
	public ServerLog createServerLogThread() {
		return null;
	}
}

serverlog类

public class ServerLog {
	private ServerLog() {}

	private static volatile ServerLog serverLog;

	public static ServerLog getInstance(){
		if (serverLog == null){
			synchronized (UserLog.class){
				if (serverLog == null){
					serverLog = new ServerLog();
				}
			}
		}
		return serverLog;
	}

	public String makeServiceLog(){

		try {
			Properties properties = Properties.class.newInstance();

		} catch (Exception e) {
			e.printStackTrace();
		}


		return null;

	}
}

userlog类

public class UserLog extends Thread {
	private static volatile UserLog userlog;

	private UserLog(){}
	private UserLog(String threadName) {
		setName(threadName);
	}

	// 单例
	public static UserLog getInstance(String threadName){
		if (userlog == null){
			synchronized (UserLog.class){
				if (userlog == null){
					userlog = new UserLog(threadName);
				}
			}
		}
		return userlog;
	}



	@Override
	public void run(){
		InfosWriter infosWriter = new InfosWriter();
		String name = Thread.currentThread().getName();
		int circle = 300;
		if (name.equalsIgnoreCase("action")){
			circle = 100000;
		}
		for (int i = 0; i < circle; i++) {
			infosWriter.writeLog(name);
		}
	}





	// 用时1分钟
	public static void main(String[] args) {
		LogFactory uselog = new LogProduct();
		uselog.createUserLogThread().start();
	}


}

echarts前端ajax页面

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Title</title>
</head>
<body>
    <div id="phone" style="width: 600px;height: 400px"></div>
    <script type="text/javascript" src="frame/jquery/jquery-3.5.1.js"></script>
    <script type="text/javascript" src="frame/echarts/echarts.min.js"></script>
    <script type="text/javascript">
        $.ajax({
            type : 'get',
            url : 'fenzu',
            dataType : 'json',
            success : function (res) {
                var mc = echarts.init(document.getElementById("phone"));
                var option = {
                    title : {
                        text : "nicedays"
                    },
                    color : ["#ff8400","#FFFF00","#eee564"],

                    legend : {
                        data : ["销量"]
                    },
                    series : [
                        {
                            name : "销量",
                            type : "pie",
                            radius : '25%',
                            center : ['50%','50%'],
                            data : res,
                            animation : true
                        }
                    ]
                };
                mc.setOption(option);
            }
        })



    </script>
</body>
</html>