storm分析系统实时日志
1. 启动zookeeper：
sh /Users/hyy044101331/java_tools/zk-jiqun-start.sh
2.
启动storm:
sh /Users/hyy044101331/java_tools/start-storm.sh
3.
启动web日志：
每3秒钟打印一条日志
4. 收集日志—启动server：
sh start-server.sh storm.starter.topology_02.server.Server
1 #!/bin/bash 2 3 ## 4 # receive client log data,then send to storm spout 5 ## 6 7 jar_path="/Users/hyy044101331/work_storm_topology/topology/target/topology-0.0.1-SNAPSHOT-jar-with-dependencies.jar" 8 9 java -classpath $jar_path $1
public class Constant {
/**
* SOCKET获取数据的ip地址
*/
public static final String DATA_IP = "127.0.0.1";
/**
* SOCKET获取数据的端口port
*/
public static final int DATA_PORT = 5678;
public static final int DATA_CLIENT_PORT = 5679;
/**
* 日志文件
*/
public static final String LOG_PATH = "/Users/hyy044101331/work_struts_03/mitty/mitty_error.log";
}
import org.apache.log4j.Logger;
import storm.starter.topology_02.Constant;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.ArrayList;
/**
* 接受client传输过来的日志数据
* <p/>
* Created by mengka
*/
public class Server {
private static final Logger log = Logger.getLogger(Server.class);
private static Socket clientSocket;
private static ArrayList<PrintWriter> outs = new ArrayList<PrintWriter>();
public static void main(String[] args) throws Exception {
/**
* 接收log数据
*/
ServerSocket serverSocket_client = new ServerSocket(Constant.DATA_CLIENT_PORT);
clientSocket = serverSocket_client.accept();
invoke(clientSocket, outs);
ServerSocket serverSocket = new ServerSocket(Constant.DATA_PORT);
while (true) {
Socket socket = serverSocket.accept();
PrintWriter pwriter = new PrintWriter(socket.getOutputStream());
outs.add(pwriter);
log.info("-----------, spout socket connected: "+socket.getInetAddress());
}
}
/**
* 转发消息给storm的spout
*
* @param client
* @param writers
* @throws Exception
*/
private static void invoke(final Socket client, final ArrayList<PrintWriter> writers) throws Exception {
new Thread(new Runnable() {
public void run() {
BufferedReader clientReader = null;
PrintWriter clientWriter = null;
PrintWriter writer = null;
try {
clientReader = new BufferedReader(new InputStreamReader(client.getInputStream()));
clientWriter = new PrintWriter(client.getOutputStream());
while (true) {
/**
* 接受数据
*/
String msg = clientReader.readLine();
System.out.println(msg);
clientWriter.println("Server received " + msg);
clientWriter.flush();
/**
* 数据转发送到多个client
*/
for (int i = 0; i < writers.size(); i++) {
writer = writers.get(i);
System.out.println(i);
System.out.println("send msg:" + msg);
writer.println(msg);
writer.flush();
}
System.out.println(client.getInetAddress());
if (msg.equals("bye")) {
break;
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
//释放资源
try {
clientReader.close();
} catch (Exception e) {
}
try {
clientWriter.close();
} catch (Exception e) {
}
try {
clientSocket.close();
} catch (Exception e) {
}
}
}
}).start();
}
}
5.
收集日志—启动client：
sh start-server.sh storm.starter.topology_02.client.Client
import org.apache.log4j.Logger;
import storm.starter.topology_02.Constant;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.RandomAccessFile;
import java.net.Socket;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* Created by mengka
*/
public class Client {
private static final Logger log = Logger.getLogger(Client.class);
private long lastTimeFileSize = 0; //上次文件大小
private RandomAccessFile randomFile = null;
public static void main(String[] args) throws Exception {
/**
* 建立连接
*/
Socket socket = new Socket(Constant.DATA_IP, Constant.DATA_CLIENT_PORT);
PrintWriter out = new PrintWriter(socket.getOutputStream());
/**
* 发送实时日志数据
*/
Client client = new Client();
client.realtimeShowLog(out);
}
/**
* @param out
* @throws IOException
*/
public void realtimeShowLog(final PrintWriter out) throws IOException {
File logFile = new File(Constant.LOG_PATH);
randomFile = new RandomAccessFile(logFile, "r");
//启动一个线程每1秒钟读取新增的日志信息
ScheduledExecutorService executorService =
Executors.newScheduledThreadPool(1);
executorService.scheduleWithFixedDelay(new RealtimeLogTask(out), 0, 1, TimeUnit.SECONDS);
}
/**
* 实时读取日志文件task
*/
public class RealtimeLogTask implements Runnable {
private PrintWriter out;
public RealtimeLogTask(PrintWriter out) {
this.out = out;
}
@Override
public void run() {
try {
//获得变化部分的
randomFile.seek(lastTimeFileSize);
String tmp = "";
while ((tmp = randomFile.readLine()) != null) {
log.info("----------------, realtimeLogTask run...");
System.out.println(new String(tmp.getBytes("ISO8859-1")));
out.println(new String(tmp.getBytes("ISO8859-1")));
out.flush();
}
lastTimeFileSize = randomFile.length();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
}
6.
提交任务：
storm jar topology-0.0.1-SNAPSHOT-jar-with-dependencies.jar storm.starter.topology_02.RealtimeLogTopology "mengka-mm"
7. 日志分析结果输出：
mengka-2:storm hyy044101331$ pwd /Users/hyy044101331/logs/storm mengka-2:storm hyy044101331$ vi data Press ENTER or type command to continue 1 apple-0 2 apple-0 3 apple-0 4 tiger-1 5 tiger-1