Storm整合Kafka、Redis、AMQ
一、需求
ogg监控Oracle数据的增删改查操作推到Kafka，Kafka推送Json数据串，Storm进行解析过滤操作，存储一份在Redis中，在推送到AMQ，前台页面利用WebSocket实时刷新取出AMQ消息缓存数据，展示页面。
二、安装Storm、Kafka、Redis、AMQ
三、创建java项目
四、引入pom依赖
1. 引入Storm
<!--引入storm -->
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>1.1.0</version>
<!--<scope>provided</scope>-->
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
2. 引入Kafka依赖
<!--引入kafka-->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>0.8.1.1</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
</exclusions>
</dependency>
3. 引入Storm-kafka整合依赖
<!--引入storm-kafka整合包-->
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-kafka</artifactId>
<version>1.1.0</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
4. 引入Redis
<!--引入Redis--> <dependency> <groupId>org.apache.storm</groupId> <artifactId>storm-redis</artifactId> <version>1.1.0</version> </dependency>
5. 引入AMQ
<!--引入ActiveMQ-->
<dependency>
<groupId>org.apache.activemq</groupId>
<artifactId>activemq-all</artifactId>
<version>5.14.4</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
6. 引入build打包插件
<!--引入build-->
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>com.test.storm.WordsTopology</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2.1</version>
<executions>
<execution>
<goals>
<goal>exec</goal>
</goals>
</execution>
</executions>
<configuration>
<executable>java</executable>
<includeProjectDependencies>true</includeProjectDependencies>
<includePluginDependencies>true</includePluginDependencies>
<mainClass></mainClass>
</configuration>
</plugin>
</plugins>
</build>
7. 引入json解析依赖
<!--引入json -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.36</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>10.0.1</version>
</dependency>
五、定义Spout
对接Kafka直接可以在Topo中new KafkaSpout()
六、定义JsonBean
定义JsonBean将传过来的Json串解析成JavaBean
package com.test.json;
import java.util.Map;
public class JsonEmpBean {
private String table;
private String op_type;
// @JSONField(format="yyyy-MM-dd HH:mm:ss")
private String op_ts;
// @JSONField(format="yyyy-MM-dd HH:mm:ss")
private String current_ts;
private String pos;
private Map<String,Object> before;
private Map<String,Object> after;
public void setTable(String table) {
this.table = table;
}
public String getTable() {
return table;
}
public void setOp_type(String op_type) {
this.op_type = op_type;
}
public String getOp_type() {
return op_type;
}
public void setOp_ts(String op_ts) {
this.op_ts = op_ts;
}
public String getOp_ts() {
return op_ts;
}
public void setCurrent_ts(String current_ts) {
if(current_ts.contains("T")){
current_ts = current_ts.replaceFirst("T"," ");
}
this.current_ts = current_ts;
}
public String getCurrent_ts() {
return current_ts;
}
public Map<String, Object> getBefore() {
return before;
}
public void setBefore(Map<String, Object> before) {
this.before = before;
}
public Map<String, Object> getAfter() {
return after;
}
public void setAfter(Map<String, Object> after) {
this.after = after;
}
public void setPos(String pos) {
this.pos = pos;
}
public String getPos() {
return pos;
}
public JsonEmpBean() {
}
public JsonEmpBean(String table, String op_type, String op_ts, String current_ts, String pos, Map<String,Object> before, Map<String,Object> after) {
this.table = table;
this.op_type = op_type;
this.op_ts = op_ts;
setCurrent_ts(current_ts);
this.pos = pos;
this.before = before;
this.after = after;
}
@Override
public String toString() {
return "JsonEmpBean{" +
"table='" + table + '\'' +
", op_type='" + op_type + '\'' +
", op_ts='" + op_ts + '\'' +
", current_ts='" + current_ts + '\'' +
", pos='" + pos + '\'' +
", before=" + before +
", after=" + after +
'}';
}
}
七、定义AMQ的生产者
package com.test.amq;
import org.apache.activemq.ActiveMQConnection;
import org.apache.activemq.ActiveMQConnectionFactory;
import javax.jms.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by Administrator on 2018/1/17.
*/
public class Producer {
//ActiveMq 的默认用户名
private static final String USERNAME = ActiveMQConnection.DEFAULT_USER;
//ActiveMq 的默认登录密码
private static final String PASSWORD = ActiveMQConnection.DEFAULT_PASSWORD;
//ActiveMQ 的链接地址
//private static final String BROKEN_URL = ActiveMQConnection.DEFAULT_BROKER_URL;
private static final String BROKEN_URL = "tcp://10.10.56.244:61616";
/*private static final String BROKEN_URL = "tcp://10.10.34.200:61616";*/
AtomicInteger count = new AtomicInteger(0);
//链接工厂
ConnectionFactory connectionFactory;
//链接对象
Connection connection;
//事务管理
Session session;
ThreadLocal<MessageProducer> threadLocal = new ThreadLocal<>();
public void init(){
try {
//创建一个链接工厂
connectionFactory = new ActiveMQConnectionFactory(USERNAME,PASSWORD,BROKEN_URL);
//从工厂中创建一个链接
connection = connectionFactory.createConnection();
//开启链接
connection.start();
//创建一个事务（这里通过参数可以设置事务的级别）
session = connection.createSession(true,Session.SESSION_TRANSACTED);
} catch (JMSException e) {
e.printStackTrace();
}
}
public void sendMessage(String message){
try {
//创建一个消息队列
Queue queue = session.createQueue("yeWuRiBao");
//消息生产者
MessageProducer messageProducer = null;
if(threadLocal.get()!=null){
messageProducer = threadLocal.get();
}else{
messageProducer = session.createProducer(queue);
threadLocal.set(messageProducer);
}
//创建一条消息
System.out.println("------------------------------------");
System.out.println(message);
TextMessage msg = session.createTextMessage(message);
//发送消息
messageProducer.send(msg);
//提交事务
session.commit();
} catch (JMSException e) {
e.printStackTrace();
}
}
}
八、定义Bolt进行处理
package com.test.storm;
import com.alibaba.fastjson.JSON;
import com.test.amq.Producer;
import com.test.json.JsonEmpBean;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.BasicOutputCollector;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseBasicBolt;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.Values;
import redis.clients.jedis.Jedis;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Created by Administrator on 2018/1/22.
*/
public class JsonBolt extends BaseBasicBolt {
private Jedis jedis;
Producer producer ;
JsonEmpBean jsonEmpBean;
@Override
public void prepare(Map stormConf, TopologyContext context) {
super.prepare(stormConf, context);
jedis = new Jedis("master",6379);
//实例化生产者
producer = new Producer();
producer.init();
}
@Override
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
String string = tuple.getString(0);
StringBuilder sb = new StringBuilder();
try{
jsonEmpBean = JSON.parseObject(string,JsonEmpBean.class);
}catch (Exception e){
return;
}
if(jsonEmpBean != null) {
String sal = "0";
String comm = "0";
Map<String, Object> after = jsonEmpBean.getAfter();
Map<String, Object> before = jsonEmpBean.getBefore();
if(StringUtils.isNotEmpty(jsonEmpBean.getOp_type())){
if(jsonEmpBean.getOp_type().equals("I")){
//增
if(after != null && after.size()>0){
String sal1 = String.valueOf(after.get("SAL"));
String comm1 = String.valueOf(after.get("COMM")) ;
if(sal1 != null ){
sal = sal1;
sb.append(sal+",");
}else{
sb.append(sal+",");
}
if(comm1 != null){
comm = comm1;
sb.append(comm+",");
}else{
sb.append(comm+",");
}
}
}else if(jsonEmpBean.getOp_type().equals("D")){
//删
if(before != null && before.size()>0){
String sal1 = String.valueOf(before.get("SAL"));
String comm1 = String.valueOf(before.get("COMM")) ;
f(sal1 != null ){
sal = sal1;
sb.append("-"+sal+",");
}else{
sb.append(sal+",");
}
if(comm1 != null){
comm = comm1;
sb.append("-"+comm+",");
}else{
sb.append(comm+",");
}
}
}else if(jsonEmpBean.getOp_type().equals("U")){
//改
if(after != null && after.size() >0 && before != null && before.size() > 0){
Set<String> afterKeys = after.keySet();
Set<String> beforeKeys = before.keySet();
if (afterKeys.equals(beforeKeys)) {
if(afterKeys.contains("SAL") && beforeKeys.contains("SAL")){
if (!after.get("SAL").equals(before.get("SAL"))) {
sal =String.valueOf(after.get("SAL"));
//jedis.zincrby("prem",1,sal);
sb.append(sal+",");
}else{
sb.append(sal+",");
}
}
if(afterKeys.contains("COMM") && beforeKeys.contains("COMM")){
if (!after.get("COMM").equals(before.get("COMM"))) {
sal =String.valueOf(after.get("COMM"));
//jedis.zincrby("prem",1,sal);
sb.append(sal+",");
}else{
sb.append(sal+",");
}
}
}
}
}
}
}else{
return;
}
String s = sb.toString();
if(null!=s && s.length()>0){
s = s.substring(0,s.length()-1);
}
System.out.println("拼成的串:"+s);
producer.sendMessage(s);
basicOutputCollector.emit(new Values(s));
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields("jsonWords"));
}
}
九、运行提交