Storm 本地模式

企业开发 2018-05-09 17:55:26 阅读次数: 2
  本地模式，是在eclipse等编译器编写strom运行文件，在于模拟storm在集群运行的结果，便于代码的编写和调试。

一、下载开发环境的zip文件，将storm相关jar包导入编辑器。注意是zip文件，不是gz运行linux文件。
      http://storm.apache.org/downloads.html

二、创建数据文件
     storm有Tail属性，只适合文本源，会对文本文件内容进行监听。 

public class GetData {

	/**
	 * 生成数据
	 * @param args
	 */
	public static void main(String[] args) {
		File logFile = new File("track.log");
		Random random = new Random();

		String[] hosts = { "www.baidu.com" };
		String[] session_id = { "ABYH6Y4V4SCVXTG6DPB4VH9U123", "XXYH6YCGFJYERTT834R52FDXV9U34", "BBYH61456FGHHJ7JL89RG5VV9UYU7",
				"CYYH6Y2345GHI899OFG4V9U567", "VVVYH6Y4V4SFXZ56JIPDPB4V678" };
		String[] time = { "2014-01-07 08:40:50", "2014-01-07 08:40:51", "2014-01-07 08:40:52", "2014-01-07 08:40:53", 
				"2014-01-07 09:40:49", "2014-01-07 10:40:49", "2014-01-07 11:40:49", "2014-01-07 12:40:49" };
		
		StringBuffer sbBuffer = new StringBuffer() ;
		for (int i = 0; i < 50; i++) {
			sbBuffer.append(hosts[0]+"\t"+session_id[random.nextInt(5)]+"\t"+time[random.nextInt(8)]+"\n");
		}
		if(! logFile.exists())
		{
			try {
				logFile.createNewFile();
			} catch (IOException e) {
				System.out.println("Create logFile fail !");
			}
		}
		byte[] b = (sbBuffer.toString()).getBytes();
		
		FileOutputStream fs;
		try {
			fs = new FileOutputStream(logFile);
			fs.write(b);
			fs.close();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

}

文件格式如下：
引用

www.baidu.com XXYH6YCGFJYERTT834R52FDXV9U34 2014-01-07 12:40:49
www.baidu.com ABYH6Y4V4SCVXTG6DPB4VH9U123 2014-01-07 08:40:51
www.baidu.com ABYH6Y4V4SCVXTG6DPB4VH9U123 2014-01-07 08:40:53
www.baidu.com BBYH61456FGHHJ7JL89RG5VV9UYU7 2014-01-07 11:40:49
www.baidu.com XXYH6YCGFJYERTT834R52FDXV9U34 2014-01-07 11:40:49
www.baidu.com XXYH6YCGFJYERTT834R52FDXV9U34 2014-01-07 12:40:49
www.baidu.com ABYH6Y4V4SCVXTG6DPB4VH9U123 2014-01-07 08:40:53
www.baidu.com CYYH6Y2345GHI899OFG4V9U567 2014-01-07 12:40:49
www.baidu.com ABYH6Y4V4SCVXTG6DPB4VH9U123 2014-01-07 10:40:49
www.baidu.com ABYH6Y4V4SCVXTG6DPB4VH9U123 2014-01-07 08:40:52
www.baidu.com BBYH61456FGHHJ7JL89RG5VV9UYU7 2014-01-07 12:40:49
www.baidu.com VVVYH6Y4V4SFXZ56JIPDPB4V678 2014-01-07 12:40:49
www.baidu.com VVVYH6Y4V4SFXZ56JIPDPB4V678 2014-01-07 10:40:49



二、编写读取数据源类MySpout
Open(配置文件，上下文，发射topology类)是初始化方法
nextTuple()循环发射数据
ack() 成功处理tuple回调方法
Fail()处理失败tuple回调方法
activate和deactivate ：spout可以被暂时激活和关闭
close方法在该spout关闭当前执行，但是并不能得到保证其一定被执行。kill -9时不执行，
Storm kill {topoName} 时执行。
superviser、numberis 用Kill -9。Topology一般不用Kill -9。 

public class MySpout implements IRichSpout{

	/**
	 * 
	 */
	private static final long serialVersionUID = 1L;

	FileInputStream fis;
	InputStreamReader isr;
	BufferedReader br;			

	SpoutOutputCollector collector = null;
	
	
	String str = null;

	
	public void nextTuple() {
		try {
			while ((str = this.br.readLine()) != null) {
				// 过滤动作
				
				collector.emit(new Values(str));
				
//				Thread.sleep(3000);
				//to do 
			}
		} catch (Exception e) {
			// TODO: handle exception
		}
		
		
	}

	
	public void open(Map conf, TopologyContext context,
			SpoutOutputCollector collector) {
		try {
			this.collector = collector;
			this.fis = new FileInputStream("track.log");
			this.isr = new InputStreamReader(fis, "UTF-8");
			this.br = new BufferedReader(isr);
		} catch (Exception e) {
			e.printStackTrace();
		}
		
		// 打开文件
		
	}

	
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		// 发射数据格式,与bolt接收数据一致
		declarer.declare(new Fields("loog"));
	}

	
	public Map<String, Object> getComponentConfiguration() {
		// 与ope方法中的map对应
		return null;
	}
	

	public void ack(Object msgId) {
		// TODO Auto-generated method stub
		
	}

	
	public void activate() {
		// TODO Auto-generated method stub
		
	}

	
	public void close() {
		// TODO Auto-generated method stub
		
	}

	
	public void deactivate() {
		// TODO Auto-generated method stub
		
	}

	
	public void fail(Object msgId) {
		// TODO Auto-generated method stub
		
	}

}


  三、编写数据处理类
   prepare方法进行初始化，传入当前执行的上下文,与Spout的open方法一样
   execute接受一个tuple进行处理，也可emit数据到下一级组件
   cleanup 同ISpout的close方法，在关闭前调用，不保证其一定执行。 

public class MyBolt implements IRichBolt {

	
	private static final long serialVersionUID = 1L;
	OutputCollector collector = null;
	
	public void cleanup() {

	}
	
	int num = 0;
	String valueString = null;
	
	public void execute(Tuple input) {
//		input.getValueByField("log");
//		input.getValue(0);
		try {
			valueString = input.getStringByField("loog") ;
			
			if(valueString != null)
			{
				num ++ ;
				System.err.println("lines  :"+num +"   session_id:"+valueString.split("\t")[1]);
			}
			collector.ack(input);
			Thread.sleep(2000);
		} catch (Exception e) {
			collector.fail(input);
			e.printStackTrace();
		}
		
	}

	
	public void prepare(Map stormConf, TopologyContext context,
			OutputCollector collector) {
		this.collector = collector ;
	}

	
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		declarer.declare(new Fields("")) ;
	}

	
	public Map<String, Object> getComponentConfiguration() {
		return null;
	}

}


四、构建topology结构并运行 

public class Main {
	
	public static void main(String[] args) {

		TopologyBuilder builder = new TopologyBuilder();

		builder.setSpout("spout", new MySpout(), 1);
		builder.setBolt("bolt", new MyBolt(), 1).shuffleGrouping("spout");//与builder中的spout名称对应

		Map conf = new HashMap();
		conf.put(Config.TOPOLOGY_WORKERS, 4);

		if (args.length > 0) {
			try {
					StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
			} catch (AlreadyAliveException e) {
				e.printStackTrace();
			} catch (InvalidTopologyException e) {
				e.printStackTrace();
			}catch (AuthorizationException e) {
				e.printStackTrace();
			}
		}else {
			LocalCluster localCluster = new LocalCluster();
			localCluster.submitTopology("mytopology", conf, builder.createTopology());
		}
		
	}

}


六、如果maven项目的pom.xml文件
引用

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>com.test</groupId>
  <artifactId>StormMavenProject</artifactId>
  <packaging>jar</packaging>
  <version>0.0.1-SNAPSHOT</version>
  <name>StormMavenProject</name>
  <url>http://maven.apache.org</url>
  <dependencies>
   
   <dependency>
    <groupId>org.ow2.asm</groupId>
    <artifactId>asm</artifactId>
    <version>5.0.3</version>
   </dependency>
<dependency>
    <groupId>org.clojure</groupId>
    <artifactId>clojure</artifactId>
    <version>1.7.0</version>
</dependency>
<dependency>
    <groupId>com.lmax</groupId>
    <artifactId>disruptor</artifactId>
    <version>3.3.2</version>
</dependency>
<dependency>
    <groupId>com.esotericsoftware</groupId>
    <artifactId>kryo</artifactId>
    <version>3.0.3</version>
</dependency>
<dependency>
    <groupId>org.apache.logging.log4j</groupId>
    <artifactId>log4j-api</artifactId>
    <version>2.8</version>
</dependency>
<dependency>
    <groupId>org.apache.logging.log4j</groupId>
    <artifactId>log4j-core</artifactId>
    <version>2.8</version>
</dependency>
<dependency>
    <groupId>org.slf4j</groupId>
    <artifactId>log4j-over-slf4j</artifactId>
    <version>1.6.6</version>
</dependency>
<dependency>
    <groupId>org.apache.logging.log4j</groupId>
    <artifactId>log4j-slf4j-impl</artifactId>
    <version>2.8</version>
</dependency>
<dependency>
    <groupId>com.esotericsoftware</groupId>
    <artifactId>minlog</artifactId>
    <version>1.3.0</version>
</dependency>
<dependency>
    <groupId>org.objenesis</groupId>
    <artifactId>objenesis</artifactId>
    <version>2.1</version>
</dependency>
<dependency>
    <groupId>com.esotericsoftware</groupId>
    <artifactId>reflectasm</artifactId>
    <version>1.10.1</version>
</dependency>

<dependency>
    <groupId>javax.servlet</groupId>
    <artifactId>servlet-api</artifactId>
    <version>2.5</version>
</dependency>
<dependency>
    <groupId>org.slf4j</groupId>
    <artifactId>slf4j-api</artifactId>
    <version>1.7.21</version>
</dependency>
<dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>storm-core</artifactId>
    <version>1.1.0</version>
</dependency>
<dependency>
    <groupId>org.apache.storm</groupId>
    <artifactId>storm-rename-hack</artifactId>
    <version>1.1.0</version>
</dependency>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>3.8.1</version>
      <scope>test</scope>
    </dependency>

<dependency>
    <groupId>ring-cors</groupId>
    <artifactId>ring-cors</artifactId>
    <version>0.1.5</version>
</dependency>

  </dependencies>
  <build>
    <finalName>StormMavenProject</finalName>
  </build>
</project>

七、运行结果
引用

lines  :2   session_id:ABYH6Y4V4SCVXTG6DPB4VH9U123
lines  :3   session_id:ABYH6Y4V4SCVXTG6DPB4VH9U123
lines  :4   session_id:BBYH61456FGHHJ7JL89RG5VV9UYU7
lines  :5   session_id:XXYH6YCGFJYERTT834R52FDXV9U34

猜你喜欢

转载自javafu.iteye.com/blog/2368280
今日推荐
面壁智能发布 Eurux-8x22B 开源大模型 —— 堪称「理科状元」
开源日报 | 谷歌扶持鸿蒙上位；开源Rabbit R1；Docker加持的安卓手机；微软的焦虑和野心；海尔电器把开放平台关了
中国码农的“35岁魔咒”
蘭雅 CorelDRAW 插件 2024.5.1 国际劳动节版，免费下载
Arc Browser for Windows 1.0 正式 GA
90后程序员开发视频搬运软件、不到一年获利超 700 万，结局很刑！
周排行
【转】spring中对控制反转和依赖注入的理解
tms webcore 安装和使用
java程序员进阶相关书籍
SpringMVC接受请求参数、
如何保存训练好的机器学习模型
MyEclipse、Eclipse设置项目JDK的三个地方
商超行业微信小程序开发定制一般多少钱 （行业技术人员解读）
Markdown编辑器语言——30分钟入门到到精通
Linux系统下MongoDB的简单安装与基本操作
Power Strings
每日归档
更多
2024-05-07(14)
2024-05-06(40)
2024-05-05(0)
2024-05-04(7)
2024-05-03(19)
2024-05-02(0)
2024-05-01(4)
2024-04-30(1)
2024-04-29(40)
2024-04-28(0)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022