storm 开发实战一

1、pom配置：

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>

	<groupId>com.youku</groupId>
	<artifactId>stormtest</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<packaging>jar</packaging>

	<name>stormtest</name>
	<url>http://maven.apache.org</url>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
	</properties>
	
	<repositories>
		<repository>
			<id>Public Repositories</id>
			<url>http://repo.data.1verge.net/nexus/content/groups/public</url>
		</repository>
	</repositories>

	<dependencies>
		<!-- storm基本配置 -->
		<dependency>
			<groupId>com.youku.data</groupId>
			<artifactId>data-storm-common</artifactId>
			<version>1.0.0</version>
		</dependency>
		
		<dependency>
			<groupId>storm</groupId>
			<artifactId>storm</artifactId>
			<version>0.8.2</version>
			<!-- keep storm out of the jar-with-dependencies -->
			<scope>provided</scope>
		</dependency>
		<dependency>
			<groupId>org.twitter4j</groupId>
			<artifactId>twitter4j-core</artifactId>
			<version>2.2.6-SNAPSHOT</version>
		</dependency>
		<dependency>
			<groupId>org.twitter4j</groupId>
			<artifactId>twitter4j-stream</artifactId>
			<version>2.2.6-SNAPSHOT</version>
		</dependency>
		
	</dependencies>
	<build>
		<sourceDirectory>src/main/java</sourceDirectory>
		<testSourceDirectory>test/main/java</testSourceDirectory>
		<resources>
			<resource>
				<directory>resource</directory>
			</resource>
		</resources>

		<plugins>
			<!-- Bind the maven-assembly-plugin to the package phase this will create 
				a jar file without the storm dependencies suitable for deployment to a cluster. -->
			<plugin>
				<artifactId>maven-assembly-plugin</artifactId>
				<configuration>
					<descriptorRefs>
						<descriptorRef>jar-with-dependencies</descriptorRef>
					</descriptorRefs>
					<archive>
						<manifest>
							<mainClass></mainClass>
						</manifest>
					</archive>
				</configuration>
				<executions>
					<execution>
						<id>make-assembly</id>
						<phase>package</phase>
						<goals>
							<goal>single</goal>
						</goals>
					</execution>
				</executions>
			</plugin>

			<plugin>
				<groupId>org.codehaus.mojo</groupId>
				<artifactId>exec-maven-plugin</artifactId>
				<version>1.2.1</version>
				<executions>
					<execution>
						<goals>
							<goal>exec</goal>
						</goals>
					</execution>
				</executions>
				<configuration>
					<executable>java</executable>
					<includeProjectDependencies>true</includeProjectDependencies>
					<includePluginDependencies>false</includePluginDependencies>
					<classpathScope>compile</classpathScope>
					<mainClass>${storm.topology}</mainClass>
				</configuration>
			</plugin>

			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<configuration>
					<source>1.6</source>
					<target>1.6</target>
				</configuration>
			</plugin>

		</plugins>
		<defaultGoal>compile</defaultGoal>
	</build>
</project>

2、WordCountTopology

package com.youku.stormtest;

import java.io.FileWriter;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import com.youku.stormtest.spouts.RandomSentenceSpout;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.StormSubmitter;
import backtype.storm.topology.BasicOutputCollector;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.topology.base.BaseBasicBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;

/**
 * This topology demonstrates Storm's stream groupings and multilang capabilities.
 */
public class WordCountTopology {
    public static class SplitSentence extends  BaseBasicBolt{
        
//        public SplitSentence() {
//            super("python", "splitsentence.py");
//        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("word"));
        }

        @Override
        public Map<String, Object> getComponentConfiguration() {
            return null;
        }

		public void execute(Tuple input, BasicOutputCollector collector) {
//			 FileWriter fw;
//				try {
//					fw = new FileWriter("splitFile", true);
//					fw.write(input.getString(0) +" splitbolt:"+ this.hashCode() +" threadID:"+Thread.currentThread().getId());
//					fw.write("\r\n");
//					fw.close();
//				} catch (IOException e) {
//					e.printStackTrace();
//				}
			collector.emit(new Values(input.getString(0)));
			
		}
    }  
    
    public static class WordCount extends BaseBasicBolt {
        Map<String, Integer> counts = new HashMap<String, Integer>();

        public void execute(Tuple tuple, BasicOutputCollector collector) {
            String word = tuple.getString(0);
            Integer count = counts.get(word);
            if(count==null) count = 0;
            count++;
            counts.put(word, count);
//            FileWriter fw;
//			try {
//				fw = new FileWriter("appendFile", true);
//				 fw.write(word + "!!!" + count + " appendBolt:"+ this.hashCode() +" threadID:"+Thread.currentThread().getId());
//				 fw.write("\r\n");
//				 fw.close();
//			} catch (IOException e) {
//				e.printStackTrace();
//			}
            collector.emit(new Values(word, count));
           
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("word", "count"));
        }
    }
    
    public static void main(String[] args) throws Exception {
        TopologyBuilder builder = new TopologyBuilder();
        
        builder.setSpout("spout", new RandomSentenceSpout(), 8);
        
        builder.setBolt("split", new SplitSentence(), 8)
                 .shuffleGrouping("spout");
        builder.setBolt("count", new WordCount(), 12)
                 .fieldsGrouping("split", new Fields("word"));

        Config conf = new Config();
        conf.setDebug(true);
        conf.put("xxx", "123");
        
        if(args!=null && args.length > 0) {
            conf.setNumWorkers(3);
            
            StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
        } else {        
            conf.setMaxTaskParallelism(2);

            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("word-count", conf, builder.createTopology());
        
            Thread.sleep(10000);

            cluster.shutdown();
        }
    }
}

3、错误目录

14/09/23 14:27:04 INFO zookeeper.ClientCnxn: Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2000
14/09/23 14:27:04 WARN zookeeper.ClientCnxn: Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
java.net.SocketException: Address family not supported by protocol family: connect
	at sun.nio.ch.Net.connect(Native Method)
	at sun.nio.ch.SocketChannelImpl.connect(SocketChannelImpl.java:532)
	at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1050)
	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1077)
14/09/23 14:27:05 INFO zookeeper.ClientCnxn: Opening socket connection to server localhost/127.0.0.1:2000

添加vm参数：-Djava.net.preferIPv4Stack=true
eclipse设立禁用IPv6：Djava.net.preferIPv4Stack=true


在windows下运行ExclamationTopology或者WordCountTopology时，报“Unable to delete file错误” nio的java.nio.channels.CancelledKeyException的解决方法
这是storm的一个bug，0.8.2版本还没有解决，这个错误在cluster.shutdown();时候抛出，没有什么好解决办法，注释掉shutdown()方法程序将不会自己停止，这个错误影响有限，忽略它吧。

4、连接数据库
pom配置：

<!-- mysql-connector-java 数据库连接-->
		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<version>5.1.18</version>
		</dependency>

Database：

package com.laifeng.control.util;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;

import org.apache.log4j.Logger;

public final class Database {

	private Connection con = null;
	private String dbUrl = null;
	private String dbClass = "com.mysql.jdbc.Driver";
	private String dbUser = null;
	private String dbPasswd = null;
	private PreparedStatement prepstmt = null;
	private boolean error = false;
	private static final Logger LOG = Logger.getLogger(Database.class);
	
	public Database(final String sqlDBUrl, final String sqlUser, final String sqlPassword){
		this.dbUrl = sqlDBUrl;
		this.dbUser = sqlUser;
		this.dbPasswd = sqlPassword;
	}
	/*
	 * get connection and return a Connection object
	 */
	private Connection getConnection(final String sqlDBUrl, final String sqlUser, final String sqlPassword) throws ClassNotFoundException, SQLException {
		StringBuilder builder = new StringBuilder();
		builder.append(sqlDBUrl).append("?user=").append(sqlUser).append("&password=").append(sqlPassword);
		dbUrl = builder.toString();
		Class.forName(dbClass);
		con = DriverManager.getConnection (dbUrl);
		return con;
	}

	public void execute(final String sql){
	    if(this.con == null){
	    	try {
				this.con = this.getConnection(this.dbUrl, this.dbUser, this.dbPasswd);
			} catch (ClassNotFoundException e) {
				// TODO Auto-generated catch block
				LOG.error("execute:" + e.getMessage());
			} catch (SQLException e) {
				// TODO Auto-generated catch block
				LOG.error("execute:" + e.getMessage());
			}
	    }
	    if(this.con == null){
	    	LOG.error("execute:conn get failed. drop sql:" + sql);
	    	return;
	    }
	    try {
			prepstmt = this.con.prepareStatement(sql);
			prepstmt.executeUpdate();
		} catch (SQLException e) {
			// TODO Auto-generated catch block
			LOG.error("execute:execute failed. message = " + e.getMessage()+" drop sql:" + sql);
			this.error = true;
		}
	    if(this.error){
	    	try {
				this.con.close();
				this.con = null;
			} catch (SQLException e) {
				// TODO Auto-generated catch block
				LOG.error("execute:conn.close failed. message = " + e.getMessage());
			}
	    }
	    this.error = false;
	}
}

在bolt中调用：

package control.playerFlowrate.bolts;

import java.util.Map;

import util.Database;
import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichBolt;
import backtype.storm.tuple.Tuple;

/**
 * 
 * 
 * @author wuzhanwei
 * 
 */
public class PlayerFlowrateBolt extends BaseRichBolt {

	private transient Database _db = null;
	private static final long serialVersionUID = 1L;

	@SuppressWarnings("rawtypes")
	public void prepare(Map stormConf, TopologyContext context,
			OutputCollector collector) {
		this._db = new Database("jdbc:mysql://10.105.7.15:3306/control_stat",
				"22222", "111111");
	}

	public void execute(Tuple tuple) {
		synchronized (this._db) {
			// 初始化语句
			this._db.execute(String
					.format("insert into t_result_control_playerflowrate_detail(roomid,provinceid,asn,minutecol,cdnid,streamid,city,cvl,pd,logversion,ip,userid,appid,seid,ext,visittime) values ('%s','%s','%s','%s','%s','%s','%s',%s,%s,'%s','%s','%s','%s','%s','%s','%s');",
							tuple.getStringByField("roomid"),
							tuple.getStringByField("provinceid"),
							tuple.getStringByField("asn"),
							tuple.getStringByField("minutecol"),
							tuple.getStringByField("cdnid"),
							tuple.getStringByField("streamid"),
							tuple.getStringByField("city"),
							tuple.getIntegerByField("cvl"),
							tuple.getIntegerByField("pd"),
							tuple.getStringByField("logversion"),
							tuple.getStringByField("ip"),
							tuple.getStringByField("userid"),
							tuple.getStringByField("appid"),
							tuple.getStringByField("seid"),
							tuple.getStringByField("ext"),
							tuple.getStringByField("visittime")));
		}
	}

	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		// TODO Auto-generated method stub

	}
}