Storm学习记录（六、分布式远程过程调用）

一、简介

DRPC 是通过一个 DRPC 服务端(DRPC server)来实现分布式 RPC 功能的。

DRPC Server 负责接收 RPC 请求，并将该请求发送到 Storm中运行的 Topology，等待接收 Topology 发送的处理结果，并将该结果返回给发送请求的客户端。（其实，从客户端的角度来说，DPRC 与普通的 RPC 调用并没有什么区别。）

DRPC设计目的：

为了充分利用Storm的计算能力实现高密度的并行实时计算。（Storm接收若干个数据流输入，数据在Topology当中运行完成，然后通过DRPC将结果进行输出。）

过程：

客户端通过向 DRPC 服务器发送待执行函数的名称以及该函数的参数来获取处理结果。实现该函数的拓扑使用一个DRPCSpout 从 DRPC 服务器中接收一个函数调用流。DRPC 服务器会为每个函数调用都标记了一个唯一的 id。随后拓扑会执行函数来计算结果，并在拓扑的最后使用一个名为 ReturnResults 的 bolt 连接到 DRPC 服务器，根据函数调用的 id 来将函数调用的结果返回。

二、本地构建drpc的方式 

/**
 * LinearDRPCTopologyBuilder自动创建传输spout和返回bolt
 */
public class BasicDRPCTopology {
	public static class ExclaimBolt extends BaseBasicBolt {
		@Override
		public void execute(Tuple tuple, BasicOutputCollector collector) {
			String input = tuple.getString(1);
			collector.emit(new Values(tuple.getValue(0), input + "!"));
		}

		@Override
		public void declareOutputFields(OutputFieldsDeclarer declarer) {
			declarer.declare(new Fields("id", "result"));
		}

	}

	public static void main(String[] args) throws Exception {
		LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation");
		builder.addBolt(new ExclaimBolt(), 3);

		Config conf = new Config();

		if (args == null || args.length == 0) {
			LocalDRPC drpc = new LocalDRPC();
			LocalCluster cluster = new LocalCluster();

			cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc));

			for (String word : new String[] { "hello", "goodbye" }) {
				System.err.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word));
			}

			cluster.shutdown();
			drpc.shutdown();
		} else {
			conf.setNumWorkers(3);
			StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createRemoteTopology());
		}
	}
}

/**
 * 手动创建spout和bolt
 */
public class ManualDRPC {
	public static class ExclamationBolt extends BaseBasicBolt {

		@Override
		public void declareOutputFields(OutputFieldsDeclarer declarer) {
			declarer.declare(new Fields("result", "return-info"));
		}

		@Override
		public void execute(Tuple tuple, BasicOutputCollector collector) {
			String arg = tuple.getString(0);
			Object retInfo = tuple.getValue(1);
			collector.emit(new Values(arg + "!!!", retInfo));
		}

	}

	public static void main(String[] args) {
		TopologyBuilder builder = new TopologyBuilder();
		LocalDRPC drpc = new LocalDRPC();

		DRPCSpout spout = new DRPCSpout("exclamation", drpc);
		builder.setSpout("drpc", spout);
		builder.setBolt("exclaim", new ExclamationBolt(), 3).shuffleGrouping("drpc");
		builder.setBolt("return", new ReturnResults(), 3).shuffleGrouping("exclaim");

		LocalCluster cluster = new LocalCluster();
		Config conf = new Config();
		cluster.submitTopology("exclaim", conf, builder.createTopology());

		System.err.println(drpc.execute("exclamation", "aaa"));
		System.err.println(drpc.execute("exclamation", "bbb"));

	}
}

三、集群部署drpc

1.修改conf下的storm.yaml文件，添加

drpc.servers:
    - "hadoop1"

2.分发yaml文件

scp -r storm.yaml hadoop2:/usr/local/storm*/conf

3.启动zookeeper和storm集群

4.启动任务

./bin/storm jar /usr/local/storm-1.0-SNAPSHOT.jar BasicDRPCTopology drpc

四、在本地远程调用drpc的任务

public class MyDRPCclient {

	public static void main(String[] args) {
		
		DRPCClient client = new DRPCClient("hadoop1", 3772);
		
		try {
			String result = client.execute("exclamation", "hello");
			
			System.out.println(result);
		} catch (TException e) {
			e.printStackTrace();
		} catch (DRPCExecutionException e) {
			e.printStackTrace();
		} 
	}
}

五、案例

统计twitter上某篇文章的阅读量：统计该作者及转发该文章的人的粉丝数

public class ReachTopology {
	public static Map<String, List<String>> TWEETERS_DB = new HashMap<String, List<String>>() {
		{
			/**
			 * 模拟url和发表或转发文章的人
			 */
			put("foo.com/blog/1", Arrays.asList("sally", "bob", "tim", "george", "nathan"));
			put("engineering.twitter.com/blog/5", Arrays.asList("adam", "david", "sally", "nathan"));
			put("tech.backtype.com/blog/123", Arrays.asList("tim", "mike", "john"));
		}
	};

	public static Map<String, List<String>> FOLLOWERS_DB = new HashMap<String, List<String>>() {
		{
			/**
			 * 模拟某个人的粉丝
			 */
			put("sally", Arrays.asList("bob", "tim", "alice", "adam", "jim", "chris", "jai"));
			put("bob", Arrays.asList("sally", "nathan", "jim", "mary", "david", "vivian"));
			put("tim", Arrays.asList("alex"));
			put("nathan", Arrays.asList("sally", "bob", "adam", "harry", "chris", "vivian", "emily", "jordan"));
			put("adam", Arrays.asList("david", "carissa"));
			put("mike", Arrays.asList("john", "bob"));
			put("john", Arrays.asList("alice", "nathan", "jim", "mike", "bob"));
		}
	};

	/**
	 * 获取某个url对应的博主
	 */
	public static class GetTweeters extends BaseBasicBolt {
		@Override
		public void execute(Tuple tuple, BasicOutputCollector collector) {
//			获取tid
			Object id = tuple.getValue(0);
			String url = tuple.getString(1);
			List<String> tweeters = TWEETERS_DB.get(url);
			if (tweeters != null) {
				for (String tweeter : tweeters) {
					collector.emit(new Values(id, tweeter));
				}
			}
		}

		@Override
		public void declareOutputFields(OutputFieldsDeclarer declarer) {
			declarer.declare(new Fields("id", "tweeter"));
		}
	}

	/**
	 * 获取某个博主对应的粉丝
	 */
	public static class GetFollowers extends BaseBasicBolt {
		@Override
		public void execute(Tuple tuple, BasicOutputCollector collector) {
			Object id = tuple.getValue(0);
			String tweeter = tuple.getString(1);
			List<String> followers = FOLLOWERS_DB.get(tweeter);
			if (followers != null) {
				for (String follower : followers) {
					collector.emit(new Values(id, follower));
				}
			}
		}

		@Override
		public void declareOutputFields(OutputFieldsDeclarer declarer) {
			declarer.declare(new Fields("id", "follower"));
		}
	}

	public static class PartialUniquer extends BaseBatchBolt {
		BatchOutputCollector _collector;
		Object _id;
		Set<String> _followers = new HashSet<String>();

		@Override
		public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) {
			_collector = collector;
			_id = id;
		}

		@Override
		public void execute(Tuple tuple) {
			_followers.add(tuple.getString(1));
		}

		@Override
		public void finishBatch() {
			_collector.emit(new Values(_id, _followers.size()));
		}

		@Override
		public void declareOutputFields(OutputFieldsDeclarer declarer) {
			declarer.declare(new Fields("id", "partial-count"));
		}
	}

	public static class CountAggregator extends BaseBatchBolt {
		BatchOutputCollector _collector;
		Object _id;
		int _count = 0;

		@Override
		public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, Object id) {
			_collector = collector;
			_id = id;
		}

		@Override
		public void execute(Tuple tuple) {
			_count += tuple.getInteger(1);
		}

		@Override
		public void finishBatch() {
			_collector.emit(new Values(_id, _count));
		}

		@Override
		public void declareOutputFields(OutputFieldsDeclarer declarer) {
			declarer.declare(new Fields("id", "reach"));
		}
	}

	public static LinearDRPCTopologyBuilder construct() {
		LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("reach");
		builder.addBolt(new GetTweeters(), 4);
		builder.addBolt(new GetFollowers(), 12).shuffleGrouping();
		builder.addBolt(new PartialUniquer(), 6).fieldsGrouping(new Fields("id", "follower"));
		builder.addBolt(new CountAggregator(), 3).fieldsGrouping(new Fields("id"));
		return builder;
	}

	public static void main(String[] args) throws Exception {
		LinearDRPCTopologyBuilder builder = construct();

		Config conf = new Config();

		if (args == null || args.length == 0) {
			conf.setMaxTaskParallelism(3);
			LocalDRPC drpc = new LocalDRPC();
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology("reach-drpc", conf, builder.createLocalTopology(drpc));

			String[] urlsToTry = new String[] { "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
			for (String url : urlsToTry) {
				System.err.println("Reach of " + url + ":   " + drpc.execute("reach", url));
			}

			cluster.shutdown();
			drpc.shutdown();
		} else {
			conf.setNumWorkers(6);
			StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createRemoteTopology());
		}
	}
}