/**
* Ordinary transaction Spout
*/
public class MyTxSpout implements ITransactionalSpout<MyMata>{
private static final long serialVersionUID = 1L;
/**
* data source
*/
Map<Long, String> dbMap = null;
public MyTxSpout()
{
Random random = new Random();
dbMap = new HashMap<Long, String> ();
String[] hosts = { "www.taobao.com" };
String[] session_id = { "ABYH6Y4V4SCVXTG6DPB4VH9U123", "XXYH6YCGFJYERTT834R52FDXV9U34", "BBYH61456FGHHJ7JL89RG5VV9UYU7",
"CYYH6Y2345GHI899OFG4V9U567", "VVVYH6Y4V4SFXZ56JIPDPB4V678" };
String[] time = { "2014-01-07 08:40:50", "2014-01-07 08:40:51", "2014-01-07 08:40:52", "2014-01-07 08:40:53",
"2014-01-07 09:40:49", "2014-01-07 10:40:49", "2014-01-07 11:40:49", "2014-01-07 12:40:49" };
for (long i = 0; i < 100; i++) {
dbMap.put(i,hosts[0]+"\t"+session_id[random.nextInt(5)]+"\t"+time[random.nextInt(8)]);
}
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("tx","log"));
}
public Map<String, Object> getComponentConfiguration() {
return null;
}
public org.apache.storm.transactional.ITransactionalSpout.Coordinator<MyMata> getCoordinator(Map conf,
TopologyContext context) {
/**
* Emit the metadata (transaction tuple) to the "batch emit" stream
*/
return new MyCoordinator();
}
public org.apache.storm.transactional.ITransactionalSpout.Emitter<MyMata> getEmitter(Map conf,
TopologyContext context) {
/**
* Emit the actual batch of tuples one by one
*/
return new MyEmitter(dbMap);
}
}
2. Transaction Spout creates a new transaction (metadata) metadata
2.1 Metadata definition
public class MyMata implements Serializable{
/**
* The metadata (metadata) contains the point from which the current transaction can replay the data, stored in the zookeeper, and the spout can serialize and deserialize the metadata from the zookeeper through Kryo.
*/
private static final long serialVersionUID = 1L;
private long beginPoint;//Transaction start position
private int num ;//The number of tuples in batch
@Override
public String toString() {
return getBeginPoint()+"----"+getNum();
}
public long getBeginPoint() {
return beginPoint;
}
public void setBeginPoint(long beginPoint) {
this.beginPoint = beginPoint;
}
public int getNum() {
return num;
}
public void setNum (int num) {
this.num = num;
}
}
2.2 Obtain (metadata) metadata, and emit the tuple of the actual batch one by one
public class MyEmitter implements ITransactionalSpout.Emitter<MyMata> {
Map<Long, String> dbMap = null;
public MyEmitter(Map<Long, String> dbMap) {
this.dbMap = dbMap;
}
//Emit the tuple of the actual batch one by one
public void emitBatch(TransactionAttempt tx, MyMata coordinatorMeta, BatchOutputCollector collector) {
long beginPoint = coordinatorMeta.getBeginPoint();// Get the start position from the previous batch
int num = coordinatorMeta.getNum();// Get the batch number from the batch
for (long i = beginPoint; i < num + beginPoint; i++) {
if (dbMap.get(i) == null) {
continue;
}
collector.emit(new Values(tx, dbMap.get(i)));
}
}
public void cleanupBefore(BigInteger txid) {
}
public void close() {
}
}
3. The transaction Bolt will receive data processing from the Emitter, complete the processing, and submit it to the finishBatch method for processing.
/**
* Transaction Bolt
*/
public class MyTransactionBolt extends BaseTransactionalBolt {
private static final long serialVersionUID = 1L;
public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, TransactionAttempt id) {
this.collector = collector;
System.err.println("MyTransactionBolt prepare " + id.getTransactionId() + " attemptid" + id.getAttemptId());
}
Integer count = 0;
BatchOutputCollector collector;
TransactionAttempt tx;
// Receive data processing from Emitter, complete the processing, and submit it to the finishBatch method for processing.
public void execute(Tuple tuple) {
tx = (TransactionAttempt) tuple.getValue(0);
System.err.println(
"MyTransactionBolt TransactionAttempt " + tx.getTransactionId() + " attemptid" + tx.getAttemptId());
String log = tuple.getString(1);
if (log != null && log.length() > 0) {
count++;
}
}
// batch submit
public void finishBatch() {
System.err.println("finishBatch " + count);
collector.emit(new Values(tx, count));
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("tx", "count"));
}
}
4. Icommitter, the batches are forced to be submitted in order
public class MyCommitter extends BaseTransactionalBolt implements ICommitter {
/**
* Interface Icommitter: identifies whether IBatchBolt or BaseTransactionalBolt is a committer
*/
private static final long serialVersionUID = 1L;
public static final String GLOBAL_KEY = "GLOBAL_KEY";
public static Map<String, DbValue> dbMap = new HashMap<String, DbValue>();
int sum = 0;
TransactionAttempt id;
BatchOutputCollector collector;
public void execute(Tuple tuple) {
sum += tuple.getInteger(1);
}
public void finishBatch() {
DbValue value = dbMap.get(GLOBAL_KEY);
DbValue newValue;
if (value == null || !value.txid.equals(id.getTransactionId())) {
// update the database
newValue = new DbValue();
newValue.txid = id.getTransactionId();
if (value == null) {
newValue.count = sum;
} else {
newValue.count = value.count + sum;
}
dbMap.put(GLOBAL_KEY, newValue);
} else {
newValue = value;
}
System.err.println("total==========================:" + dbMap.get(GLOBAL_KEY).count);
// collector.emit(tuple)
}
public void prepare(Map conf, TopologyContext context, BatchOutputCollector collector, TransactionAttempt id) {
this.id = id;
this.collector = collector;
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
public static class DbValue {
BigInteger txid;
int count = 0;
}
}
5, topo type
public class MyTopo {
/**
* Transaction topo
*/
public static void main(String[] args) {
TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("ttbId", "spoutid", new MyTxSpout(), 1);
builder.setBolt("bolt1", new MyTransactionBolt(), 3).shuffleGrouping("spoutid");
builder.setBolt("committer", new MyCommitter(), 1).shuffleGrouping("bolt1");
Config conf = new Config();
conf.setDebug(true);
if (args.length > 0) {
try {
StormSubmitter.submitTopology(args[0], conf, builder.buildTopology());
} catch (AlreadyAliveException e) {
e.printStackTrace ();
} catch (InvalidTopologyException e) {
e.printStackTrace ();
} catch (AuthorizationException e) {
e.printStackTrace ();
}
} else {
LocalCluster localCluster = new LocalCluster();
localCluster.submitTopology("mytopology", conf, builder.buildTopology());
}
}
}
6. Test results
quote
启动一个事务：0----10
MyTransactionBolt prepare 1 attemptid3005464965348344518
MyTransactionBolt prepare 1 attemptid3005464965348344518
MyTransactionBolt TransactionAttempt 1 attemptid3005464965348344518
MyTransactionBolt prepare 1 attemptid3005464965348344518
MyTransactionBolt TransactionAttempt 1 attemptid3005464965348344518
MyTransactionBolt TransactionAttempt 1 attemptid3005464965348344518
MyTransactionBolt TransactionAttempt 1 attemptid3005464965348344518
MyTransactionBolt TransactionAttempt 1 attemptid3005464965348344518
MyTransactionBolt TransactionAttempt 1 attemptid3005464965348344518
finishBatch 3
MyTransactionBolt TransactionAttempt 1 attemptid3005464965348344518
MyTransactionBolt TransactionAttempt 1 attemptid3005464965348344518
MyTransactionBolt TransactionAttempt 1 attemptid3005464965348344518
MyTransactionBolt TransactionAttempt 1 attemptid3005464965348344518
finishBatch 4
finishBatch 3
total==========================:10
启动一个事务：10----10
MyTransactionBolt prepare 2 attemptid4420908201582652570
MyTransactionBolt prepare 2 attemptid4420908201582652570
MyTransactionBolt prepare 2 attemptid4420908201582652570
MyTransactionBolt TransactionAttempt 2 attemptid4420908201582652570
MyTransactionBolt TransactionAttempt 2 attemptid4420908201582652570
MyTransactionBolt TransactionAttempt 2 attemptid4420908201582652570
MyTransactionBolt TransactionAttempt 2 attemptid4420908201582652570
MyTransactionBolt TransactionAttempt 2 attemptid4420908201582652570
MyTransactionBolt TransactionAttempt 2 attemptid4420908201582652570
MyTransactionBolt TransactionAttempt 2 attemptid4420908201582652570
MyTransactionBolt TransactionAttempt 2 attemptid4420908201582652570
MyTransactionBolt TransactionAttempt 2 attemptid4420908201582652570
finishBatch 3
MyTransactionBolt TransactionAttempt 2 attemptid4420908201582652570
finishBatch 4
finishBatch 3
total==========================:20