MapReduce——流量汇总程序案例（统计每一个手机号耗费的总上行流量、下行流量、总流量）

一、准备

（1）windows可以连接hadoop集群

（2）配置hadoop和jdk的环境变量

（3）一份要处理的数据xxx.txt

二、分析

基本思路：

Map阶段：

（1）读取一行数据，切分字段

（2）抽取手机号、上行流量、下行流量

（3）以手机号为key，bean对象为value输出，即context.write(手机号,bean);

Reduce阶段：

（1）累加上行流量和下行流量得到总流量。

（2）实现自定义的bean来封装流量信息，并将bean作为map输出的key来传输

（3）MR程序在处理数据的过程中会对数据排序(map输出的kv对传输到reduce之前，会排序)，排序的依据是map输出的key

所以，我们如果要实现自己需要的排序规则，则可以考虑将排序因素放到key中，让key实现接口：WritableComparable。

然后重写key的compareTo方法。

三、编写程序

（1）编写流量统计的bean对象

     FlowBean.java

package com.atguigu.mapreduce.flow;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.Writable;
import com.sun.javafx.font.directwrite.DWFactory;
public class FlowBean implements Writable{
	private long upFlow;     //上行流量
	private long downFlow;		//下行流量
	private long sumFlow;	//总流量
	//反序列化时需要空参构造
	public FlowBean() {
		super();
	}
	public FlowBean(long upFlow, long downFlow) {
		super();
		this.upFlow = upFlow;
		this.downFlow = downFlow;
		this.sumFlow = upFlow + downFlow;
	}
	public void set(long upFlow, long downFlow) {
		this.upFlow = upFlow;
		this.downFlow = downFlow;
		this.sumFlow = upFlow + downFlow;
	}
	public long getUpFlow() {
		return upFlow;
	}
	public void setUpFlow(long upFlow) {
		this.upFlow = upFlow;
	}
	public long getDownFlow() {
		return downFlow;
	}
	public void setDownFlow(long downFlow) {
		this.downFlow = downFlow;
	}
	//序列化
	@Override
	public void write(DataOutput out) throws IOException {
		out.writeLong(upFlow);
		out.writeLong(downFlow);
		out.writeLong(sumFlow);
	}
	//反序列化
	@Override
	public void readFields(DataInput in) throws IOException {
		this.upFlow = in.readLong();
		this.downFlow = in.readLong();
		this.sumFlow = in.readLong();
	}
	@Override
	public String toString() {
		return upFlow + "\t" + downFlow + "\t" + sumFlow;
	}
}

   FlowMapper.java

package com.atguigu.mapreduce.flow;
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class FlowMapper extends Mapper<LongWritable,Text,Text,FlowBean>{
	FlowBean bean = new FlowBean();
	Text k = new Text();
	@Override
	protected void map(LongWritable key, Text value,Context context)
			throws IOException, InterruptedException {
		//1 获取一行数据
		String line = value.toString();
		//2 截取字段
		String[] fields = line.split("\t");
		//3 封装bean对象以及获取电话号
		String phoneNum = fields[1];
		long upFlow = Long.parseLong(fields[fields.length - 3]);
		long downFlow = Long.parseLong(fields[fields.length - 2]);
		bean.set(upFlow,downFlow);
		k.set(phoneNum);
		//4 写出去
		context.write(k,bean);
	}
}

   FlowReduce.java

package com.atguigu.mapreduce.flow;
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class FlowReducer extends Reducer<Text,FlowBean,Text,FlowBean>{
	@Override
	protected void reduce(Text key, Iterable<FlowBean> values,Context context)
			throws IOException, InterruptedException {
		// 计算总的流量
		long sum_upFlow = 0;
		long sum_downFlow = 0;
		for(FlowBean bean : values) {
			sum_upFlow += bean.getUpFlow();
			sum_downFlow += bean.getDownFlow();
		}
		//输出
		context.write(key,new FlowBean(sum_upFlow,sum_downFlow));
	}
}

（2）编写mapreduce主程序

   FlowDriver.java

package com.atguigu.mapreduce.flow;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class FlowDriver {
	public static void main(String[] args) throws Exception {
		Configuration configuration = new Configuration();
		//1、获取job信息
		Job job = Job.getInstance(configuration);
		//2、获取jar的存储路径
		job.setJarByClass(FlowDriver.class);
		//3、关联map和reduce的class类
		job.setMapperClass(FlowMapper.class);
		job.setReducerClass(FlowReducer.class);
		//4、设置map阶段输出的key和value类型
		job.setMapOutputKeyClass(Text.class);
		job.setMapOutputValueClass(FlowBean.class);
		//5、设置最后输出数据的key和value类型
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(FlowBean.class);
		//6、设置输入数据的路径和输出数据的路径
		FileInputFormat.setInputPaths(job,new Path(args[0]));
		FileOutputFormat.setOutputPath(job,new Path(args[1]));
		//7、提交
		boolean result = job.waitForCompletion(true);
		System.exit(result ? 0 : 1);
	}
}

（3）执行程序