/* 说明，对于数据来源的格式不一致，获取源头一处问题，我们采取多端的输入，一个reduce的处理*/
按单词统计来说
第一步 ：App
/**
*把多个map封装到job中，启动job
*/
public class WCApp {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "file:///");
Job job = Job.getInstance(conf);
//设置job的各种属性
job.setJobName("WCAppMulti"); //作业名称
job.setJarByClass(WCApp.class); //搜索类
//多个输入
MultipleInputs.addInputPath(job,new Path("file:///d:/mr/txt"),TextInputFormat.class, WCTextMapper.class);
MultipleInputs.addInputPath(job,new Path("file:///d:/mr/seq"), SequenceFileInputFormat.class,WCSeqMapper.class);
//设置输出
FileOutputFormat.setOutputPath(job,new Path(args[0]));
job.setReducerClass(WCReducer.class); //reducer类
job.setNumReduceTasks(3); //reduce个数
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class); //
job.waitForCompletion(true);
}
}
第二步：map的处理
/**
*文本类型的文件的读取
* WCTextMapper
*/
public class WCSeqMapper extends Mapper<IntWritable, Text, Text, IntWritable>{
protected void map(IntWritable key, Text value, Context context) throws IOException, InterruptedException {
Text keyOut = new Text();
IntWritable valueOut = new IntWritable();
String[] arr = value.toString().split(" ");
for(String s : arr){
keyOut.set(s);
valueOut.set(1);
context.write(keyOut,valueOut);
}
}
}
/**
* hadoop的压缩文件读取
*SeqMapper
*/
public class WCSeqMapper extends Mapper<IntWritable, Text, Text, IntWritable>{
protected void map(IntWritable key, Text value, Context context) throws IOException, InterruptedException {
Text keyOut = new Text();
IntWritable valueOut = new IntWritable();
String[] arr = value.toString().split(" ");
for(String s : arr){
keyOut.set(s);
valueOut.set(1);
context.write(keyOut,valueOut);
}
}
}
第三步：对读取的数据集进行聚集
/**
* Reducer
*/
public class WCReducer extends Reducer<Text, IntWritable, Text, IntWritable>{
/**
* reduce
*/
protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int count = 0 ;
for(IntWritable iw : values){
count = count + iw.get() ;
}
String tno = Thread.currentThread().getName();
System.out.println(tno + " : MaxTempReducer :" + key.toString() + "=" + count);
context.write(key,new IntWritable(count));
}
}