MapReduce自定义分组比较器

版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。

本文链接： https://blog.csdn.net/qq_18505209/article/details/100550885

数据：

Input：

20 21
50 51
50 53
50 52
50 54
60 51
60 53
60 52
60 56
60 57
70 58
60 61
70 54
70 55
70 56
70 57
70 58

Output：

20 21
50 54
60 61
70 58

代码：

package MapReducer05;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.log4j.BasicConfigurator;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

public class WordCountGroup {
    public static class MyMapper extends Mapper<LongWritable, Text, MyWri, NullWritable>{

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String[] str = value.toString().split(" ");
            MyWri mw = new MyWri();
            mw.setFirst(Integer.parseInt(str[0]));
            mw.setLast(Integer.parseInt(str[1]));
            context.write(mw, NullWritable.get());
        }
    }
    public static class MyReducer extends Reducer<MyWri, NullWritable, MyWri, NullWritable> {
        @Override
        protected void reduce(MyWri key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
        	//每个分组一个Iterable，以第一个字段分组
        	//只输出第一key，即第一个字段相同中的第二个字段最大的key
            int flag = 0;
            for(NullWritable nw : values){
                context.write(key, nw);
                flag = 1;
                if(flag==1) break;
            }
            //等context.write(key, NullWritable.get());
        }
    }
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
    	//System.setProperty("hadoop.home.dir", "F:\\hadoop-2.6.4");
        Configuration conf = new Configuration();
        BasicConfigurator.configure();
        Job job = Job.getInstance(conf, "mr");

        job.setJarByClass(WordCount.class);
        job.setMapperClass(MyMapper.class);
        job.setMapOutputKeyClass(MyWri.class);
        job.setMapOutputValueClass(NullWritable.class);
		//添加分组比较器
        job.setGroupingComparatorClass(MyGroupCompartor.class);

        job.setReducerClass(MyReducer.class);
        job.setOutputKeyClass(MyWri.class);
        job.setOutputValueClass(NullWritable.class);

        FileInputFormat.addInputPath(job, new Path("C:\\Users\\Chen\\Desktop\\input\\secondarysort"));
        FileOutputFormat.setOutputPath(job, new Path("C:\\Users\\Chen\\Desktop\\20"));

        System.exit(job.waitForCompletion(true)?0:1);


    }
}
//继承WritableComparator类实现分组比较器
class MyGroupCompartor extends WritableComparator{
    public MyGroupCompartor(){
        super(MyWri.class,true);
    }
    @Override
    public int compare(WritableComparable o1, WritableComparable o2) {
    	//按第一个字段进行分组
        MyWri mw1 = (MyWri)o1;
        MyWri mw2 = (MyWri)o2;
        return mw1.getFirst()-mw2.getFirst();
    }
}
//自定义Writable类
class MyWri implements WritableComparable<MyWri>{
    private int first;
    private int last;
    @Override
    public int compareTo(MyWri o) {
        int ff = this.first-o.first;
        if(ff==0){
            return o.last-this.last;
        } else {
            return ff;
        }
    }
    public MyWri(){}
    public MyWri(int first, int last) {
        this.first = first;
        this.last = last;
    }

    @Override
    public String toString() {
        return this.first+" "+this.last;
    }

    @Override
    public void write(DataOutput out) throws IOException {
        out.writeInt(first);
        out.writeInt(last);
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        this.first = in.readInt();
        this.last = in.readInt();
    }

    public void setFirst(int first) {
        this.first = first;
    }

    public void setLast(int last) {
        this.last = last;
    }

    public int getFirst() {
        return first;
    }

    public int getLast() {
        return last;
    }
}