MapReduce 求共同好友

Step1

ComFriendsMapper01

public class ComFriendsMapper01 extends Mapper<LongWritable, Text, Text, Text> {

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] split = value.toString().split(":");
        String person = split[0];
        String[] friends = split[1].split(",");
        for (String friend : friends) {
            context.write(new Text(friend), new Text(person));
        }
    }
}

 ComFriendsReducer01

public class ComFriendsReducer01 extends Reducer<Text, Text, Text, Text> {

    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        StringBuffer buffer = new StringBuffer();
        for (Text person : values) {
            buffer.append(person).append("-");
        }
        context.write(new Text(buffer.toString()), key);
    }
}

ComFriendsJobMian01

public class ComFriendsJobMian01 extends Configured implements Tool {
    public static void main(String[] args) throws Exception {
        int run = ToolRunner.run(new Configuration(), new ComFriendsJobMian01(), args);
        System.exit(run);
    }
    @Override
    public int run(String[] args) throws Exception {
        Job job = Job.getInstance(super.getConf(), "step1");
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job, new Path("file:///F:\\hdfs\\input-mapreduce-gao\\共同好友\\input"));

        job.setMapperClass(ComFriendsMapper01.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);

        job.setReducerClass(ComFriendsReducer01.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, new Path("file:///F:\\hdfs\\input-mapreduce-gao\\共同好友\\step1_output"));
        boolean b = job.waitForCompletion(true);
        return b ? 0 : 1;
    }
}

Step2

ComFriendsMapper02

public class ComFriendsMapper02 extends Mapper<LongWritable, Text, Text, Text> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] split = value.toString().split("\t");
        String persons = split[1];
        String[] friends = split[0].split("-");
        Arrays.sort(friends);
        for (int i = 0; i < friends.length - 1; i++) {
            for (int j = i + 1; j < friends.length; j++) {
                context.write(new Text(friends[i] + "-" + friends[j]), new Text(persons));
            }
        }
    }
}

ComFriendsReducer02

public class ComFriendsReducer02 extends Reducer<Text, Text, Text, Text> {
    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        StringBuffer buffer = new StringBuffer();
        for (Text value : values) {
            buffer.append(value.toString() + "\t");
        }
        context.write(key, new Text(buffer.toString()));
    }
}

ComFriendsJobMian02

public class ComFriendsJobMian02 extends Configured implements Tool {
    public static void main(String[] args) throws Exception {
        int run = ToolRunner.run(new Configuration(), new ComFriendsJobMian02(), args);
        System.exit(run);
    }
    @Override
    public int run(String[] args) throws Exception {
        Job job = Job.getInstance(super.getConf(), "step2");
        job.setInputFormatClass(TextInputFormat.class);
        TextInputFormat.addInputPath(job, new Path("file:///F:\\hdfs\\input-mapreduce-gao\\共同好友\\step1_output"));

        job.setMapperClass(ComFriendsMapper02.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);

        job.setReducerClass(ComFriendsReducer02.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        job.setOutputFormatClass(TextOutputFormat.class);
        TextOutputFormat.setOutputPath(job, new Path("file:///F:\\hdfs\\input-mapreduce-gao\\共同好友\\step2_output"));
        boolean b = job.waitForCompletion(true);
        return b ? 0 : 1;
    }
}

扫描二维码关注公众号，回复： 11068162 查看本文章