双MapReduce框架求共同好友案例（案例二）

数据

需求

求出某两个人的共同好友
分析：第一个Map将数据处理为 B A 的格式输出
C A
D A
E A
第一个Reduce将Map输出的数据处理为 ：
A K,B,C,D,O,F,X,G,H,I
B A,J,F,E
C B,K,H,G,F,A,E
D H,G,A,F,L,E,C,K
E A,F,H,M,B,G,D,L
F M,A,D,L,G,C
key为好友，value为用户
第二个Map将第一个Map产生的数据处理为 A-B E
A-D F
A-D E
A与B的共同好友为E
第二个Reduce将第二个Map产生的数据处理为 A-D E，F的格式

代码

第一个MapReduce

public class FriendMR {
    public static class MapTask extends Mapper<LongWritable, Text, Text, Text>{
        @Override
        protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)
                throws IOException, InterruptedException {
            String[] split = value.toString().split(":");
            String user = split[0];
            String[] friends = split[1].split(",");
            for (String f : friends) {
                context.write(new Text(f), new Text(user));
            }
        }
    }
    /**
     * A    B,C,D,E   A是B，C，D，E的好友
     * @author huihui
     *
     */
    public static class ReduceTask extends Reducer<Text, Text, Text, Text>{
        @Override
        protected void reduce(Text f, Iterable<Text> users, Reducer<Text, Text, Text, Text>.Context context)
                throws IOException, InterruptedException {
            StringBuilder sb = new StringBuilder();
            for (Text user : users) {
                sb.append(user.toString()).append(",");
            }
            //输出时去掉value末尾的逗号
            context.write(f, new Text(sb.deleteCharAt(sb.length()-1).toString()));
        }
    }

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf,"friend1");
        //设置job的map和reduce是哪一个，并且设置是哪一任务做提交
        job.setMapperClass(MapTask.class);
        job.setReducerClass(ReduceTask.class);
        job.setJarByClass(FriendMR.class);
        //设置输出类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);
        //设置输入和输出目录
        FileInputFormat.addInputPath(job, new Path("D:\\a\\friend.txt"));
        FileOutputFormat.setOutputPath(job, new Path("D:\\a\\friend1-out"));


        //判断文件是否存在
        File file = new File("D:\\a\\friend1-out");
        if(file.exists()){
            FileUtils.deleteDirectory(file);
        }
        //提交后监控程序运行状态
        boolean completion = job.waitForCompletion(true);
        System.out.println(completion?"你很优秀！":"程序出错了，调bug去");
    }

}

第二个MapReduce

public class FriendMR2 {

    public static class MapTask extends Mapper<LongWritable, Text, Text, Text>{
        @Override
        protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)
                throws IOException, InterruptedException {
            String[] split = value.toString().split("\t");
            String f = split[0];
            String[] users = split[1].split(",");
            //将users排序
            Arrays.sort(users);
            //将user两两组合
            for (int i = 0; i < users.length-1; i++) {
                for (int j = i+1; j < users.length; j++) {
                    context.write(new Text(users[i]+"-"+users[j]), new Text(f));
                }
            }
        }
    }
    /**
     * A-B  C,D,E  A和B的共同好友为C,D,E
     * @author huihui
     *
     */
    public static class ReduceTask extends Reducer<Text, Text, Text, Text>{
        @Override
        protected void reduce(Text usercp, Iterable<Text> friends, Reducer<Text, Text, Text, Text>.Context context)
                throws IOException, InterruptedException {
            StringBuilder sb = new StringBuilder();
            for (Text f : friends) {
                sb.append(f.toString()).append(",");
            }
            context.write(usercp, new Text(sb.deleteCharAt(sb.length()-1).toString()));
        }
    }

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf,"friend2");

        job.setMapperClass(MapTask.class);
        job.setReducerClass(ReduceTask.class);
        job.setJarByClass(FriendMR2.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        FileInputFormat.addInputPath(job, new Path("D:\\a\\friend1-out"));
        FileOutputFormat.setOutputPath(job, new Path("D:\\a\\friend2-out"));


        //判断文件是否存在
        File file = new File("D:\\a\\friend2-out");
        if(file.exists()){
            FileUtils.deleteDirectory(file);
        }

        boolean completion = job.waitForCompletion(true);
        System.out.println(completion?"你很优秀！":"调bug");
    }

}