MapReduce多Job串联及Partitioner

版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。

本文链接： https://blog.csdn.net/qq_18505209/article/details/100565749

数据：

Input：

name hwl chinese 100 math 100 english 55
name lh chinese 99 math 45 english 54
name zs chinese 52 math 95 english 51
name bb chinese 96 math 89 english 57
name lk chinese 71 math 60 english 50
name ll chinese 32 math 12 english 72
name dd chinese 63 math 74 english 13

Output：

chinese	zs:52;ll:32;dd:63;hwl:100;bb:96;lk:71;lh:99;
english	hwl:55;zs:51;bb:57;ll:72;dd:13;lh:54;lk:50;
math	hwl:100;lh:45;zs:95;lk:60;ll:12;dd:74;bb:89;

代码：

package MapReducer05;

import com.google.inject.internal.util.$StackTraceElements;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob;
import org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.log4j.BasicConfigurator;

import java.io.IOException;

public class ScoreMR {
    public static class MyMapper01 extends Mapper<LongWritable, Text, Text, Text>{
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String[] str = value.toString().split(" ");
            context.write(new Text(str[2]), new Text(str[1]+":"+str[3]));
            context.write(new Text(str[4]), new Text(str[1]+":"+str[5]));
            context.write(new Text(str[6]), new Text(str[1]+":"+str[7]));
        }
    }
    public static class MyMapper02 extends Mapper<LongWritable, Text, Text, Text>{
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            /*InputSplit is = context.getInputSplit();
            FileSplit fsp = (FileSplit) is;
            String fname = fsp.getPath().getName();*/
            String[] str = value.toString().split("\t");
            context.write(new Text(str[0]), new Text(str[1]));
        }
    }
    public static class MyReducer02 extends Reducer<Text, Text, Text, Text>{
        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            String res = "";
            for(Text t : values){
                res += t.toString()+";";
            }
            context.write(key, new Text(res));
        }
    }

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        System.setProperty("hadoop.home.dir", "F:\\hadoop-2.6.4");
        BasicConfigurator.configure();
        Configuration conf1 = new Configuration();
        Configuration conf2 = new Configuration();

        Job job1 = Job.getInstance(conf1, "mr1");
        job1.setJarByClass(ScoreMR.class);

        job1.setMapperClass(MyMapper01.class);
        job1.setMapOutputKeyClass(Text.class);
        job1.setMapOutputValueClass(Text.class);
        //分区
        job1.setPartitionerClass(MyPartitioner.class);
        job1.setNumReduceTasks(3);

        FileInputFormat.addInputPath(job1, new Path("C:\\Users\\Chen\\Desktop\\input\\00"));
        FileOutputFormat.setOutputPath(job1, new Path("C:\\Users\\Chen\\Desktop\\30"));

        Job job2 = Job.getInstance(conf2, "mr2");
        job2.setJarByClass(ScoreMR.class);

        job2.setMapperClass(MyMapper02.class);
        job2.setMapOutputKeyClass(Text.class);
        job2.setMapOutputValueClass(Text.class);

        job2.setReducerClass(MyReducer02.class);
        job2.setOutputKeyClass(Text.class);
        job2.setOutputValueClass(Text.class);

        FileInputFormat.setInputPaths(job2, new Path("C:\\Users\\Chen\\Desktop\\30"));
        FileOutputFormat.setOutputPath(job2, new Path("C:\\Users\\Chen\\Desktop\\31"));

        JobControl jobControl = new JobControl("jobControl");
        ControlledJob cj1 = new ControlledJob(job1.getConfiguration());
        ControlledJob cj2 = new ControlledJob(job2.getConfiguration());

        cj2.addDependingJob(cj1);
        jobControl.addJob(cj1);
        jobControl.addJob(cj2);

        Thread thread = new Thread(jobControl);
        thread.start();

        while (!jobControl.allFinished()){
            thread.sleep(1000);
        }
        jobControl.stop();
        System.exit(0);
        //System.exit(job2.waitForCompletion(true) ? 0 : 1);
    }
}
class MyPartitioner extends Partitioner<Text, Text> {
    @Override
    public int getPartition(Text text, Text text2, int numPartitions) {
        String str = text.toString();
        if(str.startsWith("chinese")){
            return 0%numPartitions;//0
        } else if(str.startsWith("math")){
            return 1%numPartitions;
        } else{
            return 2%numPartitions;
        }
    }
}

中间文件：

part-r-00000：

chinese	lh:99
chinese	lk:71
chinese	bb:96
chinese	hwl:100
chinese	dd:63
chinese	ll:32
chinese	zs:52

part-r-00001：

math	bb:89
math	dd:74
math	ll:12
math	lk:60
math	zs:95
math	lh:45
math	hwl:100

part-r-00002：

english	lk:50
english	lh:54
english	dd:13
english	ll:72
english	bb:57
english	zs:51
english	hwl:55