mos可以和content一起用
package jyw.test; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.TextInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; import org.apache.hadoop.util.GenericOptionsParser; import java.io.IOException; import java.util.StringTokenizer; import org.apache.hadoop.fs.FileSystem; public class WordCountMultiDir { public static class MapClass extends Mapper<LongWritable, Text, NullWritable, Text> { private MultipleOutputs mos; @Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); mos = new MultipleOutputs(context); } @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { StringTokenizer itr = new StringTokenizer(value.toString()); while (itr.hasMoreTokens()) { // word.set(); // context.write(word, one); Text wvalue = new Text(itr.nextToken()); //方式1 mos.write(NullWritable.get(), wvalue, generateFileName(wvalue)); } } private String generateFileName(Text value) { char c = value.toString().toLowerCase().charAt(0); String dirname ; if (c >= 'a' && c <= 'g') { dirname="ag"; } else{ dirname="hz"; } return "hdfs://192.168.0.42:9000/user/jiayongwei/mul/"+ dirname + "/log"; } @Override protected void cleanup(Context context) throws IOException, InterruptedException { super.cleanup(context); mos.close(); } } public static void deleteFile(String file) throws IOException { Configuration conf = new Configuration(); // conf.addResource(new Path("/home/jiayongwei/hadoop/hadoop-0.20.0/conf/core-site.xml")); FileSystem fileSystem = FileSystem.get(conf); Path path = new Path(file); if (!fileSystem.exists(path)) { System.out.println("File " + file + " does not exists"); return; } fileSystem.delete(new Path(file), true); fileSystem.close(); } public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "MulOutput"); String[] remainingArgs = new GenericOptionsParser(conf, args) .getRemainingArgs(); if (remainingArgs.length != 1) { System.err.println("argument Error!"); System.exit(1); } Path in = new Path("hdfs://192.168.0.42:9000/user/jiayongwei/input/"); Path out = new Path(remainingArgs[0]); deleteFile("/user/jiayongwei/mul/"); deleteFile(remainingArgs[0]); FileInputFormat.setInputPaths(job, in); FileOutputFormat.setOutputPath(job, out); job.setJarByClass(WordCountMultiDir.class); job.setMapperClass(MapClass.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(Text.class); job.setNumReduceTasks(0); System.exit(job.waitForCompletion(true) ? 0 : 1); } }