本地运行wordcount
一、编写mapper：
package sk;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class MapperClass extends Mapper<LongWritable,Text,Text,LongWritable> {
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String[] strs = value.toString().split(" ");
for(String s : strs) {
context.write(new Text(s),new LongWritable(1));//将分析好的数据直接发给shuffle进行相同key的合并
}
}
}
二、编写Reducer：
package sk;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
//继承reducer，指定四个范型
public class ReducerClass extends Reducer<Text,LongWritable,Text,LongWritable>{
protected void reduce(Text key, Iterable values, Context context) throws IOException, InterruptedException {
long count = 0 ;
for(LongWritable value : values) {
count = count + value.get();
}
context.write(key,new LongWritable(count));
}
}
三、本地调用
public class Test {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
// Job 类是一个单例模式：
Job job = Job.getInstance(conf); // job -> yarn 生成job
// 分别指定Mapper 和Reducer 是哪个类
job.setMapperClass(MapperClass.class);
job.setReducerClass(ReducerClass.class);
// 制定 Mapper 的输出Key 和 Value的类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);
// 指定 Reducer的输出Key 和Value 的类型
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
// 设置输入的文件
FileInputFormat.setInputPaths(job,“D:/a.txt”);
// 设置输出的路径 ： 注意这个路径一定不要存在
FileOutputFormat.setOutputPath(job,new Path(“D:/out111”));
// 执行 true -> console打印信息， -> 默认不打印
job.waitForCompletion(true);
}
四、测试：
输入：
输出：
五、远程调用1：
public class RemoTest {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
conf.set(“fs.defaultFS”,“hdfs://master:9000/”);
//conf.set(“mapreduce.job.jar”, “test_mr_01-1.0-SNAPSHOT.jar”);
//conf.set(“mapreduce.framework.name”, “yarn”);
//conf.set(“yarn.resourcemanager.hostname”, “master”);
//conf.set(“mapreduce.app-submission.cross-platform”, “true”);
Job job = Job.getInstance(conf);
/*
* map过程中的分区数 -> 设置reduce个数
*/
//job.setNumReduceTasks(3);
//job.setJobName(“myWordCount”);
// 设置分区规则
// job.setPartitionerClass(xxx.class);
//job.setUser(“centos”);
job.setJarByClass(Test.class);
job.setMapperClass(MapperClass.class);
job.setReducerClass(ReducerClass.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
// Path inputPath = new Path(args[0]);
// Path outputPath = new Path(args[1]);
// FileInputFormat.setInputPaths(job,inputPath);
// FileOutputFormat.setOutputPath(job,outputPath);
FileInputFormat.setInputPaths(job,"/wordcount/hello.txt");
FileOutputFormat.setOutputPath(job,new Path("/wordcount/out1"));
FileInputFormat.setMaxInputSplitSize(job,1024L);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
六、测试