Mapreduce is locally submitted to the cluster to run, word statistics case

Mapreduce is locally submitted to the cluster to run, word statistics case

The yarn high-availability cluster environment is built and the cluster mode is running. The
cluster mode is running: https://blog.csdn.net/weixin_43614067/article/details/108400938 The
local mode is running: https://blog.csdn.net/weixin_43614067/article/details /108386389
modify the Runner side

package com.bjsxt.wc;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.awt.*;
public class WCRunner {
    
    
    public static void main(String[] args) throws Exception {
    
    
        //创建配置对象
        Configuration conf = new Configuration();
        //默认文件系统的名称
        conf.set("fs.defaultFS", "hdfs://node001:8020");*/
        //本地运行，读取hdfs数据，并将数据提到hdfs
        conf.set("fs.defaultFS", "hdfs://node001:8020");
        /*本地集群运行map'mapreduce */
        System.setProperty("HADOOP_USER_NAME","hadoop");
        // 项目jar包地址。先编译打包，再运行main方法
        conf.set("mapreduce.job.jar", "E:\\IDEAproject\\wordcount\\classes\\artifacts\\wordcount.jar");
        conf.set("mapreduce.framework.name", "yarn");
        //node002为ResourceManager的active节点
        //命令yarn rmadmin -getServiceState rm1查看状态
        conf.set("yarn.resourcemanager.address", "http://node002:8032");       
        conf.set("yarn.resourcemanager.scheduler.address","http://node002:8030");
        //允许跨平台提交jar包
        conf.set("mapreduce.app-submission.cross-platform", "true");
        //创建Job对象
        Job job = Job.getInstance(conf, "wordCount");
        //设置mapper类
        job.setMapperClass(WCMapper.class);
        //设置 Reduce类
        job.setReducerClass(WCReducer.class);
        //设置运行job类
        job.setJarByClass(WCRunner.class);
        //设置map输出的key,value类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        //设置reduce输出的key,value类型
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        //设置输入路径金额输出路径
       /* FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));*/
        FileInputFormat.setInputPaths(job, new Path("hdfs://node001:8020/wordcount/input"));
        FileOutputFormat.setOutputPath(job, new Path("hdfs://node001:8020/wordcount/output"));
        long startTime = System.currentTimeMillis();
        try {
    
    
            //传参是：是否显示程序运行状态及进度
            boolean b = job.waitForCompletion(true);
            if (b) {
    
    
                System.out.println("单词统计完成!");
            }
        } finally {
    
    
            // 结束的毫秒数
            long endTime = System.currentTimeMillis();
            System.out.println("Job<" + job.getJobName() + ">是否执行成功:" + job.isSuccessful() + "; 开始时间:" + startTime + "; 结束时间:" + endTime + "; 用时:" + (endTime - startTime) + "ms");
        }
    }
}

Cluster mode operation: https://blog.csdn.net/weixin_43614067/article/details/108400938
Local mode operation: https://blog.csdn.net/weixin_43614067/article/details/108386389