Java implementation of MapReduce Wordcount Case

First change pom.xml:

<project xmlns="http://maven.apache.org/POM/4.0.0"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>com.mcq</groupId>
	<artifactId>mr-1101</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<dependencies>
		<dependency>
			<groupId>jdk.tools</groupId>
			<artifactId>jdk.tools</artifactId>
			<version>1.8</version>
			<scope>system</scope>
			<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
		</dependency>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>RELEASE</version>
		</dependency>
		<dependency>
			<groupId>org.apache.logging.log4j</groupId>
			<artifactId>log4j-core</artifactId>
			<version>2.8.2</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-common</artifactId>
			<version>2.7.2</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>2.7.2</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-hdfs</artifactId>
			<version>2.7.2</version>
		</dependency>
	</dependencies>
</project>

Add File log4j.properties in the resources folder:

log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

 

 WordcountDriver.java：

package com.mcq;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordcountDriver{
	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		System.out.println("hello");
		Configuration conf=new Configuration();
		//1.获取Job对象
		Job job=Job.getInstance(conf);
		//2.设置jar存储位置
		job.setJarByClass (WordcountDriver.class); 
		// Map and Reduce. 3 associated class. 
		job.setMapperClass (WordcountMapper.class); 
		job.setReducerClass (WordcountReducer.class); 
		.. 4 // Mapper stage output data provided key and value type 
		job.setMapOutputKeyClass (Text.class); 
		job.setMapOutputValueClass (IntWritable.class); 
		// set the final output. 5 key and value types. 
		job.setOutputKeyClass (Text.class); 
		job.setOutputValueClass (IntWritable.class); 
		// 6. set input path and output path 
		FileInputFormat.setInputPaths (Job, the path new new (args [0])); 
		FileOutputFormat.setOutputPath (Job, the path new new (args [. 1])); 
		.. 7 // submit the Job 
// Job. the Submit (); 
		job.waitForCompletion (to true);  
// Boolean RES = job.waitForCompletion (to true); // print the results to true represents
// System.exit (RES 0:? 1); 
	} 
}

 WordcountMapper.java：

com.mcq Package; 

Import java.io.IOException; 

Import org.apache.hadoop.io.IntWritable; 
Import org.apache.hadoop.io.LongWritable; 
Import org.apache.hadoop.io.Text; 
Import the org.apache. hadoop.mapreduce.Mapper; 

// stage Map 
// KEYIN: a data input key (offset, such as the first line is 0 to 19, 20 to the second line 25) must be LongWritable 
// VALUEIN: input data the value (such as a text string, then fill the text) 
// KEYOUT: Key types of output data 
// VALUEOUT: output data value type 
public class WordcountMapper the extends Mapper <LongWritable, the text, the text, IntWritable> { 
	IntWritable V IntWritable new new = (. 1); 
	the Text new new K = the Text (); 
	@Override 
	protected void Map (LongWritable Key, the Text value, Mapper <LongWritable, the Text, the Text, IntWritable> .context context)
			IOException throws, InterruptedException { 
		// the TODO Auto-Generated Method Stub 
		. // Get line. 1 
		String line = value.toString ();
		// 2 cutting word 
		String [] = line.split words ( ""); 
		// write cycle. 3. 
		For (String Word: words) { 
			k.set (Word); 
			context.write (K, V); 
		} 
	} 
}

 WordcountReducer.java：

package com.mcq;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

//KEYIN、VALUEIN：map阶段输出的key和value类型
public class WordcountReducer extends Reducer<Text, IntWritable, Text, IntWritable>{
	IntWritable v=new IntWritable();
	@Override
	protected void reduce(Text key, Iterable<IntWritable> values,
			Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		int sum=0;
		for(IntWritable value:values) {
			sum+=value.get();
		}
		v.set(sum);
		context.write(key, v);
	}
}

In the run configuration parameters in plus e: /mrtest/in.txt e: /mrtest/out.txt

 

 

Encountered a bug runtime reference https://blog.csdn.net/qq_40310148/article/details/86617512 solved