彷徨 | MapReduce实例一 | 判断线段的共同点个数

版权声明：版权所有,转载请注明出处.谢谢 https://blog.csdn.net/weixin_35353187/article/details/81987014

判断线段的共同点个数,及求一个点上经过该点的线段的个数,有俩条及俩条以上的线段经过该点,就有共同点

数据模型: 分别表示起点和终点

1,4
2,5
3,4
2,5
2,4
3,4
2,6
1,4
4,7
5,8
5,9
6,11
7,12
a,b
6,10
10,15
11,16
12,18
13,17

方法一:

package hadoop_day06.zhang.line;

import java.io.File;
import java.io.IOException;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class line_version1 {
	
	public static class LineMapper extends Mapper<LongWritable, Text, IntWritable, IntWritable>{
		@Override
		protected void map(LongWritable key, Text value,
				Mapper<LongWritable, Text, IntWritable, IntWritable>.Context context)
				throws IOException, InterruptedException {
			//1,4
			String[] split = value.toString().split(",");
			
			for(int i=Integer.parseInt(split[0]);i<=Integer.parseInt(split[1]);i++) {
				context.write(new IntWritable(i), new IntWritable(1));
			}
		}
	}
	
	public static class LineReducer extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable>{
		@Override
		protected void reduce(IntWritable key, Iterable<IntWritable> values,
				Reducer<IntWritable, IntWritable, IntWritable, IntWritable>.Context context)
				throws IOException, InterruptedException {
			
			int count = 0;
			for (IntWritable v : values) {
				count += v.get();
			}
			
			if(count>1) {
				context.write(key, new IntWritable(count));
			}
		}
	}
	
	public static void main(String[] args) throws Exception {
		
		Configuration conf = new Configuration();
		Job job = Job.getInstance(conf);
		job.setJarByClass(line_version1.class);
		
		job.setMapOutputKeyClass(IntWritable.class);
		job.setMapOutputValueClass(IntWritable.class);
		
		// map输出的keyvalue类型必须设置，否则mr框架会默认认为map产生key是LongWritable类型
		job.setMapperClass(LineMapper.class);
		job.setReducerClass(LineReducer.class);
		
		//输入和输出目录
		FileInputFormat.addInputPath(job, new Path("E:\\data\\line.txt"));
		FileOutputFormat.setOutputPath(job, new Path("E:\\data\\out\\line"));
		
		//判断文件是否存在
		File file = new File("E:\\data\\out\\line");
		if(file.exists()){
			FileUtils.deleteDirectory(file);
		}
		
		boolean res = job.waitForCompletion(true);
		System.out.println(res?"你很优秀！！！":"滚去调bug！！");		
	}

}

方法二:

package hadoop_day06.zhang.line;

import java.io.File;
import java.io.IOException;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

/**
 * 统计共同点
 * @author Administrator
 *
 */
public class line_version2 {
	public static class MapTask extends Mapper<LongWritable, Text, Text, IntWritable>{
		@Override
		protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, IntWritable>.Context context)
				throws IOException, InterruptedException {
			try {
				String[] split = value.toString().split(",");
				int start = Integer.parseInt(split[0]);
				int end = Integer.parseInt(split[1]);
				for(int i = start; i<=end ; i++){
					context.write(new Text(i+""), new IntWritable(1));
				}
			} catch (Exception e) {
				
			}
		}
	}
	
	public static class ReduceTask extends Reducer<Text, IntWritable, Text, IntWritable>{
		@Override
		protected void reduce(Text arg0, Iterable<IntWritable> arg1,
				Reducer<Text, IntWritable, Text, IntWritable>.Context arg2) throws IOException, InterruptedException {
			int count = 0 ;
			for (IntWritable intWritable : arg1) {
				count +=intWritable.get();
			}
			arg2.write(arg0, new IntWritable(count));
		}
	}
	
	public static void main(String[] args) throws Exception {
		
				Configuration conf = new Configuration();
				
				Job job = Job.getInstance(conf, "line");
				
				//设置map和reduce，以及提交的jar
				job.setMapperClass(MapTask.class);
				job.setReducerClass(ReduceTask.class);
				job.setJarByClass(line_version2.class);
				
				//设置输入输出类型
				job.setMapOutputKeyClass(Text.class);
				job.setMapOutputValueClass(IntWritable.class);
				
				job.setOutputKeyClass(Text.class);
				job.setOutputValueClass(IntWritable.class);
				
				//输入和输出目录
				FileInputFormat.addInputPath(job, new Path("E:\\data\\line.txt"));
				FileOutputFormat.setOutputPath(job, new Path("E:\\data\\out\\line"));
				
				//判断文件是否存在
				File file = new File("E:\\data\\out\\line");
				if(file.exists()){
					FileUtils.deleteDirectory(file);
				}
				
				//提交任务
				boolean completion = job.waitForCompletion(true);
				System.out.println(completion?"你很优秀！！！":"滚去调bug！！");
	}
}

运行结果: