Combiner执行顺序引起的错误，无法产生结果

昨天在些Tmporal Join 的代码，算法是将时间分成以阈值为大小的一个个小的区间，之后每隔一定数目的区间为同一个Key，最后一个区间需要送入到下一个Key之中，当我完成代码对的时候发现无法产生结果，通过控制台对输出

smap1**6
rmap1**0
smap2** 6
rmap2** 0
smap1**16
rmap1**0
smap2** 16
rmap2** 0
smap1**26
rmap1**0
smap2** 26
rmap2** 0
smap1**36
rmap1**0
smap2** 36
rmap2** 0
smap1**46
rmap1**0
smap2** 46
rmap2** 0
smap1**49
rmap1**0
smap2** 49
rmap2** 0

发现两个Map总有一个是没有数值的，这样就无法做join操作，开始我以为是我Reducer错误，赋值的时候出现误差，后来通过控制台对输出是没有问题，后来想到是不是map中没有读取另一个文件，后来查看发现也不是，两个文件都读取来，这个时候开始疑问是不是MapReduce这个架构出现来问题，造成无法完成我需要的操作，后来我看到以前的程序，按照这个逻辑都是可以运行对，后来我查看我的Job文件，才发现我写了Combine函数，我用我Reduce的类去执行对Combine对逻辑，本来我以为是没有问题，我认为一台电脑的执行顺序应该是执行完所有对map函数，再执行combine，而我是一台电脑，应该读取两个文件之后再执行combine，这样应该没错呀，但是看控制台对输出发现这和我想的执行过程有区别，看控制台对输出，应该是读取完一个map之后，就马上执行combine，这样才会有一个是空的集合输出，所以应该是有一个map完成之后就马上combine

以下是我查询到对资料

Combiner会在map端的那个时期执行呢？实际上，Conbiner函数的执行时机可能会在map的merge操作完成之前，也可能在merge之后执行，这个时机由配置参数min.num.spill.for.combine(该值默认为3)，也就是说在map端产生的spill文件最少有min.num.spill.for.combine的时候，Conbiner函数会在merge操作合并最终的本机结果文件之前执行，否则在merge之后执行。通过这种方式，就可以在spill文件很多并且需要做conbine的时候，减少写入本地磁盘的数据量，同样也减少了对磁盘的读写频率，可以起到优化作业的目的。

merge有三种形式：1)内存到内存  2)内存到磁盘  3)磁盘到磁盘。默认情况下第一种形式不启用，让人比较困惑，是吧。当内存中的数据量到达一定阈值，就启动内存到磁盘的merge。与map 端类似，这也是溢写的过程，这个过程中如果你设置有Combiner，也是会启用的，然后在磁盘中生成了众多的溢写文件。第二种merge方式一直在运行，直到没有map端的数据时才结束，然后启动第三种磁盘到磁盘的merge方式生成最终的那个文件。 

总结如下：

在做map操作的时候，如果有设置有combine操作的话，为了节省空间，在map执行的过程中就会执行combine函数，不一定会等到所有的map结束后才会执行。

package org.macau.stjoin.basic.temporal;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.macau.flickr.job.TemporalSimilarityJoin;
import org.macau.flickr.util.FlickrSimilarityUtil;
import org.macau.flickr.util.FlickrValue;


/**
 * 
 * @author hadoop
 * Map: Input:  KEY  : 
 *              Value:
 *      output: KEY  :
 *      		Value:
 *      
 * Reduce: Input: KEY  :
 * 				  Value:
 */
public class TemporalJoinJob {

	public static boolean TemporalSimilarityBasicJoin(Configuration conf) throws Exception{
		
		Job basicJob = new Job(conf,"Temporal Basic Similarity Join");
		basicJob.setJarByClass(TemporalSimilarityJoin.class);
		
		basicJob.setMapperClass(TemporalJoinMapper.class);
		basicJob.setCombinerClass(TemporalJoinReducer.class);
		
		basicJob.setReducerClass(TemporalJoinReducer.class);
		
		basicJob.setMapOutputKeyClass(LongWritable.class);
		basicJob.setMapOutputValueClass(FlickrValue.class);
		
//		basicJob.setOutputKeyClass(Text.class);
//		basicJob.setOutputValueClass(Text.class);
//		basicJob.setNumReduceTasks(6);
		
		FileInputFormat.addInputPath(basicJob, new Path(FlickrSimilarityUtil.flickrInputPath));
		FileOutputFormat.setOutputPath(basicJob, new Path(FlickrSimilarityUtil.flickrOutputPath));
		
		if(basicJob.waitForCompletion(true))
			return true;
		else
			return false;
	}
}

另附map代码

package org.macau.stjoin.basic.temporal;

/**
 * The Mapper uses the temporal information
 * 
 */
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.macau.flickr.util.FlickrSimilarityUtil;
import org.macau.flickr.util.FlickrValue;

public class TemporalJoinMapper extends
	Mapper<Object, Text, LongWritable, FlickrValue>{
	
	private final LongWritable outputKey = new LongWritable();
	
	private final FlickrValue outputValue = new FlickrValue();
	
	public static String convertDateToString(Date date){
		SimpleDateFormat df=new SimpleDateFormat("yyyy-MM-dd");
		return df.format(date);
	}
	
	public static Date convertLongToDate(Long date){
		return new Date(date);
	}
	
	
	public void map(Object key, Text value, Context context)
			throws IOException, InterruptedException {
		
		InputSplit inputSplit = context.getInputSplit();
		
		//R: 0; S:1
		int tag;
		
		//get the the file name which is used for separating the different set
		String fileName = ((FileSplit)inputSplit).getPath().getName();
				
		
		
		if(fileName.contains(FlickrSimilarityUtil.R_TAG)){
			
			tag = 0;
			
		}else{
			tag = 1;
		}
		
		long id =Long.parseLong(value.toString().split(":")[0]);
		double lat = Double.parseDouble(value.toString().split(":")[2]);
		double lon = Double.parseDouble(value.toString().split(":")[3]);
		long timestamp = Long.parseLong(value.toString().split(":")[4]);
		
		
		/* Convert the timestamp to the Date
		 * use the day as key
		 * the all value as a value
		 * use the timestamp to refine and compare the distance
		 */
		
//		long previousTimeStamp = timestamp - MS_OF_ONE_DAY;
//		long laterTimestamp = timestamp + MS_OF_ONE_DAY;
		
		long timeInterval = timestamp / FlickrSimilarityUtil.TEMPORAL_THRESHOLD;
		
		
		outputValue.setTileNumber((int)timeInterval);
		
		outputValue.setId(id);
		outputValue.setLat(lat);
		outputValue.setLon(lon);
		outputValue.setTag(tag);
		outputValue.setTiles(value.toString().split(":")[5]);
		
		outputValue.setTimestamp(timestamp);
		
//		System.out.println("map" + (timeInterval/10 + 1));
		
		if(timeInterval % 10 == 9){
			
			outputKey.set(timeInterval/10 + 1);
			context.write(outputKey, outputValue);
			
		}
		
		outputKey.set(timeInterval/10);
		context.write(outputKey, outputValue);
		
		
	}
}

另附reducer代码

package org.macau.stjoin.basic.temporal;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import org.apache.commons.math.util.OpenIntToDoubleHashMap.Iterator;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;
import org.macau.flickr.util.FlickrSimilarityUtil;
import org.macau.flickr.util.FlickrValue;

import org.macau.stjoin.basic.temporal.TemporalComparator;;

public class TemporalJoinReducer extends
	Reducer<LongWritable, FlickrValue, Text, Text>{
		
		
		private final Text text = new Text();
		private final ArrayList<FlickrValue> records = new ArrayList<FlickrValue>();
		private final long MS_OF_ONE_DAY = 86400000L;
		
		private final Map<Integer,ArrayList<FlickrValue>> rMap = new HashMap<Integer,ArrayList<FlickrValue>>();
		private final Map<Integer,ArrayList<FlickrValue>> sMap = new HashMap<Integer,ArrayList<FlickrValue>>();
		
		@SuppressWarnings("unchecked")
		public void reduce(LongWritable key, Iterable<FlickrValue> values,
				Context context) throws IOException, InterruptedException{
			
			//store the R and S records

			
			
			
			for(FlickrValue value:values){
				
				FlickrValue recCopy = new FlickrValue(value);
			    records.add(recCopy);
			    
//			    System.out.println(value);
			    
			    if(value.getTag() == FlickrSimilarityUtil.R_tag){
			    	
			    	if(rMap.containsKey(value.getTileNumber())){
				    	
				    	rMap.get(value.getTileNumber()).add(new FlickrValue(value));
				    	
				    }else{
				    	
				    	ArrayList<FlickrValue> list = new ArrayList<FlickrValue>();
				    	
				    	list.add(value);
//				    	System.out.println("rmap:" + rMap.size());
				    	
				    	rMap.put(value.getTileNumber(), list);
//				    	System.out.println("rmap==" + rMap.size());
				    }
			    }else{
			    	
			    	if(sMap.containsKey(value.getTileNumber())){
				    	
				    	sMap.get(value.getTileNumber()).add(new FlickrValue(value));
				    	
				    }else{
				    	
				    	ArrayList<FlickrValue> list = new ArrayList<FlickrValue>();
				    	
				    	list.add(value);
				    	
				    	sMap.put(value.getTileNumber(), list);
				    }
			    }
			    
			    
			}
			
		
			System.out.println("smap1**" + sMap.size());
			System.out.println("rmap1**" + rMap.size());
			
			for(java.util.Iterator<Integer> i = rMap.keySet().iterator();i.hasNext();){
				
				TemporalComparator comp = new TemporalComparator();
				Collections.sort(rMap.get(i.next()),comp);
				
				
			}
			

			
			
			for(java.util.Iterator<Integer> i = sMap.keySet().iterator();i.hasNext();){
				
				TemporalComparator comp = new TemporalComparator();
				Collections.sort(sMap.get(i.next()),comp);
				
			}
			
			System.out.println("smap2** " + sMap.size());
			System.out.println("rmap2** " + rMap.size());
			
			for(java.util.Iterator<Integer> obj = rMap.keySet().iterator();obj.hasNext();){
				
				System.out.println("smap3**" + sMap.size());
				System.out.println("rmap3**" + rMap.size());
				Integer i = obj.next();
				
//				System.out.println(i + "fuck"+rMap.get(i).size());
//				System.out.println("smap" + sMap.size());
//				System.out.println(sMap.containsKey(i));
//				System.out.println(sMap.containsKey(obj));
				
				for(java.util.Iterator<Integer> o= sMap.keySet().iterator();o.hasNext();){
					
					int value = o.next();
					System.out.println("smap4** " + sMap.size());
					System.out.println("rmap4** " + rMap.size());
//					System.out.println(value + "good "+sMap.get(value).size());
					
				}
				
				if(sMap.containsKey(i)){
					
					System.out.println("size" + rMap.get(i).size());
					
					for(int j = 0;j < rMap.get(i).size();j++){
						
						FlickrValue value1 = rMap.get(i).get(j);
						
						System.out.println(i + "," + j );
						
						for(int k = 0; k < sMap.get(i).size();k++){
							FlickrValue value2 = sMap.get(i).get(k);
							
							if(FlickrSimilarityUtil.SpatialSimilarity(value1, value2)){
								long ridA = value1.getId();
					            long ridB = value2.getId();
					            if (ridA < ridB) {
					                long rid = ridA;
					                ridA = ridB;
					                ridB = rid;
					            }
					            
					            //System.out.println("" + ridA + "%" + ridB);
					            
					            text.set("" + ridA + "%" + ridB);
					            context.write(text, new Text(""));
							}
						}
						
						// for the adjacent tail
						if(sMap.containsKey(i+1)){
							for(int m = 0; m < sMap.get(i+1).size();m++){
								FlickrValue value3 = sMap.get(i+1).get(m);
								
								if(FlickrSimilarityUtil.SpatialSimilarity(value1, value3)){
									
									if(FlickrSimilarityUtil.SpatialSimilarity(value1, value3)){
										
										long ridA = value1.getId();
							            long ridB = value3.getId();
							            
							            if (ridA < ridB) {
							                long rid = ridA;
							                ridA = ridB;
							                ridB = rid;
							            }
							            
										text.set("" + ridA + "%" + ridB);
							            context.write(text, new Text(""));
									}
								}
								
							}
						}
						
					}
				}
			}

			
//			rMap.clear();
//			sMap.clear();
		}
	}