if (shouldRunNextPhase(parsedArgs, currentPhase)) { ToolRunner.run(getConf(), new PreparePreferenceMatrixJob(), new String[]{ "--input", getInputPath().toString(),//输入路径 "--output", prepPath.toString(),//输出路径 "--maxPrefsPerUser", String.valueOf(maxPrefsPerUser), "--minPrefsPerUser", String.valueOf(minPrefsPerUser), "--booleanData", String.valueOf(booleanData),//是否为boolean型的偏好数据 "--tempDir", getTempPath().toString() }); }
下边详细的分析一下PreparePreferenceMatrixJob类的实现,主要是通过三个MapReduce来完成的:
public class PreparePreferenceMatrixJob extends AbstractJob { @Override public int run(String[] args) throws Exception { //第一个MapReduce Job itemIDIndex = prepareJob(getInputPath(), getOutputPath(ITEMID_INDEX), TextInputFormat.class, ItemIDIndexMapper.class, VarIntWritable.class, VarLongWritable.class, ItemIDIndexReducer.class, VarIntWritable.class, VarLongWritable.class, SequenceFileOutputFormat.class); //第二个MapReduce Job toUserVectors = prepareJob(getInputPath(), getOutputPath(USER_VECTORS), TextInputFormat.class, ToItemPrefsMapper.class, VarLongWritable.class, booleanData ? VarLongWritable.class : EntityPrefWritable.class, ToUserVectorsReducer.class, VarLongWritable.class, VectorWritable.class, SequenceFileOutputFormat.class); //第三个MapReduce Job toItemVectors = prepareJob(getOutputPath(USER_VECTORS), getOutputPath(RATING_MATRIX), ToItemVectorsMapper.class, IntWritable.class, VectorWritable.class, ToItemVectorsReducer.class, IntWritable.class, VectorWritable.class); toItemVectors.setCombinerClass(ToItemVectorsReducer.class); }
首先说一下prepareJob()函数,他的参数列表
(输入路径, 输出路径, Mapper类,Mapper的key,Mapper的value,reducer类,reducer的key,reducer的value)
至于这函数里边的实现就不用看了,反正就是执行了一个MapReducer操作。
下边挨个分析这三个MapReducer的具体计算过程:
(1) Mapper : ItemIDIndexMapper类,将类型为Long的itemid转换成int类型的itemid_index,并输出<itemid_index, itemid>这样将所有的item先进行split
public final class ItemIDIndexMapper extends Mapper<LongWritable,Text, VarIntWritable, VarLongWritable>{ @Override protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String[] tokens = TasteHadoopUtils.splitPrefTokens(value.toString()); long itemID = Long.parseLong(tokens[transpose ? 0 : 1]); //将long类型itemid转换成int类型的itemid_index int index = TasteHadoopUtils.idToIndex(itemID); //输出(itemid_index, itemid)类型 context.write(new VarIntWritable(index), new VarLongWritable(itemID)); } }
Reducer : temIDIndexReducer类,对每一个itemid_index下的所有的itemid取最小值,并输出<itemid_index, minimumItemid>
public final class ItemIDIndexReducer extends Reducer<VarIntWritable, VarLongWritable, VarIntWritable,VarLongWritable> { //对每一个itemid_index下的所有的itemid取最小值,并输出<itemid_index, minimumItemid> @Override protected void reduce(VarIntWritable index, Iterable<VarLongWritable> possibleItemIDs, Context context) throws IOException, InterruptedException { long minimumItemID = Long.MAX_VALUE; for (VarLongWritable varLongWritable : possibleItemIDs) { long itemID = varLongWritable.get(); if (itemID < minimumItemID) { minimumItemID = itemID; } } if (minimumItemID != Long.MAX_VALUE) { context.write(index, new VarLongWritable(minimumItemID)); } } }
(2)Mapper : ToItemPrefsMapper类,继承于ToEntityPrefsMapper类,作用是从文件中读取数据,并以
userid,<itemid, pref> 的形式作为reducer的输出
public final class ToItemPrefsMapper extends ToEntityPrefsMapper { public ToItemPrefsMapper() { super(false); } } public abstract class ToEntityPrefsMapper extends Mapper<LongWritable,Text, VarLongWritable,VarLongWritable> { @Override public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { String[] tokens = DELIMITER.split(value.toString()); long userID = Long.parseLong(tokens[0]); long itemID = Long.parseLong(tokens[1]); if (itemKey ^ transpose) { // If using items as keys, and not transposing items and users, then users are items! // Or if not using items as keys (users are, as usual), but transposing items and users, // then users are items! Confused? //如果设置偏转,那么就是将userid和itemid的位置互换,也就是基于user的CF了 long temp = userID; userID = itemID; itemID = temp; } if (booleanData) { context.write(new VarLongWritable(userID), new VarLongWritable(itemID)); } else { float prefValue = tokens.length > 2 ? Float.parseFloat(tokens[2]) + ratingShift : 1.0f; //输出(userid,<itemid, pref> )类型 context.write(new VarLongWritable(userID), new EntityPrefWritable(itemID, prefValue)); } } }
Reducer : ToUserVectorsReducer类,收集同一个userid下的<itemid, pref>对,并将itemid映射成itemid_index,然后和pref组成<itemid_index, pref>对,并将同一个userid下的所有<itemid_index, pref>对保存到一个vecotr(RandomAccessSparseVector类型的)中,并输出 (userid,vector<itemid_index, pref>) 类型的结果
public final class ToUserVectorsReducer extends Reducer<VarLongWritable,VarLongWritable,VarLongWritable,VectorWritable> { @Override protected void reduce(VarLongWritable userID, Iterable<VarLongWritable> itemPrefs, Context context) throws IOException, InterruptedException { Vector userVector = new RandomAccessSparseVector(Integer.MAX_VALUE, 100); for (VarLongWritable itemPref : itemPrefs) { //将long类型的itemid映射成int类型的itemid_index int index = TasteHadoopUtils.idToIndex(itemPref.get()); float value = itemPref instanceof EntityPrefWritable ? ((EntityPrefWritable) itemPref).getPrefValue() : 1.0f; userVector.set(index, value); } if (userVector.getNumNondefaultElements() >= minPreferences) { VectorWritable vw = new VectorWritable(userVector); vw.setWritesLaxPrecision(true); context.getCounter(Counters.USERS).increment(1); context.write(userID, vw); } } }
(3)Mapper :ToItemVectorsMapper类,将上边(2)中的输出作为Mapper的输入,将Long类型userid映射为int类型的userid_index,并输出( itemid_index, <userid_index, pref> )类型
public class ToItemVectorsMapper extends Mapper<VarLongWritable,VectorWritable,IntWritable,VectorWritable> { @Override protected void map(VarLongWritable rowIndex, VectorWritable vectorWritable, Context ctx) throws IOException, InterruptedException { Vector userRatings = vectorWritable.get(); int numElementsBeforeSampling = userRatings.getNumNondefaultElements(); userRatings = Vectors.maybeSample(userRatings, sampleSize); int numElementsAfterSampling = userRatings.getNumNondefaultElements(); int column = TasteHadoopUtils.idToIndex(rowIndex.get()); VectorWritable itemVector = new VectorWritable(new RandomAccessSparseVector(Integer.MAX_VALUE, 1)); itemVector.setWritesLaxPrecision(true); Iterator<Vector.Element> iterator = userRatings.iterateNonZero(); while (iterator.hasNext()) { Vector.Element elem = iterator.next(); //这里vector.setQuick()是将键值对插入到一个map里,因为column不变,所以每次替换value itemVector.get().setQuick(column, elem.get()); //输出( itemid_index, <userid_index, pref> ) ctx.write(new IntWritable(elem.index()), itemVector); } ctx.getCounter(Elements.USER_RATINGS_USED).increment(numElementsAfterSampling); ctx.getCounter(Elements.USER_RATINGS_NEGLECTED).increment(numElementsBeforeSampling - numElementsAfterSampling); } }
Reducer :ToItemVectorsReducer类,将同一个itemid_index下的<userid_index, pref>对收集到一起,保存在一个vector中,并输出 ( itemid_index,vector<userid_index, pref> ) 类型
public class ToItemVectorsReducer extends Reducer<IntWritable,VectorWritable,IntWritable,VectorWritable> { @Override protected void reduce(IntWritable row, Iterable<VectorWritable> vectors, Context ctx) throws IOException, InterruptedException { //合并同一个itemid_index下的<userid_index, pref>对 VectorWritable vectorWritable = VectorWritable.merge(vectors.iterator()); vectorWritable.setWritesLaxPrecision(true); ctx.write(row, vectorWritable); } }
至此,在PreparePreferenceMatrixJob中的所有MapReduce计算都完成了,最终我们看到,生成的是一个
itemid_index, vector<userid_index, pref> 的偏好矩阵