微博粉丝互粉列表统计

微博粉丝互粉统计

mapper阶段

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class microMapper extends Mapper<Object, Text, Text, NullWritable>{
    Text text=new Text();
    @Override
    protected void map(Object key, Text value, Context context)
            throws IOException, InterruptedException {
        String[] lines=value.toString().trim().split("\t");
        String str1=lines[0];
        String str2=lines[1];
        if(str1.compareTo(str2)>0){

            text.set(str2+"_"+str1);
        }else {
            text.set(str1+"_"+str2);
        }

        context.write(text,NullWritable.get());
    }
}

reduce阶段

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class microReduce extends Reducer<Text,NullWritable,Text,LongWritable> {

    public static long row=1;
    @Override
    protected void reduce(Text key, Iterable<NullWritable> values, Context context)
            throws IOException, InterruptedException {
    int count=0;
        for(NullWritable dd:values){
            count++;
        }

        if(count==2){
            context.write(key,new LongWritable(row));
            row++;
        }
    }
}

runner阶段

import mapper.microMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import reduce.microReduce;

import java.io.IOException;

public class microRunner {

    public static void main(String[] args) {

        try {
            //系统设置
            Configuration con=new Configuration();
            Job job=Job.getInstance(con,"wordcount");
            job.setJarByClass(microRunner.class);
            //map阶段进行设置
            job.setMapperClass(microMapper.class);

            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(NullWritable.class);
            FileInputFormat.addInputPath(job, new Path(args[0]));
            //reduce端进行设置
            job.setReducerClass(microReduce.class);
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(LongWritable.class);
            FileOutputFormat.setOutputPath(job, new Path(args[1]));
            //返回结果
            int isok= 0;
            isok = job.waitForCompletion(true)?0:1;
            System.exit(isok);
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }

    }
}

create table if not exists com_fanslist(
mid string,
id string )
row format delimited fields terminated by '\t'
stored as textfile
location '/project/sport_micro_blog/ods_user_fanslist_out/out';

微博粉丝互粉列表统计

猜你喜欢