MR data filtering

MR：

package com.euphe.filter;

import com.euphe.util.HUtils;
import com.euphe.util.Utils;
import com.euphe.util.standardUtil.Location;
import com.euphe.util.standardUtil.StringListTools;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import static com.euphe.util.standardUtil.Shufflter.shufflter;

public class FilterJob extends Configured implements Tool {
    public static class Map extends Mapper<Object, Text, Text, Text> {
        private static Text text = new Text();

        public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
            String line = value.toString();
            List<String> FirstList = new ArrayList<String>();
            FirstList = StringListTools.StringToList(line,"\t");
            String time = FirstList.get(Location.DATE_TIME);
            context.write(new Text(time), new Text(line));
        }
    }

    public static class Reduce extends Reducer<Text, Text, NullWritable, Text> {
        public void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            try{
                String line = "";
                String result = "";
                boolean fflg = false;
                List<String> resultList = new ArrayList<String>();
                for(Text value : values){
                    line = value.toString();
                    fflg = shufflter(line);

                    if(fflg){
                        resultList.add(line);
                    }
                }
                result = StringListTools.ListToString(resultList, "\n");
                context.write(NullWritable.get(), new Text(result));
            }catch (Exception e){
                e.printStackTrace ();
            }
        }
    }

    @Override
    public int run(String[] args) throws Exception {
        Configuration conf = HUtils.getConf();
        conf.set("mapreduce.job.jar", Utils.getRootPathBasedPath("WEB-INF/jars/filter.jar"));
        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); // Parse command line parameters 
        if (otherArgs.length !=2) { // Requires two parameters, input and output paths, 
            System.err.println ("Usage: com.euphe.filter.FilterJob <in> <out>" );
            System.exit(2);
        }
        Job job =  Job.getInstance(conf,"Filter input  :"+otherArgs[0]+" to "+otherArgs[1]);
        job.setJarByClass(FilterJob.class);
        job.setMapperClass(FilterJob.Map.class);
        job.setReducerClass(FilterJob.Reduce.class);
        job.setNumReduceTasks ( 1 );

        // Set the key value of map output 
        job.setMapOutputKeyClass(Text.class ) ;
        job.setMapOutputValueClass(Text.class ) ;
         // Set the key of reducer output, value type 
        job.setOutputKeyClass(NullWritable.class ) ;
        job.setOutputValueClass(Text.class);

        FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
        FileOutputFormat.setOutputPath(job,new Path(otherArgs[1]));
        FileSystem.get(conf).delete( new Path(otherArgs[1]), true ); // Delete the output directory before calling the task 
        return job.waitForCompletion( true ) ? 0 : 1 ;
    }
}

 

shuffler function:

package com.euphe.util.standardUtil;

import java.util.ArrayList;
import java.util.List;

public class Shufflter {
    public static boolean shufflter(String line){
        boolean fflg = true;
        List<String> tmpList = new ArrayList<String>();
        tmpList = StringListTools.StringToList(line, "\t");

        try{
            String dk1 = tmpList.get(Location.DK1);
            String of = tmpList.get(Location.osFamily);
            String uf = tmpList.get(Location.uaFamily);
            String ty = tmpList.get(Location.type);
            if(dk1.equals("null")
                    && of.equals("unknown")
                    && uf.equals("unknown")
                    && ty.equals("unknown"))
                fflg = false ;

        }catch (Exception e){
            e.printStackTrace ();
        }
        return fflg;
    }
}

 

Location tool:

package com.euphe.util.standardUtil;

public  class Location {
     // The location of each attribute in the original data 
    public  static  final  int DATE_TIME = 0 ;

    // Each attribute position in the shufflter stage 
    public  static  final  int DK1 = 5 ;
     public  static  final  int osFamily = 8 ;
     public  static  final  int uaFamily = 9 ;
     public  static  final  int type = 13 ;
}

 

StringList tool:

package com.euphe.util.standardUtil;

import java.util.ArrayList;
import java.util.List;

public  class StringListTools {
     public  static List<String> StringToList(String str, String seperator){
         // The function reads a line of the log file, and saves each item into the List according to the separator 
        if (str == null )
             return  null ;

        List<String> strList = new ArrayList<String>();
        String[] strArray = str.split(seperator);
        for(String text : strArray)
            strList.add(text);
        return strList;
    }

    public  static String ListToString(List<String> tempList, String seperator) {
         // This function saves the List as String according to the separator 
        if (tempList == null )
             return  null ;

        String temp = new String();
        for(int i = 0; i < tempList.size()-1; i++){
            temp = temp + tempList.get(i) + seperator;
        }
        temp = temp + tempList.get(tempList.size()-1);
        return temp;
    }
}