MapReduce --- Data Cleansing

Result file data Description:

Ip : 106.39.41.166, (city)

DATE : 10 / Nov / 2016: 00: 01: 02 +0800, (date)

Day : 10, (number of days)

Traffic: 54 is, (traffic)

Type: video, (Type: Video video or article Article This article was )

The above mentioned id: 8701 (video or article of the above mentioned id )

Testing requirements:

1,  data cleaning: Cleaning in accordance with the data, and import data washing hive data repository .

Two-stage data cleaning:

( 1 ) First stage: the required information is extracted from the original log

ip:    199.30.25.88

time:  10/Nov/2016:00:01:03 +0800

traffic:  62

Article: Article This article was / 11325

Video: Video / 3235

( 2 ) The second stage: to do fine operation based on information extracted from the

ip ---> City City ( IP )

date--> time:2016-11-10 00:01:03

day: 10

traffic:62

type:article/video

id:11325

( . 3 ) Hive database table structure :

create table data(  ip string,  time string , day string, traffic bigint,

type string, id   string )

2 , the data processing:

· Statistical most popular video / article Top10 visits ( Video / Article This article was )

· According to the statistics of the most popular cities Top10 course ( ip )

· According to traffic statistics of the most popular Top10 course ( traffic )

3 , Data Visualization: The statistical results poured MySql database, unfolded through a graphical display mode.

 

 

Today, the main installation Hive database, complete the IP data cleaning.

 

package mapreduce;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.IntWritable;  
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.mapreduce.Job;  
import org.apache.hadoop.mapreduce.Mapper;  
import org.apache.hadoop.mapreduce.Reducer;  
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;  
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;  
public class dataclean{  
    public static List<String> ips=new ArrayList<String>();
    public static  List<String> times=new ArrayList<String>();
    public static List<String> traffic=new ArrayList<String>();
    public static  List<String> wen=new ArrayList<String>();
    public static  List<String> shi=new ArrayList<String>();
    
    public static class Map extends Mapper<Object , Text , Text,Text>{  
    private static Text Name =new Text();  
    private static Text num=new Text();  
    public void map(Object key,Text value,Context context) throws IOException, InterruptedException{  
    String line=value.toString();  
    String arr[]=line.split(",");  
        Name.set(arr[0]);
        num.set(arr[0]);
    context.write(Name,num);  
    }  
    }  
    public static class Reduce extends Reducer< Text, Text,Text, Text>{  
    private static Text result= new Text();  
    int i=0;
    public void reduce(Text key,Iterable<Text> values,Context context) throws IOException, InterruptedException{  
       
        for(Text val:values){  
            context.write(key, val);
            ips.add(val.toString());
        }
    
        
        }  
        }  
    public static int run()throws IOException, ClassNotFoundException, InterruptedException
    {
        Configuration conf=new Configuration();  
        conf.set("fs.defaultFS", "hdfs://localhost:9000");
        FileSystem fs =FileSystem.get(conf);
        Job job =new Job(conf,"OneSort");  
        job.setJarByClass(dataclean.class);  
        job.setMapperClass(Map.class);  
        job.setReducerClass(Reduce.class);  
        job.setOutputKeyClass(Text.class);  
        job.setOutputValueClass(Text.class);  
        job.setInputFormatClass(TextInputFormat.class);  
        job.setOutputFormatClass(TextOutputFormat.class);  
        Path in=new Path("hdfs://localhost:9000/test2/in/result.txt");  
        Path out=new Path("hdfs://localhost:9000/test2/out/ip/1");  
        FileInputFormat.addInputPath(job,in);  
        fs.delete(out,true);
        FileOutputFormat.setOutputPath(job,out);  
        return(job.waitForCompletion(true) ? 0 : 1);  
    }
        public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException{  
    
            run();
        }  
        }
}