使用mapreduce清洗web访问日志并导入hive数据库流程

编写一个简单的日志清洗脚本，原始访问日志如下：

192.168.18.1 - - [16/Feb/2017:13:53:49 +0800] "GET /favicon.ico HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a001 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a001 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a001 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a001 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a001 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a001 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a001 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a001 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a001 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a001 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a001 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a007 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a003 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/运动鞋/a003 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/皮鞋/b001 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/皮鞋/b002 HTTP/1.1" 404 288
192.168.18.2 - - [16/Feb/2017:13:53:49 +0800] "GET /鞋子/男鞋/皮鞋/b003 HTTP/1.1" 404 288

1，按照格式做好样式数据后，将原始数据导入到/user/hadoop/name目录中；

2，创建java数据清洗执行文件：

    vim Namecount.java

import java.lang.String;
import java.io.IOException;
import java.util.*;
import java.text.SimpleDateFormat;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.io.NullWritable;
 
public class Namecount {

         public static final SimpleDateFormat FORMAT = new SimpleDateFormat("d/MMM/yyyy:HH:mm:ss", Locale.ENGLISH); //原时间格式
         public static final SimpleDateFormat dateformat1 = new SimpleDateFormat("yyyy-MM-dd");//现时间格式
       private Date parseDateFormat(String string) {         //转换时间格式
            Date parse = null;
            try {
                parse = FORMAT.parse(string);
            } catch (Exception e) {
                e.printStackTrace();
            }
            return parse;
        }
        
        public String[] parse(String line) {
            String ip = parseIP(line);       //ip
            String time = parseTime(line);   //时间
            String url = parseURL(line);     //url
            String status = parseStatus(line); //状态
            String traffic = parseTraffic(line);//流量

            return new String[] { ip, time, url, status, traffic };
        } 
        private String parseTraffic(String line) {    //流量
            final String trim = line.substring(line.lastIndexOf("\"") + 1)
                    .trim();
            String traffic = trim.split(" ")[1];
            return traffic;
        }
       private String parseStatus(String line) {     //状态
            final String trim = line.substring(line.lastIndexOf("\"") + 1)
                    .trim();
            String status = trim.split(" ")[0];
            return status;
        }

        private String parseURL(String line) {       //url
            final int first = line.indexOf("\"");
            final int last = line.lastIndexOf("\"");
            String url = line.substring(first + 1, last);
            return url;
        }
        private String parseTime(String line) {    //时间
            final int first = line.indexOf("[");
            final int last = line.indexOf("+0800]");
            String time = line.substring(first + 1, last).trim();
            Date date = parseDateFormat(time);
            return dateformat1.format(date);
        }
        private String parseIP(String line) {     //ip
            String ip = line.split("- -")[0].trim();
            return ip;
        }
    public static class Map extends
            Mapper<LongWritable, Text, Text, IntWritable> {
                
        public void map(LongWritable key, Text value, Context context)
                throws IOException, InterruptedException {
            // 将输入的纯文本文件的数据转化成String
            Text outputValue = new Text();
            String line = value.toString();
             Namecount aa=new Namecount();
            StringTokenizer tokenizerArticle = new StringTokenizer(line, "\n");
 
            // 分别对每一行进行处理
            while (tokenizerArticle.hasMoreElements()) {
                // 每行按空格划分
              String stra=tokenizerArticle.nextToken().toString();
              String [] Newstr=aa.parse(stra);

           if (Newstr[2].startsWith("GET /")) { //过滤开头字符串
                Newstr[2] = Newstr[2].substring("GET /".length());
            } 
          else if (Newstr[2].startsWith("POST /")) {
                Newstr[2] = Newstr[2].substring("POST /".length());
            }
           if (Newstr[2].endsWith(" HTTP/1.1")) { //过滤结尾字符串
                Newstr[2] = Newstr[2].substring(0, Newstr[2].length()
                        - " HTTP/1.1".length());
            }
              String[] words = Newstr[2].split("/");
              if(words.length==4){
                  outputValue.set(Newstr[0] + "\t" + Newstr[1] + "\t" + words[0]+"\t"+words[1]+"\t"+words[2]+"\t"+words[3]+"\t"+"0");
                   context.write(outputValue,new IntWritable(1));                 
}    
    }
  }
}
 
    public static class Reduce extends
            Reducer<Text, IntWritable, Text, IntWritable> {
        // 实现reduce函数
        public void reduce(Text key, Iterable<IntWritable> values,
                Context context) throws IOException, InterruptedException {
          int sum = 0;
            Iterator<IntWritable> iterator = values.iterator();
            while (iterator.hasNext()) {
                sum += iterator.next().get();
            }
            context.write(key, new IntWritable(sum));
        }
    }
    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        
	conf.set("mapred.jar","Namecount.jar");
 
        String[] ioArgs = new String[] { "name", "name_out" };
        String[] otherArgs = new GenericOptionsParser(conf, ioArgs).getRemainingArgs();
        if (otherArgs.length != 2) {
            System.err.println("Usage: Score Average <in> <out>");
            System.exit(2);
        }
 
        Job job = new Job(conf, "name_goods_count");
        job.setJarByClass(Namecount.class);
 
        // 设置Map、Combine和Reduce处理类
        job.setMapperClass(Map.class);
        job.setCombinerClass(Reduce.class);
        job.setReducerClass(Reduce.class);
 
        // 设置输出类型
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
 
        // 将输入的数据集分割成小数据块splites，提供一个RecordReder的实现
        job.setInputFormatClass(TextInputFormat.class);
        // 提供一个RecordWriter的实现，负责数据输出
        job.setOutputFormatClass(TextOutputFormat.class);
 
        // 设置输入和输出目录
        FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
        FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

3，编译执行

[hadoop@h85 mr]$ /usr/jdk1.7.0_25/bin/javac Namecount.java
[hadoop@h85 mr]$ /usr/jdk1.7.0_25/bin/jar cvf Namecount.jar Namecount*class
[hadoop@h85 mr]$ hadoop jar Namecount.jar Namecount

输出的结果被保存在/user/hadoop/name_out/part-r-00000

4，hive中创建有相应字段的表：(字段)

例如： ip string       acc_date string    wp string   sex string（鞋子种类）   type（鞋子种类） string     nid（鞋子编号） string   quanzhong(权重) int         count int

例如：192.168.18.2    20170216            鞋子            男鞋                   运动鞋                          a001                0                    13

创建表：

  create table acc_log(ip string,acc_date string,wp string,sex string,type string,nid string,quanzhong int,count int) row format delimited fields terminated by '\t';

抽取数据：

  load data inpath '/user/hadoop/name_out/part-r-00000' into table acc_log;