1.mkdir /export/data/logs
2.vi uploadHDFS.sh
1 #!/bin/bash 2 3 4 5 #配置环境变量 6 7 export JAVA_HOME=/export/servers/jdk 8 export JRE_HOME=${JAVA_HOME}/jre 9 export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib 10 export PATH=${JAVA_HOME}/bin:$PATH 11 12 export HADOOP_HOME=/export/servers/hadoop 13 export PATH=${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:$PATH 14 15 16 # log file storage directory . 17 log_src_dir = / Export / Data / logs / log / 18 is . 19 # directory to be uploaded file storage 20 is log_toupload_dir = / Export / Data / logs / toupload / 21 is 22 is # Set the date 23 is date1 = ` Last Day + -d-DATE Y_%%% m_Low d` 24 25 # log files uploaded to the root path of hdfs 26 is hdfs_root_dir = / Data / clickLog / $ date1 / 27 28 29 30 # printing environment variable information 31 is echo " Envs: HADOOP_HOME: $ HADOOP_HOME " 32 33 # catalog reads the log file to determine whether a file to be uploaded 34 is echo " log_src_dir: " $ log_src_dir 35 36 LS $ log_src_dir | the while Read fileName 37 [ do 38 is IF [[ " $ fileName " == * the access.log. ]]; The then 39 DATE DATE = `% +% Y_% m_Low D_ H_% M_%% S ' 40 # move the file to be uploaded to the directory and rename the 41 echo " moving $ log_src_dir $ fileName to $ log_toupload_dir " xxxxx_click_log_ $ fileName " $ DATE " 42 mv $ log_src_dir $ fileName log_toupload_dir $ " xxxxx_click_log_ fileName $ " $ DATE 43 is # the path to be written to a file upload list file willDoing 44 is echo $ log_toupload_dir " xxxxx_click_log_ fileName $ " $ DATE >> $ log_toupload_dir " willDoing. " $ DATE 45 Fi 46 is DONE 47 48 # find a list of file willDoing 49 LS $ log_toupload_dir | grep by Will | grep -v " _COPY_ " | grep -v " _DONE_ " | while read line 50 do 51 52 # print information 53 echo " toupload IS in File: " $ Line 54 # will be uploaded the file list willDoing renamed willDoing_COPY_ 55 mv $ log_toupload_dir $ Line $ log_toupload_dir $ Line " _COPY_ " 56 # to read the contents of a list of files willDoing_COPY_ (a name of a file to be uploaded) 57 # path Line here is a list of files to be uploaded 58 CAT $ log_toupload_dir $ Line " _COPY_ " | the while the Read Line 59 do 60 # print information 61 echo"puting ... $line to hdfs path ..... $hdfs_root_dir" 62 hadoop fs -mkdir -p $hdfs_root_dir 63 hadoop fs -put $line $hdfs_root_dir 64 done 65 mv $log_toupload_dir$line"_COPY_" $log_toupload_dir$line"_DONE_" 66 done
3.mkdir /export/data/logs/log/
4.vi access.log access.log.1 access.log.2
5.sh uploadHDFS.sh