打印日志文件并将其上传存到HDFS中

在本地打印日志 , 对日志文件进行分析 , 把需要上传的日志文件移动到待上传目录 , 将待上传的日志文件逐个上传到HDFS , 并移动到备份目录 , 定期清理过期备份日志 .

---

1.首先生产日志文件:

  
  
/**
* 生产日志
* @author Administrator
*
*/
public class MoreLog {
public static void main(String[] args) throws Exception {
Logger logger = LogManager.getLogger(MoreLog.class);
System.out.println( "开始打印日志了");
while( true){
logger.info( "张俊杰zhangjunjie张俊杰zhangjunjie张俊杰zhangjunjie");
Thread.sleep( 1);
}
}
}
  
  

2.启动日志采集和日志清理

  
  
/**
* 启动日志采集
* 和日志清理
* @author Administrator
*
*/

public class StartUp {
public static void main(String[] args) {
Timer timer = new Timer();
//上传
timer.schedule( new ClollactionTask(), 0, 2* 60* 1000l);

//清理文件
timer.schedule( new ClearTask(), 0, 24* 60* 1000);
}
}
  
  

3.日志采集并上传HDFS

  
  
import java.io.File;
import java.io.FilenameFilter;
import java.net.InetAddress;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimerTask;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
* 日志采集的:
* 1.从日志目录里面查看哪些需要上传(1.2.3.....)
* 2.把需要上传的文件移动到待上传文件
* 3.上传到hdfs上(/log/2018-8-20/xxx.log)
* 4.将上传完成的日志文件移动到备份目录
* @author Administrator
*
*/

public class ClollactionTask extends TimerTask{

@Override
public void run() {
System.out.println( "开始执行任务");
try {
SimpleDateFormat format = new SimpleDateFormat( "yyyy-MM-dd-HH-mm");
String datetime = format.format( new Date());

// 1:查看哪些文件是需要上传的
File logDir = new File( "e:/testlog/");
File[] listFiles = logDir.listFiles( new FilenameFilter() {
// FileNameFilter 是过哪些文件能够获取的
@Override
public boolean accept(File dir, String name) {
return name.startsWith( "test.log.");
}
});

// 2:将文件移动到待上传目录
for (File file : listFiles) {
// file.renameTo(new File(""));
FileUtils.moveFileToDirectory(file, new File( "e:/waitUpLoad"), true);
}

//3:将待上传的文件逐个上传到hdfs上，并移动到备份目录
FileSystem fs = FileSystem.get( new URI( "hdfs://hadoop01:9000"), new Configuration(), "root");
Path detPath = new Path( "/log/"+datetime.substring( 0, 10));
//判断待上传的目录是否存在，不存在则创建一个（hdfs上的文件夹）
boolean exists = fs.exists(detPath);
if(!exists){
fs.mkdirs(detPath);
}

//判断备份目录是否存在
File backDir = new File( "e:/backDir/"+datetime);
boolean exists2 = backDir.exists();
if(!exists2){
backDir.mkdirs();
}

//得到上传的是哪一个服务上的日志文件
String hostName = InetAddress.getLocalHost().getHostName();

//4:遍历待上传的目录
File file = new File( "e:/waitUpLoad");
File[] listFile = file.listFiles();
for (File f : listFile) {
//上传到hdfs上
fs.copyFromLocalFile( new Path(f.getPath()), new Path(detPath,hostName+ "_"+f.getName()+ "_"+System.currentTimeMillis()));

//cp到备份目录
FileUtils.moveFileToDirectory(f, backDir, true);
}
fs.close();

} catch (Exception e) {
e.getStackTrace();
}

}

}
  
  

4.清理过期日志

  
  
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimerTask;

import org.apache.commons.io.FileUtils;
/**
* 清理过期日志
* @author Administrator
*
*/
public class ClearTask extends TimerTask {

@Override
public void run() {
try {
SimpleDateFormat format = new SimpleDateFormat( "yyyy-MM-dd-HH-mm");
Date nowdate = new Date();

File file = new File( "E:/backDir");
File[] listFiles = file.listFiles();

for (File dir : listFiles) {
String name = dir.getName();
Date dirdate = format.parse(name);
if(nowdate.getTime() - dirdate.getTime() > 24* 60* 60* 1000) {
FileUtils.deleteDirectory(dir);
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
  
  

附:log4j配置文件

  
  
log4j.rootLogger =ALL,systemOut,logRollingFile
#log4j.rootLogger =ALL,systemOut,logFile,logDailyFile,logRollingFile,logMail,logDB
#\u8F93\u51FA\u5230\u63A7\u5236\u53F0
log4j.appender.systemOut = org.apache.log4j.ConsoleAppender
log4j.appender.systemOut.layout = org.apache.log4j.PatternLayout
log4j.appender.systemOut.layout.ConversionPattern = [%- 5p][%- 22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n
log4j.appender.systemOut.Threshold = DEBUG
log4j.appender.systemOut.ImmediateFlush = TRUE
log4j.appender.systemOut.Target = System.out

#\u8F93\u51FA\u5230\u6587\u4EF6
log4j.appender.logFile = org.apache.log4j.FileAppender
log4j.appender.logFile.layout = org.apache.log4j.PatternLayout
log4j.appender.logFile.layout.ConversionPattern = [%- 5p][%- 22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n
log4j.appender.logFile.Threshold = WARN
log4j.appender.logFile.ImmediateFlush = TRUE
log4j.appender.logFile.Append = TRUE
log4j.appender.logFile.File = ../Struts2/WebRoot/ log/File/log4j_Struts.log
log4j.appender.logFile.Encoding = UTF- 8

#\u6309DatePattern\u8F93\u51FA\u5230\u6587\u4EF6
log4j.appender.logDailyFile = org.apache.log4j.DailyRollingFileAppender
log4j.appender.logDailyFile.layout = org.apache.log4j.PatternLayout
log4j.appender.logDailyFile.layout.ConversionPattern = [%- 5p][%- 22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n
log4j.appender.logDailyFile.Threshold = DEBUG
log4j.appender.logDailyFile.ImmediateFlush = TRUE
log4j.appender.logDailyFile.Append = TRUE
log4j.appender.logDailyFile.File = ../Struts2/WebRoot/ log/DailyFile/log4j_Struts
log4j.appender.logDailyFile.DatePattern = '.'yyyy-MM-dd-HH-mm '.log'
log4j.appender.logDailyFile.Encoding = UTF- 8

#\u8BBE\u5B9A\u6587\u4EF6\u5927\u5C0F\u8F93\u51FA\u5230\u6587\u4EF6
log4j.appender.logRollingFile = org.apache.log4j.RollingFileAppender
log4j.appender.logRollingFile.layout = org.apache.log4j.PatternLayout
log4j.appender.logRollingFile.layout.ConversionPattern = [%- 5p][%- 22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n
log4j.appender.logRollingFile.Threshold = DEBUG
log4j.appender.logRollingFile.ImmediateFlush = TRUE
log4j.appender.logRollingFile.Append = TRUE
log4j.appender.logRollingFile.File = e: /testlog/test.log
log4j.appender.logRollingFile.MaxFileSize = 1MB
log4j.appender.logRollingFile.MaxBackupIndex = 50
log4j.appender.logRollingFile.Encoding = UTF- 8

#\u7528Email\u53D1\u9001\u65E5\u5FD7
log4j.appender.logMail = org.apache.log4j.net.SMTPAppender
log4j.appender.logMail.layout = org.apache.log4j.HTMLLayout
log4j.appender.logMail.layout.LocationInfo = TRUE
log4j.appender.logMail.layout.Title = Struts2 Mail LogFile
log4j.appender.logMail.Threshold = DEBUG
log4j.appender.logMail.SMTPDebug = FALSE
log4j.appender.logMail.SMTPHost = SMTP. 163.com
log4j.appender.logMail.From = xly300 0@163.com
log4j.appender.logMail.To = xly300 0@gmail.com
#log4j.appender.logMail.Cc = [email protected]
#log4j.appender.logMail.Bcc = [email protected]
log4j.appender.logMail.SMTPUsername = xly300 0
log4j.appender.logMail.SMTPPassword = 1234567
log4j.appender.logMail.Subject = Log4j Log Messages
#log4j.appender.logMail.BufferSize = 1024
#log4j.appender.logMail.SMTPAuth = TRUE

#\u5C06\u65E5\u5FD7\u767B\u5F55\u5230MySQL\u6570\u636E\u5E93
log4j.appender.logDB = org.apache.log4j.jdbc.JDBCAppender
log4j.appender.logDB.layout = org.apache.log4j.PatternLayout
log4j.appender.logDB.Driver = com.mysql.jdbc.Driver
log4j.appender.logDB.URL = jdbc:mysql: // 127.0. 0. 1: 3306/xly
log4j.appender.logDB.User = root
log4j.appender.logDB.Password = 123456
log4j.appender.logDB.Sql = INSERT INTOT_log4j(project_name,create_date,level,category,file_name,thread_name,line,all_category,message) values( 'Struts2', '%d{yyyy-MM-ddHH:mm:ss}', '%p', '%c', '%F', '%t', '%L', '%l', '%m')