版权声明:版权所有,转载请注明出处.谢谢 https://blog.csdn.net/weixin_35353187/article/details/81901290
在本地打印日志 , 对日志文件进行分析 , 把需要上传的日志文件移动到待上传目录 , 将待上传的日志文件逐个上传到HDFS , 并移动到备份目录 , 定期清理过期备份日志 .
1.首先生产日志文件:
/**
* 生产日志
* @author Administrator
*
*/
public class MoreLog {
public static void main(String[] args) throws Exception {
Logger logger = LogManager.getLogger(MoreLog.class);
System.out.println("开始打印日志了");
while(true){
logger.info("张俊杰zhangjunjie张俊杰zhangjunjie张俊杰zhangjunjie");
Thread.sleep(1);
}
}
}
2.启动日志采集和日志清理
/**
* 启动日志采集
* 和日志清理
* @author Administrator
*
*/
public class StartUp {
public static void main(String[] args) {
Timer timer = new Timer();
//上传
timer.schedule(new ClollactionTask(), 0, 2*60*1000l);
//清理文件
timer.schedule(new ClearTask(), 0, 24*60*1000);
}
}
3.日志采集并上传HDFS
import java.io.File;
import java.io.FilenameFilter;
import java.net.InetAddress;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimerTask;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
/**
* 日志采集的:
* 1.从日志目录里面查看哪些需要上传(1.2.3.....)
* 2.把需要上传的文件移动到待上传文件
* 3.上传到hdfs上(/log/2018-8-20/xxx.log)
* 4.将上传完成的日志文件移动到备份目录
* @author Administrator
*
*/
public class ClollactionTask extends TimerTask{
@Override
public void run() {
System.out.println("开始执行任务");
try {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm");
String datetime = format.format(new Date());
// 1:查看哪些文件是需要上传的
File logDir = new File("e:/testlog/");
File[] listFiles = logDir.listFiles(new FilenameFilter() {
// FileNameFilter 是过哪些文件能够获取的
@Override
public boolean accept(File dir, String name) {
return name.startsWith("test.log.");
}
});
// 2:将文件移动到待上传目录
for (File file : listFiles) {
// file.renameTo(new File(""));
FileUtils.moveFileToDirectory(file, new File("e:/waitUpLoad"), true);
}
//3:将待上传的文件逐个上传到hdfs上,并移动到备份目录
FileSystem fs = FileSystem.get(new URI("hdfs://hadoop01:9000"), new Configuration(), "root");
Path detPath = new Path("/log/"+datetime.substring(0, 10));
//判断待上传的目录是否存在,不存在则创建一个(hdfs上的文件夹)
boolean exists = fs.exists(detPath);
if(!exists){
fs.mkdirs(detPath);
}
//判断备份目录是否存在
File backDir = new File("e:/backDir/"+datetime);
boolean exists2 = backDir.exists();
if(!exists2){
backDir.mkdirs();
}
//得到上传的是哪一个服务上的日志文件
String hostName = InetAddress.getLocalHost().getHostName();
//4:遍历待上传的目录
File file = new File("e:/waitUpLoad");
File[] listFile = file.listFiles();
for (File f : listFile) {
//上传到hdfs上
fs.copyFromLocalFile(new Path(f.getPath()), new Path(detPath,hostName+"_"+f.getName()+"_"+System.currentTimeMillis()));
//cp到备份目录
FileUtils.moveFileToDirectory(f, backDir, true);
}
fs.close();
} catch (Exception e) {
e.getStackTrace();
}
}
}
4.清理过期日志
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimerTask;
import org.apache.commons.io.FileUtils;
/**
* 清理过期日志
* @author Administrator
*
*/
public class ClearTask extends TimerTask {
@Override
public void run() {
try {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm");
Date nowdate = new Date();
File file = new File("E:/backDir");
File[] listFiles = file.listFiles();
for (File dir : listFiles) {
String name = dir.getName();
Date dirdate = format.parse(name);
if(nowdate.getTime() - dirdate.getTime() > 24*60*60*1000) {
FileUtils.deleteDirectory(dir);
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
附:log4j配置文件
log4j.rootLogger =ALL,systemOut,logRollingFile
#log4j.rootLogger =ALL,systemOut,logFile,logDailyFile,logRollingFile,logMail,logDB
#\u8F93\u51FA\u5230\u63A7\u5236\u53F0
log4j.appender.systemOut = org.apache.log4j.ConsoleAppender
log4j.appender.systemOut.layout = org.apache.log4j.PatternLayout
log4j.appender.systemOut.layout.ConversionPattern = [%-5p][%-22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n
log4j.appender.systemOut.Threshold = DEBUG
log4j.appender.systemOut.ImmediateFlush = TRUE
log4j.appender.systemOut.Target = System.out
#\u8F93\u51FA\u5230\u6587\u4EF6
log4j.appender.logFile = org.apache.log4j.FileAppender
log4j.appender.logFile.layout = org.apache.log4j.PatternLayout
log4j.appender.logFile.layout.ConversionPattern = [%-5p][%-22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n
log4j.appender.logFile.Threshold = WARN
log4j.appender.logFile.ImmediateFlush = TRUE
log4j.appender.logFile.Append = TRUE
log4j.appender.logFile.File = ../Struts2/WebRoot/log/File/log4j_Struts.log
log4j.appender.logFile.Encoding = UTF-8
#\u6309DatePattern\u8F93\u51FA\u5230\u6587\u4EF6
log4j.appender.logDailyFile = org.apache.log4j.DailyRollingFileAppender
log4j.appender.logDailyFile.layout = org.apache.log4j.PatternLayout
log4j.appender.logDailyFile.layout.ConversionPattern = [%-5p][%-22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n
log4j.appender.logDailyFile.Threshold = DEBUG
log4j.appender.logDailyFile.ImmediateFlush = TRUE
log4j.appender.logDailyFile.Append = TRUE
log4j.appender.logDailyFile.File = ../Struts2/WebRoot/log/DailyFile/log4j_Struts
log4j.appender.logDailyFile.DatePattern = '.'yyyy-MM-dd-HH-mm'.log'
log4j.appender.logDailyFile.Encoding = UTF-8
#\u8BBE\u5B9A\u6587\u4EF6\u5927\u5C0F\u8F93\u51FA\u5230\u6587\u4EF6
log4j.appender.logRollingFile = org.apache.log4j.RollingFileAppender
log4j.appender.logRollingFile.layout = org.apache.log4j.PatternLayout
log4j.appender.logRollingFile.layout.ConversionPattern = [%-5p][%-22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n
log4j.appender.logRollingFile.Threshold = DEBUG
log4j.appender.logRollingFile.ImmediateFlush = TRUE
log4j.appender.logRollingFile.Append = TRUE
log4j.appender.logRollingFile.File = e:/testlog/test.log
log4j.appender.logRollingFile.MaxFileSize = 1MB
log4j.appender.logRollingFile.MaxBackupIndex = 50
log4j.appender.logRollingFile.Encoding = UTF-8
#\u7528Email\u53D1\u9001\u65E5\u5FD7
log4j.appender.logMail = org.apache.log4j.net.SMTPAppender
log4j.appender.logMail.layout = org.apache.log4j.HTMLLayout
log4j.appender.logMail.layout.LocationInfo = TRUE
log4j.appender.logMail.layout.Title = Struts2 Mail LogFile
log4j.appender.logMail.Threshold = DEBUG
log4j.appender.logMail.SMTPDebug = FALSE
log4j.appender.logMail.SMTPHost = SMTP.163.com
log4j.appender.logMail.From = [email protected]
log4j.appender.logMail.To = [email protected]
#log4j.appender.logMail.Cc = [email protected]
#log4j.appender.logMail.Bcc = [email protected]
log4j.appender.logMail.SMTPUsername = xly3000
log4j.appender.logMail.SMTPPassword = 1234567
log4j.appender.logMail.Subject = Log4j Log Messages
#log4j.appender.logMail.BufferSize = 1024
#log4j.appender.logMail.SMTPAuth = TRUE
#\u5C06\u65E5\u5FD7\u767B\u5F55\u5230MySQL\u6570\u636E\u5E93
log4j.appender.logDB = org.apache.log4j.jdbc.JDBCAppender
log4j.appender.logDB.layout = org.apache.log4j.PatternLayout
log4j.appender.logDB.Driver = com.mysql.jdbc.Driver
log4j.appender.logDB.URL = jdbc:mysql://127.0.0.1:3306/xly
log4j.appender.logDB.User = root
log4j.appender.logDB.Password = 123456
log4j.appender.logDB.Sql = INSERT INTOT_log4j(project_name,create_date,level,category,file_name,thread_name,line,all_category,message)values('Struts2','%d{yyyy-MM-ddHH:mm:ss}','%p','%c','%F','%t','%L','%l','%m')