原文：https://blog.csdn.net/tangthh123/article/details/105854438
整个读取db文件的源码如下：
（1）Controller
注意：
这边我采用了异步调用的方法，因为300MB的文件解析下来需要200S ，如果前端一直访问我这个接口没用返回数据的话，直接network上报error.
异步调用原理很简单，前端调用这个接口以后，立马返回数据，后端开启一个线程去执行我解析文件的代码。
如何开启异步调用？
小编用的是SpringBoot,第一步是需要在启动类上加上@EnableAsync
其次需要在具体的方法上加上 @Async
/**
* 4.导入本地文本文件(.txt 或者db文件)解析文件数据，将其保存到数据库中
*
* @param
* @return
*/
@RequestMapping(value = "/uploadFileToNasLog")
public ResultData uploadFileToNasLog(@RequestParam("uploadFile") MultipartFile file,HttpServletRequest request) {
ResultData resultData = new ResultData();
NasLogVo vo = new NasLogVo();
vo.setFile(file);
vo.setType("1");
vo.setRequest(request);
//获取上传的文件名称
String fileName = file.getOriginalFilename();
//上传文件功能
String rootPath = request.getServletContext().getRealPath("upload");
File fileTemp = new File(rootPath);
if (!fileTemp.exists()) {
fileTemp.exists();
}
String fileUrl = rootPath + "\\" + fileName;
vo.setPath(fileUrl);
vo.setFileName(fileName);
File file1 = new File(fileUrl);
try {
file.transferTo(file1);
} catch (IOException e) {
e.printStackTrace();
}
DeferredResult<ResultData> deferredResult = new DeferredResult<>();
CompletableFuture.supplyAsync(()->nasService.uploadFileToNasLog(vo))
.whenCompleteAsync((result, throwable) -> deferredResult.setResult(result));
return resultData ;
}（2）NasService
/**
* 6.上传文本文件，数据保存在群晖日志表中
*
* @param
* @return
*/
@Override
@Async
public ResultData uploadFileToNasLog(NasLogVo vo) {
ResultData resultData = new ResultData();
Map<String, File> map = new HashMap<String, File>();
String fileUrl = vo.getPath();
String fileName = vo.getFileName();
long startTime = System.currentTimeMillis();
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
System.out.println("开始时间----" + simpleDateFormat.format(new Date()));
try {
//将上传的文件信息保存到nas_file ，以备后期删除这些临时文件数据
NasFileModel nasFileModel = new NasFileModel();
nasFileModel.setFileName(fileName);
nasFileModel.setFileUrl(fileUrl);
inserDateToNasFile(nasFileModel);
Builder builder = new Builder(fileUrl, new IHandle() {
@Override
public void handle(String line) {
}
});
builder.withTreahdSize(10)
.withCharset("gbk")
.withBufferSize(1024 * 1024)
.withFilePath(fileUrl);
BigFileReader bigFileReader = builder.build();
bigFileReader.start();
resultData.setResult("true");
resultData.setMessage("上传成功");
long endTime1 = System.currentTimeMillis();
float excTime = (float) (endTime1 - startTime) / 1000;
System.out.println("执行DB文件时间：" + excTime + "s");
} catch (Exception e) {
logger.error("文件上传失败", e);
resultData.setResult("false");
resultData.setMessage("上传失败");
}
return resultData;
}（3）读取db文件的方法
class BigFileReader {
private int threadSize;
private String charset;
private int bufferSize;
private IHandle handle;
ExecutorService executorService;
private long fileLength;
RandomAccessFile rAccessFile;
Set<StartEndPair> startEndPairs;
CyclicBarrier cyclicBarrier;
AtomicLong counter = new AtomicLong(0);
private String filePath;
private BigFileReader(File file, IHandle handle, String charset, int bufferSize, int threadSize, String filePath) {
this.fileLength = file.length();
this.handle = handle;
this.charset = charset;
this.bufferSize = bufferSize;
this.threadSize = threadSize;
this.filePath = filePath;
try {
this.rAccessFile = new RandomAccessFile(file, "r");
String url = SqliteUtil.DB_URL.replace("{url}", filePath);
Connection connection = SqliteUtil.createConnection(url);
//解析db文件中的数据
//执行当前线程
Statement statement = connection.createStatement();
// 执行查询语句
ResultSet rs = statement.executeQuery("select * from logs");
List<NasSynologyModel> nasSynologyModelList = new ArrayList<>();
while (rs.next()) {
NasSynologyModel synologyModel = new NasSynologyModel();
synologyModel.setHost(rs.getString("host"));
synologyModel.setIp(rs.getString("ip"));
synologyModel.setFac(rs.getString("fac"));
synologyModel.setPrio(rs.getString("prio"));
synologyModel.setLlevel(rs.getString("llevel"));
synologyModel.setTag(rs.getString("tag"));
synologyModel.setUtcsec(rs.getString("utcsec"));
synologyModel.setrUtcsec(rs.getString("r_utcsec"));
synologyModel.setTzoffset(rs.getString("tzoffset"));
synologyModel.setLdate(rs.getString("ldate"));
synologyModel.setLtime(rs.getString("ltime"));
synologyModel.setProg(rs.getString("prog"));
synologyModel.setMsg(rs.getString("msg"));
nasSynologyModelList.add(synologyModel);
}
//使用jdbc 的保存
insertNasSynologyByJdpcTemplate(nasSynologyModelList);
//保存完当前这个项目文件以后删除上传的文件
} catch (Exception e) {
System.err.println(e.getMessage());
}
this.executorService = Executors.newFixedThreadPool(threadSize);
startEndPairs = new HashSet<StartEndPair>();
}
public void start() {
long everySize = this.fileLength / this.threadSize;
try {
calculateStartEnd(0, everySize);
} catch (IOException e) {
e.printStackTrace();
return;
}
final long startTime = System.currentTimeMillis();
cyclicBarrier = new CyclicBarrier(startEndPairs.size(), new Runnable() {
@Override
public void run() {
}
});
for (StartEndPair pair : startEndPairs) {
this.executorService.execute(new SliceReaderTask(pair));
}
}
private void calculateStartEnd(long start, long size) throws IOException {
if (start > fileLength - 1) {
return;
}
StartEndPair pair = new StartEndPair();
pair.start = start;
long endPosition = start + size - 1;
if (endPosition >= fileLength - 1) {
pair.end = fileLength - 1;
startEndPairs.add(pair);
return;
}
rAccessFile.seek(endPosition);
byte tmp = (byte) rAccessFile.read();
while (tmp != '\n' && tmp != '\r') {
endPosition++;
if (endPosition >= fileLength - 1) {
endPosition = fileLength - 1;
break;
}
rAccessFile.seek(endPosition);
tmp = (byte) rAccessFile.read();
}
pair.end = endPosition;
startEndPairs.add(pair);
calculateStartEnd(endPosition + 1, size);
}
private void handle(byte[] bytes) throws UnsupportedEncodingException {
String line = null;
if (this.charset == null) {
line = new String(bytes);
} else {
line = new String(bytes, charset);
}
if (line != null && !"".equals(line)) {
this.handle.handle(line);
counter.incrementAndGet();
}
}
private class StartEndPair {
public long start;
public long end;
@Override
public String toString() {
return "star=" + start + ";end=" + end;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (end ^ (end >>> 32));
result = prime * result + (int) (start ^ (start >>> 32));
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
StartEndPair other = (StartEndPair) obj;
if (end != other.end) {
return false;
}
if (start != other.start) {
return false;
}
return true;
}
}
private class SliceReaderTask implements Runnable {
private long start;
private long sliceSize;
private byte[] readBuff;
public SliceReaderTask(StartEndPair pair) {
this.start = pair.start;
this.sliceSize = pair.end - pair.start + 1;
this.readBuff = new byte[bufferSize];
}
@Override
public void run() {
try {
MappedByteBuffer mapBuffer = rAccessFile.getChannel().map(FileChannel.MapMode.READ_ONLY, start, this.sliceSize);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
for (int offset = 0; offset < sliceSize; offset += bufferSize) {
int readLength;
if (offset + bufferSize <= sliceSize) {
readLength = bufferSize;
} else {
readLength = (int) (sliceSize - offset);
}
mapBuffer.get(readBuff, 0, readLength);
for (int i = 0; i < readLength; i++) {
byte tmp = readBuff[i];
if (tmp == '\n' || tmp == '\r') {
handle(bos.toByteArray());
bos.reset();
} else {
bos.write(tmp);
}
}
}
if (bos.size() > 0) {
handle(bos.toByteArray());
}
cyclicBarrier.await();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}（4）具体连接Sqlite数据库的方法（SqliteUtil）
package com.bos.common.file;
import java.sql.*;
/**
* @Author: tanghh
* Java操作Sqlite数据库
*/
public class SqliteUtil {
private static final String Class_Name = "org.sqlite.JDBC";
/**
* 要组成这样的url: jdbc:sqlite:C://uploadFile/NasLog/2019-04-03_2019-04-12.DB
*/
public static String DB_URL = "jdbc:sqlite:{url}";
/**
* 测试连接sqlite数据库的方法
* @param
* @return
*/
public static void main(String[] args) {
String url = "jdbc:sqlite:C://uploadFile/NasLog/2019-04-03_2019-04-12.DB";
connectSqlite(url);
}
public static void connectSqlite(String url) {
Connection connection = null;
try {
connection = createConnection(url);
func1(connection);
} catch (SQLException e) {
System.err.println(e.getMessage());
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (connection != null)
connection.close();
} catch (SQLException e) {
// connection close failed.
System.err.println(e);
}
}
}
/**
* 创建Sqlite数据库连接
*
* @param
* @return
*/
public static Connection createConnection(String url) throws SQLException, ClassNotFoundException {
Class.forName(Class_Name);
return DriverManager.getConnection(url);
}
public static void func1(Connection connection) throws SQLException {
Statement statement = connection.createStatement();
// set timeout to 30 sec.
statement.setQueryTimeout(30);
// 执行查询语句
ResultSet rs = statement.executeQuery("select * from logs limit 100");
while (rs.next()) {
String host = rs.getString("host");
String ip = rs.getString("ip");
String fac = rs.getString("fac");
String prio = rs.getString("prio");
String llevel = rs.getString("llevel");
String tag = rs.getString("tag");
String utcsec = rs.getString("utcsec");
String r_utcsec = rs.getString("r_utcsec");
String ldate = rs.getString("ldate");
String ltime = rs.getString("ltime");
String prog = rs.getString("prog");
String msg = rs.getString("msg");
System.out.println("host = " + host + " ip = " + ip);
// 执行插入语句操作
statement.executeUpdate("insert into nas_log(server_name,ip_address,user_name,type,time) values(host,ip,fac,info,ltime)");
// 执行更新语句
// statement1.executeUpdate("update table_name2 set 字段名1=" + 字段值1 + " where 字段名2='" + 字段值2 + "'");
}
}
}（5） NasLogVo
package com.bos.data.model.vo.setting;
import lombok.Data;
import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletRequest;
/**
* @Author: tanghh
* 群晖Nas
*/
public @Data
class NasLogVo {
private String Event;
private String Path;
private String Size;
private String User;
private String IP;
/**
* 群晖Nas上传的文件
* @param
* @return
*/
private MultipartFile file;
/**
* 文件上传类型 0为.txt 文件 1为db文件
* @param
* @return
*/
private String type;
private HttpServletRequest request;
private String fileName;
}
（6）保存数据的方法
/**
* 将数据通过jdbcTemplate的方式保存到数据库中
*
* @param nasSynologyModelList
* @return
*/
private int insertNasSynologyByJdpcTemplate(List<NasSynologyModel> nasSynologyModelList) {
String sql = "insert into nas_synology(host,ip,fac,prio,llevel,tag,utcsec,r_utcsec,tzoffset,ldate,ltime,prog,msg,is_analysis) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
try {
jdbcTemplate.batchUpdate(sql, new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
ps.setString(1, nasSynologyModelList.get(i).getHost());
ps.setString(2, nasSynologyModelList.get(i).getIp());
ps.setString(3, nasSynologyModelList.get(i).getFac());
ps.setString(4, nasSynologyModelList.get(i).getPrio());
ps.setString(5, nasSynologyModelList.get(i).getLlevel());
ps.setString(6, nasSynologyModelList.get(i).getTag());
ps.setString(7, nasSynologyModelList.get(i).getUtcsec());
ps.setString(8, nasSynologyModelList.get(i).getrUtcsec());
ps.setString(9, nasSynologyModelList.get(i).getTzoffset());
ps.setString(10, nasSynologyModelList.get(i).getLdate());
ps.setString(11, nasSynologyModelList.get(i).getLtime());
ps.setString(12, nasSynologyModelList.get(i).getProg());
ps.setString(13, nasSynologyModelList.get(i).getMsg());
ps.setString(14, "0");
}
@Override
public int getBatchSize() {
return nasSynologyModelList.size();
}
});
} catch (Exception e) {
e.printStackTrace();
}
return 0;
}（7）Builder类
class Builder {
private int threadSize = 1;
private String charset = null;
private int bufferSize = 1024 * 1024;
private IHandle handle;
private File file;
private String filePath;
public Builder(String file, IHandle handle) {
this.file = new File(file);
if (!this.file.exists()) {
throw new IllegalArgumentException("文件不存在！");
}
this.handle = handle;
}
public Builder withTreahdSize(int size) {
this.threadSize = size;
return this;
}
public Builder withCharset(String charset) {
this.charset = charset;
return this;
}
public Builder withBufferSize(int bufferSize) {
this.bufferSize = bufferSize;
return this;
}
public Builder withFilePath(String filePath) {
this.filePath = filePath;
return this;
}
public BigFileReader build() {
return new BigFileReader(this.file, this.handle, this.charset, this.bufferSize, this.threadSize, this.filePath);
}
}（8）NasFileModel
package com.bos.data.model;
import javax.persistence.*;
import java.sql.Timestamp;
import java.util.Objects;
/**
* @Author tanghh
* @Date 2020/4/23 16:02
*/
@Entity
@Table(name = "nas_file", schema = "test3", catalog = "")
public class NasFileModel {
private int id;
private String fileName;
private String fileUrl;
private Timestamp createTime;
private String isDel;
@Id
@Column(name = "id")
@GeneratedValue(strategy = GenerationType.IDENTITY)
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
@Basic
@Column(name = "file_name")
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
@Basic
@Column(name = "file_url")
public String getFileUrl() {
return fileUrl;
}
public void setFileUrl(String fileUrl) {
this.fileUrl = fileUrl;
}
@Basic
@Column(name = "create_time")
public Timestamp getCreateTime() {
return createTime;
}
public void setCreateTime(Timestamp createTime) {
this.createTime = createTime;
}
@Basic
@Column(name = "is_del")
public String getIsDel() {
return isDel;
}
public void setIsDel(String isDel) {
this.isDel = isDel;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
NasFileModel that = (NasFileModel) o;
return id == that.id &&
Objects.equals(fileName, that.fileName) &&
Objects.equals(fileUrl, that.fileUrl) &&
Objects.equals(createTime, that.createTime) &&
Objects.equals(isDel, that.isDel);
}
@Override
public int hashCode() {
return Objects.hash(id, fileName, fileUrl, createTime, isDel);
}
}
（9）NasSynologyModel
package com.bos.data.model;
import javax.persistence.*;
import java.sql.Timestamp;
import java.util.Objects;
/**
* @Author tanghh
* @Date 2020/4/22 8:52
*/
@Entity
@Table(name = "nas_synology", schema = "test3", catalog = "")
public class NasSynologyModel {
private int id;
private String host;
private String ip;
private String fac;
private String prio;
private String llevel;
private String tag;
private String utcsec;
private String rUtcsec;
private String tzoffset;
private String ldate;
private String ltime;
private String prog;
private String msg;
private Timestamp createTime;
private String isAnalysis="0";
@Id
@Column(name = "id")
@GeneratedValue(strategy = GenerationType.IDENTITY)
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
@Basic
@Column(name = "host")
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
@Basic
@Column(name = "ip")
public String getIp() {
return ip;
}
public void setIp(String ip) {
this.ip = ip;
}
@Basic
@Column(name = "fac")
public String getFac() {
return fac;
}
public void setFac(String fac) {
this.fac = fac;
}
@Basic
@Column(name = "prio")
public String getPrio() {
return prio;
}
public void setPrio(String prio) {
this.prio = prio;
}
@Basic
@Column(name = "llevel")
public String getLlevel() {
return llevel;
}
public void setLlevel(String llevel) {
this.llevel = llevel;
}
@Basic
@Column(name = "tag")
public String getTag() {
return tag;
}
public void setTag(String tag) {
this.tag = tag;
}
@Basic
@Column(name = "utcsec")
public String getUtcsec() {
return utcsec;
}
public void setUtcsec(String utcsec) {
this.utcsec = utcsec;
}
@Basic
@Column(name = "r_utcsec")
public String getrUtcsec() {
return rUtcsec;
}
public void setrUtcsec(String rUtcsec) {
this.rUtcsec = rUtcsec;
}
@Basic
@Column(name = "tzoffset")
public String getTzoffset() {
return tzoffset;
}
public void setTzoffset(String tzoffset) {
this.tzoffset = tzoffset;
}
@Basic
@Column(name = "ldate")
public String getLdate() {
return ldate;
}
public void setLdate(String ldate) {
this.ldate = ldate;
}
@Basic
@Column(name = "ltime")
public String getLtime() {
return ltime;
}
public void setLtime(String ltime) {
this.ltime = ltime;
}
@Basic
@Column(name = "prog")
public String getProg() {
return prog;
}
public void setProg(String prog) {
this.prog = prog;
}
@Basic
@Column(name = "msg")
public String getMsg() {
return msg;
}
public void setMsg(String msg) {
this.msg = msg;
}
@Basic
@Column(name = "create_time")
public Timestamp getCreateTime() {
return createTime;
}
public void setCreateTime(Timestamp createTime) {
this.createTime = createTime;
}
@Basic
@Column(name = "is_analysis")
public String getIsAnalysis() {
return isAnalysis;
}
public void setIsAnalysis(String isAnalysis) {
this.isAnalysis = isAnalysis;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
NasSynologyModel that = (NasSynologyModel) o;
return id == that.id &&
Objects.equals(host, that.host) &&
Objects.equals(ip, that.ip) &&
Objects.equals(fac, that.fac) &&
Objects.equals(prio, that.prio) &&
Objects.equals(llevel, that.llevel) &&
Objects.equals(tag, that.tag) &&
Objects.equals(utcsec, that.utcsec) &&
Objects.equals(rUtcsec, that.rUtcsec) &&
Objects.equals(tzoffset, that.tzoffset) &&
Objects.equals(ldate, that.ldate) &&
Objects.equals(ltime, that.ltime) &&
Objects.equals(prog, that.prog) &&
Objects.equals(msg, that.msg);
}
@Override
public int hashCode() {
return Objects.hash(id, host, ip, fac, prio, llevel, tag, utcsec, rUtcsec, tzoffset, ldate, ltime, prog, msg);
}
}
（10）ResultData
package com.bos.common;
import com.bos.data.model.TShowColumnModel;
import io.swagger.annotations.ApiModelProperty;
import java.util.List;
import java.util.Map;
/**
* @author luoj
* @author luojie 2018/6/8
*/
public class ResultData {
/**
* "true" or "false"
*/
@ApiModelProperty(value = "后台返回的查询结果,true=查询成功,false=查询失败")
private String result;
/**
* 消息提示
*/
@ApiModelProperty(value = "后台返回的消息,如'保存成功'。用作前端的提示消息")
private String message;
/**
* 消息提示
*/
@ApiModelProperty(value = "一个字符串")
private String str;
/**
* 存放一个实体
*/
private Object object;
/**
* 主数据集合
*/
private PageData pageData;
/**
*其他数据
*/
private List<Map> otherData;
/**
*整形
*/
private Integer number;
/**
*存放list集合的map
*/
private Map<String, List> mapList;
/**
*
*/
private Map<Object, Object> map;
/**
*存放一个list集合
*/
private List list;
/**
*专用于存放表头字段的集合
*/
private List fieldList;
/**
*专用于存放视图集合
*/
private List viewList;
/**
*专用于存放报表集合
*/
private List reportList;
/**
* 专用于存每个页面的page
* @param
* @return
*/
private String page;
/**
* 专用于存不同页面的module
* @param
* @return
*/
private String module;
/**
* 存储当前数据的上一级数据(第一级)
* @param
* @return
*/
private String level1;
/**
* 存储当前数据的上一级的上一级数据(第二级)
* @param
* @return
*/
private String level2;
/**
* 存储当前数据的上一级的上一级的上一级数据（第三级）
* @param
* @return
*/
private String level3;
/**
* map2
*/
private Map<Object, Object> map2;
private Map<String,List<TShowColumnModel>> mapCloumnList;
public List getReportList() {
return reportList;
}
public void setReportList(List reportList) {
this.reportList = reportList;
}
public List getViewList() {
return viewList;
}
public void setViewList(List viewList) {
this.viewList = viewList;
}
public List getList() {
return list;
}
public void setList(List list) {
this.list = list;
}
public Integer getNumber() {
return number;
}
public void setNumber(Integer number) {
this.number = number;
}
public Object getObject() {
return object;
}
public void setObject(Object object) {
this.object = object;
}
public Map<String, List<TShowColumnModel>> getMapCloumnList() {
return mapCloumnList;
}
public void setMapCloumnList(Map<String, List<TShowColumnModel>> mapCloumnList) {
this.mapCloumnList = mapCloumnList;
}
public ResultData() {
this.message = "查询成功";
this.result = "true";
}
/**
* 用于删除成功
* @param count
*/
public ResultData(int count) {
if(count>0){
this.message = "删除成功";
this.result = "true";
}else {
this.message = "删除失败";
this.result = "false";
}
}
public ResultData(Object object) {
this.object=object;
this.message = "查询成功";
this.result = "true";
}
/**
* 用于添加删除提示
* @param type
* @param flag
*/
public ResultData(String type,boolean flag) {
if(flag==false){
//保存和修改时候的提示
if(Constant.SAVE.equals(type)){
this.message = "保存失败";
}else{
this.message = "删除失败";
}
this.result = "false";
}else {
if(Constant.SAVE.equals(type)){
this.message = "保存成功";
}else{
this.message = "删除成功";
}
this.result = "true";
}
}
public ResultData(String result, String message, PageData pageData) {
this.result = result;
this.message = message;
this.pageData = pageData;
}
public ResultData(String result, String message, PageData pageData, List<Map> otherData) {
this.result = result;
this.message = message;
this.pageData = pageData;
this.otherData = otherData;
}
public ResultData(String result, String message, PageData pageData, List<Map> otherData, Map<String, List> mapList) {
this.result = result;
this.message = message;
this.pageData = pageData;
this.otherData = otherData;
this.mapList = mapList;
}
public ResultData(String result, String message) {
this.result = result;
this.message = message;
}
public String getResult() {
return result;
}
public void setResult(String result) {
this.result = result;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
public PageData getPageData() {
return pageData;
}
public void setPageData(PageData pageData) {
this.pageData = pageData;
}
public List<Map> getOtherData() {
return otherData;
}
public void setOtherData(List<Map> otherData) {
this.otherData = otherData;
}
public Map<String, List> getMapList() {
return mapList;
}
public void setMapList(Map<String, List> mapList) {
this.mapList = mapList;
}
public Map<Object, Object> getMap() {
return map;
}
public void setMap(Map<Object, Object> map) {
this.map = map;
}
public List getFieldList() {
return fieldList;
}
public void setFieldList(List fieldList) {
this.fieldList = fieldList;
}
public String getStr() {
return str;
}
public void setStr(String str) {
this.str = str;
}
public String getPage() {
return page;
}
public void setPage(String page) {
this.page = page;
}
public String getModule() {
return module;
}
public void setModule(String module) {
this.module = module;
}
public String getLevel1() {
return level1;
}
public void setLevel1(String level1) {
this.level1 = level1;
}
public String getLevel2() {
return level2;
}
public void setLevel2(String level2) {
this.level2 = level2;
}
public String getLevel3() {
return level3;
}
public void setLevel3(String level3) {
this.level3 = level3;
}
public Map<Object, Object> getMap2() {
return map2;
}
public void setMap2(Map<Object, Object> map2) {
this.map2 = map2;
}
}
（11）整个NasService
package com.bos.service.impl.basic;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.bos.common.PageData;
import com.bos.common.ResultData;
import com.bos.common.file.FileThreadResource;
import com.bos.common.file.FileUtil;
import com.bos.common.file.IHandle;
import com.bos.common.file.SqliteUtil;
import com.bos.data.model.NasFileModel;
import com.bos.data.model.NasLogModel;
import com.bos.data.model.NasSynologyModel;
import com.bos.data.model.vo.basic.CommonVO;
import com.bos.data.model.vo.setting.NASVo;
import com.bos.data.model.vo.setting.NasLogVo;
import com.bos.data.repositories.ColumnRepository;
import com.bos.data.repositories.NasLogRepository;
import com.bos.data.repositories.jpa.NasFileJPARepository;
import com.bos.data.repositories.jpa.NasLogJPARepository;
import com.bos.data.repositories.jpa.NasSynologyJPARepository;
import com.bos.service.basic.CommonService;
import com.bos.service.basic.NASService;
import com.bos.service.integrated_management.BosUserService;
import com.bos.util.DateUtil;
import com.bos.util.FastDFSClientWrapper;
import com.bos.util.NasUtil;
import com.bos.util.SendRequest;
import com.google.common.base.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.transaction.interceptor.TransactionAspectSupport;
import javax.transaction.Transactional;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.sql.*;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Date;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
/**
* @author luojie on 2018/11/30
*/
@Service
@Transactional(rollbackOn = Exception.class)
public class NASServiceImpl implements NASService {
private Logger logger = LoggerFactory.getLogger(NASServiceImpl.class);
@Autowired
private NasLogJPARepository nasLogJPARepository;
@Autowired
private NasLogRepository nasLogRepository;
@Autowired
private NasSynologyJPARepository nasSynologyJPARepository;
@Autowired
private JdbcTemplate jdbcTemplate;
@Autowired
private NasFileJPARepository nasFileJPARepository;
/**
* 将数据通过jdbcTemplate的方式保存到数据库中
*
* @param nasSynologyModelList
* @return
*/
private int insertNasSynologyByJdpcTemplate(List<NasSynologyModel> nasSynologyModelList) {
String sql = "insert into nas_synology(host,ip,fac,prio,llevel,tag,utcsec,r_utcsec,tzoffset,ldate,ltime,prog,msg,is_analysis) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
try {
jdbcTemplate.batchUpdate(sql, new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
ps.setString(1, nasSynologyModelList.get(i).getHost());
ps.setString(2, nasSynologyModelList.get(i).getIp());
ps.setString(3, nasSynologyModelList.get(i).getFac());
ps.setString(4, nasSynologyModelList.get(i).getPrio());
ps.setString(5, nasSynologyModelList.get(i).getLlevel());
ps.setString(6, nasSynologyModelList.get(i).getTag());
ps.setString(7, nasSynologyModelList.get(i).getUtcsec());
ps.setString(8, nasSynologyModelList.get(i).getrUtcsec());
ps.setString(9, nasSynologyModelList.get(i).getTzoffset());
ps.setString(10, nasSynologyModelList.get(i).getLdate());
ps.setString(11, nasSynologyModelList.get(i).getLtime());
ps.setString(12, nasSynologyModelList.get(i).getProg());
ps.setString(13, nasSynologyModelList.get(i).getMsg());
ps.setString(14, "0");
}
@Override
public int getBatchSize() {
return nasSynologyModelList.size();
}
});
} catch (Exception e) {
e.printStackTrace();
}
return 0;
}
/**
* 6.上传文本文件，数据保存在群晖日志表中
*
* @param
* @return
*/
@Override
@Async
public ResultData uploadFileToNasLog(NasLogVo vo) {
ResultData resultData = new ResultData();
Map<String, File> map = new HashMap<String, File>();
String fileUrl = vo.getPath();
String fileName = vo.getFileName();
long startTime = System.currentTimeMillis();
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
System.out.println("开始时间----" + simpleDateFormat.format(new Date()));
try {
//将上传的文件信息保存到nas_file ，以备后期删除这些临时文件数据
NasFileModel nasFileModel = new NasFileModel();
nasFileModel.setFileName(fileName);
nasFileModel.setFileUrl(fileUrl);
inserDateToNasFile(nasFileModel);
//将上传的文件信息保存到
if (vo.getType().equals("0")) {
FileUtil fileUtil = new FileUtil();
// 如果当前上传的文件为.txt文件, 接下来将这个大文件分割成多个小文件(提高解析文本行的效率)
fileUtil.readFile(fileUrl);
//接下来遍历整个文件夹里的31个分割后的文件
File[] files = new File(fileUrl).listFiles();
for (File fileInfo : files) {
if (fileInfo.isFile() && fileInfo.exists() && fileInfo.getName().contains("part")) {
fileName = fileInfo.getName();
map.put(fileName, fileInfo);
}
}
//创建自定义线程对象(开启10个线程来解析这三十一个文件)
FileThreadResource resource = new FileThreadResource();
for (int k = 0; k < 10; k++) {
FileThread mt = new FileThread(fileName, map, resource);
mt.run();
}
} else {
Builder builder = new Builder(fileUrl, new IHandle() {
@Override
public void handle(String line) {
}
});
builder.withTreahdSize(10)
.withCharset("gbk")
.withBufferSize(1024 * 1024)
.withFilePath(fileUrl);
BigFileReader bigFileReader = builder.build();
bigFileReader.start();
}
resultData.setResult("true");
resultData.setMessage("上传成功");
long endTime1 = System.currentTimeMillis();
float excTime = (float) (endTime1 - startTime) / 1000;
System.out.println("执行DB文件时间：" + excTime + "s");
} catch (Exception e) {
logger.error("文件上传失败", e);
resultData.setResult("false");
resultData.setMessage("上传失败");
}
return resultData;
}
/**
* 7.保存群晖日志文件数据
*
* @param
* @return
*/
@Override
public void insertDataToNasLog(NasLogModel nasLogModel) {
try {
nasLogJPARepository.save(nasLogModel);
} catch (Exception e) {
logger.error("保存群晖日志信息失败", e);
TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
}
}
@Override
public void inserDateToNasFile(NasFileModel nasFileModel) {
try {
nasFileJPARepository.save(nasFileModel);
} catch (Exception e) {
logger.error("保存群晖日志信息失败", e);
TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
}
}
@Override
public void insertAllDataToNasLogByJdbcTemplate(List<NasLogModel> nasLogModelList) {
String sql = "insert into nas_log(type,logs,time,ip,user_name,event,floder,file_size,file_name,ip_address,is_del,call_name,level,flevel1,flevel2,flevel3,flevel4,flevel5,flevel6,flevel7,flevel8,flevel9,flevel10,flevel_number,specific_name) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
try {
jdbcTemplate.batchUpdate(sql, new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
ps.setString(1, nasLogModelList.get(i).getType());
ps.setString(2, nasLogModelList.get(i).getLogs());
ps.setString(3, nasLogModelList.get(i).getTime());
ps.setString(4, nasLogModelList.get(i).getIp());
ps.setString(5, nasLogModelList.get(i).getUserName());
ps.setString(6, nasLogModelList.get(i).getEvent());
ps.setString(7, nasLogModelList.get(i).getFloder());
ps.setString(8, nasLogModelList.get(i).getFileSize());
ps.setString(9, nasLogModelList.get(i).getFileName());
ps.setString(10, nasLogModelList.get(i).getIpAddress());
ps.setString(11, "0");
ps.setString(12, nasLogModelList.get(i).getCallName());
ps.setString(13, nasLogModelList.get(i).getLevel());
ps.setString(14, nasLogModelList.get(i).getFlevel1());
ps.setString(15, nasLogModelList.get(i).getFlevel2());
ps.setString(16, nasLogModelList.get(i).getFlevel3());
ps.setString(17, nasLogModelList.get(i).getFlevel4());
ps.setString(18, nasLogModelList.get(i).getFlevel5());
ps.setString(19, nasLogModelList.get(i).getFlevel6());
ps.setString(20, nasLogModelList.get(i).getFlevel7());
ps.setString(21, nasLogModelList.get(i).getFlevel8());
ps.setString(22, nasLogModelList.get(i).getFlevel9());
ps.setString(23, nasLogModelList.get(i).getFlevel10());
ps.setInt(24, nasLogModelList.get(i).getFlevelNumber());
ps.setString(25, nasLogModelList.get(i).getSpecificName());
}
@Override
public int getBatchSize() {
return nasLogModelList.size();
}
});
} catch (Exception e) {
TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
}
}
/**
* 根据文件名称和文件路径删除文件
* @param filename
* @param filePath
* @return
*/
public boolean deleteFile(String filename,String filePath){
File file = new File(filePath);
// 如果文件路径所对应的文件存在，并且是一个文件，则直接删除
if (file.exists() && file.isFile()) {
if (file.delete()) {
return true;
} else {
return false;
}
} else {
file.delete();
return true;
}
}
class BigFileReader {
private int threadSize;
private String charset;
private int bufferSize;
private IHandle handle;
ExecutorService executorService;
private long fileLength;
RandomAccessFile rAccessFile;
Set<StartEndPair> startEndPairs;
CyclicBarrier cyclicBarrier;
AtomicLong counter = new AtomicLong(0);
private String filePath;
private BigFileReader(File file, IHandle handle, String charset, int bufferSize, int threadSize, String filePath) {
this.fileLength = file.length();
this.handle = handle;
this.charset = charset;
this.bufferSize = bufferSize;
this.threadSize = threadSize;
this.filePath = filePath;
try {
this.rAccessFile = new RandomAccessFile(file, "r");
String url = SqliteUtil.DB_URL.replace("{url}", filePath);
Connection connection = SqliteUtil.createConnection(url);
//解析db文件中的数据
//执行当前线程
Statement statement = connection.createStatement();
// 执行查询语句
ResultSet rs = statement.executeQuery("select * from logs");
List<NasSynologyModel> nasSynologyModelList = new ArrayList<>();
while (rs.next()) {
NasSynologyModel synologyModel = new NasSynologyModel();
synologyModel.setHost(rs.getString("host"));
synologyModel.setIp(rs.getString("ip"));
synologyModel.setFac(rs.getString("fac"));
synologyModel.setPrio(rs.getString("prio"));
synologyModel.setLlevel(rs.getString("llevel"));
synologyModel.setTag(rs.getString("tag"));
synologyModel.setUtcsec(rs.getString("utcsec"));
synologyModel.setrUtcsec(rs.getString("r_utcsec"));
synologyModel.setTzoffset(rs.getString("tzoffset"));
synologyModel.setLdate(rs.getString("ldate"));
synologyModel.setLtime(rs.getString("ltime"));
synologyModel.setProg(rs.getString("prog"));
synologyModel.setMsg(rs.getString("msg"));
nasSynologyModelList.add(synologyModel);
}
//使用jdbc 的保存
insertNasSynologyByJdpcTemplate(nasSynologyModelList);
//保存完当前这个项目文件以后删除上传的文件
} catch (Exception e) {
System.err.println(e.getMessage());
}
this.executorService = Executors.newFixedThreadPool(threadSize);
startEndPairs = new HashSet<StartEndPair>();
}
public void start() {
long everySize = this.fileLength / this.threadSize;
try {
calculateStartEnd(0, everySize);
} catch (IOException e) {
e.printStackTrace();
return;
}
final long startTime = System.currentTimeMillis();
cyclicBarrier = new CyclicBarrier(startEndPairs.size(), new Runnable() {
@Override
public void run() {
}
});
for (StartEndPair pair : startEndPairs) {
this.executorService.execute(new SliceReaderTask(pair));
}
}
private void calculateStartEnd(long start, long size) throws IOException {
if (start > fileLength - 1) {
return;
}
StartEndPair pair = new StartEndPair();
pair.start = start;
long endPosition = start + size - 1;
if (endPosition >= fileLength - 1) {
pair.end = fileLength - 1;
startEndPairs.add(pair);
return;
}
rAccessFile.seek(endPosition);
byte tmp = (byte) rAccessFile.read();
while (tmp != '\n' && tmp != '\r') {
endPosition++;
if (endPosition >= fileLength - 1) {
endPosition = fileLength - 1;
break;
}
rAccessFile.seek(endPosition);
tmp = (byte) rAccessFile.read();
}
pair.end = endPosition;
startEndPairs.add(pair);
calculateStartEnd(endPosition + 1, size);
}
private void handle(byte[] bytes) throws UnsupportedEncodingException {
String line = null;
if (this.charset == null) {
line = new String(bytes);
} else {
line = new String(bytes, charset);
}
if (line != null && !"".equals(line)) {
this.handle.handle(line);
counter.incrementAndGet();
}
}
private class StartEndPair {
public long start;
public long end;
@Override
public String toString() {
return "star=" + start + ";end=" + end;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + (int) (end ^ (end >>> 32));
result = prime * result + (int) (start ^ (start >>> 32));
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
StartEndPair other = (StartEndPair) obj;
if (end != other.end) {
return false;
}
if (start != other.start) {
return false;
}
return true;
}
}
private class SliceReaderTask implements Runnable {
private long start;
private long sliceSize;
private byte[] readBuff;
public SliceReaderTask(StartEndPair pair) {
this.start = pair.start;
this.sliceSize = pair.end - pair.start + 1;
this.readBuff = new byte[bufferSize];
}
@Override
public void run() {
try {
MappedByteBuffer mapBuffer = rAccessFile.getChannel().map(FileChannel.MapMode.READ_ONLY, start, this.sliceSize);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
for (int offset = 0; offset < sliceSize; offset += bufferSize) {
int readLength;
if (offset + bufferSize <= sliceSize) {
readLength = bufferSize;
} else {
readLength = (int) (sliceSize - offset);
}
mapBuffer.get(readBuff, 0, readLength);
for (int i = 0; i < readLength; i++) {
byte tmp = readBuff[i];
if (tmp == '\n' || tmp == '\r') {
handle(bos.toByteArray());
bos.reset();
} else {
bos.write(tmp);
}
}
}
if (bos.size() > 0) {
handle(bos.toByteArray());
}
cyclicBarrier.await();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
class Builder {
private int threadSize = 1;
private String charset = null;
private int bufferSize = 1024 * 1024;
private IHandle handle;
private File file;
private String filePath;
public Builder(String file, IHandle handle) {
this.file = new File(file);
if (!this.file.exists()) {
throw new IllegalArgumentException("文件不存在！");
}
this.handle = handle;
}
public Builder withTreahdSize(int size) {
this.threadSize = size;
return this;
}
public Builder withCharset(String charset) {
this.charset = charset;
return this;
}
public Builder withBufferSize(int bufferSize) {
this.bufferSize = bufferSize;
return this;
}
public Builder withFilePath(String filePath) {
this.filePath = filePath;
return this;
}
public BigFileReader build() {
return new BigFileReader(this.file, this.handle, this.charset, this.bufferSize, this.threadSize, this.filePath);
}
}
}