版权声明： https://blog.csdn.net/lichundongV5/article/details/79812720
这是对日志文件的读取，特别是大文件的读取，用了scanner来读取每行数据
package com.kentrasoft.controller.system;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Scanner;
public class test {
private static String filePathName = "D:\\virtual\\logs\\virtual_cms_trace.log";
public static void main(String[] args) throws IOException {
// TODO 自动生成的方法存根
//generateBigFile();
// readFile1();
// readFile2();
// readFile3();
// generateBigFile();
// readNLine2(filePathName,10,10);
readline(filePathName,100,10000000);
}
/**
* 读大文件
* BufferedReader + char[]
* @throws IOException
*/
public static void readFile1() throws IOException{
long start = System.currentTimeMillis();
BufferedReader br = new BufferedReader(new FileReader(filePathName));
char[] buff = new char[1024];
int len = -1;
while( (len = br.read(buff)) != -1 ){
// System.out.print(new String(buff, 0, len));
}
long end = System.currentTimeMillis();
System.out.println("读大文件 BufferedReader + char[], 耗时="+(end-start));
}
/**
* 读大文件
* FileChannel + ByteBuffer
* @throws IOException
*/
private static void readFile2() throws IOException{
long start = System.currentTimeMillis();
FileChannel fc = new FileInputStream(filePathName).getChannel();
ByteBuffer buffer = ByteBuffer.allocate(1024);
while(fc.read(buffer) != -1){
buffer.flip();
//System.out.print(Charset.forName("UTF-8").newDecoder().decode(buffer));;
buffer.clear();
}
long end = System.currentTimeMillis();
System.out.println("读大文件 FileChannel + ByteBuffer, 耗时="+(end-start));
}
/**
* 读大文件
* BufferedReader + CharBuffer
* @throws IOException
*/
public static void readFile3() throws IOException{
long start = System.currentTimeMillis();
BufferedReader br = new BufferedReader(new FileReader(filePathName));
CharBuffer buff = CharBuffer.allocate(1024);
String str = null;
int counts=0;
while((str = br.readLine()) != null){
// System.out.println(str);//此时str就保存了一行字符串
counts++;
}
System.out.println(counts);
long end = System.currentTimeMillis();
System.out.println("读大文件 BufferedReader + CharBuffer, 耗时="+(end-start));
}
// public static void readFile4() throws IOException{
//
// long start = System.currentTimeMillis();
// FileChannel fc = new FileInputStream(filePathName).getChannel();
// int begin = 0, size = 1024;
//
// MappedByteBuffer mappedByteBuffer =
// fc.map(FileChannel.MapMode.READ_ONLY, begin, size);
//
// while(mappedByteBuffer.capacity() > 0){
//
// begin += mappedByteBuffer.capacity();
// mappedByteBuffer = fc.map(FileChannel.MapMode.READ_ONLY, begin, size);
//
// }
// long end = System.currentTimeMillis();
// System.out.println("nio读大文件 FileChannel + MappedByteBuffer, 耗时="+(end-start));
// }
/**
* 生成一个大文件 a.txt
* @throws IOException
*/
private static void generateBigFile() throws IOException{
long start = System.currentTimeMillis();
File bigFile = new File(filePathName);
FileWriter fileWriter = new FileWriter(bigFile);
for(int i=0;i<100000000;i++){
fileWriter.write(Math.random()+"\r\n");
}
fileWriter.close();
long end = System.currentTimeMillis();
System.out.println("生成一个大文件 a.txt , 耗时="+(end-start));
}
public static HashMap<String, Object> readNLine2(String filenamepath,long pageSize,long numStart) throws IOException {
InputStream is = new BufferedInputStream(new FileInputStream(filenamepath));
HashMap<String, Object> PageCounts=new HashMap<String, Object>();
byte[] c = new byte[1024];
long count = 0;
int readChars = 0;
long start = System.currentTimeMillis();
while ((readChars = is.read(c)) != -1) {
for (int i = 0; i < readChars; ++i) {
if (c[i] == '\n'){
++count;
}
StringBuilder stringBuilder = new StringBuilder("");
String str = Integer.toHexString(c[i] & 0xff);
if (str.length() < 2) { // 不足两位要补0
stringBuilder.append(0);
}
stringBuilder.append(str);
stringBuilder.toString();
str= byteArrayToStr(c);
// System.out.println("单个bytec[i]:"+c[i]);
/* byte[] byteArray = new byte[1024];
byteArray=byteArray.put();
//使用readLine获取当前行
String line = byteArrayToStr(c[i]);
line=new String(line.getBytes("ISO-8859-1"),"utf-8");
//行数统计，如果到达了pageSize指定的行数，就跳出循环
//保存结果
if(count >numStart){
result.add(line);
}
if (count == pageSize+numStart)
{
break;
}*/
}
}
long end = System.currentTimeMillis();
is.close();
System.out.println("读一个大文件 , 耗时="+(end-start));
PageCounts.put("pageCount", (int)Math.ceil(count/pageSize)+1);
PageCounts.put("count", count);
return PageCounts;
}
public static String byteArrayToStr(byte[] byteArray) {
if (byteArray == null) {
return null;
}
String str = new String(byteArray);
return str;
}
public static List<String> readline(String path,long pageSize,long numStart) throws IOException {
// 定义结果集
List<String> result = new ArrayList<String>();
FileInputStream inputStream = null;
Scanner sc = null;
long count=0;
long start = System.currentTimeMillis();
try {
inputStream = new FileInputStream(path);
sc = new Scanner(inputStream, "UTF-8");
while (sc.hasNextLine()) {
String line = sc.nextLine();
count++;
// System.out.println(line);
if(count >numStart){
//使用readLine获取当前行
line=new String(line.getBytes("ISO-8859-1"),"utf-8");
result.add(line);
// System.out.println(line);
}
if (count == pageSize+numStart)
{
break;
}
}
System.out.println(count);
// note that Scanner suppresses exceptions
if (sc.ioException() != null) {
throw sc.ioException();
}
} finally {
if (inputStream != null) {
inputStream.close();
long end = System.currentTimeMillis();
System.out.println("读一个大文件 , 耗时="+(end-start)/1000);
}
if (sc != null) {
sc.close();
}
}
return result;
}
}