无非就是io流-------------------------------------------------------
实例类：
1.抽象类
import java.io.File;
import java.io.IOException;
public abstract class PartitionFile {
/**
* 单个文件设置的字节数
*/
public static long MAX_BYTE = 1024*1024*1000;
/**
* 获取可以分割的文件数
*
* @param filePath
* @param max_byte
* @return
*/
public int getPartitionFileNum(long fileByte, String filePath) {
if (MAX_BYTE < fileByte) {
if (fileByte % MAX_BYTE == 0) {
return (int) (fileByte / MAX_BYTE);
} else {
return (int) (fileByte / MAX_BYTE) + 1;
}
}
return 1;
}
/**
* 获取文件长度
*
* @param file
* @return
* @throws IOException
*/
public abstract long getFileLength(File file) throws IOException;
/**
* 分割Byte文件
*
* @param file
* @throws IOException
* @throws IOException
*/
public abstract String[] partitionFile(File outPath,File srcFile, int partitionFileNum) throws IOException;
/**
* 合并文件
* @param files
* @param newFile
* @throws IOException
*/
public abstract void uniteFile(String[] files, String newFile) throws IOException;
}
2.实现类
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
public class SplitBigTextFile extends PartitionFile{
public static void main(String[] args) throws Exception {
/*大文件分割*/
String srcFile = "H:\\proData\\datahktw3\\hk\\b_account_audit.csv";//要分割的文件
String outPath = "E:\\111111111111111111111111111\\";//分割后文件输出位置
File file = new File(srcFile);
File oPath = new File(outPath);
SplitBigTextFile partitionTextFile = new SplitBigTextFile();
int count = partitionTextFile.getPartitionFileNum(file.length(), srcFile);
String str[] = partitionTextFile.partitionFile(oPath,file, count);
System.out.println(str.length);
}
@SuppressWarnings("finally")
@Override
public long getFileLength(File file) throws IOException {
FileReader fr = null;
BufferedReader br = null;
long fileSize = 0;
try {
fr = new FileReader(file);
br = new BufferedReader(fr);
String line = br.readLine();
while (line != null) {
fileSize += line.length();
line = br.readLine();
}
} catch (FileNotFoundException ex) {
ex.printStackTrace();
} catch (IOException ex) {
ex.printStackTrace();
} finally {
if (br != null){
br.close();
}
if (fr != null)
fr.close();
return fileSize;
}
}
@Override
public String[] partitionFile(File outPath,File srcFile, int partitionFileNum) throws IOException {
if (partitionFileNum <= 0){
return null;
}
FileReader fr = null;
BufferedReader br = null;
long readNum = 0;
String[] partitions = new String[partitionFileNum];
try {
fr = new FileReader(srcFile);
br = new BufferedReader(new InputStreamReader(new FileInputStream(srcFile), "gb2312"));
int i = 1;
while (partitionFileNum > i) {
String name = "";
String fileType = "";
if (srcFile.getName().indexOf(".") != -1) {
name = srcFile.getName().substring(0,srcFile.getName().indexOf("."));
fileType = srcFile.getName().substring(srcFile.getName().lastIndexOf(".")+1,srcFile.getName().length());
} else {
name = srcFile.getName();
}
//windows文件分割
if(outPath.getAbsolutePath().contains("\\")){
partitions[i] = outPath.getParent() + "\\" + name +"\\"+name+"_" + i+".txt";
partitions[i] = partitions[i].toUpperCase();
}else{
//linux文件分割
partitions[i] = outPath.getParent() + "/" + name +"/"+name+"_" + i+"."+fileType;
}
System.out.println(partitions[i]);
File wfile = new File(partitions[i]);
if (!wfile.exists()) {
wfile.getParentFile().mkdirs();
wfile.createNewFile();
}
FileWriter fw = new FileWriter(wfile,false);
BufferedWriter bw = new BufferedWriter(fw);
String line = br.readLine();
int flush=0;
String []tem = null;
StringBuffer sb = null;
while (line != null) {
sb = new StringBuffer();
if (line.trim().length() == 0) {
line = br.readLine();
continue;
}
tem = line.split("-");
for(int n=0;n<tem.length;n++){
sb.append(tem[n]+",");
}
readNum += sb.toString().length();
if (i + 1 == partitionFileNum) {
bw.write(line);
bw.newLine();
} else {
if (readNum >= MAX_BYTE) {
bw.write(sb.toString());
bw.newLine();
break;
} else {
if(sb.length()>0){
bw.write(sb.toString());
bw.newLine();
}
}
}
line = br.readLine();
if(flush%1000==0){
bw.flush();
}
}
bw.flush();
fw.flush();
bw.close();
fw.close();
if(sb!=null && sb.length()==0){
wfile.delete();
}
readNum = 0;
i++;
}
} finally {
try {
if (br != null){
br.close();
}
if (fr != null){
fr.close();
}
} catch (IOException e) {
e.printStackTrace();
} finally {
br = null;
fr = null;
}
}
return partitions;
}
@Override
public void uniteFile(String[] files, String newFile) throws IOException {
File wFile = new File(newFile);
FileWriter writer = null;
BufferedWriter bufferedWriter = null;
try {
writer = new FileWriter(wFile,false);
bufferedWriter = new BufferedWriter(writer);
for (int i = 0; i < files.length; i++) {
File rFile = new File(files[i]);
FileReader reader = new FileReader(rFile);
BufferedReader bufferedReader = new BufferedReader(reader);
String line = bufferedReader.readLine();
while (line != null) {
if (line.trim().length() == 0) {
line = bufferedReader.readLine();
continue;
}
bufferedWriter.write(line);
bufferedWriter.newLine();
line = bufferedReader.readLine();
}
bufferedWriter.flush();
writer.flush();
}
} finally {
if (bufferedWriter != null){
bufferedWriter.close();
bufferedWriter = null;
}
if (writer != null){
writer.close();
}
writer = null;
}
}
}