On the use of RandomAccessFile to achieve multi-threaded segment copy

Preface

Recently, an idea came up suddenly, using RandomAccessFile and multithreading to realize file segment copying, and direct streaming copy, will it take less time? Originally thought, multi-thread copy at the same time, the time-consuming should be shorter.

But in fact, this is not the case. In most cases, it will be slower, no matter how many threads are opened. The reasons may be as follows:

1. RandomAccessFile is more efficient than Buffer flow

2. The more threads open, the higher the cost of thread scheduling switching

3. Due to multi-threaded segment copy, the head of the underlying disk changes frequently, which increases the disk seek time and positioning time. In other words, for multi-threaded copy and direct streaming copy, the disk data is first read into memory and then written to the disk copy. But the main reason why multi-threaded copy will be slower is: the speed of the underlying disk IO limits the speed of the CPU [ mainly ! ! ! 】

Therefore, it is meaningless to use RandomAccessFile to implement multi-threaded segmented copy . But the code is still recorded, because logical design thinking is still meaningful. In the code implementation process, the issue of multi-thread concurrency is also involved.

PS: I learned the operating system really well, but the teacher speaks well. . .

Code

package net.ysq.nio.test;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;

/**
 * 复制者。负责调度多线程复制src到dest
 *
 * @author	passerbyYSQ
 * @date	2020-11-5 16:11:40
 */
public class MultiThreadCopyer {

	private File src;
	private File dest;
	private String newName;

	private long totalSize;
	// 各个线程已经复制的大小
	private long[] copied;
	private int lastProgress;

	// 专门负责从src到dest复制工作的线程池
	// 并不是为每一个MultiThreadCopyer实例对象创建一个线程池
	// 而是所有MultiThreadCopyer实例对象都是用这一线程池，故声明为静态变量
	private static ThreadPoolExecutor threadPool;
	// 线程数量
	private int threadCount;
	private CountDownLatch countDownLatch;

	private CopyingListener listener;

	static {
		threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(32);
	}

	public MultiThreadCopyer(File src, File dest, CopyingListener listener) {
		this(src, dest, null, 14, listener);
	}

	public MultiThreadCopyer(File src, File dest, String newName, int threadCount,
							 CopyingListener listener) {
		if (src.isDirectory()) {
			throw new RuntimeException("不支持复制目录");
		}
		if (!dest.exists()) {
			boolean isCreated = dest.mkdirs();
			if (!isCreated) {
				throw new RuntimeException("创建目标目录失败");
			}
		}
		this.newName = (newName == null || "".equals(newName)) ? src.getName() : newName;
		File tempDest = new File(dest, this.newName);
		if (tempDest.exists()) {
//			boolean isDel = tempDest.delete(); // 目标目录存在同名文件，则先删除
//			if (!isDel) {
//				throw new RuntimeException("删除目标目录同名文件失败");
//			}
		}
		this.src = src;
		this.dest = tempDest;
		this.totalSize = this.src.length();
		this.threadCount = (threadCount <= 0) ? 4 : threadCount;
		this.countDownLatch = new CountDownLatch(this.threadCount);
		this.copied = new long[ this.threadCount ];
		this.listener = listener;
	}

	public void copy() {
		long startTime = System.currentTimeMillis();

		long sectionSize = (long) (totalSize / threadCount); // 取下整
//		System.out.println("totalSize=" + totalSize);
//		System.out.println("sectionSize=" + sectionSize);
		long start = 0, end = sectionSize;
		for(int i = 0; i < threadCount; i++) {
			CopyWorker worker = new CopyWorker(start, end, i);
			threadPool.execute(worker); // 交由线程池调度
//			System.out.println(i + "：start=" + start + "; end=" + end);
			start = end;
			end = (i < threadCount - 1) ? (start + sectionSize) : totalSize;
		}

		try {
			// 阻塞等待， 直至所有线程执行完成。CountDownLatch里面有线程同步
			countDownLatch.await();

			long endTime = System.currentTimeMillis();
			if (listener != null) {
				listener.onCompleted(dest, endTime - startTime);
			}
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
	}

	// 计算进度
	private void calculateProgress() {
		if (listener != null) {
			long totalCopied = 0;
			for(int i = 0; i < threadCount; i++) {
				totalCopied += copied[i];
			}
			int progress = (int) (100.0 * totalCopied / totalSize);
			if (progress > lastProgress) { // 排队前先过滤。增加效率
				synchronized (this) { // 防止多个线程同时，刷新同一个百分点，造成百分点错乱
					if (progress > lastProgress) { // 抢到锁之后，再次验证
						listener.onProgress(progress); // 回调进度
						lastProgress = progress; // 记录上一个百分点
					}
				}
			}
		}
	}

	// 复制的线程
	class CopyWorker extends Thread {

		private RandomAccessFile randSrc;
		private RandomAccessFile randDest;

		// 负责的区间[start, end)
		private long start;
		private long end;

		private int num; // 线程编号
		private String threadName;

		public CopyWorker(long start, long end, int num) {
			super(src.getName() + "=>" + newName + ": thread-" + num);
			this.start = start;
			this.end = end;
			this.num = num;
			this.threadName = "thread-" + num;
		}

		@Override
		public void run() {
			try {
				randSrc = new RandomAccessFile(src, "r");
				randDest = new RandomAccessFile(dest, "rw");

//				long startTime = System.currentTimeMillis();
				randSrc.seek(start);
				randDest.seek(start);
//				long endTime = System.currentTimeMillis();
//				System.out.println(num + " seek耗时：" + (endTime-startTime));

				byte[] buf = new byte[1024]; // 缓冲的字节数组
				long sum = 0; // 该区间内已复制的长度
				int remain;
				while (true) {
					remain = (int) Math.min((end - start) - sum, buf.length);

					int read = randSrc.read(buf, 0, remain);
					randDest.write(buf, 0, read);
					sum += read;
					copied[num] = sum;

					// 计算并回调进度
					calculateProgress();

					if (sum >= (end - start)) {
						// 该区间复制完成
						System.out.println(threadName + " 已完成区间复制");
						countDownLatch.countDown();
						break; // 不要忘了
					}
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
		}

	}

	// 监听器
	interface CopyingListener {
		void onProgress(int progress); // 回调复制进度，百分数

		void onCompleted(File dest, long millisecond); // 下载完成，回调 目标文件和耗时（毫秒）
	}

}

test

package net.ysq.nio.test;

import java.io.File;
import java.io.IOException;

import org.junit.Test;

/**
 * @author	passerbyYSQ
 * @date	2020-11-5 18:05:49
 */
public class MultiThreadUtil {
	File src = new File("E:\\Download\\cn_sql_server_2016_enterprise_x64_dvd_8699450.iso");
	File dest = new File("E:\\");

	@Test
	public void test() {

		new MultiThreadCopyer(src, dest, new MultiThreadCopyer.CopyingListener() {

			@Override
			public void onProgress(int progress) {
				System.out.println("下载进度：" + progress + "%");
			}

			@Override
			public void onCompleted(File dest, long millisecond) {
				System.out.println("下载完成：");
				System.out.println("目标文件：" + dest.getAbsolutePath());
				System.out.println("耗时：" + millisecond + " ms");
			}

		}).copy(); // 不要忘了 .copy()
	}

	@Test
	public void test2() throws IOException {
		long startTime = System.currentTimeMillis();
		dest = new File("E:\\abc.iso");
		StreamUtil.copy(src, dest);
		long endTime = System.currentTimeMillis();
		System.out.println("耗时：" + (endTime - startTime));
	}


}