#! /usr/bin/python
# -*- coding:utf-8 -*-
'''
------------------------------------------
function:
多进程复制图片
author: bingo
created: 2020-01-13
------------------------------------------
'''
from Queue import Queue
import multiprocessing
import shutil
import os
import time
count = 0
DEFAULT_PROCESS_NUM = 6
file_dir = './11-bak'
target_dir = './11_copy/'
def ensure_dir_exits(path):
try:
os.makedirs(path)
except OSError:
if not os.path.isdir(path):
raise
class Counter():
def __init__(self, current_finished):
self.current_finished = multiprocessing.Value('i', current_finished)
self.lock = multiprocessing.Lock()
def increment(self):
with self.lock:
self.current_finished.value += 1
def value(self):
with self.lock:
return (self.current_finished.value)
def generate_list(file_dir):
global count
inputs = []
for root, dirs, files in os.walk(file_dir):
for each in files:
count += 1
fullpath = os.path.join(root,each)
inputs.append(fullpath)
return inputs
def generate_file_copy_func():
def file_copy_func(file1):
file2 = target_dir + os.path.basename(file1)
shutil.copy(file1, file2)
return file_copy_func
def split_input(inputs, split_times):
outputs = []
each_len = len(inputs)/split_times
remainer = len(inputs)%split_times
for i in range(split_times):
tmp = inputs[(i*each_len):((i+1)*each_len)]
outputs.append(tmp)
outputs.append(inputs[(i+1)*each_len:])
return outputs
def work(partition, func, counter):
for each in partition:
func(each.strip())
counter.increment()
current_finished = counter.value()
if current_finished%10 == 0:
print (current_finished)
if __name__ == "__main__":
ensure_dir_exits(target_dir)
inputs = generate_list(file_dir)
partitions = split_input(inputs, DEFAULT_PROCESS_NUM)
process_list = []
func = generate_file_copy_func()
counter = Counter(0)
for i, partition in enumerate(partitions):
p = multiprocessing.Process(target = work, args = (partition, func, counter))
process_list.append(p)
p.start()
for p in process_list:
p.join()
