python 加速运算

测试代码运行的时间

import time

tic = time.time()
much_job = [x**2 for x in range(1, 1000000, 3)]
toc = time.time()
print('userd: {:.5f}s'.format(toc-tic))
# userd: 0.06204s

一、加速查找

1. 用set而非list

import time

data = [i**2+1 for i in range(1000000)]
list_data = list(data)
set_data = set(data)
# normal
tic = time.time()
s = 1098987 in list_data
toc = time.time()
print('userd: {:.5f}s'.format(toc-tic))
# speed up
tic = time.time()
ss = 1098987 in set_data
toc = time.time()
print('userd: {:.5f}s'.format(toc-tic))
# userd: 0.00833s
# userd: 0.00000s

2. 用dict而非两个list进行匹配查找

import time

list_a = [i*2-1 for i in range(1000000)]
list_b = [i**2 for i in list_a]
dict_ab = dict(zip(list_a, list_b))
# normal
tic = time.time()
a = list_b[list_a.index(876567)]
toc = time.time()
print('userd: {:.5f}s'.format(toc-tic))
# speed up
tic = time.time()
aa = dict_ab.get(876567, None)
toc = time.time()
print('userd: {:.5f}s'.format(toc-tic))
# userd: 0.00394s
# userd: 0.00000s

二、加速循环，在循环体中避免重复计算，用循环机制代替递归函数

3. 用for而非while

import time

tic = time.time()
s, i = 0, 0
while i<100000:
    i += 1
    s += i
toc = time.time()
print('userd: {:.5f}s'.format(toc-tic))


tic = time.time()
s, i = 0, 0
for i in range(1, 100001):
    i += 1
    s += i
toc = time.time()
print('userd: {:.5f}s'.format(toc-tic))

三、利用库函数进行加速

4. 用numba加速Python函数

import time

tic = time.time()
def my_power(x):
    return (x**2)

def my_power_sum(n):
    s = 0
    for i in range(1, n+1):
        s = s + my_power(i)
    return s
s = my_power_sum(1000000)
toc = time.time()
print('used: {:.5f}s'.format(toc-tic))

# speed up
from numba import jit
tic = time.time()
@jit
def my_power(x):
    return (x**2)
@jit
def my_power_sum(n):
    s = 0
    for i in range(1, n+1):
        s = s + my_power(i)
    return s
ss = my_power_sum(1000000)
toc = time.time()
print('used: {:.5f}s'.format(toc-tic))
# userd: 0.25207s
# userd: 0.09088s

代码是使用numpy做数字运算，并且常常有很多的循环，那么使用Numba就是一个很好的选择。

numba不适合字典型变量和一些非numpy的函数，尤其是上面numba不能解析pandas，上面的函数内容在运行时也就无法编译。

5. 用map加速Python函数

import time

tic = time.time()
res = [x**2 for x in range(1, 1000000, 3)]
toc = time.time()
print('used: {:.5f}s'.format(toc-tic))

# speed up

tic = time.time()
res = map(lambda x:x**2, range(1, 1000000, 3))
toc = time.time()
print('used: {:.5f}s'.format(toc-tic))
# used: 0.07043s
# used: 0.00244s

6. 用filter加速Python函数

import time

tic = time.time()
res = [x**2 for x in range(1, 1000000, 3) if x%7==0]
toc = time.time()
print('used: {:.5f}s'.format(toc-tic))

# speed up

tic = time.time()
res = filter(lambda x:x%7==0, range(1, 1000000, 3))
toc = time.time()
print('used: {:.5f}s'.format(toc-tic))
# used: 0.02840s
# used: 0.00033s

7. 用np.where加速if函数

import time

import numpy as np

array_a = np.arange(-100000, 100000)
tic = time.time()
relu = np.vectorize(lambda x: x if x>0 else 0)
arr = relu(array_a)
toc = time.time()
print('used: {:.5f}s'.format(toc-tic))

# speed up

tic = time.time()
relu = lambda x:np.where(x>0, x, 0)
arrr = relu(array_a)
toc = time.time()
print('used: {:.5f}s'.format(toc-tic))
# used: 0.02531s
# used: 0.00083s

8. 多线程thread加速

import time

import numpy as np


tic = time.time()

def writefile(i):
    with open(str(i)+'.txt', 'w') as f:
        s = ('hello %d\n'%i) * 10000000
        f.write(s)
for i in range(40,50, 1):
    writefile(i)

toc = time.time()
print('used: {:.5f}s'.format(toc-tic))

# speed up
import threading

tic = time.time()
def writefile(i):
    with open(str(i)+'.txt', 'w') as f:
        s = ('hello %d\n'%i) * 10000000
        f.write(s)

thread_list = []
for i in range(10, 20, 1):
    t = threading.Thread(target=writefile, args=(i, ))
    t.setDaemon(True)
    thread_list.append(t)

for t in thread_list:
    t.start()
for t in thread_list:
    t.join()

toc = time.time()
print('used: {:.5f}s'.format(toc-tic))
# used: 1.25816s
# used: 1.21268s

9. 多线程multiprocessing加速

import time

import numpy as np


tic = time.time()

def muchjob(x):
    time.sleep(5)
    return(x**2)

ans = [muchjob(i) for i in range(8)]
toc = time.time()
print('used: {:.5f}s'.format(toc-tic))

# speed up
import multiprocessing

tic = time.time()

def muchjob(x):
    time.sleep(5)
    return x**2
pool = multiprocessing.Pool(processes=4)
res = []
for i in range(8):
    res.append(pool.apply_async(muchjob, (i, )))
pool.close()
pool.join()

toc = time.time()
print('used: {:.5f}s'.format(toc-tic))
# used: 40.03782s
# used: 10.07268s

reference：

https://mp.weixin.qq.com/s?__biz=MzU5MDY5OTI5MA==&mid=2247484643&idx=1&sn=55ba87185102042bf8c4641e14573161&chksm=fe3b0a96c94c83802ef2a8c4f35fb2364bf732f7931739a3d09386fa244634720b271892fe5b&token=1679241518&lang=zh_CN&scene=21#wechat_redirect