Python3 substitute for loop to enhance the running speed with multithreading

[This article comes days outside the park owned by a cloud of blog]

Before and after optimization of old and new codes are as follows:

from git_tools.git_tool import get_collect_projects, QQNews_Git
from threading import Thread, Lock
import datetime

base_url = "http://git.xx.com"
project_members_commits_lang_info = {}
lock = Lock()
threads = []

'''
Author:zenkilan
'''


def count_time(func):
    def took_up_time(*args, **kwargs):
        start_time = datetime.datetime.now()
        ret = func(*args, **kwargs)
        end_time = datetime.datetime.now()
        took_up_time = (end_time - start_time).total_seconds()
        print(f"{func.__name__} execution took up time:{took_up_time}")
        return ret

    return took_up_time


def get_project_member_lang_code_lines(git, member, begin_date, end_date):
    global project_members_commits_lang_info
    global lock
    member_name = member["username"]
    r = git.get_user_info(member_name)
    if not r["id"]:
        return
    user_commits_lang_info = git.get_commits_user_lang_diff_between(r["id"], begin_date, end_date)
    if len(user_commits_lang_info) == 0:
        return
    lock.acquire()
    project_members_commits_lang_info.setdefault(git.project, dict())
    project_members_commits_lang_info[git.project][member_name] = user_commits_lang_info
    lock.release()


def get_project_lang_code_lines(project, begin_date, end_date):
    global threads
    git QQNews_Git = (Project [. 1 ], the base_url, Project [0]) 
    project_members = git.get_project_members ()
     IF len (project_members) == 0:
         return 
    for Member in project_members: 
        Thread = the Thread (target = get_project_member_lang_code_lines, args = (Git, Member, BEGIN_DATE, END_DATE)) 
        threads.append (the thread) 
        Thread.start () 


@count_time 
DEF get_projects_lang_code_lines (BEGIN_DATE, END_DATE):
     "" " 
    access to relevant statistical line code language - new methods (to improve efficiency) 
    multi-threaded alternative The for loop 
    concurrent access to shared external resources 
    : return: 
    "" " 
    , Ltd. Free Joinproject_members_commits_lang_info
     , Ltd. Free Join Threads
     for Project in get_collect_projects (): 
        the Thread = the Thread (target = get_project_lang_code_lines, args = (Project, BEGIN_DATE, END_DATE)) 
        threads.append (the Thread) 
        Thread.start () 


@count_time 
DEF get_projects_lang_code_lines_old (BEGIN_DATE, END_DATE):
     " "" 
    Get line code associated statistical language - old method (Processed severe) 
    using the basic ideas for programming 
    a double nested loop for consuming operations and each containing 
    : return: 
    "" " 
    project_members_commits_lang_info = {}
     for Project in get_collect_projects():
        git = QQNews_Git(project[1], base_url, project[0])
        project_members = git.get_project_members()
        user_commits_lang_info_dict = {}
        if len(project_members) == 0:
            continue
        for member in project_members:
            member_name = member["username"]
            r = git.get_user_info(member_name, debug=False)
            if not r["id"]:
                continue
            try:
                user_commits_lang_info = git.get_commits_user_lang_diff_between(r["id"], begin_date, end_date)
                if len(user_commits_lang_info) == 0:
                    continue
                user_commits_lang_info_dict[member_name] = user_commits_lang_info
                project_members_commits_lang_info[git.project] = user_commits_lang_info_dict
            except:
                pass
    return project_members_commits_lang_info


def test_results_equal(resultA, resultB):
    """
    测试方法
    :param resultA:
    :param resultB:
    :return:
    """
    print(resultA)
    print(resultB)
    assert len(str(resultA)) == len(str(resultB))


if __name__ == '__main__':
    from git_tools.config import begin_date, end_date

    get_projects_lang_code_lines(begin_date, end_date)
    for t in threads:
        t.join()
    old_result = get_projects_lang_code_lines_old(begin_date, end_date)
    test_results_equal(old_result, project_members_commits_lang_info)

Method for the old in the outer loop and the inner loop for consuming operations there are present:

1）git.get_project_members()

2）git.get_user_info(member_name, debug=False)

Two steps to optimize the first years after the outside or outside to inside will do. Multithreading replacement cycle for concurrent sharing of external resources to avoid conflicting write lock.

Test results, decorative displays a function of time (in seconds):

get_projects_lang_code_lines execution took up time:1.85294

get_projects_lang_code_lines_old execution took up time:108.604177

About 58 times faster