Python进程经典例题

多线程、多进程应用案例

1）题目1: 计算1～100000之间所有素数和， 要求如下:

• 编写函数判断一个数字是否为素数，
• 使用内置函数sum()统计所有素数和。
• 对比1: 对比使用多进程和不使用多进程两种方法的速度。
• 对比2：对比开启4个多进程和开启10个多进程两种方法的速度

from math import sqrt
import multiprocessing
from mytime import timeit

# def sum_prime():
#     res = [p for p in range(2, 100000) if 0 not in
#            [p % d for d in range(2, int(sqrt(p)) + 1)]]
#
#     sum(res)
def sum_prime():
    primelist=[]
    for i in range(2,100000):
        for j in range(2, i):
            if i % j == 0:
                break
        else:
            primelist.append(i)
    sum(primelist)

@timeit
def has_process_4():
    processes = []
    for i in range(4):
        p = multiprocessing.Process(target=sum_prime())
        processes.append(p)
        p.start()
    [process.join() for process in processes]

@timeit
def no_has_process():
    sum_prime()

@timeit
def has_process_10():
    processes = []
    for i in range(10):
        p = multiprocessing.Process(target=sum_prime())
        processes.append(p)
        p.start()
    [process.join() for process in processes]

if __name__ == '__main__':
    has_process_4()
    no_has_process()
    # has_process_10()

2）题目2： 使用生产者消费者模型， 多线程爬取指定个url网页信息，并多线程写入mysql数据库中;
要求如下:

• 理论上url地址信息通过其他程序获取, 此处从一个文件urls.txt中读取；
• 如果url地址访问不成功， bytesCount存储为0；
• 数据库存储的表头要求： id(唯一标识码)， url, bytesCount
• 获取url中字节数最多的10个url(自行查找相关的SQL语句);
• 附加： 对比多进程和多线程的速度

import random
import threading
from queue import Queue
from urllib.request import urlopen

import pymysql
import time


class Producer(threading.Thread):
    def __init__(self,queue):
        super(Producer, self).__init__()
        self.queue=queue
    def run(self):
        with open('urls.txt')as f:
            for line in f:
                url=line.strip()
                self.queue.put(url)
                urllist.append(url)
                id=url.split('.')[1]
                idlist.append(id)
            return idlist,urllist
class Consumer(threading.Thread):
    def __init__(self,queue):
        super(Consumer, self).__init__()
        self.queue=queue
    def run(self):
        while True:
            get_url=self.queue.get
            try:
                text=urlopen(get_url).read().decode('utf-8')
            except Exception:
                bytesCount=0
                countlist.append(bytesCount)
            else:
                bytesCount=len(text)
                countlist.append(bytesCount)
            return countlist
def write_mysql():
    conn = pymysql.connect(host='172.25.254.69', user='hello',
                           password='hello', charset='utf8', autocommit=True)
    cur=conn.cursor()
    # cur.execute('create database urls;')
    conn.select_db('urls')
    cur.execute('create table url(id varchar(30),url varchar(60),bytesCount int);')
    insert_sqli='insert into url values(%s,%s);'
    cur.executemany(insert_sqli,countlist)
    insert_sql = 'insert into url values(%s,%s);'
    cur.executemany(insert_sql,urllist)
    insert_sq = 'insert into url values(%s,%s);'
    cur.executemany(insert_sq,countlist)
    cur.close()
    conn.close()
urllist = []
idlist = []
countlist = []
if __name__ == '__main__':
    queue=Queue()
    p=Producer(queue)
    c=Consumer(queue)
    p.start()
    c.start()
    p.join()
    c.join()
    write_mysql()
    print(urllist)
    print(idlist)
    print(countlist)