python封装连接池（HTTP + Mysql + Redis）

一：HTTP连接池

# -*- coding: utf-8 -*-
import requests

pool_connections = 2
pool_maxsize = 5


def get_http_pool(pool_connections, pool_maxsize):
    # 实例化会话对象
    session = requests.Session()
    # 创建适配器
    adapter = requests.adapters.HTTPAdapter(pool_connections=pool_connections, pool_maxsize=pool_maxsize)
    # 告诉requests，http协议和https协议都使用这个适配器
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session


http_pool = get_http_pool(pool_connections=pool_connections, pool_maxsize=pool_maxsize)


def http_req(url, data='', timeout=5, method="POST", max_retries=3):
    """发送HTTP请求, 不是所有的连接异常都重试"""
    res_data = ''
    if method not in ["GET", "POST"]:
        raise Exception('post requests method error!!!')
    while max_retries > 0:
        try:
            if method == "GET":
                res_data = http_pool.get(url, params=data, timeout=timeout)
            else:
                res_data = http_pool.post(url, data=data, timeout=timeout)
        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
            max_retries -= 1
        except Exception as e:
            max_retries = 0
            logger.error('requests error is %s', e)

    if not res_data:
        raise Exception('post requests failed')
    return res_data

二： Redis连接池

import redis
from config import result_config


class DBInterface(object):
    def __init__(self):
        fraud = redis.ConnectionPool(**result_config)
        self.fraud = redis.Redis(connection_pool=fraud)

redis_db = DBInterface()

三：Mysql连接池

• config

result_config = {
    
    
    'db_host': 'XXXX',
    'db_user': 'XXX',
    'db_pwd': 'XXXX',
    'db': 'XXXX',
    'db_port': 3306,
}

• 连接池封装

# -*- coding: utf-8 -*-
"""数据库连接池的封装"""
import pymysql
from DBUtils.PooledDB import PooledDB
import multiprocessing
from config import result_config

CPU = multiprocessing.cpu_count()


class MySQLEngine(object):
    """MYSQL引擎封装"""

    __tablename__ = None
    placeholder = '%s'

    def connect(self, **kwargs):
        db_host = kwargs.get('db_host', 'localhost')
        db_port = kwargs.get('db_port', 3306)
        db_user = kwargs.get('db_user', 'root')
        db_pwd = kwargs.get('db_pwd', '')
        db = kwargs.get('db', '')
        cursor = kwargs.get('cursor', pymysql.cursors.Cursor)

        self.pool = PooledDB(pymysql, mincached=1, maxcached=CPU,
                             maxconnections=0, blocking=False, host=db_host,
                             user=db_user, passwd=db_pwd, db=db, port=db_port,
                             cursorclass=cursor, charset='utf8')

    def _execute(self, sql, values, **kwargs):
        conn = self.pool.connection()
        cur = conn.cursor()
        cur.execute(sql, values)
        conn.commit()
        return cur, conn

    def select(self, sql, values=[], **kwargs):
        cur, conn = self._execute(sql, values, **kwargs)
        conn.close()
        for row in cur:
            yield row

    def execute(self, sql, values=[], **kwargs):
        cur, conn = self._execute(sql, values, **kwargs)
        conn.close()

    def update(self, table, data, **kwargs):
        """当数据不存在的时候，执行插入，如果存在则进行修改。(更新的参数必须存在唯一的值)
        :param table: 表名
        :param data: 字典数据
        :param kwargs: 其他数据
        :return:
        """
        assert isinstance(data, dict)

        sql = '''INSERT INTO {0} ({1}) VALUES ({2}) ON DUPLICATE KEY UPDATE {3}'''
        p0 = '`' + table + '`'  # 表名
        p1 = list()  # 字段名
        p3 = list()  # 字段的值
        values = list()
        for k, v in data.items():
            if v is None:
                continue

            k = '`' + k + '`'
            p1.append(k)
            p3.append('%s=VALUES(%s)' % (k, k))
            values.append(v)

        p2 = ['%s'] * len(p1)
        sql = sql.format(p0, ', '.join(p1), ', '.join(p2), ', '.join(p3))

        self.execute(sql, values, **kwargs)


class DBInterface(object):

    def __init__(self):
        self.result_engine = MySQLEngine()
        self.result_engine.connect(**result_config)


mysql_db = DBInterface()