python mysql stored procedures to quickly generate ten million test data

Mysql configuration

Modify the configuration file: sudo vim /etc/mysql/my.cnfto avoid memory table is full error occurs when data is inserted.
Add the following:

[mysqld]
# 避免内存表if full, 自行设置
max_heap_table_size = 4096M

Restart mysql service: sudo service mysql restart

Create a data table

Creating a product data sheet goods,
classification of goods and commodities List requires a separate built,
here added to the list of goods which, in order to test sub-table
affect the operating performance of the query, please ignore.

InnoDb Engine Table

DROP TABLE IF EXISTS goods;
CREATE TABLE goods
(
   goods_id INT(11) UNSIGNED NOT NULL AUTO_INCREMENT,
   goods_name VARCHAR(30) NOT NULL DEFAULT '' COMMENT '商品名称',
   goods_price DECIMAL(10, 2) NOT NULL DEFAULT 0.0 COMMENT '商品价格',
   goods_cate VARCHAR(30) NOT NULL DEFAULT ''  COMMENT '商品分类名称',
   goods_brand VARCHAR(30) NOT NULL DEFAULT '' COMMENT '商品品牌名称',
   goods_order TINYINT(1) NOT NULL DEFAULT 0 COMMENT '排序字段',
   create_time INT(11) NOT NULL DEFaULT 0 COMMENT '创建时间',
   update_time INT(11) NOT NULL DEFaULT 0 COMMENT '更新时间',
   PRIMARY KEY(goods_id)
   # INDEX order_index (`goods_order`) 暂时不加索引，优化的时候再加， 测试性能差别
)ENGINE = INNODB AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8 COMMENT '商品表';

Memory Engine Table

DROP TABLE IF EXISTS goods_memory;
CREATE TABLE goods_memory
(
   goods_id INT(11) UNSIGNED NOT NULL AUTO_INCREMENT,
   goods_name VARCHAR(30) NOT NULL DEFAULT '' COMMENT '商品名称',
   goods_price DECIMAL(10, 2) NOT NULL DEFAULT 0.0 COMMENT '商品价格',
   goods_cate VARCHAR(30) NOT NULL DEFAULT ''  COMMENT '商品分类名称',
   goods_brand VARCHAR(30) NOT NULL DEFAULT '' COMMENT '商品品牌名称',
   goods_order TINYINT(1) NOT NULL DEFAULT 0 COMMENT '排序字段',
   create_time INT(11) NOT NULL DEFaULT 0 COMMENT '创建时间',
   update_time INT(11) NOT NULL DEFaULT 0 COMMENT '更新时间',
	 PRIMARY KEY(goods_id)
	 # INDEX order_index (goods_order) 暂时不加索引，优化的时候再加， 测试性能差别
)ENGINE = MEMORY AUTO_INCREMENT = 1 DEFAULT CHARSET = utf8 COMMENT '内存商品表';

Create a stored procedure

Use stored procedures insert faster, because the stored procedure in the pre-compiled and stored in a database, and SQL to run the process needs to go through several stages.

CREATE DEFINER=`root`@`localhost` PROCEDURE `add_goods`(
    IN g_name VARCHAR (30),
    IN g_price DECIMAL(10, 2),
    IN g_cate VARCHAR(30),
    IN g_brand VARCHAR(30),
    IN g_order TINYINT(1),
    IN c_time INT(11),
    IN u_time INT(11)
    )
BEGIN  
   INSERT INTO goods_memory(goods_name, goods_price, goods_cate, goods_brand, goods_order, create_time, update_time)
    VALUES( g_name, g_price, g_cate, g_brand, g_order, c_time, u_time);
END

The addition to not have children play the Python

# _*_coding:utf-8_*_
"""
File: main.py
Time: 2019/8/20 21:07
Author: ClassmateLin
Email: [email protected]
Desc: 
"""
import asyncio
import aiomysql
import random
import decimal
import time

pool = ''


async def consumer(num: int):
    """
    生成数据，并调用存储过程插入数据
    :param num: 需要生成的数据条数
    :return: 
    """
    brands = ['华为', '小米', '中兴', '华硕', '戴尔', '三星', '金士顿', '耐克', '新百伦']
    categories = ['手机', '电脑', '配件', '箱包', '服饰', '鞋子', '家具']
    for i in range(num):
        async with pool.acquire() as conn:
            async with conn.cursor() as cur:
                try:
                    goods_name = '商品' + str(i)
                    goods_cate = random.choice(categories)
                    goods_brand = random.choice(brands)
                    goods_order = random.randint(0, 9)
                    goods_price = decimal.Decimal(random.randint(100, 1000000))
                    create_time = update_time = time.time()
                    # 调用存储过程插入数据到内存表
                    await cur.callproc('add_goods', (goods_name, goods_price, goods_cate,
                                                     goods_brand, goods_order, create_time, update_time))
                    print('插入第{}条数据成功'.format(str(i)))
                except aiomysql.Error as e:
                    print('mysql error {}: {}'.format(e.args[0], e.args[1]))


async def main(loop, num):
    global pool
    database = {
        'host': '127.0.0.1',
        'port': 3306,
        'user': 'root',
        'password': 'root',
        'db': 'test',
        'loop': loop,
        'charset': 'utf8',
        'autocommit': True
    }
    pool = await aiomysql.create_pool(**database)
    asyncio.ensure_future(consumer(num))

if __name__ == '__main__':
    max_num = 10000 * 1000  # 1千万
    loop = asyncio.get_event_loop()
    asyncio.ensure_future(main(loop, max_num))
    loop.run_forever()

Memory table data into an ordinary table

A memory table may not exist 10 million test takes about 6G memory
can perform repetitive Python script, insert ordinary table and clear the memory table step, the memory is large enough, please ignore.
INSERT INTO goods SELECT * FROM goods_memory