爬虫数据存储

一. 使用pymongo进行数据存储:

import pymongo

MONGO_HOST = 'localhost'

MONGO_DB = '数据库名称'

MONGO_TABLE = '表名'

~~~~~~~~~~~~~~~~~~

在类的初始化函数中:

# 创建数据库客户端

self.client = pymongo.MongoClient(MONGO_HOST)

# 创建客户端对象, 连接数据库

self.db = self.client[MONGO_DB]

~~~~~~~~~~~~~~~~~~~~~~~~

定义保存数据到数据库的函数

def save_to_mongo(self, data)

# 插入一条数据

self.db['表名'].insert_one(data)

~~~~~~~~~~~~~~~~~~~~~~~~~

最后调用函数(save_to_mongo(参数))保存到mongo_db数据库(注意: 保存到mongo数据库中的数据需要是字典的形式)

二. 使用pymysql进行数据存储:

法一:

import pymysql

示例代码:(连接数据库)

    @classmethod
     def connect_db(cls):
         cls.db = pymysql.connect(host='localhost', user='root', passwd='123456', db='qsbklist', port=3306, charset='utf8')
         cls.cursor = cls.db.cursor()

     @classmethod
     def save_list_data(cls, list_data):
         # 遍历list_data，执行insert操作
         for q_name, q_age, q_href, q_content, q_smail_num, q_comment_num in list_data:
             # 表使用文章的id作为主键 /article/120510346
             q_id = q_href.split('/')[2]
             insert_sql = "INSERT INTO qsbk_list (q_id, q_name, q_age, q_href, q_content, q_smail_num, q_comment_num) VALUES (%s,%s,%s,%s,%s,%s,%s)"
             try:
                 cls.cursor.execute(insert_sql, (q_id, q_name, q_age, q_href, q_content, q_smail_num, q_comment_num))
                 cls.db.commit()
             except Exception as e:
                 print('主键冲突或者内容有表情数据,跳过...')
                 cls.db.rollback()

     @classmethod
     def connect_close(cls):
         cls.cursor.close()
         cls.db.close()

在主函数中:首先打开数据库连接->执行程序->关闭数据库连接

法二:

import MySQLdb

# 连接数据库

db = MySQLdb.connect("localhost", "root", "123456", "zhilian", charset='utf8')

# 保存到数据库

def save_data_to_db(self, res):
    
    for href, zwmc, fk_lv, gsmc, zwyx, gzdd, fbrq, dd, gsxz, gsgm, jy, gwzz in res:
        cursor = db.cursor()
        sql = "INSERT INTO python(`zwmc`, `fk_lv`, `gsmc`, `zwyx`, `gzdd`, `fbrq`, `dd`, `gsxz`, `gsgm`, `jy`, `gwzz`) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )"
        try:
            # 执行sql语句
            cursor.execute(sql, (zwmc, fk_lv, gsmc, zwyx, gzdd,  fbrq, dd, gsxz, gsgm, jy, gwzz))
            db.commit()
        except Exception as e:
            print("导入错误", e)

猜你喜欢