Save the data to MongoDB redis

Persistence operations redis # data ----> MongoDB 
Import Redis
from pymongo Import MongoClient
Import json

# reptile simple distributed: more url into the list, not inside parked URL, the value of the program cycle,
# can url into redis, the values from multiple machines in redis crawling data, simple distributed

# instantiated redis client
redis_client = redis.Redis (Host = '127.0.0.1', Port = 6379)

# instantiated MongoDB client
mongo_client = MongoClient (Host = '127.0.0.1', Port = 27017)

# links MongDB specified database, a set of
DB = mongo_client [ 'the HN']
COL = DB [ 'the HN']
# cycles using the data redis are all written to the MongoDB
the while True:
# retrieve data from the redis
#blpop out the parameter value is the name of the named list of key-value in redis,
# name system similar to the file name generated default reptile, shape as => (reptiles name: items)
Key, Data = redis_client.blpop ([ 'hn_spider: items'])
Print ( 'Key', Key)
Print ( 'Data', Data)
# json.loads () function is to convert the data format of a dictionary json
# MongoDB write data into the
col.insert_one (json.loads (data.decode ()))


# close the database
mongo_client .close ()

Guess you like

Origin www.cnblogs.com/lnd-blog/p/11706462.html