Scrapy source code analysis Twisted use of foreplay

 

 

 

from twisted.internet Import the Reactor    # event loop (termination condition, all of the socket have been removed) 
from twisted.web.client Import getPage # socket objects (if the download is complete, the time automatically removed from circulation ...) 
from Twisted .internet Import the defer      # defer.Deferred special socket object (do not send the request, manual removal) 

# 1. Using getPage create socket 
# 2. the socket added to the event loop 
# 3. start the event loop (internal transmit request, and receiving a response; socekt when all the requests is completed, a termination event loop) 

# ######################## 
# 1. Socket created using getPage 
# # ######################## 
# 1. Using getPage create socket 
#Response DEF (Content): 
#      Print (Content) 
# 
# 
# DEF Task (): 
#      URL = "http://www.baidu.com" 
#      D = the getPage (URL) 
#      d.addCallback (Response) 


# ## ###################### 
# 1. Using getPage create socket 
# 2. Add the socket to the event loop 
# ########## ############## 
# DEF the Response (Content): 
#      Print (Content) 
# 
# @ defer.inlineCallbacks 
# DEF Task (): 
#      url = "http://www.baidu. COM " 
#      D = the getPage (URL) 
#     d.addCallback (the Response) 
#      yield d 

# ######################## 
# 1. use getPage create socket 
# 2. Add the socket to the event loop in 
# 3. start the event loop (not automatically end) 
# ######################## 
# DEF the Response (Content): 
#      Print (Content) 

# defer.inlineCallbacks @ 
# DEF Task (): 
#      URL = "http://www.baidu.com" 
#      D = the getPage (url.encode ( 'UTF-. 8')) 
#      d.addCallback (Response) 
#      the yield D 
# 
# DEF DONE (* args, ** kwargs): 
#      reactor.stop ()
# 
# Task () 
# reactor.run () 
# ######################## 
# 1. use getPage create socket 
# 2. Add the socket to event loop 
# 3. start the event loop (automatically end) 
# ######################## 
# DEF the Response (Content): 
#      Print (Content) 
# @ defer.inlineCallbacks 
# DEF Task (): 
#      URL = "http://www.baidu.com" 
#      D = the getPage (url.encode ( 'UTF-. 8')) 
#      d.addCallback (Response) 
#      the yield D 
# 
# DEF DONE (* args, ** kwargs): 
#      reactor.stop ()
# 
# D = Task () 
# dd defer.DeferredList = ([D,]) 
# dd.addBoth (DONE) 
# 
# reactor.run () 



# ################ ######## 
# 1. Using getPage create socket 
# 2. Add the socket to the event loop 
# 3. start the event loop (automatically end) 
# ############## ########## 
# DEF Response (Content): 
#      Print (Content) 
# 
# @ defer.inlineCallbacks 
# DEF Task (): 
#      URL = "http://www.baidu.com" 
#      D the getPage = (url.encode ( 'UTF-. 8')) 
#      d.addCallback (Response) 
#     yield d
#     url = "http://www.baidu.com"
#     d = getPage(url.encode('utf-8'))
#     d.addCallback(response)
#     yield d
#
# def done(*args,**kwargs):
#     reactor.stop()
#
# li = []
# for i in range(10):
#     d = task()
#     li.append(d)
# dd = defer.DeferredList(li)
# dd.addBoth(done)

# reactor.run()








#########################
# 1.利用getPage创建socket
# 2.将socket添加到事件循环中
# 3.开始事件循环（自动结束）
#########################
"""
_close = None
count = 0
def response(content):

    print(content)
    global count
    count += 1
    if count == 3:
        _close.callback(None)

@defer.inlineCallbacks
def task():
    """
    每个爬虫的开始：stats_request
    :return:
    """
    url = "http://www.baidu.com"
    d1 = getPage(url.encode('utf-8'))
    d1.addCallback(response)

    url = "http://www.cnblogs.com"
    d2 = getPage(url.encode('utf-8'))
    d2.addCallback(response)

    url = "http://www.bing.com"
    d3 = getPage(url.encode('utf-8'))
    d3.addCallback(response)

    global _close
    _close = defer.Deferred()
    yield _close


def done(*args,**kwargs):
    reactor.stop()

# 每一个爬虫
spider1 = task()
spderd2 = task()
dd = defer.DeferredList([spider1,spderd2])
dd.addBoth(done)

reactor.run()
"""

Foreplay .py