今天闲着无聊，好就没用scrapy框架写爬虫了，于是就写了一个企业黄页的爬虫脚本，采集顺企网黄页数据，保存到mongodb，话不多说，简单看下代码吧。
下面部分是spider代码（习惯使用beautifulsoup了QAQ）
import scrapy
from bs4 import BeautifulSoup
from shunqi.items import ShunqiItem
import re
class EpregSpider(scrapy.Spider):
name = 'epreg'
allowed_domains = ['11467.com']
start_url = "http://b2b.11467.com/"
def start_requests(self):
yield scrapy.Request(url=self.start_url,callback=self.get_city)
def get_city(self,response):
soup = BeautifulSoup(response.text,"lxml")
dl_listtxt = soup.find(text="按城市浏览全国公司黄页").findNext("div").find_all("dl",attrs={"class":"listtxt"})
for dl in dl_listtxt:
province = dl.find("dt").text.strip()
dd_list = dl.find_all("dd")
for dd in dd_list:
city = dd.text.strip()
city_link = "https:" + dd.find("a")["href"]
yield scrapy.Request(url=city_link,callback=self.get_class,meta={"city":city,"province":province})
def get_class(self,response):
city = response.meta["city"]
province = response.meta["province"]
soup = BeautifulSoup(response.text,"lxml")
class_all = soup.find(text=re.compile("公司分类")).findNext("ul").find_all("li")
for li in class_all:
dd_all = li.find("dl").find_all("dd")
for dd in dd_all:
cor_link = "https:" + dd.find("a")["href"]
yield scrapy.Request(url=cor_link,callback=self.get_cor,meta={"city":city,"province":province})
def get_cor(self,response):
city = response.meta["city"]
province = response.meta["province"]
soup = BeautifulSoup(response.text,"lxml")
company_list = soup.find("ul",attrs={"class":"companylist"})
if company_list:
company_list1 = company_list.find_all("div",attrs={"class","f_l"})
for li in company_list1:
whole_url = "https:" + li.find("h4").findNext("a")["href"]
if "qiye" in whole_url:
whole_url = whole_url + "#contact"
else:
whole_url = "https:" + li.find("h4").findNext("a")["href"]
yield scrapy.Request(url=whole_url,callback=self.get_info,meta={"url":whole_url,"city":city,"province":province})
is_pages = soup.find("div",attrs={"class","pages"})
if is_pages:
max_page = is_pages.find(text="尾页").findParent("a")["href"].split("-")[1].replace(".htm")
page_href = is_pages.find(text="尾页").findParent("a")["href"]
for page in range(2,int(max_page)+1):
page_url = page_href.split("-")[0] + "-" + str(page) + ".htm"
yield scrapy.Request(url=page_url,callback=self.get_cor,meta={"city":city,"province":province})
def deal_item(self,soup,text,label):
try:
data = soup.find(text=re.compile("%s" % text)).findNext("%s" % label).text.strip().replace("\r\n","")
except:
data = None
return data
def get_info(self,response):
item = ShunqiItem()
whole_url = response.meta["url"]
if "qiye" in whole_url:
id = re.search("qiye\/(\d+)\.htm",whole_url,re.S).group(1)
else:
id = re.search("co\/(\d+)\.htm",whole_url,re.S).group(1)
city= response.meta["city"]
province = response.meta["province"]
soup = BeautifulSoup(response.text,"lxml")
adress = self.deal_item(soup,"公司地址：","dd")
phone = self.deal_item(soup,"固定电话：","dd")
manager = self.deal_item(soup,"经理：","dd")
if self.deal_item(soup,"手机号码：","dd") == None:
link_phone = self.deal_item(soup,"经理手机：","dd")
else:
link_phone= self.deal_item(soup,"手机号码：","dd")
email = self.deal_item(soup,"电子邮件：","dd")
cor_name = self.deal_item(soup,"法人名称：","td")
business = self.deal_item(soup,"经营范围：","td")
main_product = self.deal_item(soup,"主要经营产品：","td")
for field in item.fields:
item[field] = eval(field)
yield item
下面是采集到一些数据的截图
以上就是基本的一些代码，本文章只是提供一些采集思路，有兴趣的可以自己写一下，不是很复杂。同时欢迎访问个人博客主页… …
