scrapy 提取html 提取正文

html="".join(response.xpath('//html').extract())

content = "".join(response.xpath('//div[@id="cnblogs_post_body"]//text()').extract())

完整代码如下：

#coding=utf-8
import re
import json
import scrapy
import urllib
from scrapy.selector import Selector
try:
    from scrapy.spider import Spider
except:
    from scrapy.spider import BaseSpider as Spider
from scrapy.utils.response import get_base_url
from scrapy.utils.url import urljoin_rfc
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor as sle
from cnblogs.items import *


class CnblogsSpider(CrawlSpider):
    #定义爬虫的名称
    name = "CnblogsSpider"
    #定义允许抓取的域名,如果不是在此列表的域名则放弃抓取
    allowed_domains = ["cnblogs.com"]
    #定义抓取的入口url
    start_urls = [
       "http://www.cnblogs.com/rwxwsblog/default.html?page=1"
    ]
    # 定义爬取URL的规则，并指定回调函数为parse_item
    rules = [
        Rule(sle(allow=("/rwxwsblog/default.html\?page=2")), #\d{1,}此处要注意?号的转换，复制过来需要对?号进行转换。
          follow=True,
          callback='parse_item')
    ]
    print "**********CnblogsSpider**********"
    #定义回调函数
    #提取数据到Items里面，主要用到XPath和CSS选择器提取网页数据
    def parse_item(self, response):
        #print "-----------------"

        sel = Selector(response)
        base_url = get_base_url(response)
        postTitle = sel.css('div.day div.postTitle')
        #print "=============length======="
        postCon = sel.css('div.postCon div.c_b_p_desc')
       #标题、url和描述的结构是一个松散的结构，后期可以改进
        for index in range(len(postTitle)):
            item = CnblogsItem()
            item['title'] = postTitle[index].css("a").xpath('text()').extract()[0]
            #print item['title'] + "***************\r\n"
            item['link'] = postTitle[index].css('a').xpath('@href').extract()[0]
            item['listUrl'] = base_url
            item['desc'] = postCon[index].xpath('text()').extract()[0]
            #print base_url + "********\n"

            #print repr(item).decode("unicode-escape") + '\n'

            yield scrapy.Request(item['link'] , self.parse_details , meta={'item': item})

            # print repr(item).decode("unicode-escape") + '\n'


    def parse_details(self , response):
        items = []
        content1 = []
        item = response.meta['item']

        sel = Selector(response)
        #html =repr(sel.xpath('//html').extract()[0]).decode("unicode-escape")
        #html = repr(sel.xpath('//html').extract()[0]).decode("unicode_escape")
        #html =response.xpath('//div[@id="cnblogs_post_body"]/*').extract()
        #html =response.xpath('//div[@id="cnblogs_post_body"]//node()').extract()

        html="".join(response.xpath('//html').extract())

       # content = sel.xpath('//div[@class="post"]//text()').extract()[0].encode('utf-8')
        #content = repr(sel.xpath('//div[@class="post"]//text()')j.extract()).decode("unicode-escape").encode('utf-8')
       # content = repr(sel.xpath('//div[@class="post"]//text()').extract()).decode("unicode-escape")

        #content = "".join(response.xpath('//div[@class="post"]//text()').extract())

        content = "".join(response.xpath('//div[@id="cnblogs_post_body"]//text()').extract())


        #print 'html:'+html.decode("unicode-escape")
        #print 'html:' + html.encode('utf-8')
        #print 'content:' + content

       # print urllib.unquote(str(html))
        #print 'content:' + content.encode('raw_unicode_escape')

        #content = repr(sel.xpath('//div[@class="post"]//text()').extract())
        #
        #
        #
        # for t in content:
        #     #content1.append(t.encode("utf-8"))
        #     print(t.encode("utf-8"))
        #content = repr(sel.xpath('//div[@class="post"]//text()').extract())
       # content =repr(sel.xpath('//div[@class="post"]//text()').extract().encode('utf-8'))

        #content = repr(sel.css('div.post').xpath('text()').extract()).decode("unicode-escape")
        #content = unicode.encode( content , 'utf-8')

        #content='content'#repr(content).decode("unicode-escape")
        item['html'] =html
        item['content'] = content

        print 'link::'+item['link']
        print 'desc::'+item['desc']

        #str = json.dumps(html, ensure_ascii=False) + "\n";
        #str = unicode.decode(str, 'utf-8').encode('gb2312');

        #s=html#item['content']
        #print 'html:' + html

        #
        # if isinstance(s, unicode):
        #     # s=u"中文"
        #     print s.encode('utf-8')
        # else:
        #     # s="中文"
        #     print s.decode('utf-8').encode('gb2312')


        #print 'content:' + item['content']
        # populate more `item` fields
        items.append(item)

        return items