通过HtmlAgilityPack插件和xpath解析html完成爬虫抓取数据

1. 通过HtmlAgilityPack插件和xpath解析html完成爬虫抓取数据
   public static void GetData(string url, ref DataTable dt)
   {
   try
   {
   //WebClient获取Amazon的html会返回校验页面的html
   //WebClient wc = new WebClient();
   //string html = wc.DownloadString(url);

           //HtmlWeb方式获取html，获取多次以后，后续会加载不到html
           //HtmlWeb web = new HtmlWeb();
           //HtmlAgilityPack.HtmlDocument doc = web.Load(url);
           //找到排行的每个商品节点
   
           //通过HttpWebRequest方式获取html
           string html = WebRequestPost(url);
           HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
           doc.LoadHtml(html);
           string xpathDiv = "//div[@class='zg_itemImmersion']";//找到class=zg_itemImmersion的div节点
           HtmlNodeCollection allDivs = doc.DocumentNode.SelectNodes(xpathDiv);
           for (int i = 0; i < allDivs.Count; i++)
           {
               if (i > 2) break;
   
               //需要把allDivs里面的node重新转换为HtmlNode才能取到对应节点的信息，否则取到的一直都是第一个
               HtmlNode node = HtmlNode.CreateNode(allDivs[i].InnerHtml);
               DataRow dr = dt.NewRow();
   
               //过滤商品排名
               string xpath = "//span[@class='zg_rankNumber']";//找到class=zg_rankNumber的span节点
               string indexText = node.SelectSingleNode(xpath).InnerText.Replace(".", "").Replace("\n", "").TrimStart().TrimEnd();
               int rank = int.Parse(indexText);
               dr["排名"] = rank;
   
               //过滤商品名称
               xpath = "//div[@class='p13n-sc-truncate p13n-sc-truncated-hyphen p13n-sc-line-clamp-2']";//找到对应class的div节点
               string name = node.SelectSingleNode(xpath).InnerText.Replace("\n", "").TrimStart().TrimEnd();
               dr["商品名称"] = name;
   
               //过滤商品价格
               xpath = "//span[@class='p13n-sc-price']";//找到class=p13n-sc-price的span节点
               string price = node.SelectSingleNode(xpath).InnerText.Replace("\n", "");
               dr["售价"] = price;
   
               //过滤商品明细连接，position()从1开始
               xpath = "//a[@class='a-link-normal' and position()=1]";//找到class=a-link-normal的并且位置是第一个的a节点
               string href = node.SelectSingleNode(xpath).Attributes["href"].Value;
               href = "https://www.amazon.com" + href;
               string htmlDetail = WebRequestPost(href);
               HtmlAgilityPack.HtmlDocument docDetail = new HtmlAgilityPack.HtmlDocument();
               docDetail.LoadHtml(htmlDetail);
               xpath = "//div[@id='detailBulletsWrapper_feature_div']";//找到id=detailBulletsWrapper_feature_div的div节点
               HtmlNode nodeDetail = docDetail.DocumentNode.SelectSingleNode(xpath);
               if (nodeDetail != null)
               {
                   //过滤商品首次上架日期节点
                   //xpath = "//li[position()=5]//span[position()=2]";//不能直接取固定位置的li，因为有些商品的li数量不一致有些5个，有些6个
                   //找到包含有Date first available at Amazon.com文本内容的span节点的第一个span兄弟节点
                   xpath = "//span[contains(text(), 'Date first available at Amazon.com')]/following-sibling::span[1]";
                   string dateFrist = nodeDetail.SelectSingleNode(xpath).InnerText;
                   dr["首次上架日期"] = dateFrist;
   
                   //过滤商品分类排名信息
                   xpath = "//li[@id='SalesRank']/b/following::text()[1]";//找到id=SalesRank的li节点里面b节点相邻的第一个文本节点
                   string categoryRank = nodeDetail.SelectSingleNode(xpath).InnerText.Replace("(", "");//获取主分类排名
                   xpath = "//li[@id='SalesRank']/ul[@class='zg_hrsr']";//找到id=SalesRank的li节点里面class=zg_hrsr的ul节点
                   string detailRank = nodeDetail.SelectSingleNode(xpath).InnerText.Replace("&nbsp;", " ").Replace("&gt;", ">");//获取具体分类排名
                   dr["排名信息"] = categoryRank + detailRank;
               }
               dt.Rows.Add(dr);
           }
       }
       catch (Exception ex)
       {
           MessageBox.Show("爬虫抓取失败，失败信息：" + ex.Message);
       }
   }