一时心血来潮想做一个网页图片爬虫，从网上找了好多资料，不过大多数的都看不懂，知识面太浅，历经千辛万苦，终于实现简单的完成了！
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace 网页图片爬虫_窗体版
{
public partial class Form1 : ZHSkin.ZHForm
{
/// <summary>
/// 关键词
/// </summary>
private string SavePath = string.Empty;
private Thread thread = null;
public Form1()
{
InitializeComponent();
}
/// <summary>
/// 获取保存路径
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void btn_GetPath_Click(object sender, EventArgs e)
{
FolderBrowserDialog folderBrowser = new FolderBrowserDialog();
if (folderBrowser.ShowDialog() == DialogResult.OK)
{
txt_SavePath.Text = folderBrowser.SelectedPath;
}
}
/// <summary>
/// 开始获取
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void btn_Start_Click(object sender, EventArgs e)
{
string KeyWord = txt_KeyWords.Text.Trim();
int pageCount = (int)NUD_PageCount.Value;
SavePath = txt_SavePath.Text;
if (string.IsNullOrEmpty(KeyWord))
{
txt_ShowPath.AppendText("请输入要搜索的关键词！" + Environment.NewLine);
return;
}
if (string.IsNullOrEmpty(SavePath))
{
txt_ShowPath.AppendText("请选择要保存的路径！" + Environment.NewLine);
return;
}
if (!Directory.Exists(SavePath))//如果不存在就创建file文件夹
{
txt_ShowPath.AppendText("输入路径不正确，请核对！" + Environment.NewLine);
return;
}
if (!SavePath.EndsWith("\\"))
{
SavePath = SavePath + "\\";
}
btn_Stop.Enabled = true;//启用停止按钮
btn_Start.Enabled = false;//禁用开始按钮
txt_ShowPath.Clear();//清空日志
txt_ShowPath.AppendText("正在启动下载！" + Environment.NewLine);
msg.Text = "已启动下载...";
//启动进度条
ProgressBar.Enabled = true;
//使用多线程下载
thread = new Thread(() =>
{
ProcessDownload(KeyWord);
});
thread.Start();//启动下载
}
/// <summary>
/// 处理下载
/// </summary>
/// <param name="KeyWord"></param>
public void ProcessDownload(string KeyWord)
{
try
{
int pageCount = (int)NUD_PageCount.Value;//下载页数
//循环获取路径
for (int i = 0; i < pageCount; i++)
{
msg.Text = "正在下载第" + (i + 1) + "页，共" + pageCount + "页";
string URL = "https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=" + Uri.EscapeDataString(KeyWord) + "&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=-1&z=&ic=0&word=" + Uri.EscapeDataString(KeyWord) + "&s=&se=&tab=&width=&height=&face=0&istype=2&qc=&nc=1&fr=&cg=wallpaper&pn=" + (i + 1) * 60 + "&rn=60&gsm=3c&1525422519486=";
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URL);
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode == HttpStatusCode.OK)
{
using (Stream stream = response.GetResponseStream())
{
try
{
DownloadPage(stream, i);//下载页面
}
catch (Exception ex)
{
txt_ShowPath.BeginInvoke(new Action(() =>
{
txt_ShowPath.AppendText(ex.Message + Environment.NewLine);
}));
}
}
}
else
{
MessageBox.Show("获取第" + i + "页失败：" + response.StatusCode);
}
}
}
msg.Text = "下载完成！下载" + pageCount + "页，共" + pageCount * 60 + "张图片";
ProgressBar.Value = 0;//进度条归零
btn_Stop.Enabled = false;//禁用停止按钮
btn_Start.Enabled = true;//启用开始按钮
btn_Stop.Text = "停止";
}
catch (Exception)
{
txt_ShowPath.BeginInvoke(new Action(() =>
{
txt_ShowPath.AppendText("网络连接失败！" + Environment.NewLine);
}));
}
}
/// <summary>
/// 下载页面
/// </summary>
/// <param name="stream"></param>
private void DownloadPage(Stream stream, int index)
{
using (StreamReader reader = new StreamReader(stream))
{
string json = reader.ReadToEnd();
//txt_ShowPath.AppendText(json);
JObject objRoot = (JObject)JsonConvert.DeserializeObject(json);
JArray imgs = (JArray)objRoot["data"];
txt_ShowPath.BeginInvoke(new Action(() =>
{
txt_ShowPath.AppendText("正在下载第" + (index + 1) + "页！" + Environment.NewLine);
}));
for (int i = 0; i < imgs.Count; i++)
{
JObject img = (JObject)imgs[i];
string objUrl = (string)img["middleURL"];
try
{
DownloadImage(objUrl);//下载
SetTextMessage(100 * i / imgs.Count);
}
catch (Exception ex)
{
txt_ShowPath.BeginInvoke(new Action(() =>
{
txt_ShowPath.AppendText(ex.Message + Environment.NewLine);
}));
}
}
txt_ShowPath.BeginInvoke(new Action(() =>
{
txt_ShowPath.AppendText("第" + (index + 1) + "页下载完成！" + Environment.NewLine);
}));
}
}
/// <summary>
/// 下载图片
/// </summary>
/// <param name="url"></param>
private void DownloadImage(string objUrl)
{
txt_ShowPath.AppendText("正在下载：" + Path.GetFileName(objUrl) + "" + Environment.NewLine);
//URLRefer://这个图片是从哪个页面启动下载的
string destFile = Path.Combine(SavePath, Path.GetFileName(objUrl));//获取保存路径
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(objUrl);
request.Referer = "https://image.baidu.com/";//欺骗浏览器
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode == HttpStatusCode.OK)
{
using (Stream stream = response.GetResponseStream())
using (Stream FStream = new FileStream(destFile, FileMode.Create))
{
stream.CopyTo(FStream);
txt_ShowPath.AppendText("下载成功！" + Environment.NewLine);
}
}
else
{
throw new Exception("下载" + objUrl + "失败，错误码：" + response.StatusCode);
}
}
}
/// <summary>
/// 进度条代理
/// </summary>
/// <param name="ipos"></param>
/// <param name="vinfo"></param>
private delegate void SetPos(int ipos);
/// <summary>
/// 进度条更新函数
/// </summary>
/// <param name="ipos"></param>
/// <param name="vinfo"></param>
private void SetTextMessage(int ipos)
{
if (this.InvokeRequired)
{
SetPos pos = new SetPos(SetTextMessage);
this.Invoke(pos, new object[] { ipos });
}
else
{
this.ProgressBar.Value = Convert.ToInt32(ipos);
}
}
/// <summary>
/// 停止下载
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void btn_Stop_Click(object sender, EventArgs e)
{
if (thread != null)
{
if (btn_Stop.Text == "暂 停")
{
thread.Suspend();//挂起线程线程
txt_ShowPath.AppendText("已暂停下载！" + Environment.NewLine);
btn_Stop.Text = "继 续";
}
else if (btn_Stop.Text == "继 续")
{
thread.Resume();//挂起线程线程
txt_ShowPath.AppendText("开始下载！" + Environment.NewLine);
btn_Stop.Text = "暂 停";
}
}
}
/// <summary>
/// 窗体关闭时关闭所有线程
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void Form1_FormClosed(object sender, FormClosedEventArgs e)
{
if (thread != null)
{
thread.Abort();
}
}
/// <summary>
/// 设置滚动条
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void txt_ShowPath_TextChanged(object sender, EventArgs e)
{
txt_ShowPath.SelectionStart = txt_ShowPath.Text.Length;
txt_ShowPath.ScrollToCaret();
}
}
}