Use crawlers to download thousands of beautiful pictures! (Qt version)

I saw a blog today: Java crawler-use crawler to download thousands of beautiful pictures! , I couldn't help itching my hands, so I imitated one with Qt.

Code:

#ifndef WIDGET_H
#define WIDGET_H

#include <QWidget>

QT_BEGIN_NAMESPACE
namespace Ui { class Widget; }
QT_END_NAMESPACE

class Widget : public QWidget
{
    Q_OBJECT

public:
    Widget(QWidget *parent = nullptr);
    ~Widget();
    static void getHttpData(const QString &urlString);

private slots:
    void on_pushButton_clicked();

private:
    Ui::Widget *ui;
    QStringList urlList;//图片地址
    static QStringList failList;//下载失败的图片地址
};
#endif // WIDGET_

#include "widget.h"
#include "ui_widget.h"
#include <QJsonDocument>
#include <QJsonObject>
#include <QJsonArray>
#include <QDebug>
#include <QUrl>
#include <QNetworkReply>
#include <QNetworkAccessManager>
#include <QFuture>
#include <QtConcurrent>
#include <QMessageBox>

#define debug qDebug()<<

QStringList Widget::failList;
Widget::Widget(QWidget *parent)
    : QWidget(parent)
    , ui(new Ui::Widget)
{
    ui->setupUi(this);
}

Widget::~Widget()
{
    delete ui;
}

void Widget::getHttpData(const QString & urlString)
{
    QUrl url = QUrl::fromUserInput(urlString.toUtf8());
    if(!url.isValid())
    {
        failList << urlString;
        return;
    }
    QNetworkRequest request;
    request.setUrl(url);

    QNetworkAccessManager manager;
    QNetworkReply * reply_ = manager.get(request);
    connect(reply_, QOverload<QNetworkReply::NetworkError>::of(&QNetworkReply::error),[&](QNetworkReply::NetworkError code)
    {
        switch(static_cast<int>(code))
        {
            case QNetworkReply::ConnectionRefusedError:
                debug "远程服务器拒绝连接（服务器不接受请求）";
                break;
            case QNetworkReply::HostNotFoundError:
                debug "找不到远程主机名（无效的主机名）";
                break;
            case QNetworkReply::TimeoutError:
                debug "与远程服务器的连接超时";
                break;
            default:
                break;
        }
    });

    QEventLoop eventLoop2;
    connect(reply_, &QNetworkReply::finished, &eventLoop2, &QEventLoop::quit);
    eventLoop2.exec(QEventLoop::ExcludeUserInputEvents);

    QByteArray data = reply_->readAll();
    if(data.isEmpty())
    {
        failList << urlString;
        return;
    }

    QRegExp exp("[^/]+\.(png|jpg)$");
    exp.setMinimal(false);

    int pos = 0;
    QString fileName;
    while ((pos = exp.indexIn(urlString , pos)) != -1)
    {
        pos += exp.matchedLength();
        fileName = exp.cap(0);
    }

    QPixmap pixmap;
    pixmap.loadFromData(data);
    pixmap.save("pic/" + fileName);
}

void Widget::on_pushButton_clicked()
{
    QString dataInfo = ui->textEdit->toPlainText().trimmed();
    QJsonDocument jsonDocument = QJsonDocument::fromJson(dataInfo.toUtf8().data());
    QJsonArray array = jsonDocument.object()["data"].toObject()["items"].toArray();
    urlList.clear();
    std::for_each(array.begin(),array.end(),[this](const QJsonValue & value)
    {
        urlList << value.toObject()["picUrl"].toString();
    });
    if(!urlList.isEmpty())
    {
        debug "图片数量：" << urlList.size();
        failList.clear();
        QFuture<void> f = QtConcurrent::map(urlList,&Widget::getHttpData);
        f.waitForFinished();
        QMessageBox::information(this,"提示","全部下载完成");
        if(!failList.isEmpty())
        {
            debug "以下图片下载失败："<<endl<<failList;
        }
    }
    else
    {
        debug "没有图片";
    }
}

The code is very simple, take the picture address from the json data, and then start the multi-threaded download. After downloading, save it to the "/pic" folder.

effect:

Code: https://download.csdn.net/download/kenfan1647/15173692

Use crawlers to download thousands of beautiful pictures! (Qt version)

Guess you like