MINIO サーバーは、AWS S3 SDK ファイルのアップロードとスライスでのダウンロードに基づいています (C++ で実装)

開発 2023-12-17 15:36:13 訪問数: null

MINIO サーバーは、AWS S3 SDK ファイルのアップロードとスライスでのダウンロードに基づいています (C++ で実装)

aws-s3に基づくインストール

インストール

インストール環境の依存関係:

Debian/Ubuntu-based systems : sudo apt-get install libcurl4-openssl-dev
libssl-dev uuid-dev zlib1g-dev libpulse-dev cmake
Redhat/Fedora-based systems : sudo dnf install libcurl-devel openssl-devel
libuuid-devel pulseaudio-devel cmake

SDK ソース コードを取得してインストールします。

git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git
cd aws-sdk-cpp
mkdir build
cd build
cmake ..
make -j8
sudo make install

プロジェクト内の CMakeLists.txt 構成:

cmake_minimum_required(VERSION 3.19)
project(project_name)
set(CMAKE_CXX_STANDARD 11)
 
# Find the AWS SDK for C++ package.
find_package(AWSSDK CONFIG COMPONENTS core s3 REQUIRED)
 
include_directories(${
    
    AWSSDK_INCLUDE_DIRS})
add_executable(project_name mian.cpp)
target_link_libraries(project_name PRIVATE  aws-cpp-sdk-s3)

S3接続の初期化

S3Client InitS3Client(const Aws::String IPPort, const Aws::String& accessKeyId, const Aws::String& secretKey, std::string securityToken){
    
    
    // 初始化 S3 Client
    Aws::Client::ClientConfiguration cfg;
    cfg.endpointOverride = IPPort;
    cfg.scheme = Aws::Http::Scheme::HTTP;
    cfg.verifySSL = false;

    Aws::Auth::AWSCredentials cred(accessKeyId, secretKey, securityToken);
    S3Client client(cred, cfg,
                    Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false);
    return client;
}

ファイルをアップロードする

一般的なファイルのアップロード

/// 判断待上传的文件是否存在
inline bool file_exists(const string& name)
{
    
    
    struct stat buffer;
    return (stat(name.c_str(), &buffer) == 0);
}

/// 上传文件
bool doPutObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const string& file_name, const Aws::String &region)
{
    
    
    // 判断文件是否存在
    if (!file_exists(file_name)) {
    
    
        cout << "ERROR: 找不到这个文件，这个文件不存在"
             << endl;
        return false;
    }

    /// 初始化上传请求
    Aws::S3::Model::PutObjectRequest object_request;
    object_request.SetBucket(fromBucket);
    object_request.SetKey(objectKey);

    /// 建立文件输入串流，可为任何串流
    const shared_ptr<Aws::IOStream> input_data =
            Aws::MakeShared<Aws::FStream>("SampleAllocationTag",
                                          file_name.c_str(),
                                          ios_base::in | ios_base::binary);
    object_request.SetBody(input_data);

    /// 开始上传至S3
    Aws::S3::Model::PutObjectOutcome put_object_outcome = client.PutObject(object_request);

    if (!put_object_outcome.IsSuccess()) {
    
    
        auto error = put_object_outcome.GetError();
        cout << "ERROR: " << error.GetExceptionName() << ": "
             << error.GetMessage() << endl;
        return false;
    }else {
    
    
        cout << "success" << endl;
        return true;
    }
}

/// 启动上传程序
bool StartUpLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){
    
    
    // 初始化
    Aws::SDKOptions options;
    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
    Aws::InitAPI(options);

    /// 设置ip、acccessKey、secretKey
    Aws::String IP = Ips;
    Aws::String accessKeyId = acccessKey;
    Aws::String secretKey = secretKeyS;

    S3Client client;
    client = InitS3Client(IP, accessKeyId, secretKey, securityToken);

    Aws::InitAPI(options);
    {
    
    
        /// 设置上传的目标桶和文件路径
        const Aws::String bucket_name = bucket_names;
        const Aws::String object_name = object_names;
        if (!doPutObject(object_name, bucket_name, client, imagename, " ")) {
    
    
            std::string errorInfo =  object_name + " 上传失败";
            writeLog(L_ERROR, errorInfo.c_str());
            return false;
        }
    }
    writeLog(L_INFO, "====> 文件上传成功");
    return true;
}

分割してアップロードする

/// MINIO 切片上传
bool mainCutUpload(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){
    
    


    Aws::SDKOptions options;
    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
    Aws::InitAPI(options);

    Aws::String IP = Ips;
    Aws::String accessKeyId = acccessKey;
    Aws::String secretKey = secretKeyS;

    /// 初始化S3服务
    S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken);

    Aws::InitAPI(options);
    {
    
    
        /// 设置上传的目标桶和文件路径
        const Aws::String bucket_name = bucket_names;
        const Aws::String object_name = object_names;
        std::string bucket = bucket_name;
        std::string key = object_name;

        /// 初始化分块上传任务
        Aws::S3::Model::CreateMultipartUploadRequest create_request;
        create_request.SetBucket(bucket.c_str());
        create_request.SetKey(key.c_str());
        Aws::S3::Model::CreateMultipartUploadOutcome outcome = client.CreateMultipartUpload(create_request);

        /// 获取上传ID
        std::string upload_id = outcome.GetResult().GetUploadId();
        std::cout << "upload id is:" << upload_id << std::endl;

        /// 创建分段上传输出
        if (!outcome.IsSuccess()) {
    
    
            auto err = outcome.GetError();
            std::cout << "Error: CreateMultipartUpload: " <<
                      err.GetExceptionName() << ": " << err.GetMessage() << std::endl;
            return false;
        }
        else
            std::cout << "Success: CreateMultipartUpload: " << bucket << std::endl;

        std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetBucket() << std::endl;
        std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetKey() << std::endl;
        std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetUploadId() << std::endl;

        Aws::String UploadId = upload_id;

        /// 启动分块上传
        Aws::S3::Model::UploadPartRequest request1;
        request1.SetBucket(bucket);
        request1.SetKey(key);

        /// 每个分片大小
        long partSize = 5 * 1024 * 1024;

        /// 读取文件
        std::fstream file(imagename.c_str(),std::ios::in | std::ios::binary);
        file.seekg(0,std::ios::end);
        long fileSize = file.tellg();   /// 获取文件大小
        file.seekg(0, std::ios::beg);
        char* buffer = new char[partSize];  // 定义缓存

        /// 初始化上传变量
        long filePosition = 0;
        Aws::Vector<Aws::S3::Model::CompletedPart> completeParts;
        int partNumber = 1;

        /// 开始上传
        while(filePosition < fileSize){
    
    
            partSize = std::min(partSize,(fileSize - filePosition));
            file.read(buffer,partSize);
            Aws::S3::Model::UploadPartRequest uploadPartRequest;
            uploadPartRequest.WithBucket(bucket).WithKey(key).WithUploadId(UploadId).WithPartNumber(partNumber).WithContentLength(partSize);
            Aws::String str(buffer,partSize);
            auto input_data = Aws::MakeShared<Aws::StringStream>("UploadPartStream",str);
            uploadPartRequest.SetBody(input_data);
            filePosition += partSize;
            auto uploadPartResult = client.UploadPart(uploadPartRequest);
            completeParts.push_back(Aws::S3::Model::CompletedPart().WithETag(uploadPartResult.GetResult().GetETag()).WithPartNumber(partNumber));
            memset(buffer, 0, partSize);
            ++partNumber;
        }

        // 完成分块上传
        Aws::S3::Model::CompleteMultipartUploadRequest request2;
        request2.SetBucket(bucket);
        request2.SetKey(key);
        request2.SetUploadId(UploadId);
        Aws::S3::Model::CompletedMultipartUpload completed_multipart_upload;

        completed_multipart_upload.SetParts(completeParts);
        request2.SetMultipartUpload(completed_multipart_upload);
        Aws::S3::Model::CompleteMultipartUploadOutcome outcome3 = client.CompleteMultipartUpload(request2);

        if (outcome3.IsSuccess()) {
    
    
            std::cout << "Success: CompleteMultipartUpload " << std::endl;
            return true;
        }
        else
        {
    
    
            std::cout << "Error: CompleteMultipartUpload: " <<
                      outcome.GetError().GetMessage() << std::endl;
            return false;
        }

    }
}

ダウンロードファイル

/// 从MINIO中下载文件或数据
bool doGetObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const Aws::String savePath,const Aws::String &region)
{
    
    
    /// 初始化下载请求
    Aws::S3::Model::GetObjectRequest object_request;
    object_request.SetBucket(fromBucket);
    object_request.SetKey(objectKey);

    /// 从S3服务器中下载数据
    Aws::S3::Model::GetObjectOutcome get_object_outcome = client.GetObject(object_request);

    if (get_object_outcome.IsSuccess()) {
    
    
        /// 保存文件
        Aws::OFStream local_file;
        local_file.open(savePath, std::ios::out | std::ios::binary);
        local_file << get_object_outcome.GetResultWithOwnership().GetBody().rdbuf();
        std::cout << "Done!" << std::endl;
        return true;
    } else {
    
    
        auto err = get_object_outcome.GetError();
        std::cout << "Error: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl;
        return false;
    }
}


/// 启动下载程序
bool StartDownLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_name, std::string downPath, std::string imagePath, std::string securityToken,
    bool &flags){
    
    
    // 初始化Aws API
    Aws::SDKOptions options;
    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
    Aws::InitAPI(options);

    /// 设置ip、acccessKey、secretKey
    Aws::String IP = Ips;
    Aws::String accessKeyId = acccessKey;
    Aws::String secretKey = secretKeyS;

    /// 初始化S3服务
    S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken);

    /// 根据get接口的桶名称设置文件所在的桶
    Aws::S3::Model::Bucket temps{
    
    };
    std::string stringK = bucket_name;
    temps.SetName(stringK);

    /// 查找MINIO相关数据
    Aws::S3::Model::ListObjectsRequest objects_request;
    objects_request.WithBucket(temps.GetName());   /// 设置桶路径
    objects_request.SetPrefix(downPath);   /// 设置文件夹路径

    auto list_objects_outcome = client.ListObjects(objects_request);  /// 判断存储桶及相关连接是否有效

    if (flags){
    
    
        writeLog(L_INFO, "====> 文件下载终止");
        return false;
    }
    if (list_objects_outcome.IsSuccess())
    {
    
    
        /// 获取该目录下的前一千个文件
        Aws::Vector<Aws::S3::Model::Object> object_list =
                list_objects_outcome.GetResult().GetContents();

        if (object_list.size() == 1000){
    
    

            /// 当下载数量大于一千时 循环下载文件
            while(object_list.size() == 1000){
    
    
                auto finally_object = object_list[0];
                for (auto const &s3_object : object_list)
                {
    
    
                    if (flags){
    
    
                        writeLog(L_INFO, "====> 文件下载终止");
                        return false;
                    }
                    /// 设置桶名称
                    const Aws::String bucket_name = stringK;
                    /// 设置待下载的文件名称
                    const Aws::String object_name = s3_object.GetKey();
                    /// 设置保存文件路径及文件名称
                    std::string NewPath;
                    size_t indexNew = object_name.find_last_of('/');
                    NewPath = object_name.substr(indexNew + 1, object_name.length());
                    const Aws::String SavePath = imagePath + NewPath;
                    size_t indexK = downPath.find_last_of('/');
                    std::string newPathEs = downPath.substr(0, indexK);
                    std::string strK;
                    strK = imagePath + "/" + NewPath;
                    const Aws::String SavePaths = strK;

                    //筛选数据
                    std::string fileName;
                    size_t index = object_name.find_last_of('.');
                    fileName = object_name.substr(index + 1, object_name.length());
                    if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){
    
    
                        /// 启动下载
                        if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {
    
    
                            std::string errorInfo =  object_name + " 下载失败";
                            writeLog(L_ERROR, errorInfo.c_str());
                            continue;
                        }
                    }
                }
                finally_object = object_list[999];
                /// 重新配置相关路径
                objects_request.WithBucket(temps.GetName());   /// 设置桶路径
                objects_request.SetPrefix(finally_object.GetKey());   /// 设置新文件夹路径

                list_objects_outcome = client.ListObjects(objects_request);  /// 判断存储桶及相关连接是否有效
                if (list_objects_outcome.IsSuccess()){
    
    
                    object_list = list_objects_outcome.GetResult().GetContents();
                }else{
    
    
                    writeLog(L_ERROR, "====> 再次连接失败");
                }
            }
            for (auto const &s3_object : object_list)
            {
    
    
                if (flags){
    
    
                    writeLog(L_INFO, "====> 文件下载终止");
                    return false;
                }
                /// 设置桶名称
                const Aws::String bucket_name = stringK;
                /// 设置待下载的文件名称
                const Aws::String object_name = s3_object.GetKey();
                /// 设置保存文件路径及文件名称
                std::string NewPath;
                size_t indexNew = object_name.find_last_of('/');
                NewPath = object_name.substr(indexNew + 1, object_name.length());
                const Aws::String SavePath = imagePath + NewPath;
                size_t indexK = downPath.find_last_of('/');
                std::string newPathEs = downPath.substr(0, indexK);

                std::string strK;
                strK = imagePath + "/" + NewPath;
                const Aws::String SavePaths = strK;

                //筛选数据
                std::string fileName;
                size_t index = object_name.find_last_of('.');
                fileName = object_name.substr(index + 1, object_name.length());
                if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){
    
    
                    /// 启动下载
                    if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {
    
    
                        std::string errorInfo =  object_name + " 下载失败";
                        writeLog(L_ERROR, errorInfo.c_str());
                        continue;
                    }
                }
            }
        }else{
    
    
            for (auto const &s3_object : object_list)
            {
    
    
                if (flags){
    
    
                    writeLog(L_INFO, "====> 文件下载终止");
                    return false;
                }
                /// 设置桶名称
                const Aws::String bucket_name = stringK;
                /// 设置待下载的文件名称
                const Aws::String object_name = s3_object.GetKey();
                /// 设置保存文件路径及文件名称
                std::string NewPath;
                size_t indexNew = object_name.find_last_of('/');
                NewPath = object_name.substr(indexNew + 1, object_name.length());
                const Aws::String SavePath = imagePath +  NewPath;
                size_t indexK = downPath.find_last_of('/');
                std::string newPathEs = downPath.substr(0, indexK);
                std::string strK;
                strK = imagePath + "/" + NewPath;
                const Aws::String SavePaths = strK;

                //筛选数据
                std::string fileName;
                size_t index = object_name.find_last_of('.');
                fileName = object_name.substr(index + 1, object_name.length());
                if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){
    
    
                    /// 启动下载
                    if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {
    
    
                        std::string errorInfo =  object_name + " 下载失败";
                        writeLog(L_ERROR, errorInfo.c_str());
                        continue;
                    }
                }

            }
        }

    }
    writeLog(L_INFO, "====> 文件下载成功");
    return true;
}

参考文献

SDK ドキュメント
C++_SDK.pdf

ダウンロードリンク

ダウンロードを実装するクラス コード全体のダウンロード
具体的な内容は次のとおりです
ここに画像の説明を挿入します

おすすめ

転載: blog.csdn.net/weixin_43482623/article/details/134397818
おすすめ
GitHub のホットリストに急上昇 — オープンソースのプログラミング言語とフレームワークは、どうしてこんなにも魅力的なのでしょうか?
北京ヒューマノイドロボットイノベーションセンターが、擬人化走行用の純粋な電気駆動を備えた世界初のフルサイズヒューマノイドロボット「Tiangong」を発売
ランキング
アリクラウドサーバ_ubuntu ECSシステムディスク拡張システム_
ボタンをクリックしてクリップボードにコピーします
脅威ハンティングを使用してサイバー セキュリティを強化する
ライン画像やコンテナnginxのの建設
Python での try の使用について書く
リアルタイムのデータウェアハウスを構築する方法高速データ時代、MySQLのAnalyticDBはあなたを教えて
Docker でのネットワークおよびデータ管理 (Docker Swarm クラスターおよび Docker データ管理を含む)
Java ループの比較: 従来の for ループ、拡張された for ループ、および forEach ループ
浙江大学版「C言語プログラミング（第3版）」演習2-18
Gestion du temps RTOS - Utilisation de la fonction de crochet SysTick pour réaliser l'échantillonnage de synchronisation et les sous-sections de ce chapitre
アーカイブ
もっと
2024-04-27(29)
2024-04-26(22)
2024-04-25(31)
2024-04-24(30)
2024-04-23(31)
2024-04-22(5)
2024-04-21(0)
2024-04-20(6)
2024-04-19(5)
2024-04-18(0)

コードワールド

© 2018 codetd.com