Go 작업 Amazon S3 자습서
공식 웹사이트 API 주소: https://docs.aws.amazon.com/sdk-for-go/api/service/s3/
서문: S3 소개
1. aws-sdk-go
1 S3 구성 초기화
세션과 연결을 초기화하고 동시에 오류 로그를 인쇄하십시오.
var (
sess *session.Session
svc *s3.S3
)
func init() {
access_key := "xxxxxx"
secret_key := "xxxxxxxxxxx"
end_point := "http://xxx.xxx.xxx.xxx:xxx/" //根据自己endpint修改
sess, _ = session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(access_key, secret_key, ""),
Endpoint: aws.String(end_point),
Region: aws.String("us-east-1"),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(false), //virtual-host style方式，不要修改
})
svc = s3.New(sess)
}
func main(){
....
}
func exitErrorf(msg string, args ...interface{
}) {
fmt.Fprintf(os.Stderr, msg+"\n", args...)
os.Exit(1)
}
2 구체적인 조작
2.1 모든 버킷 정보 얻기
//获取所有桶
func ListBuckets() {
result, err := svc.ListBuckets(nil)
if err != nil {
exitErrorf("Unable to list buckets, %v", err)
}
fmt.Println("Buckets:")
for _, b := range result.Buckets {
fmt.Printf("* %s created on %s\n",
aws.StringValue(b.Name), aws.TimeValue(b.CreationDate))
}
for _, b := range result.Buckets {
fmt.Printf("%s\n", aws.StringValue(b.Name))
}
}
2.2 버킷 생성
func createBucket(bucket string) {
params := &s3.CreateBucketInput{
Bucket: aws.String(bucket),
}
_, err := svc.CreateBucket(params)
if err != nil {
exitErrorf("Unable to create bucket %q, %v", bucket, err)
}
// Wait until bucket is created before finishing
fmt.Printf("Waiting for bucket %q to be created...\n", bucket)
err = svc.WaitUntilBucketExists(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
exitErrorf("Error occurred while waiting for bucket to be created, %v", bucket)
}
fmt.Printf("Bucket %q successfully created\n", bucket)
}
2.3 버킷 삭제
func deleteBucket(bucket string) {
params := &s3.DeleteBucketInput{
Bucket: aws.String(bucket),
}
_, err := svc.DeleteBucket(params)
if err != nil {
exitErrorf("Unable to delete bucket %q, %v", bucket, err)
}
//wait until bucket is deleted
err = svc.WaitUntilBucketNotExists(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
exitErrorf("Error occurred while waiting for bucket to be deleted, %v", bucket)
}
fmt.Printf("Bucket %q successfully delete\n", bucket)
}
2.4 버킷의 파일 다운로드
func downloadFile(bucket, item string) {
file, err := os.Create(item)
if err != nil {
exitErrorf("Unable to open file %q, %v", err)
}
defer file.Close()
downloader := s3manager.NewDownloader(sess)
numBytes, err := downloader.Download(file,
&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(item),
})
if err != nil {
exitErrorf("Unable to download item %q, %v", item, err)
}
fmt.Println("Downloaded", file.Name(), numBytes, "bytes")
}
2.5 버킷의 파일 삭제
func deleteFile(bucket, obj string) {
_, err := svc.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(bucket), Key: aws.String(obj)})
if err != nil {
exitErrorf("Unable to delete object %q from bucket %q, %v", obj, bucket, err)
}
err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(obj),
})
fmt.Printf("Object %q successfully deleted\n", obj)
}
2.6 지정된 버킷에 파일 업로드
func uploadFile(bucket, filename string) {
file, err := os.Open(filename)
if err != nil {
exitErrorf("Unable to open file %q, %v", err)
}
defer file.Close()
uploader := s3manager.NewUploader(sess)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(bucket),
Key: aws.String(filename),
Body: file,
})
if err != nil {
// Print the error and exit.
exitErrorf("Unable to upload %q to %q, %v", filename, bucket, err)
}
fmt.Printf("Successfully uploaded %q to %q\n", filename, bucket)
}
3 모든 코드
package main
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
_ "github.com/aws/aws-sdk-go/service/s3/s3manager"
"os"
)
var (
sess *session.Session
svc *s3.S3
)
func init() {
access_key := "xxxxxx"
secret_key := "xxxxxxxxxxx"
end_point := "http://xxx.xxx.xxx.xxx:xxx/" //根据自己endpint修改
sess, _ = session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(access_key, secret_key, ""),
Endpoint: aws.String(end_point),
Region: aws.String("us-east-1"),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(false), //virtual-host style方式，不要修改
})
svc = s3.New(sess)
}
func main() {
//注意！！！桶名称后面一定要根`/`
//downloadFile("bucket-ziyi/", "banner.png")
//uploadFile("bucket-ziyi/", "test/test1/test2/test.csv")
//createBuckt("bucket-demo1/")
//deleteBucket("bucket-demo1/")
deleteFile("bucket-ziyi/test/", "log.txt")
}
func deleteFile(bucket, obj string) {
_, err := svc.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(bucket), Key: aws.String(obj)})
if err != nil {
exitErrorf("Unable to delete object %q from bucket %q, %v", obj, bucket, err)
}
err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(obj),
})
fmt.Printf("Object %q successfully deleted\n", obj)
}
func deleteBucket(bucket string) {
params := &s3.DeleteBucketInput{
Bucket: aws.String(bucket),
}
_, err := svc.DeleteBucket(params)
if err != nil {
exitErrorf("Unable to delete bucket %q, %v", bucket, err)
}
//wait until bucket is deleted
err = svc.WaitUntilBucketNotExists(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
exitErrorf("Error occurred while waiting for bucket to be deleted, %v", bucket)
}
fmt.Printf("Bucket %q successfully delete\n", bucket)
}
func createBucket(bucket string) {
params := &s3.CreateBucketInput{
Bucket: aws.String(bucket),
}
_, err := svc.CreateBucket(params)
if err != nil {
exitErrorf("Unable to create bucket %q, %v", bucket, err)
}
// Wait until bucket is created before finishing
fmt.Printf("Waiting for bucket %q to be created...\n", bucket)
err = svc.WaitUntilBucketExists(&s3.HeadBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
exitErrorf("Error occurred while waiting for bucket to be created, %v", bucket)
}
fmt.Printf("Bucket %q successfully created\n", bucket)
}
func uploadFile(bucket, filename string) {
file, err := os.Open(filename)
if err != nil {
exitErrorf("Unable to open file %q, %v", err)
}
defer file.Close()
uploader := s3manager.NewUploader(sess)
_, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(bucket),
Key: aws.String(filename),
Body: file,
})
if err != nil {
// Print the error and exit.
exitErrorf("Unable to upload %q to %q, %v", filename, bucket, err)
}
fmt.Printf("Successfully uploaded %q to %q\n", filename, bucket)
}
func downloadFile(bucket, item string) {
file, err := os.Create(item)
if err != nil {
exitErrorf("Unable to open file %q, %v", err)
}
defer file.Close()
downloader := s3manager.NewDownloader(sess)
numBytes, err := downloader.Download(file,
&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(item),
})
if err != nil {
exitErrorf("Unable to download item %q, %v", item, err)
}
fmt.Println("Downloaded", file.Name(), numBytes, "bytes")
}
//获取所有桶
func ListBuckets() {
result, err := svc.ListBuckets(nil)
if err != nil {
exitErrorf("Unable to list buckets, %v", err)
}
fmt.Println("Buckets:")
for _, b := range result.Buckets {
fmt.Printf("* %s created on %s\n",
aws.StringValue(b.Name), aws.TimeValue(b.CreationDate))
}
for _, b := range result.Buckets {
fmt.Printf("%s\n", aws.StringValue(b.Name))
}
}
func exitErrorf(msg string, args ...interface{
}) {
fmt.Fprintf(os.Stderr, msg+"\n", args...)
os.Exit(1)
}
二、aws-sdk-go-v2
package main
import (
"archive/zip"
"bufio"
"context"
"fmt"
"github.com/aobco/log"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"io"
"os"
"path/filepath"
"strings"
)
var (
ak = "xxxxxxxxxxxxxxx"
sk = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
endpoint = "http://xxx.xxx.xxx:xx/" //endpoint设置，不要动
client *s3.Client
)
func init() {
// 配置AWS SDK
var credentialsProvider aws.CredentialsProvider
credentialsProvider = credentials.StaticCredentialsProvider{
Value: aws.Credentials{
AccessKeyID: ak, SecretAccessKey: sk, SessionToken: ""}}
cfg := aws.Config{
Credentials: credentialsProvider,
EndpointResolverWithOptions: aws.EndpointResolverWithOptionsFunc(
func(service string, region string, options ...interface{
}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: endpoint,
HostnameImmutable: true,
}, nil
}),
}
// 创建S3客户端
client = s3.NewFromConfig(cfg)
}
func main() {
//step1 : download target file
key1 := "log1.txt"
key2 := "log/log2.txt"
bucket1 := "bucket-ziyi"
bucket2 := "bucket-ziyi"
tmpDir := "test/test1/test2/tmp"
DownloadFile(bucket1, key1, fmt.Sprintf("%v/%v", tmpDir, key1))
DownloadFile(bucket2, key2, fmt.Sprintf("%v/%v", tmpDir, key2))
//step2 : compression
zipFileName, err := compressionDir(tmpDir)
if err != nil {
log.Fatal("compression file err=", err)
return
}
//fmt.Println(zipFileName)
//step3 : upload zip
key3 := "demo.zip"
bucket3 := "bucket-ziyi"
UploadFile(bucket3, key3, zipFileName)
}
// UploadFile 上传文件到S3
func UploadFile(bucket string, key string, filePath string) {
// 打开本地文件
file, err := os.Open(filePath)
if err != nil {
log.Info("upload file err=", err)
}
defer file.Close()
// 上传文件到S3 bucket
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: &bucket,
Key: &key,
Body: file,
})
if err != nil {
log.Error("upload file err=", err)
return
}
log.Infof("upload file success, bucket=%v, name=%v", bucket, key)
}
// DownloadFile 下载S3文件到本地
func DownloadFile(bucket string, key string, filePath string) {
// 下载文件内容
resp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
})
if err != nil {
log.Error("download file err=", err)
return
}
defer resp.Body.Close()
index := strings.LastIndex(filePath, "/")
dir := filePath[:index]
err = os.MkdirAll(dir, os.ModePerm)
if err != nil {
log.Error("download file err=", err)
}
// 写入文件
file, err := os.Create(filePath)
if err != nil {
log.Error("download file err=", err)
return
}
defer file.Close()
buffer := make([]byte, 1024*2)
for {
reader := bufio.NewReader(resp.Body)
n, err := reader.Read(buffer)
if err == io.EOF {
break
}
if err != nil {
log.Fatal("download file err=", err)
return
}
file.Write(buffer[:n])
}
log.Info("download file success, filePath=", filePath)
}
func compressionDir(baseDir string) (string, error) {
zipFileName := baseDir + ".zip"
// 创建一个新的 zip 文件
zipFile, err := os.Create(zipFileName)
if err != nil {
return "", err
}
defer zipFile.Close()
// 创建一个 zip.Writer
zipWriter := zip.NewWriter(zipFile)
defer zipWriter.Close()
// 遍历目录下的所有文件和子目录
err = filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// 创建一个 zip 文件中的文件或目录
relativePath := strings.TrimPrefix(path, baseDir)
zipPath := strings.TrimLeft(filepath.Join("/", relativePath), "/")
// 如果是目录或空目录，则在 zip 文件中创建一个目录
if info.IsDir() || isEmptyDir(path) {
_, err := zipWriter.Create(zipPath + "/")
if err != nil {
return err
}
} else {
// 如果是文件，则创建一个 zip 文件中的文件
zipFile, err := zipWriter.Create(zipPath)
if err != nil {
return err
}
// 打开原始文件
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
// 将原始文件的内容拷贝到 zip 文件中
_, err = io.Copy(zipFile, file)
if err != nil {
return err
}
}
return nil
})
if err != nil {
return "", err
}
return zipFileName, nil
}
// 判断目录是否为空目录
func isEmptyDir(dirPath string) bool {
dir, err := os.Open(dirPath)
if err != nil {
return false
}
defer dir.Close()
_, err = dir.Readdirnames(1)
return err == io.EOF
}
① 버킷 디렉토리의 모든 파일을 탐색하여 다운로드
- ak, sk, endpoint 등이 맞는지 확인
ctx := context.TODO()
// 发送 ListBuckets 请求
_, err := client.ListBuckets(ctx, &s3.ListBucketsInput{
})
if err != nil {
fmt.Println("Connection failed:", err)
return
}
fmt.Println("Connection succeeded")
- 먼저 버킷이 존재하는지 확인할 수 있습니다.
// 指定要验证的存储桶名称
bucketName := "bucket-ziyi"
// 创建输入参数
input := &s3.HeadBucketInput{
Bucket: &bucketName,
}
// 发送 HeadBucket 请求
_, err := client.HeadBucket(context.TODO(), input)
if err != nil {
// 如果返回错误，说明存储桶不存在
fmt.Println("Bucket does not exist")
return
}
// 存储桶存在
fmt.Println("Bucket exists")
비즈니스 운영: 버킷 통과 및 다운로드:
var(
ak = ""
sk = ""
endpoint = ""
client *s3.Client
)
func init() {
// 配置AWS SDK
var credentialsProvider aws.CredentialsProvider
credentialsProvider = credentials.StaticCredentialsProvider{
Value: aws.Credentials{
AccessKeyID: ak, SecretAccessKey: sk, SessionToken: ""}}
cfg := aws.Config{
Credentials: credentialsProvider,
EndpointResolverWithOptions: aws.EndpointResolverWithOptionsFunc(
func(service string, region string, options ...interface{
}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: endpoint,
HostnameImmutable: true,
}, nil
}),
}
// 创建S3客户端
client = s3.NewFromConfig(cfg)
}
func main() {
// AWS 配置
bucket := "bucket-ziyi"
prefix := "log/"
// 获取对象列表
resp, err := client.ListObjects(context.TODO(), &s3.ListObjectsInput{
Bucket: &bucket,
Prefix: &prefix,
})
if err != nil {
fmt.Println("Failed to list objects:", err)
return
}
// 创建下载根目录
downloadRootDir := "./download"
if err := os.MkdirAll(downloadRootDir, 0700); err != nil {
fmt.Println("Failed to create download directory:", err)
return
}
keys := make([]string, 0)
// 遍历对象列表并下载
for i, obj := range resp.Contents {
if strings.HasSuffix(*obj.Key, "/") {
continue
}
keys = append(keys, *obj.Key)
// 构建下载路径
//downloadPath := filepath.Join(downloadRootDir, *obj.Key)
downloadPath := filepath.Join(downloadRootDir, *obj.Key)
//fmt.Println("downloadPath=", *obj.Key, i)
// 如果对象是目录，则创建对应的目录
if strings.HasSuffix(*obj.Key, "/") {
if err := os.MkdirAll(downloadPath, 0700); err != nil {
fmt.Printf("Failed to create directory '%s': %v\n", downloadPath, err)
continue
}
fmt.Printf("Created directory '%s'\n", downloadPath)
continue
}
// 下载对象
err := downloadObject(context.TODO(), client, bucket, *obj.Key, downloadPath)
if err != nil {
fmt.Printf("Failed to download object '%s': %v\n", *obj.Key, err)
} else {
fmt.Printf("Downloaded object '%s' to '%s'\n", *obj.Key, downloadPath)
}
}
fmt.Println(keys)
}
// 下载对象
func downloadObject(ctx context.Context, client *s3.Client, bucket, key, filePath string) error {
// 创建文件
file, err := os.Create(filePath)
if err != nil {
return err
}
defer file.Close()
// 下载对象
resp, err := client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
})
if err != nil {
return err
}
defer resp.Body.Close()
// 写入文件
_, err = io.Copy(file, resp.Body)
if err != nil {
return err
}
return nil
}
3. AWS 관련 버그
① 버전 버킷은 원본을 지원하지 않습니다.
해당 버킷에 버전 제어가 없습니다.
4. 업무 관련
4.1 다운로드한 파일 압축하기
파일을 임시 디렉터리에 다운로드한 다음 zip 명령 또는 golang과 함께 제공되는 압축 API를 통해 압축을 풉니다.
- zip -r -compressionLevel xxx.zip *
- compressionLevel의 범위는 0-9이며 0은 압축하지 않음을 의미하고 -r은 디렉토리가 있으면 압축된 디렉토리의 내용에도 해당함을 의미합니다.
//os.TempDir()：获取临时目录
downloadDir := filepath.Join(os.TempDir(), fmt.Sprintf("%d-%d", id1, id2))
注意：golang의 exec.Command()는 와일드카드를 지원할 수 없으므로 *filepath.Glob("*")로 모든 파일을 나열해야 합니다.
zipName := filepath.Base(dest)
command := []string{
"-r",
compressionLevel,
zipName,
}
tmp, err := filepath.Glob("*")
if len(tmp) == 0 {
log.Infof("No matching files found")
}
log.Infof("* tmp=%v", tmp)
command = append(command, tmp...)
if err != nil {
log.Error(err)
}
log.Infof("args=%v", command)
cmd := exec.Command("/usr/bin/zip", command...)
log.Infof("compression cmd=%v", cmd)
err = cmd.Run()
4.2 여러 개의 코루틴 압축 파일을 열고 크기가 초과되면 일시 중지
package main
import (
"context"
"fmt"
"github.com/aobco/log"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"os"
"strings"
"sync"
"sync/atomic"
)
var (
access_key = "xxxxxx"
secret_key = "xxxx"
end_point = "xxxxx/" //endpoint设置，不要动
client *s3.Client
maxWorkers = 3
//keys = []string{"视频/黑鹰坠落.mp4", "视频/多文件打包接口文档.md", "视频/啊1Aa~!@#$%^&()_+{}.txt"}
keys = []string{
"视频/黑鹰坠落.mp4", "视频/多文件打包接口文档.md", "goroutine/docker-wsl-cli.iso"}
fileSizeLimit int64 = 23 * 1024
totalFileSize int64 = 0
)
func init() {
// 配置AWS SDK
var credentialsProvider aws.CredentialsProvider
credentialsProvider = credentials.StaticCredentialsProvider{
Value: aws.Credentials{
AccessKeyID: access_key, SecretAccessKey: secret_key, SessionToken: ""}}
cfg := aws.Config{
Credentials: credentialsProvider,
EndpointResolverWithOptions: aws.EndpointResolverWithOptionsFunc(
func(service string, region string, options ...interface{
}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: end_point,
HostnameImmutable: true,
}, nil
}),
}
// 创建S3客户端
client = s3.NewFromConfig(cfg)
}
func main() {
wg := new(sync.WaitGroup)
bucket := "bucket-ziyi"
path := ""
ctx, cancel := context.WithCancel(context.Background())
for i, key := range keys {
fileName := strings.Split(key, "/")[1]
path = fmt.Sprintf("%s\\%d-%s", "E:\\Go\\GoPro\\src\\go_code\\gouitest\\context\\downloader", i, fileName)
wg.Add(1)
go func(key, path string, cancel context.CancelFunc) {
_, _, err := downLoadFile(ctx, client, bucket, key, path, wg, cancel)
if err != nil {
fmt.Println("err=", err)
}
}(key, path, cancel)
}
//key = "视频/多文件打包接口文档.md"
//timer := time.NewTicker(time.Millisecond * 2000)
//defer timer.Stop()
//go func(timer *time.Ticker, cancel context.CancelFunc) {
// <-timer.C
// cancel()
//}(timer, cancel)
wg.Wait()
fmt.Println("all the task is end...........")
}
func downLoadFile(ctx context.Context, client *s3.Client, bucket, key, path string, wg *sync.WaitGroup, cancel context.CancelFunc) (*os.File, int64, error) {
downloader := manager.NewDownloader(client)
input := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
}
downloadFile, err := os.Create(path)
defer func() {
wg.Done()
if err := downloadFile.Close(); err != nil {
log.Errorf("error when close file %v %v", downloadFile, err)
return
}
}()
downloadSize, err := downloader.Download(ctx, downloadFile, input, func(downloader *manager.Downloader) {
downloader.PartSize = 10 * 1024 * 1024
downloader.Concurrency = 5
})
atomic.AddInt64(&totalFileSize, downloadSize)
if totalFileSize > fileSizeLimit {
cancel()
}
if err != nil {
fmt.Errorf("download file err %v", err)
}
fmt.Println("downloadSize=", downloadSize)
return downloadFile, downloadSize, err
}