Try using S3 object upload and download with AWS SDK for Go v2

Overview

Prerequisites

Go version is v1.15

$ go version
go version go1.15 linux/amd64

Preparation

go mod init awssdkapp

go get -u github.com/aws/aws-sdk-go-v2
go get -u github.com/aws/aws-sdk-go-v2/config
go get -u github.com/aws/aws-sdk-go-v2/feature/s3/manager
go get -u github.com/aws/aws-sdk-go-v2/service/s3
touch main.go
mkdir myclient
touch myclient/myclient.go

Client code to connect to S3

Simply upload or download objects.

myclient/myclient.go


package myclient

import (
	"context"
	"io"
	"log"
	"os"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
	"github.com/aws/aws-sdk-go-v2/service/s3"
)

type MyS3Client struct {
	downloader *manager.Downloader
	uploader   *manager.Uploader
	client     *s3.Client
}

func NewMyS3Client(cfg aws.Config) *MyS3Client {
	client := s3.NewFromConfig(cfg)
	downloader := manager.NewDownloader(client)
	uploader := manager.NewUploader(client)

	return &MyS3Client{
		downloader: downloader,
		uploader:   uploader,
		client:     client,
	}
}

//Method to download object
func (c *MyS3Client) UploadSingleObject(bucket, key string, reader io.Reader) {
	_, err := c.uploader.Upload(context.Background(), &s3.PutObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
		Body:   reader,
	})

	if err != nil {
		log.Fatal(err)
	}

	log.Println("upload successed")
}

//Method to download object
func (c *MyS3Client) DownloadSingleObject(bucket, key, filename string) {
	file, _ := os.Create(filename)
	defer file.Close()

	_, err := c.downloader.Download(context.Background(), file, &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(key),
	})

	if err != nil {
		log.Fatal(err)
	}

	log.Println("download successed")

}

main.go


package main

import (
	"context"
	"flag"
	"log"
	"strings"

	"awsdkapp/myclient"

	"github.com/aws/aws-sdk-go-v2/config"
)

func main() {

	bucket := flag.String("bucket", "", "specify bucket name")
	key := flag.String("key", "", "specify key name ")
	flag.Parse()

	if *bucket == "" || *key == "" {
		log.Fatal("error invalid arugments")
	}

	//Creating a config
	cfg, err := config.LoadDefaultConfig(context.Background())
	if err != nil {
		log.Fatal(err)
	}

	client := myclient.NewMyS3Client(cfg)

	//Upload content
	client.UploadSingleObject(*bucket, *key, strings.NewReader("hello, aws sdk"))

	//Download as a file
	client.DownloadSingleObject(*bucket, *key, "hello.txt")
}

If you try it, you should be able to upload/download it.

$ go run main.go -bucket the-chocomintkusoyaro-demo -key hoge
2020/12/30 23:05:32 upload successed
2020/12/30 23:05:32 download successed

$ ls hello.txt 
hello.txt

$ cat hello.txt 
hello, aws sdk

When downloading multiple objects

Generate a pagenator with NewListObjectsV2Paginator, get the object group for each page, and download and write each object.

myclient/myclient.go


...

//Download multiple objects
func (c *MyS3Client) DownloadMultipleObjects(bucket, prefix, localDir string) {

	//Create a page nator
	paginator := s3.NewListObjectsV2Paginator(c.client, &s3.ListObjectsV2Input{
		Bucket: aws.String(bucket),
		Prefix: aws.String(prefix),
	})

	for paginator.HasMorePages() {
		page, err := paginator.NextPage(context.Background())
		if err != nil {
			log.Fatal(err)
		}

		for _, obj := range page.Contents {
			if err != nil {
				log.Fatal(err)
			}

			filepath := filepath.Join(localDir, *obj.Key)

			c.DownloadSingleObject(bucket, *obj.Key, filepath)
		}
	}
	log.Println("all file downloaded")
}

main.go


package main

import (
	"context"
	"flag"
	"log"

	"awsdkapp/myclient"

	"github.com/aws/aws-sdk-go-v2/config"
)

func main() {

	bucket := flag.String("bucket", "", "specify bucket name")
	prefix := flag.String("prefix", "", "specify prefix name")
	localDir := flag.String("localDir", "", "specify local dir ")
	flag.Parse()

	if *bucket == "" || *localDir == "" {
		log.Fatal("error invalid arugments")
	}

	//Creating a config
	cfg, err := config.LoadDefaultConfig(context.Background())
	if err != nil {
		log.Fatal(err)
	}

	client := myclient.NewMyS3Client(cfg)

	//Download the object
	client.DownloadMultipleObjects(*bucket, *prefix, *localDir)
}

Upload multiple objects

I tried to imitate the Example of the document.

myclient/myclient.go


...

//Upload multiple objects at once
func (c *MyS3Client) UploadMultipleObjects(bucket, prefix, localDir string) {
	fileWalker := make(chan string)

	//Send the local file path to the channel
	go func() {
		err := filepath.Walk(localDir, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}

			if !info.IsDir() {
				//Send file path to channel
				fileWalker <- path
			}

			return nil
		})

		if err != nil {
			log.Fatal(err)
		}

		
		//Notify channel closure
		close(fileWalker)
	}()

	//Receive file path from channel
	for path := range fileWalker {
		rel, err := filepath.Rel(localDir, path)

		if err != nil {
			log.Println("unable to get relative path", path, err)
			continue
		}

		file, err := os.Open(path)
		if err != nil {
			log.Println("failed open file", path, err)
			continue
		}
		defer file.Close()

		key := filepath.Join(prefix, rel)

		//Download the object
		c.UploadSingleObject(bucket, key, file)
	}
}

main.go


package main

import (
	"context"
	"flag"
	"log"

	"awsdkapp/myclient"

	"github.com/aws/aws-sdk-go-v2/config"
)

func main() {

	bucket := flag.String("bucket", "", "specify bucket name")
	prefix := flag.String("prefix", "", "specify prefix name")
	localDir := flag.String("localDir", "", "specify local dir ")
	flag.Parse()

	if *bucket == "" || *localDir == "" {
		log.Fatal("error invalid arugments")
	}

	//Creating a config
	cfg, err := config.LoadDefaultConfig(context.Background())
	if err != nil {
		log.Fatal(err)
	}

	client := myclient.NewMyS3Client(cfg)

	//Upload objects in bulk
	client.UploadMultipleObjects(*bucket, *prefix, *localDir)
}

Recommended Posts

Try using S3 object upload and download with AWS SDK for Go v2
Try using Dropbox API v2 with Go
[Cloudian # 1] Try to access object storage with AWS SDK for Python (boto3)
Fleet provisioning with AWS IoT SDK for Python v2
Using Lambda with AWS Amplify with Go
Upload and download images with falcon
Let's try gRPC with Go and Docker
[AWS] Link Lambda and S3 with boto3
Join Azure Using Go ~ For those who want to start and know Azure with Go ~
Upload images to S3 with GUI using tkinter
Learn Service Bus Queue with Azure SDK for Go (1)
Try a similar search for Image Search using the Python SDK [Search]
Try MD simulation with ANN potential using aenet and ASE
Try automating Start / Stop for EC2 instances with AWS Lambda
This and that for using Step Functions with CDK + Python
Upload files to Aspera that comes with IBM Cloud Object Storage (ICOS) using SDK (Python version)