使用Localstack的Golang的一个简单AWS S3例子

在这个例子中，我们将使用Localstack和Golang来与AWS简单存储服务（S3）合作。我们将创建一个新的存储桶，向存储桶上传一个对象，从存储桶中下载一个对象，从存储桶中删除一个对象，并在存储桶中列出对象。

结构

├── assets
│   ├── id.txt
│   └── logo.png
├── internal
│   ├── bucket
│   │   └── bucket.go
│   └── pkg
│       └── cloud
│           ├── aws
│           │   ├── aws.go
│           │   └── s3.go
│           ├── client.go
│           └── model.go
├── main.go
└── tmp

文件

main.go

package main

import (
	"log"
	"time"

	"github.com/you/aws/internal/bucket"
	"github.com/you/aws/internal/pkg/cloud/aws"
)

func main() {
	// Create a session instance.
	ses, err := aws.New(aws.Config{
		Address: "http://localhost:4566",
		Region:  "eu-west-1",
		Profile: "localstack",
		ID:      "test",
		Secret:  "test",
	})
	if err != nil {
		log.Fatalln(err)
	}

	// Test bucket
	bucket.Bucket(aws.NewS3(ses, time.Second*5))
}

client.go

package cloud

import (
	"context"
	"io"
)

type BucketClient interface {
	// Creates a new bucket.
	Create(ctx context.Context, bucket string) error
	// Upload a new object to a bucket and returns its URL to view/download.
	UploadObject(ctx context.Context, bucket, fileName string, body io.Reader) (string, error)
	// Downloads an existing object from a bucket.
	DownloadObject(ctx context.Context, bucket, fileName string, body io.WriterAt) error
	// Deletes an existing object from a bucket.
	DeleteObject(ctx context.Context, bucket, fileName string) error
	// Lists all objects in a bucket.
	ListObjects(ctx context.Context, bucket string) ([]*Object, error)
	// Returns an object from bucket for reading.
	FetchObject(ctx context.Context, bucket, fileName string) (io.ReadCloser, error)
}

model.go

package cloud

import "time"

type Object struct {
	Key        string
	Size       int64
	ModifiedAt time.Time
}

aws.go

package aws

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
)

type Config struct {
	Address string
	Region  string
	Profile string
	ID      string
	Secret  string
}

func New(config Config) (*session.Session, error) {
	return session.NewSessionWithOptions(
		session.Options{
			Config: aws.Config{
				Credentials:      credentials.NewStaticCredentials(config.ID, config.Secret, ""),
				Region:           aws.String(config.Region),
				Endpoint:         aws.String(config.Address),
				S3ForcePathStyle: aws.Bool(true),
			},
			Profile: config.Profile,
		},
	)
}

s3.go

package aws

import (
	"context"
	"fmt"
	"io"
	"time"

	"github.com/you/aws/internal/pkg/cloud"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

var _ cloud.BucketClient = S3{}

type S3 struct {
	timeout    time.Duration
	client     *s3.S3
	uploader   *s3manager.Uploader
	downloader *s3manager.Downloader
}

func NewS3(session *session.Session, timeout time.Duration) S3 {
	s3manager.NewUploader(session)
	return S3{
		timeout:    timeout,
		client:     s3.New(session),
		uploader:   s3manager.NewUploader(session),
		downloader: s3manager.NewDownloader(session),
	}
}

func (s S3) Create(ctx context.Context, bucket string) error {
	ctx, cancel := context.WithTimeout(ctx, s.timeout)
	defer cancel()

	if _, err := s.client.CreateBucketWithContext(ctx, &s3.CreateBucketInput{
		Bucket: aws.String(bucket),
	}); err != nil {
		return fmt.Errorf("create: %w", err)
	}

	if err := s.client.WaitUntilBucketExists(&s3.HeadBucketInput{
		Bucket: aws.String(bucket),
	}); err != nil {
		return fmt.Errorf("wait: %w", err)
	}

	return nil
}

func (s S3) UploadObject(ctx context.Context, bucket, fileName string, body io.Reader) (string, error) {
	ctx, cancel := context.WithTimeout(ctx, s.timeout)
	defer cancel()

	res, err := s.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
		Body:   body,
		Bucket: aws.String(bucket),
		Key:    aws.String(fileName),
	})
	if err != nil {
		return "", fmt.Errorf("upload: %w", err)
	}

	return res.Location, nil
}

func (s S3) DownloadObject(ctx context.Context, bucket, fileName string, body io.WriterAt) error {
	if _, err := s.downloader.DownloadWithContext(ctx, body, &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(fileName),
	}); err != nil {
		return fmt.Errorf("download: %w", err)
	}

	return nil
}

func (s S3) DeleteObject(ctx context.Context, bucket, fileName string) error {
	if _, err := s.client.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(fileName),
	}); err != nil {
		return fmt.Errorf("delete: %w", err)
	}

	if err := s.client.WaitUntilObjectNotExists(&s3.HeadObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(fileName),
	}); err != nil {
		return fmt.Errorf("wait: %w", err)
	}

	return nil
}

func (s S3) ListObjects(ctx context.Context, bucket string) ([]*cloud.Object, error) {
	ctx, cancel := context.WithTimeout(ctx, s.timeout)
	defer cancel()

	res, err := s.client.ListObjectsV2WithContext(ctx, &s3.ListObjectsV2Input{
		Bucket: aws.String(bucket),
	})
	if err != nil {
		return nil, fmt.Errorf("list: %w", err)
	}

	objects := make([]*cloud.Object, len(res.Contents))

	for i, object := range res.Contents {
		objects[i] = &cloud.Object{
			Key:        *object.Key,
			Size:       *object.Size,
			ModifiedAt: *object.LastModified,
		}
	}

	return objects, nil
}

func (s S3) FetchObject(ctx context.Context, bucket, fileName string) (io.ReadCloser, error) {
	ctx, cancel := context.WithTimeout(ctx, s.timeout)
	defer cancel()

	res, err := s.client.GetObjectWithContext(ctx, &s3.GetObjectInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(fileName),
	})
	if err != nil {
		return nil, err
	}

	return res.Body, nil
}

bucket.go

这只是一个 "肮脏 "的使用例子!

package bucket

import (
	"context"
	"fmt"
	"log"
	"os"

	"github.com/you/aws/internal/pkg/cloud"
)

func Bucket(client cloud.BucketClient) {
	ctx := context.Background()

	create(ctx, client)
	uploadObject(ctx, client)
	downloadObject(ctx, client)
	deleteObject(ctx, client)
	listObjects(ctx, client)
}

func create(ctx context.Context, client cloud.BucketClient) {
	if err := client.Create(ctx, "aws-test"); err != nil {
		log.Fatalln(err)
	}
	log.Println("create: ok")
}

func uploadObject(ctx context.Context, client cloud.BucketClient) {
	file, err := os.Open("./assets/id.txt")
	if err != nil {
		log.Fatalln(err)
	}
	defer file.Close()

	url, err := client.UploadObject(ctx, "aws-test", "id.txt", file)
	if err != nil {
		log.Fatalln(err)
	}
	log.Println("upload object:", url)
}

func downloadObject(ctx context.Context, client cloud.BucketClient) {
	file, err := os.Create("./tmp/id.txt")
	if err != nil {
		log.Fatalln(err)
	}
	defer file.Close()

	if err := client.DownloadObject(ctx, "aws-test", "id.txt", file); err != nil {
		log.Fatalln(err)
	}
	log.Println("download object: ok")
}

func deleteObject(ctx context.Context, client cloud.BucketClient) {
	if err := client.DeleteObject(ctx, "aws-test", "id.txt"); err != nil {
		log.Fatalln(err)
	}
	log.Println("delete object: ok")
}

func listObjects(ctx context.Context, client cloud.BucketClient) {
	objects, err := client.ListObjects(ctx, "aws-test")
	if err != nil {
		log.Fatalln(err)
	}
	log.Println("list objects:")
	for _, object := range objects {
		fmt.Printf("%+v\n", object)
	}
}

测试

显然，"列表对象 "部分将是空的，因为你删除了文件，但我手动在那里添加了一些东西作为示范：

$ go run --race main.go
2021/01/23 21:35:59 create: ok
2021/01/23 21:35:59 upload object: http://localhost:4566/aws-test/id.txt
2021/01/23 21:35:59 download object: ok
2021/01/23 21:35:59 delete object: ok
2021/01/23 21:35:59 list objects:
&{Key:id.txt Size:23 ModifiedAt:2021-01-23 21:36:30 +0000 UTC}