Skip to content

Commit

Permalink
example/service/s3: add example for file upload with progress (aws#1868)
Browse files Browse the repository at this point in the history
Adds example for tracking file upload progress.
  • Loading branch information
xinst authored and xibz committed Sep 10, 2018
1 parent 006d83b commit 39f2521
Show file tree
Hide file tree
Showing 2 changed files with 116 additions and 0 deletions.
14 changes: 14 additions & 0 deletions example/service/s3/putObjectWithProcess/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Example

This is an example using the AWS SDK for Go to upload object with progress.
We use CustomReader to implement it


# Usage

The example uses the bucket name provided, one key for object, and output the progress to stdout.
The Object size should larger than 5M or your will not see the progress

```sh
AWS_REGION=<region> go run putObjWithProcess.go <credential> <bucket> <key for object> <local file name>
```
102 changes: 102 additions & 0 deletions example/service/s3/putObjectWithProcess/putObjWithProcess.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
// +build example

package main

import (
"log"
"os"
"sync/atomic"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

type CustomReader struct {
fp *os.File
size int64
read int64
}

func (r *CustomReader) Read(p []byte) (int, error) {
return r.fp.Read(p)
}

func (r *CustomReader) ReadAt(p []byte, off int64) (int, error) {
n, err := r.fp.ReadAt(p, off)
if err != nil {
return n, err
}

// Got the length have read( or means has uploaded), and you can construct your message
atomic.AddInt64(&r.read, int64(n))

// I have no idea why the read length need to be div 2,
// maybe the request read once when Sign and actually send call ReadAt again
// It works for me
log.Printf("total read:%d progress:%d%%\n", r.read/2, int(float32(r.read*100/2)/float32(r.size)))

return n, err
}

func (r *CustomReader) Seek(offset int64, whence int) (int64, error) {
return r.fp.Seek(offset, whence)
}

func main() {
if len(os.Args) < 4 {
log.Println("USAGE ERROR: AWS_REGION=us-east-1 go run putObjWithProcess.go <credential> <bucket> <key for object> <local file name>")
return
}

credential := os.Args[1]
bucket := os.Args[2]
key := os.Args[3]
fileName := os.Args[4]

creds := credentials.NewSharedCredentials(credential, "default")
if _, err := creds.Get(); err != nil {
log.Println("ERROR:", err)
return
}

sess := session.New(&aws.Config{
Credentials: creds,
})

file, err := os.Open(fileName)
if err != nil {
log.Println("ERROR:", err)
return
}

fileInfo, err := file.Stat()
if err != nil {
log.Println("ERROR:", err)
return
}

reader := &CustomReader{
fp: file,
size: fileInfo.Size(),
}

uploader := s3manager.NewUploader(sess, func(u *s3manager.Uploader) {
u.PartSize = 5 * 1024 * 1024
u.LeavePartsOnError = true
})

output, err := uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: reader,
})

if err != nil {
log.Println("ERROR:", err)
return
}

log.Println(output.Location)
}

0 comments on commit 39f2521

Please sign in to comment.