command/app: add --no-sign-request global flag to access public bucke…

…ts (peak#305)

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## not released yet
 
+#### Features
+
+- Added global `--no-sign-request` flag. API requests won't be signed and credentials won't be used if this option is provided. It is useful for accessing public buckets. ([#285](https://github.com/peak/s5cmd/issues/285))
+
 #### Improvements
 
 - If retryable errors are received during command execution, users now can see what's happening under the hood. ([#261](https://github.com/peak/s5cmd/pull/261))

command/app.go

@@ -64,6 +64,10 @@ var app = &cli.App{
 			Name:  "stat",
 			Usage: "collect statistics of program execution and display it at the end",
 		},
+		&cli.BoolFlag{
+			Name:  "no-sign-request",
+			Usage: "do not sign requests: credentials will not be loaded if --no-sign-request is provided",
+		},
 	},
 	Before: func(c *cli.Context) error {
 		retryCount := c.Int("retry-count")
@@ -129,10 +133,11 @@ var app = &cli.App{
 // NewStorageOpts creates storage.Options object from the given context.
 func NewStorageOpts(c *cli.Context) storage.Options {
 	return storage.Options{
-		MaxRetries:  c.Int("retry-count"),
-		Endpoint:    c.String("endpoint-url"),
-		NoVerifySSL: c.Bool("no-verify-ssl"),
-		DryRun:      c.Bool("dry-run"),
+		MaxRetries:    c.Int("retry-count"),
+		Endpoint:      c.String("endpoint-url"),
+		NoVerifySSL:   c.Bool("no-verify-ssl"),
+		DryRun:        c.Bool("dry-run"),
+		NoSignRequest: c.Bool("no-sign-request"),
 	}
 }
 

command/cp.go

@@ -47,31 +47,34 @@ Examples:
 	03. Download all S3 objects to a directory
 		 > s5cmd {{.HelpName}} s3://bucket/* target-directory/
 
-	04. Upload a file to S3 bucket
+	04. Download an S3 object from a public bucket
+		 > s5cmd --no-sign-request {{.HelpName}} s3://bucket/prefix/object.gz .
+
+	05. Upload a file to S3 bucket
 		 > s5cmd {{.HelpName}} myfile.gz s3://bucket/
 
-	05. Upload matching files to S3 bucket
+	06. Upload matching files to S3 bucket
 		 > s5cmd {{.HelpName}} dir/*.gz s3://bucket/
 
-	06. Upload all files in a directory to S3 bucket recursively
+	07. Upload all files in a directory to S3 bucket recursively
 		 > s5cmd {{.HelpName}} dir/ s3://bucket/
 
-	07. Copy S3 object to another bucket
+	08. Copy S3 object to another bucket
 		 > s5cmd {{.HelpName}} s3://bucket/object s3://target-bucket/prefix/object
 
-	08. Copy matching S3 objects to another bucket
+	09. Copy matching S3 objects to another bucket
 		 > s5cmd {{.HelpName}} s3://bucket/*.gz s3://target-bucket/prefix/
 
-	09. Copy files in a directory to S3 prefix if not found on target
+	10. Copy files in a directory to S3 prefix if not found on target
 		 > s5cmd {{.HelpName}} -n -s -u dir/ s3://bucket/target-prefix/
 
-	10. Copy files in an S3 prefix to another S3 prefix if not found on target
+	11. Copy files in an S3 prefix to another S3 prefix if not found on target
 		 > s5cmd {{.HelpName}} -n -s -u s3://bucket/source-prefix/* s3://bucket/target-prefix/
 
-	11. Perform KMS Server Side Encryption of the object(s) at the destination
+	12. Perform KMS Server Side Encryption of the object(s) at the destination
 		> s5cmd {{.HelpName}} --sse aws:kms s3://bucket/object s3://target-bucket/prefix/object
 
-	12. Perform KMS-SSE of the object(s) at the destination using customer managed Customer Master Key (CMK) key id
+	13. Perform KMS-SSE of the object(s) at the destination using customer managed Customer Master Key (CMK) key id
 		> s5cmd {{.HelpName}} --sse aws:kms --sse-kms-key-id <your-kms-key-id> s3://bucket/object s3://target-bucket/prefix/object
 `
 

command/ls.go

@@ -36,6 +36,9 @@ Examples:
 
 	4. List all objects that matches a wildcard
 		 > s5cmd {{.HelpName}} s3://bucket/prefix/*/*.gz
+
+	5. List all objects in a public bucket
+		 > s5cmd --no-sign-request {{.HelpName}} s3://bucket/*
 `
 
 var listCommand = &cli.Command{

storage/s3.go

@@ -17,6 +17,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -623,6 +624,11 @@ func (sc *SessionCache) newSession(ctx context.Context, opts Options) (*session.
 
 	awsCfg := aws.NewConfig()
 
+	if opts.NoSignRequest {
+		// do not sign requests when making service API calls
+		awsCfg.Credentials = credentials.AnonymousCredentials
+	}
+
 	endpointURL, err := parseEndpoint(opts.Endpoint)
 	if err != nil {
 		return nil, err

storage/s3_test.go

@@ -108,6 +108,24 @@ func TestNewSessionWithRegionSetViaEnv(t *testing.T) {
 	}
 }
 
+func TestNewSessionWithNoSignRequest(t *testing.T) {
+	globalSessionCache.clear()
+
+	sess, err := globalSessionCache.newSession(context.Background(), Options{
+		NoSignRequest: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	got := sess.Config.Credentials
+	expected := credentials.AnonymousCredentials
+
+	if expected != got {
+		t.Fatalf("expected %v, got %v", expected, got)
+	}
+}
+
 func TestS3ListURL(t *testing.T) {
 	url, err := url.New("s3://bucket/key")
 	if err != nil {

storage/storage.go

@@ -48,11 +48,12 @@ func NewLocalClient(opts Options) *Filesystem {
 
 func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, error) {
 	newOpts := Options{
-		MaxRetries:  opts.MaxRetries,
-		Endpoint:    opts.Endpoint,
-		NoVerifySSL: opts.NoVerifySSL,
-		DryRun:      opts.DryRun,
-		bucket:      url.Bucket,
+		MaxRetries:    opts.MaxRetries,
+		Endpoint:      opts.Endpoint,
+		NoVerifySSL:   opts.NoVerifySSL,
+		DryRun:        opts.DryRun,
+		NoSignRequest: opts.NoSignRequest,
+		bucket:        url.Bucket,
 	}
 	return newS3Storage(ctx, newOpts)
 }
@@ -66,11 +67,12 @@ func NewClient(ctx context.Context, url *url.URL, opts Options) (Storage, error)
 
 // Options stores configuration for storage.
 type Options struct {
-	MaxRetries  int
-	Endpoint    string
-	NoVerifySSL bool
-	DryRun      bool
-	bucket      string
+	MaxRetries    int
+	Endpoint      string
+	NoVerifySSL   bool
+	DryRun        bool
+	NoSignRequest bool
+	bucket        string
 }
 
 // Object is a generic type which contains metadata for storage items.