storage: add profile flag (peak#453)

This commit adds --profile flag to allow users to specify a named profile rather than using the default.

Fixes peak#353

CHANGELOG.md

@@ -2,11 +2,14 @@
 
 ## not released yet
 
+#### Features
+- Added `--profile` flag to allow users to specify a [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html). ([#353](https://github.com/peak/s5cmd/issues/353))
+- Added `--credentials-file` flag to allow users to specify path for the AWS credentials file instead of using the [default location](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html#cli-configure-files-where). 
+
 #### Bugfixes
 - Fixed a bug where (`--stat`) prints unnecessarily when used with help and version commands ([#452](https://github.com/peak/s5cmd/issues/452))
 - Changed cp error message to be more precise. "given object not found" error message now will also include absolute path of the file. ([#463](https://github.com/peak/s5cmd/pull/463))
 
-
 #### Improvements
 - Disable AWS SDK logger if log level is not "trace"
 

README.md

@@ -413,7 +413,7 @@ s5cmd --use-list-objects-v1 ls s3://bucket/
 
 `s5cmd` uses official AWS SDK to access S3. SDK requires credentials to sign
 requests to AWS. Credentials can be provided in a variety of ways:
-
+- Command line options `--profile` to use a [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html), `--credentials-file` flag to use the specified credentials file, and `--no-sign-request` to send requests anonymously
 - Environment variables
 - AWS credentials file, including profile selection via `AWS_PROFILE` environment
   variable

command/app.go

@@ -81,6 +81,14 @@ var app = &cli.App{
 			Name:  "request-payer",
 			Usage: "who pays for request (access requester pays buckets)",
 		},
+		&cli.StringFlag{
+			Name:  "profile",
+			Usage: "use the specified profile from the credentials file",
+		},
+		&cli.StringFlag{
+			Name:  "credentials-file",
+			Usage: "use the specified credentials file instead of the default credentials file",
+		},
 	},
 	Before: func(c *cli.Context) error {
 		retryCount := c.Int("retry-count")
@@ -97,6 +105,16 @@ var app = &cli.App{
 			printError(commandFromContext(c), c.Command.Name, err)
 			return err
 		}
+		if c.Bool("no-sign-request") && c.String("profile") != "" {
+			err := fmt.Errorf(`"no-sign-request" and "profile" flags cannot be used together`)
+			printError(commandFromContext(c), c.Command.Name, err)
+			return err
+		}
+		if c.Bool("no-sign-request") && c.String("credentials-file") != "" {
+			err := fmt.Errorf(`"no-sign-request" and "credentials-file" flags cannot be used together`)
+			printError(commandFromContext(c), c.Command.Name, err)
+			return err
+		}
 
 		if isStat {
 			stat.InitStat()
@@ -162,6 +180,8 @@ func NewStorageOpts(c *cli.Context) storage.Options {
 		NoVerifySSL:      c.Bool("no-verify-ssl"),
 		RequestPayer:     c.String("request-payer"),
 		UseListObjectsV1: c.Bool("use-list-objects-v1"),
+		Profile:          c.String("profile"),
+		CredentialFile:   c.String("credentials-file"),
 		LogLevel:         log.LevelFromString(c.String("log")),
 	}
 }

storage/s3.go

@@ -770,7 +770,11 @@ func (sc *SessionCache) newSession(ctx context.Context, opts Options) (*session.
 
 	if opts.NoSignRequest {
 		// do not sign requests when making service API calls
-		awsCfg.Credentials = credentials.AnonymousCredentials
+		awsCfg = awsCfg.WithCredentials(credentials.AnonymousCredentials)
+	} else if opts.CredentialFile != "" || opts.Profile != "" {
+		awsCfg = awsCfg.WithCredentials(
+			credentials.NewSharedCredentials(opts.CredentialFile, opts.Profile),
+		)
 	}
 
 	endpointURL, err := parseEndpoint(opts.Endpoint)

storage/s3_test.go

@@ -128,6 +128,88 @@ func TestNewSessionWithNoSignRequest(t *testing.T) {
 	}
 }
 
+func TestNewSessionWithProfileFromFile(t *testing.T) {
+	// create a temporary credentials file
+	file, err := os.CreateTemp("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(file.Name())
+
+	profiles := `[default]
+aws_access_key_id = default_profile_key_id
+aws_secret_access_key = default_profile_access_key
+
+[p1]
+aws_access_key_id = p1_profile_key_id
+aws_secret_access_key = p1_profile_access_key
+
+[p2]
+aws_access_key_id = p2_profile_key_id
+aws_secret_access_key = p2_profile_access_key`
+
+	_, err = file.Write([]byte(profiles))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testcases := []struct {
+		name               string
+		fileName           string
+		profileName        string
+		expAccessKeyId     string
+		expSecretAccessKey string
+	}{
+		{
+			name:               "use default profile",
+			fileName:           file.Name(),
+			profileName:        "",
+			expAccessKeyId:     "default_profile_key_id",
+			expSecretAccessKey: "default_profile_access_key",
+		},
+		{
+			name:               "use a non-default profile",
+			fileName:           file.Name(),
+			profileName:        "p1",
+			expAccessKeyId:     "p1_profile_key_id",
+			expSecretAccessKey: "p1_profile_access_key",
+		},
+		{
+
+			name:               "use a non-existent profile",
+			fileName:           file.Name(),
+			profileName:        "non-existent-profile",
+			expAccessKeyId:     "",
+			expSecretAccessKey: "",
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			globalSessionCache.clear()
+			sess, err := globalSessionCache.newSession(context.Background(), Options{
+				Profile:        tc.profileName,
+				CredentialFile: tc.fileName,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			got, err := sess.Config.Credentials.Get()
+			if err != nil {
+				// if there should be such a profile but received an error fail,
+				// ignore the error otherwise.
+				if tc.expAccessKeyId != "" || tc.expSecretAccessKey != "" {
+					t.Fatal(err)
+				}
+			}
+
+			if got.AccessKeyID != tc.expAccessKeyId || got.SecretAccessKey != tc.expSecretAccessKey {
+				t.Errorf("Expected credentials does not match the credential we got!\nExpected: Access Key ID: %v, Secret Access Key: %v\nGot    : Access Key ID: %v, Secret Access Key: %v\n", tc.expAccessKeyId, tc.expSecretAccessKey, got.AccessKeyID, got.SecretAccessKey)
+			}
+		})
+	}
+}
+
 func TestS3ListURL(t *testing.T) {
 	url, err := url.New("s3://bucket/key")
 	if err != nil {

storage/storage.go

@@ -63,6 +63,8 @@ func NewRemoteClient(ctx context.Context, url *url.URL, opts Options) (*S3, erro
 		NoSignRequest:    opts.NoSignRequest,
 		UseListObjectsV1: opts.UseListObjectsV1,
 		RequestPayer:     opts.RequestPayer,
+		Profile:          opts.Profile,
+		CredentialFile:   opts.CredentialFile,
 		LogLevel:         opts.LogLevel,
 		bucket:           url.Bucket,
 		region:           opts.region,
@@ -87,6 +89,8 @@ type Options struct {
 	UseListObjectsV1 bool
 	LogLevel         log.LogLevel
 	RequestPayer     string
+	Profile          string
+	CredentialFile   string
 	bucket           string
 	region           string
 }