Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add mb (make-bucket) command #86

Merged
merged 3 commits into from
Mar 4, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Resolves #25
- added mb (make bucket) command
  • Loading branch information
ilkinulas committed Mar 3, 2020
commit c65912f7caa2673fdd3e31ec7fb2a2ae5a85a35c
2 changes: 2 additions & 0 deletions core/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,8 @@ var Commands = []CommandMap{

{"du", op.Size, []opt.ParamType{opt.S3ObjOrDir}, noOpts},
{"du", op.Size, []opt.ParamType{opt.S3WildObj}, noOpts},

{"mb", op.MakeBucket, []opt.ParamType{opt.S3Bucket}, noOpts},
}

// String formats the CommandMap using its Operation and ParamTypes
Expand Down
1 change: 1 addition & 0 deletions core/commands.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,5 @@ var globalCmdRegistry = map[op.Operation]commandFunc{
op.List: List,
op.ListBuckets: ListBuckets,
op.Size: Size,
op.MakeBucket: MakeBucket,
}
16 changes: 16 additions & 0 deletions core/job_exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -324,3 +324,19 @@ func Size(ctx context.Context, job *Job) *JobResponse {

return jobResponse(err)
}

func MakeBucket(ctx context.Context, job *Job) *JobResponse {
bucket := job.args[0]

client, err := storage.NewClient(bucket)
if err != nil {
return jobResponse(err)
}

err = client.MakeBucket(ctx, bucket.Bucket)
if err != nil {
return jobResponse(err)
}
log.Logger.Success(fmt.Sprintf("Bucket '%s' created successfully.", bucket))
igungor marked this conversation as resolved.
Show resolved Hide resolved
return jobResponse(nil)
}
6 changes: 5 additions & 1 deletion core/parse.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func parseArgumentByType(s string, t opt.ParamType, fnObj *objurl.ObjectURL) (*o
case opt.Unchecked, opt.UncheckedOneOrMore:
return objurl.New(s)

case opt.S3Obj, opt.S3ObjOrDir, opt.S3WildObj, opt.S3Dir, opt.S3SimpleObj:
case opt.S3Bucket, opt.S3Obj, opt.S3ObjOrDir, opt.S3WildObj, opt.S3Dir, opt.S3SimpleObj:
url, err := objurl.New(s)
if err != nil {
return nil, err
Expand All @@ -44,6 +44,10 @@ func parseArgumentByType(s string, t opt.ParamType, fnObj *objurl.ObjectURL) (*o

s = url.Absolute()

if t == opt.S3Bucket && !url.IsBucket() {
return nil, errors.New("invalid s3 bucket")
}

if (t == opt.S3Obj || t == opt.S3ObjOrDir || t == opt.S3SimpleObj) && objurl.HasGlobCharacter(url.Path) {
return nil, errors.New("s3 key cannot contain wildcards")
}
Expand Down
38 changes: 38 additions & 0 deletions e2e/mb_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package e2e

import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"gotest.tools/v3/icmd"
"testing"
igungor marked this conversation as resolved.
Show resolved Hide resolved
)

func Test_MakeBucket_success(t *testing.T) {
t.Parallel()
s3client, s5cmd, cleanup := setup(t)
defer cleanup()

bucketName := "test-bucket"
cmd := s5cmd(fmt.Sprintf("mb s3://%s", bucketName))
result := icmd.RunCmd(cmd)

result.Assert(t, icmd.Success)

_, err := s3client.HeadBucket(&s3.HeadBucketInput{Bucket: aws.String(bucketName)})
if err != nil {
t.Errorf("unexpected error %v", err)
}
}

func Test_MakeBucket_failure(t *testing.T) {
t.Parallel()
_, s5cmd, cleanup := setup(t)
defer cleanup()

bucketName := "invalid/bucket/name"
cmd := s5cmd(fmt.Sprintf("mb s3://%s", bucketName))
result := icmd.RunCmd(cmd)

result.Assert(t, icmd.Expected{ExitCode: 127})
}
5 changes: 5 additions & 0 deletions objurl/objurl.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,11 @@ func (o *ObjectURL) IsRemote() bool {
return o.Type == remoteObject
}

// IsBucket returns true if the object url contains only bucket name
func (o *ObjectURL) IsBucket() bool {
return o.IsRemote() && o.Path == ""
}

// Absolute returns the absolute URL format of the object.
func (o *ObjectURL) Absolute() string {
if !o.IsRemote() {
Expand Down
32 changes: 32 additions & 0 deletions objurl/objurl_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -364,3 +364,35 @@ func Test_parseNonBatch(t *testing.T) {
})
}
}

func TestObjectURL_IsBucket(t *testing.T) {
tests := []struct {
input string
want bool
want_error bool
}{
{"s3://bucket", true, false},
{"s3://bucket/file", false, false},
{"bucket", false, false},
{"s3://", false, true},
}
for _, tc := range tests {
url, err := New(tc.input)
if tc.want_error && err != nil {
continue
}

if tc.want_error && err == nil {
t.Errorf("expecting error for input %s", tc.input)
}

if err != nil {
t.Errorf("unexpected error: %v for input %s", err, tc.input)
continue
}

if url.IsBucket() != tc.want {
t.Errorf("isBucket should return %v for %s", tc.want, tc.input)
}
}
}
6 changes: 5 additions & 1 deletion op/op.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package op

import (
"fmt"

"github.com/peak/s5cmd/opt"
"github.com/peak/s5cmd/stats"
)
Expand All @@ -29,6 +28,7 @@ const (
LocalCopy // Copy from local to local
LocalDelete // Delete local file
AliasGet // Alias for Download
MakeBucket // Make Bucket
)

var batchOperations = []Operation{
Expand Down Expand Up @@ -106,6 +106,8 @@ func (o Operation) String() string {
return "get"
case AliasBatchGet:
return "batch-get"
case MakeBucket:
return "make-bucket"
}

return fmt.Sprintf("Unknown:%d", o)
Expand Down Expand Up @@ -166,6 +168,8 @@ func (o Operation) Describe(l opt.OptionList) string {
return "Batch copy local files"
case LocalDelete:
return "Delete local files"
case MakeBucket:
return "Creates an S3 bucket"
}

return fmt.Sprintf("Unknown:%d", o)
Expand Down
3 changes: 3 additions & 0 deletions opt/opt.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ type ParamType int
const (
Unchecked ParamType = iota // Arbitrary single parameter
UncheckedOneOrMore // One or more arbitrary parameters (special case)
S3Bucket //Bucket
igungor marked this conversation as resolved.
Show resolved Hide resolved
S3Obj // Bucket or bucket + key
S3Dir // Bucket or bucket + key + "/" (prefix)
S3ObjOrDir // Bucket or bucket + key [+ "/"]
Expand All @@ -181,6 +182,8 @@ func (p ParamType) String() string {
return "param"
case UncheckedOneOrMore:
return "param..."
case S3Bucket:
return "s3://bucket"
case S3Obj:
return "s3://bucket[/object]"
case S3SimpleObj:
Expand Down
4 changes: 4 additions & 0 deletions storage/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,10 @@ func (f *Filesystem) ListBuckets(_ context.Context, _ string) ([]Bucket, error)
return nil, f.notimplemented("ListBuckets")
}

func (f *Filesystem) MakeBucket(_ context.Context, _ string) error {
return f.notimplemented("MakeBucket")
}

func (f *Filesystem) UpdateRegion(_ string) error {
return f.notimplemented("UpdateRegion")
}
Expand Down
12 changes: 12 additions & 0 deletions storage/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,18 @@ func (s *S3) ListBuckets(ctx context.Context, prefix string) ([]Bucket, error) {
return buckets, nil
}

// MakeBucket creates an S3 bucket with the given name.
func (s *S3) MakeBucket(ctx context.Context, name string) error {
o, err := s.api.CreateBucketWithContext(ctx, &s3.CreateBucketInput{
Bucket: aws.String(name),
})
if err != nil {
igungor marked this conversation as resolved.
Show resolved Hide resolved
return err
}
fmt.Printf("make bucket output %v\n", o)
igungor marked this conversation as resolved.
Show resolved Hide resolved
return nil
}

// UpdateRegion overrides AWS session with the region of given bucket.
func (s *S3) UpdateRegion(bucket string) error {
o, err := s.api.GetBucketLocation(&s3.GetBucketLocationInput{
Expand Down
1 change: 1 addition & 0 deletions storage/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ type Storage interface {
Delete(context.Context, *objurl.ObjectURL) error
MultiDelete(context.Context, <-chan *objurl.ObjectURL) <-chan *Object
ListBuckets(context.Context, string) ([]Bucket, error)
MakeBucket(context.Context, string) error
UpdateRegion(string) error
}

Expand Down