Skip to content

Commit

Permalink
command: add arbitrary metadata support to cp and pipe (peak#621)
Browse files Browse the repository at this point in the history 
Co-authored-by: İbrahim Güngör <[email protected]>
  • Loading branch information
@denizsurmeli @igungor
denizsurmeli and igungor committed Aug 21, 2023
1 parent b7babaf commit db6b53f
Show file tree
Hide file tree
Showing 10 changed files with 450 additions and 153 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
- Added `--show-fullpath` flag to `ls`. ([#596](https://github.com/peak/s5cmd/issues/596))
- Added `pipe` command. ([#182](https://github.com/peak/s5cmd/issues/182))
- Added `--show-progress` flag to `cp` to show a progress bar. ([#51](https://github.com/peak/s5cmd/issues/51))
- Added `--metadata` flag to `cp` and `pipe` to set arbitrary metadata for the objects. ([#537](https://github.com/peak/s5cmd/issues/537))
- Added `--include` flag to `cp`, `rm` and `sync` commands. ([#516](https://github.com/peak/s5cmd/issues/516))

#### Improvements
Expand Down
71 changes: 44 additions & 27 deletions command/cp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ const (
defaultCopyConcurrency = 5
defaultPartSize = 50 // MiB
megabytes = 1024 * 1024
kilobytes = 1024
)

var copyHelpTemplate = `Name:
Expand Down Expand Up @@ -109,6 +110,9 @@ Examples:
23. Download the specific version of a remote object to working directory
> s5cmd {{.HelpName}} --version-id VERSION_ID s3://bucket/prefix/object .
24. Pass arbitrary metadata to the object during upload or copy
> s5cmd {{.HelpName}} --metadata "camera=Nixon D750" --metadata "imageSize=6032x4032" flowers.png s3://bucket/prefix/flowers.png
`

func NewSharedFlags() []cli.Flag {
Expand All @@ -133,6 +137,10 @@ func NewSharedFlags() []cli.Flag {
Value: defaultPartSize,
Usage: "size of each part transferred between host and remote server, in MiB",
},
&MapFlag{
Name: "metadata",
Usage: "set arbitrary metadata for the object, e.g. --metadata 'foo=bar' --metadata 'fizz=buzz'",
},
&cli.StringFlag{
Name: "sse",
Usage: "perform server side encryption of the data at its destination, e.g. aws:kms",
Expand Down Expand Up @@ -296,6 +304,7 @@ type Copy struct {
contentType string
contentEncoding string
contentDisposition string
metadata map[string]string
showProgress bool
progressbar progressbar.ProgressBar

Expand Down Expand Up @@ -338,6 +347,13 @@ func NewCopy(c *cli.Context, deleteSource bool) (*Copy, error) {
commandProgressBar = &progressbar.NoOp{}
}

metadata, ok := c.Value("metadata").(MapValue)
if !ok {
err := errors.New("metadata flag is not a map")
printError(fullCommand, c.Command.Name, err)
return nil, err
}

return &Copy{
src: src,
dst: dst,
Expand Down Expand Up @@ -365,6 +381,7 @@ func NewCopy(c *cli.Context, deleteSource bool) (*Copy, error) {
contentType: c.String("content-type"),
contentEncoding: c.String("content-encoding"),
contentDisposition: c.String("content-disposition"),
metadata: metadata,
showProgress: c.Bool("show-progress"),
progressbar: commandProgressBar,

Expand Down Expand Up @@ -497,11 +514,11 @@ func (c Copy) Run(ctx context.Context) error {

switch {
case srcurl.Type == c.dst.Type: // local->local or remote->remote
task = c.prepareCopyTask(ctx, srcurl, c.dst, isBatch)
task = c.prepareCopyTask(ctx, srcurl, c.dst, isBatch, c.metadata)
case srcurl.IsRemote(): // remote->local
task = c.prepareDownloadTask(ctx, srcurl, c.dst, isBatch)
case c.dst.IsRemote(): // local->remote
task = c.prepareUploadTask(ctx, srcurl, c.dst, isBatch)
task = c.prepareUploadTask(ctx, srcurl, c.dst, isBatch, c.metadata)
default:
panic("unexpected src-dst pair")
}
Expand All @@ -518,10 +535,11 @@ func (c Copy) prepareCopyTask(
srcurl *url.URL,
dsturl *url.URL,
isBatch bool,
metadata map[string]string,
) func() error {
return func() error {
dsturl = prepareRemoteDestination(srcurl, dsturl, c.flatten, isBatch)
err := c.doCopy(ctx, srcurl, dsturl)
err := c.doCopy(ctx, srcurl, dsturl, metadata)
if err != nil {
return &errorpkg.Error{
Op: c.op,
Expand Down Expand Up @@ -565,10 +583,11 @@ func (c Copy) prepareUploadTask(
srcurl *url.URL,
dsturl *url.URL,
isBatch bool,
metadata map[string]string,
) func() error {
return func() error {
dsturl = prepareRemoteDestination(srcurl, dsturl, c.flatten, isBatch)
err := c.doUpload(ctx, srcurl, dsturl)
err := c.doUpload(ctx, srcurl, dsturl, metadata)
if err != nil {
return &errorpkg.Error{
Op: c.op,
Expand Down Expand Up @@ -644,7 +663,7 @@ func (c Copy) doDownload(ctx context.Context, srcurl *url.URL, dsturl *url.URL)
return nil
}

func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL) error {
func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL, extradata map[string]string) error {
srcClient := storage.NewLocalClient(c.storageOpts)

file, err := srcClient.Open(srcurl.Absolute())
Expand All @@ -670,29 +689,29 @@ func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL) er
if err != nil {
return err
}
metadata := storage.Metadata{UserDefined: extradata}

metadata := storage.NewMetadata().
SetStorageClass(string(c.storageClass)).
SetSSE(c.encryptionMethod).
SetSSEKeyID(c.encryptionKeyID).
SetACL(c.acl).
SetCacheControl(c.cacheControl).
SetExpires(c.expires)
if c.storageClass != "" {
metadata.StorageClass = string(c.storageClass)
}

if c.contentType != "" {
metadata.SetContentType(c.contentType)
metadata.ContentType = c.contentType
} else {
metadata.SetContentType(guessContentType(file))
metadata.ContentType = guessContentType(file)
}

if c.contentEncoding != "" {
metadata.SetContentEncoding(c.contentEncoding)
metadata.ContentEncoding = c.contentEncoding
}

if c.contentDisposition != "" {
metadata.SetContentDisposition(c.contentDisposition)
metadata.ContentDisposition = c.contentDisposition
}

reader := newCountingReaderWriter(file, c.progressbar)
err = dstClient.Put(ctx, reader, dsturl, metadata, c.concurrency, c.partSize)

if err != nil {
return err
}
Expand Down Expand Up @@ -726,7 +745,7 @@ func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL) er
return nil
}

func (c Copy) doCopy(ctx context.Context, srcurl, dsturl *url.URL) error {
func (c Copy) doCopy(ctx context.Context, srcurl, dsturl *url.URL, extradata map[string]string) error {
// override destination region if set
if c.dstRegion != "" {
c.storageOpts.SetRegion(c.dstRegion)
Expand All @@ -736,22 +755,20 @@ func (c Copy) doCopy(ctx context.Context, srcurl, dsturl *url.URL) error {
return err
}

metadata := storage.NewMetadata().
SetStorageClass(string(c.storageClass)).
SetSSE(c.encryptionMethod).
SetSSEKeyID(c.encryptionKeyID).
SetACL(c.acl).
SetCacheControl(c.cacheControl).
SetExpires(c.expires)
metadata := storage.Metadata{UserDefined: extradata}
if c.storageClass != "" {
metadata.StorageClass = string(c.storageClass)
}

if c.contentType != "" {
metadata.SetContentType(c.contentType)
metadata.ContentType = c.contentType
}

if c.contentEncoding != "" {
metadata.SetContentEncoding(c.contentEncoding)
metadata.ContentEncoding = c.contentEncoding
}
if c.contentDisposition != "" {
metadata.SetContentDisposition(c.contentDisposition)
metadata.ContentDisposition = c.contentDisposition
}

err = c.shouldOverride(ctx, srcurl, dsturl)
Expand Down
127 changes: 127 additions & 0 deletions command/flag.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
package command

import (
"flag"
"fmt"
"strings"

"github.com/urfave/cli/v2"
)

type EnumValue struct {
Expand Down Expand Up @@ -41,3 +44,127 @@ func (e EnumValue) String() string {
func (e EnumValue) Get() interface{} {
return e
}

type MapValue map[string]string

func (m MapValue) String() string {
if m == nil {
m = make(map[string]string)
}

var s strings.Builder
for key, value := range m {
s.WriteString(fmt.Sprintf("%s=%s ", key, value))
}

return s.String()
}

func (m MapValue) Set(s string) error {
if m == nil {
m = make(map[string]string)
}

if len(s) == 0 {
return fmt.Errorf("flag can't be passed empty. Format: key=value")
}

tokens := strings.Split(s, "=")
if len(tokens) <= 1 {
return fmt.Errorf("the key value pair(%s) has invalid format", tokens)
}

key := tokens[0]
value := strings.Join(tokens[1:], "=")

_, ok := m[key]
if ok {
return fmt.Errorf("key %q is already defined", key)
}

m[key] = value
return nil
}

func (m MapValue) Get() interface{} {
if m == nil {
m = make(map[string]string)
}
return m
}

type MapFlag struct {
Name string

Category string
DefaultText string
FilePath string
Usage string

HasBeenSet bool
Required bool
Hidden bool

Value MapValue
}

var (
_ cli.Flag = (*MapFlag)(nil)
_ cli.RequiredFlag = (*MapFlag)(nil)
_ cli.VisibleFlag = (*MapFlag)(nil)
_ cli.DocGenerationFlag = (*MapFlag)(nil)
)

func (f *MapFlag) Apply(set *flag.FlagSet) error {
if f.Value == nil {
f.Value = make(map[string]string)
}
for _, name := range f.Names() {
set.Var(f.Value, name, f.Usage)
if len(f.Value) > 0 {
f.HasBeenSet = true
}
}

return nil
}

func (f *MapFlag) GetUsage() string {
return f.Usage
}

func (f *MapFlag) Names() []string {
return []string{f.Name}
}

func (f *MapFlag) IsSet() bool {
return f.HasBeenSet
}

func (f *MapFlag) IsVisible() bool {
return true
}

func (f *MapFlag) String() string {
return cli.FlagStringer(f)
}

func (f *MapFlag) TakesValue() bool {
return true
}

func (f *MapFlag) GetValue() string {
return f.Value.String()
}

func (f *MapFlag) GetDefaultText() string {
return ""
}

func (f *MapFlag) GetEnvVars() []string {
return []string{}
}

func (f *MapFlag) IsRequired() bool {
return f.Required
}
Loading

0 comments on commit db6b53f

Please sign in to comment.