From 65ee4f3e6e605307610305466858312af78addfb Mon Sep 17 00:00:00 2001 From: corebreaker Date: Fri, 15 Sep 2017 20:19:45 +0300 Subject: [PATCH 1/2] Add `-split` option to avoid generating big file --- config.go | 30 ++++++++-- convert.go | 67 +-------------------- convert_to_dir.go | 141 +++++++++++++++++++++++++++++++++++++++++++++ convert_to_file.go | 77 +++++++++++++++++++++++++ debug.go | 36 +++++++++++- doc.go | 13 +++++ go-bindata/main.go | 1 + release.go | 25 +++++++- 8 files changed, 318 insertions(+), 72 deletions(-) create mode 100644 convert_to_dir.go create mode 100644 convert_to_file.go diff --git a/config.go b/config.go index 4bd92fa..e63b07d 100644 --- a/config.go +++ b/config.go @@ -39,7 +39,8 @@ type Config struct { // Output defines the output file for the generated code. // If left empty, this defaults to 'bindata.go' in the current - // working directory. + // working directory and the current directory in case of having true + // to `Split` config. Output string // Prefix defines a regular expression which should used to strip @@ -142,6 +143,12 @@ type Config struct { Ignore []*regexp.Regexp Include []*regexp.Regexp + // Split the output into several files. Every embedded file is bound into + // a specific file, and a common file is also generated containing API and + // other common parts. + // If true, the output config is a directory and not a file. + Split bool + // MD5Checksum is a flag that, when set to true, indicates to calculate // MD5 checksums for files. MD5Checksum bool @@ -154,7 +161,6 @@ func NewConfig() *Config { c.NoMemCopy = false c.NoCompress = false c.Debug = false - c.Output = "./bindata.go" c.Ignore = make([]*regexp.Regexp, 0) c.Include = make([]*regexp.Regexp, 0) return c @@ -180,7 +186,11 @@ func (c *Config) validate() error { return fmt.Errorf("Unable to determine current working directory") } - c.Output = filepath.Join(cwd, "bindata.go") + if c.Split { + c.Output = cwd + } else { + c.Output = filepath.Join(cwd, "bindata.go") + } } stat, err := os.Lstat(c.Output) @@ -201,8 +211,18 @@ func (c *Config) validate() error { } } - if stat != nil && stat.IsDir() { - return fmt.Errorf("Output path is a directory") + if stat != nil { + if c.Split { + if !stat.IsDir() { + return fmt.Errorf("Output path is not a directory") + + } + } else { + if stat.IsDir() { + return fmt.Errorf("Output path is a directory") + + } + } } return nil diff --git a/convert.go b/convert.go index e4a226e..f815c08 100644 --- a/convert.go +++ b/convert.go @@ -5,7 +5,6 @@ package bindata import ( - "bufio" "fmt" "os" "path/filepath" @@ -37,76 +36,16 @@ func Translate(c *Config) error { } } - // Create output file. - fd, err := os.Create(c.Output) - if err != nil { - return err - } - - defer fd.Close() - - // Create a buffered writer for better performance. - bfd := bufio.NewWriter(fd) - defer bfd.Flush() - - // Write the header. This makes e.g. Github ignore diffs in generated files. - if _, err = fmt.Fprint(bfd, "// Code generated by go-bindata.\n"); err != nil { - return err - } - if _, err = fmt.Fprint(bfd, "// sources:\n"); err != nil { - return err - } - wd, err := os.Getwd() if err != nil { return err } - for _, asset := range toc { - relative, _ := filepath.Rel(wd, asset.Path) - if _, err = fmt.Fprintf(bfd, "// %s\n", filepath.ToSlash(relative)); err != nil { - return err - } - } - if _, err = fmt.Fprint(bfd, "// DO NOT EDIT!\n\n"); err != nil { - return err - } - - // Write build tags, if applicable. - if len(c.Tags) > 0 { - if _, err = fmt.Fprintf(bfd, "// +build %s\n\n", c.Tags); err != nil { - return err - } - } - - // Write package declaration. - _, err = fmt.Fprintf(bfd, "package %s\n\n", c.Package) - if err != nil { - return err - } - - // Write assets. - if c.Debug || c.Dev { - err = writeDebug(bfd, c, toc) + if c.Split { + return translateToDir(c, toc, wd) } else { - err = writeRelease(bfd, c, toc) + return translateToFile(c, toc, wd) } - - if err != nil { - return err - } - - // Write table of contents - if err := writeTOC(bfd, toc); err != nil { - return err - } - // Write hierarchical tree of assets - if err := writeTOCTree(bfd, toc); err != nil { - return err - } - - // Write restore procedure - return writeRestore(bfd) } // ByName implement sort.Interface for []os.FileInfo based on Name() diff --git a/convert_to_dir.go b/convert_to_dir.go new file mode 100644 index 0000000..3ca7deb --- /dev/null +++ b/convert_to_dir.go @@ -0,0 +1,141 @@ +package bindata + +import ( + "bufio" + "fmt" + "os" + "path/filepath" +) + +// translateToDir generates splited file +func translateToDir(c *Config, toc []Asset, wd string) error { + if err := generateCommonFile(c, toc); err != nil { + return err + } + + for i := range toc { + if err := generateOneAsset(c, &toc[i], wd); err != nil { + return err + } + } + + return nil +} + +func generateCommonFile(c *Config, toc []Asset) error { + // Create output file. + fd, err := os.Create(filepath.Join(c.Output, "common.go")) + if err != nil { + return err + } + + defer fd.Close() + + // Create a buffered writer for better performance. + bfd := bufio.NewWriter(fd) + defer bfd.Flush() + + // Write the header. This makes e.g. Github ignore diffs in generated files. + if _, err = fmt.Fprint(bfd, "// Code generated by go-bindata.\n"); err != nil { + return err + } + + if _, err = fmt.Fprint(bfd, "// -- Common file --\n"); err != nil { + return err + } + + if _, err = fmt.Fprint(bfd, "// DO NOT EDIT!\n\n"); err != nil { + return err + } + + // Write build tags, if applicable. + if len(c.Tags) > 0 { + if _, err = fmt.Fprintf(bfd, "// +build %s\n\n", c.Tags); err != nil { + return err + } + } + + // Write package declaration. + _, err = fmt.Fprintf(bfd, "package %s\n\n", c.Package) + if err != nil { + return err + } + + // Write assets. + if c.Debug || c.Dev { + err = writeDebugHeader(bfd) + } else { + err = writeReleaseHeader(bfd, c) + } + + if err != nil { + return err + } + + // Write table of contents + if err := writeTOC(bfd, toc); err != nil { + return err + } + // Write hierarchical tree of assets + if err := writeTOCTree(bfd, toc); err != nil { + return err + } + + // Write restore procedure + return writeRestore(bfd) + +} + +func generateOneAsset(c *Config, a *Asset, wd string) error { + // Create output file. + fd, err := os.Create(filepath.Join(c.Output, a.Func + ".go")) + if err != nil { + return err + } + + defer fd.Close() + + // Create a buffered writer for better performance. + bfd := bufio.NewWriter(fd) + defer bfd.Flush() + + // Write the header. This makes e.g. Github ignore diffs in generated files. + if _, err = fmt.Fprint(bfd, "// Code generated by go-bindata.\n"); err != nil { + return err + } + + if _, err = fmt.Fprint(bfd, "// source: "); err != nil { + return err + } + + relative, _ := filepath.Rel(wd, a.Path) + if _, err = fmt.Fprintln(bfd, filepath.ToSlash(relative)); err != nil { + return err + } + + if _, err = fmt.Fprint(bfd, "// DO NOT EDIT!\n\n"); err != nil { + return err + } + + // Write build tags, if applicable. + if len(c.Tags) > 0 { + if _, err = fmt.Fprintf(bfd, "// +build %s\n\n", c.Tags); err != nil { + return err + } + } + + // Write package declaration. + _, err = fmt.Fprintf(bfd, "package %s\n\n", c.Package) + if err != nil { + return err + } + + // Write assets. + if c.Debug || c.Dev { + err = writeOneFileDebug(bfd, c, a) + } else { + err = writeOneFileRelease(bfd, c, a) + } + + return err +} diff --git a/convert_to_file.go b/convert_to_file.go new file mode 100644 index 0000000..6b29a23 --- /dev/null +++ b/convert_to_file.go @@ -0,0 +1,77 @@ +package bindata + +import ( + "os" + "bufio" + "fmt" + "path/filepath" +) + +// translateToFile generates one single file +func translateToFile(c *Config, toc []Asset, wd string) error { + // Create output file. + fd, err := os.Create(c.Output) + if err != nil { + return err + } + + defer fd.Close() + + // Create a buffered writer for better performance. + bfd := bufio.NewWriter(fd) + defer bfd.Flush() + + // Write the header. This makes e.g. Github ignore diffs in generated files. + if _, err = fmt.Fprint(bfd, "// Code generated by go-bindata.\n"); err != nil { + return err + } + if _, err = fmt.Fprint(bfd, "// sources:\n"); err != nil { + return err + } + + for _, asset := range toc { + relative, _ := filepath.Rel(wd, asset.Path) + if _, err = fmt.Fprintf(bfd, "// %s\n", filepath.ToSlash(relative)); err != nil { + return err + } + } + if _, err = fmt.Fprint(bfd, "// DO NOT EDIT!\n\n"); err != nil { + return err + } + + // Write build tags, if applicable. + if len(c.Tags) > 0 { + if _, err = fmt.Fprintf(bfd, "// +build %s\n\n", c.Tags); err != nil { + return err + } + } + + // Write package declaration. + _, err = fmt.Fprintf(bfd, "package %s\n\n", c.Package) + if err != nil { + return err + } + + // Write assets. + if c.Debug || c.Dev { + err = writeDebug(bfd, c, toc) + } else { + err = writeRelease(bfd, c, toc) + } + + if err != nil { + return err + } + + // Write table of contents + if err := writeTOC(bfd, toc); err != nil { + return err + } + // Write hierarchical tree of assets + if err := writeTOCTree(bfd, toc); err != nil { + return err + } + + // Write restore procedure + return writeRestore(bfd) +} diff --git a/debug.go b/debug.go index 09fee78..717f03a 100644 --- a/debug.go +++ b/debug.go @@ -9,7 +9,20 @@ import ( "io" ) -// writeDebug writes the debug code file. +// writeOneFileDebug writes the debug code file for each file (when splited file). +func writeOneFileDebug(w io.Writer, c *Config, a *Asset) error { + if err := writeDebugFileHeader(w, c.Dev); err != nil { + return err + } + + if err := writeDebugAsset(w, c, a); err != nil { + return err + } + + return nil +} + +// writeDebug writes the debug code file for single file. func writeDebug(w io.Writer, c *Config, toc []Asset) error { err := writeDebugHeader(w) if err != nil { @@ -26,7 +39,26 @@ func writeDebug(w io.Writer, c *Config, toc []Asset) error { return nil } -// writeDebugHeader writes output file headers. +// writeDebugHeader writes output file headers for each file. +// This targets debug builds. +func writeDebugFileHeader(w io.Writer, dev bool) error { + add := "" + if dev { + add = ` + "path/filepath"` + } + + _, err := fmt.Fprintf(w, `import ( + "fmt" + "os"%s +) + +`, add) + + return err +} + +// writeDebugHeader writes output file headers for sigle file. // This targets debug builds. func writeDebugHeader(w io.Writer) error { _, err := fmt.Fprintf(w, `import ( diff --git a/doc.go b/doc.go index 5c6750d..9f3ef5d 100644 --- a/doc.go +++ b/doc.go @@ -126,5 +126,18 @@ format is specified at build time with the appropriate tags. The tags are appended to a `// +build` line in the beginning of the output file and must follow the build tags syntax specified by the go tool. + +Splitting generated file + +When you want to embed big files or plenty of files, then the generated output +is really big (maybe over 3Mo). Even if the generated file shouldn't be read, +you probably need use analysis tool or an editor which can become slower +with a such file. + +Generating big files can be avoided with `-split` command line option. +In that case, the given output is a directory path, the tool will generate +one source file per file to embed, and it will generate a common file +nammed `common.go` which contains commons parts like API. + */ package bindata diff --git a/go-bindata/main.go b/go-bindata/main.go index c85673c..4b8a39a 100644 --- a/go-bindata/main.go +++ b/go-bindata/main.go @@ -109,6 +109,7 @@ func initArgs() { flag.BoolVar(&cfg.NoCompress, "nocompress", cfg.NoCompress, "Assets will *not* be GZIP compressed when this flag is specified.") flag.BoolVar(&cfg.NoMemCopy, "nomemcopy", cfg.NoMemCopy, "Use a .rodata hack to get rid of unnecessary memcopies. Refer to the documentation to see what implications this carries.") flag.BoolVar(&cfg.NoMetadata, "nometadata", cfg.NoMetadata, "Assets will not preserve size, mode, and modtime info.") + flag.BoolVar(&cfg.Split, "split", cfg.NoMetadata, "Split output into several files avoiding to hava a big output file.") flag.Int64Var(&cfg.ModTime, "modtime", cfg.ModTime, "Optional modification unix timestamp override for all files.") flag.StringVar(&argPrefix, "prefix", "", "Optional path prefix to strip off asset names.") flag.StringVar(&cfg.Output, "o", cfg.Output, "Optional name of the output file to be generated.") diff --git a/release.go b/release.go index 95ef360..06bf906 100644 --- a/release.go +++ b/release.go @@ -15,7 +15,20 @@ import ( "unicode/utf8" ) -// writeRelease writes the release code file. +// writeOneFileRelease writes the release code file for each file (when splited file). +func writeOneFileRelease(w io.Writer, c *Config, a *Asset) error { + if err := fileHeader(w); err != nil { + return err + } + + if err := writeReleaseAsset(w, c, a); err != nil { + return err + } + + return nil +} + +// writeRelease writes the release code file for single file. func writeRelease(w io.Writer, c *Config, toc []Asset) error { err := writeReleaseHeader(w, c) if err != nil { @@ -98,6 +111,16 @@ func sanitize(b []byte) []byte { return bytes.Replace(b, []byte("\xEF\xBB\xBF"), []byte("`+\"\\xEF\\xBB\\xBF\"+`"), -1) } +func fileHeader(w io.Writer) error { + _, err := fmt.Fprintf(w, `import ( + "os" + "time" +) + +`) + return err +} + func headerCompressedNomemcopy(w io.Writer) error { _, err := fmt.Fprintf(w, `import ( "bytes" From 10f7a336c996c77ea632d8fd8a4953484b784d3c Mon Sep 17 00:00:00 2001 From: ZaniaDeveloper Date: Mon, 18 Sep 2017 11:10:20 +0300 Subject: [PATCH 2/2] Mofications following first code review --- convert.go | 4 ++-- convert_to_dir.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/convert.go b/convert.go index f815c08..8aad6eb 100644 --- a/convert.go +++ b/convert.go @@ -43,9 +43,9 @@ func Translate(c *Config) error { if c.Split { return translateToDir(c, toc, wd) - } else { - return translateToFile(c, toc, wd) } + + return translateToFile(c, toc, wd) } // ByName implement sort.Interface for []os.FileInfo based on Name() diff --git a/convert_to_dir.go b/convert_to_dir.go index 3ca7deb..9a07344 100644 --- a/convert_to_dir.go +++ b/convert_to_dir.go @@ -24,7 +24,7 @@ func translateToDir(c *Config, toc []Asset, wd string) error { func generateCommonFile(c *Config, toc []Asset) error { // Create output file. - fd, err := os.Create(filepath.Join(c.Output, "common.go")) + fd, err := os.Create(filepath.Join(c.Output, "bindata.go")) if err != nil { return err }