main.go

Documentation: golang.org/x/tools/cmd/bundle

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Bundle creates a single-source-file version of a source package
     6  // suitable for inclusion in a particular target package.
     7  //
     8  // Usage:
     9  //
    10  //	bundle [-o file] [-dst path] [-pkg name] [-prefix p] [-import old=new] [-tags build_constraints] <src>
    11  //
    12  // The src argument specifies the import path of the package to bundle.
    13  // The bundling of a directory of source files into a single source file
    14  // necessarily imposes a number of constraints.
    15  // The package being bundled must not use cgo; must not use conditional
    16  // file compilation, whether with build tags or system-specific file names
    17  // like code_amd64.go; must not depend on any special comments, which
    18  // may not be preserved; must not use any assembly sources;
    19  // must not use renaming imports; and must not use reflection-based APIs
    20  // that depend on the specific names of types or struct fields.
    21  //
    22  // By default, bundle writes the bundled code to standard output.
    23  // If the -o argument is given, bundle writes to the named file
    24  // and also includes a “//go:generate” comment giving the exact
    25  // command line used, for regenerating the file with “go generate.”
    26  //
    27  // Bundle customizes its output for inclusion in a particular package, the destination package.
    28  // By default bundle assumes the destination is the package in the current directory,
    29  // but the destination package can be specified explicitly using the -dst option,
    30  // which takes an import path as its argument.
    31  // If the source package imports the destination package, bundle will remove
    32  // those imports and rewrite any references to use direct references to the
    33  // corresponding symbols.
    34  // Bundle also must write a package declaration in the output and must
    35  // choose a name to use in that declaration.
    36  // If the -pkg option is given, bundle uses that name.
    37  // Otherwise, the name of the destination package is used.
    38  // Build constraints for the generated file can be specified using the -tags option.
    39  //
    40  // To avoid collisions, bundle inserts a prefix at the beginning of
    41  // every package-level const, func, type, and var identifier in src's code,
    42  // updating references accordingly. The default prefix is the package name
    43  // of the source package followed by an underscore. The -prefix option
    44  // specifies an alternate prefix.
    45  //
    46  // Occasionally it is necessary to rewrite imports during the bundling
    47  // process. The -import option, which may be repeated, specifies that
    48  // an import of "old" should be rewritten to import "new" instead.
    49  //
    50  // # Example
    51  //
    52  // Bundle archive/zip for inclusion in cmd/dist:
    53  //
    54  //	cd $GOROOT/src/cmd/dist
    55  //	bundle -o zip.go archive/zip
    56  //
    57  // Bundle golang.org/x/net/http2 for inclusion in net/http,
    58  // prefixing all identifiers by "http2" instead of "http2_", and
    59  // including a "!nethttpomithttp2" build constraint:
    60  //
    61  //	cd $GOROOT/src/net/http
    62  //	bundle -o h2_bundle.go -prefix http2 -tags '!nethttpomithttp2' golang.org/x/net/http2
    63  //
    64  // Update the http2 bundle in net/http:
    65  //
    66  //	go generate net/http
    67  //
    68  // Update all bundles in the standard library:
    69  //
    70  //	go generate -run bundle std
    71  package main
    72  
    73  import (
    74  	"bytes"
    75  	"flag"
    76  	"fmt"
    77  	"go/ast"
    78  	"go/format"
    79  	"go/printer"
    80  	"go/token"
    81  	"go/types"
    82  	"io/ioutil"
    83  	"log"
    84  	"os"
    85  	"strconv"
    86  	"strings"
    87  	"unicode"
    88  
    89  	"golang.org/x/tools/go/packages"
    90  )
    91  
    92  var (
    93  	outputFile = flag.String("o", "", "write output to `file` (default standard output)")
    94  	dstPath    = flag.String("dst", ".", "set destination import `path`")
    95  	pkgName    = flag.String("pkg", "", "set destination package `name`")
    96  	prefix     = flag.String("prefix", "&_", "set bundled identifier prefix to `p` (default is \"&_\", where & stands for the original name)")
    97  	buildTags  = flag.String("tags", "", "the build constraints to be inserted into the generated file")
    98  
    99  	importMap = map[string]string{}
   100  )
   101  
   102  func init() {
   103  	flag.Var(flagFunc(addImportMap), "import", "rewrite import using `map`, of form old=new (can be repeated)")
   104  }
   105  
   106  func addImportMap(s string) {
   107  	if strings.Count(s, "=") != 1 {
   108  		log.Fatal("-import argument must be of the form old=new")
   109  	}
   110  	i := strings.Index(s, "=")
   111  	old, new := s[:i], s[i+1:]
   112  	if old == "" || new == "" {
   113  		log.Fatal("-import argument must be of the form old=new; old and new must be non-empty")
   114  	}
   115  	importMap[old] = new
   116  }
   117  
   118  func usage() {
   119  	fmt.Fprintf(os.Stderr, "Usage: bundle [options] <src>\n")
   120  	flag.PrintDefaults()
   121  }
   122  
   123  func main() {
   124  	log.SetPrefix("bundle: ")
   125  	log.SetFlags(0)
   126  
   127  	flag.Usage = usage
   128  	flag.Parse()
   129  	args := flag.Args()
   130  	if len(args) != 1 {
   131  		usage()
   132  		os.Exit(2)
   133  	}
   134  
   135  	cfg := &packages.Config{Mode: packages.NeedName}
   136  	pkgs, err := packages.Load(cfg, *dstPath)
   137  	if err != nil {
   138  		log.Fatalf("cannot load destination package: %v", err)
   139  	}
   140  	if packages.PrintErrors(pkgs) > 0 || len(pkgs) != 1 {
   141  		log.Fatalf("failed to load destination package")
   142  	}
   143  	if *pkgName == "" {
   144  		*pkgName = pkgs[0].Name
   145  	}
   146  
   147  	code, err := bundle(args[0], pkgs[0].PkgPath, *pkgName, *prefix, *buildTags)
   148  	if err != nil {
   149  		log.Fatal(err)
   150  	}
   151  	if *outputFile != "" {
   152  		err := ioutil.WriteFile(*outputFile, code, 0666)
   153  		if err != nil {
   154  			log.Fatal(err)
   155  		}
   156  	} else {
   157  		_, err := os.Stdout.Write(code)
   158  		if err != nil {
   159  			log.Fatal(err)
   160  		}
   161  	}
   162  }
   163  
   164  // isStandardImportPath is copied from cmd/go in the standard library.
   165  func isStandardImportPath(path string) bool {
   166  	i := strings.Index(path, "/")
   167  	if i < 0 {
   168  		i = len(path)
   169  	}
   170  	elem := path[:i]
   171  	return !strings.Contains(elem, ".")
   172  }
   173  
   174  var testingOnlyPackagesConfig *packages.Config
   175  
   176  func bundle(src, dst, dstpkg, prefix, buildTags string) ([]byte, error) {
   177  	// Load the initial package.
   178  	cfg := &packages.Config{}
   179  	if testingOnlyPackagesConfig != nil {
   180  		*cfg = *testingOnlyPackagesConfig
   181  	} else {
   182  		// Bypass default vendor mode, as we need a package not available in the
   183  		// std module vendor folder.
   184  		cfg.Env = append(os.Environ(), "GOFLAGS=-mod=mod")
   185  	}
   186  	cfg.Mode = packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesInfo
   187  	pkgs, err := packages.Load(cfg, src)
   188  	if err != nil {
   189  		return nil, err
   190  	}
   191  	if packages.PrintErrors(pkgs) > 0 || len(pkgs) != 1 {
   192  		return nil, fmt.Errorf("failed to load source package")
   193  	}
   194  	pkg := pkgs[0]
   195  
   196  	if strings.Contains(prefix, "&") {
   197  		prefix = strings.Replace(prefix, "&", pkg.Syntax[0].Name.Name, -1)
   198  	}
   199  
   200  	objsToUpdate := make(map[types.Object]bool)
   201  	var rename func(from types.Object)
   202  	rename = func(from types.Object) {
   203  		if !objsToUpdate[from] {
   204  			objsToUpdate[from] = true
   205  
   206  			// Renaming a type that is used as an embedded field
   207  			// requires renaming the field too. e.g.
   208  			// 	type T int // if we rename this to U..
   209  			// 	var s struct {T}
   210  			// 	print(s.T) // ...this must change too
   211  			if _, ok := from.(*types.TypeName); ok {
   212  				for id, obj := range pkg.TypesInfo.Uses {
   213  					if obj == from {
   214  						if field := pkg.TypesInfo.Defs[id]; field != nil {
   215  							rename(field)
   216  						}
   217  					}
   218  				}
   219  			}
   220  		}
   221  	}
   222  
   223  	// Rename each package-level object.
   224  	scope := pkg.Types.Scope()
   225  	for _, name := range scope.Names() {
   226  		rename(scope.Lookup(name))
   227  	}
   228  
   229  	var out bytes.Buffer
   230  	if buildTags != "" {
   231  		fmt.Fprintf(&out, "//go:build %s\n", buildTags)
   232  		fmt.Fprintf(&out, "// +build %s\n\n", buildTags)
   233  	}
   234  
   235  	fmt.Fprintf(&out, "// Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT.\n")
   236  	if *outputFile != "" && buildTags == "" {
   237  		fmt.Fprintf(&out, "//go:generate bundle %s\n", strings.Join(quoteArgs(os.Args[1:]), " "))
   238  	} else {
   239  		fmt.Fprintf(&out, "//   $ bundle %s\n", strings.Join(os.Args[1:], " "))
   240  	}
   241  	fmt.Fprintf(&out, "\n")
   242  
   243  	// Concatenate package comments from all files...
   244  	for _, f := range pkg.Syntax {
   245  		if doc := f.Doc.Text(); strings.TrimSpace(doc) != "" {
   246  			for _, line := range strings.Split(doc, "\n") {
   247  				fmt.Fprintf(&out, "// %s\n", line)
   248  			}
   249  		}
   250  	}
   251  	// ...but don't let them become the actual package comment.
   252  	fmt.Fprintln(&out)
   253  
   254  	fmt.Fprintf(&out, "package %s\n\n", dstpkg)
   255  
   256  	// BUG(adonovan,shurcooL): bundle may generate incorrect code
   257  	// due to shadowing between identifiers and imported package names.
   258  	//
   259  	// The generated code will either fail to compile or
   260  	// (unlikely) compile successfully but have different behavior
   261  	// than the original package. The risk of this happening is higher
   262  	// when the original package has renamed imports (they're typically
   263  	// renamed in order to resolve a shadow inside that particular .go file).
   264  
   265  	// TODO(adonovan,shurcooL):
   266  	// - detect shadowing issues, and either return error or resolve them
   267  	// - preserve comments from the original import declarations.
   268  
   269  	// pkgStd and pkgExt are sets of printed import specs. This is done
   270  	// to deduplicate instances of the same import name and path.
   271  	var pkgStd = make(map[string]bool)
   272  	var pkgExt = make(map[string]bool)
   273  	for _, f := range pkg.Syntax {
   274  		for _, imp := range f.Imports {
   275  			path, err := strconv.Unquote(imp.Path.Value)
   276  			if err != nil {
   277  				log.Fatalf("invalid import path string: %v", err) // Shouldn't happen here since packages.Load succeeded.
   278  			}
   279  			if path == dst {
   280  				continue
   281  			}
   282  			if newPath, ok := importMap[path]; ok {
   283  				path = newPath
   284  			}
   285  
   286  			var name string
   287  			if imp.Name != nil {
   288  				name = imp.Name.Name
   289  			}
   290  			spec := fmt.Sprintf("%s %q", name, path)
   291  			if isStandardImportPath(path) {
   292  				pkgStd[spec] = true
   293  			} else {
   294  				pkgExt[spec] = true
   295  			}
   296  		}
   297  	}
   298  
   299  	// Print a single declaration that imports all necessary packages.
   300  	fmt.Fprintln(&out, "import (")
   301  	for p := range pkgStd {
   302  		fmt.Fprintf(&out, "\t%s\n", p)
   303  	}
   304  	if len(pkgExt) > 0 {
   305  		fmt.Fprintln(&out)
   306  	}
   307  	for p := range pkgExt {
   308  		fmt.Fprintf(&out, "\t%s\n", p)
   309  	}
   310  	fmt.Fprint(&out, ")\n\n")
   311  
   312  	// Modify and print each file.
   313  	for _, f := range pkg.Syntax {
   314  		// Update renamed identifiers.
   315  		for id, obj := range pkg.TypesInfo.Defs {
   316  			if objsToUpdate[obj] {
   317  				id.Name = prefix + obj.Name()
   318  			}
   319  		}
   320  		for id, obj := range pkg.TypesInfo.Uses {
   321  			if objsToUpdate[obj] {
   322  				id.Name = prefix + obj.Name()
   323  			}
   324  		}
   325  
   326  		// For each qualified identifier that refers to the
   327  		// destination package, remove the qualifier.
   328  		// The "@@@." strings are removed in postprocessing.
   329  		ast.Inspect(f, func(n ast.Node) bool {
   330  			if sel, ok := n.(*ast.SelectorExpr); ok {
   331  				if id, ok := sel.X.(*ast.Ident); ok {
   332  					if obj, ok := pkg.TypesInfo.Uses[id].(*types.PkgName); ok {
   333  						if obj.Imported().Path() == dst {
   334  							id.Name = "@@@"
   335  						}
   336  					}
   337  				}
   338  			}
   339  			return true
   340  		})
   341  
   342  		last := f.Package
   343  		if len(f.Imports) > 0 {
   344  			imp := f.Imports[len(f.Imports)-1]
   345  			last = imp.End()
   346  			if imp.Comment != nil {
   347  				if e := imp.Comment.End(); e > last {
   348  					last = e
   349  				}
   350  			}
   351  		}
   352  
   353  		// Pretty-print package-level declarations.
   354  		// but no package or import declarations.
   355  		var buf bytes.Buffer
   356  		for _, decl := range f.Decls {
   357  			if decl, ok := decl.(*ast.GenDecl); ok && decl.Tok == token.IMPORT {
   358  				continue
   359  			}
   360  
   361  			beg, end := sourceRange(decl)
   362  
   363  			printComments(&out, f.Comments, last, beg)
   364  
   365  			buf.Reset()
   366  			format.Node(&buf, pkg.Fset, &printer.CommentedNode{Node: decl, Comments: f.Comments})
   367  			// Remove each "@@@." in the output.
   368  			// TODO(adonovan): not hygienic.
   369  			out.Write(bytes.Replace(buf.Bytes(), []byte("@@@."), nil, -1))
   370  
   371  			last = printSameLineComment(&out, f.Comments, pkg.Fset, end)
   372  
   373  			out.WriteString("\n\n")
   374  		}
   375  
   376  		printLastComments(&out, f.Comments, last)
   377  	}
   378  
   379  	// Now format the entire thing.
   380  	result, err := format.Source(out.Bytes())
   381  	if err != nil {
   382  		log.Fatalf("formatting failed: %v", err)
   383  	}
   384  
   385  	return result, nil
   386  }
   387  
   388  // sourceRange returns the [beg, end) interval of source code
   389  // belonging to decl (incl. associated comments).
   390  func sourceRange(decl ast.Decl) (beg, end token.Pos) {
   391  	beg = decl.Pos()
   392  	end = decl.End()
   393  
   394  	var doc, com *ast.CommentGroup
   395  
   396  	switch d := decl.(type) {
   397  	case *ast.GenDecl:
   398  		doc = d.Doc
   399  		if len(d.Specs) > 0 {
   400  			switch spec := d.Specs[len(d.Specs)-1].(type) {
   401  			case *ast.ValueSpec:
   402  				com = spec.Comment
   403  			case *ast.TypeSpec:
   404  				com = spec.Comment
   405  			}
   406  		}
   407  	case *ast.FuncDecl:
   408  		doc = d.Doc
   409  	}
   410  
   411  	if doc != nil {
   412  		beg = doc.Pos()
   413  	}
   414  	if com != nil && com.End() > end {
   415  		end = com.End()
   416  	}
   417  
   418  	return beg, end
   419  }
   420  
   421  func printComments(out *bytes.Buffer, comments []*ast.CommentGroup, pos, end token.Pos) {
   422  	for _, cg := range comments {
   423  		if pos <= cg.Pos() && cg.Pos() < end {
   424  			for _, c := range cg.List {
   425  				fmt.Fprintln(out, c.Text)
   426  			}
   427  			fmt.Fprintln(out)
   428  		}
   429  	}
   430  }
   431  
   432  const infinity = 1 << 30
   433  
   434  func printLastComments(out *bytes.Buffer, comments []*ast.CommentGroup, pos token.Pos) {
   435  	printComments(out, comments, pos, infinity)
   436  }
   437  
   438  func printSameLineComment(out *bytes.Buffer, comments []*ast.CommentGroup, fset *token.FileSet, pos token.Pos) token.Pos {
   439  	tf := fset.File(pos)
   440  	for _, cg := range comments {
   441  		if pos <= cg.Pos() && tf.Line(cg.Pos()) == tf.Line(pos) {
   442  			for _, c := range cg.List {
   443  				fmt.Fprintln(out, c.Text)
   444  			}
   445  			return cg.End()
   446  		}
   447  	}
   448  	return pos
   449  }
   450  
   451  func quoteArgs(ss []string) []string {
   452  	// From go help generate:
   453  	//
   454  	// > The arguments to the directive are space-separated tokens or
   455  	// > double-quoted strings passed to the generator as individual
   456  	// > arguments when it is run.
   457  	//
   458  	// > Quoted strings use Go syntax and are evaluated before execution; a
   459  	// > quoted string appears as a single argument to the generator.
   460  	//
   461  	var qs []string
   462  	for _, s := range ss {
   463  		if s == "" || containsSpace(s) {
   464  			s = strconv.Quote(s)
   465  		}
   466  		qs = append(qs, s)
   467  	}
   468  	return qs
   469  }
   470  
   471  func containsSpace(s string) bool {
   472  	for _, r := range s {
   473  		if unicode.IsSpace(r) {
   474  			return true
   475  		}
   476  	}
   477  	return false
   478  }
   479  
   480  type flagFunc func(string)
   481  
   482  func (f flagFunc) Set(s string) error {
   483  	f(s)
   484  	return nil
   485  }
   486  
   487  func (f flagFunc) String() string { return "" }
   488
View as plain text