golist_overlay.go

Documentation: golang.org/x/tools/go/packages

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package packages
     6  
     7  import (
     8  	"encoding/json"
     9  	"fmt"
    10  	"go/parser"
    11  	"go/token"
    12  	"os"
    13  	"path/filepath"
    14  	"regexp"
    15  	"sort"
    16  	"strconv"
    17  	"strings"
    18  
    19  	"golang.org/x/tools/internal/gocommand"
    20  )
    21  
    22  // processGolistOverlay provides rudimentary support for adding
    23  // files that don't exist on disk to an overlay. The results can be
    24  // sometimes incorrect.
    25  // TODO(matloob): Handle unsupported cases, including the following:
    26  // - determining the correct package to add given a new import path
    27  func (state *golistState) processGolistOverlay(response *responseDeduper) (modifiedPkgs, needPkgs []string, err error) {
    28  	havePkgs := make(map[string]string) // importPath -> non-test package ID
    29  	needPkgsSet := make(map[string]bool)
    30  	modifiedPkgsSet := make(map[string]bool)
    31  
    32  	pkgOfDir := make(map[string][]*Package)
    33  	for _, pkg := range response.dr.Packages {
    34  		// This is an approximation of import path to id. This can be
    35  		// wrong for tests, vendored packages, and a number of other cases.
    36  		havePkgs[pkg.PkgPath] = pkg.ID
    37  		dir, err := commonDir(pkg.GoFiles)
    38  		if err != nil {
    39  			return nil, nil, err
    40  		}
    41  		if dir != "" {
    42  			pkgOfDir[dir] = append(pkgOfDir[dir], pkg)
    43  		}
    44  	}
    45  
    46  	// If no new imports are added, it is safe to avoid loading any needPkgs.
    47  	// Otherwise, it's hard to tell which package is actually being loaded
    48  	// (due to vendoring) and whether any modified package will show up
    49  	// in the transitive set of dependencies (because new imports are added,
    50  	// potentially modifying the transitive set of dependencies).
    51  	var overlayAddsImports bool
    52  
    53  	// If both a package and its test package are created by the overlay, we
    54  	// need the real package first. Process all non-test files before test
    55  	// files, and make the whole process deterministic while we're at it.
    56  	var overlayFiles []string
    57  	for opath := range state.cfg.Overlay {
    58  		overlayFiles = append(overlayFiles, opath)
    59  	}
    60  	sort.Slice(overlayFiles, func(i, j int) bool {
    61  		iTest := strings.HasSuffix(overlayFiles[i], "_test.go")
    62  		jTest := strings.HasSuffix(overlayFiles[j], "_test.go")
    63  		if iTest != jTest {
    64  			return !iTest // non-tests are before tests.
    65  		}
    66  		return overlayFiles[i] < overlayFiles[j]
    67  	})
    68  	for _, opath := range overlayFiles {
    69  		contents := state.cfg.Overlay[opath]
    70  		base := filepath.Base(opath)
    71  		dir := filepath.Dir(opath)
    72  		var pkg *Package           // if opath belongs to both a package and its test variant, this will be the test variant
    73  		var testVariantOf *Package // if opath is a test file, this is the package it is testing
    74  		var fileExists bool
    75  		isTestFile := strings.HasSuffix(opath, "_test.go")
    76  		pkgName, ok := extractPackageName(opath, contents)
    77  		if !ok {
    78  			// Don't bother adding a file that doesn't even have a parsable package statement
    79  			// to the overlay.
    80  			continue
    81  		}
    82  		// If all the overlay files belong to a different package, change the
    83  		// package name to that package.
    84  		maybeFixPackageName(pkgName, isTestFile, pkgOfDir[dir])
    85  	nextPackage:
    86  		for _, p := range response.dr.Packages {
    87  			if pkgName != p.Name && p.ID != "command-line-arguments" {
    88  				continue
    89  			}
    90  			for _, f := range p.GoFiles {
    91  				if !sameFile(filepath.Dir(f), dir) {
    92  					continue
    93  				}
    94  				// Make sure to capture information on the package's test variant, if needed.
    95  				if isTestFile && !hasTestFiles(p) {
    96  					// TODO(matloob): Are there packages other than the 'production' variant
    97  					// of a package that this can match? This shouldn't match the test main package
    98  					// because the file is generated in another directory.
    99  					testVariantOf = p
   100  					continue nextPackage
   101  				} else if !isTestFile && hasTestFiles(p) {
   102  					// We're examining a test variant, but the overlaid file is
   103  					// a non-test file. Because the overlay implementation
   104  					// (currently) only adds a file to one package, skip this
   105  					// package, so that we can add the file to the production
   106  					// variant of the package. (https://golang.org/issue/36857
   107  					// tracks handling overlays on both the production and test
   108  					// variant of a package).
   109  					continue nextPackage
   110  				}
   111  				if pkg != nil && p != pkg && pkg.PkgPath == p.PkgPath {
   112  					// We have already seen the production version of the
   113  					// for which p is a test variant.
   114  					if hasTestFiles(p) {
   115  						testVariantOf = pkg
   116  					}
   117  				}
   118  				pkg = p
   119  				if filepath.Base(f) == base {
   120  					fileExists = true
   121  				}
   122  			}
   123  		}
   124  		// The overlay could have included an entirely new package or an
   125  		// ad-hoc package. An ad-hoc package is one that we have manually
   126  		// constructed from inadequate `go list` results for a file= query.
   127  		// It will have the ID command-line-arguments.
   128  		if pkg == nil || pkg.ID == "command-line-arguments" {
   129  			// Try to find the module or gopath dir the file is contained in.
   130  			// Then for modules, add the module opath to the beginning.
   131  			pkgPath, ok, err := state.getPkgPath(dir)
   132  			if err != nil {
   133  				return nil, nil, err
   134  			}
   135  			if !ok {
   136  				break
   137  			}
   138  			var forTest string // only set for x tests
   139  			isXTest := strings.HasSuffix(pkgName, "_test")
   140  			if isXTest {
   141  				forTest = pkgPath
   142  				pkgPath += "_test"
   143  			}
   144  			id := pkgPath
   145  			if isTestFile {
   146  				if isXTest {
   147  					id = fmt.Sprintf("%s [%s.test]", pkgPath, forTest)
   148  				} else {
   149  					id = fmt.Sprintf("%s [%s.test]", pkgPath, pkgPath)
   150  				}
   151  			}
   152  			if pkg != nil {
   153  				// TODO(rstambler): We should change the package's path and ID
   154  				// here. The only issue is that this messes with the roots.
   155  			} else {
   156  				// Try to reclaim a package with the same ID, if it exists in the response.
   157  				for _, p := range response.dr.Packages {
   158  					if reclaimPackage(p, id, opath, contents) {
   159  						pkg = p
   160  						break
   161  					}
   162  				}
   163  				// Otherwise, create a new package.
   164  				if pkg == nil {
   165  					pkg = &Package{
   166  						PkgPath: pkgPath,
   167  						ID:      id,
   168  						Name:    pkgName,
   169  						Imports: make(map[string]*Package),
   170  					}
   171  					response.addPackage(pkg)
   172  					havePkgs[pkg.PkgPath] = id
   173  					// Add the production package's sources for a test variant.
   174  					if isTestFile && !isXTest && testVariantOf != nil {
   175  						pkg.GoFiles = append(pkg.GoFiles, testVariantOf.GoFiles...)
   176  						pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, testVariantOf.CompiledGoFiles...)
   177  						// Add the package under test and its imports to the test variant.
   178  						pkg.forTest = testVariantOf.PkgPath
   179  						for k, v := range testVariantOf.Imports {
   180  							pkg.Imports[k] = &Package{ID: v.ID}
   181  						}
   182  					}
   183  					if isXTest {
   184  						pkg.forTest = forTest
   185  					}
   186  				}
   187  			}
   188  		}
   189  		if !fileExists {
   190  			pkg.GoFiles = append(pkg.GoFiles, opath)
   191  			// TODO(matloob): Adding the file to CompiledGoFiles can exhibit the wrong behavior
   192  			// if the file will be ignored due to its build tags.
   193  			pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, opath)
   194  			modifiedPkgsSet[pkg.ID] = true
   195  		}
   196  		imports, err := extractImports(opath, contents)
   197  		if err != nil {
   198  			// Let the parser or type checker report errors later.
   199  			continue
   200  		}
   201  		for _, imp := range imports {
   202  			// TODO(rstambler): If the package is an x test and the import has
   203  			// a test variant, make sure to replace it.
   204  			if _, found := pkg.Imports[imp]; found {
   205  				continue
   206  			}
   207  			overlayAddsImports = true
   208  			id, ok := havePkgs[imp]
   209  			if !ok {
   210  				var err error
   211  				id, err = state.resolveImport(dir, imp)
   212  				if err != nil {
   213  					return nil, nil, err
   214  				}
   215  			}
   216  			pkg.Imports[imp] = &Package{ID: id}
   217  			// Add dependencies to the non-test variant version of this package as well.
   218  			if testVariantOf != nil {
   219  				testVariantOf.Imports[imp] = &Package{ID: id}
   220  			}
   221  		}
   222  	}
   223  
   224  	// toPkgPath guesses the package path given the id.
   225  	toPkgPath := func(sourceDir, id string) (string, error) {
   226  		if i := strings.IndexByte(id, ' '); i >= 0 {
   227  			return state.resolveImport(sourceDir, id[:i])
   228  		}
   229  		return state.resolveImport(sourceDir, id)
   230  	}
   231  
   232  	// Now that new packages have been created, do another pass to determine
   233  	// the new set of missing packages.
   234  	for _, pkg := range response.dr.Packages {
   235  		for _, imp := range pkg.Imports {
   236  			if len(pkg.GoFiles) == 0 {
   237  				return nil, nil, fmt.Errorf("cannot resolve imports for package %q with no Go files", pkg.PkgPath)
   238  			}
   239  			pkgPath, err := toPkgPath(filepath.Dir(pkg.GoFiles[0]), imp.ID)
   240  			if err != nil {
   241  				return nil, nil, err
   242  			}
   243  			if _, ok := havePkgs[pkgPath]; !ok {
   244  				needPkgsSet[pkgPath] = true
   245  			}
   246  		}
   247  	}
   248  
   249  	if overlayAddsImports {
   250  		needPkgs = make([]string, 0, len(needPkgsSet))
   251  		for pkg := range needPkgsSet {
   252  			needPkgs = append(needPkgs, pkg)
   253  		}
   254  	}
   255  	modifiedPkgs = make([]string, 0, len(modifiedPkgsSet))
   256  	for pkg := range modifiedPkgsSet {
   257  		modifiedPkgs = append(modifiedPkgs, pkg)
   258  	}
   259  	return modifiedPkgs, needPkgs, err
   260  }
   261  
   262  // resolveImport finds the ID of a package given its import path.
   263  // In particular, it will find the right vendored copy when in GOPATH mode.
   264  func (state *golistState) resolveImport(sourceDir, importPath string) (string, error) {
   265  	env, err := state.getEnv()
   266  	if err != nil {
   267  		return "", err
   268  	}
   269  	if env["GOMOD"] != "" {
   270  		return importPath, nil
   271  	}
   272  
   273  	searchDir := sourceDir
   274  	for {
   275  		vendorDir := filepath.Join(searchDir, "vendor")
   276  		exists, ok := state.vendorDirs[vendorDir]
   277  		if !ok {
   278  			info, err := os.Stat(vendorDir)
   279  			exists = err == nil && info.IsDir()
   280  			state.vendorDirs[vendorDir] = exists
   281  		}
   282  
   283  		if exists {
   284  			vendoredPath := filepath.Join(vendorDir, importPath)
   285  			if info, err := os.Stat(vendoredPath); err == nil && info.IsDir() {
   286  				// We should probably check for .go files here, but shame on anyone who fools us.
   287  				path, ok, err := state.getPkgPath(vendoredPath)
   288  				if err != nil {
   289  					return "", err
   290  				}
   291  				if ok {
   292  					return path, nil
   293  				}
   294  			}
   295  		}
   296  
   297  		// We know we've hit the top of the filesystem when we Dir / and get /,
   298  		// or C:\ and get C:\, etc.
   299  		next := filepath.Dir(searchDir)
   300  		if next == searchDir {
   301  			break
   302  		}
   303  		searchDir = next
   304  	}
   305  	return importPath, nil
   306  }
   307  
   308  func hasTestFiles(p *Package) bool {
   309  	for _, f := range p.GoFiles {
   310  		if strings.HasSuffix(f, "_test.go") {
   311  			return true
   312  		}
   313  	}
   314  	return false
   315  }
   316  
   317  // determineRootDirs returns a mapping from absolute directories that could
   318  // contain code to their corresponding import path prefixes.
   319  func (state *golistState) determineRootDirs() (map[string]string, error) {
   320  	env, err := state.getEnv()
   321  	if err != nil {
   322  		return nil, err
   323  	}
   324  	if env["GOMOD"] != "" {
   325  		state.rootsOnce.Do(func() {
   326  			state.rootDirs, state.rootDirsError = state.determineRootDirsModules()
   327  		})
   328  	} else {
   329  		state.rootsOnce.Do(func() {
   330  			state.rootDirs, state.rootDirsError = state.determineRootDirsGOPATH()
   331  		})
   332  	}
   333  	return state.rootDirs, state.rootDirsError
   334  }
   335  
   336  func (state *golistState) determineRootDirsModules() (map[string]string, error) {
   337  	// List all of the modules--the first will be the directory for the main
   338  	// module. Any replaced modules will also need to be treated as roots.
   339  	// Editing files in the module cache isn't a great idea, so we don't
   340  	// plan to ever support that.
   341  	out, err := state.invokeGo("list", "-m", "-json", "all")
   342  	if err != nil {
   343  		// 'go list all' will fail if we're outside of a module and
   344  		// GO111MODULE=on. Try falling back without 'all'.
   345  		var innerErr error
   346  		out, innerErr = state.invokeGo("list", "-m", "-json")
   347  		if innerErr != nil {
   348  			return nil, err
   349  		}
   350  	}
   351  	roots := map[string]string{}
   352  	modules := map[string]string{}
   353  	var i int
   354  	for dec := json.NewDecoder(out); dec.More(); {
   355  		mod := new(gocommand.ModuleJSON)
   356  		if err := dec.Decode(mod); err != nil {
   357  			return nil, err
   358  		}
   359  		if mod.Dir != "" && mod.Path != "" {
   360  			// This is a valid module; add it to the map.
   361  			absDir, err := filepath.Abs(mod.Dir)
   362  			if err != nil {
   363  				return nil, err
   364  			}
   365  			modules[absDir] = mod.Path
   366  			// The first result is the main module.
   367  			if i == 0 || mod.Replace != nil && mod.Replace.Path != "" {
   368  				roots[absDir] = mod.Path
   369  			}
   370  		}
   371  		i++
   372  	}
   373  	return roots, nil
   374  }
   375  
   376  func (state *golistState) determineRootDirsGOPATH() (map[string]string, error) {
   377  	m := map[string]string{}
   378  	for _, dir := range filepath.SplitList(state.mustGetEnv()["GOPATH"]) {
   379  		absDir, err := filepath.Abs(dir)
   380  		if err != nil {
   381  			return nil, err
   382  		}
   383  		m[filepath.Join(absDir, "src")] = ""
   384  	}
   385  	return m, nil
   386  }
   387  
   388  func extractImports(filename string, contents []byte) ([]string, error) {
   389  	f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.ImportsOnly) // TODO(matloob): reuse fileset?
   390  	if err != nil {
   391  		return nil, err
   392  	}
   393  	var res []string
   394  	for _, imp := range f.Imports {
   395  		quotedPath := imp.Path.Value
   396  		path, err := strconv.Unquote(quotedPath)
   397  		if err != nil {
   398  			return nil, err
   399  		}
   400  		res = append(res, path)
   401  	}
   402  	return res, nil
   403  }
   404  
   405  // reclaimPackage attempts to reuse a package that failed to load in an overlay.
   406  //
   407  // If the package has errors and has no Name, GoFiles, or Imports,
   408  // then it's possible that it doesn't yet exist on disk.
   409  func reclaimPackage(pkg *Package, id string, filename string, contents []byte) bool {
   410  	// TODO(rstambler): Check the message of the actual error?
   411  	// It differs between $GOPATH and module mode.
   412  	if pkg.ID != id {
   413  		return false
   414  	}
   415  	if len(pkg.Errors) != 1 {
   416  		return false
   417  	}
   418  	if pkg.Name != "" || pkg.ExportFile != "" {
   419  		return false
   420  	}
   421  	if len(pkg.GoFiles) > 0 || len(pkg.CompiledGoFiles) > 0 || len(pkg.OtherFiles) > 0 {
   422  		return false
   423  	}
   424  	if len(pkg.Imports) > 0 {
   425  		return false
   426  	}
   427  	pkgName, ok := extractPackageName(filename, contents)
   428  	if !ok {
   429  		return false
   430  	}
   431  	pkg.Name = pkgName
   432  	pkg.Errors = nil
   433  	return true
   434  }
   435  
   436  func extractPackageName(filename string, contents []byte) (string, bool) {
   437  	// TODO(rstambler): Check the message of the actual error?
   438  	// It differs between $GOPATH and module mode.
   439  	f, err := parser.ParseFile(token.NewFileSet(), filename, contents, parser.PackageClauseOnly) // TODO(matloob): reuse fileset?
   440  	if err != nil {
   441  		return "", false
   442  	}
   443  	return f.Name.Name, true
   444  }
   445  
   446  // commonDir returns the directory that all files are in, "" if files is empty,
   447  // or an error if they aren't in the same directory.
   448  func commonDir(files []string) (string, error) {
   449  	seen := make(map[string]bool)
   450  	for _, f := range files {
   451  		seen[filepath.Dir(f)] = true
   452  	}
   453  	if len(seen) > 1 {
   454  		return "", fmt.Errorf("files (%v) are in more than one directory: %v", files, seen)
   455  	}
   456  	for k := range seen {
   457  		// seen has only one element; return it.
   458  		return k, nil
   459  	}
   460  	return "", nil // no files
   461  }
   462  
   463  // It is possible that the files in the disk directory dir have a different package
   464  // name from newName, which is deduced from the overlays. If they all have a different
   465  // package name, and they all have the same package name, then that name becomes
   466  // the package name.
   467  // It returns true if it changes the package name, false otherwise.
   468  func maybeFixPackageName(newName string, isTestFile bool, pkgsOfDir []*Package) {
   469  	names := make(map[string]int)
   470  	for _, p := range pkgsOfDir {
   471  		names[p.Name]++
   472  	}
   473  	if len(names) != 1 {
   474  		// some files are in different packages
   475  		return
   476  	}
   477  	var oldName string
   478  	for k := range names {
   479  		oldName = k
   480  	}
   481  	if newName == oldName {
   482  		return
   483  	}
   484  	// We might have a case where all of the package names in the directory are
   485  	// the same, but the overlay file is for an x test, which belongs to its
   486  	// own package. If the x test does not yet exist on disk, we may not yet
   487  	// have its package name on disk, but we should not rename the packages.
   488  	//
   489  	// We use a heuristic to determine if this file belongs to an x test:
   490  	// The test file should have a package name whose package name has a _test
   491  	// suffix or looks like "newName_test".
   492  	maybeXTest := strings.HasPrefix(oldName+"_test", newName) || strings.HasSuffix(newName, "_test")
   493  	if isTestFile && maybeXTest {
   494  		return
   495  	}
   496  	for _, p := range pkgsOfDir {
   497  		p.Name = newName
   498  	}
   499  }
   500  
   501  // This function is copy-pasted from
   502  // https://github.com/golang/go/blob/9706f510a5e2754595d716bd64be8375997311fb/src/cmd/go/internal/search/search.go#L360.
   503  // It should be deleted when we remove support for overlays from go/packages.
   504  //
   505  // NOTE: This does not handle any ./... or ./ style queries, as this function
   506  // doesn't know the working directory.
   507  //
   508  // matchPattern(pattern)(name) reports whether
   509  // name matches pattern. Pattern is a limited glob
   510  // pattern in which '...' means 'any string' and there
   511  // is no other special syntax.
   512  // Unfortunately, there are two special cases. Quoting "go help packages":
   513  //
   514  // First, /... at the end of the pattern can match an empty string,
   515  // so that net/... matches both net and packages in its subdirectories, like net/http.
   516  // Second, any slash-separated pattern element containing a wildcard never
   517  // participates in a match of the "vendor" element in the path of a vendored
   518  // package, so that ./... does not match packages in subdirectories of
   519  // ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do.
   520  // Note, however, that a directory named vendor that itself contains code
   521  // is not a vendored package: cmd/vendor would be a command named vendor,
   522  // and the pattern cmd/... matches it.
   523  func matchPattern(pattern string) func(name string) bool {
   524  	// Convert pattern to regular expression.
   525  	// The strategy for the trailing /... is to nest it in an explicit ? expression.
   526  	// The strategy for the vendor exclusion is to change the unmatchable
   527  	// vendor strings to a disallowed code point (vendorChar) and to use
   528  	// "(anything but that codepoint)*" as the implementation of the ... wildcard.
   529  	// This is a bit complicated but the obvious alternative,
   530  	// namely a hand-written search like in most shell glob matchers,
   531  	// is too easy to make accidentally exponential.
   532  	// Using package regexp guarantees linear-time matching.
   533  
   534  	const vendorChar = "\x00"
   535  
   536  	if strings.Contains(pattern, vendorChar) {
   537  		return func(name string) bool { return false }
   538  	}
   539  
   540  	re := regexp.QuoteMeta(pattern)
   541  	re = replaceVendor(re, vendorChar)
   542  	switch {
   543  	case strings.HasSuffix(re, `/`+vendorChar+`/\.\.\.`):
   544  		re = strings.TrimSuffix(re, `/`+vendorChar+`/\.\.\.`) + `(/vendor|/` + vendorChar + `/\.\.\.)`
   545  	case re == vendorChar+`/\.\.\.`:
   546  		re = `(/vendor|/` + vendorChar + `/\.\.\.)`
   547  	case strings.HasSuffix(re, `/\.\.\.`):
   548  		re = strings.TrimSuffix(re, `/\.\.\.`) + `(/\.\.\.)?`
   549  	}
   550  	re = strings.ReplaceAll(re, `\.\.\.`, `[^`+vendorChar+`]*`)
   551  
   552  	reg := regexp.MustCompile(`^` + re + `$`)
   553  
   554  	return func(name string) bool {
   555  		if strings.Contains(name, vendorChar) {
   556  			return false
   557  		}
   558  		return reg.MatchString(replaceVendor(name, vendorChar))
   559  	}
   560  }
   561  
   562  // replaceVendor returns the result of replacing
   563  // non-trailing vendor path elements in x with repl.
   564  func replaceVendor(x, repl string) string {
   565  	if !strings.Contains(x, "vendor") {
   566  		return x
   567  	}
   568  	elem := strings.Split(x, "/")
   569  	for i := 0; i < len(elem)-1; i++ {
   570  		if elem[i] == "vendor" {
   571  			elem[i] = repl
   572  		}
   573  	}
   574  	return strings.Join(elem, "/")
   575  }
   576
View as plain text