analysistest.go

Documentation: golang.org/x/tools/go/analysis/analysistest

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package analysistest provides utilities for testing analyzers.
     6  package analysistest
     7  
     8  import (
     9  	"bytes"
    10  	"fmt"
    11  	"go/format"
    12  	"go/token"
    13  	"go/types"
    14  	"io/ioutil"
    15  	"log"
    16  	"os"
    17  	"path/filepath"
    18  	"regexp"
    19  	"sort"
    20  	"strconv"
    21  	"strings"
    22  	"testing"
    23  	"text/scanner"
    24  
    25  	"golang.org/x/tools/go/analysis"
    26  	"golang.org/x/tools/go/analysis/internal/checker"
    27  	"golang.org/x/tools/go/packages"
    28  	"golang.org/x/tools/internal/diff"
    29  	"golang.org/x/tools/internal/testenv"
    30  	"golang.org/x/tools/txtar"
    31  )
    32  
    33  // WriteFiles is a helper function that creates a temporary directory
    34  // and populates it with a GOPATH-style project using filemap (which
    35  // maps file names to contents). On success it returns the name of the
    36  // directory and a cleanup function to delete it.
    37  func WriteFiles(filemap map[string]string) (dir string, cleanup func(), err error) {
    38  	gopath, err := ioutil.TempDir("", "analysistest")
    39  	if err != nil {
    40  		return "", nil, err
    41  	}
    42  	cleanup = func() { os.RemoveAll(gopath) }
    43  
    44  	for name, content := range filemap {
    45  		filename := filepath.Join(gopath, "src", name)
    46  		os.MkdirAll(filepath.Dir(filename), 0777) // ignore error
    47  		if err := ioutil.WriteFile(filename, []byte(content), 0666); err != nil {
    48  			cleanup()
    49  			return "", nil, err
    50  		}
    51  	}
    52  	return gopath, cleanup, nil
    53  }
    54  
    55  // TestData returns the effective filename of
    56  // the program's "testdata" directory.
    57  // This function may be overridden by projects using
    58  // an alternative build system (such as Blaze) that
    59  // does not run a test in its package directory.
    60  var TestData = func() string {
    61  	testdata, err := filepath.Abs("testdata")
    62  	if err != nil {
    63  		log.Fatal(err)
    64  	}
    65  	return testdata
    66  }
    67  
    68  // Testing is an abstraction of a *testing.T.
    69  type Testing interface {
    70  	Errorf(format string, args ...interface{})
    71  }
    72  
    73  // RunWithSuggestedFixes behaves like Run, but additionally verifies suggested fixes.
    74  // It uses golden files placed alongside the source code under analysis:
    75  // suggested fixes for code in example.go will be compared against example.go.golden.
    76  //
    77  // Golden files can be formatted in one of two ways: as plain Go source code, or as txtar archives.
    78  // In the first case, all suggested fixes will be applied to the original source, which will then be compared against the golden file.
    79  // In the second case, suggested fixes will be grouped by their messages, and each set of fixes will be applied and tested separately.
    80  // Each section in the archive corresponds to a single message.
    81  //
    82  // A golden file using txtar may look like this:
    83  //
    84  //	-- turn into single negation --
    85  //	package pkg
    86  //
    87  //	func fn(b1, b2 bool) {
    88  //		if !b1 { // want `negating a boolean twice`
    89  //			println()
    90  //		}
    91  //	}
    92  //
    93  //	-- remove double negation --
    94  //	package pkg
    95  //
    96  //	func fn(b1, b2 bool) {
    97  //		if b1 { // want `negating a boolean twice`
    98  //			println()
    99  //		}
   100  //	}
   101  func RunWithSuggestedFixes(t Testing, dir string, a *analysis.Analyzer, patterns ...string) []*Result {
   102  	r := Run(t, dir, a, patterns...)
   103  
   104  	// Process each result (package) separately, matching up the suggested
   105  	// fixes into a diff, which we will compare to the .golden file.  We have
   106  	// to do this per-result in case a file appears in two packages, such as in
   107  	// packages with tests, where mypkg/a.go will appear in both mypkg and
   108  	// mypkg.test.  In that case, the analyzer may suggest the same set of
   109  	// changes to a.go for each package.  If we merge all the results, those
   110  	// changes get doubly applied, which will cause conflicts or mismatches.
   111  	// Validating the results separately means as long as the two analyses
   112  	// don't produce conflicting suggestions for a single file, everything
   113  	// should match up.
   114  	for _, act := range r {
   115  		// file -> message -> edits
   116  		fileEdits := make(map[*token.File]map[string][]diff.Edit)
   117  		fileContents := make(map[*token.File][]byte)
   118  
   119  		// Validate edits, prepare the fileEdits map and read the file contents.
   120  		for _, diag := range act.Diagnostics {
   121  			for _, sf := range diag.SuggestedFixes {
   122  				for _, edit := range sf.TextEdits {
   123  					// Validate the edit.
   124  					if edit.Pos > edit.End {
   125  						t.Errorf(
   126  							"diagnostic for analysis %v contains Suggested Fix with malformed edit: pos (%v) > end (%v)",
   127  							act.Pass.Analyzer.Name, edit.Pos, edit.End)
   128  						continue
   129  					}
   130  					file, endfile := act.Pass.Fset.File(edit.Pos), act.Pass.Fset.File(edit.End)
   131  					if file == nil || endfile == nil || file != endfile {
   132  						t.Errorf(
   133  							"diagnostic for analysis %v contains Suggested Fix with malformed spanning files %v and %v",
   134  							act.Pass.Analyzer.Name, file.Name(), endfile.Name())
   135  						continue
   136  					}
   137  					if _, ok := fileContents[file]; !ok {
   138  						contents, err := ioutil.ReadFile(file.Name())
   139  						if err != nil {
   140  							t.Errorf("error reading %s: %v", file.Name(), err)
   141  						}
   142  						fileContents[file] = contents
   143  					}
   144  					if _, ok := fileEdits[file]; !ok {
   145  						fileEdits[file] = make(map[string][]diff.Edit)
   146  					}
   147  					fileEdits[file][sf.Message] = append(fileEdits[file][sf.Message], diff.Edit{
   148  						Start: file.Offset(edit.Pos),
   149  						End:   file.Offset(edit.End),
   150  						New:   string(edit.NewText),
   151  					})
   152  				}
   153  			}
   154  		}
   155  
   156  		for file, fixes := range fileEdits {
   157  			// Get the original file contents.
   158  			orig, ok := fileContents[file]
   159  			if !ok {
   160  				t.Errorf("could not find file contents for %s", file.Name())
   161  				continue
   162  			}
   163  
   164  			// Get the golden file and read the contents.
   165  			ar, err := txtar.ParseFile(file.Name() + ".golden")
   166  			if err != nil {
   167  				t.Errorf("error reading %s.golden: %v", file.Name(), err)
   168  				continue
   169  			}
   170  
   171  			if len(ar.Files) > 0 {
   172  				// one virtual file per kind of suggested fix
   173  
   174  				if len(ar.Comment) != 0 {
   175  					// we allow either just the comment, or just virtual
   176  					// files, not both. it is not clear how "both" should
   177  					// behave.
   178  					t.Errorf("%s.golden has leading comment; we don't know what to do with it", file.Name())
   179  					continue
   180  				}
   181  
   182  				for sf, edits := range fixes {
   183  					found := false
   184  					for _, vf := range ar.Files {
   185  						if vf.Name == sf {
   186  							found = true
   187  							out, err := diff.Apply(string(orig), edits)
   188  							if err != nil {
   189  								t.Errorf("%s: error applying fixes: %v", file.Name(), err)
   190  								continue
   191  							}
   192  							// the file may contain multiple trailing
   193  							// newlines if the user places empty lines
   194  							// between files in the archive. normalize
   195  							// this to a single newline.
   196  							want := string(bytes.TrimRight(vf.Data, "\n")) + "\n"
   197  							formatted, err := format.Source([]byte(out))
   198  							if err != nil {
   199  								t.Errorf("%s: error formatting edited source: %v\n%s", file.Name(), err, out)
   200  								continue
   201  							}
   202  							if got := string(formatted); got != want {
   203  								unified := diff.Unified(fmt.Sprintf("%s.golden [%s]", file.Name(), sf), "actual", want, got)
   204  								t.Errorf("suggested fixes failed for %s:\n%s", file.Name(), unified)
   205  							}
   206  							break
   207  						}
   208  					}
   209  					if !found {
   210  						t.Errorf("no section for suggested fix %q in %s.golden", sf, file.Name())
   211  					}
   212  				}
   213  			} else {
   214  				// all suggested fixes are represented by a single file
   215  
   216  				var catchallEdits []diff.Edit
   217  				for _, edits := range fixes {
   218  					catchallEdits = append(catchallEdits, edits...)
   219  				}
   220  
   221  				out, err := diff.Apply(string(orig), catchallEdits)
   222  				if err != nil {
   223  					t.Errorf("%s: error applying fixes: %v", file.Name(), err)
   224  					continue
   225  				}
   226  				want := string(ar.Comment)
   227  
   228  				formatted, err := format.Source([]byte(out))
   229  				if err != nil {
   230  					t.Errorf("%s: error formatting resulting source: %v\n%s", file.Name(), err, out)
   231  					continue
   232  				}
   233  				if got := string(formatted); got != want {
   234  					unified := diff.Unified(file.Name()+".golden", "actual", want, got)
   235  					t.Errorf("suggested fixes failed for %s:\n%s", file.Name(), unified)
   236  				}
   237  			}
   238  		}
   239  	}
   240  	return r
   241  }
   242  
   243  // Run applies an analysis to the packages denoted by the "go list" patterns.
   244  //
   245  // It loads the packages from the specified GOPATH-style project
   246  // directory using golang.org/x/tools/go/packages, runs the analysis on
   247  // them, and checks that each analysis emits the expected diagnostics
   248  // and facts specified by the contents of '// want ...' comments in the
   249  // package's source files. It treats a comment of the form
   250  // "//...// want..." or "/*...// want... */" as if it starts at 'want'
   251  //
   252  // An expectation of a Diagnostic is specified by a string literal
   253  // containing a regular expression that must match the diagnostic
   254  // message. For example:
   255  //
   256  //	fmt.Printf("%s", 1) // want `cannot provide int 1 to %s`
   257  //
   258  // An expectation of a Fact associated with an object is specified by
   259  // 'name:"pattern"', where name is the name of the object, which must be
   260  // declared on the same line as the comment, and pattern is a regular
   261  // expression that must match the string representation of the fact,
   262  // fmt.Sprint(fact). For example:
   263  //
   264  //	func panicf(format string, args interface{}) { // want panicf:"printfWrapper"
   265  //
   266  // Package facts are specified by the name "package" and appear on
   267  // line 1 of the first source file of the package.
   268  //
   269  // A single 'want' comment may contain a mixture of diagnostic and fact
   270  // expectations, including multiple facts about the same object:
   271  //
   272  //	// want "diag" "diag2" x:"fact1" x:"fact2" y:"fact3"
   273  //
   274  // Unexpected diagnostics and facts, and unmatched expectations, are
   275  // reported as errors to the Testing.
   276  //
   277  // Run reports an error to the Testing if loading or analysis failed.
   278  // Run also returns a Result for each package for which analysis was
   279  // attempted, even if unsuccessful. It is safe for a test to ignore all
   280  // the results, but a test may use it to perform additional checks.
   281  func Run(t Testing, dir string, a *analysis.Analyzer, patterns ...string) []*Result {
   282  	if t, ok := t.(testing.TB); ok {
   283  		testenv.NeedsGoPackages(t)
   284  	}
   285  
   286  	pkgs, err := loadPackages(a, dir, patterns...)
   287  	if err != nil {
   288  		t.Errorf("loading %s: %v", patterns, err)
   289  		return nil
   290  	}
   291  
   292  	results := checker.TestAnalyzer(a, pkgs)
   293  	for _, result := range results {
   294  		if result.Err != nil {
   295  			t.Errorf("error analyzing %s: %v", result.Pass, result.Err)
   296  		} else {
   297  			check(t, dir, result.Pass, result.Diagnostics, result.Facts)
   298  		}
   299  	}
   300  	return results
   301  }
   302  
   303  // A Result holds the result of applying an analyzer to a package.
   304  type Result = checker.TestAnalyzerResult
   305  
   306  // loadPackages uses go/packages to load a specified packages (from source, with
   307  // dependencies) from dir, which is the root of a GOPATH-style project
   308  // tree. It returns an error if any package had an error, or the pattern
   309  // matched no packages.
   310  func loadPackages(a *analysis.Analyzer, dir string, patterns ...string) ([]*packages.Package, error) {
   311  	// packages.Load loads the real standard library, not a minimal
   312  	// fake version, which would be more efficient, especially if we
   313  	// have many small tests that import, say, net/http.
   314  	// However there is no easy way to make go/packages to consume
   315  	// a list of packages we generate and then do the parsing and
   316  	// typechecking, though this feature seems to be a recurring need.
   317  
   318  	mode := packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports |
   319  		packages.NeedTypes | packages.NeedTypesSizes | packages.NeedSyntax | packages.NeedTypesInfo |
   320  		packages.NeedDeps
   321  	cfg := &packages.Config{
   322  		Mode:  mode,
   323  		Dir:   dir,
   324  		Tests: true,
   325  		Env:   append(os.Environ(), "GOPATH="+dir, "GO111MODULE=off", "GOPROXY=off"),
   326  	}
   327  	pkgs, err := packages.Load(cfg, patterns...)
   328  	if err != nil {
   329  		return nil, err
   330  	}
   331  
   332  	// Do NOT print errors if the analyzer will continue running.
   333  	// It is incredibly confusing for tests to be printing to stderr
   334  	// willy-nilly instead of their test logs, especially when the
   335  	// errors are expected and are going to be fixed.
   336  	if !a.RunDespiteErrors {
   337  		packages.PrintErrors(pkgs)
   338  	}
   339  
   340  	if len(pkgs) == 0 {
   341  		return nil, fmt.Errorf("no packages matched %s", patterns)
   342  	}
   343  	return pkgs, nil
   344  }
   345  
   346  // check inspects an analysis pass on which the analysis has already
   347  // been run, and verifies that all reported diagnostics and facts match
   348  // specified by the contents of "// want ..." comments in the package's
   349  // source files, which must have been parsed with comments enabled.
   350  func check(t Testing, gopath string, pass *analysis.Pass, diagnostics []analysis.Diagnostic, facts map[types.Object][]analysis.Fact) {
   351  	type key struct {
   352  		file string
   353  		line int
   354  	}
   355  
   356  	want := make(map[key][]expectation)
   357  
   358  	// processComment parses expectations out of comments.
   359  	processComment := func(filename string, linenum int, text string) {
   360  		text = strings.TrimSpace(text)
   361  
   362  		// Any comment starting with "want" is treated
   363  		// as an expectation, even without following whitespace.
   364  		if rest := strings.TrimPrefix(text, "want"); rest != text {
   365  			lineDelta, expects, err := parseExpectations(rest)
   366  			if err != nil {
   367  				t.Errorf("%s:%d: in 'want' comment: %s", filename, linenum, err)
   368  				return
   369  			}
   370  			if expects != nil {
   371  				want[key{filename, linenum + lineDelta}] = expects
   372  			}
   373  		}
   374  	}
   375  
   376  	// Extract 'want' comments from parsed Go files.
   377  	for _, f := range pass.Files {
   378  		for _, cgroup := range f.Comments {
   379  			for _, c := range cgroup.List {
   380  
   381  				text := strings.TrimPrefix(c.Text, "//")
   382  				if text == c.Text { // not a //-comment.
   383  					text = strings.TrimPrefix(text, "/*")
   384  					text = strings.TrimSuffix(text, "*/")
   385  				}
   386  
   387  				// Hack: treat a comment of the form "//...// want..."
   388  				// or "/*...// want... */
   389  				// as if it starts at 'want'.
   390  				// This allows us to add comments on comments,
   391  				// as required when testing the buildtag analyzer.
   392  				if i := strings.Index(text, "// want"); i >= 0 {
   393  					text = text[i+len("// "):]
   394  				}
   395  
   396  				// It's tempting to compute the filename
   397  				// once outside the loop, but it's
   398  				// incorrect because it can change due
   399  				// to //line directives.
   400  				posn := pass.Fset.Position(c.Pos())
   401  				filename := sanitize(gopath, posn.Filename)
   402  				processComment(filename, posn.Line, text)
   403  			}
   404  		}
   405  	}
   406  
   407  	// Extract 'want' comments from non-Go files.
   408  	// TODO(adonovan): we may need to handle //line directives.
   409  	for _, filename := range pass.OtherFiles {
   410  		data, err := ioutil.ReadFile(filename)
   411  		if err != nil {
   412  			t.Errorf("can't read '// want' comments from %s: %v", filename, err)
   413  			continue
   414  		}
   415  		filename := sanitize(gopath, filename)
   416  		linenum := 0
   417  		for _, line := range strings.Split(string(data), "\n") {
   418  			linenum++
   419  
   420  			// Hack: treat a comment of the form "//...// want..."
   421  			// or "/*...// want... */
   422  			// as if it starts at 'want'.
   423  			// This allows us to add comments on comments,
   424  			// as required when testing the buildtag analyzer.
   425  			if i := strings.Index(line, "// want"); i >= 0 {
   426  				line = line[i:]
   427  			}
   428  
   429  			if i := strings.Index(line, "//"); i >= 0 {
   430  				line = line[i+len("//"):]
   431  				processComment(filename, linenum, line)
   432  			}
   433  		}
   434  	}
   435  
   436  	checkMessage := func(posn token.Position, kind, name, message string) {
   437  		posn.Filename = sanitize(gopath, posn.Filename)
   438  		k := key{posn.Filename, posn.Line}
   439  		expects := want[k]
   440  		var unmatched []string
   441  		for i, exp := range expects {
   442  			if exp.kind == kind && exp.name == name {
   443  				if exp.rx.MatchString(message) {
   444  					// matched: remove the expectation.
   445  					expects[i] = expects[len(expects)-1]
   446  					expects = expects[:len(expects)-1]
   447  					want[k] = expects
   448  					return
   449  				}
   450  				unmatched = append(unmatched, fmt.Sprintf("%#q", exp.rx))
   451  			}
   452  		}
   453  		if unmatched == nil {
   454  			t.Errorf("%v: unexpected %s: %v", posn, kind, message)
   455  		} else {
   456  			t.Errorf("%v: %s %q does not match pattern %s",
   457  				posn, kind, message, strings.Join(unmatched, " or "))
   458  		}
   459  	}
   460  
   461  	// Check the diagnostics match expectations.
   462  	for _, f := range diagnostics {
   463  		// TODO(matloob): Support ranges in analysistest.
   464  		posn := pass.Fset.Position(f.Pos)
   465  		checkMessage(posn, "diagnostic", "", f.Message)
   466  	}
   467  
   468  	// Check the facts match expectations.
   469  	// Report errors in lexical order for determinism.
   470  	// (It's only deterministic within each file, not across files,
   471  	// because go/packages does not guarantee file.Pos is ascending
   472  	// across the files of a single compilation unit.)
   473  	var objects []types.Object
   474  	for obj := range facts {
   475  		objects = append(objects, obj)
   476  	}
   477  	sort.Slice(objects, func(i, j int) bool {
   478  		// Package facts compare less than object facts.
   479  		ip, jp := objects[i] == nil, objects[j] == nil // whether i, j is a package fact
   480  		if ip != jp {
   481  			return ip && !jp
   482  		}
   483  		return objects[i].Pos() < objects[j].Pos()
   484  	})
   485  	for _, obj := range objects {
   486  		var posn token.Position
   487  		var name string
   488  		if obj != nil {
   489  			// Object facts are reported on the declaring line.
   490  			name = obj.Name()
   491  			posn = pass.Fset.Position(obj.Pos())
   492  		} else {
   493  			// Package facts are reported at the start of the file.
   494  			name = "package"
   495  			posn = pass.Fset.Position(pass.Files[0].Pos())
   496  			posn.Line = 1
   497  		}
   498  
   499  		for _, fact := range facts[obj] {
   500  			checkMessage(posn, "fact", name, fmt.Sprint(fact))
   501  		}
   502  	}
   503  
   504  	// Reject surplus expectations.
   505  	//
   506  	// Sometimes an Analyzer reports two similar diagnostics on a
   507  	// line with only one expectation. The reader may be confused by
   508  	// the error message.
   509  	// TODO(adonovan): print a better error:
   510  	// "got 2 diagnostics here; each one needs its own expectation".
   511  	var surplus []string
   512  	for key, expects := range want {
   513  		for _, exp := range expects {
   514  			err := fmt.Sprintf("%s:%d: no %s was reported matching %#q", key.file, key.line, exp.kind, exp.rx)
   515  			surplus = append(surplus, err)
   516  		}
   517  	}
   518  	sort.Strings(surplus)
   519  	for _, err := range surplus {
   520  		t.Errorf("%s", err)
   521  	}
   522  }
   523  
   524  type expectation struct {
   525  	kind string // either "fact" or "diagnostic"
   526  	name string // name of object to which fact belongs, or "package" ("fact" only)
   527  	rx   *regexp.Regexp
   528  }
   529  
   530  func (ex expectation) String() string {
   531  	return fmt.Sprintf("%s %s:%q", ex.kind, ex.name, ex.rx) // for debugging
   532  }
   533  
   534  // parseExpectations parses the content of a "// want ..." comment
   535  // and returns the expectations, a mixture of diagnostics ("rx") and
   536  // facts (name:"rx").
   537  func parseExpectations(text string) (lineDelta int, expects []expectation, err error) {
   538  	var scanErr string
   539  	sc := new(scanner.Scanner).Init(strings.NewReader(text))
   540  	sc.Error = func(s *scanner.Scanner, msg string) {
   541  		scanErr = msg // e.g. bad string escape
   542  	}
   543  	sc.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanInts
   544  
   545  	scanRegexp := func(tok rune) (*regexp.Regexp, error) {
   546  		if tok != scanner.String && tok != scanner.RawString {
   547  			return nil, fmt.Errorf("got %s, want regular expression",
   548  				scanner.TokenString(tok))
   549  		}
   550  		pattern, _ := strconv.Unquote(sc.TokenText()) // can't fail
   551  		return regexp.Compile(pattern)
   552  	}
   553  
   554  	for {
   555  		tok := sc.Scan()
   556  		switch tok {
   557  		case '+':
   558  			tok = sc.Scan()
   559  			if tok != scanner.Int {
   560  				return 0, nil, fmt.Errorf("got +%s, want +Int", scanner.TokenString(tok))
   561  			}
   562  			lineDelta, _ = strconv.Atoi(sc.TokenText())
   563  		case scanner.String, scanner.RawString:
   564  			rx, err := scanRegexp(tok)
   565  			if err != nil {
   566  				return 0, nil, err
   567  			}
   568  			expects = append(expects, expectation{"diagnostic", "", rx})
   569  
   570  		case scanner.Ident:
   571  			name := sc.TokenText()
   572  			tok = sc.Scan()
   573  			if tok != ':' {
   574  				return 0, nil, fmt.Errorf("got %s after %s, want ':'",
   575  					scanner.TokenString(tok), name)
   576  			}
   577  			tok = sc.Scan()
   578  			rx, err := scanRegexp(tok)
   579  			if err != nil {
   580  				return 0, nil, err
   581  			}
   582  			expects = append(expects, expectation{"fact", name, rx})
   583  
   584  		case scanner.EOF:
   585  			if scanErr != "" {
   586  				return 0, nil, fmt.Errorf("%s", scanErr)
   587  			}
   588  			return lineDelta, expects, nil
   589  
   590  		default:
   591  			return 0, nil, fmt.Errorf("unexpected %s", scanner.TokenString(tok))
   592  		}
   593  	}
   594  }
   595  
   596  // sanitize removes the GOPATH portion of the filename,
   597  // typically a gnarly /tmp directory, and returns the rest.
   598  func sanitize(gopath, filename string) string {
   599  	prefix := gopath + string(os.PathSeparator) + "src" + string(os.PathSeparator)
   600  	return filepath.ToSlash(strings.TrimPrefix(filename, prefix))
   601  }
   602
View as plain text