tests.go

Documentation: golang.org/x/tools/go/analysis/passes/tests

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package tests defines an Analyzer that checks for common mistaken
     6  // usages of tests and examples.
     7  package tests
     8  
     9  import (
    10  	"fmt"
    11  	"go/ast"
    12  	"go/token"
    13  	"go/types"
    14  	"regexp"
    15  	"strings"
    16  	"unicode"
    17  	"unicode/utf8"
    18  
    19  	"golang.org/x/tools/go/analysis"
    20  	"golang.org/x/tools/internal/analysisinternal"
    21  	"golang.org/x/tools/internal/typeparams"
    22  )
    23  
    24  const Doc = `check for common mistaken usages of tests and examples
    25  
    26  The tests checker walks Test, Benchmark and Example functions checking
    27  malformed names, wrong signatures and examples documenting non-existent
    28  identifiers.
    29  
    30  Please see the documentation for package testing in golang.org/pkg/testing
    31  for the conventions that are enforced for Tests, Benchmarks, and Examples.`
    32  
    33  var Analyzer = &analysis.Analyzer{
    34  	Name: "tests",
    35  	Doc:  Doc,
    36  	Run:  run,
    37  }
    38  
    39  var acceptedFuzzTypes = []types.Type{
    40  	types.Typ[types.String],
    41  	types.Typ[types.Bool],
    42  	types.Typ[types.Float32],
    43  	types.Typ[types.Float64],
    44  	types.Typ[types.Int],
    45  	types.Typ[types.Int8],
    46  	types.Typ[types.Int16],
    47  	types.Typ[types.Int32],
    48  	types.Typ[types.Int64],
    49  	types.Typ[types.Uint],
    50  	types.Typ[types.Uint8],
    51  	types.Typ[types.Uint16],
    52  	types.Typ[types.Uint32],
    53  	types.Typ[types.Uint64],
    54  	types.NewSlice(types.Universe.Lookup("byte").Type()),
    55  }
    56  
    57  func run(pass *analysis.Pass) (interface{}, error) {
    58  	for _, f := range pass.Files {
    59  		if !strings.HasSuffix(pass.Fset.File(f.Pos()).Name(), "_test.go") {
    60  			continue
    61  		}
    62  		for _, decl := range f.Decls {
    63  			fn, ok := decl.(*ast.FuncDecl)
    64  			if !ok || fn.Recv != nil {
    65  				// Ignore non-functions or functions with receivers.
    66  				continue
    67  			}
    68  			switch {
    69  			case strings.HasPrefix(fn.Name.Name, "Example"):
    70  				checkExampleName(pass, fn)
    71  				checkExampleOutput(pass, fn, f.Comments)
    72  			case strings.HasPrefix(fn.Name.Name, "Test"):
    73  				checkTest(pass, fn, "Test")
    74  			case strings.HasPrefix(fn.Name.Name, "Benchmark"):
    75  				checkTest(pass, fn, "Benchmark")
    76  			}
    77  			// run fuzz tests diagnostics only for 1.18 i.e. when analysisinternal.DiagnoseFuzzTests is turned on.
    78  			if strings.HasPrefix(fn.Name.Name, "Fuzz") && analysisinternal.DiagnoseFuzzTests {
    79  				checkTest(pass, fn, "Fuzz")
    80  				checkFuzz(pass, fn)
    81  			}
    82  		}
    83  	}
    84  	return nil, nil
    85  }
    86  
    87  // checkFuzz checks the contents of a fuzz function.
    88  func checkFuzz(pass *analysis.Pass, fn *ast.FuncDecl) {
    89  	params := checkFuzzCall(pass, fn)
    90  	if params != nil {
    91  		checkAddCalls(pass, fn, params)
    92  	}
    93  }
    94  
    95  // checkFuzzCall checks the arguments of f.Fuzz() calls:
    96  //
    97  //  1. f.Fuzz() should call a function and it should be of type (*testing.F).Fuzz().
    98  //  2. The called function in f.Fuzz(func(){}) should not return result.
    99  //  3. First argument of func() should be of type *testing.T
   100  //  4. Second argument onwards should be of type []byte, string, bool, byte,
   101  //     rune, float32, float64, int, int8, int16, int32, int64, uint, uint8, uint16,
   102  //     uint32, uint64
   103  //  5. func() must not call any *F methods, e.g. (*F).Log, (*F).Error, (*F).Skip
   104  //     The only *F methods that are allowed in the (*F).Fuzz function are (*F).Failed and (*F).Name.
   105  //
   106  // Returns the list of parameters to the fuzz function, if they are valid fuzz parameters.
   107  func checkFuzzCall(pass *analysis.Pass, fn *ast.FuncDecl) (params *types.Tuple) {
   108  	ast.Inspect(fn, func(n ast.Node) bool {
   109  		call, ok := n.(*ast.CallExpr)
   110  		if ok {
   111  			if !isFuzzTargetDotFuzz(pass, call) {
   112  				return true
   113  			}
   114  
   115  			// Only one argument (func) must be passed to (*testing.F).Fuzz.
   116  			if len(call.Args) != 1 {
   117  				return true
   118  			}
   119  			expr := call.Args[0]
   120  			if pass.TypesInfo.Types[expr].Type == nil {
   121  				return true
   122  			}
   123  			t := pass.TypesInfo.Types[expr].Type.Underlying()
   124  			tSign, argOk := t.(*types.Signature)
   125  			// Argument should be a function
   126  			if !argOk {
   127  				pass.ReportRangef(expr, "argument to Fuzz must be a function")
   128  				return false
   129  			}
   130  			// ff Argument function should not return
   131  			if tSign.Results().Len() != 0 {
   132  				pass.ReportRangef(expr, "fuzz target must not return any value")
   133  			}
   134  			// ff Argument function should have 1 or more argument
   135  			if tSign.Params().Len() == 0 {
   136  				pass.ReportRangef(expr, "fuzz target must have 1 or more argument")
   137  				return false
   138  			}
   139  			ok := validateFuzzArgs(pass, tSign.Params(), expr)
   140  			if ok && params == nil {
   141  				params = tSign.Params()
   142  			}
   143  			// Inspect the function that was passed as an argument to make sure that
   144  			// there are no calls to *F methods, except for Name and Failed.
   145  			ast.Inspect(expr, func(n ast.Node) bool {
   146  				if call, ok := n.(*ast.CallExpr); ok {
   147  					if !isFuzzTargetDot(pass, call, "") {
   148  						return true
   149  					}
   150  					if !isFuzzTargetDot(pass, call, "Name") && !isFuzzTargetDot(pass, call, "Failed") {
   151  						pass.ReportRangef(call, "fuzz target must not call any *F methods")
   152  					}
   153  				}
   154  				return true
   155  			})
   156  			// We do not need to look at any calls to f.Fuzz inside of a Fuzz call,
   157  			// since they are not allowed.
   158  			return false
   159  		}
   160  		return true
   161  	})
   162  	return params
   163  }
   164  
   165  // checkAddCalls checks that the arguments of f.Add calls have the same number and type of arguments as
   166  // the signature of the function passed to (*testing.F).Fuzz
   167  func checkAddCalls(pass *analysis.Pass, fn *ast.FuncDecl, params *types.Tuple) {
   168  	ast.Inspect(fn, func(n ast.Node) bool {
   169  		call, ok := n.(*ast.CallExpr)
   170  		if ok {
   171  			if !isFuzzTargetDotAdd(pass, call) {
   172  				return true
   173  			}
   174  
   175  			// The first argument to function passed to (*testing.F).Fuzz is (*testing.T).
   176  			if len(call.Args) != params.Len()-1 {
   177  				pass.ReportRangef(call, "wrong number of values in call to (*testing.F).Add: %d, fuzz target expects %d", len(call.Args), params.Len()-1)
   178  				return true
   179  			}
   180  			var mismatched []int
   181  			for i, expr := range call.Args {
   182  				if pass.TypesInfo.Types[expr].Type == nil {
   183  					return true
   184  				}
   185  				t := pass.TypesInfo.Types[expr].Type
   186  				if !types.Identical(t, params.At(i+1).Type()) {
   187  					mismatched = append(mismatched, i)
   188  				}
   189  			}
   190  			// If just one of the types is mismatched report for that
   191  			// type only. Otherwise report for the whole call to (*testing.F).Add
   192  			if len(mismatched) == 1 {
   193  				i := mismatched[0]
   194  				expr := call.Args[i]
   195  				t := pass.TypesInfo.Types[expr].Type
   196  				pass.ReportRangef(expr, fmt.Sprintf("mismatched type in call to (*testing.F).Add: %v, fuzz target expects %v", t, params.At(i+1).Type()))
   197  			} else if len(mismatched) > 1 {
   198  				var gotArgs, wantArgs []types.Type
   199  				for i := 0; i < len(call.Args); i++ {
   200  					gotArgs, wantArgs = append(gotArgs, pass.TypesInfo.Types[call.Args[i]].Type), append(wantArgs, params.At(i+1).Type())
   201  				}
   202  				pass.ReportRangef(call, fmt.Sprintf("mismatched types in call to (*testing.F).Add: %v, fuzz target expects %v", gotArgs, wantArgs))
   203  			}
   204  		}
   205  		return true
   206  	})
   207  }
   208  
   209  // isFuzzTargetDotFuzz reports whether call is (*testing.F).Fuzz().
   210  func isFuzzTargetDotFuzz(pass *analysis.Pass, call *ast.CallExpr) bool {
   211  	return isFuzzTargetDot(pass, call, "Fuzz")
   212  }
   213  
   214  // isFuzzTargetDotAdd reports whether call is (*testing.F).Add().
   215  func isFuzzTargetDotAdd(pass *analysis.Pass, call *ast.CallExpr) bool {
   216  	return isFuzzTargetDot(pass, call, "Add")
   217  }
   218  
   219  // isFuzzTargetDot reports whether call is (*testing.F).<name>().
   220  func isFuzzTargetDot(pass *analysis.Pass, call *ast.CallExpr, name string) bool {
   221  	if selExpr, ok := call.Fun.(*ast.SelectorExpr); ok {
   222  		if !isTestingType(pass.TypesInfo.Types[selExpr.X].Type, "F") {
   223  			return false
   224  		}
   225  		if name == "" || selExpr.Sel.Name == name {
   226  			return true
   227  		}
   228  	}
   229  	return false
   230  }
   231  
   232  // Validate the arguments of fuzz target.
   233  func validateFuzzArgs(pass *analysis.Pass, params *types.Tuple, expr ast.Expr) bool {
   234  	fLit, isFuncLit := expr.(*ast.FuncLit)
   235  	exprRange := expr
   236  	ok := true
   237  	if !isTestingType(params.At(0).Type(), "T") {
   238  		if isFuncLit {
   239  			exprRange = fLit.Type.Params.List[0].Type
   240  		}
   241  		pass.ReportRangef(exprRange, "the first parameter of a fuzz target must be *testing.T")
   242  		ok = false
   243  	}
   244  	for i := 1; i < params.Len(); i++ {
   245  		if !isAcceptedFuzzType(params.At(i).Type()) {
   246  			if isFuncLit {
   247  				curr := 0
   248  				for _, field := range fLit.Type.Params.List {
   249  					curr += len(field.Names)
   250  					if i < curr {
   251  						exprRange = field.Type
   252  						break
   253  					}
   254  				}
   255  			}
   256  			pass.ReportRangef(exprRange, "fuzzing arguments can only have the following types: "+formatAcceptedFuzzType())
   257  			ok = false
   258  		}
   259  	}
   260  	return ok
   261  }
   262  
   263  func isTestingType(typ types.Type, testingType string) bool {
   264  	ptr, ok := typ.(*types.Pointer)
   265  	if !ok {
   266  		return false
   267  	}
   268  	named, ok := ptr.Elem().(*types.Named)
   269  	if !ok {
   270  		return false
   271  	}
   272  	obj := named.Obj()
   273  	// obj.Pkg is nil for the error type.
   274  	return obj != nil && obj.Pkg() != nil && obj.Pkg().Path() == "testing" && obj.Name() == testingType
   275  }
   276  
   277  // Validate that fuzz target function's arguments are of accepted types.
   278  func isAcceptedFuzzType(paramType types.Type) bool {
   279  	for _, typ := range acceptedFuzzTypes {
   280  		if types.Identical(typ, paramType) {
   281  			return true
   282  		}
   283  	}
   284  	return false
   285  }
   286  
   287  func formatAcceptedFuzzType() string {
   288  	var acceptedFuzzTypesStrings []string
   289  	for _, typ := range acceptedFuzzTypes {
   290  		acceptedFuzzTypesStrings = append(acceptedFuzzTypesStrings, typ.String())
   291  	}
   292  	acceptedFuzzTypesMsg := strings.Join(acceptedFuzzTypesStrings, ", ")
   293  	return acceptedFuzzTypesMsg
   294  }
   295  
   296  func isExampleSuffix(s string) bool {
   297  	r, size := utf8.DecodeRuneInString(s)
   298  	return size > 0 && unicode.IsLower(r)
   299  }
   300  
   301  func isTestSuffix(name string) bool {
   302  	if len(name) == 0 {
   303  		// "Test" is ok.
   304  		return true
   305  	}
   306  	r, _ := utf8.DecodeRuneInString(name)
   307  	return !unicode.IsLower(r)
   308  }
   309  
   310  func isTestParam(typ ast.Expr, wantType string) bool {
   311  	ptr, ok := typ.(*ast.StarExpr)
   312  	if !ok {
   313  		// Not a pointer.
   314  		return false
   315  	}
   316  	// No easy way of making sure it's a *testing.T or *testing.B:
   317  	// ensure the name of the type matches.
   318  	if name, ok := ptr.X.(*ast.Ident); ok {
   319  		return name.Name == wantType
   320  	}
   321  	if sel, ok := ptr.X.(*ast.SelectorExpr); ok {
   322  		return sel.Sel.Name == wantType
   323  	}
   324  	return false
   325  }
   326  
   327  func lookup(pkg *types.Package, name string) []types.Object {
   328  	if o := pkg.Scope().Lookup(name); o != nil {
   329  		return []types.Object{o}
   330  	}
   331  
   332  	var ret []types.Object
   333  	// Search through the imports to see if any of them define name.
   334  	// It's hard to tell in general which package is being tested, so
   335  	// for the purposes of the analysis, allow the object to appear
   336  	// in any of the imports. This guarantees there are no false positives
   337  	// because the example needs to use the object so it must be defined
   338  	// in the package or one if its imports. On the other hand, false
   339  	// negatives are possible, but should be rare.
   340  	for _, imp := range pkg.Imports() {
   341  		if obj := imp.Scope().Lookup(name); obj != nil {
   342  			ret = append(ret, obj)
   343  		}
   344  	}
   345  	return ret
   346  }
   347  
   348  // This pattern is taken from /go/src/go/doc/example.go
   349  var outputRe = regexp.MustCompile(`(?i)^[[:space:]]*(unordered )?output:`)
   350  
   351  type commentMetadata struct {
   352  	isOutput bool
   353  	pos      token.Pos
   354  }
   355  
   356  func checkExampleOutput(pass *analysis.Pass, fn *ast.FuncDecl, fileComments []*ast.CommentGroup) {
   357  	commentsInExample := []commentMetadata{}
   358  	numOutputs := 0
   359  
   360  	// Find the comment blocks that are in the example. These comments are
   361  	// guaranteed to be in order of appearance.
   362  	for _, cg := range fileComments {
   363  		if cg.Pos() < fn.Pos() {
   364  			continue
   365  		} else if cg.End() > fn.End() {
   366  			break
   367  		}
   368  
   369  		isOutput := outputRe.MatchString(cg.Text())
   370  		if isOutput {
   371  			numOutputs++
   372  		}
   373  
   374  		commentsInExample = append(commentsInExample, commentMetadata{
   375  			isOutput: isOutput,
   376  			pos:      cg.Pos(),
   377  		})
   378  	}
   379  
   380  	// Change message based on whether there are multiple output comment blocks.
   381  	msg := "output comment block must be the last comment block"
   382  	if numOutputs > 1 {
   383  		msg = "there can only be one output comment block per example"
   384  	}
   385  
   386  	for i, cg := range commentsInExample {
   387  		// Check for output comments that are not the last comment in the example.
   388  		isLast := (i == len(commentsInExample)-1)
   389  		if cg.isOutput && !isLast {
   390  			pass.Report(
   391  				analysis.Diagnostic{
   392  					Pos:     cg.pos,
   393  					Message: msg,
   394  				},
   395  			)
   396  		}
   397  	}
   398  }
   399  
   400  func checkExampleName(pass *analysis.Pass, fn *ast.FuncDecl) {
   401  	fnName := fn.Name.Name
   402  	if params := fn.Type.Params; len(params.List) != 0 {
   403  		pass.Reportf(fn.Pos(), "%s should be niladic", fnName)
   404  	}
   405  	if results := fn.Type.Results; results != nil && len(results.List) != 0 {
   406  		pass.Reportf(fn.Pos(), "%s should return nothing", fnName)
   407  	}
   408  	if tparams := typeparams.ForFuncType(fn.Type); tparams != nil && len(tparams.List) > 0 {
   409  		pass.Reportf(fn.Pos(), "%s should not have type params", fnName)
   410  	}
   411  
   412  	if fnName == "Example" {
   413  		// Nothing more to do.
   414  		return
   415  	}
   416  
   417  	var (
   418  		exName = strings.TrimPrefix(fnName, "Example")
   419  		elems  = strings.SplitN(exName, "_", 3)
   420  		ident  = elems[0]
   421  		objs   = lookup(pass.Pkg, ident)
   422  	)
   423  	if ident != "" && len(objs) == 0 {
   424  		// Check ExampleFoo and ExampleBadFoo.
   425  		pass.Reportf(fn.Pos(), "%s refers to unknown identifier: %s", fnName, ident)
   426  		// Abort since obj is absent and no subsequent checks can be performed.
   427  		return
   428  	}
   429  	if len(elems) < 2 {
   430  		// Nothing more to do.
   431  		return
   432  	}
   433  
   434  	if ident == "" {
   435  		// Check Example_suffix and Example_BadSuffix.
   436  		if residual := strings.TrimPrefix(exName, "_"); !isExampleSuffix(residual) {
   437  			pass.Reportf(fn.Pos(), "%s has malformed example suffix: %s", fnName, residual)
   438  		}
   439  		return
   440  	}
   441  
   442  	mmbr := elems[1]
   443  	if !isExampleSuffix(mmbr) {
   444  		// Check ExampleFoo_Method and ExampleFoo_BadMethod.
   445  		found := false
   446  		// Check if Foo.Method exists in this package or its imports.
   447  		for _, obj := range objs {
   448  			if obj, _, _ := types.LookupFieldOrMethod(obj.Type(), true, obj.Pkg(), mmbr); obj != nil {
   449  				found = true
   450  				break
   451  			}
   452  		}
   453  		if !found {
   454  			pass.Reportf(fn.Pos(), "%s refers to unknown field or method: %s.%s", fnName, ident, mmbr)
   455  		}
   456  	}
   457  	if len(elems) == 3 && !isExampleSuffix(elems[2]) {
   458  		// Check ExampleFoo_Method_suffix and ExampleFoo_Method_Badsuffix.
   459  		pass.Reportf(fn.Pos(), "%s has malformed example suffix: %s", fnName, elems[2])
   460  	}
   461  }
   462  
   463  func checkTest(pass *analysis.Pass, fn *ast.FuncDecl, prefix string) {
   464  	// Want functions with 0 results and 1 parameter.
   465  	if fn.Type.Results != nil && len(fn.Type.Results.List) > 0 ||
   466  		fn.Type.Params == nil ||
   467  		len(fn.Type.Params.List) != 1 ||
   468  		len(fn.Type.Params.List[0].Names) > 1 {
   469  		return
   470  	}
   471  
   472  	// The param must look like a *testing.T or *testing.B.
   473  	if !isTestParam(fn.Type.Params.List[0].Type, prefix[:1]) {
   474  		return
   475  	}
   476  
   477  	if tparams := typeparams.ForFuncType(fn.Type); tparams != nil && len(tparams.List) > 0 {
   478  		// Note: cmd/go/internal/load also errors about TestXXX and BenchmarkXXX functions with type parameters.
   479  		// We have currently decided to also warn before compilation/package loading. This can help users in IDEs.
   480  		// TODO(adonovan): use ReportRangef(tparams).
   481  		pass.Reportf(fn.Pos(), "%s has type parameters: it will not be run by go test as a %sXXX function", fn.Name.Name, prefix)
   482  	}
   483  
   484  	if !isTestSuffix(fn.Name.Name[len(prefix):]) {
   485  		// TODO(adonovan): use ReportRangef(fn.Name).
   486  		pass.Reportf(fn.Pos(), "%s has malformed name: first letter after '%s' must not be lowercase", fn.Name.Name, prefix)
   487  	}
   488  }
   489
View as plain text