generate_test.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // This file implements a custom generator to create various go/types
     6  // source files from the corresponding types2 files.
     7  
     8  package types_test
     9  
    10  import (
    11  	"bytes"
    12  	"flag"
    13  	"fmt"
    14  	"go/ast"
    15  	"go/format"
    16  	"go/parser"
    17  	"go/token"
    18  	"internal/diff"
    19  	"os"
    20  	"path/filepath"
    21  	"runtime"
    22  	"strings"
    23  	"testing"
    24  )
    25  
    26  var filesToWrite = flag.String("write", "", `go/types files to generate, or "all" for all files`)
    27  
    28  const (
    29  	srcDir = "/src/cmd/compile/internal/types2/"
    30  	dstDir = "/src/go/types/"
    31  )
    32  
    33  // TestGenerate verifies that generated files in go/types match their types2
    34  // counterpart. If -write is set, this test actually writes the expected
    35  // content to go/types; otherwise, it just compares with the existing content.
    36  func TestGenerate(t *testing.T) {
    37  	// If filesToWrite is set, write the generated content to disk.
    38  	// In the special case of "all", write all files in filemap.
    39  	write := *filesToWrite != ""
    40  	var files []string // files to process
    41  	if *filesToWrite != "" && *filesToWrite != "all" {
    42  		files = strings.Split(*filesToWrite, ",")
    43  	} else {
    44  		for file := range filemap {
    45  			files = append(files, file)
    46  		}
    47  	}
    48  
    49  	for _, filename := range files {
    50  		generate(t, filename, write)
    51  	}
    52  }
    53  
    54  func generate(t *testing.T, filename string, write bool) {
    55  	// parse src (cmd/compile/internal/types2)
    56  	srcFilename := filepath.FromSlash(runtime.GOROOT() + srcDir + filename)
    57  	file, err := parser.ParseFile(fset, srcFilename, nil, parser.ParseComments)
    58  	if err != nil {
    59  		t.Fatal(err)
    60  	}
    61  
    62  	// fix package name
    63  	file.Name.Name = strings.ReplaceAll(file.Name.Name, "types2", "types")
    64  
    65  	// rewrite AST as needed
    66  	if action := filemap[filename]; action != nil {
    67  		action(file)
    68  	}
    69  
    70  	// format AST
    71  	var buf bytes.Buffer
    72  	rel, _ := filepath.Rel(dstDir, srcDir)
    73  	fmt.Fprintf(&buf, "// Code generated by \"go test -run=Generate -write=all\"; DO NOT EDIT.\n")
    74  	fmt.Fprintf(&buf, "// Source: %s/%s\n\n", filepath.ToSlash(rel), filename)
    75  	if err := format.Node(&buf, fset, file); err != nil {
    76  		t.Fatal(err)
    77  	}
    78  	generatedContent := buf.Bytes()
    79  
    80  	// read dst (go/types)
    81  	dstFilename := filepath.FromSlash(runtime.GOROOT() + dstDir + filename)
    82  	onDiskContent, err := os.ReadFile(dstFilename)
    83  	if err != nil {
    84  		t.Fatalf("reading %q: %v", filename, err)
    85  	}
    86  
    87  	// compare on-disk dst with buffer generated from src.
    88  	if d := diff.Diff(filename+" (on disk in "+dstDir+")", onDiskContent, filename+" (generated from "+srcDir+")", generatedContent); d != nil {
    89  		if write {
    90  			t.Logf("applying change:\n%s", d)
    91  			if err := os.WriteFile(dstFilename, generatedContent, 0o644); err != nil {
    92  				t.Fatalf("writing %q: %v", filename, err)
    93  			}
    94  		} else {
    95  			t.Errorf("file on disk in %s is stale:\n%s", dstDir, d)
    96  		}
    97  	}
    98  }
    99  
   100  type action func(in *ast.File)
   101  
   102  var filemap = map[string]action{
   103  	"alias.go": fixTokenPos,
   104  	"assignments.go": func(f *ast.File) {
   105  		renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
   106  		renameSelectorExprs(f, "syntax.Name->ast.Ident", "ident.Value->ident.Name", "ast.Pos->token.Pos") // must happen before renaming identifiers
   107  		renameIdents(f, "syntax->ast", "poser->positioner", "nopos->noposn")
   108  	},
   109  	"array.go":          nil,
   110  	"api_predicates.go": nil,
   111  	"basic.go":          nil,
   112  	"builtins.go": func(f *ast.File) {
   113  		renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
   114  		renameIdents(f, "syntax->ast")
   115  		renameSelectors(f, "ArgList->Args")
   116  		fixSelValue(f)
   117  		fixAtPosCall(f)
   118  	},
   119  	"builtins_test.go": func(f *ast.File) {
   120  		renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`, `"cmd/compile/internal/types2"->"go/types"`)
   121  		renameSelectorExprs(f, "syntax.Name->ast.Ident", "p.Value->p.Name") // must happen before renaming identifiers
   122  		renameIdents(f, "syntax->ast")
   123  	},
   124  	"chan.go":         nil,
   125  	"const.go":        fixTokenPos,
   126  	"context.go":      nil,
   127  	"context_test.go": nil,
   128  	"conversions.go":  nil,
   129  	"errors_test.go":  func(f *ast.File) { renameIdents(f, "nopos->noposn") },
   130  	"errsupport.go":   nil,
   131  	"gccgosizes.go":   nil,
   132  	"gcsizes.go":      func(f *ast.File) { renameIdents(f, "IsSyncAtomicAlign64->_IsSyncAtomicAlign64") },
   133  	"hilbert_test.go": func(f *ast.File) { renameImportPath(f, `"cmd/compile/internal/types2"->"go/types"`) },
   134  	"infer.go":        func(f *ast.File) { fixTokenPos(f); fixInferSig(f) },
   135  	"initorder.go":    nil,
   136  	// "initorder.go": fixErrErrorfCall, // disabled for now due to unresolved error_ use implications for gopls
   137  	"instantiate.go":      func(f *ast.File) { fixTokenPos(f); fixCheckErrorfCall(f) },
   138  	"instantiate_test.go": func(f *ast.File) { renameImportPath(f, `"cmd/compile/internal/types2"->"go/types"`) },
   139  	"literals.go": func(f *ast.File) {
   140  		insertImportPath(f, `"go/token"`)
   141  		renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
   142  		renameSelectorExprs(f,
   143  			"syntax.IntLit->token.INT", "syntax.FloatLit->token.FLOAT", "syntax.ImagLit->token.IMAG",
   144  			"syntax.Name->ast.Ident", "key.Value->key.Name", "atyp.Elem->atyp.Elt") // must happen before renaming identifiers
   145  		renameIdents(f, "syntax->ast")
   146  		renameSelectors(f, "ElemList->Elts")
   147  	},
   148  	"lookup.go":    func(f *ast.File) { fixTokenPos(f) },
   149  	"main_test.go": nil,
   150  	"map.go":       nil,
   151  	"mono.go": func(f *ast.File) {
   152  		fixTokenPos(f)
   153  		insertImportPath(f, `"go/ast"`)
   154  		renameSelectorExprs(f, "syntax.Expr->ast.Expr")
   155  	},
   156  	"named.go":  func(f *ast.File) { fixTokenPos(f); renameSelectors(f, "Trace->_Trace") },
   157  	"object.go": func(f *ast.File) { fixTokenPos(f); renameIdents(f, "NewTypeNameLazy->_NewTypeNameLazy") },
   158  	// TODO(gri) needs adjustments for TestObjectString - disabled for now
   159  	// "object_test.go": func(f *ast.File) { renameImportPath(f, `"cmd/compile/internal/types2"->"go/types"`) },
   160  	"objset.go": nil,
   161  	"operand.go": func(f *ast.File) {
   162  		insertImportPath(f, `"go/token"`)
   163  		renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
   164  		renameSelectorExprs(f,
   165  			"syntax.Pos->token.Pos", "syntax.LitKind->token.Token",
   166  			"syntax.IntLit->token.INT", "syntax.FloatLit->token.FLOAT",
   167  			"syntax.ImagLit->token.IMAG", "syntax.RuneLit->token.CHAR",
   168  			"syntax.StringLit->token.STRING") // must happen before renaming identifiers
   169  		renameIdents(f, "syntax->ast")
   170  	},
   171  	"package.go":    nil,
   172  	"pointer.go":    nil,
   173  	"predicates.go": nil,
   174  	"recording.go": func(f *ast.File) {
   175  		renameImportPath(f, `"cmd/compile/internal/syntax"->"go/ast"`)
   176  		renameSelectorExprs(f, "syntax.Name->ast.Ident") // must happen before renaming identifiers
   177  		renameIdents(f, "syntax->ast")
   178  		fixAtPosCall(f)
   179  	},
   180  	"scope.go":         func(f *ast.File) { fixTokenPos(f); renameIdents(f, "InsertLazy->_InsertLazy") },
   181  	"selection.go":     nil,
   182  	"sizes.go":         func(f *ast.File) { renameIdents(f, "IsSyncAtomicAlign64->_IsSyncAtomicAlign64") },
   183  	"slice.go":         nil,
   184  	"subst.go":         func(f *ast.File) { fixTokenPos(f); renameSelectors(f, "Trace->_Trace") },
   185  	"termlist.go":      nil,
   186  	"termlist_test.go": nil,
   187  	"tuple.go":         nil,
   188  	"typelists.go":     nil,
   189  	"typeset.go":       func(f *ast.File) { fixTokenPos(f); renameSelectors(f, "Trace->_Trace") },
   190  	"typeparam.go":     nil,
   191  	"typeterm_test.go": nil,
   192  	"typeterm.go":      nil,
   193  	"typestring.go":    nil,
   194  	"under.go":         nil,
   195  	"unify.go":         fixSprintf,
   196  	"universe.go":      fixGlobalTypVarDecl,
   197  	"util_test.go":     fixTokenPos,
   198  	"validtype.go":     func(f *ast.File) { fixTokenPos(f); renameSelectors(f, "Trace->_Trace") },
   199  }
   200  
   201  // TODO(gri) We should be able to make these rewriters more configurable/composable.
   202  //           For now this is a good starting point.
   203  
   204  // A renameMap maps old strings to new strings.
   205  type renameMap map[string]string
   206  
   207  // makeRenameMap returns a renameMap populates from renames entries of the form "from->to".
   208  func makeRenameMap(renames ...string) renameMap {
   209  	m := make(renameMap)
   210  	for _, r := range renames {
   211  		s := strings.Split(r, "->")
   212  		if len(s) != 2 {
   213  			panic("invalid rename entry: " + r)
   214  		}
   215  		m[s[0]] = s[1]
   216  	}
   217  	return m
   218  }
   219  
   220  // rename renames the given string s if a corresponding rename exists in m.
   221  func (m renameMap) rename(s *string) {
   222  	if r, ok := m[*s]; ok {
   223  		*s = r
   224  	}
   225  }
   226  
   227  // renameSel renames a selector expression of the form a.x to b.x (where a, b are identifiers)
   228  // if m contains the ("a.x" : "b.y") key-value pair.
   229  func (m renameMap) renameSel(n *ast.SelectorExpr) {
   230  	if a, _ := n.X.(*ast.Ident); a != nil {
   231  		a_x := a.Name + "." + n.Sel.Name
   232  		if r, ok := m[a_x]; ok {
   233  			b_y := strings.Split(r, ".")
   234  			if len(b_y) != 2 {
   235  				panic("invalid selector expression: " + r)
   236  			}
   237  			a.Name = b_y[0]
   238  			n.Sel.Name = b_y[1]
   239  		}
   240  	}
   241  }
   242  
   243  // renameIdents renames identifiers: each renames entry is of the form "from->to".
   244  // Note: This doesn't change the use of the identifiers in comments.
   245  func renameIdents(f *ast.File, renames ...string) {
   246  	m := makeRenameMap(renames...)
   247  	ast.Inspect(f, func(n ast.Node) bool {
   248  		switch n := n.(type) {
   249  		case *ast.Ident:
   250  			m.rename(&n.Name)
   251  			return false
   252  		}
   253  		return true
   254  	})
   255  }
   256  
   257  // renameSelectors is like renameIdents but only looks at selectors.
   258  func renameSelectors(f *ast.File, renames ...string) {
   259  	m := makeRenameMap(renames...)
   260  	ast.Inspect(f, func(n ast.Node) bool {
   261  		switch n := n.(type) {
   262  		case *ast.SelectorExpr:
   263  			m.rename(&n.Sel.Name)
   264  			return false
   265  		}
   266  		return true
   267  	})
   268  
   269  }
   270  
   271  // renameSelectorExprs is like renameIdents but only looks at selector expressions.
   272  // Each renames entry must be of the form "x.a->y.b".
   273  func renameSelectorExprs(f *ast.File, renames ...string) {
   274  	m := makeRenameMap(renames...)
   275  	ast.Inspect(f, func(n ast.Node) bool {
   276  		switch n := n.(type) {
   277  		case *ast.SelectorExpr:
   278  			m.renameSel(n)
   279  			return false
   280  		}
   281  		return true
   282  	})
   283  }
   284  
   285  // renameImportPath is like renameIdents but renames import paths.
   286  func renameImportPath(f *ast.File, renames ...string) {
   287  	m := makeRenameMap(renames...)
   288  	ast.Inspect(f, func(n ast.Node) bool {
   289  		switch n := n.(type) {
   290  		case *ast.ImportSpec:
   291  			if n.Path.Kind != token.STRING {
   292  				panic("invalid import path")
   293  			}
   294  			m.rename(&n.Path.Value)
   295  			return false
   296  		}
   297  		return true
   298  	})
   299  }
   300  
   301  // insertImportPath inserts the given import path.
   302  // There must be at least one import declaration present already.
   303  func insertImportPath(f *ast.File, path string) {
   304  	for _, d := range f.Decls {
   305  		if g, _ := d.(*ast.GenDecl); g != nil && g.Tok == token.IMPORT {
   306  			g.Specs = append(g.Specs, &ast.ImportSpec{Path: &ast.BasicLit{ValuePos: g.End(), Kind: token.STRING, Value: path}})
   307  			return
   308  		}
   309  	}
   310  	panic("no import declaration present")
   311  }
   312  
   313  // fixTokenPos changes imports of "cmd/compile/internal/syntax" to "go/token",
   314  // uses of syntax.Pos to token.Pos, and calls to x.IsKnown() to x.IsValid().
   315  func fixTokenPos(f *ast.File) {
   316  	m := makeRenameMap(`"cmd/compile/internal/syntax"->"go/token"`, "syntax.Pos->token.Pos", "IsKnown->IsValid")
   317  	ast.Inspect(f, func(n ast.Node) bool {
   318  		switch n := n.(type) {
   319  		case *ast.ImportSpec:
   320  			// rewrite import path "cmd/compile/internal/syntax" to "go/token"
   321  			if n.Path.Kind != token.STRING {
   322  				panic("invalid import path")
   323  			}
   324  			m.rename(&n.Path.Value)
   325  			return false
   326  		case *ast.SelectorExpr:
   327  			// rewrite syntax.Pos to token.Pos
   328  			m.renameSel(n)
   329  		case *ast.CallExpr:
   330  			// rewrite x.IsKnown() to x.IsValid()
   331  			if fun, _ := n.Fun.(*ast.SelectorExpr); fun != nil && len(n.Args) == 0 {
   332  				m.rename(&fun.Sel.Name)
   333  				return false
   334  			}
   335  		}
   336  		return true
   337  	})
   338  }
   339  
   340  // fixSelValue updates the selector x.Sel.Value to x.Sel.Name.
   341  func fixSelValue(f *ast.File) {
   342  	ast.Inspect(f, func(n ast.Node) bool {
   343  		switch n := n.(type) {
   344  		case *ast.SelectorExpr:
   345  			if n.Sel.Name == "Value" {
   346  				if selx, _ := n.X.(*ast.SelectorExpr); selx != nil && selx.Sel.Name == "Sel" {
   347  					n.Sel.Name = "Name"
   348  					return false
   349  				}
   350  			}
   351  		}
   352  		return true
   353  	})
   354  }
   355  
   356  // fixInferSig updates the Checker.infer signature to use a positioner instead of a token.Position
   357  // as first argument, renames the argument from "pos" to "posn", and updates a few internal uses of
   358  // "pos" to "posn" and "posn.Pos()" respectively.
   359  func fixInferSig(f *ast.File) {
   360  	ast.Inspect(f, func(n ast.Node) bool {
   361  		switch n := n.(type) {
   362  		case *ast.FuncDecl:
   363  			if n.Name.Name == "infer" {
   364  				// rewrite (pos token.Pos, ...) to (posn positioner, ...)
   365  				par := n.Type.Params.List[0]
   366  				if len(par.Names) == 1 && par.Names[0].Name == "pos" {
   367  					par.Names[0] = newIdent(par.Names[0].Pos(), "posn")
   368  					par.Type = newIdent(par.Type.Pos(), "positioner")
   369  					return true
   370  				}
   371  			}
   372  		case *ast.CallExpr:
   373  			if selx, _ := n.Fun.(*ast.SelectorExpr); selx != nil {
   374  				switch selx.Sel.Name {
   375  				case "renameTParams":
   376  					// rewrite check.renameTParams(pos, ... ) to check.renameTParams(posn.Pos(), ... )
   377  					if isIdent(n.Args[0], "pos") {
   378  						pos := n.Args[0].Pos()
   379  						fun := &ast.SelectorExpr{X: newIdent(pos, "posn"), Sel: newIdent(pos, "Pos")}
   380  						arg := &ast.CallExpr{Fun: fun, Lparen: pos, Args: nil, Ellipsis: token.NoPos, Rparen: pos}
   381  						n.Args[0] = arg
   382  						return false
   383  					}
   384  				case "addf":
   385  					// rewrite err.addf(pos, ...) to err.addf(posn, ...)
   386  					if isIdent(n.Args[0], "pos") {
   387  						pos := n.Args[0].Pos()
   388  						arg := newIdent(pos, "posn")
   389  						n.Args[0] = arg
   390  						return false
   391  					}
   392  				case "allowVersion":
   393  					// rewrite check.allowVersion(pos, ...) to check.allowVersion(posn, ...)
   394  					if isIdent(n.Args[0], "pos") {
   395  						pos := n.Args[0].Pos()
   396  						arg := newIdent(pos, "posn")
   397  						n.Args[0] = arg
   398  						return false
   399  					}
   400  				}
   401  			}
   402  		}
   403  		return true
   404  	})
   405  }
   406  
   407  // fixAtPosCall updates calls of the form atPos(x) to x.Pos() in argument lists of (check).dump calls.
   408  // TODO(gri) can we avoid this and just use atPos consistently in go/types and types2?
   409  func fixAtPosCall(f *ast.File) {
   410  	ast.Inspect(f, func(n ast.Node) bool {
   411  		switch n := n.(type) {
   412  		case *ast.CallExpr:
   413  			if selx, _ := n.Fun.(*ast.SelectorExpr); selx != nil && selx.Sel.Name == "dump" {
   414  				for i, arg := range n.Args {
   415  					if call, _ := arg.(*ast.CallExpr); call != nil {
   416  						// rewrite xxx.dump(..., atPos(x), ...) to xxx.dump(..., x.Pos(), ...)
   417  						if isIdent(call.Fun, "atPos") {
   418  							pos := call.Args[0].Pos()
   419  							fun := &ast.SelectorExpr{X: call.Args[0], Sel: newIdent(pos, "Pos")}
   420  							n.Args[i] = &ast.CallExpr{Fun: fun, Lparen: pos, Rparen: pos}
   421  							return false
   422  						}
   423  					}
   424  				}
   425  			}
   426  		}
   427  		return true
   428  	})
   429  }
   430  
   431  // fixErrErrorfCall updates calls of the form err.addf(obj, ...) to err.addf(obj.Pos(), ...).
   432  func fixErrErrorfCall(f *ast.File) {
   433  	ast.Inspect(f, func(n ast.Node) bool {
   434  		switch n := n.(type) {
   435  		case *ast.CallExpr:
   436  			if selx, _ := n.Fun.(*ast.SelectorExpr); selx != nil {
   437  				if isIdent(selx.X, "err") {
   438  					switch selx.Sel.Name {
   439  					case "errorf":
   440  						// rewrite err.addf(obj, ... ) to err.addf(obj.Pos(), ... )
   441  						if ident, _ := n.Args[0].(*ast.Ident); ident != nil && ident.Name == "obj" {
   442  							pos := n.Args[0].Pos()
   443  							fun := &ast.SelectorExpr{X: ident, Sel: newIdent(pos, "Pos")}
   444  							n.Args[0] = &ast.CallExpr{Fun: fun, Lparen: pos, Rparen: pos}
   445  							return false
   446  						}
   447  					}
   448  				}
   449  			}
   450  		}
   451  		return true
   452  	})
   453  }
   454  
   455  // fixCheckErrorfCall updates calls of the form check.errorf(pos, ...) to check.errorf(atPos(pos), ...).
   456  func fixCheckErrorfCall(f *ast.File) {
   457  	ast.Inspect(f, func(n ast.Node) bool {
   458  		switch n := n.(type) {
   459  		case *ast.CallExpr:
   460  			if selx, _ := n.Fun.(*ast.SelectorExpr); selx != nil {
   461  				if isIdent(selx.X, "check") {
   462  					switch selx.Sel.Name {
   463  					case "errorf":
   464  						// rewrite check.errorf(pos, ... ) to check.errorf(atPos(pos), ... )
   465  						if ident := asIdent(n.Args[0], "pos"); ident != nil {
   466  							pos := n.Args[0].Pos()
   467  							fun := newIdent(pos, "atPos")
   468  							n.Args[0] = &ast.CallExpr{Fun: fun, Lparen: pos, Args: []ast.Expr{ident}, Rparen: pos}
   469  							return false
   470  						}
   471  					}
   472  				}
   473  			}
   474  		}
   475  		return true
   476  	})
   477  }
   478  
   479  // fixGlobalTypVarDecl changes the global Typ variable from an array to a slice
   480  // (in types2 we use an array for efficiency, in go/types it's a slice and we
   481  // cannot change that).
   482  func fixGlobalTypVarDecl(f *ast.File) {
   483  	ast.Inspect(f, func(n ast.Node) bool {
   484  		switch n := n.(type) {
   485  		case *ast.ValueSpec:
   486  			// rewrite type Typ = [...]Type{...} to type Typ = []Type{...}
   487  			if len(n.Names) == 1 && n.Names[0].Name == "Typ" && len(n.Values) == 1 {
   488  				n.Values[0].(*ast.CompositeLit).Type.(*ast.ArrayType).Len = nil
   489  				return false
   490  			}
   491  		}
   492  		return true
   493  	})
   494  }
   495  
   496  // fixSprintf adds an extra nil argument for the *token.FileSet parameter in sprintf calls.
   497  func fixSprintf(f *ast.File) {
   498  	ast.Inspect(f, func(n ast.Node) bool {
   499  		switch n := n.(type) {
   500  		case *ast.CallExpr:
   501  			if isIdent(n.Fun, "sprintf") && len(n.Args) >= 4 /* ... args */ {
   502  				n.Args = insert(n.Args, 1, newIdent(n.Args[1].Pos(), "nil"))
   503  				return false
   504  			}
   505  		}
   506  		return true
   507  	})
   508  }
   509  
   510  // asIdent returns x as *ast.Ident if it is an identifier with the given name.
   511  func asIdent(x ast.Node, name string) *ast.Ident {
   512  	if ident, _ := x.(*ast.Ident); ident != nil && ident.Name == name {
   513  		return ident
   514  	}
   515  	return nil
   516  }
   517  
   518  // isIdent reports whether x is an identifier with the given name.
   519  func isIdent(x ast.Node, name string) bool {
   520  	return asIdent(x, name) != nil
   521  }
   522  
   523  // newIdent returns a new identifier with the given position and name.
   524  func newIdent(pos token.Pos, name string) *ast.Ident {
   525  	id := ast.NewIdent(name)
   526  	id.NamePos = pos
   527  	return id
   528  }
   529  
   530  // insert inserts x at list[at] and moves the remaining elements up.
   531  func insert(list []ast.Expr, at int, x ast.Expr) []ast.Expr {
   532  	list = append(list, nil)
   533  	copy(list[at+1:], list[at:])
   534  	list[at] = x
   535  	return list
   536  }
   537
View as plain text