generate.go

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package generate implements the “go generate” command.
     6  package generate
     7  
     8  import (
     9  	"bufio"
    10  	"bytes"
    11  	"context"
    12  	"fmt"
    13  	"go/parser"
    14  	"go/token"
    15  	"io"
    16  	"log"
    17  	"os"
    18  	"os/exec"
    19  	"path/filepath"
    20  	"regexp"
    21  	"slices"
    22  	"strconv"
    23  	"strings"
    24  
    25  	"cmd/go/internal/base"
    26  	"cmd/go/internal/cfg"
    27  	"cmd/go/internal/load"
    28  	"cmd/go/internal/modload"
    29  	"cmd/go/internal/str"
    30  	"cmd/go/internal/work"
    31  	"cmd/internal/pathcache"
    32  )
    33  
    34  var CmdGenerate = &base.Command{
    35  	Run:       runGenerate,
    36  	UsageLine: "go generate [-run regexp] [-n] [-v] [-x] [build flags] [file.go... | packages]",
    37  	Short:     "generate Go files by processing source",
    38  	Long: `
    39  Generate runs commands described by directives within existing
    40  files. Those commands can run any process but the intent is to
    41  create or update Go source files.
    42  
    43  Go generate is never run automatically by go build, go test,
    44  and so on. It must be run explicitly.
    45  
    46  Go generate scans the file for directives, which are lines of
    47  the form,
    48  
    49  	//go:generate command argument...
    50  
    51  (note: no leading spaces and no space in "//go") where command
    52  is the generator to be run, corresponding to an executable file
    53  that can be run locally. It must either be in the shell path
    54  (gofmt), a fully qualified path (/usr/you/bin/mytool), or a
    55  command alias, described below.
    56  
    57  Note that go generate does not parse the file, so lines that look
    58  like directives in comments or multiline strings will be treated
    59  as directives.
    60  
    61  The arguments to the directive are space-separated tokens or
    62  double-quoted strings passed to the generator as individual
    63  arguments when it is run.
    64  
    65  Quoted strings use Go syntax and are evaluated before execution; a
    66  quoted string appears as a single argument to the generator.
    67  
    68  To convey to humans and machine tools that code is generated,
    69  generated source should have a line that matches the following
    70  regular expression (in Go syntax):
    71  
    72  	^// Code generated .* DO NOT EDIT\.$
    73  
    74  This line must appear before the first non-comment, non-blank
    75  text in the file.
    76  
    77  Go generate sets several variables when it runs the generator:
    78  
    79  	$GOARCH
    80  		The execution architecture (arm, amd64, etc.)
    81  	$GOOS
    82  		The execution operating system (linux, windows, etc.)
    83  	$GOFILE
    84  		The base name of the file.
    85  	$GOLINE
    86  		The line number of the directive in the source file.
    87  	$GOPACKAGE
    88  		The name of the package of the file containing the directive.
    89  	$GOROOT
    90  		The GOROOT directory for the 'go' command that invoked the
    91  		generator, containing the Go toolchain and standard library.
    92  	$DOLLAR
    93  		A dollar sign.
    94  	$PATH
    95  		The $PATH of the parent process, with $GOROOT/bin
    96  		placed at the beginning. This causes generators
    97  		that execute 'go' commands to use the same 'go'
    98  		as the parent 'go generate' command.
    99  
   100  Other than variable substitution and quoted-string evaluation, no
   101  special processing such as "globbing" is performed on the command
   102  line.
   103  
   104  As a last step before running the command, any invocations of any
   105  environment variables with alphanumeric names, such as $GOFILE or
   106  $HOME, are expanded throughout the command line. The syntax for
   107  variable expansion is $NAME on all operating systems. Due to the
   108  order of evaluation, variables are expanded even inside quoted
   109  strings. If the variable NAME is not set, $NAME expands to the
   110  empty string.
   111  
   112  A directive of the form,
   113  
   114  	//go:generate -command xxx args...
   115  
   116  specifies, for the remainder of this source file only, that the
   117  string xxx represents the command identified by the arguments. This
   118  can be used to create aliases or to handle multiword generators.
   119  For example,
   120  
   121  	//go:generate -command foo go tool foo
   122  
   123  specifies that the command "foo" represents the generator
   124  "go tool foo".
   125  
   126  Generate processes packages in the order given on the command line,
   127  one at a time. If the command line lists .go files from a single directory,
   128  they are treated as a single package. Within a package, generate processes the
   129  source files in a package in file name order, one at a time. Within
   130  a source file, generate runs generators in the order they appear
   131  in the file, one at a time. The go generate tool also sets the build
   132  tag "generate" so that files may be examined by go generate but ignored
   133  during build.
   134  
   135  For packages with invalid code, generate processes only source files with a
   136  valid package clause.
   137  
   138  If any generator returns an error exit status, "go generate" skips
   139  all further processing for that package.
   140  
   141  The generator is run in the package's source directory.
   142  
   143  Go generate accepts two specific flags:
   144  
   145  	-run=""
   146  		if non-empty, specifies a regular expression to select
   147  		directives whose full original source text (excluding
   148  		any trailing spaces and final newline) matches the
   149  		expression.
   150  
   151  	-skip=""
   152  		if non-empty, specifies a regular expression to suppress
   153  		directives whose full original source text (excluding
   154  		any trailing spaces and final newline) matches the
   155  		expression. If a directive matches both the -run and
   156  		the -skip arguments, it is skipped.
   157  
   158  It also accepts the standard build flags including -v, -n, and -x.
   159  The -v flag prints the names of packages and files as they are
   160  processed.
   161  The -n flag prints commands that would be executed.
   162  The -x flag prints commands as they are executed.
   163  
   164  For more about build flags, see 'go help build'.
   165  
   166  For more about specifying packages, see 'go help packages'.
   167  	`,
   168  }
   169  
   170  var (
   171  	generateRunFlag string         // generate -run flag
   172  	generateRunRE   *regexp.Regexp // compiled expression for -run
   173  
   174  	generateSkipFlag string         // generate -skip flag
   175  	generateSkipRE   *regexp.Regexp // compiled expression for -skip
   176  )
   177  
   178  func init() {
   179  	work.AddBuildFlags(CmdGenerate, work.DefaultBuildFlags)
   180  	CmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "")
   181  	CmdGenerate.Flag.StringVar(&generateSkipFlag, "skip", "", "")
   182  }
   183  
   184  func runGenerate(ctx context.Context, cmd *base.Command, args []string) {
   185  	modload.InitWorkfile()
   186  
   187  	if generateRunFlag != "" {
   188  		var err error
   189  		generateRunRE, err = regexp.Compile(generateRunFlag)
   190  		if err != nil {
   191  			log.Fatalf("generate: %s", err)
   192  		}
   193  	}
   194  	if generateSkipFlag != "" {
   195  		var err error
   196  		generateSkipRE, err = regexp.Compile(generateSkipFlag)
   197  		if err != nil {
   198  			log.Fatalf("generate: %s", err)
   199  		}
   200  	}
   201  
   202  	cfg.BuildContext.BuildTags = append(cfg.BuildContext.BuildTags, "generate")
   203  
   204  	// Even if the arguments are .go files, this loop suffices.
   205  	printed := false
   206  	pkgOpts := load.PackageOpts{IgnoreImports: true}
   207  	for _, pkg := range load.PackagesAndErrors(ctx, pkgOpts, args) {
   208  		if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main {
   209  			if !printed {
   210  				fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n")
   211  				printed = true
   212  			}
   213  			continue
   214  		}
   215  
   216  		if pkg.Error != nil && len(pkg.InternalAllGoFiles()) == 0 {
   217  			// A directory only contains a Go package if it has at least
   218  			// one .go source file, so the fact that there are no files
   219  			// implies that the package couldn't be found.
   220  			base.Errorf("%v", pkg.Error)
   221  		}
   222  
   223  		for _, file := range pkg.InternalGoFiles() {
   224  			if !generate(file) {
   225  				break
   226  			}
   227  		}
   228  
   229  		for _, file := range pkg.InternalXGoFiles() {
   230  			if !generate(file) {
   231  				break
   232  			}
   233  		}
   234  	}
   235  	base.ExitIfErrors()
   236  }
   237  
   238  // generate runs the generation directives for a single file.
   239  func generate(absFile string) bool {
   240  	src, err := os.ReadFile(absFile)
   241  	if err != nil {
   242  		log.Fatalf("generate: %s", err)
   243  	}
   244  
   245  	// Parse package clause
   246  	filePkg, err := parser.ParseFile(token.NewFileSet(), "", src, parser.PackageClauseOnly)
   247  	if err != nil {
   248  		// Invalid package clause - ignore file.
   249  		return true
   250  	}
   251  
   252  	g := &Generator{
   253  		r:        bytes.NewReader(src),
   254  		path:     absFile,
   255  		pkg:      filePkg.Name.String(),
   256  		commands: make(map[string][]string),
   257  	}
   258  	return g.run()
   259  }
   260  
   261  // A Generator represents the state of a single Go source file
   262  // being scanned for generator commands.
   263  type Generator struct {
   264  	r        io.Reader
   265  	path     string // full rooted path name.
   266  	dir      string // full rooted directory of file.
   267  	file     string // base name of file.
   268  	pkg      string
   269  	commands map[string][]string
   270  	lineNum  int // current line number.
   271  	env      []string
   272  }
   273  
   274  // run runs the generators in the current file.
   275  func (g *Generator) run() (ok bool) {
   276  	// Processing below here calls g.errorf on failure, which does panic(stop).
   277  	// If we encounter an error, we abort the package.
   278  	defer func() {
   279  		e := recover()
   280  		if e != nil {
   281  			ok = false
   282  			if e != stop {
   283  				panic(e)
   284  			}
   285  			base.SetExitStatus(1)
   286  		}
   287  	}()
   288  	g.dir, g.file = filepath.Split(g.path)
   289  	g.dir = filepath.Clean(g.dir) // No final separator please.
   290  	if cfg.BuildV {
   291  		fmt.Fprintf(os.Stderr, "%s\n", base.ShortPath(g.path))
   292  	}
   293  
   294  	// Scan for lines that start "//go:generate".
   295  	// Can't use bufio.Scanner because it can't handle long lines,
   296  	// which are likely to appear when using generate.
   297  	input := bufio.NewReader(g.r)
   298  	var err error
   299  	// One line per loop.
   300  	for {
   301  		g.lineNum++ // 1-indexed.
   302  		var buf []byte
   303  		buf, err = input.ReadSlice('\n')
   304  		if err == bufio.ErrBufferFull {
   305  			// Line too long - consume and ignore.
   306  			if isGoGenerate(buf) {
   307  				g.errorf("directive too long")
   308  			}
   309  			for err == bufio.ErrBufferFull {
   310  				_, err = input.ReadSlice('\n')
   311  			}
   312  			if err != nil {
   313  				break
   314  			}
   315  			continue
   316  		}
   317  
   318  		if err != nil {
   319  			// Check for marker at EOF without final \n.
   320  			if err == io.EOF && isGoGenerate(buf) {
   321  				err = io.ErrUnexpectedEOF
   322  			}
   323  			break
   324  		}
   325  
   326  		if !isGoGenerate(buf) {
   327  			continue
   328  		}
   329  		if generateRunFlag != "" && !generateRunRE.Match(bytes.TrimSpace(buf)) {
   330  			continue
   331  		}
   332  		if generateSkipFlag != "" && generateSkipRE.Match(bytes.TrimSpace(buf)) {
   333  			continue
   334  		}
   335  
   336  		g.setEnv()
   337  		words := g.split(string(buf))
   338  		if len(words) == 0 {
   339  			g.errorf("no arguments to directive")
   340  		}
   341  		if words[0] == "-command" {
   342  			g.setShorthand(words)
   343  			continue
   344  		}
   345  		// Run the command line.
   346  		if cfg.BuildN || cfg.BuildX {
   347  			fmt.Fprintf(os.Stderr, "%s\n", strings.Join(words, " "))
   348  		}
   349  		if cfg.BuildN {
   350  			continue
   351  		}
   352  		g.exec(words)
   353  	}
   354  	if err != nil && err != io.EOF {
   355  		g.errorf("error reading %s: %s", base.ShortPath(g.path), err)
   356  	}
   357  	return true
   358  }
   359  
   360  func isGoGenerate(buf []byte) bool {
   361  	return bytes.HasPrefix(buf, []byte("//go:generate ")) || bytes.HasPrefix(buf, []byte("//go:generate\t"))
   362  }
   363  
   364  // setEnv sets the extra environment variables used when executing a
   365  // single go:generate command.
   366  func (g *Generator) setEnv() {
   367  	env := []string{
   368  		"GOROOT=" + cfg.GOROOT,
   369  		"GOARCH=" + cfg.BuildContext.GOARCH,
   370  		"GOOS=" + cfg.BuildContext.GOOS,
   371  		"GOFILE=" + g.file,
   372  		"GOLINE=" + strconv.Itoa(g.lineNum),
   373  		"GOPACKAGE=" + g.pkg,
   374  		"DOLLAR=" + "$",
   375  	}
   376  	env = base.AppendPATH(env)
   377  	env = base.AppendPWD(env, g.dir)
   378  	g.env = env
   379  }
   380  
   381  // split breaks the line into words, evaluating quoted
   382  // strings and evaluating environment variables.
   383  // The initial //go:generate element is present in line.
   384  func (g *Generator) split(line string) []string {
   385  	// Parse line, obeying quoted strings.
   386  	var words []string
   387  	line = line[len("//go:generate ") : len(line)-1] // Drop preamble and final newline.
   388  	// There may still be a carriage return.
   389  	if len(line) > 0 && line[len(line)-1] == '\r' {
   390  		line = line[:len(line)-1]
   391  	}
   392  	// One (possibly quoted) word per iteration.
   393  Words:
   394  	for {
   395  		line = strings.TrimLeft(line, " \t")
   396  		if len(line) == 0 {
   397  			break
   398  		}
   399  		if line[0] == '"' {
   400  			for i := 1; i < len(line); i++ {
   401  				c := line[i] // Only looking for ASCII so this is OK.
   402  				switch c {
   403  				case '\\':
   404  					if i+1 == len(line) {
   405  						g.errorf("bad backslash")
   406  					}
   407  					i++ // Absorb next byte (If it's a multibyte we'll get an error in Unquote).
   408  				case '"':
   409  					word, err := strconv.Unquote(line[0 : i+1])
   410  					if err != nil {
   411  						g.errorf("bad quoted string")
   412  					}
   413  					words = append(words, word)
   414  					line = line[i+1:]
   415  					// Check the next character is space or end of line.
   416  					if len(line) > 0 && line[0] != ' ' && line[0] != '\t' {
   417  						g.errorf("expect space after quoted argument")
   418  					}
   419  					continue Words
   420  				}
   421  			}
   422  			g.errorf("mismatched quoted string")
   423  		}
   424  		i := strings.IndexAny(line, " \t")
   425  		if i < 0 {
   426  			i = len(line)
   427  		}
   428  		words = append(words, line[0:i])
   429  		line = line[i:]
   430  	}
   431  	// Substitute command if required.
   432  	if len(words) > 0 && g.commands[words[0]] != nil {
   433  		// Replace 0th word by command substitution.
   434  		//
   435  		// Force a copy of the command definition to
   436  		// ensure words doesn't end up as a reference
   437  		// to the g.commands content.
   438  		tmpCmdWords := append([]string(nil), (g.commands[words[0]])...)
   439  		words = append(tmpCmdWords, words[1:]...)
   440  	}
   441  	// Substitute environment variables.
   442  	for i, word := range words {
   443  		words[i] = os.Expand(word, g.expandVar)
   444  	}
   445  	return words
   446  }
   447  
   448  var stop = fmt.Errorf("error in generation")
   449  
   450  // errorf logs an error message prefixed with the file and line number.
   451  // It then exits the program (with exit status 1) because generation stops
   452  // at the first error.
   453  func (g *Generator) errorf(format string, args ...any) {
   454  	fmt.Fprintf(os.Stderr, "%s:%d: %s\n", base.ShortPath(g.path), g.lineNum,
   455  		fmt.Sprintf(format, args...))
   456  	panic(stop)
   457  }
   458  
   459  // expandVar expands the $XXX invocation in word. It is called
   460  // by os.Expand.
   461  func (g *Generator) expandVar(word string) string {
   462  	w := word + "="
   463  	for _, e := range g.env {
   464  		if strings.HasPrefix(e, w) {
   465  			return e[len(w):]
   466  		}
   467  	}
   468  	return os.Getenv(word)
   469  }
   470  
   471  // setShorthand installs a new shorthand as defined by a -command directive.
   472  func (g *Generator) setShorthand(words []string) {
   473  	// Create command shorthand.
   474  	if len(words) == 1 {
   475  		g.errorf("no command specified for -command")
   476  	}
   477  	command := words[1]
   478  	if g.commands[command] != nil {
   479  		g.errorf("command %q multiply defined", command)
   480  	}
   481  	g.commands[command] = slices.Clip(words[2:])
   482  }
   483  
   484  // exec runs the command specified by the argument. The first word is
   485  // the command name itself.
   486  func (g *Generator) exec(words []string) {
   487  	path := words[0]
   488  	if path != "" && !strings.Contains(path, string(os.PathSeparator)) {
   489  		// If a generator says '//go:generate go run <blah>' it almost certainly
   490  		// intends to use the same 'go' as 'go generate' itself.
   491  		// Prefer to resolve the binary from GOROOT/bin, and for consistency
   492  		// prefer to resolve any other commands there too.
   493  		gorootBinPath, err := pathcache.LookPath(filepath.Join(cfg.GOROOTbin, path))
   494  		if err == nil {
   495  			path = gorootBinPath
   496  		}
   497  	}
   498  	cmd := exec.Command(path, words[1:]...)
   499  	cmd.Args[0] = words[0] // Overwrite with the original in case it was rewritten above.
   500  
   501  	// Standard in and out of generator should be the usual.
   502  	cmd.Stdout = os.Stdout
   503  	cmd.Stderr = os.Stderr
   504  	// Run the command in the package directory.
   505  	cmd.Dir = g.dir
   506  	cmd.Env = str.StringList(cfg.OrigEnv, g.env)
   507  	err := cmd.Run()
   508  	if err != nil {
   509  		g.errorf("running %q: %s", words[0], err)
   510  	}
   511  }
   512
View as plain text