Black Lives Matter. Support the Equal Justice Initiative.

Source file src/cmd/go/internal/generate/generate.go

Documentation: cmd/go/internal/generate

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package generate implements the ``go generate'' command.
     6  package generate
     7  
     8  import (
     9  	"bufio"
    10  	"bytes"
    11  	"fmt"
    12  	"go/parser"
    13  	"go/token"
    14  	"io"
    15  	"io/ioutil"
    16  	"log"
    17  	"os"
    18  	"os/exec"
    19  	"path/filepath"
    20  	"regexp"
    21  	"strconv"
    22  	"strings"
    23  
    24  	"cmd/go/internal/base"
    25  	"cmd/go/internal/cfg"
    26  	"cmd/go/internal/load"
    27  	"cmd/go/internal/modload"
    28  	"cmd/go/internal/str"
    29  	"cmd/go/internal/work"
    30  )
    31  
    32  var CmdGenerate = &base.Command{
    33  	Run:       runGenerate,
    34  	UsageLine: "go generate [-run regexp] [-n] [-v] [-x] [build flags] [file.go... | packages]",
    35  	Short:     "generate Go files by processing source",
    36  	Long: `
    37  Generate runs commands described by directives within existing
    38  files. Those commands can run any process but the intent is to
    39  create or update Go source files.
    40  
    41  Go generate is never run automatically by go build, go get, go test,
    42  and so on. It must be run explicitly.
    43  
    44  Go generate scans the file for directives, which are lines of
    45  the form,
    46  
    47  	//go:generate command argument...
    48  
    49  (note: no leading spaces and no space in "//go") where command
    50  is the generator to be run, corresponding to an executable file
    51  that can be run locally. It must either be in the shell path
    52  (gofmt), a fully qualified path (/usr/you/bin/mytool), or a
    53  command alias, described below.
    54  
    55  To convey to humans and machine tools that code is generated,
    56  generated source should have a line that matches the following
    57  regular expression (in Go syntax):
    58  
    59  	^// Code generated .* DO NOT EDIT\.$
    60  
    61  The line may appear anywhere in the file, but is typically
    62  placed near the beginning so it is easy to find.
    63  
    64  Note that go generate does not parse the file, so lines that look
    65  like directives in comments or multiline strings will be treated
    66  as directives.
    67  
    68  The arguments to the directive are space-separated tokens or
    69  double-quoted strings passed to the generator as individual
    70  arguments when it is run.
    71  
    72  Quoted strings use Go syntax and are evaluated before execution; a
    73  quoted string appears as a single argument to the generator.
    74  
    75  Go generate sets several variables when it runs the generator:
    76  
    77  	$GOARCH
    78  		The execution architecture (arm, amd64, etc.)
    79  	$GOOS
    80  		The execution operating system (linux, windows, etc.)
    81  	$GOFILE
    82  		The base name of the file.
    83  	$GOLINE
    84  		The line number of the directive in the source file.
    85  	$GOPACKAGE
    86  		The name of the package of the file containing the directive.
    87  	$DOLLAR
    88  		A dollar sign.
    89  
    90  Other than variable substitution and quoted-string evaluation, no
    91  special processing such as "globbing" is performed on the command
    92  line.
    93  
    94  As a last step before running the command, any invocations of any
    95  environment variables with alphanumeric names, such as $GOFILE or
    96  $HOME, are expanded throughout the command line. The syntax for
    97  variable expansion is $NAME on all operating systems. Due to the
    98  order of evaluation, variables are expanded even inside quoted
    99  strings. If the variable NAME is not set, $NAME expands to the
   100  empty string.
   101  
   102  A directive of the form,
   103  
   104  	//go:generate -command xxx args...
   105  
   106  specifies, for the remainder of this source file only, that the
   107  string xxx represents the command identified by the arguments. This
   108  can be used to create aliases or to handle multiword generators.
   109  For example,
   110  
   111  	//go:generate -command foo go tool foo
   112  
   113  specifies that the command "foo" represents the generator
   114  "go tool foo".
   115  
   116  Generate processes packages in the order given on the command line,
   117  one at a time. If the command line lists .go files from a single directory,
   118  they are treated as a single package. Within a package, generate processes the
   119  source files in a package in file name order, one at a time. Within
   120  a source file, generate runs generators in the order they appear
   121  in the file, one at a time. The go generate tool also sets the build
   122  tag "generate" so that files may be examined by go generate but ignored
   123  during build.
   124  
   125  For packages with invalid code, generate processes only source files with a
   126  valid package clause.
   127  
   128  If any generator returns an error exit status, "go generate" skips
   129  all further processing for that package.
   130  
   131  The generator is run in the package's source directory.
   132  
   133  Go generate accepts one specific flag:
   134  
   135  	-run=""
   136  		if non-empty, specifies a regular expression to select
   137  		directives whose full original source text (excluding
   138  		any trailing spaces and final newline) matches the
   139  		expression.
   140  
   141  It also accepts the standard build flags including -v, -n, and -x.
   142  The -v flag prints the names of packages and files as they are
   143  processed.
   144  The -n flag prints commands that would be executed.
   145  The -x flag prints commands as they are executed.
   146  
   147  For more about build flags, see 'go help build'.
   148  
   149  For more about specifying packages, see 'go help packages'.
   150  	`,
   151  }
   152  
   153  var (
   154  	generateRunFlag string         // generate -run flag
   155  	generateRunRE   *regexp.Regexp // compiled expression for -run
   156  )
   157  
   158  func init() {
   159  	work.AddBuildFlags(CmdGenerate, work.DefaultBuildFlags)
   160  	CmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "")
   161  }
   162  
   163  func runGenerate(cmd *base.Command, args []string) {
   164  	load.IgnoreImports = true
   165  
   166  	if generateRunFlag != "" {
   167  		var err error
   168  		generateRunRE, err = regexp.Compile(generateRunFlag)
   169  		if err != nil {
   170  			log.Fatalf("generate: %s", err)
   171  		}
   172  	}
   173  
   174  	cfg.BuildContext.BuildTags = append(cfg.BuildContext.BuildTags, "generate")
   175  
   176  	// Even if the arguments are .go files, this loop suffices.
   177  	printed := false
   178  	for _, pkg := range load.PackagesAndErrors(args) {
   179  		if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main {
   180  			if !printed {
   181  				fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n")
   182  				printed = true
   183  			}
   184  			continue
   185  		}
   186  
   187  		for _, file := range pkg.InternalGoFiles() {
   188  			if !generate(file) {
   189  				break
   190  			}
   191  		}
   192  
   193  		for _, file := range pkg.InternalXGoFiles() {
   194  			if !generate(file) {
   195  				break
   196  			}
   197  		}
   198  	}
   199  }
   200  
   201  // generate runs the generation directives for a single file.
   202  func generate(absFile string) bool {
   203  	src, err := ioutil.ReadFile(absFile)
   204  	if err != nil {
   205  		log.Fatalf("generate: %s", err)
   206  	}
   207  
   208  	// Parse package clause
   209  	filePkg, err := parser.ParseFile(token.NewFileSet(), "", src, parser.PackageClauseOnly)
   210  	if err != nil {
   211  		// Invalid package clause - ignore file.
   212  		return true
   213  	}
   214  
   215  	g := &Generator{
   216  		r:        bytes.NewReader(src),
   217  		path:     absFile,
   218  		pkg:      filePkg.Name.String(),
   219  		commands: make(map[string][]string),
   220  	}
   221  	return g.run()
   222  }
   223  
   224  // A Generator represents the state of a single Go source file
   225  // being scanned for generator commands.
   226  type Generator struct {
   227  	r        io.Reader
   228  	path     string // full rooted path name.
   229  	dir      string // full rooted directory of file.
   230  	file     string // base name of file.
   231  	pkg      string
   232  	commands map[string][]string
   233  	lineNum  int // current line number.
   234  	env      []string
   235  }
   236  
   237  // run runs the generators in the current file.
   238  func (g *Generator) run() (ok bool) {
   239  	// Processing below here calls g.errorf on failure, which does panic(stop).
   240  	// If we encounter an error, we abort the package.
   241  	defer func() {
   242  		e := recover()
   243  		if e != nil {
   244  			ok = false
   245  			if e != stop {
   246  				panic(e)
   247  			}
   248  			base.SetExitStatus(1)
   249  		}
   250  	}()
   251  	g.dir, g.file = filepath.Split(g.path)
   252  	g.dir = filepath.Clean(g.dir) // No final separator please.
   253  	if cfg.BuildV {
   254  		fmt.Fprintf(os.Stderr, "%s\n", base.ShortPath(g.path))
   255  	}
   256  
   257  	// Scan for lines that start "//go:generate".
   258  	// Can't use bufio.Scanner because it can't handle long lines,
   259  	// which are likely to appear when using generate.
   260  	input := bufio.NewReader(g.r)
   261  	var err error
   262  	// One line per loop.
   263  	for {
   264  		g.lineNum++ // 1-indexed.
   265  		var buf []byte
   266  		buf, err = input.ReadSlice('\n')
   267  		if err == bufio.ErrBufferFull {
   268  			// Line too long - consume and ignore.
   269  			if isGoGenerate(buf) {
   270  				g.errorf("directive too long")
   271  			}
   272  			for err == bufio.ErrBufferFull {
   273  				_, err = input.ReadSlice('\n')
   274  			}
   275  			if err != nil {
   276  				break
   277  			}
   278  			continue
   279  		}
   280  
   281  		if err != nil {
   282  			// Check for marker at EOF without final \n.
   283  			if err == io.EOF && isGoGenerate(buf) {
   284  				err = io.ErrUnexpectedEOF
   285  			}
   286  			break
   287  		}
   288  
   289  		if !isGoGenerate(buf) {
   290  			continue
   291  		}
   292  		if generateRunFlag != "" {
   293  			if !generateRunRE.Match(bytes.TrimSpace(buf)) {
   294  				continue
   295  			}
   296  		}
   297  
   298  		g.setEnv()
   299  		words := g.split(string(buf))
   300  		if len(words) == 0 {
   301  			g.errorf("no arguments to directive")
   302  		}
   303  		if words[0] == "-command" {
   304  			g.setShorthand(words)
   305  			continue
   306  		}
   307  		// Run the command line.
   308  		if cfg.BuildN || cfg.BuildX {
   309  			fmt.Fprintf(os.Stderr, "%s\n", strings.Join(words, " "))
   310  		}
   311  		if cfg.BuildN {
   312  			continue
   313  		}
   314  		g.exec(words)
   315  	}
   316  	if err != nil && err != io.EOF {
   317  		g.errorf("error reading %s: %s", base.ShortPath(g.path), err)
   318  	}
   319  	return true
   320  }
   321  
   322  func isGoGenerate(buf []byte) bool {
   323  	return bytes.HasPrefix(buf, []byte("//go:generate ")) || bytes.HasPrefix(buf, []byte("//go:generate\t"))
   324  }
   325  
   326  // setEnv sets the extra environment variables used when executing a
   327  // single go:generate command.
   328  func (g *Generator) setEnv() {
   329  	g.env = []string{
   330  		"GOARCH=" + cfg.BuildContext.GOARCH,
   331  		"GOOS=" + cfg.BuildContext.GOOS,
   332  		"GOFILE=" + g.file,
   333  		"GOLINE=" + strconv.Itoa(g.lineNum),
   334  		"GOPACKAGE=" + g.pkg,
   335  		"DOLLAR=" + "$",
   336  	}
   337  }
   338  
   339  // split breaks the line into words, evaluating quoted
   340  // strings and evaluating environment variables.
   341  // The initial //go:generate element is present in line.
   342  func (g *Generator) split(line string) []string {
   343  	// Parse line, obeying quoted strings.
   344  	var words []string
   345  	line = line[len("//go:generate ") : len(line)-1] // Drop preamble and final newline.
   346  	// There may still be a carriage return.
   347  	if len(line) > 0 && line[len(line)-1] == '\r' {
   348  		line = line[:len(line)-1]
   349  	}
   350  	// One (possibly quoted) word per iteration.
   351  Words:
   352  	for {
   353  		line = strings.TrimLeft(line, " \t")
   354  		if len(line) == 0 {
   355  			break
   356  		}
   357  		if line[0] == '"' {
   358  			for i := 1; i < len(line); i++ {
   359  				c := line[i] // Only looking for ASCII so this is OK.
   360  				switch c {
   361  				case '\\':
   362  					if i+1 == len(line) {
   363  						g.errorf("bad backslash")
   364  					}
   365  					i++ // Absorb next byte (If it's a multibyte we'll get an error in Unquote).
   366  				case '"':
   367  					word, err := strconv.Unquote(line[0 : i+1])
   368  					if err != nil {
   369  						g.errorf("bad quoted string")
   370  					}
   371  					words = append(words, word)
   372  					line = line[i+1:]
   373  					// Check the next character is space or end of line.
   374  					if len(line) > 0 && line[0] != ' ' && line[0] != '\t' {
   375  						g.errorf("expect space after quoted argument")
   376  					}
   377  					continue Words
   378  				}
   379  			}
   380  			g.errorf("mismatched quoted string")
   381  		}
   382  		i := strings.IndexAny(line, " \t")
   383  		if i < 0 {
   384  			i = len(line)
   385  		}
   386  		words = append(words, line[0:i])
   387  		line = line[i:]
   388  	}
   389  	// Substitute command if required.
   390  	if len(words) > 0 && g.commands[words[0]] != nil {
   391  		// Replace 0th word by command substitution.
   392  		//
   393  		// Force a copy of the command definition to
   394  		// ensure words doesn't end up as a reference
   395  		// to the g.commands content.
   396  		tmpCmdWords := append([]string(nil), (g.commands[words[0]])...)
   397  		words = append(tmpCmdWords, words[1:]...)
   398  	}
   399  	// Substitute environment variables.
   400  	for i, word := range words {
   401  		words[i] = os.Expand(word, g.expandVar)
   402  	}
   403  	return words
   404  }
   405  
   406  var stop = fmt.Errorf("error in generation")
   407  
   408  // errorf logs an error message prefixed with the file and line number.
   409  // It then exits the program (with exit status 1) because generation stops
   410  // at the first error.
   411  func (g *Generator) errorf(format string, args ...interface{}) {
   412  	fmt.Fprintf(os.Stderr, "%s:%d: %s\n", base.ShortPath(g.path), g.lineNum,
   413  		fmt.Sprintf(format, args...))
   414  	panic(stop)
   415  }
   416  
   417  // expandVar expands the $XXX invocation in word. It is called
   418  // by os.Expand.
   419  func (g *Generator) expandVar(word string) string {
   420  	w := word + "="
   421  	for _, e := range g.env {
   422  		if strings.HasPrefix(e, w) {
   423  			return e[len(w):]
   424  		}
   425  	}
   426  	return os.Getenv(word)
   427  }
   428  
   429  // setShorthand installs a new shorthand as defined by a -command directive.
   430  func (g *Generator) setShorthand(words []string) {
   431  	// Create command shorthand.
   432  	if len(words) == 1 {
   433  		g.errorf("no command specified for -command")
   434  	}
   435  	command := words[1]
   436  	if g.commands[command] != nil {
   437  		g.errorf("command %q multiply defined", command)
   438  	}
   439  	g.commands[command] = words[2:len(words):len(words)] // force later append to make copy
   440  }
   441  
   442  // exec runs the command specified by the argument. The first word is
   443  // the command name itself.
   444  func (g *Generator) exec(words []string) {
   445  	cmd := exec.Command(words[0], words[1:]...)
   446  	// Standard in and out of generator should be the usual.
   447  	cmd.Stdout = os.Stdout
   448  	cmd.Stderr = os.Stderr
   449  	// Run the command in the package directory.
   450  	cmd.Dir = g.dir
   451  	cmd.Env = str.StringList(cfg.OrigEnv, g.env)
   452  	err := cmd.Run()
   453  	if err != nil {
   454  		g.errorf("running %q: %s", words[0], err)
   455  	}
   456  }
   457  

View as plain text