read.go

     1  // Copyright 2012 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package build
     6  
     7  import (
     8  	"bufio"
     9  	"bytes"
    10  	"errors"
    11  	"fmt"
    12  	"go/ast"
    13  	"go/parser"
    14  	"go/scanner"
    15  	"go/token"
    16  	"io"
    17  	"strconv"
    18  	"strings"
    19  	"unicode"
    20  	"unicode/utf8"
    21  	_ "unsafe" // for linkname
    22  )
    23  
    24  type importReader struct {
    25  	b    *bufio.Reader
    26  	buf  []byte
    27  	peek byte
    28  	err  error
    29  	eof  bool
    30  	nerr int
    31  	pos  token.Position
    32  }
    33  
    34  var bom = []byte{0xef, 0xbb, 0xbf}
    35  
    36  func newImportReader(name string, r io.Reader) *importReader {
    37  	b := bufio.NewReader(r)
    38  	// Remove leading UTF-8 BOM.
    39  	// Per https://golang.org/ref/spec#Source_code_representation:
    40  	// a compiler may ignore a UTF-8-encoded byte order mark (U+FEFF)
    41  	// if it is the first Unicode code point in the source text.
    42  	if leadingBytes, err := b.Peek(3); err == nil && bytes.Equal(leadingBytes, bom) {
    43  		b.Discard(3)
    44  	}
    45  	return &importReader{
    46  		b:   b,
    47  		buf: make([]byte, 0, 1024),
    48  		pos: token.Position{
    49  			Filename: name,
    50  			Line:     1,
    51  			Column:   1,
    52  		},
    53  	}
    54  }
    55  
    56  func isIdent(c byte) bool {
    57  	return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '_' || c >= utf8.RuneSelf
    58  }
    59  
    60  var (
    61  	errSyntax = errors.New("syntax error")
    62  	errNUL    = errors.New("unexpected NUL in input")
    63  )
    64  
    65  // syntaxError records a syntax error, but only if an I/O error has not already been recorded.
    66  func (r *importReader) syntaxError() {
    67  	if r.err == nil {
    68  		r.err = errSyntax
    69  	}
    70  }
    71  
    72  // readByte reads the next byte from the input, saves it in buf, and returns it.
    73  // If an error occurs, readByte records the error in r.err and returns 0.
    74  func (r *importReader) readByte() byte {
    75  	c, err := r.b.ReadByte()
    76  	if err == nil {
    77  		r.buf = append(r.buf, c)
    78  		if c == 0 {
    79  			err = errNUL
    80  		}
    81  	}
    82  	if err != nil {
    83  		if err == io.EOF {
    84  			r.eof = true
    85  		} else if r.err == nil {
    86  			r.err = err
    87  		}
    88  		c = 0
    89  	}
    90  	return c
    91  }
    92  
    93  // readRest reads the entire rest of the file into r.buf.
    94  func (r *importReader) readRest() {
    95  	for {
    96  		if len(r.buf) == cap(r.buf) {
    97  			// Grow the buffer
    98  			r.buf = append(r.buf, 0)[:len(r.buf)]
    99  		}
   100  		n, err := r.b.Read(r.buf[len(r.buf):cap(r.buf)])
   101  		r.buf = r.buf[:len(r.buf)+n]
   102  		if err != nil {
   103  			if err == io.EOF {
   104  				r.eof = true
   105  			} else if r.err == nil {
   106  				r.err = err
   107  			}
   108  			break
   109  		}
   110  	}
   111  }
   112  
   113  // peekByte returns the next byte from the input reader but does not advance beyond it.
   114  // If skipSpace is set, peekByte skips leading spaces and comments.
   115  func (r *importReader) peekByte(skipSpace bool) byte {
   116  	if r.err != nil {
   117  		if r.nerr++; r.nerr > 10000 {
   118  			panic("go/build: import reader looping")
   119  		}
   120  		return 0
   121  	}
   122  
   123  	// Use r.peek as first input byte.
   124  	// Don't just return r.peek here: it might have been left by peekByte(false)
   125  	// and this might be peekByte(true).
   126  	c := r.peek
   127  	if c == 0 {
   128  		c = r.readByte()
   129  	}
   130  	for r.err == nil && !r.eof {
   131  		if skipSpace {
   132  			// For the purposes of this reader, semicolons are never necessary to
   133  			// understand the input and are treated as spaces.
   134  			switch c {
   135  			case ' ', '\f', '\t', '\r', '\n', ';':
   136  				c = r.readByte()
   137  				continue
   138  
   139  			case '/':
   140  				c = r.readByte()
   141  				if c == '/' {
   142  					for c != '\n' && r.err == nil && !r.eof {
   143  						c = r.readByte()
   144  					}
   145  				} else if c == '*' {
   146  					var c1 byte
   147  					for (c != '*' || c1 != '/') && r.err == nil {
   148  						if r.eof {
   149  							r.syntaxError()
   150  						}
   151  						c, c1 = c1, r.readByte()
   152  					}
   153  				} else {
   154  					r.syntaxError()
   155  				}
   156  				c = r.readByte()
   157  				continue
   158  			}
   159  		}
   160  		break
   161  	}
   162  	r.peek = c
   163  	return r.peek
   164  }
   165  
   166  // nextByte is like peekByte but advances beyond the returned byte.
   167  func (r *importReader) nextByte(skipSpace bool) byte {
   168  	c := r.peekByte(skipSpace)
   169  	r.peek = 0
   170  	return c
   171  }
   172  
   173  // readKeyword reads the given keyword from the input.
   174  // If the keyword is not present, readKeyword records a syntax error.
   175  func (r *importReader) readKeyword(kw string) {
   176  	r.peekByte(true)
   177  	for i := 0; i < len(kw); i++ {
   178  		if r.nextByte(false) != kw[i] {
   179  			r.syntaxError()
   180  			return
   181  		}
   182  	}
   183  	if isIdent(r.peekByte(false)) {
   184  		r.syntaxError()
   185  	}
   186  }
   187  
   188  // readIdent reads an identifier from the input.
   189  // If an identifier is not present, readIdent records a syntax error.
   190  func (r *importReader) readIdent() {
   191  	c := r.peekByte(true)
   192  	if !isIdent(c) {
   193  		r.syntaxError()
   194  		return
   195  	}
   196  	for isIdent(r.peekByte(false)) {
   197  		r.peek = 0
   198  	}
   199  }
   200  
   201  // readString reads a quoted string literal from the input.
   202  // If an identifier is not present, readString records a syntax error.
   203  func (r *importReader) readString() {
   204  	switch r.nextByte(true) {
   205  	case '`':
   206  		for r.err == nil {
   207  			if r.nextByte(false) == '`' {
   208  				break
   209  			}
   210  			if r.eof {
   211  				r.syntaxError()
   212  			}
   213  		}
   214  	case '"':
   215  		for r.err == nil {
   216  			c := r.nextByte(false)
   217  			if c == '"' {
   218  				break
   219  			}
   220  			if r.eof || c == '\n' {
   221  				r.syntaxError()
   222  			}
   223  			if c == '\\' {
   224  				r.nextByte(false)
   225  			}
   226  		}
   227  	default:
   228  		r.syntaxError()
   229  	}
   230  }
   231  
   232  // readImport reads an import clause - optional identifier followed by quoted string -
   233  // from the input.
   234  func (r *importReader) readImport() {
   235  	c := r.peekByte(true)
   236  	if c == '.' {
   237  		r.peek = 0
   238  	} else if isIdent(c) {
   239  		r.readIdent()
   240  	}
   241  	r.readString()
   242  }
   243  
   244  // readComments is like io.ReadAll, except that it only reads the leading
   245  // block of comments in the file.
   246  //
   247  // readComments should be an internal detail,
   248  // but widely used packages access it using linkname.
   249  // Notable members of the hall of shame include:
   250  //   - github.com/bazelbuild/bazel-gazelle
   251  //
   252  // Do not remove or change the type signature.
   253  // See go.dev/issue/67401.
   254  //
   255  //go:linkname readComments
   256  func readComments(f io.Reader) ([]byte, error) {
   257  	r := newImportReader("", f)
   258  	r.peekByte(true)
   259  	if r.err == nil && !r.eof {
   260  		// Didn't reach EOF, so must have found a non-space byte. Remove it.
   261  		r.buf = r.buf[:len(r.buf)-1]
   262  	}
   263  	return r.buf, r.err
   264  }
   265  
   266  // readGoInfo expects a Go file as input and reads the file up to and including the import section.
   267  // It records what it learned in *info.
   268  // If info.fset is non-nil, readGoInfo parses the file and sets info.parsed, info.parseErr,
   269  // info.imports and info.embeds.
   270  //
   271  // It only returns an error if there are problems reading the file,
   272  // not for syntax errors in the file itself.
   273  func readGoInfo(f io.Reader, info *fileInfo) error {
   274  	r := newImportReader(info.name, f)
   275  
   276  	r.readKeyword("package")
   277  	r.readIdent()
   278  	for r.peekByte(true) == 'i' {
   279  		r.readKeyword("import")
   280  		if r.peekByte(true) == '(' {
   281  			r.nextByte(false)
   282  			for r.peekByte(true) != ')' && r.err == nil {
   283  				r.readImport()
   284  			}
   285  			r.nextByte(false)
   286  		} else {
   287  			r.readImport()
   288  		}
   289  	}
   290  
   291  	info.header = r.buf
   292  
   293  	// If we stopped successfully before EOF, we read a byte that told us we were done.
   294  	// Return all but that last byte, which would cause a syntax error if we let it through.
   295  	if r.err == nil && !r.eof {
   296  		info.header = r.buf[:len(r.buf)-1]
   297  	}
   298  
   299  	// If we stopped for a syntax error, consume the whole file so that
   300  	// we are sure we don't change the errors that go/parser returns.
   301  	if r.err == errSyntax {
   302  		r.err = nil
   303  		r.readRest()
   304  		info.header = r.buf
   305  	}
   306  	if r.err != nil {
   307  		return r.err
   308  	}
   309  
   310  	if info.fset == nil {
   311  		return nil
   312  	}
   313  
   314  	// Parse file header & record imports.
   315  	info.parsed, info.parseErr = parser.ParseFile(info.fset, info.name, info.header, parser.ImportsOnly|parser.ParseComments|parser.SkipObjectResolution)
   316  	if info.parseErr != nil {
   317  		return nil
   318  	}
   319  
   320  	hasEmbed := false
   321  	for _, decl := range info.parsed.Decls {
   322  		d, ok := decl.(*ast.GenDecl)
   323  		if !ok {
   324  			continue
   325  		}
   326  		for _, dspec := range d.Specs {
   327  			spec, ok := dspec.(*ast.ImportSpec)
   328  			if !ok {
   329  				continue
   330  			}
   331  			quoted := spec.Path.Value
   332  			path, err := strconv.Unquote(quoted)
   333  			if err != nil {
   334  				return fmt.Errorf("parser returned invalid quoted string: <%s>", quoted)
   335  			}
   336  			if !isValidImport(path) {
   337  				// The parser used to return a parse error for invalid import paths, but
   338  				// no longer does, so check for and create the error here instead.
   339  				info.parseErr = &scanner.Error{Pos: info.fset.Position(spec.Pos()), Msg: "invalid import path: " + path}
   340  				info.imports = nil
   341  				return nil
   342  			}
   343  			if path == "embed" {
   344  				hasEmbed = true
   345  			}
   346  
   347  			doc := spec.Doc
   348  			if doc == nil && len(d.Specs) == 1 {
   349  				doc = d.Doc
   350  			}
   351  			info.imports = append(info.imports, fileImport{path, spec.Pos(), doc})
   352  		}
   353  	}
   354  
   355  	// Extract directives.
   356  	for _, group := range info.parsed.Comments {
   357  		if group.Pos() >= info.parsed.Package {
   358  			break
   359  		}
   360  		for _, c := range group.List {
   361  			if strings.HasPrefix(c.Text, "//go:") {
   362  				info.directives = append(info.directives, Directive{c.Text, info.fset.Position(c.Slash)})
   363  			}
   364  		}
   365  	}
   366  
   367  	// If the file imports "embed",
   368  	// we have to look for //go:embed comments
   369  	// in the remainder of the file.
   370  	// The compiler will enforce the mapping of comments to
   371  	// declared variables. We just need to know the patterns.
   372  	// If there were //go:embed comments earlier in the file
   373  	// (near the package statement or imports), the compiler
   374  	// will reject them. They can be (and have already been) ignored.
   375  	if hasEmbed {
   376  		r.readRest()
   377  		fset := token.NewFileSet()
   378  		file := fset.AddFile(r.pos.Filename, -1, len(r.buf))
   379  		var sc scanner.Scanner
   380  		sc.Init(file, r.buf, nil, scanner.ScanComments)
   381  		for {
   382  			pos, tok, lit := sc.Scan()
   383  			if tok == token.EOF {
   384  				break
   385  			}
   386  			if tok == token.COMMENT && strings.HasPrefix(lit, "//go:embed") {
   387  				// Ignore badly-formed lines - the compiler will report them when it finds them,
   388  				// and we can pretend they are not there to help go list succeed with what it knows.
   389  				embs, err := parseGoEmbed(fset, pos, lit)
   390  				if err == nil {
   391  					info.embeds = append(info.embeds, embs...)
   392  				}
   393  			}
   394  		}
   395  	}
   396  
   397  	return nil
   398  }
   399  
   400  // isValidImport checks if the import is a valid import using the more strict
   401  // checks allowed by the implementation restriction in https://go.dev/ref/spec#Import_declarations.
   402  // It was ported from the function of the same name that was removed from the
   403  // parser in CL 424855, when the parser stopped doing these checks.
   404  func isValidImport(s string) bool {
   405  	const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD"
   406  	for _, r := range s {
   407  		if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) {
   408  			return false
   409  		}
   410  	}
   411  	return s != ""
   412  }
   413  
   414  // parseGoEmbed parses a "//go:embed" to extract the glob patterns.
   415  // It accepts unquoted space-separated patterns as well as double-quoted and back-quoted Go strings.
   416  // This must match the behavior of cmd/compile/internal/noder/noder.go.
   417  func parseGoEmbed(fset *token.FileSet, pos token.Pos, comment string) ([]fileEmbed, error) {
   418  	dir, ok := ast.ParseDirective(pos, comment)
   419  	if !ok || dir.Tool != "go" || dir.Name != "embed" {
   420  		return nil, nil
   421  	}
   422  	args, err := dir.ParseArgs()
   423  	if err != nil {
   424  		return nil, err
   425  	}
   426  	var list []fileEmbed
   427  	for _, arg := range args {
   428  		list = append(list, fileEmbed{arg.Arg, fset.Position(arg.Pos)})
   429  	}
   430  	return list, nil
   431  }
   432
View as plain text