http.go

     1  // Copyright 2012 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build !cmd_go_bootstrap
     6  
     7  // This code is compiled into the real 'go' binary, but it is not
     8  // compiled into the binary that is built during all.bash, so as
     9  // to avoid needing to build net (and thus use cgo) during the
    10  // bootstrap process.
    11  
    12  package web
    13  
    14  import (
    15  	"crypto/tls"
    16  	"errors"
    17  	"fmt"
    18  	"io"
    19  	"mime"
    20  	"net"
    21  	"net/http"
    22  	urlpkg "net/url"
    23  	"os"
    24  	"strconv"
    25  	"strings"
    26  	"sync"
    27  	"time"
    28  
    29  	"cmd/go/internal/auth"
    30  	"cmd/go/internal/base"
    31  	"cmd/go/internal/cfg"
    32  	"cmd/go/internal/web/intercept"
    33  	"cmd/internal/browser"
    34  )
    35  
    36  const userAgent = "GoCommand/1 (+https://go.dev/cmd/go)"
    37  
    38  // impatientInsecureHTTPClient is used with GOINSECURE,
    39  // when we're connecting to https servers that might not be there
    40  // or might be using self-signed certificates.
    41  var impatientInsecureHTTPClient = &http.Client{
    42  	CheckRedirect: checkRedirect,
    43  	Timeout:       5 * time.Second,
    44  	Transport: &http.Transport{
    45  		Proxy: http.ProxyFromEnvironment,
    46  		TLSClientConfig: &tls.Config{
    47  			InsecureSkipVerify: true,
    48  		},
    49  	},
    50  }
    51  
    52  var securityPreservingDefaultClient = securityPreservingHTTPClient(http.DefaultClient)
    53  
    54  // securityPreservingHTTPClient returns a client that is like the original
    55  // but rejects redirects to plain-HTTP URLs if the original URL was secure.
    56  func securityPreservingHTTPClient(original *http.Client) *http.Client {
    57  	c := new(http.Client)
    58  	*c = *original
    59  	c.CheckRedirect = func(req *http.Request, via []*http.Request) error {
    60  		if len(via) > 0 && via[0].URL.Scheme == "https" && req.URL.Scheme != "https" {
    61  			lastHop := via[len(via)-1].URL
    62  			return fmt.Errorf("redirected from secure URL %s to insecure URL %s", lastHop, req.URL)
    63  		}
    64  		return checkRedirect(req, via)
    65  	}
    66  	return c
    67  }
    68  
    69  func checkRedirect(req *http.Request, via []*http.Request) error {
    70  	// Go's http.DefaultClient allows 10 redirects before returning an error.
    71  	// Mimic that behavior here.
    72  	if len(via) >= 10 {
    73  		return errors.New("stopped after 10 redirects")
    74  	}
    75  	hasGoGet1 := via[len(via)-1].URL.Query().Get("go-get") == "1"
    76  	if hasGoGet1 {
    77  		if len(req.URL.RawQuery) > 0 {
    78  			req.URL.RawQuery += "&"
    79  		}
    80  		req.URL.RawQuery += "go-get=1"
    81  	}
    82  
    83  	intercept.Request(req)
    84  	return nil
    85  }
    86  
    87  func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
    88  	start := time.Now()
    89  
    90  	if url.Scheme == "file" {
    91  		return getFile(url)
    92  	}
    93  
    94  	if intercept.TestHooksEnabled {
    95  		switch url.Host {
    96  		case "localhost.localdev":
    97  			return nil, fmt.Errorf("no such host localhost.localdev")
    98  
    99  		default:
   100  			if os.Getenv("TESTGONETWORK") == "panic" {
   101  				if _, ok := intercept.URL(url); !ok {
   102  					host := url.Host
   103  					if h, _, err := net.SplitHostPort(url.Host); err == nil && h != "" {
   104  						host = h
   105  					}
   106  					addr := net.ParseIP(host)
   107  					if addr == nil || (!addr.IsLoopback() && !addr.IsUnspecified()) {
   108  						panic("use of network: " + url.String())
   109  					}
   110  				}
   111  			}
   112  		}
   113  	}
   114  
   115  	var (
   116  		fetched *urlpkg.URL
   117  		res     *http.Response
   118  		err     error
   119  	)
   120  	if url.Scheme == "" || url.Scheme == "https" {
   121  		secure := new(urlpkg.URL)
   122  		*secure = *url
   123  		secure.Scheme = "https"
   124  
   125  		res, err = fetch(security, secure, 0, "")
   126  		if err == nil {
   127  			fetched = secure
   128  		} else {
   129  			if cfg.BuildX {
   130  				fmt.Fprintf(os.Stderr, "# get %s: %v\n", secure.Redacted(), err)
   131  			}
   132  			if security != Insecure || url.Scheme == "https" {
   133  				// HTTPS failed, and we can't fall back to plain HTTP.
   134  				// Report the error from the HTTPS attempt.
   135  				return nil, err
   136  			}
   137  		}
   138  	}
   139  
   140  	if res == nil {
   141  		switch url.Scheme {
   142  		case "http":
   143  			if security == SecureOnly {
   144  				if cfg.BuildX {
   145  					fmt.Fprintf(os.Stderr, "# get %s: insecure\n", url.Redacted())
   146  				}
   147  				return nil, fmt.Errorf("insecure URL: %s", url.Redacted())
   148  			}
   149  		case "":
   150  			if security != Insecure {
   151  				panic("should have returned after HTTPS failure")
   152  			}
   153  		default:
   154  			if cfg.BuildX {
   155  				fmt.Fprintf(os.Stderr, "# get %s: unsupported\n", url.Redacted())
   156  			}
   157  			return nil, fmt.Errorf("unsupported scheme: %s", url.Redacted())
   158  		}
   159  
   160  		insecure := new(urlpkg.URL)
   161  		*insecure = *url
   162  		insecure.Scheme = "http"
   163  		if insecure.User != nil && security != Insecure {
   164  			if cfg.BuildX {
   165  				fmt.Fprintf(os.Stderr, "# get %s: insecure credentials\n", insecure.Redacted())
   166  			}
   167  			return nil, fmt.Errorf("refusing to pass credentials to insecure URL: %s", insecure.Redacted())
   168  		}
   169  
   170  		res, err = fetch(security, insecure, 0, "")
   171  		if err == nil {
   172  			fetched = insecure
   173  		} else {
   174  			if cfg.BuildX {
   175  				fmt.Fprintf(os.Stderr, "# get %s: %v\n", insecure.Redacted(), err)
   176  			}
   177  			// HTTP failed, and we already tried HTTPS if applicable.
   178  			// Report the error from the HTTP attempt.
   179  			return nil, err
   180  		}
   181  	}
   182  
   183  	// Note: accepting a non-200 OK here, so people can serve a
   184  	// meta import in their http 404 page.
   185  	if cfg.BuildX {
   186  		fmt.Fprintf(os.Stderr, "# get %s: %v (%.3fs)\n", fetched.Redacted(), res.Status, time.Since(start).Seconds())
   187  	}
   188  
   189  	r := &Response{
   190  		URL:        fetched.Redacted(),
   191  		Status:     res.Status,
   192  		StatusCode: res.StatusCode,
   193  		Header:     map[string][]string(res.Header),
   194  		Body:       res.Body,
   195  	}
   196  
   197  	switch res.StatusCode {
   198  	case http.StatusOK:
   199  		r.Body = newRetryBody(security, fetched, res)
   200  	default:
   201  		contentType := res.Header.Get("Content-Type")
   202  		if mediaType, params, _ := mime.ParseMediaType(contentType); mediaType == "text/plain" {
   203  			switch charset := strings.ToLower(params["charset"]); charset {
   204  			case "us-ascii", "utf-8", "":
   205  				// Body claims to be plain text in UTF-8 or a subset thereof.
   206  				// Try to extract a useful error message from it.
   207  				r.errorDetail.r = res.Body
   208  				r.Body = &r.errorDetail
   209  			}
   210  		}
   211  	}
   212  
   213  	return r, nil
   214  }
   215  
   216  func fetch(security SecurityMode, url *urlpkg.URL, offset int64, ifRange string) (*http.Response, error) {
   217  	// Note: The -v build flag does not mean "print logging information",
   218  	// despite its historical misuse for this in GOPATH-based go get.
   219  	// We print extra logging in -x mode instead, which traces what
   220  	// commands are executed.
   221  	if cfg.BuildX {
   222  		if offset == 0 {
   223  			fmt.Fprintf(os.Stderr, "# get %s\n", url.Redacted())
   224  		} else {
   225  			fmt.Fprintf(os.Stderr, "# get %s (offset %d)\n", url.Redacted(), offset)
   226  		}
   227  	}
   228  
   229  	req, err := http.NewRequest("GET", url.String(), nil)
   230  	if err != nil {
   231  		return nil, err
   232  	}
   233  	t, intercepted := intercept.URL(req.URL)
   234  	var client *http.Client
   235  	if security == Insecure && url.Scheme == "https" {
   236  		client = impatientInsecureHTTPClient
   237  	} else if intercepted && t.Client != nil {
   238  		client = securityPreservingHTTPClient(t.Client)
   239  	} else {
   240  		client = securityPreservingDefaultClient
   241  	}
   242  	if url.Scheme == "https" {
   243  		// Use initial GOAUTH credentials.
   244  		auth.AddCredentials(client, req, nil, "")
   245  	}
   246  	if intercepted {
   247  		req.Host = req.URL.Host
   248  		req.URL.Host = t.ToHost
   249  	}
   250  	req.Header.Set("User-Agent", userAgent)
   251  
   252  	setRangeHeaders := func(req *http.Request) {
   253  		if offset <= 0 {
   254  			return
   255  		}
   256  		// Make a conditional range request: The server will only respond with
   257  		// 206 Partial Content if the resource hasn't changed since our last GET.
   258  		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
   259  		req.Header.Set("If-Range", ifRange)
   260  	}
   261  	setRangeHeaders(req)
   262  
   263  	release, err := base.AcquireNet()
   264  	if err != nil {
   265  		return nil, err
   266  	}
   267  	defer func() {
   268  		if err != nil && release != nil {
   269  			release()
   270  		}
   271  	}()
   272  	res, err := client.Do(req)
   273  	// If the initial request fails with a 4xx client error and the
   274  	// response body didn't satisfy the request
   275  	// (e.g. a valid <meta name="go-import"> tag),
   276  	// retry the request with credentials obtained by invoking GOAUTH
   277  	// with the request URL.
   278  	if url.Scheme == "https" && err == nil && res.StatusCode >= 400 && res.StatusCode < 500 {
   279  		// Close the body of the previous response since we
   280  		// are discarding it and creating a new one.
   281  		res.Body.Close()
   282  		req, err = http.NewRequest("GET", url.String(), nil)
   283  		if err != nil {
   284  			return nil, err
   285  		}
   286  		auth.AddCredentials(client, req, res, url.String())
   287  		if intercepted {
   288  			intercept.Request(req)
   289  		}
   290  		setRangeHeaders(req)
   291  		res, err = client.Do(req)
   292  	}
   293  
   294  	if err != nil {
   295  		// Per the docs for [net/http.Client.Do], “On error, any Response can be
   296  		// ignored. A non-nil Response with a non-nil error only occurs when
   297  		// CheckRedirect fails, and even then the returned Response.Body is
   298  		// already closed.”
   299  		return nil, err
   300  	}
   301  
   302  	// “If the returned error is nil, the Response will contain a non-nil Body
   303  	// which the user is expected to close.”
   304  	body := res.Body
   305  	res.Body = hookCloser{
   306  		ReadCloser: body,
   307  		afterClose: release,
   308  	}
   309  	return res, nil
   310  }
   311  
   312  type retryBody struct {
   313  	security SecurityMode
   314  	url      *urlpkg.URL
   315  	ifRange  string
   316  
   317  	mu          sync.Mutex
   318  	closed      bool
   319  	restarts    int
   320  	startOffset int64
   321  	offset      int64
   322  	size        int64
   323  	err         error
   324  	lastReadErr error
   325  	body        io.ReadCloser
   326  }
   327  
   328  func newRetryBody(security SecurityMode, u *urlpkg.URL, res *http.Response) io.ReadCloser {
   329  	if res.Uncompressed {
   330  		// The http Transport automatically added an Accept-Encoding: gzip header,
   331  		// and the server responded with Content-Encoding: gzip. We can't resume
   332  		// a broken download, because we don't know the correct content offset to
   333  		// resume at--a Range request will specify a location in the compressed
   334  		// content, and res.Body contains the uncompressed content.
   335  		//
   336  		// We can avoid this case by setting Transport.DisableCompression,
   337  		// but that requires modifying the Transport and the module proxy never
   338  		// responds with Content-Encoding: gzip anyway. Check here just in case,
   339  		// but this should never happen.
   340  		//
   341  		// (If we implement #81200, we can disable automatic decompression
   342  		// on a per-request basis and should do that instead.)
   343  		return res.Body
   344  	}
   345  
   346  	if res.ContentLength < 0 {
   347  		// The server didn't send us a Content-Length header.
   348  		// It probably can't handle Range requests if it doesn't know
   349  		// the size of the files it serves.
   350  		return res.Body
   351  	}
   352  
   353  	// We need a strong ETag or Last-Modified header to retry with a Range request.
   354  	// If we don't have one, just use the original body.
   355  	ifRange := res.Header.Get("ETag")
   356  	if !isStrongETag(ifRange) {
   357  		ifRange = res.Header.Get("Last-Modified")
   358  	}
   359  	if ifRange == "" {
   360  		return res.Body
   361  	}
   362  
   363  	return &retryBody{
   364  		// We could use res.Request.URL for the retry URL,
   365  		// which would avoid re-following redirects.
   366  		// On the other hand, following redirects might send us to a healthier destination.
   367  		// Probably doesn't actually matter either way in practice.
   368  		url: u,
   369  
   370  		security: security,
   371  		ifRange:  ifRange,
   372  		size:     res.ContentLength,
   373  		body:     res.Body,
   374  	}
   375  }
   376  
   377  func (b *retryBody) Close() error {
   378  	// This is the same mutex Read takes, so Close can't interrupt a Read.
   379  	// Not a problem in our current usage.
   380  	b.mu.Lock()
   381  	defer b.mu.Unlock()
   382  	if b.closed {
   383  		return nil
   384  	}
   385  	b.closed = true
   386  
   387  	// If we hit an error prior to reading to EOF, always return an error from Close.
   388  	// Prefer the error from b.body.Close, if we have an open body and closing it fails.
   389  	var closeErr error
   390  	if b.err != nil && b.err != io.EOF {
   391  		closeErr = fmt.Errorf("fetch error: %v", b.err)
   392  	}
   393  	if b.body != nil {
   394  		if err := b.body.Close(); err != nil {
   395  			closeErr = err
   396  		}
   397  		b.body = nil
   398  	}
   399  	return closeErr
   400  }
   401  
   402  func (b *retryBody) Read(p []byte) (n int, err error) {
   403  	b.mu.Lock()
   404  	defer b.mu.Unlock()
   405  
   406  	if b.closed {
   407  		return 0, errors.New("read from closed body")
   408  	}
   409  	if b.err != nil {
   410  		return 0, b.err
   411  	}
   412  
   413  	for {
   414  		if b.body == nil {
   415  			// The previous read returned an error,
   416  			// and we want to try resuming the download with a Range request.
   417  			if err := b.resume(); err != nil {
   418  				if cfg.BuildX {
   419  					fmt.Fprintf(os.Stderr, "# get %s: resume: %v\n", b.url.Redacted(), err)
   420  				}
   421  				b.err = b.lastReadErr
   422  				return n, b.lastReadErr
   423  			}
   424  		}
   425  
   426  		n, err = b.body.Read(p)
   427  		if n > 0 {
   428  			b.offset += int64(n)
   429  			if b.size >= 0 && b.offset > b.size {
   430  				// Can't retry this (we're past the end of the expected body).
   431  				b.err = fmt.Errorf("read %v bytes from %v-byte response",
   432  					b.offset, b.size)
   433  				return 0, b.err
   434  			}
   435  		}
   436  		if err == io.EOF && b.size >= 0 && b.offset != b.size {
   437  			err = io.ErrUnexpectedEOF
   438  		}
   439  		b.err = err
   440  
   441  		const maxRestarts = 2 // 3 total: 1 initial + 2 restarts
   442  		switch {
   443  		case err == nil || err == io.EOF:
   444  			return n, err // success
   445  		case err != nil && b.offset == b.size:
   446  			return n, io.EOF // non-EOF error at the exact end of file, call it EOF
   447  		case b.offset == b.startOffset:
   448  			return n, err // no progress
   449  		case b.restarts >= maxRestarts:
   450  			return n, err // too many restarts
   451  		}
   452  
   453  		// We've hit an error while downloading,
   454  		// and we can try to resume.
   455  		if cfg.BuildX {
   456  			fmt.Fprintf(os.Stderr, "# get %s: interrupted\n", b.url.Redacted())
   457  		}
   458  		b.lastReadErr = err
   459  		b.err = nil
   460  		b.body.Close()
   461  		b.body = nil
   462  		b.startOffset = b.offset
   463  		b.restarts++
   464  		if n != 0 {
   465  			// We did get some data, so return it before resuming.
   466  			return n, nil
   467  		}
   468  	}
   469  }
   470  
   471  func (b *retryBody) resume() error {
   472  	res, err := fetch(b.security, b.url, b.offset, b.ifRange)
   473  	if err != nil {
   474  		return err
   475  	}
   476  	defer func() {
   477  		if res.Body != nil {
   478  			res.Body.Close()
   479  		}
   480  	}()
   481  	if res.StatusCode != http.StatusPartialContent {
   482  		// We could handle a 200 response (which resends the entire response)
   483  		// and skip to where we left off. Don't bother for now.
   484  		return fmt.Errorf("non-206 response code: %v", res.StatusCode)
   485  	}
   486  	cr := res.Header.Get("Content-Range")
   487  	first, _, size, ok := parseContentRange(cr)
   488  	if !ok || first != b.offset || (b.size != -1 && size != -1 && b.size != size) {
   489  		return fmt.Errorf("invalid Content-Range: %q", cr)
   490  	}
   491  	b.body = res.Body
   492  	res.Body = nil
   493  	return nil
   494  }
   495  
   496  func isStrongETag(s string) bool {
   497  	return s != "" && !strings.HasPrefix(s, "W/")
   498  }
   499  
   500  // parseContentRange parses a Content-Range header.
   501  func parseContentRange(s string) (first, last, total int64, ok bool) {
   502  	// "bytes NNNN-NNNN/NNNN"
   503  	s = strings.ToLower(strings.TrimSpace(s))
   504  	s, ok = strings.CutPrefix(s, "bytes ")
   505  	if !ok {
   506  		return 0, 0, 0, false
   507  	}
   508  	s = strings.TrimSpace(s)
   509  	// "NNNN-NNNN/NNNN"
   510  	first, s, ok = cutInt63(s, "-")
   511  	if !ok {
   512  		return 0, 0, 0, false
   513  	}
   514  	// "NNNN/NNNN"
   515  	last, s, ok = cutInt63(s, "/")
   516  	if !ok || first > last {
   517  		return 0, 0, 0, false
   518  	}
   519  	// "NNNN"
   520  	if s == "*" {
   521  		return first, last, -1, true // "bytes NNNN-NNNN/*"
   522  	}
   523  	total, err := parseInt63(s)
   524  	if err != nil {
   525  		return 0, 0, 0, false
   526  	}
   527  	return first, last, total, true
   528  }
   529  
   530  func cutInt63(s, sep string) (int64, string, bool) {
   531  	part, rest, ok := strings.Cut(s, sep)
   532  	if !ok {
   533  		return 0, rest, false
   534  	}
   535  	n, err := parseInt63(part)
   536  	if err != nil {
   537  		return 0, rest, false
   538  	}
   539  	return n, rest, true
   540  }
   541  
   542  func parseInt63(s string) (int64, error) {
   543  	n, err := strconv.ParseUint(s, 10, 63)
   544  	if err != nil {
   545  		return 0, err
   546  	}
   547  	return int64(n), err
   548  }
   549  
   550  func getFile(u *urlpkg.URL) (*Response, error) {
   551  	path, err := urlToFilePath(u)
   552  	if err != nil {
   553  		return nil, err
   554  	}
   555  	f, err := os.Open(path)
   556  
   557  	if os.IsNotExist(err) {
   558  		return &Response{
   559  			URL:        u.Redacted(),
   560  			Status:     http.StatusText(http.StatusNotFound),
   561  			StatusCode: http.StatusNotFound,
   562  			Body:       http.NoBody,
   563  			fileErr:    err,
   564  		}, nil
   565  	}
   566  
   567  	if os.IsPermission(err) {
   568  		return &Response{
   569  			URL:        u.Redacted(),
   570  			Status:     http.StatusText(http.StatusForbidden),
   571  			StatusCode: http.StatusForbidden,
   572  			Body:       http.NoBody,
   573  			fileErr:    err,
   574  		}, nil
   575  	}
   576  
   577  	if err != nil {
   578  		return nil, err
   579  	}
   580  
   581  	return &Response{
   582  		URL:        u.Redacted(),
   583  		Status:     http.StatusText(http.StatusOK),
   584  		StatusCode: http.StatusOK,
   585  		Body:       f,
   586  	}, nil
   587  }
   588  
   589  func openBrowser(url string) bool { return browser.Open(url) }
   590  
   591  func isLocalHost(u *urlpkg.URL) bool {
   592  	// VCSTestRepoURL itself is secure, and it may redirect requests to other
   593  	// ports (such as a port serving the "svn" protocol) which should also be
   594  	// considered secure.
   595  	host, _, err := net.SplitHostPort(u.Host)
   596  	if err != nil {
   597  		host = u.Host
   598  	}
   599  	if host == "localhost" {
   600  		return true
   601  	}
   602  	if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() {
   603  		return true
   604  	}
   605  	return false
   606  }
   607  
   608  type hookCloser struct {
   609  	io.ReadCloser
   610  	afterClose func()
   611  }
   612  
   613  func (c hookCloser) Close() error {
   614  	err := c.ReadCloser.Close()
   615  	c.afterClose()
   616  	return err
   617  }
   618
View as plain text