http.go

     1  // Copyright 2012 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build !cmd_go_bootstrap
     6  
     7  // This code is compiled into the real 'go' binary, but it is not
     8  // compiled into the binary that is built during all.bash, so as
     9  // to avoid needing to build net (and thus use cgo) during the
    10  // bootstrap process.
    11  
    12  package web
    13  
    14  import (
    15  	"crypto/tls"
    16  	"errors"
    17  	"fmt"
    18  	"io"
    19  	"mime"
    20  	"net"
    21  	"net/http"
    22  	urlpkg "net/url"
    23  	"os"
    24  	"strings"
    25  	"time"
    26  
    27  	"cmd/go/internal/auth"
    28  	"cmd/go/internal/base"
    29  	"cmd/go/internal/cfg"
    30  	"cmd/go/internal/web/intercept"
    31  	"cmd/internal/browser"
    32  )
    33  
    34  const userAgent = "GoCommand/1 (+https://go.dev/cmd/go)"
    35  
    36  // impatientInsecureHTTPClient is used with GOINSECURE,
    37  // when we're connecting to https servers that might not be there
    38  // or might be using self-signed certificates.
    39  var impatientInsecureHTTPClient = &http.Client{
    40  	CheckRedirect: checkRedirect,
    41  	Timeout:       5 * time.Second,
    42  	Transport: &http.Transport{
    43  		Proxy: http.ProxyFromEnvironment,
    44  		TLSClientConfig: &tls.Config{
    45  			InsecureSkipVerify: true,
    46  		},
    47  	},
    48  }
    49  
    50  var securityPreservingDefaultClient = securityPreservingHTTPClient(http.DefaultClient)
    51  
    52  // securityPreservingHTTPClient returns a client that is like the original
    53  // but rejects redirects to plain-HTTP URLs if the original URL was secure.
    54  func securityPreservingHTTPClient(original *http.Client) *http.Client {
    55  	c := new(http.Client)
    56  	*c = *original
    57  	c.CheckRedirect = func(req *http.Request, via []*http.Request) error {
    58  		if len(via) > 0 && via[0].URL.Scheme == "https" && req.URL.Scheme != "https" {
    59  			lastHop := via[len(via)-1].URL
    60  			return fmt.Errorf("redirected from secure URL %s to insecure URL %s", lastHop, req.URL)
    61  		}
    62  		return checkRedirect(req, via)
    63  	}
    64  	return c
    65  }
    66  
    67  func checkRedirect(req *http.Request, via []*http.Request) error {
    68  	// Go's http.DefaultClient allows 10 redirects before returning an error.
    69  	// Mimic that behavior here.
    70  	if len(via) >= 10 {
    71  		return errors.New("stopped after 10 redirects")
    72  	}
    73  
    74  	intercept.Request(req)
    75  	return nil
    76  }
    77  
    78  func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
    79  	start := time.Now()
    80  
    81  	if url.Scheme == "file" {
    82  		return getFile(url)
    83  	}
    84  
    85  	if intercept.TestHooksEnabled {
    86  		switch url.Host {
    87  		case "localhost.localdev":
    88  			return nil, fmt.Errorf("no such host localhost.localdev")
    89  
    90  		default:
    91  			if os.Getenv("TESTGONETWORK") == "panic" {
    92  				if _, ok := intercept.URL(url); !ok {
    93  					host := url.Host
    94  					if h, _, err := net.SplitHostPort(url.Host); err == nil && h != "" {
    95  						host = h
    96  					}
    97  					addr := net.ParseIP(host)
    98  					if addr == nil || (!addr.IsLoopback() && !addr.IsUnspecified()) {
    99  						panic("use of network: " + url.String())
   100  					}
   101  				}
   102  			}
   103  		}
   104  	}
   105  
   106  	fetch := func(url *urlpkg.URL) (*http.Response, error) {
   107  		// Note: The -v build flag does not mean "print logging information",
   108  		// despite its historical misuse for this in GOPATH-based go get.
   109  		// We print extra logging in -x mode instead, which traces what
   110  		// commands are executed.
   111  		if cfg.BuildX {
   112  			fmt.Fprintf(os.Stderr, "# get %s\n", url.Redacted())
   113  		}
   114  
   115  		req, err := http.NewRequest("GET", url.String(), nil)
   116  		if err != nil {
   117  			return nil, err
   118  		}
   119  		t, intercepted := intercept.URL(req.URL)
   120  		var client *http.Client
   121  		if security == Insecure && url.Scheme == "https" {
   122  			client = impatientInsecureHTTPClient
   123  		} else if intercepted && t.Client != nil {
   124  			client = securityPreservingHTTPClient(t.Client)
   125  		} else {
   126  			client = securityPreservingDefaultClient
   127  		}
   128  		if url.Scheme == "https" {
   129  			// Use initial GOAUTH credentials.
   130  			auth.AddCredentials(client, req, nil, "")
   131  		}
   132  		if intercepted {
   133  			req.Host = req.URL.Host
   134  			req.URL.Host = t.ToHost
   135  		}
   136  		req.Header.Set("User-Agent", userAgent)
   137  
   138  		release, err := base.AcquireNet()
   139  		if err != nil {
   140  			return nil, err
   141  		}
   142  		defer func() {
   143  			if err != nil && release != nil {
   144  				release()
   145  			}
   146  		}()
   147  		res, err := client.Do(req)
   148  		// If the initial request fails with a 4xx client error and the
   149  		// response body didn't satisfy the request
   150  		// (e.g. a valid <meta name="go-import"> tag),
   151  		// retry the request with credentials obtained by invoking GOAUTH
   152  		// with the request URL.
   153  		if url.Scheme == "https" && err == nil && res.StatusCode >= 400 && res.StatusCode < 500 {
   154  			// Close the body of the previous response since we
   155  			// are discarding it and creating a new one.
   156  			res.Body.Close()
   157  			req, err = http.NewRequest("GET", url.String(), nil)
   158  			if err != nil {
   159  				return nil, err
   160  			}
   161  			auth.AddCredentials(client, req, res, url.String())
   162  			intercept.Request(req)
   163  			res, err = client.Do(req)
   164  		}
   165  
   166  		if err != nil {
   167  			// Per the docs for [net/http.Client.Do], “On error, any Response can be
   168  			// ignored. A non-nil Response with a non-nil error only occurs when
   169  			// CheckRedirect fails, and even then the returned Response.Body is
   170  			// already closed.”
   171  			return nil, err
   172  		}
   173  
   174  		// “If the returned error is nil, the Response will contain a non-nil Body
   175  		// which the user is expected to close.”
   176  		body := res.Body
   177  		res.Body = hookCloser{
   178  			ReadCloser: body,
   179  			afterClose: release,
   180  		}
   181  		return res, nil
   182  	}
   183  
   184  	var (
   185  		fetched *urlpkg.URL
   186  		res     *http.Response
   187  		err     error
   188  	)
   189  	if url.Scheme == "" || url.Scheme == "https" {
   190  		secure := new(urlpkg.URL)
   191  		*secure = *url
   192  		secure.Scheme = "https"
   193  
   194  		res, err = fetch(secure)
   195  		if err == nil {
   196  			fetched = secure
   197  		} else {
   198  			if cfg.BuildX {
   199  				fmt.Fprintf(os.Stderr, "# get %s: %v\n", secure.Redacted(), err)
   200  			}
   201  			if security != Insecure || url.Scheme == "https" {
   202  				// HTTPS failed, and we can't fall back to plain HTTP.
   203  				// Report the error from the HTTPS attempt.
   204  				return nil, err
   205  			}
   206  		}
   207  	}
   208  
   209  	if res == nil {
   210  		switch url.Scheme {
   211  		case "http":
   212  			if security == SecureOnly {
   213  				if cfg.BuildX {
   214  					fmt.Fprintf(os.Stderr, "# get %s: insecure\n", url.Redacted())
   215  				}
   216  				return nil, fmt.Errorf("insecure URL: %s", url.Redacted())
   217  			}
   218  		case "":
   219  			if security != Insecure {
   220  				panic("should have returned after HTTPS failure")
   221  			}
   222  		default:
   223  			if cfg.BuildX {
   224  				fmt.Fprintf(os.Stderr, "# get %s: unsupported\n", url.Redacted())
   225  			}
   226  			return nil, fmt.Errorf("unsupported scheme: %s", url.Redacted())
   227  		}
   228  
   229  		insecure := new(urlpkg.URL)
   230  		*insecure = *url
   231  		insecure.Scheme = "http"
   232  		if insecure.User != nil && security != Insecure {
   233  			if cfg.BuildX {
   234  				fmt.Fprintf(os.Stderr, "# get %s: insecure credentials\n", insecure.Redacted())
   235  			}
   236  			return nil, fmt.Errorf("refusing to pass credentials to insecure URL: %s", insecure.Redacted())
   237  		}
   238  
   239  		res, err = fetch(insecure)
   240  		if err == nil {
   241  			fetched = insecure
   242  		} else {
   243  			if cfg.BuildX {
   244  				fmt.Fprintf(os.Stderr, "# get %s: %v\n", insecure.Redacted(), err)
   245  			}
   246  			// HTTP failed, and we already tried HTTPS if applicable.
   247  			// Report the error from the HTTP attempt.
   248  			return nil, err
   249  		}
   250  	}
   251  
   252  	// Note: accepting a non-200 OK here, so people can serve a
   253  	// meta import in their http 404 page.
   254  	if cfg.BuildX {
   255  		fmt.Fprintf(os.Stderr, "# get %s: %v (%.3fs)\n", fetched.Redacted(), res.Status, time.Since(start).Seconds())
   256  	}
   257  
   258  	r := &Response{
   259  		URL:        fetched.Redacted(),
   260  		Status:     res.Status,
   261  		StatusCode: res.StatusCode,
   262  		Header:     map[string][]string(res.Header),
   263  		Body:       res.Body,
   264  	}
   265  
   266  	if res.StatusCode != http.StatusOK {
   267  		contentType := res.Header.Get("Content-Type")
   268  		if mediaType, params, _ := mime.ParseMediaType(contentType); mediaType == "text/plain" {
   269  			switch charset := strings.ToLower(params["charset"]); charset {
   270  			case "us-ascii", "utf-8", "":
   271  				// Body claims to be plain text in UTF-8 or a subset thereof.
   272  				// Try to extract a useful error message from it.
   273  				r.errorDetail.r = res.Body
   274  				r.Body = &r.errorDetail
   275  			}
   276  		}
   277  	}
   278  
   279  	return r, nil
   280  }
   281  
   282  func getFile(u *urlpkg.URL) (*Response, error) {
   283  	path, err := urlToFilePath(u)
   284  	if err != nil {
   285  		return nil, err
   286  	}
   287  	f, err := os.Open(path)
   288  
   289  	if os.IsNotExist(err) {
   290  		return &Response{
   291  			URL:        u.Redacted(),
   292  			Status:     http.StatusText(http.StatusNotFound),
   293  			StatusCode: http.StatusNotFound,
   294  			Body:       http.NoBody,
   295  			fileErr:    err,
   296  		}, nil
   297  	}
   298  
   299  	if os.IsPermission(err) {
   300  		return &Response{
   301  			URL:        u.Redacted(),
   302  			Status:     http.StatusText(http.StatusForbidden),
   303  			StatusCode: http.StatusForbidden,
   304  			Body:       http.NoBody,
   305  			fileErr:    err,
   306  		}, nil
   307  	}
   308  
   309  	if err != nil {
   310  		return nil, err
   311  	}
   312  
   313  	return &Response{
   314  		URL:        u.Redacted(),
   315  		Status:     http.StatusText(http.StatusOK),
   316  		StatusCode: http.StatusOK,
   317  		Body:       f,
   318  	}, nil
   319  }
   320  
   321  func openBrowser(url string) bool { return browser.Open(url) }
   322  
   323  func isLocalHost(u *urlpkg.URL) bool {
   324  	// VCSTestRepoURL itself is secure, and it may redirect requests to other
   325  	// ports (such as a port serving the "svn" protocol) which should also be
   326  	// considered secure.
   327  	host, _, err := net.SplitHostPort(u.Host)
   328  	if err != nil {
   329  		host = u.Host
   330  	}
   331  	if host == "localhost" {
   332  		return true
   333  	}
   334  	if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() {
   335  		return true
   336  	}
   337  	return false
   338  }
   339  
   340  type hookCloser struct {
   341  	io.ReadCloser
   342  	afterClose func()
   343  }
   344  
   345  func (c hookCloser) Close() error {
   346  	err := c.ReadCloser.Close()
   347  	c.afterClose()
   348  	return err
   349  }
   350
View as plain text