http.go

     1  // Copyright 2012 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build !cmd_go_bootstrap
     6  
     7  // This code is compiled into the real 'go' binary, but it is not
     8  // compiled into the binary that is built during all.bash, so as
     9  // to avoid needing to build net (and thus use cgo) during the
    10  // bootstrap process.
    11  
    12  package web
    13  
    14  import (
    15  	"crypto/tls"
    16  	"errors"
    17  	"fmt"
    18  	"io"
    19  	"mime"
    20  	"net"
    21  	"net/http"
    22  	urlpkg "net/url"
    23  	"os"
    24  	"strings"
    25  	"time"
    26  
    27  	"cmd/go/internal/auth"
    28  	"cmd/go/internal/base"
    29  	"cmd/go/internal/cfg"
    30  	"cmd/go/internal/web/intercept"
    31  	"cmd/internal/browser"
    32  )
    33  
    34  // impatientInsecureHTTPClient is used with GOINSECURE,
    35  // when we're connecting to https servers that might not be there
    36  // or might be using self-signed certificates.
    37  var impatientInsecureHTTPClient = &http.Client{
    38  	CheckRedirect: checkRedirect,
    39  	Timeout:       5 * time.Second,
    40  	Transport: &http.Transport{
    41  		Proxy: http.ProxyFromEnvironment,
    42  		TLSClientConfig: &tls.Config{
    43  			InsecureSkipVerify: true,
    44  		},
    45  	},
    46  }
    47  
    48  var securityPreservingDefaultClient = securityPreservingHTTPClient(http.DefaultClient)
    49  
    50  // securityPreservingHTTPClient returns a client that is like the original
    51  // but rejects redirects to plain-HTTP URLs if the original URL was secure.
    52  func securityPreservingHTTPClient(original *http.Client) *http.Client {
    53  	c := new(http.Client)
    54  	*c = *original
    55  	c.CheckRedirect = func(req *http.Request, via []*http.Request) error {
    56  		if len(via) > 0 && via[0].URL.Scheme == "https" && req.URL.Scheme != "https" {
    57  			lastHop := via[len(via)-1].URL
    58  			return fmt.Errorf("redirected from secure URL %s to insecure URL %s", lastHop, req.URL)
    59  		}
    60  		return checkRedirect(req, via)
    61  	}
    62  	return c
    63  }
    64  
    65  func checkRedirect(req *http.Request, via []*http.Request) error {
    66  	// Go's http.DefaultClient allows 10 redirects before returning an error.
    67  	// Mimic that behavior here.
    68  	if len(via) >= 10 {
    69  		return errors.New("stopped after 10 redirects")
    70  	}
    71  
    72  	intercept.Request(req)
    73  	return nil
    74  }
    75  
    76  func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
    77  	start := time.Now()
    78  
    79  	if url.Scheme == "file" {
    80  		return getFile(url)
    81  	}
    82  
    83  	if intercept.TestHooksEnabled {
    84  		switch url.Host {
    85  		case "proxy.golang.org":
    86  			if os.Getenv("TESTGOPROXY404") == "1" {
    87  				res := &Response{
    88  					URL:        url.Redacted(),
    89  					Status:     "404 testing",
    90  					StatusCode: 404,
    91  					Header:     make(map[string][]string),
    92  					Body:       http.NoBody,
    93  				}
    94  				if cfg.BuildX {
    95  					fmt.Fprintf(os.Stderr, "# get %s: %v (%.3fs)\n", url.Redacted(), res.Status, time.Since(start).Seconds())
    96  				}
    97  				return res, nil
    98  			}
    99  
   100  		case "localhost.localdev":
   101  			return nil, fmt.Errorf("no such host localhost.localdev")
   102  
   103  		default:
   104  			if os.Getenv("TESTGONETWORK") == "panic" {
   105  				if _, ok := intercept.URL(url); !ok {
   106  					host := url.Host
   107  					if h, _, err := net.SplitHostPort(url.Host); err == nil && h != "" {
   108  						host = h
   109  					}
   110  					addr := net.ParseIP(host)
   111  					if addr == nil || (!addr.IsLoopback() && !addr.IsUnspecified()) {
   112  						panic("use of network: " + url.String())
   113  					}
   114  				}
   115  			}
   116  		}
   117  	}
   118  
   119  	fetch := func(url *urlpkg.URL) (*http.Response, error) {
   120  		// Note: The -v build flag does not mean "print logging information",
   121  		// despite its historical misuse for this in GOPATH-based go get.
   122  		// We print extra logging in -x mode instead, which traces what
   123  		// commands are executed.
   124  		if cfg.BuildX {
   125  			fmt.Fprintf(os.Stderr, "# get %s\n", url.Redacted())
   126  		}
   127  
   128  		req, err := http.NewRequest("GET", url.String(), nil)
   129  		if err != nil {
   130  			return nil, err
   131  		}
   132  		t, intercepted := intercept.URL(req.URL)
   133  		var client *http.Client
   134  		if security == Insecure && url.Scheme == "https" {
   135  			client = impatientInsecureHTTPClient
   136  		} else if intercepted && t.Client != nil {
   137  			client = securityPreservingHTTPClient(t.Client)
   138  		} else {
   139  			client = securityPreservingDefaultClient
   140  		}
   141  		if url.Scheme == "https" {
   142  			// Use initial GOAUTH credentials.
   143  			auth.AddCredentials(client, req, "")
   144  		}
   145  		if intercepted {
   146  			req.Host = req.URL.Host
   147  			req.URL.Host = t.ToHost
   148  		}
   149  
   150  		release, err := base.AcquireNet()
   151  		if err != nil {
   152  			return nil, err
   153  		}
   154  		defer func() {
   155  			if err != nil && release != nil {
   156  				release()
   157  			}
   158  		}()
   159  		res, err := client.Do(req)
   160  		// If the initial request fails with a 4xx client error and the
   161  		// response body didn't satisfy the request
   162  		// (e.g. a valid <meta name="go-import"> tag),
   163  		// retry the request with credentials obtained by invoking GOAUTH
   164  		// with the request URL.
   165  		if url.Scheme == "https" && err == nil && res.StatusCode >= 400 && res.StatusCode < 500 {
   166  			// Close the body of the previous response since we
   167  			// are discarding it and creating a new one.
   168  			res.Body.Close()
   169  			req, err = http.NewRequest("GET", url.String(), nil)
   170  			if err != nil {
   171  				return nil, err
   172  			}
   173  			auth.AddCredentials(client, req, url.String())
   174  			intercept.Request(req)
   175  			res, err = client.Do(req)
   176  		}
   177  
   178  		if err != nil {
   179  			// Per the docs for [net/http.Client.Do], “On error, any Response can be
   180  			// ignored. A non-nil Response with a non-nil error only occurs when
   181  			// CheckRedirect fails, and even then the returned Response.Body is
   182  			// already closed.”
   183  			return nil, err
   184  		}
   185  
   186  		// “If the returned error is nil, the Response will contain a non-nil Body
   187  		// which the user is expected to close.”
   188  		body := res.Body
   189  		res.Body = hookCloser{
   190  			ReadCloser: body,
   191  			afterClose: release,
   192  		}
   193  		return res, nil
   194  	}
   195  
   196  	var (
   197  		fetched *urlpkg.URL
   198  		res     *http.Response
   199  		err     error
   200  	)
   201  	if url.Scheme == "" || url.Scheme == "https" {
   202  		secure := new(urlpkg.URL)
   203  		*secure = *url
   204  		secure.Scheme = "https"
   205  
   206  		res, err = fetch(secure)
   207  		if err == nil {
   208  			fetched = secure
   209  		} else {
   210  			if cfg.BuildX {
   211  				fmt.Fprintf(os.Stderr, "# get %s: %v\n", secure.Redacted(), err)
   212  			}
   213  			if security != Insecure || url.Scheme == "https" {
   214  				// HTTPS failed, and we can't fall back to plain HTTP.
   215  				// Report the error from the HTTPS attempt.
   216  				return nil, err
   217  			}
   218  		}
   219  	}
   220  
   221  	if res == nil {
   222  		switch url.Scheme {
   223  		case "http":
   224  			if security == SecureOnly {
   225  				if cfg.BuildX {
   226  					fmt.Fprintf(os.Stderr, "# get %s: insecure\n", url.Redacted())
   227  				}
   228  				return nil, fmt.Errorf("insecure URL: %s", url.Redacted())
   229  			}
   230  		case "":
   231  			if security != Insecure {
   232  				panic("should have returned after HTTPS failure")
   233  			}
   234  		default:
   235  			if cfg.BuildX {
   236  				fmt.Fprintf(os.Stderr, "# get %s: unsupported\n", url.Redacted())
   237  			}
   238  			return nil, fmt.Errorf("unsupported scheme: %s", url.Redacted())
   239  		}
   240  
   241  		insecure := new(urlpkg.URL)
   242  		*insecure = *url
   243  		insecure.Scheme = "http"
   244  		if insecure.User != nil && security != Insecure {
   245  			if cfg.BuildX {
   246  				fmt.Fprintf(os.Stderr, "# get %s: insecure credentials\n", insecure.Redacted())
   247  			}
   248  			return nil, fmt.Errorf("refusing to pass credentials to insecure URL: %s", insecure.Redacted())
   249  		}
   250  
   251  		res, err = fetch(insecure)
   252  		if err == nil {
   253  			fetched = insecure
   254  		} else {
   255  			if cfg.BuildX {
   256  				fmt.Fprintf(os.Stderr, "# get %s: %v\n", insecure.Redacted(), err)
   257  			}
   258  			// HTTP failed, and we already tried HTTPS if applicable.
   259  			// Report the error from the HTTP attempt.
   260  			return nil, err
   261  		}
   262  	}
   263  
   264  	// Note: accepting a non-200 OK here, so people can serve a
   265  	// meta import in their http 404 page.
   266  	if cfg.BuildX {
   267  		fmt.Fprintf(os.Stderr, "# get %s: %v (%.3fs)\n", fetched.Redacted(), res.Status, time.Since(start).Seconds())
   268  	}
   269  
   270  	r := &Response{
   271  		URL:        fetched.Redacted(),
   272  		Status:     res.Status,
   273  		StatusCode: res.StatusCode,
   274  		Header:     map[string][]string(res.Header),
   275  		Body:       res.Body,
   276  	}
   277  
   278  	if res.StatusCode != http.StatusOK {
   279  		contentType := res.Header.Get("Content-Type")
   280  		if mediaType, params, _ := mime.ParseMediaType(contentType); mediaType == "text/plain" {
   281  			switch charset := strings.ToLower(params["charset"]); charset {
   282  			case "us-ascii", "utf-8", "":
   283  				// Body claims to be plain text in UTF-8 or a subset thereof.
   284  				// Try to extract a useful error message from it.
   285  				r.errorDetail.r = res.Body
   286  				r.Body = &r.errorDetail
   287  			}
   288  		}
   289  	}
   290  
   291  	return r, nil
   292  }
   293  
   294  func getFile(u *urlpkg.URL) (*Response, error) {
   295  	path, err := urlToFilePath(u)
   296  	if err != nil {
   297  		return nil, err
   298  	}
   299  	f, err := os.Open(path)
   300  
   301  	if os.IsNotExist(err) {
   302  		return &Response{
   303  			URL:        u.Redacted(),
   304  			Status:     http.StatusText(http.StatusNotFound),
   305  			StatusCode: http.StatusNotFound,
   306  			Body:       http.NoBody,
   307  			fileErr:    err,
   308  		}, nil
   309  	}
   310  
   311  	if os.IsPermission(err) {
   312  		return &Response{
   313  			URL:        u.Redacted(),
   314  			Status:     http.StatusText(http.StatusForbidden),
   315  			StatusCode: http.StatusForbidden,
   316  			Body:       http.NoBody,
   317  			fileErr:    err,
   318  		}, nil
   319  	}
   320  
   321  	if err != nil {
   322  		return nil, err
   323  	}
   324  
   325  	return &Response{
   326  		URL:        u.Redacted(),
   327  		Status:     http.StatusText(http.StatusOK),
   328  		StatusCode: http.StatusOK,
   329  		Body:       f,
   330  	}, nil
   331  }
   332  
   333  func openBrowser(url string) bool { return browser.Open(url) }
   334  
   335  func isLocalHost(u *urlpkg.URL) bool {
   336  	// VCSTestRepoURL itself is secure, and it may redirect requests to other
   337  	// ports (such as a port serving the "svn" protocol) which should also be
   338  	// considered secure.
   339  	host, _, err := net.SplitHostPort(u.Host)
   340  	if err != nil {
   341  		host = u.Host
   342  	}
   343  	if host == "localhost" {
   344  		return true
   345  	}
   346  	if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() {
   347  		return true
   348  	}
   349  	return false
   350  }
   351  
   352  type hookCloser struct {
   353  	io.ReadCloser
   354  	afterClose func()
   355  }
   356  
   357  func (c hookCloser) Close() error {
   358  	err := c.ReadCloser.Close()
   359  	c.afterClose()
   360  	return err
   361  }
   362
View as plain text