Source file src/net/http/client.go
1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // HTTP client. See RFC 7230 through 7235. 6 // 7 // This is the high-level Client interface. 8 // The low-level implementation is in transport.go. 9 10 package http 11 12 import ( 13 "context" 14 "crypto/tls" 15 "encoding/base64" 16 "errors" 17 "fmt" 18 "io" 19 "log" 20 "net/http/internal/ascii" 21 "net/url" 22 "reflect" 23 "slices" 24 "strings" 25 "sync" 26 "sync/atomic" 27 "time" 28 29 "golang.org/x/net/http/httpguts" 30 ) 31 32 // A Client is an HTTP client. Its zero value ([DefaultClient]) is a 33 // usable client that uses [DefaultTransport]. 34 // 35 // The [Client.Transport] typically has internal state (cached TCP 36 // connections), so Clients should be reused instead of created as 37 // needed. Clients are safe for concurrent use by multiple goroutines. 38 // 39 // A Client is higher-level than a [RoundTripper] (such as [Transport]) 40 // and additionally handles HTTP details such as cookies and 41 // redirects. 42 // 43 // When following redirects, the Client will forward all headers set on the 44 // initial [Request] except: 45 // 46 // - when forwarding sensitive headers like "Authorization", 47 // "WWW-Authenticate", and "Cookie" to untrusted targets. 48 // These headers will be ignored when following a redirect to a domain 49 // that is not a subdomain match or exact match of the initial domain. 50 // For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" 51 // will forward the sensitive headers, but a redirect to "bar.com" will not. 52 // - when forwarding the "Cookie" header with a non-nil cookie Jar. 53 // Since each redirect may mutate the state of the cookie jar, 54 // a redirect may possibly alter a cookie set in the initial request. 55 // When forwarding the "Cookie" header, any mutated cookies will be omitted, 56 // with the expectation that the Jar will insert those mutated cookies 57 // with the updated values (assuming the origin matches). 58 // If Jar is nil, the initial cookies are forwarded without change. 59 type Client struct { 60 // Transport specifies the mechanism by which individual 61 // HTTP requests are made. 62 // If nil, DefaultTransport is used. 63 Transport RoundTripper 64 65 // CheckRedirect specifies the policy for handling redirects. 66 // If CheckRedirect is not nil, the client calls it before 67 // following an HTTP redirect. The arguments req and via are 68 // the upcoming request and the requests made already, oldest 69 // first. If CheckRedirect returns an error, the Client's Get 70 // method returns both the previous Response (with its Body 71 // closed) and CheckRedirect's error (wrapped in a url.Error) 72 // instead of issuing the Request req. 73 // As a special case, if CheckRedirect returns ErrUseLastResponse, 74 // then the most recent response is returned with its body 75 // unclosed, along with a nil error. 76 // 77 // If CheckRedirect is nil, the Client uses its default policy, 78 // which is to stop after 10 consecutive requests. 79 CheckRedirect func(req *Request, via []*Request) error 80 81 // Jar specifies the cookie jar. 82 // 83 // The Jar is used to insert relevant cookies into every 84 // outbound Request and is updated with the cookie values 85 // of every inbound Response. The Jar is consulted for every 86 // redirect that the Client follows. 87 // 88 // If Jar is nil, cookies are only sent if they are explicitly 89 // set on the Request. 90 Jar CookieJar 91 92 // Timeout specifies a time limit for requests made by this 93 // Client. The timeout includes connection time, any 94 // redirects, and reading the response body. The timer remains 95 // running after Get, Head, Post, or Do return and will 96 // interrupt reading of the Response.Body. 97 // 98 // A Timeout of zero means no timeout. 99 // 100 // The Client cancels requests to the underlying Transport 101 // as if the Request's Context ended. 102 // 103 // For compatibility, the Client will also use the deprecated 104 // CancelRequest method on Transport if found. New 105 // RoundTripper implementations should use the Request's Context 106 // for cancellation instead of implementing CancelRequest. 107 Timeout time.Duration 108 } 109 110 // DefaultClient is the default [Client] and is used by [Get], [Head], and [Post]. 111 var DefaultClient = &Client{} 112 113 // RoundTripper is an interface representing the ability to execute a 114 // single HTTP transaction, obtaining the [Response] for a given [Request]. 115 // 116 // A RoundTripper must be safe for concurrent use by multiple 117 // goroutines. 118 type RoundTripper interface { 119 // RoundTrip executes a single HTTP transaction, returning 120 // a Response for the provided Request. 121 // 122 // RoundTrip should not attempt to interpret the response. In 123 // particular, RoundTrip must return err == nil if it obtained 124 // a response, regardless of the response's HTTP status code. 125 // A non-nil err should be reserved for failure to obtain a 126 // response. Similarly, RoundTrip should not attempt to 127 // handle higher-level protocol details such as redirects, 128 // authentication, or cookies. 129 // 130 // RoundTrip should not modify the request, except for 131 // consuming and closing the Request's Body. RoundTrip may 132 // read fields of the request in a separate goroutine. Callers 133 // should not mutate or reuse the request until the Response's 134 // Body has been closed. 135 // 136 // RoundTrip must always close the body, including on errors, 137 // but depending on the implementation may do so in a separate 138 // goroutine even after RoundTrip returns. This means that 139 // callers wanting to reuse the body for subsequent requests 140 // must arrange to wait for the Close call before doing so. 141 // 142 // The Request's URL and Header fields must be initialized. 143 RoundTrip(*Request) (*Response, error) 144 } 145 146 // refererForURL returns a referer without any authentication info or 147 // an empty string if lastReq scheme is https and newReq scheme is http. 148 // If the referer was explicitly set, then it will continue to be used. 149 func refererForURL(lastReq, newReq *url.URL, explicitRef string) string { 150 // https://tools.ietf.org/html/rfc7231#section-5.5.2 151 // "Clients SHOULD NOT include a Referer header field in a 152 // (non-secure) HTTP request if the referring page was 153 // transferred with a secure protocol." 154 if lastReq.Scheme == "https" && newReq.Scheme == "http" { 155 return "" 156 } 157 if explicitRef != "" { 158 return explicitRef 159 } 160 161 referer := lastReq.String() 162 if lastReq.User != nil { 163 // This is not very efficient, but is the best we can 164 // do without: 165 // - introducing a new method on URL 166 // - creating a race condition 167 // - copying the URL struct manually, which would cause 168 // maintenance problems down the line 169 auth := lastReq.User.String() + "@" 170 referer = strings.Replace(referer, auth, "", 1) 171 } 172 return referer 173 } 174 175 // didTimeout is non-nil only if err != nil. 176 func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { 177 cookieURL := req.URL 178 if req.Host != "" { 179 cookieURL = cloneURL(cookieURL) 180 cookieURL.Host = req.Host 181 } 182 if c.Jar != nil { 183 for _, cookie := range c.Jar.Cookies(cookieURL) { 184 req.AddCookie(cookie) 185 } 186 } 187 resp, didTimeout, err = send(req, c.transport(), deadline) 188 if err != nil { 189 return nil, didTimeout, err 190 } 191 if c.Jar != nil { 192 if rc := resp.Cookies(); len(rc) > 0 { 193 c.Jar.SetCookies(cookieURL, rc) 194 } 195 } 196 return resp, nil, nil 197 } 198 199 func (c *Client) deadline() time.Time { 200 if c.Timeout > 0 { 201 return time.Now().Add(c.Timeout) 202 } 203 return time.Time{} 204 } 205 206 func (c *Client) transport() RoundTripper { 207 if c.Transport != nil { 208 return c.Transport 209 } 210 return DefaultTransport 211 } 212 213 // ErrSchemeMismatch is returned when a server returns an HTTP response to an HTTPS client. 214 var ErrSchemeMismatch = errors.New("http: server gave HTTP response to HTTPS client") 215 216 // send issues an HTTP request. 217 // Caller should close resp.Body when done reading from it. 218 func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { 219 req := ireq // req is either the original request, or a modified fork 220 221 if rt == nil { 222 req.closeBody() 223 return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport") 224 } 225 226 if req.URL == nil { 227 req.closeBody() 228 return nil, alwaysFalse, errors.New("http: nil Request.URL") 229 } 230 231 if req.RequestURI != "" { 232 req.closeBody() 233 return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests") 234 } 235 236 // forkReq forks req into a shallow clone of ireq the first 237 // time it's called. 238 forkReq := func() { 239 if ireq == req { 240 req = new(Request) 241 *req = *ireq // shallow clone 242 } 243 } 244 245 // Most of the callers of send (Get, Post, et al) don't need 246 // Headers, leaving it uninitialized. We guarantee to the 247 // Transport that this has been initialized, though. 248 if req.Header == nil { 249 forkReq() 250 req.Header = make(Header) 251 } 252 253 if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" { 254 username := u.Username() 255 password, _ := u.Password() 256 forkReq() 257 req.Header = cloneOrMakeHeader(ireq.Header) 258 req.Header.Set("Authorization", "Basic "+basicAuth(username, password)) 259 } 260 261 if !deadline.IsZero() { 262 forkReq() 263 } 264 stopTimer, didTimeout := setRequestCancel(req, rt, deadline) 265 266 resp, err = rt.RoundTrip(req) 267 if err != nil { 268 stopTimer() 269 if resp != nil { 270 log.Printf("RoundTripper returned a response & error; ignoring response") 271 } 272 if tlsErr, ok := err.(tls.RecordHeaderError); ok { 273 // If we get a bad TLS record header, check to see if the 274 // response looks like HTTP and give a more helpful error. 275 // See golang.org/issue/11111. 276 if string(tlsErr.RecordHeader[:]) == "HTTP/" { 277 err = ErrSchemeMismatch 278 } 279 } 280 return nil, didTimeout, err 281 } 282 if resp == nil { 283 return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a nil *Response with a nil error", rt) 284 } 285 if resp.Body == nil { 286 // The documentation on the Body field says “The http Client and Transport 287 // guarantee that Body is always non-nil, even on responses without a body 288 // or responses with a zero-length body.” Unfortunately, we didn't document 289 // that same constraint for arbitrary RoundTripper implementations, and 290 // RoundTripper implementations in the wild (mostly in tests) assume that 291 // they can use a nil Body to mean an empty one (similar to Request.Body). 292 // (See https://golang.org/issue/38095.) 293 // 294 // If the ContentLength allows the Body to be empty, fill in an empty one 295 // here to ensure that it is non-nil. 296 if resp.ContentLength > 0 && req.Method != "HEAD" { 297 return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a *Response with content length %d but a nil Body", rt, resp.ContentLength) 298 } 299 resp.Body = io.NopCloser(strings.NewReader("")) 300 } 301 if !deadline.IsZero() { 302 resp.Body = &cancelTimerBody{ 303 stop: stopTimer, 304 rc: resp.Body, 305 reqDidTimeout: didTimeout, 306 } 307 } 308 return resp, nil, nil 309 } 310 311 // timeBeforeContextDeadline reports whether the non-zero Time t is 312 // before ctx's deadline, if any. If ctx does not have a deadline, it 313 // always reports true (the deadline is considered infinite). 314 func timeBeforeContextDeadline(t time.Time, ctx context.Context) bool { 315 d, ok := ctx.Deadline() 316 if !ok { 317 return true 318 } 319 return t.Before(d) 320 } 321 322 // knownRoundTripperImpl reports whether rt is a RoundTripper that's 323 // maintained by the Go team and known to implement the latest 324 // optional semantics (notably contexts). The Request is used 325 // to check whether this particular request is using an alternate protocol, 326 // in which case we need to check the RoundTripper for that protocol. 327 func knownRoundTripperImpl(rt RoundTripper, req *Request) bool { 328 switch t := rt.(type) { 329 case *Transport: 330 if altRT := t.alternateRoundTripper(req); altRT != nil { 331 return knownRoundTripperImpl(altRT, req) 332 } 333 return true 334 case http2RoundTripper: 335 return true 336 } 337 // There's a very minor chance of a false positive with this. 338 // Instead of detecting our golang.org/x/net/http2.Transport, 339 // it might detect a Transport type in a different http2 340 // package. But I know of none, and the only problem would be 341 // some temporarily leaked goroutines if the transport didn't 342 // support contexts. So this is a good enough heuristic: 343 if reflect.TypeOf(rt).String() == "*http2.Transport" { 344 return true 345 } 346 return false 347 } 348 349 // setRequestCancel sets req.Cancel and adds a deadline context to req 350 // if deadline is non-zero. The RoundTripper's type is used to 351 // determine whether the legacy CancelRequest behavior should be used. 352 // 353 // As background, there are three ways to cancel a request: 354 // First was Transport.CancelRequest. (deprecated) 355 // Second was Request.Cancel. 356 // Third was Request.Context. 357 // This function populates the second and third, and uses the first if it really needs to. 358 func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) { 359 if deadline.IsZero() { 360 return nop, alwaysFalse 361 } 362 knownTransport := knownRoundTripperImpl(rt, req) 363 oldCtx := req.Context() 364 365 if req.Cancel == nil && knownTransport { 366 // If they already had a Request.Context that's 367 // expiring sooner, do nothing: 368 if !timeBeforeContextDeadline(deadline, oldCtx) { 369 return nop, alwaysFalse 370 } 371 372 var cancelCtx func() 373 req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) 374 return cancelCtx, func() bool { return time.Now().After(deadline) } 375 } 376 initialReqCancel := req.Cancel // the user's original Request.Cancel, if any 377 378 var cancelCtx func() 379 if timeBeforeContextDeadline(deadline, oldCtx) { 380 req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) 381 } 382 383 cancel := make(chan struct{}) 384 req.Cancel = cancel 385 386 doCancel := func() { 387 // The second way in the func comment above: 388 close(cancel) 389 // The first way, used only for RoundTripper 390 // implementations written before Go 1.5 or Go 1.6. 391 type canceler interface{ CancelRequest(*Request) } 392 if v, ok := rt.(canceler); ok { 393 v.CancelRequest(req) 394 } 395 } 396 397 stopTimerCh := make(chan struct{}) 398 stopTimer = sync.OnceFunc(func() { 399 close(stopTimerCh) 400 if cancelCtx != nil { 401 cancelCtx() 402 } 403 }) 404 405 timer := time.NewTimer(time.Until(deadline)) 406 var timedOut atomic.Bool 407 408 go func() { 409 select { 410 case <-initialReqCancel: 411 doCancel() 412 timer.Stop() 413 case <-timer.C: 414 timedOut.Store(true) 415 doCancel() 416 case <-stopTimerCh: 417 timer.Stop() 418 } 419 }() 420 421 return stopTimer, timedOut.Load 422 } 423 424 // See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt 425 // "To receive authorization, the client sends the userid and password, 426 // separated by a single colon (":") character, within a base64 427 // encoded string in the credentials." 428 // It is not meant to be urlencoded. 429 func basicAuth(username, password string) string { 430 auth := username + ":" + password 431 return base64.StdEncoding.EncodeToString([]byte(auth)) 432 } 433 434 // Get issues a GET to the specified URL. If the response is one of 435 // the following redirect codes, Get follows the redirect, up to a 436 // maximum of 10 redirects: 437 // 438 // 301 (Moved Permanently) 439 // 302 (Found) 440 // 303 (See Other) 441 // 307 (Temporary Redirect) 442 // 308 (Permanent Redirect) 443 // 444 // An error is returned if there were too many redirects or if there 445 // was an HTTP protocol error. A non-2xx response doesn't cause an 446 // error. Any returned error will be of type [*url.Error]. The url.Error 447 // value's Timeout method will report true if the request timed out. 448 // 449 // When err is nil, resp always contains a non-nil resp.Body. 450 // Caller should close resp.Body when done reading from it. 451 // 452 // Get is a wrapper around DefaultClient.Get. 453 // 454 // To make a request with custom headers, use [NewRequest] and 455 // DefaultClient.Do. 456 // 457 // To make a request with a specified context.Context, use [NewRequestWithContext] 458 // and DefaultClient.Do. 459 func Get(url string) (resp *Response, err error) { 460 return DefaultClient.Get(url) 461 } 462 463 // Get issues a GET to the specified URL. If the response is one of the 464 // following redirect codes, Get follows the redirect after calling the 465 // [Client.CheckRedirect] function: 466 // 467 // 301 (Moved Permanently) 468 // 302 (Found) 469 // 303 (See Other) 470 // 307 (Temporary Redirect) 471 // 308 (Permanent Redirect) 472 // 473 // An error is returned if the [Client.CheckRedirect] function fails 474 // or if there was an HTTP protocol error. A non-2xx response doesn't 475 // cause an error. Any returned error will be of type [*url.Error]. The 476 // url.Error value's Timeout method will report true if the request 477 // timed out. 478 // 479 // When err is nil, resp always contains a non-nil resp.Body. 480 // Caller should close resp.Body when done reading from it. 481 // 482 // To make a request with custom headers, use [NewRequest] and [Client.Do]. 483 // 484 // To make a request with a specified context.Context, use [NewRequestWithContext] 485 // and Client.Do. 486 func (c *Client) Get(url string) (resp *Response, err error) { 487 req, err := NewRequest("GET", url, nil) 488 if err != nil { 489 return nil, err 490 } 491 return c.Do(req) 492 } 493 494 func alwaysFalse() bool { return false } 495 496 // ErrUseLastResponse can be returned by Client.CheckRedirect hooks to 497 // control how redirects are processed. If returned, the next request 498 // is not sent and the most recent response is returned with its body 499 // unclosed. 500 var ErrUseLastResponse = errors.New("net/http: use last response") 501 502 // checkRedirect calls either the user's configured CheckRedirect 503 // function, or the default. 504 func (c *Client) checkRedirect(req *Request, via []*Request) error { 505 fn := c.CheckRedirect 506 if fn == nil { 507 fn = defaultCheckRedirect 508 } 509 return fn(req, via) 510 } 511 512 // redirectBehavior describes what should happen when the 513 // client encounters a 3xx status code from the server. 514 func redirectBehavior(reqMethod string, resp *Response, ireq *Request) (redirectMethod string, shouldRedirect, includeBody bool) { 515 switch resp.StatusCode { 516 case 301, 302, 303: 517 redirectMethod = reqMethod 518 shouldRedirect = true 519 includeBody = false 520 521 // RFC 10008, Section 2.5: QUERY is safe and idempotent, so 301 522 // and 302 preserve the method and re-send the body (as 307 and 523 // 308 do); only 303 changes the method to GET. 524 if reqMethod == "QUERY" && resp.StatusCode != 303 { 525 includeBody = true 526 if ireq.GetBody == nil && ireq.outgoingLength() != 0 { 527 // We had a request body, and 301/302 require 528 // re-sending it, but GetBody is not defined. 529 shouldRedirect = false 530 } 531 } else if reqMethod != "GET" && reqMethod != "HEAD" { 532 // RFC 2616 allowed automatic redirection only with GET and 533 // HEAD requests. RFC 7231 lifts this restriction, but we still 534 // restrict other methods to GET to maintain compatibility. 535 // See Issue 18570. 536 redirectMethod = "GET" 537 } 538 case 307, 308: 539 redirectMethod = reqMethod 540 shouldRedirect = true 541 includeBody = true 542 543 if ireq.GetBody == nil && ireq.outgoingLength() != 0 { 544 // We had a request body, and 307/308 require 545 // re-sending it, but GetBody is not defined. So just 546 // return this response to the user instead of an 547 // error, like we did in Go 1.7 and earlier. 548 shouldRedirect = false 549 } 550 } 551 return redirectMethod, shouldRedirect, includeBody 552 } 553 554 // urlErrorOp returns the (*url.Error).Op value to use for the 555 // provided (*Request).Method value. 556 func urlErrorOp(method string) string { 557 if method == "" { 558 return "Get" 559 } 560 if lowerMethod, ok := ascii.ToLower(method); ok { 561 return method[:1] + lowerMethod[1:] 562 } 563 return method 564 } 565 566 // Do sends an HTTP request and returns an HTTP response, following 567 // policy (such as redirects, cookies, auth) as configured on the 568 // client. 569 // 570 // An error is returned if caused by client policy (such as 571 // CheckRedirect), or failure to speak HTTP (such as a network 572 // connectivity problem). A non-2xx status code doesn't cause an 573 // error. 574 // 575 // If the returned error is nil, the [Response] will contain a non-nil 576 // Body which the user is expected to close. If the Body is not both 577 // read to EOF and closed, the [Client]'s underlying [RoundTripper] 578 // (typically [Transport]) may not be able to re-use a persistent TCP 579 // connection to the server for a subsequent "keep-alive" request. 580 // Note, however, that [Transport] will automatically try to read a 581 // [Response] Body to EOF asynchronously up to a conservative limit 582 // when a Body is closed. 583 // 584 // The request Body, if non-nil, will be closed by the underlying 585 // Transport, even on errors. The Body may be closed asynchronously after 586 // Do returns. 587 // 588 // On error, any Response can be ignored. A non-nil Response with a 589 // non-nil error only occurs when CheckRedirect fails, and even then 590 // the returned [Response.Body] is already closed. 591 // 592 // Generally [Get], [Post], or [PostForm] will be used instead of Do. 593 // 594 // If the server replies with a redirect, the Client first uses the 595 // CheckRedirect function to determine whether the redirect should be 596 // followed. If permitted, a 301, 302, or 303 redirect causes 597 // subsequent requests to use HTTP method GET 598 // (or HEAD if the original request was HEAD), with no body. 599 // A 307 or 308 redirect preserves the original HTTP method and body, 600 // provided that the [Request.GetBody] function is defined. 601 // The [NewRequest] function automatically sets GetBody for common 602 // standard library body types. 603 // 604 // Note that the [Client] redirect behavior does not follow the WHATWG 605 // Fetch standard. This is because it was written before established 606 // standards existed. As such, by modern standards, [Client] has a 607 // rather permissive behavior. For example, sensitive headers are 608 // retained on redirect to a subdomain or to a different scheme on the 609 // same host. 610 // 611 // Any returned error will be of type [*url.Error]. The url.Error 612 // value's Timeout method will report true if the request timed out. 613 func (c *Client) Do(req *Request) (*Response, error) { 614 return c.do(req) 615 } 616 617 var testHookClientDoResult func(retres *Response, reterr error) 618 619 func (c *Client) do(req *Request) (retres *Response, reterr error) { 620 if testHookClientDoResult != nil { 621 defer func() { testHookClientDoResult(retres, reterr) }() 622 } 623 if req.URL == nil { 624 req.closeBody() 625 return nil, &url.Error{ 626 Op: urlErrorOp(req.Method), 627 Err: errors.New("http: nil Request.URL"), 628 } 629 } 630 _ = *c // panic early if c is nil; see go.dev/issue/53521 631 632 var ( 633 deadline = c.deadline() 634 reqs []*Request 635 resp *Response 636 copyHeaders = c.makeHeadersCopier(req) 637 reqBodyClosed = false // have we closed the current req.Body? 638 639 // Redirect behavior: 640 redirectMethod string 641 includeBody = true 642 stripSensitiveHeaders = false 643 ) 644 uerr := func(err error) error { 645 // the body may have been closed already by c.send() 646 if !reqBodyClosed { 647 req.closeBody() 648 } 649 var urlStr string 650 if resp != nil && resp.Request != nil { 651 urlStr = stripPassword(resp.Request.URL) 652 } else { 653 urlStr = stripPassword(req.URL) 654 } 655 return &url.Error{ 656 Op: urlErrorOp(reqs[0].Method), 657 URL: urlStr, 658 Err: err, 659 } 660 } 661 for { 662 // For all but the first request, create the next 663 // request hop and replace req. 664 if len(reqs) > 0 { 665 loc := resp.Header.Get("Location") 666 if loc == "" { 667 // While most 3xx responses include a Location, it is not 668 // required and 3xx responses without a Location have been 669 // observed in the wild. See issues #17773 and #49281. 670 return resp, nil 671 } 672 u, err := req.URL.Parse(loc) 673 if err != nil { 674 resp.closeBody() 675 return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err)) 676 } 677 host := "" 678 if req.Host != "" && req.Host != req.URL.Host { 679 // If the caller specified a custom Host header and the 680 // redirect location is relative, preserve the Host header 681 // through the redirect. See issue #22233. 682 if u, _ := url.Parse(loc); u != nil && !u.IsAbs() { 683 host = req.Host 684 } 685 } 686 ireq := reqs[0] 687 req = &Request{ 688 Method: redirectMethod, 689 Response: resp, 690 URL: u, 691 Header: make(Header), 692 Host: host, 693 Cancel: ireq.Cancel, 694 ctx: ireq.ctx, 695 } 696 if includeBody && ireq.GetBody != nil { 697 req.Body, err = ireq.GetBody() 698 if err != nil { 699 resp.closeBody() 700 return nil, uerr(err) 701 } 702 req.GetBody = ireq.GetBody 703 req.ContentLength = ireq.ContentLength 704 } 705 706 // Copy original headers before setting the Referer, 707 // in case the user set Referer on their first request. 708 // If they really want to override, they can do it in 709 // their CheckRedirect func. 710 if !stripSensitiveHeaders && reqs[0].URL.Host != req.URL.Host { 711 if !shouldCopyHeaderOnRedirect(reqs[0].URL, req.URL) { 712 stripSensitiveHeaders = true 713 } 714 } 715 copyHeaders(req, stripSensitiveHeaders, !includeBody) 716 // Add the Referer header from the most recent 717 // request URL to the new one, if it's not https->http: 718 if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL, req.Header.Get("Referer")); ref != "" { 719 req.Header.Set("Referer", ref) 720 } 721 err = c.checkRedirect(req, reqs) 722 723 // Sentinel error to let users select the 724 // previous response, without closing its 725 // body. See Issue 10069. 726 if err == ErrUseLastResponse { 727 return resp, nil 728 } 729 730 // Close the previous response's body. But 731 // read at least some of the body so if it's 732 // small the underlying TCP connection will be 733 // re-used. No need to check for errors: if it 734 // fails, the Transport won't reuse it anyway. 735 const maxBodySlurpSize = 2 << 10 736 if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize { 737 io.CopyN(io.Discard, resp.Body, maxBodySlurpSize) 738 } 739 resp.Body.Close() 740 741 if err != nil { 742 // Special case for Go 1 compatibility: return both the response 743 // and an error if the CheckRedirect function failed. 744 // See https://golang.org/issue/3795 745 // The resp.Body has already been closed. 746 ue := uerr(err) 747 ue.(*url.Error).URL = loc 748 return resp, ue 749 } 750 } 751 752 reqs = append(reqs, req) 753 var err error 754 var didTimeout func() bool 755 if resp, didTimeout, err = c.send(req, deadline); err != nil { 756 // c.send() always closes req.Body 757 reqBodyClosed = true 758 if !deadline.IsZero() && didTimeout() { 759 err = &timeoutError{err.Error() + " (Client.Timeout exceeded while awaiting headers)"} 760 } 761 return nil, uerr(err) 762 } 763 764 var shouldRedirect, includeBodyOnHop bool 765 redirectMethod, shouldRedirect, includeBodyOnHop = redirectBehavior(req.Method, resp, reqs[0]) 766 if !shouldRedirect { 767 return resp, nil 768 } 769 if !includeBodyOnHop { 770 // Once a hop drops the body, we never send it again 771 // (because we're now handling a redirect for a request with no body). 772 includeBody = false 773 } 774 775 req.closeBody() 776 } 777 } 778 779 // makeHeadersCopier makes a function that copies headers from the 780 // initial Request, ireq. For every redirect, this function must be called 781 // so that it can copy headers into the upcoming Request. 782 func (c *Client) makeHeadersCopier(ireq *Request) func(req *Request, stripSensitiveHeaders, stripBodyHeaders bool) { 783 // The headers to copy are from the very initial request. 784 // We use a closured callback to keep a reference to these original headers. 785 var ( 786 ireqhdr = cloneOrMakeHeader(ireq.Header) 787 icookies map[string][]*Cookie 788 ) 789 if c.Jar != nil && ireq.Header.Get("Cookie") != "" { 790 icookies = make(map[string][]*Cookie) 791 for _, c := range ireq.Cookies() { 792 icookies[c.Name] = append(icookies[c.Name], c) 793 } 794 } 795 796 return func(req *Request, stripSensitiveHeaders, stripBodyHeaders bool) { 797 // If Jar is present and there was some initial cookies provided 798 // via the request header, then we may need to alter the initial 799 // cookies as we follow redirects since each redirect may end up 800 // modifying a pre-existing cookie. 801 // 802 // Since cookies already set in the request header do not contain 803 // information about the original domain and path, the logic below 804 // assumes any new set cookies override the original cookie 805 // regardless of domain or path. 806 // 807 // See https://golang.org/issue/17494 808 if c.Jar != nil && icookies != nil { 809 var changed bool 810 resp := req.Response // The response that caused the upcoming redirect 811 for _, c := range resp.Cookies() { 812 if _, ok := icookies[c.Name]; ok { 813 delete(icookies, c.Name) 814 changed = true 815 } 816 } 817 if changed { 818 ireqhdr.Del("Cookie") 819 var ss []string 820 for _, cs := range icookies { 821 for _, c := range cs { 822 ss = append(ss, c.Name+"="+c.Value) 823 } 824 } 825 slices.Sort(ss) // Ensure deterministic headers 826 ireqhdr.Set("Cookie", strings.Join(ss, "; ")) 827 } 828 } 829 830 // Copy the initial request's Header values 831 // (at least the safe ones). 832 for k, vv := range ireqhdr { 833 sensitive := false 834 body := false 835 switch CanonicalHeaderKey(k) { 836 case "Authorization", "Www-Authenticate", "Cookie", "Cookie2", 837 "Proxy-Authorization", "Proxy-Authenticate": 838 sensitive = true 839 840 case "Content-Encoding", "Content-Language", "Content-Location", 841 "Content-Type": 842 // Headers relating to the body which is removed for 843 // POST to GET redirects 844 // https://fetch.spec.whatwg.org/#http-redirect-fetch 845 body = true 846 847 } 848 if !(sensitive && stripSensitiveHeaders) && !(body && stripBodyHeaders) { 849 req.Header[k] = vv 850 } 851 } 852 } 853 } 854 855 func defaultCheckRedirect(req *Request, via []*Request) error { 856 if len(via) >= 10 { 857 return errors.New("stopped after 10 redirects") 858 } 859 return nil 860 } 861 862 // Post issues a POST to the specified URL. 863 // 864 // Caller should close resp.Body when done reading from it. 865 // 866 // If the provided body is an [io.Closer], it is closed after the 867 // request. 868 // 869 // Post is a wrapper around DefaultClient.Post. 870 // 871 // To set custom headers, use [NewRequest] and DefaultClient.Do. 872 // 873 // See the [Client.Do] method documentation for details on how redirects 874 // are handled. 875 // 876 // To make a request with a specified context.Context, use [NewRequestWithContext] 877 // and DefaultClient.Do. 878 func Post(url, contentType string, body io.Reader) (resp *Response, err error) { 879 return DefaultClient.Post(url, contentType, body) 880 } 881 882 // Post issues a POST to the specified URL. 883 // 884 // Caller should close resp.Body when done reading from it. 885 // 886 // If the provided body is an [io.Closer], it is closed after the 887 // request. 888 // 889 // To set custom headers, use [NewRequest] and [Client.Do]. 890 // 891 // To make a request with a specified context.Context, use [NewRequestWithContext] 892 // and [Client.Do]. 893 // 894 // See the [Client.Do] method documentation for details on how redirects 895 // are handled. 896 func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) { 897 req, err := NewRequest("POST", url, body) 898 if err != nil { 899 return nil, err 900 } 901 req.Header.Set("Content-Type", contentType) 902 return c.Do(req) 903 } 904 905 // PostForm issues a POST to the specified URL, with data's keys and 906 // values URL-encoded as the request body. 907 // 908 // The Content-Type header is set to application/x-www-form-urlencoded. 909 // To set other headers, use [NewRequest] and DefaultClient.Do. 910 // 911 // When err is nil, resp always contains a non-nil resp.Body. 912 // Caller should close resp.Body when done reading from it. 913 // 914 // PostForm is a wrapper around DefaultClient.PostForm. 915 // 916 // See the [Client.Do] method documentation for details on how redirects 917 // are handled. 918 // 919 // To make a request with a specified [context.Context], use [NewRequestWithContext] 920 // and DefaultClient.Do. 921 func PostForm(url string, data url.Values) (resp *Response, err error) { 922 return DefaultClient.PostForm(url, data) 923 } 924 925 // PostForm issues a POST to the specified URL, 926 // with data's keys and values URL-encoded as the request body. 927 // 928 // The Content-Type header is set to application/x-www-form-urlencoded. 929 // To set other headers, use [NewRequest] and [Client.Do]. 930 // 931 // When err is nil, resp always contains a non-nil resp.Body. 932 // Caller should close resp.Body when done reading from it. 933 // 934 // See the [Client.Do] method documentation for details on how redirects 935 // are handled. 936 // 937 // To make a request with a specified context.Context, use [NewRequestWithContext] 938 // and Client.Do. 939 func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) { 940 return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) 941 } 942 943 // Head issues a HEAD to the specified URL. If the response is one of 944 // the following redirect codes, Head follows the redirect, up to a 945 // maximum of 10 redirects: 946 // 947 // 301 (Moved Permanently) 948 // 302 (Found) 949 // 303 (See Other) 950 // 307 (Temporary Redirect) 951 // 308 (Permanent Redirect) 952 // 953 // Head is a wrapper around DefaultClient.Head. 954 // 955 // To make a request with a specified [context.Context], use [NewRequestWithContext] 956 // and DefaultClient.Do. 957 func Head(url string) (resp *Response, err error) { 958 return DefaultClient.Head(url) 959 } 960 961 // Head issues a HEAD to the specified URL. If the response is one of the 962 // following redirect codes, Head follows the redirect after calling the 963 // [Client.CheckRedirect] function: 964 // 965 // 301 (Moved Permanently) 966 // 302 (Found) 967 // 303 (See Other) 968 // 307 (Temporary Redirect) 969 // 308 (Permanent Redirect) 970 // 971 // To make a request with a specified [context.Context], use [NewRequestWithContext] 972 // and [Client.Do]. 973 func (c *Client) Head(url string) (resp *Response, err error) { 974 req, err := NewRequest("HEAD", url, nil) 975 if err != nil { 976 return nil, err 977 } 978 return c.Do(req) 979 } 980 981 // CloseIdleConnections closes any connections on its [Transport] which 982 // were previously connected from previous requests but are now 983 // sitting idle in a "keep-alive" state. It does not interrupt any 984 // connections currently in use. 985 // 986 // If [Client.Transport] does not have a [Client.CloseIdleConnections] method 987 // then this method does nothing. 988 func (c *Client) CloseIdleConnections() { 989 type closeIdler interface { 990 CloseIdleConnections() 991 } 992 if tr, ok := c.transport().(closeIdler); ok { 993 tr.CloseIdleConnections() 994 } 995 } 996 997 // cancelTimerBody is an io.ReadCloser that wraps rc with two features: 998 // 1. On Read error or close, the stop func is called. 999 // 2. On Read failure, if reqDidTimeout is true, the error is wrapped and 1000 // marked as net.Error that hit its timeout. 1001 type cancelTimerBody struct { 1002 stop func() // stops the time.Timer waiting to cancel the request 1003 rc io.ReadCloser 1004 reqDidTimeout func() bool 1005 } 1006 1007 func (b *cancelTimerBody) Read(p []byte) (n int, err error) { 1008 n, err = b.rc.Read(p) 1009 if err == nil { 1010 return n, nil 1011 } 1012 if err == io.EOF { 1013 return n, err 1014 } 1015 if b.reqDidTimeout() { 1016 err = &timeoutError{err.Error() + " (Client.Timeout or context cancellation while reading body)"} 1017 } 1018 return n, err 1019 } 1020 1021 func (b *cancelTimerBody) Close() error { 1022 err := b.rc.Close() 1023 b.stop() 1024 return err 1025 } 1026 1027 func shouldCopyHeaderOnRedirect(initial, dest *url.URL) bool { 1028 // Permit sending auth/cookie headers from "foo.com" 1029 // to "sub.foo.com". 1030 1031 // Note that we don't send all cookies to subdomains 1032 // automatically. This function is only used for 1033 // Cookies set explicitly on the initial outgoing 1034 // client request. Cookies automatically added via the 1035 // CookieJar mechanism continue to follow each 1036 // cookie's scope as set by Set-Cookie. But for 1037 // outgoing requests with the Cookie header set 1038 // directly, we don't know their scope, so we assume 1039 // it's for *.domain.com. 1040 1041 ihost, err1 := httpguts.PunycodeHostPort(initial.Hostname()) 1042 dhost, err2 := httpguts.PunycodeHostPort(dest.Hostname()) 1043 if err1 != nil || err2 != nil { 1044 return false 1045 } 1046 ihost, ok1 := ascii.ToLower(ihost) 1047 dhost, ok2 := ascii.ToLower(dhost) 1048 return ok1 && ok2 && isDomainOrSubdomain(dhost, ihost) 1049 } 1050 1051 // isDomainOrSubdomain reports whether sub is a subdomain (or exact 1052 // match) of the parent domain. 1053 // 1054 // Both domains must already be in canonical form. 1055 func isDomainOrSubdomain(sub, parent string) bool { 1056 if sub == parent { 1057 return true 1058 } 1059 // If sub contains a :, it's probably an IPv6 address (and is definitely not a hostname). 1060 // Don't check the suffix in this case, to avoid matching the contents of a IPv6 zone. 1061 // For example, "::1%.www.example.com" is not a subdomain of "www.example.com". 1062 if strings.ContainsAny(sub, ":%") { 1063 return false 1064 } 1065 // If sub is "foo.example.com" and parent is "example.com", 1066 // that means sub must end in "."+parent. 1067 // Do it without allocating. 1068 if !strings.HasSuffix(sub, parent) { 1069 return false 1070 } 1071 return sub[len(sub)-len(parent)-1] == '.' 1072 } 1073 1074 func stripPassword(u *url.URL) string { 1075 _, passSet := u.User.Password() 1076 if passSet { 1077 return strings.Replace(u.String(), u.User.String()+"@", u.User.Username()+":***@", 1) 1078 } 1079 return u.String() 1080 } 1081