Source file src/net/http/client.go
1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // HTTP client. See RFC 7230 through 7235. 6 // 7 // This is the high-level Client interface. 8 // The low-level implementation is in transport.go. 9 10 package http 11 12 import ( 13 "context" 14 "crypto/tls" 15 "encoding/base64" 16 "errors" 17 "fmt" 18 "io" 19 "log" 20 "net/http/internal/ascii" 21 "net/url" 22 "reflect" 23 "slices" 24 "strings" 25 "sync" 26 "sync/atomic" 27 "time" 28 ) 29 30 // A Client is an HTTP client. Its zero value ([DefaultClient]) is a 31 // usable client that uses [DefaultTransport]. 32 // 33 // The [Client.Transport] typically has internal state (cached TCP 34 // connections), so Clients should be reused instead of created as 35 // needed. Clients are safe for concurrent use by multiple goroutines. 36 // 37 // A Client is higher-level than a [RoundTripper] (such as [Transport]) 38 // and additionally handles HTTP details such as cookies and 39 // redirects. 40 // 41 // When following redirects, the Client will forward all headers set on the 42 // initial [Request] except: 43 // 44 // - when forwarding sensitive headers like "Authorization", 45 // "WWW-Authenticate", and "Cookie" to untrusted targets. 46 // These headers will be ignored when following a redirect to a domain 47 // that is not a subdomain match or exact match of the initial domain. 48 // For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" 49 // will forward the sensitive headers, but a redirect to "bar.com" will not. 50 // - when forwarding the "Cookie" header with a non-nil cookie Jar. 51 // Since each redirect may mutate the state of the cookie jar, 52 // a redirect may possibly alter a cookie set in the initial request. 53 // When forwarding the "Cookie" header, any mutated cookies will be omitted, 54 // with the expectation that the Jar will insert those mutated cookies 55 // with the updated values (assuming the origin matches). 56 // If Jar is nil, the initial cookies are forwarded without change. 57 type Client struct { 58 // Transport specifies the mechanism by which individual 59 // HTTP requests are made. 60 // If nil, DefaultTransport is used. 61 Transport RoundTripper 62 63 // CheckRedirect specifies the policy for handling redirects. 64 // If CheckRedirect is not nil, the client calls it before 65 // following an HTTP redirect. The arguments req and via are 66 // the upcoming request and the requests made already, oldest 67 // first. If CheckRedirect returns an error, the Client's Get 68 // method returns both the previous Response (with its Body 69 // closed) and CheckRedirect's error (wrapped in a url.Error) 70 // instead of issuing the Request req. 71 // As a special case, if CheckRedirect returns ErrUseLastResponse, 72 // then the most recent response is returned with its body 73 // unclosed, along with a nil error. 74 // 75 // If CheckRedirect is nil, the Client uses its default policy, 76 // which is to stop after 10 consecutive requests. 77 CheckRedirect func(req *Request, via []*Request) error 78 79 // Jar specifies the cookie jar. 80 // 81 // The Jar is used to insert relevant cookies into every 82 // outbound Request and is updated with the cookie values 83 // of every inbound Response. The Jar is consulted for every 84 // redirect that the Client follows. 85 // 86 // If Jar is nil, cookies are only sent if they are explicitly 87 // set on the Request. 88 Jar CookieJar 89 90 // Timeout specifies a time limit for requests made by this 91 // Client. The timeout includes connection time, any 92 // redirects, and reading the response body. The timer remains 93 // running after Get, Head, Post, or Do return and will 94 // interrupt reading of the Response.Body. 95 // 96 // A Timeout of zero means no timeout. 97 // 98 // The Client cancels requests to the underlying Transport 99 // as if the Request's Context ended. 100 // 101 // For compatibility, the Client will also use the deprecated 102 // CancelRequest method on Transport if found. New 103 // RoundTripper implementations should use the Request's Context 104 // for cancellation instead of implementing CancelRequest. 105 Timeout time.Duration 106 } 107 108 // DefaultClient is the default [Client] and is used by [Get], [Head], and [Post]. 109 var DefaultClient = &Client{} 110 111 // RoundTripper is an interface representing the ability to execute a 112 // single HTTP transaction, obtaining the [Response] for a given [Request]. 113 // 114 // A RoundTripper must be safe for concurrent use by multiple 115 // goroutines. 116 type RoundTripper interface { 117 // RoundTrip executes a single HTTP transaction, returning 118 // a Response for the provided Request. 119 // 120 // RoundTrip should not attempt to interpret the response. In 121 // particular, RoundTrip must return err == nil if it obtained 122 // a response, regardless of the response's HTTP status code. 123 // A non-nil err should be reserved for failure to obtain a 124 // response. Similarly, RoundTrip should not attempt to 125 // handle higher-level protocol details such as redirects, 126 // authentication, or cookies. 127 // 128 // RoundTrip should not modify the request, except for 129 // consuming and closing the Request's Body. RoundTrip may 130 // read fields of the request in a separate goroutine. Callers 131 // should not mutate or reuse the request until the Response's 132 // Body has been closed. 133 // 134 // RoundTrip must always close the body, including on errors, 135 // but depending on the implementation may do so in a separate 136 // goroutine even after RoundTrip returns. This means that 137 // callers wanting to reuse the body for subsequent requests 138 // must arrange to wait for the Close call before doing so. 139 // 140 // The Request's URL and Header fields must be initialized. 141 RoundTrip(*Request) (*Response, error) 142 } 143 144 // refererForURL returns a referer without any authentication info or 145 // an empty string if lastReq scheme is https and newReq scheme is http. 146 // If the referer was explicitly set, then it will continue to be used. 147 func refererForURL(lastReq, newReq *url.URL, explicitRef string) string { 148 // https://tools.ietf.org/html/rfc7231#section-5.5.2 149 // "Clients SHOULD NOT include a Referer header field in a 150 // (non-secure) HTTP request if the referring page was 151 // transferred with a secure protocol." 152 if lastReq.Scheme == "https" && newReq.Scheme == "http" { 153 return "" 154 } 155 if explicitRef != "" { 156 return explicitRef 157 } 158 159 referer := lastReq.String() 160 if lastReq.User != nil { 161 // This is not very efficient, but is the best we can 162 // do without: 163 // - introducing a new method on URL 164 // - creating a race condition 165 // - copying the URL struct manually, which would cause 166 // maintenance problems down the line 167 auth := lastReq.User.String() + "@" 168 referer = strings.Replace(referer, auth, "", 1) 169 } 170 return referer 171 } 172 173 // didTimeout is non-nil only if err != nil. 174 func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { 175 if c.Jar != nil { 176 for _, cookie := range c.Jar.Cookies(req.URL) { 177 req.AddCookie(cookie) 178 } 179 } 180 resp, didTimeout, err = send(req, c.transport(), deadline) 181 if err != nil { 182 return nil, didTimeout, err 183 } 184 if c.Jar != nil { 185 if rc := resp.Cookies(); len(rc) > 0 { 186 c.Jar.SetCookies(req.URL, rc) 187 } 188 } 189 return resp, nil, nil 190 } 191 192 func (c *Client) deadline() time.Time { 193 if c.Timeout > 0 { 194 return time.Now().Add(c.Timeout) 195 } 196 return time.Time{} 197 } 198 199 func (c *Client) transport() RoundTripper { 200 if c.Transport != nil { 201 return c.Transport 202 } 203 return DefaultTransport 204 } 205 206 // ErrSchemeMismatch is returned when a server returns an HTTP response to an HTTPS client. 207 var ErrSchemeMismatch = errors.New("http: server gave HTTP response to HTTPS client") 208 209 // send issues an HTTP request. 210 // Caller should close resp.Body when done reading from it. 211 func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { 212 req := ireq // req is either the original request, or a modified fork 213 214 if rt == nil { 215 req.closeBody() 216 return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport") 217 } 218 219 if req.URL == nil { 220 req.closeBody() 221 return nil, alwaysFalse, errors.New("http: nil Request.URL") 222 } 223 224 if req.RequestURI != "" { 225 req.closeBody() 226 return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests") 227 } 228 229 // forkReq forks req into a shallow clone of ireq the first 230 // time it's called. 231 forkReq := func() { 232 if ireq == req { 233 req = new(Request) 234 *req = *ireq // shallow clone 235 } 236 } 237 238 // Most the callers of send (Get, Post, et al) don't need 239 // Headers, leaving it uninitialized. We guarantee to the 240 // Transport that this has been initialized, though. 241 if req.Header == nil { 242 forkReq() 243 req.Header = make(Header) 244 } 245 246 if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" { 247 username := u.Username() 248 password, _ := u.Password() 249 forkReq() 250 req.Header = cloneOrMakeHeader(ireq.Header) 251 req.Header.Set("Authorization", "Basic "+basicAuth(username, password)) 252 } 253 254 if !deadline.IsZero() { 255 forkReq() 256 } 257 stopTimer, didTimeout := setRequestCancel(req, rt, deadline) 258 259 resp, err = rt.RoundTrip(req) 260 if err != nil { 261 stopTimer() 262 if resp != nil { 263 log.Printf("RoundTripper returned a response & error; ignoring response") 264 } 265 if tlsErr, ok := err.(tls.RecordHeaderError); ok { 266 // If we get a bad TLS record header, check to see if the 267 // response looks like HTTP and give a more helpful error. 268 // See golang.org/issue/11111. 269 if string(tlsErr.RecordHeader[:]) == "HTTP/" { 270 err = ErrSchemeMismatch 271 } 272 } 273 return nil, didTimeout, err 274 } 275 if resp == nil { 276 return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a nil *Response with a nil error", rt) 277 } 278 if resp.Body == nil { 279 // The documentation on the Body field says “The http Client and Transport 280 // guarantee that Body is always non-nil, even on responses without a body 281 // or responses with a zero-length body.” Unfortunately, we didn't document 282 // that same constraint for arbitrary RoundTripper implementations, and 283 // RoundTripper implementations in the wild (mostly in tests) assume that 284 // they can use a nil Body to mean an empty one (similar to Request.Body). 285 // (See https://golang.org/issue/38095.) 286 // 287 // If the ContentLength allows the Body to be empty, fill in an empty one 288 // here to ensure that it is non-nil. 289 if resp.ContentLength > 0 && req.Method != "HEAD" { 290 return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a *Response with content length %d but a nil Body", rt, resp.ContentLength) 291 } 292 resp.Body = io.NopCloser(strings.NewReader("")) 293 } 294 if !deadline.IsZero() { 295 resp.Body = &cancelTimerBody{ 296 stop: stopTimer, 297 rc: resp.Body, 298 reqDidTimeout: didTimeout, 299 } 300 } 301 return resp, nil, nil 302 } 303 304 // timeBeforeContextDeadline reports whether the non-zero Time t is 305 // before ctx's deadline, if any. If ctx does not have a deadline, it 306 // always reports true (the deadline is considered infinite). 307 func timeBeforeContextDeadline(t time.Time, ctx context.Context) bool { 308 d, ok := ctx.Deadline() 309 if !ok { 310 return true 311 } 312 return t.Before(d) 313 } 314 315 // knownRoundTripperImpl reports whether rt is a RoundTripper that's 316 // maintained by the Go team and known to implement the latest 317 // optional semantics (notably contexts). The Request is used 318 // to check whether this particular request is using an alternate protocol, 319 // in which case we need to check the RoundTripper for that protocol. 320 func knownRoundTripperImpl(rt RoundTripper, req *Request) bool { 321 switch t := rt.(type) { 322 case *Transport: 323 if altRT := t.alternateRoundTripper(req); altRT != nil { 324 return knownRoundTripperImpl(altRT, req) 325 } 326 return true 327 case *http2Transport, http2noDialH2RoundTripper: 328 return true 329 } 330 // There's a very minor chance of a false positive with this. 331 // Instead of detecting our golang.org/x/net/http2.Transport, 332 // it might detect a Transport type in a different http2 333 // package. But I know of none, and the only problem would be 334 // some temporarily leaked goroutines if the transport didn't 335 // support contexts. So this is a good enough heuristic: 336 if reflect.TypeOf(rt).String() == "*http2.Transport" { 337 return true 338 } 339 return false 340 } 341 342 // setRequestCancel sets req.Cancel and adds a deadline context to req 343 // if deadline is non-zero. The RoundTripper's type is used to 344 // determine whether the legacy CancelRequest behavior should be used. 345 // 346 // As background, there are three ways to cancel a request: 347 // First was Transport.CancelRequest. (deprecated) 348 // Second was Request.Cancel. 349 // Third was Request.Context. 350 // This function populates the second and third, and uses the first if it really needs to. 351 func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) { 352 if deadline.IsZero() { 353 return nop, alwaysFalse 354 } 355 knownTransport := knownRoundTripperImpl(rt, req) 356 oldCtx := req.Context() 357 358 if req.Cancel == nil && knownTransport { 359 // If they already had a Request.Context that's 360 // expiring sooner, do nothing: 361 if !timeBeforeContextDeadline(deadline, oldCtx) { 362 return nop, alwaysFalse 363 } 364 365 var cancelCtx func() 366 req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) 367 return cancelCtx, func() bool { return time.Now().After(deadline) } 368 } 369 initialReqCancel := req.Cancel // the user's original Request.Cancel, if any 370 371 var cancelCtx func() 372 if timeBeforeContextDeadline(deadline, oldCtx) { 373 req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) 374 } 375 376 cancel := make(chan struct{}) 377 req.Cancel = cancel 378 379 doCancel := func() { 380 // The second way in the func comment above: 381 close(cancel) 382 // The first way, used only for RoundTripper 383 // implementations written before Go 1.5 or Go 1.6. 384 type canceler interface{ CancelRequest(*Request) } 385 if v, ok := rt.(canceler); ok { 386 v.CancelRequest(req) 387 } 388 } 389 390 stopTimerCh := make(chan struct{}) 391 stopTimer = sync.OnceFunc(func() { 392 close(stopTimerCh) 393 if cancelCtx != nil { 394 cancelCtx() 395 } 396 }) 397 398 timer := time.NewTimer(time.Until(deadline)) 399 var timedOut atomic.Bool 400 401 go func() { 402 select { 403 case <-initialReqCancel: 404 doCancel() 405 timer.Stop() 406 case <-timer.C: 407 timedOut.Store(true) 408 doCancel() 409 case <-stopTimerCh: 410 timer.Stop() 411 } 412 }() 413 414 return stopTimer, timedOut.Load 415 } 416 417 // See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt 418 // "To receive authorization, the client sends the userid and password, 419 // separated by a single colon (":") character, within a base64 420 // encoded string in the credentials." 421 // It is not meant to be urlencoded. 422 func basicAuth(username, password string) string { 423 auth := username + ":" + password 424 return base64.StdEncoding.EncodeToString([]byte(auth)) 425 } 426 427 // Get issues a GET to the specified URL. If the response is one of 428 // the following redirect codes, Get follows the redirect, up to a 429 // maximum of 10 redirects: 430 // 431 // 301 (Moved Permanently) 432 // 302 (Found) 433 // 303 (See Other) 434 // 307 (Temporary Redirect) 435 // 308 (Permanent Redirect) 436 // 437 // An error is returned if there were too many redirects or if there 438 // was an HTTP protocol error. A non-2xx response doesn't cause an 439 // error. Any returned error will be of type [*url.Error]. The url.Error 440 // value's Timeout method will report true if the request timed out. 441 // 442 // When err is nil, resp always contains a non-nil resp.Body. 443 // Caller should close resp.Body when done reading from it. 444 // 445 // Get is a wrapper around DefaultClient.Get. 446 // 447 // To make a request with custom headers, use [NewRequest] and 448 // DefaultClient.Do. 449 // 450 // To make a request with a specified context.Context, use [NewRequestWithContext] 451 // and DefaultClient.Do. 452 func Get(url string) (resp *Response, err error) { 453 return DefaultClient.Get(url) 454 } 455 456 // Get issues a GET to the specified URL. If the response is one of the 457 // following redirect codes, Get follows the redirect after calling the 458 // [Client.CheckRedirect] function: 459 // 460 // 301 (Moved Permanently) 461 // 302 (Found) 462 // 303 (See Other) 463 // 307 (Temporary Redirect) 464 // 308 (Permanent Redirect) 465 // 466 // An error is returned if the [Client.CheckRedirect] function fails 467 // or if there was an HTTP protocol error. A non-2xx response doesn't 468 // cause an error. Any returned error will be of type [*url.Error]. The 469 // url.Error value's Timeout method will report true if the request 470 // timed out. 471 // 472 // When err is nil, resp always contains a non-nil resp.Body. 473 // Caller should close resp.Body when done reading from it. 474 // 475 // To make a request with custom headers, use [NewRequest] and [Client.Do]. 476 // 477 // To make a request with a specified context.Context, use [NewRequestWithContext] 478 // and Client.Do. 479 func (c *Client) Get(url string) (resp *Response, err error) { 480 req, err := NewRequest("GET", url, nil) 481 if err != nil { 482 return nil, err 483 } 484 return c.Do(req) 485 } 486 487 func alwaysFalse() bool { return false } 488 489 // ErrUseLastResponse can be returned by Client.CheckRedirect hooks to 490 // control how redirects are processed. If returned, the next request 491 // is not sent and the most recent response is returned with its body 492 // unclosed. 493 var ErrUseLastResponse = errors.New("net/http: use last response") 494 495 // checkRedirect calls either the user's configured CheckRedirect 496 // function, or the default. 497 func (c *Client) checkRedirect(req *Request, via []*Request) error { 498 fn := c.CheckRedirect 499 if fn == nil { 500 fn = defaultCheckRedirect 501 } 502 return fn(req, via) 503 } 504 505 // redirectBehavior describes what should happen when the 506 // client encounters a 3xx status code from the server. 507 func redirectBehavior(reqMethod string, resp *Response, ireq *Request) (redirectMethod string, shouldRedirect, includeBody bool) { 508 switch resp.StatusCode { 509 case 301, 302, 303: 510 redirectMethod = reqMethod 511 shouldRedirect = true 512 includeBody = false 513 514 // RFC 2616 allowed automatic redirection only with GET and 515 // HEAD requests. RFC 7231 lifts this restriction, but we still 516 // restrict other methods to GET to maintain compatibility. 517 // See Issue 18570. 518 if reqMethod != "GET" && reqMethod != "HEAD" { 519 redirectMethod = "GET" 520 } 521 case 307, 308: 522 redirectMethod = reqMethod 523 shouldRedirect = true 524 includeBody = true 525 526 if ireq.GetBody == nil && ireq.outgoingLength() != 0 { 527 // We had a request body, and 307/308 require 528 // re-sending it, but GetBody is not defined. So just 529 // return this response to the user instead of an 530 // error, like we did in Go 1.7 and earlier. 531 shouldRedirect = false 532 } 533 } 534 return redirectMethod, shouldRedirect, includeBody 535 } 536 537 // urlErrorOp returns the (*url.Error).Op value to use for the 538 // provided (*Request).Method value. 539 func urlErrorOp(method string) string { 540 if method == "" { 541 return "Get" 542 } 543 if lowerMethod, ok := ascii.ToLower(method); ok { 544 return method[:1] + lowerMethod[1:] 545 } 546 return method 547 } 548 549 // Do sends an HTTP request and returns an HTTP response, following 550 // policy (such as redirects, cookies, auth) as configured on the 551 // client. 552 // 553 // An error is returned if caused by client policy (such as 554 // CheckRedirect), or failure to speak HTTP (such as a network 555 // connectivity problem). A non-2xx status code doesn't cause an 556 // error. 557 // 558 // If the returned error is nil, the [Response] will contain a non-nil 559 // Body which the user is expected to close. If the Body is not both 560 // read to EOF and closed, the [Client]'s underlying [RoundTripper] 561 // (typically [Transport]) may not be able to re-use a persistent TCP 562 // connection to the server for a subsequent "keep-alive" request. 563 // 564 // The request Body, if non-nil, will be closed by the underlying 565 // Transport, even on errors. The Body may be closed asynchronously after 566 // Do returns. 567 // 568 // On error, any Response can be ignored. A non-nil Response with a 569 // non-nil error only occurs when CheckRedirect fails, and even then 570 // the returned [Response.Body] is already closed. 571 // 572 // Generally [Get], [Post], or [PostForm] will be used instead of Do. 573 // 574 // If the server replies with a redirect, the Client first uses the 575 // CheckRedirect function to determine whether the redirect should be 576 // followed. If permitted, a 301, 302, or 303 redirect causes 577 // subsequent requests to use HTTP method GET 578 // (or HEAD if the original request was HEAD), with no body. 579 // A 307 or 308 redirect preserves the original HTTP method and body, 580 // provided that the [Request.GetBody] function is defined. 581 // The [NewRequest] function automatically sets GetBody for common 582 // standard library body types. 583 // 584 // Any returned error will be of type [*url.Error]. The url.Error 585 // value's Timeout method will report true if the request timed out. 586 func (c *Client) Do(req *Request) (*Response, error) { 587 return c.do(req) 588 } 589 590 var testHookClientDoResult func(retres *Response, reterr error) 591 592 func (c *Client) do(req *Request) (retres *Response, reterr error) { 593 if testHookClientDoResult != nil { 594 defer func() { testHookClientDoResult(retres, reterr) }() 595 } 596 if req.URL == nil { 597 req.closeBody() 598 return nil, &url.Error{ 599 Op: urlErrorOp(req.Method), 600 Err: errors.New("http: nil Request.URL"), 601 } 602 } 603 _ = *c // panic early if c is nil; see go.dev/issue/53521 604 605 var ( 606 deadline = c.deadline() 607 reqs []*Request 608 resp *Response 609 copyHeaders = c.makeHeadersCopier(req) 610 reqBodyClosed = false // have we closed the current req.Body? 611 612 // Redirect behavior: 613 redirectMethod string 614 includeBody = true 615 ) 616 uerr := func(err error) error { 617 // the body may have been closed already by c.send() 618 if !reqBodyClosed { 619 req.closeBody() 620 } 621 var urlStr string 622 if resp != nil && resp.Request != nil { 623 urlStr = stripPassword(resp.Request.URL) 624 } else { 625 urlStr = stripPassword(req.URL) 626 } 627 return &url.Error{ 628 Op: urlErrorOp(reqs[0].Method), 629 URL: urlStr, 630 Err: err, 631 } 632 } 633 for { 634 // For all but the first request, create the next 635 // request hop and replace req. 636 if len(reqs) > 0 { 637 loc := resp.Header.Get("Location") 638 if loc == "" { 639 // While most 3xx responses include a Location, it is not 640 // required and 3xx responses without a Location have been 641 // observed in the wild. See issues #17773 and #49281. 642 return resp, nil 643 } 644 u, err := req.URL.Parse(loc) 645 if err != nil { 646 resp.closeBody() 647 return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err)) 648 } 649 host := "" 650 if req.Host != "" && req.Host != req.URL.Host { 651 // If the caller specified a custom Host header and the 652 // redirect location is relative, preserve the Host header 653 // through the redirect. See issue #22233. 654 if u, _ := url.Parse(loc); u != nil && !u.IsAbs() { 655 host = req.Host 656 } 657 } 658 ireq := reqs[0] 659 req = &Request{ 660 Method: redirectMethod, 661 Response: resp, 662 URL: u, 663 Header: make(Header), 664 Host: host, 665 Cancel: ireq.Cancel, 666 ctx: ireq.ctx, 667 } 668 if includeBody && ireq.GetBody != nil { 669 req.Body, err = ireq.GetBody() 670 if err != nil { 671 resp.closeBody() 672 return nil, uerr(err) 673 } 674 req.ContentLength = ireq.ContentLength 675 } 676 677 // Copy original headers before setting the Referer, 678 // in case the user set Referer on their first request. 679 // If they really want to override, they can do it in 680 // their CheckRedirect func. 681 copyHeaders(req) 682 683 // Add the Referer header from the most recent 684 // request URL to the new one, if it's not https->http: 685 if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL, req.Header.Get("Referer")); ref != "" { 686 req.Header.Set("Referer", ref) 687 } 688 err = c.checkRedirect(req, reqs) 689 690 // Sentinel error to let users select the 691 // previous response, without closing its 692 // body. See Issue 10069. 693 if err == ErrUseLastResponse { 694 return resp, nil 695 } 696 697 // Close the previous response's body. But 698 // read at least some of the body so if it's 699 // small the underlying TCP connection will be 700 // re-used. No need to check for errors: if it 701 // fails, the Transport won't reuse it anyway. 702 const maxBodySlurpSize = 2 << 10 703 if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize { 704 io.CopyN(io.Discard, resp.Body, maxBodySlurpSize) 705 } 706 resp.Body.Close() 707 708 if err != nil { 709 // Special case for Go 1 compatibility: return both the response 710 // and an error if the CheckRedirect function failed. 711 // See https://golang.org/issue/3795 712 // The resp.Body has already been closed. 713 ue := uerr(err) 714 ue.(*url.Error).URL = loc 715 return resp, ue 716 } 717 } 718 719 reqs = append(reqs, req) 720 var err error 721 var didTimeout func() bool 722 if resp, didTimeout, err = c.send(req, deadline); err != nil { 723 // c.send() always closes req.Body 724 reqBodyClosed = true 725 if !deadline.IsZero() && didTimeout() { 726 err = &timeoutError{err.Error() + " (Client.Timeout exceeded while awaiting headers)"} 727 } 728 return nil, uerr(err) 729 } 730 731 var shouldRedirect, includeBodyOnHop bool 732 redirectMethod, shouldRedirect, includeBodyOnHop = redirectBehavior(req.Method, resp, reqs[0]) 733 if !shouldRedirect { 734 return resp, nil 735 } 736 if !includeBodyOnHop { 737 // Once a hop drops the body, we never send it again 738 // (because we're now handling a redirect for a request with no body). 739 includeBody = false 740 } 741 742 req.closeBody() 743 } 744 } 745 746 // makeHeadersCopier makes a function that copies headers from the 747 // initial Request, ireq. For every redirect, this function must be called 748 // so that it can copy headers into the upcoming Request. 749 func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) { 750 // The headers to copy are from the very initial request. 751 // We use a closured callback to keep a reference to these original headers. 752 var ( 753 ireqhdr = cloneOrMakeHeader(ireq.Header) 754 icookies map[string][]*Cookie 755 ) 756 if c.Jar != nil && ireq.Header.Get("Cookie") != "" { 757 icookies = make(map[string][]*Cookie) 758 for _, c := range ireq.Cookies() { 759 icookies[c.Name] = append(icookies[c.Name], c) 760 } 761 } 762 763 preq := ireq // The previous request 764 return func(req *Request) { 765 // If Jar is present and there was some initial cookies provided 766 // via the request header, then we may need to alter the initial 767 // cookies as we follow redirects since each redirect may end up 768 // modifying a pre-existing cookie. 769 // 770 // Since cookies already set in the request header do not contain 771 // information about the original domain and path, the logic below 772 // assumes any new set cookies override the original cookie 773 // regardless of domain or path. 774 // 775 // See https://golang.org/issue/17494 776 if c.Jar != nil && icookies != nil { 777 var changed bool 778 resp := req.Response // The response that caused the upcoming redirect 779 for _, c := range resp.Cookies() { 780 if _, ok := icookies[c.Name]; ok { 781 delete(icookies, c.Name) 782 changed = true 783 } 784 } 785 if changed { 786 ireqhdr.Del("Cookie") 787 var ss []string 788 for _, cs := range icookies { 789 for _, c := range cs { 790 ss = append(ss, c.Name+"="+c.Value) 791 } 792 } 793 slices.Sort(ss) // Ensure deterministic headers 794 ireqhdr.Set("Cookie", strings.Join(ss, "; ")) 795 } 796 } 797 798 // Copy the initial request's Header values 799 // (at least the safe ones). 800 for k, vv := range ireqhdr { 801 if shouldCopyHeaderOnRedirect(k, preq.URL, req.URL) { 802 req.Header[k] = vv 803 } 804 } 805 806 preq = req // Update previous Request with the current request 807 } 808 } 809 810 func defaultCheckRedirect(req *Request, via []*Request) error { 811 if len(via) >= 10 { 812 return errors.New("stopped after 10 redirects") 813 } 814 return nil 815 } 816 817 // Post issues a POST to the specified URL. 818 // 819 // Caller should close resp.Body when done reading from it. 820 // 821 // If the provided body is an [io.Closer], it is closed after the 822 // request. 823 // 824 // Post is a wrapper around DefaultClient.Post. 825 // 826 // To set custom headers, use [NewRequest] and DefaultClient.Do. 827 // 828 // See the [Client.Do] method documentation for details on how redirects 829 // are handled. 830 // 831 // To make a request with a specified context.Context, use [NewRequestWithContext] 832 // and DefaultClient.Do. 833 func Post(url, contentType string, body io.Reader) (resp *Response, err error) { 834 return DefaultClient.Post(url, contentType, body) 835 } 836 837 // Post issues a POST to the specified URL. 838 // 839 // Caller should close resp.Body when done reading from it. 840 // 841 // If the provided body is an [io.Closer], it is closed after the 842 // request. 843 // 844 // To set custom headers, use [NewRequest] and [Client.Do]. 845 // 846 // To make a request with a specified context.Context, use [NewRequestWithContext] 847 // and [Client.Do]. 848 // 849 // See the Client.Do method documentation for details on how redirects 850 // are handled. 851 func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) { 852 req, err := NewRequest("POST", url, body) 853 if err != nil { 854 return nil, err 855 } 856 req.Header.Set("Content-Type", contentType) 857 return c.Do(req) 858 } 859 860 // PostForm issues a POST to the specified URL, with data's keys and 861 // values URL-encoded as the request body. 862 // 863 // The Content-Type header is set to application/x-www-form-urlencoded. 864 // To set other headers, use [NewRequest] and DefaultClient.Do. 865 // 866 // When err is nil, resp always contains a non-nil resp.Body. 867 // Caller should close resp.Body when done reading from it. 868 // 869 // PostForm is a wrapper around DefaultClient.PostForm. 870 // 871 // See the [Client.Do] method documentation for details on how redirects 872 // are handled. 873 // 874 // To make a request with a specified [context.Context], use [NewRequestWithContext] 875 // and DefaultClient.Do. 876 func PostForm(url string, data url.Values) (resp *Response, err error) { 877 return DefaultClient.PostForm(url, data) 878 } 879 880 // PostForm issues a POST to the specified URL, 881 // with data's keys and values URL-encoded as the request body. 882 // 883 // The Content-Type header is set to application/x-www-form-urlencoded. 884 // To set other headers, use [NewRequest] and [Client.Do]. 885 // 886 // When err is nil, resp always contains a non-nil resp.Body. 887 // Caller should close resp.Body when done reading from it. 888 // 889 // See the Client.Do method documentation for details on how redirects 890 // are handled. 891 // 892 // To make a request with a specified context.Context, use [NewRequestWithContext] 893 // and Client.Do. 894 func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) { 895 return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) 896 } 897 898 // Head issues a HEAD to the specified URL. If the response is one of 899 // the following redirect codes, Head follows the redirect, up to a 900 // maximum of 10 redirects: 901 // 902 // 301 (Moved Permanently) 903 // 302 (Found) 904 // 303 (See Other) 905 // 307 (Temporary Redirect) 906 // 308 (Permanent Redirect) 907 // 908 // Head is a wrapper around DefaultClient.Head. 909 // 910 // To make a request with a specified [context.Context], use [NewRequestWithContext] 911 // and DefaultClient.Do. 912 func Head(url string) (resp *Response, err error) { 913 return DefaultClient.Head(url) 914 } 915 916 // Head issues a HEAD to the specified URL. If the response is one of the 917 // following redirect codes, Head follows the redirect after calling the 918 // [Client.CheckRedirect] function: 919 // 920 // 301 (Moved Permanently) 921 // 302 (Found) 922 // 303 (See Other) 923 // 307 (Temporary Redirect) 924 // 308 (Permanent Redirect) 925 // 926 // To make a request with a specified [context.Context], use [NewRequestWithContext] 927 // and [Client.Do]. 928 func (c *Client) Head(url string) (resp *Response, err error) { 929 req, err := NewRequest("HEAD", url, nil) 930 if err != nil { 931 return nil, err 932 } 933 return c.Do(req) 934 } 935 936 // CloseIdleConnections closes any connections on its [Transport] which 937 // were previously connected from previous requests but are now 938 // sitting idle in a "keep-alive" state. It does not interrupt any 939 // connections currently in use. 940 // 941 // If [Client.Transport] does not have a [Client.CloseIdleConnections] method 942 // then this method does nothing. 943 func (c *Client) CloseIdleConnections() { 944 type closeIdler interface { 945 CloseIdleConnections() 946 } 947 if tr, ok := c.transport().(closeIdler); ok { 948 tr.CloseIdleConnections() 949 } 950 } 951 952 // cancelTimerBody is an io.ReadCloser that wraps rc with two features: 953 // 1. On Read error or close, the stop func is called. 954 // 2. On Read failure, if reqDidTimeout is true, the error is wrapped and 955 // marked as net.Error that hit its timeout. 956 type cancelTimerBody struct { 957 stop func() // stops the time.Timer waiting to cancel the request 958 rc io.ReadCloser 959 reqDidTimeout func() bool 960 } 961 962 func (b *cancelTimerBody) Read(p []byte) (n int, err error) { 963 n, err = b.rc.Read(p) 964 if err == nil { 965 return n, nil 966 } 967 if err == io.EOF { 968 return n, err 969 } 970 if b.reqDidTimeout() { 971 err = &timeoutError{err.Error() + " (Client.Timeout or context cancellation while reading body)"} 972 } 973 return n, err 974 } 975 976 func (b *cancelTimerBody) Close() error { 977 err := b.rc.Close() 978 b.stop() 979 return err 980 } 981 982 func shouldCopyHeaderOnRedirect(headerKey string, initial, dest *url.URL) bool { 983 switch CanonicalHeaderKey(headerKey) { 984 case "Authorization", "Www-Authenticate", "Cookie", "Cookie2": 985 // Permit sending auth/cookie headers from "foo.com" 986 // to "sub.foo.com". 987 988 // Note that we don't send all cookies to subdomains 989 // automatically. This function is only used for 990 // Cookies set explicitly on the initial outgoing 991 // client request. Cookies automatically added via the 992 // CookieJar mechanism continue to follow each 993 // cookie's scope as set by Set-Cookie. But for 994 // outgoing requests with the Cookie header set 995 // directly, we don't know their scope, so we assume 996 // it's for *.domain.com. 997 998 ihost := idnaASCIIFromURL(initial) 999 dhost := idnaASCIIFromURL(dest) 1000 return isDomainOrSubdomain(dhost, ihost) 1001 } 1002 // All other headers are copied: 1003 return true 1004 } 1005 1006 // isDomainOrSubdomain reports whether sub is a subdomain (or exact 1007 // match) of the parent domain. 1008 // 1009 // Both domains must already be in canonical form. 1010 func isDomainOrSubdomain(sub, parent string) bool { 1011 if sub == parent { 1012 return true 1013 } 1014 // If sub contains a :, it's probably an IPv6 address (and is definitely not a hostname). 1015 // Don't check the suffix in this case, to avoid matching the contents of a IPv6 zone. 1016 // For example, "::1%.www.example.com" is not a subdomain of "www.example.com". 1017 if strings.ContainsAny(sub, ":%") { 1018 return false 1019 } 1020 // If sub is "foo.example.com" and parent is "example.com", 1021 // that means sub must end in "."+parent. 1022 // Do it without allocating. 1023 if !strings.HasSuffix(sub, parent) { 1024 return false 1025 } 1026 return sub[len(sub)-len(parent)-1] == '.' 1027 } 1028 1029 func stripPassword(u *url.URL) string { 1030 _, passSet := u.User.Password() 1031 if passSet { 1032 return strings.Replace(u.String(), u.User.String()+"@", u.User.Username()+":***@", 1) 1033 } 1034 return u.String() 1035 } 1036