1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // HTTP client. See RFC 7230 through 7235. 6 // 7 // This is the high-level Client interface. 8 // The low-level implementation is in transport.go. 9 10 package http 11 12 import ( 13 "context" 14 "crypto/tls" 15 "encoding/base64" 16 "errors" 17 "fmt" 18 "io" 19 "log" 20 "net/http/internal/ascii" 21 "net/url" 22 "reflect" 23 "sort" 24 "strings" 25 "sync" 26 "time" 27 ) 28 29 // A Client is an HTTP client. Its zero value (DefaultClient) is a 30 // usable client that uses DefaultTransport. 31 // 32 // The Client's Transport typically has internal state (cached TCP 33 // connections), so Clients should be reused instead of created as 34 // needed. Clients are safe for concurrent use by multiple goroutines. 35 // 36 // A Client is higher-level than a RoundTripper (such as Transport) 37 // and additionally handles HTTP details such as cookies and 38 // redirects. 39 // 40 // When following redirects, the Client will forward all headers set on the 41 // initial Request except: 42 // 43 // • when forwarding sensitive headers like "Authorization", 44 // "WWW-Authenticate", and "Cookie" to untrusted targets. 45 // These headers will be ignored when following a redirect to a domain 46 // that is not a subdomain match or exact match of the initial domain. 47 // For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" 48 // will forward the sensitive headers, but a redirect to "bar.com" will not. 49 // 50 // • when forwarding the "Cookie" header with a non-nil cookie Jar. 51 // Since each redirect may mutate the state of the cookie jar, 52 // a redirect may possibly alter a cookie set in the initial request. 53 // When forwarding the "Cookie" header, any mutated cookies will be omitted, 54 // with the expectation that the Jar will insert those mutated cookies 55 // with the updated values (assuming the origin matches). 56 // If Jar is nil, the initial cookies are forwarded without change. 57 type Client struct { 58 // Transport specifies the mechanism by which individual 59 // HTTP requests are made. 60 // If nil, DefaultTransport is used. 61 Transport RoundTripper 62 63 // CheckRedirect specifies the policy for handling redirects. 64 // If CheckRedirect is not nil, the client calls it before 65 // following an HTTP redirect. The arguments req and via are 66 // the upcoming request and the requests made already, oldest 67 // first. If CheckRedirect returns an error, the Client's Get 68 // method returns both the previous Response (with its Body 69 // closed) and CheckRedirect's error (wrapped in a url.Error) 70 // instead of issuing the Request req. 71 // As a special case, if CheckRedirect returns ErrUseLastResponse, 72 // then the most recent response is returned with its body 73 // unclosed, along with a nil error. 74 // 75 // If CheckRedirect is nil, the Client uses its default policy, 76 // which is to stop after 10 consecutive requests. 77 CheckRedirect func(req *Request, via []*Request) error 78 79 // Jar specifies the cookie jar. 80 // 81 // The Jar is used to insert relevant cookies into every 82 // outbound Request and is updated with the cookie values 83 // of every inbound Response. The Jar is consulted for every 84 // redirect that the Client follows. 85 // 86 // If Jar is nil, cookies are only sent if they are explicitly 87 // set on the Request. 88 Jar CookieJar 89 90 // Timeout specifies a time limit for requests made by this 91 // Client. The timeout includes connection time, any 92 // redirects, and reading the response body. The timer remains 93 // running after Get, Head, Post, or Do return and will 94 // interrupt reading of the Response.Body. 95 // 96 // A Timeout of zero means no timeout. 97 // 98 // The Client cancels requests to the underlying Transport 99 // as if the Request's Context ended. 100 // 101 // For compatibility, the Client will also use the deprecated 102 // CancelRequest method on Transport if found. New 103 // RoundTripper implementations should use the Request's Context 104 // for cancellation instead of implementing CancelRequest. 105 Timeout time.Duration 106 } 107 108 // DefaultClient is the default Client and is used by Get, Head, and Post. 109 var DefaultClient = &Client{} 110 111 // RoundTripper is an interface representing the ability to execute a 112 // single HTTP transaction, obtaining the Response for a given Request. 113 // 114 // A RoundTripper must be safe for concurrent use by multiple 115 // goroutines. 116 type RoundTripper interface { 117 // RoundTrip executes a single HTTP transaction, returning 118 // a Response for the provided Request. 119 // 120 // RoundTrip should not attempt to interpret the response. In 121 // particular, RoundTrip must return err == nil if it obtained 122 // a response, regardless of the response's HTTP status code. 123 // A non-nil err should be reserved for failure to obtain a 124 // response. Similarly, RoundTrip should not attempt to 125 // handle higher-level protocol details such as redirects, 126 // authentication, or cookies. 127 // 128 // RoundTrip should not modify the request, except for 129 // consuming and closing the Request's Body. RoundTrip may 130 // read fields of the request in a separate goroutine. Callers 131 // should not mutate or reuse the request until the Response's 132 // Body has been closed. 133 // 134 // RoundTrip must always close the body, including on errors, 135 // but depending on the implementation may do so in a separate 136 // goroutine even after RoundTrip returns. This means that 137 // callers wanting to reuse the body for subsequent requests 138 // must arrange to wait for the Close call before doing so. 139 // 140 // The Request's URL and Header fields must be initialized. 141 RoundTrip(*Request) (*Response, error) 142 } 143 144 // refererForURL returns a referer without any authentication info or 145 // an empty string if lastReq scheme is https and newReq scheme is http. 146 func refererForURL(lastReq, newReq *url.URL) string { 147 // https://tools.ietf.org/html/rfc7231#section-5.5.2 148 // "Clients SHOULD NOT include a Referer header field in a 149 // (non-secure) HTTP request if the referring page was 150 // transferred with a secure protocol." 151 if lastReq.Scheme == "https" && newReq.Scheme == "http" { 152 return "" 153 } 154 referer := lastReq.String() 155 if lastReq.User != nil { 156 // This is not very efficient, but is the best we can 157 // do without: 158 // - introducing a new method on URL 159 // - creating a race condition 160 // - copying the URL struct manually, which would cause 161 // maintenance problems down the line 162 auth := lastReq.User.String() + "@" 163 referer = strings.Replace(referer, auth, "", 1) 164 } 165 return referer 166 } 167 168 // didTimeout is non-nil only if err != nil. 169 func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { 170 if c.Jar != nil { 171 for _, cookie := range c.Jar.Cookies(req.URL) { 172 req.AddCookie(cookie) 173 } 174 } 175 resp, didTimeout, err = send(req, c.transport(), deadline) 176 if err != nil { 177 return nil, didTimeout, err 178 } 179 if c.Jar != nil { 180 if rc := resp.Cookies(); len(rc) > 0 { 181 c.Jar.SetCookies(req.URL, rc) 182 } 183 } 184 return resp, nil, nil 185 } 186 187 func (c *Client) deadline() time.Time { 188 if c.Timeout > 0 { 189 return time.Now().Add(c.Timeout) 190 } 191 return time.Time{} 192 } 193 194 func (c *Client) transport() RoundTripper { 195 if c.Transport != nil { 196 return c.Transport 197 } 198 return DefaultTransport 199 } 200 201 // send issues an HTTP request. 202 // Caller should close resp.Body when done reading from it. 203 func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { 204 req := ireq // req is either the original request, or a modified fork 205 206 if rt == nil { 207 req.closeBody() 208 return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport") 209 } 210 211 if req.URL == nil { 212 req.closeBody() 213 return nil, alwaysFalse, errors.New("http: nil Request.URL") 214 } 215 216 if req.RequestURI != "" { 217 req.closeBody() 218 return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests") 219 } 220 221 // forkReq forks req into a shallow clone of ireq the first 222 // time it's called. 223 forkReq := func() { 224 if ireq == req { 225 req = new(Request) 226 *req = *ireq // shallow clone 227 } 228 } 229 230 // Most the callers of send (Get, Post, et al) don't need 231 // Headers, leaving it uninitialized. We guarantee to the 232 // Transport that this has been initialized, though. 233 if req.Header == nil { 234 forkReq() 235 req.Header = make(Header) 236 } 237 238 if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" { 239 username := u.Username() 240 password, _ := u.Password() 241 forkReq() 242 req.Header = cloneOrMakeHeader(ireq.Header) 243 req.Header.Set("Authorization", "Basic "+basicAuth(username, password)) 244 } 245 246 if !deadline.IsZero() { 247 forkReq() 248 } 249 stopTimer, didTimeout := setRequestCancel(req, rt, deadline) 250 251 resp, err = rt.RoundTrip(req) 252 if err != nil { 253 stopTimer() 254 if resp != nil { 255 log.Printf("RoundTripper returned a response & error; ignoring response") 256 } 257 if tlsErr, ok := err.(tls.RecordHeaderError); ok { 258 // If we get a bad TLS record header, check to see if the 259 // response looks like HTTP and give a more helpful error. 260 // See golang.org/issue/11111. 261 if string(tlsErr.RecordHeader[:]) == "HTTP/" { 262 err = errors.New("http: server gave HTTP response to HTTPS client") 263 } 264 } 265 return nil, didTimeout, err 266 } 267 if resp == nil { 268 return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a nil *Response with a nil error", rt) 269 } 270 if resp.Body == nil { 271 // The documentation on the Body field says “The http Client and Transport 272 // guarantee that Body is always non-nil, even on responses without a body 273 // or responses with a zero-length body.” Unfortunately, we didn't document 274 // that same constraint for arbitrary RoundTripper implementations, and 275 // RoundTripper implementations in the wild (mostly in tests) assume that 276 // they can use a nil Body to mean an empty one (similar to Request.Body). 277 // (See https://golang.org/issue/38095.) 278 // 279 // If the ContentLength allows the Body to be empty, fill in an empty one 280 // here to ensure that it is non-nil. 281 if resp.ContentLength > 0 && req.Method != "HEAD" { 282 return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a *Response with content length %d but a nil Body", rt, resp.ContentLength) 283 } 284 resp.Body = io.NopCloser(strings.NewReader("")) 285 } 286 if !deadline.IsZero() { 287 resp.Body = &cancelTimerBody{ 288 stop: stopTimer, 289 rc: resp.Body, 290 reqDidTimeout: didTimeout, 291 } 292 } 293 return resp, nil, nil 294 } 295 296 // timeBeforeContextDeadline reports whether the non-zero Time t is 297 // before ctx's deadline, if any. If ctx does not have a deadline, it 298 // always reports true (the deadline is considered infinite). 299 func timeBeforeContextDeadline(t time.Time, ctx context.Context) bool { 300 d, ok := ctx.Deadline() 301 if !ok { 302 return true 303 } 304 return t.Before(d) 305 } 306 307 // knownRoundTripperImpl reports whether rt is a RoundTripper that's 308 // maintained by the Go team and known to implement the latest 309 // optional semantics (notably contexts). The Request is used 310 // to check whether this particular request is using an alternate protocol, 311 // in which case we need to check the RoundTripper for that protocol. 312 func knownRoundTripperImpl(rt RoundTripper, req *Request) bool { 313 switch t := rt.(type) { 314 case *Transport: 315 if altRT := t.alternateRoundTripper(req); altRT != nil { 316 return knownRoundTripperImpl(altRT, req) 317 } 318 return true 319 case *http2Transport, http2noDialH2RoundTripper: 320 return true 321 } 322 // There's a very minor chance of a false positive with this. 323 // Instead of detecting our golang.org/x/net/http2.Transport, 324 // it might detect a Transport type in a different http2 325 // package. But I know of none, and the only problem would be 326 // some temporarily leaked goroutines if the transport didn't 327 // support contexts. So this is a good enough heuristic: 328 if reflect.TypeOf(rt).String() == "*http2.Transport" { 329 return true 330 } 331 return false 332 } 333 334 // setRequestCancel sets req.Cancel and adds a deadline context to req 335 // if deadline is non-zero. The RoundTripper's type is used to 336 // determine whether the legacy CancelRequest behavior should be used. 337 // 338 // As background, there are three ways to cancel a request: 339 // First was Transport.CancelRequest. (deprecated) 340 // Second was Request.Cancel. 341 // Third was Request.Context. 342 // This function populates the second and third, and uses the first if it really needs to. 343 func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) { 344 if deadline.IsZero() { 345 return nop, alwaysFalse 346 } 347 knownTransport := knownRoundTripperImpl(rt, req) 348 oldCtx := req.Context() 349 350 if req.Cancel == nil && knownTransport { 351 // If they already had a Request.Context that's 352 // expiring sooner, do nothing: 353 if !timeBeforeContextDeadline(deadline, oldCtx) { 354 return nop, alwaysFalse 355 } 356 357 var cancelCtx func() 358 req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) 359 return cancelCtx, func() bool { return time.Now().After(deadline) } 360 } 361 initialReqCancel := req.Cancel // the user's original Request.Cancel, if any 362 363 var cancelCtx func() 364 if oldCtx := req.Context(); timeBeforeContextDeadline(deadline, oldCtx) { 365 req.ctx, cancelCtx = context.WithDeadline(oldCtx, deadline) 366 } 367 368 cancel := make(chan struct{}) 369 req.Cancel = cancel 370 371 doCancel := func() { 372 // The second way in the func comment above: 373 close(cancel) 374 // The first way, used only for RoundTripper 375 // implementations written before Go 1.5 or Go 1.6. 376 type canceler interface{ CancelRequest(*Request) } 377 if v, ok := rt.(canceler); ok { 378 v.CancelRequest(req) 379 } 380 } 381 382 stopTimerCh := make(chan struct{}) 383 var once sync.Once 384 stopTimer = func() { 385 once.Do(func() { 386 close(stopTimerCh) 387 if cancelCtx != nil { 388 cancelCtx() 389 } 390 }) 391 } 392 393 timer := time.NewTimer(time.Until(deadline)) 394 var timedOut atomicBool 395 396 go func() { 397 select { 398 case <-initialReqCancel: 399 doCancel() 400 timer.Stop() 401 case <-timer.C: 402 timedOut.setTrue() 403 doCancel() 404 case <-stopTimerCh: 405 timer.Stop() 406 } 407 }() 408 409 return stopTimer, timedOut.isSet 410 } 411 412 // See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt 413 // "To receive authorization, the client sends the userid and password, 414 // separated by a single colon (":") character, within a base64 415 // encoded string in the credentials." 416 // It is not meant to be urlencoded. 417 func basicAuth(username, password string) string { 418 auth := username + ":" + password 419 return base64.StdEncoding.EncodeToString([]byte(auth)) 420 } 421 422 // Get issues a GET to the specified URL. If the response is one of 423 // the following redirect codes, Get follows the redirect, up to a 424 // maximum of 10 redirects: 425 // 426 // 301 (Moved Permanently) 427 // 302 (Found) 428 // 303 (See Other) 429 // 307 (Temporary Redirect) 430 // 308 (Permanent Redirect) 431 // 432 // An error is returned if there were too many redirects or if there 433 // was an HTTP protocol error. A non-2xx response doesn't cause an 434 // error. Any returned error will be of type *url.Error. The url.Error 435 // value's Timeout method will report true if the request timed out. 436 // 437 // When err is nil, resp always contains a non-nil resp.Body. 438 // Caller should close resp.Body when done reading from it. 439 // 440 // Get is a wrapper around DefaultClient.Get. 441 // 442 // To make a request with custom headers, use NewRequest and 443 // DefaultClient.Do. 444 // 445 // To make a request with a specified context.Context, use NewRequestWithContext 446 // and DefaultClient.Do. 447 func Get(url string) (resp *Response, err error) { 448 return DefaultClient.Get(url) 449 } 450 451 // Get issues a GET to the specified URL. If the response is one of the 452 // following redirect codes, Get follows the redirect after calling the 453 // Client's CheckRedirect function: 454 // 455 // 301 (Moved Permanently) 456 // 302 (Found) 457 // 303 (See Other) 458 // 307 (Temporary Redirect) 459 // 308 (Permanent Redirect) 460 // 461 // An error is returned if the Client's CheckRedirect function fails 462 // or if there was an HTTP protocol error. A non-2xx response doesn't 463 // cause an error. Any returned error will be of type *url.Error. The 464 // url.Error value's Timeout method will report true if the request 465 // timed out. 466 // 467 // When err is nil, resp always contains a non-nil resp.Body. 468 // Caller should close resp.Body when done reading from it. 469 // 470 // To make a request with custom headers, use NewRequest and Client.Do. 471 // 472 // To make a request with a specified context.Context, use NewRequestWithContext 473 // and Client.Do. 474 func (c *Client) Get(url string) (resp *Response, err error) { 475 req, err := NewRequest("GET", url, nil) 476 if err != nil { 477 return nil, err 478 } 479 return c.Do(req) 480 } 481 482 func alwaysFalse() bool { return false } 483 484 // ErrUseLastResponse can be returned by Client.CheckRedirect hooks to 485 // control how redirects are processed. If returned, the next request 486 // is not sent and the most recent response is returned with its body 487 // unclosed. 488 var ErrUseLastResponse = errors.New("net/http: use last response") 489 490 // checkRedirect calls either the user's configured CheckRedirect 491 // function, or the default. 492 func (c *Client) checkRedirect(req *Request, via []*Request) error { 493 fn := c.CheckRedirect 494 if fn == nil { 495 fn = defaultCheckRedirect 496 } 497 return fn(req, via) 498 } 499 500 // redirectBehavior describes what should happen when the 501 // client encounters a 3xx status code from the server 502 func redirectBehavior(reqMethod string, resp *Response, ireq *Request) (redirectMethod string, shouldRedirect, includeBody bool) { 503 switch resp.StatusCode { 504 case 301, 302, 303: 505 redirectMethod = reqMethod 506 shouldRedirect = true 507 includeBody = false 508 509 // RFC 2616 allowed automatic redirection only with GET and 510 // HEAD requests. RFC 7231 lifts this restriction, but we still 511 // restrict other methods to GET to maintain compatibility. 512 // See Issue 18570. 513 if reqMethod != "GET" && reqMethod != "HEAD" { 514 redirectMethod = "GET" 515 } 516 case 307, 308: 517 redirectMethod = reqMethod 518 shouldRedirect = true 519 includeBody = true 520 521 if ireq.GetBody == nil && ireq.outgoingLength() != 0 { 522 // We had a request body, and 307/308 require 523 // re-sending it, but GetBody is not defined. So just 524 // return this response to the user instead of an 525 // error, like we did in Go 1.7 and earlier. 526 shouldRedirect = false 527 } 528 } 529 return redirectMethod, shouldRedirect, includeBody 530 } 531 532 // urlErrorOp returns the (*url.Error).Op value to use for the 533 // provided (*Request).Method value. 534 func urlErrorOp(method string) string { 535 if method == "" { 536 return "Get" 537 } 538 if lowerMethod, ok := ascii.ToLower(method); ok { 539 return method[:1] + lowerMethod[1:] 540 } 541 return method 542 } 543 544 // Do sends an HTTP request and returns an HTTP response, following 545 // policy (such as redirects, cookies, auth) as configured on the 546 // client. 547 // 548 // An error is returned if caused by client policy (such as 549 // CheckRedirect), or failure to speak HTTP (such as a network 550 // connectivity problem). A non-2xx status code doesn't cause an 551 // error. 552 // 553 // If the returned error is nil, the Response will contain a non-nil 554 // Body which the user is expected to close. If the Body is not both 555 // read to EOF and closed, the Client's underlying RoundTripper 556 // (typically Transport) may not be able to re-use a persistent TCP 557 // connection to the server for a subsequent "keep-alive" request. 558 // 559 // The request Body, if non-nil, will be closed by the underlying 560 // Transport, even on errors. 561 // 562 // On error, any Response can be ignored. A non-nil Response with a 563 // non-nil error only occurs when CheckRedirect fails, and even then 564 // the returned Response.Body is already closed. 565 // 566 // Generally Get, Post, or PostForm will be used instead of Do. 567 // 568 // If the server replies with a redirect, the Client first uses the 569 // CheckRedirect function to determine whether the redirect should be 570 // followed. If permitted, a 301, 302, or 303 redirect causes 571 // subsequent requests to use HTTP method GET 572 // (or HEAD if the original request was HEAD), with no body. 573 // A 307 or 308 redirect preserves the original HTTP method and body, 574 // provided that the Request.GetBody function is defined. 575 // The NewRequest function automatically sets GetBody for common 576 // standard library body types. 577 // 578 // Any returned error will be of type *url.Error. The url.Error 579 // value's Timeout method will report true if the request timed out. 580 func (c *Client) Do(req *Request) (*Response, error) { 581 return c.do(req) 582 } 583 584 var testHookClientDoResult func(retres *Response, reterr error) 585 586 func (c *Client) do(req *Request) (retres *Response, reterr error) { 587 if testHookClientDoResult != nil { 588 defer func() { testHookClientDoResult(retres, reterr) }() 589 } 590 if req.URL == nil { 591 req.closeBody() 592 return nil, &url.Error{ 593 Op: urlErrorOp(req.Method), 594 Err: errors.New("http: nil Request.URL"), 595 } 596 } 597 598 var ( 599 deadline = c.deadline() 600 reqs []*Request 601 resp *Response 602 copyHeaders = c.makeHeadersCopier(req) 603 reqBodyClosed = false // have we closed the current req.Body? 604 605 // Redirect behavior: 606 redirectMethod string 607 includeBody bool 608 ) 609 uerr := func(err error) error { 610 // the body may have been closed already by c.send() 611 if !reqBodyClosed { 612 req.closeBody() 613 } 614 var urlStr string 615 if resp != nil && resp.Request != nil { 616 urlStr = stripPassword(resp.Request.URL) 617 } else { 618 urlStr = stripPassword(req.URL) 619 } 620 return &url.Error{ 621 Op: urlErrorOp(reqs[0].Method), 622 URL: urlStr, 623 Err: err, 624 } 625 } 626 for { 627 // For all but the first request, create the next 628 // request hop and replace req. 629 if len(reqs) > 0 { 630 loc := resp.Header.Get("Location") 631 if loc == "" { 632 // While most 3xx responses include a Location, it is not 633 // required and 3xx responses without a Location have been 634 // observed in the wild. See issues #17773 and #49281. 635 return resp, nil 636 } 637 u, err := req.URL.Parse(loc) 638 if err != nil { 639 resp.closeBody() 640 return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err)) 641 } 642 host := "" 643 if req.Host != "" && req.Host != req.URL.Host { 644 // If the caller specified a custom Host header and the 645 // redirect location is relative, preserve the Host header 646 // through the redirect. See issue #22233. 647 if u, _ := url.Parse(loc); u != nil && !u.IsAbs() { 648 host = req.Host 649 } 650 } 651 ireq := reqs[0] 652 req = &Request{ 653 Method: redirectMethod, 654 Response: resp, 655 URL: u, 656 Header: make(Header), 657 Host: host, 658 Cancel: ireq.Cancel, 659 ctx: ireq.ctx, 660 } 661 if includeBody && ireq.GetBody != nil { 662 req.Body, err = ireq.GetBody() 663 if err != nil { 664 resp.closeBody() 665 return nil, uerr(err) 666 } 667 req.ContentLength = ireq.ContentLength 668 } 669 670 // Copy original headers before setting the Referer, 671 // in case the user set Referer on their first request. 672 // If they really want to override, they can do it in 673 // their CheckRedirect func. 674 copyHeaders(req) 675 676 // Add the Referer header from the most recent 677 // request URL to the new one, if it's not https->http: 678 if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL); ref != "" { 679 req.Header.Set("Referer", ref) 680 } 681 err = c.checkRedirect(req, reqs) 682 683 // Sentinel error to let users select the 684 // previous response, without closing its 685 // body. See Issue 10069. 686 if err == ErrUseLastResponse { 687 return resp, nil 688 } 689 690 // Close the previous response's body. But 691 // read at least some of the body so if it's 692 // small the underlying TCP connection will be 693 // re-used. No need to check for errors: if it 694 // fails, the Transport won't reuse it anyway. 695 const maxBodySlurpSize = 2 << 10 696 if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize { 697 io.CopyN(io.Discard, resp.Body, maxBodySlurpSize) 698 } 699 resp.Body.Close() 700 701 if err != nil { 702 // Special case for Go 1 compatibility: return both the response 703 // and an error if the CheckRedirect function failed. 704 // See https://golang.org/issue/3795 705 // The resp.Body has already been closed. 706 ue := uerr(err) 707 ue.(*url.Error).URL = loc 708 return resp, ue 709 } 710 } 711 712 reqs = append(reqs, req) 713 var err error 714 var didTimeout func() bool 715 if resp, didTimeout, err = c.send(req, deadline); err != nil { 716 // c.send() always closes req.Body 717 reqBodyClosed = true 718 if !deadline.IsZero() && didTimeout() { 719 err = &httpError{ 720 err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", 721 timeout: true, 722 } 723 } 724 return nil, uerr(err) 725 } 726 727 var shouldRedirect bool 728 redirectMethod, shouldRedirect, includeBody = redirectBehavior(req.Method, resp, reqs[0]) 729 if !shouldRedirect { 730 return resp, nil 731 } 732 733 req.closeBody() 734 } 735 } 736 737 // makeHeadersCopier makes a function that copies headers from the 738 // initial Request, ireq. For every redirect, this function must be called 739 // so that it can copy headers into the upcoming Request. 740 func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) { 741 // The headers to copy are from the very initial request. 742 // We use a closured callback to keep a reference to these original headers. 743 var ( 744 ireqhdr = cloneOrMakeHeader(ireq.Header) 745 icookies map[string][]*Cookie 746 ) 747 if c.Jar != nil && ireq.Header.Get("Cookie") != "" { 748 icookies = make(map[string][]*Cookie) 749 for _, c := range ireq.Cookies() { 750 icookies[c.Name] = append(icookies[c.Name], c) 751 } 752 } 753 754 preq := ireq // The previous request 755 return func(req *Request) { 756 // If Jar is present and there was some initial cookies provided 757 // via the request header, then we may need to alter the initial 758 // cookies as we follow redirects since each redirect may end up 759 // modifying a pre-existing cookie. 760 // 761 // Since cookies already set in the request header do not contain 762 // information about the original domain and path, the logic below 763 // assumes any new set cookies override the original cookie 764 // regardless of domain or path. 765 // 766 // See https://golang.org/issue/17494 767 if c.Jar != nil && icookies != nil { 768 var changed bool 769 resp := req.Response // The response that caused the upcoming redirect 770 for _, c := range resp.Cookies() { 771 if _, ok := icookies[c.Name]; ok { 772 delete(icookies, c.Name) 773 changed = true 774 } 775 } 776 if changed { 777 ireqhdr.Del("Cookie") 778 var ss []string 779 for _, cs := range icookies { 780 for _, c := range cs { 781 ss = append(ss, c.Name+"="+c.Value) 782 } 783 } 784 sort.Strings(ss) // Ensure deterministic headers 785 ireqhdr.Set("Cookie", strings.Join(ss, "; ")) 786 } 787 } 788 789 // Copy the initial request's Header values 790 // (at least the safe ones). 791 for k, vv := range ireqhdr { 792 if shouldCopyHeaderOnRedirect(k, preq.URL, req.URL) { 793 req.Header[k] = vv 794 } 795 } 796 797 preq = req // Update previous Request with the current request 798 } 799 } 800 801 func defaultCheckRedirect(req *Request, via []*Request) error { 802 if len(via) >= 10 { 803 return errors.New("stopped after 10 redirects") 804 } 805 return nil 806 } 807 808 // Post issues a POST to the specified URL. 809 // 810 // Caller should close resp.Body when done reading from it. 811 // 812 // If the provided body is an io.Closer, it is closed after the 813 // request. 814 // 815 // Post is a wrapper around DefaultClient.Post. 816 // 817 // To set custom headers, use NewRequest and DefaultClient.Do. 818 // 819 // See the Client.Do method documentation for details on how redirects 820 // are handled. 821 // 822 // To make a request with a specified context.Context, use NewRequestWithContext 823 // and DefaultClient.Do. 824 func Post(url, contentType string, body io.Reader) (resp *Response, err error) { 825 return DefaultClient.Post(url, contentType, body) 826 } 827 828 // Post issues a POST to the specified URL. 829 // 830 // Caller should close resp.Body when done reading from it. 831 // 832 // If the provided body is an io.Closer, it is closed after the 833 // request. 834 // 835 // To set custom headers, use NewRequest and Client.Do. 836 // 837 // To make a request with a specified context.Context, use NewRequestWithContext 838 // and Client.Do. 839 // 840 // See the Client.Do method documentation for details on how redirects 841 // are handled. 842 func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) { 843 req, err := NewRequest("POST", url, body) 844 if err != nil { 845 return nil, err 846 } 847 req.Header.Set("Content-Type", contentType) 848 return c.Do(req) 849 } 850 851 // PostForm issues a POST to the specified URL, with data's keys and 852 // values URL-encoded as the request body. 853 // 854 // The Content-Type header is set to application/x-www-form-urlencoded. 855 // To set other headers, use NewRequest and DefaultClient.Do. 856 // 857 // When err is nil, resp always contains a non-nil resp.Body. 858 // Caller should close resp.Body when done reading from it. 859 // 860 // PostForm is a wrapper around DefaultClient.PostForm. 861 // 862 // See the Client.Do method documentation for details on how redirects 863 // are handled. 864 // 865 // To make a request with a specified context.Context, use NewRequestWithContext 866 // and DefaultClient.Do. 867 func PostForm(url string, data url.Values) (resp *Response, err error) { 868 return DefaultClient.PostForm(url, data) 869 } 870 871 // PostForm issues a POST to the specified URL, 872 // with data's keys and values URL-encoded as the request body. 873 // 874 // The Content-Type header is set to application/x-www-form-urlencoded. 875 // To set other headers, use NewRequest and Client.Do. 876 // 877 // When err is nil, resp always contains a non-nil resp.Body. 878 // Caller should close resp.Body when done reading from it. 879 // 880 // See the Client.Do method documentation for details on how redirects 881 // are handled. 882 // 883 // To make a request with a specified context.Context, use NewRequestWithContext 884 // and Client.Do. 885 func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) { 886 return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) 887 } 888 889 // Head issues a HEAD to the specified URL. If the response is one of 890 // the following redirect codes, Head follows the redirect, up to a 891 // maximum of 10 redirects: 892 // 893 // 301 (Moved Permanently) 894 // 302 (Found) 895 // 303 (See Other) 896 // 307 (Temporary Redirect) 897 // 308 (Permanent Redirect) 898 // 899 // Head is a wrapper around DefaultClient.Head. 900 // 901 // To make a request with a specified context.Context, use NewRequestWithContext 902 // and DefaultClient.Do. 903 func Head(url string) (resp *Response, err error) { 904 return DefaultClient.Head(url) 905 } 906 907 // Head issues a HEAD to the specified URL. If the response is one of the 908 // following redirect codes, Head follows the redirect after calling the 909 // Client's CheckRedirect function: 910 // 911 // 301 (Moved Permanently) 912 // 302 (Found) 913 // 303 (See Other) 914 // 307 (Temporary Redirect) 915 // 308 (Permanent Redirect) 916 // 917 // To make a request with a specified context.Context, use NewRequestWithContext 918 // and Client.Do. 919 func (c *Client) Head(url string) (resp *Response, err error) { 920 req, err := NewRequest("HEAD", url, nil) 921 if err != nil { 922 return nil, err 923 } 924 return c.Do(req) 925 } 926 927 // CloseIdleConnections closes any connections on its Transport which 928 // were previously connected from previous requests but are now 929 // sitting idle in a "keep-alive" state. It does not interrupt any 930 // connections currently in use. 931 // 932 // If the Client's Transport does not have a CloseIdleConnections method 933 // then this method does nothing. 934 func (c *Client) CloseIdleConnections() { 935 type closeIdler interface { 936 CloseIdleConnections() 937 } 938 if tr, ok := c.transport().(closeIdler); ok { 939 tr.CloseIdleConnections() 940 } 941 } 942 943 // cancelTimerBody is an io.ReadCloser that wraps rc with two features: 944 // 1. On Read error or close, the stop func is called. 945 // 2. On Read failure, if reqDidTimeout is true, the error is wrapped and 946 // marked as net.Error that hit its timeout. 947 type cancelTimerBody struct { 948 stop func() // stops the time.Timer waiting to cancel the request 949 rc io.ReadCloser 950 reqDidTimeout func() bool 951 } 952 953 func (b *cancelTimerBody) Read(p []byte) (n int, err error) { 954 n, err = b.rc.Read(p) 955 if err == nil { 956 return n, nil 957 } 958 if err == io.EOF { 959 return n, err 960 } 961 if b.reqDidTimeout() { 962 err = &httpError{ 963 err: err.Error() + " (Client.Timeout or context cancellation while reading body)", 964 timeout: true, 965 } 966 } 967 return n, err 968 } 969 970 func (b *cancelTimerBody) Close() error { 971 err := b.rc.Close() 972 b.stop() 973 return err 974 } 975 976 func shouldCopyHeaderOnRedirect(headerKey string, initial, dest *url.URL) bool { 977 switch CanonicalHeaderKey(headerKey) { 978 case "Authorization", "Www-Authenticate", "Cookie", "Cookie2": 979 // Permit sending auth/cookie headers from "foo.com" 980 // to "sub.foo.com". 981 982 // Note that we don't send all cookies to subdomains 983 // automatically. This function is only used for 984 // Cookies set explicitly on the initial outgoing 985 // client request. Cookies automatically added via the 986 // CookieJar mechanism continue to follow each 987 // cookie's scope as set by Set-Cookie. But for 988 // outgoing requests with the Cookie header set 989 // directly, we don't know their scope, so we assume 990 // it's for *.domain.com. 991 992 ihost := canonicalAddr(initial) 993 dhost := canonicalAddr(dest) 994 return isDomainOrSubdomain(dhost, ihost) 995 } 996 // All other headers are copied: 997 return true 998 } 999 1000 // isDomainOrSubdomain reports whether sub is a subdomain (or exact 1001 // match) of the parent domain. 1002 // 1003 // Both domains must already be in canonical form. 1004 func isDomainOrSubdomain(sub, parent string) bool { 1005 if sub == parent { 1006 return true 1007 } 1008 // If sub is "foo.example.com" and parent is "example.com", 1009 // that means sub must end in "."+parent. 1010 // Do it without allocating. 1011 if !strings.HasSuffix(sub, parent) { 1012 return false 1013 } 1014 return sub[len(sub)-len(parent)-1] == '.' 1015 } 1016 1017 func stripPassword(u *url.URL) string { 1018 _, passSet := u.User.Password() 1019 if passSet { 1020 return strings.Replace(u.String(), u.User.String()+"@", u.User.Username()+":***@", 1) 1021 } 1022 return u.String() 1023 } 1024