...
Run Format

Source file src/net/http/client.go

Documentation: net/http

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // HTTP client. See RFC 7230 through 7235.
     6  //
     7  // This is the high-level Client interface.
     8  // The low-level implementation is in transport.go.
     9  
    10  package http
    11  
    12  import (
    13  	"crypto/tls"
    14  	"encoding/base64"
    15  	"errors"
    16  	"fmt"
    17  	"io"
    18  	"io/ioutil"
    19  	"log"
    20  	"net/url"
    21  	"sort"
    22  	"strings"
    23  	"sync"
    24  	"time"
    25  )
    26  
    27  // A Client is an HTTP client. Its zero value (DefaultClient) is a
    28  // usable client that uses DefaultTransport.
    29  //
    30  // The Client's Transport typically has internal state (cached TCP
    31  // connections), so Clients should be reused instead of created as
    32  // needed. Clients are safe for concurrent use by multiple goroutines.
    33  //
    34  // A Client is higher-level than a RoundTripper (such as Transport)
    35  // and additionally handles HTTP details such as cookies and
    36  // redirects.
    37  //
    38  // When following redirects, the Client will forward all headers set on the
    39  // initial Request except:
    40  //
    41  // • when forwarding sensitive headers like "Authorization",
    42  // "WWW-Authenticate", and "Cookie" to untrusted targets.
    43  // These headers will be ignored when following a redirect to a domain
    44  // that is not a subdomain match or exact match of the initial domain.
    45  // For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com"
    46  // will forward the sensitive headers, but a redirect to "bar.com" will not.
    47  //
    48  // • when forwarding the "Cookie" header with a non-nil cookie Jar.
    49  // Since each redirect may mutate the state of the cookie jar,
    50  // a redirect may possibly alter a cookie set in the initial request.
    51  // When forwarding the "Cookie" header, any mutated cookies will be omitted,
    52  // with the expectation that the Jar will insert those mutated cookies
    53  // with the updated values (assuming the origin matches).
    54  // If Jar is nil, the initial cookies are forwarded without change.
    55  //
    56  type Client struct {
    57  	// Transport specifies the mechanism by which individual
    58  	// HTTP requests are made.
    59  	// If nil, DefaultTransport is used.
    60  	Transport RoundTripper
    61  
    62  	// CheckRedirect specifies the policy for handling redirects.
    63  	// If CheckRedirect is not nil, the client calls it before
    64  	// following an HTTP redirect. The arguments req and via are
    65  	// the upcoming request and the requests made already, oldest
    66  	// first. If CheckRedirect returns an error, the Client's Get
    67  	// method returns both the previous Response (with its Body
    68  	// closed) and CheckRedirect's error (wrapped in a url.Error)
    69  	// instead of issuing the Request req.
    70  	// As a special case, if CheckRedirect returns ErrUseLastResponse,
    71  	// then the most recent response is returned with its body
    72  	// unclosed, along with a nil error.
    73  	//
    74  	// If CheckRedirect is nil, the Client uses its default policy,
    75  	// which is to stop after 10 consecutive requests.
    76  	CheckRedirect func(req *Request, via []*Request) error
    77  
    78  	// Jar specifies the cookie jar.
    79  	//
    80  	// The Jar is used to insert relevant cookies into every
    81  	// outbound Request and is updated with the cookie values
    82  	// of every inbound Response. The Jar is consulted for every
    83  	// redirect that the Client follows.
    84  	//
    85  	// If Jar is nil, cookies are only sent if they are explicitly
    86  	// set on the Request.
    87  	Jar CookieJar
    88  
    89  	// Timeout specifies a time limit for requests made by this
    90  	// Client. The timeout includes connection time, any
    91  	// redirects, and reading the response body. The timer remains
    92  	// running after Get, Head, Post, or Do return and will
    93  	// interrupt reading of the Response.Body.
    94  	//
    95  	// A Timeout of zero means no timeout.
    96  	//
    97  	// The Client cancels requests to the underlying Transport
    98  	// as if the Request's Context ended.
    99  	//
   100  	// For compatibility, the Client will also use the deprecated
   101  	// CancelRequest method on Transport if found. New
   102  	// RoundTripper implementations should use the Request's Context
   103  	// for cancelation instead of implementing CancelRequest.
   104  	Timeout time.Duration
   105  }
   106  
   107  // DefaultClient is the default Client and is used by Get, Head, and Post.
   108  var DefaultClient = &Client{}
   109  
   110  // RoundTripper is an interface representing the ability to execute a
   111  // single HTTP transaction, obtaining the Response for a given Request.
   112  //
   113  // A RoundTripper must be safe for concurrent use by multiple
   114  // goroutines.
   115  type RoundTripper interface {
   116  	// RoundTrip executes a single HTTP transaction, returning
   117  	// a Response for the provided Request.
   118  	//
   119  	// RoundTrip should not attempt to interpret the response. In
   120  	// particular, RoundTrip must return err == nil if it obtained
   121  	// a response, regardless of the response's HTTP status code.
   122  	// A non-nil err should be reserved for failure to obtain a
   123  	// response. Similarly, RoundTrip should not attempt to
   124  	// handle higher-level protocol details such as redirects,
   125  	// authentication, or cookies.
   126  	//
   127  	// RoundTrip should not modify the request, except for
   128  	// consuming and closing the Request's Body. RoundTrip may
   129  	// read fields of the request in a separate goroutine. Callers
   130  	// should not mutate or reuse the request until the Response's
   131  	// Body has been closed.
   132  	//
   133  	// RoundTrip must always close the body, including on errors,
   134  	// but depending on the implementation may do so in a separate
   135  	// goroutine even after RoundTrip returns. This means that
   136  	// callers wanting to reuse the body for subsequent requests
   137  	// must arrange to wait for the Close call before doing so.
   138  	//
   139  	// The Request's URL and Header fields must be initialized.
   140  	RoundTrip(*Request) (*Response, error)
   141  }
   142  
   143  // refererForURL returns a referer without any authentication info or
   144  // an empty string if lastReq scheme is https and newReq scheme is http.
   145  func refererForURL(lastReq, newReq *url.URL) string {
   146  	// https://tools.ietf.org/html/rfc7231#section-5.5.2
   147  	//   "Clients SHOULD NOT include a Referer header field in a
   148  	//    (non-secure) HTTP request if the referring page was
   149  	//    transferred with a secure protocol."
   150  	if lastReq.Scheme == "https" && newReq.Scheme == "http" {
   151  		return ""
   152  	}
   153  	referer := lastReq.String()
   154  	if lastReq.User != nil {
   155  		// This is not very efficient, but is the best we can
   156  		// do without:
   157  		// - introducing a new method on URL
   158  		// - creating a race condition
   159  		// - copying the URL struct manually, which would cause
   160  		//   maintenance problems down the line
   161  		auth := lastReq.User.String() + "@"
   162  		referer = strings.Replace(referer, auth, "", 1)
   163  	}
   164  	return referer
   165  }
   166  
   167  // didTimeout is non-nil only if err != nil.
   168  func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) {
   169  	if c.Jar != nil {
   170  		for _, cookie := range c.Jar.Cookies(req.URL) {
   171  			req.AddCookie(cookie)
   172  		}
   173  	}
   174  	resp, didTimeout, err = send(req, c.transport(), deadline)
   175  	if err != nil {
   176  		return nil, didTimeout, err
   177  	}
   178  	if c.Jar != nil {
   179  		if rc := resp.Cookies(); len(rc) > 0 {
   180  			c.Jar.SetCookies(req.URL, rc)
   181  		}
   182  	}
   183  	return resp, nil, nil
   184  }
   185  
   186  func (c *Client) deadline() time.Time {
   187  	if c.Timeout > 0 {
   188  		return time.Now().Add(c.Timeout)
   189  	}
   190  	return time.Time{}
   191  }
   192  
   193  func (c *Client) transport() RoundTripper {
   194  	if c.Transport != nil {
   195  		return c.Transport
   196  	}
   197  	return DefaultTransport
   198  }
   199  
   200  // send issues an HTTP request.
   201  // Caller should close resp.Body when done reading from it.
   202  func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) {
   203  	req := ireq // req is either the original request, or a modified fork
   204  
   205  	if rt == nil {
   206  		req.closeBody()
   207  		return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport")
   208  	}
   209  
   210  	if req.URL == nil {
   211  		req.closeBody()
   212  		return nil, alwaysFalse, errors.New("http: nil Request.URL")
   213  	}
   214  
   215  	if req.RequestURI != "" {
   216  		req.closeBody()
   217  		return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests.")
   218  	}
   219  
   220  	// forkReq forks req into a shallow clone of ireq the first
   221  	// time it's called.
   222  	forkReq := func() {
   223  		if ireq == req {
   224  			req = new(Request)
   225  			*req = *ireq // shallow clone
   226  		}
   227  	}
   228  
   229  	// Most the callers of send (Get, Post, et al) don't need
   230  	// Headers, leaving it uninitialized. We guarantee to the
   231  	// Transport that this has been initialized, though.
   232  	if req.Header == nil {
   233  		forkReq()
   234  		req.Header = make(Header)
   235  	}
   236  
   237  	if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" {
   238  		username := u.Username()
   239  		password, _ := u.Password()
   240  		forkReq()
   241  		req.Header = cloneHeader(ireq.Header)
   242  		req.Header.Set("Authorization", "Basic "+basicAuth(username, password))
   243  	}
   244  
   245  	if !deadline.IsZero() {
   246  		forkReq()
   247  	}
   248  	stopTimer, didTimeout := setRequestCancel(req, rt, deadline)
   249  
   250  	resp, err = rt.RoundTrip(req)
   251  	if err != nil {
   252  		stopTimer()
   253  		if resp != nil {
   254  			log.Printf("RoundTripper returned a response & error; ignoring response")
   255  		}
   256  		if tlsErr, ok := err.(tls.RecordHeaderError); ok {
   257  			// If we get a bad TLS record header, check to see if the
   258  			// response looks like HTTP and give a more helpful error.
   259  			// See golang.org/issue/11111.
   260  			if string(tlsErr.RecordHeader[:]) == "HTTP/" {
   261  				err = errors.New("http: server gave HTTP response to HTTPS client")
   262  			}
   263  		}
   264  		return nil, didTimeout, err
   265  	}
   266  	if !deadline.IsZero() {
   267  		resp.Body = &cancelTimerBody{
   268  			stop:          stopTimer,
   269  			rc:            resp.Body,
   270  			reqDidTimeout: didTimeout,
   271  		}
   272  	}
   273  	return resp, nil, nil
   274  }
   275  
   276  // setRequestCancel sets the Cancel field of req, if deadline is
   277  // non-zero. The RoundTripper's type is used to determine whether the legacy
   278  // CancelRequest behavior should be used.
   279  //
   280  // As background, there are three ways to cancel a request:
   281  // First was Transport.CancelRequest. (deprecated)
   282  // Second was Request.Cancel (this mechanism).
   283  // Third was Request.Context.
   284  func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) {
   285  	if deadline.IsZero() {
   286  		return nop, alwaysFalse
   287  	}
   288  
   289  	initialReqCancel := req.Cancel // the user's original Request.Cancel, if any
   290  
   291  	cancel := make(chan struct{})
   292  	req.Cancel = cancel
   293  
   294  	doCancel := func() {
   295  		// The newer way (the second way in the func comment):
   296  		close(cancel)
   297  
   298  		// The legacy compatibility way, used only
   299  		// for RoundTripper implementations written
   300  		// before Go 1.5 or Go 1.6.
   301  		type canceler interface {
   302  			CancelRequest(*Request)
   303  		}
   304  		switch v := rt.(type) {
   305  		case *Transport, *http2Transport:
   306  			// Do nothing. The net/http package's transports
   307  			// support the new Request.Cancel channel
   308  		case canceler:
   309  			v.CancelRequest(req)
   310  		}
   311  	}
   312  
   313  	stopTimerCh := make(chan struct{})
   314  	var once sync.Once
   315  	stopTimer = func() { once.Do(func() { close(stopTimerCh) }) }
   316  
   317  	timer := time.NewTimer(time.Until(deadline))
   318  	var timedOut atomicBool
   319  
   320  	go func() {
   321  		select {
   322  		case <-initialReqCancel:
   323  			doCancel()
   324  			timer.Stop()
   325  		case <-timer.C:
   326  			timedOut.setTrue()
   327  			doCancel()
   328  		case <-stopTimerCh:
   329  			timer.Stop()
   330  		}
   331  	}()
   332  
   333  	return stopTimer, timedOut.isSet
   334  }
   335  
   336  // See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt
   337  // "To receive authorization, the client sends the userid and password,
   338  // separated by a single colon (":") character, within a base64
   339  // encoded string in the credentials."
   340  // It is not meant to be urlencoded.
   341  func basicAuth(username, password string) string {
   342  	auth := username + ":" + password
   343  	return base64.StdEncoding.EncodeToString([]byte(auth))
   344  }
   345  
   346  // Get issues a GET to the specified URL. If the response is one of
   347  // the following redirect codes, Get follows the redirect, up to a
   348  // maximum of 10 redirects:
   349  //
   350  //    301 (Moved Permanently)
   351  //    302 (Found)
   352  //    303 (See Other)
   353  //    307 (Temporary Redirect)
   354  //    308 (Permanent Redirect)
   355  //
   356  // An error is returned if there were too many redirects or if there
   357  // was an HTTP protocol error. A non-2xx response doesn't cause an
   358  // error. Any returned error will be of type *url.Error. The url.Error
   359  // value's Timeout method will report true if request timed out or was
   360  // canceled.
   361  //
   362  // When err is nil, resp always contains a non-nil resp.Body.
   363  // Caller should close resp.Body when done reading from it.
   364  //
   365  // Get is a wrapper around DefaultClient.Get.
   366  //
   367  // To make a request with custom headers, use NewRequest and
   368  // DefaultClient.Do.
   369  func Get(url string) (resp *Response, err error) {
   370  	return DefaultClient.Get(url)
   371  }
   372  
   373  // Get issues a GET to the specified URL. If the response is one of the
   374  // following redirect codes, Get follows the redirect after calling the
   375  // Client's CheckRedirect function:
   376  //
   377  //    301 (Moved Permanently)
   378  //    302 (Found)
   379  //    303 (See Other)
   380  //    307 (Temporary Redirect)
   381  //    308 (Permanent Redirect)
   382  //
   383  // An error is returned if the Client's CheckRedirect function fails
   384  // or if there was an HTTP protocol error. A non-2xx response doesn't
   385  // cause an error. Any returned error will be of type *url.Error. The
   386  // url.Error value's Timeout method will report true if request timed
   387  // out or was canceled.
   388  //
   389  // When err is nil, resp always contains a non-nil resp.Body.
   390  // Caller should close resp.Body when done reading from it.
   391  //
   392  // To make a request with custom headers, use NewRequest and Client.Do.
   393  func (c *Client) Get(url string) (resp *Response, err error) {
   394  	req, err := NewRequest("GET", url, nil)
   395  	if err != nil {
   396  		return nil, err
   397  	}
   398  	return c.Do(req)
   399  }
   400  
   401  func alwaysFalse() bool { return false }
   402  
   403  // ErrUseLastResponse can be returned by Client.CheckRedirect hooks to
   404  // control how redirects are processed. If returned, the next request
   405  // is not sent and the most recent response is returned with its body
   406  // unclosed.
   407  var ErrUseLastResponse = errors.New("net/http: use last response")
   408  
   409  // checkRedirect calls either the user's configured CheckRedirect
   410  // function, or the default.
   411  func (c *Client) checkRedirect(req *Request, via []*Request) error {
   412  	fn := c.CheckRedirect
   413  	if fn == nil {
   414  		fn = defaultCheckRedirect
   415  	}
   416  	return fn(req, via)
   417  }
   418  
   419  // redirectBehavior describes what should happen when the
   420  // client encounters a 3xx status code from the server
   421  func redirectBehavior(reqMethod string, resp *Response, ireq *Request) (redirectMethod string, shouldRedirect, includeBody bool) {
   422  	switch resp.StatusCode {
   423  	case 301, 302, 303:
   424  		redirectMethod = reqMethod
   425  		shouldRedirect = true
   426  		includeBody = false
   427  
   428  		// RFC 2616 allowed automatic redirection only with GET and
   429  		// HEAD requests. RFC 7231 lifts this restriction, but we still
   430  		// restrict other methods to GET to maintain compatibility.
   431  		// See Issue 18570.
   432  		if reqMethod != "GET" && reqMethod != "HEAD" {
   433  			redirectMethod = "GET"
   434  		}
   435  	case 307, 308:
   436  		redirectMethod = reqMethod
   437  		shouldRedirect = true
   438  		includeBody = true
   439  
   440  		// Treat 307 and 308 specially, since they're new in
   441  		// Go 1.8, and they also require re-sending the request body.
   442  		if resp.Header.Get("Location") == "" {
   443  			// 308s have been observed in the wild being served
   444  			// without Location headers. Since Go 1.7 and earlier
   445  			// didn't follow these codes, just stop here instead
   446  			// of returning an error.
   447  			// See Issue 17773.
   448  			shouldRedirect = false
   449  			break
   450  		}
   451  		if ireq.GetBody == nil && ireq.outgoingLength() != 0 {
   452  			// We had a request body, and 307/308 require
   453  			// re-sending it, but GetBody is not defined. So just
   454  			// return this response to the user instead of an
   455  			// error, like we did in Go 1.7 and earlier.
   456  			shouldRedirect = false
   457  		}
   458  	}
   459  	return redirectMethod, shouldRedirect, includeBody
   460  }
   461  
   462  // urlErrorOp returns the (*url.Error).Op value to use for the
   463  // provided (*Request).Method value.
   464  func urlErrorOp(method string) string {
   465  	if method == "" {
   466  		return "Get"
   467  	}
   468  	return method[:1] + strings.ToLower(method[1:])
   469  }
   470  
   471  // Do sends an HTTP request and returns an HTTP response, following
   472  // policy (such as redirects, cookies, auth) as configured on the
   473  // client.
   474  //
   475  // An error is returned if caused by client policy (such as
   476  // CheckRedirect), or failure to speak HTTP (such as a network
   477  // connectivity problem). A non-2xx status code doesn't cause an
   478  // error.
   479  //
   480  // If the returned error is nil, the Response will contain a non-nil
   481  // Body which the user is expected to close. If the Body is not
   482  // closed, the Client's underlying RoundTripper (typically Transport)
   483  // may not be able to re-use a persistent TCP connection to the server
   484  // for a subsequent "keep-alive" request.
   485  //
   486  // The request Body, if non-nil, will be closed by the underlying
   487  // Transport, even on errors.
   488  //
   489  // On error, any Response can be ignored. A non-nil Response with a
   490  // non-nil error only occurs when CheckRedirect fails, and even then
   491  // the returned Response.Body is already closed.
   492  //
   493  // Generally Get, Post, or PostForm will be used instead of Do.
   494  //
   495  // If the server replies with a redirect, the Client first uses the
   496  // CheckRedirect function to determine whether the redirect should be
   497  // followed. If permitted, a 301, 302, or 303 redirect causes
   498  // subsequent requests to use HTTP method GET
   499  // (or HEAD if the original request was HEAD), with no body.
   500  // A 307 or 308 redirect preserves the original HTTP method and body,
   501  // provided that the Request.GetBody function is defined.
   502  // The NewRequest function automatically sets GetBody for common
   503  // standard library body types.
   504  //
   505  // Any returned error will be of type *url.Error. The url.Error
   506  // value's Timeout method will report true if request timed out or was
   507  // canceled.
   508  func (c *Client) Do(req *Request) (*Response, error) {
   509  	return c.do(req)
   510  }
   511  
   512  var testHookClientDoResult func(retres *Response, reterr error)
   513  
   514  func (c *Client) do(req *Request) (retres *Response, reterr error) {
   515  	if testHookClientDoResult != nil {
   516  		defer func() { testHookClientDoResult(retres, reterr) }()
   517  	}
   518  	if req.URL == nil {
   519  		req.closeBody()
   520  		return nil, &url.Error{
   521  			Op:  urlErrorOp(req.Method),
   522  			Err: errors.New("http: nil Request.URL"),
   523  		}
   524  	}
   525  
   526  	var (
   527  		deadline      = c.deadline()
   528  		reqs          []*Request
   529  		resp          *Response
   530  		copyHeaders   = c.makeHeadersCopier(req)
   531  		reqBodyClosed = false // have we closed the current req.Body?
   532  
   533  		// Redirect behavior:
   534  		redirectMethod string
   535  		includeBody    bool
   536  	)
   537  	uerr := func(err error) error {
   538  		// the body may have been closed already by c.send()
   539  		if !reqBodyClosed {
   540  			req.closeBody()
   541  		}
   542  		var urlStr string
   543  		if resp != nil && resp.Request != nil {
   544  			urlStr = stripPassword(resp.Request.URL)
   545  		} else {
   546  			urlStr = stripPassword(req.URL)
   547  		}
   548  		return &url.Error{
   549  			Op:  urlErrorOp(reqs[0].Method),
   550  			URL: urlStr,
   551  			Err: err,
   552  		}
   553  	}
   554  	for {
   555  		// For all but the first request, create the next
   556  		// request hop and replace req.
   557  		if len(reqs) > 0 {
   558  			loc := resp.Header.Get("Location")
   559  			if loc == "" {
   560  				resp.closeBody()
   561  				return nil, uerr(fmt.Errorf("%d response missing Location header", resp.StatusCode))
   562  			}
   563  			u, err := req.URL.Parse(loc)
   564  			if err != nil {
   565  				resp.closeBody()
   566  				return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err))
   567  			}
   568  			host := ""
   569  			if req.Host != "" && req.Host != req.URL.Host {
   570  				// If the caller specified a custom Host header and the
   571  				// redirect location is relative, preserve the Host header
   572  				// through the redirect. See issue #22233.
   573  				if u, _ := url.Parse(loc); u != nil && !u.IsAbs() {
   574  					host = req.Host
   575  				}
   576  			}
   577  			ireq := reqs[0]
   578  			req = &Request{
   579  				Method:   redirectMethod,
   580  				Response: resp,
   581  				URL:      u,
   582  				Header:   make(Header),
   583  				Host:     host,
   584  				Cancel:   ireq.Cancel,
   585  				ctx:      ireq.ctx,
   586  			}
   587  			if includeBody && ireq.GetBody != nil {
   588  				req.Body, err = ireq.GetBody()
   589  				if err != nil {
   590  					resp.closeBody()
   591  					return nil, uerr(err)
   592  				}
   593  				req.ContentLength = ireq.ContentLength
   594  			}
   595  
   596  			// Copy original headers before setting the Referer,
   597  			// in case the user set Referer on their first request.
   598  			// If they really want to override, they can do it in
   599  			// their CheckRedirect func.
   600  			copyHeaders(req)
   601  
   602  			// Add the Referer header from the most recent
   603  			// request URL to the new one, if it's not https->http:
   604  			if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL); ref != "" {
   605  				req.Header.Set("Referer", ref)
   606  			}
   607  			err = c.checkRedirect(req, reqs)
   608  
   609  			// Sentinel error to let users select the
   610  			// previous response, without closing its
   611  			// body. See Issue 10069.
   612  			if err == ErrUseLastResponse {
   613  				return resp, nil
   614  			}
   615  
   616  			// Close the previous response's body. But
   617  			// read at least some of the body so if it's
   618  			// small the underlying TCP connection will be
   619  			// re-used. No need to check for errors: if it
   620  			// fails, the Transport won't reuse it anyway.
   621  			const maxBodySlurpSize = 2 << 10
   622  			if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize {
   623  				io.CopyN(ioutil.Discard, resp.Body, maxBodySlurpSize)
   624  			}
   625  			resp.Body.Close()
   626  
   627  			if err != nil {
   628  				// Special case for Go 1 compatibility: return both the response
   629  				// and an error if the CheckRedirect function failed.
   630  				// See https://golang.org/issue/3795
   631  				// The resp.Body has already been closed.
   632  				ue := uerr(err)
   633  				ue.(*url.Error).URL = loc
   634  				return resp, ue
   635  			}
   636  		}
   637  
   638  		reqs = append(reqs, req)
   639  		var err error
   640  		var didTimeout func() bool
   641  		if resp, didTimeout, err = c.send(req, deadline); err != nil {
   642  			// c.send() always closes req.Body
   643  			reqBodyClosed = true
   644  			if !deadline.IsZero() && didTimeout() {
   645  				err = &httpError{
   646  					// TODO: early in cycle: s/Client.Timeout exceeded/timeout or context cancelation/
   647  					err:     err.Error() + " (Client.Timeout exceeded while awaiting headers)",
   648  					timeout: true,
   649  				}
   650  			}
   651  			return nil, uerr(err)
   652  		}
   653  
   654  		var shouldRedirect bool
   655  		redirectMethod, shouldRedirect, includeBody = redirectBehavior(req.Method, resp, reqs[0])
   656  		if !shouldRedirect {
   657  			return resp, nil
   658  		}
   659  
   660  		req.closeBody()
   661  	}
   662  }
   663  
   664  // makeHeadersCopier makes a function that copies headers from the
   665  // initial Request, ireq. For every redirect, this function must be called
   666  // so that it can copy headers into the upcoming Request.
   667  func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) {
   668  	// The headers to copy are from the very initial request.
   669  	// We use a closured callback to keep a reference to these original headers.
   670  	var (
   671  		ireqhdr  = ireq.Header.clone()
   672  		icookies map[string][]*Cookie
   673  	)
   674  	if c.Jar != nil && ireq.Header.Get("Cookie") != "" {
   675  		icookies = make(map[string][]*Cookie)
   676  		for _, c := range ireq.Cookies() {
   677  			icookies[c.Name] = append(icookies[c.Name], c)
   678  		}
   679  	}
   680  
   681  	preq := ireq // The previous request
   682  	return func(req *Request) {
   683  		// If Jar is present and there was some initial cookies provided
   684  		// via the request header, then we may need to alter the initial
   685  		// cookies as we follow redirects since each redirect may end up
   686  		// modifying a pre-existing cookie.
   687  		//
   688  		// Since cookies already set in the request header do not contain
   689  		// information about the original domain and path, the logic below
   690  		// assumes any new set cookies override the original cookie
   691  		// regardless of domain or path.
   692  		//
   693  		// See https://golang.org/issue/17494
   694  		if c.Jar != nil && icookies != nil {
   695  			var changed bool
   696  			resp := req.Response // The response that caused the upcoming redirect
   697  			for _, c := range resp.Cookies() {
   698  				if _, ok := icookies[c.Name]; ok {
   699  					delete(icookies, c.Name)
   700  					changed = true
   701  				}
   702  			}
   703  			if changed {
   704  				ireqhdr.Del("Cookie")
   705  				var ss []string
   706  				for _, cs := range icookies {
   707  					for _, c := range cs {
   708  						ss = append(ss, c.Name+"="+c.Value)
   709  					}
   710  				}
   711  				sort.Strings(ss) // Ensure deterministic headers
   712  				ireqhdr.Set("Cookie", strings.Join(ss, "; "))
   713  			}
   714  		}
   715  
   716  		// Copy the initial request's Header values
   717  		// (at least the safe ones).
   718  		for k, vv := range ireqhdr {
   719  			if shouldCopyHeaderOnRedirect(k, preq.URL, req.URL) {
   720  				req.Header[k] = vv
   721  			}
   722  		}
   723  
   724  		preq = req // Update previous Request with the current request
   725  	}
   726  }
   727  
   728  func defaultCheckRedirect(req *Request, via []*Request) error {
   729  	if len(via) >= 10 {
   730  		return errors.New("stopped after 10 redirects")
   731  	}
   732  	return nil
   733  }
   734  
   735  // Post issues a POST to the specified URL.
   736  //
   737  // Caller should close resp.Body when done reading from it.
   738  //
   739  // If the provided body is an io.Closer, it is closed after the
   740  // request.
   741  //
   742  // Post is a wrapper around DefaultClient.Post.
   743  //
   744  // To set custom headers, use NewRequest and DefaultClient.Do.
   745  //
   746  // See the Client.Do method documentation for details on how redirects
   747  // are handled.
   748  func Post(url, contentType string, body io.Reader) (resp *Response, err error) {
   749  	return DefaultClient.Post(url, contentType, body)
   750  }
   751  
   752  // Post issues a POST to the specified URL.
   753  //
   754  // Caller should close resp.Body when done reading from it.
   755  //
   756  // If the provided body is an io.Closer, it is closed after the
   757  // request.
   758  //
   759  // To set custom headers, use NewRequest and Client.Do.
   760  //
   761  // See the Client.Do method documentation for details on how redirects
   762  // are handled.
   763  func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) {
   764  	req, err := NewRequest("POST", url, body)
   765  	if err != nil {
   766  		return nil, err
   767  	}
   768  	req.Header.Set("Content-Type", contentType)
   769  	return c.Do(req)
   770  }
   771  
   772  // PostForm issues a POST to the specified URL, with data's keys and
   773  // values URL-encoded as the request body.
   774  //
   775  // The Content-Type header is set to application/x-www-form-urlencoded.
   776  // To set other headers, use NewRequest and DefaultClient.Do.
   777  //
   778  // When err is nil, resp always contains a non-nil resp.Body.
   779  // Caller should close resp.Body when done reading from it.
   780  //
   781  // PostForm is a wrapper around DefaultClient.PostForm.
   782  //
   783  // See the Client.Do method documentation for details on how redirects
   784  // are handled.
   785  func PostForm(url string, data url.Values) (resp *Response, err error) {
   786  	return DefaultClient.PostForm(url, data)
   787  }
   788  
   789  // PostForm issues a POST to the specified URL,
   790  // with data's keys and values URL-encoded as the request body.
   791  //
   792  // The Content-Type header is set to application/x-www-form-urlencoded.
   793  // To set other headers, use NewRequest and Client.Do.
   794  //
   795  // When err is nil, resp always contains a non-nil resp.Body.
   796  // Caller should close resp.Body when done reading from it.
   797  //
   798  // See the Client.Do method documentation for details on how redirects
   799  // are handled.
   800  func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) {
   801  	return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
   802  }
   803  
   804  // Head issues a HEAD to the specified URL. If the response is one of
   805  // the following redirect codes, Head follows the redirect, up to a
   806  // maximum of 10 redirects:
   807  //
   808  //    301 (Moved Permanently)
   809  //    302 (Found)
   810  //    303 (See Other)
   811  //    307 (Temporary Redirect)
   812  //    308 (Permanent Redirect)
   813  //
   814  // Head is a wrapper around DefaultClient.Head
   815  func Head(url string) (resp *Response, err error) {
   816  	return DefaultClient.Head(url)
   817  }
   818  
   819  // Head issues a HEAD to the specified URL. If the response is one of the
   820  // following redirect codes, Head follows the redirect after calling the
   821  // Client's CheckRedirect function:
   822  //
   823  //    301 (Moved Permanently)
   824  //    302 (Found)
   825  //    303 (See Other)
   826  //    307 (Temporary Redirect)
   827  //    308 (Permanent Redirect)
   828  func (c *Client) Head(url string) (resp *Response, err error) {
   829  	req, err := NewRequest("HEAD", url, nil)
   830  	if err != nil {
   831  		return nil, err
   832  	}
   833  	return c.Do(req)
   834  }
   835  
   836  // cancelTimerBody is an io.ReadCloser that wraps rc with two features:
   837  // 1) on Read error or close, the stop func is called.
   838  // 2) On Read failure, if reqDidTimeout is true, the error is wrapped and
   839  //    marked as net.Error that hit its timeout.
   840  type cancelTimerBody struct {
   841  	stop          func() // stops the time.Timer waiting to cancel the request
   842  	rc            io.ReadCloser
   843  	reqDidTimeout func() bool
   844  }
   845  
   846  func (b *cancelTimerBody) Read(p []byte) (n int, err error) {
   847  	n, err = b.rc.Read(p)
   848  	if err == nil {
   849  		return n, nil
   850  	}
   851  	b.stop()
   852  	if err == io.EOF {
   853  		return n, err
   854  	}
   855  	if b.reqDidTimeout() {
   856  		err = &httpError{
   857  			// TODO: early in cycle: s/Client.Timeout exceeded/timeout or context cancelation/
   858  			err:     err.Error() + " (Client.Timeout exceeded while reading body)",
   859  			timeout: true,
   860  		}
   861  	}
   862  	return n, err
   863  }
   864  
   865  func (b *cancelTimerBody) Close() error {
   866  	err := b.rc.Close()
   867  	b.stop()
   868  	return err
   869  }
   870  
   871  func shouldCopyHeaderOnRedirect(headerKey string, initial, dest *url.URL) bool {
   872  	switch CanonicalHeaderKey(headerKey) {
   873  	case "Authorization", "Www-Authenticate", "Cookie", "Cookie2":
   874  		// Permit sending auth/cookie headers from "foo.com"
   875  		// to "sub.foo.com".
   876  
   877  		// Note that we don't send all cookies to subdomains
   878  		// automatically. This function is only used for
   879  		// Cookies set explicitly on the initial outgoing
   880  		// client request. Cookies automatically added via the
   881  		// CookieJar mechanism continue to follow each
   882  		// cookie's scope as set by Set-Cookie. But for
   883  		// outgoing requests with the Cookie header set
   884  		// directly, we don't know their scope, so we assume
   885  		// it's for *.domain.com.
   886  
   887  		ihost := canonicalAddr(initial)
   888  		dhost := canonicalAddr(dest)
   889  		return isDomainOrSubdomain(dhost, ihost)
   890  	}
   891  	// All other headers are copied:
   892  	return true
   893  }
   894  
   895  // isDomainOrSubdomain reports whether sub is a subdomain (or exact
   896  // match) of the parent domain.
   897  //
   898  // Both domains must already be in canonical form.
   899  func isDomainOrSubdomain(sub, parent string) bool {
   900  	if sub == parent {
   901  		return true
   902  	}
   903  	// If sub is "foo.example.com" and parent is "example.com",
   904  	// that means sub must end in "."+parent.
   905  	// Do it without allocating.
   906  	if !strings.HasSuffix(sub, parent) {
   907  		return false
   908  	}
   909  	return sub[len(sub)-len(parent)-1] == '.'
   910  }
   911  
   912  func stripPassword(u *url.URL) string {
   913  	pass, passSet := u.User.Password()
   914  	if passSet {
   915  		return strings.Replace(u.String(), pass+"@", "***@", 1)
   916  	}
   917  
   918  	return u.String()
   919  }
   920  

View as plain text