...
Run Format

Source file src/net/http/client.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// HTTP client. See RFC 2616.
     6	//
     7	// This is the high-level Client interface.
     8	// The low-level implementation is in transport.go.
     9	
    10	package http
    11	
    12	import (
    13		"crypto/tls"
    14		"encoding/base64"
    15		"errors"
    16		"fmt"
    17		"io"
    18		"io/ioutil"
    19		"log"
    20		"net/url"
    21		"strings"
    22		"sync"
    23		"time"
    24	)
    25	
    26	// A Client is an HTTP client. Its zero value (DefaultClient) is a
    27	// usable client that uses DefaultTransport.
    28	//
    29	// The Client's Transport typically has internal state (cached TCP
    30	// connections), so Clients should be reused instead of created as
    31	// needed. Clients are safe for concurrent use by multiple goroutines.
    32	//
    33	// A Client is higher-level than a RoundTripper (such as Transport)
    34	// and additionally handles HTTP details such as cookies and
    35	// redirects.
    36	type Client struct {
    37		// Transport specifies the mechanism by which individual
    38		// HTTP requests are made.
    39		// If nil, DefaultTransport is used.
    40		Transport RoundTripper
    41	
    42		// CheckRedirect specifies the policy for handling redirects.
    43		// If CheckRedirect is not nil, the client calls it before
    44		// following an HTTP redirect. The arguments req and via are
    45		// the upcoming request and the requests made already, oldest
    46		// first. If CheckRedirect returns an error, the Client's Get
    47		// method returns both the previous Response (with its Body
    48		// closed) and CheckRedirect's error (wrapped in a url.Error)
    49		// instead of issuing the Request req.
    50		// As a special case, if CheckRedirect returns ErrUseLastResponse,
    51		// then the most recent response is returned with its body
    52		// unclosed, along with a nil error.
    53		//
    54		// If CheckRedirect is nil, the Client uses its default policy,
    55		// which is to stop after 10 consecutive requests.
    56		CheckRedirect func(req *Request, via []*Request) error
    57	
    58		// Jar specifies the cookie jar.
    59		// If Jar is nil, cookies are not sent in requests and ignored
    60		// in responses.
    61		Jar CookieJar
    62	
    63		// Timeout specifies a time limit for requests made by this
    64		// Client. The timeout includes connection time, any
    65		// redirects, and reading the response body. The timer remains
    66		// running after Get, Head, Post, or Do return and will
    67		// interrupt reading of the Response.Body.
    68		//
    69		// A Timeout of zero means no timeout.
    70		//
    71		// The Client cancels requests to the underlying Transport
    72		// using the Request.Cancel mechanism. Requests passed
    73		// to Client.Do may still set Request.Cancel; both will
    74		// cancel the request.
    75		//
    76		// For compatibility, the Client will also use the deprecated
    77		// CancelRequest method on Transport if found. New
    78		// RoundTripper implementations should use Request.Cancel
    79		// instead of implementing CancelRequest.
    80		Timeout time.Duration
    81	}
    82	
    83	// DefaultClient is the default Client and is used by Get, Head, and Post.
    84	var DefaultClient = &Client{}
    85	
    86	// RoundTripper is an interface representing the ability to execute a
    87	// single HTTP transaction, obtaining the Response for a given Request.
    88	//
    89	// A RoundTripper must be safe for concurrent use by multiple
    90	// goroutines.
    91	type RoundTripper interface {
    92		// RoundTrip executes a single HTTP transaction, returning
    93		// a Response for the provided Request.
    94		//
    95		// RoundTrip should not attempt to interpret the response. In
    96		// particular, RoundTrip must return err == nil if it obtained
    97		// a response, regardless of the response's HTTP status code.
    98		// A non-nil err should be reserved for failure to obtain a
    99		// response. Similarly, RoundTrip should not attempt to
   100		// handle higher-level protocol details such as redirects,
   101		// authentication, or cookies.
   102		//
   103		// RoundTrip should not modify the request, except for
   104		// consuming and closing the Request's Body.
   105		//
   106		// RoundTrip must always close the body, including on errors,
   107		// but depending on the implementation may do so in a separate
   108		// goroutine even after RoundTrip returns. This means that
   109		// callers wanting to reuse the body for subsequent requests
   110		// must arrange to wait for the Close call before doing so.
   111		//
   112		// The Request's URL and Header fields must be initialized.
   113		RoundTrip(*Request) (*Response, error)
   114	}
   115	
   116	// refererForURL returns a referer without any authentication info or
   117	// an empty string if lastReq scheme is https and newReq scheme is http.
   118	func refererForURL(lastReq, newReq *url.URL) string {
   119		// https://tools.ietf.org/html/rfc7231#section-5.5.2
   120		//   "Clients SHOULD NOT include a Referer header field in a
   121		//    (non-secure) HTTP request if the referring page was
   122		//    transferred with a secure protocol."
   123		if lastReq.Scheme == "https" && newReq.Scheme == "http" {
   124			return ""
   125		}
   126		referer := lastReq.String()
   127		if lastReq.User != nil {
   128			// This is not very efficient, but is the best we can
   129			// do without:
   130			// - introducing a new method on URL
   131			// - creating a race condition
   132			// - copying the URL struct manually, which would cause
   133			//   maintenance problems down the line
   134			auth := lastReq.User.String() + "@"
   135			referer = strings.Replace(referer, auth, "", 1)
   136		}
   137		return referer
   138	}
   139	
   140	func (c *Client) send(req *Request, deadline time.Time) (*Response, error) {
   141		if c.Jar != nil {
   142			for _, cookie := range c.Jar.Cookies(req.URL) {
   143				req.AddCookie(cookie)
   144			}
   145		}
   146		resp, err := send(req, c.transport(), deadline)
   147		if err != nil {
   148			return nil, err
   149		}
   150		if c.Jar != nil {
   151			if rc := resp.Cookies(); len(rc) > 0 {
   152				c.Jar.SetCookies(req.URL, rc)
   153			}
   154		}
   155		return resp, nil
   156	}
   157	
   158	// Do sends an HTTP request and returns an HTTP response, following
   159	// policy (such as redirects, cookies, auth) as configured on the
   160	// client.
   161	//
   162	// An error is returned if caused by client policy (such as
   163	// CheckRedirect), or failure to speak HTTP (such as a network
   164	// connectivity problem). A non-2xx status code doesn't cause an
   165	// error.
   166	//
   167	// If the returned error is nil, the Response will contain a non-nil
   168	// Body which the user is expected to close. If the Body is not
   169	// closed, the Client's underlying RoundTripper (typically Transport)
   170	// may not be able to re-use a persistent TCP connection to the server
   171	// for a subsequent "keep-alive" request.
   172	//
   173	// The request Body, if non-nil, will be closed by the underlying
   174	// Transport, even on errors.
   175	//
   176	// On error, any Response can be ignored. A non-nil Response with a
   177	// non-nil error only occurs when CheckRedirect fails, and even then
   178	// the returned Response.Body is already closed.
   179	//
   180	// Generally Get, Post, or PostForm will be used instead of Do.
   181	func (c *Client) Do(req *Request) (*Response, error) {
   182		method := valueOrDefault(req.Method, "GET")
   183		if method == "GET" || method == "HEAD" {
   184			return c.doFollowingRedirects(req, shouldRedirectGet)
   185		}
   186		if method == "POST" || method == "PUT" {
   187			return c.doFollowingRedirects(req, shouldRedirectPost)
   188		}
   189		return c.send(req, c.deadline())
   190	}
   191	
   192	func (c *Client) deadline() time.Time {
   193		if c.Timeout > 0 {
   194			return time.Now().Add(c.Timeout)
   195		}
   196		return time.Time{}
   197	}
   198	
   199	func (c *Client) transport() RoundTripper {
   200		if c.Transport != nil {
   201			return c.Transport
   202		}
   203		return DefaultTransport
   204	}
   205	
   206	// send issues an HTTP request.
   207	// Caller should close resp.Body when done reading from it.
   208	func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) {
   209		req := ireq // req is either the original request, or a modified fork
   210	
   211		if rt == nil {
   212			req.closeBody()
   213			return nil, errors.New("http: no Client.Transport or DefaultTransport")
   214		}
   215	
   216		if req.URL == nil {
   217			req.closeBody()
   218			return nil, errors.New("http: nil Request.URL")
   219		}
   220	
   221		if req.RequestURI != "" {
   222			req.closeBody()
   223			return nil, errors.New("http: Request.RequestURI can't be set in client requests.")
   224		}
   225	
   226		// forkReq forks req into a shallow clone of ireq the first
   227		// time it's called.
   228		forkReq := func() {
   229			if ireq == req {
   230				req = new(Request)
   231				*req = *ireq // shallow clone
   232			}
   233		}
   234	
   235		// Most the callers of send (Get, Post, et al) don't need
   236		// Headers, leaving it uninitialized. We guarantee to the
   237		// Transport that this has been initialized, though.
   238		if req.Header == nil {
   239			forkReq()
   240			req.Header = make(Header)
   241		}
   242	
   243		if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" {
   244			username := u.Username()
   245			password, _ := u.Password()
   246			forkReq()
   247			req.Header = cloneHeader(ireq.Header)
   248			req.Header.Set("Authorization", "Basic "+basicAuth(username, password))
   249		}
   250	
   251		if !deadline.IsZero() {
   252			forkReq()
   253		}
   254		stopTimer, wasCanceled := setRequestCancel(req, rt, deadline)
   255	
   256		resp, err := rt.RoundTrip(req)
   257		if err != nil {
   258			stopTimer()
   259			if resp != nil {
   260				log.Printf("RoundTripper returned a response & error; ignoring response")
   261			}
   262			if tlsErr, ok := err.(tls.RecordHeaderError); ok {
   263				// If we get a bad TLS record header, check to see if the
   264				// response looks like HTTP and give a more helpful error.
   265				// See golang.org/issue/11111.
   266				if string(tlsErr.RecordHeader[:]) == "HTTP/" {
   267					err = errors.New("http: server gave HTTP response to HTTPS client")
   268				}
   269			}
   270			return nil, err
   271		}
   272		if !deadline.IsZero() {
   273			resp.Body = &cancelTimerBody{
   274				stop:           stopTimer,
   275				rc:             resp.Body,
   276				reqWasCanceled: wasCanceled,
   277			}
   278		}
   279		return resp, nil
   280	}
   281	
   282	// setRequestCancel sets the Cancel field of req, if deadline is
   283	// non-zero. The RoundTripper's type is used to determine whether the legacy
   284	// CancelRequest behavior should be used.
   285	func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), wasCanceled func() bool) {
   286		if deadline.IsZero() {
   287			return nop, alwaysFalse
   288		}
   289	
   290		initialReqCancel := req.Cancel // the user's original Request.Cancel, if any
   291	
   292		cancel := make(chan struct{})
   293		req.Cancel = cancel
   294	
   295		wasCanceled = func() bool {
   296			select {
   297			case <-cancel:
   298				return true
   299			default:
   300				return false
   301			}
   302		}
   303	
   304		doCancel := func() {
   305			// The new way:
   306			close(cancel)
   307	
   308			// The legacy compatibility way, used only
   309			// for RoundTripper implementations written
   310			// before Go 1.5 or Go 1.6.
   311			type canceler interface {
   312				CancelRequest(*Request)
   313			}
   314			switch v := rt.(type) {
   315			case *Transport, *http2Transport:
   316				// Do nothing. The net/http package's transports
   317				// support the new Request.Cancel channel
   318			case canceler:
   319				v.CancelRequest(req)
   320			}
   321		}
   322	
   323		stopTimerCh := make(chan struct{})
   324		var once sync.Once
   325		stopTimer = func() { once.Do(func() { close(stopTimerCh) }) }
   326	
   327		timer := time.NewTimer(deadline.Sub(time.Now()))
   328		go func() {
   329			select {
   330			case <-initialReqCancel:
   331				doCancel()
   332			case <-timer.C:
   333				doCancel()
   334			case <-stopTimerCh:
   335				timer.Stop()
   336			}
   337		}()
   338	
   339		return stopTimer, wasCanceled
   340	}
   341	
   342	// See 2 (end of page 4) http://www.ietf.org/rfc/rfc2617.txt
   343	// "To receive authorization, the client sends the userid and password,
   344	// separated by a single colon (":") character, within a base64
   345	// encoded string in the credentials."
   346	// It is not meant to be urlencoded.
   347	func basicAuth(username, password string) string {
   348		auth := username + ":" + password
   349		return base64.StdEncoding.EncodeToString([]byte(auth))
   350	}
   351	
   352	// True if the specified HTTP status code is one for which the Get utility should
   353	// automatically redirect.
   354	func shouldRedirectGet(statusCode int) bool {
   355		switch statusCode {
   356		case StatusMovedPermanently, StatusFound, StatusSeeOther, StatusTemporaryRedirect:
   357			return true
   358		}
   359		return false
   360	}
   361	
   362	// True if the specified HTTP status code is one for which the Post utility should
   363	// automatically redirect.
   364	func shouldRedirectPost(statusCode int) bool {
   365		switch statusCode {
   366		case StatusFound, StatusSeeOther:
   367			return true
   368		}
   369		return false
   370	}
   371	
   372	// Get issues a GET to the specified URL. If the response is one of
   373	// the following redirect codes, Get follows the redirect, up to a
   374	// maximum of 10 redirects:
   375	//
   376	//    301 (Moved Permanently)
   377	//    302 (Found)
   378	//    303 (See Other)
   379	//    307 (Temporary Redirect)
   380	//
   381	// An error is returned if there were too many redirects or if there
   382	// was an HTTP protocol error. A non-2xx response doesn't cause an
   383	// error.
   384	//
   385	// When err is nil, resp always contains a non-nil resp.Body.
   386	// Caller should close resp.Body when done reading from it.
   387	//
   388	// Get is a wrapper around DefaultClient.Get.
   389	//
   390	// To make a request with custom headers, use NewRequest and
   391	// DefaultClient.Do.
   392	func Get(url string) (resp *Response, err error) {
   393		return DefaultClient.Get(url)
   394	}
   395	
   396	// Get issues a GET to the specified URL. If the response is one of the
   397	// following redirect codes, Get follows the redirect after calling the
   398	// Client's CheckRedirect function:
   399	//
   400	//    301 (Moved Permanently)
   401	//    302 (Found)
   402	//    303 (See Other)
   403	//    307 (Temporary Redirect)
   404	//
   405	// An error is returned if the Client's CheckRedirect function fails
   406	// or if there was an HTTP protocol error. A non-2xx response doesn't
   407	// cause an error.
   408	//
   409	// When err is nil, resp always contains a non-nil resp.Body.
   410	// Caller should close resp.Body when done reading from it.
   411	//
   412	// To make a request with custom headers, use NewRequest and Client.Do.
   413	func (c *Client) Get(url string) (resp *Response, err error) {
   414		req, err := NewRequest("GET", url, nil)
   415		if err != nil {
   416			return nil, err
   417		}
   418		return c.doFollowingRedirects(req, shouldRedirectGet)
   419	}
   420	
   421	func alwaysFalse() bool { return false }
   422	
   423	// ErrUseLastResponse can be returned by Client.CheckRedirect hooks to
   424	// control how redirects are processed. If returned, the next request
   425	// is not sent and the most recent response is returned with its body
   426	// unclosed.
   427	var ErrUseLastResponse = errors.New("net/http: use last response")
   428	
   429	// checkRedirect calls either the user's configured CheckRedirect
   430	// function, or the default.
   431	func (c *Client) checkRedirect(req *Request, via []*Request) error {
   432		fn := c.CheckRedirect
   433		if fn == nil {
   434			fn = defaultCheckRedirect
   435		}
   436		return fn(req, via)
   437	}
   438	
   439	func (c *Client) doFollowingRedirects(req *Request, shouldRedirect func(int) bool) (*Response, error) {
   440		if req.URL == nil {
   441			req.closeBody()
   442			return nil, errors.New("http: nil Request.URL")
   443		}
   444	
   445		var (
   446			deadline = c.deadline()
   447			reqs     []*Request
   448			resp     *Response
   449		)
   450		uerr := func(err error) error {
   451			req.closeBody()
   452			method := valueOrDefault(reqs[0].Method, "GET")
   453			var urlStr string
   454			if resp != nil && resp.Request != nil {
   455				urlStr = resp.Request.URL.String()
   456			} else {
   457				urlStr = req.URL.String()
   458			}
   459			return &url.Error{
   460				Op:  method[:1] + strings.ToLower(method[1:]),
   461				URL: urlStr,
   462				Err: err,
   463			}
   464		}
   465		for {
   466			// For all but the first request, create the next
   467			// request hop and replace req.
   468			if len(reqs) > 0 {
   469				loc := resp.Header.Get("Location")
   470				if loc == "" {
   471					return nil, uerr(fmt.Errorf("%d response missing Location header", resp.StatusCode))
   472				}
   473				u, err := req.URL.Parse(loc)
   474				if err != nil {
   475					return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err))
   476				}
   477				ireq := reqs[0]
   478				req = &Request{
   479					Method:   ireq.Method,
   480					Response: resp,
   481					URL:      u,
   482					Header:   make(Header),
   483					Cancel:   ireq.Cancel,
   484					ctx:      ireq.ctx,
   485				}
   486				if ireq.Method == "POST" || ireq.Method == "PUT" {
   487					req.Method = "GET"
   488				}
   489				// Add the Referer header from the most recent
   490				// request URL to the new one, if it's not https->http:
   491				if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL); ref != "" {
   492					req.Header.Set("Referer", ref)
   493				}
   494				err = c.checkRedirect(req, reqs)
   495	
   496				// Sentinel error to let users select the
   497				// previous response, without closing its
   498				// body. See Issue 10069.
   499				if err == ErrUseLastResponse {
   500					return resp, nil
   501				}
   502	
   503				// Close the previous response's body. But
   504				// read at least some of the body so if it's
   505				// small the underlying TCP connection will be
   506				// re-used. No need to check for errors: if it
   507				// fails, the Transport won't reuse it anyway.
   508				const maxBodySlurpSize = 2 << 10
   509				if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize {
   510					io.CopyN(ioutil.Discard, resp.Body, maxBodySlurpSize)
   511				}
   512				resp.Body.Close()
   513	
   514				if err != nil {
   515					// Special case for Go 1 compatibility: return both the response
   516					// and an error if the CheckRedirect function failed.
   517					// See https://golang.org/issue/3795
   518					// The resp.Body has already been closed.
   519					ue := uerr(err)
   520					ue.(*url.Error).URL = loc
   521					return resp, ue
   522				}
   523			}
   524	
   525			reqs = append(reqs, req)
   526	
   527			var err error
   528			if resp, err = c.send(req, deadline); err != nil {
   529				if !deadline.IsZero() && !time.Now().Before(deadline) {
   530					err = &httpError{
   531						err:     err.Error() + " (Client.Timeout exceeded while awaiting headers)",
   532						timeout: true,
   533					}
   534				}
   535				return nil, uerr(err)
   536			}
   537	
   538			if !shouldRedirect(resp.StatusCode) {
   539				return resp, nil
   540			}
   541		}
   542	}
   543	
   544	func defaultCheckRedirect(req *Request, via []*Request) error {
   545		if len(via) >= 10 {
   546			return errors.New("stopped after 10 redirects")
   547		}
   548		return nil
   549	}
   550	
   551	// Post issues a POST to the specified URL.
   552	//
   553	// Caller should close resp.Body when done reading from it.
   554	//
   555	// If the provided body is an io.Closer, it is closed after the
   556	// request.
   557	//
   558	// Post is a wrapper around DefaultClient.Post.
   559	//
   560	// To set custom headers, use NewRequest and DefaultClient.Do.
   561	func Post(url string, bodyType string, body io.Reader) (resp *Response, err error) {
   562		return DefaultClient.Post(url, bodyType, body)
   563	}
   564	
   565	// Post issues a POST to the specified URL.
   566	//
   567	// Caller should close resp.Body when done reading from it.
   568	//
   569	// If the provided body is an io.Closer, it is closed after the
   570	// request.
   571	//
   572	// To set custom headers, use NewRequest and Client.Do.
   573	func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *Response, err error) {
   574		req, err := NewRequest("POST", url, body)
   575		if err != nil {
   576			return nil, err
   577		}
   578		req.Header.Set("Content-Type", bodyType)
   579		return c.doFollowingRedirects(req, shouldRedirectPost)
   580	}
   581	
   582	// PostForm issues a POST to the specified URL, with data's keys and
   583	// values URL-encoded as the request body.
   584	//
   585	// The Content-Type header is set to application/x-www-form-urlencoded.
   586	// To set other headers, use NewRequest and DefaultClient.Do.
   587	//
   588	// When err is nil, resp always contains a non-nil resp.Body.
   589	// Caller should close resp.Body when done reading from it.
   590	//
   591	// PostForm is a wrapper around DefaultClient.PostForm.
   592	func PostForm(url string, data url.Values) (resp *Response, err error) {
   593		return DefaultClient.PostForm(url, data)
   594	}
   595	
   596	// PostForm issues a POST to the specified URL,
   597	// with data's keys and values URL-encoded as the request body.
   598	//
   599	// The Content-Type header is set to application/x-www-form-urlencoded.
   600	// To set other headers, use NewRequest and DefaultClient.Do.
   601	//
   602	// When err is nil, resp always contains a non-nil resp.Body.
   603	// Caller should close resp.Body when done reading from it.
   604	func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) {
   605		return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
   606	}
   607	
   608	// Head issues a HEAD to the specified URL.  If the response is one of
   609	// the following redirect codes, Head follows the redirect, up to a
   610	// maximum of 10 redirects:
   611	//
   612	//    301 (Moved Permanently)
   613	//    302 (Found)
   614	//    303 (See Other)
   615	//    307 (Temporary Redirect)
   616	//
   617	// Head is a wrapper around DefaultClient.Head
   618	func Head(url string) (resp *Response, err error) {
   619		return DefaultClient.Head(url)
   620	}
   621	
   622	// Head issues a HEAD to the specified URL.  If the response is one of the
   623	// following redirect codes, Head follows the redirect after calling the
   624	// Client's CheckRedirect function:
   625	//
   626	//    301 (Moved Permanently)
   627	//    302 (Found)
   628	//    303 (See Other)
   629	//    307 (Temporary Redirect)
   630	func (c *Client) Head(url string) (resp *Response, err error) {
   631		req, err := NewRequest("HEAD", url, nil)
   632		if err != nil {
   633			return nil, err
   634		}
   635		return c.doFollowingRedirects(req, shouldRedirectGet)
   636	}
   637	
   638	// cancelTimerBody is an io.ReadCloser that wraps rc with two features:
   639	// 1) on Read error or close, the stop func is called.
   640	// 2) On Read failure, if reqWasCanceled is true, the error is wrapped and
   641	//    marked as net.Error that hit its timeout.
   642	type cancelTimerBody struct {
   643		stop           func() // stops the time.Timer waiting to cancel the request
   644		rc             io.ReadCloser
   645		reqWasCanceled func() bool
   646	}
   647	
   648	func (b *cancelTimerBody) Read(p []byte) (n int, err error) {
   649		n, err = b.rc.Read(p)
   650		if err == nil {
   651			return n, nil
   652		}
   653		b.stop()
   654		if err == io.EOF {
   655			return n, err
   656		}
   657		if b.reqWasCanceled() {
   658			err = &httpError{
   659				err:     err.Error() + " (Client.Timeout exceeded while reading body)",
   660				timeout: true,
   661			}
   662		}
   663		return n, err
   664	}
   665	
   666	func (b *cancelTimerBody) Close() error {
   667		err := b.rc.Close()
   668		b.stop()
   669		return err
   670	}
   671	

View as plain text