...
Run Format

Source file src/net/http/client.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// HTTP client. See RFC 2616.
     6	//
     7	// This is the high-level Client interface.
     8	// The low-level implementation is in transport.go.
     9	
    10	package http
    11	
    12	import (
    13		"crypto/tls"
    14		"encoding/base64"
    15		"errors"
    16		"fmt"
    17		"io"
    18		"io/ioutil"
    19		"log"
    20		"net/url"
    21		"strings"
    22		"sync"
    23		"time"
    24	)
    25	
    26	// A Client is an HTTP client. Its zero value (DefaultClient) is a
    27	// usable client that uses DefaultTransport.
    28	//
    29	// The Client's Transport typically has internal state (cached TCP
    30	// connections), so Clients should be reused instead of created as
    31	// needed. Clients are safe for concurrent use by multiple goroutines.
    32	//
    33	// A Client is higher-level than a RoundTripper (such as Transport)
    34	// and additionally handles HTTP details such as cookies and
    35	// redirects.
    36	type Client struct {
    37		// Transport specifies the mechanism by which individual
    38		// HTTP requests are made.
    39		// If nil, DefaultTransport is used.
    40		Transport RoundTripper
    41	
    42		// CheckRedirect specifies the policy for handling redirects.
    43		// If CheckRedirect is not nil, the client calls it before
    44		// following an HTTP redirect. The arguments req and via are
    45		// the upcoming request and the requests made already, oldest
    46		// first. If CheckRedirect returns an error, the Client's Get
    47		// method returns both the previous Response and
    48		// CheckRedirect's error (wrapped in a url.Error) instead of
    49		// issuing the Request req.
    50		//
    51		// If CheckRedirect is nil, the Client uses its default policy,
    52		// which is to stop after 10 consecutive requests.
    53		CheckRedirect func(req *Request, via []*Request) error
    54	
    55		// Jar specifies the cookie jar.
    56		// If Jar is nil, cookies are not sent in requests and ignored
    57		// in responses.
    58		Jar CookieJar
    59	
    60		// Timeout specifies a time limit for requests made by this
    61		// Client. The timeout includes connection time, any
    62		// redirects, and reading the response body. The timer remains
    63		// running after Get, Head, Post, or Do return and will
    64		// interrupt reading of the Response.Body.
    65		//
    66		// A Timeout of zero means no timeout.
    67		//
    68		// The Client cancels requests to the underlying Transport
    69		// using the Request.Cancel mechanism. Requests passed
    70		// to Client.Do may still set Request.Cancel; both will
    71		// cancel the request.
    72		//
    73		// For compatibility, the Client will also use the deprecated
    74		// CancelRequest method on Transport if found. New
    75		// RoundTripper implementations should use Request.Cancel
    76		// instead of implementing CancelRequest.
    77		Timeout time.Duration
    78	}
    79	
    80	// DefaultClient is the default Client and is used by Get, Head, and Post.
    81	var DefaultClient = &Client{}
    82	
    83	// RoundTripper is an interface representing the ability to execute a
    84	// single HTTP transaction, obtaining the Response for a given Request.
    85	//
    86	// A RoundTripper must be safe for concurrent use by multiple
    87	// goroutines.
    88	type RoundTripper interface {
    89		// RoundTrip executes a single HTTP transaction, returning
    90		// a Response for the provided Request.
    91		//
    92		// RoundTrip should not attempt to interpret the response. In
    93		// particular, RoundTrip must return err == nil if it obtained
    94		// a response, regardless of the response's HTTP status code.
    95		// A non-nil err should be reserved for failure to obtain a
    96		// response. Similarly, RoundTrip should not attempt to
    97		// handle higher-level protocol details such as redirects,
    98		// authentication, or cookies.
    99		//
   100		// RoundTrip should not modify the request, except for
   101		// consuming and closing the Request's Body.
   102		//
   103		// RoundTrip must always close the body, including on errors,
   104		// but depending on the implementation may do so in a separate
   105		// goroutine even after RoundTrip returns. This means that
   106		// callers wanting to reuse the body for subsequent requests
   107		// must arrange to wait for the Close call before doing so.
   108		//
   109		// The Request's URL and Header fields must be initialized.
   110		RoundTrip(*Request) (*Response, error)
   111	}
   112	
   113	// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
   114	// return true if the string includes a port.
   115	func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
   116	
   117	// refererForURL returns a referer without any authentication info or
   118	// an empty string if lastReq scheme is https and newReq scheme is http.
   119	func refererForURL(lastReq, newReq *url.URL) string {
   120		// https://tools.ietf.org/html/rfc7231#section-5.5.2
   121		//   "Clients SHOULD NOT include a Referer header field in a
   122		//    (non-secure) HTTP request if the referring page was
   123		//    transferred with a secure protocol."
   124		if lastReq.Scheme == "https" && newReq.Scheme == "http" {
   125			return ""
   126		}
   127		referer := lastReq.String()
   128		if lastReq.User != nil {
   129			// This is not very efficient, but is the best we can
   130			// do without:
   131			// - introducing a new method on URL
   132			// - creating a race condition
   133			// - copying the URL struct manually, which would cause
   134			//   maintenance problems down the line
   135			auth := lastReq.User.String() + "@"
   136			referer = strings.Replace(referer, auth, "", 1)
   137		}
   138		return referer
   139	}
   140	
   141	// Used in Send to implement io.ReadCloser by bundling together the
   142	// bufio.Reader through which we read the response, and the underlying
   143	// network connection.
   144	type readClose struct {
   145		io.Reader
   146		io.Closer
   147	}
   148	
   149	func (c *Client) send(req *Request, deadline time.Time) (*Response, error) {
   150		if c.Jar != nil {
   151			for _, cookie := range c.Jar.Cookies(req.URL) {
   152				req.AddCookie(cookie)
   153			}
   154		}
   155		resp, err := send(req, c.transport(), deadline)
   156		if err != nil {
   157			return nil, err
   158		}
   159		if c.Jar != nil {
   160			if rc := resp.Cookies(); len(rc) > 0 {
   161				c.Jar.SetCookies(req.URL, rc)
   162			}
   163		}
   164		return resp, err
   165	}
   166	
   167	// Do sends an HTTP request and returns an HTTP response, following
   168	// policy (e.g. redirects, cookies, auth) as configured on the client.
   169	//
   170	// An error is returned if caused by client policy (such as
   171	// CheckRedirect), or if there was an HTTP protocol error.
   172	// A non-2xx response doesn't cause an error.
   173	//
   174	// When err is nil, resp always contains a non-nil resp.Body.
   175	//
   176	// Callers should close resp.Body when done reading from it. If
   177	// resp.Body is not closed, the Client's underlying RoundTripper
   178	// (typically Transport) may not be able to re-use a persistent TCP
   179	// connection to the server for a subsequent "keep-alive" request.
   180	//
   181	// The request Body, if non-nil, will be closed by the underlying
   182	// Transport, even on errors.
   183	//
   184	// Generally Get, Post, or PostForm will be used instead of Do.
   185	func (c *Client) Do(req *Request) (resp *Response, err error) {
   186		method := valueOrDefault(req.Method, "GET")
   187		if method == "GET" || method == "HEAD" {
   188			return c.doFollowingRedirects(req, shouldRedirectGet)
   189		}
   190		if method == "POST" || method == "PUT" {
   191			return c.doFollowingRedirects(req, shouldRedirectPost)
   192		}
   193		return c.send(req, c.deadline())
   194	}
   195	
   196	func (c *Client) deadline() time.Time {
   197		if c.Timeout > 0 {
   198			return time.Now().Add(c.Timeout)
   199		}
   200		return time.Time{}
   201	}
   202	
   203	func (c *Client) transport() RoundTripper {
   204		if c.Transport != nil {
   205			return c.Transport
   206		}
   207		return DefaultTransport
   208	}
   209	
   210	// send issues an HTTP request.
   211	// Caller should close resp.Body when done reading from it.
   212	func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) {
   213		req := ireq // req is either the original request, or a modified fork
   214	
   215		if rt == nil {
   216			req.closeBody()
   217			return nil, errors.New("http: no Client.Transport or DefaultTransport")
   218		}
   219	
   220		if req.URL == nil {
   221			req.closeBody()
   222			return nil, errors.New("http: nil Request.URL")
   223		}
   224	
   225		if req.RequestURI != "" {
   226			req.closeBody()
   227			return nil, errors.New("http: Request.RequestURI can't be set in client requests.")
   228		}
   229	
   230		// forkReq forks req into a shallow clone of ireq the first
   231		// time it's called.
   232		forkReq := func() {
   233			if ireq == req {
   234				req = new(Request)
   235				*req = *ireq // shallow clone
   236			}
   237		}
   238	
   239		// Most the callers of send (Get, Post, et al) don't need
   240		// Headers, leaving it uninitialized.  We guarantee to the
   241		// Transport that this has been initialized, though.
   242		if req.Header == nil {
   243			forkReq()
   244			req.Header = make(Header)
   245		}
   246	
   247		if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" {
   248			username := u.Username()
   249			password, _ := u.Password()
   250			forkReq()
   251			req.Header = cloneHeader(ireq.Header)
   252			req.Header.Set("Authorization", "Basic "+basicAuth(username, password))
   253		}
   254	
   255		if !deadline.IsZero() {
   256			forkReq()
   257		}
   258		stopTimer, wasCanceled := setRequestCancel(req, rt, deadline)
   259	
   260		resp, err := rt.RoundTrip(req)
   261		if err != nil {
   262			stopTimer()
   263			if resp != nil {
   264				log.Printf("RoundTripper returned a response & error; ignoring response")
   265			}
   266			if tlsErr, ok := err.(tls.RecordHeaderError); ok {
   267				// If we get a bad TLS record header, check to see if the
   268				// response looks like HTTP and give a more helpful error.
   269				// See golang.org/issue/11111.
   270				if string(tlsErr.RecordHeader[:]) == "HTTP/" {
   271					err = errors.New("http: server gave HTTP response to HTTPS client")
   272				}
   273			}
   274			return nil, err
   275		}
   276		if !deadline.IsZero() {
   277			resp.Body = &cancelTimerBody{
   278				stop:           stopTimer,
   279				rc:             resp.Body,
   280				reqWasCanceled: wasCanceled,
   281			}
   282		}
   283		return resp, nil
   284	}
   285	
   286	// setRequestCancel sets the Cancel field of req, if deadline is
   287	// non-zero. The RoundTripper's type is used to determine whether the legacy
   288	// CancelRequest behavior should be used.
   289	func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), wasCanceled func() bool) {
   290		if deadline.IsZero() {
   291			return nop, alwaysFalse
   292		}
   293	
   294		initialReqCancel := req.Cancel // the user's original Request.Cancel, if any
   295	
   296		cancel := make(chan struct{})
   297		req.Cancel = cancel
   298	
   299		wasCanceled = func() bool {
   300			select {
   301			case <-cancel:
   302				return true
   303			default:
   304				return false
   305			}
   306		}
   307	
   308		doCancel := func() {
   309			// The new way:
   310			close(cancel)
   311	
   312			// The legacy compatibility way, used only
   313			// for RoundTripper implementations written
   314			// before Go 1.5 or Go 1.6.
   315			type canceler interface {
   316				CancelRequest(*Request)
   317			}
   318			switch v := rt.(type) {
   319			case *Transport, *http2Transport:
   320				// Do nothing. The net/http package's transports
   321				// support the new Request.Cancel channel
   322			case canceler:
   323				v.CancelRequest(req)
   324			}
   325		}
   326	
   327		stopTimerCh := make(chan struct{})
   328		var once sync.Once
   329		stopTimer = func() { once.Do(func() { close(stopTimerCh) }) }
   330	
   331		timer := time.NewTimer(deadline.Sub(time.Now()))
   332		go func() {
   333			select {
   334			case <-initialReqCancel:
   335				doCancel()
   336			case <-timer.C:
   337				doCancel()
   338			case <-stopTimerCh:
   339				timer.Stop()
   340			}
   341		}()
   342	
   343		return stopTimer, wasCanceled
   344	}
   345	
   346	// See 2 (end of page 4) http://www.ietf.org/rfc/rfc2617.txt
   347	// "To receive authorization, the client sends the userid and password,
   348	// separated by a single colon (":") character, within a base64
   349	// encoded string in the credentials."
   350	// It is not meant to be urlencoded.
   351	func basicAuth(username, password string) string {
   352		auth := username + ":" + password
   353		return base64.StdEncoding.EncodeToString([]byte(auth))
   354	}
   355	
   356	// True if the specified HTTP status code is one for which the Get utility should
   357	// automatically redirect.
   358	func shouldRedirectGet(statusCode int) bool {
   359		switch statusCode {
   360		case StatusMovedPermanently, StatusFound, StatusSeeOther, StatusTemporaryRedirect:
   361			return true
   362		}
   363		return false
   364	}
   365	
   366	// True if the specified HTTP status code is one for which the Post utility should
   367	// automatically redirect.
   368	func shouldRedirectPost(statusCode int) bool {
   369		switch statusCode {
   370		case StatusFound, StatusSeeOther:
   371			return true
   372		}
   373		return false
   374	}
   375	
   376	// Get issues a GET to the specified URL. If the response is one of
   377	// the following redirect codes, Get follows the redirect, up to a
   378	// maximum of 10 redirects:
   379	//
   380	//    301 (Moved Permanently)
   381	//    302 (Found)
   382	//    303 (See Other)
   383	//    307 (Temporary Redirect)
   384	//
   385	// An error is returned if there were too many redirects or if there
   386	// was an HTTP protocol error. A non-2xx response doesn't cause an
   387	// error.
   388	//
   389	// When err is nil, resp always contains a non-nil resp.Body.
   390	// Caller should close resp.Body when done reading from it.
   391	//
   392	// Get is a wrapper around DefaultClient.Get.
   393	//
   394	// To make a request with custom headers, use NewRequest and
   395	// DefaultClient.Do.
   396	func Get(url string) (resp *Response, err error) {
   397		return DefaultClient.Get(url)
   398	}
   399	
   400	// Get issues a GET to the specified URL. If the response is one of the
   401	// following redirect codes, Get follows the redirect after calling the
   402	// Client's CheckRedirect function:
   403	//
   404	//    301 (Moved Permanently)
   405	//    302 (Found)
   406	//    303 (See Other)
   407	//    307 (Temporary Redirect)
   408	//
   409	// An error is returned if the Client's CheckRedirect function fails
   410	// or if there was an HTTP protocol error. A non-2xx response doesn't
   411	// cause an error.
   412	//
   413	// When err is nil, resp always contains a non-nil resp.Body.
   414	// Caller should close resp.Body when done reading from it.
   415	//
   416	// To make a request with custom headers, use NewRequest and Client.Do.
   417	func (c *Client) Get(url string) (resp *Response, err error) {
   418		req, err := NewRequest("GET", url, nil)
   419		if err != nil {
   420			return nil, err
   421		}
   422		return c.doFollowingRedirects(req, shouldRedirectGet)
   423	}
   424	
   425	func alwaysFalse() bool { return false }
   426	
   427	func (c *Client) doFollowingRedirects(ireq *Request, shouldRedirect func(int) bool) (resp *Response, err error) {
   428		var base *url.URL
   429		redirectChecker := c.CheckRedirect
   430		if redirectChecker == nil {
   431			redirectChecker = defaultCheckRedirect
   432		}
   433		var via []*Request
   434	
   435		if ireq.URL == nil {
   436			ireq.closeBody()
   437			return nil, errors.New("http: nil Request.URL")
   438		}
   439	
   440		req := ireq
   441		deadline := c.deadline()
   442	
   443		urlStr := "" // next relative or absolute URL to fetch (after first request)
   444		redirectFailed := false
   445		for redirect := 0; ; redirect++ {
   446			if redirect != 0 {
   447				nreq := new(Request)
   448				nreq.Cancel = ireq.Cancel
   449				nreq.Method = ireq.Method
   450				if ireq.Method == "POST" || ireq.Method == "PUT" {
   451					nreq.Method = "GET"
   452				}
   453				nreq.Header = make(Header)
   454				nreq.URL, err = base.Parse(urlStr)
   455				if err != nil {
   456					break
   457				}
   458				if len(via) > 0 {
   459					// Add the Referer header.
   460					lastReq := via[len(via)-1]
   461					if ref := refererForURL(lastReq.URL, nreq.URL); ref != "" {
   462						nreq.Header.Set("Referer", ref)
   463					}
   464	
   465					err = redirectChecker(nreq, via)
   466					if err != nil {
   467						redirectFailed = true
   468						break
   469					}
   470				}
   471				req = nreq
   472			}
   473	
   474			urlStr = req.URL.String()
   475			if resp, err = c.send(req, deadline); err != nil {
   476				if !deadline.IsZero() && !time.Now().Before(deadline) {
   477					err = &httpError{
   478						err:     err.Error() + " (Client.Timeout exceeded while awaiting headers)",
   479						timeout: true,
   480					}
   481				}
   482				break
   483			}
   484	
   485			if shouldRedirect(resp.StatusCode) {
   486				// Read the body if small so underlying TCP connection will be re-used.
   487				// No need to check for errors: if it fails, Transport won't reuse it anyway.
   488				const maxBodySlurpSize = 2 << 10
   489				if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize {
   490					io.CopyN(ioutil.Discard, resp.Body, maxBodySlurpSize)
   491				}
   492				resp.Body.Close()
   493				if urlStr = resp.Header.Get("Location"); urlStr == "" {
   494					err = fmt.Errorf("%d response missing Location header", resp.StatusCode)
   495					break
   496				}
   497				base = req.URL
   498				via = append(via, req)
   499				continue
   500			}
   501			return resp, nil
   502		}
   503	
   504		method := valueOrDefault(ireq.Method, "GET")
   505		urlErr := &url.Error{
   506			Op:  method[:1] + strings.ToLower(method[1:]),
   507			URL: urlStr,
   508			Err: err,
   509		}
   510	
   511		if redirectFailed {
   512			// Special case for Go 1 compatibility: return both the response
   513			// and an error if the CheckRedirect function failed.
   514			// See https://golang.org/issue/3795
   515			return resp, urlErr
   516		}
   517	
   518		if resp != nil {
   519			resp.Body.Close()
   520		}
   521		return nil, urlErr
   522	}
   523	
   524	func defaultCheckRedirect(req *Request, via []*Request) error {
   525		if len(via) >= 10 {
   526			return errors.New("stopped after 10 redirects")
   527		}
   528		return nil
   529	}
   530	
   531	// Post issues a POST to the specified URL.
   532	//
   533	// Caller should close resp.Body when done reading from it.
   534	//
   535	// If the provided body is an io.Closer, it is closed after the
   536	// request.
   537	//
   538	// Post is a wrapper around DefaultClient.Post.
   539	//
   540	// To set custom headers, use NewRequest and DefaultClient.Do.
   541	func Post(url string, bodyType string, body io.Reader) (resp *Response, err error) {
   542		return DefaultClient.Post(url, bodyType, body)
   543	}
   544	
   545	// Post issues a POST to the specified URL.
   546	//
   547	// Caller should close resp.Body when done reading from it.
   548	//
   549	// If the provided body is an io.Closer, it is closed after the
   550	// request.
   551	//
   552	// To set custom headers, use NewRequest and Client.Do.
   553	func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *Response, err error) {
   554		req, err := NewRequest("POST", url, body)
   555		if err != nil {
   556			return nil, err
   557		}
   558		req.Header.Set("Content-Type", bodyType)
   559		return c.doFollowingRedirects(req, shouldRedirectPost)
   560	}
   561	
   562	// PostForm issues a POST to the specified URL, with data's keys and
   563	// values URL-encoded as the request body.
   564	//
   565	// The Content-Type header is set to application/x-www-form-urlencoded.
   566	// To set other headers, use NewRequest and DefaultClient.Do.
   567	//
   568	// When err is nil, resp always contains a non-nil resp.Body.
   569	// Caller should close resp.Body when done reading from it.
   570	//
   571	// PostForm is a wrapper around DefaultClient.PostForm.
   572	func PostForm(url string, data url.Values) (resp *Response, err error) {
   573		return DefaultClient.PostForm(url, data)
   574	}
   575	
   576	// PostForm issues a POST to the specified URL,
   577	// with data's keys and values URL-encoded as the request body.
   578	//
   579	// The Content-Type header is set to application/x-www-form-urlencoded.
   580	// To set other headers, use NewRequest and DefaultClient.Do.
   581	//
   582	// When err is nil, resp always contains a non-nil resp.Body.
   583	// Caller should close resp.Body when done reading from it.
   584	func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) {
   585		return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
   586	}
   587	
   588	// Head issues a HEAD to the specified URL.  If the response is one of
   589	// the following redirect codes, Head follows the redirect, up to a
   590	// maximum of 10 redirects:
   591	//
   592	//    301 (Moved Permanently)
   593	//    302 (Found)
   594	//    303 (See Other)
   595	//    307 (Temporary Redirect)
   596	//
   597	// Head is a wrapper around DefaultClient.Head
   598	func Head(url string) (resp *Response, err error) {
   599		return DefaultClient.Head(url)
   600	}
   601	
   602	// Head issues a HEAD to the specified URL.  If the response is one of the
   603	// following redirect codes, Head follows the redirect after calling the
   604	// Client's CheckRedirect function:
   605	//
   606	//    301 (Moved Permanently)
   607	//    302 (Found)
   608	//    303 (See Other)
   609	//    307 (Temporary Redirect)
   610	func (c *Client) Head(url string) (resp *Response, err error) {
   611		req, err := NewRequest("HEAD", url, nil)
   612		if err != nil {
   613			return nil, err
   614		}
   615		return c.doFollowingRedirects(req, shouldRedirectGet)
   616	}
   617	
   618	// cancelTimerBody is an io.ReadCloser that wraps rc with two features:
   619	// 1) on Read error or close, the stop func is called.
   620	// 2) On Read failure, if reqWasCanceled is true, the error is wrapped and
   621	//    marked as net.Error that hit its timeout.
   622	type cancelTimerBody struct {
   623		stop           func() // stops the time.Timer waiting to cancel the request
   624		rc             io.ReadCloser
   625		reqWasCanceled func() bool
   626	}
   627	
   628	func (b *cancelTimerBody) Read(p []byte) (n int, err error) {
   629		n, err = b.rc.Read(p)
   630		if err == nil {
   631			return n, nil
   632		}
   633		b.stop()
   634		if err == io.EOF {
   635			return n, err
   636		}
   637		if b.reqWasCanceled() {
   638			err = &httpError{
   639				err:     err.Error() + " (Client.Timeout exceeded while reading body)",
   640				timeout: true,
   641			}
   642		}
   643		return n, err
   644	}
   645	
   646	func (b *cancelTimerBody) Close() error {
   647		err := b.rc.Close()
   648		b.stop()
   649		return err
   650	}
   651	

View as plain text