...
Run Format

Source file src/net/http/client.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// HTTP client. See RFC 2616.
     6	//
     7	// This is the high-level Client interface.
     8	// The low-level implementation is in transport.go.
     9	
    10	package http
    11	
    12	import (
    13		"encoding/base64"
    14		"errors"
    15		"fmt"
    16		"io"
    17		"io/ioutil"
    18		"log"
    19		"net/url"
    20		"strings"
    21		"sync"
    22		"sync/atomic"
    23		"time"
    24	)
    25	
    26	// A Client is an HTTP client. Its zero value (DefaultClient) is a
    27	// usable client that uses DefaultTransport.
    28	//
    29	// The Client's Transport typically has internal state (cached TCP
    30	// connections), so Clients should be reused instead of created as
    31	// needed. Clients are safe for concurrent use by multiple goroutines.
    32	//
    33	// A Client is higher-level than a RoundTripper (such as Transport)
    34	// and additionally handles HTTP details such as cookies and
    35	// redirects.
    36	type Client struct {
    37		// Transport specifies the mechanism by which individual
    38		// HTTP requests are made.
    39		// If nil, DefaultTransport is used.
    40		Transport RoundTripper
    41	
    42		// CheckRedirect specifies the policy for handling redirects.
    43		// If CheckRedirect is not nil, the client calls it before
    44		// following an HTTP redirect. The arguments req and via are
    45		// the upcoming request and the requests made already, oldest
    46		// first. If CheckRedirect returns an error, the Client's Get
    47		// method returns both the previous Response and
    48		// CheckRedirect's error (wrapped in a url.Error) instead of
    49		// issuing the Request req.
    50		//
    51		// If CheckRedirect is nil, the Client uses its default policy,
    52		// which is to stop after 10 consecutive requests.
    53		CheckRedirect func(req *Request, via []*Request) error
    54	
    55		// Jar specifies the cookie jar.
    56		// If Jar is nil, cookies are not sent in requests and ignored
    57		// in responses.
    58		Jar CookieJar
    59	
    60		// Timeout specifies a time limit for requests made by this
    61		// Client. The timeout includes connection time, any
    62		// redirects, and reading the response body. The timer remains
    63		// running after Get, Head, Post, or Do return and will
    64		// interrupt reading of the Response.Body.
    65		//
    66		// A Timeout of zero means no timeout.
    67		//
    68		// The Client's Transport must support the CancelRequest
    69		// method or Client will return errors when attempting to make
    70		// a request with Get, Head, Post, or Do. Client's default
    71		// Transport (DefaultTransport) supports CancelRequest.
    72		Timeout time.Duration
    73	}
    74	
    75	// DefaultClient is the default Client and is used by Get, Head, and Post.
    76	var DefaultClient = &Client{}
    77	
    78	// RoundTripper is an interface representing the ability to execute a
    79	// single HTTP transaction, obtaining the Response for a given Request.
    80	//
    81	// A RoundTripper must be safe for concurrent use by multiple
    82	// goroutines.
    83	type RoundTripper interface {
    84		// RoundTrip executes a single HTTP transaction, returning
    85		// the Response for the request req.  RoundTrip should not
    86		// attempt to interpret the response.  In particular,
    87		// RoundTrip must return err == nil if it obtained a response,
    88		// regardless of the response's HTTP status code.  A non-nil
    89		// err should be reserved for failure to obtain a response.
    90		// Similarly, RoundTrip should not attempt to handle
    91		// higher-level protocol details such as redirects,
    92		// authentication, or cookies.
    93		//
    94		// RoundTrip should not modify the request, except for
    95		// consuming and closing the Body, including on errors. The
    96		// request's URL and Header fields are guaranteed to be
    97		// initialized.
    98		RoundTrip(*Request) (*Response, error)
    99	}
   100	
   101	// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
   102	// return true if the string includes a port.
   103	func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
   104	
   105	// refererForURL returns a referer without any authentication info or
   106	// an empty string if lastReq scheme is https and newReq scheme is http.
   107	func refererForURL(lastReq, newReq *url.URL) string {
   108		// https://tools.ietf.org/html/rfc7231#section-5.5.2
   109		//   "Clients SHOULD NOT include a Referer header field in a
   110		//    (non-secure) HTTP request if the referring page was
   111		//    transferred with a secure protocol."
   112		if lastReq.Scheme == "https" && newReq.Scheme == "http" {
   113			return ""
   114		}
   115		referer := lastReq.String()
   116		if lastReq.User != nil {
   117			// This is not very efficient, but is the best we can
   118			// do without:
   119			// - introducing a new method on URL
   120			// - creating a race condition
   121			// - copying the URL struct manually, which would cause
   122			//   maintenance problems down the line
   123			auth := lastReq.User.String() + "@"
   124			referer = strings.Replace(referer, auth, "", 1)
   125		}
   126		return referer
   127	}
   128	
   129	// Used in Send to implement io.ReadCloser by bundling together the
   130	// bufio.Reader through which we read the response, and the underlying
   131	// network connection.
   132	type readClose struct {
   133		io.Reader
   134		io.Closer
   135	}
   136	
   137	func (c *Client) send(req *Request) (*Response, error) {
   138		if c.Jar != nil {
   139			for _, cookie := range c.Jar.Cookies(req.URL) {
   140				req.AddCookie(cookie)
   141			}
   142		}
   143		resp, err := send(req, c.transport())
   144		if err != nil {
   145			return nil, err
   146		}
   147		if c.Jar != nil {
   148			if rc := resp.Cookies(); len(rc) > 0 {
   149				c.Jar.SetCookies(req.URL, rc)
   150			}
   151		}
   152		return resp, err
   153	}
   154	
   155	// Do sends an HTTP request and returns an HTTP response, following
   156	// policy (e.g. redirects, cookies, auth) as configured on the client.
   157	//
   158	// An error is returned if caused by client policy (such as
   159	// CheckRedirect), or if there was an HTTP protocol error.
   160	// A non-2xx response doesn't cause an error.
   161	//
   162	// When err is nil, resp always contains a non-nil resp.Body.
   163	//
   164	// Callers should close resp.Body when done reading from it. If
   165	// resp.Body is not closed, the Client's underlying RoundTripper
   166	// (typically Transport) may not be able to re-use a persistent TCP
   167	// connection to the server for a subsequent "keep-alive" request.
   168	//
   169	// The request Body, if non-nil, will be closed by the underlying
   170	// Transport, even on errors.
   171	//
   172	// Generally Get, Post, or PostForm will be used instead of Do.
   173	func (c *Client) Do(req *Request) (resp *Response, err error) {
   174		if req.Method == "GET" || req.Method == "HEAD" {
   175			return c.doFollowingRedirects(req, shouldRedirectGet)
   176		}
   177		if req.Method == "POST" || req.Method == "PUT" {
   178			return c.doFollowingRedirects(req, shouldRedirectPost)
   179		}
   180		return c.send(req)
   181	}
   182	
   183	func (c *Client) transport() RoundTripper {
   184		if c.Transport != nil {
   185			return c.Transport
   186		}
   187		return DefaultTransport
   188	}
   189	
   190	// send issues an HTTP request.
   191	// Caller should close resp.Body when done reading from it.
   192	func send(req *Request, t RoundTripper) (resp *Response, err error) {
   193		if t == nil {
   194			req.closeBody()
   195			return nil, errors.New("http: no Client.Transport or DefaultTransport")
   196		}
   197	
   198		if req.URL == nil {
   199			req.closeBody()
   200			return nil, errors.New("http: nil Request.URL")
   201		}
   202	
   203		if req.RequestURI != "" {
   204			req.closeBody()
   205			return nil, errors.New("http: Request.RequestURI can't be set in client requests.")
   206		}
   207	
   208		// Most the callers of send (Get, Post, et al) don't need
   209		// Headers, leaving it uninitialized.  We guarantee to the
   210		// Transport that this has been initialized, though.
   211		if req.Header == nil {
   212			req.Header = make(Header)
   213		}
   214	
   215		if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" {
   216			username := u.Username()
   217			password, _ := u.Password()
   218			req.Header.Set("Authorization", "Basic "+basicAuth(username, password))
   219		}
   220		resp, err = t.RoundTrip(req)
   221		if err != nil {
   222			if resp != nil {
   223				log.Printf("RoundTripper returned a response & error; ignoring response")
   224			}
   225			return nil, err
   226		}
   227		return resp, nil
   228	}
   229	
   230	// See 2 (end of page 4) http://www.ietf.org/rfc/rfc2617.txt
   231	// "To receive authorization, the client sends the userid and password,
   232	// separated by a single colon (":") character, within a base64
   233	// encoded string in the credentials."
   234	// It is not meant to be urlencoded.
   235	func basicAuth(username, password string) string {
   236		auth := username + ":" + password
   237		return base64.StdEncoding.EncodeToString([]byte(auth))
   238	}
   239	
   240	// True if the specified HTTP status code is one for which the Get utility should
   241	// automatically redirect.
   242	func shouldRedirectGet(statusCode int) bool {
   243		switch statusCode {
   244		case StatusMovedPermanently, StatusFound, StatusSeeOther, StatusTemporaryRedirect:
   245			return true
   246		}
   247		return false
   248	}
   249	
   250	// True if the specified HTTP status code is one for which the Post utility should
   251	// automatically redirect.
   252	func shouldRedirectPost(statusCode int) bool {
   253		switch statusCode {
   254		case StatusFound, StatusSeeOther:
   255			return true
   256		}
   257		return false
   258	}
   259	
   260	// Get issues a GET to the specified URL. If the response is one of
   261	// the following redirect codes, Get follows the redirect, up to a
   262	// maximum of 10 redirects:
   263	//
   264	//    301 (Moved Permanently)
   265	//    302 (Found)
   266	//    303 (See Other)
   267	//    307 (Temporary Redirect)
   268	//
   269	// An error is returned if there were too many redirects or if there
   270	// was an HTTP protocol error. A non-2xx response doesn't cause an
   271	// error.
   272	//
   273	// When err is nil, resp always contains a non-nil resp.Body.
   274	// Caller should close resp.Body when done reading from it.
   275	//
   276	// Get is a wrapper around DefaultClient.Get.
   277	//
   278	// To make a request with custom headers, use NewRequest and
   279	// DefaultClient.Do.
   280	func Get(url string) (resp *Response, err error) {
   281		return DefaultClient.Get(url)
   282	}
   283	
   284	// Get issues a GET to the specified URL. If the response is one of the
   285	// following redirect codes, Get follows the redirect after calling the
   286	// Client's CheckRedirect function:
   287	//
   288	//    301 (Moved Permanently)
   289	//    302 (Found)
   290	//    303 (See Other)
   291	//    307 (Temporary Redirect)
   292	//
   293	// An error is returned if the Client's CheckRedirect function fails
   294	// or if there was an HTTP protocol error. A non-2xx response doesn't
   295	// cause an error.
   296	//
   297	// When err is nil, resp always contains a non-nil resp.Body.
   298	// Caller should close resp.Body when done reading from it.
   299	//
   300	// To make a request with custom headers, use NewRequest and Client.Do.
   301	func (c *Client) Get(url string) (resp *Response, err error) {
   302		req, err := NewRequest("GET", url, nil)
   303		if err != nil {
   304			return nil, err
   305		}
   306		return c.doFollowingRedirects(req, shouldRedirectGet)
   307	}
   308	
   309	func alwaysFalse() bool { return false }
   310	
   311	func (c *Client) doFollowingRedirects(ireq *Request, shouldRedirect func(int) bool) (resp *Response, err error) {
   312		var base *url.URL
   313		redirectChecker := c.CheckRedirect
   314		if redirectChecker == nil {
   315			redirectChecker = defaultCheckRedirect
   316		}
   317		var via []*Request
   318	
   319		if ireq.URL == nil {
   320			ireq.closeBody()
   321			return nil, errors.New("http: nil Request.URL")
   322		}
   323	
   324		var reqmu sync.Mutex // guards req
   325		req := ireq
   326	
   327		var timer *time.Timer
   328		var atomicWasCanceled int32 // atomic bool (1 or 0)
   329		var wasCanceled = alwaysFalse
   330		if c.Timeout > 0 {
   331			wasCanceled = func() bool { return atomic.LoadInt32(&atomicWasCanceled) != 0 }
   332			type canceler interface {
   333				CancelRequest(*Request)
   334			}
   335			tr, ok := c.transport().(canceler)
   336			if !ok {
   337				return nil, fmt.Errorf("net/http: Client Transport of type %T doesn't support CancelRequest; Timeout not supported", c.transport())
   338			}
   339			timer = time.AfterFunc(c.Timeout, func() {
   340				atomic.StoreInt32(&atomicWasCanceled, 1)
   341				reqmu.Lock()
   342				defer reqmu.Unlock()
   343				tr.CancelRequest(req)
   344			})
   345		}
   346	
   347		urlStr := "" // next relative or absolute URL to fetch (after first request)
   348		redirectFailed := false
   349		for redirect := 0; ; redirect++ {
   350			if redirect != 0 {
   351				nreq := new(Request)
   352				nreq.Method = ireq.Method
   353				if ireq.Method == "POST" || ireq.Method == "PUT" {
   354					nreq.Method = "GET"
   355				}
   356				nreq.Header = make(Header)
   357				nreq.URL, err = base.Parse(urlStr)
   358				if err != nil {
   359					break
   360				}
   361				if len(via) > 0 {
   362					// Add the Referer header.
   363					lastReq := via[len(via)-1]
   364					if ref := refererForURL(lastReq.URL, nreq.URL); ref != "" {
   365						nreq.Header.Set("Referer", ref)
   366					}
   367	
   368					err = redirectChecker(nreq, via)
   369					if err != nil {
   370						redirectFailed = true
   371						break
   372					}
   373				}
   374				reqmu.Lock()
   375				req = nreq
   376				reqmu.Unlock()
   377			}
   378	
   379			urlStr = req.URL.String()
   380			if resp, err = c.send(req); err != nil {
   381				if wasCanceled() {
   382					err = &httpError{
   383						err:     err.Error() + " (Client.Timeout exceeded while awaiting headers)",
   384						timeout: true,
   385					}
   386				}
   387				break
   388			}
   389	
   390			if shouldRedirect(resp.StatusCode) {
   391				// Read the body if small so underlying TCP connection will be re-used.
   392				// No need to check for errors: if it fails, Transport won't reuse it anyway.
   393				const maxBodySlurpSize = 2 << 10
   394				if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize {
   395					io.CopyN(ioutil.Discard, resp.Body, maxBodySlurpSize)
   396				}
   397				resp.Body.Close()
   398				if urlStr = resp.Header.Get("Location"); urlStr == "" {
   399					err = fmt.Errorf("%d response missing Location header", resp.StatusCode)
   400					break
   401				}
   402				base = req.URL
   403				via = append(via, req)
   404				continue
   405			}
   406			if timer != nil {
   407				resp.Body = &cancelTimerBody{
   408					t:              timer,
   409					rc:             resp.Body,
   410					reqWasCanceled: wasCanceled,
   411				}
   412			}
   413			return resp, nil
   414		}
   415	
   416		method := ireq.Method
   417		urlErr := &url.Error{
   418			Op:  method[0:1] + strings.ToLower(method[1:]),
   419			URL: urlStr,
   420			Err: err,
   421		}
   422	
   423		if redirectFailed {
   424			// Special case for Go 1 compatibility: return both the response
   425			// and an error if the CheckRedirect function failed.
   426			// See https://golang.org/issue/3795
   427			return resp, urlErr
   428		}
   429	
   430		if resp != nil {
   431			resp.Body.Close()
   432		}
   433		return nil, urlErr
   434	}
   435	
   436	func defaultCheckRedirect(req *Request, via []*Request) error {
   437		if len(via) >= 10 {
   438			return errors.New("stopped after 10 redirects")
   439		}
   440		return nil
   441	}
   442	
   443	// Post issues a POST to the specified URL.
   444	//
   445	// Caller should close resp.Body when done reading from it.
   446	//
   447	// If the provided body is an io.Closer, it is closed after the
   448	// request.
   449	//
   450	// Post is a wrapper around DefaultClient.Post.
   451	//
   452	// To set custom headers, use NewRequest and DefaultClient.Do.
   453	func Post(url string, bodyType string, body io.Reader) (resp *Response, err error) {
   454		return DefaultClient.Post(url, bodyType, body)
   455	}
   456	
   457	// Post issues a POST to the specified URL.
   458	//
   459	// Caller should close resp.Body when done reading from it.
   460	//
   461	// If the provided body is an io.Closer, it is closed after the
   462	// request.
   463	//
   464	// To set custom headers, use NewRequest and Client.Do.
   465	func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *Response, err error) {
   466		req, err := NewRequest("POST", url, body)
   467		if err != nil {
   468			return nil, err
   469		}
   470		req.Header.Set("Content-Type", bodyType)
   471		return c.doFollowingRedirects(req, shouldRedirectPost)
   472	}
   473	
   474	// PostForm issues a POST to the specified URL, with data's keys and
   475	// values URL-encoded as the request body.
   476	//
   477	// The Content-Type header is set to application/x-www-form-urlencoded.
   478	// To set other headers, use NewRequest and DefaultClient.Do.
   479	//
   480	// When err is nil, resp always contains a non-nil resp.Body.
   481	// Caller should close resp.Body when done reading from it.
   482	//
   483	// PostForm is a wrapper around DefaultClient.PostForm.
   484	func PostForm(url string, data url.Values) (resp *Response, err error) {
   485		return DefaultClient.PostForm(url, data)
   486	}
   487	
   488	// PostForm issues a POST to the specified URL,
   489	// with data's keys and values URL-encoded as the request body.
   490	//
   491	// The Content-Type header is set to application/x-www-form-urlencoded.
   492	// To set other headers, use NewRequest and DefaultClient.Do.
   493	//
   494	// When err is nil, resp always contains a non-nil resp.Body.
   495	// Caller should close resp.Body when done reading from it.
   496	func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) {
   497		return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
   498	}
   499	
   500	// Head issues a HEAD to the specified URL.  If the response is one of
   501	// the following redirect codes, Head follows the redirect, up to a
   502	// maximum of 10 redirects:
   503	//
   504	//    301 (Moved Permanently)
   505	//    302 (Found)
   506	//    303 (See Other)
   507	//    307 (Temporary Redirect)
   508	//
   509	// Head is a wrapper around DefaultClient.Head
   510	func Head(url string) (resp *Response, err error) {
   511		return DefaultClient.Head(url)
   512	}
   513	
   514	// Head issues a HEAD to the specified URL.  If the response is one of the
   515	// following redirect codes, Head follows the redirect after calling the
   516	// Client's CheckRedirect function:
   517	//
   518	//    301 (Moved Permanently)
   519	//    302 (Found)
   520	//    303 (See Other)
   521	//    307 (Temporary Redirect)
   522	func (c *Client) Head(url string) (resp *Response, err error) {
   523		req, err := NewRequest("HEAD", url, nil)
   524		if err != nil {
   525			return nil, err
   526		}
   527		return c.doFollowingRedirects(req, shouldRedirectGet)
   528	}
   529	
   530	// cancelTimerBody is an io.ReadCloser that wraps rc with two features:
   531	// 1) on Read EOF or Close, the timer t is Stopped,
   532	// 2) On Read failure, if reqWasCanceled is true, the error is wrapped and
   533	//    marked as net.Error that hit its timeout.
   534	type cancelTimerBody struct {
   535		t              *time.Timer
   536		rc             io.ReadCloser
   537		reqWasCanceled func() bool
   538	}
   539	
   540	func (b *cancelTimerBody) Read(p []byte) (n int, err error) {
   541		n, err = b.rc.Read(p)
   542		if err == io.EOF {
   543			b.t.Stop()
   544		} else if err != nil && b.reqWasCanceled() {
   545			return n, &httpError{
   546				err:     err.Error() + " (Client.Timeout exceeded while reading body)",
   547				timeout: true,
   548			}
   549		}
   550		return
   551	}
   552	
   553	func (b *cancelTimerBody) Close() error {
   554		err := b.rc.Close()
   555		b.t.Stop()
   556		return err
   557	}
   558	

View as plain text