...
Run Format

Source file src/net/http/client.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// HTTP client. See RFC 2616.
     6	//
     7	// This is the high-level Client interface.
     8	// The low-level implementation is in transport.go.
     9	
    10	package http
    11	
    12	import (
    13		"encoding/base64"
    14		"errors"
    15		"fmt"
    16		"io"
    17		"io/ioutil"
    18		"log"
    19		"net/url"
    20		"strings"
    21		"sync"
    22		"time"
    23	)
    24	
    25	// A Client is an HTTP client. Its zero value (DefaultClient) is a
    26	// usable client that uses DefaultTransport.
    27	//
    28	// The Client's Transport typically has internal state (cached TCP
    29	// connections), so Clients should be reused instead of created as
    30	// needed. Clients are safe for concurrent use by multiple goroutines.
    31	//
    32	// A Client is higher-level than a RoundTripper (such as Transport)
    33	// and additionally handles HTTP details such as cookies and
    34	// redirects.
    35	type Client struct {
    36		// Transport specifies the mechanism by which individual
    37		// HTTP requests are made.
    38		// If nil, DefaultTransport is used.
    39		Transport RoundTripper
    40	
    41		// CheckRedirect specifies the policy for handling redirects.
    42		// If CheckRedirect is not nil, the client calls it before
    43		// following an HTTP redirect. The arguments req and via are
    44		// the upcoming request and the requests made already, oldest
    45		// first. If CheckRedirect returns an error, the Client's Get
    46		// method returns both the previous Response and
    47		// CheckRedirect's error (wrapped in a url.Error) instead of
    48		// issuing the Request req.
    49		//
    50		// If CheckRedirect is nil, the Client uses its default policy,
    51		// which is to stop after 10 consecutive requests.
    52		CheckRedirect func(req *Request, via []*Request) error
    53	
    54		// Jar specifies the cookie jar.
    55		// If Jar is nil, cookies are not sent in requests and ignored
    56		// in responses.
    57		Jar CookieJar
    58	
    59		// Timeout specifies a time limit for requests made by this
    60		// Client. The timeout includes connection time, any
    61		// redirects, and reading the response body. The timer remains
    62		// running after Get, Head, Post, or Do return and will
    63		// interrupt reading of the Response.Body.
    64		//
    65		// A Timeout of zero means no timeout.
    66		//
    67		// The Client's Transport must support the CancelRequest
    68		// method or Client will return errors when attempting to make
    69		// a request with Get, Head, Post, or Do. Client's default
    70		// Transport (DefaultTransport) supports CancelRequest.
    71		Timeout time.Duration
    72	}
    73	
    74	// DefaultClient is the default Client and is used by Get, Head, and Post.
    75	var DefaultClient = &Client{}
    76	
    77	// RoundTripper is an interface representing the ability to execute a
    78	// single HTTP transaction, obtaining the Response for a given Request.
    79	//
    80	// A RoundTripper must be safe for concurrent use by multiple
    81	// goroutines.
    82	type RoundTripper interface {
    83		// RoundTrip executes a single HTTP transaction, returning
    84		// the Response for the request req.  RoundTrip should not
    85		// attempt to interpret the response.  In particular,
    86		// RoundTrip must return err == nil if it obtained a response,
    87		// regardless of the response's HTTP status code.  A non-nil
    88		// err should be reserved for failure to obtain a response.
    89		// Similarly, RoundTrip should not attempt to handle
    90		// higher-level protocol details such as redirects,
    91		// authentication, or cookies.
    92		//
    93		// RoundTrip should not modify the request, except for
    94		// consuming and closing the Body, including on errors. The
    95		// request's URL and Header fields are guaranteed to be
    96		// initialized.
    97		RoundTrip(*Request) (*Response, error)
    98	}
    99	
   100	// Given a string of the form "host", "host:port", or "[ipv6::address]:port",
   101	// return true if the string includes a port.
   102	func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
   103	
   104	// refererForURL returns a referer without any authentication info or
   105	// an empty string if lastReq scheme is https and newReq scheme is http.
   106	func refererForURL(lastReq, newReq *url.URL) string {
   107		// https://tools.ietf.org/html/rfc7231#section-5.5.2
   108		//   "Clients SHOULD NOT include a Referer header field in a
   109		//    (non-secure) HTTP request if the referring page was
   110		//    transferred with a secure protocol."
   111		if lastReq.Scheme == "https" && newReq.Scheme == "http" {
   112			return ""
   113		}
   114		referer := lastReq.String()
   115		if lastReq.User != nil {
   116			// This is not very efficient, but is the best we can
   117			// do without:
   118			// - introducing a new method on URL
   119			// - creating a race condition
   120			// - copying the URL struct manually, which would cause
   121			//   maintenance problems down the line
   122			auth := lastReq.User.String() + "@"
   123			referer = strings.Replace(referer, auth, "", 1)
   124		}
   125		return referer
   126	}
   127	
   128	// Used in Send to implement io.ReadCloser by bundling together the
   129	// bufio.Reader through which we read the response, and the underlying
   130	// network connection.
   131	type readClose struct {
   132		io.Reader
   133		io.Closer
   134	}
   135	
   136	func (c *Client) send(req *Request) (*Response, error) {
   137		if c.Jar != nil {
   138			for _, cookie := range c.Jar.Cookies(req.URL) {
   139				req.AddCookie(cookie)
   140			}
   141		}
   142		resp, err := send(req, c.transport())
   143		if err != nil {
   144			return nil, err
   145		}
   146		if c.Jar != nil {
   147			if rc := resp.Cookies(); len(rc) > 0 {
   148				c.Jar.SetCookies(req.URL, rc)
   149			}
   150		}
   151		return resp, err
   152	}
   153	
   154	// Do sends an HTTP request and returns an HTTP response, following
   155	// policy (e.g. redirects, cookies, auth) as configured on the client.
   156	//
   157	// An error is returned if caused by client policy (such as
   158	// CheckRedirect), or if there was an HTTP protocol error.
   159	// A non-2xx response doesn't cause an error.
   160	//
   161	// When err is nil, resp always contains a non-nil resp.Body.
   162	//
   163	// Callers should close resp.Body when done reading from it. If
   164	// resp.Body is not closed, the Client's underlying RoundTripper
   165	// (typically Transport) may not be able to re-use a persistent TCP
   166	// connection to the server for a subsequent "keep-alive" request.
   167	//
   168	// The request Body, if non-nil, will be closed by the underlying
   169	// Transport, even on errors.
   170	//
   171	// Generally Get, Post, or PostForm will be used instead of Do.
   172	func (c *Client) Do(req *Request) (resp *Response, err error) {
   173		if req.Method == "GET" || req.Method == "HEAD" {
   174			return c.doFollowingRedirects(req, shouldRedirectGet)
   175		}
   176		if req.Method == "POST" || req.Method == "PUT" {
   177			return c.doFollowingRedirects(req, shouldRedirectPost)
   178		}
   179		return c.send(req)
   180	}
   181	
   182	func (c *Client) transport() RoundTripper {
   183		if c.Transport != nil {
   184			return c.Transport
   185		}
   186		return DefaultTransport
   187	}
   188	
   189	// send issues an HTTP request.
   190	// Caller should close resp.Body when done reading from it.
   191	func send(req *Request, t RoundTripper) (resp *Response, err error) {
   192		if t == nil {
   193			req.closeBody()
   194			return nil, errors.New("http: no Client.Transport or DefaultTransport")
   195		}
   196	
   197		if req.URL == nil {
   198			req.closeBody()
   199			return nil, errors.New("http: nil Request.URL")
   200		}
   201	
   202		if req.RequestURI != "" {
   203			req.closeBody()
   204			return nil, errors.New("http: Request.RequestURI can't be set in client requests.")
   205		}
   206	
   207		// Most the callers of send (Get, Post, et al) don't need
   208		// Headers, leaving it uninitialized.  We guarantee to the
   209		// Transport that this has been initialized, though.
   210		if req.Header == nil {
   211			req.Header = make(Header)
   212		}
   213	
   214		if u := req.URL.User; u != nil {
   215			username := u.Username()
   216			password, _ := u.Password()
   217			req.Header.Set("Authorization", "Basic "+basicAuth(username, password))
   218		}
   219		resp, err = t.RoundTrip(req)
   220		if err != nil {
   221			if resp != nil {
   222				log.Printf("RoundTripper returned a response & error; ignoring response")
   223			}
   224			return nil, err
   225		}
   226		return resp, nil
   227	}
   228	
   229	// See 2 (end of page 4) http://www.ietf.org/rfc/rfc2617.txt
   230	// "To receive authorization, the client sends the userid and password,
   231	// separated by a single colon (":") character, within a base64
   232	// encoded string in the credentials."
   233	// It is not meant to be urlencoded.
   234	func basicAuth(username, password string) string {
   235		auth := username + ":" + password
   236		return base64.StdEncoding.EncodeToString([]byte(auth))
   237	}
   238	
   239	// True if the specified HTTP status code is one for which the Get utility should
   240	// automatically redirect.
   241	func shouldRedirectGet(statusCode int) bool {
   242		switch statusCode {
   243		case StatusMovedPermanently, StatusFound, StatusSeeOther, StatusTemporaryRedirect:
   244			return true
   245		}
   246		return false
   247	}
   248	
   249	// True if the specified HTTP status code is one for which the Post utility should
   250	// automatically redirect.
   251	func shouldRedirectPost(statusCode int) bool {
   252		switch statusCode {
   253		case StatusFound, StatusSeeOther:
   254			return true
   255		}
   256		return false
   257	}
   258	
   259	// Get issues a GET to the specified URL.  If the response is one of the following
   260	// redirect codes, Get follows the redirect, up to a maximum of 10 redirects:
   261	//
   262	//    301 (Moved Permanently)
   263	//    302 (Found)
   264	//    303 (See Other)
   265	//    307 (Temporary Redirect)
   266	//
   267	// An error is returned if there were too many redirects or if there
   268	// was an HTTP protocol error. A non-2xx response doesn't cause an
   269	// error.
   270	//
   271	// When err is nil, resp always contains a non-nil resp.Body.
   272	// Caller should close resp.Body when done reading from it.
   273	//
   274	// Get is a wrapper around DefaultClient.Get.
   275	func Get(url string) (resp *Response, err error) {
   276		return DefaultClient.Get(url)
   277	}
   278	
   279	// Get issues a GET to the specified URL.  If the response is one of the
   280	// following redirect codes, Get follows the redirect after calling the
   281	// Client's CheckRedirect function.
   282	//
   283	//    301 (Moved Permanently)
   284	//    302 (Found)
   285	//    303 (See Other)
   286	//    307 (Temporary Redirect)
   287	//
   288	// An error is returned if the Client's CheckRedirect function fails
   289	// or if there was an HTTP protocol error. A non-2xx response doesn't
   290	// cause an error.
   291	//
   292	// When err is nil, resp always contains a non-nil resp.Body.
   293	// Caller should close resp.Body when done reading from it.
   294	func (c *Client) Get(url string) (resp *Response, err error) {
   295		req, err := NewRequest("GET", url, nil)
   296		if err != nil {
   297			return nil, err
   298		}
   299		return c.doFollowingRedirects(req, shouldRedirectGet)
   300	}
   301	
   302	func (c *Client) doFollowingRedirects(ireq *Request, shouldRedirect func(int) bool) (resp *Response, err error) {
   303		var base *url.URL
   304		redirectChecker := c.CheckRedirect
   305		if redirectChecker == nil {
   306			redirectChecker = defaultCheckRedirect
   307		}
   308		var via []*Request
   309	
   310		if ireq.URL == nil {
   311			ireq.closeBody()
   312			return nil, errors.New("http: nil Request.URL")
   313		}
   314	
   315		var reqmu sync.Mutex // guards req
   316		req := ireq
   317	
   318		var timer *time.Timer
   319		if c.Timeout > 0 {
   320			type canceler interface {
   321				CancelRequest(*Request)
   322			}
   323			tr, ok := c.transport().(canceler)
   324			if !ok {
   325				return nil, fmt.Errorf("net/http: Client Transport of type %T doesn't support CancelRequest; Timeout not supported", c.transport())
   326			}
   327			timer = time.AfterFunc(c.Timeout, func() {
   328				reqmu.Lock()
   329				defer reqmu.Unlock()
   330				tr.CancelRequest(req)
   331			})
   332		}
   333	
   334		urlStr := "" // next relative or absolute URL to fetch (after first request)
   335		redirectFailed := false
   336		for redirect := 0; ; redirect++ {
   337			if redirect != 0 {
   338				nreq := new(Request)
   339				nreq.Method = ireq.Method
   340				if ireq.Method == "POST" || ireq.Method == "PUT" {
   341					nreq.Method = "GET"
   342				}
   343				nreq.Header = make(Header)
   344				nreq.URL, err = base.Parse(urlStr)
   345				if err != nil {
   346					break
   347				}
   348				if len(via) > 0 {
   349					// Add the Referer header.
   350					lastReq := via[len(via)-1]
   351					if ref := refererForURL(lastReq.URL, nreq.URL); ref != "" {
   352						nreq.Header.Set("Referer", ref)
   353					}
   354	
   355					err = redirectChecker(nreq, via)
   356					if err != nil {
   357						redirectFailed = true
   358						break
   359					}
   360				}
   361				reqmu.Lock()
   362				req = nreq
   363				reqmu.Unlock()
   364			}
   365	
   366			urlStr = req.URL.String()
   367			if resp, err = c.send(req); err != nil {
   368				break
   369			}
   370	
   371			if shouldRedirect(resp.StatusCode) {
   372				// Read the body if small so underlying TCP connection will be re-used.
   373				// No need to check for errors: if it fails, Transport won't reuse it anyway.
   374				const maxBodySlurpSize = 2 << 10
   375				if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize {
   376					io.CopyN(ioutil.Discard, resp.Body, maxBodySlurpSize)
   377				}
   378				resp.Body.Close()
   379				if urlStr = resp.Header.Get("Location"); urlStr == "" {
   380					err = errors.New(fmt.Sprintf("%d response missing Location header", resp.StatusCode))
   381					break
   382				}
   383				base = req.URL
   384				via = append(via, req)
   385				continue
   386			}
   387			if timer != nil {
   388				resp.Body = &cancelTimerBody{timer, resp.Body}
   389			}
   390			return resp, nil
   391		}
   392	
   393		method := ireq.Method
   394		urlErr := &url.Error{
   395			Op:  method[0:1] + strings.ToLower(method[1:]),
   396			URL: urlStr,
   397			Err: err,
   398		}
   399	
   400		if redirectFailed {
   401			// Special case for Go 1 compatibility: return both the response
   402			// and an error if the CheckRedirect function failed.
   403			// See http://golang.org/issue/3795
   404			return resp, urlErr
   405		}
   406	
   407		if resp != nil {
   408			resp.Body.Close()
   409		}
   410		return nil, urlErr
   411	}
   412	
   413	func defaultCheckRedirect(req *Request, via []*Request) error {
   414		if len(via) >= 10 {
   415			return errors.New("stopped after 10 redirects")
   416		}
   417		return nil
   418	}
   419	
   420	// Post issues a POST to the specified URL.
   421	//
   422	// Caller should close resp.Body when done reading from it.
   423	//
   424	// Post is a wrapper around DefaultClient.Post
   425	func Post(url string, bodyType string, body io.Reader) (resp *Response, err error) {
   426		return DefaultClient.Post(url, bodyType, body)
   427	}
   428	
   429	// Post issues a POST to the specified URL.
   430	//
   431	// Caller should close resp.Body when done reading from it.
   432	//
   433	// If the provided body is also an io.Closer, it is closed after the
   434	// request.
   435	func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *Response, err error) {
   436		req, err := NewRequest("POST", url, body)
   437		if err != nil {
   438			return nil, err
   439		}
   440		req.Header.Set("Content-Type", bodyType)
   441		return c.doFollowingRedirects(req, shouldRedirectPost)
   442	}
   443	
   444	// PostForm issues a POST to the specified URL, with data's keys and
   445	// values URL-encoded as the request body.
   446	//
   447	// When err is nil, resp always contains a non-nil resp.Body.
   448	// Caller should close resp.Body when done reading from it.
   449	//
   450	// PostForm is a wrapper around DefaultClient.PostForm
   451	func PostForm(url string, data url.Values) (resp *Response, err error) {
   452		return DefaultClient.PostForm(url, data)
   453	}
   454	
   455	// PostForm issues a POST to the specified URL,
   456	// with data's keys and values urlencoded as the request body.
   457	//
   458	// When err is nil, resp always contains a non-nil resp.Body.
   459	// Caller should close resp.Body when done reading from it.
   460	func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) {
   461		return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
   462	}
   463	
   464	// Head issues a HEAD to the specified URL.  If the response is one of the
   465	// following redirect codes, Head follows the redirect after calling the
   466	// Client's CheckRedirect function.
   467	//
   468	//    301 (Moved Permanently)
   469	//    302 (Found)
   470	//    303 (See Other)
   471	//    307 (Temporary Redirect)
   472	//
   473	// Head is a wrapper around DefaultClient.Head
   474	func Head(url string) (resp *Response, err error) {
   475		return DefaultClient.Head(url)
   476	}
   477	
   478	// Head issues a HEAD to the specified URL.  If the response is one of the
   479	// following redirect codes, Head follows the redirect after calling the
   480	// Client's CheckRedirect function.
   481	//
   482	//    301 (Moved Permanently)
   483	//    302 (Found)
   484	//    303 (See Other)
   485	//    307 (Temporary Redirect)
   486	func (c *Client) Head(url string) (resp *Response, err error) {
   487		req, err := NewRequest("HEAD", url, nil)
   488		if err != nil {
   489			return nil, err
   490		}
   491		return c.doFollowingRedirects(req, shouldRedirectGet)
   492	}
   493	
   494	type cancelTimerBody struct {
   495		t  *time.Timer
   496		rc io.ReadCloser
   497	}
   498	
   499	func (b *cancelTimerBody) Read(p []byte) (n int, err error) {
   500		n, err = b.rc.Read(p)
   501		if err == io.EOF {
   502			b.t.Stop()
   503		}
   504		return
   505	}
   506	
   507	func (b *cancelTimerBody) Close() error {
   508		err := b.rc.Close()
   509		b.t.Stop()
   510		return err
   511	}
   512	

View as plain text