...
Run Format

Source file src/net/http/transport.go

     1	// Copyright 2011 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// HTTP client implementation. See RFC 2616.
     6	//
     7	// This is the low-level Transport implementation of RoundTripper.
     8	// The high-level interface is in client.go.
     9	
    10	package http
    11	
    12	import (
    13		"bufio"
    14		"compress/gzip"
    15		"crypto/tls"
    16		"errors"
    17		"fmt"
    18		"io"
    19		"log"
    20		"net"
    21		"net/url"
    22		"os"
    23		"strings"
    24		"sync"
    25		"time"
    26	)
    27	
    28	// DefaultTransport is the default implementation of Transport and is
    29	// used by DefaultClient. It establishes network connections as needed
    30	// and caches them for reuse by subsequent calls. It uses HTTP proxies
    31	// as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and
    32	// $no_proxy) environment variables.
    33	var DefaultTransport RoundTripper = &Transport{
    34		Proxy: ProxyFromEnvironment,
    35		Dial: (&net.Dialer{
    36			Timeout:   30 * time.Second,
    37			KeepAlive: 30 * time.Second,
    38		}).Dial,
    39		TLSHandshakeTimeout:   10 * time.Second,
    40		ExpectContinueTimeout: 1 * time.Second,
    41	}
    42	
    43	// DefaultMaxIdleConnsPerHost is the default value of Transport's
    44	// MaxIdleConnsPerHost.
    45	const DefaultMaxIdleConnsPerHost = 2
    46	
    47	// Transport is an implementation of RoundTripper that supports HTTP,
    48	// HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT).
    49	//
    50	// By default, Transport caches connections for future re-use.
    51	// This may leave many open connections when accessing many hosts.
    52	// This behavior can be managed using Transport's CloseIdleConnections method
    53	// and the MaxIdleConnsPerHost and DisableKeepAlives fields.
    54	//
    55	// Transports should be reused instead of created as needed.
    56	// Transports are safe for concurrent use by multiple goroutines.
    57	//
    58	// A Transport is a low-level primitive for making HTTP and HTTPS requests.
    59	// For high-level functionality, such as cookies and redirects, see Client.
    60	//
    61	// Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2
    62	// for HTTPS URLs, depending on whether the server supports HTTP/2.
    63	// See the package docs for more about HTTP/2.
    64	type Transport struct {
    65		idleMu     sync.Mutex
    66		wantIdle   bool // user has requested to close all idle conns
    67		idleConn   map[connectMethodKey][]*persistConn
    68		idleConnCh map[connectMethodKey]chan *persistConn
    69	
    70		reqMu       sync.Mutex
    71		reqCanceler map[*Request]func()
    72	
    73		altMu    sync.RWMutex
    74		altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper
    75	
    76		// Proxy specifies a function to return a proxy for a given
    77		// Request. If the function returns a non-nil error, the
    78		// request is aborted with the provided error.
    79		// If Proxy is nil or returns a nil *URL, no proxy is used.
    80		Proxy func(*Request) (*url.URL, error)
    81	
    82		// Dial specifies the dial function for creating unencrypted
    83		// TCP connections.
    84		// If Dial is nil, net.Dial is used.
    85		Dial func(network, addr string) (net.Conn, error)
    86	
    87		// DialTLS specifies an optional dial function for creating
    88		// TLS connections for non-proxied HTTPS requests.
    89		//
    90		// If DialTLS is nil, Dial and TLSClientConfig are used.
    91		//
    92		// If DialTLS is set, the Dial hook is not used for HTTPS
    93		// requests and the TLSClientConfig and TLSHandshakeTimeout
    94		// are ignored. The returned net.Conn is assumed to already be
    95		// past the TLS handshake.
    96		DialTLS func(network, addr string) (net.Conn, error)
    97	
    98		// TLSClientConfig specifies the TLS configuration to use with
    99		// tls.Client. If nil, the default configuration is used.
   100		TLSClientConfig *tls.Config
   101	
   102		// TLSHandshakeTimeout specifies the maximum amount of time waiting to
   103		// wait for a TLS handshake. Zero means no timeout.
   104		TLSHandshakeTimeout time.Duration
   105	
   106		// DisableKeepAlives, if true, prevents re-use of TCP connections
   107		// between different HTTP requests.
   108		DisableKeepAlives bool
   109	
   110		// DisableCompression, if true, prevents the Transport from
   111		// requesting compression with an "Accept-Encoding: gzip"
   112		// request header when the Request contains no existing
   113		// Accept-Encoding value. If the Transport requests gzip on
   114		// its own and gets a gzipped response, it's transparently
   115		// decoded in the Response.Body. However, if the user
   116		// explicitly requested gzip it is not automatically
   117		// uncompressed.
   118		DisableCompression bool
   119	
   120		// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
   121		// (keep-alive) to keep per-host.  If zero,
   122		// DefaultMaxIdleConnsPerHost is used.
   123		MaxIdleConnsPerHost int
   124	
   125		// ResponseHeaderTimeout, if non-zero, specifies the amount of
   126		// time to wait for a server's response headers after fully
   127		// writing the request (including its body, if any). This
   128		// time does not include the time to read the response body.
   129		ResponseHeaderTimeout time.Duration
   130	
   131		// ExpectContinueTimeout, if non-zero, specifies the amount of
   132		// time to wait for a server's first response headers after fully
   133		// writing the request headers if the request has an
   134		// "Expect: 100-continue" header. Zero means no timeout.
   135		// This time does not include the time to send the request header.
   136		ExpectContinueTimeout time.Duration
   137	
   138		// TLSNextProto specifies how the Transport switches to an
   139		// alternate protocol (such as HTTP/2) after a TLS NPN/ALPN
   140		// protocol negotiation.  If Transport dials an TLS connection
   141		// with a non-empty protocol name and TLSNextProto contains a
   142		// map entry for that key (such as "h2"), then the func is
   143		// called with the request's authority (such as "example.com"
   144		// or "example.com:1234") and the TLS connection. The function
   145		// must return a RoundTripper that then handles the request.
   146		// If TLSNextProto is nil, HTTP/2 support is enabled automatically.
   147		TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper
   148	
   149		// nextProtoOnce guards initialization of TLSNextProto and
   150		// h2transport (via onceSetNextProtoDefaults)
   151		nextProtoOnce sync.Once
   152		h2transport   *http2Transport // non-nil if http2 wired up
   153	
   154		// TODO: tunable on global max cached connections
   155		// TODO: tunable on timeout on cached connections
   156		// TODO: tunable on max per-host TCP dials in flight (Issue 13957)
   157	}
   158	
   159	// onceSetNextProtoDefaults initializes TLSNextProto.
   160	// It must be called via t.nextProtoOnce.Do.
   161	func (t *Transport) onceSetNextProtoDefaults() {
   162		if strings.Contains(os.Getenv("GODEBUG"), "http2client=0") {
   163			return
   164		}
   165		if t.TLSNextProto != nil {
   166			// This is the documented way to disable http2 on a
   167			// Transport.
   168			return
   169		}
   170		if t.TLSClientConfig != nil {
   171			// Be conservative for now (for Go 1.6) at least and
   172			// don't automatically enable http2 if they've
   173			// specified a custom TLS config. Let them opt-in
   174			// themselves via http2.ConfigureTransport so we don't
   175			// surprise them by modifying their tls.Config.
   176			// Issue 14275.
   177			return
   178		}
   179		if t.ExpectContinueTimeout != 0 && t != DefaultTransport {
   180			// ExpectContinueTimeout is unsupported in http2, so
   181			// if they explicitly asked for it (as opposed to just
   182			// using the DefaultTransport, which sets it), then
   183			// disable http2 for now.
   184			//
   185			// Issue 13851. (and changed in Issue 14391)
   186			return
   187		}
   188		t2, err := http2configureTransport(t)
   189		if err != nil {
   190			log.Printf("Error enabling Transport HTTP/2 support: %v", err)
   191		} else {
   192			t.h2transport = t2
   193		}
   194	}
   195	
   196	// ProxyFromEnvironment returns the URL of the proxy to use for a
   197	// given request, as indicated by the environment variables
   198	// HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions
   199	// thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https
   200	// requests.
   201	//
   202	// The environment values may be either a complete URL or a
   203	// "host[:port]", in which case the "http" scheme is assumed.
   204	// An error is returned if the value is a different form.
   205	//
   206	// A nil URL and nil error are returned if no proxy is defined in the
   207	// environment, or a proxy should not be used for the given request,
   208	// as defined by NO_PROXY.
   209	//
   210	// As a special case, if req.URL.Host is "localhost" (with or without
   211	// a port number), then a nil URL and nil error will be returned.
   212	func ProxyFromEnvironment(req *Request) (*url.URL, error) {
   213		var proxy string
   214		if req.URL.Scheme == "https" {
   215			proxy = httpsProxyEnv.Get()
   216		}
   217		if proxy == "" {
   218			proxy = httpProxyEnv.Get()
   219		}
   220		if proxy == "" {
   221			return nil, nil
   222		}
   223		if !useProxy(canonicalAddr(req.URL)) {
   224			return nil, nil
   225		}
   226		proxyURL, err := url.Parse(proxy)
   227		if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") {
   228			// proxy was bogus. Try prepending "http://" to it and
   229			// see if that parses correctly. If not, we fall
   230			// through and complain about the original one.
   231			if proxyURL, err := url.Parse("http://" + proxy); err == nil {
   232				return proxyURL, nil
   233			}
   234		}
   235		if err != nil {
   236			return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
   237		}
   238		return proxyURL, nil
   239	}
   240	
   241	// ProxyURL returns a proxy function (for use in a Transport)
   242	// that always returns the same URL.
   243	func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) {
   244		return func(*Request) (*url.URL, error) {
   245			return fixedURL, nil
   246		}
   247	}
   248	
   249	// transportRequest is a wrapper around a *Request that adds
   250	// optional extra headers to write.
   251	type transportRequest struct {
   252		*Request        // original request, not to be mutated
   253		extra    Header // extra headers to write, or nil
   254	}
   255	
   256	func (tr *transportRequest) extraHeaders() Header {
   257		if tr.extra == nil {
   258			tr.extra = make(Header)
   259		}
   260		return tr.extra
   261	}
   262	
   263	// RoundTrip implements the RoundTripper interface.
   264	//
   265	// For higher-level HTTP client support (such as handling of cookies
   266	// and redirects), see Get, Post, and the Client type.
   267	func (t *Transport) RoundTrip(req *Request) (*Response, error) {
   268		t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
   269		if req.URL == nil {
   270			req.closeBody()
   271			return nil, errors.New("http: nil Request.URL")
   272		}
   273		if req.Header == nil {
   274			req.closeBody()
   275			return nil, errors.New("http: nil Request.Header")
   276		}
   277		// TODO(bradfitz): switch to atomic.Value for this map instead of RWMutex
   278		t.altMu.RLock()
   279		altRT := t.altProto[req.URL.Scheme]
   280		t.altMu.RUnlock()
   281		if altRT != nil {
   282			if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol {
   283				return resp, err
   284			}
   285		}
   286		if s := req.URL.Scheme; s != "http" && s != "https" {
   287			req.closeBody()
   288			return nil, &badStringError{"unsupported protocol scheme", s}
   289		}
   290		if req.Method != "" && !validMethod(req.Method) {
   291			return nil, fmt.Errorf("net/http: invalid method %q", req.Method)
   292		}
   293		if req.URL.Host == "" {
   294			req.closeBody()
   295			return nil, errors.New("http: no Host in request URL")
   296		}
   297	
   298		for {
   299			// treq gets modified by roundTrip, so we need to recreate for each retry.
   300			treq := &transportRequest{Request: req}
   301			cm, err := t.connectMethodForRequest(treq)
   302			if err != nil {
   303				req.closeBody()
   304				return nil, err
   305			}
   306	
   307			// Get the cached or newly-created connection to either the
   308			// host (for http or https), the http proxy, or the http proxy
   309			// pre-CONNECTed to https server.  In any case, we'll be ready
   310			// to send it requests.
   311			pconn, err := t.getConn(req, cm)
   312			if err != nil {
   313				t.setReqCanceler(req, nil)
   314				req.closeBody()
   315				return nil, err
   316			}
   317	
   318			var resp *Response
   319			if pconn.alt != nil {
   320				// HTTP/2 path.
   321				t.setReqCanceler(req, nil) // not cancelable with CancelRequest
   322				resp, err = pconn.alt.RoundTrip(req)
   323			} else {
   324				resp, err = pconn.roundTrip(treq)
   325			}
   326			if err == nil {
   327				return resp, nil
   328			}
   329			if err := checkTransportResend(err, req, pconn); err != nil {
   330				return nil, err
   331			}
   332			testHookRoundTripRetried()
   333		}
   334	}
   335	
   336	// checkTransportResend checks whether a failed HTTP request can be
   337	// resent on a new connection. The non-nil input error is the error from
   338	// roundTrip, which might be wrapped in a beforeRespHeaderError error.
   339	//
   340	// The return value is err or the unwrapped error inside a
   341	// beforeRespHeaderError.
   342	func checkTransportResend(err error, req *Request, pconn *persistConn) error {
   343		brhErr, ok := err.(beforeRespHeaderError)
   344		if !ok {
   345			return err
   346		}
   347		err = brhErr.error // unwrap the custom error in case we return it
   348		if err != errMissingHost && pconn.isReused() && req.isReplayable() {
   349			// If we try to reuse a connection that the server is in the process of
   350			// closing, we may end up successfully writing out our request (or a
   351			// portion of our request) only to find a connection error when we try to
   352			// read from (or finish writing to) the socket.
   353	
   354			// There can be a race between the socket pool checking whether a socket
   355			// is still connected, receiving the FIN, and sending/reading data on a
   356			// reused socket. If we receive the FIN between the connectedness check
   357			// and writing/reading from the socket, we may first learn the socket is
   358			// disconnected when we get a ERR_SOCKET_NOT_CONNECTED. This will most
   359			// likely happen when trying to retrieve its IP address. See
   360			// http://crbug.com/105824 for more details.
   361	
   362			// We resend a request only if we reused a keep-alive connection and did
   363			// not yet receive any header data. This automatically prevents an
   364			// infinite resend loop because we'll run out of the cached keep-alive
   365			// connections eventually.
   366			return nil
   367		}
   368		return err
   369	}
   370	
   371	// ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol.
   372	var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol")
   373	
   374	// RegisterProtocol registers a new protocol with scheme.
   375	// The Transport will pass requests using the given scheme to rt.
   376	// It is rt's responsibility to simulate HTTP request semantics.
   377	//
   378	// RegisterProtocol can be used by other packages to provide
   379	// implementations of protocol schemes like "ftp" or "file".
   380	//
   381	// If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will
   382	// handle the RoundTrip itself for that one request, as if the
   383	// protocol were not registered.
   384	func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) {
   385		t.altMu.Lock()
   386		defer t.altMu.Unlock()
   387		if t.altProto == nil {
   388			t.altProto = make(map[string]RoundTripper)
   389		}
   390		if _, exists := t.altProto[scheme]; exists {
   391			panic("protocol " + scheme + " already registered")
   392		}
   393		t.altProto[scheme] = rt
   394	}
   395	
   396	// CloseIdleConnections closes any connections which were previously
   397	// connected from previous requests but are now sitting idle in
   398	// a "keep-alive" state. It does not interrupt any connections currently
   399	// in use.
   400	func (t *Transport) CloseIdleConnections() {
   401		t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
   402		t.idleMu.Lock()
   403		m := t.idleConn
   404		t.idleConn = nil
   405		t.idleConnCh = nil
   406		t.wantIdle = true
   407		t.idleMu.Unlock()
   408		for _, conns := range m {
   409			for _, pconn := range conns {
   410				pconn.close(errCloseIdleConns)
   411			}
   412		}
   413		if t2 := t.h2transport; t2 != nil {
   414			t2.CloseIdleConnections()
   415		}
   416	}
   417	
   418	// CancelRequest cancels an in-flight request by closing its connection.
   419	// CancelRequest should only be called after RoundTrip has returned.
   420	//
   421	// Deprecated: Use Request.Cancel instead. CancelRequest can not cancel
   422	// HTTP/2 requests.
   423	func (t *Transport) CancelRequest(req *Request) {
   424		t.reqMu.Lock()
   425		cancel := t.reqCanceler[req]
   426		delete(t.reqCanceler, req)
   427		t.reqMu.Unlock()
   428		if cancel != nil {
   429			cancel()
   430		}
   431	}
   432	
   433	//
   434	// Private implementation past this point.
   435	//
   436	
   437	var (
   438		httpProxyEnv = &envOnce{
   439			names: []string{"HTTP_PROXY", "http_proxy"},
   440		}
   441		httpsProxyEnv = &envOnce{
   442			names: []string{"HTTPS_PROXY", "https_proxy"},
   443		}
   444		noProxyEnv = &envOnce{
   445			names: []string{"NO_PROXY", "no_proxy"},
   446		}
   447	)
   448	
   449	// envOnce looks up an environment variable (optionally by multiple
   450	// names) once. It mitigates expensive lookups on some platforms
   451	// (e.g. Windows).
   452	type envOnce struct {
   453		names []string
   454		once  sync.Once
   455		val   string
   456	}
   457	
   458	func (e *envOnce) Get() string {
   459		e.once.Do(e.init)
   460		return e.val
   461	}
   462	
   463	func (e *envOnce) init() {
   464		for _, n := range e.names {
   465			e.val = os.Getenv(n)
   466			if e.val != "" {
   467				return
   468			}
   469		}
   470	}
   471	
   472	// reset is used by tests
   473	func (e *envOnce) reset() {
   474		e.once = sync.Once{}
   475		e.val = ""
   476	}
   477	
   478	func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) {
   479		cm.targetScheme = treq.URL.Scheme
   480		cm.targetAddr = canonicalAddr(treq.URL)
   481		if t.Proxy != nil {
   482			cm.proxyURL, err = t.Proxy(treq.Request)
   483		}
   484		return cm, err
   485	}
   486	
   487	// proxyAuth returns the Proxy-Authorization header to set
   488	// on requests, if applicable.
   489	func (cm *connectMethod) proxyAuth() string {
   490		if cm.proxyURL == nil {
   491			return ""
   492		}
   493		if u := cm.proxyURL.User; u != nil {
   494			username := u.Username()
   495			password, _ := u.Password()
   496			return "Basic " + basicAuth(username, password)
   497		}
   498		return ""
   499	}
   500	
   501	// error values for debugging and testing, not seen by users.
   502	var (
   503		errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled")
   504		errConnBroken         = errors.New("http: putIdleConn: connection is in bad state")
   505		errWantIdle           = errors.New("http: putIdleConn: CloseIdleConnections was called")
   506		errTooManyIdle        = errors.New("http: putIdleConn: too many idle connections")
   507		errCloseIdleConns     = errors.New("http: CloseIdleConnections called")
   508		errReadLoopExiting    = errors.New("http: persistConn.readLoop exiting")
   509		errServerClosedIdle   = errors.New("http: server closed idle conn")
   510	)
   511	
   512	func (t *Transport) putOrCloseIdleConn(pconn *persistConn) {
   513		if err := t.tryPutIdleConn(pconn); err != nil {
   514			pconn.close(err)
   515		}
   516	}
   517	
   518	// tryPutIdleConn adds pconn to the list of idle persistent connections awaiting
   519	// a new request.
   520	// If pconn is no longer needed or not in a good state, tryPutIdleConn returns
   521	// an error explaining why it wasn't registered.
   522	// tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that.
   523	func (t *Transport) tryPutIdleConn(pconn *persistConn) error {
   524		if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 {
   525			return errKeepAlivesDisabled
   526		}
   527		if pconn.isBroken() {
   528			return errConnBroken
   529		}
   530		key := pconn.cacheKey
   531		max := t.MaxIdleConnsPerHost
   532		if max == 0 {
   533			max = DefaultMaxIdleConnsPerHost
   534		}
   535		pconn.markReused()
   536		t.idleMu.Lock()
   537	
   538		waitingDialer := t.idleConnCh[key]
   539		select {
   540		case waitingDialer <- pconn:
   541			// We're done with this pconn and somebody else is
   542			// currently waiting for a conn of this type (they're
   543			// actively dialing, but this conn is ready
   544			// first). Chrome calls this socket late binding.  See
   545			// https://insouciant.org/tech/connection-management-in-chromium/
   546			t.idleMu.Unlock()
   547			return nil
   548		default:
   549			if waitingDialer != nil {
   550				// They had populated this, but their dial won
   551				// first, so we can clean up this map entry.
   552				delete(t.idleConnCh, key)
   553			}
   554		}
   555		if t.wantIdle {
   556			t.idleMu.Unlock()
   557			return errWantIdle
   558		}
   559		if t.idleConn == nil {
   560			t.idleConn = make(map[connectMethodKey][]*persistConn)
   561		}
   562		if len(t.idleConn[key]) >= max {
   563			t.idleMu.Unlock()
   564			return errTooManyIdle
   565		}
   566		for _, exist := range t.idleConn[key] {
   567			if exist == pconn {
   568				log.Fatalf("dup idle pconn %p in freelist", pconn)
   569			}
   570		}
   571		t.idleConn[key] = append(t.idleConn[key], pconn)
   572		t.idleMu.Unlock()
   573		return nil
   574	}
   575	
   576	// getIdleConnCh returns a channel to receive and return idle
   577	// persistent connection for the given connectMethod.
   578	// It may return nil, if persistent connections are not being used.
   579	func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn {
   580		if t.DisableKeepAlives {
   581			return nil
   582		}
   583		key := cm.key()
   584		t.idleMu.Lock()
   585		defer t.idleMu.Unlock()
   586		t.wantIdle = false
   587		if t.idleConnCh == nil {
   588			t.idleConnCh = make(map[connectMethodKey]chan *persistConn)
   589		}
   590		ch, ok := t.idleConnCh[key]
   591		if !ok {
   592			ch = make(chan *persistConn)
   593			t.idleConnCh[key] = ch
   594		}
   595		return ch
   596	}
   597	
   598	func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn) {
   599		key := cm.key()
   600		t.idleMu.Lock()
   601		defer t.idleMu.Unlock()
   602		if t.idleConn == nil {
   603			return nil
   604		}
   605		for {
   606			pconns, ok := t.idleConn[key]
   607			if !ok {
   608				return nil
   609			}
   610			if len(pconns) == 1 {
   611				pconn = pconns[0]
   612				delete(t.idleConn, key)
   613			} else {
   614				// 2 or more cached connections; pop last
   615				// TODO: queue?
   616				pconn = pconns[len(pconns)-1]
   617				t.idleConn[key] = pconns[:len(pconns)-1]
   618			}
   619			if !pconn.isBroken() {
   620				return
   621			}
   622		}
   623	}
   624	
   625	func (t *Transport) setReqCanceler(r *Request, fn func()) {
   626		t.reqMu.Lock()
   627		defer t.reqMu.Unlock()
   628		if t.reqCanceler == nil {
   629			t.reqCanceler = make(map[*Request]func())
   630		}
   631		if fn != nil {
   632			t.reqCanceler[r] = fn
   633		} else {
   634			delete(t.reqCanceler, r)
   635		}
   636	}
   637	
   638	// replaceReqCanceler replaces an existing cancel function. If there is no cancel function
   639	// for the request, we don't set the function and return false.
   640	// Since CancelRequest will clear the canceler, we can use the return value to detect if
   641	// the request was canceled since the last setReqCancel call.
   642	func (t *Transport) replaceReqCanceler(r *Request, fn func()) bool {
   643		t.reqMu.Lock()
   644		defer t.reqMu.Unlock()
   645		_, ok := t.reqCanceler[r]
   646		if !ok {
   647			return false
   648		}
   649		if fn != nil {
   650			t.reqCanceler[r] = fn
   651		} else {
   652			delete(t.reqCanceler, r)
   653		}
   654		return true
   655	}
   656	
   657	func (t *Transport) dial(network, addr string) (net.Conn, error) {
   658		if t.Dial != nil {
   659			c, err := t.Dial(network, addr)
   660			if c == nil && err == nil {
   661				err = errors.New("net/http: Transport.Dial hook returned (nil, nil)")
   662			}
   663			return c, err
   664		}
   665		return net.Dial(network, addr)
   666	}
   667	
   668	// getConn dials and creates a new persistConn to the target as
   669	// specified in the connectMethod.  This includes doing a proxy CONNECT
   670	// and/or setting up TLS.  If this doesn't return an error, the persistConn
   671	// is ready to write requests to.
   672	func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error) {
   673		if pc := t.getIdleConn(cm); pc != nil {
   674			// set request canceler to some non-nil function so we
   675			// can detect whether it was cleared between now and when
   676			// we enter roundTrip
   677			t.setReqCanceler(req, func() {})
   678			return pc, nil
   679		}
   680	
   681		type dialRes struct {
   682			pc  *persistConn
   683			err error
   684		}
   685		dialc := make(chan dialRes)
   686	
   687		// Copy these hooks so we don't race on the postPendingDial in
   688		// the goroutine we launch. Issue 11136.
   689		testHookPrePendingDial := testHookPrePendingDial
   690		testHookPostPendingDial := testHookPostPendingDial
   691	
   692		handlePendingDial := func() {
   693			testHookPrePendingDial()
   694			go func() {
   695				if v := <-dialc; v.err == nil {
   696					t.putOrCloseIdleConn(v.pc)
   697				}
   698				testHookPostPendingDial()
   699			}()
   700		}
   701	
   702		cancelc := make(chan struct{})
   703		t.setReqCanceler(req, func() { close(cancelc) })
   704	
   705		go func() {
   706			pc, err := t.dialConn(cm)
   707			dialc <- dialRes{pc, err}
   708		}()
   709	
   710		idleConnCh := t.getIdleConnCh(cm)
   711		select {
   712		case v := <-dialc:
   713			// Our dial finished.
   714			return v.pc, v.err
   715		case pc := <-idleConnCh:
   716			// Another request finished first and its net.Conn
   717			// became available before our dial. Or somebody
   718			// else's dial that they didn't use.
   719			// But our dial is still going, so give it away
   720			// when it finishes:
   721			handlePendingDial()
   722			return pc, nil
   723		case <-req.Cancel:
   724			handlePendingDial()
   725			return nil, errRequestCanceledConn
   726		case <-cancelc:
   727			handlePendingDial()
   728			return nil, errRequestCanceledConn
   729		}
   730	}
   731	
   732	func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) {
   733		pconn := &persistConn{
   734			t:          t,
   735			cacheKey:   cm.key(),
   736			reqch:      make(chan requestAndChan, 1),
   737			writech:    make(chan writeRequest, 1),
   738			closech:    make(chan struct{}),
   739			writeErrCh: make(chan error, 1),
   740		}
   741		tlsDial := t.DialTLS != nil && cm.targetScheme == "https" && cm.proxyURL == nil
   742		if tlsDial {
   743			var err error
   744			pconn.conn, err = t.DialTLS("tcp", cm.addr())
   745			if err != nil {
   746				return nil, err
   747			}
   748			if pconn.conn == nil {
   749				return nil, errors.New("net/http: Transport.DialTLS returned (nil, nil)")
   750			}
   751			if tc, ok := pconn.conn.(*tls.Conn); ok {
   752				// Handshake here, in case DialTLS didn't. TLSNextProto below
   753				// depends on it for knowing the connection state.
   754				if err := tc.Handshake(); err != nil {
   755					go pconn.conn.Close()
   756					return nil, err
   757				}
   758				cs := tc.ConnectionState()
   759				pconn.tlsState = &cs
   760			}
   761		} else {
   762			conn, err := t.dial("tcp", cm.addr())
   763			if err != nil {
   764				if cm.proxyURL != nil {
   765					err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err)
   766				}
   767				return nil, err
   768			}
   769			pconn.conn = conn
   770		}
   771	
   772		// Proxy setup.
   773		switch {
   774		case cm.proxyURL == nil:
   775			// Do nothing. Not using a proxy.
   776		case cm.targetScheme == "http":
   777			pconn.isProxy = true
   778			if pa := cm.proxyAuth(); pa != "" {
   779				pconn.mutateHeaderFunc = func(h Header) {
   780					h.Set("Proxy-Authorization", pa)
   781				}
   782			}
   783		case cm.targetScheme == "https":
   784			conn := pconn.conn
   785			connectReq := &Request{
   786				Method: "CONNECT",
   787				URL:    &url.URL{Opaque: cm.targetAddr},
   788				Host:   cm.targetAddr,
   789				Header: make(Header),
   790			}
   791			if pa := cm.proxyAuth(); pa != "" {
   792				connectReq.Header.Set("Proxy-Authorization", pa)
   793			}
   794			connectReq.Write(conn)
   795	
   796			// Read response.
   797			// Okay to use and discard buffered reader here, because
   798			// TLS server will not speak until spoken to.
   799			br := bufio.NewReader(conn)
   800			resp, err := ReadResponse(br, connectReq)
   801			if err != nil {
   802				conn.Close()
   803				return nil, err
   804			}
   805			if resp.StatusCode != 200 {
   806				f := strings.SplitN(resp.Status, " ", 2)
   807				conn.Close()
   808				return nil, errors.New(f[1])
   809			}
   810		}
   811	
   812		if cm.targetScheme == "https" && !tlsDial {
   813			// Initiate TLS and check remote host name against certificate.
   814			cfg := cloneTLSClientConfig(t.TLSClientConfig)
   815			if cfg.ServerName == "" {
   816				cfg.ServerName = cm.tlsHost()
   817			}
   818			plainConn := pconn.conn
   819			tlsConn := tls.Client(plainConn, cfg)
   820			errc := make(chan error, 2)
   821			var timer *time.Timer // for canceling TLS handshake
   822			if d := t.TLSHandshakeTimeout; d != 0 {
   823				timer = time.AfterFunc(d, func() {
   824					errc <- tlsHandshakeTimeoutError{}
   825				})
   826			}
   827			go func() {
   828				err := tlsConn.Handshake()
   829				if timer != nil {
   830					timer.Stop()
   831				}
   832				errc <- err
   833			}()
   834			if err := <-errc; err != nil {
   835				plainConn.Close()
   836				return nil, err
   837			}
   838			if !cfg.InsecureSkipVerify {
   839				if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil {
   840					plainConn.Close()
   841					return nil, err
   842				}
   843			}
   844			cs := tlsConn.ConnectionState()
   845			pconn.tlsState = &cs
   846			pconn.conn = tlsConn
   847		}
   848	
   849		if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" {
   850			if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok {
   851				return &persistConn{alt: next(cm.targetAddr, pconn.conn.(*tls.Conn))}, nil
   852			}
   853		}
   854	
   855		pconn.br = bufio.NewReader(noteEOFReader{pconn.conn, &pconn.sawEOF})
   856		pconn.bw = bufio.NewWriter(pconn.conn)
   857		go pconn.readLoop()
   858		go pconn.writeLoop()
   859		return pconn, nil
   860	}
   861	
   862	// useProxy reports whether requests to addr should use a proxy,
   863	// according to the NO_PROXY or no_proxy environment variable.
   864	// addr is always a canonicalAddr with a host and port.
   865	func useProxy(addr string) bool {
   866		if len(addr) == 0 {
   867			return true
   868		}
   869		host, _, err := net.SplitHostPort(addr)
   870		if err != nil {
   871			return false
   872		}
   873		if host == "localhost" {
   874			return false
   875		}
   876		if ip := net.ParseIP(host); ip != nil {
   877			if ip.IsLoopback() {
   878				return false
   879			}
   880		}
   881	
   882		no_proxy := noProxyEnv.Get()
   883		if no_proxy == "*" {
   884			return false
   885		}
   886	
   887		addr = strings.ToLower(strings.TrimSpace(addr))
   888		if hasPort(addr) {
   889			addr = addr[:strings.LastIndex(addr, ":")]
   890		}
   891	
   892		for _, p := range strings.Split(no_proxy, ",") {
   893			p = strings.ToLower(strings.TrimSpace(p))
   894			if len(p) == 0 {
   895				continue
   896			}
   897			if hasPort(p) {
   898				p = p[:strings.LastIndex(p, ":")]
   899			}
   900			if addr == p {
   901				return false
   902			}
   903			if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) {
   904				// no_proxy ".foo.com" matches "bar.foo.com" or "foo.com"
   905				return false
   906			}
   907			if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' {
   908				// no_proxy "foo.com" matches "bar.foo.com"
   909				return false
   910			}
   911		}
   912		return true
   913	}
   914	
   915	// connectMethod is the map key (in its String form) for keeping persistent
   916	// TCP connections alive for subsequent HTTP requests.
   917	//
   918	// A connect method may be of the following types:
   919	//
   920	// Cache key form                Description
   921	// -----------------             -------------------------
   922	// |http|foo.com                 http directly to server, no proxy
   923	// |https|foo.com                https directly to server, no proxy
   924	// http://proxy.com|https|foo.com  http to proxy, then CONNECT to foo.com
   925	// http://proxy.com|http           http to proxy, http to anywhere after that
   926	//
   927	// Note: no support to https to the proxy yet.
   928	//
   929	type connectMethod struct {
   930		proxyURL     *url.URL // nil for no proxy, else full proxy URL
   931		targetScheme string   // "http" or "https"
   932		targetAddr   string   // Not used if proxy + http targetScheme (4th example in table)
   933	}
   934	
   935	func (cm *connectMethod) key() connectMethodKey {
   936		proxyStr := ""
   937		targetAddr := cm.targetAddr
   938		if cm.proxyURL != nil {
   939			proxyStr = cm.proxyURL.String()
   940			if cm.targetScheme == "http" {
   941				targetAddr = ""
   942			}
   943		}
   944		return connectMethodKey{
   945			proxy:  proxyStr,
   946			scheme: cm.targetScheme,
   947			addr:   targetAddr,
   948		}
   949	}
   950	
   951	// addr returns the first hop "host:port" to which we need to TCP connect.
   952	func (cm *connectMethod) addr() string {
   953		if cm.proxyURL != nil {
   954			return canonicalAddr(cm.proxyURL)
   955		}
   956		return cm.targetAddr
   957	}
   958	
   959	// tlsHost returns the host name to match against the peer's
   960	// TLS certificate.
   961	func (cm *connectMethod) tlsHost() string {
   962		h := cm.targetAddr
   963		if hasPort(h) {
   964			h = h[:strings.LastIndex(h, ":")]
   965		}
   966		return h
   967	}
   968	
   969	// connectMethodKey is the map key version of connectMethod, with a
   970	// stringified proxy URL (or the empty string) instead of a pointer to
   971	// a URL.
   972	type connectMethodKey struct {
   973		proxy, scheme, addr string
   974	}
   975	
   976	func (k connectMethodKey) String() string {
   977		// Only used by tests.
   978		return fmt.Sprintf("%s|%s|%s", k.proxy, k.scheme, k.addr)
   979	}
   980	
   981	// persistConn wraps a connection, usually a persistent one
   982	// (but may be used for non-keep-alive requests as well)
   983	type persistConn struct {
   984		// alt optionally specifies the TLS NextProto RoundTripper.
   985		// This is used for HTTP/2 today and future protocol laters.
   986		// If it's non-nil, the rest of the fields are unused.
   987		alt RoundTripper
   988	
   989		t        *Transport
   990		cacheKey connectMethodKey
   991		conn     net.Conn
   992		tlsState *tls.ConnectionState
   993		br       *bufio.Reader       // from conn
   994		sawEOF   bool                // whether we've seen EOF from conn; owned by readLoop
   995		bw       *bufio.Writer       // to conn
   996		reqch    chan requestAndChan // written by roundTrip; read by readLoop
   997		writech  chan writeRequest   // written by roundTrip; read by writeLoop
   998		closech  chan struct{}       // closed when conn closed
   999		isProxy  bool
  1000		// writeErrCh passes the request write error (usually nil)
  1001		// from the writeLoop goroutine to the readLoop which passes
  1002		// it off to the res.Body reader, which then uses it to decide
  1003		// whether or not a connection can be reused. Issue 7569.
  1004		writeErrCh chan error
  1005	
  1006		lk                   sync.Mutex // guards following fields
  1007		numExpectedResponses int
  1008		closed               error // set non-nil when conn is closed, before closech is closed
  1009		broken               bool  // an error has happened on this connection; marked broken so it's not reused.
  1010		canceled             bool  // whether this conn was broken due a CancelRequest
  1011		reused               bool  // whether conn has had successful request/response and is being reused.
  1012		// mutateHeaderFunc is an optional func to modify extra
  1013		// headers on each outbound request before it's written. (the
  1014		// original Request given to RoundTrip is not modified)
  1015		mutateHeaderFunc func(Header)
  1016	}
  1017	
  1018	// isBroken reports whether this connection is in a known broken state.
  1019	func (pc *persistConn) isBroken() bool {
  1020		pc.lk.Lock()
  1021		b := pc.broken
  1022		pc.lk.Unlock()
  1023		return b
  1024	}
  1025	
  1026	// isCanceled reports whether this connection was closed due to CancelRequest.
  1027	func (pc *persistConn) isCanceled() bool {
  1028		pc.lk.Lock()
  1029		defer pc.lk.Unlock()
  1030		return pc.canceled
  1031	}
  1032	
  1033	// isReused reports whether this connection is in a known broken state.
  1034	func (pc *persistConn) isReused() bool {
  1035		pc.lk.Lock()
  1036		r := pc.reused
  1037		pc.lk.Unlock()
  1038		return r
  1039	}
  1040	
  1041	func (pc *persistConn) cancelRequest() {
  1042		pc.lk.Lock()
  1043		defer pc.lk.Unlock()
  1044		pc.canceled = true
  1045		pc.closeLocked(errRequestCanceled)
  1046	}
  1047	
  1048	func (pc *persistConn) readLoop() {
  1049		closeErr := errReadLoopExiting // default value, if not changed below
  1050		defer func() { pc.close(closeErr) }()
  1051	
  1052		tryPutIdleConn := func() bool {
  1053			if err := pc.t.tryPutIdleConn(pc); err != nil {
  1054				closeErr = err
  1055				return false
  1056			}
  1057			return true
  1058		}
  1059	
  1060		// eofc is used to block caller goroutines reading from Response.Body
  1061		// at EOF until this goroutines has (potentially) added the connection
  1062		// back to the idle pool.
  1063		eofc := make(chan struct{})
  1064		defer close(eofc) // unblock reader on errors
  1065	
  1066		// Read this once, before loop starts. (to avoid races in tests)
  1067		testHookMu.Lock()
  1068		testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead
  1069		testHookMu.Unlock()
  1070	
  1071		alive := true
  1072		for alive {
  1073			_, err := pc.br.Peek(1)
  1074			if err != nil {
  1075				err = beforeRespHeaderError{err}
  1076			}
  1077	
  1078			pc.lk.Lock()
  1079			if pc.numExpectedResponses == 0 {
  1080				pc.readLoopPeekFailLocked(err)
  1081				pc.lk.Unlock()
  1082				return
  1083			}
  1084			pc.lk.Unlock()
  1085	
  1086			rc := <-pc.reqch
  1087	
  1088			var resp *Response
  1089			if err == nil {
  1090				resp, err = pc.readResponse(rc)
  1091			}
  1092	
  1093			if err != nil {
  1094				// If we won't be able to retry this request later (from the
  1095				// roundTrip goroutine), mark it as done now.
  1096				// BEFORE the send on rc.ch, as the client might re-use the
  1097				// same *Request pointer, and we don't want to set call
  1098				// t.setReqCanceler from this persistConn while the Transport
  1099				// potentially spins up a different persistConn for the
  1100				// caller's subsequent request.
  1101				if checkTransportResend(err, rc.req, pc) != nil {
  1102					pc.t.setReqCanceler(rc.req, nil)
  1103				}
  1104				select {
  1105				case rc.ch <- responseAndError{err: err}:
  1106				case <-rc.callerGone:
  1107					return
  1108				}
  1109				return
  1110			}
  1111	
  1112			pc.lk.Lock()
  1113			pc.numExpectedResponses--
  1114			pc.lk.Unlock()
  1115	
  1116			hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0
  1117	
  1118			if resp.Close || rc.req.Close || resp.StatusCode <= 199 {
  1119				// Don't do keep-alive on error if either party requested a close
  1120				// or we get an unexpected informational (1xx) response.
  1121				// StatusCode 100 is already handled above.
  1122				alive = false
  1123			}
  1124	
  1125			if !hasBody {
  1126				pc.t.setReqCanceler(rc.req, nil)
  1127	
  1128				// Put the idle conn back into the pool before we send the response
  1129				// so if they process it quickly and make another request, they'll
  1130				// get this same conn. But we use the unbuffered channel 'rc'
  1131				// to guarantee that persistConn.roundTrip got out of its select
  1132				// potentially waiting for this persistConn to close.
  1133				// but after
  1134				alive = alive &&
  1135					!pc.sawEOF &&
  1136					pc.wroteRequest() &&
  1137					tryPutIdleConn()
  1138	
  1139				select {
  1140				case rc.ch <- responseAndError{res: resp}:
  1141				case <-rc.callerGone:
  1142					return
  1143				}
  1144	
  1145				// Now that they've read from the unbuffered channel, they're safely
  1146				// out of the select that also waits on this goroutine to die, so
  1147				// we're allowed to exit now if needed (if alive is false)
  1148				testHookReadLoopBeforeNextRead()
  1149				continue
  1150			}
  1151	
  1152			if rc.addedGzip {
  1153				maybeUngzipResponse(resp)
  1154			}
  1155			resp.Body = &bodyEOFSignal{body: resp.Body}
  1156	
  1157			waitForBodyRead := make(chan bool, 2)
  1158			resp.Body.(*bodyEOFSignal).earlyCloseFn = func() error {
  1159				waitForBodyRead <- false
  1160				return nil
  1161			}
  1162			resp.Body.(*bodyEOFSignal).fn = func(err error) error {
  1163				isEOF := err == io.EOF
  1164				waitForBodyRead <- isEOF
  1165				if isEOF {
  1166					<-eofc // see comment above eofc declaration
  1167				} else if err != nil && pc.isCanceled() {
  1168					return errRequestCanceled
  1169				}
  1170				return err
  1171			}
  1172	
  1173			select {
  1174			case rc.ch <- responseAndError{res: resp}:
  1175			case <-rc.callerGone:
  1176				return
  1177			}
  1178	
  1179			// Before looping back to the top of this function and peeking on
  1180			// the bufio.Reader, wait for the caller goroutine to finish
  1181			// reading the response body. (or for cancelation or death)
  1182			select {
  1183			case bodyEOF := <-waitForBodyRead:
  1184				pc.t.setReqCanceler(rc.req, nil) // before pc might return to idle pool
  1185				alive = alive &&
  1186					bodyEOF &&
  1187					!pc.sawEOF &&
  1188					pc.wroteRequest() &&
  1189					tryPutIdleConn()
  1190				if bodyEOF {
  1191					eofc <- struct{}{}
  1192				}
  1193			case <-rc.req.Cancel:
  1194				alive = false
  1195				pc.t.CancelRequest(rc.req)
  1196			case <-pc.closech:
  1197				alive = false
  1198			}
  1199	
  1200			testHookReadLoopBeforeNextRead()
  1201		}
  1202	}
  1203	
  1204	func maybeUngzipResponse(resp *Response) {
  1205		if resp.Header.Get("Content-Encoding") == "gzip" {
  1206			resp.Header.Del("Content-Encoding")
  1207			resp.Header.Del("Content-Length")
  1208			resp.ContentLength = -1
  1209			resp.Body = &gzipReader{body: resp.Body}
  1210		}
  1211	}
  1212	
  1213	func (pc *persistConn) readLoopPeekFailLocked(peekErr error) {
  1214		if pc.closed != nil {
  1215			return
  1216		}
  1217		if n := pc.br.Buffered(); n > 0 {
  1218			buf, _ := pc.br.Peek(n)
  1219			log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr)
  1220		}
  1221		if peekErr == io.EOF {
  1222			// common case.
  1223			pc.closeLocked(errServerClosedIdle)
  1224		} else {
  1225			pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %v", peekErr))
  1226		}
  1227	}
  1228	
  1229	// readResponse reads an HTTP response (or two, in the case of "Expect:
  1230	// 100-continue") from the server. It returns the final non-100 one.
  1231	func (pc *persistConn) readResponse(rc requestAndChan) (resp *Response, err error) {
  1232		resp, err = ReadResponse(pc.br, rc.req)
  1233		if err != nil {
  1234			return
  1235		}
  1236		if rc.continueCh != nil {
  1237			if resp.StatusCode == 100 {
  1238				rc.continueCh <- struct{}{}
  1239			} else {
  1240				close(rc.continueCh)
  1241			}
  1242		}
  1243		if resp.StatusCode == 100 {
  1244			resp, err = ReadResponse(pc.br, rc.req)
  1245			if err != nil {
  1246				return
  1247			}
  1248		}
  1249		resp.TLS = pc.tlsState
  1250		return
  1251	}
  1252	
  1253	// waitForContinue returns the function to block until
  1254	// any response, timeout or connection close. After any of them,
  1255	// the function returns a bool which indicates if the body should be sent.
  1256	func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool {
  1257		if continueCh == nil {
  1258			return nil
  1259		}
  1260		return func() bool {
  1261			timer := time.NewTimer(pc.t.ExpectContinueTimeout)
  1262			defer timer.Stop()
  1263	
  1264			select {
  1265			case _, ok := <-continueCh:
  1266				return ok
  1267			case <-timer.C:
  1268				return true
  1269			case <-pc.closech:
  1270				return false
  1271			}
  1272		}
  1273	}
  1274	
  1275	func (pc *persistConn) writeLoop() {
  1276		for {
  1277			select {
  1278			case wr := <-pc.writech:
  1279				if pc.isBroken() {
  1280					wr.ch <- errors.New("http: can't write HTTP request on broken connection")
  1281					continue
  1282				}
  1283				err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh))
  1284				if err == nil {
  1285					err = pc.bw.Flush()
  1286				}
  1287				if err != nil {
  1288					pc.markBroken()
  1289					wr.req.Request.closeBody()
  1290				}
  1291				pc.writeErrCh <- err // to the body reader, which might recycle us
  1292				wr.ch <- err         // to the roundTrip function
  1293			case <-pc.closech:
  1294				return
  1295			}
  1296		}
  1297	}
  1298	
  1299	// wroteRequest is a check before recycling a connection that the previous write
  1300	// (from writeLoop above) happened and was successful.
  1301	func (pc *persistConn) wroteRequest() bool {
  1302		select {
  1303		case err := <-pc.writeErrCh:
  1304			// Common case: the write happened well before the response, so
  1305			// avoid creating a timer.
  1306			return err == nil
  1307		default:
  1308			// Rare case: the request was written in writeLoop above but
  1309			// before it could send to pc.writeErrCh, the reader read it
  1310			// all, processed it, and called us here. In this case, give the
  1311			// write goroutine a bit of time to finish its send.
  1312			//
  1313			// Less rare case: We also get here in the legitimate case of
  1314			// Issue 7569, where the writer is still writing (or stalled),
  1315			// but the server has already replied. In this case, we don't
  1316			// want to wait too long, and we want to return false so this
  1317			// connection isn't re-used.
  1318			select {
  1319			case err := <-pc.writeErrCh:
  1320				return err == nil
  1321			case <-time.After(50 * time.Millisecond):
  1322				return false
  1323			}
  1324		}
  1325	}
  1326	
  1327	// responseAndError is how the goroutine reading from an HTTP/1 server
  1328	// communicates with the goroutine doing the RoundTrip.
  1329	type responseAndError struct {
  1330		res *Response // else use this response (see res method)
  1331		err error
  1332	}
  1333	
  1334	type requestAndChan struct {
  1335		req *Request
  1336		ch  chan responseAndError // unbuffered; always send in select on callerGone
  1337	
  1338		// did the Transport (as opposed to the client code) add an
  1339		// Accept-Encoding gzip header? only if it we set it do
  1340		// we transparently decode the gzip.
  1341		addedGzip bool
  1342	
  1343		// Optional blocking chan for Expect: 100-continue (for send).
  1344		// If the request has an "Expect: 100-continue" header and
  1345		// the server responds 100 Continue, readLoop send a value
  1346		// to writeLoop via this chan.
  1347		continueCh chan<- struct{}
  1348	
  1349		callerGone <-chan struct{} // closed when roundTrip caller has returned
  1350	}
  1351	
  1352	// A writeRequest is sent by the readLoop's goroutine to the
  1353	// writeLoop's goroutine to write a request while the read loop
  1354	// concurrently waits on both the write response and the server's
  1355	// reply.
  1356	type writeRequest struct {
  1357		req *transportRequest
  1358		ch  chan<- error
  1359	
  1360		// Optional blocking chan for Expect: 100-continue (for recieve).
  1361		// If not nil, writeLoop blocks sending request body until
  1362		// it receives from this chan.
  1363		continueCh <-chan struct{}
  1364	}
  1365	
  1366	type httpError struct {
  1367		err     string
  1368		timeout bool
  1369	}
  1370	
  1371	func (e *httpError) Error() string   { return e.err }
  1372	func (e *httpError) Timeout() bool   { return e.timeout }
  1373	func (e *httpError) Temporary() bool { return true }
  1374	
  1375	var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true}
  1376	var errClosed error = &httpError{err: "net/http: server closed connection before response was received"}
  1377	var errRequestCanceled = errors.New("net/http: request canceled")
  1378	var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify?
  1379	
  1380	func nop() {}
  1381	
  1382	// testHooks. Always non-nil.
  1383	var (
  1384		testHookEnterRoundTrip   = nop
  1385		testHookWaitResLoop      = nop
  1386		testHookRoundTripRetried = nop
  1387		testHookPrePendingDial   = nop
  1388		testHookPostPendingDial  = nop
  1389	
  1390		testHookMu                     sync.Locker = fakeLocker{} // guards following
  1391		testHookReadLoopBeforeNextRead             = nop
  1392	)
  1393	
  1394	// beforeRespHeaderError is used to indicate when an IO error has occurred before
  1395	// any header data was received.
  1396	type beforeRespHeaderError struct {
  1397		error
  1398	}
  1399	
  1400	func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) {
  1401		testHookEnterRoundTrip()
  1402		if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) {
  1403			pc.t.putOrCloseIdleConn(pc)
  1404			return nil, errRequestCanceled
  1405		}
  1406		pc.lk.Lock()
  1407		pc.numExpectedResponses++
  1408		headerFn := pc.mutateHeaderFunc
  1409		pc.lk.Unlock()
  1410	
  1411		if headerFn != nil {
  1412			headerFn(req.extraHeaders())
  1413		}
  1414	
  1415		// Ask for a compressed version if the caller didn't set their
  1416		// own value for Accept-Encoding. We only attempt to
  1417		// uncompress the gzip stream if we were the layer that
  1418		// requested it.
  1419		requestedGzip := false
  1420		if !pc.t.DisableCompression &&
  1421			req.Header.Get("Accept-Encoding") == "" &&
  1422			req.Header.Get("Range") == "" &&
  1423			req.Method != "HEAD" {
  1424			// Request gzip only, not deflate. Deflate is ambiguous and
  1425			// not as universally supported anyway.
  1426			// See: http://www.gzip.org/zlib/zlib_faq.html#faq38
  1427			//
  1428			// Note that we don't request this for HEAD requests,
  1429			// due to a bug in nginx:
  1430			//   http://trac.nginx.org/nginx/ticket/358
  1431			//   https://golang.org/issue/5522
  1432			//
  1433			// We don't request gzip if the request is for a range, since
  1434			// auto-decoding a portion of a gzipped document will just fail
  1435			// anyway. See https://golang.org/issue/8923
  1436			requestedGzip = true
  1437			req.extraHeaders().Set("Accept-Encoding", "gzip")
  1438		}
  1439	
  1440		var continueCh chan struct{}
  1441		if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() {
  1442			continueCh = make(chan struct{}, 1)
  1443		}
  1444	
  1445		if pc.t.DisableKeepAlives {
  1446			req.extraHeaders().Set("Connection", "close")
  1447		}
  1448	
  1449		gone := make(chan struct{})
  1450		defer close(gone)
  1451	
  1452		// Write the request concurrently with waiting for a response,
  1453		// in case the server decides to reply before reading our full
  1454		// request body.
  1455		writeErrCh := make(chan error, 1)
  1456		pc.writech <- writeRequest{req, writeErrCh, continueCh}
  1457	
  1458		resc := make(chan responseAndError)
  1459		pc.reqch <- requestAndChan{
  1460			req:        req.Request,
  1461			ch:         resc,
  1462			addedGzip:  requestedGzip,
  1463			continueCh: continueCh,
  1464			callerGone: gone,
  1465		}
  1466	
  1467		var re responseAndError
  1468		var respHeaderTimer <-chan time.Time
  1469		cancelChan := req.Request.Cancel
  1470	WaitResponse:
  1471		for {
  1472			testHookWaitResLoop()
  1473			select {
  1474			case err := <-writeErrCh:
  1475				if err != nil {
  1476					if pc.isCanceled() {
  1477						err = errRequestCanceled
  1478					}
  1479					re = responseAndError{err: beforeRespHeaderError{err}}
  1480					pc.close(fmt.Errorf("write error: %v", err))
  1481					break WaitResponse
  1482				}
  1483				if d := pc.t.ResponseHeaderTimeout; d > 0 {
  1484					timer := time.NewTimer(d)
  1485					defer timer.Stop() // prevent leaks
  1486					respHeaderTimer = timer.C
  1487				}
  1488			case <-pc.closech:
  1489				var err error
  1490				if pc.isCanceled() {
  1491					err = errRequestCanceled
  1492				} else {
  1493					err = beforeRespHeaderError{fmt.Errorf("net/http: HTTP/1 transport connection broken: %v", pc.closed)}
  1494				}
  1495				re = responseAndError{err: err}
  1496				break WaitResponse
  1497			case <-respHeaderTimer:
  1498				pc.close(errTimeout)
  1499				re = responseAndError{err: errTimeout}
  1500				break WaitResponse
  1501			case re = <-resc:
  1502				if re.err != nil && pc.isCanceled() {
  1503					re.err = errRequestCanceled
  1504				}
  1505				break WaitResponse
  1506			case <-cancelChan:
  1507				pc.t.CancelRequest(req.Request)
  1508				cancelChan = nil
  1509			}
  1510		}
  1511	
  1512		if re.err != nil {
  1513			pc.t.setReqCanceler(req.Request, nil)
  1514		}
  1515		if (re.res == nil) == (re.err == nil) {
  1516			panic("internal error: exactly one of res or err should be set")
  1517		}
  1518		return re.res, re.err
  1519	}
  1520	
  1521	// markBroken marks a connection as broken (so it's not reused).
  1522	// It differs from close in that it doesn't close the underlying
  1523	// connection for use when it's still being read.
  1524	func (pc *persistConn) markBroken() {
  1525		pc.lk.Lock()
  1526		defer pc.lk.Unlock()
  1527		pc.broken = true
  1528	}
  1529	
  1530	// markReused marks this connection as having been successfully used for a
  1531	// request and response.
  1532	func (pc *persistConn) markReused() {
  1533		pc.lk.Lock()
  1534		pc.reused = true
  1535		pc.lk.Unlock()
  1536	}
  1537	
  1538	// close closes the underlying TCP connection and closes
  1539	// the pc.closech channel.
  1540	//
  1541	// The provided err is only for testing and debugging; in normal
  1542	// circumstances it should never be seen by users.
  1543	func (pc *persistConn) close(err error) {
  1544		pc.lk.Lock()
  1545		defer pc.lk.Unlock()
  1546		pc.closeLocked(err)
  1547	}
  1548	
  1549	func (pc *persistConn) closeLocked(err error) {
  1550		if err == nil {
  1551			panic("nil error")
  1552		}
  1553		pc.broken = true
  1554		if pc.closed == nil {
  1555			pc.closed = err
  1556			if pc.alt != nil {
  1557				// Do nothing; can only get here via getConn's
  1558				// handlePendingDial's putOrCloseIdleConn when
  1559				// it turns out the abandoned connection in
  1560				// flight ended up negotiating an alternate
  1561				// protocol.  We don't use the connection
  1562				// freelist for http2. That's done by the
  1563				// alternate protocol's RoundTripper.
  1564			} else {
  1565				pc.conn.Close()
  1566				close(pc.closech)
  1567			}
  1568		}
  1569		pc.mutateHeaderFunc = nil
  1570	}
  1571	
  1572	var portMap = map[string]string{
  1573		"http":  "80",
  1574		"https": "443",
  1575	}
  1576	
  1577	// canonicalAddr returns url.Host but always with a ":port" suffix
  1578	func canonicalAddr(url *url.URL) string {
  1579		addr := url.Host
  1580		if !hasPort(addr) {
  1581			return addr + ":" + portMap[url.Scheme]
  1582		}
  1583		return addr
  1584	}
  1585	
  1586	// bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most
  1587	// once, right before its final (error-producing) Read or Close call
  1588	// returns. fn should return the new error to return from Read or Close.
  1589	//
  1590	// If earlyCloseFn is non-nil and Close is called before io.EOF is
  1591	// seen, earlyCloseFn is called instead of fn, and its return value is
  1592	// the return value from Close.
  1593	type bodyEOFSignal struct {
  1594		body         io.ReadCloser
  1595		mu           sync.Mutex        // guards following 4 fields
  1596		closed       bool              // whether Close has been called
  1597		rerr         error             // sticky Read error
  1598		fn           func(error) error // err will be nil on Read io.EOF
  1599		earlyCloseFn func() error      // optional alt Close func used if io.EOF not seen
  1600	}
  1601	
  1602	func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
  1603		es.mu.Lock()
  1604		closed, rerr := es.closed, es.rerr
  1605		es.mu.Unlock()
  1606		if closed {
  1607			return 0, errors.New("http: read on closed response body")
  1608		}
  1609		if rerr != nil {
  1610			return 0, rerr
  1611		}
  1612	
  1613		n, err = es.body.Read(p)
  1614		if err != nil {
  1615			es.mu.Lock()
  1616			defer es.mu.Unlock()
  1617			if es.rerr == nil {
  1618				es.rerr = err
  1619			}
  1620			err = es.condfn(err)
  1621		}
  1622		return
  1623	}
  1624	
  1625	func (es *bodyEOFSignal) Close() error {
  1626		es.mu.Lock()
  1627		defer es.mu.Unlock()
  1628		if es.closed {
  1629			return nil
  1630		}
  1631		es.closed = true
  1632		if es.earlyCloseFn != nil && es.rerr != io.EOF {
  1633			return es.earlyCloseFn()
  1634		}
  1635		err := es.body.Close()
  1636		return es.condfn(err)
  1637	}
  1638	
  1639	// caller must hold es.mu.
  1640	func (es *bodyEOFSignal) condfn(err error) error {
  1641		if es.fn == nil {
  1642			return err
  1643		}
  1644		err = es.fn(err)
  1645		es.fn = nil
  1646		return err
  1647	}
  1648	
  1649	// gzipReader wraps a response body so it can lazily
  1650	// call gzip.NewReader on the first call to Read
  1651	type gzipReader struct {
  1652		body io.ReadCloser // underlying Response.Body
  1653		zr   io.Reader     // lazily-initialized gzip reader
  1654	}
  1655	
  1656	func (gz *gzipReader) Read(p []byte) (n int, err error) {
  1657		if gz.zr == nil {
  1658			gz.zr, err = gzip.NewReader(gz.body)
  1659			if err != nil {
  1660				return 0, err
  1661			}
  1662		}
  1663		return gz.zr.Read(p)
  1664	}
  1665	
  1666	func (gz *gzipReader) Close() error {
  1667		return gz.body.Close()
  1668	}
  1669	
  1670	type readerAndCloser struct {
  1671		io.Reader
  1672		io.Closer
  1673	}
  1674	
  1675	type tlsHandshakeTimeoutError struct{}
  1676	
  1677	func (tlsHandshakeTimeoutError) Timeout() bool   { return true }
  1678	func (tlsHandshakeTimeoutError) Temporary() bool { return true }
  1679	func (tlsHandshakeTimeoutError) Error() string   { return "net/http: TLS handshake timeout" }
  1680	
  1681	type noteEOFReader struct {
  1682		r      io.Reader
  1683		sawEOF *bool
  1684	}
  1685	
  1686	func (nr noteEOFReader) Read(p []byte) (n int, err error) {
  1687		n, err = nr.r.Read(p)
  1688		if err == io.EOF {
  1689			*nr.sawEOF = true
  1690		}
  1691		return
  1692	}
  1693	
  1694	// fakeLocker is a sync.Locker which does nothing. It's used to guard
  1695	// test-only fields when not under test, to avoid runtime atomic
  1696	// overhead.
  1697	type fakeLocker struct{}
  1698	
  1699	func (fakeLocker) Lock()   {}
  1700	func (fakeLocker) Unlock() {}
  1701	
  1702	func isNetWriteError(err error) bool {
  1703		switch e := err.(type) {
  1704		case *url.Error:
  1705			return isNetWriteError(e.Err)
  1706		case *net.OpError:
  1707			return e.Op == "write"
  1708		default:
  1709			return false
  1710		}
  1711	}
  1712	
  1713	// cloneTLSConfig returns a shallow clone of the exported
  1714	// fields of cfg, ignoring the unexported sync.Once, which
  1715	// contains a mutex and must not be copied.
  1716	//
  1717	// The cfg must not be in active use by tls.Server, or else
  1718	// there can still be a race with tls.Server updating SessionTicketKey
  1719	// and our copying it, and also a race with the server setting
  1720	// SessionTicketsDisabled=false on failure to set the random
  1721	// ticket key.
  1722	//
  1723	// If cfg is nil, a new zero tls.Config is returned.
  1724	func cloneTLSConfig(cfg *tls.Config) *tls.Config {
  1725		if cfg == nil {
  1726			return &tls.Config{}
  1727		}
  1728		return &tls.Config{
  1729			Rand:                     cfg.Rand,
  1730			Time:                     cfg.Time,
  1731			Certificates:             cfg.Certificates,
  1732			NameToCertificate:        cfg.NameToCertificate,
  1733			GetCertificate:           cfg.GetCertificate,
  1734			RootCAs:                  cfg.RootCAs,
  1735			NextProtos:               cfg.NextProtos,
  1736			ServerName:               cfg.ServerName,
  1737			ClientAuth:               cfg.ClientAuth,
  1738			ClientCAs:                cfg.ClientCAs,
  1739			InsecureSkipVerify:       cfg.InsecureSkipVerify,
  1740			CipherSuites:             cfg.CipherSuites,
  1741			PreferServerCipherSuites: cfg.PreferServerCipherSuites,
  1742			SessionTicketsDisabled:   cfg.SessionTicketsDisabled,
  1743			SessionTicketKey:         cfg.SessionTicketKey,
  1744			ClientSessionCache:       cfg.ClientSessionCache,
  1745			MinVersion:               cfg.MinVersion,
  1746			MaxVersion:               cfg.MaxVersion,
  1747			CurvePreferences:         cfg.CurvePreferences,
  1748		}
  1749	}
  1750	
  1751	// cloneTLSClientConfig is like cloneTLSConfig but omits
  1752	// the fields SessionTicketsDisabled and SessionTicketKey.
  1753	// This makes it safe to call cloneTLSClientConfig on a config
  1754	// in active use by a server.
  1755	func cloneTLSClientConfig(cfg *tls.Config) *tls.Config {
  1756		if cfg == nil {
  1757			return &tls.Config{}
  1758		}
  1759		return &tls.Config{
  1760			Rand:                     cfg.Rand,
  1761			Time:                     cfg.Time,
  1762			Certificates:             cfg.Certificates,
  1763			NameToCertificate:        cfg.NameToCertificate,
  1764			GetCertificate:           cfg.GetCertificate,
  1765			RootCAs:                  cfg.RootCAs,
  1766			NextProtos:               cfg.NextProtos,
  1767			ServerName:               cfg.ServerName,
  1768			ClientAuth:               cfg.ClientAuth,
  1769			ClientCAs:                cfg.ClientCAs,
  1770			InsecureSkipVerify:       cfg.InsecureSkipVerify,
  1771			CipherSuites:             cfg.CipherSuites,
  1772			PreferServerCipherSuites: cfg.PreferServerCipherSuites,
  1773			ClientSessionCache:       cfg.ClientSessionCache,
  1774			MinVersion:               cfg.MinVersion,
  1775			MaxVersion:               cfg.MaxVersion,
  1776			CurvePreferences:         cfg.CurvePreferences,
  1777		}
  1778	}
  1779	

View as plain text