...
Run Format

Source file src/net/http/transport.go

     1	// Copyright 2011 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// HTTP client implementation. See RFC 2616.
     6	//
     7	// This is the low-level Transport implementation of RoundTripper.
     8	// The high-level interface is in client.go.
     9	
    10	package http
    11	
    12	import (
    13		"bufio"
    14		"compress/gzip"
    15		"crypto/tls"
    16		"errors"
    17		"fmt"
    18		"io"
    19		"log"
    20		"net"
    21		"net/url"
    22		"os"
    23		"strings"
    24		"sync"
    25		"time"
    26	)
    27	
    28	// DefaultTransport is the default implementation of Transport and is
    29	// used by DefaultClient. It establishes network connections as needed
    30	// and caches them for reuse by subsequent calls. It uses HTTP proxies
    31	// as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and
    32	// $no_proxy) environment variables.
    33	var DefaultTransport RoundTripper = &Transport{
    34		Proxy: ProxyFromEnvironment,
    35		Dial: (&net.Dialer{
    36			Timeout:   30 * time.Second,
    37			KeepAlive: 30 * time.Second,
    38		}).Dial,
    39		TLSHandshakeTimeout:   10 * time.Second,
    40		ExpectContinueTimeout: 1 * time.Second,
    41	}
    42	
    43	// DefaultMaxIdleConnsPerHost is the default value of Transport's
    44	// MaxIdleConnsPerHost.
    45	const DefaultMaxIdleConnsPerHost = 2
    46	
    47	// Transport is an implementation of RoundTripper that supports HTTP,
    48	// HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT).
    49	//
    50	// By default, Transport caches connections for future re-use.
    51	// This may leave many open connections when accessing many hosts.
    52	// This behavior can be managed using Transport's CloseIdleConnections method
    53	// and the MaxIdleConnsPerHost and DisableKeepAlives fields.
    54	//
    55	// Transports should be reused instead of created as needed.
    56	// Transports are safe for concurrent use by multiple goroutines.
    57	//
    58	// A Transport is a low-level primitive for making HTTP and HTTPS requests.
    59	// For high-level functionality, such as cookies and redirects, see Client.
    60	//
    61	// Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2
    62	// for HTTPS URLs, depending on whether the server supports HTTP/2.
    63	// See the package docs for more about HTTP/2.
    64	type Transport struct {
    65		idleMu     sync.Mutex
    66		wantIdle   bool // user has requested to close all idle conns
    67		idleConn   map[connectMethodKey][]*persistConn
    68		idleConnCh map[connectMethodKey]chan *persistConn
    69	
    70		reqMu       sync.Mutex
    71		reqCanceler map[*Request]func()
    72	
    73		altMu    sync.RWMutex
    74		altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper
    75	
    76		// Proxy specifies a function to return a proxy for a given
    77		// Request. If the function returns a non-nil error, the
    78		// request is aborted with the provided error.
    79		// If Proxy is nil or returns a nil *URL, no proxy is used.
    80		Proxy func(*Request) (*url.URL, error)
    81	
    82		// Dial specifies the dial function for creating unencrypted
    83		// TCP connections.
    84		// If Dial is nil, net.Dial is used.
    85		Dial func(network, addr string) (net.Conn, error)
    86	
    87		// DialTLS specifies an optional dial function for creating
    88		// TLS connections for non-proxied HTTPS requests.
    89		//
    90		// If DialTLS is nil, Dial and TLSClientConfig are used.
    91		//
    92		// If DialTLS is set, the Dial hook is not used for HTTPS
    93		// requests and the TLSClientConfig and TLSHandshakeTimeout
    94		// are ignored. The returned net.Conn is assumed to already be
    95		// past the TLS handshake.
    96		DialTLS func(network, addr string) (net.Conn, error)
    97	
    98		// TLSClientConfig specifies the TLS configuration to use with
    99		// tls.Client. If nil, the default configuration is used.
   100		TLSClientConfig *tls.Config
   101	
   102		// TLSHandshakeTimeout specifies the maximum amount of time waiting to
   103		// wait for a TLS handshake. Zero means no timeout.
   104		TLSHandshakeTimeout time.Duration
   105	
   106		// DisableKeepAlives, if true, prevents re-use of TCP connections
   107		// between different HTTP requests.
   108		DisableKeepAlives bool
   109	
   110		// DisableCompression, if true, prevents the Transport from
   111		// requesting compression with an "Accept-Encoding: gzip"
   112		// request header when the Request contains no existing
   113		// Accept-Encoding value. If the Transport requests gzip on
   114		// its own and gets a gzipped response, it's transparently
   115		// decoded in the Response.Body. However, if the user
   116		// explicitly requested gzip it is not automatically
   117		// uncompressed.
   118		DisableCompression bool
   119	
   120		// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
   121		// (keep-alive) to keep per-host.  If zero,
   122		// DefaultMaxIdleConnsPerHost is used.
   123		MaxIdleConnsPerHost int
   124	
   125		// ResponseHeaderTimeout, if non-zero, specifies the amount of
   126		// time to wait for a server's response headers after fully
   127		// writing the request (including its body, if any). This
   128		// time does not include the time to read the response body.
   129		ResponseHeaderTimeout time.Duration
   130	
   131		// ExpectContinueTimeout, if non-zero, specifies the amount of
   132		// time to wait for a server's first response headers after fully
   133		// writing the request headers if the request has an
   134		// "Expect: 100-continue" header. Zero means no timeout.
   135		// This time does not include the time to send the request header.
   136		ExpectContinueTimeout time.Duration
   137	
   138		// TLSNextProto specifies how the Transport switches to an
   139		// alternate protocol (such as HTTP/2) after a TLS NPN/ALPN
   140		// protocol negotiation.  If Transport dials an TLS connection
   141		// with a non-empty protocol name and TLSNextProto contains a
   142		// map entry for that key (such as "h2"), then the func is
   143		// called with the request's authority (such as "example.com"
   144		// or "example.com:1234") and the TLS connection. The function
   145		// must return a RoundTripper that then handles the request.
   146		// If TLSNextProto is nil, HTTP/2 support is enabled automatically.
   147		TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper
   148	
   149		// nextProtoOnce guards initialization of TLSNextProto and
   150		// h2transport (via onceSetNextProtoDefaults)
   151		nextProtoOnce sync.Once
   152		h2transport   *http2Transport // non-nil if http2 wired up
   153	
   154		// TODO: tunable on global max cached connections
   155		// TODO: tunable on timeout on cached connections
   156		// TODO: tunable on max per-host TCP dials in flight (Issue 13957)
   157	}
   158	
   159	// onceSetNextProtoDefaults initializes TLSNextProto.
   160	// It must be called via t.nextProtoOnce.Do.
   161	func (t *Transport) onceSetNextProtoDefaults() {
   162		if strings.Contains(os.Getenv("GODEBUG"), "http2client=0") {
   163			return
   164		}
   165		if t.TLSNextProto != nil {
   166			// This is the documented way to disable http2 on a
   167			// Transport.
   168			return
   169		}
   170		if t.TLSClientConfig != nil {
   171			// Be conservative for now (for Go 1.6) at least and
   172			// don't automatically enable http2 if they've
   173			// specified a custom TLS config. Let them opt-in
   174			// themselves via http2.ConfigureTransport so we don't
   175			// surprise them by modifying their tls.Config.
   176			// Issue 14275.
   177			return
   178		}
   179		if t.ExpectContinueTimeout != 0 && t != DefaultTransport {
   180			// ExpectContinueTimeout is unsupported in http2, so
   181			// if they explicitly asked for it (as opposed to just
   182			// using the DefaultTransport, which sets it), then
   183			// disable http2 for now.
   184			//
   185			// Issue 13851. (and changed in Issue 14391)
   186			return
   187		}
   188		t2, err := http2configureTransport(t)
   189		if err != nil {
   190			log.Printf("Error enabling Transport HTTP/2 support: %v", err)
   191		} else {
   192			t.h2transport = t2
   193		}
   194	}
   195	
   196	// ProxyFromEnvironment returns the URL of the proxy to use for a
   197	// given request, as indicated by the environment variables
   198	// HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions
   199	// thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https
   200	// requests.
   201	//
   202	// The environment values may be either a complete URL or a
   203	// "host[:port]", in which case the "http" scheme is assumed.
   204	// An error is returned if the value is a different form.
   205	//
   206	// A nil URL and nil error are returned if no proxy is defined in the
   207	// environment, or a proxy should not be used for the given request,
   208	// as defined by NO_PROXY.
   209	//
   210	// As a special case, if req.URL.Host is "localhost" (with or without
   211	// a port number), then a nil URL and nil error will be returned.
   212	func ProxyFromEnvironment(req *Request) (*url.URL, error) {
   213		var proxy string
   214		if req.URL.Scheme == "https" {
   215			proxy = httpsProxyEnv.Get()
   216		}
   217		if proxy == "" {
   218			proxy = httpProxyEnv.Get()
   219			if proxy != "" && os.Getenv("REQUEST_METHOD") != "" {
   220				return nil, errors.New("net/http: refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy")
   221			}
   222		}
   223		if proxy == "" {
   224			return nil, nil
   225		}
   226		if !useProxy(canonicalAddr(req.URL)) {
   227			return nil, nil
   228		}
   229		proxyURL, err := url.Parse(proxy)
   230		if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") {
   231			// proxy was bogus. Try prepending "http://" to it and
   232			// see if that parses correctly. If not, we fall
   233			// through and complain about the original one.
   234			if proxyURL, err := url.Parse("http://" + proxy); err == nil {
   235				return proxyURL, nil
   236			}
   237		}
   238		if err != nil {
   239			return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
   240		}
   241		return proxyURL, nil
   242	}
   243	
   244	// ProxyURL returns a proxy function (for use in a Transport)
   245	// that always returns the same URL.
   246	func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) {
   247		return func(*Request) (*url.URL, error) {
   248			return fixedURL, nil
   249		}
   250	}
   251	
   252	// transportRequest is a wrapper around a *Request that adds
   253	// optional extra headers to write.
   254	type transportRequest struct {
   255		*Request        // original request, not to be mutated
   256		extra    Header // extra headers to write, or nil
   257	}
   258	
   259	func (tr *transportRequest) extraHeaders() Header {
   260		if tr.extra == nil {
   261			tr.extra = make(Header)
   262		}
   263		return tr.extra
   264	}
   265	
   266	// RoundTrip implements the RoundTripper interface.
   267	//
   268	// For higher-level HTTP client support (such as handling of cookies
   269	// and redirects), see Get, Post, and the Client type.
   270	func (t *Transport) RoundTrip(req *Request) (*Response, error) {
   271		t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
   272		if req.URL == nil {
   273			req.closeBody()
   274			return nil, errors.New("http: nil Request.URL")
   275		}
   276		if req.Header == nil {
   277			req.closeBody()
   278			return nil, errors.New("http: nil Request.Header")
   279		}
   280		// TODO(bradfitz): switch to atomic.Value for this map instead of RWMutex
   281		t.altMu.RLock()
   282		altRT := t.altProto[req.URL.Scheme]
   283		t.altMu.RUnlock()
   284		if altRT != nil {
   285			if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol {
   286				return resp, err
   287			}
   288		}
   289		if s := req.URL.Scheme; s != "http" && s != "https" {
   290			req.closeBody()
   291			return nil, &badStringError{"unsupported protocol scheme", s}
   292		}
   293		if req.Method != "" && !validMethod(req.Method) {
   294			return nil, fmt.Errorf("net/http: invalid method %q", req.Method)
   295		}
   296		if req.URL.Host == "" {
   297			req.closeBody()
   298			return nil, errors.New("http: no Host in request URL")
   299		}
   300	
   301		for {
   302			// treq gets modified by roundTrip, so we need to recreate for each retry.
   303			treq := &transportRequest{Request: req}
   304			cm, err := t.connectMethodForRequest(treq)
   305			if err != nil {
   306				req.closeBody()
   307				return nil, err
   308			}
   309	
   310			// Get the cached or newly-created connection to either the
   311			// host (for http or https), the http proxy, or the http proxy
   312			// pre-CONNECTed to https server.  In any case, we'll be ready
   313			// to send it requests.
   314			pconn, err := t.getConn(req, cm)
   315			if err != nil {
   316				t.setReqCanceler(req, nil)
   317				req.closeBody()
   318				return nil, err
   319			}
   320	
   321			var resp *Response
   322			if pconn.alt != nil {
   323				// HTTP/2 path.
   324				t.setReqCanceler(req, nil) // not cancelable with CancelRequest
   325				resp, err = pconn.alt.RoundTrip(req)
   326			} else {
   327				resp, err = pconn.roundTrip(treq)
   328			}
   329			if err == nil {
   330				return resp, nil
   331			}
   332			if err := checkTransportResend(err, req, pconn); err != nil {
   333				return nil, err
   334			}
   335			testHookRoundTripRetried()
   336		}
   337	}
   338	
   339	// checkTransportResend checks whether a failed HTTP request can be
   340	// resent on a new connection. The non-nil input error is the error from
   341	// roundTrip, which might be wrapped in a beforeRespHeaderError error.
   342	//
   343	// The return value is err or the unwrapped error inside a
   344	// beforeRespHeaderError.
   345	func checkTransportResend(err error, req *Request, pconn *persistConn) error {
   346		brhErr, ok := err.(beforeRespHeaderError)
   347		if !ok {
   348			return err
   349		}
   350		err = brhErr.error // unwrap the custom error in case we return it
   351		if err != errMissingHost && pconn.isReused() && req.isReplayable() {
   352			// If we try to reuse a connection that the server is in the process of
   353			// closing, we may end up successfully writing out our request (or a
   354			// portion of our request) only to find a connection error when we try to
   355			// read from (or finish writing to) the socket.
   356	
   357			// There can be a race between the socket pool checking whether a socket
   358			// is still connected, receiving the FIN, and sending/reading data on a
   359			// reused socket. If we receive the FIN between the connectedness check
   360			// and writing/reading from the socket, we may first learn the socket is
   361			// disconnected when we get a ERR_SOCKET_NOT_CONNECTED. This will most
   362			// likely happen when trying to retrieve its IP address. See
   363			// http://crbug.com/105824 for more details.
   364	
   365			// We resend a request only if we reused a keep-alive connection and did
   366			// not yet receive any header data. This automatically prevents an
   367			// infinite resend loop because we'll run out of the cached keep-alive
   368			// connections eventually.
   369			return nil
   370		}
   371		return err
   372	}
   373	
   374	// ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol.
   375	var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol")
   376	
   377	// RegisterProtocol registers a new protocol with scheme.
   378	// The Transport will pass requests using the given scheme to rt.
   379	// It is rt's responsibility to simulate HTTP request semantics.
   380	//
   381	// RegisterProtocol can be used by other packages to provide
   382	// implementations of protocol schemes like "ftp" or "file".
   383	//
   384	// If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will
   385	// handle the RoundTrip itself for that one request, as if the
   386	// protocol were not registered.
   387	func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) {
   388		t.altMu.Lock()
   389		defer t.altMu.Unlock()
   390		if t.altProto == nil {
   391			t.altProto = make(map[string]RoundTripper)
   392		}
   393		if _, exists := t.altProto[scheme]; exists {
   394			panic("protocol " + scheme + " already registered")
   395		}
   396		t.altProto[scheme] = rt
   397	}
   398	
   399	// CloseIdleConnections closes any connections which were previously
   400	// connected from previous requests but are now sitting idle in
   401	// a "keep-alive" state. It does not interrupt any connections currently
   402	// in use.
   403	func (t *Transport) CloseIdleConnections() {
   404		t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
   405		t.idleMu.Lock()
   406		m := t.idleConn
   407		t.idleConn = nil
   408		t.idleConnCh = nil
   409		t.wantIdle = true
   410		t.idleMu.Unlock()
   411		for _, conns := range m {
   412			for _, pconn := range conns {
   413				pconn.close(errCloseIdleConns)
   414			}
   415		}
   416		if t2 := t.h2transport; t2 != nil {
   417			t2.CloseIdleConnections()
   418		}
   419	}
   420	
   421	// CancelRequest cancels an in-flight request by closing its connection.
   422	// CancelRequest should only be called after RoundTrip has returned.
   423	//
   424	// Deprecated: Use Request.Cancel instead. CancelRequest can not cancel
   425	// HTTP/2 requests.
   426	func (t *Transport) CancelRequest(req *Request) {
   427		t.reqMu.Lock()
   428		cancel := t.reqCanceler[req]
   429		delete(t.reqCanceler, req)
   430		t.reqMu.Unlock()
   431		if cancel != nil {
   432			cancel()
   433		}
   434	}
   435	
   436	//
   437	// Private implementation past this point.
   438	//
   439	
   440	var (
   441		httpProxyEnv = &envOnce{
   442			names: []string{"HTTP_PROXY", "http_proxy"},
   443		}
   444		httpsProxyEnv = &envOnce{
   445			names: []string{"HTTPS_PROXY", "https_proxy"},
   446		}
   447		noProxyEnv = &envOnce{
   448			names: []string{"NO_PROXY", "no_proxy"},
   449		}
   450	)
   451	
   452	// envOnce looks up an environment variable (optionally by multiple
   453	// names) once. It mitigates expensive lookups on some platforms
   454	// (e.g. Windows).
   455	type envOnce struct {
   456		names []string
   457		once  sync.Once
   458		val   string
   459	}
   460	
   461	func (e *envOnce) Get() string {
   462		e.once.Do(e.init)
   463		return e.val
   464	}
   465	
   466	func (e *envOnce) init() {
   467		for _, n := range e.names {
   468			e.val = os.Getenv(n)
   469			if e.val != "" {
   470				return
   471			}
   472		}
   473	}
   474	
   475	// reset is used by tests
   476	func (e *envOnce) reset() {
   477		e.once = sync.Once{}
   478		e.val = ""
   479	}
   480	
   481	func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) {
   482		cm.targetScheme = treq.URL.Scheme
   483		cm.targetAddr = canonicalAddr(treq.URL)
   484		if t.Proxy != nil {
   485			cm.proxyURL, err = t.Proxy(treq.Request)
   486		}
   487		return cm, err
   488	}
   489	
   490	// proxyAuth returns the Proxy-Authorization header to set
   491	// on requests, if applicable.
   492	func (cm *connectMethod) proxyAuth() string {
   493		if cm.proxyURL == nil {
   494			return ""
   495		}
   496		if u := cm.proxyURL.User; u != nil {
   497			username := u.Username()
   498			password, _ := u.Password()
   499			return "Basic " + basicAuth(username, password)
   500		}
   501		return ""
   502	}
   503	
   504	// error values for debugging and testing, not seen by users.
   505	var (
   506		errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled")
   507		errConnBroken         = errors.New("http: putIdleConn: connection is in bad state")
   508		errWantIdle           = errors.New("http: putIdleConn: CloseIdleConnections was called")
   509		errTooManyIdle        = errors.New("http: putIdleConn: too many idle connections")
   510		errCloseIdleConns     = errors.New("http: CloseIdleConnections called")
   511		errReadLoopExiting    = errors.New("http: persistConn.readLoop exiting")
   512		errServerClosedIdle   = errors.New("http: server closed idle conn")
   513	)
   514	
   515	func (t *Transport) putOrCloseIdleConn(pconn *persistConn) {
   516		if err := t.tryPutIdleConn(pconn); err != nil {
   517			pconn.close(err)
   518		}
   519	}
   520	
   521	// tryPutIdleConn adds pconn to the list of idle persistent connections awaiting
   522	// a new request.
   523	// If pconn is no longer needed or not in a good state, tryPutIdleConn returns
   524	// an error explaining why it wasn't registered.
   525	// tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that.
   526	func (t *Transport) tryPutIdleConn(pconn *persistConn) error {
   527		if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 {
   528			return errKeepAlivesDisabled
   529		}
   530		if pconn.isBroken() {
   531			return errConnBroken
   532		}
   533		key := pconn.cacheKey
   534		max := t.MaxIdleConnsPerHost
   535		if max == 0 {
   536			max = DefaultMaxIdleConnsPerHost
   537		}
   538		pconn.markReused()
   539		t.idleMu.Lock()
   540	
   541		waitingDialer := t.idleConnCh[key]
   542		select {
   543		case waitingDialer <- pconn:
   544			// We're done with this pconn and somebody else is
   545			// currently waiting for a conn of this type (they're
   546			// actively dialing, but this conn is ready
   547			// first). Chrome calls this socket late binding.  See
   548			// https://insouciant.org/tech/connection-management-in-chromium/
   549			t.idleMu.Unlock()
   550			return nil
   551		default:
   552			if waitingDialer != nil {
   553				// They had populated this, but their dial won
   554				// first, so we can clean up this map entry.
   555				delete(t.idleConnCh, key)
   556			}
   557		}
   558		if t.wantIdle {
   559			t.idleMu.Unlock()
   560			return errWantIdle
   561		}
   562		if t.idleConn == nil {
   563			t.idleConn = make(map[connectMethodKey][]*persistConn)
   564		}
   565		if len(t.idleConn[key]) >= max {
   566			t.idleMu.Unlock()
   567			return errTooManyIdle
   568		}
   569		for _, exist := range t.idleConn[key] {
   570			if exist == pconn {
   571				log.Fatalf("dup idle pconn %p in freelist", pconn)
   572			}
   573		}
   574		t.idleConn[key] = append(t.idleConn[key], pconn)
   575		t.idleMu.Unlock()
   576		return nil
   577	}
   578	
   579	// getIdleConnCh returns a channel to receive and return idle
   580	// persistent connection for the given connectMethod.
   581	// It may return nil, if persistent connections are not being used.
   582	func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn {
   583		if t.DisableKeepAlives {
   584			return nil
   585		}
   586		key := cm.key()
   587		t.idleMu.Lock()
   588		defer t.idleMu.Unlock()
   589		t.wantIdle = false
   590		if t.idleConnCh == nil {
   591			t.idleConnCh = make(map[connectMethodKey]chan *persistConn)
   592		}
   593		ch, ok := t.idleConnCh[key]
   594		if !ok {
   595			ch = make(chan *persistConn)
   596			t.idleConnCh[key] = ch
   597		}
   598		return ch
   599	}
   600	
   601	func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn) {
   602		key := cm.key()
   603		t.idleMu.Lock()
   604		defer t.idleMu.Unlock()
   605		if t.idleConn == nil {
   606			return nil
   607		}
   608		for {
   609			pconns, ok := t.idleConn[key]
   610			if !ok {
   611				return nil
   612			}
   613			if len(pconns) == 1 {
   614				pconn = pconns[0]
   615				delete(t.idleConn, key)
   616			} else {
   617				// 2 or more cached connections; pop last
   618				// TODO: queue?
   619				pconn = pconns[len(pconns)-1]
   620				t.idleConn[key] = pconns[:len(pconns)-1]
   621			}
   622			if !pconn.isBroken() {
   623				return
   624			}
   625		}
   626	}
   627	
   628	func (t *Transport) setReqCanceler(r *Request, fn func()) {
   629		t.reqMu.Lock()
   630		defer t.reqMu.Unlock()
   631		if t.reqCanceler == nil {
   632			t.reqCanceler = make(map[*Request]func())
   633		}
   634		if fn != nil {
   635			t.reqCanceler[r] = fn
   636		} else {
   637			delete(t.reqCanceler, r)
   638		}
   639	}
   640	
   641	// replaceReqCanceler replaces an existing cancel function. If there is no cancel function
   642	// for the request, we don't set the function and return false.
   643	// Since CancelRequest will clear the canceler, we can use the return value to detect if
   644	// the request was canceled since the last setReqCancel call.
   645	func (t *Transport) replaceReqCanceler(r *Request, fn func()) bool {
   646		t.reqMu.Lock()
   647		defer t.reqMu.Unlock()
   648		_, ok := t.reqCanceler[r]
   649		if !ok {
   650			return false
   651		}
   652		if fn != nil {
   653			t.reqCanceler[r] = fn
   654		} else {
   655			delete(t.reqCanceler, r)
   656		}
   657		return true
   658	}
   659	
   660	func (t *Transport) dial(network, addr string) (net.Conn, error) {
   661		if t.Dial != nil {
   662			c, err := t.Dial(network, addr)
   663			if c == nil && err == nil {
   664				err = errors.New("net/http: Transport.Dial hook returned (nil, nil)")
   665			}
   666			return c, err
   667		}
   668		return net.Dial(network, addr)
   669	}
   670	
   671	// getConn dials and creates a new persistConn to the target as
   672	// specified in the connectMethod.  This includes doing a proxy CONNECT
   673	// and/or setting up TLS.  If this doesn't return an error, the persistConn
   674	// is ready to write requests to.
   675	func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error) {
   676		if pc := t.getIdleConn(cm); pc != nil {
   677			// set request canceler to some non-nil function so we
   678			// can detect whether it was cleared between now and when
   679			// we enter roundTrip
   680			t.setReqCanceler(req, func() {})
   681			return pc, nil
   682		}
   683	
   684		type dialRes struct {
   685			pc  *persistConn
   686			err error
   687		}
   688		dialc := make(chan dialRes)
   689	
   690		// Copy these hooks so we don't race on the postPendingDial in
   691		// the goroutine we launch. Issue 11136.
   692		testHookPrePendingDial := testHookPrePendingDial
   693		testHookPostPendingDial := testHookPostPendingDial
   694	
   695		handlePendingDial := func() {
   696			testHookPrePendingDial()
   697			go func() {
   698				if v := <-dialc; v.err == nil {
   699					t.putOrCloseIdleConn(v.pc)
   700				}
   701				testHookPostPendingDial()
   702			}()
   703		}
   704	
   705		cancelc := make(chan struct{})
   706		t.setReqCanceler(req, func() { close(cancelc) })
   707	
   708		go func() {
   709			pc, err := t.dialConn(cm)
   710			dialc <- dialRes{pc, err}
   711		}()
   712	
   713		idleConnCh := t.getIdleConnCh(cm)
   714		select {
   715		case v := <-dialc:
   716			// Our dial finished.
   717			return v.pc, v.err
   718		case pc := <-idleConnCh:
   719			// Another request finished first and its net.Conn
   720			// became available before our dial. Or somebody
   721			// else's dial that they didn't use.
   722			// But our dial is still going, so give it away
   723			// when it finishes:
   724			handlePendingDial()
   725			return pc, nil
   726		case <-req.Cancel:
   727			handlePendingDial()
   728			return nil, errRequestCanceledConn
   729		case <-cancelc:
   730			handlePendingDial()
   731			return nil, errRequestCanceledConn
   732		}
   733	}
   734	
   735	func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) {
   736		pconn := &persistConn{
   737			t:          t,
   738			cacheKey:   cm.key(),
   739			reqch:      make(chan requestAndChan, 1),
   740			writech:    make(chan writeRequest, 1),
   741			closech:    make(chan struct{}),
   742			writeErrCh: make(chan error, 1),
   743		}
   744		tlsDial := t.DialTLS != nil && cm.targetScheme == "https" && cm.proxyURL == nil
   745		if tlsDial {
   746			var err error
   747			pconn.conn, err = t.DialTLS("tcp", cm.addr())
   748			if err != nil {
   749				return nil, err
   750			}
   751			if pconn.conn == nil {
   752				return nil, errors.New("net/http: Transport.DialTLS returned (nil, nil)")
   753			}
   754			if tc, ok := pconn.conn.(*tls.Conn); ok {
   755				// Handshake here, in case DialTLS didn't. TLSNextProto below
   756				// depends on it for knowing the connection state.
   757				if err := tc.Handshake(); err != nil {
   758					go pconn.conn.Close()
   759					return nil, err
   760				}
   761				cs := tc.ConnectionState()
   762				pconn.tlsState = &cs
   763			}
   764		} else {
   765			conn, err := t.dial("tcp", cm.addr())
   766			if err != nil {
   767				if cm.proxyURL != nil {
   768					err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err)
   769				}
   770				return nil, err
   771			}
   772			pconn.conn = conn
   773		}
   774	
   775		// Proxy setup.
   776		switch {
   777		case cm.proxyURL == nil:
   778			// Do nothing. Not using a proxy.
   779		case cm.targetScheme == "http":
   780			pconn.isProxy = true
   781			if pa := cm.proxyAuth(); pa != "" {
   782				pconn.mutateHeaderFunc = func(h Header) {
   783					h.Set("Proxy-Authorization", pa)
   784				}
   785			}
   786		case cm.targetScheme == "https":
   787			conn := pconn.conn
   788			connectReq := &Request{
   789				Method: "CONNECT",
   790				URL:    &url.URL{Opaque: cm.targetAddr},
   791				Host:   cm.targetAddr,
   792				Header: make(Header),
   793			}
   794			if pa := cm.proxyAuth(); pa != "" {
   795				connectReq.Header.Set("Proxy-Authorization", pa)
   796			}
   797			connectReq.Write(conn)
   798	
   799			// Read response.
   800			// Okay to use and discard buffered reader here, because
   801			// TLS server will not speak until spoken to.
   802			br := bufio.NewReader(conn)
   803			resp, err := ReadResponse(br, connectReq)
   804			if err != nil {
   805				conn.Close()
   806				return nil, err
   807			}
   808			if resp.StatusCode != 200 {
   809				f := strings.SplitN(resp.Status, " ", 2)
   810				conn.Close()
   811				return nil, errors.New(f[1])
   812			}
   813		}
   814	
   815		if cm.targetScheme == "https" && !tlsDial {
   816			// Initiate TLS and check remote host name against certificate.
   817			cfg := cloneTLSClientConfig(t.TLSClientConfig)
   818			if cfg.ServerName == "" {
   819				cfg.ServerName = cm.tlsHost()
   820			}
   821			plainConn := pconn.conn
   822			tlsConn := tls.Client(plainConn, cfg)
   823			errc := make(chan error, 2)
   824			var timer *time.Timer // for canceling TLS handshake
   825			if d := t.TLSHandshakeTimeout; d != 0 {
   826				timer = time.AfterFunc(d, func() {
   827					errc <- tlsHandshakeTimeoutError{}
   828				})
   829			}
   830			go func() {
   831				err := tlsConn.Handshake()
   832				if timer != nil {
   833					timer.Stop()
   834				}
   835				errc <- err
   836			}()
   837			if err := <-errc; err != nil {
   838				plainConn.Close()
   839				return nil, err
   840			}
   841			if !cfg.InsecureSkipVerify {
   842				if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil {
   843					plainConn.Close()
   844					return nil, err
   845				}
   846			}
   847			cs := tlsConn.ConnectionState()
   848			pconn.tlsState = &cs
   849			pconn.conn = tlsConn
   850		}
   851	
   852		if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" {
   853			if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok {
   854				return &persistConn{alt: next(cm.targetAddr, pconn.conn.(*tls.Conn))}, nil
   855			}
   856		}
   857	
   858		pconn.br = bufio.NewReader(noteEOFReader{pconn.conn, &pconn.sawEOF})
   859		pconn.bw = bufio.NewWriter(pconn.conn)
   860		go pconn.readLoop()
   861		go pconn.writeLoop()
   862		return pconn, nil
   863	}
   864	
   865	// useProxy reports whether requests to addr should use a proxy,
   866	// according to the NO_PROXY or no_proxy environment variable.
   867	// addr is always a canonicalAddr with a host and port.
   868	func useProxy(addr string) bool {
   869		if len(addr) == 0 {
   870			return true
   871		}
   872		host, _, err := net.SplitHostPort(addr)
   873		if err != nil {
   874			return false
   875		}
   876		if host == "localhost" {
   877			return false
   878		}
   879		if ip := net.ParseIP(host); ip != nil {
   880			if ip.IsLoopback() {
   881				return false
   882			}
   883		}
   884	
   885		no_proxy := noProxyEnv.Get()
   886		if no_proxy == "*" {
   887			return false
   888		}
   889	
   890		addr = strings.ToLower(strings.TrimSpace(addr))
   891		if hasPort(addr) {
   892			addr = addr[:strings.LastIndex(addr, ":")]
   893		}
   894	
   895		for _, p := range strings.Split(no_proxy, ",") {
   896			p = strings.ToLower(strings.TrimSpace(p))
   897			if len(p) == 0 {
   898				continue
   899			}
   900			if hasPort(p) {
   901				p = p[:strings.LastIndex(p, ":")]
   902			}
   903			if addr == p {
   904				return false
   905			}
   906			if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) {
   907				// no_proxy ".foo.com" matches "bar.foo.com" or "foo.com"
   908				return false
   909			}
   910			if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' {
   911				// no_proxy "foo.com" matches "bar.foo.com"
   912				return false
   913			}
   914		}
   915		return true
   916	}
   917	
   918	// connectMethod is the map key (in its String form) for keeping persistent
   919	// TCP connections alive for subsequent HTTP requests.
   920	//
   921	// A connect method may be of the following types:
   922	//
   923	// Cache key form                Description
   924	// -----------------             -------------------------
   925	// |http|foo.com                 http directly to server, no proxy
   926	// |https|foo.com                https directly to server, no proxy
   927	// http://proxy.com|https|foo.com  http to proxy, then CONNECT to foo.com
   928	// http://proxy.com|http           http to proxy, http to anywhere after that
   929	//
   930	// Note: no support to https to the proxy yet.
   931	//
   932	type connectMethod struct {
   933		proxyURL     *url.URL // nil for no proxy, else full proxy URL
   934		targetScheme string   // "http" or "https"
   935		targetAddr   string   // Not used if proxy + http targetScheme (4th example in table)
   936	}
   937	
   938	func (cm *connectMethod) key() connectMethodKey {
   939		proxyStr := ""
   940		targetAddr := cm.targetAddr
   941		if cm.proxyURL != nil {
   942			proxyStr = cm.proxyURL.String()
   943			if cm.targetScheme == "http" {
   944				targetAddr = ""
   945			}
   946		}
   947		return connectMethodKey{
   948			proxy:  proxyStr,
   949			scheme: cm.targetScheme,
   950			addr:   targetAddr,
   951		}
   952	}
   953	
   954	// addr returns the first hop "host:port" to which we need to TCP connect.
   955	func (cm *connectMethod) addr() string {
   956		if cm.proxyURL != nil {
   957			return canonicalAddr(cm.proxyURL)
   958		}
   959		return cm.targetAddr
   960	}
   961	
   962	// tlsHost returns the host name to match against the peer's
   963	// TLS certificate.
   964	func (cm *connectMethod) tlsHost() string {
   965		h := cm.targetAddr
   966		if hasPort(h) {
   967			h = h[:strings.LastIndex(h, ":")]
   968		}
   969		return h
   970	}
   971	
   972	// connectMethodKey is the map key version of connectMethod, with a
   973	// stringified proxy URL (or the empty string) instead of a pointer to
   974	// a URL.
   975	type connectMethodKey struct {
   976		proxy, scheme, addr string
   977	}
   978	
   979	func (k connectMethodKey) String() string {
   980		// Only used by tests.
   981		return fmt.Sprintf("%s|%s|%s", k.proxy, k.scheme, k.addr)
   982	}
   983	
   984	// persistConn wraps a connection, usually a persistent one
   985	// (but may be used for non-keep-alive requests as well)
   986	type persistConn struct {
   987		// alt optionally specifies the TLS NextProto RoundTripper.
   988		// This is used for HTTP/2 today and future protocol laters.
   989		// If it's non-nil, the rest of the fields are unused.
   990		alt RoundTripper
   991	
   992		t        *Transport
   993		cacheKey connectMethodKey
   994		conn     net.Conn
   995		tlsState *tls.ConnectionState
   996		br       *bufio.Reader       // from conn
   997		sawEOF   bool                // whether we've seen EOF from conn; owned by readLoop
   998		bw       *bufio.Writer       // to conn
   999		reqch    chan requestAndChan // written by roundTrip; read by readLoop
  1000		writech  chan writeRequest   // written by roundTrip; read by writeLoop
  1001		closech  chan struct{}       // closed when conn closed
  1002		isProxy  bool
  1003		// writeErrCh passes the request write error (usually nil)
  1004		// from the writeLoop goroutine to the readLoop which passes
  1005		// it off to the res.Body reader, which then uses it to decide
  1006		// whether or not a connection can be reused. Issue 7569.
  1007		writeErrCh chan error
  1008	
  1009		lk                   sync.Mutex // guards following fields
  1010		numExpectedResponses int
  1011		closed               error // set non-nil when conn is closed, before closech is closed
  1012		broken               bool  // an error has happened on this connection; marked broken so it's not reused.
  1013		canceled             bool  // whether this conn was broken due a CancelRequest
  1014		reused               bool  // whether conn has had successful request/response and is being reused.
  1015		// mutateHeaderFunc is an optional func to modify extra
  1016		// headers on each outbound request before it's written. (the
  1017		// original Request given to RoundTrip is not modified)
  1018		mutateHeaderFunc func(Header)
  1019	}
  1020	
  1021	// isBroken reports whether this connection is in a known broken state.
  1022	func (pc *persistConn) isBroken() bool {
  1023		pc.lk.Lock()
  1024		b := pc.broken
  1025		pc.lk.Unlock()
  1026		return b
  1027	}
  1028	
  1029	// isCanceled reports whether this connection was closed due to CancelRequest.
  1030	func (pc *persistConn) isCanceled() bool {
  1031		pc.lk.Lock()
  1032		defer pc.lk.Unlock()
  1033		return pc.canceled
  1034	}
  1035	
  1036	// isReused reports whether this connection is in a known broken state.
  1037	func (pc *persistConn) isReused() bool {
  1038		pc.lk.Lock()
  1039		r := pc.reused
  1040		pc.lk.Unlock()
  1041		return r
  1042	}
  1043	
  1044	func (pc *persistConn) cancelRequest() {
  1045		pc.lk.Lock()
  1046		defer pc.lk.Unlock()
  1047		pc.canceled = true
  1048		pc.closeLocked(errRequestCanceled)
  1049	}
  1050	
  1051	func (pc *persistConn) readLoop() {
  1052		closeErr := errReadLoopExiting // default value, if not changed below
  1053		defer func() { pc.close(closeErr) }()
  1054	
  1055		tryPutIdleConn := func() bool {
  1056			if err := pc.t.tryPutIdleConn(pc); err != nil {
  1057				closeErr = err
  1058				return false
  1059			}
  1060			return true
  1061		}
  1062	
  1063		// eofc is used to block caller goroutines reading from Response.Body
  1064		// at EOF until this goroutines has (potentially) added the connection
  1065		// back to the idle pool.
  1066		eofc := make(chan struct{})
  1067		defer close(eofc) // unblock reader on errors
  1068	
  1069		// Read this once, before loop starts. (to avoid races in tests)
  1070		testHookMu.Lock()
  1071		testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead
  1072		testHookMu.Unlock()
  1073	
  1074		alive := true
  1075		for alive {
  1076			_, err := pc.br.Peek(1)
  1077			if err != nil {
  1078				err = beforeRespHeaderError{err}
  1079			}
  1080	
  1081			pc.lk.Lock()
  1082			if pc.numExpectedResponses == 0 {
  1083				pc.readLoopPeekFailLocked(err)
  1084				pc.lk.Unlock()
  1085				return
  1086			}
  1087			pc.lk.Unlock()
  1088	
  1089			rc := <-pc.reqch
  1090	
  1091			var resp *Response
  1092			if err == nil {
  1093				resp, err = pc.readResponse(rc)
  1094			}
  1095	
  1096			if err != nil {
  1097				// If we won't be able to retry this request later (from the
  1098				// roundTrip goroutine), mark it as done now.
  1099				// BEFORE the send on rc.ch, as the client might re-use the
  1100				// same *Request pointer, and we don't want to set call
  1101				// t.setReqCanceler from this persistConn while the Transport
  1102				// potentially spins up a different persistConn for the
  1103				// caller's subsequent request.
  1104				if checkTransportResend(err, rc.req, pc) != nil {
  1105					pc.t.setReqCanceler(rc.req, nil)
  1106				}
  1107				select {
  1108				case rc.ch <- responseAndError{err: err}:
  1109				case <-rc.callerGone:
  1110					return
  1111				}
  1112				return
  1113			}
  1114	
  1115			pc.lk.Lock()
  1116			pc.numExpectedResponses--
  1117			pc.lk.Unlock()
  1118	
  1119			hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0
  1120	
  1121			if resp.Close || rc.req.Close || resp.StatusCode <= 199 {
  1122				// Don't do keep-alive on error if either party requested a close
  1123				// or we get an unexpected informational (1xx) response.
  1124				// StatusCode 100 is already handled above.
  1125				alive = false
  1126			}
  1127	
  1128			if !hasBody {
  1129				pc.t.setReqCanceler(rc.req, nil)
  1130	
  1131				// Put the idle conn back into the pool before we send the response
  1132				// so if they process it quickly and make another request, they'll
  1133				// get this same conn. But we use the unbuffered channel 'rc'
  1134				// to guarantee that persistConn.roundTrip got out of its select
  1135				// potentially waiting for this persistConn to close.
  1136				// but after
  1137				alive = alive &&
  1138					!pc.sawEOF &&
  1139					pc.wroteRequest() &&
  1140					tryPutIdleConn()
  1141	
  1142				select {
  1143				case rc.ch <- responseAndError{res: resp}:
  1144				case <-rc.callerGone:
  1145					return
  1146				}
  1147	
  1148				// Now that they've read from the unbuffered channel, they're safely
  1149				// out of the select that also waits on this goroutine to die, so
  1150				// we're allowed to exit now if needed (if alive is false)
  1151				testHookReadLoopBeforeNextRead()
  1152				continue
  1153			}
  1154	
  1155			if rc.addedGzip {
  1156				maybeUngzipResponse(resp)
  1157			}
  1158			resp.Body = &bodyEOFSignal{body: resp.Body}
  1159	
  1160			waitForBodyRead := make(chan bool, 2)
  1161			resp.Body.(*bodyEOFSignal).earlyCloseFn = func() error {
  1162				waitForBodyRead <- false
  1163				return nil
  1164			}
  1165			resp.Body.(*bodyEOFSignal).fn = func(err error) error {
  1166				isEOF := err == io.EOF
  1167				waitForBodyRead <- isEOF
  1168				if isEOF {
  1169					<-eofc // see comment above eofc declaration
  1170				} else if err != nil && pc.isCanceled() {
  1171					return errRequestCanceled
  1172				}
  1173				return err
  1174			}
  1175	
  1176			select {
  1177			case rc.ch <- responseAndError{res: resp}:
  1178			case <-rc.callerGone:
  1179				return
  1180			}
  1181	
  1182			// Before looping back to the top of this function and peeking on
  1183			// the bufio.Reader, wait for the caller goroutine to finish
  1184			// reading the response body. (or for cancelation or death)
  1185			select {
  1186			case bodyEOF := <-waitForBodyRead:
  1187				pc.t.setReqCanceler(rc.req, nil) // before pc might return to idle pool
  1188				alive = alive &&
  1189					bodyEOF &&
  1190					!pc.sawEOF &&
  1191					pc.wroteRequest() &&
  1192					tryPutIdleConn()
  1193				if bodyEOF {
  1194					eofc <- struct{}{}
  1195				}
  1196			case <-rc.req.Cancel:
  1197				alive = false
  1198				pc.t.CancelRequest(rc.req)
  1199			case <-pc.closech:
  1200				alive = false
  1201			}
  1202	
  1203			testHookReadLoopBeforeNextRead()
  1204		}
  1205	}
  1206	
  1207	func maybeUngzipResponse(resp *Response) {
  1208		if resp.Header.Get("Content-Encoding") == "gzip" {
  1209			resp.Header.Del("Content-Encoding")
  1210			resp.Header.Del("Content-Length")
  1211			resp.ContentLength = -1
  1212			resp.Body = &gzipReader{body: resp.Body}
  1213		}
  1214	}
  1215	
  1216	func (pc *persistConn) readLoopPeekFailLocked(peekErr error) {
  1217		if pc.closed != nil {
  1218			return
  1219		}
  1220		if n := pc.br.Buffered(); n > 0 {
  1221			buf, _ := pc.br.Peek(n)
  1222			log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr)
  1223		}
  1224		if peekErr == io.EOF {
  1225			// common case.
  1226			pc.closeLocked(errServerClosedIdle)
  1227		} else {
  1228			pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %v", peekErr))
  1229		}
  1230	}
  1231	
  1232	// readResponse reads an HTTP response (or two, in the case of "Expect:
  1233	// 100-continue") from the server. It returns the final non-100 one.
  1234	func (pc *persistConn) readResponse(rc requestAndChan) (resp *Response, err error) {
  1235		resp, err = ReadResponse(pc.br, rc.req)
  1236		if err != nil {
  1237			return
  1238		}
  1239		if rc.continueCh != nil {
  1240			if resp.StatusCode == 100 {
  1241				rc.continueCh <- struct{}{}
  1242			} else {
  1243				close(rc.continueCh)
  1244			}
  1245		}
  1246		if resp.StatusCode == 100 {
  1247			resp, err = ReadResponse(pc.br, rc.req)
  1248			if err != nil {
  1249				return
  1250			}
  1251		}
  1252		resp.TLS = pc.tlsState
  1253		return
  1254	}
  1255	
  1256	// waitForContinue returns the function to block until
  1257	// any response, timeout or connection close. After any of them,
  1258	// the function returns a bool which indicates if the body should be sent.
  1259	func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool {
  1260		if continueCh == nil {
  1261			return nil
  1262		}
  1263		return func() bool {
  1264			timer := time.NewTimer(pc.t.ExpectContinueTimeout)
  1265			defer timer.Stop()
  1266	
  1267			select {
  1268			case _, ok := <-continueCh:
  1269				return ok
  1270			case <-timer.C:
  1271				return true
  1272			case <-pc.closech:
  1273				return false
  1274			}
  1275		}
  1276	}
  1277	
  1278	func (pc *persistConn) writeLoop() {
  1279		for {
  1280			select {
  1281			case wr := <-pc.writech:
  1282				if pc.isBroken() {
  1283					wr.ch <- errors.New("http: can't write HTTP request on broken connection")
  1284					continue
  1285				}
  1286				err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh))
  1287				if err == nil {
  1288					err = pc.bw.Flush()
  1289				}
  1290				if err != nil {
  1291					pc.markBroken()
  1292					wr.req.Request.closeBody()
  1293				}
  1294				pc.writeErrCh <- err // to the body reader, which might recycle us
  1295				wr.ch <- err         // to the roundTrip function
  1296			case <-pc.closech:
  1297				return
  1298			}
  1299		}
  1300	}
  1301	
  1302	// wroteRequest is a check before recycling a connection that the previous write
  1303	// (from writeLoop above) happened and was successful.
  1304	func (pc *persistConn) wroteRequest() bool {
  1305		select {
  1306		case err := <-pc.writeErrCh:
  1307			// Common case: the write happened well before the response, so
  1308			// avoid creating a timer.
  1309			return err == nil
  1310		default:
  1311			// Rare case: the request was written in writeLoop above but
  1312			// before it could send to pc.writeErrCh, the reader read it
  1313			// all, processed it, and called us here. In this case, give the
  1314			// write goroutine a bit of time to finish its send.
  1315			//
  1316			// Less rare case: We also get here in the legitimate case of
  1317			// Issue 7569, where the writer is still writing (or stalled),
  1318			// but the server has already replied. In this case, we don't
  1319			// want to wait too long, and we want to return false so this
  1320			// connection isn't re-used.
  1321			select {
  1322			case err := <-pc.writeErrCh:
  1323				return err == nil
  1324			case <-time.After(50 * time.Millisecond):
  1325				return false
  1326			}
  1327		}
  1328	}
  1329	
  1330	// responseAndError is how the goroutine reading from an HTTP/1 server
  1331	// communicates with the goroutine doing the RoundTrip.
  1332	type responseAndError struct {
  1333		res *Response // else use this response (see res method)
  1334		err error
  1335	}
  1336	
  1337	type requestAndChan struct {
  1338		req *Request
  1339		ch  chan responseAndError // unbuffered; always send in select on callerGone
  1340	
  1341		// did the Transport (as opposed to the client code) add an
  1342		// Accept-Encoding gzip header? only if it we set it do
  1343		// we transparently decode the gzip.
  1344		addedGzip bool
  1345	
  1346		// Optional blocking chan for Expect: 100-continue (for send).
  1347		// If the request has an "Expect: 100-continue" header and
  1348		// the server responds 100 Continue, readLoop send a value
  1349		// to writeLoop via this chan.
  1350		continueCh chan<- struct{}
  1351	
  1352		callerGone <-chan struct{} // closed when roundTrip caller has returned
  1353	}
  1354	
  1355	// A writeRequest is sent by the readLoop's goroutine to the
  1356	// writeLoop's goroutine to write a request while the read loop
  1357	// concurrently waits on both the write response and the server's
  1358	// reply.
  1359	type writeRequest struct {
  1360		req *transportRequest
  1361		ch  chan<- error
  1362	
  1363		// Optional blocking chan for Expect: 100-continue (for recieve).
  1364		// If not nil, writeLoop blocks sending request body until
  1365		// it receives from this chan.
  1366		continueCh <-chan struct{}
  1367	}
  1368	
  1369	type httpError struct {
  1370		err     string
  1371		timeout bool
  1372	}
  1373	
  1374	func (e *httpError) Error() string   { return e.err }
  1375	func (e *httpError) Timeout() bool   { return e.timeout }
  1376	func (e *httpError) Temporary() bool { return true }
  1377	
  1378	var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true}
  1379	var errClosed error = &httpError{err: "net/http: server closed connection before response was received"}
  1380	var errRequestCanceled = errors.New("net/http: request canceled")
  1381	var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify?
  1382	
  1383	func nop() {}
  1384	
  1385	// testHooks. Always non-nil.
  1386	var (
  1387		testHookEnterRoundTrip   = nop
  1388		testHookWaitResLoop      = nop
  1389		testHookRoundTripRetried = nop
  1390		testHookPrePendingDial   = nop
  1391		testHookPostPendingDial  = nop
  1392	
  1393		testHookMu                     sync.Locker = fakeLocker{} // guards following
  1394		testHookReadLoopBeforeNextRead             = nop
  1395	)
  1396	
  1397	// beforeRespHeaderError is used to indicate when an IO error has occurred before
  1398	// any header data was received.
  1399	type beforeRespHeaderError struct {
  1400		error
  1401	}
  1402	
  1403	func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) {
  1404		testHookEnterRoundTrip()
  1405		if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) {
  1406			pc.t.putOrCloseIdleConn(pc)
  1407			return nil, errRequestCanceled
  1408		}
  1409		pc.lk.Lock()
  1410		pc.numExpectedResponses++
  1411		headerFn := pc.mutateHeaderFunc
  1412		pc.lk.Unlock()
  1413	
  1414		if headerFn != nil {
  1415			headerFn(req.extraHeaders())
  1416		}
  1417	
  1418		// Ask for a compressed version if the caller didn't set their
  1419		// own value for Accept-Encoding. We only attempt to
  1420		// uncompress the gzip stream if we were the layer that
  1421		// requested it.
  1422		requestedGzip := false
  1423		if !pc.t.DisableCompression &&
  1424			req.Header.Get("Accept-Encoding") == "" &&
  1425			req.Header.Get("Range") == "" &&
  1426			req.Method != "HEAD" {
  1427			// Request gzip only, not deflate. Deflate is ambiguous and
  1428			// not as universally supported anyway.
  1429			// See: http://www.gzip.org/zlib/zlib_faq.html#faq38
  1430			//
  1431			// Note that we don't request this for HEAD requests,
  1432			// due to a bug in nginx:
  1433			//   http://trac.nginx.org/nginx/ticket/358
  1434			//   https://golang.org/issue/5522
  1435			//
  1436			// We don't request gzip if the request is for a range, since
  1437			// auto-decoding a portion of a gzipped document will just fail
  1438			// anyway. See https://golang.org/issue/8923
  1439			requestedGzip = true
  1440			req.extraHeaders().Set("Accept-Encoding", "gzip")
  1441		}
  1442	
  1443		var continueCh chan struct{}
  1444		if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() {
  1445			continueCh = make(chan struct{}, 1)
  1446		}
  1447	
  1448		if pc.t.DisableKeepAlives {
  1449			req.extraHeaders().Set("Connection", "close")
  1450		}
  1451	
  1452		gone := make(chan struct{})
  1453		defer close(gone)
  1454	
  1455		// Write the request concurrently with waiting for a response,
  1456		// in case the server decides to reply before reading our full
  1457		// request body.
  1458		writeErrCh := make(chan error, 1)
  1459		pc.writech <- writeRequest{req, writeErrCh, continueCh}
  1460	
  1461		resc := make(chan responseAndError)
  1462		pc.reqch <- requestAndChan{
  1463			req:        req.Request,
  1464			ch:         resc,
  1465			addedGzip:  requestedGzip,
  1466			continueCh: continueCh,
  1467			callerGone: gone,
  1468		}
  1469	
  1470		var re responseAndError
  1471		var respHeaderTimer <-chan time.Time
  1472		cancelChan := req.Request.Cancel
  1473	WaitResponse:
  1474		for {
  1475			testHookWaitResLoop()
  1476			select {
  1477			case err := <-writeErrCh:
  1478				if err != nil {
  1479					if pc.isCanceled() {
  1480						err = errRequestCanceled
  1481					}
  1482					re = responseAndError{err: beforeRespHeaderError{err}}
  1483					pc.close(fmt.Errorf("write error: %v", err))
  1484					break WaitResponse
  1485				}
  1486				if d := pc.t.ResponseHeaderTimeout; d > 0 {
  1487					timer := time.NewTimer(d)
  1488					defer timer.Stop() // prevent leaks
  1489					respHeaderTimer = timer.C
  1490				}
  1491			case <-pc.closech:
  1492				var err error
  1493				if pc.isCanceled() {
  1494					err = errRequestCanceled
  1495				} else {
  1496					err = beforeRespHeaderError{fmt.Errorf("net/http: HTTP/1 transport connection broken: %v", pc.closed)}
  1497				}
  1498				re = responseAndError{err: err}
  1499				break WaitResponse
  1500			case <-respHeaderTimer:
  1501				pc.close(errTimeout)
  1502				re = responseAndError{err: errTimeout}
  1503				break WaitResponse
  1504			case re = <-resc:
  1505				if re.err != nil && pc.isCanceled() {
  1506					re.err = errRequestCanceled
  1507				}
  1508				break WaitResponse
  1509			case <-cancelChan:
  1510				pc.t.CancelRequest(req.Request)
  1511				cancelChan = nil
  1512			}
  1513		}
  1514	
  1515		if re.err != nil {
  1516			pc.t.setReqCanceler(req.Request, nil)
  1517		}
  1518		if (re.res == nil) == (re.err == nil) {
  1519			panic("internal error: exactly one of res or err should be set")
  1520		}
  1521		return re.res, re.err
  1522	}
  1523	
  1524	// markBroken marks a connection as broken (so it's not reused).
  1525	// It differs from close in that it doesn't close the underlying
  1526	// connection for use when it's still being read.
  1527	func (pc *persistConn) markBroken() {
  1528		pc.lk.Lock()
  1529		defer pc.lk.Unlock()
  1530		pc.broken = true
  1531	}
  1532	
  1533	// markReused marks this connection as having been successfully used for a
  1534	// request and response.
  1535	func (pc *persistConn) markReused() {
  1536		pc.lk.Lock()
  1537		pc.reused = true
  1538		pc.lk.Unlock()
  1539	}
  1540	
  1541	// close closes the underlying TCP connection and closes
  1542	// the pc.closech channel.
  1543	//
  1544	// The provided err is only for testing and debugging; in normal
  1545	// circumstances it should never be seen by users.
  1546	func (pc *persistConn) close(err error) {
  1547		pc.lk.Lock()
  1548		defer pc.lk.Unlock()
  1549		pc.closeLocked(err)
  1550	}
  1551	
  1552	func (pc *persistConn) closeLocked(err error) {
  1553		if err == nil {
  1554			panic("nil error")
  1555		}
  1556		pc.broken = true
  1557		if pc.closed == nil {
  1558			pc.closed = err
  1559			if pc.alt != nil {
  1560				// Do nothing; can only get here via getConn's
  1561				// handlePendingDial's putOrCloseIdleConn when
  1562				// it turns out the abandoned connection in
  1563				// flight ended up negotiating an alternate
  1564				// protocol.  We don't use the connection
  1565				// freelist for http2. That's done by the
  1566				// alternate protocol's RoundTripper.
  1567			} else {
  1568				pc.conn.Close()
  1569				close(pc.closech)
  1570			}
  1571		}
  1572		pc.mutateHeaderFunc = nil
  1573	}
  1574	
  1575	var portMap = map[string]string{
  1576		"http":  "80",
  1577		"https": "443",
  1578	}
  1579	
  1580	// canonicalAddr returns url.Host but always with a ":port" suffix
  1581	func canonicalAddr(url *url.URL) string {
  1582		addr := url.Host
  1583		if !hasPort(addr) {
  1584			return addr + ":" + portMap[url.Scheme]
  1585		}
  1586		return addr
  1587	}
  1588	
  1589	// bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most
  1590	// once, right before its final (error-producing) Read or Close call
  1591	// returns. fn should return the new error to return from Read or Close.
  1592	//
  1593	// If earlyCloseFn is non-nil and Close is called before io.EOF is
  1594	// seen, earlyCloseFn is called instead of fn, and its return value is
  1595	// the return value from Close.
  1596	type bodyEOFSignal struct {
  1597		body         io.ReadCloser
  1598		mu           sync.Mutex        // guards following 4 fields
  1599		closed       bool              // whether Close has been called
  1600		rerr         error             // sticky Read error
  1601		fn           func(error) error // err will be nil on Read io.EOF
  1602		earlyCloseFn func() error      // optional alt Close func used if io.EOF not seen
  1603	}
  1604	
  1605	func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
  1606		es.mu.Lock()
  1607		closed, rerr := es.closed, es.rerr
  1608		es.mu.Unlock()
  1609		if closed {
  1610			return 0, errors.New("http: read on closed response body")
  1611		}
  1612		if rerr != nil {
  1613			return 0, rerr
  1614		}
  1615	
  1616		n, err = es.body.Read(p)
  1617		if err != nil {
  1618			es.mu.Lock()
  1619			defer es.mu.Unlock()
  1620			if es.rerr == nil {
  1621				es.rerr = err
  1622			}
  1623			err = es.condfn(err)
  1624		}
  1625		return
  1626	}
  1627	
  1628	func (es *bodyEOFSignal) Close() error {
  1629		es.mu.Lock()
  1630		defer es.mu.Unlock()
  1631		if es.closed {
  1632			return nil
  1633		}
  1634		es.closed = true
  1635		if es.earlyCloseFn != nil && es.rerr != io.EOF {
  1636			return es.earlyCloseFn()
  1637		}
  1638		err := es.body.Close()
  1639		return es.condfn(err)
  1640	}
  1641	
  1642	// caller must hold es.mu.
  1643	func (es *bodyEOFSignal) condfn(err error) error {
  1644		if es.fn == nil {
  1645			return err
  1646		}
  1647		err = es.fn(err)
  1648		es.fn = nil
  1649		return err
  1650	}
  1651	
  1652	// gzipReader wraps a response body so it can lazily
  1653	// call gzip.NewReader on the first call to Read
  1654	type gzipReader struct {
  1655		body io.ReadCloser // underlying Response.Body
  1656		zr   io.Reader     // lazily-initialized gzip reader
  1657	}
  1658	
  1659	func (gz *gzipReader) Read(p []byte) (n int, err error) {
  1660		if gz.zr == nil {
  1661			gz.zr, err = gzip.NewReader(gz.body)
  1662			if err != nil {
  1663				return 0, err
  1664			}
  1665		}
  1666		return gz.zr.Read(p)
  1667	}
  1668	
  1669	func (gz *gzipReader) Close() error {
  1670		return gz.body.Close()
  1671	}
  1672	
  1673	type readerAndCloser struct {
  1674		io.Reader
  1675		io.Closer
  1676	}
  1677	
  1678	type tlsHandshakeTimeoutError struct{}
  1679	
  1680	func (tlsHandshakeTimeoutError) Timeout() bool   { return true }
  1681	func (tlsHandshakeTimeoutError) Temporary() bool { return true }
  1682	func (tlsHandshakeTimeoutError) Error() string   { return "net/http: TLS handshake timeout" }
  1683	
  1684	type noteEOFReader struct {
  1685		r      io.Reader
  1686		sawEOF *bool
  1687	}
  1688	
  1689	func (nr noteEOFReader) Read(p []byte) (n int, err error) {
  1690		n, err = nr.r.Read(p)
  1691		if err == io.EOF {
  1692			*nr.sawEOF = true
  1693		}
  1694		return
  1695	}
  1696	
  1697	// fakeLocker is a sync.Locker which does nothing. It's used to guard
  1698	// test-only fields when not under test, to avoid runtime atomic
  1699	// overhead.
  1700	type fakeLocker struct{}
  1701	
  1702	func (fakeLocker) Lock()   {}
  1703	func (fakeLocker) Unlock() {}
  1704	
  1705	func isNetWriteError(err error) bool {
  1706		switch e := err.(type) {
  1707		case *url.Error:
  1708			return isNetWriteError(e.Err)
  1709		case *net.OpError:
  1710			return e.Op == "write"
  1711		default:
  1712			return false
  1713		}
  1714	}
  1715	
  1716	// cloneTLSConfig returns a shallow clone of the exported
  1717	// fields of cfg, ignoring the unexported sync.Once, which
  1718	// contains a mutex and must not be copied.
  1719	//
  1720	// The cfg must not be in active use by tls.Server, or else
  1721	// there can still be a race with tls.Server updating SessionTicketKey
  1722	// and our copying it, and also a race with the server setting
  1723	// SessionTicketsDisabled=false on failure to set the random
  1724	// ticket key.
  1725	//
  1726	// If cfg is nil, a new zero tls.Config is returned.
  1727	func cloneTLSConfig(cfg *tls.Config) *tls.Config {
  1728		if cfg == nil {
  1729			return &tls.Config{}
  1730		}
  1731		return &tls.Config{
  1732			Rand:                     cfg.Rand,
  1733			Time:                     cfg.Time,
  1734			Certificates:             cfg.Certificates,
  1735			NameToCertificate:        cfg.NameToCertificate,
  1736			GetCertificate:           cfg.GetCertificate,
  1737			RootCAs:                  cfg.RootCAs,
  1738			NextProtos:               cfg.NextProtos,
  1739			ServerName:               cfg.ServerName,
  1740			ClientAuth:               cfg.ClientAuth,
  1741			ClientCAs:                cfg.ClientCAs,
  1742			InsecureSkipVerify:       cfg.InsecureSkipVerify,
  1743			CipherSuites:             cfg.CipherSuites,
  1744			PreferServerCipherSuites: cfg.PreferServerCipherSuites,
  1745			SessionTicketsDisabled:   cfg.SessionTicketsDisabled,
  1746			SessionTicketKey:         cfg.SessionTicketKey,
  1747			ClientSessionCache:       cfg.ClientSessionCache,
  1748			MinVersion:               cfg.MinVersion,
  1749			MaxVersion:               cfg.MaxVersion,
  1750			CurvePreferences:         cfg.CurvePreferences,
  1751		}
  1752	}
  1753	
  1754	// cloneTLSClientConfig is like cloneTLSConfig but omits
  1755	// the fields SessionTicketsDisabled and SessionTicketKey.
  1756	// This makes it safe to call cloneTLSClientConfig on a config
  1757	// in active use by a server.
  1758	func cloneTLSClientConfig(cfg *tls.Config) *tls.Config {
  1759		if cfg == nil {
  1760			return &tls.Config{}
  1761		}
  1762		return &tls.Config{
  1763			Rand:                     cfg.Rand,
  1764			Time:                     cfg.Time,
  1765			Certificates:             cfg.Certificates,
  1766			NameToCertificate:        cfg.NameToCertificate,
  1767			GetCertificate:           cfg.GetCertificate,
  1768			RootCAs:                  cfg.RootCAs,
  1769			NextProtos:               cfg.NextProtos,
  1770			ServerName:               cfg.ServerName,
  1771			ClientAuth:               cfg.ClientAuth,
  1772			ClientCAs:                cfg.ClientCAs,
  1773			InsecureSkipVerify:       cfg.InsecureSkipVerify,
  1774			CipherSuites:             cfg.CipherSuites,
  1775			PreferServerCipherSuites: cfg.PreferServerCipherSuites,
  1776			ClientSessionCache:       cfg.ClientSessionCache,
  1777			MinVersion:               cfg.MinVersion,
  1778			MaxVersion:               cfg.MaxVersion,
  1779			CurvePreferences:         cfg.CurvePreferences,
  1780		}
  1781	}
  1782	

View as plain text