Source file src/net/http/transport.go

Documentation: net/http

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // HTTP client implementation. See RFC 7230 through 7235.
     6  //
     7  // This is the low-level Transport implementation of RoundTripper.
     8  // The high-level interface is in client.go.
     9  
    10  package http
    11  
    12  import (
    13  	"bufio"
    14  	"compress/gzip"
    15  	"container/list"
    16  	"context"
    17  	"crypto/tls"
    18  	"errors"
    19  	"fmt"
    20  	"io"
    21  	"log"
    22  	"net"
    23  	"net/http/httptrace"
    24  	"net/textproto"
    25  	"net/url"
    26  	"os"
    27  	"reflect"
    28  	"strings"
    29  	"sync"
    30  	"sync/atomic"
    31  	"time"
    32  
    33  	"golang.org/x/net/http/httpguts"
    34  	"golang.org/x/net/http/httpproxy"
    35  )
    36  
    37  // DefaultTransport is the default implementation of Transport and is
    38  // used by DefaultClient. It establishes network connections as needed
    39  // and caches them for reuse by subsequent calls. It uses HTTP proxies
    40  // as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and
    41  // $no_proxy) environment variables.
    42  var DefaultTransport RoundTripper = &Transport{
    43  	Proxy: ProxyFromEnvironment,
    44  	DialContext: (&net.Dialer{
    45  		Timeout:   30 * time.Second,
    46  		KeepAlive: 30 * time.Second,
    47  		DualStack: true,
    48  	}).DialContext,
    49  	ForceAttemptHTTP2:     true,
    50  	MaxIdleConns:          100,
    51  	IdleConnTimeout:       90 * time.Second,
    52  	TLSHandshakeTimeout:   10 * time.Second,
    53  	ExpectContinueTimeout: 1 * time.Second,
    54  }
    55  
    56  // DefaultMaxIdleConnsPerHost is the default value of Transport's
    57  // MaxIdleConnsPerHost.
    58  const DefaultMaxIdleConnsPerHost = 2
    59  
    60  // Transport is an implementation of RoundTripper that supports HTTP,
    61  // HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT).
    62  //
    63  // By default, Transport caches connections for future re-use.
    64  // This may leave many open connections when accessing many hosts.
    65  // This behavior can be managed using Transport's CloseIdleConnections method
    66  // and the MaxIdleConnsPerHost and DisableKeepAlives fields.
    67  //
    68  // Transports should be reused instead of created as needed.
    69  // Transports are safe for concurrent use by multiple goroutines.
    70  //
    71  // A Transport is a low-level primitive for making HTTP and HTTPS requests.
    72  // For high-level functionality, such as cookies and redirects, see Client.
    73  //
    74  // Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2
    75  // for HTTPS URLs, depending on whether the server supports HTTP/2,
    76  // and how the Transport is configured. The DefaultTransport supports HTTP/2.
    77  // To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2
    78  // and call ConfigureTransport. See the package docs for more about HTTP/2.
    79  //
    80  // Responses with status codes in the 1xx range are either handled
    81  // automatically (100 expect-continue) or ignored. The one
    82  // exception is HTTP status code 101 (Switching Protocols), which is
    83  // considered a terminal status and returned by RoundTrip. To see the
    84  // ignored 1xx responses, use the httptrace trace package's
    85  // ClientTrace.Got1xxResponse.
    86  //
    87  // Transport only retries a request upon encountering a network error
    88  // if the request is idempotent and either has no body or has its
    89  // Request.GetBody defined. HTTP requests are considered idempotent if
    90  // they have HTTP methods GET, HEAD, OPTIONS, or TRACE; or if their
    91  // Header map contains an "Idempotency-Key" or "X-Idempotency-Key"
    92  // entry. If the idempotency key value is an zero-length slice, the
    93  // request is treated as idempotent but the header is not sent on the
    94  // wire.
    95  type Transport struct {
    96  	idleMu       sync.Mutex
    97  	closeIdle    bool                                // user has requested to close all idle conns
    98  	idleConn     map[connectMethodKey][]*persistConn // most recently used at end
    99  	idleConnWait map[connectMethodKey]wantConnQueue  // waiting getConns
   100  	idleLRU      connLRU
   101  
   102  	reqMu       sync.Mutex
   103  	reqCanceler map[*Request]func(error)
   104  
   105  	altMu    sync.Mutex   // guards changing altProto only
   106  	altProto atomic.Value // of nil or map[string]RoundTripper, key is URI scheme
   107  
   108  	connsPerHostMu   sync.Mutex
   109  	connsPerHost     map[connectMethodKey]int
   110  	connsPerHostWait map[connectMethodKey]wantConnQueue // waiting getConns
   111  
   112  	// Proxy specifies a function to return a proxy for a given
   113  	// Request. If the function returns a non-nil error, the
   114  	// request is aborted with the provided error.
   115  	//
   116  	// The proxy type is determined by the URL scheme. "http",
   117  	// "https", and "socks5" are supported. If the scheme is empty,
   118  	// "http" is assumed.
   119  	//
   120  	// If Proxy is nil or returns a nil *URL, no proxy is used.
   121  	Proxy func(*Request) (*url.URL, error)
   122  
   123  	// DialContext specifies the dial function for creating unencrypted TCP connections.
   124  	// If DialContext is nil (and the deprecated Dial below is also nil),
   125  	// then the transport dials using package net.
   126  	//
   127  	// DialContext runs concurrently with calls to RoundTrip.
   128  	// A RoundTrip call that initiates a dial may end up using
   129  	// a connection dialed previously when the earlier connection
   130  	// becomes idle before the later DialContext completes.
   131  	DialContext func(ctx context.Context, network, addr string) (net.Conn, error)
   132  
   133  	// Dial specifies the dial function for creating unencrypted TCP connections.
   134  	//
   135  	// Dial runs concurrently with calls to RoundTrip.
   136  	// A RoundTrip call that initiates a dial may end up using
   137  	// a connection dialed previously when the earlier connection
   138  	// becomes idle before the later Dial completes.
   139  	//
   140  	// Deprecated: Use DialContext instead, which allows the transport
   141  	// to cancel dials as soon as they are no longer needed.
   142  	// If both are set, DialContext takes priority.
   143  	Dial func(network, addr string) (net.Conn, error)
   144  
   145  	// DialTLS specifies an optional dial function for creating
   146  	// TLS connections for non-proxied HTTPS requests.
   147  	//
   148  	// If DialTLS is nil, Dial and TLSClientConfig are used.
   149  	//
   150  	// If DialTLS is set, the Dial hook is not used for HTTPS
   151  	// requests and the TLSClientConfig and TLSHandshakeTimeout
   152  	// are ignored. The returned net.Conn is assumed to already be
   153  	// past the TLS handshake.
   154  	DialTLS func(network, addr string) (net.Conn, error)
   155  
   156  	// TLSClientConfig specifies the TLS configuration to use with
   157  	// tls.Client.
   158  	// If nil, the default configuration is used.
   159  	// If non-nil, HTTP/2 support may not be enabled by default.
   160  	TLSClientConfig *tls.Config
   161  
   162  	// TLSHandshakeTimeout specifies the maximum amount of time waiting to
   163  	// wait for a TLS handshake. Zero means no timeout.
   164  	TLSHandshakeTimeout time.Duration
   165  
   166  	// DisableKeepAlives, if true, disables HTTP keep-alives and
   167  	// will only use the connection to the server for a single
   168  	// HTTP request.
   169  	//
   170  	// This is unrelated to the similarly named TCP keep-alives.
   171  	DisableKeepAlives bool
   172  
   173  	// DisableCompression, if true, prevents the Transport from
   174  	// requesting compression with an "Accept-Encoding: gzip"
   175  	// request header when the Request contains no existing
   176  	// Accept-Encoding value. If the Transport requests gzip on
   177  	// its own and gets a gzipped response, it's transparently
   178  	// decoded in the Response.Body. However, if the user
   179  	// explicitly requested gzip it is not automatically
   180  	// uncompressed.
   181  	DisableCompression bool
   182  
   183  	// MaxIdleConns controls the maximum number of idle (keep-alive)
   184  	// connections across all hosts. Zero means no limit.
   185  	MaxIdleConns int
   186  
   187  	// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
   188  	// (keep-alive) connections to keep per-host. If zero,
   189  	// DefaultMaxIdleConnsPerHost is used.
   190  	MaxIdleConnsPerHost int
   191  
   192  	// MaxConnsPerHost optionally limits the total number of
   193  	// connections per host, including connections in the dialing,
   194  	// active, and idle states. On limit violation, dials will block.
   195  	//
   196  	// Zero means no limit.
   197  	MaxConnsPerHost int
   198  
   199  	// IdleConnTimeout is the maximum amount of time an idle
   200  	// (keep-alive) connection will remain idle before closing
   201  	// itself.
   202  	// Zero means no limit.
   203  	IdleConnTimeout time.Duration
   204  
   205  	// ResponseHeaderTimeout, if non-zero, specifies the amount of
   206  	// time to wait for a server's response headers after fully
   207  	// writing the request (including its body, if any). This
   208  	// time does not include the time to read the response body.
   209  	ResponseHeaderTimeout time.Duration
   210  
   211  	// ExpectContinueTimeout, if non-zero, specifies the amount of
   212  	// time to wait for a server's first response headers after fully
   213  	// writing the request headers if the request has an
   214  	// "Expect: 100-continue" header. Zero means no timeout and
   215  	// causes the body to be sent immediately, without
   216  	// waiting for the server to approve.
   217  	// This time does not include the time to send the request header.
   218  	ExpectContinueTimeout time.Duration
   219  
   220  	// TLSNextProto specifies how the Transport switches to an
   221  	// alternate protocol (such as HTTP/2) after a TLS NPN/ALPN
   222  	// protocol negotiation. If Transport dials an TLS connection
   223  	// with a non-empty protocol name and TLSNextProto contains a
   224  	// map entry for that key (such as "h2"), then the func is
   225  	// called with the request's authority (such as "example.com"
   226  	// or "example.com:1234") and the TLS connection. The function
   227  	// must return a RoundTripper that then handles the request.
   228  	// If TLSNextProto is not nil, HTTP/2 support is not enabled
   229  	// automatically.
   230  	TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper
   231  
   232  	// ProxyConnectHeader optionally specifies headers to send to
   233  	// proxies during CONNECT requests.
   234  	ProxyConnectHeader Header
   235  
   236  	// MaxResponseHeaderBytes specifies a limit on how many
   237  	// response bytes are allowed in the server's response
   238  	// header.
   239  	//
   240  	// Zero means to use a default limit.
   241  	MaxResponseHeaderBytes int64
   242  
   243  	// WriteBufferSize specifies the size of the write buffer used
   244  	// when writing to the transport.
   245  	// If zero, a default (currently 4KB) is used.
   246  	WriteBufferSize int
   247  
   248  	// ReadBufferSize specifies the size of the read buffer used
   249  	// when reading from the transport.
   250  	// If zero, a default (currently 4KB) is used.
   251  	ReadBufferSize int
   252  
   253  	// nextProtoOnce guards initialization of TLSNextProto and
   254  	// h2transport (via onceSetNextProtoDefaults)
   255  	nextProtoOnce      sync.Once
   256  	h2transport        h2Transport // non-nil if http2 wired up
   257  	tlsNextProtoWasNil bool        // whether TLSNextProto was nil when the Once fired
   258  
   259  	// ForceAttemptHTTP2 controls whether HTTP/2 is enabled when a non-zero
   260  	// Dial, DialTLS, or DialContext func or TLSClientConfig is provided.
   261  	// By default, use of any those fields conservatively disables HTTP/2.
   262  	// To use a custom dialer or TLS config and still attempt HTTP/2
   263  	// upgrades, set this to true.
   264  	ForceAttemptHTTP2 bool
   265  }
   266  
   267  func (t *Transport) writeBufferSize() int {
   268  	if t.WriteBufferSize > 0 {
   269  		return t.WriteBufferSize
   270  	}
   271  	return 4 << 10
   272  }
   273  
   274  func (t *Transport) readBufferSize() int {
   275  	if t.ReadBufferSize > 0 {
   276  		return t.ReadBufferSize
   277  	}
   278  	return 4 << 10
   279  }
   280  
   281  // Clone returns a deep copy of t's exported fields.
   282  func (t *Transport) Clone() *Transport {
   283  	t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
   284  	t2 := &Transport{
   285  		Proxy:                  t.Proxy,
   286  		DialContext:            t.DialContext,
   287  		Dial:                   t.Dial,
   288  		DialTLS:                t.DialTLS,
   289  		TLSClientConfig:        t.TLSClientConfig.Clone(),
   290  		TLSHandshakeTimeout:    t.TLSHandshakeTimeout,
   291  		DisableKeepAlives:      t.DisableKeepAlives,
   292  		DisableCompression:     t.DisableCompression,
   293  		MaxIdleConns:           t.MaxIdleConns,
   294  		MaxIdleConnsPerHost:    t.MaxIdleConnsPerHost,
   295  		MaxConnsPerHost:        t.MaxConnsPerHost,
   296  		IdleConnTimeout:        t.IdleConnTimeout,
   297  		ResponseHeaderTimeout:  t.ResponseHeaderTimeout,
   298  		ExpectContinueTimeout:  t.ExpectContinueTimeout,
   299  		ProxyConnectHeader:     t.ProxyConnectHeader.Clone(),
   300  		MaxResponseHeaderBytes: t.MaxResponseHeaderBytes,
   301  		ForceAttemptHTTP2:      t.ForceAttemptHTTP2,
   302  		WriteBufferSize:        t.WriteBufferSize,
   303  		ReadBufferSize:         t.ReadBufferSize,
   304  	}
   305  	if !t.tlsNextProtoWasNil {
   306  		npm := map[string]func(authority string, c *tls.Conn) RoundTripper{}
   307  		for k, v := range t.TLSNextProto {
   308  			npm[k] = v
   309  		}
   310  		t2.TLSNextProto = npm
   311  	}
   312  	return t2
   313  }
   314  
   315  // h2Transport is the interface we expect to be able to call from
   316  // net/http against an *http2.Transport that's either bundled into
   317  // h2_bundle.go or supplied by the user via x/net/http2.
   318  //
   319  // We name it with the "h2" prefix to stay out of the "http2" prefix
   320  // namespace used by x/tools/cmd/bundle for h2_bundle.go.
   321  type h2Transport interface {
   322  	CloseIdleConnections()
   323  }
   324  
   325  // onceSetNextProtoDefaults initializes TLSNextProto.
   326  // It must be called via t.nextProtoOnce.Do.
   327  func (t *Transport) onceSetNextProtoDefaults() {
   328  	t.tlsNextProtoWasNil = (t.TLSNextProto == nil)
   329  	if strings.Contains(os.Getenv("GODEBUG"), "http2client=0") {
   330  		return
   331  	}
   332  
   333  	// If they've already configured http2 with
   334  	// golang.org/x/net/http2 instead of the bundled copy, try to
   335  	// get at its http2.Transport value (via the "https"
   336  	// altproto map) so we can call CloseIdleConnections on it if
   337  	// requested. (Issue 22891)
   338  	altProto, _ := t.altProto.Load().(map[string]RoundTripper)
   339  	if rv := reflect.ValueOf(altProto["https"]); rv.IsValid() && rv.Type().Kind() == reflect.Struct && rv.Type().NumField() == 1 {
   340  		if v := rv.Field(0); v.CanInterface() {
   341  			if h2i, ok := v.Interface().(h2Transport); ok {
   342  				t.h2transport = h2i
   343  				return
   344  			}
   345  		}
   346  	}
   347  
   348  	if t.TLSNextProto != nil {
   349  		// This is the documented way to disable http2 on a
   350  		// Transport.
   351  		return
   352  	}
   353  	if !t.ForceAttemptHTTP2 && (t.TLSClientConfig != nil || t.Dial != nil || t.DialTLS != nil || t.DialContext != nil) {
   354  		// Be conservative and don't automatically enable
   355  		// http2 if they've specified a custom TLS config or
   356  		// custom dialers. Let them opt-in themselves via
   357  		// http2.ConfigureTransport so we don't surprise them
   358  		// by modifying their tls.Config. Issue 14275.
   359  		// However, if ForceAttemptHTTP2 is true, it overrides the above checks.
   360  		return
   361  	}
   362  	t2, err := http2configureTransport(t)
   363  	if err != nil {
   364  		log.Printf("Error enabling Transport HTTP/2 support: %v", err)
   365  		return
   366  	}
   367  	t.h2transport = t2
   368  
   369  	// Auto-configure the http2.Transport's MaxHeaderListSize from
   370  	// the http.Transport's MaxResponseHeaderBytes. They don't
   371  	// exactly mean the same thing, but they're close.
   372  	//
   373  	// TODO: also add this to x/net/http2.Configure Transport, behind
   374  	// a +build go1.7 build tag:
   375  	if limit1 := t.MaxResponseHeaderBytes; limit1 != 0 && t2.MaxHeaderListSize == 0 {
   376  		const h2max = 1<<32 - 1
   377  		if limit1 >= h2max {
   378  			t2.MaxHeaderListSize = h2max
   379  		} else {
   380  			t2.MaxHeaderListSize = uint32(limit1)
   381  		}
   382  	}
   383  }
   384  
   385  // ProxyFromEnvironment returns the URL of the proxy to use for a
   386  // given request, as indicated by the environment variables
   387  // HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions
   388  // thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https
   389  // requests.
   390  //
   391  // The environment values may be either a complete URL or a
   392  // "host[:port]", in which case the "http" scheme is assumed.
   393  // An error is returned if the value is a different form.
   394  //
   395  // A nil URL and nil error are returned if no proxy is defined in the
   396  // environment, or a proxy should not be used for the given request,
   397  // as defined by NO_PROXY.
   398  //
   399  // As a special case, if req.URL.Host is "localhost" (with or without
   400  // a port number), then a nil URL and nil error will be returned.
   401  func ProxyFromEnvironment(req *Request) (*url.URL, error) {
   402  	return envProxyFunc()(req.URL)
   403  }
   404  
   405  // ProxyURL returns a proxy function (for use in a Transport)
   406  // that always returns the same URL.
   407  func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) {
   408  	return func(*Request) (*url.URL, error) {
   409  		return fixedURL, nil
   410  	}
   411  }
   412  
   413  // transportRequest is a wrapper around a *Request that adds
   414  // optional extra headers to write and stores any error to return
   415  // from roundTrip.
   416  type transportRequest struct {
   417  	*Request                        // original request, not to be mutated
   418  	extra    Header                 // extra headers to write, or nil
   419  	trace    *httptrace.ClientTrace // optional
   420  
   421  	mu  sync.Mutex // guards err
   422  	err error      // first setError value for mapRoundTripError to consider
   423  }
   424  
   425  func (tr *transportRequest) extraHeaders() Header {
   426  	if tr.extra == nil {
   427  		tr.extra = make(Header)
   428  	}
   429  	return tr.extra
   430  }
   431  
   432  func (tr *transportRequest) setError(err error) {
   433  	tr.mu.Lock()
   434  	if tr.err == nil {
   435  		tr.err = err
   436  	}
   437  	tr.mu.Unlock()
   438  }
   439  
   440  // useRegisteredProtocol reports whether an alternate protocol (as reqistered
   441  // with Transport.RegisterProtocol) should be respected for this request.
   442  func (t *Transport) useRegisteredProtocol(req *Request) bool {
   443  	if req.URL.Scheme == "https" && req.requiresHTTP1() {
   444  		// If this request requires HTTP/1, don't use the
   445  		// "https" alternate protocol, which is used by the
   446  		// HTTP/2 code to take over requests if there's an
   447  		// existing cached HTTP/2 connection.
   448  		return false
   449  	}
   450  	return true
   451  }
   452  
   453  // roundTrip implements a RoundTripper over HTTP.
   454  func (t *Transport) roundTrip(req *Request) (*Response, error) {
   455  	t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
   456  	ctx := req.Context()
   457  	trace := httptrace.ContextClientTrace(ctx)
   458  
   459  	if req.URL == nil {
   460  		req.closeBody()
   461  		return nil, errors.New("http: nil Request.URL")
   462  	}
   463  	if req.Header == nil {
   464  		req.closeBody()
   465  		return nil, errors.New("http: nil Request.Header")
   466  	}
   467  	scheme := req.URL.Scheme
   468  	isHTTP := scheme == "http" || scheme == "https"
   469  	if isHTTP {
   470  		for k, vv := range req.Header {
   471  			if !httpguts.ValidHeaderFieldName(k) {
   472  				return nil, fmt.Errorf("net/http: invalid header field name %q", k)
   473  			}
   474  			for _, v := range vv {
   475  				if !httpguts.ValidHeaderFieldValue(v) {
   476  					return nil, fmt.Errorf("net/http: invalid header field value %q for key %v", v, k)
   477  				}
   478  			}
   479  		}
   480  	}
   481  
   482  	if t.useRegisteredProtocol(req) {
   483  		altProto, _ := t.altProto.Load().(map[string]RoundTripper)
   484  		if altRT := altProto[scheme]; altRT != nil {
   485  			if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol {
   486  				return resp, err
   487  			}
   488  		}
   489  	}
   490  	if !isHTTP {
   491  		req.closeBody()
   492  		return nil, &badStringError{"unsupported protocol scheme", scheme}
   493  	}
   494  	if req.Method != "" && !validMethod(req.Method) {
   495  		return nil, fmt.Errorf("net/http: invalid method %q", req.Method)
   496  	}
   497  	if req.URL.Host == "" {
   498  		req.closeBody()
   499  		return nil, errors.New("http: no Host in request URL")
   500  	}
   501  
   502  	for {
   503  		select {
   504  		case <-ctx.Done():
   505  			req.closeBody()
   506  			return nil, ctx.Err()
   507  		default:
   508  		}
   509  
   510  		// treq gets modified by roundTrip, so we need to recreate for each retry.
   511  		treq := &transportRequest{Request: req, trace: trace}
   512  		cm, err := t.connectMethodForRequest(treq)
   513  		if err != nil {
   514  			req.closeBody()
   515  			return nil, err
   516  		}
   517  
   518  		// Get the cached or newly-created connection to either the
   519  		// host (for http or https), the http proxy, or the http proxy
   520  		// pre-CONNECTed to https server. In any case, we'll be ready
   521  		// to send it requests.
   522  		pconn, err := t.getConn(treq, cm)
   523  		if err != nil {
   524  			t.setReqCanceler(req, nil)
   525  			req.closeBody()
   526  			return nil, err
   527  		}
   528  
   529  		var resp *Response
   530  		if pconn.alt != nil {
   531  			// HTTP/2 path.
   532  			t.setReqCanceler(req, nil) // not cancelable with CancelRequest
   533  			resp, err = pconn.alt.RoundTrip(req)
   534  		} else {
   535  			resp, err = pconn.roundTrip(treq)
   536  		}
   537  		if err == nil {
   538  			return resp, nil
   539  		}
   540  
   541  		// Failed. Clean up and determine whether to retry.
   542  
   543  		_, isH2DialError := pconn.alt.(http2erringRoundTripper)
   544  		if http2isNoCachedConnError(err) || isH2DialError {
   545  			if t.removeIdleConn(pconn) {
   546  				t.decConnsPerHost(pconn.cacheKey)
   547  			}
   548  		}
   549  		if !pconn.shouldRetryRequest(req, err) {
   550  			// Issue 16465: return underlying net.Conn.Read error from peek,
   551  			// as we've historically done.
   552  			if e, ok := err.(transportReadFromServerError); ok {
   553  				err = e.err
   554  			}
   555  			return nil, err
   556  		}
   557  		testHookRoundTripRetried()
   558  
   559  		// Rewind the body if we're able to.
   560  		if req.GetBody != nil {
   561  			newReq := *req
   562  			var err error
   563  			newReq.Body, err = req.GetBody()
   564  			if err != nil {
   565  				return nil, err
   566  			}
   567  			req = &newReq
   568  		}
   569  	}
   570  }
   571  
   572  // shouldRetryRequest reports whether we should retry sending a failed
   573  // HTTP request on a new connection. The non-nil input error is the
   574  // error from roundTrip.
   575  func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool {
   576  	if http2isNoCachedConnError(err) {
   577  		// Issue 16582: if the user started a bunch of
   578  		// requests at once, they can all pick the same conn
   579  		// and violate the server's max concurrent streams.
   580  		// Instead, match the HTTP/1 behavior for now and dial
   581  		// again to get a new TCP connection, rather than failing
   582  		// this request.
   583  		return true
   584  	}
   585  	if err == errMissingHost {
   586  		// User error.
   587  		return false
   588  	}
   589  	if !pc.isReused() {
   590  		// This was a fresh connection. There's no reason the server
   591  		// should've hung up on us.
   592  		//
   593  		// Also, if we retried now, we could loop forever
   594  		// creating new connections and retrying if the server
   595  		// is just hanging up on us because it doesn't like
   596  		// our request (as opposed to sending an error).
   597  		return false
   598  	}
   599  	if _, ok := err.(nothingWrittenError); ok {
   600  		// We never wrote anything, so it's safe to retry, if there's no body or we
   601  		// can "rewind" the body with GetBody.
   602  		return req.outgoingLength() == 0 || req.GetBody != nil
   603  	}
   604  	if !req.isReplayable() {
   605  		// Don't retry non-idempotent requests.
   606  		return false
   607  	}
   608  	if _, ok := err.(transportReadFromServerError); ok {
   609  		// We got some non-EOF net.Conn.Read failure reading
   610  		// the 1st response byte from the server.
   611  		return true
   612  	}
   613  	if err == errServerClosedIdle {
   614  		// The server replied with io.EOF while we were trying to
   615  		// read the response. Probably an unfortunately keep-alive
   616  		// timeout, just as the client was writing a request.
   617  		return true
   618  	}
   619  	return false // conservatively
   620  }
   621  
   622  // ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol.
   623  var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol")
   624  
   625  // RegisterProtocol registers a new protocol with scheme.
   626  // The Transport will pass requests using the given scheme to rt.
   627  // It is rt's responsibility to simulate HTTP request semantics.
   628  //
   629  // RegisterProtocol can be used by other packages to provide
   630  // implementations of protocol schemes like "ftp" or "file".
   631  //
   632  // If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will
   633  // handle the RoundTrip itself for that one request, as if the
   634  // protocol were not registered.
   635  func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) {
   636  	t.altMu.Lock()
   637  	defer t.altMu.Unlock()
   638  	oldMap, _ := t.altProto.Load().(map[string]RoundTripper)
   639  	if _, exists := oldMap[scheme]; exists {
   640  		panic("protocol " + scheme + " already registered")
   641  	}
   642  	newMap := make(map[string]RoundTripper)
   643  	for k, v := range oldMap {
   644  		newMap[k] = v
   645  	}
   646  	newMap[scheme] = rt
   647  	t.altProto.Store(newMap)
   648  }
   649  
   650  // CloseIdleConnections closes any connections which were previously
   651  // connected from previous requests but are now sitting idle in
   652  // a "keep-alive" state. It does not interrupt any connections currently
   653  // in use.
   654  func (t *Transport) CloseIdleConnections() {
   655  	t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
   656  	t.idleMu.Lock()
   657  	m := t.idleConn
   658  	t.idleConn = nil
   659  	t.closeIdle = true // close newly idle connections
   660  	t.idleLRU = connLRU{}
   661  	t.idleMu.Unlock()
   662  	for _, conns := range m {
   663  		for _, pconn := range conns {
   664  			pconn.close(errCloseIdleConns)
   665  		}
   666  	}
   667  	if t2 := t.h2transport; t2 != nil {
   668  		t2.CloseIdleConnections()
   669  	}
   670  }
   671  
   672  // CancelRequest cancels an in-flight request by closing its connection.
   673  // CancelRequest should only be called after RoundTrip has returned.
   674  //
   675  // Deprecated: Use Request.WithContext to create a request with a
   676  // cancelable context instead. CancelRequest cannot cancel HTTP/2
   677  // requests.
   678  func (t *Transport) CancelRequest(req *Request) {
   679  	t.cancelRequest(req, errRequestCanceled)
   680  }
   681  
   682  // Cancel an in-flight request, recording the error value.
   683  func (t *Transport) cancelRequest(req *Request, err error) {
   684  	t.reqMu.Lock()
   685  	cancel := t.reqCanceler[req]
   686  	delete(t.reqCanceler, req)
   687  	t.reqMu.Unlock()
   688  	if cancel != nil {
   689  		cancel(err)
   690  	}
   691  }
   692  
   693  //
   694  // Private implementation past this point.
   695  //
   696  
   697  var (
   698  	// proxyConfigOnce guards proxyConfig
   699  	envProxyOnce      sync.Once
   700  	envProxyFuncValue func(*url.URL) (*url.URL, error)
   701  )
   702  
   703  // defaultProxyConfig returns a ProxyConfig value looked up
   704  // from the environment. This mitigates expensive lookups
   705  // on some platforms (e.g. Windows).
   706  func envProxyFunc() func(*url.URL) (*url.URL, error) {
   707  	envProxyOnce.Do(func() {
   708  		envProxyFuncValue = httpproxy.FromEnvironment().ProxyFunc()
   709  	})
   710  	return envProxyFuncValue
   711  }
   712  
   713  // resetProxyConfig is used by tests.
   714  func resetProxyConfig() {
   715  	envProxyOnce = sync.Once{}
   716  	envProxyFuncValue = nil
   717  }
   718  
   719  func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) {
   720  	// TODO: the validPort check is redundant after CL 189258, as url.URL.Port
   721  	// only returns valid ports now. golang.org/issue/33600
   722  	if port := treq.URL.Port(); !validPort(port) {
   723  		return cm, fmt.Errorf("invalid URL port %q", port)
   724  	}
   725  	cm.targetScheme = treq.URL.Scheme
   726  	cm.targetAddr = canonicalAddr(treq.URL)
   727  	if t.Proxy != nil {
   728  		cm.proxyURL, err = t.Proxy(treq.Request)
   729  		if err == nil && cm.proxyURL != nil {
   730  			if port := cm.proxyURL.Port(); !validPort(port) {
   731  				return cm, fmt.Errorf("invalid proxy URL port %q", port)
   732  			}
   733  		}
   734  	}
   735  	cm.onlyH1 = treq.requiresHTTP1()
   736  	return cm, err
   737  }
   738  
   739  // proxyAuth returns the Proxy-Authorization header to set
   740  // on requests, if applicable.
   741  func (cm *connectMethod) proxyAuth() string {
   742  	if cm.proxyURL == nil {
   743  		return ""
   744  	}
   745  	if u := cm.proxyURL.User; u != nil {
   746  		username := u.Username()
   747  		password, _ := u.Password()
   748  		return "Basic " + basicAuth(username, password)
   749  	}
   750  	return ""
   751  }
   752  
   753  // error values for debugging and testing, not seen by users.
   754  var (
   755  	errKeepAlivesDisabled = errors.New("http: putIdleConn: keep alives disabled")
   756  	errConnBroken         = errors.New("http: putIdleConn: connection is in bad state")
   757  	errCloseIdle          = errors.New("http: putIdleConn: CloseIdleConnections was called")
   758  	errTooManyIdle        = errors.New("http: putIdleConn: too many idle connections")
   759  	errTooManyIdleHost    = errors.New("http: putIdleConn: too many idle connections for host")
   760  	errCloseIdleConns     = errors.New("http: CloseIdleConnections called")
   761  	errReadLoopExiting    = errors.New("http: persistConn.readLoop exiting")
   762  	errIdleConnTimeout    = errors.New("http: idle connection timeout")
   763  	errNotCachingH2Conn   = errors.New("http: not caching alternate protocol's connections")
   764  
   765  	// errServerClosedIdle is not seen by users for idempotent requests, but may be
   766  	// seen by a user if the server shuts down an idle connection and sends its FIN
   767  	// in flight with already-written POST body bytes from the client.
   768  	// See https://github.com/golang/go/issues/19943#issuecomment-355607646
   769  	errServerClosedIdle = errors.New("http: server closed idle connection")
   770  )
   771  
   772  // transportReadFromServerError is used by Transport.readLoop when the
   773  // 1 byte peek read fails and we're actually anticipating a response.
   774  // Usually this is just due to the inherent keep-alive shut down race,
   775  // where the server closed the connection at the same time the client
   776  // wrote. The underlying err field is usually io.EOF or some
   777  // ECONNRESET sort of thing which varies by platform. But it might be
   778  // the user's custom net.Conn.Read error too, so we carry it along for
   779  // them to return from Transport.RoundTrip.
   780  type transportReadFromServerError struct {
   781  	err error
   782  }
   783  
   784  func (e transportReadFromServerError) Unwrap() error { return e.err }
   785  
   786  func (e transportReadFromServerError) Error() string {
   787  	return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err)
   788  }
   789  
   790  func (t *Transport) putOrCloseIdleConn(pconn *persistConn) {
   791  	if err := t.tryPutIdleConn(pconn); err != nil {
   792  		pconn.close(err)
   793  	}
   794  }
   795  
   796  func (t *Transport) maxIdleConnsPerHost() int {
   797  	if v := t.MaxIdleConnsPerHost; v != 0 {
   798  		return v
   799  	}
   800  	return DefaultMaxIdleConnsPerHost
   801  }
   802  
   803  // tryPutIdleConn adds pconn to the list of idle persistent connections awaiting
   804  // a new request.
   805  // If pconn is no longer needed or not in a good state, tryPutIdleConn returns
   806  // an error explaining why it wasn't registered.
   807  // tryPutIdleConn does not close pconn. Use putOrCloseIdleConn instead for that.
   808  func (t *Transport) tryPutIdleConn(pconn *persistConn) error {
   809  	if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 {
   810  		return errKeepAlivesDisabled
   811  	}
   812  	if pconn.isBroken() {
   813  		return errConnBroken
   814  	}
   815  	pconn.markReused()
   816  
   817  	t.idleMu.Lock()
   818  	defer t.idleMu.Unlock()
   819  
   820  	// HTTP/2 (pconn.alt != nil) connections do not come out of the idle list,
   821  	// because multiple goroutines can use them simultaneously.
   822  	// If this is an HTTP/2 connection being “returned,” we're done.
   823  	if pconn.alt != nil && t.idleLRU.m[pconn] != nil {
   824  		return nil
   825  	}
   826  
   827  	// Deliver pconn to goroutine waiting for idle connection, if any.
   828  	// (They may be actively dialing, but this conn is ready first.
   829  	// Chrome calls this socket late binding.
   830  	// See https://insouciant.org/tech/connection-management-in-chromium/.)
   831  	key := pconn.cacheKey
   832  	if q, ok := t.idleConnWait[key]; ok {
   833  		done := false
   834  		if pconn.alt == nil {
   835  			// HTTP/1.
   836  			// Loop over the waiting list until we find a w that isn't done already, and hand it pconn.
   837  			for q.len() > 0 {
   838  				w := q.popFront()
   839  				if w.tryDeliver(pconn, nil) {
   840  					done = true
   841  					break
   842  				}
   843  			}
   844  		} else {
   845  			// HTTP/2.
   846  			// Can hand the same pconn to everyone in the waiting list,
   847  			// and we still won't be done: we want to put it in the idle
   848  			// list unconditionally, for any future clients too.
   849  			for q.len() > 0 {
   850  				w := q.popFront()
   851  				w.tryDeliver(pconn, nil)
   852  			}
   853  		}
   854  		if q.len() == 0 {
   855  			delete(t.idleConnWait, key)
   856  		} else {
   857  			t.idleConnWait[key] = q
   858  		}
   859  		if done {
   860  			return nil
   861  		}
   862  	}
   863  
   864  	if t.closeIdle {
   865  		return errCloseIdle
   866  	}
   867  	if t.idleConn == nil {
   868  		t.idleConn = make(map[connectMethodKey][]*persistConn)
   869  	}
   870  	idles := t.idleConn[key]
   871  	if len(idles) >= t.maxIdleConnsPerHost() {
   872  		return errTooManyIdleHost
   873  	}
   874  	for _, exist := range idles {
   875  		if exist == pconn {
   876  			log.Fatalf("dup idle pconn %p in freelist", pconn)
   877  		}
   878  	}
   879  	t.idleConn[key] = append(idles, pconn)
   880  	t.idleLRU.add(pconn)
   881  	if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns {
   882  		oldest := t.idleLRU.removeOldest()
   883  		oldest.close(errTooManyIdle)
   884  		t.removeIdleConnLocked(oldest)
   885  	}
   886  
   887  	// Set idle timer, but only for HTTP/1 (pconn.alt == nil).
   888  	// The HTTP/2 implementation manages the idle timer itself
   889  	// (see idleConnTimeout in h2_bundle.go).
   890  	if t.IdleConnTimeout > 0 && pconn.alt == nil {
   891  		if pconn.idleTimer != nil {
   892  			pconn.idleTimer.Reset(t.IdleConnTimeout)
   893  		} else {
   894  			pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle)
   895  		}
   896  	}
   897  	pconn.idleAt = time.Now()
   898  	return nil
   899  }
   900  
   901  // queueForIdleConn queues w to receive the next idle connection for w.cm.
   902  // As an optimization hint to the caller, queueForIdleConn reports whether
   903  // it successfully delivered an already-idle connection.
   904  func (t *Transport) queueForIdleConn(w *wantConn) (delivered bool) {
   905  	if t.DisableKeepAlives {
   906  		return false
   907  	}
   908  
   909  	t.idleMu.Lock()
   910  	defer t.idleMu.Unlock()
   911  
   912  	// Stop closing connections that become idle - we might want one.
   913  	// (That is, undo the effect of t.CloseIdleConnections.)
   914  	t.closeIdle = false
   915  
   916  	if w == nil {
   917  		// Happens in test hook.
   918  		return false
   919  	}
   920  
   921  	// Look for most recently-used idle connection.
   922  	if list, ok := t.idleConn[w.key]; ok {
   923  		stop := false
   924  		delivered := false
   925  		for len(list) > 0 && !stop {
   926  			pconn := list[len(list)-1]
   927  			if pconn.isBroken() {
   928  				// persistConn.readLoop has marked the connection broken,
   929  				// but Transport.removeIdleConn has not yet removed it from the idle list.
   930  				// Drop on floor on behalf of Transport.removeIdleConn.
   931  				list = list[:len(list)-1]
   932  				continue
   933  			}
   934  			delivered = w.tryDeliver(pconn, nil)
   935  			if delivered {
   936  				if pconn.alt != nil {
   937  					// HTTP/2: multiple clients can share pconn.
   938  					// Leave it in the list.
   939  				} else {
   940  					// HTTP/1: only one client can use pconn.
   941  					// Remove it from the list.
   942  					t.idleLRU.remove(pconn)
   943  					list = list[:len(list)-1]
   944  				}
   945  			}
   946  			stop = true
   947  		}
   948  		if len(list) > 0 {
   949  			t.idleConn[w.key] = list
   950  		} else {
   951  			delete(t.idleConn, w.key)
   952  		}
   953  		if stop {
   954  			return delivered
   955  		}
   956  	}
   957  
   958  	// Register to receive next connection that becomes idle.
   959  	if t.idleConnWait == nil {
   960  		t.idleConnWait = make(map[connectMethodKey]wantConnQueue)
   961  	}
   962  	q := t.idleConnWait[w.key]
   963  	q.cleanFront()
   964  	q.pushBack(w)
   965  	t.idleConnWait[w.key] = q
   966  	return false
   967  }
   968  
   969  // removeIdleConn marks pconn as dead.
   970  func (t *Transport) removeIdleConn(pconn *persistConn) bool {
   971  	t.idleMu.Lock()
   972  	defer t.idleMu.Unlock()
   973  	return t.removeIdleConnLocked(pconn)
   974  }
   975  
   976  // t.idleMu must be held.
   977  func (t *Transport) removeIdleConnLocked(pconn *persistConn) bool {
   978  	if pconn.idleTimer != nil {
   979  		pconn.idleTimer.Stop()
   980  	}
   981  	t.idleLRU.remove(pconn)
   982  	key := pconn.cacheKey
   983  	pconns := t.idleConn[key]
   984  	var removed bool
   985  	switch len(pconns) {
   986  	case 0:
   987  		// Nothing
   988  	case 1:
   989  		if pconns[0] == pconn {
   990  			delete(t.idleConn, key)
   991  			removed = true
   992  		}
   993  	default:
   994  		for i, v := range pconns {
   995  			if v != pconn {
   996  				continue
   997  			}
   998  			// Slide down, keeping most recently-used
   999  			// conns at the end.
  1000  			copy(pconns[i:], pconns[i+1:])
  1001  			t.idleConn[key] = pconns[:len(pconns)-1]
  1002  			removed = true
  1003  			break
  1004  		}
  1005  	}
  1006  	return removed
  1007  }
  1008  
  1009  func (t *Transport) setReqCanceler(r *Request, fn func(error)) {
  1010  	t.reqMu.Lock()
  1011  	defer t.reqMu.Unlock()
  1012  	if t.reqCanceler == nil {
  1013  		t.reqCanceler = make(map[*Request]func(error))
  1014  	}
  1015  	if fn != nil {
  1016  		t.reqCanceler[r] = fn
  1017  	} else {
  1018  		delete(t.reqCanceler, r)
  1019  	}
  1020  }
  1021  
  1022  // replaceReqCanceler replaces an existing cancel function. If there is no cancel function
  1023  // for the request, we don't set the function and return false.
  1024  // Since CancelRequest will clear the canceler, we can use the return value to detect if
  1025  // the request was canceled since the last setReqCancel call.
  1026  func (t *Transport) replaceReqCanceler(r *Request, fn func(error)) bool {
  1027  	t.reqMu.Lock()
  1028  	defer t.reqMu.Unlock()
  1029  	_, ok := t.reqCanceler[r]
  1030  	if !ok {
  1031  		return false
  1032  	}
  1033  	if fn != nil {
  1034  		t.reqCanceler[r] = fn
  1035  	} else {
  1036  		delete(t.reqCanceler, r)
  1037  	}
  1038  	return true
  1039  }
  1040  
  1041  var zeroDialer net.Dialer
  1042  
  1043  func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) {
  1044  	if t.DialContext != nil {
  1045  		return t.DialContext(ctx, network, addr)
  1046  	}
  1047  	if t.Dial != nil {
  1048  		c, err := t.Dial(network, addr)
  1049  		if c == nil && err == nil {
  1050  			err = errors.New("net/http: Transport.Dial hook returned (nil, nil)")
  1051  		}
  1052  		return c, err
  1053  	}
  1054  	return zeroDialer.DialContext(ctx, network, addr)
  1055  }
  1056  
  1057  // A wantConn records state about a wanted connection
  1058  // (that is, an active call to getConn).
  1059  // The conn may be gotten by dialing or by finding an idle connection,
  1060  // or a cancellation may make the conn no longer wanted.
  1061  // These three options are racing against each other and use
  1062  // wantConn to coordinate and agree about the winning outcome.
  1063  type wantConn struct {
  1064  	cm    connectMethod
  1065  	key   connectMethodKey // cm.key()
  1066  	ctx   context.Context  // context for dial
  1067  	ready chan struct{}    // closed when pc, err pair is delivered
  1068  
  1069  	// hooks for testing to know when dials are done
  1070  	// beforeDial is called in the getConn goroutine when the dial is queued.
  1071  	// afterDial is called when the dial is completed or cancelled.
  1072  	beforeDial func()
  1073  	afterDial  func()
  1074  
  1075  	mu  sync.Mutex // protects pc, err, close(ready)
  1076  	pc  *persistConn
  1077  	err error
  1078  }
  1079  
  1080  // waiting reports whether w is still waiting for an answer (connection or error).
  1081  func (w *wantConn) waiting() bool {
  1082  	select {
  1083  	case <-w.ready:
  1084  		return false
  1085  	default:
  1086  		return true
  1087  	}
  1088  }
  1089  
  1090  // tryDeliver attempts to deliver pc, err to w and reports whether it succeeded.
  1091  func (w *wantConn) tryDeliver(pc *persistConn, err error) bool {
  1092  	w.mu.Lock()
  1093  	defer w.mu.Unlock()
  1094  
  1095  	if w.pc != nil || w.err != nil {
  1096  		return false
  1097  	}
  1098  
  1099  	w.pc = pc
  1100  	w.err = err
  1101  	if w.pc == nil && w.err == nil {
  1102  		panic("net/http: internal error: misuse of tryDeliver")
  1103  	}
  1104  	close(w.ready)
  1105  	return true
  1106  }
  1107  
  1108  // cancel marks w as no longer wanting a result (for example, due to cancellation).
  1109  // If a connection has been delivered already, cancel returns it with t.putOrCloseIdleConn.
  1110  func (w *wantConn) cancel(t *Transport, err error) {
  1111  	w.mu.Lock()
  1112  	if w.pc == nil && w.err == nil {
  1113  		close(w.ready) // catch misbehavior in future delivery
  1114  	}
  1115  	pc := w.pc
  1116  	w.pc = nil
  1117  	w.err = err
  1118  	w.mu.Unlock()
  1119  
  1120  	if pc != nil {
  1121  		t.putOrCloseIdleConn(pc)
  1122  	}
  1123  }
  1124  
  1125  // A wantConnQueue is a queue of wantConns.
  1126  type wantConnQueue struct {
  1127  	// This is a queue, not a deque.
  1128  	// It is split into two stages - head[headPos:] and tail.
  1129  	// popFront is trivial (headPos++) on the first stage, and
  1130  	// pushBack is trivial (append) on the second stage.
  1131  	// If the first stage is empty, popFront can swap the
  1132  	// first and second stages to remedy the situation.
  1133  	//
  1134  	// This two-stage split is analogous to the use of two lists
  1135  	// in Okasaki's purely functional queue but without the
  1136  	// overhead of reversing the list when swapping stages.
  1137  	head    []*wantConn
  1138  	headPos int
  1139  	tail    []*wantConn
  1140  }
  1141  
  1142  // len returns the number of items in the queue.
  1143  func (q *wantConnQueue) len() int {
  1144  	return len(q.head) - q.headPos + len(q.tail)
  1145  }
  1146  
  1147  // pushBack adds w to the back of the queue.
  1148  func (q *wantConnQueue) pushBack(w *wantConn) {
  1149  	q.tail = append(q.tail, w)
  1150  }
  1151  
  1152  // popFront removes and returns the wantConn at the front of the queue.
  1153  func (q *wantConnQueue) popFront() *wantConn {
  1154  	if q.headPos >= len(q.head) {
  1155  		if len(q.tail) == 0 {
  1156  			return nil
  1157  		}
  1158  		// Pick up tail as new head, clear tail.
  1159  		q.head, q.headPos, q.tail = q.tail, 0, q.head[:0]
  1160  	}
  1161  	w := q.head[q.headPos]
  1162  	q.head[q.headPos] = nil
  1163  	q.headPos++
  1164  	return w
  1165  }
  1166  
  1167  // peekFront returns the wantConn at the front of the queue without removing it.
  1168  func (q *wantConnQueue) peekFront() *wantConn {
  1169  	if q.headPos < len(q.head) {
  1170  		return q.head[q.headPos]
  1171  	}
  1172  	if len(q.tail) > 0 {
  1173  		return q.tail[0]
  1174  	}
  1175  	return nil
  1176  }
  1177  
  1178  // cleanFront pops any wantConns that are no longer waiting from the head of the
  1179  // queue, reporting whether any were popped.
  1180  func (q *wantConnQueue) cleanFront() (cleaned bool) {
  1181  	for {
  1182  		w := q.peekFront()
  1183  		if w == nil || w.waiting() {
  1184  			return cleaned
  1185  		}
  1186  		q.popFront()
  1187  		cleaned = true
  1188  	}
  1189  }
  1190  
  1191  // getConn dials and creates a new persistConn to the target as
  1192  // specified in the connectMethod. This includes doing a proxy CONNECT
  1193  // and/or setting up TLS.  If this doesn't return an error, the persistConn
  1194  // is ready to write requests to.
  1195  func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (pc *persistConn, err error) {
  1196  	req := treq.Request
  1197  	trace := treq.trace
  1198  	ctx := req.Context()
  1199  	if trace != nil && trace.GetConn != nil {
  1200  		trace.GetConn(cm.addr())
  1201  	}
  1202  
  1203  	w := &wantConn{
  1204  		cm:         cm,
  1205  		key:        cm.key(),
  1206  		ctx:        ctx,
  1207  		ready:      make(chan struct{}, 1),
  1208  		beforeDial: testHookPrePendingDial,
  1209  		afterDial:  testHookPostPendingDial,
  1210  	}
  1211  	defer func() {
  1212  		if err != nil {
  1213  			w.cancel(t, err)
  1214  		}
  1215  	}()
  1216  
  1217  	// Queue for idle connection.
  1218  	if delivered := t.queueForIdleConn(w); delivered {
  1219  		pc := w.pc
  1220  		// Trace only for HTTP/1.
  1221  		// HTTP/2 calls trace.GotConn itself.
  1222  		if pc.alt == nil && trace != nil && trace.GotConn != nil {
  1223  			trace.GotConn(pc.gotIdleConnTrace(pc.idleAt))
  1224  		}
  1225  		// set request canceler to some non-nil function so we
  1226  		// can detect whether it was cleared between now and when
  1227  		// we enter roundTrip
  1228  		t.setReqCanceler(req, func(error) {})
  1229  		return pc, nil
  1230  	}
  1231  
  1232  	cancelc := make(chan error, 1)
  1233  	t.setReqCanceler(req, func(err error) { cancelc <- err })
  1234  
  1235  	// Queue for permission to dial.
  1236  	t.queueForDial(w)
  1237  
  1238  	// Wait for completion or cancellation.
  1239  	select {
  1240  	case <-w.ready:
  1241  		// Trace success but only for HTTP/1.
  1242  		// HTTP/2 calls trace.GotConn itself.
  1243  		if w.pc != nil && w.pc.alt == nil && trace != nil && trace.GotConn != nil {
  1244  			trace.GotConn(httptrace.GotConnInfo{Conn: w.pc.conn, Reused: w.pc.isReused()})
  1245  		}
  1246  		if w.err != nil {
  1247  			// If the request has been cancelled, that's probably
  1248  			// what caused w.err; if so, prefer to return the
  1249  			// cancellation error (see golang.org/issue/16049).
  1250  			select {
  1251  			case <-req.Cancel:
  1252  				return nil, errRequestCanceledConn
  1253  			case <-req.Context().Done():
  1254  				return nil, req.Context().Err()
  1255  			case err := <-cancelc:
  1256  				if err == errRequestCanceled {
  1257  					err = errRequestCanceledConn
  1258  				}
  1259  				return nil, err
  1260  			default:
  1261  				// return below
  1262  			}
  1263  		}
  1264  		return w.pc, w.err
  1265  	case <-req.Cancel:
  1266  		return nil, errRequestCanceledConn
  1267  	case <-req.Context().Done():
  1268  		return nil, req.Context().Err()
  1269  	case err := <-cancelc:
  1270  		if err == errRequestCanceled {
  1271  			err = errRequestCanceledConn
  1272  		}
  1273  		return nil, err
  1274  	}
  1275  }
  1276  
  1277  // queueForDial queues w to wait for permission to begin dialing.
  1278  // Once w receives permission to dial, it will do so in a separate goroutine.
  1279  func (t *Transport) queueForDial(w *wantConn) {
  1280  	w.beforeDial()
  1281  	if t.MaxConnsPerHost <= 0 {
  1282  		go t.dialConnFor(w)
  1283  		return
  1284  	}
  1285  
  1286  	t.connsPerHostMu.Lock()
  1287  	defer t.connsPerHostMu.Unlock()
  1288  
  1289  	if n := t.connsPerHost[w.key]; n < t.MaxConnsPerHost {
  1290  		if t.connsPerHost == nil {
  1291  			t.connsPerHost = make(map[connectMethodKey]int)
  1292  		}
  1293  		t.connsPerHost[w.key] = n + 1
  1294  		go t.dialConnFor(w)
  1295  		return
  1296  	}
  1297  
  1298  	if t.connsPerHostWait == nil {
  1299  		t.connsPerHostWait = make(map[connectMethodKey]wantConnQueue)
  1300  	}
  1301  	q := t.connsPerHostWait[w.key]
  1302  	q.cleanFront()
  1303  	q.pushBack(w)
  1304  	t.connsPerHostWait[w.key] = q
  1305  }
  1306  
  1307  // dialConnFor dials on behalf of w and delivers the result to w.
  1308  // dialConnFor has received permission to dial w.cm and is counted in t.connCount[w.cm.key()].
  1309  // If the dial is cancelled or unsuccessful, dialConnFor decrements t.connCount[w.cm.key()].
  1310  func (t *Transport) dialConnFor(w *wantConn) {
  1311  	defer w.afterDial()
  1312  
  1313  	pc, err := t.dialConn(w.ctx, w.cm)
  1314  	delivered := w.tryDeliver(pc, err)
  1315  	if err == nil && (!delivered || pc.alt != nil) {
  1316  		// pconn was not passed to w,
  1317  		// or it is HTTP/2 and can be shared.
  1318  		// Add to the idle connection pool.
  1319  		t.putOrCloseIdleConn(pc)
  1320  	}
  1321  	if err != nil {
  1322  		t.decConnsPerHost(w.key)
  1323  	}
  1324  }
  1325  
  1326  // decConnsPerHost decrements the per-host connection count for key,
  1327  // which may in turn give a different waiting goroutine permission to dial.
  1328  func (t *Transport) decConnsPerHost(key connectMethodKey) {
  1329  	if t.MaxConnsPerHost <= 0 {
  1330  		return
  1331  	}
  1332  
  1333  	t.connsPerHostMu.Lock()
  1334  	defer t.connsPerHostMu.Unlock()
  1335  	n := t.connsPerHost[key]
  1336  	if n == 0 {
  1337  		// Shouldn't happen, but if it does, the counting is buggy and could
  1338  		// easily lead to a silent deadlock, so report the problem loudly.
  1339  		panic("net/http: internal error: connCount underflow")
  1340  	}
  1341  
  1342  	// Can we hand this count to a goroutine still waiting to dial?
  1343  	// (Some goroutines on the wait list may have timed out or
  1344  	// gotten a connection another way. If they're all gone,
  1345  	// we don't want to kick off any spurious dial operations.)
  1346  	if q := t.connsPerHostWait[key]; q.len() > 0 {
  1347  		done := false
  1348  		for q.len() > 0 {
  1349  			w := q.popFront()
  1350  			if w.waiting() {
  1351  				go t.dialConnFor(w)
  1352  				done = true
  1353  				break
  1354  			}
  1355  		}
  1356  		if q.len() == 0 {
  1357  			delete(t.connsPerHostWait, key)
  1358  		} else {
  1359  			// q is a value (like a slice), so we have to store
  1360  			// the updated q back into the map.
  1361  			t.connsPerHostWait[key] = q
  1362  		}
  1363  		if done {
  1364  			return
  1365  		}
  1366  	}
  1367  
  1368  	// Otherwise, decrement the recorded count.
  1369  	if n--; n == 0 {
  1370  		delete(t.connsPerHost, key)
  1371  	} else {
  1372  		t.connsPerHost[key] = n
  1373  	}
  1374  }
  1375  
  1376  // The connect method and the transport can both specify a TLS
  1377  // Host name.  The transport's name takes precedence if present.
  1378  func chooseTLSHost(cm connectMethod, t *Transport) string {
  1379  	tlsHost := ""
  1380  	if t.TLSClientConfig != nil {
  1381  		tlsHost = t.TLSClientConfig.ServerName
  1382  	}
  1383  	if tlsHost == "" {
  1384  		tlsHost = cm.tlsHost()
  1385  	}
  1386  	return tlsHost
  1387  }
  1388  
  1389  // Add TLS to a persistent connection, i.e. negotiate a TLS session. If pconn is already a TLS
  1390  // tunnel, this function establishes a nested TLS session inside the encrypted channel.
  1391  // The remote endpoint's name may be overridden by TLSClientConfig.ServerName.
  1392  func (pconn *persistConn) addTLS(name string, trace *httptrace.ClientTrace) error {
  1393  	// Initiate TLS and check remote host name against certificate.
  1394  	cfg := cloneTLSConfig(pconn.t.TLSClientConfig)
  1395  	if cfg.ServerName == "" {
  1396  		cfg.ServerName = name
  1397  	}
  1398  	if pconn.cacheKey.onlyH1 {
  1399  		cfg.NextProtos = nil
  1400  	}
  1401  	plainConn := pconn.conn
  1402  	tlsConn := tls.Client(plainConn, cfg)
  1403  	errc := make(chan error, 2)
  1404  	var timer *time.Timer // for canceling TLS handshake
  1405  	if d := pconn.t.TLSHandshakeTimeout; d != 0 {
  1406  		timer = time.AfterFunc(d, func() {
  1407  			errc <- tlsHandshakeTimeoutError{}
  1408  		})
  1409  	}
  1410  	go func() {
  1411  		if trace != nil && trace.TLSHandshakeStart != nil {
  1412  			trace.TLSHandshakeStart()
  1413  		}
  1414  		err := tlsConn.Handshake()
  1415  		if timer != nil {
  1416  			timer.Stop()
  1417  		}
  1418  		errc <- err
  1419  	}()
  1420  	if err := <-errc; err != nil {
  1421  		plainConn.Close()
  1422  		if trace != nil && trace.TLSHandshakeDone != nil {
  1423  			trace.TLSHandshakeDone(tls.ConnectionState{}, err)
  1424  		}
  1425  		return err
  1426  	}
  1427  	cs := tlsConn.ConnectionState()
  1428  	if trace != nil && trace.TLSHandshakeDone != nil {
  1429  		trace.TLSHandshakeDone(cs, nil)
  1430  	}
  1431  	pconn.tlsState = &cs
  1432  	pconn.conn = tlsConn
  1433  	return nil
  1434  }
  1435  
  1436  func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (pconn *persistConn, err error) {
  1437  	pconn = &persistConn{
  1438  		t:             t,
  1439  		cacheKey:      cm.key(),
  1440  		reqch:         make(chan requestAndChan, 1),
  1441  		writech:       make(chan writeRequest, 1),
  1442  		closech:       make(chan struct{}),
  1443  		writeErrCh:    make(chan error, 1),
  1444  		writeLoopDone: make(chan struct{}),
  1445  	}
  1446  	trace := httptrace.ContextClientTrace(ctx)
  1447  	wrapErr := func(err error) error {
  1448  		if cm.proxyURL != nil {
  1449  			// Return a typed error, per Issue 16997
  1450  			return &net.OpError{Op: "proxyconnect", Net: "tcp", Err: err}
  1451  		}
  1452  		return err
  1453  	}
  1454  	if cm.scheme() == "https" && t.DialTLS != nil {
  1455  		var err error
  1456  		pconn.conn, err = t.DialTLS("tcp", cm.addr())
  1457  		if err != nil {
  1458  			return nil, wrapErr(err)
  1459  		}
  1460  		if pconn.conn == nil {
  1461  			return nil, wrapErr(errors.New("net/http: Transport.DialTLS returned (nil, nil)"))
  1462  		}
  1463  		if tc, ok := pconn.conn.(*tls.Conn); ok {
  1464  			// Handshake here, in case DialTLS didn't. TLSNextProto below
  1465  			// depends on it for knowing the connection state.
  1466  			if trace != nil && trace.TLSHandshakeStart != nil {
  1467  				trace.TLSHandshakeStart()
  1468  			}
  1469  			if err := tc.Handshake(); err != nil {
  1470  				go pconn.conn.Close()
  1471  				if trace != nil && trace.TLSHandshakeDone != nil {
  1472  					trace.TLSHandshakeDone(tls.ConnectionState{}, err)
  1473  				}
  1474  				return nil, err
  1475  			}
  1476  			cs := tc.ConnectionState()
  1477  			if trace != nil && trace.TLSHandshakeDone != nil {
  1478  				trace.TLSHandshakeDone(cs, nil)
  1479  			}
  1480  			pconn.tlsState = &cs
  1481  		}
  1482  	} else {
  1483  		conn, err := t.dial(ctx, "tcp", cm.addr())
  1484  		if err != nil {
  1485  			return nil, wrapErr(err)
  1486  		}
  1487  		pconn.conn = conn
  1488  		if cm.scheme() == "https" {
  1489  			var firstTLSHost string
  1490  			if firstTLSHost, _, err = net.SplitHostPort(cm.addr()); err != nil {
  1491  				return nil, wrapErr(err)
  1492  			}
  1493  			if err = pconn.addTLS(firstTLSHost, trace); err != nil {
  1494  				return nil, wrapErr(err)
  1495  			}
  1496  		}
  1497  	}
  1498  
  1499  	// Proxy setup.
  1500  	switch {
  1501  	case cm.proxyURL == nil:
  1502  		// Do nothing. Not using a proxy.
  1503  	case cm.proxyURL.Scheme == "socks5":
  1504  		conn := pconn.conn
  1505  		d := socksNewDialer("tcp", conn.RemoteAddr().String())
  1506  		if u := cm.proxyURL.User; u != nil {
  1507  			auth := &socksUsernamePassword{
  1508  				Username: u.Username(),
  1509  			}
  1510  			auth.Password, _ = u.Password()
  1511  			d.AuthMethods = []socksAuthMethod{
  1512  				socksAuthMethodNotRequired,
  1513  				socksAuthMethodUsernamePassword,
  1514  			}
  1515  			d.Authenticate = auth.Authenticate
  1516  		}
  1517  		if _, err := d.DialWithConn(ctx, conn, "tcp", cm.targetAddr); err != nil {
  1518  			conn.Close()
  1519  			return nil, err
  1520  		}
  1521  	case cm.targetScheme == "http":
  1522  		pconn.isProxy = true
  1523  		if pa := cm.proxyAuth(); pa != "" {
  1524  			pconn.mutateHeaderFunc = func(h Header) {
  1525  				h.Set("Proxy-Authorization", pa)
  1526  			}
  1527  		}
  1528  	case cm.targetScheme == "https":
  1529  		conn := pconn.conn
  1530  		hdr := t.ProxyConnectHeader
  1531  		if hdr == nil {
  1532  			hdr = make(Header)
  1533  		}
  1534  		if pa := cm.proxyAuth(); pa != "" {
  1535  			hdr = hdr.Clone()
  1536  			hdr.Set("Proxy-Authorization", pa)
  1537  		}
  1538  		connectReq := &Request{
  1539  			Method: "CONNECT",
  1540  			URL:    &url.URL{Opaque: cm.targetAddr},
  1541  			Host:   cm.targetAddr,
  1542  			Header: hdr,
  1543  		}
  1544  		connectReq.Write(conn)
  1545  
  1546  		// Read response.
  1547  		// Okay to use and discard buffered reader here, because
  1548  		// TLS server will not speak until spoken to.
  1549  		br := bufio.NewReader(conn)
  1550  		resp, err := ReadResponse(br, connectReq)
  1551  		if err != nil {
  1552  			conn.Close()
  1553  			return nil, err
  1554  		}
  1555  		if resp.StatusCode != 200 {
  1556  			f := strings.SplitN(resp.Status, " ", 2)
  1557  			conn.Close()
  1558  			if len(f) < 2 {
  1559  				return nil, errors.New("unknown status code")
  1560  			}
  1561  			return nil, errors.New(f[1])
  1562  		}
  1563  	}
  1564  
  1565  	if cm.proxyURL != nil && cm.targetScheme == "https" {
  1566  		if err := pconn.addTLS(cm.tlsHost(), trace); err != nil {
  1567  			return nil, err
  1568  		}
  1569  	}
  1570  
  1571  	if s := pconn.tlsState; s != nil && s.NegotiatedProtocolIsMutual && s.NegotiatedProtocol != "" {
  1572  		if next, ok := t.TLSNextProto[s.NegotiatedProtocol]; ok {
  1573  			return &persistConn{t: t, cacheKey: pconn.cacheKey, alt: next(cm.targetAddr, pconn.conn.(*tls.Conn))}, nil
  1574  		}
  1575  	}
  1576  
  1577  	pconn.br = bufio.NewReaderSize(pconn, t.readBufferSize())
  1578  	pconn.bw = bufio.NewWriterSize(persistConnWriter{pconn}, t.writeBufferSize())
  1579  
  1580  	go pconn.readLoop()
  1581  	go pconn.writeLoop()
  1582  	return pconn, nil
  1583  }
  1584  
  1585  // persistConnWriter is the io.Writer written to by pc.bw.
  1586  // It accumulates the number of bytes written to the underlying conn,
  1587  // so the retry logic can determine whether any bytes made it across
  1588  // the wire.
  1589  // This is exactly 1 pointer field wide so it can go into an interface
  1590  // without allocation.
  1591  type persistConnWriter struct {
  1592  	pc *persistConn
  1593  }
  1594  
  1595  func (w persistConnWriter) Write(p []byte) (n int, err error) {
  1596  	n, err = w.pc.conn.Write(p)
  1597  	w.pc.nwrite += int64(n)
  1598  	return
  1599  }
  1600  
  1601  // ReadFrom exposes persistConnWriter's underlying Conn to io.Copy and if
  1602  // the Conn implements io.ReaderFrom, it can take advantage of optimizations
  1603  // such as sendfile.
  1604  func (w persistConnWriter) ReadFrom(r io.Reader) (n int64, err error) {
  1605  	n, err = io.Copy(w.pc.conn, r)
  1606  	w.pc.nwrite += n
  1607  	return
  1608  }
  1609  
  1610  var _ io.ReaderFrom = (*persistConnWriter)(nil)
  1611  
  1612  // connectMethod is the map key (in its String form) for keeping persistent
  1613  // TCP connections alive for subsequent HTTP requests.
  1614  //
  1615  // A connect method may be of the following types:
  1616  //
  1617  //	connectMethod.key().String()      Description
  1618  //	------------------------------    -------------------------
  1619  //	|http|foo.com                     http directly to server, no proxy
  1620  //	|https|foo.com                    https directly to server, no proxy
  1621  //	|https,h1|foo.com                 https directly to server w/o HTTP/2, no proxy
  1622  //	http://proxy.com|https|foo.com    http to proxy, then CONNECT to foo.com
  1623  //	http://proxy.com|http             http to proxy, http to anywhere after that
  1624  //	socks5://proxy.com|http|foo.com   socks5 to proxy, then http to foo.com
  1625  //	socks5://proxy.com|https|foo.com  socks5 to proxy, then https to foo.com
  1626  //	https://proxy.com|https|foo.com   https to proxy, then CONNECT to foo.com
  1627  //	https://proxy.com|http            https to proxy, http to anywhere after that
  1628  //
  1629  type connectMethod struct {
  1630  	proxyURL     *url.URL // nil for no proxy, else full proxy URL
  1631  	targetScheme string   // "http" or "https"
  1632  	// If proxyURL specifies an http or https proxy, and targetScheme is http (not https),
  1633  	// then targetAddr is not included in the connect method key, because the socket can
  1634  	// be reused for different targetAddr values.
  1635  	targetAddr string
  1636  	onlyH1     bool // whether to disable HTTP/2 and force HTTP/1
  1637  }
  1638  
  1639  func (cm *connectMethod) key() connectMethodKey {
  1640  	proxyStr := ""
  1641  	targetAddr := cm.targetAddr
  1642  	if cm.proxyURL != nil {
  1643  		proxyStr = cm.proxyURL.String()
  1644  		if (cm.proxyURL.Scheme == "http" || cm.proxyURL.Scheme == "https") && cm.targetScheme == "http" {
  1645  			targetAddr = ""
  1646  		}
  1647  	}
  1648  	return connectMethodKey{
  1649  		proxy:  proxyStr,
  1650  		scheme: cm.targetScheme,
  1651  		addr:   targetAddr,
  1652  		onlyH1: cm.onlyH1,
  1653  	}
  1654  }
  1655  
  1656  // scheme returns the first hop scheme: http, https, or socks5
  1657  func (cm *connectMethod) scheme() string {
  1658  	if cm.proxyURL != nil {
  1659  		return cm.proxyURL.Scheme
  1660  	}
  1661  	return cm.targetScheme
  1662  }
  1663  
  1664  // addr returns the first hop "host:port" to which we need to TCP connect.
  1665  func (cm *connectMethod) addr() string {
  1666  	if cm.proxyURL != nil {
  1667  		return canonicalAddr(cm.proxyURL)
  1668  	}
  1669  	return cm.targetAddr
  1670  }
  1671  
  1672  // tlsHost returns the host name to match against the peer's
  1673  // TLS certificate.
  1674  func (cm *connectMethod) tlsHost() string {
  1675  	h := cm.targetAddr
  1676  	if hasPort(h) {
  1677  		h = h[:strings.LastIndex(h, ":")]
  1678  	}
  1679  	return h
  1680  }
  1681  
  1682  // connectMethodKey is the map key version of connectMethod, with a
  1683  // stringified proxy URL (or the empty string) instead of a pointer to
  1684  // a URL.
  1685  type connectMethodKey struct {
  1686  	proxy, scheme, addr string
  1687  	onlyH1              bool
  1688  }
  1689  
  1690  func (k connectMethodKey) String() string {
  1691  	// Only used by tests.
  1692  	var h1 string
  1693  	if k.onlyH1 {
  1694  		h1 = ",h1"
  1695  	}
  1696  	return fmt.Sprintf("%s|%s%s|%s", k.proxy, k.scheme, h1, k.addr)
  1697  }
  1698  
  1699  // persistConn wraps a connection, usually a persistent one
  1700  // (but may be used for non-keep-alive requests as well)
  1701  type persistConn struct {
  1702  	// alt optionally specifies the TLS NextProto RoundTripper.
  1703  	// This is used for HTTP/2 today and future protocols later.
  1704  	// If it's non-nil, the rest of the fields are unused.
  1705  	alt RoundTripper
  1706  
  1707  	t         *Transport
  1708  	cacheKey  connectMethodKey
  1709  	conn      net.Conn
  1710  	tlsState  *tls.ConnectionState
  1711  	br        *bufio.Reader       // from conn
  1712  	bw        *bufio.Writer       // to conn
  1713  	nwrite    int64               // bytes written
  1714  	reqch     chan requestAndChan // written by roundTrip; read by readLoop
  1715  	writech   chan writeRequest   // written by roundTrip; read by writeLoop
  1716  	closech   chan struct{}       // closed when conn closed
  1717  	isProxy   bool
  1718  	sawEOF    bool  // whether we've seen EOF from conn; owned by readLoop
  1719  	readLimit int64 // bytes allowed to be read; owned by readLoop
  1720  	// writeErrCh passes the request write error (usually nil)
  1721  	// from the writeLoop goroutine to the readLoop which passes
  1722  	// it off to the res.Body reader, which then uses it to decide
  1723  	// whether or not a connection can be reused. Issue 7569.
  1724  	writeErrCh chan error
  1725  
  1726  	writeLoopDone chan struct{} // closed when write loop ends
  1727  
  1728  	// Both guarded by Transport.idleMu:
  1729  	idleAt    time.Time   // time it last become idle
  1730  	idleTimer *time.Timer // holding an AfterFunc to close it
  1731  
  1732  	mu                   sync.Mutex // guards following fields
  1733  	numExpectedResponses int
  1734  	closed               error // set non-nil when conn is closed, before closech is closed
  1735  	canceledErr          error // set non-nil if conn is canceled
  1736  	broken               bool  // an error has happened on this connection; marked broken so it's not reused.
  1737  	reused               bool  // whether conn has had successful request/response and is being reused.
  1738  	// mutateHeaderFunc is an optional func to modify extra
  1739  	// headers on each outbound request before it's written. (the
  1740  	// original Request given to RoundTrip is not modified)
  1741  	mutateHeaderFunc func(Header)
  1742  }
  1743  
  1744  func (pc *persistConn) maxHeaderResponseSize() int64 {
  1745  	if v := pc.t.MaxResponseHeaderBytes; v != 0 {
  1746  		return v
  1747  	}
  1748  	return 10 << 20 // conservative default; same as http2
  1749  }
  1750  
  1751  func (pc *persistConn) Read(p []byte) (n int, err error) {
  1752  	if pc.readLimit <= 0 {
  1753  		return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize())
  1754  	}
  1755  	if int64(len(p)) > pc.readLimit {
  1756  		p = p[:pc.readLimit]
  1757  	}
  1758  	n, err = pc.conn.Read(p)
  1759  	if err == io.EOF {
  1760  		pc.sawEOF = true
  1761  	}
  1762  	pc.readLimit -= int64(n)
  1763  	return
  1764  }
  1765  
  1766  // isBroken reports whether this connection is in a known broken state.
  1767  func (pc *persistConn) isBroken() bool {
  1768  	pc.mu.Lock()
  1769  	b := pc.closed != nil
  1770  	pc.mu.Unlock()
  1771  	return b
  1772  }
  1773  
  1774  // canceled returns non-nil if the connection was closed due to
  1775  // CancelRequest or due to context cancellation.
  1776  func (pc *persistConn) canceled() error {
  1777  	pc.mu.Lock()
  1778  	defer pc.mu.Unlock()
  1779  	return pc.canceledErr
  1780  }
  1781  
  1782  // isReused reports whether this connection has been used before.
  1783  func (pc *persistConn) isReused() bool {
  1784  	pc.mu.Lock()
  1785  	r := pc.reused
  1786  	pc.mu.Unlock()
  1787  	return r
  1788  }
  1789  
  1790  func (pc *persistConn) gotIdleConnTrace(idleAt time.Time) (t httptrace.GotConnInfo) {
  1791  	pc.mu.Lock()
  1792  	defer pc.mu.Unlock()
  1793  	t.Reused = pc.reused
  1794  	t.Conn = pc.conn
  1795  	t.WasIdle = true
  1796  	if !idleAt.IsZero() {
  1797  		t.IdleTime = time.Since(idleAt)
  1798  	}
  1799  	return
  1800  }
  1801  
  1802  func (pc *persistConn) cancelRequest(err error) {
  1803  	pc.mu.Lock()
  1804  	defer pc.mu.Unlock()
  1805  	pc.canceledErr = err
  1806  	pc.closeLocked(errRequestCanceled)
  1807  }
  1808  
  1809  // closeConnIfStillIdle closes the connection if it's still sitting idle.
  1810  // This is what's called by the persistConn's idleTimer, and is run in its
  1811  // own goroutine.
  1812  func (pc *persistConn) closeConnIfStillIdle() {
  1813  	t := pc.t
  1814  	t.idleMu.Lock()
  1815  	defer t.idleMu.Unlock()
  1816  	if _, ok := t.idleLRU.m[pc]; !ok {
  1817  		// Not idle.
  1818  		return
  1819  	}
  1820  	t.removeIdleConnLocked(pc)
  1821  	pc.close(errIdleConnTimeout)
  1822  }
  1823  
  1824  // mapRoundTripError returns the appropriate error value for
  1825  // persistConn.roundTrip.
  1826  //
  1827  // The provided err is the first error that (*persistConn).roundTrip
  1828  // happened to receive from its select statement.
  1829  //
  1830  // The startBytesWritten value should be the value of pc.nwrite before the roundTrip
  1831  // started writing the request.
  1832  func (pc *persistConn) mapRoundTripError(req *transportRequest, startBytesWritten int64, err error) error {
  1833  	if err == nil {
  1834  		return nil
  1835  	}
  1836  
  1837  	// If the request was canceled, that's better than network
  1838  	// failures that were likely the result of tearing down the
  1839  	// connection.
  1840  	if cerr := pc.canceled(); cerr != nil {
  1841  		return cerr
  1842  	}
  1843  
  1844  	// See if an error was set explicitly.
  1845  	req.mu.Lock()
  1846  	reqErr := req.err
  1847  	req.mu.Unlock()
  1848  	if reqErr != nil {
  1849  		return reqErr
  1850  	}
  1851  
  1852  	if err == errServerClosedIdle {
  1853  		// Don't decorate
  1854  		return err
  1855  	}
  1856  
  1857  	if _, ok := err.(transportReadFromServerError); ok {
  1858  		// Don't decorate
  1859  		return err
  1860  	}
  1861  	if pc.isBroken() {
  1862  		<-pc.writeLoopDone
  1863  		if pc.nwrite == startBytesWritten {
  1864  			return nothingWrittenError{err}
  1865  		}
  1866  		return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %v", err)
  1867  	}
  1868  	return err
  1869  }
  1870  
  1871  // errCallerOwnsConn is an internal sentinel error used when we hand
  1872  // off a writable response.Body to the caller. We use this to prevent
  1873  // closing a net.Conn that is now owned by the caller.
  1874  var errCallerOwnsConn = errors.New("read loop ending; caller owns writable underlying conn")
  1875  
  1876  func (pc *persistConn) readLoop() {
  1877  	closeErr := errReadLoopExiting // default value, if not changed below
  1878  	defer func() {
  1879  		pc.close(closeErr)
  1880  		pc.t.removeIdleConn(pc)
  1881  	}()
  1882  
  1883  	tryPutIdleConn := func(trace *httptrace.ClientTrace) bool {
  1884  		if err := pc.t.tryPutIdleConn(pc); err != nil {
  1885  			closeErr = err
  1886  			if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled {
  1887  				trace.PutIdleConn(err)
  1888  			}
  1889  			return false
  1890  		}
  1891  		if trace != nil && trace.PutIdleConn != nil {
  1892  			trace.PutIdleConn(nil)
  1893  		}
  1894  		return true
  1895  	}
  1896  
  1897  	// eofc is used to block caller goroutines reading from Response.Body
  1898  	// at EOF until this goroutines has (potentially) added the connection
  1899  	// back to the idle pool.
  1900  	eofc := make(chan struct{})
  1901  	defer close(eofc) // unblock reader on errors
  1902  
  1903  	// Read this once, before loop starts. (to avoid races in tests)
  1904  	testHookMu.Lock()
  1905  	testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead
  1906  	testHookMu.Unlock()
  1907  
  1908  	alive := true
  1909  	for alive {
  1910  		pc.readLimit = pc.maxHeaderResponseSize()
  1911  		_, err := pc.br.Peek(1)
  1912  
  1913  		pc.mu.Lock()
  1914  		if pc.numExpectedResponses == 0 {
  1915  			pc.readLoopPeekFailLocked(err)
  1916  			pc.mu.Unlock()
  1917  			return
  1918  		}
  1919  		pc.mu.Unlock()
  1920  
  1921  		rc := <-pc.reqch
  1922  		trace := httptrace.ContextClientTrace(rc.req.Context())
  1923  
  1924  		var resp *Response
  1925  		if err == nil {
  1926  			resp, err = pc.readResponse(rc, trace)
  1927  		} else {
  1928  			err = transportReadFromServerError{err}
  1929  			closeErr = err
  1930  		}
  1931  
  1932  		if err != nil {
  1933  			if pc.readLimit <= 0 {
  1934  				err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize())
  1935  			}
  1936  
  1937  			select {
  1938  			case rc.ch <- responseAndError{err: err}:
  1939  			case <-rc.callerGone:
  1940  				return
  1941  			}
  1942  			return
  1943  		}
  1944  		pc.readLimit = maxInt64 // effictively no limit for response bodies
  1945  
  1946  		pc.mu.Lock()
  1947  		pc.numExpectedResponses--
  1948  		pc.mu.Unlock()
  1949  
  1950  		bodyWritable := resp.bodyIsWritable()
  1951  		hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0
  1952  
  1953  		if resp.Close || rc.req.Close || resp.StatusCode <= 199 || bodyWritable {
  1954  			// Don't do keep-alive on error if either party requested a close
  1955  			// or we get an unexpected informational (1xx) response.
  1956  			// StatusCode 100 is already handled above.
  1957  			alive = false
  1958  		}
  1959  
  1960  		if !hasBody || bodyWritable {
  1961  			pc.t.setReqCanceler(rc.req, nil)
  1962  
  1963  			// Put the idle conn back into the pool before we send the response
  1964  			// so if they process it quickly and make another request, they'll
  1965  			// get this same conn. But we use the unbuffered channel 'rc'
  1966  			// to guarantee that persistConn.roundTrip got out of its select
  1967  			// potentially waiting for this persistConn to close.
  1968  			// but after
  1969  			alive = alive &&
  1970  				!pc.sawEOF &&
  1971  				pc.wroteRequest() &&
  1972  				tryPutIdleConn(trace)
  1973  
  1974  			if bodyWritable {
  1975  				closeErr = errCallerOwnsConn
  1976  			}
  1977  
  1978  			select {
  1979  			case rc.ch <- responseAndError{res: resp}:
  1980  			case <-rc.callerGone:
  1981  				return
  1982  			}
  1983  
  1984  			// Now that they've read from the unbuffered channel, they're safely
  1985  			// out of the select that also waits on this goroutine to die, so
  1986  			// we're allowed to exit now if needed (if alive is false)
  1987  			testHookReadLoopBeforeNextRead()
  1988  			continue
  1989  		}
  1990  
  1991  		waitForBodyRead := make(chan bool, 2)
  1992  		body := &bodyEOFSignal{
  1993  			body: resp.Body,
  1994  			earlyCloseFn: func() error {
  1995  				waitForBodyRead <- false
  1996  				<-eofc // will be closed by deferred call at the end of the function
  1997  				return nil
  1998  
  1999  			},
  2000  			fn: func(err error) error {
  2001  				isEOF := err == io.EOF
  2002  				waitForBodyRead <- isEOF
  2003  				if isEOF {
  2004  					<-eofc // see comment above eofc declaration
  2005  				} else if err != nil {
  2006  					if cerr := pc.canceled(); cerr != nil {
  2007  						return cerr
  2008  					}
  2009  				}
  2010  				return err
  2011  			},
  2012  		}
  2013  
  2014  		resp.Body = body
  2015  		if rc.addedGzip && strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
  2016  			resp.Body = &gzipReader{body: body}
  2017  			resp.Header.Del("Content-Encoding")
  2018  			resp.Header.Del("Content-Length")
  2019  			resp.ContentLength = -1
  2020  			resp.Uncompressed = true
  2021  		}
  2022  
  2023  		select {
  2024  		case rc.ch <- responseAndError{res: resp}:
  2025  		case <-rc.callerGone:
  2026  			return
  2027  		}
  2028  
  2029  		// Before looping back to the top of this function and peeking on
  2030  		// the bufio.Reader, wait for the caller goroutine to finish
  2031  		// reading the response body. (or for cancellation or death)
  2032  		select {
  2033  		case bodyEOF := <-waitForBodyRead:
  2034  			pc.t.setReqCanceler(rc.req, nil) // before pc might return to idle pool
  2035  			alive = alive &&
  2036  				bodyEOF &&
  2037  				!pc.sawEOF &&
  2038  				pc.wroteRequest() &&
  2039  				tryPutIdleConn(trace)
  2040  			if bodyEOF {
  2041  				eofc <- struct{}{}
  2042  			}
  2043  		case <-rc.req.Cancel:
  2044  			alive = false
  2045  			pc.t.CancelRequest(rc.req)
  2046  		case <-rc.req.Context().Done():
  2047  			alive = false
  2048  			pc.t.cancelRequest(rc.req, rc.req.Context().Err())
  2049  		case <-pc.closech:
  2050  			alive = false
  2051  		}
  2052  
  2053  		testHookReadLoopBeforeNextRead()
  2054  	}
  2055  }
  2056  
  2057  func (pc *persistConn) readLoopPeekFailLocked(peekErr error) {
  2058  	if pc.closed != nil {
  2059  		return
  2060  	}
  2061  	if n := pc.br.Buffered(); n > 0 {
  2062  		buf, _ := pc.br.Peek(n)
  2063  		if is408Message(buf) {
  2064  			pc.closeLocked(errServerClosedIdle)
  2065  			return
  2066  		} else {
  2067  			log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", buf, peekErr)
  2068  		}
  2069  	}
  2070  	if peekErr == io.EOF {
  2071  		// common case.
  2072  		pc.closeLocked(errServerClosedIdle)
  2073  	} else {
  2074  		pc.closeLocked(fmt.Errorf("readLoopPeekFailLocked: %v", peekErr))
  2075  	}
  2076  }
  2077  
  2078  // is408Message reports whether buf has the prefix of an
  2079  // HTTP 408 Request Timeout response.
  2080  // See golang.org/issue/32310.
  2081  func is408Message(buf []byte) bool {
  2082  	if len(buf) < len("HTTP/1.x 408") {
  2083  		return false
  2084  	}
  2085  	if string(buf[:7]) != "HTTP/1." {
  2086  		return false
  2087  	}
  2088  	return string(buf[8:12]) == " 408"
  2089  }
  2090  
  2091  // readResponse reads an HTTP response (or two, in the case of "Expect:
  2092  // 100-continue") from the server. It returns the final non-100 one.
  2093  // trace is optional.
  2094  func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTrace) (resp *Response, err error) {
  2095  	if trace != nil && trace.GotFirstResponseByte != nil {
  2096  		if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 {
  2097  			trace.GotFirstResponseByte()
  2098  		}
  2099  	}
  2100  	num1xx := 0               // number of informational 1xx headers received
  2101  	const max1xxResponses = 5 // arbitrary bound on number of informational responses
  2102  
  2103  	continueCh := rc.continueCh
  2104  	for {
  2105  		resp, err = ReadResponse(pc.br, rc.req)
  2106  		if err != nil {
  2107  			return
  2108  		}
  2109  		resCode := resp.StatusCode
  2110  		if continueCh != nil {
  2111  			if resCode == 100 {
  2112  				if trace != nil && trace.Got100Continue != nil {
  2113  					trace.Got100Continue()
  2114  				}
  2115  				continueCh <- struct{}{}
  2116  				continueCh = nil
  2117  			} else if resCode >= 200 {
  2118  				close(continueCh)
  2119  				continueCh = nil
  2120  			}
  2121  		}
  2122  		is1xx := 100 <= resCode && resCode <= 199
  2123  		// treat 101 as a terminal status, see issue 26161
  2124  		is1xxNonTerminal := is1xx && resCode != StatusSwitchingProtocols
  2125  		if is1xxNonTerminal {
  2126  			num1xx++
  2127  			if num1xx > max1xxResponses {
  2128  				return nil, errors.New("net/http: too many 1xx informational responses")
  2129  			}
  2130  			pc.readLimit = pc.maxHeaderResponseSize() // reset the limit
  2131  			if trace != nil && trace.Got1xxResponse != nil {
  2132  				if err := trace.Got1xxResponse(resCode, textproto.MIMEHeader(resp.Header)); err != nil {
  2133  					return nil, err
  2134  				}
  2135  			}
  2136  			continue
  2137  		}
  2138  		break
  2139  	}
  2140  	if resp.isProtocolSwitch() {
  2141  		resp.Body = newReadWriteCloserBody(pc.br, pc.conn)
  2142  	}
  2143  
  2144  	resp.TLS = pc.tlsState
  2145  	return
  2146  }
  2147  
  2148  // waitForContinue returns the function to block until
  2149  // any response, timeout or connection close. After any of them,
  2150  // the function returns a bool which indicates if the body should be sent.
  2151  func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool {
  2152  	if continueCh == nil {
  2153  		return nil
  2154  	}
  2155  	return func() bool {
  2156  		timer := time.NewTimer(pc.t.ExpectContinueTimeout)
  2157  		defer timer.Stop()
  2158  
  2159  		select {
  2160  		case _, ok := <-continueCh:
  2161  			return ok
  2162  		case <-timer.C:
  2163  			return true
  2164  		case <-pc.closech:
  2165  			return false
  2166  		}
  2167  	}
  2168  }
  2169  
  2170  func newReadWriteCloserBody(br *bufio.Reader, rwc io.ReadWriteCloser) io.ReadWriteCloser {
  2171  	body := &readWriteCloserBody{ReadWriteCloser: rwc}
  2172  	if br.Buffered() != 0 {
  2173  		body.br = br
  2174  	}
  2175  	return body
  2176  }
  2177  
  2178  // readWriteCloserBody is the Response.Body type used when we want to
  2179  // give users write access to the Body through the underlying
  2180  // connection (TCP, unless using custom dialers). This is then
  2181  // the concrete type for a Response.Body on the 101 Switching
  2182  // Protocols response, as used by WebSockets, h2c, etc.
  2183  type readWriteCloserBody struct {
  2184  	br *bufio.Reader // used until empty
  2185  	io.ReadWriteCloser
  2186  }
  2187  
  2188  func (b *readWriteCloserBody) Read(p []byte) (n int, err error) {
  2189  	if b.br != nil {
  2190  		if n := b.br.Buffered(); len(p) > n {
  2191  			p = p[:n]
  2192  		}
  2193  		n, err = b.br.Read(p)
  2194  		if b.br.Buffered() == 0 {
  2195  			b.br = nil
  2196  		}
  2197  		return n, err
  2198  	}
  2199  	return b.ReadWriteCloser.Read(p)
  2200  }
  2201  
  2202  // nothingWrittenError wraps a write errors which ended up writing zero bytes.
  2203  type nothingWrittenError struct {
  2204  	error
  2205  }
  2206  
  2207  func (pc *persistConn) writeLoop() {
  2208  	defer close(pc.writeLoopDone)
  2209  	for {
  2210  		select {
  2211  		case wr := <-pc.writech:
  2212  			startBytesWritten := pc.nwrite
  2213  			err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh))
  2214  			if bre, ok := err.(requestBodyReadError); ok {
  2215  				err = bre.error
  2216  				// Errors reading from the user's
  2217  				// Request.Body are high priority.
  2218  				// Set it here before sending on the
  2219  				// channels below or calling
  2220  				// pc.close() which tears town
  2221  				// connections and causes other
  2222  				// errors.
  2223  				wr.req.setError(err)
  2224  			}
  2225  			if err == nil {
  2226  				err = pc.bw.Flush()
  2227  			}
  2228  			if err != nil {
  2229  				wr.req.Request.closeBody()
  2230  				if pc.nwrite == startBytesWritten {
  2231  					err = nothingWrittenError{err}
  2232  				}
  2233  			}
  2234  			pc.writeErrCh <- err // to the body reader, which might recycle us
  2235  			wr.ch <- err         // to the roundTrip function
  2236  			if err != nil {
  2237  				pc.close(err)
  2238  				return
  2239  			}
  2240  		case <-pc.closech:
  2241  			return
  2242  		}
  2243  	}
  2244  }
  2245  
  2246  // maxWriteWaitBeforeConnReuse is how long the a Transport RoundTrip
  2247  // will wait to see the Request's Body.Write result after getting a
  2248  // response from the server. See comments in (*persistConn).wroteRequest.
  2249  const maxWriteWaitBeforeConnReuse = 50 * time.Millisecond
  2250  
  2251  // wroteRequest is a check before recycling a connection that the previous write
  2252  // (from writeLoop above) happened and was successful.
  2253  func (pc *persistConn) wroteRequest() bool {
  2254  	select {
  2255  	case err := <-pc.writeErrCh:
  2256  		// Common case: the write happened well before the response, so
  2257  		// avoid creating a timer.
  2258  		return err == nil
  2259  	default:
  2260  		// Rare case: the request was written in writeLoop above but
  2261  		// before it could send to pc.writeErrCh, the reader read it
  2262  		// all, processed it, and called us here. In this case, give the
  2263  		// write goroutine a bit of time to finish its send.
  2264  		//
  2265  		// Less rare case: We also get here in the legitimate case of
  2266  		// Issue 7569, where the writer is still writing (or stalled),
  2267  		// but the server has already replied. In this case, we don't
  2268  		// want to wait too long, and we want to return false so this
  2269  		// connection isn't re-used.
  2270  		t := time.NewTimer(maxWriteWaitBeforeConnReuse)
  2271  		defer t.Stop()
  2272  		select {
  2273  		case err := <-pc.writeErrCh:
  2274  			return err == nil
  2275  		case <-t.C:
  2276  			return false
  2277  		}
  2278  	}
  2279  }
  2280  
  2281  // responseAndError is how the goroutine reading from an HTTP/1 server
  2282  // communicates with the goroutine doing the RoundTrip.
  2283  type responseAndError struct {
  2284  	res *Response // else use this response (see res method)
  2285  	err error
  2286  }
  2287  
  2288  type requestAndChan struct {
  2289  	req *Request
  2290  	ch  chan responseAndError // unbuffered; always send in select on callerGone
  2291  
  2292  	// whether the Transport (as opposed to the user client code)
  2293  	// added the Accept-Encoding gzip header. If the Transport
  2294  	// set it, only then do we transparently decode the gzip.
  2295  	addedGzip bool
  2296  
  2297  	// Optional blocking chan for Expect: 100-continue (for send).
  2298  	// If the request has an "Expect: 100-continue" header and
  2299  	// the server responds 100 Continue, readLoop send a value
  2300  	// to writeLoop via this chan.
  2301  	continueCh chan<- struct{}
  2302  
  2303  	callerGone <-chan struct{} // closed when roundTrip caller has returned
  2304  }
  2305  
  2306  // A writeRequest is sent by the readLoop's goroutine to the
  2307  // writeLoop's goroutine to write a request while the read loop
  2308  // concurrently waits on both the write response and the server's
  2309  // reply.
  2310  type writeRequest struct {
  2311  	req *transportRequest
  2312  	ch  chan<- error
  2313  
  2314  	// Optional blocking chan for Expect: 100-continue (for receive).
  2315  	// If not nil, writeLoop blocks sending request body until
  2316  	// it receives from this chan.
  2317  	continueCh <-chan struct{}
  2318  }
  2319  
  2320  type httpError struct {
  2321  	err     string
  2322  	timeout bool
  2323  }
  2324  
  2325  func (e *httpError) Error() string   { return e.err }
  2326  func (e *httpError) Timeout() bool   { return e.timeout }
  2327  func (e *httpError) Temporary() bool { return true }
  2328  
  2329  var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true}
  2330  
  2331  // errRequestCanceled is set to be identical to the one from h2 to facilitate
  2332  // testing.
  2333  var errRequestCanceled = http2errRequestCanceled
  2334  var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify?
  2335  
  2336  func nop() {}
  2337  
  2338  // testHooks. Always non-nil.
  2339  var (
  2340  	testHookEnterRoundTrip   = nop
  2341  	testHookWaitResLoop      = nop
  2342  	testHookRoundTripRetried = nop
  2343  	testHookPrePendingDial   = nop
  2344  	testHookPostPendingDial  = nop
  2345  
  2346  	testHookMu                     sync.Locker = fakeLocker{} // guards following
  2347  	testHookReadLoopBeforeNextRead             = nop
  2348  )
  2349  
  2350  func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) {
  2351  	testHookEnterRoundTrip()
  2352  	if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) {
  2353  		pc.t.putOrCloseIdleConn(pc)
  2354  		return nil, errRequestCanceled
  2355  	}
  2356  	pc.mu.Lock()
  2357  	pc.numExpectedResponses++
  2358  	headerFn := pc.mutateHeaderFunc
  2359  	pc.mu.Unlock()
  2360  
  2361  	if headerFn != nil {
  2362  		headerFn(req.extraHeaders())
  2363  	}
  2364  
  2365  	// Ask for a compressed version if the caller didn't set their
  2366  	// own value for Accept-Encoding. We only attempt to
  2367  	// uncompress the gzip stream if we were the layer that
  2368  	// requested it.
  2369  	requestedGzip := false
  2370  	if !pc.t.DisableCompression &&
  2371  		req.Header.Get("Accept-Encoding") == "" &&
  2372  		req.Header.Get("Range") == "" &&
  2373  		req.Method != "HEAD" {
  2374  		// Request gzip only, not deflate. Deflate is ambiguous and
  2375  		// not as universally supported anyway.
  2376  		// See: https://zlib.net/zlib_faq.html#faq39
  2377  		//
  2378  		// Note that we don't request this for HEAD requests,
  2379  		// due to a bug in nginx:
  2380  		//   https://trac.nginx.org/nginx/ticket/358
  2381  		//   https://golang.org/issue/5522
  2382  		//
  2383  		// We don't request gzip if the request is for a range, since
  2384  		// auto-decoding a portion of a gzipped document will just fail
  2385  		// anyway. See https://golang.org/issue/8923
  2386  		requestedGzip = true
  2387  		req.extraHeaders().Set("Accept-Encoding", "gzip")
  2388  	}
  2389  
  2390  	var continueCh chan struct{}
  2391  	if req.ProtoAtLeast(1, 1) && req.Body != nil && req.expectsContinue() {
  2392  		continueCh = make(chan struct{}, 1)
  2393  	}
  2394  
  2395  	if pc.t.DisableKeepAlives && !req.wantsClose() {
  2396  		req.extraHeaders().Set("Connection", "close")
  2397  	}
  2398  
  2399  	gone := make(chan struct{})
  2400  	defer close(gone)
  2401  
  2402  	defer func() {
  2403  		if err != nil {
  2404  			pc.t.setReqCanceler(req.Request, nil)
  2405  		}
  2406  	}()
  2407  
  2408  	const debugRoundTrip = false
  2409  
  2410  	// Write the request concurrently with waiting for a response,
  2411  	// in case the server decides to reply before reading our full
  2412  	// request body.
  2413  	startBytesWritten := pc.nwrite
  2414  	writeErrCh := make(chan error, 1)
  2415  	pc.writech <- writeRequest{req, writeErrCh, continueCh}
  2416  
  2417  	resc := make(chan responseAndError)
  2418  	pc.reqch <- requestAndChan{
  2419  		req:        req.Request,
  2420  		ch:         resc,
  2421  		addedGzip:  requestedGzip,
  2422  		continueCh: continueCh,
  2423  		callerGone: gone,
  2424  	}
  2425  
  2426  	var respHeaderTimer <-chan time.Time
  2427  	cancelChan := req.Request.Cancel
  2428  	ctxDoneChan := req.Context().Done()
  2429  	for {
  2430  		testHookWaitResLoop()
  2431  		select {
  2432  		case err := <-writeErrCh:
  2433  			if debugRoundTrip {
  2434  				req.logf("writeErrCh resv: %T/%#v", err, err)
  2435  			}
  2436  			if err != nil {
  2437  				pc.close(fmt.Errorf("write error: %v", err))
  2438  				return nil, pc.mapRoundTripError(req, startBytesWritten, err)
  2439  			}
  2440  			if d := pc.t.ResponseHeaderTimeout; d > 0 {
  2441  				if debugRoundTrip {
  2442  					req.logf("starting timer for %v", d)
  2443  				}
  2444  				timer := time.NewTimer(d)
  2445  				defer timer.Stop() // prevent leaks
  2446  				respHeaderTimer = timer.C
  2447  			}
  2448  		case <-pc.closech:
  2449  			if debugRoundTrip {
  2450  				req.logf("closech recv: %T %#v", pc.closed, pc.closed)
  2451  			}
  2452  			return nil, pc.mapRoundTripError(req, startBytesWritten, pc.closed)
  2453  		case <-respHeaderTimer:
  2454  			if debugRoundTrip {
  2455  				req.logf("timeout waiting for response headers.")
  2456  			}
  2457  			pc.close(errTimeout)
  2458  			return nil, errTimeout
  2459  		case re := <-resc:
  2460  			if (re.res == nil) == (re.err == nil) {
  2461  				panic(fmt.Sprintf("internal error: exactly one of res or err should be set; nil=%v", re.res == nil))
  2462  			}
  2463  			if debugRoundTrip {
  2464  				req.logf("resc recv: %p, %T/%#v", re.res, re.err, re.err)
  2465  			}
  2466  			if re.err != nil {
  2467  				return nil, pc.mapRoundTripError(req, startBytesWritten, re.err)
  2468  			}
  2469  			return re.res, nil
  2470  		case <-cancelChan:
  2471  			pc.t.CancelRequest(req.Request)
  2472  			cancelChan = nil
  2473  		case <-ctxDoneChan:
  2474  			pc.t.cancelRequest(req.Request, req.Context().Err())
  2475  			cancelChan = nil
  2476  			ctxDoneChan = nil
  2477  		}
  2478  	}
  2479  }
  2480  
  2481  // tLogKey is a context WithValue key for test debugging contexts containing
  2482  // a t.Logf func. See export_test.go's Request.WithT method.
  2483  type tLogKey struct{}
  2484  
  2485  func (tr *transportRequest) logf(format string, args ...interface{}) {
  2486  	if logf, ok := tr.Request.Context().Value(tLogKey{}).(func(string, ...interface{})); ok {
  2487  		logf(time.Now().Format(time.RFC3339Nano)+": "+format, args...)
  2488  	}
  2489  }
  2490  
  2491  // markReused marks this connection as having been successfully used for a
  2492  // request and response.
  2493  func (pc *persistConn) markReused() {
  2494  	pc.mu.Lock()
  2495  	pc.reused = true
  2496  	pc.mu.Unlock()
  2497  }
  2498  
  2499  // close closes the underlying TCP connection and closes
  2500  // the pc.closech channel.
  2501  //
  2502  // The provided err is only for testing and debugging; in normal
  2503  // circumstances it should never be seen by users.
  2504  func (pc *persistConn) close(err error) {
  2505  	pc.mu.Lock()
  2506  	defer pc.mu.Unlock()
  2507  	pc.closeLocked(err)
  2508  }
  2509  
  2510  func (pc *persistConn) closeLocked(err error) {
  2511  	if err == nil {
  2512  		panic("nil error")
  2513  	}
  2514  	pc.broken = true
  2515  	if pc.closed == nil {
  2516  		pc.closed = err
  2517  		pc.t.decConnsPerHost(pc.cacheKey)
  2518  		// Close HTTP/1 (pc.alt == nil) connection.
  2519  		// HTTP/2 closes its connection itself.
  2520  		if pc.alt == nil {
  2521  			if err != errCallerOwnsConn {
  2522  				pc.conn.Close()
  2523  			}
  2524  			close(pc.closech)
  2525  		}
  2526  	}
  2527  	pc.mutateHeaderFunc = nil
  2528  }
  2529  
  2530  var portMap = map[string]string{
  2531  	"http":   "80",
  2532  	"https":  "443",
  2533  	"socks5": "1080",
  2534  }
  2535  
  2536  // canonicalAddr returns url.Host but always with a ":port" suffix
  2537  func canonicalAddr(url *url.URL) string {
  2538  	addr := url.Hostname()
  2539  	if v, err := idnaASCII(addr); err == nil {
  2540  		addr = v
  2541  	}
  2542  	port := url.Port()
  2543  	if port == "" {
  2544  		port = portMap[url.Scheme]
  2545  	}
  2546  	return net.JoinHostPort(addr, port)
  2547  }
  2548  
  2549  // bodyEOFSignal is used by the HTTP/1 transport when reading response
  2550  // bodies to make sure we see the end of a response body before
  2551  // proceeding and reading on the connection again.
  2552  //
  2553  // It wraps a ReadCloser but runs fn (if non-nil) at most
  2554  // once, right before its final (error-producing) Read or Close call
  2555  // returns. fn should return the new error to return from Read or Close.
  2556  //
  2557  // If earlyCloseFn is non-nil and Close is called before io.EOF is
  2558  // seen, earlyCloseFn is called instead of fn, and its return value is
  2559  // the return value from Close.
  2560  type bodyEOFSignal struct {
  2561  	body         io.ReadCloser
  2562  	mu           sync.Mutex        // guards following 4 fields
  2563  	closed       bool              // whether Close has been called
  2564  	rerr         error             // sticky Read error
  2565  	fn           func(error) error // err will be nil on Read io.EOF
  2566  	earlyCloseFn func() error      // optional alt Close func used if io.EOF not seen
  2567  }
  2568  
  2569  var errReadOnClosedResBody = errors.New("http: read on closed response body")
  2570  
  2571  func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
  2572  	es.mu.Lock()
  2573  	closed, rerr := es.closed, es.rerr
  2574  	es.mu.Unlock()
  2575  	if closed {
  2576  		return 0, errReadOnClosedResBody
  2577  	}
  2578  	if rerr != nil {
  2579  		return 0, rerr
  2580  	}
  2581  
  2582  	n, err = es.body.Read(p)
  2583  	if err != nil {
  2584  		es.mu.Lock()
  2585  		defer es.mu.Unlock()
  2586  		if es.rerr == nil {
  2587  			es.rerr = err
  2588  		}
  2589  		err = es.condfn(err)
  2590  	}
  2591  	return
  2592  }
  2593  
  2594  func (es *bodyEOFSignal) Close() error {
  2595  	es.mu.Lock()
  2596  	defer es.mu.Unlock()
  2597  	if es.closed {
  2598  		return nil
  2599  	}
  2600  	es.closed = true
  2601  	if es.earlyCloseFn != nil && es.rerr != io.EOF {
  2602  		return es.earlyCloseFn()
  2603  	}
  2604  	err := es.body.Close()
  2605  	return es.condfn(err)
  2606  }
  2607  
  2608  // caller must hold es.mu.
  2609  func (es *bodyEOFSignal) condfn(err error) error {
  2610  	if es.fn == nil {
  2611  		return err
  2612  	}
  2613  	err = es.fn(err)
  2614  	es.fn = nil
  2615  	return err
  2616  }
  2617  
  2618  // gzipReader wraps a response body so it can lazily
  2619  // call gzip.NewReader on the first call to Read
  2620  type gzipReader struct {
  2621  	body *bodyEOFSignal // underlying HTTP/1 response body framing
  2622  	zr   *gzip.Reader   // lazily-initialized gzip reader
  2623  	zerr error          // any error from gzip.NewReader; sticky
  2624  }
  2625  
  2626  func (gz *gzipReader) Read(p []byte) (n int, err error) {
  2627  	if gz.zr == nil {
  2628  		if gz.zerr == nil {
  2629  			gz.zr, gz.zerr = gzip.NewReader(gz.body)
  2630  		}
  2631  		if gz.zerr != nil {
  2632  			return 0, gz.zerr
  2633  		}
  2634  	}
  2635  
  2636  	gz.body.mu.Lock()
  2637  	if gz.body.closed {
  2638  		err = errReadOnClosedResBody
  2639  	}
  2640  	gz.body.mu.Unlock()
  2641  
  2642  	if err != nil {
  2643  		return 0, err
  2644  	}
  2645  	return gz.zr.Read(p)
  2646  }
  2647  
  2648  func (gz *gzipReader) Close() error {
  2649  	return gz.body.Close()
  2650  }
  2651  
  2652  type readerAndCloser struct {
  2653  	io.Reader
  2654  	io.Closer
  2655  }
  2656  
  2657  type tlsHandshakeTimeoutError struct{}
  2658  
  2659  func (tlsHandshakeTimeoutError) Timeout() bool   { return true }
  2660  func (tlsHandshakeTimeoutError) Temporary() bool { return true }
  2661  func (tlsHandshakeTimeoutError) Error() string   { return "net/http: TLS handshake timeout" }
  2662  
  2663  // fakeLocker is a sync.Locker which does nothing. It's used to guard
  2664  // test-only fields when not under test, to avoid runtime atomic
  2665  // overhead.
  2666  type fakeLocker struct{}
  2667  
  2668  func (fakeLocker) Lock()   {}
  2669  func (fakeLocker) Unlock() {}
  2670  
  2671  // cloneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if
  2672  // cfg is nil. This is safe to call even if cfg is in active use by a TLS
  2673  // client or server.
  2674  func cloneTLSConfig(cfg *tls.Config) *tls.Config {
  2675  	if cfg == nil {
  2676  		return &tls.Config{}
  2677  	}
  2678  	return cfg.Clone()
  2679  }
  2680  
  2681  type connLRU struct {
  2682  	ll *list.List // list.Element.Value type is of *persistConn
  2683  	m  map[*persistConn]*list.Element
  2684  }
  2685  
  2686  // add adds pc to the head of the linked list.
  2687  func (cl *connLRU) add(pc *persistConn) {
  2688  	if cl.ll == nil {
  2689  		cl.ll = list.New()
  2690  		cl.m = make(map[*persistConn]*list.Element)
  2691  	}
  2692  	ele := cl.ll.PushFront(pc)
  2693  	if _, ok := cl.m[pc]; ok {
  2694  		panic("persistConn was already in LRU")
  2695  	}
  2696  	cl.m[pc] = ele
  2697  }
  2698  
  2699  func (cl *connLRU) removeOldest() *persistConn {
  2700  	ele := cl.ll.Back()
  2701  	pc := ele.Value.(*persistConn)
  2702  	cl.ll.Remove(ele)
  2703  	delete(cl.m, pc)
  2704  	return pc
  2705  }
  2706  
  2707  // remove removes pc from cl.
  2708  func (cl *connLRU) remove(pc *persistConn) {
  2709  	if ele, ok := cl.m[pc]; ok {
  2710  		cl.ll.Remove(ele)
  2711  		delete(cl.m, pc)
  2712  	}
  2713  }
  2714  
  2715  // len returns the number of items in the cache.
  2716  func (cl *connLRU) len() int {
  2717  	return len(cl.m)
  2718  }
  2719  
  2720  // validPort reports whether p (without the colon) is a valid port in
  2721  // a URL, per RFC 3986 Section 3.2.3, which says the port may be
  2722  // empty, or only contain digits.
  2723  func validPort(p string) bool {
  2724  	for _, r := range []byte(p) {
  2725  		if r < '0' || r > '9' {
  2726  			return false
  2727  		}
  2728  	}
  2729  	return true
  2730  }
  2731  

View as plain text