...
Run Format

Source file src/net/http/transport.go

     1	// Copyright 2011 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// HTTP client implementation. See RFC 2616.
     6	//
     7	// This is the low-level Transport implementation of RoundTripper.
     8	// The high-level interface is in client.go.
     9	
    10	package http
    11	
    12	import (
    13		"bufio"
    14		"compress/gzip"
    15		"crypto/tls"
    16		"errors"
    17		"fmt"
    18		"io"
    19		"log"
    20		"net"
    21		"net/url"
    22		"os"
    23		"strings"
    24		"sync"
    25		"time"
    26	)
    27	
    28	// DefaultTransport is the default implementation of Transport and is
    29	// used by DefaultClient. It establishes network connections as needed
    30	// and caches them for reuse by subsequent calls. It uses HTTP proxies
    31	// as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and
    32	// $no_proxy) environment variables.
    33	var DefaultTransport RoundTripper = &Transport{
    34		Proxy: ProxyFromEnvironment,
    35		Dial: (&net.Dialer{
    36			Timeout:   30 * time.Second,
    37			KeepAlive: 30 * time.Second,
    38		}).Dial,
    39		TLSHandshakeTimeout: 10 * time.Second,
    40	}
    41	
    42	// DefaultMaxIdleConnsPerHost is the default value of Transport's
    43	// MaxIdleConnsPerHost.
    44	const DefaultMaxIdleConnsPerHost = 2
    45	
    46	// Transport is an implementation of RoundTripper that supports HTTP,
    47	// HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT).
    48	// Transport can also cache connections for future re-use.
    49	type Transport struct {
    50		idleMu     sync.Mutex
    51		wantIdle   bool // user has requested to close all idle conns
    52		idleConn   map[connectMethodKey][]*persistConn
    53		idleConnCh map[connectMethodKey]chan *persistConn
    54	
    55		reqMu       sync.Mutex
    56		reqCanceler map[*Request]func()
    57	
    58		altMu    sync.RWMutex
    59		altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper
    60	
    61		// Proxy specifies a function to return a proxy for a given
    62		// Request. If the function returns a non-nil error, the
    63		// request is aborted with the provided error.
    64		// If Proxy is nil or returns a nil *URL, no proxy is used.
    65		Proxy func(*Request) (*url.URL, error)
    66	
    67		// Dial specifies the dial function for creating unencrypted
    68		// TCP connections.
    69		// If Dial is nil, net.Dial is used.
    70		Dial func(network, addr string) (net.Conn, error)
    71	
    72		// DialTLS specifies an optional dial function for creating
    73		// TLS connections for non-proxied HTTPS requests.
    74		//
    75		// If DialTLS is nil, Dial and TLSClientConfig are used.
    76		//
    77		// If DialTLS is set, the Dial hook is not used for HTTPS
    78		// requests and the TLSClientConfig and TLSHandshakeTimeout
    79		// are ignored. The returned net.Conn is assumed to already be
    80		// past the TLS handshake.
    81		DialTLS func(network, addr string) (net.Conn, error)
    82	
    83		// TLSClientConfig specifies the TLS configuration to use with
    84		// tls.Client. If nil, the default configuration is used.
    85		TLSClientConfig *tls.Config
    86	
    87		// TLSHandshakeTimeout specifies the maximum amount of time waiting to
    88		// wait for a TLS handshake. Zero means no timeout.
    89		TLSHandshakeTimeout time.Duration
    90	
    91		// DisableKeepAlives, if true, prevents re-use of TCP connections
    92		// between different HTTP requests.
    93		DisableKeepAlives bool
    94	
    95		// DisableCompression, if true, prevents the Transport from
    96		// requesting compression with an "Accept-Encoding: gzip"
    97		// request header when the Request contains no existing
    98		// Accept-Encoding value. If the Transport requests gzip on
    99		// its own and gets a gzipped response, it's transparently
   100		// decoded in the Response.Body. However, if the user
   101		// explicitly requested gzip it is not automatically
   102		// uncompressed.
   103		DisableCompression bool
   104	
   105		// MaxIdleConnsPerHost, if non-zero, controls the maximum idle
   106		// (keep-alive) to keep per-host.  If zero,
   107		// DefaultMaxIdleConnsPerHost is used.
   108		MaxIdleConnsPerHost int
   109	
   110		// ResponseHeaderTimeout, if non-zero, specifies the amount of
   111		// time to wait for a server's response headers after fully
   112		// writing the request (including its body, if any). This
   113		// time does not include the time to read the response body.
   114		ResponseHeaderTimeout time.Duration
   115	
   116		// TODO: tunable on global max cached connections
   117		// TODO: tunable on timeout on cached connections
   118	}
   119	
   120	// ProxyFromEnvironment returns the URL of the proxy to use for a
   121	// given request, as indicated by the environment variables
   122	// HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the lowercase versions
   123	// thereof). HTTPS_PROXY takes precedence over HTTP_PROXY for https
   124	// requests.
   125	//
   126	// The environment values may be either a complete URL or a
   127	// "host[:port]", in which case the "http" scheme is assumed.
   128	// An error is returned if the value is a different form.
   129	//
   130	// A nil URL and nil error are returned if no proxy is defined in the
   131	// environment, or a proxy should not be used for the given request,
   132	// as defined by NO_PROXY.
   133	//
   134	// As a special case, if req.URL.Host is "localhost" (with or without
   135	// a port number), then a nil URL and nil error will be returned.
   136	func ProxyFromEnvironment(req *Request) (*url.URL, error) {
   137		var proxy string
   138		if req.URL.Scheme == "https" {
   139			proxy = httpsProxyEnv.Get()
   140		}
   141		if proxy == "" {
   142			proxy = httpProxyEnv.Get()
   143		}
   144		if proxy == "" {
   145			return nil, nil
   146		}
   147		if !useProxy(canonicalAddr(req.URL)) {
   148			return nil, nil
   149		}
   150		proxyURL, err := url.Parse(proxy)
   151		if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") {
   152			// proxy was bogus. Try prepending "http://" to it and
   153			// see if that parses correctly. If not, we fall
   154			// through and complain about the original one.
   155			if proxyURL, err := url.Parse("http://" + proxy); err == nil {
   156				return proxyURL, nil
   157			}
   158		}
   159		if err != nil {
   160			return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err)
   161		}
   162		return proxyURL, nil
   163	}
   164	
   165	// ProxyURL returns a proxy function (for use in a Transport)
   166	// that always returns the same URL.
   167	func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) {
   168		return func(*Request) (*url.URL, error) {
   169			return fixedURL, nil
   170		}
   171	}
   172	
   173	// transportRequest is a wrapper around a *Request that adds
   174	// optional extra headers to write.
   175	type transportRequest struct {
   176		*Request        // original request, not to be mutated
   177		extra    Header // extra headers to write, or nil
   178	}
   179	
   180	func (tr *transportRequest) extraHeaders() Header {
   181		if tr.extra == nil {
   182			tr.extra = make(Header)
   183		}
   184		return tr.extra
   185	}
   186	
   187	// RoundTrip implements the RoundTripper interface.
   188	//
   189	// For higher-level HTTP client support (such as handling of cookies
   190	// and redirects), see Get, Post, and the Client type.
   191	func (t *Transport) RoundTrip(req *Request) (resp *Response, err error) {
   192		if req.URL == nil {
   193			req.closeBody()
   194			return nil, errors.New("http: nil Request.URL")
   195		}
   196		if req.Header == nil {
   197			req.closeBody()
   198			return nil, errors.New("http: nil Request.Header")
   199		}
   200		if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
   201			t.altMu.RLock()
   202			var rt RoundTripper
   203			if t.altProto != nil {
   204				rt = t.altProto[req.URL.Scheme]
   205			}
   206			t.altMu.RUnlock()
   207			if rt == nil {
   208				req.closeBody()
   209				return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme}
   210			}
   211			return rt.RoundTrip(req)
   212		}
   213		if req.URL.Host == "" {
   214			req.closeBody()
   215			return nil, errors.New("http: no Host in request URL")
   216		}
   217		treq := &transportRequest{Request: req}
   218		cm, err := t.connectMethodForRequest(treq)
   219		if err != nil {
   220			req.closeBody()
   221			return nil, err
   222		}
   223	
   224		// Get the cached or newly-created connection to either the
   225		// host (for http or https), the http proxy, or the http proxy
   226		// pre-CONNECTed to https server.  In any case, we'll be ready
   227		// to send it requests.
   228		pconn, err := t.getConn(req, cm)
   229		if err != nil {
   230			t.setReqCanceler(req, nil)
   231			req.closeBody()
   232			return nil, err
   233		}
   234	
   235		return pconn.roundTrip(treq)
   236	}
   237	
   238	// RegisterProtocol registers a new protocol with scheme.
   239	// The Transport will pass requests using the given scheme to rt.
   240	// It is rt's responsibility to simulate HTTP request semantics.
   241	//
   242	// RegisterProtocol can be used by other packages to provide
   243	// implementations of protocol schemes like "ftp" or "file".
   244	func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) {
   245		if scheme == "http" || scheme == "https" {
   246			panic("protocol " + scheme + " already registered")
   247		}
   248		t.altMu.Lock()
   249		defer t.altMu.Unlock()
   250		if t.altProto == nil {
   251			t.altProto = make(map[string]RoundTripper)
   252		}
   253		if _, exists := t.altProto[scheme]; exists {
   254			panic("protocol " + scheme + " already registered")
   255		}
   256		t.altProto[scheme] = rt
   257	}
   258	
   259	// CloseIdleConnections closes any connections which were previously
   260	// connected from previous requests but are now sitting idle in
   261	// a "keep-alive" state. It does not interrupt any connections currently
   262	// in use.
   263	func (t *Transport) CloseIdleConnections() {
   264		t.idleMu.Lock()
   265		m := t.idleConn
   266		t.idleConn = nil
   267		t.idleConnCh = nil
   268		t.wantIdle = true
   269		t.idleMu.Unlock()
   270		for _, conns := range m {
   271			for _, pconn := range conns {
   272				pconn.close()
   273			}
   274		}
   275	}
   276	
   277	// CancelRequest cancels an in-flight request by closing its
   278	// connection.
   279	func (t *Transport) CancelRequest(req *Request) {
   280		t.reqMu.Lock()
   281		cancel := t.reqCanceler[req]
   282		t.reqMu.Unlock()
   283		if cancel != nil {
   284			cancel()
   285		}
   286	}
   287	
   288	//
   289	// Private implementation past this point.
   290	//
   291	
   292	var (
   293		httpProxyEnv = &envOnce{
   294			names: []string{"HTTP_PROXY", "http_proxy"},
   295		}
   296		httpsProxyEnv = &envOnce{
   297			names: []string{"HTTPS_PROXY", "https_proxy"},
   298		}
   299		noProxyEnv = &envOnce{
   300			names: []string{"NO_PROXY", "no_proxy"},
   301		}
   302	)
   303	
   304	// envOnce looks up an environment variable (optionally by multiple
   305	// names) once. It mitigates expensive lookups on some platforms
   306	// (e.g. Windows).
   307	type envOnce struct {
   308		names []string
   309		once  sync.Once
   310		val   string
   311	}
   312	
   313	func (e *envOnce) Get() string {
   314		e.once.Do(e.init)
   315		return e.val
   316	}
   317	
   318	func (e *envOnce) init() {
   319		for _, n := range e.names {
   320			e.val = os.Getenv(n)
   321			if e.val != "" {
   322				return
   323			}
   324		}
   325	}
   326	
   327	// reset is used by tests
   328	func (e *envOnce) reset() {
   329		e.once = sync.Once{}
   330		e.val = ""
   331	}
   332	
   333	func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) {
   334		cm.targetScheme = treq.URL.Scheme
   335		cm.targetAddr = canonicalAddr(treq.URL)
   336		if t.Proxy != nil {
   337			cm.proxyURL, err = t.Proxy(treq.Request)
   338		}
   339		return cm, err
   340	}
   341	
   342	// proxyAuth returns the Proxy-Authorization header to set
   343	// on requests, if applicable.
   344	func (cm *connectMethod) proxyAuth() string {
   345		if cm.proxyURL == nil {
   346			return ""
   347		}
   348		if u := cm.proxyURL.User; u != nil {
   349			username := u.Username()
   350			password, _ := u.Password()
   351			return "Basic " + basicAuth(username, password)
   352		}
   353		return ""
   354	}
   355	
   356	// putIdleConn adds pconn to the list of idle persistent connections awaiting
   357	// a new request.
   358	// If pconn is no longer needed or not in a good state, putIdleConn
   359	// returns false.
   360	func (t *Transport) putIdleConn(pconn *persistConn) bool {
   361		if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 {
   362			pconn.close()
   363			return false
   364		}
   365		if pconn.isBroken() {
   366			return false
   367		}
   368		key := pconn.cacheKey
   369		max := t.MaxIdleConnsPerHost
   370		if max == 0 {
   371			max = DefaultMaxIdleConnsPerHost
   372		}
   373		t.idleMu.Lock()
   374	
   375		waitingDialer := t.idleConnCh[key]
   376		select {
   377		case waitingDialer <- pconn:
   378			// We're done with this pconn and somebody else is
   379			// currently waiting for a conn of this type (they're
   380			// actively dialing, but this conn is ready
   381			// first). Chrome calls this socket late binding.  See
   382			// https://insouciant.org/tech/connection-management-in-chromium/
   383			t.idleMu.Unlock()
   384			return true
   385		default:
   386			if waitingDialer != nil {
   387				// They had populated this, but their dial won
   388				// first, so we can clean up this map entry.
   389				delete(t.idleConnCh, key)
   390			}
   391		}
   392		if t.wantIdle {
   393			t.idleMu.Unlock()
   394			pconn.close()
   395			return false
   396		}
   397		if t.idleConn == nil {
   398			t.idleConn = make(map[connectMethodKey][]*persistConn)
   399		}
   400		if len(t.idleConn[key]) >= max {
   401			t.idleMu.Unlock()
   402			pconn.close()
   403			return false
   404		}
   405		for _, exist := range t.idleConn[key] {
   406			if exist == pconn {
   407				log.Fatalf("dup idle pconn %p in freelist", pconn)
   408			}
   409		}
   410		t.idleConn[key] = append(t.idleConn[key], pconn)
   411		t.idleMu.Unlock()
   412		return true
   413	}
   414	
   415	// getIdleConnCh returns a channel to receive and return idle
   416	// persistent connection for the given connectMethod.
   417	// It may return nil, if persistent connections are not being used.
   418	func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn {
   419		if t.DisableKeepAlives {
   420			return nil
   421		}
   422		key := cm.key()
   423		t.idleMu.Lock()
   424		defer t.idleMu.Unlock()
   425		t.wantIdle = false
   426		if t.idleConnCh == nil {
   427			t.idleConnCh = make(map[connectMethodKey]chan *persistConn)
   428		}
   429		ch, ok := t.idleConnCh[key]
   430		if !ok {
   431			ch = make(chan *persistConn)
   432			t.idleConnCh[key] = ch
   433		}
   434		return ch
   435	}
   436	
   437	func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn) {
   438		key := cm.key()
   439		t.idleMu.Lock()
   440		defer t.idleMu.Unlock()
   441		if t.idleConn == nil {
   442			return nil
   443		}
   444		for {
   445			pconns, ok := t.idleConn[key]
   446			if !ok {
   447				return nil
   448			}
   449			if len(pconns) == 1 {
   450				pconn = pconns[0]
   451				delete(t.idleConn, key)
   452			} else {
   453				// 2 or more cached connections; pop last
   454				// TODO: queue?
   455				pconn = pconns[len(pconns)-1]
   456				t.idleConn[key] = pconns[:len(pconns)-1]
   457			}
   458			if !pconn.isBroken() {
   459				return
   460			}
   461		}
   462	}
   463	
   464	func (t *Transport) setReqCanceler(r *Request, fn func()) {
   465		t.reqMu.Lock()
   466		defer t.reqMu.Unlock()
   467		if t.reqCanceler == nil {
   468			t.reqCanceler = make(map[*Request]func())
   469		}
   470		if fn != nil {
   471			t.reqCanceler[r] = fn
   472		} else {
   473			delete(t.reqCanceler, r)
   474		}
   475	}
   476	
   477	func (t *Transport) dial(network, addr string) (c net.Conn, err error) {
   478		if t.Dial != nil {
   479			return t.Dial(network, addr)
   480		}
   481		return net.Dial(network, addr)
   482	}
   483	
   484	// Testing hooks:
   485	var prePendingDial, postPendingDial func()
   486	
   487	// getConn dials and creates a new persistConn to the target as
   488	// specified in the connectMethod.  This includes doing a proxy CONNECT
   489	// and/or setting up TLS.  If this doesn't return an error, the persistConn
   490	// is ready to write requests to.
   491	func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error) {
   492		if pc := t.getIdleConn(cm); pc != nil {
   493			return pc, nil
   494		}
   495	
   496		type dialRes struct {
   497			pc  *persistConn
   498			err error
   499		}
   500		dialc := make(chan dialRes)
   501	
   502		handlePendingDial := func() {
   503			if prePendingDial != nil {
   504				prePendingDial()
   505			}
   506			go func() {
   507				if v := <-dialc; v.err == nil {
   508					t.putIdleConn(v.pc)
   509				}
   510				if postPendingDial != nil {
   511					postPendingDial()
   512				}
   513			}()
   514		}
   515	
   516		cancelc := make(chan struct{})
   517		t.setReqCanceler(req, func() { close(cancelc) })
   518	
   519		go func() {
   520			pc, err := t.dialConn(cm)
   521			dialc <- dialRes{pc, err}
   522		}()
   523	
   524		idleConnCh := t.getIdleConnCh(cm)
   525		select {
   526		case v := <-dialc:
   527			// Our dial finished.
   528			return v.pc, v.err
   529		case pc := <-idleConnCh:
   530			// Another request finished first and its net.Conn
   531			// became available before our dial. Or somebody
   532			// else's dial that they didn't use.
   533			// But our dial is still going, so give it away
   534			// when it finishes:
   535			handlePendingDial()
   536			return pc, nil
   537		case <-cancelc:
   538			handlePendingDial()
   539			return nil, errors.New("net/http: request canceled while waiting for connection")
   540		}
   541	}
   542	
   543	func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) {
   544		pconn := &persistConn{
   545			t:          t,
   546			cacheKey:   cm.key(),
   547			reqch:      make(chan requestAndChan, 1),
   548			writech:    make(chan writeRequest, 1),
   549			closech:    make(chan struct{}),
   550			writeErrCh: make(chan error, 1),
   551		}
   552		tlsDial := t.DialTLS != nil && cm.targetScheme == "https" && cm.proxyURL == nil
   553		if tlsDial {
   554			var err error
   555			pconn.conn, err = t.DialTLS("tcp", cm.addr())
   556			if err != nil {
   557				return nil, err
   558			}
   559			if tc, ok := pconn.conn.(*tls.Conn); ok {
   560				cs := tc.ConnectionState()
   561				pconn.tlsState = &cs
   562			}
   563		} else {
   564			conn, err := t.dial("tcp", cm.addr())
   565			if err != nil {
   566				if cm.proxyURL != nil {
   567					err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err)
   568				}
   569				return nil, err
   570			}
   571			pconn.conn = conn
   572		}
   573	
   574		// Proxy setup.
   575		switch {
   576		case cm.proxyURL == nil:
   577			// Do nothing. Not using a proxy.
   578		case cm.targetScheme == "http":
   579			pconn.isProxy = true
   580			if pa := cm.proxyAuth(); pa != "" {
   581				pconn.mutateHeaderFunc = func(h Header) {
   582					h.Set("Proxy-Authorization", pa)
   583				}
   584			}
   585		case cm.targetScheme == "https":
   586			conn := pconn.conn
   587			connectReq := &Request{
   588				Method: "CONNECT",
   589				URL:    &url.URL{Opaque: cm.targetAddr},
   590				Host:   cm.targetAddr,
   591				Header: make(Header),
   592			}
   593			if pa := cm.proxyAuth(); pa != "" {
   594				connectReq.Header.Set("Proxy-Authorization", pa)
   595			}
   596			connectReq.Write(conn)
   597	
   598			// Read response.
   599			// Okay to use and discard buffered reader here, because
   600			// TLS server will not speak until spoken to.
   601			br := bufio.NewReader(conn)
   602			resp, err := ReadResponse(br, connectReq)
   603			if err != nil {
   604				conn.Close()
   605				return nil, err
   606			}
   607			if resp.StatusCode != 200 {
   608				f := strings.SplitN(resp.Status, " ", 2)
   609				conn.Close()
   610				return nil, errors.New(f[1])
   611			}
   612		}
   613	
   614		if cm.targetScheme == "https" && !tlsDial {
   615			// Initiate TLS and check remote host name against certificate.
   616			cfg := t.TLSClientConfig
   617			if cfg == nil || cfg.ServerName == "" {
   618				host := cm.tlsHost()
   619				if cfg == nil {
   620					cfg = &tls.Config{ServerName: host}
   621				} else {
   622					clone := *cfg // shallow clone
   623					clone.ServerName = host
   624					cfg = &clone
   625				}
   626			}
   627			plainConn := pconn.conn
   628			tlsConn := tls.Client(plainConn, cfg)
   629			errc := make(chan error, 2)
   630			var timer *time.Timer // for canceling TLS handshake
   631			if d := t.TLSHandshakeTimeout; d != 0 {
   632				timer = time.AfterFunc(d, func() {
   633					errc <- tlsHandshakeTimeoutError{}
   634				})
   635			}
   636			go func() {
   637				err := tlsConn.Handshake()
   638				if timer != nil {
   639					timer.Stop()
   640				}
   641				errc <- err
   642			}()
   643			if err := <-errc; err != nil {
   644				plainConn.Close()
   645				return nil, err
   646			}
   647			if !cfg.InsecureSkipVerify {
   648				if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil {
   649					plainConn.Close()
   650					return nil, err
   651				}
   652			}
   653			cs := tlsConn.ConnectionState()
   654			pconn.tlsState = &cs
   655			pconn.conn = tlsConn
   656		}
   657	
   658		pconn.br = bufio.NewReader(noteEOFReader{pconn.conn, &pconn.sawEOF})
   659		pconn.bw = bufio.NewWriter(pconn.conn)
   660		go pconn.readLoop()
   661		go pconn.writeLoop()
   662		return pconn, nil
   663	}
   664	
   665	// useProxy returns true if requests to addr should use a proxy,
   666	// according to the NO_PROXY or no_proxy environment variable.
   667	// addr is always a canonicalAddr with a host and port.
   668	func useProxy(addr string) bool {
   669		if len(addr) == 0 {
   670			return true
   671		}
   672		host, _, err := net.SplitHostPort(addr)
   673		if err != nil {
   674			return false
   675		}
   676		if host == "localhost" {
   677			return false
   678		}
   679		if ip := net.ParseIP(host); ip != nil {
   680			if ip.IsLoopback() {
   681				return false
   682			}
   683		}
   684	
   685		no_proxy := noProxyEnv.Get()
   686		if no_proxy == "*" {
   687			return false
   688		}
   689	
   690		addr = strings.ToLower(strings.TrimSpace(addr))
   691		if hasPort(addr) {
   692			addr = addr[:strings.LastIndex(addr, ":")]
   693		}
   694	
   695		for _, p := range strings.Split(no_proxy, ",") {
   696			p = strings.ToLower(strings.TrimSpace(p))
   697			if len(p) == 0 {
   698				continue
   699			}
   700			if hasPort(p) {
   701				p = p[:strings.LastIndex(p, ":")]
   702			}
   703			if addr == p {
   704				return false
   705			}
   706			if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) {
   707				// no_proxy ".foo.com" matches "bar.foo.com" or "foo.com"
   708				return false
   709			}
   710			if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' {
   711				// no_proxy "foo.com" matches "bar.foo.com"
   712				return false
   713			}
   714		}
   715		return true
   716	}
   717	
   718	// connectMethod is the map key (in its String form) for keeping persistent
   719	// TCP connections alive for subsequent HTTP requests.
   720	//
   721	// A connect method may be of the following types:
   722	//
   723	// Cache key form                Description
   724	// -----------------             -------------------------
   725	// |http|foo.com                 http directly to server, no proxy
   726	// |https|foo.com                https directly to server, no proxy
   727	// http://proxy.com|https|foo.com  http to proxy, then CONNECT to foo.com
   728	// http://proxy.com|http           http to proxy, http to anywhere after that
   729	//
   730	// Note: no support to https to the proxy yet.
   731	//
   732	type connectMethod struct {
   733		proxyURL     *url.URL // nil for no proxy, else full proxy URL
   734		targetScheme string   // "http" or "https"
   735		targetAddr   string   // Not used if proxy + http targetScheme (4th example in table)
   736	}
   737	
   738	func (cm *connectMethod) key() connectMethodKey {
   739		proxyStr := ""
   740		targetAddr := cm.targetAddr
   741		if cm.proxyURL != nil {
   742			proxyStr = cm.proxyURL.String()
   743			if cm.targetScheme == "http" {
   744				targetAddr = ""
   745			}
   746		}
   747		return connectMethodKey{
   748			proxy:  proxyStr,
   749			scheme: cm.targetScheme,
   750			addr:   targetAddr,
   751		}
   752	}
   753	
   754	// addr returns the first hop "host:port" to which we need to TCP connect.
   755	func (cm *connectMethod) addr() string {
   756		if cm.proxyURL != nil {
   757			return canonicalAddr(cm.proxyURL)
   758		}
   759		return cm.targetAddr
   760	}
   761	
   762	// tlsHost returns the host name to match against the peer's
   763	// TLS certificate.
   764	func (cm *connectMethod) tlsHost() string {
   765		h := cm.targetAddr
   766		if hasPort(h) {
   767			h = h[:strings.LastIndex(h, ":")]
   768		}
   769		return h
   770	}
   771	
   772	// connectMethodKey is the map key version of connectMethod, with a
   773	// stringified proxy URL (or the empty string) instead of a pointer to
   774	// a URL.
   775	type connectMethodKey struct {
   776		proxy, scheme, addr string
   777	}
   778	
   779	func (k connectMethodKey) String() string {
   780		// Only used by tests.
   781		return fmt.Sprintf("%s|%s|%s", k.proxy, k.scheme, k.addr)
   782	}
   783	
   784	// persistConn wraps a connection, usually a persistent one
   785	// (but may be used for non-keep-alive requests as well)
   786	type persistConn struct {
   787		t        *Transport
   788		cacheKey connectMethodKey
   789		conn     net.Conn
   790		tlsState *tls.ConnectionState
   791		br       *bufio.Reader       // from conn
   792		sawEOF   bool                // whether we've seen EOF from conn; owned by readLoop
   793		bw       *bufio.Writer       // to conn
   794		reqch    chan requestAndChan // written by roundTrip; read by readLoop
   795		writech  chan writeRequest   // written by roundTrip; read by writeLoop
   796		closech  chan struct{}       // closed when conn closed
   797		isProxy  bool
   798		// writeErrCh passes the request write error (usually nil)
   799		// from the writeLoop goroutine to the readLoop which passes
   800		// it off to the res.Body reader, which then uses it to decide
   801		// whether or not a connection can be reused. Issue 7569.
   802		writeErrCh chan error
   803	
   804		lk                   sync.Mutex // guards following fields
   805		numExpectedResponses int
   806		closed               bool // whether conn has been closed
   807		broken               bool // an error has happened on this connection; marked broken so it's not reused.
   808		// mutateHeaderFunc is an optional func to modify extra
   809		// headers on each outbound request before it's written. (the
   810		// original Request given to RoundTrip is not modified)
   811		mutateHeaderFunc func(Header)
   812	}
   813	
   814	// isBroken reports whether this connection is in a known broken state.
   815	func (pc *persistConn) isBroken() bool {
   816		pc.lk.Lock()
   817		b := pc.broken
   818		pc.lk.Unlock()
   819		return b
   820	}
   821	
   822	func (pc *persistConn) cancelRequest() {
   823		pc.conn.Close()
   824	}
   825	
   826	var remoteSideClosedFunc func(error) bool // or nil to use default
   827	
   828	func remoteSideClosed(err error) bool {
   829		if err == io.EOF {
   830			return true
   831		}
   832		if remoteSideClosedFunc != nil {
   833			return remoteSideClosedFunc(err)
   834		}
   835		return false
   836	}
   837	
   838	func (pc *persistConn) readLoop() {
   839		alive := true
   840	
   841		for alive {
   842			pb, err := pc.br.Peek(1)
   843	
   844			pc.lk.Lock()
   845			if pc.numExpectedResponses == 0 {
   846				if !pc.closed {
   847					pc.closeLocked()
   848					if len(pb) > 0 {
   849						log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v",
   850							string(pb), err)
   851					}
   852				}
   853				pc.lk.Unlock()
   854				return
   855			}
   856			pc.lk.Unlock()
   857	
   858			rc := <-pc.reqch
   859	
   860			var resp *Response
   861			if err == nil {
   862				resp, err = ReadResponse(pc.br, rc.req)
   863				if err == nil && resp.StatusCode == 100 {
   864					// Skip any 100-continue for now.
   865					// TODO(bradfitz): if rc.req had "Expect: 100-continue",
   866					// actually block the request body write and signal the
   867					// writeLoop now to begin sending it. (Issue 2184) For now we
   868					// eat it, since we're never expecting one.
   869					resp, err = ReadResponse(pc.br, rc.req)
   870				}
   871			}
   872	
   873			if resp != nil {
   874				resp.TLS = pc.tlsState
   875			}
   876	
   877			hasBody := resp != nil && rc.req.Method != "HEAD" && resp.ContentLength != 0
   878	
   879			if err != nil {
   880				pc.close()
   881			} else {
   882				if rc.addedGzip && hasBody && resp.Header.Get("Content-Encoding") == "gzip" {
   883					resp.Header.Del("Content-Encoding")
   884					resp.Header.Del("Content-Length")
   885					resp.ContentLength = -1
   886					resp.Body = &gzipReader{body: resp.Body}
   887				}
   888				resp.Body = &bodyEOFSignal{body: resp.Body}
   889			}
   890	
   891			if err != nil || resp.Close || rc.req.Close || resp.StatusCode <= 199 {
   892				// Don't do keep-alive on error if either party requested a close
   893				// or we get an unexpected informational (1xx) response.
   894				// StatusCode 100 is already handled above.
   895				alive = false
   896			}
   897	
   898			var waitForBodyRead chan bool
   899			if hasBody {
   900				waitForBodyRead = make(chan bool, 2)
   901				resp.Body.(*bodyEOFSignal).earlyCloseFn = func() error {
   902					// Sending false here sets alive to
   903					// false and closes the connection
   904					// below.
   905					waitForBodyRead <- false
   906					return nil
   907				}
   908				resp.Body.(*bodyEOFSignal).fn = func(err error) {
   909					waitForBodyRead <- alive &&
   910						err == nil &&
   911						!pc.sawEOF &&
   912						pc.wroteRequest() &&
   913						pc.t.putIdleConn(pc)
   914				}
   915			}
   916	
   917			if alive && !hasBody {
   918				alive = !pc.sawEOF &&
   919					pc.wroteRequest() &&
   920					pc.t.putIdleConn(pc)
   921			}
   922	
   923			rc.ch <- responseAndError{resp, err}
   924	
   925			// Wait for the just-returned response body to be fully consumed
   926			// before we race and peek on the underlying bufio reader.
   927			if waitForBodyRead != nil {
   928				select {
   929				case alive = <-waitForBodyRead:
   930				case <-pc.closech:
   931					alive = false
   932				}
   933			}
   934	
   935			pc.t.setReqCanceler(rc.req, nil)
   936	
   937			if !alive {
   938				pc.close()
   939			}
   940		}
   941	}
   942	
   943	func (pc *persistConn) writeLoop() {
   944		for {
   945			select {
   946			case wr := <-pc.writech:
   947				if pc.isBroken() {
   948					wr.ch <- errors.New("http: can't write HTTP request on broken connection")
   949					continue
   950				}
   951				err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra)
   952				if err == nil {
   953					err = pc.bw.Flush()
   954				}
   955				if err != nil {
   956					pc.markBroken()
   957					wr.req.Request.closeBody()
   958				}
   959				pc.writeErrCh <- err // to the body reader, which might recycle us
   960				wr.ch <- err         // to the roundTrip function
   961			case <-pc.closech:
   962				return
   963			}
   964		}
   965	}
   966	
   967	// wroteRequest is a check before recycling a connection that the previous write
   968	// (from writeLoop above) happened and was successful.
   969	func (pc *persistConn) wroteRequest() bool {
   970		select {
   971		case err := <-pc.writeErrCh:
   972			// Common case: the write happened well before the response, so
   973			// avoid creating a timer.
   974			return err == nil
   975		default:
   976			// Rare case: the request was written in writeLoop above but
   977			// before it could send to pc.writeErrCh, the reader read it
   978			// all, processed it, and called us here. In this case, give the
   979			// write goroutine a bit of time to finish its send.
   980			//
   981			// Less rare case: We also get here in the legitimate case of
   982			// Issue 7569, where the writer is still writing (or stalled),
   983			// but the server has already replied. In this case, we don't
   984			// want to wait too long, and we want to return false so this
   985			// connection isn't re-used.
   986			select {
   987			case err := <-pc.writeErrCh:
   988				return err == nil
   989			case <-time.After(50 * time.Millisecond):
   990				return false
   991			}
   992		}
   993	}
   994	
   995	type responseAndError struct {
   996		res *Response
   997		err error
   998	}
   999	
  1000	type requestAndChan struct {
  1001		req *Request
  1002		ch  chan responseAndError
  1003	
  1004		// did the Transport (as opposed to the client code) add an
  1005		// Accept-Encoding gzip header? only if it we set it do
  1006		// we transparently decode the gzip.
  1007		addedGzip bool
  1008	}
  1009	
  1010	// A writeRequest is sent by the readLoop's goroutine to the
  1011	// writeLoop's goroutine to write a request while the read loop
  1012	// concurrently waits on both the write response and the server's
  1013	// reply.
  1014	type writeRequest struct {
  1015		req *transportRequest
  1016		ch  chan<- error
  1017	}
  1018	
  1019	type httpError struct {
  1020		err     string
  1021		timeout bool
  1022	}
  1023	
  1024	func (e *httpError) Error() string   { return e.err }
  1025	func (e *httpError) Timeout() bool   { return e.timeout }
  1026	func (e *httpError) Temporary() bool { return true }
  1027	
  1028	var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true}
  1029	var errClosed error = &httpError{err: "net/http: transport closed before response was received"}
  1030	
  1031	func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) {
  1032		pc.t.setReqCanceler(req.Request, pc.cancelRequest)
  1033		pc.lk.Lock()
  1034		pc.numExpectedResponses++
  1035		headerFn := pc.mutateHeaderFunc
  1036		pc.lk.Unlock()
  1037	
  1038		if headerFn != nil {
  1039			headerFn(req.extraHeaders())
  1040		}
  1041	
  1042		// Ask for a compressed version if the caller didn't set their
  1043		// own value for Accept-Encoding. We only attempt to
  1044		// uncompress the gzip stream if we were the layer that
  1045		// requested it.
  1046		requestedGzip := false
  1047		if !pc.t.DisableCompression &&
  1048			req.Header.Get("Accept-Encoding") == "" &&
  1049			req.Header.Get("Range") == "" &&
  1050			req.Method != "HEAD" {
  1051			// Request gzip only, not deflate. Deflate is ambiguous and
  1052			// not as universally supported anyway.
  1053			// See: http://www.gzip.org/zlib/zlib_faq.html#faq38
  1054			//
  1055			// Note that we don't request this for HEAD requests,
  1056			// due to a bug in nginx:
  1057			//   http://trac.nginx.org/nginx/ticket/358
  1058			//   http://golang.org/issue/5522
  1059			//
  1060			// We don't request gzip if the request is for a range, since
  1061			// auto-decoding a portion of a gzipped document will just fail
  1062			// anyway. See http://golang.org/issue/8923
  1063			requestedGzip = true
  1064			req.extraHeaders().Set("Accept-Encoding", "gzip")
  1065		}
  1066	
  1067		// Write the request concurrently with waiting for a response,
  1068		// in case the server decides to reply before reading our full
  1069		// request body.
  1070		writeErrCh := make(chan error, 1)
  1071		pc.writech <- writeRequest{req, writeErrCh}
  1072	
  1073		resc := make(chan responseAndError, 1)
  1074		pc.reqch <- requestAndChan{req.Request, resc, requestedGzip}
  1075	
  1076		var re responseAndError
  1077		var pconnDeadCh = pc.closech
  1078		var failTicker <-chan time.Time
  1079		var respHeaderTimer <-chan time.Time
  1080	WaitResponse:
  1081		for {
  1082			select {
  1083			case err := <-writeErrCh:
  1084				if err != nil {
  1085					re = responseAndError{nil, err}
  1086					pc.close()
  1087					break WaitResponse
  1088				}
  1089				if d := pc.t.ResponseHeaderTimeout; d > 0 {
  1090					respHeaderTimer = time.After(d)
  1091				}
  1092			case <-pconnDeadCh:
  1093				// The persist connection is dead. This shouldn't
  1094				// usually happen (only with Connection: close responses
  1095				// with no response bodies), but if it does happen it
  1096				// means either a) the remote server hung up on us
  1097				// prematurely, or b) the readLoop sent us a response &
  1098				// closed its closech at roughly the same time, and we
  1099				// selected this case first, in which case a response
  1100				// might still be coming soon.
  1101				//
  1102				// We can't avoid the select race in b) by using a unbuffered
  1103				// resc channel instead, because then goroutines can
  1104				// leak if we exit due to other errors.
  1105				pconnDeadCh = nil                               // avoid spinning
  1106				failTicker = time.After(100 * time.Millisecond) // arbitrary time to wait for resc
  1107			case <-failTicker:
  1108				re = responseAndError{err: errClosed}
  1109				break WaitResponse
  1110			case <-respHeaderTimer:
  1111				pc.close()
  1112				re = responseAndError{err: errTimeout}
  1113				break WaitResponse
  1114			case re = <-resc:
  1115				break WaitResponse
  1116			}
  1117		}
  1118	
  1119		pc.lk.Lock()
  1120		pc.numExpectedResponses--
  1121		pc.lk.Unlock()
  1122	
  1123		if re.err != nil {
  1124			pc.t.setReqCanceler(req.Request, nil)
  1125		}
  1126		return re.res, re.err
  1127	}
  1128	
  1129	// markBroken marks a connection as broken (so it's not reused).
  1130	// It differs from close in that it doesn't close the underlying
  1131	// connection for use when it's still being read.
  1132	func (pc *persistConn) markBroken() {
  1133		pc.lk.Lock()
  1134		defer pc.lk.Unlock()
  1135		pc.broken = true
  1136	}
  1137	
  1138	func (pc *persistConn) close() {
  1139		pc.lk.Lock()
  1140		defer pc.lk.Unlock()
  1141		pc.closeLocked()
  1142	}
  1143	
  1144	func (pc *persistConn) closeLocked() {
  1145		pc.broken = true
  1146		if !pc.closed {
  1147			pc.conn.Close()
  1148			pc.closed = true
  1149			close(pc.closech)
  1150		}
  1151		pc.mutateHeaderFunc = nil
  1152	}
  1153	
  1154	var portMap = map[string]string{
  1155		"http":  "80",
  1156		"https": "443",
  1157	}
  1158	
  1159	// canonicalAddr returns url.Host but always with a ":port" suffix
  1160	func canonicalAddr(url *url.URL) string {
  1161		addr := url.Host
  1162		if !hasPort(addr) {
  1163			return addr + ":" + portMap[url.Scheme]
  1164		}
  1165		return addr
  1166	}
  1167	
  1168	// bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most
  1169	// once, right before its final (error-producing) Read or Close call
  1170	// returns. If earlyCloseFn is non-nil and Close is called before
  1171	// io.EOF is seen, earlyCloseFn is called instead of fn, and its
  1172	// return value is the return value from Close.
  1173	type bodyEOFSignal struct {
  1174		body         io.ReadCloser
  1175		mu           sync.Mutex   // guards following 4 fields
  1176		closed       bool         // whether Close has been called
  1177		rerr         error        // sticky Read error
  1178		fn           func(error)  // error will be nil on Read io.EOF
  1179		earlyCloseFn func() error // optional alt Close func used if io.EOF not seen
  1180	}
  1181	
  1182	func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
  1183		es.mu.Lock()
  1184		closed, rerr := es.closed, es.rerr
  1185		es.mu.Unlock()
  1186		if closed {
  1187			return 0, errors.New("http: read on closed response body")
  1188		}
  1189		if rerr != nil {
  1190			return 0, rerr
  1191		}
  1192	
  1193		n, err = es.body.Read(p)
  1194		if err != nil {
  1195			es.mu.Lock()
  1196			defer es.mu.Unlock()
  1197			if es.rerr == nil {
  1198				es.rerr = err
  1199			}
  1200			es.condfn(err)
  1201		}
  1202		return
  1203	}
  1204	
  1205	func (es *bodyEOFSignal) Close() error {
  1206		es.mu.Lock()
  1207		defer es.mu.Unlock()
  1208		if es.closed {
  1209			return nil
  1210		}
  1211		es.closed = true
  1212		if es.earlyCloseFn != nil && es.rerr != io.EOF {
  1213			return es.earlyCloseFn()
  1214		}
  1215		err := es.body.Close()
  1216		es.condfn(err)
  1217		return err
  1218	}
  1219	
  1220	// caller must hold es.mu.
  1221	func (es *bodyEOFSignal) condfn(err error) {
  1222		if es.fn == nil {
  1223			return
  1224		}
  1225		if err == io.EOF {
  1226			err = nil
  1227		}
  1228		es.fn(err)
  1229		es.fn = nil
  1230	}
  1231	
  1232	// gzipReader wraps a response body so it can lazily
  1233	// call gzip.NewReader on the first call to Read
  1234	type gzipReader struct {
  1235		body io.ReadCloser // underlying Response.Body
  1236		zr   io.Reader     // lazily-initialized gzip reader
  1237	}
  1238	
  1239	func (gz *gzipReader) Read(p []byte) (n int, err error) {
  1240		if gz.zr == nil {
  1241			gz.zr, err = gzip.NewReader(gz.body)
  1242			if err != nil {
  1243				return 0, err
  1244			}
  1245		}
  1246		return gz.zr.Read(p)
  1247	}
  1248	
  1249	func (gz *gzipReader) Close() error {
  1250		return gz.body.Close()
  1251	}
  1252	
  1253	type readerAndCloser struct {
  1254		io.Reader
  1255		io.Closer
  1256	}
  1257	
  1258	type tlsHandshakeTimeoutError struct{}
  1259	
  1260	func (tlsHandshakeTimeoutError) Timeout() bool   { return true }
  1261	func (tlsHandshakeTimeoutError) Temporary() bool { return true }
  1262	func (tlsHandshakeTimeoutError) Error() string   { return "net/http: TLS handshake timeout" }
  1263	
  1264	type noteEOFReader struct {
  1265		r      io.Reader
  1266		sawEOF *bool
  1267	}
  1268	
  1269	func (nr noteEOFReader) Read(p []byte) (n int, err error) {
  1270		n, err = nr.r.Read(p)
  1271		if err == io.EOF {
  1272			*nr.sawEOF = true
  1273		}
  1274		return
  1275	}
  1276	

View as plain text