http.go

     1  // Copyright 2012 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build !cmd_go_bootstrap
     6  
     7  // This code is compiled into the real 'go' binary, but it is not
     8  // compiled into the binary that is built during all.bash, so as
     9  // to avoid needing to build net (and thus use cgo) during the
    10  // bootstrap process.
    11  
    12  package web
    13  
    14  import (
    15  	"crypto/tls"
    16  	"errors"
    17  	"fmt"
    18  	"io"
    19  	"mime"
    20  	"net"
    21  	"net/http"
    22  	urlpkg "net/url"
    23  	"os"
    24  	"strings"
    25  	"time"
    26  
    27  	"cmd/go/internal/auth"
    28  	"cmd/go/internal/base"
    29  	"cmd/go/internal/cfg"
    30  	"cmd/internal/browser"
    31  )
    32  
    33  // impatientInsecureHTTPClient is used with GOINSECURE,
    34  // when we're connecting to https servers that might not be there
    35  // or might be using self-signed certificates.
    36  var impatientInsecureHTTPClient = &http.Client{
    37  	CheckRedirect: checkRedirect,
    38  	Timeout:       5 * time.Second,
    39  	Transport: &http.Transport{
    40  		Proxy: http.ProxyFromEnvironment,
    41  		TLSClientConfig: &tls.Config{
    42  			InsecureSkipVerify: true,
    43  		},
    44  	},
    45  }
    46  
    47  var securityPreservingDefaultClient = securityPreservingHTTPClient(http.DefaultClient)
    48  
    49  // securityPreservingHTTPClient returns a client that is like the original
    50  // but rejects redirects to plain-HTTP URLs if the original URL was secure.
    51  func securityPreservingHTTPClient(original *http.Client) *http.Client {
    52  	c := new(http.Client)
    53  	*c = *original
    54  	c.CheckRedirect = func(req *http.Request, via []*http.Request) error {
    55  		if len(via) > 0 && via[0].URL.Scheme == "https" && req.URL.Scheme != "https" {
    56  			lastHop := via[len(via)-1].URL
    57  			return fmt.Errorf("redirected from secure URL %s to insecure URL %s", lastHop, req.URL)
    58  		}
    59  		return checkRedirect(req, via)
    60  	}
    61  	return c
    62  }
    63  
    64  func checkRedirect(req *http.Request, via []*http.Request) error {
    65  	// Go's http.DefaultClient allows 10 redirects before returning an error.
    66  	// Mimic that behavior here.
    67  	if len(via) >= 10 {
    68  		return errors.New("stopped after 10 redirects")
    69  	}
    70  
    71  	interceptRequest(req)
    72  	return nil
    73  }
    74  
    75  type Interceptor struct {
    76  	Scheme   string
    77  	FromHost string
    78  	ToHost   string
    79  	Client   *http.Client
    80  }
    81  
    82  func EnableTestHooks(interceptors []Interceptor) error {
    83  	if enableTestHooks {
    84  		return errors.New("web: test hooks already enabled")
    85  	}
    86  
    87  	for _, t := range interceptors {
    88  		if t.FromHost == "" {
    89  			panic("EnableTestHooks: missing FromHost")
    90  		}
    91  		if t.ToHost == "" {
    92  			panic("EnableTestHooks: missing ToHost")
    93  		}
    94  	}
    95  
    96  	testInterceptors = interceptors
    97  	enableTestHooks = true
    98  	return nil
    99  }
   100  
   101  func DisableTestHooks() {
   102  	if !enableTestHooks {
   103  		panic("web: test hooks not enabled")
   104  	}
   105  	enableTestHooks = false
   106  	testInterceptors = nil
   107  }
   108  
   109  var (
   110  	enableTestHooks  = false
   111  	testInterceptors []Interceptor
   112  )
   113  
   114  func interceptURL(u *urlpkg.URL) (*Interceptor, bool) {
   115  	if !enableTestHooks {
   116  		return nil, false
   117  	}
   118  	for i, t := range testInterceptors {
   119  		if u.Host == t.FromHost && (u.Scheme == "" || u.Scheme == t.Scheme) {
   120  			return &testInterceptors[i], true
   121  		}
   122  	}
   123  	return nil, false
   124  }
   125  
   126  func interceptRequest(req *http.Request) {
   127  	if t, ok := interceptURL(req.URL); ok {
   128  		req.Host = req.URL.Host
   129  		req.URL.Host = t.ToHost
   130  	}
   131  }
   132  
   133  func get(security SecurityMode, url *urlpkg.URL) (*Response, error) {
   134  	start := time.Now()
   135  
   136  	if url.Scheme == "file" {
   137  		return getFile(url)
   138  	}
   139  
   140  	if enableTestHooks {
   141  		switch url.Host {
   142  		case "proxy.golang.org":
   143  			if os.Getenv("TESTGOPROXY404") == "1" {
   144  				res := &Response{
   145  					URL:        url.Redacted(),
   146  					Status:     "404 testing",
   147  					StatusCode: 404,
   148  					Header:     make(map[string][]string),
   149  					Body:       http.NoBody,
   150  				}
   151  				if cfg.BuildX {
   152  					fmt.Fprintf(os.Stderr, "# get %s: %v (%.3fs)\n", url.Redacted(), res.Status, time.Since(start).Seconds())
   153  				}
   154  				return res, nil
   155  			}
   156  
   157  		case "localhost.localdev":
   158  			return nil, fmt.Errorf("no such host localhost.localdev")
   159  
   160  		default:
   161  			if os.Getenv("TESTGONETWORK") == "panic" {
   162  				if _, ok := interceptURL(url); !ok {
   163  					host := url.Host
   164  					if h, _, err := net.SplitHostPort(url.Host); err == nil && h != "" {
   165  						host = h
   166  					}
   167  					addr := net.ParseIP(host)
   168  					if addr == nil || (!addr.IsLoopback() && !addr.IsUnspecified()) {
   169  						panic("use of network: " + url.String())
   170  					}
   171  				}
   172  			}
   173  		}
   174  	}
   175  
   176  	fetch := func(url *urlpkg.URL) (*http.Response, error) {
   177  		// Note: The -v build flag does not mean "print logging information",
   178  		// despite its historical misuse for this in GOPATH-based go get.
   179  		// We print extra logging in -x mode instead, which traces what
   180  		// commands are executed.
   181  		if cfg.BuildX {
   182  			fmt.Fprintf(os.Stderr, "# get %s\n", url.Redacted())
   183  		}
   184  
   185  		req, err := http.NewRequest("GET", url.String(), nil)
   186  		if err != nil {
   187  			return nil, err
   188  		}
   189  		if url.Scheme == "https" {
   190  			auth.AddCredentials(req)
   191  		}
   192  		t, intercepted := interceptURL(req.URL)
   193  		if intercepted {
   194  			req.Host = req.URL.Host
   195  			req.URL.Host = t.ToHost
   196  		}
   197  
   198  		release, err := base.AcquireNet()
   199  		if err != nil {
   200  			return nil, err
   201  		}
   202  
   203  		var res *http.Response
   204  		if security == Insecure && url.Scheme == "https" { // fail earlier
   205  			res, err = impatientInsecureHTTPClient.Do(req)
   206  		} else {
   207  			if intercepted && t.Client != nil {
   208  				client := securityPreservingHTTPClient(t.Client)
   209  				res, err = client.Do(req)
   210  			} else {
   211  				res, err = securityPreservingDefaultClient.Do(req)
   212  			}
   213  		}
   214  
   215  		if err != nil {
   216  			// Per the docs for [net/http.Client.Do], “On error, any Response can be
   217  			// ignored. A non-nil Response with a non-nil error only occurs when
   218  			// CheckRedirect fails, and even then the returned Response.Body is
   219  			// already closed.”
   220  			release()
   221  			return nil, err
   222  		}
   223  
   224  		// “If the returned error is nil, the Response will contain a non-nil Body
   225  		// which the user is expected to close.”
   226  		body := res.Body
   227  		res.Body = hookCloser{
   228  			ReadCloser: body,
   229  			afterClose: release,
   230  		}
   231  		return res, err
   232  	}
   233  
   234  	var (
   235  		fetched *urlpkg.URL
   236  		res     *http.Response
   237  		err     error
   238  	)
   239  	if url.Scheme == "" || url.Scheme == "https" {
   240  		secure := new(urlpkg.URL)
   241  		*secure = *url
   242  		secure.Scheme = "https"
   243  
   244  		res, err = fetch(secure)
   245  		if err == nil {
   246  			fetched = secure
   247  		} else {
   248  			if cfg.BuildX {
   249  				fmt.Fprintf(os.Stderr, "# get %s: %v\n", secure.Redacted(), err)
   250  			}
   251  			if security != Insecure || url.Scheme == "https" {
   252  				// HTTPS failed, and we can't fall back to plain HTTP.
   253  				// Report the error from the HTTPS attempt.
   254  				return nil, err
   255  			}
   256  		}
   257  	}
   258  
   259  	if res == nil {
   260  		switch url.Scheme {
   261  		case "http":
   262  			if security == SecureOnly {
   263  				if cfg.BuildX {
   264  					fmt.Fprintf(os.Stderr, "# get %s: insecure\n", url.Redacted())
   265  				}
   266  				return nil, fmt.Errorf("insecure URL: %s", url.Redacted())
   267  			}
   268  		case "":
   269  			if security != Insecure {
   270  				panic("should have returned after HTTPS failure")
   271  			}
   272  		default:
   273  			if cfg.BuildX {
   274  				fmt.Fprintf(os.Stderr, "# get %s: unsupported\n", url.Redacted())
   275  			}
   276  			return nil, fmt.Errorf("unsupported scheme: %s", url.Redacted())
   277  		}
   278  
   279  		insecure := new(urlpkg.URL)
   280  		*insecure = *url
   281  		insecure.Scheme = "http"
   282  		if insecure.User != nil && security != Insecure {
   283  			if cfg.BuildX {
   284  				fmt.Fprintf(os.Stderr, "# get %s: insecure credentials\n", insecure.Redacted())
   285  			}
   286  			return nil, fmt.Errorf("refusing to pass credentials to insecure URL: %s", insecure.Redacted())
   287  		}
   288  
   289  		res, err = fetch(insecure)
   290  		if err == nil {
   291  			fetched = insecure
   292  		} else {
   293  			if cfg.BuildX {
   294  				fmt.Fprintf(os.Stderr, "# get %s: %v\n", insecure.Redacted(), err)
   295  			}
   296  			// HTTP failed, and we already tried HTTPS if applicable.
   297  			// Report the error from the HTTP attempt.
   298  			return nil, err
   299  		}
   300  	}
   301  
   302  	// Note: accepting a non-200 OK here, so people can serve a
   303  	// meta import in their http 404 page.
   304  	if cfg.BuildX {
   305  		fmt.Fprintf(os.Stderr, "# get %s: %v (%.3fs)\n", fetched.Redacted(), res.Status, time.Since(start).Seconds())
   306  	}
   307  
   308  	r := &Response{
   309  		URL:        fetched.Redacted(),
   310  		Status:     res.Status,
   311  		StatusCode: res.StatusCode,
   312  		Header:     map[string][]string(res.Header),
   313  		Body:       res.Body,
   314  	}
   315  
   316  	if res.StatusCode != http.StatusOK {
   317  		contentType := res.Header.Get("Content-Type")
   318  		if mediaType, params, _ := mime.ParseMediaType(contentType); mediaType == "text/plain" {
   319  			switch charset := strings.ToLower(params["charset"]); charset {
   320  			case "us-ascii", "utf-8", "":
   321  				// Body claims to be plain text in UTF-8 or a subset thereof.
   322  				// Try to extract a useful error message from it.
   323  				r.errorDetail.r = res.Body
   324  				r.Body = &r.errorDetail
   325  			}
   326  		}
   327  	}
   328  
   329  	return r, nil
   330  }
   331  
   332  func getFile(u *urlpkg.URL) (*Response, error) {
   333  	path, err := urlToFilePath(u)
   334  	if err != nil {
   335  		return nil, err
   336  	}
   337  	f, err := os.Open(path)
   338  
   339  	if os.IsNotExist(err) {
   340  		return &Response{
   341  			URL:        u.Redacted(),
   342  			Status:     http.StatusText(http.StatusNotFound),
   343  			StatusCode: http.StatusNotFound,
   344  			Body:       http.NoBody,
   345  			fileErr:    err,
   346  		}, nil
   347  	}
   348  
   349  	if os.IsPermission(err) {
   350  		return &Response{
   351  			URL:        u.Redacted(),
   352  			Status:     http.StatusText(http.StatusForbidden),
   353  			StatusCode: http.StatusForbidden,
   354  			Body:       http.NoBody,
   355  			fileErr:    err,
   356  		}, nil
   357  	}
   358  
   359  	if err != nil {
   360  		return nil, err
   361  	}
   362  
   363  	return &Response{
   364  		URL:        u.Redacted(),
   365  		Status:     http.StatusText(http.StatusOK),
   366  		StatusCode: http.StatusOK,
   367  		Body:       f,
   368  	}, nil
   369  }
   370  
   371  func openBrowser(url string) bool { return browser.Open(url) }
   372  
   373  func isLocalHost(u *urlpkg.URL) bool {
   374  	// VCSTestRepoURL itself is secure, and it may redirect requests to other
   375  	// ports (such as a port serving the "svn" protocol) which should also be
   376  	// considered secure.
   377  	host, _, err := net.SplitHostPort(u.Host)
   378  	if err != nil {
   379  		host = u.Host
   380  	}
   381  	if host == "localhost" {
   382  		return true
   383  	}
   384  	if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() {
   385  		return true
   386  	}
   387  	return false
   388  }
   389  
   390  type hookCloser struct {
   391  	io.ReadCloser
   392  	afterClose func()
   393  }
   394  
   395  func (c hookCloser) Close() error {
   396  	err := c.ReadCloser.Close()
   397  	c.afterClose()
   398  	return err
   399  }
   400
View as plain text