sumdb.go

     1  // Copyright 2019 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Go checksum database lookup
     6  
     7  //go:build !cmd_go_bootstrap
     8  
     9  package modfetch
    10  
    11  import (
    12  	"bytes"
    13  	"errors"
    14  	"fmt"
    15  	"io"
    16  	"io/fs"
    17  	"net/url"
    18  	"os"
    19  	"path/filepath"
    20  	"strings"
    21  	"sync"
    22  	"time"
    23  
    24  	"cmd/go/internal/base"
    25  	"cmd/go/internal/cfg"
    26  	"cmd/go/internal/lockedfile"
    27  	"cmd/go/internal/web"
    28  
    29  	"golang.org/x/mod/module"
    30  	"golang.org/x/mod/sumdb"
    31  	"golang.org/x/mod/sumdb/note"
    32  )
    33  
    34  // useSumDB reports whether to use the Go checksum database for the given module.
    35  func useSumDB(mod module.Version) bool {
    36  	if mod.Path == "golang.org/toolchain" {
    37  		must := true
    38  		// Downloaded toolchains cannot be listed in go.sum,
    39  		// so we require checksum database lookups even if
    40  		// GOSUMDB=off or GONOSUMDB matches the pattern.
    41  		// If GOSUMDB=off, then the eventual lookup will fail
    42  		// with a good error message.
    43  
    44  		// Exception #1: using GOPROXY=file:// to test a distpack.
    45  		if strings.HasPrefix(cfg.GOPROXY, "file://") && !strings.ContainsAny(cfg.GOPROXY, ",|") {
    46  			must = false
    47  		}
    48  		// Exception #2: the Go proxy+checksum database cannot check itself
    49  		// while doing the initial download.
    50  		if strings.Contains(os.Getenv("GIT_HTTP_USER_AGENT"), "proxy.golang.org") {
    51  			must = false
    52  		}
    53  
    54  		// Another potential exception would be GOPROXY=direct,
    55  		// but that would make toolchain downloads only as secure
    56  		// as HTTPS, and in particular they'd be susceptible to MITM
    57  		// attacks on systems with less-than-trustworthy root certificates.
    58  		// The checksum database provides a stronger guarantee,
    59  		// so we don't make that exception.
    60  
    61  		// Otherwise, require the checksum database.
    62  		if must {
    63  			return true
    64  		}
    65  	}
    66  	return cfg.GOSUMDB != "off" && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path)
    67  }
    68  
    69  // lookupSumDB returns the Go checksum database's go.sum lines for the given module,
    70  // along with the name of the database.
    71  func lookupSumDB(mod module.Version) (dbname string, lines []string, err error) {
    72  	dbOnce.Do(func() {
    73  		dbName, db, dbErr = dbDial()
    74  	})
    75  	if dbErr != nil {
    76  		return "", nil, dbErr
    77  	}
    78  	lines, err = db.Lookup(mod.Path, mod.Version)
    79  	return dbName, lines, err
    80  }
    81  
    82  var (
    83  	dbOnce sync.Once
    84  	dbName string
    85  	db     *sumdb.Client
    86  	dbErr  error
    87  )
    88  
    89  func dbDial() (dbName string, db *sumdb.Client, err error) {
    90  	// $GOSUMDB can be "key" or "key url",
    91  	// and the key can be a full verifier key
    92  	// or a host on our list of known keys.
    93  
    94  	// Special case: sum.golang.google.cn
    95  	// is an alias, reachable inside mainland China,
    96  	// for sum.golang.org. If there are more
    97  	// of these we should add a map like knownGOSUMDB.
    98  	gosumdb := cfg.GOSUMDB
    99  	if gosumdb == "sum.golang.google.cn" {
   100  		gosumdb = "sum.golang.org https://sum.golang.google.cn"
   101  	}
   102  
   103  	if gosumdb == "off" {
   104  		return "", nil, fmt.Errorf("checksum database disabled by GOSUMDB=off")
   105  	}
   106  
   107  	key := strings.Fields(gosumdb)
   108  	if len(key) >= 1 {
   109  		if k := knownGOSUMDB[key[0]]; k != "" {
   110  			key[0] = k
   111  		}
   112  	}
   113  	if len(key) == 0 {
   114  		return "", nil, fmt.Errorf("missing GOSUMDB")
   115  	}
   116  	if len(key) > 2 {
   117  		return "", nil, fmt.Errorf("invalid GOSUMDB: too many fields")
   118  	}
   119  	vkey, err := note.NewVerifier(key[0])
   120  	if err != nil {
   121  		return "", nil, fmt.Errorf("invalid GOSUMDB: %v", err)
   122  	}
   123  	name := vkey.Name()
   124  
   125  	// No funny business in the database name.
   126  	direct, err := url.Parse("https://" + name)
   127  	if err != nil || strings.HasSuffix(name, "/") || *direct != (url.URL{Scheme: "https", Host: direct.Host, Path: direct.Path, RawPath: direct.RawPath}) || direct.RawPath != "" || direct.Host == "" {
   128  		return "", nil, fmt.Errorf("invalid sumdb name (must be host[/path]): %s %+v", name, *direct)
   129  	}
   130  
   131  	// Determine how to get to database.
   132  	var base *url.URL
   133  	if len(key) >= 2 {
   134  		// Use explicit alternate URL listed in $GOSUMDB,
   135  		// bypassing both the default URL derivation and any proxies.
   136  		u, err := url.Parse(key[1])
   137  		if err != nil {
   138  			return "", nil, fmt.Errorf("invalid GOSUMDB URL: %v", err)
   139  		}
   140  		base = u
   141  	}
   142  
   143  	return name, sumdb.NewClient(&dbClient{key: key[0], name: name, direct: direct, base: base}), nil
   144  }
   145  
   146  type dbClient struct {
   147  	key    string
   148  	name   string
   149  	direct *url.URL
   150  
   151  	once    sync.Once
   152  	base    *url.URL
   153  	baseErr error
   154  }
   155  
   156  func (c *dbClient) ReadRemote(path string) ([]byte, error) {
   157  	c.once.Do(c.initBase)
   158  	if c.baseErr != nil {
   159  		return nil, c.baseErr
   160  	}
   161  
   162  	var data []byte
   163  	start := time.Now()
   164  	targ := web.Join(c.base, path)
   165  	data, err := web.GetBytes(targ)
   166  	if false {
   167  		fmt.Fprintf(os.Stderr, "%.3fs %s\n", time.Since(start).Seconds(), targ.Redacted())
   168  	}
   169  	return data, err
   170  }
   171  
   172  // initBase determines the base URL for connecting to the database.
   173  // Determining the URL requires sending network traffic to proxies,
   174  // so this work is delayed until we need to download something from
   175  // the database. If everything we need is in the local cache and
   176  // c.ReadRemote is never called, we will never do this work.
   177  func (c *dbClient) initBase() {
   178  	if c.base != nil {
   179  		return
   180  	}
   181  
   182  	// Try proxies in turn until we find out how to connect to this database.
   183  	//
   184  	// Before accessing any checksum database URL using a proxy, the proxy
   185  	// client should first fetch <proxyURL>/sumdb/<sumdb-name>/supported.
   186  	//
   187  	// If that request returns a successful (HTTP 200) response, then the proxy
   188  	// supports proxying checksum database requests. In that case, the client
   189  	// should use the proxied access method only, never falling back to a direct
   190  	// connection to the database.
   191  	//
   192  	// If the /sumdb/<sumdb-name>/supported check fails with a “not found” (HTTP
   193  	// 404) or “gone” (HTTP 410) response, or if the proxy is configured to fall
   194  	// back on errors, the client will try the next proxy. If there are no
   195  	// proxies left or if the proxy is "direct" or "off", the client should
   196  	// connect directly to that database.
   197  	//
   198  	// Any other response is treated as the database being unavailable.
   199  	//
   200  	// See https://golang.org/design/25530-sumdb#proxying-a-checksum-database.
   201  	err := TryProxies(func(proxy string) error {
   202  		switch proxy {
   203  		case "noproxy":
   204  			return errUseProxy
   205  		case "direct", "off":
   206  			return errProxyOff
   207  		default:
   208  			proxyURL, err := url.Parse(proxy)
   209  			if err != nil {
   210  				return err
   211  			}
   212  			if _, err := web.GetBytes(web.Join(proxyURL, "sumdb/"+c.name+"/supported")); err != nil {
   213  				return err
   214  			}
   215  			// Success! This proxy will help us.
   216  			c.base = web.Join(proxyURL, "sumdb/"+c.name)
   217  			return nil
   218  		}
   219  	})
   220  	if errors.Is(err, fs.ErrNotExist) {
   221  		// No proxies, or all proxies failed (with 404, 410, or were allowed
   222  		// to fall back), or we reached an explicit "direct" or "off".
   223  		c.base = c.direct
   224  	} else if err != nil {
   225  		c.baseErr = err
   226  	}
   227  }
   228  
   229  // ReadConfig reads the key from c.key
   230  // and otherwise reads the config (a latest tree head) from GOPATH/pkg/sumdb/<file>.
   231  func (c *dbClient) ReadConfig(file string) (data []byte, err error) {
   232  	if file == "key" {
   233  		return []byte(c.key), nil
   234  	}
   235  
   236  	if cfg.SumdbDir == "" {
   237  		return nil, fmt.Errorf("could not locate sumdb file: missing $GOPATH: %s",
   238  			cfg.GoPathError)
   239  	}
   240  	targ := filepath.Join(cfg.SumdbDir, file)
   241  	data, err = lockedfile.Read(targ)
   242  	if errors.Is(err, fs.ErrNotExist) {
   243  		// Treat non-existent as empty, to bootstrap the "latest" file
   244  		// the first time we connect to a given database.
   245  		return []byte{}, nil
   246  	}
   247  	return data, err
   248  }
   249  
   250  // WriteConfig rewrites the latest tree head.
   251  func (*dbClient) WriteConfig(file string, old, new []byte) error {
   252  	if file == "key" {
   253  		// Should not happen.
   254  		return fmt.Errorf("cannot write key")
   255  	}
   256  	if cfg.SumdbDir == "" {
   257  		return fmt.Errorf("could not locate sumdb file: missing $GOPATH: %s",
   258  			cfg.GoPathError)
   259  	}
   260  	targ := filepath.Join(cfg.SumdbDir, file)
   261  	os.MkdirAll(filepath.Dir(targ), 0777)
   262  	f, err := lockedfile.Edit(targ)
   263  	if err != nil {
   264  		return err
   265  	}
   266  	defer f.Close()
   267  	data, err := io.ReadAll(f)
   268  	if err != nil {
   269  		return err
   270  	}
   271  	if len(data) > 0 && !bytes.Equal(data, old) {
   272  		return sumdb.ErrWriteConflict
   273  	}
   274  	if _, err := f.Seek(0, 0); err != nil {
   275  		return err
   276  	}
   277  	if err := f.Truncate(0); err != nil {
   278  		return err
   279  	}
   280  	if _, err := f.Write(new); err != nil {
   281  		return err
   282  	}
   283  	return f.Close()
   284  }
   285  
   286  // ReadCache reads cached lookups or tiles from
   287  // GOPATH/pkg/mod/cache/download/sumdb,
   288  // which will be deleted by "go clean -modcache".
   289  func (*dbClient) ReadCache(file string) ([]byte, error) {
   290  	targ := filepath.Join(cfg.GOMODCACHE, "cache/download/sumdb", file)
   291  	data, err := lockedfile.Read(targ)
   292  	// lockedfile.Write does not atomically create the file with contents.
   293  	// There is a moment between file creation and locking the file for writing,
   294  	// during which the empty file can be locked for reading.
   295  	// Treat observing an empty file as file not found.
   296  	if err == nil && len(data) == 0 {
   297  		err = &fs.PathError{Op: "read", Path: targ, Err: fs.ErrNotExist}
   298  	}
   299  	return data, err
   300  }
   301  
   302  // WriteCache updates cached lookups or tiles.
   303  func (*dbClient) WriteCache(file string, data []byte) {
   304  	targ := filepath.Join(cfg.GOMODCACHE, "cache/download/sumdb", file)
   305  	os.MkdirAll(filepath.Dir(targ), 0777)
   306  	lockedfile.Write(targ, bytes.NewReader(data), 0666)
   307  }
   308  
   309  func (*dbClient) Log(msg string) {
   310  	// nothing for now
   311  }
   312  
   313  func (*dbClient) SecurityError(msg string) {
   314  	base.Fatalf("%s", msg)
   315  }
   316
View as plain text