...
Run Format

Source file src/net/http/transfer.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package http
     6	
     7	import (
     8		"bufio"
     9		"bytes"
    10		"errors"
    11		"fmt"
    12		"io"
    13		"io/ioutil"
    14		"net/http/internal"
    15		"net/textproto"
    16		"sort"
    17		"strconv"
    18		"strings"
    19		"sync"
    20		"time"
    21	
    22		"golang_org/x/net/lex/httplex"
    23	)
    24	
    25	// ErrLineTooLong is returned when reading request or response bodies
    26	// with malformed chunked encoding.
    27	var ErrLineTooLong = internal.ErrLineTooLong
    28	
    29	type errorReader struct {
    30		err error
    31	}
    32	
    33	func (r errorReader) Read(p []byte) (n int, err error) {
    34		return 0, r.err
    35	}
    36	
    37	type byteReader struct {
    38		b    byte
    39		done bool
    40	}
    41	
    42	func (br *byteReader) Read(p []byte) (n int, err error) {
    43		if br.done {
    44			return 0, io.EOF
    45		}
    46		if len(p) == 0 {
    47			return 0, nil
    48		}
    49		br.done = true
    50		p[0] = br.b
    51		return 1, io.EOF
    52	}
    53	
    54	// transferWriter inspects the fields of a user-supplied Request or Response,
    55	// sanitizes them without changing the user object and provides methods for
    56	// writing the respective header, body and trailer in wire format.
    57	type transferWriter struct {
    58		Method           string
    59		Body             io.Reader
    60		BodyCloser       io.Closer
    61		ResponseToHEAD   bool
    62		ContentLength    int64 // -1 means unknown, 0 means exactly none
    63		Close            bool
    64		TransferEncoding []string
    65		Trailer          Header
    66		IsResponse       bool
    67	
    68		FlushHeaders bool            // flush headers to network before body
    69		ByteReadCh   chan readResult // non-nil if probeRequestBody called
    70	}
    71	
    72	func newTransferWriter(r interface{}) (t *transferWriter, err error) {
    73		t = &transferWriter{}
    74	
    75		// Extract relevant fields
    76		atLeastHTTP11 := false
    77		switch rr := r.(type) {
    78		case *Request:
    79			if rr.ContentLength != 0 && rr.Body == nil {
    80				return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength)
    81			}
    82			t.Method = valueOrDefault(rr.Method, "GET")
    83			t.Close = rr.Close
    84			t.TransferEncoding = rr.TransferEncoding
    85			t.Trailer = rr.Trailer
    86			atLeastHTTP11 = rr.protoAtLeastOutgoing(1, 1)
    87			t.Body = rr.Body
    88			t.BodyCloser = rr.Body
    89			t.ContentLength = rr.outgoingLength()
    90			if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && atLeastHTTP11 && t.shouldSendChunkedRequestBody() {
    91				t.TransferEncoding = []string{"chunked"}
    92			}
    93		case *Response:
    94			t.IsResponse = true
    95			if rr.Request != nil {
    96				t.Method = rr.Request.Method
    97			}
    98			t.Body = rr.Body
    99			t.BodyCloser = rr.Body
   100			t.ContentLength = rr.ContentLength
   101			t.Close = rr.Close
   102			t.TransferEncoding = rr.TransferEncoding
   103			t.Trailer = rr.Trailer
   104			atLeastHTTP11 = rr.ProtoAtLeast(1, 1)
   105			t.ResponseToHEAD = noResponseBodyExpected(t.Method)
   106		}
   107	
   108		// Sanitize Body,ContentLength,TransferEncoding
   109		if t.ResponseToHEAD {
   110			t.Body = nil
   111			if chunked(t.TransferEncoding) {
   112				t.ContentLength = -1
   113			}
   114		} else {
   115			if !atLeastHTTP11 || t.Body == nil {
   116				t.TransferEncoding = nil
   117			}
   118			if chunked(t.TransferEncoding) {
   119				t.ContentLength = -1
   120			} else if t.Body == nil { // no chunking, no body
   121				t.ContentLength = 0
   122			}
   123		}
   124	
   125		// Sanitize Trailer
   126		if !chunked(t.TransferEncoding) {
   127			t.Trailer = nil
   128		}
   129	
   130		return t, nil
   131	}
   132	
   133	// shouldSendChunkedRequestBody reports whether we should try to send a
   134	// chunked request body to the server. In particular, the case we really
   135	// want to prevent is sending a GET or other typically-bodyless request to a
   136	// server with a chunked body when the body has zero bytes, since GETs with
   137	// bodies (while acceptable according to specs), even zero-byte chunked
   138	// bodies, are approximately never seen in the wild and confuse most
   139	// servers. See Issue 18257, as one example.
   140	//
   141	// The only reason we'd send such a request is if the user set the Body to a
   142	// non-nil value (say, ioutil.NopCloser(bytes.NewReader(nil))) and didn't
   143	// set ContentLength, or NewRequest set it to -1 (unknown), so then we assume
   144	// there's bytes to send.
   145	//
   146	// This code tries to read a byte from the Request.Body in such cases to see
   147	// whether the body actually has content (super rare) or is actually just
   148	// a non-nil content-less ReadCloser (the more common case). In that more
   149	// common case, we act as if their Body were nil instead, and don't send
   150	// a body.
   151	func (t *transferWriter) shouldSendChunkedRequestBody() bool {
   152		// Note that t.ContentLength is the corrected content length
   153		// from rr.outgoingLength, so 0 actually means zero, not unknown.
   154		if t.ContentLength >= 0 || t.Body == nil { // redundant checks; caller did them
   155			return false
   156		}
   157		if requestMethodUsuallyLacksBody(t.Method) {
   158			// Only probe the Request.Body for GET/HEAD/DELETE/etc
   159			// requests, because it's only those types of requests
   160			// that confuse servers.
   161			t.probeRequestBody() // adjusts t.Body, t.ContentLength
   162			return t.Body != nil
   163		}
   164		// For all other request types (PUT, POST, PATCH, or anything
   165		// made-up we've never heard of), assume it's normal and the server
   166		// can deal with a chunked request body. Maybe we'll adjust this
   167		// later.
   168		return true
   169	}
   170	
   171	// probeRequestBody reads a byte from t.Body to see whether it's empty
   172	// (returns io.EOF right away).
   173	//
   174	// But because we've had problems with this blocking users in the past
   175	// (issue 17480) when the body is a pipe (perhaps waiting on the response
   176	// headers before the pipe is fed data), we need to be careful and bound how
   177	// long we wait for it. This delay will only affect users if all the following
   178	// are true:
   179	//   * the request body blocks
   180	//   * the content length is not set (or set to -1)
   181	//   * the method doesn't usually have a body (GET, HEAD, DELETE, ...)
   182	//   * there is no transfer-encoding=chunked already set.
   183	// In other words, this delay will not normally affect anybody, and there
   184	// are workarounds if it does.
   185	func (t *transferWriter) probeRequestBody() {
   186		t.ByteReadCh = make(chan readResult, 1)
   187		go func(body io.Reader) {
   188			var buf [1]byte
   189			var rres readResult
   190			rres.n, rres.err = body.Read(buf[:])
   191			if rres.n == 1 {
   192				rres.b = buf[0]
   193			}
   194			t.ByteReadCh <- rres
   195		}(t.Body)
   196		timer := time.NewTimer(200 * time.Millisecond)
   197		select {
   198		case rres := <-t.ByteReadCh:
   199			timer.Stop()
   200			if rres.n == 0 && rres.err == io.EOF {
   201				// It was empty.
   202				t.Body = nil
   203				t.ContentLength = 0
   204			} else if rres.n == 1 {
   205				if rres.err != nil {
   206					t.Body = io.MultiReader(&byteReader{b: rres.b}, errorReader{rres.err})
   207				} else {
   208					t.Body = io.MultiReader(&byteReader{b: rres.b}, t.Body)
   209				}
   210			} else if rres.err != nil {
   211				t.Body = errorReader{rres.err}
   212			}
   213		case <-timer.C:
   214			// Too slow. Don't wait. Read it later, and keep
   215			// assuming that this is ContentLength == -1
   216			// (unknown), which means we'll send a
   217			// "Transfer-Encoding: chunked" header.
   218			t.Body = io.MultiReader(finishAsyncByteRead{t}, t.Body)
   219			// Request that Request.Write flush the headers to the
   220			// network before writing the body, since our body may not
   221			// become readable until it's seen the response headers.
   222			t.FlushHeaders = true
   223		}
   224	}
   225	
   226	func noResponseBodyExpected(requestMethod string) bool {
   227		return requestMethod == "HEAD"
   228	}
   229	
   230	func (t *transferWriter) shouldSendContentLength() bool {
   231		if chunked(t.TransferEncoding) {
   232			return false
   233		}
   234		if t.ContentLength > 0 {
   235			return true
   236		}
   237		if t.ContentLength < 0 {
   238			return false
   239		}
   240		// Many servers expect a Content-Length for these methods
   241		if t.Method == "POST" || t.Method == "PUT" {
   242			return true
   243		}
   244		if t.ContentLength == 0 && isIdentity(t.TransferEncoding) {
   245			if t.Method == "GET" || t.Method == "HEAD" {
   246				return false
   247			}
   248			return true
   249		}
   250	
   251		return false
   252	}
   253	
   254	func (t *transferWriter) WriteHeader(w io.Writer) error {
   255		if t.Close {
   256			if _, err := io.WriteString(w, "Connection: close\r\n"); err != nil {
   257				return err
   258			}
   259		}
   260	
   261		// Write Content-Length and/or Transfer-Encoding whose values are a
   262		// function of the sanitized field triple (Body, ContentLength,
   263		// TransferEncoding)
   264		if t.shouldSendContentLength() {
   265			if _, err := io.WriteString(w, "Content-Length: "); err != nil {
   266				return err
   267			}
   268			if _, err := io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n"); err != nil {
   269				return err
   270			}
   271		} else if chunked(t.TransferEncoding) {
   272			if _, err := io.WriteString(w, "Transfer-Encoding: chunked\r\n"); err != nil {
   273				return err
   274			}
   275		}
   276	
   277		// Write Trailer header
   278		if t.Trailer != nil {
   279			keys := make([]string, 0, len(t.Trailer))
   280			for k := range t.Trailer {
   281				k = CanonicalHeaderKey(k)
   282				switch k {
   283				case "Transfer-Encoding", "Trailer", "Content-Length":
   284					return &badStringError{"invalid Trailer key", k}
   285				}
   286				keys = append(keys, k)
   287			}
   288			if len(keys) > 0 {
   289				sort.Strings(keys)
   290				// TODO: could do better allocation-wise here, but trailers are rare,
   291				// so being lazy for now.
   292				if _, err := io.WriteString(w, "Trailer: "+strings.Join(keys, ",")+"\r\n"); err != nil {
   293					return err
   294				}
   295			}
   296		}
   297	
   298		return nil
   299	}
   300	
   301	func (t *transferWriter) WriteBody(w io.Writer) error {
   302		var err error
   303		var ncopy int64
   304	
   305		// Write body
   306		if t.Body != nil {
   307			if chunked(t.TransferEncoding) {
   308				if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse {
   309					w = &internal.FlushAfterChunkWriter{Writer: bw}
   310				}
   311				cw := internal.NewChunkedWriter(w)
   312				_, err = io.Copy(cw, t.Body)
   313				if err == nil {
   314					err = cw.Close()
   315				}
   316			} else if t.ContentLength == -1 {
   317				ncopy, err = io.Copy(w, t.Body)
   318			} else {
   319				ncopy, err = io.Copy(w, io.LimitReader(t.Body, t.ContentLength))
   320				if err != nil {
   321					return err
   322				}
   323				var nextra int64
   324				nextra, err = io.Copy(ioutil.Discard, t.Body)
   325				ncopy += nextra
   326			}
   327			if err != nil {
   328				return err
   329			}
   330		}
   331		if t.BodyCloser != nil {
   332			if err := t.BodyCloser.Close(); err != nil {
   333				return err
   334			}
   335		}
   336	
   337		if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy {
   338			return fmt.Errorf("http: ContentLength=%d with Body length %d",
   339				t.ContentLength, ncopy)
   340		}
   341	
   342		if chunked(t.TransferEncoding) {
   343			// Write Trailer header
   344			if t.Trailer != nil {
   345				if err := t.Trailer.Write(w); err != nil {
   346					return err
   347				}
   348			}
   349			// Last chunk, empty trailer
   350			_, err = io.WriteString(w, "\r\n")
   351		}
   352		return err
   353	}
   354	
   355	type transferReader struct {
   356		// Input
   357		Header        Header
   358		StatusCode    int
   359		RequestMethod string
   360		ProtoMajor    int
   361		ProtoMinor    int
   362		// Output
   363		Body             io.ReadCloser
   364		ContentLength    int64
   365		TransferEncoding []string
   366		Close            bool
   367		Trailer          Header
   368	}
   369	
   370	func (t *transferReader) protoAtLeast(m, n int) bool {
   371		return t.ProtoMajor > m || (t.ProtoMajor == m && t.ProtoMinor >= n)
   372	}
   373	
   374	// bodyAllowedForStatus reports whether a given response status code
   375	// permits a body. See RFC 2616, section 4.4.
   376	func bodyAllowedForStatus(status int) bool {
   377		switch {
   378		case status >= 100 && status <= 199:
   379			return false
   380		case status == 204:
   381			return false
   382		case status == 304:
   383			return false
   384		}
   385		return true
   386	}
   387	
   388	var (
   389		suppressedHeaders304    = []string{"Content-Type", "Content-Length", "Transfer-Encoding"}
   390		suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"}
   391	)
   392	
   393	func suppressedHeaders(status int) []string {
   394		switch {
   395		case status == 304:
   396			// RFC 2616 section 10.3.5: "the response MUST NOT include other entity-headers"
   397			return suppressedHeaders304
   398		case !bodyAllowedForStatus(status):
   399			return suppressedHeadersNoBody
   400		}
   401		return nil
   402	}
   403	
   404	// msg is *Request or *Response.
   405	func readTransfer(msg interface{}, r *bufio.Reader) (err error) {
   406		t := &transferReader{RequestMethod: "GET"}
   407	
   408		// Unify input
   409		isResponse := false
   410		switch rr := msg.(type) {
   411		case *Response:
   412			t.Header = rr.Header
   413			t.StatusCode = rr.StatusCode
   414			t.ProtoMajor = rr.ProtoMajor
   415			t.ProtoMinor = rr.ProtoMinor
   416			t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true)
   417			isResponse = true
   418			if rr.Request != nil {
   419				t.RequestMethod = rr.Request.Method
   420			}
   421		case *Request:
   422			t.Header = rr.Header
   423			t.RequestMethod = rr.Method
   424			t.ProtoMajor = rr.ProtoMajor
   425			t.ProtoMinor = rr.ProtoMinor
   426			// Transfer semantics for Requests are exactly like those for
   427			// Responses with status code 200, responding to a GET method
   428			t.StatusCode = 200
   429			t.Close = rr.Close
   430		default:
   431			panic("unexpected type")
   432		}
   433	
   434		// Default to HTTP/1.1
   435		if t.ProtoMajor == 0 && t.ProtoMinor == 0 {
   436			t.ProtoMajor, t.ProtoMinor = 1, 1
   437		}
   438	
   439		// Transfer encoding, content length
   440		err = t.fixTransferEncoding()
   441		if err != nil {
   442			return err
   443		}
   444	
   445		realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.TransferEncoding)
   446		if err != nil {
   447			return err
   448		}
   449		if isResponse && t.RequestMethod == "HEAD" {
   450			if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil {
   451				return err
   452			} else {
   453				t.ContentLength = n
   454			}
   455		} else {
   456			t.ContentLength = realLength
   457		}
   458	
   459		// Trailer
   460		t.Trailer, err = fixTrailer(t.Header, t.TransferEncoding)
   461		if err != nil {
   462			return err
   463		}
   464	
   465		// If there is no Content-Length or chunked Transfer-Encoding on a *Response
   466		// and the status is not 1xx, 204 or 304, then the body is unbounded.
   467		// See RFC 2616, section 4.4.
   468		switch msg.(type) {
   469		case *Response:
   470			if realLength == -1 &&
   471				!chunked(t.TransferEncoding) &&
   472				bodyAllowedForStatus(t.StatusCode) {
   473				// Unbounded body.
   474				t.Close = true
   475			}
   476		}
   477	
   478		// Prepare body reader. ContentLength < 0 means chunked encoding
   479		// or close connection when finished, since multipart is not supported yet
   480		switch {
   481		case chunked(t.TransferEncoding):
   482			if noResponseBodyExpected(t.RequestMethod) {
   483				t.Body = NoBody
   484			} else {
   485				t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close}
   486			}
   487		case realLength == 0:
   488			t.Body = NoBody
   489		case realLength > 0:
   490			t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close}
   491		default:
   492			// realLength < 0, i.e. "Content-Length" not mentioned in header
   493			if t.Close {
   494				// Close semantics (i.e. HTTP/1.0)
   495				t.Body = &body{src: r, closing: t.Close}
   496			} else {
   497				// Persistent connection (i.e. HTTP/1.1)
   498				t.Body = NoBody
   499			}
   500		}
   501	
   502		// Unify output
   503		switch rr := msg.(type) {
   504		case *Request:
   505			rr.Body = t.Body
   506			rr.ContentLength = t.ContentLength
   507			rr.TransferEncoding = t.TransferEncoding
   508			rr.Close = t.Close
   509			rr.Trailer = t.Trailer
   510		case *Response:
   511			rr.Body = t.Body
   512			rr.ContentLength = t.ContentLength
   513			rr.TransferEncoding = t.TransferEncoding
   514			rr.Close = t.Close
   515			rr.Trailer = t.Trailer
   516		}
   517	
   518		return nil
   519	}
   520	
   521	// Checks whether chunked is part of the encodings stack
   522	func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" }
   523	
   524	// Checks whether the encoding is explicitly "identity".
   525	func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" }
   526	
   527	// fixTransferEncoding sanitizes t.TransferEncoding, if needed.
   528	func (t *transferReader) fixTransferEncoding() error {
   529		raw, present := t.Header["Transfer-Encoding"]
   530		if !present {
   531			return nil
   532		}
   533		delete(t.Header, "Transfer-Encoding")
   534	
   535		// Issue 12785; ignore Transfer-Encoding on HTTP/1.0 requests.
   536		if !t.protoAtLeast(1, 1) {
   537			return nil
   538		}
   539	
   540		encodings := strings.Split(raw[0], ",")
   541		te := make([]string, 0, len(encodings))
   542		// TODO: Even though we only support "identity" and "chunked"
   543		// encodings, the loop below is designed with foresight. One
   544		// invariant that must be maintained is that, if present,
   545		// chunked encoding must always come first.
   546		for _, encoding := range encodings {
   547			encoding = strings.ToLower(strings.TrimSpace(encoding))
   548			// "identity" encoding is not recorded
   549			if encoding == "identity" {
   550				break
   551			}
   552			if encoding != "chunked" {
   553				return &badStringError{"unsupported transfer encoding", encoding}
   554			}
   555			te = te[0 : len(te)+1]
   556			te[len(te)-1] = encoding
   557		}
   558		if len(te) > 1 {
   559			return &badStringError{"too many transfer encodings", strings.Join(te, ",")}
   560		}
   561		if len(te) > 0 {
   562			// RFC 7230 3.3.2 says "A sender MUST NOT send a
   563			// Content-Length header field in any message that
   564			// contains a Transfer-Encoding header field."
   565			//
   566			// but also:
   567			// "If a message is received with both a
   568			// Transfer-Encoding and a Content-Length header
   569			// field, the Transfer-Encoding overrides the
   570			// Content-Length. Such a message might indicate an
   571			// attempt to perform request smuggling (Section 9.5)
   572			// or response splitting (Section 9.4) and ought to be
   573			// handled as an error. A sender MUST remove the
   574			// received Content-Length field prior to forwarding
   575			// such a message downstream."
   576			//
   577			// Reportedly, these appear in the wild.
   578			delete(t.Header, "Content-Length")
   579			t.TransferEncoding = te
   580			return nil
   581		}
   582	
   583		return nil
   584	}
   585	
   586	// Determine the expected body length, using RFC 2616 Section 4.4. This
   587	// function is not a method, because ultimately it should be shared by
   588	// ReadResponse and ReadRequest.
   589	func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) {
   590		isRequest := !isResponse
   591		contentLens := header["Content-Length"]
   592	
   593		// Hardening against HTTP request smuggling
   594		if len(contentLens) > 1 {
   595			// Per RFC 7230 Section 3.3.2, prevent multiple
   596			// Content-Length headers if they differ in value.
   597			// If there are dups of the value, remove the dups.
   598			// See Issue 16490.
   599			first := strings.TrimSpace(contentLens[0])
   600			for _, ct := range contentLens[1:] {
   601				if first != strings.TrimSpace(ct) {
   602					return 0, fmt.Errorf("http: message cannot contain multiple Content-Length headers; got %q", contentLens)
   603				}
   604			}
   605	
   606			// deduplicate Content-Length
   607			header.Del("Content-Length")
   608			header.Add("Content-Length", first)
   609	
   610			contentLens = header["Content-Length"]
   611		}
   612	
   613		// Logic based on response type or status
   614		if noResponseBodyExpected(requestMethod) {
   615			// For HTTP requests, as part of hardening against request
   616			// smuggling (RFC 7230), don't allow a Content-Length header for
   617			// methods which don't permit bodies. As an exception, allow
   618			// exactly one Content-Length header if its value is "0".
   619			if isRequest && len(contentLens) > 0 && !(len(contentLens) == 1 && contentLens[0] == "0") {
   620				return 0, fmt.Errorf("http: method cannot contain a Content-Length; got %q", contentLens)
   621			}
   622			return 0, nil
   623		}
   624		if status/100 == 1 {
   625			return 0, nil
   626		}
   627		switch status {
   628		case 204, 304:
   629			return 0, nil
   630		}
   631	
   632		// Logic based on Transfer-Encoding
   633		if chunked(te) {
   634			return -1, nil
   635		}
   636	
   637		// Logic based on Content-Length
   638		var cl string
   639		if len(contentLens) == 1 {
   640			cl = strings.TrimSpace(contentLens[0])
   641		}
   642		if cl != "" {
   643			n, err := parseContentLength(cl)
   644			if err != nil {
   645				return -1, err
   646			}
   647			return n, nil
   648		} else {
   649			header.Del("Content-Length")
   650		}
   651	
   652		if isRequest {
   653			// RFC 2616 neither explicitly permits nor forbids an
   654			// entity-body on a GET request so we permit one if
   655			// declared, but we default to 0 here (not -1 below)
   656			// if there's no mention of a body.
   657			// Likewise, all other request methods are assumed to have
   658			// no body if neither Transfer-Encoding chunked nor a
   659			// Content-Length are set.
   660			return 0, nil
   661		}
   662	
   663		// Body-EOF logic based on other methods (like closing, or chunked coding)
   664		return -1, nil
   665	}
   666	
   667	// Determine whether to hang up after sending a request and body, or
   668	// receiving a response and body
   669	// 'header' is the request headers
   670	func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool {
   671		if major < 1 {
   672			return true
   673		}
   674	
   675		conv := header["Connection"]
   676		hasClose := httplex.HeaderValuesContainsToken(conv, "close")
   677		if major == 1 && minor == 0 {
   678			return hasClose || !httplex.HeaderValuesContainsToken(conv, "keep-alive")
   679		}
   680	
   681		if hasClose && removeCloseHeader {
   682			header.Del("Connection")
   683		}
   684	
   685		return hasClose
   686	}
   687	
   688	// Parse the trailer header
   689	func fixTrailer(header Header, te []string) (Header, error) {
   690		vv, ok := header["Trailer"]
   691		if !ok {
   692			return nil, nil
   693		}
   694		header.Del("Trailer")
   695	
   696		trailer := make(Header)
   697		var err error
   698		for _, v := range vv {
   699			foreachHeaderElement(v, func(key string) {
   700				key = CanonicalHeaderKey(key)
   701				switch key {
   702				case "Transfer-Encoding", "Trailer", "Content-Length":
   703					if err == nil {
   704						err = &badStringError{"bad trailer key", key}
   705						return
   706					}
   707				}
   708				trailer[key] = nil
   709			})
   710		}
   711		if err != nil {
   712			return nil, err
   713		}
   714		if len(trailer) == 0 {
   715			return nil, nil
   716		}
   717		if !chunked(te) {
   718			// Trailer and no chunking
   719			return nil, ErrUnexpectedTrailer
   720		}
   721		return trailer, nil
   722	}
   723	
   724	// body turns a Reader into a ReadCloser.
   725	// Close ensures that the body has been fully read
   726	// and then reads the trailer if necessary.
   727	type body struct {
   728		src          io.Reader
   729		hdr          interface{}   // non-nil (Response or Request) value means read trailer
   730		r            *bufio.Reader // underlying wire-format reader for the trailer
   731		closing      bool          // is the connection to be closed after reading body?
   732		doEarlyClose bool          // whether Close should stop early
   733	
   734		mu         sync.Mutex // guards following, and calls to Read and Close
   735		sawEOF     bool
   736		closed     bool
   737		earlyClose bool   // Close called and we didn't read to the end of src
   738		onHitEOF   func() // if non-nil, func to call when EOF is Read
   739	}
   740	
   741	// ErrBodyReadAfterClose is returned when reading a Request or Response
   742	// Body after the body has been closed. This typically happens when the body is
   743	// read after an HTTP Handler calls WriteHeader or Write on its
   744	// ResponseWriter.
   745	var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body")
   746	
   747	func (b *body) Read(p []byte) (n int, err error) {
   748		b.mu.Lock()
   749		defer b.mu.Unlock()
   750		if b.closed {
   751			return 0, ErrBodyReadAfterClose
   752		}
   753		return b.readLocked(p)
   754	}
   755	
   756	// Must hold b.mu.
   757	func (b *body) readLocked(p []byte) (n int, err error) {
   758		if b.sawEOF {
   759			return 0, io.EOF
   760		}
   761		n, err = b.src.Read(p)
   762	
   763		if err == io.EOF {
   764			b.sawEOF = true
   765			// Chunked case. Read the trailer.
   766			if b.hdr != nil {
   767				if e := b.readTrailer(); e != nil {
   768					err = e
   769					// Something went wrong in the trailer, we must not allow any
   770					// further reads of any kind to succeed from body, nor any
   771					// subsequent requests on the server connection. See
   772					// golang.org/issue/12027
   773					b.sawEOF = false
   774					b.closed = true
   775				}
   776				b.hdr = nil
   777			} else {
   778				// If the server declared the Content-Length, our body is a LimitedReader
   779				// and we need to check whether this EOF arrived early.
   780				if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 {
   781					err = io.ErrUnexpectedEOF
   782				}
   783			}
   784		}
   785	
   786		// If we can return an EOF here along with the read data, do
   787		// so. This is optional per the io.Reader contract, but doing
   788		// so helps the HTTP transport code recycle its connection
   789		// earlier (since it will see this EOF itself), even if the
   790		// client doesn't do future reads or Close.
   791		if err == nil && n > 0 {
   792			if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 {
   793				err = io.EOF
   794				b.sawEOF = true
   795			}
   796		}
   797	
   798		if b.sawEOF && b.onHitEOF != nil {
   799			b.onHitEOF()
   800		}
   801	
   802		return n, err
   803	}
   804	
   805	var (
   806		singleCRLF = []byte("\r\n")
   807		doubleCRLF = []byte("\r\n\r\n")
   808	)
   809	
   810	func seeUpcomingDoubleCRLF(r *bufio.Reader) bool {
   811		for peekSize := 4; ; peekSize++ {
   812			// This loop stops when Peek returns an error,
   813			// which it does when r's buffer has been filled.
   814			buf, err := r.Peek(peekSize)
   815			if bytes.HasSuffix(buf, doubleCRLF) {
   816				return true
   817			}
   818			if err != nil {
   819				break
   820			}
   821		}
   822		return false
   823	}
   824	
   825	var errTrailerEOF = errors.New("http: unexpected EOF reading trailer")
   826	
   827	func (b *body) readTrailer() error {
   828		// The common case, since nobody uses trailers.
   829		buf, err := b.r.Peek(2)
   830		if bytes.Equal(buf, singleCRLF) {
   831			b.r.Discard(2)
   832			return nil
   833		}
   834		if len(buf) < 2 {
   835			return errTrailerEOF
   836		}
   837		if err != nil {
   838			return err
   839		}
   840	
   841		// Make sure there's a header terminator coming up, to prevent
   842		// a DoS with an unbounded size Trailer. It's not easy to
   843		// slip in a LimitReader here, as textproto.NewReader requires
   844		// a concrete *bufio.Reader. Also, we can't get all the way
   845		// back up to our conn's LimitedReader that *might* be backing
   846		// this bufio.Reader. Instead, a hack: we iteratively Peek up
   847		// to the bufio.Reader's max size, looking for a double CRLF.
   848		// This limits the trailer to the underlying buffer size, typically 4kB.
   849		if !seeUpcomingDoubleCRLF(b.r) {
   850			return errors.New("http: suspiciously long trailer after chunked body")
   851		}
   852	
   853		hdr, err := textproto.NewReader(b.r).ReadMIMEHeader()
   854		if err != nil {
   855			if err == io.EOF {
   856				return errTrailerEOF
   857			}
   858			return err
   859		}
   860		switch rr := b.hdr.(type) {
   861		case *Request:
   862			mergeSetHeader(&rr.Trailer, Header(hdr))
   863		case *Response:
   864			mergeSetHeader(&rr.Trailer, Header(hdr))
   865		}
   866		return nil
   867	}
   868	
   869	func mergeSetHeader(dst *Header, src Header) {
   870		if *dst == nil {
   871			*dst = src
   872			return
   873		}
   874		for k, vv := range src {
   875			(*dst)[k] = vv
   876		}
   877	}
   878	
   879	// unreadDataSizeLocked returns the number of bytes of unread input.
   880	// It returns -1 if unknown.
   881	// b.mu must be held.
   882	func (b *body) unreadDataSizeLocked() int64 {
   883		if lr, ok := b.src.(*io.LimitedReader); ok {
   884			return lr.N
   885		}
   886		return -1
   887	}
   888	
   889	func (b *body) Close() error {
   890		b.mu.Lock()
   891		defer b.mu.Unlock()
   892		if b.closed {
   893			return nil
   894		}
   895		var err error
   896		switch {
   897		case b.sawEOF:
   898			// Already saw EOF, so no need going to look for it.
   899		case b.hdr == nil && b.closing:
   900			// no trailer and closing the connection next.
   901			// no point in reading to EOF.
   902		case b.doEarlyClose:
   903			// Read up to maxPostHandlerReadBytes bytes of the body, looking for
   904			// for EOF (and trailers), so we can re-use this connection.
   905			if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > maxPostHandlerReadBytes {
   906				// There was a declared Content-Length, and we have more bytes remaining
   907				// than our maxPostHandlerReadBytes tolerance. So, give up.
   908				b.earlyClose = true
   909			} else {
   910				var n int64
   911				// Consume the body, or, which will also lead to us reading
   912				// the trailer headers after the body, if present.
   913				n, err = io.CopyN(ioutil.Discard, bodyLocked{b}, maxPostHandlerReadBytes)
   914				if err == io.EOF {
   915					err = nil
   916				}
   917				if n == maxPostHandlerReadBytes {
   918					b.earlyClose = true
   919				}
   920			}
   921		default:
   922			// Fully consume the body, which will also lead to us reading
   923			// the trailer headers after the body, if present.
   924			_, err = io.Copy(ioutil.Discard, bodyLocked{b})
   925		}
   926		b.closed = true
   927		return err
   928	}
   929	
   930	func (b *body) didEarlyClose() bool {
   931		b.mu.Lock()
   932		defer b.mu.Unlock()
   933		return b.earlyClose
   934	}
   935	
   936	// bodyRemains reports whether future Read calls might
   937	// yield data.
   938	func (b *body) bodyRemains() bool {
   939		b.mu.Lock()
   940		defer b.mu.Unlock()
   941		return !b.sawEOF
   942	}
   943	
   944	func (b *body) registerOnHitEOF(fn func()) {
   945		b.mu.Lock()
   946		defer b.mu.Unlock()
   947		b.onHitEOF = fn
   948	}
   949	
   950	// bodyLocked is a io.Reader reading from a *body when its mutex is
   951	// already held.
   952	type bodyLocked struct {
   953		b *body
   954	}
   955	
   956	func (bl bodyLocked) Read(p []byte) (n int, err error) {
   957		if bl.b.closed {
   958			return 0, ErrBodyReadAfterClose
   959		}
   960		return bl.b.readLocked(p)
   961	}
   962	
   963	// parseContentLength trims whitespace from s and returns -1 if no value
   964	// is set, or the value if it's >= 0.
   965	func parseContentLength(cl string) (int64, error) {
   966		cl = strings.TrimSpace(cl)
   967		if cl == "" {
   968			return -1, nil
   969		}
   970		n, err := strconv.ParseInt(cl, 10, 64)
   971		if err != nil || n < 0 {
   972			return 0, &badStringError{"bad Content-Length", cl}
   973		}
   974		return n, nil
   975	
   976	}
   977	
   978	// finishAsyncByteRead finishes reading the 1-byte sniff
   979	// from the ContentLength==0, Body!=nil case.
   980	type finishAsyncByteRead struct {
   981		tw *transferWriter
   982	}
   983	
   984	func (fr finishAsyncByteRead) Read(p []byte) (n int, err error) {
   985		if len(p) == 0 {
   986			return
   987		}
   988		rres := <-fr.tw.ByteReadCh
   989		n, err = rres.n, rres.err
   990		if n == 1 {
   991			p[0] = rres.b
   992		}
   993		return
   994	}
   995	

View as plain text