Source file src/pkg/net/http/transfer.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package http
     6	
     7	import (
     8		"bufio"
     9		"bytes"
    10		"errors"
    11		"fmt"
    12		"io"
    13		"io/ioutil"
    14		"net/textproto"
    15		"strconv"
    16		"strings"
    17	)
    18	
    19	// transferWriter inspects the fields of a user-supplied Request or Response,
    20	// sanitizes them without changing the user object and provides methods for
    21	// writing the respective header, body and trailer in wire format.
    22	type transferWriter struct {
    23		Method           string
    24		Body             io.Reader
    25		BodyCloser       io.Closer
    26		ResponseToHEAD   bool
    27		ContentLength    int64 // -1 means unknown, 0 means exactly none
    28		Close            bool
    29		TransferEncoding []string
    30		Trailer          Header
    31	}
    32	
    33	func newTransferWriter(r interface{}) (t *transferWriter, err error) {
    34		t = &transferWriter{}
    35	
    36		// Extract relevant fields
    37		atLeastHTTP11 := false
    38		switch rr := r.(type) {
    39		case *Request:
    40			if rr.ContentLength != 0 && rr.Body == nil {
    41				return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength)
    42			}
    43			t.Method = rr.Method
    44			t.Body = rr.Body
    45			t.BodyCloser = rr.Body
    46			t.ContentLength = rr.ContentLength
    47			t.Close = rr.Close
    48			t.TransferEncoding = rr.TransferEncoding
    49			t.Trailer = rr.Trailer
    50			atLeastHTTP11 = rr.ProtoAtLeast(1, 1)
    51			if t.Body != nil && len(t.TransferEncoding) == 0 && atLeastHTTP11 {
    52				if t.ContentLength == 0 {
    53					// Test to see if it's actually zero or just unset.
    54					var buf [1]byte
    55					n, _ := io.ReadFull(t.Body, buf[:])
    56					if n == 1 {
    57						// Oh, guess there is data in this Body Reader after all.
    58						// The ContentLength field just wasn't set.
    59						// Stich the Body back together again, re-attaching our
    60						// consumed byte.
    61						t.ContentLength = -1
    62						t.Body = io.MultiReader(bytes.NewBuffer(buf[:]), t.Body)
    63					} else {
    64						// Body is actually empty.
    65						t.Body = nil
    66						t.BodyCloser = nil
    67					}
    68				}
    69				if t.ContentLength < 0 {
    70					t.TransferEncoding = []string{"chunked"}
    71				}
    72			}
    73		case *Response:
    74			if rr.Request != nil {
    75				t.Method = rr.Request.Method
    76			}
    77			t.Body = rr.Body
    78			t.BodyCloser = rr.Body
    79			t.ContentLength = rr.ContentLength
    80			t.Close = rr.Close
    81			t.TransferEncoding = rr.TransferEncoding
    82			t.Trailer = rr.Trailer
    83			atLeastHTTP11 = rr.ProtoAtLeast(1, 1)
    84			t.ResponseToHEAD = noBodyExpected(t.Method)
    85		}
    86	
    87		// Sanitize Body,ContentLength,TransferEncoding
    88		if t.ResponseToHEAD {
    89			t.Body = nil
    90			if chunked(t.TransferEncoding) {
    91				t.ContentLength = -1
    92			}
    93		} else {
    94			if !atLeastHTTP11 || t.Body == nil {
    95				t.TransferEncoding = nil
    96			}
    97			if chunked(t.TransferEncoding) {
    98				t.ContentLength = -1
    99			} else if t.Body == nil { // no chunking, no body
   100				t.ContentLength = 0
   101			}
   102		}
   103	
   104		// Sanitize Trailer
   105		if !chunked(t.TransferEncoding) {
   106			t.Trailer = nil
   107		}
   108	
   109		return t, nil
   110	}
   111	
   112	func noBodyExpected(requestMethod string) bool {
   113		return requestMethod == "HEAD"
   114	}
   115	
   116	func (t *transferWriter) shouldSendContentLength() bool {
   117		if chunked(t.TransferEncoding) {
   118			return false
   119		}
   120		if t.ContentLength > 0 {
   121			return true
   122		}
   123		// Many servers expect a Content-Length for these methods
   124		if t.Method == "POST" || t.Method == "PUT" {
   125			return true
   126		}
   127		if t.ContentLength == 0 && isIdentity(t.TransferEncoding) {
   128			return true
   129		}
   130	
   131		return false
   132	}
   133	
   134	func (t *transferWriter) WriteHeader(w io.Writer) (err error) {
   135		if t.Close {
   136			_, err = io.WriteString(w, "Connection: close\r\n")
   137			if err != nil {
   138				return
   139			}
   140		}
   141	
   142		// Write Content-Length and/or Transfer-Encoding whose values are a
   143		// function of the sanitized field triple (Body, ContentLength,
   144		// TransferEncoding)
   145		if t.shouldSendContentLength() {
   146			io.WriteString(w, "Content-Length: ")
   147			_, err = io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n")
   148			if err != nil {
   149				return
   150			}
   151		} else if chunked(t.TransferEncoding) {
   152			_, err = io.WriteString(w, "Transfer-Encoding: chunked\r\n")
   153			if err != nil {
   154				return
   155			}
   156		}
   157	
   158		// Write Trailer header
   159		if t.Trailer != nil {
   160			// TODO: At some point, there should be a generic mechanism for
   161			// writing long headers, using HTTP line splitting
   162			io.WriteString(w, "Trailer: ")
   163			needComma := false
   164			for k := range t.Trailer {
   165				k = CanonicalHeaderKey(k)
   166				switch k {
   167				case "Transfer-Encoding", "Trailer", "Content-Length":
   168					return &badStringError{"invalid Trailer key", k}
   169				}
   170				if needComma {
   171					io.WriteString(w, ",")
   172				}
   173				io.WriteString(w, k)
   174				needComma = true
   175			}
   176			_, err = io.WriteString(w, "\r\n")
   177		}
   178	
   179		return
   180	}
   181	
   182	func (t *transferWriter) WriteBody(w io.Writer) (err error) {
   183		var ncopy int64
   184	
   185		// Write body
   186		if t.Body != nil {
   187			if chunked(t.TransferEncoding) {
   188				cw := newChunkedWriter(w)
   189				_, err = io.Copy(cw, t.Body)
   190				if err == nil {
   191					err = cw.Close()
   192				}
   193			} else if t.ContentLength == -1 {
   194				ncopy, err = io.Copy(w, t.Body)
   195			} else {
   196				ncopy, err = io.Copy(w, io.LimitReader(t.Body, t.ContentLength))
   197				if err != nil {
   198					return err
   199				}
   200				var nextra int64
   201				nextra, err = io.Copy(ioutil.Discard, t.Body)
   202				ncopy += nextra
   203			}
   204			if err != nil {
   205				return err
   206			}
   207			if err = t.BodyCloser.Close(); err != nil {
   208				return err
   209			}
   210		}
   211	
   212		if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy {
   213			return fmt.Errorf("http: Request.ContentLength=%d with Body length %d",
   214				t.ContentLength, ncopy)
   215		}
   216	
   217		// TODO(petar): Place trailer writer code here.
   218		if chunked(t.TransferEncoding) {
   219			// Last chunk, empty trailer
   220			_, err = io.WriteString(w, "\r\n")
   221		}
   222	
   223		return
   224	}
   225	
   226	type transferReader struct {
   227		// Input
   228		Header        Header
   229		StatusCode    int
   230		RequestMethod string
   231		ProtoMajor    int
   232		ProtoMinor    int
   233		// Output
   234		Body             io.ReadCloser
   235		ContentLength    int64
   236		TransferEncoding []string
   237		Close            bool
   238		Trailer          Header
   239	}
   240	
   241	// bodyAllowedForStatus returns whether a given response status code
   242	// permits a body.  See RFC2616, section 4.4.
   243	func bodyAllowedForStatus(status int) bool {
   244		switch {
   245		case status >= 100 && status <= 199:
   246			return false
   247		case status == 204:
   248			return false
   249		case status == 304:
   250			return false
   251		}
   252		return true
   253	}
   254	
   255	// msg is *Request or *Response.
   256	func readTransfer(msg interface{}, r *bufio.Reader) (err error) {
   257		t := &transferReader{}
   258	
   259		// Unify input
   260		isResponse := false
   261		switch rr := msg.(type) {
   262		case *Response:
   263			t.Header = rr.Header
   264			t.StatusCode = rr.StatusCode
   265			t.RequestMethod = rr.Request.Method
   266			t.ProtoMajor = rr.ProtoMajor
   267			t.ProtoMinor = rr.ProtoMinor
   268			t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header)
   269			isResponse = true
   270		case *Request:
   271			t.Header = rr.Header
   272			t.ProtoMajor = rr.ProtoMajor
   273			t.ProtoMinor = rr.ProtoMinor
   274			// Transfer semantics for Requests are exactly like those for
   275			// Responses with status code 200, responding to a GET method
   276			t.StatusCode = 200
   277			t.RequestMethod = "GET"
   278		default:
   279			panic("unexpected type")
   280		}
   281	
   282		// Default to HTTP/1.1
   283		if t.ProtoMajor == 0 && t.ProtoMinor == 0 {
   284			t.ProtoMajor, t.ProtoMinor = 1, 1
   285		}
   286	
   287		// Transfer encoding, content length
   288		t.TransferEncoding, err = fixTransferEncoding(t.RequestMethod, t.Header)
   289		if err != nil {
   290			return err
   291		}
   292	
   293		realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.TransferEncoding)
   294		if err != nil {
   295			return err
   296		}
   297		if isResponse && t.RequestMethod == "HEAD" {
   298			if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil {
   299				return err
   300			} else {
   301				t.ContentLength = n
   302			}
   303		} else {
   304			t.ContentLength = realLength
   305		}
   306	
   307		// Trailer
   308		t.Trailer, err = fixTrailer(t.Header, t.TransferEncoding)
   309		if err != nil {
   310			return err
   311		}
   312	
   313		// If there is no Content-Length or chunked Transfer-Encoding on a *Response
   314		// and the status is not 1xx, 204 or 304, then the body is unbounded.
   315		// See RFC2616, section 4.4.
   316		switch msg.(type) {
   317		case *Response:
   318			if realLength == -1 &&
   319				!chunked(t.TransferEncoding) &&
   320				bodyAllowedForStatus(t.StatusCode) {
   321				// Unbounded body.
   322				t.Close = true
   323			}
   324		}
   325	
   326		// Prepare body reader.  ContentLength < 0 means chunked encoding
   327		// or close connection when finished, since multipart is not supported yet
   328		switch {
   329		case chunked(t.TransferEncoding):
   330			if noBodyExpected(t.RequestMethod) {
   331				t.Body = &body{Reader: eofReader, closing: t.Close}
   332			} else {
   333				t.Body = &body{Reader: newChunkedReader(r), hdr: msg, r: r, closing: t.Close}
   334			}
   335		case realLength == 0:
   336			t.Body = &body{Reader: eofReader, closing: t.Close}
   337		case realLength > 0:
   338			t.Body = &body{Reader: io.LimitReader(r, realLength), closing: t.Close}
   339		default:
   340			// realLength < 0, i.e. "Content-Length" not mentioned in header
   341			if t.Close {
   342				// Close semantics (i.e. HTTP/1.0)
   343				t.Body = &body{Reader: r, closing: t.Close}
   344			} else {
   345				// Persistent connection (i.e. HTTP/1.1)
   346				t.Body = &body{Reader: eofReader, closing: t.Close}
   347			}
   348		}
   349	
   350		// Unify output
   351		switch rr := msg.(type) {
   352		case *Request:
   353			rr.Body = t.Body
   354			rr.ContentLength = t.ContentLength
   355			rr.TransferEncoding = t.TransferEncoding
   356			rr.Close = t.Close
   357			rr.Trailer = t.Trailer
   358		case *Response:
   359			rr.Body = t.Body
   360			rr.ContentLength = t.ContentLength
   361			rr.TransferEncoding = t.TransferEncoding
   362			rr.Close = t.Close
   363			rr.Trailer = t.Trailer
   364		}
   365	
   366		return nil
   367	}
   368	
   369	// Checks whether chunked is part of the encodings stack
   370	func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" }
   371	
   372	// Checks whether the encoding is explicitly "identity".
   373	func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" }
   374	
   375	// Sanitize transfer encoding
   376	func fixTransferEncoding(requestMethod string, header Header) ([]string, error) {
   377		raw, present := header["Transfer-Encoding"]
   378		if !present {
   379			return nil, nil
   380		}
   381	
   382		delete(header, "Transfer-Encoding")
   383	
   384		encodings := strings.Split(raw[0], ",")
   385		te := make([]string, 0, len(encodings))
   386		// TODO: Even though we only support "identity" and "chunked"
   387		// encodings, the loop below is designed with foresight. One
   388		// invariant that must be maintained is that, if present,
   389		// chunked encoding must always come first.
   390		for _, encoding := range encodings {
   391			encoding = strings.ToLower(strings.TrimSpace(encoding))
   392			// "identity" encoding is not recorded
   393			if encoding == "identity" {
   394				break
   395			}
   396			if encoding != "chunked" {
   397				return nil, &badStringError{"unsupported transfer encoding", encoding}
   398			}
   399			te = te[0 : len(te)+1]
   400			te[len(te)-1] = encoding
   401		}
   402		if len(te) > 1 {
   403			return nil, &badStringError{"too many transfer encodings", strings.Join(te, ",")}
   404		}
   405		if len(te) > 0 {
   406			// Chunked encoding trumps Content-Length. See RFC 2616
   407			// Section 4.4. Currently len(te) > 0 implies chunked
   408			// encoding.
   409			delete(header, "Content-Length")
   410			return te, nil
   411		}
   412	
   413		return nil, nil
   414	}
   415	
   416	// Determine the expected body length, using RFC 2616 Section 4.4. This
   417	// function is not a method, because ultimately it should be shared by
   418	// ReadResponse and ReadRequest.
   419	func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) {
   420	
   421		// Logic based on response type or status
   422		if noBodyExpected(requestMethod) {
   423			return 0, nil
   424		}
   425		if status/100 == 1 {
   426			return 0, nil
   427		}
   428		switch status {
   429		case 204, 304:
   430			return 0, nil
   431		}
   432	
   433		// Logic based on Transfer-Encoding
   434		if chunked(te) {
   435			return -1, nil
   436		}
   437	
   438		// Logic based on Content-Length
   439		cl := strings.TrimSpace(header.get("Content-Length"))
   440		if cl != "" {
   441			n, err := parseContentLength(cl)
   442			if err != nil {
   443				return -1, err
   444			}
   445			return n, nil
   446		} else {
   447			header.Del("Content-Length")
   448		}
   449	
   450		if !isResponse && requestMethod == "GET" {
   451			// RFC 2616 doesn't explicitly permit nor forbid an
   452			// entity-body on a GET request so we permit one if
   453			// declared, but we default to 0 here (not -1 below)
   454			// if there's no mention of a body.
   455			return 0, nil
   456		}
   457	
   458		// Body-EOF logic based on other methods (like closing, or chunked coding)
   459		return -1, nil
   460	}
   461	
   462	// Determine whether to hang up after sending a request and body, or
   463	// receiving a response and body
   464	// 'header' is the request headers
   465	func shouldClose(major, minor int, header Header) bool {
   466		if major < 1 {
   467			return true
   468		} else if major == 1 && minor == 0 {
   469			if !strings.Contains(strings.ToLower(header.get("Connection")), "keep-alive") {
   470				return true
   471			}
   472			return false
   473		} else {
   474			// TODO: Should split on commas, toss surrounding white space,
   475			// and check each field.
   476			if strings.ToLower(header.get("Connection")) == "close" {
   477				header.Del("Connection")
   478				return true
   479			}
   480		}
   481		return false
   482	}
   483	
   484	// Parse the trailer header
   485	func fixTrailer(header Header, te []string) (Header, error) {
   486		raw := header.get("Trailer")
   487		if raw == "" {
   488			return nil, nil
   489		}
   490	
   491		header.Del("Trailer")
   492		trailer := make(Header)
   493		keys := strings.Split(raw, ",")
   494		for _, key := range keys {
   495			key = CanonicalHeaderKey(strings.TrimSpace(key))
   496			switch key {
   497			case "Transfer-Encoding", "Trailer", "Content-Length":
   498				return nil, &badStringError{"bad trailer key", key}
   499			}
   500			trailer.Del(key)
   501		}
   502		if len(trailer) == 0 {
   503			return nil, nil
   504		}
   505		if !chunked(te) {
   506			// Trailer and no chunking
   507			return nil, ErrUnexpectedTrailer
   508		}
   509		return trailer, nil
   510	}
   511	
   512	// body turns a Reader into a ReadCloser.
   513	// Close ensures that the body has been fully read
   514	// and then reads the trailer if necessary.
   515	type body struct {
   516		io.Reader
   517		hdr     interface{}   // non-nil (Response or Request) value means read trailer
   518		r       *bufio.Reader // underlying wire-format reader for the trailer
   519		closing bool          // is the connection to be closed after reading body?
   520		closed  bool
   521	
   522		res *response // response writer for server requests, else nil
   523	}
   524	
   525	// ErrBodyReadAfterClose is returned when reading a Request or Response
   526	// Body after the body has been closed. This typically happens when the body is
   527	// read after an HTTP Handler calls WriteHeader or Write on its
   528	// ResponseWriter.
   529	var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body")
   530	
   531	func (b *body) Read(p []byte) (n int, err error) {
   532		if b.closed {
   533			return 0, ErrBodyReadAfterClose
   534		}
   535		n, err = b.Reader.Read(p)
   536	
   537		// Read the final trailer once we hit EOF.
   538		if err == io.EOF && b.hdr != nil {
   539			if e := b.readTrailer(); e != nil {
   540				err = e
   541			}
   542			b.hdr = nil
   543		}
   544		return n, err
   545	}
   546	
   547	var (
   548		singleCRLF = []byte("\r\n")
   549		doubleCRLF = []byte("\r\n\r\n")
   550	)
   551	
   552	func seeUpcomingDoubleCRLF(r *bufio.Reader) bool {
   553		for peekSize := 4; ; peekSize++ {
   554			// This loop stops when Peek returns an error,
   555			// which it does when r's buffer has been filled.
   556			buf, err := r.Peek(peekSize)
   557			if bytes.HasSuffix(buf, doubleCRLF) {
   558				return true
   559			}
   560			if err != nil {
   561				break
   562			}
   563		}
   564		return false
   565	}
   566	
   567	var errTrailerEOF = errors.New("http: unexpected EOF reading trailer")
   568	
   569	func (b *body) readTrailer() error {
   570		// The common case, since nobody uses trailers.
   571		buf, err := b.r.Peek(2)
   572		if bytes.Equal(buf, singleCRLF) {
   573			b.r.ReadByte()
   574			b.r.ReadByte()
   575			return nil
   576		}
   577		if len(buf) < 2 {
   578			return errTrailerEOF
   579		}
   580		if err != nil {
   581			return err
   582		}
   583	
   584		// Make sure there's a header terminator coming up, to prevent
   585		// a DoS with an unbounded size Trailer.  It's not easy to
   586		// slip in a LimitReader here, as textproto.NewReader requires
   587		// a concrete *bufio.Reader.  Also, we can't get all the way
   588		// back up to our conn's LimitedReader that *might* be backing
   589		// this bufio.Reader.  Instead, a hack: we iteratively Peek up
   590		// to the bufio.Reader's max size, looking for a double CRLF.
   591		// This limits the trailer to the underlying buffer size, typically 4kB.
   592		if !seeUpcomingDoubleCRLF(b.r) {
   593			return errors.New("http: suspiciously long trailer after chunked body")
   594		}
   595	
   596		hdr, err := textproto.NewReader(b.r).ReadMIMEHeader()
   597		if err != nil {
   598			if err == io.EOF {
   599				return errTrailerEOF
   600			}
   601			return err
   602		}
   603		switch rr := b.hdr.(type) {
   604		case *Request:
   605			rr.Trailer = Header(hdr)
   606		case *Response:
   607			rr.Trailer = Header(hdr)
   608		}
   609		return nil
   610	}
   611	
   612	func (b *body) Close() error {
   613		if b.closed {
   614			return nil
   615		}
   616		var err error
   617		switch {
   618		case b.hdr == nil && b.closing:
   619			// no trailer and closing the connection next.
   620			// no point in reading to EOF.
   621		case b.res != nil && b.res.requestBodyLimitHit:
   622			// In a server request, don't continue reading from the client
   623			// if we've already hit the maximum body size set by the
   624			// handler. If this is set, that also means the TCP connection
   625			// is about to be closed, so getting to the next HTTP request
   626			// in the stream is not necessary.
   627		case b.Reader == eofReader:
   628			// Nothing to read. No need to io.Copy from it.
   629		default:
   630			// Fully consume the body, which will also lead to us reading
   631			// the trailer headers after the body, if present.
   632			_, err = io.Copy(ioutil.Discard, b)
   633		}
   634		b.closed = true
   635		return err
   636	}
   637	
   638	// parseContentLength trims whitespace from s and returns -1 if no value
   639	// is set, or the value if it's >= 0.
   640	func parseContentLength(cl string) (int64, error) {
   641		cl = strings.TrimSpace(cl)
   642		if cl == "" {
   643			return -1, nil
   644		}
   645		n, err := strconv.ParseInt(cl, 10, 64)
   646		if err != nil || n < 0 {
   647			return 0, &badStringError{"bad Content-Length", cl}
   648		}
   649		return n, nil
   650	
   651	}

View as plain text