Run Format

Source file src/pkg/net/http/transfer.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package http
     6	
     7	import (
     8		"bufio"
     9		"bytes"
    10		"errors"
    11		"fmt"
    12		"io"
    13		"io/ioutil"
    14		"net/textproto"
    15		"strconv"
    16		"strings"
    17	)
    18	
    19	// transferWriter inspects the fields of a user-supplied Request or Response,
    20	// sanitizes them without changing the user object and provides methods for
    21	// writing the respective header, body and trailer in wire format.
    22	type transferWriter struct {
    23		Method           string
    24		Body             io.Reader
    25		BodyCloser       io.Closer
    26		ResponseToHEAD   bool
    27		ContentLength    int64 // -1 means unknown, 0 means exactly none
    28		Close            bool
    29		TransferEncoding []string
    30		Trailer          Header
    31	}
    32	
    33	func newTransferWriter(r interface{}) (t *transferWriter, err error) {
    34		t = &transferWriter{}
    35	
    36		// Extract relevant fields
    37		atLeastHTTP11 := false
    38		switch rr := r.(type) {
    39		case *Request:
    40			if rr.ContentLength != 0 && rr.Body == nil {
    41				return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength)
    42			}
    43			t.Method = rr.Method
    44			t.Body = rr.Body
    45			t.BodyCloser = rr.Body
    46			t.ContentLength = rr.ContentLength
    47			t.Close = rr.Close
    48			t.TransferEncoding = rr.TransferEncoding
    49			t.Trailer = rr.Trailer
    50			atLeastHTTP11 = rr.ProtoAtLeast(1, 1)
    51			if t.Body != nil && len(t.TransferEncoding) == 0 && atLeastHTTP11 {
    52				if t.ContentLength == 0 {
    53					// Test to see if it's actually zero or just unset.
    54					var buf [1]byte
    55					n, _ := io.ReadFull(t.Body, buf[:])
    56					if n == 1 {
    57						// Oh, guess there is data in this Body Reader after all.
    58						// The ContentLength field just wasn't set.
    59						// Stich the Body back together again, re-attaching our
    60						// consumed byte.
    61						t.ContentLength = -1
    62						t.Body = io.MultiReader(bytes.NewBuffer(buf[:]), t.Body)
    63					} else {
    64						// Body is actually empty.
    65						t.Body = nil
    66						t.BodyCloser = nil
    67					}
    68				}
    69				if t.ContentLength < 0 {
    70					t.TransferEncoding = []string{"chunked"}
    71				}
    72			}
    73		case *Response:
    74			if rr.Request != nil {
    75				t.Method = rr.Request.Method
    76			}
    77			t.Body = rr.Body
    78			t.BodyCloser = rr.Body
    79			t.ContentLength = rr.ContentLength
    80			t.Close = rr.Close
    81			t.TransferEncoding = rr.TransferEncoding
    82			t.Trailer = rr.Trailer
    83			atLeastHTTP11 = rr.ProtoAtLeast(1, 1)
    84			t.ResponseToHEAD = noBodyExpected(t.Method)
    85		}
    86	
    87		// Sanitize Body,ContentLength,TransferEncoding
    88		if t.ResponseToHEAD {
    89			t.Body = nil
    90			if chunked(t.TransferEncoding) {
    91				t.ContentLength = -1
    92			}
    93		} else {
    94			if !atLeastHTTP11 || t.Body == nil {
    95				t.TransferEncoding = nil
    96			}
    97			if chunked(t.TransferEncoding) {
    98				t.ContentLength = -1
    99			} else if t.Body == nil { // no chunking, no body
   100				t.ContentLength = 0
   101			}
   102		}
   103	
   104		// Sanitize Trailer
   105		if !chunked(t.TransferEncoding) {
   106			t.Trailer = nil
   107		}
   108	
   109		return t, nil
   110	}
   111	
   112	func noBodyExpected(requestMethod string) bool {
   113		return requestMethod == "HEAD"
   114	}
   115	
   116	func (t *transferWriter) shouldSendContentLength() bool {
   117		if chunked(t.TransferEncoding) {
   118			return false
   119		}
   120		if t.ContentLength > 0 {
   121			return true
   122		}
   123		// Many servers expect a Content-Length for these methods
   124		if t.Method == "POST" || t.Method == "PUT" {
   125			return true
   126		}
   127		if t.ContentLength == 0 && isIdentity(t.TransferEncoding) {
   128			return true
   129		}
   130	
   131		return false
   132	}
   133	
   134	func (t *transferWriter) WriteHeader(w io.Writer) (err error) {
   135		if t.Close {
   136			_, err = io.WriteString(w, "Connection: close\r\n")
   137			if err != nil {
   138				return
   139			}
   140		}
   141	
   142		// Write Content-Length and/or Transfer-Encoding whose values are a
   143		// function of the sanitized field triple (Body, ContentLength,
   144		// TransferEncoding)
   145		if t.shouldSendContentLength() {
   146			io.WriteString(w, "Content-Length: ")
   147			_, err = io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n")
   148			if err != nil {
   149				return
   150			}
   151		} else if chunked(t.TransferEncoding) {
   152			_, err = io.WriteString(w, "Transfer-Encoding: chunked\r\n")
   153			if err != nil {
   154				return
   155			}
   156		}
   157	
   158		// Write Trailer header
   159		if t.Trailer != nil {
   160			// TODO: At some point, there should be a generic mechanism for
   161			// writing long headers, using HTTP line splitting
   162			io.WriteString(w, "Trailer: ")
   163			needComma := false
   164			for k := range t.Trailer {
   165				k = CanonicalHeaderKey(k)
   166				switch k {
   167				case "Transfer-Encoding", "Trailer", "Content-Length":
   168					return &badStringError{"invalid Trailer key", k}
   169				}
   170				if needComma {
   171					io.WriteString(w, ",")
   172				}
   173				io.WriteString(w, k)
   174				needComma = true
   175			}
   176			_, err = io.WriteString(w, "\r\n")
   177		}
   178	
   179		return
   180	}
   181	
   182	func (t *transferWriter) WriteBody(w io.Writer) (err error) {
   183		var ncopy int64
   184	
   185		// Write body
   186		if t.Body != nil {
   187			if chunked(t.TransferEncoding) {
   188				cw := newChunkedWriter(w)
   189				_, err = io.Copy(cw, t.Body)
   190				if err == nil {
   191					err = cw.Close()
   192				}
   193			} else if t.ContentLength == -1 {
   194				ncopy, err = io.Copy(w, t.Body)
   195			} else {
   196				ncopy, err = io.Copy(w, io.LimitReader(t.Body, t.ContentLength))
   197				if err != nil {
   198					return err
   199				}
   200				var nextra int64
   201				nextra, err = io.Copy(ioutil.Discard, t.Body)
   202				ncopy += nextra
   203			}
   204			if err != nil {
   205				return err
   206			}
   207			if err = t.BodyCloser.Close(); err != nil {
   208				return err
   209			}
   210		}
   211	
   212		if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy {
   213			return fmt.Errorf("http: Request.ContentLength=%d with Body length %d",
   214				t.ContentLength, ncopy)
   215		}
   216	
   217		// TODO(petar): Place trailer writer code here.
   218		if chunked(t.TransferEncoding) {
   219			// Last chunk, empty trailer
   220			_, err = io.WriteString(w, "\r\n")
   221		}
   222	
   223		return
   224	}
   225	
   226	type transferReader struct {
   227		// Input
   228		Header        Header
   229		StatusCode    int
   230		RequestMethod string
   231		ProtoMajor    int
   232		ProtoMinor    int
   233		// Output
   234		Body             io.ReadCloser
   235		ContentLength    int64
   236		TransferEncoding []string
   237		Close            bool
   238		Trailer          Header
   239	}
   240	
   241	// bodyAllowedForStatus reports whether a given response status code
   242	// permits a body.  See RFC2616, section 4.4.
   243	func bodyAllowedForStatus(status int) bool {
   244		switch {
   245		case status >= 100 && status <= 199:
   246			return false
   247		case status == 204:
   248			return false
   249		case status == 304:
   250			return false
   251		}
   252		return true
   253	}
   254	
   255	// msg is *Request or *Response.
   256	func readTransfer(msg interface{}, r *bufio.Reader) (err error) {
   257		t := &transferReader{RequestMethod: "GET"}
   258	
   259		// Unify input
   260		isResponse := false
   261		switch rr := msg.(type) {
   262		case *Response:
   263			t.Header = rr.Header
   264			t.StatusCode = rr.StatusCode
   265			t.ProtoMajor = rr.ProtoMajor
   266			t.ProtoMinor = rr.ProtoMinor
   267			t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header)
   268			isResponse = true
   269			if rr.Request != nil {
   270				t.RequestMethod = rr.Request.Method
   271			}
   272		case *Request:
   273			t.Header = rr.Header
   274			t.ProtoMajor = rr.ProtoMajor
   275			t.ProtoMinor = rr.ProtoMinor
   276			// Transfer semantics for Requests are exactly like those for
   277			// Responses with status code 200, responding to a GET method
   278			t.StatusCode = 200
   279		default:
   280			panic("unexpected type")
   281		}
   282	
   283		// Default to HTTP/1.1
   284		if t.ProtoMajor == 0 && t.ProtoMinor == 0 {
   285			t.ProtoMajor, t.ProtoMinor = 1, 1
   286		}
   287	
   288		// Transfer encoding, content length
   289		t.TransferEncoding, err = fixTransferEncoding(t.RequestMethod, t.Header)
   290		if err != nil {
   291			return err
   292		}
   293	
   294		realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.TransferEncoding)
   295		if err != nil {
   296			return err
   297		}
   298		if isResponse && t.RequestMethod == "HEAD" {
   299			if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil {
   300				return err
   301			} else {
   302				t.ContentLength = n
   303			}
   304		} else {
   305			t.ContentLength = realLength
   306		}
   307	
   308		// Trailer
   309		t.Trailer, err = fixTrailer(t.Header, t.TransferEncoding)
   310		if err != nil {
   311			return err
   312		}
   313	
   314		// If there is no Content-Length or chunked Transfer-Encoding on a *Response
   315		// and the status is not 1xx, 204 or 304, then the body is unbounded.
   316		// See RFC2616, section 4.4.
   317		switch msg.(type) {
   318		case *Response:
   319			if realLength == -1 &&
   320				!chunked(t.TransferEncoding) &&
   321				bodyAllowedForStatus(t.StatusCode) {
   322				// Unbounded body.
   323				t.Close = true
   324			}
   325		}
   326	
   327		// Prepare body reader.  ContentLength < 0 means chunked encoding
   328		// or close connection when finished, since multipart is not supported yet
   329		switch {
   330		case chunked(t.TransferEncoding):
   331			if noBodyExpected(t.RequestMethod) {
   332				t.Body = eofReader
   333			} else {
   334				t.Body = &body{Reader: newChunkedReader(r), hdr: msg, r: r, closing: t.Close}
   335			}
   336		case realLength == 0:
   337			t.Body = eofReader
   338		case realLength > 0:
   339			t.Body = &body{Reader: io.LimitReader(r, realLength), closing: t.Close}
   340		default:
   341			// realLength < 0, i.e. "Content-Length" not mentioned in header
   342			if t.Close {
   343				// Close semantics (i.e. HTTP/1.0)
   344				t.Body = &body{Reader: r, closing: t.Close}
   345			} else {
   346				// Persistent connection (i.e. HTTP/1.1)
   347				t.Body = eofReader
   348			}
   349		}
   350	
   351		// Unify output
   352		switch rr := msg.(type) {
   353		case *Request:
   354			rr.Body = t.Body
   355			rr.ContentLength = t.ContentLength
   356			rr.TransferEncoding = t.TransferEncoding
   357			rr.Close = t.Close
   358			rr.Trailer = t.Trailer
   359		case *Response:
   360			rr.Body = t.Body
   361			rr.ContentLength = t.ContentLength
   362			rr.TransferEncoding = t.TransferEncoding
   363			rr.Close = t.Close
   364			rr.Trailer = t.Trailer
   365		}
   366	
   367		return nil
   368	}
   369	
   370	// Checks whether chunked is part of the encodings stack
   371	func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" }
   372	
   373	// Checks whether the encoding is explicitly "identity".
   374	func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" }
   375	
   376	// Sanitize transfer encoding
   377	func fixTransferEncoding(requestMethod string, header Header) ([]string, error) {
   378		raw, present := header["Transfer-Encoding"]
   379		if !present {
   380			return nil, nil
   381		}
   382	
   383		delete(header, "Transfer-Encoding")
   384	
   385		encodings := strings.Split(raw[0], ",")
   386		te := make([]string, 0, len(encodings))
   387		// TODO: Even though we only support "identity" and "chunked"
   388		// encodings, the loop below is designed with foresight. One
   389		// invariant that must be maintained is that, if present,
   390		// chunked encoding must always come first.
   391		for _, encoding := range encodings {
   392			encoding = strings.ToLower(strings.TrimSpace(encoding))
   393			// "identity" encoding is not recorded
   394			if encoding == "identity" {
   395				break
   396			}
   397			if encoding != "chunked" {
   398				return nil, &badStringError{"unsupported transfer encoding", encoding}
   399			}
   400			te = te[0 : len(te)+1]
   401			te[len(te)-1] = encoding
   402		}
   403		if len(te) > 1 {
   404			return nil, &badStringError{"too many transfer encodings", strings.Join(te, ",")}
   405		}
   406		if len(te) > 0 {
   407			// Chunked encoding trumps Content-Length. See RFC 2616
   408			// Section 4.4. Currently len(te) > 0 implies chunked
   409			// encoding.
   410			delete(header, "Content-Length")
   411			return te, nil
   412		}
   413	
   414		return nil, nil
   415	}
   416	
   417	// Determine the expected body length, using RFC 2616 Section 4.4. This
   418	// function is not a method, because ultimately it should be shared by
   419	// ReadResponse and ReadRequest.
   420	func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) {
   421	
   422		// Logic based on response type or status
   423		if noBodyExpected(requestMethod) {
   424			return 0, nil
   425		}
   426		if status/100 == 1 {
   427			return 0, nil
   428		}
   429		switch status {
   430		case 204, 304:
   431			return 0, nil
   432		}
   433	
   434		// Logic based on Transfer-Encoding
   435		if chunked(te) {
   436			return -1, nil
   437		}
   438	
   439		// Logic based on Content-Length
   440		cl := strings.TrimSpace(header.get("Content-Length"))
   441		if cl != "" {
   442			n, err := parseContentLength(cl)
   443			if err != nil {
   444				return -1, err
   445			}
   446			return n, nil
   447		} else {
   448			header.Del("Content-Length")
   449		}
   450	
   451		if !isResponse && requestMethod == "GET" {
   452			// RFC 2616 doesn't explicitly permit nor forbid an
   453			// entity-body on a GET request so we permit one if
   454			// declared, but we default to 0 here (not -1 below)
   455			// if there's no mention of a body.
   456			return 0, nil
   457		}
   458	
   459		// Body-EOF logic based on other methods (like closing, or chunked coding)
   460		return -1, nil
   461	}
   462	
   463	// Determine whether to hang up after sending a request and body, or
   464	// receiving a response and body
   465	// 'header' is the request headers
   466	func shouldClose(major, minor int, header Header) bool {
   467		if major < 1 {
   468			return true
   469		} else if major == 1 && minor == 0 {
   470			if !strings.Contains(strings.ToLower(header.get("Connection")), "keep-alive") {
   471				return true
   472			}
   473			return false
   474		} else {
   475			// TODO: Should split on commas, toss surrounding white space,
   476			// and check each field.
   477			if strings.ToLower(header.get("Connection")) == "close" {
   478				header.Del("Connection")
   479				return true
   480			}
   481		}
   482		return false
   483	}
   484	
   485	// Parse the trailer header
   486	func fixTrailer(header Header, te []string) (Header, error) {
   487		raw := header.get("Trailer")
   488		if raw == "" {
   489			return nil, nil
   490		}
   491	
   492		header.Del("Trailer")
   493		trailer := make(Header)
   494		keys := strings.Split(raw, ",")
   495		for _, key := range keys {
   496			key = CanonicalHeaderKey(strings.TrimSpace(key))
   497			switch key {
   498			case "Transfer-Encoding", "Trailer", "Content-Length":
   499				return nil, &badStringError{"bad trailer key", key}
   500			}
   501			trailer.Del(key)
   502		}
   503		if len(trailer) == 0 {
   504			return nil, nil
   505		}
   506		if !chunked(te) {
   507			// Trailer and no chunking
   508			return nil, ErrUnexpectedTrailer
   509		}
   510		return trailer, nil
   511	}
   512	
   513	// body turns a Reader into a ReadCloser.
   514	// Close ensures that the body has been fully read
   515	// and then reads the trailer if necessary.
   516	type body struct {
   517		io.Reader
   518		hdr     interface{}   // non-nil (Response or Request) value means read trailer
   519		r       *bufio.Reader // underlying wire-format reader for the trailer
   520		closing bool          // is the connection to be closed after reading body?
   521		closed  bool
   522	}
   523	
   524	// ErrBodyReadAfterClose is returned when reading a Request or Response
   525	// Body after the body has been closed. This typically happens when the body is
   526	// read after an HTTP Handler calls WriteHeader or Write on its
   527	// ResponseWriter.
   528	var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body")
   529	
   530	func (b *body) Read(p []byte) (n int, err error) {
   531		if b.closed {
   532			return 0, ErrBodyReadAfterClose
   533		}
   534		n, err = b.Reader.Read(p)
   535	
   536		if err == io.EOF {
   537			// Chunked case. Read the trailer.
   538			if b.hdr != nil {
   539				if e := b.readTrailer(); e != nil {
   540					err = e
   541				}
   542				b.hdr = nil
   543			} else {
   544				// If the server declared the Content-Length, our body is a LimitedReader
   545				// and we need to check whether this EOF arrived early.
   546				if lr, ok := b.Reader.(*io.LimitedReader); ok && lr.N > 0 {
   547					err = io.ErrUnexpectedEOF
   548				}
   549			}
   550		}
   551	
   552		return n, err
   553	}
   554	
   555	var (
   556		singleCRLF = []byte("\r\n")
   557		doubleCRLF = []byte("\r\n\r\n")
   558	)
   559	
   560	func seeUpcomingDoubleCRLF(r *bufio.Reader) bool {
   561		for peekSize := 4; ; peekSize++ {
   562			// This loop stops when Peek returns an error,
   563			// which it does when r's buffer has been filled.
   564			buf, err := r.Peek(peekSize)
   565			if bytes.HasSuffix(buf, doubleCRLF) {
   566				return true
   567			}
   568			if err != nil {
   569				break
   570			}
   571		}
   572		return false
   573	}
   574	
   575	var errTrailerEOF = errors.New("http: unexpected EOF reading trailer")
   576	
   577	func (b *body) readTrailer() error {
   578		// The common case, since nobody uses trailers.
   579		buf, err := b.r.Peek(2)
   580		if bytes.Equal(buf, singleCRLF) {
   581			b.r.ReadByte()
   582			b.r.ReadByte()
   583			return nil
   584		}
   585		if len(buf) < 2 {
   586			return errTrailerEOF
   587		}
   588		if err != nil {
   589			return err
   590		}
   591	
   592		// Make sure there's a header terminator coming up, to prevent
   593		// a DoS with an unbounded size Trailer.  It's not easy to
   594		// slip in a LimitReader here, as textproto.NewReader requires
   595		// a concrete *bufio.Reader.  Also, we can't get all the way
   596		// back up to our conn's LimitedReader that *might* be backing
   597		// this bufio.Reader.  Instead, a hack: we iteratively Peek up
   598		// to the bufio.Reader's max size, looking for a double CRLF.
   599		// This limits the trailer to the underlying buffer size, typically 4kB.
   600		if !seeUpcomingDoubleCRLF(b.r) {
   601			return errors.New("http: suspiciously long trailer after chunked body")
   602		}
   603	
   604		hdr, err := textproto.NewReader(b.r).ReadMIMEHeader()
   605		if err != nil {
   606			if err == io.EOF {
   607				return errTrailerEOF
   608			}
   609			return err
   610		}
   611		switch rr := b.hdr.(type) {
   612		case *Request:
   613			rr.Trailer = Header(hdr)
   614		case *Response:
   615			rr.Trailer = Header(hdr)
   616		}
   617		return nil
   618	}
   619	
   620	func (b *body) Close() error {
   621		if b.closed {
   622			return nil
   623		}
   624		var err error
   625		switch {
   626		case b.hdr == nil && b.closing:
   627			// no trailer and closing the connection next.
   628			// no point in reading to EOF.
   629		default:
   630			// Fully consume the body, which will also lead to us reading
   631			// the trailer headers after the body, if present.
   632			_, err = io.Copy(ioutil.Discard, b)
   633		}
   634		b.closed = true
   635		return err
   636	}
   637	
   638	// parseContentLength trims whitespace from s and returns -1 if no value
   639	// is set, or the value if it's >= 0.
   640	func parseContentLength(cl string) (int64, error) {
   641		cl = strings.TrimSpace(cl)
   642		if cl == "" {
   643			return -1, nil
   644		}
   645		n, err := strconv.ParseInt(cl, 10, 64)
   646		if err != nil || n < 0 {
   647			return 0, &badStringError{"bad Content-Length", cl}
   648		}
   649		return n, nil
   650	
   651	}

View as plain text