1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package http 6 7 import ( 8 "bufio" 9 "bytes" 10 "errors" 11 "fmt" 12 "io" 13 "io/ioutil" 14 "net/http/httptrace" 15 "net/http/internal" 16 "net/textproto" 17 "reflect" 18 "sort" 19 "strconv" 20 "strings" 21 "sync" 22 "time" 23 24 "golang.org/x/net/http/httpguts" 25 ) 26 27 // ErrLineTooLong is returned when reading request or response bodies 28 // with malformed chunked encoding. 29 var ErrLineTooLong = internal.ErrLineTooLong 30 31 type errorReader struct { 32 err error 33 } 34 35 func (r errorReader) Read(p []byte) (n int, err error) { 36 return 0, r.err 37 } 38 39 type byteReader struct { 40 b byte 41 done bool 42 } 43 44 func (br *byteReader) Read(p []byte) (n int, err error) { 45 if br.done { 46 return 0, io.EOF 47 } 48 if len(p) == 0 { 49 return 0, nil 50 } 51 br.done = true 52 p[0] = br.b 53 return 1, io.EOF 54 } 55 56 // transferWriter inspects the fields of a user-supplied Request or Response, 57 // sanitizes them without changing the user object and provides methods for 58 // writing the respective header, body and trailer in wire format. 59 type transferWriter struct { 60 Method string 61 Body io.Reader 62 BodyCloser io.Closer 63 ResponseToHEAD bool 64 ContentLength int64 // -1 means unknown, 0 means exactly none 65 Close bool 66 TransferEncoding []string 67 Header Header 68 Trailer Header 69 IsResponse bool 70 bodyReadError error // any non-EOF error from reading Body 71 72 FlushHeaders bool // flush headers to network before body 73 ByteReadCh chan readResult // non-nil if probeRequestBody called 74 } 75 76 func newTransferWriter(r interface{}) (t *transferWriter, err error) { 77 t = &transferWriter{} 78 79 // Extract relevant fields 80 atLeastHTTP11 := false 81 switch rr := r.(type) { 82 case *Request: 83 if rr.ContentLength != 0 && rr.Body == nil { 84 return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength) 85 } 86 t.Method = valueOrDefault(rr.Method, "GET") 87 t.Close = rr.Close 88 t.TransferEncoding = rr.TransferEncoding 89 t.Header = rr.Header 90 t.Trailer = rr.Trailer 91 t.Body = rr.Body 92 t.BodyCloser = rr.Body 93 t.ContentLength = rr.outgoingLength() 94 if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && t.shouldSendChunkedRequestBody() { 95 t.TransferEncoding = []string{"chunked"} 96 } 97 // If there's a body, conservatively flush the headers 98 // to any bufio.Writer we're writing to, just in case 99 // the server needs the headers early, before we copy 100 // the body and possibly block. We make an exception 101 // for the common standard library in-memory types, 102 // though, to avoid unnecessary TCP packets on the 103 // wire. (Issue 22088.) 104 if t.ContentLength != 0 && !isKnownInMemoryReader(t.Body) { 105 t.FlushHeaders = true 106 } 107 108 atLeastHTTP11 = true // Transport requests are always 1.1 or 2.0 109 case *Response: 110 t.IsResponse = true 111 if rr.Request != nil { 112 t.Method = rr.Request.Method 113 } 114 t.Body = rr.Body 115 t.BodyCloser = rr.Body 116 t.ContentLength = rr.ContentLength 117 t.Close = rr.Close 118 t.TransferEncoding = rr.TransferEncoding 119 t.Header = rr.Header 120 t.Trailer = rr.Trailer 121 atLeastHTTP11 = rr.ProtoAtLeast(1, 1) 122 t.ResponseToHEAD = noResponseBodyExpected(t.Method) 123 } 124 125 // Sanitize Body,ContentLength,TransferEncoding 126 if t.ResponseToHEAD { 127 t.Body = nil 128 if chunked(t.TransferEncoding) { 129 t.ContentLength = -1 130 } 131 } else { 132 if !atLeastHTTP11 || t.Body == nil { 133 t.TransferEncoding = nil 134 } 135 if chunked(t.TransferEncoding) { 136 t.ContentLength = -1 137 } else if t.Body == nil { // no chunking, no body 138 t.ContentLength = 0 139 } 140 } 141 142 // Sanitize Trailer 143 if !chunked(t.TransferEncoding) { 144 t.Trailer = nil 145 } 146 147 return t, nil 148 } 149 150 // shouldSendChunkedRequestBody reports whether we should try to send a 151 // chunked request body to the server. In particular, the case we really 152 // want to prevent is sending a GET or other typically-bodyless request to a 153 // server with a chunked body when the body has zero bytes, since GETs with 154 // bodies (while acceptable according to specs), even zero-byte chunked 155 // bodies, are approximately never seen in the wild and confuse most 156 // servers. See Issue 18257, as one example. 157 // 158 // The only reason we'd send such a request is if the user set the Body to a 159 // non-nil value (say, ioutil.NopCloser(bytes.NewReader(nil))) and didn't 160 // set ContentLength, or NewRequest set it to -1 (unknown), so then we assume 161 // there's bytes to send. 162 // 163 // This code tries to read a byte from the Request.Body in such cases to see 164 // whether the body actually has content (super rare) or is actually just 165 // a non-nil content-less ReadCloser (the more common case). In that more 166 // common case, we act as if their Body were nil instead, and don't send 167 // a body. 168 func (t *transferWriter) shouldSendChunkedRequestBody() bool { 169 // Note that t.ContentLength is the corrected content length 170 // from rr.outgoingLength, so 0 actually means zero, not unknown. 171 if t.ContentLength >= 0 || t.Body == nil { // redundant checks; caller did them 172 return false 173 } 174 if t.Method == "CONNECT" { 175 return false 176 } 177 if requestMethodUsuallyLacksBody(t.Method) { 178 // Only probe the Request.Body for GET/HEAD/DELETE/etc 179 // requests, because it's only those types of requests 180 // that confuse servers. 181 t.probeRequestBody() // adjusts t.Body, t.ContentLength 182 return t.Body != nil 183 } 184 // For all other request types (PUT, POST, PATCH, or anything 185 // made-up we've never heard of), assume it's normal and the server 186 // can deal with a chunked request body. Maybe we'll adjust this 187 // later. 188 return true 189 } 190 191 // probeRequestBody reads a byte from t.Body to see whether it's empty 192 // (returns io.EOF right away). 193 // 194 // But because we've had problems with this blocking users in the past 195 // (issue 17480) when the body is a pipe (perhaps waiting on the response 196 // headers before the pipe is fed data), we need to be careful and bound how 197 // long we wait for it. This delay will only affect users if all the following 198 // are true: 199 // * the request body blocks 200 // * the content length is not set (or set to -1) 201 // * the method doesn't usually have a body (GET, HEAD, DELETE, ...) 202 // * there is no transfer-encoding=chunked already set. 203 // In other words, this delay will not normally affect anybody, and there 204 // are workarounds if it does. 205 func (t *transferWriter) probeRequestBody() { 206 t.ByteReadCh = make(chan readResult, 1) 207 go func(body io.Reader) { 208 var buf [1]byte 209 var rres readResult 210 rres.n, rres.err = body.Read(buf[:]) 211 if rres.n == 1 { 212 rres.b = buf[0] 213 } 214 t.ByteReadCh <- rres 215 }(t.Body) 216 timer := time.NewTimer(200 * time.Millisecond) 217 select { 218 case rres := <-t.ByteReadCh: 219 timer.Stop() 220 if rres.n == 0 && rres.err == io.EOF { 221 // It was empty. 222 t.Body = nil 223 t.ContentLength = 0 224 } else if rres.n == 1 { 225 if rres.err != nil { 226 t.Body = io.MultiReader(&byteReader{b: rres.b}, errorReader{rres.err}) 227 } else { 228 t.Body = io.MultiReader(&byteReader{b: rres.b}, t.Body) 229 } 230 } else if rres.err != nil { 231 t.Body = errorReader{rres.err} 232 } 233 case <-timer.C: 234 // Too slow. Don't wait. Read it later, and keep 235 // assuming that this is ContentLength == -1 236 // (unknown), which means we'll send a 237 // "Transfer-Encoding: chunked" header. 238 t.Body = io.MultiReader(finishAsyncByteRead{t}, t.Body) 239 // Request that Request.Write flush the headers to the 240 // network before writing the body, since our body may not 241 // become readable until it's seen the response headers. 242 t.FlushHeaders = true 243 } 244 } 245 246 func noResponseBodyExpected(requestMethod string) bool { 247 return requestMethod == "HEAD" 248 } 249 250 func (t *transferWriter) shouldSendContentLength() bool { 251 if chunked(t.TransferEncoding) { 252 return false 253 } 254 if t.ContentLength > 0 { 255 return true 256 } 257 if t.ContentLength < 0 { 258 return false 259 } 260 // Many servers expect a Content-Length for these methods 261 if t.Method == "POST" || t.Method == "PUT" { 262 return true 263 } 264 if t.ContentLength == 0 && isIdentity(t.TransferEncoding) { 265 if t.Method == "GET" || t.Method == "HEAD" { 266 return false 267 } 268 return true 269 } 270 271 return false 272 } 273 274 func (t *transferWriter) writeHeader(w io.Writer, trace *httptrace.ClientTrace) error { 275 if t.Close && !hasToken(t.Header.get("Connection"), "close") { 276 if _, err := io.WriteString(w, "Connection: close\r\n"); err != nil { 277 return err 278 } 279 if trace != nil && trace.WroteHeaderField != nil { 280 trace.WroteHeaderField("Connection", []string{"close"}) 281 } 282 } 283 284 // Write Content-Length and/or Transfer-Encoding whose values are a 285 // function of the sanitized field triple (Body, ContentLength, 286 // TransferEncoding) 287 if t.shouldSendContentLength() { 288 if _, err := io.WriteString(w, "Content-Length: "); err != nil { 289 return err 290 } 291 if _, err := io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n"); err != nil { 292 return err 293 } 294 if trace != nil && trace.WroteHeaderField != nil { 295 trace.WroteHeaderField("Content-Length", []string{strconv.FormatInt(t.ContentLength, 10)}) 296 } 297 } else if chunked(t.TransferEncoding) { 298 if _, err := io.WriteString(w, "Transfer-Encoding: chunked\r\n"); err != nil { 299 return err 300 } 301 if trace != nil && trace.WroteHeaderField != nil { 302 trace.WroteHeaderField("Transfer-Encoding", []string{"chunked"}) 303 } 304 } 305 306 // Write Trailer header 307 if t.Trailer != nil { 308 keys := make([]string, 0, len(t.Trailer)) 309 for k := range t.Trailer { 310 k = CanonicalHeaderKey(k) 311 switch k { 312 case "Transfer-Encoding", "Trailer", "Content-Length": 313 return &badStringError{"invalid Trailer key", k} 314 } 315 keys = append(keys, k) 316 } 317 if len(keys) > 0 { 318 sort.Strings(keys) 319 // TODO: could do better allocation-wise here, but trailers are rare, 320 // so being lazy for now. 321 if _, err := io.WriteString(w, "Trailer: "+strings.Join(keys, ",")+"\r\n"); err != nil { 322 return err 323 } 324 if trace != nil && trace.WroteHeaderField != nil { 325 trace.WroteHeaderField("Trailer", keys) 326 } 327 } 328 } 329 330 return nil 331 } 332 333 func (t *transferWriter) writeBody(w io.Writer) error { 334 var err error 335 var ncopy int64 336 337 // Write body. We "unwrap" the body first if it was wrapped in a 338 // nopCloser. This is to ensure that we can take advantage of 339 // OS-level optimizations in the event that the body is an 340 // *os.File. 341 if t.Body != nil { 342 var body = t.unwrapBody() 343 if chunked(t.TransferEncoding) { 344 if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse { 345 w = &internal.FlushAfterChunkWriter{Writer: bw} 346 } 347 cw := internal.NewChunkedWriter(w) 348 _, err = t.doBodyCopy(cw, body) 349 if err == nil { 350 err = cw.Close() 351 } 352 } else if t.ContentLength == -1 { 353 dst := w 354 if t.Method == "CONNECT" { 355 dst = bufioFlushWriter{dst} 356 } 357 ncopy, err = t.doBodyCopy(dst, body) 358 } else { 359 ncopy, err = t.doBodyCopy(w, io.LimitReader(body, t.ContentLength)) 360 if err != nil { 361 return err 362 } 363 var nextra int64 364 nextra, err = t.doBodyCopy(ioutil.Discard, body) 365 ncopy += nextra 366 } 367 if err != nil { 368 return err 369 } 370 } 371 if t.BodyCloser != nil { 372 if err := t.BodyCloser.Close(); err != nil { 373 return err 374 } 375 } 376 377 if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy { 378 return fmt.Errorf("http: ContentLength=%d with Body length %d", 379 t.ContentLength, ncopy) 380 } 381 382 if chunked(t.TransferEncoding) { 383 // Write Trailer header 384 if t.Trailer != nil { 385 if err := t.Trailer.Write(w); err != nil { 386 return err 387 } 388 } 389 // Last chunk, empty trailer 390 _, err = io.WriteString(w, "\r\n") 391 } 392 return err 393 } 394 395 // doBodyCopy wraps a copy operation, with any resulting error also 396 // being saved in bodyReadError. 397 // 398 // This function is only intended for use in writeBody. 399 func (t *transferWriter) doBodyCopy(dst io.Writer, src io.Reader) (n int64, err error) { 400 n, err = io.Copy(dst, src) 401 if err != nil && err != io.EOF { 402 t.bodyReadError = err 403 } 404 return 405 } 406 407 // unwrapBodyReader unwraps the body's inner reader if it's a 408 // nopCloser. This is to ensure that body writes sourced from local 409 // files (*os.File types) are properly optimized. 410 // 411 // This function is only intended for use in writeBody. 412 func (t *transferWriter) unwrapBody() io.Reader { 413 if reflect.TypeOf(t.Body) == nopCloserType { 414 return reflect.ValueOf(t.Body).Field(0).Interface().(io.Reader) 415 } 416 417 return t.Body 418 } 419 420 type transferReader struct { 421 // Input 422 Header Header 423 StatusCode int 424 RequestMethod string 425 ProtoMajor int 426 ProtoMinor int 427 // Output 428 Body io.ReadCloser 429 ContentLength int64 430 TransferEncoding []string 431 Close bool 432 Trailer Header 433 } 434 435 func (t *transferReader) protoAtLeast(m, n int) bool { 436 return t.ProtoMajor > m || (t.ProtoMajor == m && t.ProtoMinor >= n) 437 } 438 439 // bodyAllowedForStatus reports whether a given response status code 440 // permits a body. See RFC 7230, section 3.3. 441 func bodyAllowedForStatus(status int) bool { 442 switch { 443 case status >= 100 && status <= 199: 444 return false 445 case status == 204: 446 return false 447 case status == 304: 448 return false 449 } 450 return true 451 } 452 453 var ( 454 suppressedHeaders304 = []string{"Content-Type", "Content-Length", "Transfer-Encoding"} 455 suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"} 456 ) 457 458 func suppressedHeaders(status int) []string { 459 switch { 460 case status == 304: 461 // RFC 7232 section 4.1 462 return suppressedHeaders304 463 case !bodyAllowedForStatus(status): 464 return suppressedHeadersNoBody 465 } 466 return nil 467 } 468 469 // msg is *Request or *Response. 470 func readTransfer(msg interface{}, r *bufio.Reader) (err error) { 471 t := &transferReader{RequestMethod: "GET"} 472 473 // Unify input 474 isResponse := false 475 switch rr := msg.(type) { 476 case *Response: 477 t.Header = rr.Header 478 t.StatusCode = rr.StatusCode 479 t.ProtoMajor = rr.ProtoMajor 480 t.ProtoMinor = rr.ProtoMinor 481 t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true) 482 isResponse = true 483 if rr.Request != nil { 484 t.RequestMethod = rr.Request.Method 485 } 486 case *Request: 487 t.Header = rr.Header 488 t.RequestMethod = rr.Method 489 t.ProtoMajor = rr.ProtoMajor 490 t.ProtoMinor = rr.ProtoMinor 491 // Transfer semantics for Requests are exactly like those for 492 // Responses with status code 200, responding to a GET method 493 t.StatusCode = 200 494 t.Close = rr.Close 495 default: 496 panic("unexpected type") 497 } 498 499 // Default to HTTP/1.1 500 if t.ProtoMajor == 0 && t.ProtoMinor == 0 { 501 t.ProtoMajor, t.ProtoMinor = 1, 1 502 } 503 504 // Transfer encoding, content length 505 err = t.fixTransferEncoding() 506 if err != nil { 507 return err 508 } 509 510 realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.TransferEncoding) 511 if err != nil { 512 return err 513 } 514 if isResponse && t.RequestMethod == "HEAD" { 515 if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil { 516 return err 517 } else { 518 t.ContentLength = n 519 } 520 } else { 521 t.ContentLength = realLength 522 } 523 524 // Trailer 525 t.Trailer, err = fixTrailer(t.Header, t.TransferEncoding) 526 if err != nil { 527 return err 528 } 529 530 // If there is no Content-Length or chunked Transfer-Encoding on a *Response 531 // and the status is not 1xx, 204 or 304, then the body is unbounded. 532 // See RFC 7230, section 3.3. 533 switch msg.(type) { 534 case *Response: 535 if realLength == -1 && 536 !chunked(t.TransferEncoding) && 537 bodyAllowedForStatus(t.StatusCode) { 538 // Unbounded body. 539 t.Close = true 540 } 541 } 542 543 // Prepare body reader. ContentLength < 0 means chunked encoding 544 // or close connection when finished, since multipart is not supported yet 545 switch { 546 case chunked(t.TransferEncoding): 547 if noResponseBodyExpected(t.RequestMethod) || !bodyAllowedForStatus(t.StatusCode) { 548 t.Body = NoBody 549 } else { 550 t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close} 551 } 552 case realLength == 0: 553 t.Body = NoBody 554 case realLength > 0: 555 t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close} 556 default: 557 // realLength < 0, i.e. "Content-Length" not mentioned in header 558 if t.Close { 559 // Close semantics (i.e. HTTP/1.0) 560 t.Body = &body{src: r, closing: t.Close} 561 } else { 562 // Persistent connection (i.e. HTTP/1.1) 563 t.Body = NoBody 564 } 565 } 566 567 // Unify output 568 switch rr := msg.(type) { 569 case *Request: 570 rr.Body = t.Body 571 rr.ContentLength = t.ContentLength 572 rr.TransferEncoding = t.TransferEncoding 573 rr.Close = t.Close 574 rr.Trailer = t.Trailer 575 case *Response: 576 rr.Body = t.Body 577 rr.ContentLength = t.ContentLength 578 rr.TransferEncoding = t.TransferEncoding 579 rr.Close = t.Close 580 rr.Trailer = t.Trailer 581 } 582 583 return nil 584 } 585 586 // Checks whether chunked is part of the encodings stack 587 func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } 588 589 // Checks whether the encoding is explicitly "identity". 590 func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } 591 592 // unsupportedTEError reports unsupported transfer-encodings. 593 type unsupportedTEError struct { 594 err string 595 } 596 597 func (uste *unsupportedTEError) Error() string { 598 return uste.err 599 } 600 601 // isUnsupportedTEError checks if the error is of type 602 // unsupportedTEError. It is usually invoked with a non-nil err. 603 func isUnsupportedTEError(err error) bool { 604 _, ok := err.(*unsupportedTEError) 605 return ok 606 } 607 608 // fixTransferEncoding sanitizes t.TransferEncoding, if needed. 609 func (t *transferReader) fixTransferEncoding() error { 610 raw, present := t.Header["Transfer-Encoding"] 611 if !present { 612 return nil 613 } 614 delete(t.Header, "Transfer-Encoding") 615 616 // Issue 12785; ignore Transfer-Encoding on HTTP/1.0 requests. 617 if !t.protoAtLeast(1, 1) { 618 return nil 619 } 620 621 encodings := strings.Split(raw[0], ",") 622 te := make([]string, 0, len(encodings)) 623 // TODO: Even though we only support "identity" and "chunked" 624 // encodings, the loop below is designed with foresight. One 625 // invariant that must be maintained is that, if present, 626 // chunked encoding must always come first. 627 for _, encoding := range encodings { 628 encoding = strings.ToLower(strings.TrimSpace(encoding)) 629 // "identity" encoding is not recorded 630 if encoding == "identity" { 631 break 632 } 633 if encoding != "chunked" { 634 return &unsupportedTEError{fmt.Sprintf("unsupported transfer encoding: %q", encoding)} 635 } 636 te = te[0 : len(te)+1] 637 te[len(te)-1] = encoding 638 } 639 if len(te) > 1 { 640 return &badStringError{"too many transfer encodings", strings.Join(te, ",")} 641 } 642 if len(te) > 0 { 643 // RFC 7230 3.3.2 says "A sender MUST NOT send a 644 // Content-Length header field in any message that 645 // contains a Transfer-Encoding header field." 646 // 647 // but also: 648 // "If a message is received with both a 649 // Transfer-Encoding and a Content-Length header 650 // field, the Transfer-Encoding overrides the 651 // Content-Length. Such a message might indicate an 652 // attempt to perform request smuggling (Section 9.5) 653 // or response splitting (Section 9.4) and ought to be 654 // handled as an error. A sender MUST remove the 655 // received Content-Length field prior to forwarding 656 // such a message downstream." 657 // 658 // Reportedly, these appear in the wild. 659 delete(t.Header, "Content-Length") 660 t.TransferEncoding = te 661 return nil 662 } 663 664 return nil 665 } 666 667 // Determine the expected body length, using RFC 7230 Section 3.3. This 668 // function is not a method, because ultimately it should be shared by 669 // ReadResponse and ReadRequest. 670 func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) { 671 isRequest := !isResponse 672 contentLens := header["Content-Length"] 673 674 // Hardening against HTTP request smuggling 675 if len(contentLens) > 1 { 676 // Per RFC 7230 Section 3.3.2, prevent multiple 677 // Content-Length headers if they differ in value. 678 // If there are dups of the value, remove the dups. 679 // See Issue 16490. 680 first := strings.TrimSpace(contentLens[0]) 681 for _, ct := range contentLens[1:] { 682 if first != strings.TrimSpace(ct) { 683 return 0, fmt.Errorf("http: message cannot contain multiple Content-Length headers; got %q", contentLens) 684 } 685 } 686 687 // deduplicate Content-Length 688 header.Del("Content-Length") 689 header.Add("Content-Length", first) 690 691 contentLens = header["Content-Length"] 692 } 693 694 // Logic based on response type or status 695 if noResponseBodyExpected(requestMethod) { 696 // For HTTP requests, as part of hardening against request 697 // smuggling (RFC 7230), don't allow a Content-Length header for 698 // methods which don't permit bodies. As an exception, allow 699 // exactly one Content-Length header if its value is "0". 700 if isRequest && len(contentLens) > 0 && !(len(contentLens) == 1 && contentLens[0] == "0") { 701 return 0, fmt.Errorf("http: method cannot contain a Content-Length; got %q", contentLens) 702 } 703 return 0, nil 704 } 705 if status/100 == 1 { 706 return 0, nil 707 } 708 switch status { 709 case 204, 304: 710 return 0, nil 711 } 712 713 // Logic based on Transfer-Encoding 714 if chunked(te) { 715 return -1, nil 716 } 717 718 // Logic based on Content-Length 719 var cl string 720 if len(contentLens) == 1 { 721 cl = strings.TrimSpace(contentLens[0]) 722 } 723 if cl != "" { 724 n, err := parseContentLength(cl) 725 if err != nil { 726 return -1, err 727 } 728 return n, nil 729 } 730 header.Del("Content-Length") 731 732 if isRequest { 733 // RFC 7230 neither explicitly permits nor forbids an 734 // entity-body on a GET request so we permit one if 735 // declared, but we default to 0 here (not -1 below) 736 // if there's no mention of a body. 737 // Likewise, all other request methods are assumed to have 738 // no body if neither Transfer-Encoding chunked nor a 739 // Content-Length are set. 740 return 0, nil 741 } 742 743 // Body-EOF logic based on other methods (like closing, or chunked coding) 744 return -1, nil 745 } 746 747 // Determine whether to hang up after sending a request and body, or 748 // receiving a response and body 749 // 'header' is the request headers 750 func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool { 751 if major < 1 { 752 return true 753 } 754 755 conv := header["Connection"] 756 hasClose := httpguts.HeaderValuesContainsToken(conv, "close") 757 if major == 1 && minor == 0 { 758 return hasClose || !httpguts.HeaderValuesContainsToken(conv, "keep-alive") 759 } 760 761 if hasClose && removeCloseHeader { 762 header.Del("Connection") 763 } 764 765 return hasClose 766 } 767 768 // Parse the trailer header 769 func fixTrailer(header Header, te []string) (Header, error) { 770 vv, ok := header["Trailer"] 771 if !ok { 772 return nil, nil 773 } 774 if !chunked(te) { 775 // Trailer and no chunking: 776 // this is an invalid use case for trailer header. 777 // Nevertheless, no error will be returned and we 778 // let users decide if this is a valid HTTP message. 779 // The Trailer header will be kept in Response.Header 780 // but not populate Response.Trailer. 781 // See issue #27197. 782 return nil, nil 783 } 784 header.Del("Trailer") 785 786 trailer := make(Header) 787 var err error 788 for _, v := range vv { 789 foreachHeaderElement(v, func(key string) { 790 key = CanonicalHeaderKey(key) 791 switch key { 792 case "Transfer-Encoding", "Trailer", "Content-Length": 793 if err == nil { 794 err = &badStringError{"bad trailer key", key} 795 return 796 } 797 } 798 trailer[key] = nil 799 }) 800 } 801 if err != nil { 802 return nil, err 803 } 804 if len(trailer) == 0 { 805 return nil, nil 806 } 807 return trailer, nil 808 } 809 810 // body turns a Reader into a ReadCloser. 811 // Close ensures that the body has been fully read 812 // and then reads the trailer if necessary. 813 type body struct { 814 src io.Reader 815 hdr interface{} // non-nil (Response or Request) value means read trailer 816 r *bufio.Reader // underlying wire-format reader for the trailer 817 closing bool // is the connection to be closed after reading body? 818 doEarlyClose bool // whether Close should stop early 819 820 mu sync.Mutex // guards following, and calls to Read and Close 821 sawEOF bool 822 closed bool 823 earlyClose bool // Close called and we didn't read to the end of src 824 onHitEOF func() // if non-nil, func to call when EOF is Read 825 } 826 827 // ErrBodyReadAfterClose is returned when reading a Request or Response 828 // Body after the body has been closed. This typically happens when the body is 829 // read after an HTTP Handler calls WriteHeader or Write on its 830 // ResponseWriter. 831 var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body") 832 833 func (b *body) Read(p []byte) (n int, err error) { 834 b.mu.Lock() 835 defer b.mu.Unlock() 836 if b.closed { 837 return 0, ErrBodyReadAfterClose 838 } 839 return b.readLocked(p) 840 } 841 842 // Must hold b.mu. 843 func (b *body) readLocked(p []byte) (n int, err error) { 844 if b.sawEOF { 845 return 0, io.EOF 846 } 847 n, err = b.src.Read(p) 848 849 if err == io.EOF { 850 b.sawEOF = true 851 // Chunked case. Read the trailer. 852 if b.hdr != nil { 853 if e := b.readTrailer(); e != nil { 854 err = e 855 // Something went wrong in the trailer, we must not allow any 856 // further reads of any kind to succeed from body, nor any 857 // subsequent requests on the server connection. See 858 // golang.org/issue/12027 859 b.sawEOF = false 860 b.closed = true 861 } 862 b.hdr = nil 863 } else { 864 // If the server declared the Content-Length, our body is a LimitedReader 865 // and we need to check whether this EOF arrived early. 866 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 { 867 err = io.ErrUnexpectedEOF 868 } 869 } 870 } 871 872 // If we can return an EOF here along with the read data, do 873 // so. This is optional per the io.Reader contract, but doing 874 // so helps the HTTP transport code recycle its connection 875 // earlier (since it will see this EOF itself), even if the 876 // client doesn't do future reads or Close. 877 if err == nil && n > 0 { 878 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 { 879 err = io.EOF 880 b.sawEOF = true 881 } 882 } 883 884 if b.sawEOF && b.onHitEOF != nil { 885 b.onHitEOF() 886 } 887 888 return n, err 889 } 890 891 var ( 892 singleCRLF = []byte("\r\n") 893 doubleCRLF = []byte("\r\n\r\n") 894 ) 895 896 func seeUpcomingDoubleCRLF(r *bufio.Reader) bool { 897 for peekSize := 4; ; peekSize++ { 898 // This loop stops when Peek returns an error, 899 // which it does when r's buffer has been filled. 900 buf, err := r.Peek(peekSize) 901 if bytes.HasSuffix(buf, doubleCRLF) { 902 return true 903 } 904 if err != nil { 905 break 906 } 907 } 908 return false 909 } 910 911 var errTrailerEOF = errors.New("http: unexpected EOF reading trailer") 912 913 func (b *body) readTrailer() error { 914 // The common case, since nobody uses trailers. 915 buf, err := b.r.Peek(2) 916 if bytes.Equal(buf, singleCRLF) { 917 b.r.Discard(2) 918 return nil 919 } 920 if len(buf) < 2 { 921 return errTrailerEOF 922 } 923 if err != nil { 924 return err 925 } 926 927 // Make sure there's a header terminator coming up, to prevent 928 // a DoS with an unbounded size Trailer. It's not easy to 929 // slip in a LimitReader here, as textproto.NewReader requires 930 // a concrete *bufio.Reader. Also, we can't get all the way 931 // back up to our conn's LimitedReader that *might* be backing 932 // this bufio.Reader. Instead, a hack: we iteratively Peek up 933 // to the bufio.Reader's max size, looking for a double CRLF. 934 // This limits the trailer to the underlying buffer size, typically 4kB. 935 if !seeUpcomingDoubleCRLF(b.r) { 936 return errors.New("http: suspiciously long trailer after chunked body") 937 } 938 939 hdr, err := textproto.NewReader(b.r).ReadMIMEHeader() 940 if err != nil { 941 if err == io.EOF { 942 return errTrailerEOF 943 } 944 return err 945 } 946 switch rr := b.hdr.(type) { 947 case *Request: 948 mergeSetHeader(&rr.Trailer, Header(hdr)) 949 case *Response: 950 mergeSetHeader(&rr.Trailer, Header(hdr)) 951 } 952 return nil 953 } 954 955 func mergeSetHeader(dst *Header, src Header) { 956 if *dst == nil { 957 *dst = src 958 return 959 } 960 for k, vv := range src { 961 (*dst)[k] = vv 962 } 963 } 964 965 // unreadDataSizeLocked returns the number of bytes of unread input. 966 // It returns -1 if unknown. 967 // b.mu must be held. 968 func (b *body) unreadDataSizeLocked() int64 { 969 if lr, ok := b.src.(*io.LimitedReader); ok { 970 return lr.N 971 } 972 return -1 973 } 974 975 func (b *body) Close() error { 976 b.mu.Lock() 977 defer b.mu.Unlock() 978 if b.closed { 979 return nil 980 } 981 var err error 982 switch { 983 case b.sawEOF: 984 // Already saw EOF, so no need going to look for it. 985 case b.hdr == nil && b.closing: 986 // no trailer and closing the connection next. 987 // no point in reading to EOF. 988 case b.doEarlyClose: 989 // Read up to maxPostHandlerReadBytes bytes of the body, looking 990 // for EOF (and trailers), so we can re-use this connection. 991 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > maxPostHandlerReadBytes { 992 // There was a declared Content-Length, and we have more bytes remaining 993 // than our maxPostHandlerReadBytes tolerance. So, give up. 994 b.earlyClose = true 995 } else { 996 var n int64 997 // Consume the body, or, which will also lead to us reading 998 // the trailer headers after the body, if present. 999 n, err = io.CopyN(ioutil.Discard, bodyLocked{b}, maxPostHandlerReadBytes) 1000 if err == io.EOF { 1001 err = nil 1002 } 1003 if n == maxPostHandlerReadBytes { 1004 b.earlyClose = true 1005 } 1006 } 1007 default: 1008 // Fully consume the body, which will also lead to us reading 1009 // the trailer headers after the body, if present. 1010 _, err = io.Copy(ioutil.Discard, bodyLocked{b}) 1011 } 1012 b.closed = true 1013 return err 1014 } 1015 1016 func (b *body) didEarlyClose() bool { 1017 b.mu.Lock() 1018 defer b.mu.Unlock() 1019 return b.earlyClose 1020 } 1021 1022 // bodyRemains reports whether future Read calls might 1023 // yield data. 1024 func (b *body) bodyRemains() bool { 1025 b.mu.Lock() 1026 defer b.mu.Unlock() 1027 return !b.sawEOF 1028 } 1029 1030 func (b *body) registerOnHitEOF(fn func()) { 1031 b.mu.Lock() 1032 defer b.mu.Unlock() 1033 b.onHitEOF = fn 1034 } 1035 1036 // bodyLocked is a io.Reader reading from a *body when its mutex is 1037 // already held. 1038 type bodyLocked struct { 1039 b *body 1040 } 1041 1042 func (bl bodyLocked) Read(p []byte) (n int, err error) { 1043 if bl.b.closed { 1044 return 0, ErrBodyReadAfterClose 1045 } 1046 return bl.b.readLocked(p) 1047 } 1048 1049 // parseContentLength trims whitespace from s and returns -1 if no value 1050 // is set, or the value if it's >= 0. 1051 func parseContentLength(cl string) (int64, error) { 1052 cl = strings.TrimSpace(cl) 1053 if cl == "" { 1054 return -1, nil 1055 } 1056 n, err := strconv.ParseInt(cl, 10, 64) 1057 if err != nil || n < 0 { 1058 return 0, &badStringError{"bad Content-Length", cl} 1059 } 1060 return n, nil 1061 1062 } 1063 1064 // finishAsyncByteRead finishes reading the 1-byte sniff 1065 // from the ContentLength==0, Body!=nil case. 1066 type finishAsyncByteRead struct { 1067 tw *transferWriter 1068 } 1069 1070 func (fr finishAsyncByteRead) Read(p []byte) (n int, err error) { 1071 if len(p) == 0 { 1072 return 1073 } 1074 rres := <-fr.tw.ByteReadCh 1075 n, err = rres.n, rres.err 1076 if n == 1 { 1077 p[0] = rres.b 1078 } 1079 return 1080 } 1081 1082 var nopCloserType = reflect.TypeOf(ioutil.NopCloser(nil)) 1083 1084 // isKnownInMemoryReader reports whether r is a type known to not 1085 // block on Read. Its caller uses this as an optional optimization to 1086 // send fewer TCP packets. 1087 func isKnownInMemoryReader(r io.Reader) bool { 1088 switch r.(type) { 1089 case *bytes.Reader, *bytes.Buffer, *strings.Reader: 1090 return true 1091 } 1092 if reflect.TypeOf(r) == nopCloserType { 1093 return isKnownInMemoryReader(reflect.ValueOf(r).Field(0).Interface().(io.Reader)) 1094 } 1095 return false 1096 } 1097 1098 // bufioFlushWriter is an io.Writer wrapper that flushes all writes 1099 // on its wrapped writer if it's a *bufio.Writer. 1100 type bufioFlushWriter struct{ w io.Writer } 1101 1102 func (fw bufioFlushWriter) Write(p []byte) (n int, err error) { 1103 n, err = fw.w.Write(p) 1104 if bw, ok := fw.w.(*bufio.Writer); n > 0 && ok { 1105 ferr := bw.Flush() 1106 if ferr != nil && err == nil { 1107 err = ferr 1108 } 1109 } 1110 return 1111 } 1112
View as plain text