1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package http 6 7 import ( 8 "bufio" 9 "bytes" 10 "errors" 11 "fmt" 12 "io" 13 "io/ioutil" 14 "net/http/httptrace" 15 "net/http/internal" 16 "net/textproto" 17 "reflect" 18 "sort" 19 "strconv" 20 "strings" 21 "sync" 22 "time" 23 24 "internal/x/net/http/httpguts" 25 ) 26 27 // ErrLineTooLong is returned when reading request or response bodies 28 // with malformed chunked encoding. 29 var ErrLineTooLong = internal.ErrLineTooLong 30 31 type errorReader struct { 32 err error 33 } 34 35 func (r errorReader) Read(p []byte) (n int, err error) { 36 return 0, r.err 37 } 38 39 type byteReader struct { 40 b byte 41 done bool 42 } 43 44 func (br *byteReader) Read(p []byte) (n int, err error) { 45 if br.done { 46 return 0, io.EOF 47 } 48 if len(p) == 0 { 49 return 0, nil 50 } 51 br.done = true 52 p[0] = br.b 53 return 1, io.EOF 54 } 55 56 // transferBodyReader is an io.Reader that reads from tw.Body 57 // and records any non-EOF error in tw.bodyReadError. 58 // It is exactly 1 pointer wide to avoid allocations into interfaces. 59 type transferBodyReader struct{ tw *transferWriter } 60 61 func (br transferBodyReader) Read(p []byte) (n int, err error) { 62 n, err = br.tw.Body.Read(p) 63 if err != nil && err != io.EOF { 64 br.tw.bodyReadError = err 65 } 66 return 67 } 68 69 // transferWriter inspects the fields of a user-supplied Request or Response, 70 // sanitizes them without changing the user object and provides methods for 71 // writing the respective header, body and trailer in wire format. 72 type transferWriter struct { 73 Method string 74 Body io.Reader 75 BodyCloser io.Closer 76 ResponseToHEAD bool 77 ContentLength int64 // -1 means unknown, 0 means exactly none 78 Close bool 79 TransferEncoding []string 80 Header Header 81 Trailer Header 82 IsResponse bool 83 bodyReadError error // any non-EOF error from reading Body 84 85 FlushHeaders bool // flush headers to network before body 86 ByteReadCh chan readResult // non-nil if probeRequestBody called 87 } 88 89 func newTransferWriter(r interface{}) (t *transferWriter, err error) { 90 t = &transferWriter{} 91 92 // Extract relevant fields 93 atLeastHTTP11 := false 94 switch rr := r.(type) { 95 case *Request: 96 if rr.ContentLength != 0 && rr.Body == nil { 97 return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength) 98 } 99 t.Method = valueOrDefault(rr.Method, "GET") 100 t.Close = rr.Close 101 t.TransferEncoding = rr.TransferEncoding 102 t.Header = rr.Header 103 t.Trailer = rr.Trailer 104 t.Body = rr.Body 105 t.BodyCloser = rr.Body 106 t.ContentLength = rr.outgoingLength() 107 if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && t.shouldSendChunkedRequestBody() { 108 t.TransferEncoding = []string{"chunked"} 109 } 110 // If there's a body, conservatively flush the headers 111 // to any bufio.Writer we're writing to, just in case 112 // the server needs the headers early, before we copy 113 // the body and possibly block. We make an exception 114 // for the common standard library in-memory types, 115 // though, to avoid unnecessary TCP packets on the 116 // wire. (Issue 22088.) 117 if t.ContentLength != 0 && !isKnownInMemoryReader(t.Body) { 118 t.FlushHeaders = true 119 } 120 121 atLeastHTTP11 = true // Transport requests are always 1.1 or 2.0 122 case *Response: 123 t.IsResponse = true 124 if rr.Request != nil { 125 t.Method = rr.Request.Method 126 } 127 t.Body = rr.Body 128 t.BodyCloser = rr.Body 129 t.ContentLength = rr.ContentLength 130 t.Close = rr.Close 131 t.TransferEncoding = rr.TransferEncoding 132 t.Header = rr.Header 133 t.Trailer = rr.Trailer 134 atLeastHTTP11 = rr.ProtoAtLeast(1, 1) 135 t.ResponseToHEAD = noResponseBodyExpected(t.Method) 136 } 137 138 // Sanitize Body,ContentLength,TransferEncoding 139 if t.ResponseToHEAD { 140 t.Body = nil 141 if chunked(t.TransferEncoding) { 142 t.ContentLength = -1 143 } 144 } else { 145 if !atLeastHTTP11 || t.Body == nil { 146 t.TransferEncoding = nil 147 } 148 if chunked(t.TransferEncoding) { 149 t.ContentLength = -1 150 } else if t.Body == nil { // no chunking, no body 151 t.ContentLength = 0 152 } 153 } 154 155 // Sanitize Trailer 156 if !chunked(t.TransferEncoding) { 157 t.Trailer = nil 158 } 159 160 return t, nil 161 } 162 163 // shouldSendChunkedRequestBody reports whether we should try to send a 164 // chunked request body to the server. In particular, the case we really 165 // want to prevent is sending a GET or other typically-bodyless request to a 166 // server with a chunked body when the body has zero bytes, since GETs with 167 // bodies (while acceptable according to specs), even zero-byte chunked 168 // bodies, are approximately never seen in the wild and confuse most 169 // servers. See Issue 18257, as one example. 170 // 171 // The only reason we'd send such a request is if the user set the Body to a 172 // non-nil value (say, ioutil.NopCloser(bytes.NewReader(nil))) and didn't 173 // set ContentLength, or NewRequest set it to -1 (unknown), so then we assume 174 // there's bytes to send. 175 // 176 // This code tries to read a byte from the Request.Body in such cases to see 177 // whether the body actually has content (super rare) or is actually just 178 // a non-nil content-less ReadCloser (the more common case). In that more 179 // common case, we act as if their Body were nil instead, and don't send 180 // a body. 181 func (t *transferWriter) shouldSendChunkedRequestBody() bool { 182 // Note that t.ContentLength is the corrected content length 183 // from rr.outgoingLength, so 0 actually means zero, not unknown. 184 if t.ContentLength >= 0 || t.Body == nil { // redundant checks; caller did them 185 return false 186 } 187 if t.Method == "CONNECT" { 188 return false 189 } 190 if requestMethodUsuallyLacksBody(t.Method) { 191 // Only probe the Request.Body for GET/HEAD/DELETE/etc 192 // requests, because it's only those types of requests 193 // that confuse servers. 194 t.probeRequestBody() // adjusts t.Body, t.ContentLength 195 return t.Body != nil 196 } 197 // For all other request types (PUT, POST, PATCH, or anything 198 // made-up we've never heard of), assume it's normal and the server 199 // can deal with a chunked request body. Maybe we'll adjust this 200 // later. 201 return true 202 } 203 204 // probeRequestBody reads a byte from t.Body to see whether it's empty 205 // (returns io.EOF right away). 206 // 207 // But because we've had problems with this blocking users in the past 208 // (issue 17480) when the body is a pipe (perhaps waiting on the response 209 // headers before the pipe is fed data), we need to be careful and bound how 210 // long we wait for it. This delay will only affect users if all the following 211 // are true: 212 // * the request body blocks 213 // * the content length is not set (or set to -1) 214 // * the method doesn't usually have a body (GET, HEAD, DELETE, ...) 215 // * there is no transfer-encoding=chunked already set. 216 // In other words, this delay will not normally affect anybody, and there 217 // are workarounds if it does. 218 func (t *transferWriter) probeRequestBody() { 219 t.ByteReadCh = make(chan readResult, 1) 220 go func(body io.Reader) { 221 var buf [1]byte 222 var rres readResult 223 rres.n, rres.err = body.Read(buf[:]) 224 if rres.n == 1 { 225 rres.b = buf[0] 226 } 227 t.ByteReadCh <- rres 228 }(t.Body) 229 timer := time.NewTimer(200 * time.Millisecond) 230 select { 231 case rres := <-t.ByteReadCh: 232 timer.Stop() 233 if rres.n == 0 && rres.err == io.EOF { 234 // It was empty. 235 t.Body = nil 236 t.ContentLength = 0 237 } else if rres.n == 1 { 238 if rres.err != nil { 239 t.Body = io.MultiReader(&byteReader{b: rres.b}, errorReader{rres.err}) 240 } else { 241 t.Body = io.MultiReader(&byteReader{b: rres.b}, t.Body) 242 } 243 } else if rres.err != nil { 244 t.Body = errorReader{rres.err} 245 } 246 case <-timer.C: 247 // Too slow. Don't wait. Read it later, and keep 248 // assuming that this is ContentLength == -1 249 // (unknown), which means we'll send a 250 // "Transfer-Encoding: chunked" header. 251 t.Body = io.MultiReader(finishAsyncByteRead{t}, t.Body) 252 // Request that Request.Write flush the headers to the 253 // network before writing the body, since our body may not 254 // become readable until it's seen the response headers. 255 t.FlushHeaders = true 256 } 257 } 258 259 func noResponseBodyExpected(requestMethod string) bool { 260 return requestMethod == "HEAD" 261 } 262 263 func (t *transferWriter) shouldSendContentLength() bool { 264 if chunked(t.TransferEncoding) { 265 return false 266 } 267 if t.ContentLength > 0 { 268 return true 269 } 270 if t.ContentLength < 0 { 271 return false 272 } 273 // Many servers expect a Content-Length for these methods 274 if t.Method == "POST" || t.Method == "PUT" { 275 return true 276 } 277 if t.ContentLength == 0 && isIdentity(t.TransferEncoding) { 278 if t.Method == "GET" || t.Method == "HEAD" { 279 return false 280 } 281 return true 282 } 283 284 return false 285 } 286 287 func (t *transferWriter) writeHeader(w io.Writer, trace *httptrace.ClientTrace) error { 288 if t.Close && !hasToken(t.Header.get("Connection"), "close") { 289 if _, err := io.WriteString(w, "Connection: close\r\n"); err != nil { 290 return err 291 } 292 if trace != nil && trace.WroteHeaderField != nil { 293 trace.WroteHeaderField("Connection", []string{"close"}) 294 } 295 } 296 297 // Write Content-Length and/or Transfer-Encoding whose values are a 298 // function of the sanitized field triple (Body, ContentLength, 299 // TransferEncoding) 300 if t.shouldSendContentLength() { 301 if _, err := io.WriteString(w, "Content-Length: "); err != nil { 302 return err 303 } 304 if _, err := io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n"); err != nil { 305 return err 306 } 307 if trace != nil && trace.WroteHeaderField != nil { 308 trace.WroteHeaderField("Content-Length", []string{strconv.FormatInt(t.ContentLength, 10)}) 309 } 310 } else if chunked(t.TransferEncoding) { 311 if _, err := io.WriteString(w, "Transfer-Encoding: chunked\r\n"); err != nil { 312 return err 313 } 314 if trace != nil && trace.WroteHeaderField != nil { 315 trace.WroteHeaderField("Transfer-Encoding", []string{"chunked"}) 316 } 317 } 318 319 // Write Trailer header 320 if t.Trailer != nil { 321 keys := make([]string, 0, len(t.Trailer)) 322 for k := range t.Trailer { 323 k = CanonicalHeaderKey(k) 324 switch k { 325 case "Transfer-Encoding", "Trailer", "Content-Length": 326 return &badStringError{"invalid Trailer key", k} 327 } 328 keys = append(keys, k) 329 } 330 if len(keys) > 0 { 331 sort.Strings(keys) 332 // TODO: could do better allocation-wise here, but trailers are rare, 333 // so being lazy for now. 334 if _, err := io.WriteString(w, "Trailer: "+strings.Join(keys, ",")+"\r\n"); err != nil { 335 return err 336 } 337 if trace != nil && trace.WroteHeaderField != nil { 338 trace.WroteHeaderField("Trailer", keys) 339 } 340 } 341 } 342 343 return nil 344 } 345 346 func (t *transferWriter) writeBody(w io.Writer) error { 347 var err error 348 var ncopy int64 349 350 // Write body 351 if t.Body != nil { 352 var body = transferBodyReader{t} 353 if chunked(t.TransferEncoding) { 354 if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse { 355 w = &internal.FlushAfterChunkWriter{Writer: bw} 356 } 357 cw := internal.NewChunkedWriter(w) 358 _, err = io.Copy(cw, body) 359 if err == nil { 360 err = cw.Close() 361 } 362 } else if t.ContentLength == -1 { 363 dst := w 364 if t.Method == "CONNECT" { 365 dst = bufioFlushWriter{dst} 366 } 367 ncopy, err = io.Copy(dst, body) 368 } else { 369 ncopy, err = io.Copy(w, io.LimitReader(body, t.ContentLength)) 370 if err != nil { 371 return err 372 } 373 var nextra int64 374 nextra, err = io.Copy(ioutil.Discard, body) 375 ncopy += nextra 376 } 377 if err != nil { 378 return err 379 } 380 } 381 if t.BodyCloser != nil { 382 if err := t.BodyCloser.Close(); err != nil { 383 return err 384 } 385 } 386 387 if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy { 388 return fmt.Errorf("http: ContentLength=%d with Body length %d", 389 t.ContentLength, ncopy) 390 } 391 392 if chunked(t.TransferEncoding) { 393 // Write Trailer header 394 if t.Trailer != nil { 395 if err := t.Trailer.Write(w); err != nil { 396 return err 397 } 398 } 399 // Last chunk, empty trailer 400 _, err = io.WriteString(w, "\r\n") 401 } 402 return err 403 } 404 405 type transferReader struct { 406 // Input 407 Header Header 408 StatusCode int 409 RequestMethod string 410 ProtoMajor int 411 ProtoMinor int 412 // Output 413 Body io.ReadCloser 414 ContentLength int64 415 TransferEncoding []string 416 Close bool 417 Trailer Header 418 } 419 420 func (t *transferReader) protoAtLeast(m, n int) bool { 421 return t.ProtoMajor > m || (t.ProtoMajor == m && t.ProtoMinor >= n) 422 } 423 424 // bodyAllowedForStatus reports whether a given response status code 425 // permits a body. See RFC 7230, section 3.3. 426 func bodyAllowedForStatus(status int) bool { 427 switch { 428 case status >= 100 && status <= 199: 429 return false 430 case status == 204: 431 return false 432 case status == 304: 433 return false 434 } 435 return true 436 } 437 438 var ( 439 suppressedHeaders304 = []string{"Content-Type", "Content-Length", "Transfer-Encoding"} 440 suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"} 441 ) 442 443 func suppressedHeaders(status int) []string { 444 switch { 445 case status == 304: 446 // RFC 7232 section 4.1 447 return suppressedHeaders304 448 case !bodyAllowedForStatus(status): 449 return suppressedHeadersNoBody 450 } 451 return nil 452 } 453 454 // msg is *Request or *Response. 455 func readTransfer(msg interface{}, r *bufio.Reader) (err error) { 456 t := &transferReader{RequestMethod: "GET"} 457 458 // Unify input 459 isResponse := false 460 switch rr := msg.(type) { 461 case *Response: 462 t.Header = rr.Header 463 t.StatusCode = rr.StatusCode 464 t.ProtoMajor = rr.ProtoMajor 465 t.ProtoMinor = rr.ProtoMinor 466 t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true) 467 isResponse = true 468 if rr.Request != nil { 469 t.RequestMethod = rr.Request.Method 470 } 471 case *Request: 472 t.Header = rr.Header 473 t.RequestMethod = rr.Method 474 t.ProtoMajor = rr.ProtoMajor 475 t.ProtoMinor = rr.ProtoMinor 476 // Transfer semantics for Requests are exactly like those for 477 // Responses with status code 200, responding to a GET method 478 t.StatusCode = 200 479 t.Close = rr.Close 480 default: 481 panic("unexpected type") 482 } 483 484 // Default to HTTP/1.1 485 if t.ProtoMajor == 0 && t.ProtoMinor == 0 { 486 t.ProtoMajor, t.ProtoMinor = 1, 1 487 } 488 489 // Transfer encoding, content length 490 err = t.fixTransferEncoding() 491 if err != nil { 492 return err 493 } 494 495 realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.TransferEncoding) 496 if err != nil { 497 return err 498 } 499 if isResponse && t.RequestMethod == "HEAD" { 500 if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil { 501 return err 502 } else { 503 t.ContentLength = n 504 } 505 } else { 506 t.ContentLength = realLength 507 } 508 509 // Trailer 510 t.Trailer, err = fixTrailer(t.Header, t.TransferEncoding) 511 if err != nil { 512 return err 513 } 514 515 // If there is no Content-Length or chunked Transfer-Encoding on a *Response 516 // and the status is not 1xx, 204 or 304, then the body is unbounded. 517 // See RFC 7230, section 3.3. 518 switch msg.(type) { 519 case *Response: 520 if realLength == -1 && 521 !chunked(t.TransferEncoding) && 522 bodyAllowedForStatus(t.StatusCode) { 523 // Unbounded body. 524 t.Close = true 525 } 526 } 527 528 // Prepare body reader. ContentLength < 0 means chunked encoding 529 // or close connection when finished, since multipart is not supported yet 530 switch { 531 case chunked(t.TransferEncoding): 532 if noResponseBodyExpected(t.RequestMethod) || !bodyAllowedForStatus(t.StatusCode) { 533 t.Body = NoBody 534 } else { 535 t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close} 536 } 537 case realLength == 0: 538 t.Body = NoBody 539 case realLength > 0: 540 t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close} 541 default: 542 // realLength < 0, i.e. "Content-Length" not mentioned in header 543 if t.Close { 544 // Close semantics (i.e. HTTP/1.0) 545 t.Body = &body{src: r, closing: t.Close} 546 } else { 547 // Persistent connection (i.e. HTTP/1.1) 548 t.Body = NoBody 549 } 550 } 551 552 // Unify output 553 switch rr := msg.(type) { 554 case *Request: 555 rr.Body = t.Body 556 rr.ContentLength = t.ContentLength 557 rr.TransferEncoding = t.TransferEncoding 558 rr.Close = t.Close 559 rr.Trailer = t.Trailer 560 case *Response: 561 rr.Body = t.Body 562 rr.ContentLength = t.ContentLength 563 rr.TransferEncoding = t.TransferEncoding 564 rr.Close = t.Close 565 rr.Trailer = t.Trailer 566 } 567 568 return nil 569 } 570 571 // Checks whether chunked is part of the encodings stack 572 func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } 573 574 // Checks whether the encoding is explicitly "identity". 575 func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } 576 577 // fixTransferEncoding sanitizes t.TransferEncoding, if needed. 578 func (t *transferReader) fixTransferEncoding() error { 579 raw, present := t.Header["Transfer-Encoding"] 580 if !present { 581 return nil 582 } 583 delete(t.Header, "Transfer-Encoding") 584 585 // Issue 12785; ignore Transfer-Encoding on HTTP/1.0 requests. 586 if !t.protoAtLeast(1, 1) { 587 return nil 588 } 589 590 encodings := strings.Split(raw[0], ",") 591 te := make([]string, 0, len(encodings)) 592 // TODO: Even though we only support "identity" and "chunked" 593 // encodings, the loop below is designed with foresight. One 594 // invariant that must be maintained is that, if present, 595 // chunked encoding must always come first. 596 for _, encoding := range encodings { 597 encoding = strings.ToLower(strings.TrimSpace(encoding)) 598 // "identity" encoding is not recorded 599 if encoding == "identity" { 600 break 601 } 602 if encoding != "chunked" { 603 return &badStringError{"unsupported transfer encoding", encoding} 604 } 605 te = te[0 : len(te)+1] 606 te[len(te)-1] = encoding 607 } 608 if len(te) > 1 { 609 return &badStringError{"too many transfer encodings", strings.Join(te, ",")} 610 } 611 if len(te) > 0 { 612 // RFC 7230 3.3.2 says "A sender MUST NOT send a 613 // Content-Length header field in any message that 614 // contains a Transfer-Encoding header field." 615 // 616 // but also: 617 // "If a message is received with both a 618 // Transfer-Encoding and a Content-Length header 619 // field, the Transfer-Encoding overrides the 620 // Content-Length. Such a message might indicate an 621 // attempt to perform request smuggling (Section 9.5) 622 // or response splitting (Section 9.4) and ought to be 623 // handled as an error. A sender MUST remove the 624 // received Content-Length field prior to forwarding 625 // such a message downstream." 626 // 627 // Reportedly, these appear in the wild. 628 delete(t.Header, "Content-Length") 629 t.TransferEncoding = te 630 return nil 631 } 632 633 return nil 634 } 635 636 // Determine the expected body length, using RFC 7230 Section 3.3. This 637 // function is not a method, because ultimately it should be shared by 638 // ReadResponse and ReadRequest. 639 func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) { 640 isRequest := !isResponse 641 contentLens := header["Content-Length"] 642 643 // Hardening against HTTP request smuggling 644 if len(contentLens) > 1 { 645 // Per RFC 7230 Section 3.3.2, prevent multiple 646 // Content-Length headers if they differ in value. 647 // If there are dups of the value, remove the dups. 648 // See Issue 16490. 649 first := strings.TrimSpace(contentLens[0]) 650 for _, ct := range contentLens[1:] { 651 if first != strings.TrimSpace(ct) { 652 return 0, fmt.Errorf("http: message cannot contain multiple Content-Length headers; got %q", contentLens) 653 } 654 } 655 656 // deduplicate Content-Length 657 header.Del("Content-Length") 658 header.Add("Content-Length", first) 659 660 contentLens = header["Content-Length"] 661 } 662 663 // Logic based on response type or status 664 if noResponseBodyExpected(requestMethod) { 665 // For HTTP requests, as part of hardening against request 666 // smuggling (RFC 7230), don't allow a Content-Length header for 667 // methods which don't permit bodies. As an exception, allow 668 // exactly one Content-Length header if its value is "0". 669 if isRequest && len(contentLens) > 0 && !(len(contentLens) == 1 && contentLens[0] == "0") { 670 return 0, fmt.Errorf("http: method cannot contain a Content-Length; got %q", contentLens) 671 } 672 return 0, nil 673 } 674 if status/100 == 1 { 675 return 0, nil 676 } 677 switch status { 678 case 204, 304: 679 return 0, nil 680 } 681 682 // Logic based on Transfer-Encoding 683 if chunked(te) { 684 return -1, nil 685 } 686 687 // Logic based on Content-Length 688 var cl string 689 if len(contentLens) == 1 { 690 cl = strings.TrimSpace(contentLens[0]) 691 } 692 if cl != "" { 693 n, err := parseContentLength(cl) 694 if err != nil { 695 return -1, err 696 } 697 return n, nil 698 } 699 header.Del("Content-Length") 700 701 if isRequest { 702 // RFC 7230 neither explicitly permits nor forbids an 703 // entity-body on a GET request so we permit one if 704 // declared, but we default to 0 here (not -1 below) 705 // if there's no mention of a body. 706 // Likewise, all other request methods are assumed to have 707 // no body if neither Transfer-Encoding chunked nor a 708 // Content-Length are set. 709 return 0, nil 710 } 711 712 // Body-EOF logic based on other methods (like closing, or chunked coding) 713 return -1, nil 714 } 715 716 // Determine whether to hang up after sending a request and body, or 717 // receiving a response and body 718 // 'header' is the request headers 719 func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool { 720 if major < 1 { 721 return true 722 } 723 724 conv := header["Connection"] 725 hasClose := httpguts.HeaderValuesContainsToken(conv, "close") 726 if major == 1 && minor == 0 { 727 return hasClose || !httpguts.HeaderValuesContainsToken(conv, "keep-alive") 728 } 729 730 if hasClose && removeCloseHeader { 731 header.Del("Connection") 732 } 733 734 return hasClose 735 } 736 737 // Parse the trailer header 738 func fixTrailer(header Header, te []string) (Header, error) { 739 vv, ok := header["Trailer"] 740 if !ok { 741 return nil, nil 742 } 743 if !chunked(te) { 744 // Trailer and no chunking: 745 // this is an invalid use case for trailer header. 746 // Nevertheless, no error will be returned and we 747 // let users decide if this is a valid HTTP message. 748 // The Trailer header will be kept in Response.Header 749 // but not populate Response.Trailer. 750 // See issue #27197. 751 return nil, nil 752 } 753 header.Del("Trailer") 754 755 trailer := make(Header) 756 var err error 757 for _, v := range vv { 758 foreachHeaderElement(v, func(key string) { 759 key = CanonicalHeaderKey(key) 760 switch key { 761 case "Transfer-Encoding", "Trailer", "Content-Length": 762 if err == nil { 763 err = &badStringError{"bad trailer key", key} 764 return 765 } 766 } 767 trailer[key] = nil 768 }) 769 } 770 if err != nil { 771 return nil, err 772 } 773 if len(trailer) == 0 { 774 return nil, nil 775 } 776 return trailer, nil 777 } 778 779 // body turns a Reader into a ReadCloser. 780 // Close ensures that the body has been fully read 781 // and then reads the trailer if necessary. 782 type body struct { 783 src io.Reader 784 hdr interface{} // non-nil (Response or Request) value means read trailer 785 r *bufio.Reader // underlying wire-format reader for the trailer 786 closing bool // is the connection to be closed after reading body? 787 doEarlyClose bool // whether Close should stop early 788 789 mu sync.Mutex // guards following, and calls to Read and Close 790 sawEOF bool 791 closed bool 792 earlyClose bool // Close called and we didn't read to the end of src 793 onHitEOF func() // if non-nil, func to call when EOF is Read 794 } 795 796 // ErrBodyReadAfterClose is returned when reading a Request or Response 797 // Body after the body has been closed. This typically happens when the body is 798 // read after an HTTP Handler calls WriteHeader or Write on its 799 // ResponseWriter. 800 var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body") 801 802 func (b *body) Read(p []byte) (n int, err error) { 803 b.mu.Lock() 804 defer b.mu.Unlock() 805 if b.closed { 806 return 0, ErrBodyReadAfterClose 807 } 808 return b.readLocked(p) 809 } 810 811 // Must hold b.mu. 812 func (b *body) readLocked(p []byte) (n int, err error) { 813 if b.sawEOF { 814 return 0, io.EOF 815 } 816 n, err = b.src.Read(p) 817 818 if err == io.EOF { 819 b.sawEOF = true 820 // Chunked case. Read the trailer. 821 if b.hdr != nil { 822 if e := b.readTrailer(); e != nil { 823 err = e 824 // Something went wrong in the trailer, we must not allow any 825 // further reads of any kind to succeed from body, nor any 826 // subsequent requests on the server connection. See 827 // golang.org/issue/12027 828 b.sawEOF = false 829 b.closed = true 830 } 831 b.hdr = nil 832 } else { 833 // If the server declared the Content-Length, our body is a LimitedReader 834 // and we need to check whether this EOF arrived early. 835 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 { 836 err = io.ErrUnexpectedEOF 837 } 838 } 839 } 840 841 // If we can return an EOF here along with the read data, do 842 // so. This is optional per the io.Reader contract, but doing 843 // so helps the HTTP transport code recycle its connection 844 // earlier (since it will see this EOF itself), even if the 845 // client doesn't do future reads or Close. 846 if err == nil && n > 0 { 847 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 { 848 err = io.EOF 849 b.sawEOF = true 850 } 851 } 852 853 if b.sawEOF && b.onHitEOF != nil { 854 b.onHitEOF() 855 } 856 857 return n, err 858 } 859 860 var ( 861 singleCRLF = []byte("\r\n") 862 doubleCRLF = []byte("\r\n\r\n") 863 ) 864 865 func seeUpcomingDoubleCRLF(r *bufio.Reader) bool { 866 for peekSize := 4; ; peekSize++ { 867 // This loop stops when Peek returns an error, 868 // which it does when r's buffer has been filled. 869 buf, err := r.Peek(peekSize) 870 if bytes.HasSuffix(buf, doubleCRLF) { 871 return true 872 } 873 if err != nil { 874 break 875 } 876 } 877 return false 878 } 879 880 var errTrailerEOF = errors.New("http: unexpected EOF reading trailer") 881 882 func (b *body) readTrailer() error { 883 // The common case, since nobody uses trailers. 884 buf, err := b.r.Peek(2) 885 if bytes.Equal(buf, singleCRLF) { 886 b.r.Discard(2) 887 return nil 888 } 889 if len(buf) < 2 { 890 return errTrailerEOF 891 } 892 if err != nil { 893 return err 894 } 895 896 // Make sure there's a header terminator coming up, to prevent 897 // a DoS with an unbounded size Trailer. It's not easy to 898 // slip in a LimitReader here, as textproto.NewReader requires 899 // a concrete *bufio.Reader. Also, we can't get all the way 900 // back up to our conn's LimitedReader that *might* be backing 901 // this bufio.Reader. Instead, a hack: we iteratively Peek up 902 // to the bufio.Reader's max size, looking for a double CRLF. 903 // This limits the trailer to the underlying buffer size, typically 4kB. 904 if !seeUpcomingDoubleCRLF(b.r) { 905 return errors.New("http: suspiciously long trailer after chunked body") 906 } 907 908 hdr, err := textproto.NewReader(b.r).ReadMIMEHeader() 909 if err != nil { 910 if err == io.EOF { 911 return errTrailerEOF 912 } 913 return err 914 } 915 switch rr := b.hdr.(type) { 916 case *Request: 917 mergeSetHeader(&rr.Trailer, Header(hdr)) 918 case *Response: 919 mergeSetHeader(&rr.Trailer, Header(hdr)) 920 } 921 return nil 922 } 923 924 func mergeSetHeader(dst *Header, src Header) { 925 if *dst == nil { 926 *dst = src 927 return 928 } 929 for k, vv := range src { 930 (*dst)[k] = vv 931 } 932 } 933 934 // unreadDataSizeLocked returns the number of bytes of unread input. 935 // It returns -1 if unknown. 936 // b.mu must be held. 937 func (b *body) unreadDataSizeLocked() int64 { 938 if lr, ok := b.src.(*io.LimitedReader); ok { 939 return lr.N 940 } 941 return -1 942 } 943 944 func (b *body) Close() error { 945 b.mu.Lock() 946 defer b.mu.Unlock() 947 if b.closed { 948 return nil 949 } 950 var err error 951 switch { 952 case b.sawEOF: 953 // Already saw EOF, so no need going to look for it. 954 case b.hdr == nil && b.closing: 955 // no trailer and closing the connection next. 956 // no point in reading to EOF. 957 case b.doEarlyClose: 958 // Read up to maxPostHandlerReadBytes bytes of the body, looking 959 // for EOF (and trailers), so we can re-use this connection. 960 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > maxPostHandlerReadBytes { 961 // There was a declared Content-Length, and we have more bytes remaining 962 // than our maxPostHandlerReadBytes tolerance. So, give up. 963 b.earlyClose = true 964 } else { 965 var n int64 966 // Consume the body, or, which will also lead to us reading 967 // the trailer headers after the body, if present. 968 n, err = io.CopyN(ioutil.Discard, bodyLocked{b}, maxPostHandlerReadBytes) 969 if err == io.EOF { 970 err = nil 971 } 972 if n == maxPostHandlerReadBytes { 973 b.earlyClose = true 974 } 975 } 976 default: 977 // Fully consume the body, which will also lead to us reading 978 // the trailer headers after the body, if present. 979 _, err = io.Copy(ioutil.Discard, bodyLocked{b}) 980 } 981 b.closed = true 982 return err 983 } 984 985 func (b *body) didEarlyClose() bool { 986 b.mu.Lock() 987 defer b.mu.Unlock() 988 return b.earlyClose 989 } 990 991 // bodyRemains reports whether future Read calls might 992 // yield data. 993 func (b *body) bodyRemains() bool { 994 b.mu.Lock() 995 defer b.mu.Unlock() 996 return !b.sawEOF 997 } 998 999 func (b *body) registerOnHitEOF(fn func()) { 1000 b.mu.Lock() 1001 defer b.mu.Unlock() 1002 b.onHitEOF = fn 1003 } 1004 1005 // bodyLocked is a io.Reader reading from a *body when its mutex is 1006 // already held. 1007 type bodyLocked struct { 1008 b *body 1009 } 1010 1011 func (bl bodyLocked) Read(p []byte) (n int, err error) { 1012 if bl.b.closed { 1013 return 0, ErrBodyReadAfterClose 1014 } 1015 return bl.b.readLocked(p) 1016 } 1017 1018 // parseContentLength trims whitespace from s and returns -1 if no value 1019 // is set, or the value if it's >= 0. 1020 func parseContentLength(cl string) (int64, error) { 1021 cl = strings.TrimSpace(cl) 1022 if cl == "" { 1023 return -1, nil 1024 } 1025 n, err := strconv.ParseInt(cl, 10, 64) 1026 if err != nil || n < 0 { 1027 return 0, &badStringError{"bad Content-Length", cl} 1028 } 1029 return n, nil 1030 1031 } 1032 1033 // finishAsyncByteRead finishes reading the 1-byte sniff 1034 // from the ContentLength==0, Body!=nil case. 1035 type finishAsyncByteRead struct { 1036 tw *transferWriter 1037 } 1038 1039 func (fr finishAsyncByteRead) Read(p []byte) (n int, err error) { 1040 if len(p) == 0 { 1041 return 1042 } 1043 rres := <-fr.tw.ByteReadCh 1044 n, err = rres.n, rres.err 1045 if n == 1 { 1046 p[0] = rres.b 1047 } 1048 return 1049 } 1050 1051 var nopCloserType = reflect.TypeOf(ioutil.NopCloser(nil)) 1052 1053 // isKnownInMemoryReader reports whether r is a type known to not 1054 // block on Read. Its caller uses this as an optional optimization to 1055 // send fewer TCP packets. 1056 func isKnownInMemoryReader(r io.Reader) bool { 1057 switch r.(type) { 1058 case *bytes.Reader, *bytes.Buffer, *strings.Reader: 1059 return true 1060 } 1061 if reflect.TypeOf(r) == nopCloserType { 1062 return isKnownInMemoryReader(reflect.ValueOf(r).Field(0).Interface().(io.Reader)) 1063 } 1064 return false 1065 } 1066 1067 // bufioFlushWriter is an io.Writer wrapper that flushes all writes 1068 // on its wrapped writer if it's a *bufio.Writer. 1069 type bufioFlushWriter struct{ w io.Writer } 1070 1071 func (fw bufioFlushWriter) Write(p []byte) (n int, err error) { 1072 n, err = fw.w.Write(p) 1073 if bw, ok := fw.w.(*bufio.Writer); n > 0 && ok { 1074 ferr := bw.Flush() 1075 if ferr != nil && err == nil { 1076 err = ferr 1077 } 1078 } 1079 return 1080 } 1081
View as plain text