1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // HTTP Request reading and parsing. 6 7 package http 8 9 import ( 10 "bufio" 11 "bytes" 12 "context" 13 "crypto/tls" 14 "encoding/base64" 15 "errors" 16 "fmt" 17 "io" 18 "io/ioutil" 19 "mime" 20 "mime/multipart" 21 "net" 22 "net/http/httptrace" 23 "net/textproto" 24 "net/url" 25 "strconv" 26 "strings" 27 "sync" 28 29 "golang_org/x/net/idna" 30 ) 31 32 const ( 33 defaultMaxMemory = 32 << 20 // 32 MB 34 ) 35 36 // ErrMissingFile is returned by FormFile when the provided file field name 37 // is either not present in the request or not a file field. 38 var ErrMissingFile = errors.New("http: no such file") 39 40 // ProtocolError represents an HTTP protocol error. 41 // 42 // Deprecated: Not all errors in the http package related to protocol errors 43 // are of type ProtocolError. 44 type ProtocolError struct { 45 ErrorString string 46 } 47 48 func (pe *ProtocolError) Error() string { return pe.ErrorString } 49 50 var ( 51 // ErrNotSupported is returned by the Push method of Pusher 52 // implementations to indicate that HTTP/2 Push support is not 53 // available. 54 ErrNotSupported = &ProtocolError{"feature not supported"} 55 56 // ErrUnexpectedTrailer is returned by the Transport when a server 57 // replies with a Trailer header, but without a chunked reply. 58 ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"} 59 60 // ErrMissingBoundary is returned by Request.MultipartReader when the 61 // request's Content-Type does not include a "boundary" parameter. 62 ErrMissingBoundary = &ProtocolError{"no multipart boundary param in Content-Type"} 63 64 // ErrNotMultipart is returned by Request.MultipartReader when the 65 // request's Content-Type is not multipart/form-data. 66 ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"} 67 68 // Deprecated: ErrHeaderTooLong is no longer returned by 69 // anything in the net/http package. Callers should not 70 // compare errors against this variable. 71 ErrHeaderTooLong = &ProtocolError{"header too long"} 72 73 // Deprecated: ErrShortBody is no longer returned by 74 // anything in the net/http package. Callers should not 75 // compare errors against this variable. 76 ErrShortBody = &ProtocolError{"entity body too short"} 77 78 // Deprecated: ErrMissingContentLength is no longer returned by 79 // anything in the net/http package. Callers should not 80 // compare errors against this variable. 81 ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"} 82 ) 83 84 type badStringError struct { 85 what string 86 str string 87 } 88 89 func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) } 90 91 // Headers that Request.Write handles itself and should be skipped. 92 var reqWriteExcludeHeader = map[string]bool{ 93 "Host": true, // not in Header map anyway 94 "User-Agent": true, 95 "Content-Length": true, 96 "Transfer-Encoding": true, 97 "Trailer": true, 98 } 99 100 // A Request represents an HTTP request received by a server 101 // or to be sent by a client. 102 // 103 // The field semantics differ slightly between client and server 104 // usage. In addition to the notes on the fields below, see the 105 // documentation for Request.Write and RoundTripper. 106 type Request struct { 107 // Method specifies the HTTP method (GET, POST, PUT, etc.). 108 // For client requests an empty string means GET. 109 // 110 // Go's HTTP client does not support sending a request with 111 // the CONNECT method. See the documentation on Transport for 112 // details. 113 Method string 114 115 // URL specifies either the URI being requested (for server 116 // requests) or the URL to access (for client requests). 117 // 118 // For server requests the URL is parsed from the URI 119 // supplied on the Request-Line as stored in RequestURI. For 120 // most requests, fields other than Path and RawQuery will be 121 // empty. (See RFC 7230, Section 5.3) 122 // 123 // For client requests, the URL's Host specifies the server to 124 // connect to, while the Request's Host field optionally 125 // specifies the Host header value to send in the HTTP 126 // request. 127 URL *url.URL 128 129 // The protocol version for incoming server requests. 130 // 131 // For client requests these fields are ignored. The HTTP 132 // client code always uses either HTTP/1.1 or HTTP/2. 133 // See the docs on Transport for details. 134 Proto string // "HTTP/1.0" 135 ProtoMajor int // 1 136 ProtoMinor int // 0 137 138 // Header contains the request header fields either received 139 // by the server or to be sent by the client. 140 // 141 // If a server received a request with header lines, 142 // 143 // Host: example.com 144 // accept-encoding: gzip, deflate 145 // Accept-Language: en-us 146 // fOO: Bar 147 // foo: two 148 // 149 // then 150 // 151 // Header = map[string][]string{ 152 // "Accept-Encoding": {"gzip, deflate"}, 153 // "Accept-Language": {"en-us"}, 154 // "Foo": {"Bar", "two"}, 155 // } 156 // 157 // For incoming requests, the Host header is promoted to the 158 // Request.Host field and removed from the Header map. 159 // 160 // HTTP defines that header names are case-insensitive. The 161 // request parser implements this by using CanonicalHeaderKey, 162 // making the first character and any characters following a 163 // hyphen uppercase and the rest lowercase. 164 // 165 // For client requests, certain headers such as Content-Length 166 // and Connection are automatically written when needed and 167 // values in Header may be ignored. See the documentation 168 // for the Request.Write method. 169 Header Header 170 171 // Body is the request's body. 172 // 173 // For client requests a nil body means the request has no 174 // body, such as a GET request. The HTTP Client's Transport 175 // is responsible for calling the Close method. 176 // 177 // For server requests the Request Body is always non-nil 178 // but will return EOF immediately when no body is present. 179 // The Server will close the request body. The ServeHTTP 180 // Handler does not need to. 181 Body io.ReadCloser 182 183 // GetBody defines an optional func to return a new copy of 184 // Body. It is used for client requests when a redirect requires 185 // reading the body more than once. Use of GetBody still 186 // requires setting Body. 187 // 188 // For server requests it is unused. 189 GetBody func() (io.ReadCloser, error) 190 191 // ContentLength records the length of the associated content. 192 // The value -1 indicates that the length is unknown. 193 // Values >= 0 indicate that the given number of bytes may 194 // be read from Body. 195 // For client requests, a value of 0 with a non-nil Body is 196 // also treated as unknown. 197 ContentLength int64 198 199 // TransferEncoding lists the transfer encodings from outermost to 200 // innermost. An empty list denotes the "identity" encoding. 201 // TransferEncoding can usually be ignored; chunked encoding is 202 // automatically added and removed as necessary when sending and 203 // receiving requests. 204 TransferEncoding []string 205 206 // Close indicates whether to close the connection after 207 // replying to this request (for servers) or after sending this 208 // request and reading its response (for clients). 209 // 210 // For server requests, the HTTP server handles this automatically 211 // and this field is not needed by Handlers. 212 // 213 // For client requests, setting this field prevents re-use of 214 // TCP connections between requests to the same hosts, as if 215 // Transport.DisableKeepAlives were set. 216 Close bool 217 218 // For server requests Host specifies the host on which the URL 219 // is sought. Per RFC 7230, section 5.4, this is either the value 220 // of the "Host" header or the host name given in the URL itself. 221 // It may be of the form "host:port". For international domain 222 // names, Host may be in Punycode or Unicode form. Use 223 // golang.org/x/net/idna to convert it to either format if 224 // needed. 225 // To prevent DNS rebinding attacks, server Handlers should 226 // validate that the Host header has a value for which the 227 // Handler considers itself authoritative. The included 228 // ServeMux supports patterns registered to particular host 229 // names and thus protects its registered Handlers. 230 // 231 // For client requests Host optionally overrides the Host 232 // header to send. If empty, the Request.Write method uses 233 // the value of URL.Host. Host may contain an international 234 // domain name. 235 Host string 236 237 // Form contains the parsed form data, including both the URL 238 // field's query parameters and the POST or PUT form data. 239 // This field is only available after ParseForm is called. 240 // The HTTP client ignores Form and uses Body instead. 241 Form url.Values 242 243 // PostForm contains the parsed form data from POST, PATCH, 244 // or PUT body parameters. 245 // 246 // This field is only available after ParseForm is called. 247 // The HTTP client ignores PostForm and uses Body instead. 248 PostForm url.Values 249 250 // MultipartForm is the parsed multipart form, including file uploads. 251 // This field is only available after ParseMultipartForm is called. 252 // The HTTP client ignores MultipartForm and uses Body instead. 253 MultipartForm *multipart.Form 254 255 // Trailer specifies additional headers that are sent after the request 256 // body. 257 // 258 // For server requests the Trailer map initially contains only the 259 // trailer keys, with nil values. (The client declares which trailers it 260 // will later send.) While the handler is reading from Body, it must 261 // not reference Trailer. After reading from Body returns EOF, Trailer 262 // can be read again and will contain non-nil values, if they were sent 263 // by the client. 264 // 265 // For client requests Trailer must be initialized to a map containing 266 // the trailer keys to later send. The values may be nil or their final 267 // values. The ContentLength must be 0 or -1, to send a chunked request. 268 // After the HTTP request is sent the map values can be updated while 269 // the request body is read. Once the body returns EOF, the caller must 270 // not mutate Trailer. 271 // 272 // Few HTTP clients, servers, or proxies support HTTP trailers. 273 Trailer Header 274 275 // RemoteAddr allows HTTP servers and other software to record 276 // the network address that sent the request, usually for 277 // logging. This field is not filled in by ReadRequest and 278 // has no defined format. The HTTP server in this package 279 // sets RemoteAddr to an "IP:port" address before invoking a 280 // handler. 281 // This field is ignored by the HTTP client. 282 RemoteAddr string 283 284 // RequestURI is the unmodified request-target of the 285 // Request-Line (RFC 7230, Section 3.1.1) as sent by the client 286 // to a server. Usually the URL field should be used instead. 287 // It is an error to set this field in an HTTP client request. 288 RequestURI string 289 290 // TLS allows HTTP servers and other software to record 291 // information about the TLS connection on which the request 292 // was received. This field is not filled in by ReadRequest. 293 // The HTTP server in this package sets the field for 294 // TLS-enabled connections before invoking a handler; 295 // otherwise it leaves the field nil. 296 // This field is ignored by the HTTP client. 297 TLS *tls.ConnectionState 298 299 // Cancel is an optional channel whose closure indicates that the client 300 // request should be regarded as canceled. Not all implementations of 301 // RoundTripper may support Cancel. 302 // 303 // For server requests, this field is not applicable. 304 // 305 // Deprecated: Use the Context and WithContext methods 306 // instead. If a Request's Cancel field and context are both 307 // set, it is undefined whether Cancel is respected. 308 Cancel <-chan struct{} 309 310 // Response is the redirect response which caused this request 311 // to be created. This field is only populated during client 312 // redirects. 313 Response *Response 314 315 // ctx is either the client or server context. It should only 316 // be modified via copying the whole Request using WithContext. 317 // It is unexported to prevent people from using Context wrong 318 // and mutating the contexts held by callers of the same request. 319 ctx context.Context 320 } 321 322 // Context returns the request's context. To change the context, use 323 // WithContext. 324 // 325 // The returned context is always non-nil; it defaults to the 326 // background context. 327 // 328 // For outgoing client requests, the context controls cancelation. 329 // 330 // For incoming server requests, the context is canceled when the 331 // client's connection closes, the request is canceled (with HTTP/2), 332 // or when the ServeHTTP method returns. 333 func (r *Request) Context() context.Context { 334 if r.ctx != nil { 335 return r.ctx 336 } 337 return context.Background() 338 } 339 340 // WithContext returns a shallow copy of r with its context changed 341 // to ctx. The provided ctx must be non-nil. 342 // 343 // For outgoing client request, the context controls the entire 344 // lifetime of a request and its response: obtaining a connection, 345 // sending the request, and reading the response headers and body. 346 func (r *Request) WithContext(ctx context.Context) *Request { 347 if ctx == nil { 348 panic("nil context") 349 } 350 r2 := new(Request) 351 *r2 = *r 352 r2.ctx = ctx 353 354 // Deep copy the URL because it isn't 355 // a map and the URL is mutable by users 356 // of WithContext. 357 if r.URL != nil { 358 r2URL := new(url.URL) 359 *r2URL = *r.URL 360 r2.URL = r2URL 361 } 362 363 return r2 364 } 365 366 // ProtoAtLeast reports whether the HTTP protocol used 367 // in the request is at least major.minor. 368 func (r *Request) ProtoAtLeast(major, minor int) bool { 369 return r.ProtoMajor > major || 370 r.ProtoMajor == major && r.ProtoMinor >= minor 371 } 372 373 // UserAgent returns the client's User-Agent, if sent in the request. 374 func (r *Request) UserAgent() string { 375 return r.Header.Get("User-Agent") 376 } 377 378 // Cookies parses and returns the HTTP cookies sent with the request. 379 func (r *Request) Cookies() []*Cookie { 380 return readCookies(r.Header, "") 381 } 382 383 // ErrNoCookie is returned by Request's Cookie method when a cookie is not found. 384 var ErrNoCookie = errors.New("http: named cookie not present") 385 386 // Cookie returns the named cookie provided in the request or 387 // ErrNoCookie if not found. 388 // If multiple cookies match the given name, only one cookie will 389 // be returned. 390 func (r *Request) Cookie(name string) (*Cookie, error) { 391 for _, c := range readCookies(r.Header, name) { 392 return c, nil 393 } 394 return nil, ErrNoCookie 395 } 396 397 // AddCookie adds a cookie to the request. Per RFC 6265 section 5.4, 398 // AddCookie does not attach more than one Cookie header field. That 399 // means all cookies, if any, are written into the same line, 400 // separated by semicolon. 401 func (r *Request) AddCookie(c *Cookie) { 402 s := fmt.Sprintf("%s=%s", sanitizeCookieName(c.Name), sanitizeCookieValue(c.Value)) 403 if c := r.Header.Get("Cookie"); c != "" { 404 r.Header.Set("Cookie", c+"; "+s) 405 } else { 406 r.Header.Set("Cookie", s) 407 } 408 } 409 410 // Referer returns the referring URL, if sent in the request. 411 // 412 // Referer is misspelled as in the request itself, a mistake from the 413 // earliest days of HTTP. This value can also be fetched from the 414 // Header map as Header["Referer"]; the benefit of making it available 415 // as a method is that the compiler can diagnose programs that use the 416 // alternate (correct English) spelling req.Referrer() but cannot 417 // diagnose programs that use Header["Referrer"]. 418 func (r *Request) Referer() string { 419 return r.Header.Get("Referer") 420 } 421 422 // multipartByReader is a sentinel value. 423 // Its presence in Request.MultipartForm indicates that parsing of the request 424 // body has been handed off to a MultipartReader instead of ParseMultipartFrom. 425 var multipartByReader = &multipart.Form{ 426 Value: make(map[string][]string), 427 File: make(map[string][]*multipart.FileHeader), 428 } 429 430 // MultipartReader returns a MIME multipart reader if this is a 431 // multipart/form-data or a multipart/mixed POST request, else returns nil and an error. 432 // Use this function instead of ParseMultipartForm to 433 // process the request body as a stream. 434 func (r *Request) MultipartReader() (*multipart.Reader, error) { 435 if r.MultipartForm == multipartByReader { 436 return nil, errors.New("http: MultipartReader called twice") 437 } 438 if r.MultipartForm != nil { 439 return nil, errors.New("http: multipart handled by ParseMultipartForm") 440 } 441 r.MultipartForm = multipartByReader 442 return r.multipartReader(true) 443 } 444 445 func (r *Request) multipartReader(allowMixed bool) (*multipart.Reader, error) { 446 v := r.Header.Get("Content-Type") 447 if v == "" { 448 return nil, ErrNotMultipart 449 } 450 d, params, err := mime.ParseMediaType(v) 451 if err != nil || !(d == "multipart/form-data" || allowMixed && d == "multipart/mixed") { 452 return nil, ErrNotMultipart 453 } 454 boundary, ok := params["boundary"] 455 if !ok { 456 return nil, ErrMissingBoundary 457 } 458 return multipart.NewReader(r.Body, boundary), nil 459 } 460 461 // isH2Upgrade reports whether r represents the http2 "client preface" 462 // magic string. 463 func (r *Request) isH2Upgrade() bool { 464 return r.Method == "PRI" && len(r.Header) == 0 && r.URL.Path == "*" && r.Proto == "HTTP/2.0" 465 } 466 467 // Return value if nonempty, def otherwise. 468 func valueOrDefault(value, def string) string { 469 if value != "" { 470 return value 471 } 472 return def 473 } 474 475 // NOTE: This is not intended to reflect the actual Go version being used. 476 // It was changed at the time of Go 1.1 release because the former User-Agent 477 // had ended up on a blacklist for some intrusion detection systems. 478 // See https://codereview.appspot.com/7532043. 479 const defaultUserAgent = "Go-http-client/1.1" 480 481 // Write writes an HTTP/1.1 request, which is the header and body, in wire format. 482 // This method consults the following fields of the request: 483 // Host 484 // URL 485 // Method (defaults to "GET") 486 // Header 487 // ContentLength 488 // TransferEncoding 489 // Body 490 // 491 // If Body is present, Content-Length is <= 0 and TransferEncoding 492 // hasn't been set to "identity", Write adds "Transfer-Encoding: 493 // chunked" to the header. Body is closed after it is sent. 494 func (r *Request) Write(w io.Writer) error { 495 return r.write(w, false, nil, nil) 496 } 497 498 // WriteProxy is like Write but writes the request in the form 499 // expected by an HTTP proxy. In particular, WriteProxy writes the 500 // initial Request-URI line of the request with an absolute URI, per 501 // section 5.3 of RFC 7230, including the scheme and host. 502 // In either case, WriteProxy also writes a Host header, using 503 // either r.Host or r.URL.Host. 504 func (r *Request) WriteProxy(w io.Writer) error { 505 return r.write(w, true, nil, nil) 506 } 507 508 // errMissingHost is returned by Write when there is no Host or URL present in 509 // the Request. 510 var errMissingHost = errors.New("http: Request.Write on Request with no Host or URL set") 511 512 // extraHeaders may be nil 513 // waitForContinue may be nil 514 func (r *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, waitForContinue func() bool) (err error) { 515 trace := httptrace.ContextClientTrace(r.Context()) 516 if trace != nil && trace.WroteRequest != nil { 517 defer func() { 518 trace.WroteRequest(httptrace.WroteRequestInfo{ 519 Err: err, 520 }) 521 }() 522 } 523 524 // Find the target host. Prefer the Host: header, but if that 525 // is not given, use the host from the request URL. 526 // 527 // Clean the host, in case it arrives with unexpected stuff in it. 528 host := cleanHost(r.Host) 529 if host == "" { 530 if r.URL == nil { 531 return errMissingHost 532 } 533 host = cleanHost(r.URL.Host) 534 } 535 536 // According to RFC 6874, an HTTP client, proxy, or other 537 // intermediary must remove any IPv6 zone identifier attached 538 // to an outgoing URI. 539 host = removeZone(host) 540 541 ruri := r.URL.RequestURI() 542 if usingProxy && r.URL.Scheme != "" && r.URL.Opaque == "" { 543 ruri = r.URL.Scheme + "://" + host + ruri 544 } else if r.Method == "CONNECT" && r.URL.Path == "" { 545 // CONNECT requests normally give just the host and port, not a full URL. 546 ruri = host 547 } 548 if stringContainsCTLByte(ruri) { 549 return errors.New("net/http: can't write control character in Request.URL") 550 } 551 // TODO: validate r.Method too? At least it's less likely to 552 // come from an attacker (more likely to be a constant in 553 // code). 554 555 // Wrap the writer in a bufio Writer if it's not already buffered. 556 // Don't always call NewWriter, as that forces a bytes.Buffer 557 // and other small bufio Writers to have a minimum 4k buffer 558 // size. 559 var bw *bufio.Writer 560 if _, ok := w.(io.ByteWriter); !ok { 561 bw = bufio.NewWriter(w) 562 w = bw 563 } 564 565 _, err = fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(r.Method, "GET"), ruri) 566 if err != nil { 567 return err 568 } 569 570 // Header lines 571 _, err = fmt.Fprintf(w, "Host: %s\r\n", host) 572 if err != nil { 573 return err 574 } 575 if trace != nil && trace.WroteHeaderField != nil { 576 trace.WroteHeaderField("Host", []string{host}) 577 } 578 579 // Use the defaultUserAgent unless the Header contains one, which 580 // may be blank to not send the header. 581 userAgent := defaultUserAgent 582 if _, ok := r.Header["User-Agent"]; ok { 583 userAgent = r.Header.Get("User-Agent") 584 } 585 if userAgent != "" { 586 _, err = fmt.Fprintf(w, "User-Agent: %s\r\n", userAgent) 587 if err != nil { 588 return err 589 } 590 if trace != nil && trace.WroteHeaderField != nil { 591 trace.WroteHeaderField("User-Agent", []string{userAgent}) 592 } 593 } 594 595 // Process Body,ContentLength,Close,Trailer 596 tw, err := newTransferWriter(r) 597 if err != nil { 598 return err 599 } 600 err = tw.writeHeader(w, trace) 601 if err != nil { 602 return err 603 } 604 605 err = r.Header.writeSubset(w, reqWriteExcludeHeader, trace) 606 if err != nil { 607 return err 608 } 609 610 if extraHeaders != nil { 611 err = extraHeaders.write(w, trace) 612 if err != nil { 613 return err 614 } 615 } 616 617 _, err = io.WriteString(w, "\r\n") 618 if err != nil { 619 return err 620 } 621 622 if trace != nil && trace.WroteHeaders != nil { 623 trace.WroteHeaders() 624 } 625 626 // Flush and wait for 100-continue if expected. 627 if waitForContinue != nil { 628 if bw, ok := w.(*bufio.Writer); ok { 629 err = bw.Flush() 630 if err != nil { 631 return err 632 } 633 } 634 if trace != nil && trace.Wait100Continue != nil { 635 trace.Wait100Continue() 636 } 637 if !waitForContinue() { 638 r.closeBody() 639 return nil 640 } 641 } 642 643 if bw, ok := w.(*bufio.Writer); ok && tw.FlushHeaders { 644 if err := bw.Flush(); err != nil { 645 return err 646 } 647 } 648 649 // Write body and trailer 650 err = tw.writeBody(w) 651 if err != nil { 652 if tw.bodyReadError == err { 653 err = requestBodyReadError{err} 654 } 655 return err 656 } 657 658 if bw != nil { 659 return bw.Flush() 660 } 661 return nil 662 } 663 664 // requestBodyReadError wraps an error from (*Request).write to indicate 665 // that the error came from a Read call on the Request.Body. 666 // This error type should not escape the net/http package to users. 667 type requestBodyReadError struct{ error } 668 669 func idnaASCII(v string) (string, error) { 670 // TODO: Consider removing this check after verifying performance is okay. 671 // Right now punycode verification, length checks, context checks, and the 672 // permissible character tests are all omitted. It also prevents the ToASCII 673 // call from salvaging an invalid IDN, when possible. As a result it may be 674 // possible to have two IDNs that appear identical to the user where the 675 // ASCII-only version causes an error downstream whereas the non-ASCII 676 // version does not. 677 // Note that for correct ASCII IDNs ToASCII will only do considerably more 678 // work, but it will not cause an allocation. 679 if isASCII(v) { 680 return v, nil 681 } 682 return idna.Lookup.ToASCII(v) 683 } 684 685 // cleanHost cleans up the host sent in request's Host header. 686 // 687 // It both strips anything after '/' or ' ', and puts the value 688 // into Punycode form, if necessary. 689 // 690 // Ideally we'd clean the Host header according to the spec: 691 // https://tools.ietf.org/html/rfc7230#section-5.4 (Host = uri-host [ ":" port ]") 692 // https://tools.ietf.org/html/rfc7230#section-2.7 (uri-host -> rfc3986's host) 693 // https://tools.ietf.org/html/rfc3986#section-3.2.2 (definition of host) 694 // But practically, what we are trying to avoid is the situation in 695 // issue 11206, where a malformed Host header used in the proxy context 696 // would create a bad request. So it is enough to just truncate at the 697 // first offending character. 698 func cleanHost(in string) string { 699 if i := strings.IndexAny(in, " /"); i != -1 { 700 in = in[:i] 701 } 702 host, port, err := net.SplitHostPort(in) 703 if err != nil { // input was just a host 704 a, err := idnaASCII(in) 705 if err != nil { 706 return in // garbage in, garbage out 707 } 708 return a 709 } 710 a, err := idnaASCII(host) 711 if err != nil { 712 return in // garbage in, garbage out 713 } 714 return net.JoinHostPort(a, port) 715 } 716 717 // removeZone removes IPv6 zone identifier from host. 718 // E.g., "[fe80::1%en0]:8080" to "[fe80::1]:8080" 719 func removeZone(host string) string { 720 if !strings.HasPrefix(host, "[") { 721 return host 722 } 723 i := strings.LastIndex(host, "]") 724 if i < 0 { 725 return host 726 } 727 j := strings.LastIndex(host[:i], "%") 728 if j < 0 { 729 return host 730 } 731 return host[:j] + host[i:] 732 } 733 734 // ParseHTTPVersion parses a HTTP version string. 735 // "HTTP/1.0" returns (1, 0, true). 736 func ParseHTTPVersion(vers string) (major, minor int, ok bool) { 737 const Big = 1000000 // arbitrary upper bound 738 switch vers { 739 case "HTTP/1.1": 740 return 1, 1, true 741 case "HTTP/1.0": 742 return 1, 0, true 743 } 744 if !strings.HasPrefix(vers, "HTTP/") { 745 return 0, 0, false 746 } 747 dot := strings.Index(vers, ".") 748 if dot < 0 { 749 return 0, 0, false 750 } 751 major, err := strconv.Atoi(vers[5:dot]) 752 if err != nil || major < 0 || major > Big { 753 return 0, 0, false 754 } 755 minor, err = strconv.Atoi(vers[dot+1:]) 756 if err != nil || minor < 0 || minor > Big { 757 return 0, 0, false 758 } 759 return major, minor, true 760 } 761 762 func validMethod(method string) bool { 763 /* 764 Method = "OPTIONS" ; Section 9.2 765 | "GET" ; Section 9.3 766 | "HEAD" ; Section 9.4 767 | "POST" ; Section 9.5 768 | "PUT" ; Section 9.6 769 | "DELETE" ; Section 9.7 770 | "TRACE" ; Section 9.8 771 | "CONNECT" ; Section 9.9 772 | extension-method 773 extension-method = token 774 token = 1*<any CHAR except CTLs or separators> 775 */ 776 return len(method) > 0 && strings.IndexFunc(method, isNotToken) == -1 777 } 778 779 // NewRequest returns a new Request given a method, URL, and optional body. 780 // 781 // If the provided body is also an io.Closer, the returned 782 // Request.Body is set to body and will be closed by the Client 783 // methods Do, Post, and PostForm, and Transport.RoundTrip. 784 // 785 // NewRequest returns a Request suitable for use with Client.Do or 786 // Transport.RoundTrip. To create a request for use with testing a 787 // Server Handler, either use the NewRequest function in the 788 // net/http/httptest package, use ReadRequest, or manually update the 789 // Request fields. See the Request type's documentation for the 790 // difference between inbound and outbound request fields. 791 // 792 // If body is of type *bytes.Buffer, *bytes.Reader, or 793 // *strings.Reader, the returned request's ContentLength is set to its 794 // exact value (instead of -1), GetBody is populated (so 307 and 308 795 // redirects can replay the body), and Body is set to NoBody if the 796 // ContentLength is 0. 797 func NewRequest(method, url string, body io.Reader) (*Request, error) { 798 if method == "" { 799 // We document that "" means "GET" for Request.Method, and people have 800 // relied on that from NewRequest, so keep that working. 801 // We still enforce validMethod for non-empty methods. 802 method = "GET" 803 } 804 if !validMethod(method) { 805 return nil, fmt.Errorf("net/http: invalid method %q", method) 806 } 807 u, err := parseURL(url) // Just url.Parse (url is shadowed for godoc). 808 if err != nil { 809 return nil, err 810 } 811 rc, ok := body.(io.ReadCloser) 812 if !ok && body != nil { 813 rc = ioutil.NopCloser(body) 814 } 815 // The host's colon:port should be normalized. See Issue 14836. 816 u.Host = removeEmptyPort(u.Host) 817 req := &Request{ 818 Method: method, 819 URL: u, 820 Proto: "HTTP/1.1", 821 ProtoMajor: 1, 822 ProtoMinor: 1, 823 Header: make(Header), 824 Body: rc, 825 Host: u.Host, 826 } 827 if body != nil { 828 switch v := body.(type) { 829 case *bytes.Buffer: 830 req.ContentLength = int64(v.Len()) 831 buf := v.Bytes() 832 req.GetBody = func() (io.ReadCloser, error) { 833 r := bytes.NewReader(buf) 834 return ioutil.NopCloser(r), nil 835 } 836 case *bytes.Reader: 837 req.ContentLength = int64(v.Len()) 838 snapshot := *v 839 req.GetBody = func() (io.ReadCloser, error) { 840 r := snapshot 841 return ioutil.NopCloser(&r), nil 842 } 843 case *strings.Reader: 844 req.ContentLength = int64(v.Len()) 845 snapshot := *v 846 req.GetBody = func() (io.ReadCloser, error) { 847 r := snapshot 848 return ioutil.NopCloser(&r), nil 849 } 850 default: 851 // This is where we'd set it to -1 (at least 852 // if body != NoBody) to mean unknown, but 853 // that broke people during the Go 1.8 testing 854 // period. People depend on it being 0 I 855 // guess. Maybe retry later. See Issue 18117. 856 } 857 // For client requests, Request.ContentLength of 0 858 // means either actually 0, or unknown. The only way 859 // to explicitly say that the ContentLength is zero is 860 // to set the Body to nil. But turns out too much code 861 // depends on NewRequest returning a non-nil Body, 862 // so we use a well-known ReadCloser variable instead 863 // and have the http package also treat that sentinel 864 // variable to mean explicitly zero. 865 if req.GetBody != nil && req.ContentLength == 0 { 866 req.Body = NoBody 867 req.GetBody = func() (io.ReadCloser, error) { return NoBody, nil } 868 } 869 } 870 871 return req, nil 872 } 873 874 // BasicAuth returns the username and password provided in the request's 875 // Authorization header, if the request uses HTTP Basic Authentication. 876 // See RFC 2617, Section 2. 877 func (r *Request) BasicAuth() (username, password string, ok bool) { 878 auth := r.Header.Get("Authorization") 879 if auth == "" { 880 return 881 } 882 return parseBasicAuth(auth) 883 } 884 885 // parseBasicAuth parses an HTTP Basic Authentication string. 886 // "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" returns ("Aladdin", "open sesame", true). 887 func parseBasicAuth(auth string) (username, password string, ok bool) { 888 const prefix = "Basic " 889 // Case insensitive prefix match. See Issue 22736. 890 if len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix) { 891 return 892 } 893 c, err := base64.StdEncoding.DecodeString(auth[len(prefix):]) 894 if err != nil { 895 return 896 } 897 cs := string(c) 898 s := strings.IndexByte(cs, ':') 899 if s < 0 { 900 return 901 } 902 return cs[:s], cs[s+1:], true 903 } 904 905 // SetBasicAuth sets the request's Authorization header to use HTTP 906 // Basic Authentication with the provided username and password. 907 // 908 // With HTTP Basic Authentication the provided username and password 909 // are not encrypted. 910 func (r *Request) SetBasicAuth(username, password string) { 911 r.Header.Set("Authorization", "Basic "+basicAuth(username, password)) 912 } 913 914 // parseRequestLine parses "GET /foo HTTP/1.1" into its three parts. 915 func parseRequestLine(line string) (method, requestURI, proto string, ok bool) { 916 s1 := strings.Index(line, " ") 917 s2 := strings.Index(line[s1+1:], " ") 918 if s1 < 0 || s2 < 0 { 919 return 920 } 921 s2 += s1 + 1 922 return line[:s1], line[s1+1 : s2], line[s2+1:], true 923 } 924 925 var textprotoReaderPool sync.Pool 926 927 func newTextprotoReader(br *bufio.Reader) *textproto.Reader { 928 if v := textprotoReaderPool.Get(); v != nil { 929 tr := v.(*textproto.Reader) 930 tr.R = br 931 return tr 932 } 933 return textproto.NewReader(br) 934 } 935 936 func putTextprotoReader(r *textproto.Reader) { 937 r.R = nil 938 textprotoReaderPool.Put(r) 939 } 940 941 // ReadRequest reads and parses an incoming request from b. 942 // 943 // ReadRequest is a low-level function and should only be used for 944 // specialized applications; most code should use the Server to read 945 // requests and handle them via the Handler interface. ReadRequest 946 // only supports HTTP/1.x requests. For HTTP/2, use golang.org/x/net/http2. 947 func ReadRequest(b *bufio.Reader) (*Request, error) { 948 return readRequest(b, deleteHostHeader) 949 } 950 951 // Constants for readRequest's deleteHostHeader parameter. 952 const ( 953 deleteHostHeader = true 954 keepHostHeader = false 955 ) 956 957 func readRequest(b *bufio.Reader, deleteHostHeader bool) (req *Request, err error) { 958 tp := newTextprotoReader(b) 959 req = new(Request) 960 961 // First line: GET /index.html HTTP/1.0 962 var s string 963 if s, err = tp.ReadLine(); err != nil { 964 return nil, err 965 } 966 defer func() { 967 putTextprotoReader(tp) 968 if err == io.EOF { 969 err = io.ErrUnexpectedEOF 970 } 971 }() 972 973 var ok bool 974 req.Method, req.RequestURI, req.Proto, ok = parseRequestLine(s) 975 if !ok { 976 return nil, &badStringError{"malformed HTTP request", s} 977 } 978 if !validMethod(req.Method) { 979 return nil, &badStringError{"invalid method", req.Method} 980 } 981 rawurl := req.RequestURI 982 if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok { 983 return nil, &badStringError{"malformed HTTP version", req.Proto} 984 } 985 986 // CONNECT requests are used two different ways, and neither uses a full URL: 987 // The standard use is to tunnel HTTPS through an HTTP proxy. 988 // It looks like "CONNECT www.google.com:443 HTTP/1.1", and the parameter is 989 // just the authority section of a URL. This information should go in req.URL.Host. 990 // 991 // The net/rpc package also uses CONNECT, but there the parameter is a path 992 // that starts with a slash. It can be parsed with the regular URL parser, 993 // and the path will end up in req.URL.Path, where it needs to be in order for 994 // RPC to work. 995 justAuthority := req.Method == "CONNECT" && !strings.HasPrefix(rawurl, "/") 996 if justAuthority { 997 rawurl = "http://" + rawurl 998 } 999 1000 if req.URL, err = url.ParseRequestURI(rawurl); err != nil { 1001 return nil, err 1002 } 1003 1004 if justAuthority { 1005 // Strip the bogus "http://" back off. 1006 req.URL.Scheme = "" 1007 } 1008 1009 // Subsequent lines: Key: value. 1010 mimeHeader, err := tp.ReadMIMEHeader() 1011 if err != nil { 1012 return nil, err 1013 } 1014 req.Header = Header(mimeHeader) 1015 1016 // RFC 7230, section 5.3: Must treat 1017 // GET /index.html HTTP/1.1 1018 // Host: www.google.com 1019 // and 1020 // GET http://www.google.com/index.html HTTP/1.1 1021 // Host: doesntmatter 1022 // the same. In the second case, any Host line is ignored. 1023 req.Host = req.URL.Host 1024 if req.Host == "" { 1025 req.Host = req.Header.get("Host") 1026 } 1027 if deleteHostHeader { 1028 delete(req.Header, "Host") 1029 } 1030 1031 fixPragmaCacheControl(req.Header) 1032 1033 req.Close = shouldClose(req.ProtoMajor, req.ProtoMinor, req.Header, false) 1034 1035 err = readTransfer(req, b) 1036 if err != nil { 1037 return nil, err 1038 } 1039 1040 if req.isH2Upgrade() { 1041 // Because it's neither chunked, nor declared: 1042 req.ContentLength = -1 1043 1044 // We want to give handlers a chance to hijack the 1045 // connection, but we need to prevent the Server from 1046 // dealing with the connection further if it's not 1047 // hijacked. Set Close to ensure that: 1048 req.Close = true 1049 } 1050 return req, nil 1051 } 1052 1053 // MaxBytesReader is similar to io.LimitReader but is intended for 1054 // limiting the size of incoming request bodies. In contrast to 1055 // io.LimitReader, MaxBytesReader's result is a ReadCloser, returns a 1056 // non-EOF error for a Read beyond the limit, and closes the 1057 // underlying reader when its Close method is called. 1058 // 1059 // MaxBytesReader prevents clients from accidentally or maliciously 1060 // sending a large request and wasting server resources. 1061 func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser { 1062 return &maxBytesReader{w: w, r: r, n: n} 1063 } 1064 1065 type maxBytesReader struct { 1066 w ResponseWriter 1067 r io.ReadCloser // underlying reader 1068 n int64 // max bytes remaining 1069 err error // sticky error 1070 } 1071 1072 func (l *maxBytesReader) Read(p []byte) (n int, err error) { 1073 if l.err != nil { 1074 return 0, l.err 1075 } 1076 if len(p) == 0 { 1077 return 0, nil 1078 } 1079 // If they asked for a 32KB byte read but only 5 bytes are 1080 // remaining, no need to read 32KB. 6 bytes will answer the 1081 // question of the whether we hit the limit or go past it. 1082 if int64(len(p)) > l.n+1 { 1083 p = p[:l.n+1] 1084 } 1085 n, err = l.r.Read(p) 1086 1087 if int64(n) <= l.n { 1088 l.n -= int64(n) 1089 l.err = err 1090 return n, err 1091 } 1092 1093 n = int(l.n) 1094 l.n = 0 1095 1096 // The server code and client code both use 1097 // maxBytesReader. This "requestTooLarge" check is 1098 // only used by the server code. To prevent binaries 1099 // which only using the HTTP Client code (such as 1100 // cmd/go) from also linking in the HTTP server, don't 1101 // use a static type assertion to the server 1102 // "*response" type. Check this interface instead: 1103 type requestTooLarger interface { 1104 requestTooLarge() 1105 } 1106 if res, ok := l.w.(requestTooLarger); ok { 1107 res.requestTooLarge() 1108 } 1109 l.err = errors.New("http: request body too large") 1110 return n, l.err 1111 } 1112 1113 func (l *maxBytesReader) Close() error { 1114 return l.r.Close() 1115 } 1116 1117 func copyValues(dst, src url.Values) { 1118 for k, vs := range src { 1119 for _, value := range vs { 1120 dst.Add(k, value) 1121 } 1122 } 1123 } 1124 1125 func parsePostForm(r *Request) (vs url.Values, err error) { 1126 if r.Body == nil { 1127 err = errors.New("missing form body") 1128 return 1129 } 1130 ct := r.Header.Get("Content-Type") 1131 // RFC 7231, section 3.1.1.5 - empty type 1132 // MAY be treated as application/octet-stream 1133 if ct == "" { 1134 ct = "application/octet-stream" 1135 } 1136 ct, _, err = mime.ParseMediaType(ct) 1137 switch { 1138 case ct == "application/x-www-form-urlencoded": 1139 var reader io.Reader = r.Body 1140 maxFormSize := int64(1<<63 - 1) 1141 if _, ok := r.Body.(*maxBytesReader); !ok { 1142 maxFormSize = int64(10 << 20) // 10 MB is a lot of text. 1143 reader = io.LimitReader(r.Body, maxFormSize+1) 1144 } 1145 b, e := ioutil.ReadAll(reader) 1146 if e != nil { 1147 if err == nil { 1148 err = e 1149 } 1150 break 1151 } 1152 if int64(len(b)) > maxFormSize { 1153 err = errors.New("http: POST too large") 1154 return 1155 } 1156 vs, e = url.ParseQuery(string(b)) 1157 if err == nil { 1158 err = e 1159 } 1160 case ct == "multipart/form-data": 1161 // handled by ParseMultipartForm (which is calling us, or should be) 1162 // TODO(bradfitz): there are too many possible 1163 // orders to call too many functions here. 1164 // Clean this up and write more tests. 1165 // request_test.go contains the start of this, 1166 // in TestParseMultipartFormOrder and others. 1167 } 1168 return 1169 } 1170 1171 // ParseForm populates r.Form and r.PostForm. 1172 // 1173 // For all requests, ParseForm parses the raw query from the URL and updates 1174 // r.Form. 1175 // 1176 // For POST, PUT, and PATCH requests, it also parses the request body as a form 1177 // and puts the results into both r.PostForm and r.Form. Request body parameters 1178 // take precedence over URL query string values in r.Form. 1179 // 1180 // For other HTTP methods, or when the Content-Type is not 1181 // application/x-www-form-urlencoded, the request Body is not read, and 1182 // r.PostForm is initialized to a non-nil, empty value. 1183 // 1184 // If the request Body's size has not already been limited by MaxBytesReader, 1185 // the size is capped at 10MB. 1186 // 1187 // ParseMultipartForm calls ParseForm automatically. 1188 // ParseForm is idempotent. 1189 func (r *Request) ParseForm() error { 1190 var err error 1191 if r.PostForm == nil { 1192 if r.Method == "POST" || r.Method == "PUT" || r.Method == "PATCH" { 1193 r.PostForm, err = parsePostForm(r) 1194 } 1195 if r.PostForm == nil { 1196 r.PostForm = make(url.Values) 1197 } 1198 } 1199 if r.Form == nil { 1200 if len(r.PostForm) > 0 { 1201 r.Form = make(url.Values) 1202 copyValues(r.Form, r.PostForm) 1203 } 1204 var newValues url.Values 1205 if r.URL != nil { 1206 var e error 1207 newValues, e = url.ParseQuery(r.URL.RawQuery) 1208 if err == nil { 1209 err = e 1210 } 1211 } 1212 if newValues == nil { 1213 newValues = make(url.Values) 1214 } 1215 if r.Form == nil { 1216 r.Form = newValues 1217 } else { 1218 copyValues(r.Form, newValues) 1219 } 1220 } 1221 return err 1222 } 1223 1224 // ParseMultipartForm parses a request body as multipart/form-data. 1225 // The whole request body is parsed and up to a total of maxMemory bytes of 1226 // its file parts are stored in memory, with the remainder stored on 1227 // disk in temporary files. 1228 // ParseMultipartForm calls ParseForm if necessary. 1229 // After one call to ParseMultipartForm, subsequent calls have no effect. 1230 func (r *Request) ParseMultipartForm(maxMemory int64) error { 1231 if r.MultipartForm == multipartByReader { 1232 return errors.New("http: multipart handled by MultipartReader") 1233 } 1234 if r.Form == nil { 1235 err := r.ParseForm() 1236 if err != nil { 1237 return err 1238 } 1239 } 1240 if r.MultipartForm != nil { 1241 return nil 1242 } 1243 1244 mr, err := r.multipartReader(false) 1245 if err != nil { 1246 return err 1247 } 1248 1249 f, err := mr.ReadForm(maxMemory) 1250 if err != nil { 1251 return err 1252 } 1253 1254 if r.PostForm == nil { 1255 r.PostForm = make(url.Values) 1256 } 1257 for k, v := range f.Value { 1258 r.Form[k] = append(r.Form[k], v...) 1259 // r.PostForm should also be populated. See Issue 9305. 1260 r.PostForm[k] = append(r.PostForm[k], v...) 1261 } 1262 1263 r.MultipartForm = f 1264 1265 return nil 1266 } 1267 1268 // FormValue returns the first value for the named component of the query. 1269 // POST and PUT body parameters take precedence over URL query string values. 1270 // FormValue calls ParseMultipartForm and ParseForm if necessary and ignores 1271 // any errors returned by these functions. 1272 // If key is not present, FormValue returns the empty string. 1273 // To access multiple values of the same key, call ParseForm and 1274 // then inspect Request.Form directly. 1275 func (r *Request) FormValue(key string) string { 1276 if r.Form == nil { 1277 r.ParseMultipartForm(defaultMaxMemory) 1278 } 1279 if vs := r.Form[key]; len(vs) > 0 { 1280 return vs[0] 1281 } 1282 return "" 1283 } 1284 1285 // PostFormValue returns the first value for the named component of the POST, 1286 // PATCH, or PUT request body. URL query parameters are ignored. 1287 // PostFormValue calls ParseMultipartForm and ParseForm if necessary and ignores 1288 // any errors returned by these functions. 1289 // If key is not present, PostFormValue returns the empty string. 1290 func (r *Request) PostFormValue(key string) string { 1291 if r.PostForm == nil { 1292 r.ParseMultipartForm(defaultMaxMemory) 1293 } 1294 if vs := r.PostForm[key]; len(vs) > 0 { 1295 return vs[0] 1296 } 1297 return "" 1298 } 1299 1300 // FormFile returns the first file for the provided form key. 1301 // FormFile calls ParseMultipartForm and ParseForm if necessary. 1302 func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, error) { 1303 if r.MultipartForm == multipartByReader { 1304 return nil, nil, errors.New("http: multipart handled by MultipartReader") 1305 } 1306 if r.MultipartForm == nil { 1307 err := r.ParseMultipartForm(defaultMaxMemory) 1308 if err != nil { 1309 return nil, nil, err 1310 } 1311 } 1312 if r.MultipartForm != nil && r.MultipartForm.File != nil { 1313 if fhs := r.MultipartForm.File[key]; len(fhs) > 0 { 1314 f, err := fhs[0].Open() 1315 return f, fhs[0], err 1316 } 1317 } 1318 return nil, nil, ErrMissingFile 1319 } 1320 1321 func (r *Request) expectsContinue() bool { 1322 return hasToken(r.Header.get("Expect"), "100-continue") 1323 } 1324 1325 func (r *Request) wantsHttp10KeepAlive() bool { 1326 if r.ProtoMajor != 1 || r.ProtoMinor != 0 { 1327 return false 1328 } 1329 return hasToken(r.Header.get("Connection"), "keep-alive") 1330 } 1331 1332 func (r *Request) wantsClose() bool { 1333 return hasToken(r.Header.get("Connection"), "close") 1334 } 1335 1336 func (r *Request) closeBody() { 1337 if r.Body != nil { 1338 r.Body.Close() 1339 } 1340 } 1341 1342 func (r *Request) isReplayable() bool { 1343 if r.Body == nil || r.Body == NoBody || r.GetBody != nil { 1344 switch valueOrDefault(r.Method, "GET") { 1345 case "GET", "HEAD", "OPTIONS", "TRACE": 1346 return true 1347 } 1348 } 1349 return false 1350 } 1351 1352 // outgoingLength reports the Content-Length of this outgoing (Client) request. 1353 // It maps 0 into -1 (unknown) when the Body is non-nil. 1354 func (r *Request) outgoingLength() int64 { 1355 if r.Body == nil || r.Body == NoBody { 1356 return 0 1357 } 1358 if r.ContentLength != 0 { 1359 return r.ContentLength 1360 } 1361 return -1 1362 } 1363 1364 // requestMethodUsuallyLacksBody reports whether the given request 1365 // method is one that typically does not involve a request body. 1366 // This is used by the Transport (via 1367 // transferWriter.shouldSendChunkedRequestBody) to determine whether 1368 // we try to test-read a byte from a non-nil Request.Body when 1369 // Request.outgoingLength() returns -1. See the comments in 1370 // shouldSendChunkedRequestBody. 1371 func requestMethodUsuallyLacksBody(method string) bool { 1372 switch method { 1373 case "GET", "HEAD", "DELETE", "OPTIONS", "PROPFIND", "SEARCH": 1374 return true 1375 } 1376 return false 1377 } 1378