1
2
3
4
5
6
7
8
9 package http
10
11 import (
12 "bufio"
13 "bytes"
14 "crypto/tls"
15 "encoding/base64"
16 "fmt"
17 "io"
18 "io/ioutil"
19 "mime"
20 "mime/multipart"
21 "net/textproto"
22 "os"
23 "strconv"
24 "strings"
25 "url"
26 )
27
28 const (
29 maxLineLength = 4096
30 maxValueLength = 4096
31 maxHeaderLines = 1024
32 chunkSize = 4 << 10
33 defaultMaxMemory = 32 << 20
34 )
35
36
37
38 var ErrMissingFile = os.NewError("http: no such file")
39
40
41 type ProtocolError struct {
42 ErrorString string
43 }
44
45 func (err *ProtocolError) String() string { return err.ErrorString }
46
47 var (
48 ErrLineTooLong = &ProtocolError{"header line too long"}
49 ErrHeaderTooLong = &ProtocolError{"header too long"}
50 ErrShortBody = &ProtocolError{"entity body too short"}
51 ErrNotSupported = &ProtocolError{"feature not supported"}
52 ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"}
53 ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"}
54 ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"}
55 ErrMissingBoundary = &ProtocolError{"no multipart boundary param Content-Type"}
56 )
57
58 type badStringError struct {
59 what string
60 str string
61 }
62
63 func (e *badStringError) String() string { return fmt.Sprintf("%s %q", e.what, e.str) }
64
65
66 var reqWriteExcludeHeader = map[string]bool{
67 "Host": true,
68 "User-Agent": true,
69 "Content-Length": true,
70 "Transfer-Encoding": true,
71 "Trailer": true,
72 }
73
74
75 type Request struct {
76 Method string
77 RawURL string
78 URL *url.URL
79
80
81
82 Proto string
83 ProtoMajor int
84 ProtoMinor int
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105 Header Header
106
107
108 Body io.ReadCloser
109
110
111
112
113 ContentLength int64
114
115
116
117 TransferEncoding []string
118
119
120 Close bool
121
122
123
124
125 Host string
126
127
128 Form url.Values
129
130
131
132 MultipartForm *multipart.Form
133
134
135
136
137 Trailer Header
138
139
140
141
142
143
144
145 RemoteAddr string
146
147
148
149
150
151
152
153 TLS *tls.ConnectionState
154 }
155
156
157
158 func (r *Request) ProtoAtLeast(major, minor int) bool {
159 return r.ProtoMajor > major ||
160 r.ProtoMajor == major && r.ProtoMinor >= minor
161 }
162
163
164 func (r *Request) UserAgent() string {
165 return r.Header.Get("User-Agent")
166 }
167
168
169 func (r *Request) Cookies() []*Cookie {
170 return readCookies(r.Header, "")
171 }
172
173 var ErrNoCookie = os.NewError("http: named cookied not present")
174
175
176
177 func (r *Request) Cookie(name string) (*Cookie, os.Error) {
178 for _, c := range readCookies(r.Header, name) {
179 return c, nil
180 }
181 return nil, ErrNoCookie
182 }
183
184
185
186
187
188 func (r *Request) AddCookie(c *Cookie) {
189 s := fmt.Sprintf("%s=%s", sanitizeName(c.Name), sanitizeValue(c.Value))
190 if c := r.Header.Get("Cookie"); c != "" {
191 r.Header.Set("Cookie", c+"; "+s)
192 } else {
193 r.Header.Set("Cookie", s)
194 }
195 }
196
197
198
199
200
201
202
203
204
205 func (r *Request) Referer() string {
206 return r.Header.Get("Referer")
207 }
208
209
210
211
212 var multipartByReader = &multipart.Form{
213 Value: make(map[string][]string),
214 File: make(map[string][]*multipart.FileHeader),
215 }
216
217
218
219
220
221 func (r *Request) MultipartReader() (*multipart.Reader, os.Error) {
222 if r.MultipartForm == multipartByReader {
223 return nil, os.NewError("http: MultipartReader called twice")
224 }
225 if r.MultipartForm != nil {
226 return nil, os.NewError("http: multipart handled by ParseMultipartForm")
227 }
228 r.MultipartForm = multipartByReader
229 return r.multipartReader()
230 }
231
232 func (r *Request) multipartReader() (*multipart.Reader, os.Error) {
233 v := r.Header.Get("Content-Type")
234 if v == "" {
235 return nil, ErrNotMultipart
236 }
237 d, params := mime.ParseMediaType(v)
238 if d != "multipart/form-data" {
239 return nil, ErrNotMultipart
240 }
241 boundary, ok := params["boundary"]
242 if !ok {
243 return nil, ErrMissingBoundary
244 }
245 return multipart.NewReader(r.Body, boundary), nil
246 }
247
248
249 func valueOrDefault(value, def string) string {
250 if value != "" {
251 return value
252 }
253 return def
254 }
255
256 const defaultUserAgent = "Go http package"
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271 func (req *Request) Write(w io.Writer) os.Error {
272 return req.write(w, false)
273 }
274
275
276
277
278
279
280 func (req *Request) WriteProxy(w io.Writer) os.Error {
281 return req.write(w, true)
282 }
283
284 func (req *Request) write(w io.Writer, usingProxy bool) os.Error {
285 host := req.Host
286 if host == "" {
287 if req.URL == nil {
288 return os.NewError("http: Request.Write on Request with no Host or URL set")
289 }
290 host = req.URL.Host
291 }
292
293 urlStr := req.RawURL
294 if urlStr == "" {
295 urlStr = valueOrDefault(req.URL.EncodedPath(), "/")
296 if req.URL.RawQuery != "" {
297 urlStr += "?" + req.URL.RawQuery
298 }
299 if usingProxy {
300 if urlStr == "" || urlStr[0] != '/' {
301 urlStr = "/" + urlStr
302 }
303 urlStr = req.URL.Scheme + "://" + host + urlStr
304 }
305 }
306
307 bw := bufio.NewWriter(w)
308 fmt.Fprintf(bw, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), urlStr)
309
310
311 fmt.Fprintf(bw, "Host: %s\r\n", host)
312
313
314
315 userAgent := defaultUserAgent
316 if req.Header != nil {
317 if ua := req.Header["User-Agent"]; len(ua) > 0 {
318 userAgent = ua[0]
319 }
320 }
321 if userAgent != "" {
322 fmt.Fprintf(bw, "User-Agent: %s\r\n", userAgent)
323 }
324
325
326 tw, err := newTransferWriter(req)
327 if err != nil {
328 return err
329 }
330 err = tw.WriteHeader(bw)
331 if err != nil {
332 return err
333 }
334
335
336 err = req.Header.WriteSubset(bw, reqWriteExcludeHeader)
337 if err != nil {
338 return err
339 }
340
341 io.WriteString(bw, "\r\n")
342
343
344 err = tw.WriteBody(bw)
345 if err != nil {
346 return err
347 }
348 bw.Flush()
349 return nil
350 }
351
352
353
354
355
356 func readLineBytes(b *bufio.Reader) (p []byte, err os.Error) {
357 if p, err = b.ReadSlice('\n'); err != nil {
358
359
360 if err == os.EOF {
361 err = io.ErrUnexpectedEOF
362 } else if err == bufio.ErrBufferFull {
363 err = ErrLineTooLong
364 }
365 return nil, err
366 }
367 if len(p) >= maxLineLength {
368 return nil, ErrLineTooLong
369 }
370
371
372 var i int
373 for i = len(p); i > 0; i-- {
374 if c := p[i-1]; c != ' ' && c != '\r' && c != '\t' && c != '\n' {
375 break
376 }
377 }
378 return p[0:i], nil
379 }
380
381
382 func readLine(b *bufio.Reader) (s string, err os.Error) {
383 p, e := readLineBytes(b)
384 if e != nil {
385 return "", e
386 }
387 return string(p), nil
388 }
389
390
391
392
393 func atoi(s string, i int) (n, i1 int, ok bool) {
394 const Big = 1000000
395 if i >= len(s) || s[i] < '0' || s[i] > '9' {
396 return 0, 0, false
397 }
398 n = 0
399 for ; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ {
400 n = n*10 + int(s[i]-'0')
401 if n > Big {
402 return 0, 0, false
403 }
404 }
405 return n, i, true
406 }
407
408
409
410 func ParseHTTPVersion(vers string) (major, minor int, ok bool) {
411 if len(vers) < 5 || vers[0:5] != "HTTP/" {
412 return 0, 0, false
413 }
414 major, i, ok := atoi(vers, 5)
415 if !ok || i >= len(vers) || vers[i] != '.' {
416 return 0, 0, false
417 }
418 minor, i, ok = atoi(vers, i+1)
419 if !ok || i != len(vers) {
420 return 0, 0, false
421 }
422 return major, minor, true
423 }
424
425 type chunkedReader struct {
426 r *bufio.Reader
427 n uint64
428 err os.Error
429 }
430
431 func (cr *chunkedReader) beginChunk() {
432
433 var line string
434 line, cr.err = readLine(cr.r)
435 if cr.err != nil {
436 return
437 }
438 cr.n, cr.err = strconv.Btoui64(line, 16)
439 if cr.err != nil {
440 return
441 }
442 if cr.n == 0 {
443
444 for {
445 line, cr.err = readLine(cr.r)
446 if cr.err != nil {
447 return
448 }
449 if line == "" {
450 break
451 }
452 }
453 cr.err = os.EOF
454 }
455 }
456
457 func (cr *chunkedReader) Read(b []uint8) (n int, err os.Error) {
458 if cr.err != nil {
459 return 0, cr.err
460 }
461 if cr.n == 0 {
462 cr.beginChunk()
463 if cr.err != nil {
464 return 0, cr.err
465 }
466 }
467 if uint64(len(b)) > cr.n {
468 b = b[0:cr.n]
469 }
470 n, cr.err = cr.r.Read(b)
471 cr.n -= uint64(n)
472 if cr.n == 0 && cr.err == nil {
473
474 b := make([]byte, 2)
475 if _, cr.err = io.ReadFull(cr.r, b); cr.err == nil {
476 if b[0] != '\r' || b[1] != '\n' {
477 cr.err = os.NewError("malformed chunked encoding")
478 }
479 }
480 }
481 return n, cr.err
482 }
483
484
485 func NewRequest(method, urlStr string, body io.Reader) (*Request, os.Error) {
486 u, err := url.Parse(urlStr)
487 if err != nil {
488 return nil, err
489 }
490 rc, ok := body.(io.ReadCloser)
491 if !ok && body != nil {
492 rc = ioutil.NopCloser(body)
493 }
494 req := &Request{
495 Method: method,
496 URL: u,
497 Proto: "HTTP/1.1",
498 ProtoMajor: 1,
499 ProtoMinor: 1,
500 Header: make(Header),
501 Body: rc,
502 Host: u.Host,
503 }
504 if body != nil {
505 switch v := body.(type) {
506 case *strings.Reader:
507 req.ContentLength = int64(v.Len())
508 case *bytes.Buffer:
509 req.ContentLength = int64(v.Len())
510 }
511 }
512
513 return req, nil
514 }
515
516
517
518
519
520
521 func (r *Request) SetBasicAuth(username, password string) {
522 s := username + ":" + password
523 r.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(s)))
524 }
525
526
527 func ReadRequest(b *bufio.Reader) (req *Request, err os.Error) {
528
529 tp := textproto.NewReader(b)
530 req = new(Request)
531
532
533 var s string
534 if s, err = tp.ReadLine(); err != nil {
535 if err == os.EOF {
536 err = io.ErrUnexpectedEOF
537 }
538 return nil, err
539 }
540
541 var f []string
542 if f = strings.SplitN(s, " ", 3); len(f) < 3 {
543 return nil, &badStringError{"malformed HTTP request", s}
544 }
545 req.Method, req.RawURL, req.Proto = f[0], f[1], f[2]
546 var ok bool
547 if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok {
548 return nil, &badStringError{"malformed HTTP version", req.Proto}
549 }
550
551 if req.URL, err = url.ParseRequest(req.RawURL); err != nil {
552 return nil, err
553 }
554
555
556 mimeHeader, err := tp.ReadMIMEHeader()
557 if err != nil {
558 return nil, err
559 }
560 req.Header = Header(mimeHeader)
561
562
563
564
565
566
567
568
569 req.Host = req.URL.Host
570 if req.Host == "" {
571 req.Host = req.Header.Get("Host")
572 }
573 req.Header.Del("Host")
574
575 fixPragmaCacheControl(req.Header)
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603 err = readTransfer(req, b)
604 if err != nil {
605 return nil, err
606 }
607
608 return req, nil
609 }
610
611
612
613
614
615 func (r *Request) ParseForm() (err os.Error) {
616 if r.Form != nil {
617 return
618 }
619
620 if r.URL != nil {
621 r.Form, err = url.ParseQuery(r.URL.RawQuery)
622 }
623 if r.Method == "POST" {
624 if r.Body == nil {
625 return os.NewError("missing form body")
626 }
627 ct := r.Header.Get("Content-Type")
628 switch strings.SplitN(ct, ";", 2)[0] {
629 case "text/plain", "application/x-www-form-urlencoded", "":
630 const maxFormSize = int64(10 << 20)
631 b, e := ioutil.ReadAll(io.LimitReader(r.Body, maxFormSize+1))
632 if e != nil {
633 if err == nil {
634 err = e
635 }
636 break
637 }
638 if int64(len(b)) > maxFormSize {
639 return os.NewError("http: POST too large")
640 }
641 var newValues url.Values
642 newValues, e = url.ParseQuery(string(b))
643 if err == nil {
644 err = e
645 }
646 if r.Form == nil {
647 r.Form = make(url.Values)
648 }
649
650 for k, vs := range newValues {
651 for _, value := range vs {
652 r.Form.Add(k, value)
653 }
654 }
655 case "multipart/form-data":
656
657 default:
658 return &badStringError{"unknown Content-Type", ct}
659 }
660 }
661 return err
662 }
663
664
665
666
667
668
669
670 func (r *Request) ParseMultipartForm(maxMemory int64) os.Error {
671 if r.MultipartForm == multipartByReader {
672 return os.NewError("http: multipart handled by MultipartReader")
673 }
674 if r.Form == nil {
675 err := r.ParseForm()
676 if err != nil {
677 return err
678 }
679 }
680 if r.MultipartForm != nil {
681 return nil
682 }
683
684 mr, err := r.multipartReader()
685 if err == ErrNotMultipart {
686 return nil
687 } else if err != nil {
688 return err
689 }
690
691 f, err := mr.ReadForm(maxMemory)
692 if err != nil {
693 return err
694 }
695 for k, v := range f.Value {
696 r.Form[k] = append(r.Form[k], v...)
697 }
698 r.MultipartForm = f
699
700 return nil
701 }
702
703
704
705 func (r *Request) FormValue(key string) string {
706 if r.Form == nil {
707 r.ParseMultipartForm(defaultMaxMemory)
708 }
709 if vs := r.Form[key]; len(vs) > 0 {
710 return vs[0]
711 }
712 return ""
713 }
714
715
716
717 func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, os.Error) {
718 if r.MultipartForm == multipartByReader {
719 return nil, nil, os.NewError("http: multipart handled by MultipartReader")
720 }
721 if r.MultipartForm == nil {
722 err := r.ParseMultipartForm(defaultMaxMemory)
723 if err != nil {
724 return nil, nil, err
725 }
726 }
727 if r.MultipartForm != nil && r.MultipartForm.File != nil {
728 if fhs := r.MultipartForm.File[key]; len(fhs) > 0 {
729 f, err := fhs[0].Open()
730 return f, fhs[0], err
731 }
732 }
733 return nil, nil, ErrMissingFile
734 }
735
736 func (r *Request) expectsContinue() bool {
737 return strings.ToLower(r.Header.Get("Expect")) == "100-continue"
738 }
739
740 func (r *Request) wantsHttp10KeepAlive() bool {
741 if r.ProtoMajor != 1 || r.ProtoMinor != 0 {
742 return false
743 }
744 return strings.Contains(strings.ToLower(r.Header.Get("Connection")), "keep-alive")
745 }