1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
21 "golang_org/x/net/lex/httplex"
24 // ErrLineTooLong is returned when reading request or response bodies
25 // with malformed chunked encoding.
26 var ErrLineTooLong = internal.ErrLineTooLong
28 type errorReader struct {
32 func (r errorReader) Read(p []byte) (n int, err error) {
36 // transferWriter inspects the fields of a user-supplied Request or Response,
37 // sanitizes them without changing the user object and provides methods for
38 // writing the respective header, body and trailer in wire format.
39 type transferWriter struct {
44 ContentLength int64 // -1 means unknown, 0 means exactly none
46 TransferEncoding []string
51 func newTransferWriter(r interface{}) (t *transferWriter, err error) {
54 // Extract relevant fields
55 atLeastHTTP11 := false
56 switch rr := r.(type) {
58 if rr.ContentLength != 0 && rr.Body == nil {
59 return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength)
61 t.Method = valueOrDefault(rr.Method, "GET")
63 t.BodyCloser = rr.Body
64 t.ContentLength = rr.ContentLength
66 t.TransferEncoding = rr.TransferEncoding
67 t.Trailer = rr.Trailer
68 atLeastHTTP11 = rr.ProtoAtLeast(1, 1)
69 if t.Body != nil && len(t.TransferEncoding) == 0 && atLeastHTTP11 {
70 if t.ContentLength == 0 {
71 // Test to see if it's actually zero or just unset.
73 n, rerr := io.ReadFull(t.Body, buf[:])
74 if rerr != nil && rerr != io.EOF {
76 t.Body = errorReader{rerr}
78 // Oh, guess there is data in this Body Reader after all.
79 // The ContentLength field just wasn't set.
80 // Stich the Body back together again, re-attaching our
83 t.Body = io.MultiReader(bytes.NewReader(buf[:]), t.Body)
85 // Body is actually empty.
90 if t.ContentLength < 0 {
91 t.TransferEncoding = []string{"chunked"}
96 if rr.Request != nil {
97 t.Method = rr.Request.Method
100 t.BodyCloser = rr.Body
101 t.ContentLength = rr.ContentLength
103 t.TransferEncoding = rr.TransferEncoding
104 t.Trailer = rr.Trailer
105 atLeastHTTP11 = rr.ProtoAtLeast(1, 1)
106 t.ResponseToHEAD = noBodyExpected(t.Method)
109 // Sanitize Body,ContentLength,TransferEncoding
110 if t.ResponseToHEAD {
112 if chunked(t.TransferEncoding) {
116 if !atLeastHTTP11 || t.Body == nil {
117 t.TransferEncoding = nil
119 if chunked(t.TransferEncoding) {
121 } else if t.Body == nil { // no chunking, no body
127 if !chunked(t.TransferEncoding) {
134 func noBodyExpected(requestMethod string) bool {
135 return requestMethod == "HEAD"
138 func (t *transferWriter) shouldSendContentLength() bool {
139 if chunked(t.TransferEncoding) {
142 if t.ContentLength > 0 {
145 if t.ContentLength < 0 {
148 // Many servers expect a Content-Length for these methods
149 if t.Method == "POST" || t.Method == "PUT" {
152 if t.ContentLength == 0 && isIdentity(t.TransferEncoding) {
153 if t.Method == "GET" || t.Method == "HEAD" {
162 func (t *transferWriter) WriteHeader(w io.Writer) error {
164 if _, err := io.WriteString(w, "Connection: close\r\n"); err != nil {
169 // Write Content-Length and/or Transfer-Encoding whose values are a
170 // function of the sanitized field triple (Body, ContentLength,
172 if t.shouldSendContentLength() {
173 if _, err := io.WriteString(w, "Content-Length: "); err != nil {
176 if _, err := io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n"); err != nil {
179 } else if chunked(t.TransferEncoding) {
180 if _, err := io.WriteString(w, "Transfer-Encoding: chunked\r\n"); err != nil {
185 // Write Trailer header
186 if t.Trailer != nil {
187 keys := make([]string, 0, len(t.Trailer))
188 for k := range t.Trailer {
189 k = CanonicalHeaderKey(k)
191 case "Transfer-Encoding", "Trailer", "Content-Length":
192 return &badStringError{"invalid Trailer key", k}
194 keys = append(keys, k)
198 // TODO: could do better allocation-wise here, but trailers are rare,
199 // so being lazy for now.
200 if _, err := io.WriteString(w, "Trailer: "+strings.Join(keys, ",")+"\r\n"); err != nil {
209 func (t *transferWriter) WriteBody(w io.Writer) error {
215 if chunked(t.TransferEncoding) {
216 if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse {
217 w = &internal.FlushAfterChunkWriter{bw}
219 cw := internal.NewChunkedWriter(w)
220 _, err = io.Copy(cw, t.Body)
224 } else if t.ContentLength == -1 {
225 ncopy, err = io.Copy(w, t.Body)
227 ncopy, err = io.Copy(w, io.LimitReader(t.Body, t.ContentLength))
232 nextra, err = io.Copy(ioutil.Discard, t.Body)
238 if err = t.BodyCloser.Close(); err != nil {
243 if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy {
244 return fmt.Errorf("http: ContentLength=%d with Body length %d",
245 t.ContentLength, ncopy)
248 if chunked(t.TransferEncoding) {
249 // Write Trailer header
250 if t.Trailer != nil {
251 if err := t.Trailer.Write(w); err != nil {
255 // Last chunk, empty trailer
256 _, err = io.WriteString(w, "\r\n")
261 type transferReader struct {
271 TransferEncoding []string
276 func (t *transferReader) protoAtLeast(m, n int) bool {
277 return t.ProtoMajor > m || (t.ProtoMajor == m && t.ProtoMinor >= n)
280 // bodyAllowedForStatus reports whether a given response status code
281 // permits a body. See RFC 2616, section 4.4.
282 func bodyAllowedForStatus(status int) bool {
284 case status >= 100 && status <= 199:
295 suppressedHeaders304 = []string{"Content-Type", "Content-Length", "Transfer-Encoding"}
296 suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"}
299 func suppressedHeaders(status int) []string {
302 // RFC 2616 section 10.3.5: "the response MUST NOT include other entity-headers"
303 return suppressedHeaders304
304 case !bodyAllowedForStatus(status):
305 return suppressedHeadersNoBody
310 // msg is *Request or *Response.
311 func readTransfer(msg interface{}, r *bufio.Reader) (err error) {
312 t := &transferReader{RequestMethod: "GET"}
316 switch rr := msg.(type) {
319 t.StatusCode = rr.StatusCode
320 t.ProtoMajor = rr.ProtoMajor
321 t.ProtoMinor = rr.ProtoMinor
322 t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true)
324 if rr.Request != nil {
325 t.RequestMethod = rr.Request.Method
329 t.RequestMethod = rr.Method
330 t.ProtoMajor = rr.ProtoMajor
331 t.ProtoMinor = rr.ProtoMinor
332 // Transfer semantics for Requests are exactly like those for
333 // Responses with status code 200, responding to a GET method
337 panic("unexpected type")
340 // Default to HTTP/1.1
341 if t.ProtoMajor == 0 && t.ProtoMinor == 0 {
342 t.ProtoMajor, t.ProtoMinor = 1, 1
345 // Transfer encoding, content length
346 err = t.fixTransferEncoding()
351 realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.TransferEncoding)
355 if isResponse && t.RequestMethod == "HEAD" {
356 if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil {
362 t.ContentLength = realLength
366 t.Trailer, err = fixTrailer(t.Header, t.TransferEncoding)
371 // If there is no Content-Length or chunked Transfer-Encoding on a *Response
372 // and the status is not 1xx, 204 or 304, then the body is unbounded.
373 // See RFC 2616, section 4.4.
376 if realLength == -1 &&
377 !chunked(t.TransferEncoding) &&
378 bodyAllowedForStatus(t.StatusCode) {
384 // Prepare body reader. ContentLength < 0 means chunked encoding
385 // or close connection when finished, since multipart is not supported yet
387 case chunked(t.TransferEncoding):
388 if noBodyExpected(t.RequestMethod) {
391 t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close}
393 case realLength == 0:
396 t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close}
398 // realLength < 0, i.e. "Content-Length" not mentioned in header
400 // Close semantics (i.e. HTTP/1.0)
401 t.Body = &body{src: r, closing: t.Close}
403 // Persistent connection (i.e. HTTP/1.1)
409 switch rr := msg.(type) {
412 rr.ContentLength = t.ContentLength
413 rr.TransferEncoding = t.TransferEncoding
415 rr.Trailer = t.Trailer
418 rr.ContentLength = t.ContentLength
419 rr.TransferEncoding = t.TransferEncoding
421 rr.Trailer = t.Trailer
427 // Checks whether chunked is part of the encodings stack
428 func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" }
430 // Checks whether the encoding is explicitly "identity".
431 func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" }
433 // fixTransferEncoding sanitizes t.TransferEncoding, if needed.
434 func (t *transferReader) fixTransferEncoding() error {
435 raw, present := t.Header["Transfer-Encoding"]
439 delete(t.Header, "Transfer-Encoding")
441 // Issue 12785; ignore Transfer-Encoding on HTTP/1.0 requests.
442 if !t.protoAtLeast(1, 1) {
446 encodings := strings.Split(raw[0], ",")
447 te := make([]string, 0, len(encodings))
448 // TODO: Even though we only support "identity" and "chunked"
449 // encodings, the loop below is designed with foresight. One
450 // invariant that must be maintained is that, if present,
451 // chunked encoding must always come first.
452 for _, encoding := range encodings {
453 encoding = strings.ToLower(strings.TrimSpace(encoding))
454 // "identity" encoding is not recorded
455 if encoding == "identity" {
458 if encoding != "chunked" {
459 return &badStringError{"unsupported transfer encoding", encoding}
461 te = te[0 : len(te)+1]
462 te[len(te)-1] = encoding
465 return &badStringError{"too many transfer encodings", strings.Join(te, ",")}
468 // RFC 7230 3.3.2 says "A sender MUST NOT send a
469 // Content-Length header field in any message that
470 // contains a Transfer-Encoding header field."
473 // "If a message is received with both a
474 // Transfer-Encoding and a Content-Length header
475 // field, the Transfer-Encoding overrides the
476 // Content-Length. Such a message might indicate an
477 // attempt to perform request smuggling (Section 9.5)
478 // or response splitting (Section 9.4) and ought to be
479 // handled as an error. A sender MUST remove the
480 // received Content-Length field prior to forwarding
481 // such a message downstream."
483 // Reportedly, these appear in the wild.
484 delete(t.Header, "Content-Length")
485 t.TransferEncoding = te
492 // Determine the expected body length, using RFC 2616 Section 4.4. This
493 // function is not a method, because ultimately it should be shared by
494 // ReadResponse and ReadRequest.
495 func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) {
496 contentLens := header["Content-Length"]
497 isRequest := !isResponse
498 // Logic based on response type or status
499 if noBodyExpected(requestMethod) {
500 // For HTTP requests, as part of hardening against request
501 // smuggling (RFC 7230), don't allow a Content-Length header for
502 // methods which don't permit bodies. As an exception, allow
503 // exactly one Content-Length header if its value is "0".
504 if isRequest && len(contentLens) > 0 && !(len(contentLens) == 1 && contentLens[0] == "0") {
505 return 0, fmt.Errorf("http: method cannot contain a Content-Length; got %q", contentLens)
517 if len(contentLens) > 1 {
518 // harden against HTTP request smuggling. See RFC 7230.
519 return 0, errors.New("http: message cannot contain multiple Content-Length headers")
522 // Logic based on Transfer-Encoding
527 // Logic based on Content-Length
529 if len(contentLens) == 1 {
530 cl = strings.TrimSpace(contentLens[0])
533 n, err := parseContentLength(cl)
539 header.Del("Content-Length")
543 // RFC 2616 neither explicitly permits nor forbids an
544 // entity-body on a GET request so we permit one if
545 // declared, but we default to 0 here (not -1 below)
546 // if there's no mention of a body.
547 // Likewise, all other request methods are assumed to have
548 // no body if neither Transfer-Encoding chunked nor a
549 // Content-Length are set.
553 // Body-EOF logic based on other methods (like closing, or chunked coding)
557 // Determine whether to hang up after sending a request and body, or
558 // receiving a response and body
559 // 'header' is the request headers
560 func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool {
565 conv := header["Connection"]
566 hasClose := httplex.HeaderValuesContainsToken(conv, "close")
567 if major == 1 && minor == 0 {
568 return hasClose || !httplex.HeaderValuesContainsToken(conv, "keep-alive")
571 if hasClose && removeCloseHeader {
572 header.Del("Connection")
578 // Parse the trailer header
579 func fixTrailer(header Header, te []string) (Header, error) {
580 vv, ok := header["Trailer"]
584 header.Del("Trailer")
586 trailer := make(Header)
588 for _, v := range vv {
589 foreachHeaderElement(v, func(key string) {
590 key = CanonicalHeaderKey(key)
592 case "Transfer-Encoding", "Trailer", "Content-Length":
594 err = &badStringError{"bad trailer key", key}
604 if len(trailer) == 0 {
608 // Trailer and no chunking
609 return nil, ErrUnexpectedTrailer
614 // body turns a Reader into a ReadCloser.
615 // Close ensures that the body has been fully read
616 // and then reads the trailer if necessary.
619 hdr interface{} // non-nil (Response or Request) value means read trailer
620 r *bufio.Reader // underlying wire-format reader for the trailer
621 closing bool // is the connection to be closed after reading body?
622 doEarlyClose bool // whether Close should stop early
624 mu sync.Mutex // guards following, and calls to Read and Close
627 earlyClose bool // Close called and we didn't read to the end of src
628 onHitEOF func() // if non-nil, func to call when EOF is Read
631 // ErrBodyReadAfterClose is returned when reading a Request or Response
632 // Body after the body has been closed. This typically happens when the body is
633 // read after an HTTP Handler calls WriteHeader or Write on its
635 var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body")
637 func (b *body) Read(p []byte) (n int, err error) {
641 return 0, ErrBodyReadAfterClose
643 return b.readLocked(p)
647 func (b *body) readLocked(p []byte) (n int, err error) {
651 n, err = b.src.Read(p)
655 // Chunked case. Read the trailer.
657 if e := b.readTrailer(); e != nil {
659 // Something went wrong in the trailer, we must not allow any
660 // further reads of any kind to succeed from body, nor any
661 // subsequent requests on the server connection. See
662 // golang.org/issue/12027
668 // If the server declared the Content-Length, our body is a LimitedReader
669 // and we need to check whether this EOF arrived early.
670 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 {
671 err = io.ErrUnexpectedEOF
676 // If we can return an EOF here along with the read data, do
677 // so. This is optional per the io.Reader contract, but doing
678 // so helps the HTTP transport code recycle its connection
679 // earlier (since it will see this EOF itself), even if the
680 // client doesn't do future reads or Close.
681 if err == nil && n > 0 {
682 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 {
688 if b.sawEOF && b.onHitEOF != nil {
696 singleCRLF = []byte("\r\n")
697 doubleCRLF = []byte("\r\n\r\n")
700 func seeUpcomingDoubleCRLF(r *bufio.Reader) bool {
701 for peekSize := 4; ; peekSize++ {
702 // This loop stops when Peek returns an error,
703 // which it does when r's buffer has been filled.
704 buf, err := r.Peek(peekSize)
705 if bytes.HasSuffix(buf, doubleCRLF) {
715 var errTrailerEOF = errors.New("http: unexpected EOF reading trailer")
717 func (b *body) readTrailer() error {
718 // The common case, since nobody uses trailers.
719 buf, err := b.r.Peek(2)
720 if bytes.Equal(buf, singleCRLF) {
731 // Make sure there's a header terminator coming up, to prevent
732 // a DoS with an unbounded size Trailer. It's not easy to
733 // slip in a LimitReader here, as textproto.NewReader requires
734 // a concrete *bufio.Reader. Also, we can't get all the way
735 // back up to our conn's LimitedReader that *might* be backing
736 // this bufio.Reader. Instead, a hack: we iteratively Peek up
737 // to the bufio.Reader's max size, looking for a double CRLF.
738 // This limits the trailer to the underlying buffer size, typically 4kB.
739 if !seeUpcomingDoubleCRLF(b.r) {
740 return errors.New("http: suspiciously long trailer after chunked body")
743 hdr, err := textproto.NewReader(b.r).ReadMIMEHeader()
750 switch rr := b.hdr.(type) {
752 mergeSetHeader(&rr.Trailer, Header(hdr))
754 mergeSetHeader(&rr.Trailer, Header(hdr))
759 func mergeSetHeader(dst *Header, src Header) {
764 for k, vv := range src {
769 // unreadDataSizeLocked returns the number of bytes of unread input.
770 // It returns -1 if unknown.
771 // b.mu must be held.
772 func (b *body) unreadDataSizeLocked() int64 {
773 if lr, ok := b.src.(*io.LimitedReader); ok {
779 func (b *body) Close() error {
788 // Already saw EOF, so no need going to look for it.
789 case b.hdr == nil && b.closing:
790 // no trailer and closing the connection next.
791 // no point in reading to EOF.
793 // Read up to maxPostHandlerReadBytes bytes of the body, looking for
794 // for EOF (and trailers), so we can re-use this connection.
795 if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > maxPostHandlerReadBytes {
796 // There was a declared Content-Length, and we have more bytes remaining
797 // than our maxPostHandlerReadBytes tolerance. So, give up.
801 // Consume the body, or, which will also lead to us reading
802 // the trailer headers after the body, if present.
803 n, err = io.CopyN(ioutil.Discard, bodyLocked{b}, maxPostHandlerReadBytes)
807 if n == maxPostHandlerReadBytes {
812 // Fully consume the body, which will also lead to us reading
813 // the trailer headers after the body, if present.
814 _, err = io.Copy(ioutil.Discard, bodyLocked{b})
820 func (b *body) didEarlyClose() bool {
826 // bodyRemains reports whether future Read calls might
828 func (b *body) bodyRemains() bool {
834 func (b *body) registerOnHitEOF(fn func()) {
840 // bodyLocked is a io.Reader reading from a *body when its mutex is
842 type bodyLocked struct {
846 func (bl bodyLocked) Read(p []byte) (n int, err error) {
848 return 0, ErrBodyReadAfterClose
850 return bl.b.readLocked(p)
853 // parseContentLength trims whitespace from s and returns -1 if no value
854 // is set, or the value if it's >= 0.
855 func parseContentLength(cl string) (int64, error) {
856 cl = strings.TrimSpace(cl)
860 n, err := strconv.ParseInt(cl, 10, 64)
861 if err != nil || n < 0 {
862 return 0, &badStringError{"bad Content-Length", cl}