Significant extension to http.Response, which now adheres to the

usage pattern of http.Request and paves the way to persistent connection
handling.

R=rsc
CC=golang-dev
https://golang.org/cl/185043

src/pkg/http/Makefile

@@ -6,9 +6,11 @@ include ../../Make.$(GOARCH)
 
 TARG=http
 GOFILES=\
+	chunked.go\
 	client.go\
 	fs.go\
 	request.go\
+	response.go\
 	server.go\
 	status.go\
 	url.go\

src/pkg/http/chunked.go

@@ -0,0 +1,56 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package http
+
+import (
+	"io"
+	"os"
+	"strconv"
+)
+
+// NewChunkedWriter returns a new writer that translates writes into HTTP
+// "chunked" format before writing them to w.  Closing the returned writer
+// sends the final 0-length chunk that marks the end of the stream.
+func NewChunkedWriter(w io.Writer) io.WriteCloser {
+	return &chunkedWriter{w}
+}
+
+// Writing to ChunkedWriter translates to writing in HTTP chunked Transfer
+// Encoding wire format to the undering Wire writer.
+type chunkedWriter struct {
+	Wire io.Writer
+}
+
+// Write the contents of data as one chunk to Wire.
+// NOTE: Note that the corresponding chunk-writing procedure in Conn.Write has
+// a bug since it does not check for success of io.WriteString
+func (cw *chunkedWriter) Write(data []byte) (n int, err os.Error) {
+
+	// Don't send 0-length data. It looks like EOF for chunked encoding.
+	if len(data) == 0 {
+		return 0, nil
+	}
+
+	head := strconv.Itob(len(data), 16) + "\r\n"
+
+	if _, err = io.WriteString(cw.Wire, head); err != nil {
+		return 0, err
+	}
+	if n, err = cw.Wire.Write(data); err != nil {
+		return
+	}
+	if n != len(data) {
+		err = io.ErrShortWrite
+		return
+	}
+	_, err = io.WriteString(cw.Wire, "\r\n")
+
+	return
+}
+
+func (cw *chunkedWriter) Close() os.Error {
+	_, err := io.WriteString(cw.Wire, "0\r\n")
+	return err
+}

src/pkg/http/client.go

@@ -13,49 +13,9 @@ import (
 	"io"
 	"net"
 	"os"
-	"strconv"
 	"strings"
 )
 
-// Response represents the response from an HTTP request.
-type Response struct {
-	Status     string // e.g. "200 OK"
-	StatusCode int    // e.g. 200
-
-	// Header maps header keys to values.  If the response had multiple
-	// headers with the same key, they will be concatenated, with comma
-	// delimiters.  (Section 4.2 of RFC 2616 requires that multiple headers
-	// be semantically equivalent to a comma-delimited sequence.)
-	//
-	// Keys in the map are canonicalized (see CanonicalHeaderKey).
-	Header map[string]string
-
-	// Stream from which the response body can be read.
-	Body io.ReadCloser
-}
-
-// GetHeader returns the value of the response header with the given
-// key, and true.  If there were multiple headers with this key, their
-// values are concatenated, with a comma delimiter.  If there were no
-// response headers with the given key, it returns the empty string and
-// false.  Keys are not case sensitive.
-func (r *Response) GetHeader(key string) (value string) {
-	value, _ = r.Header[CanonicalHeaderKey(key)]
-	return
-}
-
-// AddHeader adds a value under the given key.  Keys are not case sensitive.
-func (r *Response) AddHeader(key, value string) {
-	key = CanonicalHeaderKey(key)
-
-	oldValues, oldValuesPresent := r.Header[key]
-	if oldValuesPresent {
-		r.Header[key] = oldValues + "," + value
-	} else {
-		r.Header[key] = value
-	}
-}
-
 // Given a string of the form "host", "host:port", or "[ipv6::address]:port",
 // return true if the string includes a port.
 func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
@@ -68,43 +28,6 @@ type readClose struct {
 	io.Closer
 }
 
-// ReadResponse reads and returns an HTTP response from r.
-func ReadResponse(r *bufio.Reader) (*Response, os.Error) {
-	resp := new(Response)
-
-	// Parse the first line of the response.
-	resp.Header = make(map[string]string)
-
-	line, err := readLine(r)
-	if err != nil {
-		return nil, err
-	}
-	f := strings.Split(line, " ", 3)
-	if len(f) < 3 {
-		return nil, &badStringError{"malformed HTTP response", line}
-	}
-	resp.Status = f[1] + " " + f[2]
-	resp.StatusCode, err = strconv.Atoi(f[1])
-	if err != nil {
-		return nil, &badStringError{"malformed HTTP status code", f[1]}
-	}
-
-	// Parse the response headers.
-	for {
-		key, value, err := readKeyValue(r)
-		if err != nil {
-			return nil, err
-		}
-		if key == "" {
-			break // end of response header
-		}
-		resp.AddHeader(key, value)
-	}
-
-	return resp, nil
-}
-
-
 // Send issues an HTTP request.  Caller should close resp.Body when done reading it.
 //
 // TODO: support persistent connections (multiple requests on a single connection).
@@ -141,23 +64,13 @@ func send(req *Request) (resp *Response, err os.Error) {
 	}
 
 	reader := bufio.NewReader(conn)
-	resp, err = ReadResponse(reader)
+	resp, err = ReadResponse(reader, req.Method)
 	if err != nil {
 		conn.Close()
 		return nil, err
 	}
 
-	r := io.Reader(reader)
+	resp.Body = readClose{resp.Body, conn}
-	if v := resp.GetHeader("Transfer-Encoding"); v == "chunked" {
-		r = newChunkedReader(reader)
-	} else if v := resp.GetHeader("Content-Length"); v != "" {
-		n, err := strconv.Atoi64(v)
-		if err != nil {
-			return nil, &badStringError{"invalid Content-Length", v}
-		}
-		r = io.LimitReader(r, n)
-	}
-	resp.Body = readClose{r, conn}
 
 	return
 }
@@ -180,8 +93,8 @@ func shouldRedirect(statusCode int) bool {
 //    303 (See Other)
 //    307 (Temporary Redirect)
 //
-// finalURL is the URL from which the response was fetched -- identical to the input
+// finalURL is the URL from which the response was fetched -- identical to the
-// URL unless redirects were followed.
+// input URL unless redirects were followed.
 //
 // Caller should close r.Body when done reading it.
 func Get(url string) (r *Response, finalURL string, err os.Error) {

src/pkg/http/request.go

@@ -34,9 +34,12 @@ type ProtocolError struct {
 }
 
 var (
-	ErrLineTooLong   = &ProtocolError{"header line too long"}
+	ErrLineTooLong          = &ProtocolError{"header line too long"}
-	ErrHeaderTooLong = &ProtocolError{"header too long"}
+	ErrHeaderTooLong        = &ProtocolError{"header too long"}
-	ErrShortBody     = &ProtocolError{"entity body too short"}
+	ErrShortBody            = &ProtocolError{"entity body too short"}
+	ErrNotSupported         = &ProtocolError{"feature not supported"}
+	ErrUnexpectedTrailer    = &ProtocolError{"trailer header without chunked transfer encoding"}
+	ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"}
 )
 
 type badStringError struct {

src/pkg/http/response.go

@@ -0,0 +1,480 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// HTTP Response reading and parsing.
+
+package http
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+)
+
+// Response represents the response from an HTTP request.
+//
+type Response struct {
+	Status     string // e.g. "200 OK"
+	StatusCode int    // e.g. 200
+	Proto      string // e.g. "HTTP/1.0"
+	ProtoMajor int    // e.g. 1
+	ProtoMinor int    // e.g. 0
+
+	// RequestMethod records the method used in the HTTP request.
+	// Header fields such as Content-Length have method-specific meaning.
+	RequestMethod string // e.g. "HEAD", "CONNECT", "GET", etc.
+
+	// Header maps header keys to values.  If the response had multiple
+	// headers with the same key, they will be concatenated, with comma
+	// delimiters.  (Section 4.2 of RFC 2616 requires that multiple headers
+	// be semantically equivalent to a comma-delimited sequence.) Values
+	// duplicated by other fields in this struct (e.g., ContentLength) are
+	// omitted from Header.
+	//
+	// Keys in the map are canonicalized (see CanonicalHeaderKey).
+	Header map[string]string
+
+	// Body represents the response body.
+	Body io.ReadCloser
+
+	// ContentLength records the length of the associated content.  The
+	// value -1 indicates that the length is unknown.  Unless RequestMethod
+	// is "HEAD", values >= 0 indicate that the given number of bytes may
+	// be read from Body.
+	ContentLength int64
+
+	// Contains transfer encodings from outer-most to inner-most. Value is
+	// nil, means that "identity" encoding is used.
+	TransferEncoding []string
+
+	// Close records whether the header directed that the connection be
+	// closed after reading Body.  The value is advice for clients: neither
+	// ReadResponse nor Response.Write ever closes a connection.
+	Close bool
+
+	// Trailer maps trailer keys to values.  Like for Header, if the
+	// response has multiple trailer lines with the same key, they will be
+	// concatenated, delimited by commas.
+	Trailer map[string]string
+}
+
+// ReadResponse reads and returns an HTTP response from r.  The RequestMethod
+// parameter specifies the method used in the corresponding request (e.g.,
+// "GET", "HEAD").  Clients must call resp.Body.Close when finished reading
+// resp.Body.  After that call, clients can inspect resp.Trailer to find
+// key/value pairs included in the response trailer.
+func ReadResponse(r *bufio.Reader, requestMethod string) (resp *Response, err os.Error) {
+
+	resp = new(Response)
+
+	resp.RequestMethod = strings.ToUpper(requestMethod)
+
+	// Parse the first line of the response.
+	line, err := readLine(r)
+	if err != nil {
+		return nil, err
+	}
+	f := strings.Split(line, " ", 3)
+	if len(f) < 3 {
+		return nil, &badStringError{"malformed HTTP response", line}
+	}
+	resp.Status = f[1] + " " + f[2]
+	resp.StatusCode, err = strconv.Atoi(f[1])
+	if err != nil {
+		return nil, &badStringError{"malformed HTTP status code", f[1]}
+	}
+
+	resp.Proto = f[0]
+	var ok bool
+	if resp.ProtoMajor, resp.ProtoMinor, ok = parseHTTPVersion(resp.Proto); !ok {
+		return nil, &badStringError{"malformed HTTP version", resp.Proto}
+	}
+
+	// Parse the response headers.
+	nheader := 0
+	resp.Header = make(map[string]string)
+	for {
+		key, value, err := readKeyValue(r)
+		if err != nil {
+			return nil, err
+		}
+		if key == "" {
+			break // end of response header
+		}
+		if nheader++; nheader >= maxHeaderLines {
+			return nil, ErrHeaderTooLong
+		}
+		resp.AddHeader(key, value)
+	}
+
+	fixPragmaCacheControl(resp.Header)
+
+	resp.TransferEncoding, err = fixTransferEncoding(resp.Header)
+	if err != nil {
+		return nil, err
+	}
+
+	resp.ContentLength, err = fixLength(resp.StatusCode, resp.RequestMethod,
+		resp.Header, resp.TransferEncoding)
+	if err != nil {
+		return nil, err
+	}
+
+	resp.Close = shouldClose(resp.ProtoMajor, resp.ProtoMinor, resp.Header)
+
+	resp.Trailer, err = fixTrailer(resp.Header, resp.TransferEncoding)
+	if err != nil {
+		return nil, err
+	}
+
+	// Prepare body reader.  ContentLength < 0 means chunked encoding,
+	// since multipart is not supported yet
+	if resp.ContentLength < 0 {
+		resp.Body = &body{newChunkedReader(r), resp, r}
+	} else {
+		resp.Body = &body{io.LimitReader(r, resp.ContentLength), nil, nil}
+	}
+
+	return resp, nil
+}
+
+// ffwdClose (fast-forward close) adds a Close method to a Reader which skips
+// ahead until EOF
+type body struct {
+	io.Reader
+	resp *Response     // non-nil value means read trailer
+	r    *bufio.Reader // underlying wire-format reader for the trailer
+}
+
+func (b *body) Close() os.Error {
+	trashBuf := make([]byte, 1024) // local for thread safety
+	for {
+		_, err := b.Read(trashBuf)
+		if err == nil {
+			continue
+		}
+		if err == os.EOF {
+			break
+		}
+		return err
+	}
+	if b.resp == nil { // not reading trailer
+		return nil
+	}
+
+	// TODO(petar): Put trailer reader code here
+
+	return nil
+}
+
+// RFC2616: Should treat
+//	Pragma: no-cache
+// like
+//	Cache-Control: no-cache
+func fixPragmaCacheControl(header map[string]string) {
+	if v, present := header["Pragma"]; present && v == "no-cache" {
+		if _, presentcc := header["Cache-Control"]; !presentcc {
+			header["Cache-Control"] = "no-cache"
+		}
+	}
+}
+
+// Parse the trailer header
+func fixTrailer(header map[string]string, te []string) (map[string]string, os.Error) {
+	raw, present := header["Trailer"]
+	if !present {
+		return nil, nil
+	}
+
+	header["Trailer"] = "", false
+	trailer := make(map[string]string)
+	keys := strings.Split(raw, ",", 0)
+	for _, key := range keys {
+		key = CanonicalHeaderKey(strings.TrimSpace(key))
+		switch key {
+		case "Transfer-Encoding", "Trailer", "Content-Length":
+			return nil, &badStringError{"bad trailer key", key}
+		}
+		trailer[key] = ""
+	}
+	if len(trailer) == 0 {
+		return nil, nil
+	}
+	if !chunked(te) {
+		// Trailer and no chunking
+		return nil, ErrUnexpectedTrailer
+	}
+	return trailer, nil
+}
+
+// Sanitize transfer encoding
+func fixTransferEncoding(header map[string]string) ([]string, os.Error) {
+	raw, present := header["Transfer-Encoding"]
+	if !present {
+		return nil, nil
+	}
+
+	header["Transfer-Encoding"] = "", false
+	encodings := strings.Split(raw, ",", 0)
+	te := make([]string, 0, len(encodings))
+	// TODO: Even though we only support "identity" and "chunked"
+	// encodings, the loop below is designed with foresight. One
+	// invariant that must be maintained is that, if present,
+	// chunked encoding must always come first.
+	for _, encoding := range encodings {
+		encoding = strings.ToLower(strings.TrimSpace(encoding))
+		// "identity" encoding is not recored
+		if encoding == "identity" {
+			break
+		}
+		if encoding != "chunked" {
+			return nil, &badStringError{"unsupported transfer encoding", encoding}
+		}
+		te = te[0 : len(te)+1]
+		te[len(te)-1] = encoding
+	}
+	if len(te) > 1 {
+		return nil, &badStringError{"too many transfer encodings", strings.Join(te, ",")}
+	}
+	if len(te) > 0 {
+		// Chunked encoding trumps Content-Length. See RFC 2616
+		// Section 4.4. Currently len(te) > 0 implies chunked
+		// encoding.
+		header["Content-Length"] = "", false
+		return te, nil
+	}
+
+	return nil, nil
+}
+
+func noBodyExpected(requestMethod string) bool {
+	return requestMethod == "HEAD"
+}
+
+// Determine the expected body length, using RFC 2616 Section 4.4. This
+// function is not a method, because ultimately it should be shared by
+// ReadResponse and ReadRequest.
+func fixLength(status int, requestMethod string, header map[string]string, te []string) (int64, os.Error) {
+
+	// Logic based on response type or status
+	if noBodyExpected(requestMethod) {
+		return 0, nil
+	}
+	if status/100 == 1 {
+		return 0, nil
+	}
+	switch status {
+	case 204, 304:
+		return 0, nil
+	}
+
+	// Logic based on Transfer-Encoding
+	if chunked(te) {
+		return -1, nil
+	}
+
+	// Logic based on Content-Length
+	if cl, present := header["Content-Length"]; present {
+		cl = strings.TrimSpace(cl)
+		if cl != "" {
+			n, err := strconv.Atoi64(cl)
+			if err != nil || n < 0 {
+				return -1, &badStringError{"bad Content-Length", cl}
+			}
+			return n, nil
+		} else {
+			header["Content-Length"] = "", false
+		}
+	}
+
+	// Logic based on media type. The purpose of the following code is just
+	// to detect whether the unsupported "multipart/byteranges" is being
+	// used. A proper Content-Type parser is needed in the future.
+	if ct, present := header["Content-Type"]; present {
+		ct = strings.ToLower(ct)
+		if strings.Index(ct, "multipart/byteranges") >= 0 {
+			return -1, ErrNotSupported
+		}
+	}
+
+
+	// Logic based on close
+	return -1, nil
+}
+
+// Determine whether to hang up after sending a request and body, or
+// receiving a response and body
+func shouldClose(major, minor int, header map[string]string) bool {
+	if major < 1 || (major == 1 && minor < 1) {
+		return true
+	} else if v, present := header["Connection"]; present {
+		// TODO: Should split on commas, toss surrounding white space,
+		// and check each field.
+		if v == "close" {
+			return true
+		}
+	}
+	return false
+}
+
+// Checks whether chunked is part of the encodings stack
+func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" }
+
+// AddHeader adds a value under the given key.  Keys are not case sensitive.
+func (r *Response) AddHeader(key, value string) {
+	key = CanonicalHeaderKey(key)
+
+	oldValues, oldValuesPresent := r.Header[key]
+	if oldValuesPresent {
+		r.Header[key] = oldValues + "," + value
+	} else {
+		r.Header[key] = value
+	}
+}
+
+// GetHeader returns the value of the response header with the given
+// key, and true.  If there were multiple headers with this key, their
+// values are concatenated, with a comma delimiter.  If there were no
+// response headers with the given key, it returns the empty string and
+// false.  Keys are not case sensitive.
+func (r *Response) GetHeader(key string) (value string) {
+	value, _ = r.Header[CanonicalHeaderKey(key)]
+	return
+}
+
+// ProtoAtLeast returns whether the HTTP protocol used
+// in the response is at least major.minor.
+func (r *Response) ProtoAtLeast(major, minor int) bool {
+	return r.ProtoMajor > major ||
+		r.ProtoMajor == major && r.ProtoMinor >= minor
+}
+
+// Writes the response (header, body and trailer) in wire format. This method
+// consults the following fields of resp:
+//
+//  StatusCode
+//  ProtoMajor
+//  ProtoMinor
+//  RequestMethod
+//  TransferEncoding
+//  Trailer
+//  Body
+//  ContentLength
+//  Header, values for non-canonical keys will have unpredictable behavior
+//
+func (resp *Response) Write(w io.Writer) os.Error {
+
+	// RequestMethod should be lower-case
+	resp.RequestMethod = strings.ToUpper(resp.RequestMethod)
+
+	// Status line
+	text, ok := statusText[resp.StatusCode]
+	if !ok {
+		text = "status code " + strconv.Itoa(resp.StatusCode)
+	}
+	io.WriteString(w, "HTTP/"+strconv.Itoa(resp.ProtoMajor)+".")
+	io.WriteString(w, strconv.Itoa(resp.ProtoMinor)+" ")
+	io.WriteString(w, strconv.Itoa(resp.StatusCode)+" "+text+"\r\n")
+
+	// Sanitize the field triple (Body, ContentLength, TransferEncoding)
+	if noBodyExpected(resp.RequestMethod) {
+		resp.Body = nil
+		resp.TransferEncoding = nil
+		// resp.ContentLength is expected to hold Content-Length
+		if resp.ContentLength < 0 {
+			return ErrMissingContentLength
+		}
+	} else {
+		if !resp.ProtoAtLeast(1, 1) || resp.Body == nil {
+			resp.TransferEncoding = nil
+		}
+		if chunked(resp.TransferEncoding) {
+			resp.ContentLength = -1
+		} else if resp.Body != nil {
+			// For safety, consider sending a 0-length body an
+			// error
+			if resp.ContentLength <= 0 {
+				return &ProtocolError{"zero body length"}
+			}
+		} else { // no chunking, no body
+			resp.ContentLength = 0
+		}
+	}
+
+	// Write Content-Length and/or Transfer-Encoding whose values are a
+	// function of the sanitized field triple (Body, ContentLength,
+	// TransferEncoding)
+	if chunked(resp.TransferEncoding) {
+		io.WriteString(w, "Transfer-Encoding: chunked\r\n")
+	} else {
+		io.WriteString(w, "Content-Length: ")
+		io.WriteString(w, strconv.Itoa64(resp.ContentLength)+"\r\n")
+	}
+	if resp.Header != nil {
+		resp.Header["Content-Length"] = "", false
+		resp.Header["Transfer-Encoding"] = "", false
+	}
+
+	// Sanitize Trailer
+	if !chunked(resp.TransferEncoding) {
+		resp.Trailer = nil
+	} else if resp.Trailer != nil {
+		// TODO: At some point, there should be a generic mechanism for
+		// writing long headers, using HTTP line splitting
+		io.WriteString(w, "Trailer: ")
+		needComma := false
+		for k, _ := range resp.Trailer {
+			k = CanonicalHeaderKey(k)
+			switch k {
+			case "Transfer-Encoding", "Trailer", "Content-Length":
+				return &badStringError{"invalid Trailer key", k}
+			}
+			if needComma {
+				io.WriteString(w, ",")
+			}
+			io.WriteString(w, k)
+			needComma = true
+		}
+		io.WriteString(w, "\r\n")
+	}
+	if resp.Header != nil {
+		resp.Header["Trailer"] = "", false
+	}
+
+	// Rest of header
+	for k, v := range resp.Header {
+		io.WriteString(w, k+": "+v+"\r\n")
+	}
+
+	// End-of-header
+	io.WriteString(w, "\r\n")
+
+	// Write body
+	if resp.Body != nil {
+		var err os.Error
+		if chunked(resp.TransferEncoding) {
+			cw := NewChunkedWriter(w)
+			_, err = io.Copy(cw, resp.Body)
+			if err == nil {
+				err = cw.Close()
+			}
+		} else {
+			_, err = io.Copy(w, io.LimitReader(resp.Body, resp.ContentLength))
+		}
+		if err != nil {
+			return err
+		}
+	}
+
+	// TODO(petar): Place trailer writer code here.
+	if chunked(resp.TransferEncoding) {
+		// Last chunk, empty trailer
+		io.WriteString(w, "\r\n")
+	}
+
+	// Success
+	return nil
+}

src/pkg/rpc/client.go

@@ -126,7 +126,7 @@ func DialHTTP(network, address string) (*Client, os.Error) {
 
 	// Require successful HTTP response
 	// before switching to RPC protocol.
-	resp, err := http.ReadResponse(bufio.NewReader(conn))
+	resp, err := http.ReadResponse(bufio.NewReader(conn), "CONNECT")
 	if err == nil && resp.Status == connected {
 		return NewClient(conn), nil
 	}

src/pkg/websocket/client.go

@@ -90,7 +90,7 @@ func handshake(resourceName, host, origin, location, protocol string, br *bufio.
 	}
 	bw.WriteString("\r\n")
 	bw.Flush()
-	resp, err := http.ReadResponse(br)
+	resp, err := http.ReadResponse(br, "GET")
 	if err != nil {
 		return
 	}