mime: speed up ParseMediaType

Eschew UTF-8 decoding and strings.IndexFunc where possible, and rely
on 128-bit bitmaps instead. Eliminate some bounds checks.

Some benchmark results (no changes to allocations):

goos: darwin
goarch: amd64
pkg: mime
cpu: Intel(R) Core(TM) i7-6700HQ CPU @ 2.60GHz
                      │     old     │                 new                 │
                      │   sec/op    │   sec/op     vs base                │
ParseMediaType-8        71.75µ ± 0%   55.53µ ± 0%  -22.60% (p=0.000 n=20)
ParseMediaTypeBogus-8   5.330µ ± 0%   3.603µ ± 0%  -32.41% (p=0.000 n=20)
geomean                 19.56µ        14.14µ       -27.67%

src/mime/grammar.go

@@ -4,22 +4,68 @@
 
 package mime
 
-import (
+// isTSpecial reports whether c is in 'tspecials' as defined by RFC
-	"strings"
-)
-
-// isTSpecial reports whether rune is in 'tspecials' as defined by RFC
 // 1521 and RFC 2045.
-func isTSpecial(r rune) bool {
+func isTSpecial(c byte) bool {
-	return strings.ContainsRune(`()<>@,;:\"/[]?=`, r)
+	// tspecials :=  "(" / ")" / "<" / ">" / "@" /
+	//               "," / ";" / ":" / "\" / <">
+	//               "/" / "[" / "]" / "?" / "="
+	//
+	// mask is a 128-bit bitmap with 1s for allowed bytes,
+	// so that the byte c can be tested with a shift and an and.
+	// If c >= 128, then 1<<c and 1<<(c-64) will both be zero,
+	// and this function will return false.
+	const mask = 0 |
+		1<<'(' |
+		1<<')' |
+		1<<'<' |
+		1<<'>' |
+		1<<'@' |
+		1<<',' |
+		1<<';' |
+		1<<':' |
+		1<<'\\' |
+		1<<'"' |
+		1<<'/' |
+		1<<'[' |
+		1<<']' |
+		1<<'?' |
+		1<<'='
+	return ((uint64(1)<<c)&(mask&(1<<64-1)) |
+		(uint64(1)<<(c-64))&(mask>>64)) != 0
 }
 
-// isTokenChar reports whether rune is in 'token' as defined by RFC
+// isTokenChar reports whether c is in 'token' as defined by RFC
 // 1521 and RFC 2045.
-func isTokenChar(r rune) bool {
+func isTokenChar(c byte) bool {
 	// token := 1*<any (US-ASCII) CHAR except SPACE, CTLs,
 	//             or tspecials>
-	return r > 0x20 && r < 0x7f && !isTSpecial(r)
+	//
+	// mask is a 128-bit bitmap with 1s for allowed bytes,
+	// so that the byte c can be tested with a shift and an and.
+	// If c >= 128, then 1<<c and 1<<(c-64) will both be zero,
+	// and this function will return false.
+	const mask = 0 |
+		(1<<(10)-1)<<'0' |
+		(1<<(26)-1)<<'a' |
+		(1<<(26)-1)<<'A' |
+		1<<'!' |
+		1<<'#' |
+		1<<'$' |
+		1<<'%' |
+		1<<'&' |
+		1<<'\'' |
+		1<<'*' |
+		1<<'+' |
+		1<<'-' |
+		1<<'.' |
+		1<<'^' |
+		1<<'_' |
+		1<<'`' |
+		1<<'|' |
+		1<<'~'
+	return ((uint64(1)<<c)&(mask&(1<<64-1)) |
+		(uint64(1)<<(c-64))&(mask>>64)) != 0
 }
 
 // isToken reports whether s is a 'token' as defined by RFC 1521
@@ -28,5 +74,10 @@ func isToken(s string) bool {
 	if s == "" {
 		return false
 	}
-	return strings.IndexFunc(s, isNotTokenChar) < 0
+	for _, c := range []byte(s) {
+		if !isTokenChar(c) {
+			return false
+		}
+	}
+	return true
 }

src/mime/mediatype.go

@@ -60,7 +60,7 @@ func FormatMediaType(t string, param map[string]string) string {
 				// attribute-char := <any (US-ASCII) CHAR except SPACE, CTLs, "*", "'", "%", or tspecials>
 				if ch <= ' ' || ch >= 0x7F ||
 					ch == '*' || ch == '\'' || ch == '%' ||
-					isTSpecial(rune(ch)) {
+					isTSpecial(ch) {
 
 					b.WriteString(value[offset:index])
 					offset = index + 1
@@ -250,23 +250,17 @@ func decode2231Enc(v string) (string, bool) {
 	return encv, true
 }
 
-func isNotTokenChar(r rune) bool {
-	return !isTokenChar(r)
-}
-
 // consumeToken consumes a token from the beginning of provided
 // string, per RFC 2045 section 5.1 (referenced from 2183), and return
 // the token consumed and the rest of the string. Returns ("", v) on
 // failure to consume at least one character.
 func consumeToken(v string) (token, rest string) {
-	notPos := strings.IndexFunc(v, isNotTokenChar)
+	for i := range len(v) {
-	if notPos == -1 {
+		if !isTokenChar(v[i]) {
-		return v, ""
+			return v[:i], v[i:]
+		}
 	}
-	if notPos == 0 {
+	return v, ""
-		return "", v
-	}
-	return v[0:notPos], v[notPos:]
 }
 
 // consumeValue consumes a "value" per RFC 2045, where a value is
@@ -299,7 +293,7 @@ func consumeValue(v string) (value, rest string) {
 		// and intended as a literal backslash. This makes Go servers deal better
 		// with MSIE without affecting the way they handle conforming MIME
 		// generators.
-		if r == '\\' && i+1 < len(v) && isTSpecial(rune(v[i+1])) {
+		if r == '\\' && i+1 < len(v) && isTSpecial(v[i+1]) {
 			buffer.WriteByte(v[i+1])
 			i++
 			continue