Fix #1679: "0x" was taken as a valid integer literal.

Fixes the tokenizer, tokenize.py and int() to reject this.
Patches by Malte Helmert.

Lib/test/test_builtin.py

@@ -816,6 +816,11 @@ def test_int(self):
         self.assertEqual(int('0123', 0), 83)
         self.assertEqual(int('0x123', 16), 291)
 
+        # Bug 1679: "0x" is not a valid hex literal
+        self.assertRaises(ValueError, int, "0x", 16)
+        self.assertRaises(ValueError, int, "0x", 0)
+
+
         # SF bug 1334662: int(string, base) wrong answers
         # Various representations of 2**32 evaluated to 0
         # rather than 2**32 in previous versions

Lib/test/test_grammar.py

@@ -30,6 +30,8 @@ def testPlainIntegers(self):
         self.assertEquals(0xff, 255)
         self.assertEquals(0377, 255)
         self.assertEquals(2147483647, 017777777777)
+        # "0x" is not a valid literal
+        self.assertRaises(SyntaxError, eval, "0x")
         from sys import maxint
         if maxint == 2147483647:
             self.assertEquals(-2147483647-1, -020000000000)

Lib/tokenize.py

@@ -50,7 +50,7 @@ def maybe(*choices): return group(*choices) + '?'
 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
 Name = r'[a-zA-Z_]\w*'
 
-Hexnumber = r'0[xX][\da-fA-F]*[lL]?'
+Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
 Octnumber = r'0[0-7]*[lL]?'
 Decnumber = r'[1-9]\d*[lL]?'
 Intnumber = group(Hexnumber, Octnumber, Decnumber)

Misc/NEWS

@@ -12,6 +12,8 @@ What's New in Python 2.6 alpha 1?
 Core and builtins
 -----------------
 
+- Issue #1679: "0x" was taken as a valid integer literal.
+
 - Issue #1865: Bytes as an alias for str and b"" as an alias "" were
   added.
 

Parser/tokenizer.c

@@ -1332,7 +1332,14 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
 				goto imaginary;
 #endif
 			if (c == 'x' || c == 'X') {
+
 				/* Hex */
+				c = tok_nextc(tok);
+				if (!isxdigit(c)) {
+					tok->done = E_TOKEN;
+					tok_backup(tok, c);
+					return ERRORTOKEN;
+				}
 				do {
 					c = tok_nextc(tok);
 				} while (isxdigit(c));

Python/mystrtoul.c

@@ -116,6 +116,12 @@ PyOS_strtoul(register char *str, char **ptr, int base)
 		if (*str == '0') {
 			++str;
 			if (*str == 'x' || *str == 'X') {
+				/* there must be at least one digit after 0x */
+				if (_PyLong_DigitValue[Py_CHARMASK(str[1])] >= 16) {
+					if (ptr)
+						*ptr = str;
+					return 0;
+				}
 				++str;
 				base = 16;
 			}
@@ -129,9 +135,16 @@ PyOS_strtoul(register char *str, char **ptr, int base)
 	case 16:	/* skip leading 0x or 0X */
 		if (*str == '0') {
 			++str;
-				if (*str == 'x' || *str == 'X')
+			if (*str == 'x' || *str == 'X') {
+				/* there must be at least one digit after 0x */
+				if (_PyLong_DigitValue[Py_CHARMASK(str[1])] >= 16) {
+					if (ptr)
+						*ptr = str;
+					return 0;
+				}
 				++str;
 			}
+		}
 		break;
 	}