bpo-29568: Disable any characters between two percents for escaped percent "%%" in the format string for classic string formatting. (GH-513)

Author: serhiy-storchaka

Lib/test/test_format.py

@@ -70,12 +70,34 @@ def testcommon(formatstr, args, output=None, limit=None, overflowok=False):
     testformat(b_format, b_args, b_output, limit, overflowok)
     testformat(ba_format, b_args, ba_output, limit, overflowok)
 
+def test_exc(formatstr, args, exception, excmsg):
+    try:
+        testformat(formatstr, args)
+    except exception as exc:
+        if str(exc) == excmsg:
+            if verbose:
+                print("yes")
+        else:
+            if verbose: print('no')
+            print('Unexpected ', exception, ':', repr(str(exc)))
+    except:
+        if verbose: print('no')
+        print('Unexpected exception')
+        raise
+    else:
+        raise TestFailed('did not get expected exception: %s' % excmsg)
+
+def test_exc_common(formatstr, args, exception, excmsg):
+    # test str and bytes
+    test_exc(formatstr, args, exception, excmsg)
+    test_exc(formatstr.encode('ascii'), args, exception, excmsg)
 
 class FormatTest(unittest.TestCase):
 
     def test_common_format(self):
         # test the format identifiers that work the same across
         # str, bytes, and bytearrays (integer, float, oct, hex)
+        testcommon("%%", (), "%")
         testcommon("%.1d", (1,), "1")
         testcommon("%.*d", (sys.maxsize,1), overflowok=True)  # expect overflow
         testcommon("%.100d", (1,), '00000000000000000000000000000000000000'
@@ -246,6 +268,20 @@ def test_common_format(self):
         testcommon('%g', 1.1, '1.1')
         testcommon('%#g', 1.1, '1.10000')
 
+        if verbose:
+            print('Testing exceptions')
+        test_exc_common('%', (), ValueError, "incomplete format")
+        test_exc_common('% %s', 1, ValueError,
+                        "unsupported format character '%' (0x25) at index 2")
+        test_exc_common('%d', '1', TypeError,
+                        "%d format: a number is required, not str")
+        test_exc_common('%d', b'1', TypeError,
+                        "%d format: a number is required, not bytes")
+        test_exc_common('%x', '1', TypeError,
+                        "%x format: an integer is required, not str")
+        test_exc_common('%x', 3.14, TypeError,
+                        "%x format: an integer is required, not float")
+
     def test_str_format(self):
         testformat("%r", "\u0378", "'\\u0378'")  # non printable
         testformat("%a", "\u0378", "'\\u0378'")  # non printable
@@ -255,29 +291,10 @@ def test_str_format(self):
         # Test exception for unknown format characters, etc.
         if verbose:
             print('Testing exceptions')
-        def test_exc(formatstr, args, exception, excmsg):
-            try:
-                testformat(formatstr, args)
-            except exception as exc:
-                if str(exc) == excmsg:
-                    if verbose:
-                        print("yes")
-                else:
-                    if verbose: print('no')
-                    print('Unexpected ', exception, ':', repr(str(exc)))
-            except:
-                if verbose: print('no')
-                print('Unexpected exception')
-                raise
-            else:
-                raise TestFailed('did not get expected exception: %s' % excmsg)
         test_exc('abc %b', 1, ValueError,
                  "unsupported format character 'b' (0x62) at index 5")
         #test_exc(unicode('abc %\u3000','raw-unicode-escape'), 1, ValueError,
         #         "unsupported format character '?' (0x3000) at index 5")
-        test_exc('%d', '1', TypeError, "%d format: a number is required, not str")
-        test_exc('%x', '1', TypeError, "%x format: an integer is required, not str")
-        test_exc('%x', 3.14, TypeError, "%x format: an integer is required, not float")
         test_exc('%g', '1', TypeError, "must be real number, not str")
         test_exc('no format', '1', TypeError,
                  "not all arguments converted during string formatting")
@@ -334,28 +351,6 @@ def __bytes__(self):
         # Test exception for unknown format characters, etc.
         if verbose:
             print('Testing exceptions')
-        def test_exc(formatstr, args, exception, excmsg):
-            try:
-                testformat(formatstr, args)
-            except exception as exc:
-                if str(exc) == excmsg:
-                    if verbose:
-                        print("yes")
-                else:
-                    if verbose: print('no')
-                    print('Unexpected ', exception, ':', repr(str(exc)))
-            except:
-                if verbose: print('no')
-                print('Unexpected exception')
-                raise
-            else:
-                raise TestFailed('did not get expected exception: %s' % excmsg)
-        test_exc(b'%d', '1', TypeError,
-                "%d format: a number is required, not str")
-        test_exc(b'%d', b'1', TypeError,
-                "%d format: a number is required, not bytes")
-        test_exc(b'%x', 3.14, TypeError,
-                "%x format: an integer is required, not float")
         test_exc(b'%g', '1', TypeError, "float argument required, not str")
         test_exc(b'%g', b'1', TypeError, "float argument required, not bytes")
         test_exc(b'no format', 7, TypeError,

Misc/NEWS

@@ -10,6 +10,9 @@ What's New in Python 3.7.0 alpha 1?
 Core and Builtins
 -----------------
 
+- bpo-29568: Escaped percent "%%" in the format string for classic string
+  formatting no longer allows any characters between two percents.
+
 - bpo-29714: Fix a regression that bytes format may fail when containing zero
   bytes inside.
 

Objects/bytesobject.c

@@ -650,6 +650,12 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
 #endif
 
             fmt++;
+            if (*fmt == '%') {
+                *res++ = '%';
+                fmt++;
+                fmtcnt--;
+                continue;
+            }
             if (*fmt == '(') {
                 const char *keystart;
                 Py_ssize_t keylen;
@@ -794,11 +800,9 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
                                 "incomplete format");
                 goto error;
             }
-            if (c != '%') {
-                v = getnextarg(args, arglen, &argidx);
-                if (v == NULL)
-                    goto error;
-            }
+            v = getnextarg(args, arglen, &argidx);
+            if (v == NULL)
+                goto error;
 
             if (fmtcnt < 0) {
                 /* last writer: disable writer overallocation */
@@ -808,10 +812,6 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
             sign = 0;
             fill = ' ';
             switch (c) {
-            case '%':
-                *res++ = '%';
-                continue;
-
             case 'r':
                 // %r is only for 2/3 code; 3 only code should use %a
             case 'a':
@@ -1017,7 +1017,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
                 res += (width - len);
             }
 
-            if (dict && (argidx < arglen) && c != '%') {
+            if (dict && (argidx < arglen)) {
                 PyErr_SetString(PyExc_TypeError,
                            "not all arguments converted during bytes formatting");
                 Py_XDECREF(temp);

Objects/unicodeobject.c

@@ -14617,12 +14617,6 @@ unicode_format_arg_format(struct unicode_formatter_t *ctx,
     if (ctx->fmtcnt == 0)
         ctx->writer.overallocate = 0;
 
-    if (arg->ch == '%') {
-        if (_PyUnicodeWriter_WriteCharInline(writer, '%') < 0)
-            return -1;
-        return 1;
-    }
-
     v = unicode_format_getnextarg(ctx);
     if (v == NULL)
         return -1;
@@ -14882,6 +14876,13 @@ unicode_format_arg(struct unicode_formatter_t *ctx)
     int ret;
 
     arg.ch = PyUnicode_READ(ctx->fmtkind, ctx->fmtdata, ctx->fmtpos);
+    if (arg.ch == '%') {
+        ctx->fmtpos++;
+        ctx->fmtcnt--;
+        if (_PyUnicodeWriter_WriteCharInline(&ctx->writer, '%') < 0)
+            return -1;
+        return 0;
+    }
     arg.flags = 0;
     arg.width = -1;
     arg.prec = -1;
@@ -14903,7 +14904,7 @@ unicode_format_arg(struct unicode_formatter_t *ctx)
             return -1;
     }
 
-    if (ctx->dict && (ctx->argidx < ctx->arglen) && arg.ch != '%') {
+    if (ctx->dict && (ctx->argidx < ctx->arglen)) {
         PyErr_SetString(PyExc_TypeError,
                         "not all arguments converted during string formatting");
         return -1;