fixup! Add tests for the C tokenizer and expose it as a private module

pablogsal · pablogsal · commit 0e4795803d25 · 2021-08-24T16:30:30.000+01:00
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
@@ -72,9 +72,9 @@ tokenizeriter_next(tokenizeriterobject* it)
     }
     PyObject* str = NULL;
     if (start == NULL || end == NULL) {
-        str = PyUnicode_FromStringAndSize(start, end - start);
-    } else {
         str = PyUnicode_FromString("");
+    } else {
+        str = PyUnicode_FromStringAndSize(start, end - start);
     }
     if (str == NULL) {
         return NULL;
@@ -192,4 +192,4 @@ PyMODINIT_FUNC
 PyInit__tokenize(void)
 {
     return PyModuleDef_Init(&_tokenizemodule);
-}
+}

Original file line number	Diff line number	Diff line change
`@@ -72,9 +72,9 @@ tokenizeriter_next(tokenizeriterobject* it)`
`72`	`72`	`}`
`73`	`73`	`PyObject* str = NULL;`
`74`	`74`	`if (start == NULL \|\| end == NULL) {`
`75`		`- str = PyUnicode_FromStringAndSize(start, end - start);`
`76`		`- } else {`
`77`	`75`	`str = PyUnicode_FromString("");`
	`76`	`+ } else {`
	`77`	`+ str = PyUnicode_FromStringAndSize(start, end - start);`
`78`	`78`	`}`
`79`	`79`	`if (str == NULL) {`
`80`	`80`	`return NULL;`
`@@ -192,4 +192,4 @@ PyMODINIT_FUNC`
`192`	`192`	`PyInit__tokenize(void)`
`193`	`193`	`{`
`194`	`194`	`return PyModuleDef_Init(&_tokenizemodule);`
`195`		`-}`
	`195`	`+}`