Merge pull request RustPython#3014 from leesungbin/underscore_grammar

Solve some lexical error with underscore

Author: youknowone

Lib/test/test_grammar.py

@@ -179,8 +179,6 @@ def test_float_exponent_tokenization(self):
         self.assertEqual(1 if 0else 0, 0)
         self.assertRaises(SyntaxError, eval, "0 if 1Else 0")
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_underscore_literals(self):
         for lit in VALID_UNDERSCORE_LITERALS:
             self.assertEqual(eval(lit), eval(lit.replace('_', '')))

parser/src/lexer.rs

@@ -316,10 +316,21 @@ where
 
             // 1e6 for example:
             if self.chr0 == Some('e') || self.chr0 == Some('E') {
+                if self.chr1 == Some('_') {
+                    return Err(LexicalError {
+                        error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),
+                        location: self.get_pos(),
+                    });
+                }
                 value_text.push(self.next_char().unwrap().to_ascii_lowercase());
-
                 // Optional +/-
                 if self.chr0 == Some('-') || self.chr0 == Some('+') {
+                    if self.chr1 == Some('_') {
+                        return Err(LexicalError {
+                            error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),
+                            location: self.get_pos(),
+                        });
+                    }
                     value_text.push(self.next_char().unwrap());
                 }