PHP 8.0 | Tokenizer/PHP: add support for nullsafe object operator

package.xml

@@ -121,6 +121,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
       <file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
       <file baseinstalldir="" name="BackfillNumericSeparatorTest.inc" role="test" />
       <file baseinstalldir="" name="BackfillNumericSeparatorTest.php" role="test" />
+      <file baseinstalldir="" name="NullsafeObjectOperatorTest.inc" role="test" />
+      <file baseinstalldir="" name="NullsafeObjectOperatorTest.php" role="test" />
       <file baseinstalldir="" name="ShortArrayTest.inc" role="test" />
       <file baseinstalldir="" name="ShortArrayTest.php" role="test" />
       <file baseinstalldir="" name="StableCommentWhitespaceTest.inc" role="test" />
@@ -1991,6 +1993,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/ShortArrayTest.php" name="tests/Core/Tokenizer/ShortArrayTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/ShortArrayTest.inc" name="tests/Core/Tokenizer/ShortArrayTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.php" />
@@ -2050,6 +2054,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/ShortArrayTest.php" name="tests/Core/Tokenizer/ShortArrayTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/ShortArrayTest.inc" name="tests/Core/Tokenizer/ShortArrayTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.php" />

src/Tokenizers/PHP.php

@@ -370,6 +370,7 @@ class PHP extends Tokenizer
         T_NS_C                     => 13,
         T_NS_SEPARATOR             => 1,
         T_NEW                      => 3,
+        T_NULLSAFE_OBJECT_OPERATOR => 3,
         T_OBJECT_OPERATOR          => 2,
         T_OPEN_TAG_WITH_ECHO       => 3,
         T_OR_EQUAL                 => 2,
@@ -1015,6 +1016,29 @@ protected function tokenize($string)
                 continue;
             }
 
+            /*
+                Before PHP 8, the ?-> operator was tokenized as
+                T_INLINE_THEN followed by T_OBJECT_OPERATOR.
+                So look for and combine these tokens in earlier versions.
+            */
+
+            if ($tokenIsArray === false
+                && $token[0] === '?'
+                && isset($tokens[($stackPtr + 1)]) === true
+                && is_array($tokens[($stackPtr + 1)]) === true
+                && $tokens[($stackPtr + 1)][0] === T_OBJECT_OPERATOR
+            ) {
+                $newToken            = [];
+                $newToken['code']    = T_NULLSAFE_OBJECT_OPERATOR;
+                $newToken['type']    = 'T_NULLSAFE_OBJECT_OPERATOR';
+                $newToken['content'] = '?->';
+                $finalTokens[$newStackPtr] = $newToken;
+
+                $newStackPtr++;
+                $stackPtr++;
+                continue;
+            }
+
             /*
                 Before PHP 7.4, underscores inside T_LNUMBER and T_DNUMBER
                 tokens split the token with a T_STRING. So look for
@@ -1510,17 +1534,18 @@ function return types. We want to keep the parenthesis map clean,
                     // Some T_STRING tokens should remain that way
                     // due to their context.
                     $context = [
-                        T_OBJECT_OPERATOR      => true,
-                        T_FUNCTION             => true,
-                        T_CLASS                => true,
-                        T_EXTENDS              => true,
-                        T_IMPLEMENTS           => true,
-                        T_NEW                  => true,
-                        T_CONST                => true,
-                        T_NS_SEPARATOR         => true,
-                        T_USE                  => true,
-                        T_NAMESPACE            => true,
-                        T_PAAMAYIM_NEKUDOTAYIM => true,
+                        T_OBJECT_OPERATOR          => true,
+                        T_NULLSAFE_OBJECT_OPERATOR => true,
+                        T_FUNCTION                 => true,
+                        T_CLASS                    => true,
+                        T_EXTENDS                  => true,
+                        T_IMPLEMENTS               => true,
+                        T_NEW                      => true,
+                        T_CONST                    => true,
+                        T_NS_SEPARATOR             => true,
+                        T_USE                      => true,
+                        T_NAMESPACE                => true,
+                        T_PAAMAYIM_NEKUDOTAYIM     => true,
                     ];
 
                     if (isset($context[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
@@ -2012,6 +2037,7 @@ protected function processAdditional()
                     T_CLOSE_PARENTHESIS        => T_CLOSE_PARENTHESIS,
                     T_VARIABLE                 => T_VARIABLE,
                     T_OBJECT_OPERATOR          => T_OBJECT_OPERATOR,
+                    T_NULLSAFE_OBJECT_OPERATOR => T_NULLSAFE_OBJECT_OPERATOR,
                     T_STRING                   => T_STRING,
                     T_CONSTANT_ENCAPSED_STRING => T_CONSTANT_ENCAPSED_STRING,
                 ];
@@ -2081,9 +2107,10 @@ protected function processAdditional()
                 }
 
                 $context = [
-                    T_OBJECT_OPERATOR      => true,
-                    T_NS_SEPARATOR         => true,
-                    T_PAAMAYIM_NEKUDOTAYIM => true,
+                    T_OBJECT_OPERATOR          => true,
+                    T_NULLSAFE_OBJECT_OPERATOR => true,
+                    T_NS_SEPARATOR             => true,
+                    T_PAAMAYIM_NEKUDOTAYIM     => true,
                 ];
                 if (isset($context[$this->tokens[$x]['code']]) === true) {
                     if (PHP_CODESNIFFER_VERBOSITY > 1) {

src/Tokenizers/Tokenizer.php

@@ -1291,11 +1291,12 @@ private function recurseScopeMap($stackPtr, $depth=1, &$ignore=0)
                                 // a new statement, it isn't a scope opener.
                                 $disallowed  = Util\Tokens::$assignmentTokens;
                                 $disallowed += [
-                                    T_DOLLAR           => true,
-                                    T_VARIABLE         => true,
-                                    T_OBJECT_OPERATOR  => true,
-                                    T_COMMA            => true,
-                                    T_OPEN_PARENTHESIS => true,
+                                    T_DOLLAR                   => true,
+                                    T_VARIABLE                 => true,
+                                    T_OBJECT_OPERATOR          => true,
+                                    T_NULLSAFE_OBJECT_OPERATOR => true,
+                                    T_COMMA                    => true,
+                                    T_OPEN_PARENTHESIS         => true,
                                 ];
 
                                 if (isset($disallowed[$this->tokens[$x]['code']]) === true) {

src/Util/Tokens.php

@@ -124,6 +124,11 @@
     define('T_FN', 'PHPCS_T_FN');
 }
 
+// Some PHP 8.0 tokens, replicated for lower versions.
+if (defined('T_NULLSAFE_OBJECT_OPERATOR') === false) {
+    define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR');
+}
+
 // Tokens used for parsing doc blocks.
 define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
 define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');

tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc

@@ -0,0 +1,29 @@
+<?php
+
+/*
+ * Null safe operator.
+ */
+
+/* testObjectOperator */
+echo $obj->foo;
+
+/* testNullsafeObjectOperator */
+echo $obj?->foo;
+
+/* testNullsafeObjectOperatorWriteContext */
+// Intentional parse error, but not the concern of the tokenizer.
+$foo?->bar->baz = 'baz';
+
+/* testTernaryThen */
+echo $obj ? $obj->prop : $other->prop;
+
+/* testParseErrorWhitespaceNotAllowed */
+echo $obj ?
+    -> foo;
+
+/* testParseErrorCommentNotAllowed */
+echo $obj ?/*comment*/-> foo;
+
+/* testLiveCoding */
+// Intentional parse error. This has to be the last test in the file.
+echo $obj?

tests/Core/Tokenizer/NullsafeObjectOperatorTest.php

@@ -0,0 +1,140 @@
+<?php
+/**
+ * Tests the backfill for the PHP >= 8.0 nullsafe object operator.
+ *
+ * @author    Juliette Reinders Folmer <[email protected]>
+ * @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600)
+ * @license   https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+
+use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
+use PHP_CodeSniffer\Util\Tokens;
+
+class NullsafeObjectOperatorTest extends AbstractMethodUnitTest
+{
+
+    /**
+     * Tokens to search for.
+     *
+     * @var array
+     */
+    protected $find = [
+        T_NULLSAFE_OBJECT_OPERATOR,
+        T_OBJECT_OPERATOR,
+        T_INLINE_THEN,
+    ];
+
+
+    /**
+     * Test that a normal object operator is still tokenized as such.
+     *
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
+     *
+     * @return void
+     */
+    public function testObjectOperator()
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $operator = $this->getTargetToken('/* testObjectOperator */', $this->find);
+        $this->assertSame(T_OBJECT_OPERATOR, $tokens[$operator]['code'], 'Failed asserting code is object operator');
+        $this->assertSame('T_OBJECT_OPERATOR', $tokens[$operator]['type'], 'Failed asserting type is object operator');
+
+    }//end testObjectOperator()
+
+
+    /**
+     * Test that a nullsafe object operator is tokenized as such.
+     *
+     * @param string $testMarker The comment which prefaces the target token in the test file.
+     *
+     * @dataProvider dataNullsafeObjectOperator
+     * @covers       PHP_CodeSniffer\Tokenizers\PHP::tokenize
+     *
+     * @return void
+     */
+    public function testNullsafeObjectOperator($testMarker)
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $operator = $this->getTargetToken($testMarker, $this->find);
+        $this->assertSame(T_NULLSAFE_OBJECT_OPERATOR, $tokens[$operator]['code'], 'Failed asserting code is nullsafe object operator');
+        $this->assertSame('T_NULLSAFE_OBJECT_OPERATOR', $tokens[$operator]['type'], 'Failed asserting type is nullsafe object operator');
+
+    }//end testNullsafeObjectOperator()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testNullsafeObjectOperator()
+     *
+     * @return array
+     */
+    public function dataNullsafeObjectOperator()
+    {
+        return [
+            ['/* testNullsafeObjectOperator */'],
+            ['/* testNullsafeObjectOperatorWriteContext */'],
+        ];
+
+    }//end dataNullsafeObjectOperator()
+
+
+    /**
+     * Test that a question mark not followed by an object operator is tokenized as T_TERNARY_THEN.
+     *
+     * @param string $testMarker         The comment which prefaces the target token in the test file.
+     * @param bool   $testObjectOperator Whether to test for the next non-empty token being tokenized
+     *                                   as an object operator.
+     *
+     * @dataProvider dataTernaryThen
+     * @covers       PHP_CodeSniffer\Tokenizers\PHP::tokenize
+     *
+     * @return void
+     */
+    public function testTernaryThen($testMarker, $testObjectOperator=false)
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $operator = $this->getTargetToken($testMarker, $this->find);
+        $this->assertSame(T_INLINE_THEN, $tokens[$operator]['code'], 'Failed asserting code is inline then');
+        $this->assertSame('T_INLINE_THEN', $tokens[$operator]['type'], 'Failed asserting type is inline then');
+
+        if ($testObjectOperator === true) {
+            $next = self::$phpcsFile->findNext(Tokens::$emptyTokens, ($operator + 1), null, true);
+            $this->assertSame(T_OBJECT_OPERATOR, $tokens[$next]['code'], 'Failed asserting code is object operator');
+            $this->assertSame('T_OBJECT_OPERATOR', $tokens[$next]['type'], 'Failed asserting type is object operator');
+        }
+
+    }//end testTernaryThen()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testTernaryThen()
+     *
+     * @return array
+     */
+    public function dataTernaryThen()
+    {
+        return [
+            ['/* testTernaryThen */'],
+            [
+                '/* testParseErrorWhitespaceNotAllowed */',
+                true,
+            ],
+            [
+                '/* testParseErrorCommentNotAllowed */',
+                true,
+            ],
+            ['/* testLiveCoding */'],
+        ];
+
+    }//end dataTernaryThen()
+
+
+}//end class

tests/Core/Tokenizer/ShortArrayTest.inc

@@ -62,6 +62,8 @@ $a = (new Foo( array(1, array(4, 5), 3) ))[1][0];
 /* testClassMemberDereferencingOnClone */
 echo (clone $iterable)[20];
 
+/* testNullsafeMethodCallDereferencing */
+$var = $obj?->function_call()[$x];
 
 /*
  * Short array brackets.

tests/Core/Tokenizer/ShortArrayTest.php

@@ -72,6 +72,7 @@ public function dataSquareBrackets()
             ['/* testClassMemberDereferencingOnInstantiation1 */'],
             ['/* testClassMemberDereferencingOnInstantiation2 */'],
             ['/* testClassMemberDereferencingOnClone */'],
+            ['/* testNullsafeMethodCallDereferencing */'],
             ['/* testLiveCoding */'],
         ];