Tokenizer: improve tokenization of T_NULLABLE vs T_INLINE_THEN

This is an attempt to make the differentiation between T_NULLABLE and T_INLINE_THEN more stable.

There is only a limited set of tokens which can be used for type declarations. If a `?` is not followed by one of these, we can be certain that it is a ternary operator `T_INLINE_THEN`.

The more resource intensive "walking back" check will now only be done when it could be either.
The "walking back" token logic could probably still do with an additional check for `T_INSTANCEOF`, but the changes now made, should prevent the majority of issues where `T_INLINE_THEN` is misidentified as `T_NULLABLE`.

Includes unit tests via the PSR12.Functions.NullableTypeDeclaration sniff.

Author: jrfnl

src/Standards/PSR12/Tests/Functions/NullableTypeDeclarationUnitTest.inc

@@ -66,7 +66,19 @@ class TestTokenizingOfNullableVsInlineThen {
     }
 }
 
+// Issue #2641.
 $foo = new static(
     is_null($a) ? foo($a) : $a,
     is_null($b) ? $b : $c
 );
+
+// Issue #2791.
+class testInstanceOf() {
+    function testIt() {
+        $foo = $value instanceof static ? '(' . $value . ')' : $value;
+        $bar = $value instanceof static ? function_call($value) : $value;
+        $baz = $value instanceof static ? array($value) : $value;
+        $bal = $value instanceof static ? \className::$property : $value;
+        $bal = $value instanceof static ? CONSTANT_NAME : $value;
+    }
+}

src/Standards/PSR12/Tests/Functions/NullableTypeDeclarationUnitTest.inc.fixed

@@ -64,7 +64,19 @@ class TestTokenizingOfNullableVsInlineThen {
     }
 }
 
+// Issue #2641.
 $foo = new static(
     is_null($a) ? foo($a) : $a,
     is_null($b) ? $b : $c
 );
+
+// Issue #2791.
+class testInstanceOf() {
+    function testIt() {
+        $foo = $value instanceof static ? '(' . $value . ')' : $value;
+        $bar = $value instanceof static ? function_call($value) : $value;
+        $baz = $value instanceof static ? array($value) : $value;
+        $bal = $value instanceof static ? \className::$property : $value;
+        $bal = $value instanceof static ? CONSTANT_NAME : $value;
+    }
+}

src/Tokenizers/PHP.php

@@ -1038,6 +1038,65 @@ protected function tokenize($string)
                 $newToken            = [];
                 $newToken['content'] = '?';
 
+                /*
+                 * Check if the next non-empty token is one of the tokens which can be used
+                 * in type declarations. If not, it's definitely a ternary.
+                 * At this point, the only token types which need to be taken into consideration
+                 * as potential type declarations are T_STRING, T_ARRAY, T_CALLABLE and T_NS_SEPARATOR.
+                 */
+
+                $lastRelevantNonEmpty = null;
+
+                for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
+                    if (is_array($tokens[$i]) === true) {
+                        $tokenType = $tokens[$i][0];
+                    } else {
+                        $tokenType = $tokens[$i];
+                    }
+
+                    if (isset(Util\Tokens::$emptyTokens[$tokenType]) === true) {
+                        continue;
+                    }
+
+                    if ($tokenType === T_STRING
+                        || $tokenType === T_ARRAY
+                        || $tokenType === T_NS_SEPARATOR
+                    ) {
+                        $lastRelevantNonEmpty = $tokenType;
+                        continue;
+                    }
+
+                    if (($tokenType !== T_CALLABLE
+                        && isset($lastRelevantNonEmpty) === false)
+                        || ($lastRelevantNonEmpty === T_ARRAY
+                        && $tokenType === '(')
+                        || ($lastRelevantNonEmpty === T_STRING
+                        && ($tokenType === T_DOUBLE_COLON
+                        || $tokenType === '('
+                        || $tokenType === ':'))
+                    ) {
+                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                            echo "\t\t* token $stackPtr changed from ? to T_INLINE_THEN".PHP_EOL;
+                        }
+
+                        $newToken['code'] = T_INLINE_THEN;
+                        $newToken['type'] = 'T_INLINE_THEN';
+
+                        $insideInlineIf[] = $stackPtr;
+
+                        $finalTokens[$newStackPtr] = $newToken;
+                        $newStackPtr++;
+                        continue 2;
+                    }
+
+                    break;
+                }//end for
+
+                /*
+                 * This can still be a nullable type or a ternary.
+                 * Do additional checking.
+                 */
+
                 $prevNonEmpty     = null;
                 $lastSeenNonEmpty = null;