Skip to content

Commit 5aa9f1a

Browse files
committed
PHP 8.0 | Tokenizer/PHP: add support for nullsafe object operator
PHP 8 introduces a new object chaining operator `?->` which short-circuits moving to the next expression if the left-hand side evaluates to `null`. This operator can not be used in write-context, but that is not the concern of the PHPCS `Tokenizers\PHP` class. This commit: * Defines the token constant for PHP < 8.0. * Adds a backfill for the nullsafe object operator for PHP < 8.0 to the PHP tokenizer. * Adds the token to applicable token lists in the PHP and base tokenizer class, like the one used in the short array re-tokenization. * Adds perfunctory unit tests for the nullsafe object operator backfill. * Adds a unit test using the operator to the tokenizer tests for the short array re-tokenization. Refs: * https://wiki.php.net/rfc/nullsafe_operator * php/php-src@9bf1198
1 parent 2b8c1b3 commit 5aa9f1a

File tree

8 files changed

+230
-19
lines changed

8 files changed

+230
-19
lines changed

package.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
121121
<file baseinstalldir="" name="BackfillFnTokenTest.php" role="test" />
122122
<file baseinstalldir="" name="BackfillNumericSeparatorTest.inc" role="test" />
123123
<file baseinstalldir="" name="BackfillNumericSeparatorTest.php" role="test" />
124+
<file baseinstalldir="" name="NullsafeObjectOperatorTest.inc" role="test" />
125+
<file baseinstalldir="" name="NullsafeObjectOperatorTest.php" role="test" />
124126
<file baseinstalldir="" name="ShortArrayTest.inc" role="test" />
125127
<file baseinstalldir="" name="ShortArrayTest.php" role="test" />
126128
<file baseinstalldir="" name="StableCommentWhitespaceTest.inc" role="test" />
@@ -1991,6 +1993,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
19911993
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
19921994
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
19931995
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
1996+
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
1997+
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
19941998
<install as="CodeSniffer/Core/Tokenizer/ShortArrayTest.php" name="tests/Core/Tokenizer/ShortArrayTest.php" />
19951999
<install as="CodeSniffer/Core/Tokenizer/ShortArrayTest.inc" name="tests/Core/Tokenizer/ShortArrayTest.inc" />
19962000
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.php" />
@@ -2050,6 +2054,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
20502054
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.inc" name="tests/Core/Tokenizer/BackfillFnTokenTest.inc" />
20512055
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.php" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.php" />
20522056
<install as="CodeSniffer/Core/Tokenizer/BackfillNumericSeparatorTest.inc" name="tests/Core/Tokenizer/BackfillNumericSeparatorTest.inc" />
2057+
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
2058+
<install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.inc" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.inc" />
20532059
<install as="CodeSniffer/Core/Tokenizer/ShortArrayTest.php" name="tests/Core/Tokenizer/ShortArrayTest.php" />
20542060
<install as="CodeSniffer/Core/Tokenizer/ShortArrayTest.inc" name="tests/Core/Tokenizer/ShortArrayTest.inc" />
20552061
<install as="CodeSniffer/Core/Tokenizer/StableCommentWhitespaceTest.php" name="tests/Core/Tokenizer/StableCommentWhitespaceTest.php" />

src/Tokenizers/PHP.php

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,7 @@ class PHP extends Tokenizer
370370
T_NS_C => 13,
371371
T_NS_SEPARATOR => 1,
372372
T_NEW => 3,
373+
T_NULLSAFE_OBJECT_OPERATOR => 3,
373374
T_OBJECT_OPERATOR => 2,
374375
T_OPEN_TAG_WITH_ECHO => 3,
375376
T_OR_EQUAL => 2,
@@ -1015,6 +1016,29 @@ protected function tokenize($string)
10151016
continue;
10161017
}
10171018

1019+
/*
1020+
Before PHP 8, the ?-> operator was tokenized as
1021+
T_INLINE_THEN followed by T_OBJECT_OPERATOR.
1022+
So look for and combine these tokens in earlier versions.
1023+
*/
1024+
1025+
if ($tokenIsArray === false
1026+
&& $token[0] === '?'
1027+
&& isset($tokens[($stackPtr + 1)]) === true
1028+
&& is_array($tokens[($stackPtr + 1)]) === true
1029+
&& $tokens[($stackPtr + 1)][0] === T_OBJECT_OPERATOR
1030+
) {
1031+
$newToken = [];
1032+
$newToken['code'] = T_NULLSAFE_OBJECT_OPERATOR;
1033+
$newToken['type'] = 'T_NULLSAFE_OBJECT_OPERATOR';
1034+
$newToken['content'] = '?->';
1035+
$finalTokens[$newStackPtr] = $newToken;
1036+
1037+
$newStackPtr++;
1038+
$stackPtr++;
1039+
continue;
1040+
}
1041+
10181042
/*
10191043
Before PHP 7.4, underscores inside T_LNUMBER and T_DNUMBER
10201044
tokens split the token with a T_STRING. So look for
@@ -1510,17 +1534,18 @@ function return types. We want to keep the parenthesis map clean,
15101534
// Some T_STRING tokens should remain that way
15111535
// due to their context.
15121536
$context = [
1513-
T_OBJECT_OPERATOR => true,
1514-
T_FUNCTION => true,
1515-
T_CLASS => true,
1516-
T_EXTENDS => true,
1517-
T_IMPLEMENTS => true,
1518-
T_NEW => true,
1519-
T_CONST => true,
1520-
T_NS_SEPARATOR => true,
1521-
T_USE => true,
1522-
T_NAMESPACE => true,
1523-
T_PAAMAYIM_NEKUDOTAYIM => true,
1537+
T_OBJECT_OPERATOR => true,
1538+
T_NULLSAFE_OBJECT_OPERATOR => true,
1539+
T_FUNCTION => true,
1540+
T_CLASS => true,
1541+
T_EXTENDS => true,
1542+
T_IMPLEMENTS => true,
1543+
T_NEW => true,
1544+
T_CONST => true,
1545+
T_NS_SEPARATOR => true,
1546+
T_USE => true,
1547+
T_NAMESPACE => true,
1548+
T_PAAMAYIM_NEKUDOTAYIM => true,
15241549
];
15251550

15261551
if (isset($context[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
@@ -2012,6 +2037,7 @@ protected function processAdditional()
20122037
T_CLOSE_PARENTHESIS => T_CLOSE_PARENTHESIS,
20132038
T_VARIABLE => T_VARIABLE,
20142039
T_OBJECT_OPERATOR => T_OBJECT_OPERATOR,
2040+
T_NULLSAFE_OBJECT_OPERATOR => T_NULLSAFE_OBJECT_OPERATOR,
20152041
T_STRING => T_STRING,
20162042
T_CONSTANT_ENCAPSED_STRING => T_CONSTANT_ENCAPSED_STRING,
20172043
];
@@ -2081,9 +2107,10 @@ protected function processAdditional()
20812107
}
20822108

20832109
$context = [
2084-
T_OBJECT_OPERATOR => true,
2085-
T_NS_SEPARATOR => true,
2086-
T_PAAMAYIM_NEKUDOTAYIM => true,
2110+
T_OBJECT_OPERATOR => true,
2111+
T_NULLSAFE_OBJECT_OPERATOR => true,
2112+
T_NS_SEPARATOR => true,
2113+
T_PAAMAYIM_NEKUDOTAYIM => true,
20872114
];
20882115
if (isset($context[$this->tokens[$x]['code']]) === true) {
20892116
if (PHP_CODESNIFFER_VERBOSITY > 1) {

src/Tokenizers/Tokenizer.php

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1291,11 +1291,12 @@ private function recurseScopeMap($stackPtr, $depth=1, &$ignore=0)
12911291
// a new statement, it isn't a scope opener.
12921292
$disallowed = Util\Tokens::$assignmentTokens;
12931293
$disallowed += [
1294-
T_DOLLAR => true,
1295-
T_VARIABLE => true,
1296-
T_OBJECT_OPERATOR => true,
1297-
T_COMMA => true,
1298-
T_OPEN_PARENTHESIS => true,
1294+
T_DOLLAR => true,
1295+
T_VARIABLE => true,
1296+
T_OBJECT_OPERATOR => true,
1297+
T_NULLSAFE_OBJECT_OPERATOR => true,
1298+
T_COMMA => true,
1299+
T_OPEN_PARENTHESIS => true,
12991300
];
13001301

13011302
if (isset($disallowed[$this->tokens[$x]['code']]) === true) {

src/Util/Tokens.php

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,11 @@
124124
define('T_FN', 'PHPCS_T_FN');
125125
}
126126

127+
// Some PHP 8.0 tokens, replicated for lower versions.
128+
if (defined('T_NULLSAFE_OBJECT_OPERATOR') === false) {
129+
define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR');
130+
}
131+
127132
// Tokens used for parsing doc blocks.
128133
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
129134
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<?php
2+
3+
/*
4+
* Null safe operator.
5+
*/
6+
7+
/* testObjectOperator */
8+
echo $obj->foo;
9+
10+
/* testNullsafeObjectOperator */
11+
echo $obj?->foo;
12+
13+
/* testNullsafeObjectOperatorWriteContext */
14+
// Intentional parse error, but not the concern of the tokenizer.
15+
$foo?->bar->baz = 'baz';
16+
17+
/* testTernaryThen */
18+
echo $obj ? $obj->prop : $other->prop;
19+
20+
/* testParseErrorWhitespaceNotAllowed */
21+
echo $obj ?
22+
-> foo;
23+
24+
/* testParseErrorCommentNotAllowed */
25+
echo $obj ?/*comment*/-> foo;
26+
27+
/* testLiveCoding */
28+
// Intentional parse error. This has to be the last test in the file.
29+
echo $obj?
Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
<?php
2+
/**
3+
* Tests the backfill for the PHP >= 8.0 nullsafe object operator.
4+
*
5+
* @author Juliette Reinders Folmer <[email protected]>
6+
* @copyright 2020 Squiz Pty Ltd (ABN 77 084 670 600)
7+
* @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
11+
12+
use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
13+
use PHP_CodeSniffer\Util\Tokens;
14+
15+
class NullsafeObjectOperatorTest extends AbstractMethodUnitTest
16+
{
17+
18+
/**
19+
* Tokens to search for.
20+
*
21+
* @var array
22+
*/
23+
protected $find = [
24+
T_NULLSAFE_OBJECT_OPERATOR,
25+
T_OBJECT_OPERATOR,
26+
T_INLINE_THEN,
27+
];
28+
29+
30+
/**
31+
* Test that a normal object operator is still tokenized as such.
32+
*
33+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
34+
*
35+
* @return void
36+
*/
37+
public function testObjectOperator()
38+
{
39+
$tokens = self::$phpcsFile->getTokens();
40+
41+
$operator = $this->getTargetToken('/* testObjectOperator */', $this->find);
42+
$this->assertSame(T_OBJECT_OPERATOR, $tokens[$operator]['code'], 'Failed asserting code is object operator');
43+
$this->assertSame('T_OBJECT_OPERATOR', $tokens[$operator]['type'], 'Failed asserting type is object operator');
44+
45+
}//end testObjectOperator()
46+
47+
48+
/**
49+
* Test that a nullsafe object operator is tokenized as such.
50+
*
51+
* @param string $testMarker The comment which prefaces the target token in the test file.
52+
*
53+
* @dataProvider dataNullsafeObjectOperator
54+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
55+
*
56+
* @return void
57+
*/
58+
public function testNullsafeObjectOperator($testMarker)
59+
{
60+
$tokens = self::$phpcsFile->getTokens();
61+
62+
$operator = $this->getTargetToken($testMarker, $this->find);
63+
$this->assertSame(T_NULLSAFE_OBJECT_OPERATOR, $tokens[$operator]['code'], 'Failed asserting code is nullsafe object operator');
64+
$this->assertSame('T_NULLSAFE_OBJECT_OPERATOR', $tokens[$operator]['type'], 'Failed asserting type is nullsafe object operator');
65+
66+
}//end testNullsafeObjectOperator()
67+
68+
69+
/**
70+
* Data provider.
71+
*
72+
* @see testNullsafeObjectOperator()
73+
*
74+
* @return array
75+
*/
76+
public function dataNullsafeObjectOperator()
77+
{
78+
return [
79+
['/* testNullsafeObjectOperator */'],
80+
['/* testNullsafeObjectOperatorWriteContext */'],
81+
];
82+
83+
}//end dataNullsafeObjectOperator()
84+
85+
86+
/**
87+
* Test that a question mark not followed by an object operator is tokenized as T_TERNARY_THEN.
88+
*
89+
* @param string $testMarker The comment which prefaces the target token in the test file.
90+
* @param bool $testObjectOperator Whether to test for the next non-empty token being tokenized
91+
* as an object operator.
92+
*
93+
* @dataProvider dataTernaryThen
94+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
95+
*
96+
* @return void
97+
*/
98+
public function testTernaryThen($testMarker, $testObjectOperator=false)
99+
{
100+
$tokens = self::$phpcsFile->getTokens();
101+
102+
$operator = $this->getTargetToken($testMarker, $this->find);
103+
$this->assertSame(T_INLINE_THEN, $tokens[$operator]['code'], 'Failed asserting code is inline then');
104+
$this->assertSame('T_INLINE_THEN', $tokens[$operator]['type'], 'Failed asserting type is inline then');
105+
106+
if ($testObjectOperator === true) {
107+
$next = self::$phpcsFile->findNext(Tokens::$emptyTokens, ($operator + 1), null, true);
108+
$this->assertSame(T_OBJECT_OPERATOR, $tokens[$next]['code'], 'Failed asserting code is object operator');
109+
$this->assertSame('T_OBJECT_OPERATOR', $tokens[$next]['type'], 'Failed asserting type is object operator');
110+
}
111+
112+
}//end testTernaryThen()
113+
114+
115+
/**
116+
* Data provider.
117+
*
118+
* @see testTernaryThen()
119+
*
120+
* @return array
121+
*/
122+
public function dataTernaryThen()
123+
{
124+
return [
125+
['/* testTernaryThen */'],
126+
[
127+
'/* testParseErrorWhitespaceNotAllowed */',
128+
true,
129+
],
130+
[
131+
'/* testParseErrorCommentNotAllowed */',
132+
true,
133+
],
134+
['/* testLiveCoding */'],
135+
];
136+
137+
}//end dataTernaryThen()
138+
139+
140+
}//end class

tests/Core/Tokenizer/ShortArrayTest.inc

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ $a = (new Foo( array(1, array(4, 5), 3) ))[1][0];
6262
/* testClassMemberDereferencingOnClone */
6363
echo (clone $iterable)[20];
6464

65+
/* testNullsafeMethodCallDereferencing */
66+
$var = $obj?->function_call()[$x];
6567

6668
/*
6769
* Short array brackets.

tests/Core/Tokenizer/ShortArrayTest.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ public function dataSquareBrackets()
7272
['/* testClassMemberDereferencingOnInstantiation1 */'],
7373
['/* testClassMemberDereferencingOnInstantiation2 */'],
7474
['/* testClassMemberDereferencingOnClone */'],
75+
['/* testNullsafeMethodCallDereferencing */'],
7576
['/* testLiveCoding */'],
7677
];
7778

0 commit comments

Comments
 (0)