Skip to content

Commit 7062488

Browse files
jrfnljrfnl
committed
PHP 8.3 | Tokenizer/PHP: add support for typed OO constants
PHP 8.3 introduced typed OO constants, where the type is between the `const` keyword and the constant name. All type variations are supported, including nullable types, union types, intersection types, with the exception of `callable`, `void` and `never`. `self` and `static` types are only allowed in Enum constants. This PR adds support for typed OO constants in the Tokenizer layer of PHPCS. The following issues had to be fixed to support typed constants: 1. Consistently tokenizing the constant _name_ as `T_STRING`, even if the name mirrors a reserved keyword, like `foreach` or a special keyword, like `self` or `true`. 2. Tokenizing a `?` at the start of a constant type declaration as `T_NULLABLE`. 3. Tokenizing a `|` and `&` operators within a constant type declaration as `T_TYPE_UNION` and `T_TYPE_INTERSECTION` respectively. Each and every part of the above has been covered by extensive tests. Includes additional tests safeguarding that the `array` keyword when used in a type declaration for a constant is tokenized as `T_STRING`. Ref: https://wiki.php.net/rfc/typed_class_constants
1 parent 6da9eeb commit 7062488

13 files changed

+963
-8
lines changed

‎src/Tokenizers/PHP.php

Lines changed: 98 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -526,8 +526,9 @@ protected function tokenize($string)
526526
$numTokens = count($tokens);
527527
$lastNotEmptyToken = 0;
528528

529-
$insideInlineIf = [];
530-
$insideUseGroup = false;
529+
$insideInlineIf = [];
530+
$insideUseGroup = false;
531+
$insideConstDeclaration = false;
531532

532533
$commentTokenizer = new Comment();
533534

@@ -608,7 +609,8 @@ protected function tokenize($string)
608609
if ($tokenIsArray === true
609610
&& isset(Util\Tokens::$contextSensitiveKeywords[$token[0]]) === true
610611
&& (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
611-
|| $finalTokens[$lastNotEmptyToken]['content'] === '&')
612+
|| $finalTokens[$lastNotEmptyToken]['content'] === '&'
613+
|| $insideConstDeclaration === true)
612614
) {
613615
if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
614616
$preserveKeyword = false;
@@ -648,6 +650,30 @@ protected function tokenize($string)
648650
}
649651
}//end if
650652

653+
// Types in typed constants should not be touched, but the constant name should be.
654+
if ((isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
655+
&& $finalTokens[$lastNotEmptyToken]['code'] === T_CONST)
656+
|| $insideConstDeclaration === true
657+
) {
658+
$preserveKeyword = true;
659+
660+
// Find the next non-empty token.
661+
for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
662+
if (is_array($tokens[$i]) === true
663+
&& isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true
664+
) {
665+
continue;
666+
}
667+
668+
break;
669+
}
670+
671+
if ($tokens[$i] === '=' || $tokens[$i] === ';') {
672+
$preserveKeyword = false;
673+
$insideConstDeclaration = false;
674+
}
675+
}//end if
676+
651677
if ($finalTokens[$lastNotEmptyToken]['content'] === '&') {
652678
$preserveKeyword = true;
653679

@@ -681,6 +707,26 @@ protected function tokenize($string)
681707
}
682708
}//end if
683709

710+
/*
711+
Mark the start of a constant declaration to allow for handling keyword to T_STRING
712+
convertion for constant names using reserved keywords.
713+
*/
714+
715+
if ($tokenIsArray === true && $token[0] === T_CONST) {
716+
$insideConstDeclaration = true;
717+
}
718+
719+
/*
720+
Close an open "inside constant declaration" marker when no keyword convertion was needed.
721+
*/
722+
723+
if ($insideConstDeclaration === true
724+
&& $tokenIsArray === false
725+
&& ($token[0] === '=' || $token[0] === ';')
726+
) {
727+
$insideConstDeclaration = false;
728+
}
729+
684730
/*
685731
Special case for `static` used as a function name, i.e. `static()`.
686732
*/
@@ -1851,6 +1897,20 @@ protected function tokenize($string)
18511897
$newToken = [];
18521898
$newToken['content'] = '?';
18531899

1900+
// For typed constants, we only need to check the token before the ? to be sure.
1901+
if ($finalTokens[$lastNotEmptyToken]['code'] === T_CONST) {
1902+
$newToken['code'] = T_NULLABLE;
1903+
$newToken['type'] = 'T_NULLABLE';
1904+
1905+
if (PHP_CODESNIFFER_VERBOSITY > 1) {
1906+
echo "\t\t* token $stackPtr changed from ? to T_NULLABLE".PHP_EOL;
1907+
}
1908+
1909+
$finalTokens[$newStackPtr] = $newToken;
1910+
$newStackPtr++;
1911+
continue;
1912+
}
1913+
18541914
/*
18551915
* Check if the next non-empty token is one of the tokens which can be used
18561916
* in type declarations. If not, it's definitely a ternary.
@@ -2218,7 +2278,30 @@ function return types. We want to keep the parenthesis map clean,
22182278
if ($tokenIsArray === true && $token[0] === T_STRING) {
22192279
$preserveTstring = false;
22202280

2221-
if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
2281+
// True/false/parent/self/static in typed constants should be fixed to their own token,
2282+
// but the constant name should not be.
2283+
if ((isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
2284+
&& $finalTokens[$lastNotEmptyToken]['code'] === T_CONST)
2285+
|| $insideConstDeclaration === true
2286+
) {
2287+
// Find the next non-empty token.
2288+
for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
2289+
if (is_array($tokens[$i]) === true
2290+
&& isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true
2291+
) {
2292+
continue;
2293+
}
2294+
2295+
break;
2296+
}
2297+
2298+
if ($tokens[$i] === '=') {
2299+
$preserveTstring = true;
2300+
$insideConstDeclaration = false;
2301+
}
2302+
} else if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
2303+
&& $finalTokens[$lastNotEmptyToken]['code'] !== T_CONST
2304+
) {
22222305
$preserveTstring = true;
22232306

22242307
// Special case for syntax like: return new self/new parent
@@ -2990,6 +3073,12 @@ protected function processAdditional()
29903073
$suspectedType = 'return';
29913074
}
29923075

3076+
if ($this->tokens[$x]['code'] === T_EQUAL) {
3077+
// Possible constant declaration, the `T_STRING` name will have been skipped over already.
3078+
$suspectedType = 'constant';
3079+
break;
3080+
}
3081+
29933082
break;
29943083
}//end for
29953084

@@ -3031,6 +3120,11 @@ protected function processAdditional()
30313120
break;
30323121
}
30333122

3123+
if ($suspectedType === 'constant' && $this->tokens[$x]['code'] === T_CONST) {
3124+
$confirmed = true;
3125+
break;
3126+
}
3127+
30343128
if ($suspectedType === 'property or parameter'
30353129
&& (isset(Util\Tokens::$scopeModifiers[$this->tokens[$x]['code']]) === true
30363130
|| $this->tokens[$x]['code'] === T_VAR

‎tests/Core/Tokenizer/ArrayKeywordTest.inc

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,21 @@ $var = array(
2121
);
2222

2323
/* testFunctionDeclarationParamType */
24-
function foo(array $a) {}
24+
function typedParam(array $a) {}
2525

2626
/* testFunctionDeclarationReturnType */
27-
function foo($a) : int|array|null {}
27+
function returnType($a) : int|array|null {}
2828

2929
class Bar {
3030
/* testClassConst */
3131
const ARRAY = [];
3232

3333
/* testClassMethod */
3434
public function array() {}
35+
36+
/* testOOConstType */
37+
const array /* testTypedOOConstName */ ARRAY = /* testOOConstDefault */ array();
38+
39+
/* testOOPropertyType */
40+
protected array $property;
3541
}

‎tests/Core/Tokenizer/ArrayKeywordTest.php

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ public static function dataArrayKeyword()
6868
'nested: inner array' => [
6969
'testMarker' => '/* testNestedArray */',
7070
],
71+
'OO constant default value' => [
72+
'testMarker' => '/* testOOConstDefault */',
73+
],
7174
];
7275

7376
}//end dataArrayKeyword()
@@ -122,6 +125,12 @@ public static function dataArrayType()
122125
'function union return type' => [
123126
'testMarker' => '/* testFunctionDeclarationReturnType */',
124127
],
128+
'OO constant type' => [
129+
'testMarker' => '/* testOOConstType */',
130+
],
131+
'OO property type' => [
132+
'testMarker' => '/* testOOPropertyType */',
133+
],
125134
];
126135

127136
}//end dataArrayType()
@@ -167,13 +176,17 @@ public function testNotArrayKeyword($testMarker, $testContent='array')
167176
public static function dataNotArrayKeyword()
168177
{
169178
return [
170-
'class-constant-name' => [
179+
'class-constant-name' => [
171180
'testMarker' => '/* testClassConst */',
172181
'testContent' => 'ARRAY',
173182
],
174-
'class-method-name' => [
183+
'class-method-name' => [
175184
'testMarker' => '/* testClassMethod */',
176185
],
186+
'class-constant-name-after-type' => [
187+
'testMarker' => '/* testTypedOOConstName */',
188+
'testContent' => 'ARRAY',
189+
],
177190
];
178191

179192
}//end dataNotArrayKeyword()

‎tests/Core/Tokenizer/BitwiseOrTest.inc

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,30 @@ $result = $value | $test /* testBitwiseOr2 */ | $another;
99

1010
class TypeUnion
1111
{
12+
/* testTypeUnionOOConstSimple */
13+
public const Foo|Bar SIMPLE = new Foo;
14+
15+
/* testTypeUnionOOConstReverseModifierOrder */
16+
protected final const int|float MODIFIERS_REVERSED /* testBitwiseOrOOConstDefaultValue */ = E_WARNING | E_NOTICE;
17+
18+
const
19+
/* testTypeUnionOOConstMulti1 */
20+
array |
21+
/* testTypeUnionOOConstMulti2 */
22+
Traversable | // phpcs:ignore Stnd.Cat.Sniff
23+
false
24+
/* testTypeUnionOOConstMulti3 */
25+
| null MULTI_UNION = false;
26+
27+
/* testTypeUnionOOConstNamespaceRelative */
28+
final protected const namespace\Sub\NameA|namespace\Sub\NameB NAMESPACE_RELATIVE = new namespace\Sub\NameB;
29+
30+
/* testTypeUnionOOConstPartiallyQualified */
31+
const Partially\Qualified\NameA|Partially\Qualified\NameB PARTIALLY_QUALIFIED = new Partially\Qualified\NameA;
32+
33+
/* testTypeUnionOOConstFullyQualified */
34+
const \Fully\Qualified\NameA|\Fully\Qualified\NameB FULLY_QUALIFIED = new \Fully\Qualified\NameB();
35+
1236
/* testTypeUnionPropertySimple */
1337
public static Foo|Bar $obj;
1438

‎tests/Core/Tokenizer/BitwiseOrTest.php

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ public static function dataBitwiseOr()
4747
return [
4848
'in simple assignment 1' => ['/* testBitwiseOr1 */'],
4949
'in simple assignment 2' => ['/* testBitwiseOr2 */'],
50+
'in OO constant default value' => ['/* testBitwiseOrOOConstDefaultValue */'],
5051
'in property default value' => ['/* testBitwiseOrPropertyDefaultValue */'],
5152
'in method parameter default value' => ['/* testBitwiseOrParamDefaultValue */'],
5253
'in return statement' => ['/* testBitwiseOr3 */'],
@@ -97,6 +98,14 @@ public function testTypeUnion($testMarker)
9798
public static function dataTypeUnion()
9899
{
99100
return [
101+
'type for OO constant' => ['/* testTypeUnionOOConstSimple */'],
102+
'type for OO constant, reversed modifier order' => ['/* testTypeUnionOOConstReverseModifierOrder */'],
103+
'type for OO constant, first of multi-union' => ['/* testTypeUnionOOConstMulti1 */'],
104+
'type for OO constant, middle of multi-union + comments' => ['/* testTypeUnionOOConstMulti2 */'],
105+
'type for OO constant, last of multi-union' => ['/* testTypeUnionOOConstMulti3 */'],
106+
'type for OO constant, using namespace relative names' => ['/* testTypeUnionOOConstNamespaceRelative */'],
107+
'type for OO constant, using partially qualified names' => ['/* testTypeUnionOOConstPartiallyQualified */'],
108+
'type for OO constant, using fully qualified names' => ['/* testTypeUnionOOConstFullyQualified */'],
100109
'type for static property' => ['/* testTypeUnionPropertySimple */'],
101110
'type for static property, reversed modifier order' => ['/* testTypeUnionPropertyReverseModifierOrder */'],
102111
'type for property, first of multi-union' => ['/* testTypeUnionPropertyMulti1 */'],

‎tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,12 @@ class ContextSensitiveKeywords
7676
const /* testAnd */ AND = 'LOGICAL_AND';
7777
const /* testOr */ OR = 'LOGICAL_OR';
7878
const /* testXor */ XOR = 'LOGICAL_XOR';
79+
80+
const /* testArrayIsTstringInConstType */ array /* testArrayNameForTypedConstant */ ARRAY = /* testArrayIsKeywordInConstDefault */ array();
81+
const /* testStaticIsKeywordAsConstType */ static /* testStaticIsNameForTypedConstant */ STATIC = new /* testStaticIsKeywordAsConstDefault */ static;
82+
83+
const int|bool /* testPrivateNameForUnionTypedConstant */ PRIVATE = 'PRIVATE';
84+
const Foo&Bar /* testFinalNameForIntersectionTypedConstant */ FINAL = 'FINAL';
7985
}
8086

8187
namespace /* testKeywordAfterNamespaceShouldBeString */ class;

‎tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,12 @@ public static function dataStrings()
118118
'constant declaration: or' => ['/* testOr */'],
119119
'constant declaration: xor' => ['/* testXor */'],
120120

121+
'constant declaration: array in type' => ['/* testArrayIsTstringInConstType */'],
122+
'constant declaration: array, name after type' => ['/* testArrayNameForTypedConstant */'],
123+
'constant declaration: static, name after type' => ['/* testStaticIsNameForTypedConstant */'],
124+
'constant declaration: private, name after type' => ['/* testPrivateNameForUnionTypedConstant */'],
125+
'constant declaration: final, name after type' => ['/* testFinalNameForIntersectionTypedConstant */'],
126+
121127
'namespace declaration: class' => ['/* testKeywordAfterNamespaceShouldBeString */'],
122128
'namespace declaration (partial): my' => ['/* testNamespaceNameIsString1 */'],
123129
'namespace declaration (partial): class' => ['/* testNamespaceNameIsString2 */'],
@@ -179,6 +185,19 @@ public static function dataKeywords()
179185
'testMarker' => '/* testNamespaceIsKeyword */',
180186
'expectedTokenType' => 'T_NAMESPACE',
181187
],
188+
'array: default value in const decl' => [
189+
'testMarker' => '/* testArrayIsKeywordInConstDefault */',
190+
'expectedTokenType' => 'T_ARRAY',
191+
],
192+
'static: type in constant declaration' => [
193+
'testMarker' => '/* testStaticIsKeywordAsConstType */',
194+
'expectedTokenType' => 'T_STATIC',
195+
],
196+
'static: value in constant declaration' => [
197+
'testMarker' => '/* testStaticIsKeywordAsConstDefault */',
198+
'expectedTokenType' => 'T_STATIC',
199+
],
200+
182201
'abstract: class declaration' => [
183202
'testMarker' => '/* testAbstractIsKeyword */',
184203
'expectedTokenType' => 'T_ABSTRACT',

‎tests/Core/Tokenizer/OtherContextSensitiveKeywordsTest.inc

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,17 @@ function standAloneFalseTrueNullTypesAndMore(
5151
|| $a === /* testNullIsKeywordInComparison */ null
5252
) {}
5353
}
54+
55+
class TypedConstProp {
56+
const /* testFalseIsKeywordAsConstType */ false /* testFalseIsNameForTypedConstant */ FALSE = /* testFalseIsKeywordAsConstDefault */ false;
57+
const /* testTrueIsKeywordAsConstType */ true /* testTrueIsNameForTypedConstant */ TRUE = /* testTrueIsKeywordAsConstDefault */ true;
58+
const /* testNullIsKeywordAsConstType */ null /* testNullIsNameForTypedConstant */ NULL = /* testNullIsKeywordAsConstDefault */ null;
59+
const /* testSelfIsKeywordAsConstType */ self /* testSelfIsNameForTypedConstant */ SELF = new /* testSelfIsKeywordAsConstDefault */ self;
60+
const /* testParentIsKeywordAsConstType */ parent /* testParentIsNameForTypedConstant */ PARENT = new /* testParentIsKeywordAsConstDefault */ parent;
61+
62+
public /* testFalseIsKeywordAsPropertyType */ false $false = /* testFalseIsKeywordAsPropertyDefault */ false;
63+
protected readonly /* testTrueIsKeywordAsPropertyType */ true $true = /* testTrueIsKeywordAsPropertyDefault */ true;
64+
static private /* testNullIsKeywordAsPropertyType */ null $null = /* testNullIsKeywordAsPropertyDefault */ null;
65+
var /* testSelfIsKeywordAsPropertyType */ self $self = new /* testSelfIsKeywordAsPropertyDefault */ self;
66+
protected /* testParentIsKeywordAsPropertyType */ parent $parent = new /* testParentIsKeywordAsPropertyDefault */ parent;
67+
}

0 commit comments

Comments
 (0)