Skip to content

Commit b10c327

Browse files
gsherwoodgsherwood
committed
Merge branch 'feature/tokenizer-php-fix-performance-leak' of https://github.com/jrfnl/PHP_CodeSniffer 
2 parents 21cd2e4 + a6daa05 commit b10c327

File tree

6 files changed

+225
-28
lines changed

6 files changed

+225
-28
lines changed

‎package.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
175175
<dir name="Tokenizer">
176176
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.inc" role="test" />
177177
<file baseinstalldir="" name="AnonClassParenthesisOwnerTest.php" role="test" />
178+
<file baseinstalldir="" name="ArrayKeywordTest.inc" role="test" />
179+
<file baseinstalldir="" name="ArrayKeywordTest.php" role="test" />
178180
<file baseinstalldir="" name="AttributesTest.inc" role="test" />
179181
<file baseinstalldir="" name="AttributesTest.php" role="test" />
180182
<file baseinstalldir="" name="BackfillFnTokenTest.inc" role="test" />
@@ -2124,6 +2126,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
21242126
<install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
21252127
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
21262128
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
2129+
<install as="CodeSniffer/Core/Tokenizer/ArrayKeywordTest.php" name="tests/Core/Tokenizer/ArrayKeywordTest.php" />
2130+
<install as="CodeSniffer/Core/Tokenizer/ArrayKeywordTest.inc" name="tests/Core/Tokenizer/ArrayKeywordTest.inc" />
21272131
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
21282132
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
21292133
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />
@@ -2212,6 +2216,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
22122216
<install as="CodeSniffer/Core/Sniffs/AbstractArraySniffTestable.php" name="tests/Core/Sniffs/AbstractArraySniffTestable.php" />
22132217
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.php" />
22142218
<install as="CodeSniffer/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" name="tests/Core/Tokenizer/AnonClassParenthesisOwnerTest.inc" />
2219+
<install as="CodeSniffer/Core/Tokenizer/ArrayKeywordTest.php" name="tests/Core/Tokenizer/ArrayKeywordTest.php" />
2220+
<install as="CodeSniffer/Core/Tokenizer/ArrayKeywordTest.inc" name="tests/Core/Tokenizer/ArrayKeywordTest.inc" />
22152221
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.php" name="tests/Core/Tokenizer/AttributesTest.php" />
22162222
<install as="CodeSniffer/Core/Tokenizer/AttributesTest.inc" name="tests/Core/Tokenizer/AttributesTest.inc" />
22172223
<install as="CodeSniffer/Core/Tokenizer/BackfillFnTokenTest.php" name="tests/Core/Tokenizer/BackfillFnTokenTest.php" />

‎src/Standards/Generic/Tests/Arrays/DisallowLongArraySyntaxUnitTest.php

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@ public function getErrorList($testFile='')
3535
6 => 1,
3636
7 => 1,
3737
12 => 1,
38-
13 => 1,
3938
];
4039
case 'DisallowLongArraySyntaxUnitTest.2.inc':
4140
return [

‎src/Standards/Squiz/Tests/PHP/CommentedOutCodeUnitTest.php

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ public function getWarningList($testFile='CommentedOutCodeUnitTest.inc')
4949
8 => 1,
5050
15 => 1,
5151
19 => 1,
52-
35 => 1,
5352
87 => 1,
5453
91 => 1,
5554
97 => 1,

‎src/Tokenizers/PHP.php

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1796,23 +1796,6 @@ function return types. We want to keep the parenthesis map clean,
17961796

17971797
break;
17981798
}//end for
1799-
1800-
// Any T_ARRAY tokens we find between here and the next
1801-
// token that can't be part of the return type, need to be
1802-
// converted to T_STRING tokens.
1803-
for ($x; $x < $numTokens; $x++) {
1804-
if ((is_array($tokens[$x]) === false && $tokens[$x] !== '|')
1805-
|| (is_array($tokens[$x]) === true && isset($allowed[$tokens[$x][0]]) === false)
1806-
) {
1807-
break;
1808-
} else if (is_array($tokens[$x]) === true && $tokens[$x][0] === T_ARRAY) {
1809-
$tokens[$x][0] = T_STRING;
1810-
1811-
if (PHP_CODESNIFFER_VERBOSITY > 1) {
1812-
echo "\t\t* token $x changed from T_ARRAY to T_STRING".PHP_EOL;
1813-
}
1814-
}
1815-
}
18161799
}//end if
18171800
}//end if
18181801
}//end if
@@ -2075,20 +2058,25 @@ function return types. We want to keep the parenthesis map clean,
20752058
}
20762059
}//end if
20772060

2078-
// This is a special condition for T_ARRAY tokens used for
2079-
// type hinting function arguments as being arrays. We want to keep
2080-
// the parenthesis map clean, so let's tag these tokens as
2061+
// This is a special condition for T_ARRAY tokens used for anything else
2062+
// but array declarations, like type hinting function arguments as
2063+
// being arrays.
2064+
// We want to keep the parenthesis map clean, so let's tag these tokens as
20812065
// T_STRING.
20822066
if ($newToken['code'] === T_ARRAY) {
2083-
for ($i = $stackPtr; $i < $numTokens; $i++) {
2084-
if ($tokens[$i] === '(') {
2085-
break;
2086-
} else if ($tokens[$i][0] === T_VARIABLE) {
2087-
$newToken['code'] = T_STRING;
2088-
$newToken['type'] = 'T_STRING';
2067+
for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
2068+
if (is_array($tokens[$i]) === false
2069+
|| isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === false
2070+
) {
2071+
// Non-empty content.
20892072
break;
20902073
}
20912074
}
2075+
2076+
if ($tokens[$i] !== '(' && $i !== $numTokens) {
2077+
$newToken['code'] = T_STRING;
2078+
$newToken['type'] = 'T_STRING';
2079+
}
20922080
}
20932081

20942082
// This is a special case when checking PHP 5.5+ code in PHP < 5.5

‎tests/Core/Tokenizer/ArrayKeywordTest.inc

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
<?php
2+
3+
/* testEmptyArray */
4+
$var = array();
5+
6+
/* testArrayWithSpace */
7+
$var = array (1 => 10);
8+
9+
/* testArrayWithComment */
10+
$var = Array /*comment*/ (1 => 10);
11+
12+
/* testNestingArray */
13+
$var = array(
14+
/* testNestedArray */
15+
array(
16+
'key' => 'value',
17+
18+
/* testClosureReturnType */
19+
'closure' => function($a) use($global) : Array {},
20+
),
21+
);
22+
23+
/* testFunctionDeclarationParamType */
24+
function foo(array $a) {}
25+
26+
/* testFunctionDeclarationReturnType */
27+
function foo($a) : int|array|null {}
28+
29+
class Bar {
30+
/* testClassConst */
31+
const ARRAY = [];
32+
33+
/* testClassMethod */
34+
public function array() {}
35+
}

‎tests/Core/Tokenizer/ArrayKeywordTest.php

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
<?php
2+
/**
3+
* Tests that the array keyword is tokenized correctly.
4+
*
5+
* @author Juliette Reinders Folmer <[email protected]>
6+
* @copyright 2021 Squiz Pty Ltd (ABN 77 084 670 600)
7+
* @license https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
11+
12+
use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
13+
14+
class ArrayKeywordTest extends AbstractMethodUnitTest
15+
{
16+
17+
18+
/**
19+
* Test that the array keyword is correctly tokenized as `T_ARRAY`.
20+
*
21+
* @param string $testMarker The comment prefacing the target token.
22+
* @param string $testContent Optional. The token content to look for.
23+
*
24+
* @dataProvider dataArrayKeyword
25+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
26+
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createTokenMap
27+
*
28+
* @return void
29+
*/
30+
public function testArrayKeyword($testMarker, $testContent='array')
31+
{
32+
$tokens = self::$phpcsFile->getTokens();
33+
34+
$token = $this->getTargetToken($testMarker, [T_ARRAY, T_STRING], $testContent);
35+
$tokenArray = $tokens[$token];
36+
37+
$this->assertSame(T_ARRAY, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_ARRAY (code)');
38+
$this->assertSame('T_ARRAY', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_ARRAY (type)');
39+
40+
$this->assertArrayHasKey('parenthesis_owner', $tokenArray, 'Parenthesis owner is not set');
41+
$this->assertArrayHasKey('parenthesis_opener', $tokenArray, 'Parenthesis opener is not set');
42+
$this->assertArrayHasKey('parenthesis_closer', $tokenArray, 'Parenthesis closer is not set');
43+
44+
}//end testArrayKeyword()
45+
46+
47+
/**
48+
* Data provider.
49+
*
50+
* @see testArrayKeyword()
51+
*
52+
* @return array
53+
*/
54+
public function dataArrayKeyword()
55+
{
56+
return [
57+
'empty array' => ['/* testEmptyArray */'],
58+
'array with space before parenthesis' => ['/* testArrayWithSpace */'],
59+
'array with comment before parenthesis' => [
60+
'/* testArrayWithComment */',
61+
'Array',
62+
],
63+
'nested: outer array' => ['/* testNestingArray */'],
64+
'nested: inner array' => ['/* testNestedArray */'],
65+
];
66+
67+
}//end dataArrayKeyword()
68+
69+
70+
/**
71+
* Test that the array keyword when used in a type declaration is correctly tokenized as `T_STRING`.
72+
*
73+
* @param string $testMarker The comment prefacing the target token.
74+
* @param string $testContent Optional. The token content to look for.
75+
*
76+
* @dataProvider dataArrayType
77+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
78+
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createTokenMap
79+
*
80+
* @return void
81+
*/
82+
public function testArrayType($testMarker, $testContent='array')
83+
{
84+
$tokens = self::$phpcsFile->getTokens();
85+
86+
$token = $this->getTargetToken($testMarker, [T_ARRAY, T_STRING], $testContent);
87+
$tokenArray = $tokens[$token];
88+
89+
$this->assertSame(T_STRING, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (code)');
90+
$this->assertSame('T_STRING', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (type)');
91+
92+
$this->assertArrayNotHasKey('parenthesis_owner', $tokenArray, 'Parenthesis owner is set');
93+
$this->assertArrayNotHasKey('parenthesis_opener', $tokenArray, 'Parenthesis opener is set');
94+
$this->assertArrayNotHasKey('parenthesis_closer', $tokenArray, 'Parenthesis closer is set');
95+
96+
}//end testArrayType()
97+
98+
99+
/**
100+
* Data provider.
101+
*
102+
* @see testArrayType()
103+
*
104+
* @return array
105+
*/
106+
public function dataArrayType()
107+
{
108+
return [
109+
'closure return type' => [
110+
'/* testClosureReturnType */',
111+
'Array',
112+
],
113+
'function param type' => ['/* testFunctionDeclarationParamType */'],
114+
'function union return type' => ['/* testFunctionDeclarationReturnType */'],
115+
];
116+
117+
}//end dataArrayType()
118+
119+
120+
/**
121+
* Verify that the retokenization of `T_ARRAY` tokens to `T_STRING` is handled correctly
122+
* for tokens with the contents 'array' which aren't in actual fact the array keyword.
123+
*
124+
* @param string $testMarker The comment prefacing the target token.
125+
* @param string $testContent The token content to look for.
126+
*
127+
* @dataProvider dataNotArrayKeyword
128+
* @covers PHP_CodeSniffer\Tokenizers\PHP::tokenize
129+
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createTokenMap
130+
*
131+
* @return void
132+
*/
133+
public function testNotArrayKeyword($testMarker, $testContent='array')
134+
{
135+
$tokens = self::$phpcsFile->getTokens();
136+
137+
$token = $this->getTargetToken($testMarker, [T_ARRAY, T_STRING], $testContent);
138+
$tokenArray = $tokens[$token];
139+
140+
$this->assertSame(T_STRING, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (code)');
141+
$this->assertSame('T_STRING', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_STRING (type)');
142+
143+
$this->assertArrayNotHasKey('parenthesis_owner', $tokenArray, 'Parenthesis owner is set');
144+
$this->assertArrayNotHasKey('parenthesis_opener', $tokenArray, 'Parenthesis opener is set');
145+
$this->assertArrayNotHasKey('parenthesis_closer', $tokenArray, 'Parenthesis closer is set');
146+
147+
}//end testNotArrayKeyword()
148+
149+
150+
/**
151+
* Data provider.
152+
*
153+
* @see testNotArrayKeyword()
154+
*
155+
* @return array
156+
*/
157+
public function dataNotArrayKeyword()
158+
{
159+
return [
160+
'class-constant-name' => [
161+
'/* testClassConst */',
162+
'ARRAY',
163+
],
164+
'class-method-name' => ['/* testClassMethod */'],
165+
];
166+
167+
}//end dataNotArrayKeyword()
168+
169+
170+
}//end class

0 commit comments

Comments
 (0)