Skip to content

Commit b7c2356

Browse files
jrfnljrfnl
authored
Merge pull request #494 from PHPCSStandards/feature/tokenizer-comments-add-tests
Tokenizers/Comment: add tests + fix two edge case bugs
2 parents 4866ee3 + bef6fff commit b7c2356

17 files changed

+1946
-10
lines changed

‎src/Tokenizers/Comment.php

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ class Comment
2525
* @param string $eolChar The EOL character to use for splitting strings.
2626
* @param int $stackPtr The position of the first token in the file.
2727
*
28-
* @return array
28+
* @return array<int, array<string, string|int|array<int>>>
2929
*/
3030
public function tokenizeString($string, $eolChar, $stackPtr)
3131
{
@@ -41,9 +41,16 @@ public function tokenizeString($string, $eolChar, $stackPtr)
4141
extra star when they are used for function and class comments.
4242
*/
4343

44-
$char = ($numChars - strlen(ltrim($string, '/*')));
45-
$openTag = substr($string, 0, $char);
46-
$string = ltrim($string, '/*');
44+
$char = ($numChars - strlen(ltrim($string, '/*')));
45+
$lastChars = substr($string, -2);
46+
if ($char === $numChars && $lastChars === '*/') {
47+
// Edge case: docblock without whitespace or contents.
48+
$openTag = substr($string, 0, -2);
49+
$string = $lastChars;
50+
} else {
51+
$openTag = substr($string, 0, $char);
52+
$string = ltrim($string, '/*');
53+
}
4754

4855
$tokens[$stackPtr] = [
4956
'content' => $openTag,
@@ -74,6 +81,7 @@ public function tokenizeString($string, $eolChar, $stackPtr)
7481
];
7582

7683
if ($closeTag['content'] === false) {
84+
// In PHP < 8.0 substr() can return `false` instead of always returning a string.
7785
$closeTag['content'] = '';
7886
}
7987

@@ -171,7 +179,7 @@ public function tokenizeString($string, $eolChar, $stackPtr)
171179
* @param int $start The position in the string to start processing.
172180
* @param int $end The position in the string to end processing.
173181
*
174-
* @return array
182+
* @return array<int, array<string, string|int>>
175183
*/
176184
private function processLine($string, $eolChar, $start, $end)
177185
{
@@ -246,7 +254,7 @@ private function processLine($string, $eolChar, $start, $end)
246254
* @param int $start The position in the string to start processing.
247255
* @param int $end The position in the string to end processing.
248256
*
249-
* @return array|null
257+
* @return array<string, string|int>|null
250258
*/
251259
private function collectWhitespace($string, $start, $end)
252260
{
@@ -263,14 +271,12 @@ private function collectWhitespace($string, $start, $end)
263271
return null;
264272
}
265273

266-
$token = [
274+
return [
267275
'content' => $space,
268276
'code' => T_DOC_COMMENT_WHITESPACE,
269277
'type' => 'T_DOC_COMMENT_WHITESPACE',
270278
];
271279

272-
return $token;
273-
274280
}//end collectWhitespace()
275281

276282

‎src/Tokenizers/PHP.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -786,7 +786,7 @@ protected function tokenize($string)
786786

787787
if ($tokenIsArray === true
788788
&& ($token[0] === T_DOC_COMMENT
789-
|| ($token[0] === T_COMMENT && strpos($token[1], '/**') === 0))
789+
|| ($token[0] === T_COMMENT && strpos($token[1], '/**') === 0 && $token[1] !== '/**/'))
790790
) {
791791
$commentTokens = $commentTokenizer->tokenizeString($token[1], $this->eolChar, $newStackPtr);
792792
foreach ($commentTokens as $commentToken) {

‎tests/Core/Tokenizer/Comment/CommentTestCase.php

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
<?php
2+
/**
3+
* Base class for testing DocBlock comment tokenization.
4+
*
5+
* @author Juliette Reinders Folmer <[email protected]>
6+
* @copyright 2024 PHPCSStandards and contributors
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Comment;
11+
12+
use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
13+
use PHP_CodeSniffer\Util\Tokens;
14+
15+
/**
16+
* Base class for testing DocBlock comment tokenization.
17+
*
18+
* @covers PHP_CodeSniffer\Tokenizers\Comment
19+
*/
20+
abstract class CommentTestCase extends AbstractTokenizerTestCase
21+
{
22+
23+
24+
/**
25+
* Test whether the docblock opener and closer have the expected extra keys.
26+
*
27+
* @param string $marker The comment prefacing the target token.
28+
* @param int $closerOffset The offset of the closer from the opener.
29+
* @param array<int> $expectedTags The expected tags offsets array.
30+
*
31+
* @dataProvider dataDocblockOpenerCloser
32+
*
33+
* @return void
34+
*/
35+
public function testDocblockOpenerCloser($marker, $closerOffset, $expectedTags)
36+
{
37+
$tokens = $this->phpcsFile->getTokens();
38+
$target = $this->getTargetToken($marker, [T_DOC_COMMENT_OPEN_TAG]);
39+
40+
$opener = $tokens[$target];
41+
42+
$this->assertArrayHasKey('comment_closer', $opener, 'Comment opener: comment_closer index is not set');
43+
$this->assertArrayHasKey('comment_tags', $opener, 'Comment opener: comment_tags index is not set');
44+
45+
$expectedCloser = ($target + $closerOffset);
46+
$this->assertSame($expectedCloser, $opener['comment_closer'], 'Comment opener: comment_closer not set to the expected stack pointer');
47+
48+
// Update the tags expectations.
49+
foreach ($expectedTags as $i => $ptr) {
50+
$expectedTags[$i] += $target;
51+
}
52+
53+
$this->assertSame($expectedTags, $opener['comment_tags'], 'Comment opener: recorded tags do not match expected tags');
54+
55+
$closer = $tokens[$opener['comment_closer']];
56+
57+
$this->assertArrayHasKey('comment_opener', $closer, 'Comment closer: comment_opener index is not set');
58+
$this->assertSame($target, $closer['comment_opener'], 'Comment closer: comment_opener not set to the expected stack pointer');
59+
60+
}//end testDocblockOpenerCloser()
61+
62+
63+
/**
64+
* Data provider.
65+
*
66+
* @see testDocblockOpenerCloser()
67+
*
68+
* @return array<string, array<string, string|int|array<int>>>
69+
*/
70+
abstract public static function dataDocblockOpenerCloser();
71+
72+
73+
/**
74+
* Test helper. Check a token sequence complies with an expected token sequence.
75+
*
76+
* @param int $startPtr The position in the file to start checking from.
77+
* @param array<array<int|string, string>> $expectedSequence The consecutive token constants and their contents to expect.
78+
*
79+
* @return void
80+
*/
81+
protected function checkTokenSequence($startPtr, array $expectedSequence)
82+
{
83+
$tokens = $this->phpcsFile->getTokens();
84+
85+
$sequenceKey = 0;
86+
$sequenceCount = count($expectedSequence);
87+
88+
for ($i = $startPtr; $sequenceKey < $sequenceCount; $i++, $sequenceKey++) {
89+
$currentItem = $expectedSequence[$sequenceKey];
90+
$expectedCode = key($currentItem);
91+
$expectedType = Tokens::tokenName($expectedCode);
92+
$expectedContent = current($currentItem);
93+
$errorMsgSuffix = PHP_EOL.'(StackPtr: '.$i.' | Position in sequence: '.$sequenceKey.' | Expected: '.$expectedType.')';
94+
95+
$this->assertSame(
96+
$expectedCode,
97+
$tokens[$i]['code'],
98+
'Token tokenized as '.Tokens::tokenName($tokens[$i]['code']).', not '.$expectedType.' (code)'.$errorMsgSuffix
99+
);
100+
101+
$this->assertSame(
102+
$expectedType,
103+
$tokens[$i]['type'],
104+
'Token tokenized as '.$tokens[$i]['type'].', not '.$expectedType.' (type)'.$errorMsgSuffix
105+
);
106+
107+
$this->assertSame(
108+
$expectedContent,
109+
$tokens[$i]['content'],
110+
'Token content did not match expectations'.$errorMsgSuffix
111+
);
112+
}//end for
113+
114+
}//end checkTokenSequence()
115+
116+
117+
}//end class

‎tests/Core/Tokenizer/Comment/LiveCoding1Test.inc

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<?php
2+
3+
/* testLiveCoding */
4+
/**
5+
* Unclosed docblock, live coding.... with no blank line at end of file.
6+
*

‎tests/Core/Tokenizer/Comment/LiveCoding1Test.php

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
<?php
2+
/**
3+
* Tests that unclosed docblocks during live coding are handled correctly.
4+
*
5+
* @author Juliette Reinders Folmer <[email protected]>
6+
* @copyright 2024 PHPCSStandards and contributors
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Comment;
11+
12+
/**
13+
* Tests that unclosed docblocks during live coding are handled correctly.
14+
*
15+
* @covers PHP_CodeSniffer\Tokenizers\Comment
16+
*/
17+
final class LiveCoding1Test extends CommentTestCase
18+
{
19+
20+
21+
/**
22+
* Data provider.
23+
*
24+
* @see testDocblockOpenerCloser()
25+
*
26+
* @return array<string, array<string, string|int|array<int>>>
27+
*/
28+
public static function dataDocblockOpenerCloser()
29+
{
30+
return [
31+
'live coding: unclosed docblock, no blank line at end of file' => [
32+
'marker' => '/* testLiveCoding */',
33+
'closerOffset' => 8,
34+
'expectedTags' => [],
35+
],
36+
];
37+
38+
}//end dataDocblockOpenerCloser()
39+
40+
41+
/**
42+
* Verify tokenization of the DocBlock.
43+
*
44+
* @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment.
45+
*
46+
* @return void
47+
*/
48+
public function testLiveCoding()
49+
{
50+
$expectedSequence = [
51+
[T_DOC_COMMENT_OPEN_TAG => '/**'],
52+
[T_DOC_COMMENT_WHITESPACE => "\n"],
53+
[T_DOC_COMMENT_WHITESPACE => ' '],
54+
[T_DOC_COMMENT_STAR => '*'],
55+
[T_DOC_COMMENT_WHITESPACE => ' '],
56+
[T_DOC_COMMENT_STRING => 'Unclosed docblock, live coding.... with no blank line at end of file.'],
57+
[T_DOC_COMMENT_WHITESPACE => "\n"],
58+
[T_DOC_COMMENT_WHITESPACE => ' '],
59+
[T_DOC_COMMENT_CLOSE_TAG => '*'],
60+
];
61+
62+
$target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG);
63+
64+
$this->checkTokenSequence($target, $expectedSequence);
65+
66+
}//end testLiveCoding()
67+
68+
69+
}//end class

‎tests/Core/Tokenizer/Comment/LiveCoding2Test.inc

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
<?php
2+
3+
/* testLiveCoding */
4+
/**
5+
* Unclosed docblock, live coding.... with a blank line at end of file.

‎tests/Core/Tokenizer/Comment/LiveCoding2Test.php

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
<?php
2+
/**
3+
* Tests that unclosed docblocks during live coding are handled correctly.
4+
*
5+
* @author Juliette Reinders Folmer <[email protected]>
6+
* @copyright 2024 PHPCSStandards and contributors
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Comment;
11+
12+
/**
13+
* Tests that unclosed docblocks during live coding are handled correctly.
14+
*
15+
* @covers PHP_CodeSniffer\Tokenizers\Comment
16+
*/
17+
final class LiveCoding2Test extends CommentTestCase
18+
{
19+
20+
21+
/**
22+
* Data provider.
23+
*
24+
* @see testDocblockOpenerCloser()
25+
*
26+
* @return array<string, array<string, string|int|array<int>>>
27+
*/
28+
public static function dataDocblockOpenerCloser()
29+
{
30+
return [
31+
'live coding: unclosed docblock with blank line at end of file' => [
32+
'marker' => '/* testLiveCoding */',
33+
'closerOffset' => 7,
34+
'expectedTags' => [],
35+
],
36+
];
37+
38+
}//end dataDocblockOpenerCloser()
39+
40+
41+
/**
42+
* Verify tokenization of the DocBlock.
43+
*
44+
* @phpcs:disable Squiz.Arrays.ArrayDeclaration.SpaceBeforeDoubleArrow -- Readability is better with alignment.
45+
*
46+
* @return void
47+
*/
48+
public function testLiveCoding()
49+
{
50+
$expectedSequence = [
51+
[T_DOC_COMMENT_OPEN_TAG => '/**'],
52+
[T_DOC_COMMENT_WHITESPACE => "\n"],
53+
[T_DOC_COMMENT_WHITESPACE => ' '],
54+
[T_DOC_COMMENT_STAR => '*'],
55+
[T_DOC_COMMENT_WHITESPACE => ' '],
56+
[T_DOC_COMMENT_STRING => 'Unclosed docblock, live coding.... with a blank line at end of file.'],
57+
[T_DOC_COMMENT_WHITESPACE => "\n"],
58+
[T_DOC_COMMENT_CLOSE_TAG => ''],
59+
];
60+
61+
$target = $this->getTargetToken('/* '.__FUNCTION__.' */', T_DOC_COMMENT_OPEN_TAG);
62+
63+
$this->checkTokenSequence($target, $expectedSequence);
64+
65+
}//end testLiveCoding()
66+
67+
68+
}//end class

‎tests/Core/Tokenizer/Comment/LiveCoding3Test.inc

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
<?php
2+
3+
/* testLiveCoding */
4+
/**

0 commit comments

Comments
 (0)