Skip to content

Commit 072615e

Browse files
nikicnikic
committed
Allow extending PhpToken
Return instances of "static". Make the constructor final to allow this.
1 parent 3e97ab0 commit 072615e

File tree

5 files changed

+65
-21
lines changed

5 files changed

+65
-21
lines changed

‎ext/tokenizer/tests/PhpToken_extension.phpt

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
--TEST--
2+
Extending the PhpToken class
3+
--FILE--
4+
<?php
5+
6+
$code = <<<'PHP'
7+
<?PHP
8+
FUNCTION FOO() {
9+
ECHO "bar";
10+
}
11+
PHP;
12+
13+
class MyPhpToken extends PhpToken {
14+
public function getLoweredText(): string {
15+
return strtolower($this->text);
16+
}
17+
}
18+
19+
foreach (MyPhpToken::getAll($code) as $token) {
20+
echo $token->getLoweredText();
21+
}
22+
23+
?>
24+
--EXPECT--
25+
<?php
26+
function foo() {
27+
echo "bar";
28+
}

‎ext/tokenizer/tests/PhpToken_final_constructor.phpt

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
--TEST--
2+
Check that the PhpToken constructor is final
3+
--FILE--
4+
<?php
5+
6+
class MyPhpToken extends PhpToken {
7+
public function __construct() {
8+
}
9+
}
10+
11+
?>
12+
--EXPECTF--
13+
Fatal error: Cannot override final method PhpToken::__construct() in %s on line %d

‎ext/tokenizer/tests/PhpToken_getAll.phpt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
--TEST--
2-
TOKEN_AS_OBJECT mode
2+
PhpToken::getAll() method
33
--FILE--
44
<?php
55

‎ext/tokenizer/tokenizer.c

Lines changed: 21 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -101,20 +101,22 @@ static zval *php_token_get_text(zval *obj) {
101101
}
102102

103103
static zend_bool tokenize_common(
104-
zval *return_value, zend_string *source, zend_long flags, zend_bool as_object);
104+
zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class);
105105

106106
PHP_METHOD(PhpToken, getAll)
107107
{
108108
zend_string *source;
109109
zend_long flags = 0;
110+
zend_class_entry *token_class;
110111

111112
ZEND_PARSE_PARAMETERS_START(1, 2)
112113
Z_PARAM_STR(source)
113114
Z_PARAM_OPTIONAL
114115
Z_PARAM_LONG(flags)
115116
ZEND_PARSE_PARAMETERS_END();
116117

117-
if (tokenize_common(return_value, source, flags, /* as_object */ 1) == FAILURE) {
118+
token_class = zend_get_called_scope(execute_data);
119+
if (tokenize_common(return_value, source, flags, token_class) == FAILURE) {
118120
RETURN_THROWS();
119121
}
120122
}
@@ -231,7 +233,7 @@ PHP_METHOD(PhpToken, getTokenName)
231233

232234
static const zend_function_entry php_token_methods[] = {
233235
PHP_ME(PhpToken, getAll, arginfo_class_PhpToken_getAll, ZEND_ACC_PUBLIC|ZEND_ACC_STATIC)
234-
PHP_ME(PhpToken, __construct, arginfo_class_PhpToken___construct, ZEND_ACC_PUBLIC)
236+
PHP_ME(PhpToken, __construct, arginfo_class_PhpToken___construct, ZEND_ACC_PUBLIC|ZEND_ACC_FINAL)
235237
PHP_ME(PhpToken, is, arginfo_class_PhpToken_is, ZEND_ACC_PUBLIC)
236238
PHP_ME(PhpToken, isIgnorable, arginfo_class_PhpToken_isIgnorable, ZEND_ACC_PUBLIC)
237239
PHP_ME(PhpToken, getTokenName, arginfo_class_PhpToken_getTokenName, ZEND_ACC_PUBLIC)
@@ -296,10 +298,10 @@ static inline zend_string *make_str(unsigned char *text, size_t leng) {
296298
}
297299

298300
static void add_token(zval *return_value, int token_type,
299-
unsigned char *text, size_t leng, int lineno, zend_bool as_object) {
301+
unsigned char *text, size_t leng, int lineno, zend_class_entry *token_class) {
300302
zval token;
301-
if (as_object) {
302-
zend_object *obj = zend_objects_new(php_token_ce);
303+
if (token_class) {
304+
zend_object *obj = zend_objects_new(token_class);
303305
ZVAL_OBJ(&token, obj);
304306
ZVAL_LONG(OBJ_PROP_NUM(obj, 0), token_type);
305307
ZVAL_STR(OBJ_PROP_NUM(obj, 1), make_str(text, leng));
@@ -316,7 +318,7 @@ static void add_token(zval *return_value, int token_type,
316318
zend_hash_next_index_insert_new(Z_ARRVAL_P(return_value), &token);
317319
}
318320

319-
static zend_bool tokenize(zval *return_value, zend_string *source, zend_bool as_object)
321+
static zend_bool tokenize(zval *return_value, zend_string *source, zend_class_entry *token_class)
320322
{
321323
zval source_zval;
322324
zend_lex_state original_lex_state;
@@ -337,7 +339,7 @@ static zend_bool tokenize(zval *return_value, zend_string *source, zend_bool as_
337339
array_init(return_value);
338340

339341
while ((token_type = lex_scan(&token, NULL))) {
340-
add_token(return_value, token_type, zendtext, zendleng, token_line, as_object);
342+
add_token(return_value, token_type, zendtext, zendleng, token_line, token_class);
341343

342344
if (Z_TYPE(token) != IS_UNDEF) {
343345
zval_ptr_dtor_nogc(&token);
@@ -353,7 +355,7 @@ static zend_bool tokenize(zval *return_value, zend_string *source, zend_bool as_
353355
/* fetch the rest into a T_INLINE_HTML */
354356
if (zendcursor != zendlimit) {
355357
add_token(return_value, T_INLINE_HTML,
356-
zendcursor, zendlimit - zendcursor, token_line, as_object);
358+
zendcursor, zendlimit - zendcursor, token_line, token_class);
357359
}
358360
break;
359361
}
@@ -377,7 +379,7 @@ static zend_bool tokenize(zval *return_value, zend_string *source, zend_bool as_
377379

378380
struct event_context {
379381
zval *tokens;
380-
zend_bool as_object;
382+
zend_class_entry *token_class;
381383
};
382384

383385
void on_event(zend_php_scanner_event event, int token, int line, void *context)
@@ -396,7 +398,7 @@ void on_event(zend_php_scanner_event event, int token, int line, void *context)
396398
token = T_OPEN_TAG_WITH_ECHO;
397399
}
398400
add_token(ctx->tokens, token,
399-
LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line, ctx->as_object);
401+
LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line, ctx->token_class);
400402
break;
401403
case ON_FEEDBACK:
402404
tokens_ht = Z_ARRVAL_P(ctx->tokens);
@@ -411,13 +413,14 @@ void on_event(zend_php_scanner_event event, int token, int line, void *context)
411413
case ON_STOP:
412414
if (LANG_SCNG(yy_cursor) != LANG_SCNG(yy_limit)) {
413415
add_token(ctx->tokens, T_INLINE_HTML, LANG_SCNG(yy_cursor),
414-
LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor), CG(zend_lineno), ctx->as_object);
416+
LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor), CG(zend_lineno), ctx->token_class);
415417
}
416418
break;
417419
}
418420
}
419421

420-
static zend_bool tokenize_parse(zval *return_value, zend_string *source, zend_bool as_object)
422+
static zend_bool tokenize_parse(
423+
zval *return_value, zend_string *source, zend_class_entry *token_class)
421424
{
422425
zval source_zval;
423426
zend_lex_state original_lex_state;
@@ -436,7 +439,7 @@ static zend_bool tokenize_parse(zval *return_value, zend_string *source, zend_bo
436439
array_init(&token_stream);
437440

438441
ctx.tokens = &token_stream;
439-
ctx.as_object = as_object;
442+
ctx.token_class = token_class;
440443

441444
CG(ast) = NULL;
442445
CG(ast_arena) = zend_arena_create(1024 * 32);
@@ -464,12 +467,12 @@ static zend_bool tokenize_parse(zval *return_value, zend_string *source, zend_bo
464467
}
465468

466469
static zend_bool tokenize_common(
467-
zval *return_value, zend_string *source, zend_long flags, zend_bool as_object)
470+
zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class)
468471
{
469472
if (flags & TOKEN_PARSE) {
470-
return tokenize_parse(return_value, source, as_object);
473+
return tokenize_parse(return_value, source, token_class);
471474
} else {
472-
int success = tokenize(return_value, source, as_object);
475+
int success = tokenize(return_value, source, token_class);
473476
/* Normal token_get_all() should not throw. */
474477
zend_clear_exception();
475478
return success;
@@ -491,7 +494,7 @@ PHP_FUNCTION(token_get_all)
491494
Z_PARAM_LONG(flags)
492495
ZEND_PARSE_PARAMETERS_END();
493496

494-
if (tokenize_common(return_value, source, flags, /* as_object */ 0) == FAILURE) {
497+
if (tokenize_common(return_value, source, flags, /* token_class */ NULL) == FAILURE) {
495498
RETURN_THROWS();
496499
}
497500
}

‎ext/tokenizer/tokenizer.stub.php

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ function token_get_all(string $source, int $flags = 0): array {}
55
function token_name(int $token): string {}
66

77
class PhpToken {
8-
/** @return PhpToken[] */
8+
/** @return static[] */
99
public static function getAll(string $code, int $flags = 0): array;
1010

11-
public function __construct(int $id, string $text, int $line = -1, int $pos = -1);
11+
public final function __construct(int $id, string $text, int $line = -1, int $pos = -1);
1212

1313
/** @param int|string|array $kind */
1414
public function is($kind): bool;

0 commit comments

Comments
 (0)