Introduce PhpToken::getAll() instead of flag

Author: nikic

ext/tokenizer/tests/token_get_all_TOKEN_AS_OBJECT_001.phpt renamed to ext/tokenizer/tests/PhpToken_getAll.phpt

@@ -9,8 +9,8 @@ function foo() {
     echo "bar";
 }
 PHP;
-var_dump(token_get_all($code, TOKEN_AS_OBJECT));
-var_dump(token_get_all($code, TOKEN_AS_OBJECT|TOKEN_PARSE));
+var_dump(PhpToken::getAll($code));
+var_dump(PhpToken::getAll($code, TOKEN_PARSE));
 
 ?>
 --EXPECT--

ext/tokenizer/tests/PhpToken_methods.phpt

@@ -13,7 +13,7 @@ function foo() {
 PHP;
 
 // Token names and ignorability.
-$tokens = token_get_all($code, TOKEN_AS_OBJECT);
+$tokens = PhpToken::getAll($code);
 foreach ($tokens as $i => $token) {
     printf("[%2d] %-26s %s\n", $i, $token->getTokenName(),
         $token->isIgnorable() ? "ignorable" : "meaningful");

ext/tokenizer/tokenizer.c

@@ -35,14 +35,12 @@
 #define zendcursor LANG_SCNG(yy_cursor)
 #define zendlimit  LANG_SCNG(yy_limit)
 
-#define TOKEN_PARSE     (1 << 0)
-#define TOKEN_AS_OBJECT (1 << 1)
+#define TOKEN_PARSE (1 << 0)
 
 zend_class_entry *php_token_ce;
 
 void tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("TOKEN_PARSE", TOKEN_PARSE, CONST_CS|CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("TOKEN_AS_OBJECT", TOKEN_AS_OBJECT, CONST_CS|CONST_PERSISTENT);
 }
 
 /* {{{ tokenizer_functions[]
@@ -102,6 +100,25 @@ static zval *php_token_get_text(zval *obj) {
 	return text;
 }
 
+static zend_bool tokenize_common(
+		zval *return_value, zend_string *source, zend_long flags, zend_bool as_object);
+
+PHP_METHOD(PhpToken, getAll)
+{
+	zend_string *source;
+	zend_long flags = 0;
+
+	ZEND_PARSE_PARAMETERS_START(1, 2)
+		Z_PARAM_STR(source)
+		Z_PARAM_OPTIONAL
+		Z_PARAM_LONG(flags)
+	ZEND_PARSE_PARAMETERS_END();
+
+	if (tokenize_common(return_value, source, flags, /* as_object */ 1) == FAILURE) {
+		RETURN_THROWS();
+	}
+}
+
 PHP_METHOD(PhpToken, is)
 {
 	zval *kind;
@@ -190,6 +207,7 @@ PHP_METHOD(PhpToken, getTokenName)
 }
 
 static const zend_function_entry php_token_methods[] = {
+	PHP_ME(PhpToken, getAll, arginfo_class_PhpToken_getAll, ZEND_ACC_PUBLIC|ZEND_ACC_STATIC)
 	PHP_ME(PhpToken, is, arginfo_class_PhpToken_is, ZEND_ACC_PUBLIC)
 	PHP_ME(PhpToken, isIgnorable, arginfo_class_PhpToken_isIgnorable, ZEND_ACC_PUBLIC)
 	PHP_ME(PhpToken, getTokenName, arginfo_class_PhpToken_getTokenName, ZEND_ACC_PUBLIC)
@@ -421,6 +439,19 @@ static zend_bool tokenize_parse(zval *return_value, zend_string *source, zend_bo
 	return success;
 }
 
+static zend_bool tokenize_common(
+		zval *return_value, zend_string *source, zend_long flags, zend_bool as_object)
+{
+	if (flags & TOKEN_PARSE) {
+		return tokenize_parse(return_value, source, as_object);
+	} else {
+		int success = tokenize(return_value, source, as_object);
+		/* Normal token_get_all() should not throw. */
+		zend_clear_exception();
+		return success;
+	}
+}
+
 /* }}} */
 
 /* {{{ proto array token_get_all(string source [, int flags])
@@ -429,23 +460,14 @@ PHP_FUNCTION(token_get_all)
 {
 	zend_string *source;
 	zend_long flags = 0;
-	zend_bool success;
 
 	ZEND_PARSE_PARAMETERS_START(1, 2)
 		Z_PARAM_STR(source)
 		Z_PARAM_OPTIONAL
 		Z_PARAM_LONG(flags)
 	ZEND_PARSE_PARAMETERS_END();
 
-	if (flags & TOKEN_PARSE) {
-		success = tokenize_parse(return_value, source, (flags & TOKEN_AS_OBJECT) != 0);
-	} else {
-		success = tokenize(return_value, source, (flags & TOKEN_AS_OBJECT) != 0);
-		/* Normal token_get_all() should not throw. */
-		zend_clear_exception();
-	}
-
-	if (!success) {
+	if (tokenize_common(return_value, source, flags, /* as_object */ 0) == FAILURE) {
 		RETURN_THROWS();
 	}
 }

ext/tokenizer/tokenizer.stub.php

@@ -5,6 +5,9 @@ function token_get_all(string $source, int $flags = 0): array {}
 function token_name(int $token): string {}
 
 class PhpToken {
+    /** @return PhpToken[] */
+    public static function getAll(string $code, int $flags = 0): array;
+
     /** @param int|string|array $kind */
     public function is($kind): bool;
 

ext/tokenizer/tokenizer_arginfo.h

@@ -9,6 +9,11 @@ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_token_name, 0, 1, IS_STRING, 0)
 	ZEND_ARG_TYPE_INFO(0, token, IS_LONG, 0)
 ZEND_END_ARG_INFO()
 
+ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_PhpToken_getAll, 0, 1, IS_ARRAY, 0)
+	ZEND_ARG_TYPE_INFO(0, code, IS_STRING, 0)
+	ZEND_ARG_TYPE_INFO(0, flags, IS_LONG, 0)
+ZEND_END_ARG_INFO()
+
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_PhpToken_is, 0, 1, _IS_BOOL, 0)
 	ZEND_ARG_INFO(0, kind)
 ZEND_END_ARG_INFO()