Return null from getTokenName() if token unknown

Author: nikic

ext/tokenizer/tests/PhpToken_methods.phpt

@@ -70,6 +70,10 @@ try {
     echo $e->getMessage(), "\n";
 }
 
+echo "\nName of unknown token:\n";
+$token = new PhpToken(100000, "foo");
+var_dump($token->getTokenName());
+
 ?>
 --EXPECT--
 [ 0] T_OPEN_TAG                 ignorable
@@ -110,3 +114,6 @@ Typed property PhpToken::$id must not be accessed before initialization
 Typed property PhpToken::$text must not be accessed before initialization
 Typed property PhpToken::$id must not be accessed before initialization
 Typed property PhpToken::$text must not be accessed before initialization
+
+Name of unknown token:
+NULL

ext/tokenizer/tokenizer.c

@@ -237,7 +237,12 @@ PHP_METHOD(PhpToken, getTokenName)
 	if (Z_LVAL_P(id_zval) < 256) {
 		RETURN_INTERNED_STR(ZSTR_CHAR(Z_LVAL_P(id_zval)));
 	} else {
-		RETURN_STRING(get_token_type_name(Z_LVAL_P(id_zval)));
+		const char *token_name = get_token_type_name(Z_LVAL_P(id_zval));
+		if (!token_name) {
+			RETURN_NULL();
+		}
+
+		RETURN_STRING(token_name);
 	}
 }
 
@@ -531,6 +536,10 @@ PHP_FUNCTION(token_name)
 		Z_PARAM_LONG(type)
 	ZEND_PARSE_PARAMETERS_END();
 
-	RETVAL_STRING(get_token_type_name(type));
+	const char *token_name = get_token_type_name(type);
+	if (!token_name) {
+		token_name = "UNKNOWN";
+	}
+	RETURN_STRING(token_name);
 }
 /* }}} */

ext/tokenizer/tokenizer.stub.php

@@ -15,5 +15,5 @@ public function is($kind): bool;
 
     public function isIgnorable(): bool;
 
-    public function getTokenName(): string;
+    public function getTokenName(): ?string;
 }

ext/tokenizer/tokenizer_arginfo.h

@@ -28,5 +28,5 @@ ZEND_END_ARG_INFO()
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_PhpToken_isIgnorable, 0, 0, _IS_BOOL, 0)
 ZEND_END_ARG_INFO()
 
-ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_PhpToken_getTokenName, 0, 0, IS_STRING, 0)
+ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_PhpToken_getTokenName, 0, 0, IS_STRING, 1)
 ZEND_END_ARG_INFO()

ext/tokenizer/tokenizer_data.c

@@ -70,7 +70,6 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_BOOL_CAST", T_BOOL_CAST, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_UNSET_CAST", T_UNSET_CAST, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_POW", T_POW, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_NEW", T_NEW, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_CLONE", T_CLONE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ELSEIF", T_ELSEIF, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ELSE", T_ELSE, CONST_CS | CONST_PERSISTENT);
@@ -86,6 +85,7 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_EVAL", T_EVAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_INC", T_INC, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DEC", T_DEC, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_NEW", T_NEW, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_EXIT", T_EXIT, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_IF", T_IF, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ENDIF", T_ENDIF, CONST_CS | CONST_PERSISTENT);
@@ -213,7 +213,6 @@ char *get_token_type_name(int token_type)
 		case T_BOOL_CAST: return "T_BOOL_CAST";
 		case T_UNSET_CAST: return "T_UNSET_CAST";
 		case T_POW: return "T_POW";
-		case T_NEW: return "T_NEW";
 		case T_CLONE: return "T_CLONE";
 		case T_ELSEIF: return "T_ELSEIF";
 		case T_ELSE: return "T_ELSE";
@@ -229,6 +228,7 @@ char *get_token_type_name(int token_type)
 		case T_EVAL: return "T_EVAL";
 		case T_INC: return "T_INC";
 		case T_DEC: return "T_DEC";
+		case T_NEW: return "T_NEW";
 		case T_EXIT: return "T_EXIT";
 		case T_IF: return "T_IF";
 		case T_ENDIF: return "T_ENDIF";
@@ -306,6 +306,6 @@ char *get_token_type_name(int token_type)
 		case T_BAD_CHARACTER: return "T_BAD_CHARACTER";
 
 	}
-	return "UNKNOWN";
+	return NULL;
 }
 

ext/tokenizer/tokenizer_data_gen.sh

@@ -71,7 +71,7 @@ awk '
 
 echo '
 	}
-	return "UNKNOWN";
+	return NULL;
 }
 ' >> $outfile