Locally intern strings in token_get_all()

Author: nikic

ext/tokenizer/tokenizer.c

@@ -125,31 +125,40 @@ PHP_MINFO_FUNCTION(tokenizer)
 }
 /* }}} */
 
-static inline zend_string *make_str(unsigned char *text, size_t leng) {
+static zend_string *make_str(unsigned char *text, size_t leng, HashTable *interned_strings) {
 	if (leng == 1) {
 		return ZSTR_CHAR(text[0]);
+	} else if (interned_strings) {
+		zend_string *interned_str = zend_hash_str_find_ptr(interned_strings, (char *) text, leng);
+		if (interned_str) {
+			return zend_string_copy(interned_str);
+		}
+		interned_str = zend_string_init((char *) text, leng, 0);
+		zend_hash_add_new_ptr(interned_strings, interned_str, interned_str);
+		return interned_str;
 	} else {
 		return zend_string_init((char *) text, leng, 0);
 	}
 }
 
-static void add_token(zval *return_value, int token_type,
-		unsigned char *text, size_t leng, int lineno, zend_bool as_object) {
+static void add_token(
+		zval *return_value, int token_type, unsigned char *text, size_t leng, int lineno,
+		zend_bool as_object, HashTable *interned_strings) {
 	zval token;
 	if (as_object) {
 		zend_object *obj = zend_objects_new(php_token_ce);
 		ZVAL_OBJ(&token, obj);
 		ZVAL_LONG(OBJ_PROP_NUM(obj, 0), token_type);
-		ZVAL_STR(OBJ_PROP_NUM(obj, 1), make_str(text, leng));
+		ZVAL_STR(OBJ_PROP_NUM(obj, 1), make_str(text, leng, interned_strings));
 		ZVAL_LONG(OBJ_PROP_NUM(obj, 2), lineno);
 		ZVAL_LONG(OBJ_PROP_NUM(obj, 3), text - LANG_SCNG(yy_start));
 	} else if (token_type >= 256) {
 		array_init(&token);
 		add_next_index_long(&token, token_type);
-		add_next_index_str(&token, make_str(text, leng));
+		add_next_index_str(&token, make_str(text, leng, interned_strings));
 		add_next_index_long(&token, lineno);
 	} else {
-		ZVAL_STR(&token, make_str(text, leng));
+		ZVAL_STR(&token, make_str(text, leng, interned_strings));
 	}
 	zend_hash_next_index_insert_new(Z_ARRVAL_P(return_value), &token);
 }
@@ -162,6 +171,7 @@ static zend_bool tokenize(zval *return_value, zend_string *source, zend_bool as_
 	int token_type;
 	int token_line = 1;
 	int need_tokens = -1; /* for __halt_compiler lexing. -1 = disabled */
+	HashTable interned_strings;
 
 	ZVAL_STR_COPY(&source_zval, source);
 	zend_save_lexical_state(&original_lex_state);
@@ -172,10 +182,12 @@ static zend_bool tokenize(zval *return_value, zend_string *source, zend_bool as_
 	}
 
 	LANG_SCNG(yy_state) = yycINITIAL;
+	zend_hash_init(&interned_strings, 0, NULL, NULL, 0);
 	array_init(return_value);
 
 	while ((token_type = lex_scan(&token, NULL))) {
-		add_token(return_value, token_type, zendtext, zendleng, token_line, as_object);
+		add_token(return_value, token_type, zendtext, zendleng, token_line, as_object,
+			&interned_strings);
 
 		if (Z_TYPE(token) != IS_UNDEF) {
 			zval_ptr_dtor_nogc(&token);
@@ -191,7 +203,8 @@ static zend_bool tokenize(zval *return_value, zend_string *source, zend_bool as_
 				/* fetch the rest into a T_INLINE_HTML */
 				if (zendcursor != zendlimit) {
 					add_token(return_value, T_INLINE_HTML,
-						zendcursor, zendlimit - zendcursor, token_line, as_object);
+						zendcursor, zendlimit - zendcursor, token_line, as_object,
+						&interned_strings);
 				}
 				break;
 			}
@@ -209,6 +222,7 @@ static zend_bool tokenize(zval *return_value, zend_string *source, zend_bool as_
 
 	zval_ptr_dtor_str(&source_zval);
 	zend_restore_lexical_state(&original_lex_state);
+	zend_hash_destroy(&interned_strings);
 
 	return 1;
 }
@@ -234,7 +248,7 @@ void on_event(zend_php_scanner_event event, int token, int line, void *context)
 				token = T_OPEN_TAG_WITH_ECHO;
 			}
 			add_token(ctx->tokens, token,
-				LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line, ctx->as_object);
+				LANG_SCNG(yy_text), LANG_SCNG(yy_leng), line, ctx->as_object, NULL);
 			break;
 		case ON_FEEDBACK:
 			tokens_ht = Z_ARRVAL_P(ctx->tokens);
@@ -249,7 +263,8 @@ void on_event(zend_php_scanner_event event, int token, int line, void *context)
 		case ON_STOP:
 			if (LANG_SCNG(yy_cursor) != LANG_SCNG(yy_limit)) {
 				add_token(ctx->tokens, T_INLINE_HTML, LANG_SCNG(yy_cursor),
-					LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor), CG(zend_lineno), ctx->as_object);
+					LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor), CG(zend_lineno),
+					ctx->as_object, NULL);
 			}
 			break;
 	}