Bug#25516881: JSON_UNQUOTE RETURNS WRONG RESULT WHEN INPUT IS UTF-16

The JSON_UNQUOTE function did not unquote the string if it was UTF-16
encoded.

This happened because the function had some code to check if the
string started and ended with double-quotes, which assumed that the
input was UTF-8 encoded. This made it think that UTF-16 strings always
started with '\0', so it never treated them as quoted strings.

Fix: Convert the input string to utf8mb4, if needed, before attempting
to analyze the contents of the string.

Change-Id: I1c3fd717cb14ea8fcd57d99f5e9f7a61da798dce

Author: kahatlen

mysql-test/suite/json/r/json_no_table.result

@@ -3308,3 +3308,15 @@ NULL
 #
 DO BIT_COUNT(COMPRESS(1) << JSON_DEPTH(1));
 ERROR 22032: Invalid data type for JSON data in argument 1 to function json_depth; a JSON string or JSON type is required.
+#
+# Bug#25516881: JSON_UNQUOTE RETURNS WRONG RESULT WHEN INPUT IS UTF-16
+#
+SELECT JSON_UNQUOTE(CAST('"abc"' AS CHAR CHARSET utf16));
+JSON_UNQUOTE(CAST('"abc"' AS CHAR CHARSET utf16))
+abc
+SELECT JSON_UNQUOTE(CAST('ABCD' AS BINARY));
+ERROR 22032: Cannot create a JSON value from a string with CHARACTER SET 'binary'.
+SELECT HEX(JSON_UNQUOTE(CAST('"abc"' AS CHAR CHARSET utf16))),
+HEX(JSON_UNQUOTE(CAST('abc' AS CHAR CHARSET utf16)));
+HEX(JSON_UNQUOTE(CAST('"abc"' AS CHAR CHARSET utf16)))	HEX(JSON_UNQUOTE(CAST('abc' AS CHAR CHARSET utf16)))
+616263	616263

mysql-test/suite/json/t/json_no_table.test

@@ -2306,3 +2306,15 @@ SELECT MIN(CASE 0 WHEN 1 THEN JSON_ARRAY() ELSE NULL END);
 --echo #
 --error ER_INVALID_TYPE_FOR_JSON
 DO BIT_COUNT(COMPRESS(1) << JSON_DEPTH(1));
+
+--echo #
+--echo # Bug#25516881: JSON_UNQUOTE RETURNS WRONG RESULT WHEN INPUT IS UTF-16
+--echo #
+# Used to return the quoted string unchanged.
+SELECT JSON_UNQUOTE(CAST('"abc"' AS CHAR CHARSET utf16));
+# Used to accept binary strings.
+--error ER_INVALID_JSON_CHARSET
+SELECT JSON_UNQUOTE(CAST('ABCD' AS BINARY));
+# JSON_UNQUOTE should return utf8mb4 encoded results, but used to return utf16.
+SELECT HEX(JSON_UNQUOTE(CAST('"abc"' AS CHAR CHARSET utf16))),
+       HEX(JSON_UNQUOTE(CAST('abc' AS CHAR CHARSET utf16)));

sql/item_json_func.cc

@@ -3518,15 +3518,27 @@ String *Item_func_json_unquote::val_str(String *str)
       return error_str();
     }
 
-    if (res->length() < 2 || *res->ptr() != '"' ||
-        res->ptr()[res->length() - 1] != '"')
+    StringBuffer<STRING_BUFFER_USUAL_SIZE> buf;
+    const char *utf8text;
+    size_t utf8len;
+    if (ensure_utf8mb4(res, &buf, &utf8text, &utf8len, true))
+      return error_str();
+    String *utf8str= (res->ptr() == utf8text) ? res : &buf;
+    DBUG_ASSERT(utf8text == utf8str->ptr());
+
+    if (utf8len < 2 || utf8text[0] != '"' || utf8text[utf8len - 1] != '"')
     {
       null_value= false;
-      return res; // return string unchanged
+      // Return string unchanged, but convert to utf8mb4 if needed.
+      if (res == utf8str)
+        return res;
+      if (str->copy(utf8text, utf8len, collation.collation))
+        return error_str();                     /* purecov: inspected */
+      return str;
     }
 
     bool parse_error= false;
-    if (parse_json(res, 0, func_name(), &dom, true, &parse_error))
+    if (parse_json(utf8str, 0, func_name(), &dom, true, &parse_error))
     {
       return error_str();
     }