Bug#20642505: HENRY SPENCER REGULAR EXPRESSIONS (REGEX) LIBRARY

The MySQL server uses Henry Spencer's library for regular
expressions to support the REGEXP/RLIKE string operator.
This changeset adapts a recent fix from the upstream for
better 32-bit compatiblity. (Note that we cannot simply use
the current upstream version as a drop-in replacement
for the version used by the server as the latter has
been extended to understand MySQL charsets etc.)

(Adds unittest to 5.5 mergeset.)

Author: Tatiana Azundris Nuernberg

regex/regcomp.c

@@ -1,3 +1,11 @@
+/* Copyright 1992, 1993, 1994 Henry Spencer.  All rights reserved.
+   See file COPYRIGHT for details.
+
+   This file was modified by Oracle on 2015-05-18 for 32-bit compatibility.
+
+   Modifications copyright (c) 2015, Oracle and/or its affiliates. All rights
+   reserved. */
+
 #include <my_global.h>
 #include <m_string.h>
 #include <m_ctype.h>
@@ -133,12 +141,26 @@ const CHARSET_INFO *charset;
 	} else
 		len = strlen((char *)pattern);
 
+	/*
+	 Find the maximum len we can safely process
+	 without a rollover and a mis-malloc.
+	 p->ssize is a sopno is a long (32+ bit signed);
+	 size_t is 16+ bit unsigned.
+	*/
+	{
+	  size_t new_ssize = len / (size_t)2 * (size_t)3 + (size_t)1; /* ugh */
+	  if ((new_ssize < len) ||	/* size_t rolled over */
+	      ((SIZE_T_MAX / sizeof(sop)) < new_ssize) ||	/* malloc arg */
+	      (new_ssize > LONG_MAX))	/* won't fit in ssize */
+		return(MY_REG_ESPACE);	/* MY_REG_ESPACE or MY_REG_INVARG */
+	  p->ssize = new_ssize;
+	}
+
 	/* do the mallocs early so failure handling is easy */
 	g = (struct re_guts *)malloc(sizeof(struct re_guts) +
 							(NC-1)*sizeof(cat_t));
 	if (g == NULL)
 		return(MY_REG_ESPACE);
-	p->ssize = (long) (len/(size_t)2*(size_t)3 + (size_t)1); /* ugh */
 	p->strip = (sop *)malloc(p->ssize * sizeof(sop));
 	p->slen = 0;
 	if (p->strip == NULL) {

unittest/gunit/my_regex-t.cc

@@ -1,4 +1,4 @@
-/* Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved. 
+/* Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved. 
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -80,4 +80,39 @@ TEST_F(RegexTest, BasicTest)
   }
 }
 
+/*
+  Bug#20642505: HENRY SPENCER REGULAR EXPRESSIONS (REGEX) LIBRARY
+
+  We have our own variant of the regex code that understands MySQL charsets.
+  This test is hear to make sure that we never checkpoint or cherrypick from
+  the upstream and end up with a version that isn't patched against a
+  potential overflow.
+*/
+TEST_F(RegexTest, Bug20642505)
+{
+  my_regex_t  re;
+  char       *pattern;
+  int         err;
+  size_t      len= 684 * 1024 * 1024;
+
+  if ((sizeof(size_t) > 4) && (sizeof(long) > 4))
+    return;
+
+  /* set up an empty C string as pattern as regcomp() will strlen() this */
+  pattern= (char *) malloc(len);
+  EXPECT_FALSE(pattern == NULL);
+  memset(pattern, (int) ' ', len);
+  pattern[len - 1]= '\0';
+
+  err= my_regcomp(&re, pattern, MY_REG_BASIC,
+                  &my_charset_latin1);
+
+  my_regfree(&re);
+  free(pattern);
+
+  EXPECT_EQ(err, MY_REG_ESPACE)
+    << "my_regcomp returned " << err
+    << " instead of MY_REG_ESPACE (" << MY_REG_ESPACE << ")";
+}
+
 }  // namespace