Skip to content

Commit d25da89

Browse files
picnixzpicnixz
authored
[3.12] gh-127667: fix memory leaks in hashlib (GH-127668) (#130783)
gh-127667: fix memory leaks in `hashlib` (GH-127668) - Correctly handle `NULL` values returned by `EVP_MD_CTX_md`. - Correctly free resources in error branches. - Consistently suppress `_setException()` return value when needed. - Collapse `_setException() + return NULL` into a single statement. (cherry-picked from commit 0978465)
1 parent a22cfd4 commit d25da89

File tree

1 file changed

+78
-56
lines changed

1 file changed

+78
-56
lines changed

‎Modules/_hashopenssl.c

Lines changed: 78 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -409,7 +409,8 @@ py_digest_by_name(PyObject *module, const char *name, enum Py_hash_type py_ht)
409409
}
410410
}
411411
if (digest == NULL) {
412-
_setException(state->unsupported_digestmod_error, "unsupported hash type %s", name);
412+
(void)_setException(state->unsupported_digestmod_error,
413+
"unsupported hash type %s", name);
413414
return NULL;
414415
}
415416
return digest;
@@ -424,7 +425,6 @@ py_digest_by_name(PyObject *module, const char *name, enum Py_hash_type py_ht)
424425
*/
425426
static PY_EVP_MD*
426427
py_digest_by_digestmod(PyObject *module, PyObject *digestmod, enum Py_hash_type py_ht) {
427-
PY_EVP_MD* evp;
428428
PyObject *name_obj = NULL;
429429
const char *name;
430430

@@ -450,12 +450,7 @@ py_digest_by_digestmod(PyObject *module, PyObject *digestmod, enum Py_hash_type
450450
return NULL;
451451
}
452452

453-
evp = py_digest_by_name(module, name, py_ht);
454-
if (evp == NULL) {
455-
return NULL;
456-
}
457-
458-
return evp;
453+
return py_digest_by_name(module, name, py_ht);
459454
}
460455

461456
static EVPobject *
@@ -489,7 +484,7 @@ EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len)
489484
else
490485
process = Py_SAFE_DOWNCAST(len, Py_ssize_t, unsigned int);
491486
if (!EVP_DigestUpdate(self->ctx, (const void*)cp, process)) {
492-
_setException(PyExc_ValueError, NULL);
487+
(void)_setException(PyExc_ValueError, NULL);
493488
return -1;
494489
}
495490
len -= process;
@@ -567,17 +562,20 @@ EVP_digest_impl(EVPobject *self)
567562
}
568563

569564
if (!locked_EVP_MD_CTX_copy(temp_ctx, self)) {
570-
return _setException(PyExc_ValueError, NULL);
565+
goto error;
571566
}
572567
digest_size = EVP_MD_CTX_size(temp_ctx);
573568
if (!EVP_DigestFinal(temp_ctx, digest, NULL)) {
574-
_setException(PyExc_ValueError, NULL);
575-
return NULL;
569+
goto error;
576570
}
577571

578572
retval = PyBytes_FromStringAndSize((const char *)digest, digest_size);
579573
EVP_MD_CTX_free(temp_ctx);
580574
return retval;
575+
576+
error:
577+
EVP_MD_CTX_free(temp_ctx);
578+
return _setException(PyExc_ValueError, NULL);
581579
}
582580

583581
/*[clinic input]
@@ -602,17 +600,20 @@ EVP_hexdigest_impl(EVPobject *self)
602600

603601
/* Get the raw (binary) digest value */
604602
if (!locked_EVP_MD_CTX_copy(temp_ctx, self)) {
605-
return _setException(PyExc_ValueError, NULL);
603+
goto error;
606604
}
607605
digest_size = EVP_MD_CTX_size(temp_ctx);
608606
if (!EVP_DigestFinal(temp_ctx, digest, NULL)) {
609-
_setException(PyExc_ValueError, NULL);
610-
return NULL;
607+
goto error;
611608
}
612609

613610
EVP_MD_CTX_free(temp_ctx);
614611

615612
return _Py_strhex((const char *)digest, (Py_ssize_t)digest_size);
613+
614+
error:
615+
EVP_MD_CTX_free(temp_ctx);
616+
return _setException(PyExc_ValueError, NULL);
616617
}
617618

618619
/*[clinic input]
@@ -682,7 +683,11 @@ EVP_get_digest_size(EVPobject *self, void *closure)
682683
static PyObject *
683684
EVP_get_name(EVPobject *self, void *closure)
684685
{
685-
return py_digest_name(EVP_MD_CTX_md(self->ctx));
686+
const EVP_MD *md = EVP_MD_CTX_md(self->ctx);
687+
if (md == NULL) {
688+
return _setException(PyExc_ValueError, NULL);
689+
}
690+
return py_digest_name(md);
686691
}
687692

688693
static PyGetSetDef EVP_getseters[] = {
@@ -780,21 +785,22 @@ EVPXOF_digest_impl(EVPobject *self, Py_ssize_t length)
780785
}
781786

782787
if (!locked_EVP_MD_CTX_copy(temp_ctx, self)) {
783-
Py_DECREF(retval);
784-
EVP_MD_CTX_free(temp_ctx);
785-
return _setException(PyExc_ValueError, NULL);
788+
goto error;
786789
}
787790
if (!EVP_DigestFinalXOF(temp_ctx,
788791
(unsigned char*)PyBytes_AS_STRING(retval),
789-
length)) {
790-
Py_DECREF(retval);
791-
EVP_MD_CTX_free(temp_ctx);
792-
_setException(PyExc_ValueError, NULL);
793-
return NULL;
792+
length))
793+
{
794+
goto error;
794795
}
795796

796797
EVP_MD_CTX_free(temp_ctx);
797798
return retval;
799+
800+
error:
801+
Py_DECREF(retval);
802+
EVP_MD_CTX_free(temp_ctx);
803+
return _setException(PyExc_ValueError, NULL);
798804
}
799805

800806
/*[clinic input]
@@ -828,22 +834,22 @@ EVPXOF_hexdigest_impl(EVPobject *self, Py_ssize_t length)
828834

829835
/* Get the raw (binary) digest value */
830836
if (!locked_EVP_MD_CTX_copy(temp_ctx, self)) {
831-
PyMem_Free(digest);
832-
EVP_MD_CTX_free(temp_ctx);
833-
return _setException(PyExc_ValueError, NULL);
837+
goto error;
834838
}
835839
if (!EVP_DigestFinalXOF(temp_ctx, digest, length)) {
836-
PyMem_Free(digest);
837-
EVP_MD_CTX_free(temp_ctx);
838-
_setException(PyExc_ValueError, NULL);
839-
return NULL;
840+
goto error;
840841
}
841842

842843
EVP_MD_CTX_free(temp_ctx);
843844

844845
retval = _Py_strhex((const char *)digest, length);
845846
PyMem_Free(digest);
846847
return retval;
848+
849+
error:
850+
PyMem_Free(digest);
851+
EVP_MD_CTX_free(temp_ctx);
852+
return _setException(PyExc_ValueError, NULL);
847853
}
848854

849855
static PyMethodDef EVPXOF_methods[] = {
@@ -944,7 +950,7 @@ py_evp_fromname(PyObject *module, const char *digestname, PyObject *data_obj,
944950

945951
int result = EVP_DigestInit_ex(self->ctx, digest, NULL);
946952
if (!result) {
947-
_setException(PyExc_ValueError, NULL);
953+
(void)_setException(PyExc_ValueError, NULL);
948954
Py_CLEAR(self);
949955
goto exit;
950956
}
@@ -965,7 +971,7 @@ py_evp_fromname(PyObject *module, const char *digestname, PyObject *data_obj,
965971
}
966972
}
967973

968-
exit:
974+
exit:
969975
if (data_obj != NULL) {
970976
PyBuffer_Release(&view);
971977
}
@@ -1410,14 +1416,14 @@ _hashlib_scrypt_impl(PyObject *module, Py_buffer *password, Py_buffer *salt,
14101416
r = PyLong_AsUnsignedLong(r_obj);
14111417
if (r == (unsigned long) -1 && PyErr_Occurred()) {
14121418
PyErr_SetString(PyExc_TypeError,
1413-
"r is required and must be an unsigned int");
1419+
"r is required and must be an unsigned int");
14141420
return NULL;
14151421
}
14161422

14171423
p = PyLong_AsUnsignedLong(p_obj);
14181424
if (p == (unsigned long) -1 && PyErr_Occurred()) {
14191425
PyErr_SetString(PyExc_TypeError,
1420-
"p is required and must be an unsigned int");
1426+
"p is required and must be an unsigned int");
14211427
return NULL;
14221428
}
14231429

@@ -1426,22 +1432,22 @@ _hashlib_scrypt_impl(PyObject *module, Py_buffer *password, Py_buffer *salt,
14261432
future. The maxmem constant is private to OpenSSL. */
14271433
PyErr_Format(PyExc_ValueError,
14281434
"maxmem must be positive and smaller than %d",
1429-
INT_MAX);
1435+
INT_MAX);
14301436
return NULL;
14311437
}
14321438

14331439
if (dklen < 1 || dklen > INT_MAX) {
14341440
PyErr_Format(PyExc_ValueError,
1435-
"dklen must be greater than 0 and smaller than %d",
1436-
INT_MAX);
1441+
"dklen must be greater than 0 and smaller than %d",
1442+
INT_MAX);
14371443
return NULL;
14381444
}
14391445

14401446
/* let OpenSSL validate the rest */
14411447
retval = EVP_PBE_scrypt(NULL, 0, NULL, 0, n, r, p, maxmem, NULL, 0);
14421448
if (!retval) {
1443-
_setException(PyExc_ValueError, "Invalid parameter combination for n, r, p, maxmem.");
1444-
return NULL;
1449+
return _setException(PyExc_ValueError,
1450+
"Invalid parameter combination for n, r, p, maxmem.");
14451451
}
14461452

14471453
key_obj = PyBytes_FromStringAndSize(NULL, dklen);
@@ -1461,8 +1467,7 @@ _hashlib_scrypt_impl(PyObject *module, Py_buffer *password, Py_buffer *salt,
14611467

14621468
if (!retval) {
14631469
Py_CLEAR(key_obj);
1464-
_setException(PyExc_ValueError, NULL);
1465-
return NULL;
1470+
return _setException(PyExc_ValueError, NULL);
14661471
}
14671472
return key_obj;
14681473
}
@@ -1518,8 +1523,7 @@ _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key,
15181523
PY_EVP_MD_free(evp);
15191524

15201525
if (result == NULL) {
1521-
_setException(PyExc_ValueError, NULL);
1522-
return NULL;
1526+
return _setException(PyExc_ValueError, NULL);
15231527
}
15241528
return PyBytes_FromStringAndSize((const char*)md, md_len);
15251529
}
@@ -1568,14 +1572,15 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj,
15681572

15691573
ctx = HMAC_CTX_new();
15701574
if (ctx == NULL) {
1575+
PY_EVP_MD_free(digest);
15711576
PyErr_NoMemory();
15721577
goto error;
15731578
}
15741579

15751580
r = HMAC_Init_ex(ctx, key->buf, (int)key->len, digest, NULL /* impl */);
15761581
PY_EVP_MD_free(digest);
15771582
if (r == 0) {
1578-
_setException(PyExc_ValueError, NULL);
1583+
(void)_setException(PyExc_ValueError, NULL);
15791584
goto error;
15801585
}
15811586

@@ -1613,11 +1618,20 @@ locked_HMAC_CTX_copy(HMAC_CTX *new_ctx_p, HMACobject *self)
16131618
return result;
16141619
}
16151620

1621+
/* returning 0 means that an error occurred and an exception is set */
16161622
static unsigned int
16171623
_hmac_digest_size(HMACobject *self)
16181624
{
1619-
unsigned int digest_size = EVP_MD_size(HMAC_CTX_get_md(self->ctx));
1625+
const EVP_MD *md = HMAC_CTX_get_md(self->ctx);
1626+
if (md == NULL) {
1627+
(void)_setException(PyExc_ValueError, NULL);
1628+
return 0;
1629+
}
1630+
unsigned int digest_size = EVP_MD_size(md);
16201631
assert(digest_size <= EVP_MAX_MD_SIZE);
1632+
if (digest_size == 0) {
1633+
(void)_setException(PyExc_ValueError, NULL);
1634+
}
16211635
return digest_size;
16221636
}
16231637

@@ -1647,7 +1661,7 @@ _hmac_update(HMACobject *self, PyObject *obj)
16471661
PyBuffer_Release(&view);
16481662

16491663
if (r == 0) {
1650-
_setException(PyExc_ValueError, NULL);
1664+
(void)_setException(PyExc_ValueError, NULL);
16511665
return 0;
16521666
}
16531667
return 1;
@@ -1703,7 +1717,11 @@ _hmac_dealloc(HMACobject *self)
17031717
static PyObject *
17041718
_hmac_repr(HMACobject *self)
17051719
{
1706-
PyObject *digest_name = py_digest_name(HMAC_CTX_get_md(self->ctx));
1720+
const EVP_MD *md = HMAC_CTX_get_md(self->ctx);
1721+
if (md == NULL) {
1722+
return _setException(PyExc_ValueError, NULL);
1723+
}
1724+
PyObject *digest_name = py_digest_name(md);
17071725
if (digest_name == NULL) {
17081726
return NULL;
17091727
}
@@ -1736,18 +1754,18 @@ _hmac_digest(HMACobject *self, unsigned char *buf, unsigned int len)
17361754
{
17371755
HMAC_CTX *temp_ctx = HMAC_CTX_new();
17381756
if (temp_ctx == NULL) {
1739-
PyErr_NoMemory();
1757+
(void)PyErr_NoMemory();
17401758
return 0;
17411759
}
17421760
if (!locked_HMAC_CTX_copy(temp_ctx, self)) {
17431761
HMAC_CTX_free(temp_ctx);
1744-
_setException(PyExc_ValueError, NULL);
1762+
(void)_setException(PyExc_ValueError, NULL);
17451763
return 0;
17461764
}
17471765
int r = HMAC_Final(temp_ctx, buf, &len);
17481766
HMAC_CTX_free(temp_ctx);
17491767
if (r == 0) {
1750-
_setException(PyExc_ValueError, NULL);
1768+
(void)_setException(PyExc_ValueError, NULL);
17511769
return 0;
17521770
}
17531771
return 1;
@@ -1765,7 +1783,7 @@ _hashlib_HMAC_digest_impl(HMACobject *self)
17651783
unsigned char digest[EVP_MAX_MD_SIZE];
17661784
unsigned int digest_size = _hmac_digest_size(self);
17671785
if (digest_size == 0) {
1768-
return _setException(PyExc_ValueError, NULL);
1786+
return NULL;
17691787
}
17701788
int r = _hmac_digest(self, digest, digest_size);
17711789
if (r == 0) {
@@ -1790,7 +1808,7 @@ _hashlib_HMAC_hexdigest_impl(HMACobject *self)
17901808
unsigned char digest[EVP_MAX_MD_SIZE];
17911809
unsigned int digest_size = _hmac_digest_size(self);
17921810
if (digest_size == 0) {
1793-
return _setException(PyExc_ValueError, NULL);
1811+
return NULL;
17941812
}
17951813
int r = _hmac_digest(self, digest, digest_size);
17961814
if (r == 0) {
@@ -1804,7 +1822,7 @@ _hashlib_hmac_get_digest_size(HMACobject *self, void *closure)
18041822
{
18051823
unsigned int digest_size = _hmac_digest_size(self);
18061824
if (digest_size == 0) {
1807-
return _setException(PyExc_ValueError, NULL);
1825+
return NULL;
18081826
}
18091827
return PyLong_FromLong(digest_size);
18101828
}
@@ -1822,7 +1840,11 @@ _hashlib_hmac_get_block_size(HMACobject *self, void *closure)
18221840
static PyObject *
18231841
_hashlib_hmac_get_name(HMACobject *self, void *closure)
18241842
{
1825-
PyObject *digest_name = py_digest_name(HMAC_CTX_get_md(self->ctx));
1843+
const EVP_MD *md = HMAC_CTX_get_md(self->ctx);
1844+
if (md == NULL) {
1845+
return _setException(PyExc_ValueError, NULL);
1846+
}
1847+
PyObject *digest_name = py_digest_name(md);
18261848
if (digest_name == NULL) {
18271849
return NULL;
18281850
}
@@ -1978,7 +2000,7 @@ _hashlib_get_fips_mode_impl(PyObject *module)
19782000
// But 0 is also a valid result value.
19792001
unsigned long errcode = ERR_peek_last_error();
19802002
if (errcode) {
1981-
_setException(PyExc_ValueError, NULL);
2003+
(void)_setException(PyExc_ValueError, NULL);
19822004
return -1;
19832005
}
19842006
}

0 commit comments

Comments
 (0)