Skip to content

Commit 1ae035b

Browse files
authored
bpo-40302: Add pycore_byteswap.h header file (GH-19552)
Add a new internal pycore_byteswap.h header file with the following functions: * _Py_bswap16() * _Py_bswap32() * _Py_bswap64() Use these functions in _ctypes, sha256 and sha512 modules, and also use in the UTF-32 encoder. sha256, sha512 and _ctypes modules are now built with the internal C API.
1 parent 485e715 commit 1ae035b

File tree

13 files changed

+219
-112
lines changed

13 files changed

+219
-112
lines changed

Include/internal/pycore_byteswap.h

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/* Bytes swap functions, reverse order of bytes:
2+
3+
- _Py_bswap16(uint16_t)
4+
- _Py_bswap32(uint32_t)
5+
- _Py_bswap64(uint64_t)
6+
*/
7+
8+
#ifndef Py_INTERNAL_BSWAP_H
9+
#define Py_INTERNAL_BSWAP_H
10+
#ifdef __cplusplus
11+
extern "C" {
12+
#endif
13+
14+
#ifndef Py_BUILD_CORE
15+
# error "this header requires Py_BUILD_CORE define"
16+
#endif
17+
18+
#if defined(__clang__) || \
19+
(defined(__GNUC__) && \
20+
((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 8)))
21+
/* __builtin_bswap16() is available since GCC 4.8,
22+
__builtin_bswap32() is available since GCC 4.3,
23+
__builtin_bswap64() is available since GCC 4.3. */
24+
# define _PY_HAVE_BUILTIN_BSWAP
25+
#endif
26+
27+
#ifdef _MSC_VER
28+
/* Get _byteswap_ushort(), _byteswap_ulong(), _byteswap_uint64() */
29+
# include <intrin.h>
30+
#endif
31+
32+
static inline uint16_t
33+
_Py_bswap16(uint16_t word)
34+
{
35+
#ifdef _PY_HAVE_BUILTIN_BSWAP
36+
return __builtin_bswap16(word);
37+
#elif defined(_MSC_VER)
38+
Py_BUILD_ASSERT(sizeof(word) == sizeof(unsigned short));
39+
return _byteswap_ushort(word);
40+
#else
41+
// Portable implementation which doesn't rely on circular bit shift
42+
return ( ((word & UINT16_C(0x00FF)) << 8)
43+
| ((word & UINT16_C(0xFF00)) >> 8));
44+
#endif
45+
}
46+
47+
static inline uint32_t
48+
_Py_bswap32(uint32_t word)
49+
{
50+
#ifdef _PY_HAVE_BUILTIN_BSWAP
51+
return __builtin_bswap32(word);
52+
#elif defined(_MSC_VER)
53+
Py_BUILD_ASSERT(sizeof(word) == sizeof(unsigned long));
54+
return _byteswap_ulong(word);
55+
#else
56+
// Portable implementation which doesn't rely on circular bit shift
57+
return ( ((word & UINT32_C(0x000000FF)) << 24)
58+
| ((word & UINT32_C(0x0000FF00)) << 8)
59+
| ((word & UINT32_C(0x00FF0000)) >> 8)
60+
| ((word & UINT32_C(0xFF000000)) >> 24));
61+
#endif
62+
}
63+
64+
static inline uint64_t
65+
_Py_bswap64(uint64_t word)
66+
{
67+
#ifdef _PY_HAVE_BUILTIN_BSWAP
68+
return __builtin_bswap64(word);
69+
#elif defined(_MSC_VER)
70+
return _byteswap_uint64(word);
71+
#else
72+
// Portable implementation which doesn't rely on circular bit shift
73+
return ( ((word & UINT64_C(0x00000000000000FF)) << 56)
74+
| ((word & UINT64_C(0x000000000000FF00)) << 40)
75+
| ((word & UINT64_C(0x0000000000FF0000)) << 24)
76+
| ((word & UINT64_C(0x00000000FF000000)) << 8)
77+
| ((word & UINT64_C(0x000000FF00000000)) >> 8)
78+
| ((word & UINT64_C(0x0000FF0000000000)) >> 24)
79+
| ((word & UINT64_C(0x00FF000000000000)) >> 40)
80+
| ((word & UINT64_C(0xFF00000000000000)) >> 56));
81+
#endif
82+
}
83+
84+
85+
#ifdef __cplusplus
86+
}
87+
#endif
88+
#endif /* !Py_INTERNAL_BSWAP_H */
89+

Include/pyport.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -768,11 +768,11 @@ extern char * _getpty(int *, int, mode_t, int);
768768
*/
769769

770770
#ifdef WORDS_BIGENDIAN
771-
#define PY_BIG_ENDIAN 1
772-
#define PY_LITTLE_ENDIAN 0
771+
# define PY_BIG_ENDIAN 1
772+
# define PY_LITTLE_ENDIAN 0
773773
#else
774-
#define PY_BIG_ENDIAN 0
775-
#define PY_LITTLE_ENDIAN 1
774+
# define PY_BIG_ENDIAN 0
775+
# define PY_LITTLE_ENDIAN 1
776776
#endif
777777

778778
#ifdef Py_BUILD_CORE

Lib/test/test_capi.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
# Skip this test if the _testcapi module isn't available.
2525
_testcapi = support.import_module('_testcapi')
2626

27+
import _testinternalcapi
28+
2729
# Were we compiled --with-pydebug or with #define Py_DEBUG?
2830
Py_DEBUG = hasattr(sys, 'gettotalrefcount')
2931

@@ -658,6 +660,12 @@ class Test_testcapi(unittest.TestCase):
658660
if name.startswith('test_') and not name.endswith('_code'))
659661

660662

663+
class Test_testinternalcapi(unittest.TestCase):
664+
locals().update((name, getattr(_testinternalcapi, name))
665+
for name in dir(_testinternalcapi)
666+
if name.startswith('test_'))
667+
668+
661669
class PyMemDebugTests(unittest.TestCase):
662670
PYTHONMALLOC = 'debug'
663671
# '0x04c06e0' or '04C06E0'

Makefile.pre.in

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1083,6 +1083,7 @@ PYTHON_HEADERS= \
10831083
$(srcdir)/Include/internal/pycore_abstract.h \
10841084
$(srcdir)/Include/internal/pycore_accu.h \
10851085
$(srcdir)/Include/internal/pycore_atomic.h \
1086+
$(srcdir)/Include/internal/pycore_byteswap.h \
10861087
$(srcdir)/Include/internal/pycore_bytes_methods.h \
10871088
$(srcdir)/Include/internal/pycore_call.h \
10881089
$(srcdir)/Include/internal/pycore_ceval.h \

Modules/Setup

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -247,8 +247,8 @@ _symtable symtablemodule.c
247247
# The _sha module implements the SHA checksum algorithms.
248248
# (NIST's Secure Hash Algorithms.)
249249
#_sha1 sha1module.c
250-
#_sha256 sha256module.c
251-
#_sha512 sha512module.c
250+
#_sha256 sha256module.c -DPy_BUILD_CORE_BUILTIN
251+
#_sha512 sha512module.c -DPy_BUILD_CORE_BUILTIN
252252
#_sha3 _sha3/sha3module.c
253253

254254
# _blake module

Modules/_ctypes/cfield.c

Lines changed: 48 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#include "Python.h"
2+
#include "pycore_byteswap.h" // _Py_bswap32()
23

34
#include <ffi.h>
45
#ifdef MS_WIN32
@@ -448,46 +449,32 @@ get_ulonglong(PyObject *v, unsigned long long *p)
448449
( ( (type)x & ~(BIT_MASK(type, size) << LOW_BIT(size)) ) | ( ((type)v & BIT_MASK(type, size)) << LOW_BIT(size) ) ) \
449450
: (type)v)
450451

451-
/* byte swapping macros */
452-
#define SWAP_2(v) \
453-
( ( (v >> 8) & 0x00FF) | \
454-
( (v << 8) & 0xFF00) )
455-
456-
#define SWAP_4(v) \
457-
( ( (v & 0x000000FF) << 24 ) | \
458-
( (v & 0x0000FF00) << 8 ) | \
459-
( (v & 0x00FF0000) >> 8 ) | \
460-
( ((v >> 24) & 0xFF)) )
461-
462-
#ifdef _MSC_VER
463-
#define SWAP_8(v) \
464-
( ( (v & 0x00000000000000FFL) << 56 ) | \
465-
( (v & 0x000000000000FF00L) << 40 ) | \
466-
( (v & 0x0000000000FF0000L) << 24 ) | \
467-
( (v & 0x00000000FF000000L) << 8 ) | \
468-
( (v & 0x000000FF00000000L) >> 8 ) | \
469-
( (v & 0x0000FF0000000000L) >> 24 ) | \
470-
( (v & 0x00FF000000000000L) >> 40 ) | \
471-
( ((v >> 56) & 0xFF)) )
452+
#if SIZEOF_SHORT == 2
453+
# define SWAP_SHORT _Py_bswap16
472454
#else
473-
#define SWAP_8(v) \
474-
( ( (v & 0x00000000000000FFLL) << 56 ) | \
475-
( (v & 0x000000000000FF00LL) << 40 ) | \
476-
( (v & 0x0000000000FF0000LL) << 24 ) | \
477-
( (v & 0x00000000FF000000LL) << 8 ) | \
478-
( (v & 0x000000FF00000000LL) >> 8 ) | \
479-
( (v & 0x0000FF0000000000LL) >> 24 ) | \
480-
( (v & 0x00FF000000000000LL) >> 40 ) | \
481-
( ((v >> 56) & 0xFF)) )
455+
# error "unsupported short size"
482456
#endif
483457

484-
#define SWAP_INT SWAP_4
458+
#if SIZEOF_INT == 4
459+
# define SWAP_INT _Py_bswap32
460+
#else
461+
# error "unsupported int size"
462+
#endif
485463

486464
#if SIZEOF_LONG == 4
487-
# define SWAP_LONG SWAP_4
465+
# define SWAP_LONG _Py_bswap32
488466
#elif SIZEOF_LONG == 8
489-
# define SWAP_LONG SWAP_8
467+
# define SWAP_LONG _Py_bswap64
468+
#else
469+
# error "unsupported long size"
470+
#endif
471+
472+
#if SIZEOF_LONG_LONG == 8
473+
# define SWAP_LONG_LONG _Py_bswap64
474+
#else
475+
# error "unsupported long long size"
490476
#endif
477+
491478
/*****************************************************************
492479
* The setter methods return an object which must be kept alive, to keep the
493480
* data valid which has been stored in the memory block. The ctypes object
@@ -569,12 +556,13 @@ h_set_sw(void *ptr, PyObject *value, Py_ssize_t size)
569556
{
570557
long val;
571558
short field;
572-
if (get_long(value, &val) < 0)
559+
if (get_long(value, &val) < 0) {
573560
return NULL;
561+
}
574562
memcpy(&field, ptr, sizeof(field));
575-
field = SWAP_2(field);
563+
field = SWAP_SHORT(field);
576564
field = SET(short, field, val, size);
577-
field = SWAP_2(field);
565+
field = SWAP_SHORT(field);
578566
memcpy(ptr, &field, sizeof(field));
579567
_RET(value);
580568
}
@@ -593,7 +581,7 @@ h_get_sw(void *ptr, Py_ssize_t size)
593581
{
594582
short val;
595583
memcpy(&val, ptr, sizeof(val));
596-
val = SWAP_2(val);
584+
val = SWAP_SHORT(val);
597585
GET_BITFIELD(val, size);
598586
return PyLong_FromLong(val);
599587
}
@@ -616,12 +604,13 @@ H_set_sw(void *ptr, PyObject *value, Py_ssize_t size)
616604
{
617605
unsigned long val;
618606
unsigned short field;
619-
if (get_ulong(value, &val) < 0)
607+
if (get_ulong(value, &val) < 0) {
620608
return NULL;
609+
}
621610
memcpy(&field, ptr, sizeof(field));
622-
field = SWAP_2(field);
611+
field = SWAP_SHORT(field);
623612
field = SET(unsigned short, field, val, size);
624-
field = SWAP_2(field);
613+
field = SWAP_SHORT(field);
625614
memcpy(ptr, &field, sizeof(field));
626615
_RET(value);
627616
}
@@ -641,7 +630,7 @@ H_get_sw(void *ptr, Py_ssize_t size)
641630
{
642631
unsigned short val;
643632
memcpy(&val, ptr, sizeof(val));
644-
val = SWAP_2(val);
633+
val = SWAP_SHORT(val);
645634
GET_BITFIELD(val, size);
646635
return PyLong_FromLong(val);
647636
}
@@ -664,8 +653,9 @@ i_set_sw(void *ptr, PyObject *value, Py_ssize_t size)
664653
{
665654
long val;
666655
int field;
667-
if (get_long(value, &val) < 0)
656+
if (get_long(value, &val) < 0) {
668657
return NULL;
658+
}
669659
memcpy(&field, ptr, sizeof(field));
670660
field = SWAP_INT(field);
671661
field = SET(int, field, val, size);
@@ -757,8 +747,9 @@ I_set_sw(void *ptr, PyObject *value, Py_ssize_t size)
757747
{
758748
unsigned long val;
759749
unsigned int field;
760-
if (get_ulong(value, &val) < 0)
750+
if (get_ulong(value, &val) < 0) {
761751
return NULL;
752+
}
762753
memcpy(&field, ptr, sizeof(field));
763754
field = SWAP_INT(field);
764755
field = SET(unsigned int, field, (unsigned int)val, size);
@@ -805,8 +796,9 @@ l_set_sw(void *ptr, PyObject *value, Py_ssize_t size)
805796
{
806797
long val;
807798
long field;
808-
if (get_long(value, &val) < 0)
799+
if (get_long(value, &val) < 0) {
809800
return NULL;
801+
}
810802
memcpy(&field, ptr, sizeof(field));
811803
field = SWAP_LONG(field);
812804
field = SET(long, field, val, size);
@@ -853,8 +845,9 @@ L_set_sw(void *ptr, PyObject *value, Py_ssize_t size)
853845
{
854846
unsigned long val;
855847
unsigned long field;
856-
if (get_ulong(value, &val) < 0)
848+
if (get_ulong(value, &val) < 0) {
857849
return NULL;
850+
}
858851
memcpy(&field, ptr, sizeof(field));
859852
field = SWAP_LONG(field);
860853
field = SET(unsigned long, field, val, size);
@@ -901,12 +894,13 @@ q_set_sw(void *ptr, PyObject *value, Py_ssize_t size)
901894
{
902895
long long val;
903896
long long field;
904-
if (get_longlong(value, &val) < 0)
897+
if (get_longlong(value, &val) < 0) {
905898
return NULL;
899+
}
906900
memcpy(&field, ptr, sizeof(field));
907-
field = SWAP_8(field);
901+
field = SWAP_LONG_LONG(field);
908902
field = SET(long long, field, val, size);
909-
field = SWAP_8(field);
903+
field = SWAP_LONG_LONG(field);
910904
memcpy(ptr, &field, sizeof(field));
911905
_RET(value);
912906
}
@@ -925,7 +919,7 @@ q_get_sw(void *ptr, Py_ssize_t size)
925919
{
926920
long long val;
927921
memcpy(&val, ptr, sizeof(val));
928-
val = SWAP_8(val);
922+
val = SWAP_LONG_LONG(val);
929923
GET_BITFIELD(val, size);
930924
return PyLong_FromLongLong(val);
931925
}
@@ -948,12 +942,13 @@ Q_set_sw(void *ptr, PyObject *value, Py_ssize_t size)
948942
{
949943
unsigned long long val;
950944
unsigned long long field;
951-
if (get_ulonglong(value, &val) < 0)
945+
if (get_ulonglong(value, &val) < 0) {
952946
return NULL;
947+
}
953948
memcpy(&field, ptr, sizeof(field));
954-
field = SWAP_8(field);
949+
field = SWAP_LONG_LONG(field);
955950
field = SET(unsigned long long, field, val, size);
956-
field = SWAP_8(field);
951+
field = SWAP_LONG_LONG(field);
957952
memcpy(ptr, &field, sizeof(field));
958953
_RET(value);
959954
}
@@ -972,7 +967,7 @@ Q_get_sw(void *ptr, Py_ssize_t size)
972967
{
973968
unsigned long long val;
974969
memcpy(&val, ptr, sizeof(val));
975-
val = SWAP_8(val);
970+
val = SWAP_LONG_LONG(val);
976971
GET_BITFIELD(val, size);
977972
return PyLong_FromUnsignedLongLong(val);
978973
}

0 commit comments

Comments
 (0)