Skip to content

Commit fb8986e

Browse files
Dave Watsonherbertx
Dave Watson
authored and
herbertx
committed
crypto: aesni - Introduce scatter/gather asm function stubs
The asm macros are all set up now, introduce entry points. GCM_INIT and GCM_COMPLETE have arguments supplied, so that the new scatter/gather entry points don't have to take all the arguments, and only the ones they need. Signed-off-by: Dave Watson <[email protected]> Signed-off-by: Herbert Xu <[email protected]>
1 parent 933d6ae commit fb8986e

File tree

2 files changed

+106
-26
lines changed

2 files changed

+106
-26
lines changed

‎arch/x86/crypto/aesni-intel_asm.S

Lines changed: 90 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -201,8 +201,8 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff
201201
# Output: HashKeys stored in gcm_context_data. Only needs to be called
202202
# once per key.
203203
# clobbers r12, and tmp xmm registers.
204-
.macro PRECOMPUTE TMP1 TMP2 TMP3 TMP4 TMP5 TMP6 TMP7
205-
mov arg7, %r12
204+
.macro PRECOMPUTE SUBKEY TMP1 TMP2 TMP3 TMP4 TMP5 TMP6 TMP7
205+
mov \SUBKEY, %r12
206206
movdqu (%r12), \TMP3
207207
movdqa SHUF_MASK(%rip), \TMP2
208208
PSHUFB_XMM \TMP2, \TMP3
@@ -255,26 +255,26 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff
255255

256256
# GCM_INIT initializes a gcm_context struct to prepare for encoding/decoding.
257257
# Clobbers rax, r10-r13 and xmm0-xmm6, %xmm13
258-
.macro GCM_INIT
259-
mov arg9, %r11
258+
.macro GCM_INIT Iv SUBKEY AAD AADLEN
259+
mov \AADLEN, %r11
260260
mov %r11, AadLen(%arg2) # ctx_data.aad_length = aad_length
261261
xor %r11, %r11
262262
mov %r11, InLen(%arg2) # ctx_data.in_length = 0
263263
mov %r11, PBlockLen(%arg2) # ctx_data.partial_block_length = 0
264264
mov %r11, PBlockEncKey(%arg2) # ctx_data.partial_block_enc_key = 0
265-
mov %arg6, %rax
265+
mov \Iv, %rax
266266
movdqu (%rax), %xmm0
267267
movdqu %xmm0, OrigIV(%arg2) # ctx_data.orig_IV = iv
268268

269269
movdqa SHUF_MASK(%rip), %xmm2
270270
PSHUFB_XMM %xmm2, %xmm0
271271
movdqu %xmm0, CurCount(%arg2) # ctx_data.current_counter = iv
272272

273-
PRECOMPUTE %xmm1 %xmm2 %xmm3 %xmm4 %xmm5 %xmm6 %xmm7
273+
PRECOMPUTE \SUBKEY, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
274274
movdqa HashKey(%arg2), %xmm13
275275

276-
CALC_AAD_HASH %xmm13 %xmm0 %xmm1 %xmm2 %xmm3 %xmm4 \
277-
%xmm5 %xmm6
276+
CALC_AAD_HASH %xmm13, \AAD, \AADLEN, %xmm0, %xmm1, %xmm2, %xmm3, \
277+
%xmm4, %xmm5, %xmm6
278278
.endm
279279

280280
# GCM_ENC_DEC Encodes/Decodes given data. Assumes that the passed gcm_context
@@ -436,7 +436,7 @@ _multiple_of_16_bytes_\@:
436436
# GCM_COMPLETE Finishes update of tag of last partial block
437437
# Output: Authorization Tag (AUTH_TAG)
438438
# Clobbers rax, r10-r12, and xmm0, xmm1, xmm5-xmm15
439-
.macro GCM_COMPLETE
439+
.macro GCM_COMPLETE AUTHTAG AUTHTAGLEN
440440
movdqu AadHash(%arg2), %xmm8
441441
movdqu HashKey(%arg2), %xmm13
442442

@@ -467,8 +467,8 @@ _partial_done\@:
467467
ENCRYPT_SINGLE_BLOCK %xmm0, %xmm1 # E(K, Y0)
468468
pxor %xmm8, %xmm0
469469
_return_T_\@:
470-
mov arg10, %r10 # %r10 = authTag
471-
mov arg11, %r11 # %r11 = auth_tag_len
470+
mov \AUTHTAG, %r10 # %r10 = authTag
471+
mov \AUTHTAGLEN, %r11 # %r11 = auth_tag_len
472472
cmp $16, %r11
473473
je _T_16_\@
474474
cmp $8, %r11
@@ -600,11 +600,11 @@ _done_read_partial_block_\@:
600600

601601
# CALC_AAD_HASH: Calculates the hash of the data which will not be encrypted.
602602
# clobbers r10-11, xmm14
603-
.macro CALC_AAD_HASH HASHKEY TMP1 TMP2 TMP3 TMP4 TMP5 \
603+
.macro CALC_AAD_HASH HASHKEY AAD AADLEN TMP1 TMP2 TMP3 TMP4 TMP5 \
604604
TMP6 TMP7
605605
MOVADQ SHUF_MASK(%rip), %xmm14
606-
mov arg8, %r10 # %r10 = AAD
607-
mov arg9, %r11 # %r11 = aadLen
606+
mov \AAD, %r10 # %r10 = AAD
607+
mov \AADLEN, %r11 # %r11 = aadLen
608608
pxor \TMP7, \TMP7
609609
pxor \TMP6, \TMP6
610610

@@ -1104,18 +1104,18 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
11041104
mov keysize,%eax
11051105
shr $2,%eax # 128->4, 192->6, 256->8
11061106
sub $4,%eax # 128->0, 192->2, 256->4
1107-
jz aes_loop_par_enc_done
1107+
jz aes_loop_par_enc_done\@
11081108

1109-
aes_loop_par_enc:
1109+
aes_loop_par_enc\@:
11101110
MOVADQ (%r10),\TMP3
11111111
.irpc index, 1234
11121112
AESENC \TMP3, %xmm\index
11131113
.endr
11141114
add $16,%r10
11151115
sub $1,%eax
1116-
jnz aes_loop_par_enc
1116+
jnz aes_loop_par_enc\@
11171117

1118-
aes_loop_par_enc_done:
1118+
aes_loop_par_enc_done\@:
11191119
MOVADQ (%r10), \TMP3
11201120
AESENCLAST \TMP3, \XMM1 # Round 10
11211121
AESENCLAST \TMP3, \XMM2
@@ -1312,18 +1312,18 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
13121312
mov keysize,%eax
13131313
shr $2,%eax # 128->4, 192->6, 256->8
13141314
sub $4,%eax # 128->0, 192->2, 256->4
1315-
jz aes_loop_par_dec_done
1315+
jz aes_loop_par_dec_done\@
13161316

1317-
aes_loop_par_dec:
1317+
aes_loop_par_dec\@:
13181318
MOVADQ (%r10),\TMP3
13191319
.irpc index, 1234
13201320
AESENC \TMP3, %xmm\index
13211321
.endr
13221322
add $16,%r10
13231323
sub $1,%eax
1324-
jnz aes_loop_par_dec
1324+
jnz aes_loop_par_dec\@
13251325

1326-
aes_loop_par_dec_done:
1326+
aes_loop_par_dec_done\@:
13271327
MOVADQ (%r10), \TMP3
13281328
AESENCLAST \TMP3, \XMM1 # last round
13291329
AESENCLAST \TMP3, \XMM2
@@ -1599,9 +1599,9 @@ _esb_loop_\@:
15991599
ENTRY(aesni_gcm_dec)
16001600
FUNC_SAVE
16011601

1602-
GCM_INIT
1602+
GCM_INIT %arg6, arg7, arg8, arg9
16031603
GCM_ENC_DEC dec
1604-
GCM_COMPLETE
1604+
GCM_COMPLETE arg10, arg11
16051605
FUNC_RESTORE
16061606
ret
16071607
ENDPROC(aesni_gcm_dec)
@@ -1687,13 +1687,77 @@ ENDPROC(aesni_gcm_dec)
16871687
ENTRY(aesni_gcm_enc)
16881688
FUNC_SAVE
16891689

1690-
GCM_INIT
1690+
GCM_INIT %arg6, arg7, arg8, arg9
16911691
GCM_ENC_DEC enc
1692-
GCM_COMPLETE
1692+
1693+
GCM_COMPLETE arg10, arg11
16931694
FUNC_RESTORE
16941695
ret
16951696
ENDPROC(aesni_gcm_enc)
16961697

1698+
/*****************************************************************************
1699+
* void aesni_gcm_init(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1700+
* struct gcm_context_data *data,
1701+
* // context data
1702+
* u8 *iv, // Pre-counter block j0: 4 byte salt (from Security Association)
1703+
* // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload)
1704+
* // concatenated with 0x00000001. 16-byte aligned pointer.
1705+
* u8 *hash_subkey, // H, the Hash sub key input. Data starts on a 16-byte boundary.
1706+
* const u8 *aad, // Additional Authentication Data (AAD)
1707+
* u64 aad_len) // Length of AAD in bytes.
1708+
*/
1709+
ENTRY(aesni_gcm_init)
1710+
FUNC_SAVE
1711+
GCM_INIT %arg3, %arg4,%arg5, %arg6
1712+
FUNC_RESTORE
1713+
ret
1714+
ENDPROC(aesni_gcm_init)
1715+
1716+
/*****************************************************************************
1717+
* void aesni_gcm_enc_update(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1718+
* struct gcm_context_data *data,
1719+
* // context data
1720+
* u8 *out, // Ciphertext output. Encrypt in-place is allowed.
1721+
* const u8 *in, // Plaintext input
1722+
* u64 plaintext_len, // Length of data in bytes for encryption.
1723+
*/
1724+
ENTRY(aesni_gcm_enc_update)
1725+
FUNC_SAVE
1726+
GCM_ENC_DEC enc
1727+
FUNC_RESTORE
1728+
ret
1729+
ENDPROC(aesni_gcm_enc_update)
1730+
1731+
/*****************************************************************************
1732+
* void aesni_gcm_dec_update(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1733+
* struct gcm_context_data *data,
1734+
* // context data
1735+
* u8 *out, // Ciphertext output. Encrypt in-place is allowed.
1736+
* const u8 *in, // Plaintext input
1737+
* u64 plaintext_len, // Length of data in bytes for encryption.
1738+
*/
1739+
ENTRY(aesni_gcm_dec_update)
1740+
FUNC_SAVE
1741+
GCM_ENC_DEC dec
1742+
FUNC_RESTORE
1743+
ret
1744+
ENDPROC(aesni_gcm_dec_update)
1745+
1746+
/*****************************************************************************
1747+
* void aesni_gcm_finalize(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary.
1748+
* struct gcm_context_data *data,
1749+
* // context data
1750+
* u8 *auth_tag, // Authenticated Tag output.
1751+
* u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16 (most likely),
1752+
* // 12 or 8.
1753+
*/
1754+
ENTRY(aesni_gcm_finalize)
1755+
FUNC_SAVE
1756+
GCM_COMPLETE %arg3 %arg4
1757+
FUNC_RESTORE
1758+
ret
1759+
ENDPROC(aesni_gcm_finalize)
1760+
16971761
#endif
16981762

16991763

‎arch/x86/crypto/aesni-intel_glue.c

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,22 @@ asmlinkage void aesni_gcm_dec(void *ctx,
161161
u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
162162
u8 *auth_tag, unsigned long auth_tag_len);
163163

164+
/* Scatter / Gather routines, with args similar to above */
165+
asmlinkage void aesni_gcm_init(void *ctx,
166+
struct gcm_context_data *gdata,
167+
u8 *iv,
168+
u8 *hash_subkey, const u8 *aad,
169+
unsigned long aad_len);
170+
asmlinkage void aesni_gcm_enc_update(void *ctx,
171+
struct gcm_context_data *gdata, u8 *out,
172+
const u8 *in, unsigned long plaintext_len);
173+
asmlinkage void aesni_gcm_dec_update(void *ctx,
174+
struct gcm_context_data *gdata, u8 *out,
175+
const u8 *in,
176+
unsigned long ciphertext_len);
177+
asmlinkage void aesni_gcm_finalize(void *ctx,
178+
struct gcm_context_data *gdata,
179+
u8 *auth_tag, unsigned long auth_tag_len);
164180

165181
#ifdef CONFIG_AS_AVX
166182
asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv,

0 commit comments

Comments
 (0)