Skip to content

Commit 79e7ce2

Browse files
borkmannborkmann
committed
Merge branch 'for-next/insn' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Will Deacon says: ==================== On Tue, Feb 22, 2022 at 10:38:02PM +0000, Will Deacon wrote: > On Thu, 17 Feb 2022 15:22:28 +0800, Hou Tao wrote: > > Atomics support in bpf has already been done by "Atomics for eBPF" > > patch series [1], but it only adds support for x86, and this patchset > > adds support for arm64. > > > > Patch #1 & patch #2 are arm64 related. Patch #1 moves the common used > > macro AARCH64_BREAK_FAULT into insn-def.h for insn.h. Patch #2 adds > > necessary encoder helpers for atomic operations. > > > > [...] > > Applied to arm64 (for-next/insn), thanks! > > [1/4] arm64: move AARCH64_BREAK_FAULT into insn-def.h > https://git.kernel.org/arm64/c/97e58e395e9c > [2/4] arm64: insn: add encoders for atomic operations > https://git.kernel.org/arm64/c/fa1114d9eba5 Daniel -- let's give this a day or so in -next, then if nothing catches fire you're more than welcome to pull this branch as a base for the rest of the series. ==================== Signed-off-by: Daniel Borkmann <[email protected]> Link: https://lore.kernel.org/bpf/20220222224211.GB16976@willie-the-truck
2 parents c62dd8a + fa1114d commit 79e7ce2

File tree

5 files changed

+268
-36
lines changed

5 files changed

+268
-36
lines changed

‎arch/arm64/include/asm/debug-monitors.h

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -34,18 +34,6 @@
3434
*/
3535
#define BREAK_INSTR_SIZE AARCH64_INSN_SIZE
3636

37-
/*
38-
* BRK instruction encoding
39-
* The #imm16 value should be placed at bits[20:5] within BRK ins
40-
*/
41-
#define AARCH64_BREAK_MON 0xd4200000
42-
43-
/*
44-
* BRK instruction for provoking a fault on purpose
45-
* Unlike kgdb, #imm16 value with unallocated handler is used for faulting.
46-
*/
47-
#define AARCH64_BREAK_FAULT (AARCH64_BREAK_MON | (FAULT_BRK_IMM << 5))
48-
4937
#define AARCH64_BREAK_KGDB_DYN_DBG \
5038
(AARCH64_BREAK_MON | (KGDB_DYN_DBG_BRK_IMM << 5))
5139

‎arch/arm64/include/asm/insn-def.h

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,21 @@
33
#ifndef __ASM_INSN_DEF_H
44
#define __ASM_INSN_DEF_H
55

6+
#include <asm/brk-imm.h>
7+
68
/* A64 instructions are always 32 bits. */
79
#define AARCH64_INSN_SIZE 4
810

11+
/*
12+
* BRK instruction encoding
13+
* The #imm16 value should be placed at bits[20:5] within BRK ins
14+
*/
15+
#define AARCH64_BREAK_MON 0xd4200000
16+
17+
/*
18+
* BRK instruction for provoking a fault on purpose
19+
* Unlike kgdb, #imm16 value with unallocated handler is used for faulting.
20+
*/
21+
#define AARCH64_BREAK_FAULT (AARCH64_BREAK_MON | (FAULT_BRK_IMM << 5))
22+
923
#endif /* __ASM_INSN_DEF_H */

‎arch/arm64/include/asm/insn.h

Lines changed: 73 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,9 @@ enum aarch64_insn_ldst_type {
205205
AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX,
206206
AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX,
207207
AARCH64_INSN_LDST_LOAD_EX,
208+
AARCH64_INSN_LDST_LOAD_ACQ_EX,
208209
AARCH64_INSN_LDST_STORE_EX,
210+
AARCH64_INSN_LDST_STORE_REL_EX,
209211
};
210212

211213
enum aarch64_insn_adsb_type {
@@ -280,6 +282,36 @@ enum aarch64_insn_adr_type {
280282
AARCH64_INSN_ADR_TYPE_ADR,
281283
};
282284

285+
enum aarch64_insn_mem_atomic_op {
286+
AARCH64_INSN_MEM_ATOMIC_ADD,
287+
AARCH64_INSN_MEM_ATOMIC_CLR,
288+
AARCH64_INSN_MEM_ATOMIC_EOR,
289+
AARCH64_INSN_MEM_ATOMIC_SET,
290+
AARCH64_INSN_MEM_ATOMIC_SWP,
291+
};
292+
293+
enum aarch64_insn_mem_order_type {
294+
AARCH64_INSN_MEM_ORDER_NONE,
295+
AARCH64_INSN_MEM_ORDER_ACQ,
296+
AARCH64_INSN_MEM_ORDER_REL,
297+
AARCH64_INSN_MEM_ORDER_ACQREL,
298+
};
299+
300+
enum aarch64_insn_mb_type {
301+
AARCH64_INSN_MB_SY,
302+
AARCH64_INSN_MB_ST,
303+
AARCH64_INSN_MB_LD,
304+
AARCH64_INSN_MB_ISH,
305+
AARCH64_INSN_MB_ISHST,
306+
AARCH64_INSN_MB_ISHLD,
307+
AARCH64_INSN_MB_NSH,
308+
AARCH64_INSN_MB_NSHST,
309+
AARCH64_INSN_MB_NSHLD,
310+
AARCH64_INSN_MB_OSH,
311+
AARCH64_INSN_MB_OSHST,
312+
AARCH64_INSN_MB_OSHLD,
313+
};
314+
283315
#define __AARCH64_INSN_FUNCS(abbr, mask, val) \
284316
static __always_inline bool aarch64_insn_is_##abbr(u32 code) \
285317
{ \
@@ -303,6 +335,11 @@ __AARCH64_INSN_FUNCS(store_post, 0x3FE00C00, 0x38000400)
303335
__AARCH64_INSN_FUNCS(load_post, 0x3FE00C00, 0x38400400)
304336
__AARCH64_INSN_FUNCS(str_reg, 0x3FE0EC00, 0x38206800)
305337
__AARCH64_INSN_FUNCS(ldadd, 0x3F20FC00, 0x38200000)
338+
__AARCH64_INSN_FUNCS(ldclr, 0x3F20FC00, 0x38201000)
339+
__AARCH64_INSN_FUNCS(ldeor, 0x3F20FC00, 0x38202000)
340+
__AARCH64_INSN_FUNCS(ldset, 0x3F20FC00, 0x38203000)
341+
__AARCH64_INSN_FUNCS(swp, 0x3F20FC00, 0x38208000)
342+
__AARCH64_INSN_FUNCS(cas, 0x3FA07C00, 0x08A07C00)
306343
__AARCH64_INSN_FUNCS(ldr_reg, 0x3FE0EC00, 0x38606800)
307344
__AARCH64_INSN_FUNCS(ldr_lit, 0xBF000000, 0x18000000)
308345
__AARCH64_INSN_FUNCS(ldrsw_lit, 0xFF000000, 0x98000000)
@@ -474,13 +511,6 @@ u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
474511
enum aarch64_insn_register state,
475512
enum aarch64_insn_size_type size,
476513
enum aarch64_insn_ldst_type type);
477-
u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result,
478-
enum aarch64_insn_register address,
479-
enum aarch64_insn_register value,
480-
enum aarch64_insn_size_type size);
481-
u32 aarch64_insn_gen_stadd(enum aarch64_insn_register address,
482-
enum aarch64_insn_register value,
483-
enum aarch64_insn_size_type size);
484514
u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,
485515
enum aarch64_insn_register src,
486516
int imm, enum aarch64_insn_variant variant,
@@ -541,6 +571,42 @@ u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base,
541571
enum aarch64_insn_prfm_type type,
542572
enum aarch64_insn_prfm_target target,
543573
enum aarch64_insn_prfm_policy policy);
574+
#ifdef CONFIG_ARM64_LSE_ATOMICS
575+
u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,
576+
enum aarch64_insn_register address,
577+
enum aarch64_insn_register value,
578+
enum aarch64_insn_size_type size,
579+
enum aarch64_insn_mem_atomic_op op,
580+
enum aarch64_insn_mem_order_type order);
581+
u32 aarch64_insn_gen_cas(enum aarch64_insn_register result,
582+
enum aarch64_insn_register address,
583+
enum aarch64_insn_register value,
584+
enum aarch64_insn_size_type size,
585+
enum aarch64_insn_mem_order_type order);
586+
#else
587+
static inline
588+
u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,
589+
enum aarch64_insn_register address,
590+
enum aarch64_insn_register value,
591+
enum aarch64_insn_size_type size,
592+
enum aarch64_insn_mem_atomic_op op,
593+
enum aarch64_insn_mem_order_type order)
594+
{
595+
return AARCH64_BREAK_FAULT;
596+
}
597+
598+
static inline
599+
u32 aarch64_insn_gen_cas(enum aarch64_insn_register result,
600+
enum aarch64_insn_register address,
601+
enum aarch64_insn_register value,
602+
enum aarch64_insn_size_type size,
603+
enum aarch64_insn_mem_order_type order)
604+
{
605+
return AARCH64_BREAK_FAULT;
606+
}
607+
#endif
608+
u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type);
609+
544610
s32 aarch64_get_branch_offset(u32 insn);
545611
u32 aarch64_set_branch_offset(u32 insn, s32 offset);
546612

‎arch/arm64/lib/insn.c

Lines changed: 172 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -578,10 +578,16 @@ u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
578578

579579
switch (type) {
580580
case AARCH64_INSN_LDST_LOAD_EX:
581+
case AARCH64_INSN_LDST_LOAD_ACQ_EX:
581582
insn = aarch64_insn_get_load_ex_value();
583+
if (type == AARCH64_INSN_LDST_LOAD_ACQ_EX)
584+
insn |= BIT(15);
582585
break;
583586
case AARCH64_INSN_LDST_STORE_EX:
587+
case AARCH64_INSN_LDST_STORE_REL_EX:
584588
insn = aarch64_insn_get_store_ex_value();
589+
if (type == AARCH64_INSN_LDST_STORE_REL_EX)
590+
insn |= BIT(15);
585591
break;
586592
default:
587593
pr_err("%s: unknown load/store exclusive encoding %d\n", __func__, type);
@@ -603,12 +609,65 @@ u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
603609
state);
604610
}
605611

606-
u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result,
607-
enum aarch64_insn_register address,
608-
enum aarch64_insn_register value,
609-
enum aarch64_insn_size_type size)
612+
#ifdef CONFIG_ARM64_LSE_ATOMICS
613+
static u32 aarch64_insn_encode_ldst_order(enum aarch64_insn_mem_order_type type,
614+
u32 insn)
610615
{
611-
u32 insn = aarch64_insn_get_ldadd_value();
616+
u32 order;
617+
618+
switch (type) {
619+
case AARCH64_INSN_MEM_ORDER_NONE:
620+
order = 0;
621+
break;
622+
case AARCH64_INSN_MEM_ORDER_ACQ:
623+
order = 2;
624+
break;
625+
case AARCH64_INSN_MEM_ORDER_REL:
626+
order = 1;
627+
break;
628+
case AARCH64_INSN_MEM_ORDER_ACQREL:
629+
order = 3;
630+
break;
631+
default:
632+
pr_err("%s: unknown mem order %d\n", __func__, type);
633+
return AARCH64_BREAK_FAULT;
634+
}
635+
636+
insn &= ~GENMASK(23, 22);
637+
insn |= order << 22;
638+
639+
return insn;
640+
}
641+
642+
u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,
643+
enum aarch64_insn_register address,
644+
enum aarch64_insn_register value,
645+
enum aarch64_insn_size_type size,
646+
enum aarch64_insn_mem_atomic_op op,
647+
enum aarch64_insn_mem_order_type order)
648+
{
649+
u32 insn;
650+
651+
switch (op) {
652+
case AARCH64_INSN_MEM_ATOMIC_ADD:
653+
insn = aarch64_insn_get_ldadd_value();
654+
break;
655+
case AARCH64_INSN_MEM_ATOMIC_CLR:
656+
insn = aarch64_insn_get_ldclr_value();
657+
break;
658+
case AARCH64_INSN_MEM_ATOMIC_EOR:
659+
insn = aarch64_insn_get_ldeor_value();
660+
break;
661+
case AARCH64_INSN_MEM_ATOMIC_SET:
662+
insn = aarch64_insn_get_ldset_value();
663+
break;
664+
case AARCH64_INSN_MEM_ATOMIC_SWP:
665+
insn = aarch64_insn_get_swp_value();
666+
break;
667+
default:
668+
pr_err("%s: unimplemented mem atomic op %d\n", __func__, op);
669+
return AARCH64_BREAK_FAULT;
670+
}
612671

613672
switch (size) {
614673
case AARCH64_INSN_SIZE_32:
@@ -621,6 +680,8 @@ u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result,
621680

622681
insn = aarch64_insn_encode_ldst_size(size, insn);
623682

683+
insn = aarch64_insn_encode_ldst_order(order, insn);
684+
624685
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
625686
result);
626687

@@ -631,17 +692,68 @@ u32 aarch64_insn_gen_ldadd(enum aarch64_insn_register result,
631692
value);
632693
}
633694

634-
u32 aarch64_insn_gen_stadd(enum aarch64_insn_register address,
635-
enum aarch64_insn_register value,
636-
enum aarch64_insn_size_type size)
695+
static u32 aarch64_insn_encode_cas_order(enum aarch64_insn_mem_order_type type,
696+
u32 insn)
637697
{
638-
/*
639-
* STADD is simply encoded as an alias for LDADD with XZR as
640-
* the destination register.
641-
*/
642-
return aarch64_insn_gen_ldadd(AARCH64_INSN_REG_ZR, address,
643-
value, size);
698+
u32 order;
699+
700+
switch (type) {
701+
case AARCH64_INSN_MEM_ORDER_NONE:
702+
order = 0;
703+
break;
704+
case AARCH64_INSN_MEM_ORDER_ACQ:
705+
order = BIT(22);
706+
break;
707+
case AARCH64_INSN_MEM_ORDER_REL:
708+
order = BIT(15);
709+
break;
710+
case AARCH64_INSN_MEM_ORDER_ACQREL:
711+
order = BIT(15) | BIT(22);
712+
break;
713+
default:
714+
pr_err("%s: unknown mem order %d\n", __func__, type);
715+
return AARCH64_BREAK_FAULT;
716+
}
717+
718+
insn &= ~(BIT(15) | BIT(22));
719+
insn |= order;
720+
721+
return insn;
722+
}
723+
724+
u32 aarch64_insn_gen_cas(enum aarch64_insn_register result,
725+
enum aarch64_insn_register address,
726+
enum aarch64_insn_register value,
727+
enum aarch64_insn_size_type size,
728+
enum aarch64_insn_mem_order_type order)
729+
{
730+
u32 insn;
731+
732+
switch (size) {
733+
case AARCH64_INSN_SIZE_32:
734+
case AARCH64_INSN_SIZE_64:
735+
break;
736+
default:
737+
pr_err("%s: unimplemented size encoding %d\n", __func__, size);
738+
return AARCH64_BREAK_FAULT;
739+
}
740+
741+
insn = aarch64_insn_get_cas_value();
742+
743+
insn = aarch64_insn_encode_ldst_size(size, insn);
744+
745+
insn = aarch64_insn_encode_cas_order(order, insn);
746+
747+
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
748+
result);
749+
750+
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
751+
address);
752+
753+
return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
754+
value);
644755
}
756+
#endif
645757

646758
static u32 aarch64_insn_encode_prfm_imm(enum aarch64_insn_prfm_type type,
647759
enum aarch64_insn_prfm_target target,
@@ -1379,7 +1491,7 @@ static u32 aarch64_encode_immediate(u64 imm,
13791491
* Compute the rotation to get a continuous set of
13801492
* ones, with the first bit set at position 0
13811493
*/
1382-
ror = fls(~imm);
1494+
ror = fls64(~imm);
13831495
}
13841496

13851497
/*
@@ -1456,3 +1568,48 @@ u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant,
14561568
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn);
14571569
return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, Rm);
14581570
}
1571+
1572+
u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type)
1573+
{
1574+
u32 opt;
1575+
u32 insn;
1576+
1577+
switch (type) {
1578+
case AARCH64_INSN_MB_SY:
1579+
opt = 0xf;
1580+
break;
1581+
case AARCH64_INSN_MB_ST:
1582+
opt = 0xe;
1583+
break;
1584+
case AARCH64_INSN_MB_LD:
1585+
opt = 0xd;
1586+
break;
1587+
case AARCH64_INSN_MB_ISH:
1588+
opt = 0xb;
1589+
break;
1590+
case AARCH64_INSN_MB_ISHST:
1591+
opt = 0xa;
1592+
break;
1593+
case AARCH64_INSN_MB_ISHLD:
1594+
opt = 0x9;
1595+
break;
1596+
case AARCH64_INSN_MB_NSH:
1597+
opt = 0x7;
1598+
break;
1599+
case AARCH64_INSN_MB_NSHST:
1600+
opt = 0x6;
1601+
break;
1602+
case AARCH64_INSN_MB_NSHLD:
1603+
opt = 0x5;
1604+
break;
1605+
default:
1606+
pr_err("%s: unknown dmb type %d\n", __func__, type);
1607+
return AARCH64_BREAK_FAULT;
1608+
}
1609+
1610+
insn = aarch64_insn_get_dmb_value();
1611+
insn &= ~GENMASK(11, 8);
1612+
insn |= (opt << 8);
1613+
1614+
return insn;
1615+
}

0 commit comments

Comments
 (0)