Skip to content

Commit 692d130

Browse files
peilin-yepeilin-ye
committed
[BPF] Add load-acquire and store-release instructions under -mcpu=v5
As discussed in [1], introduce BPF instructions with load-acquire and store-release semantics under -mcpu=v5. A "load_acquire" is a BPF_LDX instruction with a new mode modifier, BPF_MEMACQ ("acquiring atomic load"). Similarly, a "store_release" is a BPF_STX instruction with another new mode modifier, BPF_MEMREL ("releasing atomic store"). BPF_MEMACQ and BPF_MEMREL share the same numeric value, 0x7 (or 0b111). For example: long foo(long *ptr) { return __atomic_load_n(ptr, __ATOMIC_ACQUIRE); } foo() can be compiled to: f9 10 00 00 00 00 00 00 r0 = load_acquire((u64 *)(r1 + 0x0)) 95 00 00 00 00 00 00 00 exit Opcode 0xf9, or 0b11111001, can be decoded as: 0b 111 11 001 BPF_MEMACQ BPF_DW BPF_LDX Similarly: void bar(short *ptr, short val) { __atomic_store_n(ptr, val, __ATOMIC_RELEASE); } bar() can be compiled to: eb 21 00 00 00 00 00 00 store_release((u16 *)(r1 + 0x0), w2) 95 00 00 00 00 00 00 00 exit Opcode 0xeb, or 0b11101011, can be decoded as: 0b 111 01 011 BPF_MEMREL BPF_H BPF_STX Inline assembly is also supported. For example: asm volatile("%0 = load_acquire((u64 *)(%1 + 0x0))" : "=r"(ret) : "r"(ptr) : "memory"); Let 'llvm-objdump -d' use -mcpu=v5 by default, just like commit 0395868 ("[BPF] Make llvm-objdump disasm default cpu v4 (#102166)"). Add two macros, __BPF_FEATURE_LOAD_ACQUIRE and __BPF_FEATURE_STORE_RELEASE, to let developers detect these new features in source code. They can also be disabled using two new llc options, -disable-load-acquire and -disable-store-release, respectively. Also use ACQUIRE or RELEASE if user requested weaker memory orders (RELAXED or CONSUME) until we actually support them. Requesting a stronger memory order (i.e. SEQ_CST) will cause an error. [1] https://lore.kernel.org/all/[email protected]/
1 parent 5b64851 commit 692d130

File tree

14 files changed

+257
-10
lines changed

14 files changed

+257
-10
lines changed

‎clang/lib/Basic/Targets/BPF.cpp

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,10 +67,15 @@ void BPFTargetInfo::getTargetDefines(const LangOptions &Opts,
6767
Builder.defineMacro("__BPF_FEATURE_GOTOL");
6868
Builder.defineMacro("__BPF_FEATURE_ST");
6969
}
70+
71+
if (CpuVerNum >= 5) {
72+
Builder.defineMacro("__BPF_FEATURE_LOAD_ACQUIRE");
73+
Builder.defineMacro("__BPF_FEATURE_STORE_RELEASE");
74+
}
7075
}
7176

72-
static constexpr llvm::StringLiteral ValidCPUNames[] = {"generic", "v1", "v2",
73-
"v3", "v4", "probe"};
77+
static constexpr llvm::StringLiteral ValidCPUNames[] = {
78+
"generic", "v1", "v2", "v3", "v4", "v5", "probe"};
7479

7580
bool BPFTargetInfo::isValidCPUName(StringRef Name) const {
7681
return llvm::is_contained(ValidCPUNames, Name);

‎clang/lib/Basic/Targets/BPF.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ class LLVM_LIBRARY_VISIBILITY BPFTargetInfo : public TargetInfo {
106106
void fillValidCPUList(SmallVectorImpl<StringRef> &Values) const override;
107107

108108
bool setCPU(const std::string &Name) override {
109-
if (Name == "v3" || Name == "v4") {
109+
if (Name == "v3" || Name == "v4" || Name == "v5") {
110110
HasAlu32 = true;
111111
}
112112

‎clang/test/Misc/target-invalid-cpu-note/bpf.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
// CHECK-SAME: {{^}}, v2
1111
// CHECK-SAME: {{^}}, v3
1212
// CHECK-SAME: {{^}}, v4
13+
// CHECK-SAME: {{^}}, v5
1314
// CHECK-SAME: {{^}}, probe
1415
// CHECK-SAME: {{$}}
1516

‎llvm/lib/Object/ELFObjectFile.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -442,7 +442,7 @@ std::optional<StringRef> ELFObjectFileBase::tryGetCPUName() const {
442442
case ELF::EM_PPC64:
443443
return StringRef("future");
444444
case ELF::EM_BPF:
445-
return StringRef("v4");
445+
return StringRef("v5");
446446
default:
447447
return std::nullopt;
448448
}

‎llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,7 @@ struct BPFOperand : public MCParsedAsmOperand {
237237
.Case("exit", true)
238238
.Case("lock", true)
239239
.Case("ld_pseudo", true)
240+
.Case("store_release", true)
240241
.Default(false);
241242
}
242243

@@ -273,6 +274,7 @@ struct BPFOperand : public MCParsedAsmOperand {
273274
.Case("cmpxchg_64", true)
274275
.Case("cmpxchg32_32", true)
275276
.Case("addr_space_cast", true)
277+
.Case("load_acquire", true)
276278
.Default(false);
277279
}
278280
};

‎llvm/lib/Target/BPF/BPF.td

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ def : Proc<"v1", []>;
3232
def : Proc<"v2", []>;
3333
def : Proc<"v3", [ALU32]>;
3434
def : Proc<"v4", [ALU32]>;
35+
def : Proc<"v5", [ALU32]>;
3536
def : Proc<"probe", []>;
3637

3738
def BPFInstPrinter : AsmWriter {

‎llvm/lib/Target/BPF/BPFInstrFormats.td

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,8 @@ def BPF_IND : BPFModeModifer<0x2>;
9494
def BPF_MEM : BPFModeModifer<0x3>;
9595
def BPF_MEMSX : BPFModeModifer<0x4>;
9696
def BPF_ATOMIC : BPFModeModifer<0x6>;
97+
def BPF_MEMACQ : BPFModeModifer<0x7>;
98+
def BPF_MEMREL : BPFModeModifer<0x7>;
9799

98100
class BPFAtomicFlag<bits<4> val> {
99101
bits<4> Value = val;

‎llvm/lib/Target/BPF/BPFInstrInfo.td

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ def BPFHasSdivSmod : Predicate<"Subtarget->hasSdivSmod()">;
6060
def BPFNoMovsx : Predicate<"!Subtarget->hasMovsx()">;
6161
def BPFNoBswap : Predicate<"!Subtarget->hasBswap()">;
6262
def BPFHasStoreImm : Predicate<"Subtarget->hasStoreImm()">;
63+
def BPFHasLoadAcquire : Predicate<"Subtarget->hasLoadAcquire()">;
64+
def BPFHasStoreRelease : Predicate<"Subtarget->hasStoreRelease()">;
6365

6466
class ImmediateAsmOperand<string name> : AsmOperandClass {
6567
let Name = name;
@@ -514,13 +516,38 @@ class STORE<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string AsmString, list
514516
class STOREi64<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode>
515517
: STORE<Opc, BPF_MEM, "*("#OpcodeStr#" *)($addr) = $src", [(OpNode GPR:$src, ADDRri:$addr)]>;
516518

519+
class STORE_RELEASEi64<BPFWidthModifer Opc, string OpcodeStr>
520+
: STORE<Opc, BPF_MEMREL, "store_release(("#OpcodeStr#" *)($addr), $src)", []>;
521+
517522
let Predicates = [BPFNoALU32] in {
518523
def STW : STOREi64<BPF_W, "u32", truncstorei32>;
519524
def STH : STOREi64<BPF_H, "u16", truncstorei16>;
520525
def STB : STOREi64<BPF_B, "u8", truncstorei8>;
521526
}
522527
def STD : STOREi64<BPF_DW, "u64", store>;
523528

529+
class relaxed_store<PatFrag base>
530+
: PatFrag<(ops node:$val, node:$ptr), (base node:$val, node:$ptr)> {
531+
let IsAtomic = 1;
532+
let IsAtomicOrderingReleaseOrStronger = 0;
533+
}
534+
535+
class releasing_store<PatFrag base>
536+
: PatFrag<(ops node:$val, node:$ptr), (base node:$val, node:$ptr)> {
537+
let IsAtomic = 1;
538+
let IsAtomicOrderingRelease = 1;
539+
}
540+
541+
let Predicates = [BPFHasStoreRelease] in {
542+
def STDREL : STORE_RELEASEi64<BPF_DW, "u64">;
543+
544+
foreach P = [[relaxed_store<atomic_store_64>, STDREL],
545+
[releasing_store<atomic_store_64>, STDREL],
546+
] in {
547+
def : Pat<(P[0] GPR:$val, ADDRri:$addr), (P[1] GPR:$val, ADDRri:$addr)>;
548+
}
549+
}
550+
524551
class STORE_imm<BPFWidthModifer SizeOp,
525552
string OpcodeStr, dag Pattern>
526553
: TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value,
@@ -584,6 +611,9 @@ class LOADi64<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, Pa
584611
: LOAD<SizeOp, ModOp, "$dst = *("#OpcodeStr#" *)($addr)",
585612
[(set i64:$dst, (OpNode ADDRri:$addr))]>;
586613

614+
class LOAD_ACQUIREi64<BPFWidthModifer SizeOp, string OpcodeStr>
615+
: LOAD<SizeOp, BPF_MEMACQ, "$dst = load_acquire(("#OpcodeStr#" *)($addr))", []>;
616+
587617
let isCodeGenOnly = 1 in {
588618
class CORE_LD<RegisterClass RegClass, string Sz>
589619
: TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value,
@@ -621,6 +651,28 @@ let Predicates = [BPFHasLdsx] in {
621651

622652
def LDD : LOADi64<BPF_DW, BPF_MEM, "u64", load>;
623653

654+
class relaxed_load<PatFrags base>
655+
: PatFrag<(ops node:$ptr), (base node:$ptr)> {
656+
let IsAtomic = 1;
657+
let IsAtomicOrderingAcquireOrStronger = 0;
658+
}
659+
660+
class acquiring_load<PatFrags base>
661+
: PatFrag<(ops node:$ptr), (base node:$ptr)> {
662+
let IsAtomic = 1;
663+
let IsAtomicOrderingAcquire = 1;
664+
}
665+
666+
let Predicates = [BPFHasLoadAcquire] in {
667+
def LDDACQ : LOAD_ACQUIREi64<BPF_DW, "u64">;
668+
669+
foreach P = [[relaxed_load<atomic_load_64>, LDDACQ],
670+
[acquiring_load<atomic_load_64>, LDDACQ],
671+
] in {
672+
def : Pat<(P[0] ADDRri:$addr), (P[1] ADDRri:$addr)>;
673+
}
674+
}
675+
624676
class BRANCH<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
625677
: TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
626678
(outs),
@@ -1086,10 +1138,19 @@ class STOREi32<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode>
10861138
: STORE32<Opc, BPF_MEM, "*("#OpcodeStr#" *)($addr) = $src",
10871139
[(OpNode GPR32:$src, ADDRri:$addr)]>;
10881140

1141+
class STORE_RELEASEi32<BPFWidthModifer Opc, string OpcodeStr>
1142+
: STORE32<Opc, BPF_MEMREL, "store_release(("#OpcodeStr#" *)($addr), $src)", []>;
1143+
10891144
let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
10901145
def STW32 : STOREi32<BPF_W, "u32", store>;
10911146
def STH32 : STOREi32<BPF_H, "u16", truncstorei16>;
10921147
def STB32 : STOREi32<BPF_B, "u8", truncstorei8>;
1148+
1149+
let Predicates = [BPFHasStoreRelease] in {
1150+
def STWREL32 : STORE_RELEASEi32<BPF_W, "u32">;
1151+
def STHREL32 : STORE_RELEASEi32<BPF_H, "u16">;
1152+
def STBREL32 : STORE_RELEASEi32<BPF_B, "u8">;
1153+
}
10931154
}
10941155

10951156
class LOAD32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string AsmString, list<dag> Pattern>
@@ -1110,10 +1171,19 @@ class LOADi32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, Pa
11101171
: LOAD32<SizeOp, ModOp, "$dst = *("#OpcodeStr#" *)($addr)",
11111172
[(set i32:$dst, (OpNode ADDRri:$addr))]>;
11121173

1174+
class LOAD_ACQUIREi32<BPFWidthModifer SizeOp, string OpcodeStr>
1175+
: LOAD32<SizeOp, BPF_MEMACQ, "$dst = load_acquire(("#OpcodeStr#" *)($addr))", []>;
1176+
11131177
let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in {
11141178
def LDW32 : LOADi32<BPF_W, BPF_MEM, "u32", load>;
11151179
def LDH32 : LOADi32<BPF_H, BPF_MEM, "u16", zextloadi16>;
11161180
def LDB32 : LOADi32<BPF_B, BPF_MEM, "u8", zextloadi8>;
1181+
1182+
let Predicates = [BPFHasLoadAcquire] in {
1183+
def LDWACQ32 : LOAD_ACQUIREi32<BPF_W, "u32">;
1184+
def LDHACQ32 : LOAD_ACQUIREi32<BPF_H, "u16">;
1185+
def LDBACQ32 : LOAD_ACQUIREi32<BPF_B, "u8">;
1186+
}
11171187
}
11181188

11191189
let Predicates = [BPFHasALU32] in {
@@ -1143,6 +1213,30 @@ let Predicates = [BPFHasALU32] in {
11431213
(SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>;
11441214
def : Pat<(i64 (extloadi32 ADDRri:$src)),
11451215
(SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>;
1216+
1217+
let Predicates = [BPFHasLoadAcquire] in {
1218+
foreach P = [[relaxed_load<atomic_load_32>, LDWACQ32],
1219+
[relaxed_load<atomic_load_az_16>, LDHACQ32],
1220+
[relaxed_load<atomic_load_az_8>, LDBACQ32],
1221+
[acquiring_load<atomic_load_32>, LDWACQ32],
1222+
[acquiring_load<atomic_load_az_16>, LDHACQ32],
1223+
[acquiring_load<atomic_load_az_8>, LDBACQ32],
1224+
] in {
1225+
def : Pat<(P[0] ADDRri:$addr), (P[1] ADDRri:$addr)>;
1226+
}
1227+
}
1228+
1229+
let Predicates = [BPFHasStoreRelease] in {
1230+
foreach P = [[relaxed_store<atomic_store_32>, STWREL32],
1231+
[relaxed_store<atomic_store_16>, STHREL32],
1232+
[relaxed_store<atomic_store_8>, STBREL32],
1233+
[releasing_store<atomic_store_32>, STWREL32],
1234+
[releasing_store<atomic_store_16>, STHREL32],
1235+
[releasing_store<atomic_store_8>, STBREL32],
1236+
] in {
1237+
def : Pat<(P[0] GPR32:$val, ADDRri:$addr), (P[1] GPR32:$val, ADDRri:$addr)>;
1238+
}
1239+
}
11461240
}
11471241

11481242
let usesCustomInserter = 1, isCodeGenOnly = 1 in {

‎llvm/lib/Target/BPF/BPFMISimplifyPatchable.cpp

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -100,21 +100,25 @@ static bool isST(unsigned Opcode) {
100100
}
101101

102102
static bool isSTX32(unsigned Opcode) {
103-
return Opcode == BPF::STB32 || Opcode == BPF::STH32 || Opcode == BPF::STW32;
103+
return Opcode == BPF::STB32 || Opcode == BPF::STH32 || Opcode == BPF::STW32 ||
104+
Opcode == BPF::STBREL32 || Opcode == BPF::STHREL32 ||
105+
Opcode == BPF::STWREL32;
104106
}
105107

106108
static bool isSTX64(unsigned Opcode) {
107109
return Opcode == BPF::STB || Opcode == BPF::STH || Opcode == BPF::STW ||
108-
Opcode == BPF::STD;
110+
Opcode == BPF::STD || Opcode == BPF::STDREL;
109111
}
110112

111113
static bool isLDX32(unsigned Opcode) {
112-
return Opcode == BPF::LDB32 || Opcode == BPF::LDH32 || Opcode == BPF::LDW32;
114+
return Opcode == BPF::LDB32 || Opcode == BPF::LDH32 || Opcode == BPF::LDW32 ||
115+
Opcode == BPF::LDBACQ32 || Opcode == BPF::LDHACQ32 ||
116+
Opcode == BPF::LDWACQ32;
113117
}
114118

115119
static bool isLDX64(unsigned Opcode) {
116120
return Opcode == BPF::LDB || Opcode == BPF::LDH || Opcode == BPF::LDW ||
117-
Opcode == BPF::LDD;
121+
Opcode == BPF::LDD || Opcode == BPF::LDDACQ;
118122
}
119123

120124
static bool isLDSX(unsigned Opcode) {

‎llvm/lib/Target/BPF/BPFSubtarget.cpp

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@ static cl::opt<bool> Disable_gotol("disable-gotol", cl::Hidden, cl::init(false),
4040
static cl::opt<bool>
4141
Disable_StoreImm("disable-storeimm", cl::Hidden, cl::init(false),
4242
cl::desc("Disable BPF_ST (immediate store) insn"));
43+
static cl::opt<bool>
44+
Disable_load_acquire("disable-load-acquire", cl::Hidden, cl::init(false),
45+
cl::desc("Disable load-acquire insns"));
46+
static cl::opt<bool>
47+
Disable_store_release("disable-store-release", cl::Hidden, cl::init(false),
48+
cl::desc("Disable store-release insns"));
4349

4450
void BPFSubtarget::anchor() {}
4551

@@ -62,6 +68,8 @@ void BPFSubtarget::initializeEnvironment() {
6268
HasSdivSmod = false;
6369
HasGotol = false;
6470
HasStoreImm = false;
71+
HasLoadAcquire = false;
72+
HasStoreRelease = false;
6573
}
6674

6775
void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
@@ -89,6 +97,11 @@ void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
8997
HasGotol = !Disable_gotol;
9098
HasStoreImm = !Disable_StoreImm;
9199
}
100+
101+
if (CpuVerNum >= 5) {
102+
HasLoadAcquire = !Disable_load_acquire;
103+
HasStoreRelease = !Disable_store_release;
104+
}
92105
}
93106

94107
BPFSubtarget::BPFSubtarget(const Triple &TT, const std::string &CPU,

‎llvm/lib/Target/BPF/BPFSubtarget.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ class BPFSubtarget : public BPFGenSubtargetInfo {
6666
// whether cpu v4 insns are enabled.
6767
bool HasLdsx, HasMovsx, HasBswap, HasSdivSmod, HasGotol, HasStoreImm;
6868

69+
// whether cpu v5 insns are enabled.
70+
bool HasLoadAcquire, HasStoreRelease;
71+
6972
std::unique_ptr<CallLowering> CallLoweringInfo;
7073
std::unique_ptr<InstructionSelector> InstSelector;
7174
std::unique_ptr<LegalizerInfo> Legalizer;
@@ -92,6 +95,8 @@ class BPFSubtarget : public BPFGenSubtargetInfo {
9295
bool hasSdivSmod() const { return HasSdivSmod; }
9396
bool hasGotol() const { return HasGotol; }
9497
bool hasStoreImm() const { return HasStoreImm; }
98+
bool hasLoadAcquire() const { return HasLoadAcquire; }
99+
bool hasStoreRelease() const { return HasStoreRelease; }
95100

96101
bool isLittleEndian() const { return IsLittleEndian; }
97102

‎llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,9 @@ class BPFDisassembler : public MCDisassembler {
5858
BPF_IND = 0x2,
5959
BPF_MEM = 0x3,
6060
BPF_MEMSX = 0x4,
61-
BPF_ATOMIC = 0x6
61+
BPF_ATOMIC = 0x6,
62+
BPF_MEMACQ = 0x7,
63+
BPF_MEMREL = 0x7
6264
};
6365

6466
BPFDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx)
@@ -177,7 +179,8 @@ DecodeStatus BPFDisassembler::getInstruction(MCInst &Instr, uint64_t &Size,
177179
uint8_t InstMode = getInstMode(Insn);
178180
if ((InstClass == BPF_LDX || InstClass == BPF_STX) &&
179181
getInstSize(Insn) != BPF_DW &&
180-
(InstMode == BPF_MEM || InstMode == BPF_ATOMIC) &&
182+
(InstMode == BPF_MEM || InstMode == BPF_ATOMIC ||
183+
InstMode == BPF_MEMACQ /* or BPF_MEMREL */) &&
181184
STI.hasFeature(BPF::ALU32))
182185
Result = decodeInstruction(DecoderTableBPFALU3264, Instr, Insn, Address,
183186
this, STI);

0 commit comments

Comments
 (0)