@@ -334,9 +334,9 @@ static Value downcastSelectAndUpcast(OpBuilder &builder, Location loc,
334
334
// /
335
335
// / Result:
336
336
// / linearizedMemref = |2|2|3|3| : <4xi2> (<1xi8>)
337
- static void atomicStore (OpBuilder &builder, Location loc,
338
- MemRefValue linearizedMemref, Value storeIdx,
339
- VectorValue valueToStore, Value mask) {
337
+ static void atomicRMWStore (OpBuilder &builder, Location loc,
338
+ MemRefValue linearizedMemref, Value storeIdx,
339
+ VectorValue valueToStore, Value mask) {
340
340
assert (valueToStore.getType ().getRank () == 1 && " expected 1-D vector" );
341
341
342
342
// Create an atomic load-modify-write region using
@@ -364,10 +364,11 @@ static void atomicStore(OpBuilder &builder, Location loc,
364
364
}
365
365
366
366
// / Generate a non-atomic read-modify-write sequence for subbyte storing.
367
- // / It has similar logic to `atomicStore`, but without atomicity.
368
- static void rmwStore (OpBuilder &builder, Location loc,
369
- MemRefValue linearizedMemref, Value linearizedIndex,
370
- VectorValue valueToStore, Value mask) {
367
+ // / It has similar logic to `atomicRMWStore`, but without atomicity.
368
+ static void nonAtomicRMWStore (OpBuilder &builder, Location loc,
369
+ MemRefValue linearizedMemref,
370
+ Value linearizedIndex, VectorValue valueToStore,
371
+ Value mask) {
371
372
assert (valueToStore.getType ().getRank () == 1 && " expected 1-D vector" );
372
373
373
374
auto oneElemVecType =
@@ -580,8 +581,10 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
580
581
extractSliceIntoByte (rewriter, loc, valueToStore, 0 ,
581
582
frontSubWidthStoreElem, *foldedNumFrontPadElems);
582
583
583
- subEmulatedWidthStore (rewriter, loc, memrefBase, currentDestIndex,
584
- cast<VectorValue>(value), frontMask.getResult ());
584
+ auto storeFunc = useAtomicWrites_ ? atomicRMWStore : nonAtomicRMWStore;
585
+
586
+ storeFunc (rewriter, loc, memrefBase, currentDestIndex,
587
+ cast<VectorValue>(value), frontMask.getResult ());
585
588
}
586
589
587
590
if (currentSourceIndex >= origElements) {
@@ -645,20 +648,6 @@ struct ConvertVectorStore final : OpConversionPattern<vector::StoreOp> {
645
648
return success ();
646
649
}
647
650
648
- // / Store a subbyte-sized value to memory, with a mask. Depending on the
649
- // / configuration, it could be an atomic store or a non-atomic RMW sequence.
650
- template <typename ... Args>
651
- void subEmulatedWidthStore (Args &&...args) const {
652
- static_assert (
653
- std::is_same_v<decltype (atomicStore), decltype (rmwStore)> &&
654
- " `atomicStore` and `rmwStore` must have same signature, as per "
655
- " the design to keep the code clean, which one to call is "
656
- " determined by the `useAtomicWrites` flag." );
657
- std::function<decltype (atomicStore)> storeFunc =
658
- useAtomicWrites_ ? atomicStore : rmwStore;
659
- storeFunc (std::forward<Args>(args)...);
660
- }
661
-
662
651
private:
663
652
const bool useAtomicWrites_;
664
653
};
0 commit comments