|
| 1 | +//===- VectorToROCDL.cpp - Vector to ROCDL lowering passes ------===// |
| 2 | +// |
| 3 | +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | +// See https://llvm.org/LICENSE.txt for license information. |
| 5 | +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | +// |
| 7 | +//===----------------------------------------------------------------------===// |
| 8 | +// |
| 9 | +// This file implements a pass to generate ROCDLIR operations for higher-level |
| 10 | +// Vector operations. |
| 11 | +// |
| 12 | +//===----------------------------------------------------------------------===// |
| 13 | + |
| 14 | +#include "mlir/Conversion/VectorToROCDL/VectorToROCDL.h" |
| 15 | + |
| 16 | +#include "../PassDetail.h" |
| 17 | +#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h" |
| 18 | +#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h" |
| 19 | +#include "mlir/Dialect/GPU/GPUDialect.h" |
| 20 | +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" |
| 21 | +#include "mlir/Dialect/LLVMIR/ROCDLDialect.h" |
| 22 | +#include "mlir/Dialect/StandardOps/IR/Ops.h" |
| 23 | +#include "mlir/Dialect/Vector/VectorOps.h" |
| 24 | +#include "mlir/Pass/Pass.h" |
| 25 | +#include "mlir/Transforms/DialectConversion.h" |
| 26 | + |
| 27 | +using namespace mlir; |
| 28 | +using namespace mlir::vector; |
| 29 | + |
| 30 | +static TransferReadOpOperandAdaptor |
| 31 | +getTransferOpAdapter(TransferReadOp xferOp, ArrayRef<Value> operands) { |
| 32 | + return OperandAdaptor<TransferReadOp>(operands); |
| 33 | +} |
| 34 | + |
| 35 | +static TransferWriteOpOperandAdaptor |
| 36 | +getTransferOpAdapter(TransferWriteOp xferOp, ArrayRef<Value> operands) { |
| 37 | + return OperandAdaptor<TransferWriteOp>(operands); |
| 38 | +} |
| 39 | + |
| 40 | +static LogicalResult replaceTransferOpWithMubuf( |
| 41 | + ConversionPatternRewriter &rewriter, ArrayRef<Value> operands, |
| 42 | + LLVMTypeConverter &typeConverter, Location loc, TransferReadOp xferOp, |
| 43 | + LLVM::LLVMType &vecTy, Value &dwordConfig, Value &vindex, |
| 44 | + Value &offsetSizeInBytes, Value &glc, Value &slc) { |
| 45 | + rewriter.replaceOpWithNewOp<ROCDL::MubufLoadOp>( |
| 46 | + xferOp, vecTy, dwordConfig, vindex, offsetSizeInBytes, glc, slc); |
| 47 | + return success(); |
| 48 | +} |
| 49 | + |
| 50 | +static LogicalResult replaceTransferOpWithMubuf( |
| 51 | + ConversionPatternRewriter &rewriter, ArrayRef<Value> operands, |
| 52 | + LLVMTypeConverter &typeConverter, Location loc, TransferWriteOp xferOp, |
| 53 | + LLVM::LLVMType &vecTy, Value &dwordConfig, Value &vindex, |
| 54 | + Value &offsetSizeInBytes, Value &glc, Value &slc) { |
| 55 | + auto adaptor = TransferWriteOpOperandAdaptor(operands); |
| 56 | + rewriter.replaceOpWithNewOp<ROCDL::MubufStoreOp>(xferOp, adaptor.vector(), |
| 57 | + dwordConfig, vindex, |
| 58 | + offsetSizeInBytes, glc, slc); |
| 59 | + return success(); |
| 60 | +} |
| 61 | + |
| 62 | +namespace { |
| 63 | +/// Conversion pattern that converts a 1-D vector transfer read/write. |
| 64 | +/// Note that this conversion pass only converts vector x2 or x4 f32 |
| 65 | +/// types. For unsupported cases, they will fall back to the vector to |
| 66 | +/// llvm conversion pattern. |
| 67 | +template <typename ConcreteOp> |
| 68 | +class VectorTransferConversion : public ConvertToLLVMPattern { |
| 69 | +public: |
| 70 | + explicit VectorTransferConversion(MLIRContext *context, |
| 71 | + LLVMTypeConverter &typeConv) |
| 72 | + : ConvertToLLVMPattern(ConcreteOp::getOperationName(), context, |
| 73 | + typeConv) {} |
| 74 | + |
| 75 | + LogicalResult |
| 76 | + matchAndRewrite(Operation *op, ArrayRef<Value> operands, |
| 77 | + ConversionPatternRewriter &rewriter) const override { |
| 78 | + auto xferOp = cast<ConcreteOp>(op); |
| 79 | + auto adaptor = getTransferOpAdapter(xferOp, operands); |
| 80 | + |
| 81 | + if (xferOp.getVectorType().getRank() > 1 || |
| 82 | + llvm::size(xferOp.indices()) == 0) |
| 83 | + return failure(); |
| 84 | + |
| 85 | + if (!AffineMap::isMinorIdentity(xferOp.permutation_map())) |
| 86 | + return failure(); |
| 87 | + |
| 88 | + // Have it handled in vector->llvm conversion pass. |
| 89 | + if (!xferOp.isMaskedDim(0)) |
| 90 | + return failure(); |
| 91 | + |
| 92 | + auto toLLVMTy = [&](Type t) { return typeConverter.convertType(t); }; |
| 93 | + LLVM::LLVMType vecTy = |
| 94 | + toLLVMTy(xferOp.getVectorType()).template cast<LLVM::LLVMType>(); |
| 95 | + unsigned vecWidth = vecTy.getVectorNumElements(); |
| 96 | + Location loc = op->getLoc(); |
| 97 | + |
| 98 | + // The backend result vector scalarization have trouble scalarize |
| 99 | + // <1 x ty> result, exclude the x1 width from the lowering. |
| 100 | + if (vecWidth != 2 && vecWidth != 4) |
| 101 | + return failure(); |
| 102 | + |
| 103 | + // Obtain dataPtr and elementType from the memref. |
| 104 | + MemRefType memRefType = xferOp.getMemRefType(); |
| 105 | + // MUBUF instruction operate only on addresspace 0(unified) or 1(global) |
| 106 | + // In case of 3(LDS): fall back to vector->llvm pass |
| 107 | + // In case of 5(VGPR): wrong |
| 108 | + if ((memRefType.getMemorySpace() != 0) && |
| 109 | + (memRefType.getMemorySpace() != 1)) |
| 110 | + return failure(); |
| 111 | + |
| 112 | + // Note that the dataPtr starts at the offset address specified by |
| 113 | + // indices, so no need to calculat offset size in bytes again in |
| 114 | + // the MUBUF instruction. |
| 115 | + Value dataPtr = getDataPtr(loc, memRefType, adaptor.memref(), |
| 116 | + adaptor.indices(), rewriter, getModule()); |
| 117 | + |
| 118 | + // 1. Create and fill a <4 x i32> dwordConfig with: |
| 119 | + // 1st two elements holding the address of dataPtr. |
| 120 | + // 3rd element: -1. |
| 121 | + // 4th element: 0x27000. |
| 122 | + SmallVector<int32_t, 4> constConfigAttr{0, 0, -1, 0x27000}; |
| 123 | + Type i32Ty = rewriter.getIntegerType(32); |
| 124 | + VectorType i32Vecx4 = VectorType::get(4, i32Ty); |
| 125 | + Value constConfig = rewriter.create<LLVM::ConstantOp>( |
| 126 | + loc, toLLVMTy(i32Vecx4), |
| 127 | + DenseElementsAttr::get(i32Vecx4, ArrayRef<int32_t>(constConfigAttr))); |
| 128 | + |
| 129 | + // Treat first two element of <4 x i32> as i64, and save the dataPtr |
| 130 | + // to it. |
| 131 | + Type i64Ty = rewriter.getIntegerType(64); |
| 132 | + Value i64x2Ty = rewriter.create<LLVM::BitcastOp>( |
| 133 | + loc, |
| 134 | + LLVM::LLVMType::getVectorTy( |
| 135 | + toLLVMTy(i64Ty).template cast<LLVM::LLVMType>(), 2), |
| 136 | + constConfig); |
| 137 | + Value dataPtrAsI64 = rewriter.create<LLVM::PtrToIntOp>( |
| 138 | + loc, toLLVMTy(i64Ty).template cast<LLVM::LLVMType>(), dataPtr); |
| 139 | + Value zero = createIndexConstant(rewriter, loc, 0); |
| 140 | + Value dwordConfig = rewriter.create<LLVM::InsertElementOp>( |
| 141 | + loc, |
| 142 | + LLVM::LLVMType::getVectorTy( |
| 143 | + toLLVMTy(i64Ty).template cast<LLVM::LLVMType>(), 2), |
| 144 | + i64x2Ty, dataPtrAsI64, zero); |
| 145 | + dwordConfig = |
| 146 | + rewriter.create<LLVM::BitcastOp>(loc, toLLVMTy(i32Vecx4), dwordConfig); |
| 147 | + |
| 148 | + // 2. Rewrite op as a buffer read or write. |
| 149 | + Value int1False = rewriter.create<LLVM::ConstantOp>( |
| 150 | + loc, toLLVMTy(rewriter.getIntegerType(1)), |
| 151 | + rewriter.getIntegerAttr(rewriter.getIntegerType(1), 0)); |
| 152 | + Value int32Zero = rewriter.create<LLVM::ConstantOp>( |
| 153 | + loc, toLLVMTy(i32Ty), |
| 154 | + rewriter.getIntegerAttr(rewriter.getIntegerType(32), 0)); |
| 155 | + return replaceTransferOpWithMubuf(rewriter, operands, typeConverter, loc, |
| 156 | + xferOp, vecTy, dwordConfig, int32Zero, |
| 157 | + int32Zero, int1False, int1False); |
| 158 | + } |
| 159 | +}; |
| 160 | +} // end anonymous namespace |
| 161 | + |
| 162 | +void mlir::populateVectorToROCDLConversionPatterns( |
| 163 | + LLVMTypeConverter &converter, OwningRewritePatternList &patterns) { |
| 164 | + MLIRContext *ctx = converter.getDialect()->getContext(); |
| 165 | + patterns.insert<VectorTransferConversion<TransferReadOp>, |
| 166 | + VectorTransferConversion<TransferWriteOp>>(ctx, converter); |
| 167 | +} |
| 168 | + |
| 169 | +namespace { |
| 170 | +struct LowerVectorToROCDLPass |
| 171 | + : public ConvertVectorToROCDLBase<LowerVectorToROCDLPass> { |
| 172 | + void runOnOperation() override; |
| 173 | +}; |
| 174 | +} // namespace |
| 175 | + |
| 176 | +void LowerVectorToROCDLPass::runOnOperation() { |
| 177 | + LLVMTypeConverter converter(&getContext()); |
| 178 | + OwningRewritePatternList patterns; |
| 179 | + |
| 180 | + populateVectorToROCDLConversionPatterns(converter, patterns); |
| 181 | + populateStdToLLVMConversionPatterns(converter, patterns); |
| 182 | + |
| 183 | + LLVMConversionTarget target(getContext()); |
| 184 | + target.addLegalDialect<ROCDL::ROCDLDialect>(); |
| 185 | + |
| 186 | + if (failed(applyPartialConversion(getOperation(), target, patterns, |
| 187 | + &converter))) { |
| 188 | + signalPassFailure(); |
| 189 | + } |
| 190 | +} |
| 191 | + |
| 192 | +std::unique_ptr<OperationPass<ModuleOp>> |
| 193 | +mlir::createConvertVectorToROCDLPass() { |
| 194 | + return std::make_unique<LowerVectorToROCDLPass>(); |
| 195 | +} |
0 commit comments