LSC 16c32b atomic fix

.

Author: KorovinVlad

IGC/VectorCompiler/lib/GenXCodeGen/GenXLoadStoreLowering.cpp

@@ -1181,10 +1181,19 @@ Instruction *GenXLoadStoreLowering::createLSCAtomicImpl(
                   ElementSize == LSC_DATA_SIZE_64b);
 
   if (ElementSize == LSC_DATA_SIZE_16c32b) {
-    Src0 = Builder.CreateBitCast(Src0, Builder.getInt16Ty());
-    Src0 = Builder.CreateZExt(Src0, Builder.getInt32Ty());
-    Src1 = Builder.CreateBitCast(Src1, Builder.getInt16Ty());
-    Src1 = Builder.CreateZExt(Src1, Builder.getInt32Ty());
+    // We must preserve undef operands but
+    // zext/sext casts make them zero.
+    if (!isa<UndefValue>(Src0)) {
+      Src0 = Builder.CreateBitCast(Src0, Builder.getInt16Ty());
+      Src0 = Builder.CreateZExt(Src0, Builder.getInt32Ty());
+    } else
+      Src0 = UndefValue::get(Builder.getInt32Ty());
+
+    if (!isa<UndefValue>(Src1)) {
+      Src1 = Builder.CreateBitCast(Src1, Builder.getInt16Ty());
+      Src1 = Builder.CreateZExt(Src1, Builder.getInt32Ty());
+    } else
+      Src1 = UndefValue::get(Builder.getInt32Ty());
   }
 
   auto AddrSize = IID == vc::InternalIntrinsic::lsc_atomic_ugm

IGC/VectorCompiler/test/LoadStoreLowering/atomic-int16-global.ll

@@ -0,0 +1,33 @@
+;=========================== begin_copyright_notice ============================
+;
+; Copyright (C) 2024 Intel Corporation
+;
+; SPDX-License-Identifier: MIT
+;
+;============================ end_copyright_notice =============================
+
+; RUN: %opt %use_old_pass_manager% -enable-debugify -GenXLoadStoreLowering -march=genx64 -mcpu=XeHPC -mtriple=spir64-unknown-unknown -enable-ldst-lowering=true -mattr=+ocl_runtime -S < %s 2>&1 | FileCheck --check-prefix=CHECK-LSC %s
+
+define i16 @inc_i16(i16 addrspace(1)* %ptr) {
+  ; CHECK-LSC: [[INC_ADDR:%[^ ]+]] = ptrtoint i16 addrspace(1)* %ptr to i64
+  ; CHECK-LSC: [[INC_VADDR:%[^ ]+]] = bitcast i64 [[INC_ADDR]] to <1 x i64>
+  ; CHECK-LSC: call void @llvm.genx.lsc.fence.i1(i1 true, i8 0, i8 0, i8 0)
+  ; CHECK-LSC: [[INC_VRES:%[^ ]+]] = call <1 x i32> @llvm.vc.internal.lsc.atomic.ugm.v1i32.v1i1.v2i8.v1i64(<1 x i1> <i1 true>, i8 8, i8 3, i8 6, <2 x i8> zeroinitializer, i64 0, <1 x i64> [[INC_VADDR]], i16 1, i32 0, <1 x i32> undef, <1 x i32> undef, <1 x i32> undef)
+  ; CHECK-LSC: call void @llvm.genx.lsc.fence.i1(i1 true, i8 0, i8 0, i8 0)
+  ; CHECK-LSC: [[INC_BCAST:%[^ ]+]] = bitcast <1 x i32> [[INC_VRES]] to i32
+  ; CHECK-LSC: %res = trunc i32 [[INC_BCAST]] to i16
+  %res = atomicrmw add i16 addrspace(1)* %ptr, i16 1 syncscope("subgroup") seq_cst
+  ret i16 %res
+}
+
+define i16 @dec_i16(i16 addrspace(1)* %ptr) {
+  ; CHECK-LSC: [[DEC_ADDR:%[^ ]+]] = ptrtoint i16 addrspace(1)* %ptr to i64
+  ; CHECK-LSC: [[DEC_VADDR:%[^ ]+]] = bitcast i64 [[DEC_ADDR]] to <1 x i64>
+  ; CHECK-LSC: call void @llvm.genx.lsc.fence.i1(i1 true, i8 0, i8 0, i8 1)
+  ; CHECK-LSC: [[DEC_VRES:%[^ ]+]] = call <1 x i32> @llvm.vc.internal.lsc.atomic.ugm.v1i32.v1i1.v2i8.v1i64(<1 x i1> <i1 true>, i8 9, i8 3, i8 6, <2 x i8> zeroinitializer, i64 0, <1 x i64> [[DEC_VADDR]], i16 1, i32 0, <1 x i32> undef, <1 x i32> undef, <1 x i32> undef)
+  ; CHECK-LSC: call void @llvm.genx.lsc.fence.i1(i1 true, i8 0, i8 0, i8 1)
+  ; CHECK-LSC: [[DEC_BCAST:%[^ ]+]] = bitcast <1 x i32> [[DEC_VRES]] to i32
+  ; CHECK-LSC: %res = trunc i32 [[DEC_BCAST]] to i16
+  %res = atomicrmw sub i16 addrspace(1)* %ptr, i16 1 syncscope("workgroup") seq_cst
+  ret i16 %res
+}