gpt-fast running now with intra-node comm

Author: pragupta

torch/csrc/distributed/c10d/intra_node_comm.cpp

@@ -330,7 +330,7 @@ bool IntraNodeComm::rendezvous() {
   auto ret = rsmi_init(0);
   if (ret != RSMI_STATUS_SUCCESS) {
     LOG(ERROR) << "IntraNodeComm:: rendezvous failed in rsmi_init, ret=" << ret;
-    return nullptr;
+    return false;
   }
 #endif
 

torch/csrc/distributed/c10d/intra_node_comm.cu

@@ -120,6 +120,7 @@ DEVICE_INLINE void releaseSignal(uint32_t* addr) {
   CUDA_KERNEL_ASSERT(false);
 #else
   atomicAdd_system(addr, 1);
+  __threadfence_system();
 #endif
 }