File tree Expand file tree Collapse file tree 1 file changed +8
-1
lines changed Expand file tree Collapse file tree 1 file changed +8
-1
lines changed Original file line number Diff line number Diff line change @@ -661,7 +661,14 @@ ModelInstanceState::ModelInstanceState(
661
661
662
662
if (Kind () == TRITONSERVER_INSTANCEGROUPKIND_MODEL) {
663
663
#ifdef TRITON_ENABLE_GPU
664
- // Create a CUDA stream for every availble device.
664
+ // Since we cannot determine the exact devices used by the model, we create
665
+ // a CUDA stream for every available device to ensure proper synchronization
666
+ // of CUDA streams. This approach may have implications when a timestamp is
667
+ // captured on a device that is not used by the model. Currently, this issue
668
+ // is addressed by synchronizing the CUDA streams before recording
669
+ // timestamps to prevent timestamp skewing. However, in the future, any
670
+ // modifications to the CUDA stream synchronization logic should be handled
671
+ // with caution.
665
672
for (int i = 0 ; i < torch::cuda::device_count (); i++) {
666
673
cudaStream_t stream;
667
674
THROW_IF_BACKEND_INSTANCE_ERROR (
You can’t perform that action at this time.
0 commit comments