File tree Expand file tree Collapse file tree 1 file changed +8
-1
lines changed Expand file tree Collapse file tree 1 file changed +8
-1
lines changed Original file line number Diff line number Diff line change @@ -660,7 +660,14 @@ ModelInstanceState::ModelInstanceState(
660
660
661
661
if (Kind () == TRITONSERVER_INSTANCEGROUPKIND_MODEL) {
662
662
#ifdef TRITON_ENABLE_GPU
663
- // Create a CUDA stream for every availble device.
663
+ // Since we cannot determine the exact devices used by the model, we create
664
+ // a CUDA stream for every available device to ensure proper synchronization
665
+ // of CUDA streams. This approach may have implications when a timestamp is
666
+ // captured on a device that is not used by the model. Currently, this issue
667
+ // is addressed by synchronizing the CUDA streams before recording
668
+ // timestamps to prevent timestamp skewing. However, in the future, any
669
+ // modifications to the CUDA stream synchronization logic should be handled
670
+ // with caution.
664
671
for (int i = 0 ; i < torch::cuda::device_count (); i++) {
665
672
cudaStream_t stream;
666
673
THROW_IF_BACKEND_INSTANCE_ERROR (
You can’t perform that action at this time.
0 commit comments