|
| 1 | +//***************************************************************************** |
| 2 | +// Copyright (c) 2024, Intel Corporation |
| 3 | +// All rights reserved. |
| 4 | +// |
| 5 | +// Redistribution and use in source and binary forms, with or without |
| 6 | +// modification, are permitted provided that the following conditions are met: |
| 7 | +// - Redistributions of source code must retain the above copyright notice, |
| 8 | +// this list of conditions and the following disclaimer. |
| 9 | +// - Redistributions in binary form must reproduce the above copyright notice, |
| 10 | +// this list of conditions and the following disclaimer in the documentation |
| 11 | +// and/or other materials provided with the distribution. |
| 12 | +// |
| 13 | +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
| 14 | +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
| 15 | +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
| 16 | +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE |
| 17 | +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
| 18 | +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
| 19 | +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
| 20 | +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
| 21 | +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
| 22 | +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF |
| 23 | +// THE POSSIBILITY OF SUCH DAMAGE. |
| 24 | +//***************************************************************************** |
| 25 | + |
| 26 | +#pragma once |
| 27 | + |
| 28 | +#include <pybind11/pybind11.h> |
| 29 | + |
| 30 | +#include <oneapi/mkl/rng/device.hpp> |
| 31 | +#include <sycl/sycl.hpp> |
| 32 | + |
| 33 | +namespace dpnp |
| 34 | +{ |
| 35 | +namespace backend |
| 36 | +{ |
| 37 | +namespace ext |
| 38 | +{ |
| 39 | +namespace rng |
| 40 | +{ |
| 41 | +namespace device |
| 42 | +{ |
| 43 | +namespace details |
| 44 | +{ |
| 45 | +namespace py = pybind11; |
| 46 | + |
| 47 | +namespace mkl_rng_dev = oneapi::mkl::rng::device; |
| 48 | + |
| 49 | +template <typename EngineBuilderT, |
| 50 | + typename DistributorBuilderT, |
| 51 | + unsigned int items_per_wi = 4, |
| 52 | + bool enable_sg_load = true> |
| 53 | +struct RngContigFunctor |
| 54 | +{ |
| 55 | +private: |
| 56 | + using DataT = typename DistributorBuilderT::result_type; |
| 57 | + |
| 58 | + EngineBuilderT engine_; |
| 59 | + DistributorBuilderT distr_; |
| 60 | + DataT *const res_ = nullptr; |
| 61 | + const std::size_t nelems_; |
| 62 | + |
| 63 | +public: |
| 64 | + RngContigFunctor(EngineBuilderT &engine, |
| 65 | + DistributorBuilderT &distr, |
| 66 | + DataT *res, |
| 67 | + const std::size_t n_elems) |
| 68 | + : engine_(engine), distr_(distr), res_(res), nelems_(n_elems) |
| 69 | + { |
| 70 | + } |
| 71 | + |
| 72 | + void operator()(sycl::nd_item<1> nd_it) const |
| 73 | + { |
| 74 | + auto sg = nd_it.get_sub_group(); |
| 75 | + const std::uint8_t sg_size = sg.get_local_range()[0]; |
| 76 | + const std::uint8_t max_sg_size = sg.get_max_local_range()[0]; |
| 77 | + |
| 78 | + using EngineT = typename EngineBuilderT::EngineType; |
| 79 | + using DistrT = typename DistributorBuilderT::distr_type; |
| 80 | + |
| 81 | + constexpr std::size_t vec_sz = EngineT::vec_size; |
| 82 | + constexpr std::size_t vi_per_wi = vec_sz * items_per_wi; |
| 83 | + |
| 84 | + EngineT engine = engine_(nd_it.get_global_id() * vi_per_wi); |
| 85 | + DistrT distr = distr_(); |
| 86 | + |
| 87 | + if constexpr (enable_sg_load) { |
| 88 | + const std::size_t base = |
| 89 | + vi_per_wi * (nd_it.get_group(0) * nd_it.get_local_range(0) + |
| 90 | + sg.get_group_id()[0] * max_sg_size); |
| 91 | + |
| 92 | + if ((sg_size == max_sg_size) && |
| 93 | + (base + vi_per_wi * sg_size < nelems_)) { |
| 94 | +#pragma unroll |
| 95 | + for (std::uint16_t it = 0; it < vi_per_wi; it += vec_sz) { |
| 96 | + std::size_t offset = |
| 97 | + base + static_cast<std::size_t>(it) * |
| 98 | + static_cast<std::size_t>(sg_size); |
| 99 | + auto out_multi_ptr = sycl::address_space_cast< |
| 100 | + sycl::access::address_space::global_space, |
| 101 | + sycl::access::decorated::yes>(&res_[offset]); |
| 102 | + |
| 103 | + sycl::vec<DataT, vec_sz> rng_val_vec = |
| 104 | + mkl_rng_dev::generate<DistrT, EngineT>(distr, engine); |
| 105 | + sg.store<vec_sz>(out_multi_ptr, rng_val_vec); |
| 106 | + } |
| 107 | + } |
| 108 | + else { |
| 109 | + for (std::size_t offset = base + sg.get_local_id()[0]; |
| 110 | + offset < nelems_; offset += sg_size) |
| 111 | + { |
| 112 | + res_[offset] = |
| 113 | + mkl_rng_dev::generate_single<DistrT, EngineT>(distr, |
| 114 | + engine); |
| 115 | + } |
| 116 | + } |
| 117 | + } |
| 118 | + else { |
| 119 | + std::size_t base = nd_it.get_global_linear_id(); |
| 120 | + |
| 121 | + base = (base / sg_size) * sg_size * vi_per_wi + (base % sg_size); |
| 122 | + for (std::size_t offset = base; |
| 123 | + offset < std::min(nelems_, base + sg_size * vi_per_wi); |
| 124 | + offset += sg_size) |
| 125 | + { |
| 126 | + res_[offset] = mkl_rng_dev::generate_single<DistrT, EngineT>( |
| 127 | + distr, engine); |
| 128 | + } |
| 129 | + } |
| 130 | + } |
| 131 | +}; |
| 132 | +} // namespace details |
| 133 | +} // namespace device |
| 134 | +} // namespace rng |
| 135 | +} // namespace ext |
| 136 | +} // namespace backend |
| 137 | +} // namespace dpnp |
0 commit comments