batched_reduction.h
Go to the documentation of this file.
/***************************************************************************************************
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#if !defined(__CUDACC_RTC__)
#include <cuda.h>
#endif
#include "cutlass/coord.h"
#include "cutlass/util/platform.h"
#include "cutlass/fragment.h"
namespace cutlass {
namespace reduction {
template <typename batched_reduction_>
__global__ __launch_bounds__(batched_reduction_::Traits::kThreads, 1) void batched_reduction_kernel(typename batched_reduction_::Params params) {
// Construct the batched_reduction object
batched_reduction_ batched_reduction(params);
batched_reduction.run();
}
template <typename BatchedReductionTraits_>
struct BatchedReduction {
typedef BatchedReduction<BatchedReductionTraits_> This_;
typedef BatchedReductionTraits_ Traits;
typedef typename Traits::Params Params;
typedef typename Traits::Functor Functor;
CUTLASS_DEVICE BatchedReduction(Params const &params_)
: params(params_), functor(params_.functorParams) {}
CUTLASS_DEVICE void run() {
#if (__CUDA_ARCH__ >= 600)
// Swizzle the IDs of the block
typename Traits::BlockSwizzle block_swizzle;
Coord<3> threadblock_offset =
block_swizzle.get_threadblock_offset(make_Coord_from_shape<Traits::SubTile>());
int subTileSize = gridDim.x * Traits::SubTile::kW;
int tileSize = params.problem_size[1] * params.problem_size[2];
int subTileOffset = threadblock_offset[2] + threadIdx.x * Traits::ThreadShape::kW;
int subTileBase = 0;
typename Traits::ScalarA inRegs[Traits::maxInReg];
typename Traits::ScalarAccum AccumRegs[Traits::maxOutReg];
#pragma unroll
for (int subTile = 0; subTile < tileSize; subTile += subTileSize) {
int tileOffset = subTileBase + subTileOffset;
// Init AccumRegs
#pragma unroll
for (int i = 0; i < Traits::ThreadShape::kW; i++)
AccumRegs[i] = static_cast<typename Traits::ScalarAccum>(0.0f);
// Fetch c0
typename Traits::ScalarAccum c0[Traits::ThreadShape::kW];
#pragma unroll
for (int i = 0; i< Traits::ThreadShape::kW; i++)
c0[i] = static_cast<typename Traits::ScalarAccum>(params.d_c[tileOffset + i]);
// Fetch partial sums from A
#pragma unroll
for (int s = 0; s < Traits::ReductionSize; s++) {
int inRegOffset = s * Traits::ThreadShape::kW;
int dOffset = (s * tileSize) + tileOffset;
#pragma unroll
for (int i = 0; i< Traits::ThreadShape::kW; i++) {
inRegs[inRegOffset + i] = params.d_a[dOffset + i];
}
}
// Accumulate
#pragma unroll
for (int s = 0; s < Traits::ReductionSize; s++) {
int inRegOffset = s * Traits::ThreadShape::kW;
#pragma unroll
for (int i = 0; i < Traits::ThreadShape::kW; i++) {
//AccumRegs[i] = cuFma(params.alpha, inRegs[inRegOffset + i], AccumRegs[i]);
//AccumRegs[i] = params.alpha * inRegs[inRegOffset + i] + AccumRegs[i];
AccumRegs[i] = static_cast<typename Traits::ScalarAccum>(inRegs[inRegOffset + i]) + AccumRegs[i];
}
}
// calling functor
functor_caller<Traits::ThreadShapeMultiple2>(AccumRegs, c0, AccumRegs);
// Store AccumRegs to D
#pragma unroll
for (int i = 0; i < Traits::ThreadShape::kW; i++) {
params.d_d[tileOffset + i] = static_cast<typename Traits::ScalarD>(AccumRegs[i]);
}
// Advance sub-tile pointer
subTileBase += subTileSize;
} // end for loop
#endif //#if (__CUDA_ARCH__ >= 600)
}
template<bool ThreadShapeMultiple2>
CUTLASS_DEVICE void functor_caller(typename Traits::ScalarAccum const *accum, typename Traits::ScalarAccum const *old, typename Traits::ScalarAccum *output) {
if (ThreadShapeMultiple2 == true) {
#pragma unroll
for (int i = 0; i < Traits::ThreadShape::kW / 2; i++) {
functor.template evaluate<typename Traits::ScalarAccum, typename Traits::ScalarAccum, 2>(&accum[2 * i], &old[2 * i], &output[2 * i]);
}
}
else {
#pragma unroll
for (int i = 0; i < Traits::ThreadShape::kW; i++) {
functor.template evaluate<typename Traits::ScalarAccum, typename Traits::ScalarAccum, 1>(&accum[i], &old[i], &output[i]);
}
}
}
//
// Static function members
//
#if !defined(__CUDACC_RTC__)
static __host__ cudaError_t launch(Params const& params,
cudaStream_t stream = cudaStreamDefault) {
// Setup the grid.
typename Traits::BlockSwizzle block_swizzle;
dim3 grid = block_swizzle.get_grid_layout(params.problem_size,
make_Coord_from_shape<typename Traits::OutputTile>());
dim3 block;
block.x = Traits::kThreads;
batched_reduction_kernel<This_><<<grid, block, 0, stream>>>(params);
return cudaGetLastError();
}
#endif
//
// Data members
//
Params const& params;
// The functor.
Functor functor;
};
} // namespace reduction
} // namespace cutlass
cutlass
Definition: aligned_buffer.h:35
cutlass::reduction::BatchedReduction::params
Params const & params
The params.
Definition: batched_reduction.h:173
cutlass::reduction::__launch_bounds__
__global__ __launch_bounds__(batched_reduction_::Traits::kThreads, 1) void batched_reduction_kernel(typename batched_reduction_
Definition: batched_reduction.h:45
coord.h
A Coord is a coordinate of arbitrary rank into a tensor or matrix.
cutlass::reduction::BatchedReduction::run
CUTLASS_DEVICE void run()
Definition: batched_reduction.h:68
cutlass::reduction::BatchedReduction::This_
BatchedReduction< BatchedReductionTraits_ > This_
This class.
Definition: batched_reduction.h:54
cutlass::reduction::BatchedReduction::functor
Functor functor
Definition: batched_reduction.h:175
cutlass::reduction::BatchedReduction
Definition: batched_reduction.h:52
cutlass::Coord< 3 >
cutlass::reduction::BatchedReduction::BatchedReduction
CUTLASS_DEVICE BatchedReduction(Params const &params_)
ctor
Definition: batched_reduction.h:63
cutlass::reduction::BatchedReduction::Params
Traits::Params Params
Params.
Definition: batched_reduction.h:58
cutlass::reduction::BatchedReduction::functor_caller
CUTLASS_DEVICE void functor_caller(typename Traits::ScalarAccum const *accum, typename Traits::ScalarAccum const *old, typename Traits::ScalarAccum *output)
Definition: batched_reduction.h:134
cutlass::reduction::BatchedReduction::Functor
Traits::Functor Functor
functor
Definition: batched_reduction.h:60
cutlass::reduction::BatchedReduction::Traits
BatchedReductionTraits_ Traits
The traits.
Definition: batched_reduction.h:56
cutlass::reduction::BatchedReduction::launch
static __host__ cudaError_t launch(Params const &params, cudaStream_t stream=cudaStreamDefault)
Launch the kernel.
Definition: batched_reduction.h:154