diff options
68 files changed, 16351 insertions, 0 deletions
diff --git a/cutlass-example b/cutlass-example deleted file mode 160000 -Subproject e69115d598bef1e5372c9134322972b2ea450d9 diff --git a/cutlass-example/Makefile b/cutlass-example/Makefile new file mode 100644 index 0000000..8156847 --- /dev/null +++ b/cutlass-example/Makefile @@ -0,0 +1,37 @@ +# Copyright (c) 1993-2017, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +CC = nvcc +src = $(wildcard *.cu) +obj = $(src:.c=.o) +INC_DIR = ./ + + +#LDFLAGS = -g -G --gpu-architecture=compute_70 --gpu-code=compute_70 -lcudart -I$(INC_DIR) +#LDFLAGS = -O0 -Xcicc -O0 -Xptxas -O0 --gpu-architecture=compute_70 --gpu-code=compute_70 -lcudart -I$(INC_DIR) +LDFLAGS = --gpu-architecture=compute_70 --gpu-code=compute_70 -lcudart -I$(INC_DIR) + +myprog: $(obj) + $(CC) -o $@ $^ $(LDFLAGS) diff --git a/cutlass-example/command_line.h b/cutlass-example/command_line.h new file mode 100644 index 0000000..8f2b17a --- /dev/null +++ b/cutlass-example/command_line.h @@ -0,0 +1,254 @@ +/****************************************************************************** + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are not permitted. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +/** + * \file + * Utility for parsing command line arguments + */ + +#include <iostream> +#include <limits> +#include <sstream> +#include <string> +#include <vector> + +#include <cuda_runtime.h> + +namespace cutlass { + +/****************************************************************************** + * command_line + ******************************************************************************/ + +/** + * Utility for parsing command line arguments + */ +struct CommandLine { + std::vector<std::string> keys; + std::vector<std::string> values; + std::vector<std::string> args; + + /** + * Constructor + */ + CommandLine(int argc, const char** argv) : keys(10), values(10) { + using namespace std; + + for (int i = 1; i < argc; i++) { + string arg = argv[i]; + + if ((arg[0] != '-') || (arg[1] != '-')) { + args.push_back(arg); + continue; + } + + string::size_type pos; + string key, val; + if ((pos = arg.find('=')) == string::npos) { + key = string(arg, 2, arg.length() - 2); + val = ""; + } else { + key = string(arg, 2, pos - 2); + val = string(arg, pos + 1, arg.length() - 1); + } + + keys.push_back(key); + values.push_back(val); + } + } + + /** + * Checks whether a flag "--<flag>" is present in the commandline + */ + bool check_cmd_line_flag(const char* arg_name) const { + using namespace std; + + for (int i = 0; i < int(keys.size()); ++i) { + if (keys[i] == string(arg_name)) return true; + } + return false; + } + + /** + * Returns number of naked (non-flag and non-key-value) commandline parameters + */ + template <typename value_t> + int num_naked_args() const { + return args.size(); + } + + /** + * Returns the commandline parameter for a given index (not including flags) + */ + template <typename value_t> + void get_cmd_line_argument(int index, value_t& val) const { + using namespace std; + if (index < args.size()) { + istringstream str_stream(args[index]); + str_stream >> val; + } + } + + /** + * Returns the commandline parameter for a given index (not including flags) + */ + void get_cmd_line_argument(const char* arg_name, bool& val, bool _default = true) const { + val = _default; + if (check_cmd_line_flag(arg_name)) { + std::string value; + get_cmd_line_argument(arg_name, value); + + val = !(value == "0" || value == "false"); + } + } + + /** + * Returns the value specified for a given commandline parameter --<flag>=<value> + */ + template <typename value_t> + void get_cmd_line_argument(const char* arg_name, + value_t& val, + value_t const& _default = value_t()) const { + using namespace std; + + val = _default; + + for (int i = 0; i < int(keys.size()); ++i) { + if (keys[i] == string(arg_name)) { + istringstream str_stream(values[i]); + str_stream >> val; + } + } + } + + /** + * Returns the values specified for a given commandline parameter --<flag>=<value>,<value>* + */ + template <typename value_t> + void get_cmd_line_arguments(const char* arg_name, + std::vector<value_t>& vals, + char sep = ',') const { + using namespace std; + + if (check_cmd_line_flag(arg_name)) { + // Clear any default values + vals.clear(); + + // Recover from multi-value string + for (int i = 0; i < keys.size(); ++i) { + if (keys[i] == string(arg_name)) { + string val_string(values[i]); + istringstream str_stream(val_string); + string::size_type old_pos = 0; + string::size_type new_pos = 0; + + // Iterate <sep>-delimited values + value_t val; + while ((new_pos = val_string.find(sep, old_pos)) != string::npos) { + if (new_pos != old_pos) { + str_stream.width(new_pos - old_pos); + str_stream >> val; + vals.push_back(val); + } + + // skip over delimiter + str_stream.ignore(1); + old_pos = new_pos + 1; + } + + // Read last value + str_stream >> val; + vals.push_back(val); + } + } + } + } + + /** + * Returns the values specified for a given commandline parameter + * --<flag>=<key:value>,<key:value>* + */ + void get_cmd_line_argument_pairs(const char* arg_name, + std::vector<std::pair<std::string, std::string> >& tokens, + char delim = ',', + char sep = ':') const { + if (check_cmd_line_flag(arg_name)) { + std::string value; + get_cmd_line_argument(arg_name, value); + + tokenize(tokens, value, delim, sep); + } + } + + /** + * The number of pairs parsed + */ + int parsed_argc() const { return (int)keys.size(); } + + //------------------------------------------------------------------------- + // Utility functions + //------------------------------------------------------------------------- + + /// Tokenizes a comma-delimited list of string pairs delimited by ':' + static void tokenize(std::vector<std::pair<std::string, std::string> >& tokens, + std::string const& str, + char delim = ',', + char sep = ':') { + // Home-built to avoid Boost dependency + size_t s_idx = 0; + size_t d_idx = std::string::npos; + while (s_idx < str.size()) { + d_idx = str.find_first_of(delim, s_idx); + + size_t end_idx = (d_idx != std::string::npos ? d_idx : str.size()); + size_t sep_idx = str.find_first_of(sep, s_idx); + size_t offset = 1; + if (sep_idx == std::string::npos || sep_idx >= end_idx) { + sep_idx = end_idx; + offset = 0; + } + + std::pair<std::string, std::string> item( + str.substr(s_idx, sep_idx - s_idx), + str.substr(sep_idx + offset, end_idx - sep_idx - offset)); + + tokens.push_back(item); + s_idx = end_idx + 1; + } + } + + /// Tokenizes a comma-delimited list of string pairs delimited by ':' + static void tokenize(std::vector<std::string>& tokens, + std::string const& str, + char delim = ',', + char sep = ':') { + typedef std::vector<std::pair<std::string, std::string> > TokenVector; + typedef TokenVector::const_iterator token_iterator; + + std::vector<std::pair<std::string, std::string> > token_pairs; + tokenize(token_pairs, str, delim, sep); + for (token_iterator tok = token_pairs.begin(); tok != token_pairs.end(); ++tok) { + tokens.push_back(tok->first); + } + } +}; + +} // namespace cutlass diff --git a/cutlass-example/config_fermi_islip.icnt b/cutlass-example/config_fermi_islip.icnt new file mode 100644 index 0000000..3b8b496 --- /dev/null +++ b/cutlass-example/config_fermi_islip.icnt @@ -0,0 +1,70 @@ +//21*1 fly with 32 flits per packet under gpgpusim injection mode +use_map = 0; +flit_size = 32; + +// currently we do not use this, see subnets below +network_count = 2; + +// Topology +topology = fly; +k = 102; +n = 1; + +// Routing + +routing_function = dest_tag; + +// Flow control + +num_vcs = 1; +vc_buf_size = 8; + +wait_for_tail_credit = 0; + +// Router architecture + +vc_allocator = islip; //separable_input_first; +sw_allocator = islip; //separable_input_first; +alloc_iters = 1; + +credit_delay = 0; +routing_delay = 0; +vc_alloc_delay = 1; +sw_alloc_delay = 1; + +input_speedup = 2; +output_speedup = 1; +internal_speedup = 1.0; + +// Traffic, GPGPU-Sim does not use this + +traffic = uniform; +packet_size ={{1,2,3,4},{10,20}}; +packet_size_rate={{1,1,1,1},{2,1}}; + +// Simulation - Don't change + +sim_type = gpgpusim; +//sim_type = latency; +injection_rate = 0.1; + +subnets = 2; + +// Always use read and write no matter following line +//use_read_write = 1; + + +read_request_subnet = 0; +read_reply_subnet = 1; +write_request_subnet = 0; +write_reply_subnet = 1; + +read_request_begin_vc = 0; +read_request_end_vc = 0; +write_request_begin_vc = 0; +write_request_end_vc = 0; +read_reply_begin_vc = 0; +read_reply_end_vc = 0; +write_reply_begin_vc = 0; +write_reply_end_vc = 0; + diff --git a/cutlass-example/cutlass/convert.h b/cutlass-example/cutlass/convert.h new file mode 100644 index 0000000..933d68a --- /dev/null +++ b/cutlass-example/cutlass/convert.h @@ -0,0 +1,102 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! + \file + \brief Defines conversion operations among Fragments of different base type. +*/ +#pragma once + +#include <cutlass/fragment.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename InputFragment_, typename OutputFragment_> +struct Convert {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename InputScalar_, typename OutputScalar_, int kScalars_> +struct Convert<Fragment<InputScalar_, kScalars_>, Fragment<OutputScalar_, kScalars_> > { + /// The input fragment. + typedef Fragment<InputScalar_, kScalars_> InputFragment; + /// The output fragment. + typedef Fragment<OutputScalar_, kScalars_> OutputFragment; + + /// Ctor. + CUTLASS_DEVICE Convert() {} + + /// Transform a fragment. + CUTLASS_DEVICE void transform(InputFragment const& src, OutputFragment& dst) { + transform(src, 0, dst); + } + + /// Transform a fragment. + template <typename Fragment_> + CUTLASS_DEVICE void transform(Fragment_ const& src, int offset, OutputFragment& dst) { + for (int i = 0; i < kScalars_; ++i) { + dst[i] = static_cast<OutputScalar_>(src[i + offset]); + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Fragment_> +struct Copy { + /// The input fragment. + typedef Fragment_ InputFragment; + /// The output fragment. + typedef Fragment_ OutputFragment; + + /// Ctor. + CUTLASS_DEVICE Copy() {} + + /// Transform a fragment. + CUTLASS_DEVICE void transform(Fragment_ const& src, Fragment_& dst) { transform(src, 0, dst); } + + /// Transform a fragment. + template <typename InputFragment_> + CUTLASS_DEVICE void transform(InputFragment_ const& src, int offset, Fragment_& dst) { + if (sizeof(typename Fragment_::Element) == 8) { + uint64_t const* src_ptr = reinterpret_cast<uint64_t const*>(&src[offset]); + uint64_t* dst_ptr = reinterpret_cast<uint64_t*>(&dst[0]); + for (int i = 0; i < sizeof(Fragment_) / 8; ++i) { + dst_ptr[i] = src_ptr[i]; + } + } else { + uint32_t const* src_ptr = reinterpret_cast<uint32_t const*>(&src[offset]); + uint32_t* dst_ptr = reinterpret_cast<uint32_t*>(&dst[0]); + for (int i = 0; i < sizeof(Fragment_) / 4; ++i) { + dst_ptr[i] = src_ptr[i]; + } + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/coord.h b/cutlass-example/cutlass/coord.h new file mode 100644 index 0000000..431c9bf --- /dev/null +++ b/cutlass-example/cutlass/coord.h @@ -0,0 +1,287 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief A Coord is a coordinate of arbitrary rank into a tensor or matrix +*/ + +#pragma once + +#include <cutlass/cutlass.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Describes identity elements +struct Identity { + /// Enumeration describing identity elements. Value assignments are significant. + /// Feel free to add or multiply by these, respectively. + enum Kind { Additive = 0, Multiplicative = 1 }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Statically-sized array specifying Coords within a tensor +template <int N_> +struct Coord { + // + // Type and constant definitions + // + + static int const N = N_; + + // + // Data members + // + + /// Indices + int idx[N]; + + // + // Methods + // + + /// Default ctor initializes uniformly + CUTLASS_HOST_DEVICE + Coord(int value = 0) { + for (int i = 0; i < N; ++i) { + idx[i] = value; + } + } + + /// Constructs from an array of integers + CUTLASS_HOST_DEVICE + Coord(int _idx[]) { + for (int i = 0; i < N; ++i) { + idx[i] = _idx[i]; + } + } + + /// Element-wise addition + CUTLASS_HOST_DEVICE + Coord operator+(Coord const& b) const { + Coord c; + for (int i = 0; i < N; ++i) { + c.idx[i] = idx[i] + b.idx[i]; + } + return c; + } + + /// Element-wise subtraction + CUTLASS_HOST_DEVICE + Coord operator-(Coord const& b) const { + Coord c; + for (int i = 0; i < N; ++i) { + c.idx[i] = idx[i] - b.idx[i]; + } + return c; + } + + /// Element-wise multiplication + CUTLASS_HOST_DEVICE + Coord operator*(Coord const& b) const { + Coord c; + for (int i = 0; i < N; ++i) { + c.idx[i] = idx[i] * b.idx[i]; + } + return c; + } + + /// Element-wise division + CUTLASS_HOST_DEVICE + Coord operator/(Coord const& b) const { + Coord c; + for (int i = 0; i < N; ++i) { + c.idx[i] = idx[i] / b.idx[i]; + } + return c; + } + + /// In-place addition + CUTLASS_HOST_DEVICE + Coord& operator+=(Coord const& b) { + for (int i = 0; i < N; ++i) { + idx[i] += b.idx[i]; + } + return *this; + } + + /// In-place subtraction + CUTLASS_HOST_DEVICE + Coord& operator-=(Coord const& b) { + for (int i = 0; i < N; ++i) { + idx[i] -= b.idx[i]; + } + return *this; + } + + /// In-place multiplication + CUTLASS_HOST_DEVICE + Coord& operator*=(Coord const& b) { + for (int i = 0; i < N; ++i) { + idx[i] *= b.idx[i]; + } + return *this; + } + + /// In-place division + CUTLASS_HOST_DEVICE + Coord& operator/=(Coord const& b) { + for (int i = 0; i < N; ++i) { + idx[i] /= b.idx[i]; + } + return *this; + } + + /// Member access operator + CUTLASS_HOST_DEVICE int& operator[](int dim) { return idx[dim]; } + + /// Member access operator + CUTLASS_HOST_DEVICE int const& operator[](int dim) const { return idx[dim]; } + + /// Computes the dot product of two Coord instances + template <typename T> + CUTLASS_HOST_DEVICE T dot(Coord const& b, T sum) const { + for (int i = 0; i < N; ++i) { + sum += idx[i] * b.idx[i]; + } + return sum; + } + + /// Computes the dot product of two Coord instances + template <typename T> + CUTLASS_HOST_DEVICE T dot(Coord const& b) const { + T sum = T(0); + for (int i = 0; i < N; ++i) { + sum += idx[i] * b.idx[i]; + } + return sum; + } + + /// Gets the index of a given Coord element + template <int Dim> + CUTLASS_HOST_DEVICE int& at() { + return idx[Dim]; + } + + /// Access via index; may limit unrolling potential + CUTLASS_HOST_DEVICE + int& at(int dim) { return idx[dim]; } + + /// Gets the index of a given Coord element + template <int Dim> + CUTLASS_HOST_DEVICE int const& at() const { + return idx[Dim]; + } + + /// Access via index; may limit unrolling potential + CUTLASS_HOST_DEVICE + int const& at(int dim) const { return idx[dim]; } + + /// Determines if two Coord<> objects are equal + CUTLASS_HOST_DEVICE + bool operator==(Coord<N> const& b) const { + bool equal = true; + for (int i = 0; equal && i < N; ++i) { + equal = (idx[i] == b.idx[i]); + } + return equal; + } + + /// Not equal + CUTLASS_HOST_DEVICE + bool operator!=(Coord<N> const& b) const { return !(*this == b); } + + /// Clamps a coordinate to a range specified by maximum and minimum values + CUTLASS_HOST_DEVICE + Coord& clamp(Coord<N> const& max, Coord<N> const& min = Coord<N>()) { + for (int i = 0; i < N; ++i) { + idx[i] = __NV_STD_MAX(__NV_STD_MIN(idx[i], max.idx[i]), min.idx[i]); + } + return *this; + } + + /// Returns the product of all elements + CUTLASS_HOST_DEVICE + int count() const { + int product = idx[0]; + for (int i = 1; i < N; ++i) { + product *= idx[i]; + } + return product; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Helper to make a 2-element coordinate +CUTLASS_HOST_DEVICE +Coord<1> make_Coord(int _0) { + int values[1] = {_0}; + return Coord<1>(values); +} + +/// Helper to make a 2-element coordinate +CUTLASS_HOST_DEVICE +Coord<2> make_Coord(int _0, int _1) { + int values[2] = {_0, _1}; + return Coord<2>(values); +} + +/// Helper to make a 3-element coordinate +CUTLASS_HOST_DEVICE +Coord<3> make_Coord(int _0, int _1, int _2) { + int values[3] = {_0, _1, _2}; + return Coord<3>(values); +} + +/// Helper to make a 4-element coordinate +CUTLASS_HOST_DEVICE +Coord<4> make_Coord(int _0, int _1, int _2, int _3) { + int values[4] = {_0, _1, _2, _3}; + return Coord<4>(values); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Getter +CUTLASS_HOST_DEVICE +Coord<2> get_Coord_hw(Coord<3> const& coord) { return make_Coord(coord[1], coord[2]); } + +/// Getter +CUTLASS_HOST_DEVICE +Coord<2> get_Coord_hw(Coord<4> const& coord) { return make_Coord(coord[1], coord[2]); } + +/// Getter +CUTLASS_HOST_DEVICE +Coord<3> get_Coord_hwc(Coord<4> const& coord) { return make_Coord(coord[1], coord[2], coord[3]); } + +/// Getter +CUTLASS_HOST_DEVICE +Coord<3> get_Coord_dhw(Coord<4> const& coord) { return make_Coord(coord[0], coord[1], coord[2]); } + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/core_io.h b/cutlass-example/cutlass/core_io.h new file mode 100644 index 0000000..cceea4c --- /dev/null +++ b/cutlass-example/cutlass/core_io.h @@ -0,0 +1,44 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +#pragma once + +/*! \file + \brief Helpers for printing cutlass/core objects +*/ + +#pragma once + +#include <iosfwd> +#include <typeinfo> + +#include <cutlass/coord.h> + +template <int Rank> +std::ostream& operator<<(std::ostream& out, cutlass::Coord<Rank> const& coord) { + for (int i = 0; i < Rank; ++i) { + out << (i ? ", " : "") << coord.idx[i]; + } + return out; +} diff --git a/cutlass-example/cutlass/cutlass.h b/cutlass-example/cutlass/cutlass.h new file mode 100644 index 0000000..19600ec --- /dev/null +++ b/cutlass-example/cutlass/cutlass.h @@ -0,0 +1,73 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +/*! \file + \brief Basic include for CUTLASS macros +*/ + +#pragma once + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#define CUTLASS_MAJOR 1 +#define CUTLASS_MINOR 0 +#define CUTLASS_PATCH 1 +#define CUTLASS_VERSION ((CUTLASS_MAJOR)*100 + (CUTLASS_MINOR)*10 + CUTLASS_PATCH) + +#ifdef __NVCC__ +#define CUTLASS_HOST_DEVICE __forceinline__ __device__ __host__ +#define CUTLASS_DEVICE __forceinline__ __device__ +#elif defined(__CUDACC_RTC__) +#define CUTLASS_HOST_DEVICE __forceinline__ __device__ +#define CUTLASS_DEVICE __forceinline__ __device__ +#else +#define CUTLASS_HOST_DEVICE +// CUTLASS_DEVICE is an error if not compiling device code +#endif + +// CUTLASS_PRAGMA_UNROLL inserts a CUTLASS_PRAGMA_UNROLL if supported by the compiler +#if defined(__CUDA_ARCH__) +#if defined(_MSC_VER) +#define CUTLASS_PRAGMA_UNROLL __pragma("unroll") +#define CUTLASS_PRAGMA_NO_UNROLL __pragma("unroll 1") +#else +#define CUTLASS_PRAGMA_UNROLL _Pragma("unroll") +#define CUTLASS_PRAGMA_NO_UNROLL _Pragma("unroll 1") +#endif +#else +#define CUTLASS_PRAGMA_UNROLL +#define CUTLASS_PRAGMA_NO_UNROLL +#endif + +#define CUTLASS_ASSERT(x) assert(x) + +namespace cutlass { + +/// NVIDIA GPU Warp size +static const int kWarpSize = 32; + +} // namespace cutlass + +//////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/cutlass-example/cutlass/fragment.h b/cutlass-example/cutlass/fragment.h new file mode 100644 index 0000000..886b114 --- /dev/null +++ b/cutlass-example/cutlass/fragment.h @@ -0,0 +1,278 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines Fragment, a statically-sized array for storing parts of matrices within a + thread's registers. +*/ +#pragma once + +#include <assert.h> +#include <cutlass/shape.h> +#include <cutlass/util/cutlass_math.h> +#include <cutlass/vector.h> + +namespace cutlass { + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/*!@defgroup fragment_concept Fragment Concept +@{ + +\ref fragment_concept is a statically sized array for storing parts of tiles held by individual CUDA +threads. + +@par \ref fragment_concept + Types satisfying \ref fragment_concept define the following members + - <b>Element</b> - type of each access held within the fragment + - <b>kElements</b> - number of elements stored by the fragment + - <b>clear()</b> - overwrites the fragment storage with zeros + - <b>Element & operator[](int i)</b> - by-reference access of the ith element + - <b>Element const & operator[](int i) const</b> - const by-reference access of the ith element +@} +*/ + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/*!@defgroup fragment_iterator_concept Fragment Iterator Concept +@{ + +\ref fragment_iterator_concept provides structured access to the elements within a fragment with an +optional bitcast to the desired access type + +@par \ref fragment_iterator_concept + Types satisfying \ref fragment_iterator_concept define the following members + - <b>AccessType& operator[](int i)</b> - provides access to the ith element of the fragment + - <b>AccessType& at(int d, int h, int w, int c)</b> - applies \ref layout_concept to fragment and +provides access to element at (d, h, w, c) + +@} +*/ + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <int kAlignment_> +struct StorageType { + typedef uint64_t Type; +}; +template <> +struct StorageType<4> { + typedef uint32_t Type; +}; +template <> +struct StorageType<2> { + typedef uint16_t Type; +}; +template <> +struct StorageType<1> { + typedef uint8_t Type; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief A template defining \ref fragment_concept +* @concept{fragment_concept} +*/ +template <typename Element_, int kElements_, size_t kAlignment_ = 16> +struct Fragment : public AlignedStruct<kAlignment_> { + /// Make sure the alignment makes sense wrt the size of elements. + static_assert(kAlignment_ == 16 || kAlignment_ >= sizeof(Element_), "Alignment is too small"); + /// Alignment must be a power of two + static_assert(is_pow2<kAlignment_>::value, "Alignment must be a power of two"); + + /// This class. + typedef Fragment<Element_, kElements_> This_; + /// The element. + typedef Element_ Element; + /// The number of elements. + static int const kElements = kElements_; + + /// Clear a fragment. + CUTLASS_DEVICE void clear() { + // Avoid element-wise access for sub 32b element type + if (kAlignment_ >= 8 && (kElements * sizeof(Element)) % 8 == 0) { + uint64_t* ptr = reinterpret_cast<uint64_t*>(storage); + for (int i = 0; i < (kElements * sizeof(Element)) / 8; ++i) { + ptr[i] = uint64_t(0); + } + } else if (kAlignment_ >= 4 && (kElements * sizeof(Element)) % 4 == 0) { + uint32_t* ptr = reinterpret_cast<uint32_t*>(storage); + for (int i = 0; i < (kElements * sizeof(Element)) / 4; ++i) { + ptr[i] = uint32_t(0); + } + } else if (kAlignment_ >= 2 && (kElements * sizeof(Element)) % 2 == 0) { + uint16_t* ptr = reinterpret_cast<uint16_t*>(storage); + for (int i = 0; i < (kElements * sizeof(Element)) / 2; ++i) { + ptr[i] = uint16_t(0); + } + } else { + for (int i = 0; i < kElements; ++i) { + storage[i] = 0; + } + } + } + + /// The accessor. + CUTLASS_DEVICE Element& operator[](int i) { + assert(i < kElements_); + return reinterpret_cast<Element*>(storage)[i]; + } + + /// The accessor. + CUTLASS_DEVICE Element const& operator[](int i) const { + assert(i < kElements_); + return reinterpret_cast<Element const*>(storage)[i]; + } + + private: + /// Storage type to use for Elements + typedef typename StorageType<kAlignment_>::Type StorageType; + + /// Number of elements in the storage + static int const kStorageCount = + (sizeof(Element_) * kElements_ + sizeof(StorageType) - 1) / sizeof(StorageType); + /// The storage. + StorageType storage[kStorageCount]; + + /// Ensure that there's enough storage for all elements + static_assert(sizeof(StorageType) <= kAlignment_, "StorageType is too big for given alignment"); +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief A template defining \ref fragment_iterator_concept +* @concept{fragment_iterator_concept} +*/ +template <typename Fragment_, typename Iterations_, typename AccessType_> +struct FragmentIterator { + /// This class. + typedef FragmentIterator<Fragment_, Iterations_, AccessType_> This_; + /// The fragment. + typedef Fragment_ Fragment; + /// The number of iterations. + typedef Iterations_ Iterations; + /// The access type. + typedef AccessType_ AccessType; + + /// The element. + typedef typename Fragment::Element Element; + /// The number of elements per access. + static int const kElementsPerAccess = (int)(sizeof(AccessType) / sizeof(Element)); + /// The shape of the the fragment. + typedef typename ShapeMul<Iterations, Shape<1, 1, 1, kElementsPerAccess> >::Shape FragmentShape; + /// The linear strides for iterations. + typedef typename ShapeStrides<FragmentShape, kElementsPerAccess>::Shape Strides; + + /// Ctor. + template <typename OtherFragment_> + CUTLASS_DEVICE FragmentIterator(OtherFragment_& fragment, int offset = 0) + : pointer(reinterpret_cast<Element*>(&fragment[offset])) { + static_assert(OtherFragment_::kElements >= Fragment::kElements, ""); + } + + /// The accessor. + CUTLASS_DEVICE AccessType const& at(int d, int h, int w, int c = 0) const { + int const imm = ComputeOffsetFromStrides<Strides>::get(d, h, w, c); + return reinterpret_cast<AccessType const&>(pointer[imm]); + } + + /// The accessor. + CUTLASS_DEVICE AccessType& at(int d, int h, int w, int c = 0) { + int const imm = ComputeOffsetFromStrides<Strides>::get(d, h, w, c); + return reinterpret_cast<AccessType&>(pointer[imm]); + } + + /// The accessor. + CUTLASS_DEVICE AccessType const& operator[](int i) const { + return reinterpret_cast<AccessType const&>(pointer[i * kElementsPerAccess]); + } + + /// The accessor. + CUTLASS_DEVICE AccessType& operator[](int i) { + return reinterpret_cast<AccessType&>(pointer[i * kElementsPerAccess]); + } + + /// Is the iterator valid? + CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { return true; } + + /// The pointer. + Element* pointer; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Fragment_, typename Iterations_, typename AccessType_> +struct FragmentConstIterator { + /// This class. + typedef FragmentIterator<Fragment_, Iterations_, AccessType_> This_; + /// The fragment. + typedef Fragment_ Fragment; + /// The number of iterations. + typedef Iterations_ Iterations; + /// The access type. + typedef AccessType_ AccessType; + + /// The element. + typedef typename Fragment::Element Element; + /// The number of elements per access. + static int const kElementsPerAccess = (int)(sizeof(AccessType) / sizeof(Element)); + /// The shape of the the fragment. + typedef typename ShapeMul<Iterations, Shape<1, 1, 1, kElementsPerAccess> >::Shape FragmentShape; + /// The linear strides for iterations. + typedef typename ShapeStrides<FragmentShape, kElementsPerAccess>::Shape IterationsStrides; + + /// Ctor. + template <typename OtherFragment_> + CUTLASS_DEVICE FragmentConstIterator(OtherFragment_& fragment, int offset = 0) + : pointer(reinterpret_cast<Element const*>(&fragment[offset])) { + static_assert(OtherFragment_::kElements >= Fragment::kElements, ""); + } + /// Create from non-constant FragmentIterator + CUTLASS_DEVICE FragmentConstIterator( + FragmentIterator<Fragment_, Iterations_, AccessType_> const& rhs_) + : pointer(reinterpret_cast<Element const*>(rhs_.offset)) {} + + /// The accessor. + CUTLASS_DEVICE AccessType const& at(int d, int h, int w, int c = 0) const { + int const imm = ComputeOffsetFromStrides<IterationsStrides>::get(d, h, w, c); + return reinterpret_cast<AccessType const&>(pointer[imm]); + } + + /// The accessor. + CUTLASS_DEVICE AccessType const& operator[](int i) const { + return reinterpret_cast<AccessType const&>(pointer[i * kElementsPerAccess]); + } + + /// Is the iterator valid? + CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { return true; } + + /// The pointer. + Element const* pointer; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/fragment_load_store.h b/cutlass-example/cutlass/fragment_load_store.h new file mode 100644 index 0000000..a7d272e --- /dev/null +++ b/cutlass-example/cutlass/fragment_load_store.h @@ -0,0 +1,135 @@ +/*************************************************************************************************** + * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines accessors for loading and storing fragments to memory efficiently. +*/ +#pragma once + +#include <cutlass/load_store.h> +#include <cutlass/vector.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <IteratorFragment::Kind kIteratorFragment, + int kAccessSize, + typename Scalar_, + MemorySpace::Kind Memory_, + typename FragmentElement_, + int kStride> +struct FragmentLoad {}; + +template <int kAccessSize, + typename Scalar_, + MemorySpace::Kind Memory_, + typename FragmentElement_, + int kStride> +struct FragmentLoad<IteratorFragment::kWmmaMatrix, + kAccessSize, + Scalar_, + Memory_, + FragmentElement_, + kStride> { + /// The output type. + typedef FragmentElement_ AccessType; + + /// The load function. + static CUTLASS_DEVICE void load(AccessType& value, Scalar_ const* pointer, int offset) { + value.load(&pointer[offset], kStride); + } +}; + +template <int kAccessSize, + typename Scalar_, + MemorySpace::Kind Memory_, + typename FragmentElement_, + int kStride> +struct FragmentLoad<IteratorFragment::kScalar, + kAccessSize, + Scalar_, + Memory_, + FragmentElement_, + kStride> { + /// The output type. + typedef typename Vectorize<Scalar_, kAccessSize>::Type AccessType; + + /// The load function. + static CUTLASS_DEVICE void load(AccessType& value, Scalar_ const* pointer, int offset) { + Load<Scalar_, kAccessSize, Memory_>::load(value, pointer, offset); + } +}; + +template <IteratorFragment::Kind kIteratorFragment, + int kAccessSize, + typename Scalar_, + MemorySpace::Kind Memory_, + typename FragmentElement_, + int kStride> +struct FragmentStore {}; + +template <int kAccessSize, + typename Scalar_, + MemorySpace::Kind Memory_, + typename FragmentElement_, + int kStride> +struct FragmentStore<IteratorFragment::kWmmaMatrix, + kAccessSize, + Scalar_, + Memory_, + FragmentElement_, + kStride> { + /// The input type. + typedef FragmentElement_ AccessType; + + /// The store function. + static CUTLASS_DEVICE void store(AccessType const& value, Scalar_* pointer, int offset) { + value.store(&pointer[offset], kStride); + } +}; + +template <int kAccessSize, + typename Scalar_, + MemorySpace::Kind Memory_, + typename FragmentElement_, + int kStride> +struct FragmentStore<IteratorFragment::kScalar, + kAccessSize, + Scalar_, + Memory_, + FragmentElement_, + kStride> { + /// The input type. + typedef typename Vectorize<Scalar_, kAccessSize>::Type AccessType; + + /// The store function. + static CUTLASS_DEVICE void store(AccessType const& value, Scalar_* pointer, int offset) { + Store<Scalar_, kAccessSize, Memory_>::store(value, pointer, offset); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} /// namespace cutlass diff --git a/cutlass-example/cutlass/fragment_multiply_add.h b/cutlass-example/cutlass/fragment_multiply_add.h new file mode 100644 index 0000000..36a4d6f --- /dev/null +++ b/cutlass-example/cutlass/fragment_multiply_add.h @@ -0,0 +1,149 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines multiply-add operations on fragments within a thread. +*/ +#pragma once + +#include <cutlass/fragment.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_> +struct FragmentMultiplyAdd { + /// The shape of the instruction. + typedef Shape<1, 1, 1, 1> InstructionShape; + /// The type for A. + typedef Scalar_ ScalarA; + /// The type for B. + typedef Scalar_ ScalarB; + /// The type for C and D. + typedef Scalar_ ScalarC; + + /// Ctor. + CUTLASS_DEVICE FragmentMultiplyAdd() {} + + /// Multiply : d = a*b. + template <typename FragmentB_, typename FragmentCd_> + CUTLASS_DEVICE void multiply(Scalar_ a, FragmentB_ const& b, FragmentCd_& d) { + int const kReduction = FragmentB_::kElements / FragmentCd_::kElements; + for (int j = 0; j < FragmentCd_::kElements; ++j) { + d[j] = a * b[j * kReduction + 0]; + for (int k = 1; k < kReduction; ++k) { + d[j] += a * b[j * kReduction + k]; + } + } + } + + /// Multiply : d = a*b + c. + template <typename FragmentB_, typename FragmentCd_> + CUTLASS_DEVICE void multiply_add(Scalar_ a, + FragmentB_ const& b, + FragmentCd_ const& c, + FragmentCd_& d) { + int const kReduction = FragmentB_::kElements / FragmentCd_::kElements; + for (int j = 0; j < FragmentCd_::kElements; ++j) { + d[j] = a * b[j * kReduction + 0] + c[j]; + for (int k = 1; k < kReduction; ++k) { + d[j] += a * b[j * kReduction + k]; + } + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#if !defined(__CUDACC_RTC__) || defined(CUTLASS_NVRTC_HAS_FP16) +template <> +struct FragmentMultiplyAdd<half> { + /// The shape of the instruction. + typedef Shape<1, 1, 2, 1> InstructionShape; + /// The type for A. + typedef half ScalarA; + /// The type for B. + typedef half ScalarB; + /// The type for C and D. + typedef half ScalarC; + + /// Ctor. + CUTLASS_DEVICE FragmentMultiplyAdd() {} + + /// Multiply : d = a*b. + template <typename FragmentB_, typename FragmentCd_> + CUTLASS_DEVICE void multiply(half a, FragmentB_ const& b, FragmentCd_& d) { +#if defined(__CUDACC__) && __CUDA_ARCH__ >= 530 + + // Assemble a half2 from a. + __half2 const a_half2 = __half2half2(a); + // The input. + __half2 const* b_half2 = reinterpret_cast<__half2 const*>(&b[0]); + // The output. + __half2* d_half2 = reinterpret_cast<__half2*>(&d[0]); + + int const kReduction = FragmentB_::kElements / FragmentCd_::kElements; + for (int j = 0; j < FragmentCd_::kElements / 2; ++j) { + d_half2[j] = __hmul2(a_half2, b_half2[j * kReduction + 0]); + for (int k = 1; k < kReduction; ++k) { + d_half2[j] = __hfma2(a_half2, b_half2[j * kReduction + k], d_half2[j]); + } + } +#endif + } + + /// Multiply : d = a*b + c. + template <typename FragmentB_, typename FragmentCd_> + CUTLASS_DEVICE void multiply_add(half a, + FragmentB_ const& b, + FragmentCd_ const& c, + FragmentCd_& d) { +#if defined(__CUDACC__) && __CUDA_ARCH__ >= 530 + // Assemble a half2 from a. + __half2 const a_half2 = __half2half2(a); + // The inputs. + __half2 const* b_half2 = reinterpret_cast<__half2 const*>(&b[0]); + __half2 const* c_half2 = reinterpret_cast<__half2 const*>(&c[0]); + // The output. + __half2* d_half2 = reinterpret_cast<__half2*>(&d[0]); + + int const kReduction = (FragmentB_::kElements / FragmentCd_::kElements); + for (int j = 0; j < FragmentCd_::kElements / 2; ++j) { + d_half2[j] = __hfma2(a_half2, b_half2[j * kReduction + 0], c_half2[j]); + for (int k = 1; k < kReduction; ++k) { + d_half2[j] = __hfma2(a_half2, b_half2[j * kReduction + k], d_half2[j]); + } + } +#endif + } +}; + +#endif + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/clear_accumulators.h b/cutlass-example/cutlass/gemm/clear_accumulators.h new file mode 100644 index 0000000..441370f --- /dev/null +++ b/cutlass-example/cutlass/gemm/clear_accumulators.h @@ -0,0 +1,57 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines abstractions for efficiently clearing accumulator tiles. +*/ +#pragma once + +#include <cutlass/vector.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, int kLanes_ = 1> +struct ClearAccumulators { + /// The shared storage. + struct SharedStorage {}; + + /// Ctor. + CUTLASS_DEVICE ClearAccumulators() {} + /// Ctor. + CUTLASS_DEVICE ClearAccumulators(SharedStorage& shared_storage) {} + + /// Clear the fragment. + template <typename Fragment_> + CUTLASS_DEVICE void clear(Fragment_& fragment) { + fragment.clear(); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/dgemm_traits.h b/cutlass-example/cutlass/gemm/dgemm_traits.h new file mode 100644 index 0000000..0bbc221 --- /dev/null +++ b/cutlass-example/cutlass/gemm/dgemm_traits.h @@ -0,0 +1,127 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines structural traits of double-precision GEMM. +*/ +#pragma once + +#include <cutlass/gemm/gemm.h> +#include <cutlass/gemm/gemm_epilogue.h> +#include <cutlass/gemm/gemm_epilogue_traits.h> +#include <cutlass/gemm/gemm_global_tile.h> +#include <cutlass/gemm/gemm_shared_tile.h> +#include <cutlass/gemm/gemm_traits.h> +#include <cutlass/gemm/thread_multiply_add.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The tile size for the GEMM KxNxM. + typename OutputTile_, + /// The number of accumulators per thread. + typename AccumulatorsPerThread_, + /// The number of scalars per LDG for A. + int kScalarsPerLdgA_ = 1, + /// The number of scalars per LDG for B. + int kScalarsPerLdgB_ = 1> +struct DgemmConfig + : public GemmConfig< + /// The scalar type for A. + double, + /// The scalar type for B. + double, + /// The scalar type for C. + double, + /// The scalar type for D. + double, + /// The tile size for the GEMM KxNxM. + OutputTile_, + /// The functor to do the math in the main loop. + ThreadMultiplyAdd<AccumulatorsPerThread_, Shape<1, 4, 8>, double, double, double>, + /// The number of scalars per LDG for A. + kScalarsPerLdgA_, + /// The number of scalars per STS for A. + kScalarsPerLdgA_, + /// The number of scalars per LDS for A. + 2, + /// The number of scalars per LDG for B. + kScalarsPerLdgB_, + /// The number of scalars per STS for B. + kScalarsPerLdgB_, + /// The number of scalars per LDS for B. + 2, + /// The number of scalars per LDG for C and STG for D. + 1, + /// The number of scalars per STS for D. + 2, + /// The number of scalars per LDS for D. + 1, + /// The number of stages in shared memory. + 2> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The layout for A. + MatrixLayout::Kind kLayoutA_, + /// The layout for B. + MatrixLayout::Kind kLayoutB_, + /// The output tile. + typename OutputTile_ = Shape<8, 64, 128>, + /// The functor to use in the epilogue. + typename EpilogueFunctor_ = LinearScaling<double>, + /// The number of accumulators per thread. + typename AccumulatorsPerThread_ = Shape<8, 8, 8>, + /// The number of doubles loaded in one LDG for A. + int kScalarsPerLdgA_ = 1, + /// The number of doubles loaded in one LDG for B. + int kScalarsPerLdgB_ = 1, + /// The index. + typename Index_ = int, + /// The DGEMM config. + typename GemmConfig_ = + DgemmConfig<OutputTile_, AccumulatorsPerThread_, kScalarsPerLdgA_, kScalarsPerLdgB_>, + /// The traits class for the epilogue. + typename GemmEpilogueTraits_ = + SimplifiedGemmEpilogueTraits<GemmConfig_, EpilogueFunctor_, Index_> > +struct DgemmTraits : public SimplifiedGemmTraits< + // The layout for A. + kLayoutA_, + // The layout for B. + kLayoutB_, + // The config. + GemmConfig_, + // The epilogue. + GemmEpilogue<GemmEpilogueTraits_>, + // The index. + Index_> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm.h b/cutlass-example/cutlass/gemm/gemm.h new file mode 100644 index 0000000..c50a3f0 --- /dev/null +++ b/cutlass-example/cutlass/gemm/gemm.h @@ -0,0 +1,344 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Implements a software-pipelined efficient GEMM. +*/ +#pragma once + +#if !defined(__CUDACC_RTC__) +#include <cuda.h> +#endif + +#include <cutlass/coord.h> +#include <cutlass/util/platform.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Gemm_> +__global__ /*__launch_bounds__(Gemm_::kThreads)*/ void gemm_kernel(typename Gemm_::Params params) { + // Declare shared memory. + __shared__ typename Gemm_::SharedStorage shared_storage; + + // Construct the GEMM object. + Gemm_ gemm(params, shared_storage); + // Run GEMM. + gemm.multiply_add(); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, typename Index_ = int> +struct GemmDesc { + /// The dimensions of the GEMM. + Index_ m, n, k; + /// The alpha/beta scaling values. + Scalar_ alpha, beta; + /// The source matrix A. + void const* d_a; + /// The stride for A. + Index_ lda; + /// The source matrix B. + void const* d_b; + /// The stride for B. + Index_ ldb; + /// The source matrix C. + void const* d_c; + /// The stride for C. + Index_ ldc; + /// The destination matrix D. + void* d_d; + /// The stride for D. + Index_ ldd; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmTraits_> +struct Gemm { + /// This class. + typedef Gemm<GemmTraits_> This_; + /// The traits. + typedef GemmTraits_ Traits; + /// The shared storage. + typedef typename Traits::SharedStorage SharedStorage; + + /// The scalar for A. + typedef typename Traits::ScalarA ScalarA; + /// The scalar for B. + typedef typename Traits::ScalarB ScalarB; + /// The scalar in the epilogue. + typedef typename Traits::Epilogue::Scalar ScalarEpilogue; + /// The scalar for C. + typedef typename Traits::Epilogue::ScalarC ScalarC; + /// The scalar for D. + typedef typename Traits::Epilogue::ScalarD ScalarD; + /// The index. + typedef typename Traits::Index Index; + + /// The number of threads. + static int const kThreads = Traits::GemmConfig::kThreads; + + /// The params. + struct Params : public Traits::Params { + CUTLASS_HOST_DEVICE int initialize(Index m, + Index n, + Index k, + ScalarEpilogue alpha, + ScalarA const* d_a, + Index lda, + ScalarB const* d_b, + Index ldb, + ScalarEpilogue beta, + ScalarC const* d_c, + Index ldc, + ScalarD* d_d, + Index ldd) { + GemmDesc<ScalarEpilogue, Index> desc; + desc.m = m; + desc.n = n; + desc.k = k; + desc.alpha = alpha; + desc.beta = beta; + desc.d_a = reinterpret_cast<void const*>(d_a); + desc.lda = lda; + desc.d_b = reinterpret_cast<void const*>(d_b); + desc.ldb = ldb; + desc.d_c = reinterpret_cast<void const*>(d_c); + desc.ldc = ldc; + desc.d_d = reinterpret_cast<void*>(d_d); + desc.ldd = ldd; + return Traits::Params::initialize(desc); + } + }; + +#if !defined(__CUDACC_RTC__) + /// Launch the kernel. + static __host__ cudaError_t launch(Params const& params, + cudaStream_t stream = cudaStreamDefault) { + // Setup the grid. + dim3 grid; + grid.x = (params.m + Traits::OutputTile::kW - 1) / Traits::OutputTile::kW; + grid.y = (params.n + Traits::OutputTile::kH - 1) / Traits::OutputTile::kH; + + // The number of threads. + dim3 block; + block.x = kThreads; + + // Launch the kernel. + void const* params_ = reinterpret_cast<void const*>(¶ms); + + return cudaLaunchKernel(reinterpret_cast<void*>(&gemm_kernel<This_>), + grid, + block, + const_cast<void**>(¶ms_), + 0, + stream); + } + + /// Launch the kernel. + static __host__ cudaError_t launch(CUfunction kernel, + Params const& params, + CUstream stream = CU_STREAM_LEGACY) { + // Setup the grid. + dim3 grid; + grid.x = (params.m + Traits::OutputTile::kW - 1) / Traits::OutputTile::kW; + grid.y = (params.n + Traits::OutputTile::kH - 1) / Traits::OutputTile::kH; + + // The number of threads. + dim3 block; + block.x = kThreads; + + // Launch the kernel. + void* params_[] = {const_cast<void*>(reinterpret_cast<void const*>(¶ms))}; + + // return cudaLaunchKernel(reinterpret_cast<void*>(&gemm_kernel<This_>), grid, block, + // const_cast<void**>(¶ms_), 0, stream); + CUresult result = cuLaunchKernel( + kernel, grid.x, grid.y, grid.z, block.x, block.y, block.z, 0, stream, params_, 0); + + if (result != CUDA_SUCCESS) { + return cudaErrorLaunchFailure; + } + return cudaSuccess; + } + +#endif + + /// Ctor. + CUTLASS_DEVICE Gemm(Params const& params_, SharedStorage& shared_storage_) + : params(params_), shared_storage(shared_storage_) {} + + /// Consume a single iteration of the loop. + template <bool kIsLastIteration> + CUTLASS_DEVICE void consume_tile(typename Traits::GlobalLoadStream& global_stream, + typename Traits::SharedLoadStream& shared_load_stream, + typename Traits::MultiplyAdd::Accumulators& accumulators, + Index outer_k) { + // If that's the last "load iteration" update the predicates. + if (!kIsLastIteration) { + global_stream.move_to_residue<false>(outer_k); + } + + // Load data for the next iteration of the main loop. + if (!kIsLastIteration) { + global_stream.copy(); + } + + // The unrolling steps for the main loop. + int const kUnrollingSteps = + Traits::MultiplyAdd::AccumulatorsPerWarp::kD / Traits::MultiplyAdd::InstructionShape::kD; + + CUTLASS_PRAGMA_UNROLL + for (int step = 0; step < kUnrollingSteps - 1; ++step) { + // Trigger the copy from shared memory for the next A/B values. + shared_load_stream.copy(step + 1); + // Make sure the values are available for the current iteration to do the multiply-add. + shared_load_stream.commit(step); + + // Do the math on the fragments of the current iteration. + typename Traits::MultiplyAdd multiply_add; + multiply_add.multiply_add(shared_load_stream.fragment_a(step), + shared_load_stream.fragment_b(step), + accumulators, + accumulators); + } + + // Make sure the data from shared memory has been entirely consumed. + Traits::shared_load_fence(true); + + // Commit the data in shared memory for A/B. + if (!kIsLastIteration) { + global_stream.commit(); + } + + // Make sure the data is in shared memory. + Traits::shared_store_fence(true); + + // Trigger the loads for the next iteration (if needed). + if (!kIsLastIteration) { + // Move to the next stage for the load (if it makes sense). + shared_load_stream.inc_stage(); + // Trigger the copy from shared memory for the next loop iteration. + shared_load_stream.copy(0); + } + + // Make sure the values are available for the current iteration to do the multiply-add. + shared_load_stream.commit(kUnrollingSteps - 1); + + // Do the math on the fragments of the current iteration. + typename Traits::MultiplyAdd multiply_add; + multiply_add.multiply_add(shared_load_stream.fragment_a(kUnrollingSteps - 1), + shared_load_stream.fragment_b(kUnrollingSteps - 1), + accumulators, + accumulators); + } + + /// Do the GEMM. + CUTLASS_DEVICE void multiply_add() { + // Swizzle the IDs of the block (to enable better cache behavior). + typename Traits::BlockSwizzle block_swizzle; + dim3 block = block_swizzle.swizzle(); + + // Scale the id. + block.x *= Traits::OutputTile::kW; + block.y *= Traits::OutputTile::kH; + + // We may want to use shared memory to clear the registers. + typedef typename Traits::ClearAccumulators ClearAccumulators; + + // The streams to read A/B from global memory to shared memory. + typename Traits::GlobalLoadStream global_stream(params, shared_storage, block); + + // Create the accumulator clear. + ClearAccumulators clear(shared_storage.main_loop.clear); + + // By how much we unroll the main loop. + Index const kUnroll = static_cast<Index>(Traits::OutputTile::kD); + + // If we do not have enough steps in the main loop, trigger the residue code. + global_stream.move_to_residue<true>(params.k); + + // Fetch the fragments for A and B from global memory. + global_stream.copy(); + + // Copy the elements to shared memory (after transformation if needed). + global_stream.commit(); + + // Make sure the data is in shared memory. + Traits::shared_store_fence(false); + + // Rollback to the beginning of the GEMM-K dimension. It may have no impact. + global_stream.rollback(); + + // The unrolling steps for the main loop. + int const kUnrollingSteps = + Traits::MultiplyAdd::AccumulatorsPerWarp::kD / Traits::MultiplyAdd::InstructionShape::kD; + + // Make sure we have at least 2 unrolling steps or our pipeling is not going to work. + static_assert(kUnrollingSteps >= 2, "The pipelining assumes at least two steps"); + + // The stream of data from shared memory to fragments. + typename Traits::SharedLoadStream shared_load_stream(params, shared_storage); + + // Trigger the copy from shared memory for the 1st stream. + shared_load_stream.copy(0); + + // Allocate the accumulators. + typename Traits::MultiplyAdd::Accumulators accumulators; + // Clear the accumulators. + clear.clear(accumulators); + + // The loop index. + Index outer_k = params.k - kUnroll; + + // Enter the main loop and iterate. + for (; outer_k > 0; outer_k -= kUnroll) { + consume_tile<false>(global_stream, shared_load_stream, accumulators, outer_k); + } + + // Residual loop. + for (; outer_k > -kUnroll; outer_k -= kUnroll) { + consume_tile<true>(global_stream, shared_load_stream, accumulators, outer_k); + } + + // Epilogue. + typedef typename Traits::Epilogue Epilogue; + Epilogue epilogue(params.epilogue, shared_storage.epilogue, params.m, params.n); + epilogue.epilogue(cutlass::make_Coord(0, block.y, block.x), accumulators); + } + + /// The params. + Params const& params; + /// The shared storage. + SharedStorage& shared_storage; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_epilogue.h b/cutlass-example/cutlass/gemm/gemm_epilogue.h new file mode 100644 index 0000000..bc25307 --- /dev/null +++ b/cutlass-example/cutlass/gemm/gemm_epilogue.h @@ -0,0 +1,231 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Implements the epilogue phase of the GEMM kernel that efficiently updates global memory + with + the computed matrix product. +*/ +#pragma once + +#include <cutlass/convert.h> +#include <cutlass/coord.h> +#include <cutlass/fragment.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename T> +CUTLASS_DEVICE bool is_zero(T x) { + return x == T(0); +} + +#if !defined(__CUDACC_RTC__) || defined(CUTLASS_NVRTC_HAS_FP16) +CUTLASS_DEVICE bool is_zero(half x) { return reinterpret_cast<int16_t&>(x) == int16_t(0); } +#endif + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmEpilogueTraits_> +struct GemmEpilogue { + /// The traits class. + typedef GemmEpilogueTraits_ Traits; + /// The params. + typedef typename Traits::Params Params; + /// The shared storage. + typedef typename Traits::SharedStorage SharedStorage; + + /// The output tile. + typedef typename Traits::OutputTile OutputTile; + /// The number of iterations. + typedef typename Traits::Iterations Iterations; + /// The accumulators. + typedef typename Traits::Accumulators Accumulators; + /// The scalar. + typedef typename Traits::Scalar Scalar; + /// The functor in charge of the math. + typedef typename Traits::Functor Functor; + + /// We do not support 3D or 4D shapes. + static_assert(Iterations::kD == 1 && Iterations::kC == 1, "Unsupported 3D/4D shapes"); + + /// The iterator for C in global memory. + typedef typename Traits::GlobalLoadIteratorC GlobalLoadIteratorC; + /// The transformer for C. + typedef typename Traits::GlobalTransformerC GlobalTransformerC; + /// The transformer for D. + typedef typename Traits::GlobalTransformerD GlobalTransformerD; + /// The iterator for D in global memory. + typedef typename Traits::GlobalStoreIteratorD GlobalStoreIteratorD; + /// The iterator to store D in shared memory. + typedef typename Traits::SharedStoreIteratorD SharedStoreIteratorD; + /// The shared store transformer for D. + typedef typename Traits::SharedStoreTransformerD SharedStoreTransformerD; + /// The iterator to load D in shared memory. + typedef typename Traits::SharedLoadIteratorD SharedLoadIteratorD; + /// The shared load transformer for D. + typedef Copy<typename SharedLoadIteratorD::Fragment> SharedLoadTransformerD; + + /// The index. + typedef typename Traits::Index Index; + + /// The scalar for C. + typedef typename GlobalLoadIteratorC::Scalar ScalarC; + /// The scalar for D. + typedef typename GlobalStoreIteratorD::Scalar ScalarD; + + /// Ctor. + CUTLASS_DEVICE GemmEpilogue(Params const& params_, + SharedStorage& shared_storage_, + Index m_, + Index n_) + : params(params_), shared_storage(shared_storage_), m(m_), n(n_) {} + + /// Execute the epilogue. + CUTLASS_DEVICE void epilogue(Coord<3> const& block, Accumulators& accumulators) { + if (is_zero(params.functor.beta)) { + epilogue_with_or_without_beta<true>(block, accumulators); + } else { + epilogue_with_or_without_beta<false>(block, accumulators); + } + } + + template <bool kBetaIsZero_> + CUTLASS_DEVICE void epilogue_with_or_without_beta(Coord<3> const& block, + Accumulators& accumulators) { + + // The problem size. + Coord<3> const bounds = cutlass::make_Coord(0, n, m); + + // The functor. + Functor functor(params.functor); + // The C fragment. + typename GlobalLoadIteratorC::Fragment fragment_c; + // The transformed C fragment. + typename GlobalTransformerC::OutputFragment transformed_c; + + CUTLASS_PRAGMA_UNROLL + for (int h = 0; h < Iterations::kH; ++h) { + // Compute pointer and predicate offsets for C and D global iterators. + int const pointer_offset = + ((params.iterator_d.inc_h * (GlobalStoreIteratorD::Iterations::kH - 1) + + params.iterator_d.inc_advance) * + Iterations::kW + + params.stride_h) * + h; + int const predicate_offset = + ((params.iterator_d.predicate_inc_h * (GlobalStoreIteratorD::Iterations::kH - 1) + + params.iterator_d.predicate_inc_advance) * + Iterations::kW + + Traits::Delta::kH) * + h; + + // The iterator to load the elements of the C matrix. + GlobalLoadIteratorC global_load_iterator( + params.iterator_c, bounds, block, pointer_offset, predicate_offset); + // The transformer for C. + GlobalTransformerC transformer_c; + // The transformer for D. + GlobalTransformerD transformer_d; + // The iterator to store into the D matrix. + GlobalStoreIteratorD global_store_iterator( + params.iterator_d, bounds, block, pointer_offset, predicate_offset); + + // The transformer to transform before storing to shared memory. + SharedStoreTransformerD shared_store_transformer; + typename SharedStoreTransformerD::OutputFragment shared_store_transformed_d; + + // The iterator to store to shared memory. + SharedStoreIteratorD shared_store_iterator(params.shared_store_iterator_d, + shared_storage.shared_stream.store); + + // The iterator to load from shared memory. TODO: Use a stream. + SharedLoadIteratorD shared_load_iterator(params.shared_load_iterator_d, + shared_storage.shared_stream.load); + + CUTLASS_PRAGMA_UNROLL + for (int w = 0; w < Iterations::kW; ++w) { + // Load the C matrix into fragment. + if (!kBetaIsZero_) { + iterator_load(global_load_iterator, fragment_c); + } + + // Make sure we can write to shared memory. + shared_load_fence(); + + // Copy the accumulators to shared memory. + int const offset = (h * Iterations::kW + w) * SharedStoreIteratorD::Fragment::kElements; + + shared_store_transformer.transform(accumulators, offset, shared_store_transformed_d); + shared_iterator_store(shared_store_iterator, shared_store_transformed_d); + + // Make sure the data is in shared memory. + shared_store_fence(); + + // Copy the accumulators back to registers from shared memory. + typename SharedLoadIteratorD::Fragment fetched_d; + shared_iterator_load(shared_load_iterator, fetched_d); + + // Do the math. + typename GlobalTransformerD::InputFragment fragment_d; + + if (kBetaIsZero_) { + functor.evaluate(fetched_d, fragment_d); + } else { + // Transform C fragment. + transformer_c.transform(fragment_c, transformed_c); + // Do the math. + functor.evaluate(fetched_d, transformed_c, fragment_d); + } + + // Transform D fragment. + typename GlobalTransformerD::OutputFragment transformed_d; + transformer_d.transform(fragment_d, transformed_d); + + // Copy the results to global memory. + iterator_store(global_store_iterator, transformed_d); + } + } + } + + /// The memory fence for shared loads. + CUTLASS_DEVICE void shared_load_fence() { __syncthreads(); } + + /// The memory fence for shared stores. + CUTLASS_DEVICE void shared_store_fence() { __syncthreads(); } + + /// The params. + Params const& params; + /// The shared storage. + SharedStorage& shared_storage; + /// The dimensions of the GEMM. + Index m, n; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_epilogue_traits.h b/cutlass-example/cutlass/gemm/gemm_epilogue_traits.h new file mode 100644 index 0000000..c06fc25 --- /dev/null +++ b/cutlass-example/cutlass/gemm/gemm_epilogue_traits.h @@ -0,0 +1,331 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines structural properties of the GEMM epilogue. +*/ +#pragma once + +#include <cutlass/convert.h> +#include <cutlass/coord.h> +#include <cutlass/gemm/gemm_global_stream.h> +#include <cutlass/gemm/gemm_shared_stream.h> +#include <cutlass/gemm/linear_scaling.h> +#include <cutlass/reshape_tile.h> +#include <cutlass/tile_iterator.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The output tile. + typename OutputTile_, + /// The accumulators. + typename Accumulators_, + /// The iterator to load C from global memory. + typename GlobalLoadIteratorC_, + /// The transformer for C. + typename GlobalTransformerC_, + /// The transformer for D. + typename GlobalTransformerD_, + /// The iterator to store D to global memory. + typename GlobalStoreIteratorD_, + /// The iterator to store D to shared memory. + typename SharedStoreIteratorD_, + /// The shared store transformer for D. + typename SharedStoreTransformerD_, + /// The iterator to load D from shared memory. + typename SharedLoadIteratorD_, + /// The number of iterations in the epilogue. + typename Iterations_, + /// The iterations strides. + typename Delta_, + /// The functor to be used in the epilogue. + typename Functor_, + /// The index. + typename Index_ = int> +struct GemmEpilogueTraits { + // + /// The output tile. + typedef OutputTile_ OutputTile; + /// The number of iterations. + /// The accumulators. + typedef Accumulators_ Accumulators; + /// The iterator for C in global memory. + typedef GlobalLoadIteratorC_ GlobalLoadIteratorC; + /// The transformer for C. + typedef GlobalTransformerC_ GlobalTransformerC; + /// The transformer for D. + typedef GlobalTransformerD_ GlobalTransformerD; + /// The iterator for D in global memory. + typedef GlobalStoreIteratorD_ GlobalStoreIteratorD; + /// The iterator to store D in shared memory. + typedef SharedStoreIteratorD_ SharedStoreIteratorD; + /// The shared store transformer for D. + typedef SharedStoreTransformerD_ SharedStoreTransformerD; + /// The iterator to store D in shared memory. + typedef SharedLoadIteratorD_ SharedLoadIteratorD; + /// typedef typename GemmConfig::EpilogueIterations Iterations; + typedef Iterations_ Iterations; + /// The iterations strides. + typedef Delta_ Delta; + + /// The functor in charge of the math. + typedef Functor_ Functor; + /// The index. + typedef Index_ Index; + + /// We do not support 3D or 4D shapes. + static_assert(Iterations::kD == 1 && Iterations::kC == 1, "Unsupported 3D/4D shapes"); + + /// The scalar. + typedef typename Functor::Scalar Scalar; + /// The scalar for C. + typedef typename GlobalLoadIteratorC::Scalar ScalarC; + /// The scalar for D. + typedef typename GlobalStoreIteratorD::Scalar ScalarD; + + /// The params. + struct Params { + /// The strides for H and W in the different iterations of the epilogue. + Index stride_h, stride_w; + /// The params for the C iterator. + typename GlobalLoadIteratorC::Params iterator_c; + /// The params for the D global iterator. + typename GlobalStoreIteratorD::Params iterator_d; + /// The params for the D shared store iterator. + typename SharedStoreIteratorD::Params shared_store_iterator_d; + /// The params for the D shared load iterator. + typename SharedLoadIteratorD::Params shared_load_iterator_d; + /// The functor params. + typename Functor::Params functor; + + /// Setup the params. + template <typename GemmDesc_> + CUTLASS_HOST_DEVICE int initialize(GemmDesc_ const& desc) { + // The parameters for the functor. + int error_code = functor.initialize(desc); + if (error_code) { + return error_code; + } + + // At the end of the H iteration, we jump over a number of columns. + this->stride_h = desc.ldd * Delta::kH; + // Nothing to do here. + this->stride_w = 0; + + // Setup the params for the global memory iterator for C. + error_code = iterator_c.initialize( + reinterpret_cast<ScalarC const*>(desc.d_c), desc.ldc, desc.n, stride_w, Delta::kW); + if (error_code) { + return error_code; + } + + // Setup the params for the global memory iterator for D. + return iterator_d.initialize( + reinterpret_cast<ScalarD*>(desc.d_d), desc.ldd, desc.n, stride_w, Delta::kW); + } + }; + + /// The shared memory storage to exchange data. + union StreamSharedStorage { + // The storage for the store iterator. + typename SharedStoreIteratorD::SharedStorage store; + // The storage for the store iterator. + typename SharedLoadIteratorD::SharedStorage load; + }; + + /// The shared memory to swizzle the data in the epilogue. + struct SharedStorage { + // The storage for the shared stream D. + StreamSharedStorage shared_stream; + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_, typename EpilogueFunctor_, typename Index_ = int> +struct GemmEpilogueTraitsHelper { + /// The scalar. + typedef typename EpilogueFunctor_::Scalar Scalar; + /// The output tile. + typedef typename GemmConfig_::OutputTile OutputTile; + + /// The number of iterations in the epilogue. + typedef Shape<1, + GemmConfig_::MultiplyAdd::AccumulatorsPerThread::kH / + GemmConfig_::kAccumulatorsPerLdsB, + GemmConfig_::kAccumulatorsPerLdsB> + Iterations; + // The iteration strides in the H/W dimension. + typedef Shape<0, + GemmConfig_::kAccumulatorsPerLdsB*( + GemmConfig_::Warps::kH* GemmConfig_::MultiplyAdd::ThreadsPerWarp::kH - 1), + 0> + Delta; + /// The functor to do the math in the epilogue. + typedef EpilogueFunctor_ Functor; + + /// The traits class to build the iterator to store to shared memory for D. + typedef GemmSharedStoreTileDTraits< + // The pointer is float. + typename Functor::Scalar, + // The output tile size. + typename GemmConfig_::OutputTile, + // The number of warps. + typename GemmConfig_::Warps, + // The number of threads per warp. + typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, + // The number of scalars per STS. + GemmConfig_::kScalarsPerStsD, + // The skew -- 128 / sizeof(ScalarD) / kScalarsPerStsD is the number of threads involved in + // a single STS. We divide by 2 as our objective is to add a skew to the odd threads to + // avoid bank conflicts between odd and even threads. + 128 / sizeof(typename GemmConfig_::ScalarD) / GemmConfig_::kScalarsPerStsD / 2 * + GemmConfig_::kScalarsPerStsD> + SharedStoreTileTraits; + + /// The iterator to store D to shared memory. + typedef TileStoreIterator<SharedStoreTileTraits, + typename SharedStoreTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedStoreIteratorD; + + /// The shared store transformer for D. + typedef Copy<typename SharedStoreIteratorD::Fragment> SharedStoreTransformerD; + + /// The traits class to build the iterator to load from shared memory for D. + typedef GemmSharedLoadTileDTraits< + // The pointer is float. + typename Functor::Scalar, + // The output tile size. + typename GemmConfig_::OutputTile, + // The number of warps. + typename GemmConfig_::Warps, + // The number of threads per warp. + typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, + // The number of columns of the output tile written by iteration. + GemmConfig_::OutputTile::kH / ShapeCount<Iterations>::kCount, + // The number of scalars per LDS. + GemmConfig_::kScalarsPerLdsD, + // The skew. + SharedStoreTileTraits::kSkew> + SharedLoadTileTraits; + + /// The iterator to load D from shared memory. + typedef TileLoadIterator<SharedLoadTileTraits, + typename SharedLoadTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedLoadIteratorD; + + /// The traits class to build the iterator to load data from global memory for C^N. + typedef GemmGlobalTileCdTraits< + // The pointer is float const. + typename GemmConfig_::ScalarC const, + // The tile has size (N / Iterations)xM in GEMM's terminology. + Shape<1, + GemmConfig_::OutputTile::kH / ShapeCount<Iterations>::kCount, + GemmConfig_::OutputTile::kW>, + // The threads are distributed as warps x 32 (the traits may reorganize). + Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, + // How many elements do we jump over at each iteration? + Iterations::kW, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerLdgC> + GlobalLoadTileTraits; + + /// The iterator to load C. + typedef GemmGlobalIteratorCd<GlobalLoadTileTraits, Index_> GlobalLoadIteratorC; + /// The transformer for C. + typedef Copy<typename GlobalLoadIteratorC::Fragment> GlobalTransformerC; + + /// The traits class to build the iterator to store data to global memory for D^N. + typedef GemmGlobalTileCdTraits< + // The pointer is float. + typename GemmConfig_::ScalarD, + // The tile has size (N / Iterations)xM in GEMM's terminology. + Shape<1, + GemmConfig_::OutputTile::kH / ShapeCount<Iterations>::kCount, + GemmConfig_::OutputTile::kW>, + // The threads are distributed as warps x 32 (the traits may reorganize). + Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, + // How many elements do we jump over at each iteration? + Iterations::kW, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerStgD> + GlobalStoreTileTraits; + + /// The iterator to store D. + typedef GemmGlobalIteratorCd<GlobalStoreTileTraits, Index_> GlobalStoreIteratorD; + /// The transformer for D. + typedef Copy<typename GlobalStoreIteratorD::Fragment> GlobalTransformerD; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The GEMM config. + typename GemmConfig_, + /// The epilogue functor to do the math in the epilogue. + typename EpilogueFunctor_, + /// The index. + typename Index_ = int, + /// The helper to create the traits class. + typename Helper_ = GemmEpilogueTraitsHelper<GemmConfig_, EpilogueFunctor_, Index_> > +struct SimplifiedGemmEpilogueTraits : public GemmEpilogueTraits< + // The output tile. + typename GemmConfig_::OutputTile, + // The accumulators. + typename GemmConfig_::Accumulators, + // The global iterator for C. + typename Helper_::GlobalLoadIteratorC, + // The transformer for C. + typename Helper_::GlobalTransformerC, + // The transformer for D. + typename Helper_::GlobalTransformerD, + // The global iterator for D. + typename Helper_::GlobalStoreIteratorD, + // The iterator to store D to shared memory. + typename Helper_::SharedStoreIteratorD, + // The shared store transformer for D. + typename Helper_::SharedStoreTransformerD, + // The iterator to load D from shared memory. + typename Helper_::SharedLoadIteratorD, + // The number of iterations. + typename Helper_::Iterations, + // The strides between iterations. + typename Helper_::Delta, + // The functor to be used in the epilogue. + EpilogueFunctor_, + // The index. + Index_> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_global_stream.h b/cutlass-example/cutlass/gemm/gemm_global_stream.h new file mode 100644 index 0000000..ec675a3 --- /dev/null +++ b/cutlass-example/cutlass/gemm/gemm_global_stream.h @@ -0,0 +1,182 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Implements efficient loading of the thread block-level tile from global memory and + storing + to shared memory. +*/ +#pragma once + +#include <cutlass/convert.h> +#include <cutlass/gemm/gemm_global_tile.h> +#include <cutlass/iterator_access.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The load iterator. + typename LoadIterator_, + /// The store iterator to copy to shared memory. + typename StoreIterator_, + /// The transformer to be applied after the data has been copied from global memory. + typename Transformer_> + +struct GlobalLoadStreamBase { + /// The load iterator. + typedef LoadIterator_ LoadIterator; + /// The transformer. + typedef Transformer_ Transformer; + /// The store iterator to write to shared memory. + typedef StoreIterator_ StoreIterator; + + /// The fragment that is copied from shared memory. + typedef typename LoadIterator::Fragment FetchedFragment; + /// The fragment that is obtained after the transformation by the transformer. + typedef typename Transformer::OutputFragment TransformedFragment; + /// Make sure the fragments match. + static_assert((platform::is_same<FetchedFragment, typename Transformer::InputFragment>::value), + ""); + /// The output fragment. + typedef TransformedFragment Fragment; + /// Make sure the transformed fragment is the same as the store fragment. + static_assert((platform::is_same<TransformedFragment, typename StoreIterator::Fragment>::value), + ""); + + /// The layout. + static MatrixLayout::Kind const kLayout = LoadIterator::kLayout; + /// The scalar type of the iterator. + typedef typename LoadIterator::Scalar Scalar; + /// The pointer. + typedef typename LoadIterator::Pointer Pointer; + /// The index. + typedef typename LoadIterator::Index Index; + + /// The params. + struct Params { + // The load iterator. + typename LoadIterator::Params load_iterator; + // The store iterator. + typename StoreIterator::Params store_iterator; + + /// Setup the params. + template <typename GemmDesc_> + CUTLASS_HOST_DEVICE int initialize(GemmDesc_ const& desc, Pointer pointer, Index ld) { + int error_code = load_iterator.initialize(desc, pointer, ld); + if (error_code) { + return error_code; + } + + return store_iterator.initialize(); + } + }; + + /// The amount of storage in shared memory needed to store the tile. + typedef typename StoreIterator::SharedStorage SharedStoreStorage; + + /// The storage in shared memory needed by that stream. + union SharedStorage { + // The load iterator. + typename LoadIterator::SharedStorage load_iterator; + // The store iterator. + SharedStoreStorage store_iterator; + }; + + /// Ctor. + CUTLASS_DEVICE GlobalLoadStreamBase(Params const& params, + SharedStorage& shared_storage, + Coord<3> const bounds, + Coord<3> const& block) + : load_iterator(params.load_iterator, bounds, block), + transformer(), + store_iterator(params.store_iterator, shared_storage.store_iterator) + + { + fetched_fragment.clear(); + } + + /// Load the data from shared memory to the fetch fragment. + CUTLASS_DEVICE void copy() { iterator_load(load_iterator, fetched_fragment); } + + /// Commit the data. + CUTLASS_DEVICE void commit() { + transformer.transform(fetched_fragment, transformed_fragment); + iterator_store(store_iterator, transformed_fragment); + store_iterator.inc_stage(); + } + + /// Move to the beginning of the residue code. That's a new code path in CUTLASS 1.0.1. + CUTLASS_DEVICE void move_to_residue(Index k) { load_iterator.move_to_residue(k); } + + /// Execute the residue code. + CUTLASS_DEVICE void residue(Index k, bool skip_clear = false) { + load_iterator.residue(k); + if (!skip_clear) { + fetched_fragment.clear(); + } + } + + /// Rollback to the beginning of the GEMM-k dimension. + CUTLASS_DEVICE void rollback() { load_iterator.rollback(); } + + /// The iterator. + LoadIterator load_iterator; + /// The fragment to fetch from shared memory. + FetchedFragment fetched_fragment; + /// The transformer. + Transformer transformer; + /// The fragment to convert the data after it has been fetched from shared memory. + TransformedFragment transformed_fragment; + /// The store iterator. + StoreIterator store_iterator; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The load iterator. + typename LoadIterator_, + /// The store iterator to copy to shared memory. + typename StoreIterator_, + /// The transformer to be applied after the data has been copied from global memory. + typename Transformer_ = Copy<typename LoadIterator_::Fragment> > + +struct GlobalLoadStream : public GlobalLoadStreamBase<LoadIterator_, StoreIterator_, Transformer_> { + /// The base class. + typedef GlobalLoadStreamBase<LoadIterator_, StoreIterator_, Transformer_> Base; + + /// Ctor. + CUTLASS_DEVICE GlobalLoadStream(typename Base::Params const& params, + typename Base::SharedStorage& shared_storage, + Coord<3> const& bounds, + Coord<3> const& block) + : Base(params, shared_storage, bounds, block) {} +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_global_tile.h b/cutlass-example/cutlass/gemm/gemm_global_tile.h new file mode 100644 index 0000000..1cc3b33 --- /dev/null +++ b/cutlass-example/cutlass/gemm/gemm_global_tile.h @@ -0,0 +1,541 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines iterators for efficiently loading and storing to global memory. +*/ +#pragma once + +#include <cutlass/coord.h> +#include <cutlass/util/platform.h> + +#include <cutlass/gemm/gemm_operand.h> +#include <cutlass/matrix_traits.h> +#include <cutlass/predicate_vector.h> +#include <cutlass/reshape_tile.h> +#include <cutlass/tile_iterator.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// The following functor reshapes a tile of threads to match a tile of data. The idea is that when +// the user wants to build the iterator traits, he/she may want to specify the tile independently +// from the number of scalars loaded/stored per instruction. For example, in the row-major version +// with a tile of size 128x8 - the user may want to that the iterator works with 32x8 threads if +// each thread loads 1 scalar per LDG. If the user changes to 4 scalars per LDG, then the tile of +// threads has to change. The code below detects that and correct the code automatically - it is +// a helper when the user does not specify the right configuration. + +template <typename Tile_, typename Threads_, bool = (Tile_::kW < Threads_::kW)> +struct ReshapeThreads { + typedef Threads_ Threads; +}; + +template <typename Tile_, typename Threads_> +struct ReshapeThreads<Tile_, Threads_, true> { + typedef Shape<Threads_::kD, Threads_::kH * Threads_::kW / Tile_::kW, Tile_::kW, 1> Threads; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <GemmOperand::Kind kOperand_, + MatrixLayout::Kind kLayout_, + typename Scalar_, + typename Tile_, + typename Threads_, + int kAccessSize_> +struct GemmGlobalTileTraits { + /// Identity of the operand + static GemmOperand::Kind const kOperand = kOperand_; + /// The layout. + static MatrixLayout::Kind const kLayout = kLayout_; + /// The scalar. + typedef typename platform::remove_const<Scalar_>::type Scalar; + /// The pointer. + typedef Scalar_* Pointer; + /// The number of scalars per LDG/STG. + static int const kAccessSize = kAccessSize_; + /// The memory space. + static MemorySpace::Kind const kMemorySpace = MemorySpace::kGlobal; + + /// The tile shape + typedef typename ReshapeTile<Tile_, kAccessSize_>::Tile Tile; + /// The threads shape + typedef typename ReshapeThreads<Tile, Threads_>::Threads Threads; + /// The relative offset between two elements in the H/W dimension in adjacent threads. + typedef Shape<1, 1, Tile::kC> ThreadsDelta; + + /// The strides in each dimension between different loads/stores. + typedef Shape<0, Threads::kH, Threads::kW * kAccessSize> Delta; + /// Strides for immediate offset computation + typedef Shape<0, 0, Threads::kW * ThreadsDelta::kW, kAccessSize> ImmediateOffsetStrides; + /// The number of iterations needed to load/store the tile. + typedef Shape<1, Tile::kH / Threads::kH, Tile::kW / Threads::kW, Tile::kC / kAccessSize> + Iterations; + + typedef GemmMultiplicandTraits<Tile, kOperand, kLayout> MultiplicandTraits; + + /// Computes the thread offset in (H, W) based on thread ID + struct ThreadOffset { + CUTLASS_HOST_DEVICE + Coord<4> operator()() const { + int thread_offset_h = threadIdx.x / Threads::kW * ThreadsDelta::kH; + int thread_offset_w = threadIdx.x % Threads::kW * ThreadsDelta::kW; + + return make_Coord(0, thread_offset_h, thread_offset_w, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, typename Tile_, typename Threads_, int kStrideH_, int kAccessSize_> +struct GemmGlobalTileCdTraits : public GemmGlobalTileTraits<GemmOperand::kC, + MatrixLayout::kColumnMajor, + Scalar_, + Tile_, + Threads_, + kAccessSize_> { + /// The base class. + typedef GemmGlobalTileTraits<GemmOperand::kC, + MatrixLayout::kColumnMajor, + Scalar_, + Tile_, + Threads_, + kAccessSize_> + Base; + + /// The stride in the H dimension. + static int const kStrideH = kStrideH_; + /// Override the strides in each dimension between different loads/stores. + typedef Shape<0, 0, Base::Delta::kW, Base::Delta::kC> Delta; + + typedef typename Base::Iterations Iterations; + + typedef typename Base::Threads Threads; + + typedef typename Base::ThreadsDelta ThreadsDelta; + + typedef typename Base::ImmediateOffsetStrides ImmediateOffsetStrides; + + /// Computes the thread offset in (H, W) based on thread ID + struct ThreadOffset { + CUTLASS_HOST_DEVICE + Coord<4> operator()() const { + int thread_offset_h = threadIdx.x / Threads::kW * kStrideH * Iterations::kH; + int thread_offset_w = threadIdx.x % Threads::kW * ThreadsDelta::kW; + + return make_Coord(0, thread_offset_h, thread_offset_w, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename TileTraits_, typename Index_ = int> +struct GemmGlobalIteratorAb + : public TileLoadIterator<TileTraits_, + typename TileTraits_::Scalar, + TileTraits_::MultiplicandTraits::kKstrided ? IteratorAdvance::kH + : IteratorAdvance::kW, + MemorySpace::kGlobal, + Index_> { + /// This class. + typedef GemmGlobalIteratorAb<TileTraits_, Index_> This_; /// The base class. + + typedef TileLoadIterator<TileTraits_, + typename TileTraits_::Scalar, + TileTraits_::MultiplicandTraits::kKstrided ? IteratorAdvance::kH + : IteratorAdvance::kW, + MemorySpace::kGlobal, + Index_> + Base; + /// The layout. + static MatrixLayout::Kind const kLayout = TileTraits_::kLayout; + /// Fragment type loaded by the iterator + typedef typename Base::Fragment Fragment; + /// The scalar. + typedef typename TileTraits_::Scalar Scalar; + /// The threads. + typedef typename TileTraits_::Threads Threads; + /// The index. + typedef Index_ Index; + /// The thread offset + typedef typename TileTraits_::ThreadOffset ThreadOffset; + /// Specifies in which dimension post-increment accesses advance. + static IteratorAdvance::Kind const kAdvance = Base::kAdvance; + + typedef cutlass::PredicateVector<ShapeCount<typename Base::Iterations>::kCount> PredicateVector; + + /// Iterator parameters type + typedef typename Base::Params BaseParams; + + struct Params : public BaseParams { + /// Initializes params to load a strip-mined tile, given pointer and stride_h. + template <typename GemmDesc_> + CUTLASS_HOST_DEVICE int initialize(GemmDesc_ const& desc, Scalar const* ptr, Index stride_h) { + Index inc_d = 0; + Index inc_advance = 0; + // Move by some columns for each iteration in the H dimension. + Index inc_h = Base::Delta::kH * stride_h; + + // Move by some more columns in the number of iterations if the D dimension is > 1. + if (Base::Delta::kD > 0) { + inc_d = Base::Delta::kD * stride_h - (Base::Iterations::kH - 1) * inc_h; + } + + // Move to the beginning of the next iteration. + if (kAdvance == IteratorAdvance::kH && Base::Delta::kD > 0) { + inc_advance = inc_d; + } else if (kAdvance == IteratorAdvance::kH) { + inc_advance = inc_h; + } else if (Base::Delta::kD > 0) { + inc_advance = (Base::Iterations::kW + 0) * ShapeCount<typename Base::Delta>::kWc - + (Base::Iterations::kH - 1) * inc_h - + (Base::Iterations::kD - 1) * Base::Delta::kD * stride_h; + } else { + inc_advance = (Base::Iterations::kW + 0) * ShapeCount<typename Base::Delta>::kWc - + (Base::Iterations::kH - 1) * inc_h; + } + + // The dimensions of the tile. + int const kH = TileTraits_::Tile::kH; + int const kW = TileTraits_::Tile::kW * TileTraits_::kAccessSize; + + // Move to the residue. + Index const kBlock = kAdvance == IteratorAdvance::kH ? kH : kW; + // The jump in the gemm-k dimension. + Index const stride = kAdvance == IteratorAdvance::kH ? stride_h : 1; + + // Compute the offset to the residue and how to "come" back. + Index const kResidue = desc.k % kBlock; + if (kResidue > 0) { + move_to_residue_offset = (desc.k - kResidue) * stride; + } else { + move_to_residue_offset = (desc.k - kBlock) * stride; + } + + Base::Params::initialize(ptr, 0, stride_h, 1, inc_d, inc_h, 0, inc_advance); + return 0; + } + + // The extra offset to control moving to the residue. + Index move_to_residue_offset; + }; + + /// Ctor. + CUTLASS_DEVICE GemmGlobalIteratorAb(Params const& _params, + const Coord<3>& bounds, + const Coord<3>& block, + ThreadOffset thread_offset_func = ThreadOffset()) + : params(_params) { + thread_offset = thread_offset_func(); + // The column. + Index block_h = thread_offset[1]; + // The contiguous dimension. + Index block_w = thread_offset[2]; + + // Add the blocks indices. + if (kAdvance == IteratorAdvance::kH) { + block_h += block[1]; + block_w += block[2]; + + } else { + block_h += block[2]; + block_w += block[1]; + } + + // Setup the pointer. + params.pointer += (block_h * params.stride_h + block_w); + + // Initialize predicates + initialize_predicates(bounds, make_Coord(0, block_h, block_w)); + } + + /// The accessor. + CUTLASS_DEVICE void get(typename Base::AccessType& value, int d, int h, int w, int c) const { + int const imm = + ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(0, 0, w, c); + Load<Scalar, TileTraits_::kAccessSize, MemorySpace::kGlobal>::load(value, params.pointer, imm); + } + + /// Increment the pointer in the H dimension. + CUTLASS_DEVICE void inc_h() { params.pointer += params.inc_h; } + /// Increment the pointer in the D dimension. + CUTLASS_DEVICE void inc_d() { params.pointer += params.inc_d; } + /// Increment the pointer to move to the next iteration. + CUTLASS_DEVICE void inc_advance() { params.pointer += params.inc_advance; } + + /// Initialize the predicates. + CUTLASS_DEVICE void initialize_predicates(const Coord<3>& bounds, const Coord<3>& block) { + // Setup the masks to control loads. + predicates.fill(0); + + int bounds_h, bounds_w; + if (kAdvance == IteratorAdvance::kH) { + bounds_w = bounds[2] - block[2]; + bounds_h = bounds[1]; + + } else { + bounds_w = bounds[1]; + bounds_h = bounds[2] - block[1]; + } + + // Fill in the bits of the predicate vector. + for (int d = 0; d < Base::Iterations::kD; ++d) { + for (int h = 0; h < Base::Iterations::kH; ++h) { + for (int w = 0; w < Base::Iterations::kW; ++w) { + for (int c = 0; c < Base::Iterations::kC; ++c) { + bool flag = w * Base::Delta::kW < bounds_w; + if (kAdvance == IteratorAdvance::kH) { + flag = flag && (h * Base::Delta::kH + d * Base::Delta::kD) < bounds_h; + } else { + flag = flag && (h * Base::Delta::kH) < bounds_h; + } + int const bit = ComputeOffsetFromShape<typename Base::Iterations>::get(d, h, w, c); + predicates.set(bit, flag); + } + } + } + } + } + + /// Move to residue portion. + CUTLASS_DEVICE void move_to_residue(Index k) { + // Store the pointer and the predicates. + stored_pointer = params.pointer; + stored_predicates = predicates; + + // Move the pointer to the residue. + params.pointer += params.move_to_residue_offset; + + // The dimensions of the tile. + int const kH = TileTraits_::Tile::kH; + int const kW = TileTraits_::Tile::kW * TileTraits_::kAccessSize; + + // The unrolling factor. + int const kUnroll = kAdvance == IteratorAdvance::kH ? kH : kW; + + // Clear the predicates for the residue. TODO: We can do something smarter. + int const kResidue = (int)(k % (Index)kUnroll); + if (kResidue > 0) { + residue(kResidue); + } + } + + /// That's the residue! Update the predicates. + CUTLASS_DEVICE void residue(Index k) { + // The coordinates of the thread. + Index block_h = thread_offset[1]; + // The contiguous dimension. + Index block_w = thread_offset[2]; + + // Update the predicate vector. + for (int d = 0; d < Base::Iterations::kD; ++d) { + for (int h = 0; h < Base::Iterations::kH; ++h) { + for (int w = 0; w < Base::Iterations::kW; ++w) { + for (int c = 0; c < Base::Iterations::kC; ++c) { + Index offset = 0; + if (kAdvance == IteratorAdvance::kH) { + offset += block_h + h * Base::Delta::kH + d * Base::Delta::kD; + } else { + offset += block_w + w * Base::Delta::kW; + } + + int const bit = ComputeOffsetFromShape<typename Base::Iterations>::get(d, h, w, c); + if (offset >= k) { + predicates.set(bit, false); + } + } + } + } + } + } + + /// Rollback to beginning of first tile and initialize predicates. + CUTLASS_DEVICE void rollback() { + params.pointer = stored_pointer; + predicates = stored_predicates; + } + + /// Is the iterator valid? + CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { + int const bit = ComputeOffsetFromShape<typename Base::Iterations>::get(d, h, w, c); + return predicates[bit]; + } + + /// Offset of an individual lane from the start of the tile + Coord<4> thread_offset; + /// The parameters + Params params; + /// The pointer. + typename Base::Scalar const* stored_pointer; + /// The predicates. + PredicateVector predicates, stored_predicates; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename TileTraits_, typename Index_ = int> +struct GemmGlobalIteratorCd : public TileIteratorBase<TileTraits_, + typename TileTraits_::Scalar, + IteratorAdvance::kH, + MemorySpace::kGlobal, + Index_> { + /// This class. + typedef GemmGlobalIteratorCd<TileTraits_, Index_> This_; + /// The base class. + typedef TileIteratorBase<TileTraits_, + typename TileTraits_::Scalar, + IteratorAdvance::kH, + MemorySpace::kGlobal, + Index_> + Base; + + /// The layout. + static MatrixLayout::Kind const kLayout = TileTraits_::kLayout; + + /// The scalar. + typedef typename TileTraits_::Scalar Scalar; + /// The pointer. + typedef typename TileTraits_::Pointer Pointer; + /// The threads. + typedef typename TileTraits_::Threads Threads; + /// The index. + typedef Index_ Index; + /// The thread offset + typedef typename TileTraits_::ThreadOffset ThreadOffset; + + /// The params. + struct Params { + /// The pointer. + Pointer pointer; + /// The stride in the H dimension to setup the thread in the block. + Index stride_h; + /// The strides to increment the pointer. + Index inc_advance, inc_h; + /// The strides to increment the predicate offset + Index predicate_inc_advance, predicate_inc_h; + /// The column offset to compute the predicate for the columns. + Index predicate_offset; + + /// Setup the params. + CUTLASS_HOST_DEVICE int initialize( + Pointer pointer, Index ld, Index bound, Index epilogue_stride_w, Index epilogue_delta_w) { + // The pointer. + this->pointer = pointer; + // Each column of the matrix. + stride_h = TileTraits_::ThreadsDelta::kH * ld; + // Each thread output 1 column per iteration. The stride between columns is given by the + // number of scalars that are loaded per LDS for B. + inc_h = ld * TileTraits_::kStrideH; + inc_advance = + (ld - ld * TileTraits_::kStrideH * (Base::Iterations::kH - 1)) + epilogue_stride_w; + + predicate_offset = bound; + predicate_inc_h = TileTraits_::kStrideH; + predicate_inc_advance = + -((TileTraits_::kStrideH * (Base::Iterations::kH - 1) - 1) + epilogue_delta_w); + + return 0; + } + }; + + Params params; + /// Offset of an individual lane from the start of the tile + Coord<4> thread_offset; + + /// Ctor. + CUTLASS_DEVICE GemmGlobalIteratorCd() {} + + /// Ctor. + CUTLASS_DEVICE GemmGlobalIteratorCd(Params const& params, + const Coord<3>& bounds, + const Coord<3>& block, + int offset = 0, + int pred_offset = 0, + ThreadOffset thread_offset_func = ThreadOffset()) + : params(params) { + thread_offset = thread_offset_func(); + // Each warp works on a different column of the tile. + int const h = thread_offset[1] + block[1]; + // Each lane writes a different element. + int const w = thread_offset[2] + block[2]; + // Setup the pointer. + this->params.pointer += ((h * params.stride_h + w) + offset); + + // Prepare the vector of predicates. + for (int i = 0; i < Base::Iterations::kW; ++i) { + predicates.set(i, w + i * Base::Delta::kW < bounds[2]); + } + this->params.predicate_offset -= (h + pred_offset); + } + + /// The accessor. + CUTLASS_DEVICE void get(typename Base::AccessType& value, int d, int h, int w, int c) const { + int const imm = + ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(0, 0, w, c); + Load<Scalar, TileTraits_::kAccessSize, MemorySpace::kGlobal>::load(value, params.pointer, imm); + } + + /// Increment the pointer in the C dimension. + CUTLASS_DEVICE void inc_c() {} + /// Increment the pointer in the W dimension. + CUTLASS_DEVICE void inc_w() {} + /// Increment the pointer in the H dimension. + CUTLASS_DEVICE void inc_h() { + params.pointer += params.inc_h; + params.predicate_offset -= params.predicate_inc_h; + } + /// Increment the pointer in the D dimension. + CUTLASS_DEVICE void inc_d() {} + /// Increment the pointer to move to the next iteration. + CUTLASS_DEVICE void inc_advance() { + params.pointer += params.inc_advance; + this->params.predicate_offset -= params.predicate_inc_advance; + } + + /// The accessor. + CUTLASS_DEVICE void set(typename Base::AccessType const& value, int d, int h, int w, int c) { + int const imm = + ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(0, 0, w, c); + Store<Scalar, TileTraits_::kAccessSize, MemorySpace::kGlobal>::store( + value, params.pointer, imm); + } + + /// Test the validity of the iterator. + CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { + return predicates.at(w) && params.predicate_offset > 0; + } + + /// The predicates for the row. + cutlass::PredicateVector<Base::Iterations::kW> predicates; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_operand.h b/cutlass-example/cutlass/gemm/gemm_operand.h new file mode 100644 index 0000000..737f993 --- /dev/null +++ b/cutlass-example/cutlass/gemm/gemm_operand.h @@ -0,0 +1,141 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines constant expressions for mapping GEMM problem size and strides onto pitch-linear + memory. +*/ +#pragma once + +#include <cutlass/matrix_traits.h> +#include <cutlass/reshape_tile.h> +#include <cutlass/util/platform.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Helper to describe attributes of GEMM matrix operands +template <GemmOperand::Kind kOperand_, MatrixLayout::Kind kLayout_> +struct GemmOperandTraitsAb { + static const bool Congruous = + (kOperand_ == GemmOperand::kA ^ kLayout_ == MatrixLayout::kRowMajor); +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmOperand::Kind kOperand_, typename Tile_> +struct GetExtent; + +template <typename Tile_> +struct GetExtent<GemmOperand::kA, Tile_> { + static const int kExtent = Tile_::kW; +}; + +template <typename Tile_> +struct GetExtent<GemmOperand::kB, Tile_> { + static const int kExtent = Tile_::kH; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Determines the shape of a multiplicand tile in terms of strided (H) and contiguous (W) +/// dimensions +template <typename ThreadBlockTile_, GemmOperand::Kind Usage, MatrixLayout::Kind Layout> +struct GemmMultiplicandTraits { + // Only defined for A or B + static_assert(Usage == GemmOperand::kA || Usage == GemmOperand::kB, + "MultiplicandTileShape defined only for A or B operands."); + + /// Shape of GEMM thread block tile (K, N, M) + typedef ThreadBlockTile_ ThreadBlockTile; + + /// Identifies multiplicand + static GemmOperand::Kind const kUsage = Usage; + + /// Layout of tile + static MatrixLayout::Kind const kLayout = Layout; + + // True if K is the strided dimension + static bool const kKstrided = (kUsage == GemmOperand::kA ^ kLayout == MatrixLayout::kRowMajor); + + /// Map the ThreadBlockShape onto (kH, kW) dimensions for A and B operand + typedef typename platform::conditional< + kKstrided, + Shape<1, ThreadBlockTile::kD, GetExtent<Usage, ThreadBlockTile>::kExtent>, + Shape<1, GetExtent<Usage, ThreadBlockTile>::kExtent, ThreadBlockTile::kD> >::type Shape; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Project's a coordinate (K, N, M) onto inner and outer dimensions defined for each +/// operand. +template <GemmOperand::Kind operand, bool Kstrided = true> +struct ProjectOperand; + +/// Project A operand - (0, K, M) +template <bool Kstrided> +struct ProjectOperand<GemmOperand::kA, Kstrided> { + CUTLASS_HOST_DEVICE + static Coord<3> project(Coord<3> const &coord) { + if (Kstrided) { + return make_Coord(0, coord[0], coord[2]); + } else { + return make_Coord(0, coord[2], coord[0]); + } + } +}; + +/// Project B operand - (0, K, N) +template <bool Kstrided> +struct ProjectOperand<GemmOperand::kB, Kstrided> { + CUTLASS_HOST_DEVICE + static Coord<3> project(Coord<3> const &coord) { + if (Kstrided) { + return make_Coord(0, coord[0], coord[1]); + } else { + return make_Coord(0, coord[1], coord[0]); + } + } +}; + +/// Project C operand - (0, N, M) +template <> +struct ProjectOperand<GemmOperand::kC, true> { + CUTLASS_HOST_DEVICE + static Coord<3> project(Coord<3> const &coord) { return make_Coord(0, coord[1], coord[2]); } +}; + +/// Project D operand - (0, N, M) +template <> +struct ProjectOperand<GemmOperand::kD, true> { + CUTLASS_HOST_DEVICE + static Coord<3> project(Coord<3> const &coord) { return make_Coord(0, coord[1], coord[2]); } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_shared_stream.h b/cutlass-example/cutlass/gemm/gemm_shared_stream.h new file mode 100644 index 0000000..c6ff7bd --- /dev/null +++ b/cutlass-example/cutlass/gemm/gemm_shared_stream.h @@ -0,0 +1,113 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines abstractions for managing loading and storing fragments to shared memory in the + efficient GEMM pipeline. +*/ +#pragma once + +#include <cutlass/gemm/gemm_shared_tile.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The load iterator. + typename Iterator_, + /// The transformer to be applied after the data has been copied from shared memory. + typename Transformer_ = Copy<typename Iterator_::Fragment> > + +struct SharedLoadStream { + /// The load iterator. + typedef Iterator_ Iterator; + /// The transformer. + typedef Transformer_ Transformer; + + /// The fragment that is copied from shared memory. + typedef typename Iterator::Fragment FetchedFragment; + /// The fragment that is obtained after the transformation by the transformer. + typedef typename Transformer::OutputFragment TransformedFragment; + /// Make sure the fragments match. + static_assert((platform::is_same<FetchedFragment, typename Transformer::InputFragment>::value), + ""); + /// The output fragment. + typedef TransformedFragment Fragment; + + /// The params. + struct Params { + /// The iterator params. + typename Iterator::Params iterator; + + /// Setup the params. + CUTLASS_HOST_DEVICE int initialize() { return iterator.initialize(); } + }; + + /// The storage in shared memory needed by that stream. + typedef typename Iterator::Storage SharedStorage; + + /// Ctor. + CUTLASS_DEVICE SharedLoadStream() {} + + /// Ctor. + CUTLASS_DEVICE SharedLoadStream(Params const ¶ms, SharedStorage &shared_storage) { + this->initialize(params, shared_storage); + } + + /// Initialize the stream. + CUTLASS_DEVICE void initialize(Params const ¶ms, SharedStorage &shared_storage) { + // The iterator. + iterator = Iterator(params.iterator, shared_storage); + // The transformer. + transformer = Transformer(); + } + + /// Load the data from shared memory to the fetch fragment. + CUTLASS_DEVICE void copy(FetchedFragment &fetched) { shared_iterator_load(iterator, fetched); } + + /// Load the data from shared memory to the fetch fragment. + CUTLASS_DEVICE void copy(int d, FetchedFragment &fetched) { + shared_iterator_load(iterator, fetched, d); + } + + /// Commit the data. + CUTLASS_DEVICE void commit(FetchedFragment &fetched, TransformedFragment &transformed) { + transformer.transform(fetched, transformed); + } + + /// Increment the stage. + CUTLASS_DEVICE void inc_stage() { iterator.inc_stage(); } + + /// The iterator. + Iterator iterator; + /// The transformer. + Transformer transformer; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_shared_tile.h b/cutlass-example/cutlass/gemm/gemm_shared_tile.h new file mode 100644 index 0000000..7c61e02 --- /dev/null +++ b/cutlass-example/cutlass/gemm/gemm_shared_tile.h @@ -0,0 +1,417 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines iterators for efficiently loading and storing tiles to and from shared memory. +*/ +#pragma once + +#include <cutlass/gemm/gemm_operand.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, typename Tile_, typename Threads_, int kScalarsPerSts_> +struct GemmSharedStoreTileAbTraits { + /// The scalar. + typedef typename platform::remove_const<Scalar_>::type Scalar; + /// The pointer. + typedef Scalar_* Pointer; + /// The tile. + typedef typename ReshapeTile<Tile_, kScalarsPerSts_>::Tile Tile; + /// The threads. + typedef Threads_ Threads; + /// The strides to compute the base position of the thread. + typedef Shape<0, ShapeCount<Tile>::kWc, Tile::kC, kScalarsPerSts_> ThreadsStrides; + /// The skew. + static int const kSkew = 0; + /// The number of scalars per LDG/STG. + static int const kAccessSize = kScalarsPerSts_; + /// The memory space. + static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; + + /// The number of iterations needed to load/store the tile. + typedef Shape<1, + Tile::kH / Threads::kH, + Tile::kW / Threads::kW, + Tile::kC / Threads::kC / kAccessSize> + Iterations; + /// The strides in each dimension between different loads/stores. + typedef Shape<0, Threads::kH * ShapeCount<Tile>::kWc, Threads::kW * kAccessSize> Delta; + /// The strides in each dimension between different loads/stores. + typedef Shape<0, Threads::kH * ShapeCount<Tile>::kWc, Threads::kW * kAccessSize> + ImmediateOffsetStrides; + + struct ThreadOffset { + CUTLASS_HOST_DEVICE + Coord<4> operator()() const { + int offset = ComputeThreadOffsetFromStrides<Threads, ThreadsStrides>::get(); + return make_Coord(0, 0, offset, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, typename Tile_, typename Threads_, int kScalarsPerSts_, int kSkew_> +struct GemmSharedStoreWithSkewTileAbTraits { + /// The scalar. + typedef typename platform::remove_const<Scalar_>::type Scalar; + /// The pointer. + typedef Scalar_* Pointer; + /// The tile without skews. + typedef typename ReshapeTile<Tile_, kScalarsPerSts_>::Tile TileWithoutSkew; + /// The tile. + typedef typename ReshapeTile<Shape<Tile_::kD, Tile_::kH, Tile_::kW + kSkew_>, + kScalarsPerSts_>::Tile Tile; + /// The threads. + typedef Threads_ Threads; + /// The skew. + static int const kSkew = kSkew_; + /// The number of scalars per STS. + static int const kAccessSize = kScalarsPerSts_; + /// The memory space. + static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; + + /// The number of iterations needed to load/store the tile. + typedef Shape<1, TileWithoutSkew::kH / Threads::kW, TileWithoutSkew::kW / Threads::kH> Iterations; + /// The strides in each dimension between different loads/stores. + typedef Shape<0, ShapeCount<Tile>::kWc, Threads::kH * kAccessSize> Delta; + /// The strides in each dimension between different loads/stores. + typedef Shape<0, ShapeCount<Tile>::kWc, Threads::kH * kAccessSize> ImmediateOffsetStrides; + + struct ThreadOffset { + CUTLASS_HOST_DEVICE Coord<4> operator()() const { + int offset = ComputeThreadOffsetFromStrides<Threads, ThreadsStrides>::get(); + return make_Coord(0, 0, offset, 0); + } + }; + + protected: + /// The strides to compute the base position of the thread. + typedef Shape<0, kScalarsPerSts_, ShapeCount<Tile>::kHwc / Threads::kW> ThreadsStrides; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, + typename OutputTile_, + typename Warps_, + typename ThreadsPerWarp_, + typename InstructionShape_, + int kStages_, + int kScalarsPerLds_, + int kSkew_ = 0> +struct GemmSharedLoadTileATraits { + static GemmOperand::Kind const kOperand = GemmOperand::kA; + /// The scalar. + typedef typename platform::remove_const<Scalar_>::type Scalar; + /// The pointer. + typedef Scalar_* Pointer; + /// The tile without skew. + typedef Shape<kStages_, + OutputTile_::kD / InstructionShape_::kD, + GetExtent<kOperand, OutputTile_>::kExtent * InstructionShape_::kD> + TileWithoutSkew_; + /// The tile with skew. + typedef Shape<kStages_, TileWithoutSkew_::kH, TileWithoutSkew_::kW + kSkew_> TileWithSkew; + /// The tile without skew after reshaping. + typedef typename ReshapeTile<TileWithoutSkew_, kScalarsPerLds_>::Tile TileWithoutSkew; + /// The tile. + typedef typename ReshapeTile<TileWithSkew, kScalarsPerLds_>::Tile Tile; + /// The number of warps. + typedef Warps_ Warps; + /// The threads in a warp. + typedef ThreadsPerWarp_ ThreadsPerWarp; + /// The number of scalars per LDG/STG. + // static int const kScalarsPerLds = kScalarsPerLds_; + static int const kAccessSize = kScalarsPerLds_; + /// The skew. + static int const kSkew = kSkew_; + /// The memory space. + static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; + + /// The number of warps. + static int const kWarps = GetExtent<kOperand, Warps>::kExtent; + /// The number of threads in one dimension of the warp. + static int const kThreadsPerWarp = GetExtent<kOperand, ThreadsPerWarp>::kExtent; + + /// The number of iterations needed to load/store the tile. + typedef Shape<1, 1, TileWithoutSkew::kW / kWarps / kThreadsPerWarp /* / kScalarsPerLds*/> + Iterations; + /// The strides in each dimension between different loads/stores. + typedef Shape<TileWithSkew::kW * Warps::kD, 0, kWarps * kThreadsPerWarp * kAccessSize, 0> + ImmediateOffsetStrides; + typedef Shape<TileWithSkew::kW * Warps::kD, 0, kWarps * kThreadsPerWarp * kAccessSize, 0> Delta; + + /// Computes the thread offset in (H, W) based on thread ID + struct ThreadOffset { + CUTLASS_HOST_DEVICE Coord<4> operator()() const { + // Extract the warp. + int const warp = threadIdx.x / kWarpSize; + // Extract the slice. + int const slice = warp / (Warps::kH * Warps::kW); + // Compute the row offset for each warp. + int const warp_row = warp % Warps::kW; + // Compute the row offset for each thread. + int const lane_row = (threadIdx.x & 0x0e) / 2; + // The offset. + int const offset = + slice * Tile::kW * Tile::kC + (warp_row * ThreadsPerWarp::kW + lane_row) * kAccessSize; + // Embed the offset in a 4D coordinate vector. + return make_Coord(0, 0, offset, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, + typename OutputTile_, + typename Warps_, + typename ThreadsPerWarp_, + typename InstructionShape_, + int kStages_, + int kScalarsPerLds_, + int kSkew_ = 0> +struct GemmSharedLoadTileBTraits { + static GemmOperand::Kind const kOperand = GemmOperand::kB; + /// The scalar. + typedef typename platform::remove_const<Scalar_>::type Scalar; + /// The pointer. + typedef Scalar_* Pointer; + /// The tile without skew. + typedef Shape<kStages_, + OutputTile_::kD / InstructionShape_::kD, + GetExtent<kOperand, OutputTile_>::kExtent * InstructionShape_::kD> + TileWithoutSkew_; + /// The tile with skew. + typedef Shape<kStages_, TileWithoutSkew_::kH, TileWithoutSkew_::kW + kSkew_> TileWithSkew; + /// The tile without skew after reshaping. + typedef typename ReshapeTile<TileWithoutSkew_, kScalarsPerLds_>::Tile TileWithoutSkew; + /// The tile. + typedef typename ReshapeTile<TileWithSkew, kScalarsPerLds_>::Tile Tile; + /// The number of warps. + typedef Warps_ Warps; + /// The threads in a warp. + typedef ThreadsPerWarp_ ThreadsPerWarp; + /// The number of scalars per LDG/STG. + static int const kAccessSize = kScalarsPerLds_; + /// The skew. + static int const kSkew = kSkew_; + /// The memory space. + static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; + + /// The number of warps. + static int const kWarps = GetExtent<kOperand, Warps>::kExtent; + /// The number of threads in one dimension of the warp. + static int const kThreadsPerWarp = GetExtent<kOperand, ThreadsPerWarp>::kExtent; + + /// The number of iterations needed to load/store the tile. + typedef Shape<1, 1, TileWithoutSkew::kW / kWarps / kThreadsPerWarp /* / kAccessSize*/> Iterations; + /// The strides in each dimension between different loads/stores. + typedef Shape<TileWithSkew::kW * Warps::kD, 0, kWarps * kThreadsPerWarp * kAccessSize, 0> + ImmediateOffsetStrides; + typedef Shape<TileWithSkew::kW * Warps::kD, 0, kWarps * kThreadsPerWarp * kAccessSize, 0> Delta; + + /// Computes the thread offset in (H, W) based on thread ID + struct ThreadOffset { + CUTLASS_HOST_DEVICE Coord<4> operator()() const { + // Extract the warp. + int const warp = threadIdx.x / kWarpSize; + // Extract the slice. + int const slice = warp / (Warps::kH * Warps::kW); + // The warp in the slice. + int const warp_in_slice = warp % (Warps::kH * Warps::kW); + // Compute the row offset for each warp. + int const warp_col = warp_in_slice / Warps::kW; + // Compute the row offset for each thread. + int const lane_col = (threadIdx.x & 0x10) / 8 + (threadIdx.x & 0x01); + // The offset. + int const offset = + slice * Tile::kW * Tile::kC + (warp_col * ThreadsPerWarp::kH + lane_col) * kAccessSize; + // Embed the offset in a 4D coordinate. + return make_Coord(0, 0, offset, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, + typename OutputTile_, + typename Warps_, + typename ThreadsPerWarp_, + int kScalarsPerSts_, + int kSkew_ = 0> +struct GemmSharedStoreTileDTraits { + /// The scalar. + typedef typename platform::remove_const<Scalar_>::type Scalar; + /// The pointer. + typedef Scalar_* Pointer; + /// The dimension of the output tile. + typedef OutputTile_ OutputTile; + /// The warps in the tile. + typedef Warps_ Warps; + /// The threads in the warps. + typedef ThreadsPerWarp_ ThreadsPerWarp; + /// The number of scalars per LDG/STG. + static int const kAccessSize = kScalarsPerSts_; + /// The skew. + static int const kSkew = kSkew_; + /// The memory space. + static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; + + /// The number of scalars per thread. + static int const kScalarsPerThread = OutputTile_::kW / Warps::kW / ThreadsPerWarp::kW; + /// The number of threads. + static int const kThreads = ShapeCount<Warps>::kCount * kWarpSize; + /// The number of scalars per row. We build a tile with 2 rows (to avoid bank conflicts). + static int const kScalarsPerRow = kThreads / 2 * kScalarsPerThread + kSkew; + + /// The tile. + typedef Shape<1, 2, kScalarsPerRow / kAccessSize, kAccessSize> Tile; + /// The number of iterations needed to store the tile. + typedef Shape<1, 1, kScalarsPerThread / kAccessSize> Iterations; + /// The strides in each dimension between different loads/stores. + typedef Shape<0, 0, Warps::kW * ThreadsPerWarp::kW * kAccessSize> Delta; + /// The strides in each dimension between different loads/stores. + typedef Shape<0, 0, Warps::kW * ThreadsPerWarp::kW * kAccessSize> ImmediateOffsetStrides; + + /// Computes the thread offset in (H, W) based on thread ID + struct ThreadOffset { + CUTLASS_HOST_DEVICE Coord<4> operator()() const { + // The warp. + int const warp = threadIdx.x / kWarpSize; + + // The position of the warp in the 2D tile. + int const warp_row = warp % Warps::kW; + int const warp_col = warp / Warps::kW; + + // We assume that the elements are distributed in a warps as 4 columns of 8 elements. The + // columns are stored in threads col0=[0, 2, 4, 6, 8, 10, 12, 14], col1=[1, 3, 5, 7, .., 15], + // col2=[16, 18, 20, ..., 30] and col3=[17, 19, ..., 31]. + int hi_halfwarp_offset = ((threadIdx.x >> 4) & 0x1) * OutputTile::kW; + int lo_halfwarp_offset = ((threadIdx.x >> 1) & 0x7) + ThreadsPerWarp::kW * warp_row; + + // Odd threads go to the second half of shared memory. + int const row = threadIdx.x & 0x01; + int col = warp_col * (ThreadsPerWarp::kH / 2) * OutputTile::kW + + lo_halfwarp_offset * kAccessSize + hi_halfwarp_offset; + // Embed the offset in a 4D coords. + return make_Coord(0, 0, row * kScalarsPerRow + col, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, + typename OutputTile_, + typename Warps_, + typename ThreadsPerWarp_, + int kTileH_, + int kScalarsPerLds_, + int kSkew_ = 0> +struct GemmSharedLoadTileDTraits { + /// The scalar. + typedef typename platform::remove_const<Scalar_>::type Scalar; + /// The pointer. + typedef Scalar_* Pointer; + /// The dimension of the output tile. + typedef OutputTile_ OutputTile; + /// The warps in the tile. + typedef Warps_ Warps; + /// The threads in the warps. + typedef ThreadsPerWarp_ ThreadsPerWarp; + /// The number of scalars per LDG/STG. + static int const kAccessSize = kScalarsPerLds_; + /// The skew. + static int const kSkew = kSkew_; + /// The memory space. + static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; + + /// The number of scalars per thread. + static int const kScalarsPerThread = OutputTile_::kW / Warps::kW / ThreadsPerWarp::kW; + /// The number of threads. + static int const kThreads = ShapeCount<Warps>::kCount * kWarpSize; + /// The number of scalars per row. We build a tile with 2 rows (to avoid bank conflicts). + static int const kScalarsPerRow = kThreads / 2 * kScalarsPerThread + kSkew; + + /// The tile. We have 2 rows of scalars. We use those two rows to make sure we do not have bank + /// conflicts in the epilogue. + typedef Shape<1, 2, kScalarsPerRow / kAccessSize, kAccessSize> Tile; + + // Compute the number of iterations per warp in the Tile::kH dimension. + static int const kIterationsInHPerWarp = kTileH_ / ShapeCount<Warps>::kCount; + + // As explained above, the shared memory tile is composed of 2 rows and each rows is made of + // kScalarsPerRow. A warp is expected to read from the 1st row, then move to the 2nd row and go + // back to the 1st row. To model that scheme we define the Iterations shape as Shape<X, 2, ...>. + // However, in some cases, we have only 1 iteration per warp. In that case, we must define the + // shape as Shape<1, 1, ...>. The following code does that except that we hijack the kH dimension + // to keep the number of elements to reduce for split-K. + static int const kIterationsH = kIterationsInHPerWarp == 1 ? 1 : 2; + // As soon as we know kIterationsH, it is trivial to compute kIterationsD: + static int const kIterationsD = kIterationsInHPerWarp / kIterationsH; + + // If we have split-K enabled, we have to jump over the elements from the "odd/even" column of + // threads to grab the other elements. + static int const kSplitK = OutputTile::kW * ThreadsPerWarp::kH / 2 * Warps::kH; + + /// The number of iterations needed to store the tile. + typedef Shape<kIterationsD, kIterationsH, OutputTile::kW / kWarpSize / kAccessSize, Warps::kD> + Iterations; + /// The strides in each dimension between different loads/stores. + typedef Shape<OutputTile::kW, kScalarsPerRow, kWarpSize * kAccessSize, kSplitK> + ImmediateOffsetStrides; + /// The strides in each dimension between different loads/stores. + typedef Shape<OutputTile::kW, kScalarsPerRow, kWarpSize * kAccessSize, kSplitK> Delta; + + /// Computes the thread offset in (H, W) based on thread ID + struct ThreadOffset { + CUTLASS_HOST_DEVICE Coord<4> operator()() const { + // Each warp works on a different column. + int const h = threadIdx.x / kWarpSize; + // Compute the row. + int const w = (threadIdx.x & (kWarpSize - 1)) * kAccessSize; + int offset = 0; + if (Iterations::kH == 1) { + int const row = h & 0x1; + int const col = h / 2; + offset = row * ShapeCount<Tile>::kWc + col * OutputTile::kW * Iterations::kD + w; + } else { + offset = h * OutputTile::kW * Iterations::kD + w; + } + return make_Coord(0, 0, offset, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_traits.h b/cutlass-example/cutlass/gemm/gemm_traits.h new file mode 100644 index 0000000..cb57c4d --- /dev/null +++ b/cutlass-example/cutlass/gemm/gemm_traits.h @@ -0,0 +1,818 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines structural properties of complete GEMM computation. +*/ +#pragma once + +#include <cutlass/convert.h> +#include <cutlass/gemm/clear_accumulators.h> +#include <cutlass/gemm/gemm_global_stream.h> +#include <cutlass/gemm/gemm_operand.h> +#include <cutlass/gemm/gemm_shared_stream.h> +#include <cutlass/gemm/identity_block_swizzle.h> +#include <cutlass/matrix_traits.h> +#include <cutlass/reshape_tile.h> +#include <cutlass/tile_iterator.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The scalar type for A. + typename ScalarA_, + /// The scalar type for B. + typename ScalarB_, + /// The scalar type for C. + typename ScalarC_, + /// The scalar type for D. + typename ScalarD_, + /// The output tile size for the GEMM KxNxM. + typename OutputTile_, + /// The functor to do the math. + typename MultiplyAdd_, + /// The number of scalars per LDG for A. + int kScalarsPerLdgA_, + /// The number of scalars per STS for A. + int kScalarsPerStsA_, + /// The number of scalars per LDG for A. + int kScalarsPerLdsA_, + /// The number of scalars per LDG for B. + int kScalarsPerLdgB_, + /// The number of scalars per STS for B. + int kScalarsPerStsB_, + /// The number of scalars per LDS for B. + int kScalarsPerLdsB_, + /// The number of scalars per LDG for C and STG for D. + int kScalarsPerLdgCAndStgD_, + /// The number of scalars per STS for D. + int kScalarsPerStsD_, + /// The number of scalars per LDS for D. + int kScalarsPerLdsD_, + /// The number of stages in shared memory to do single/double/triple-buffering. + int kStages_, + /// Do we do the residue in the prologue? + bool kResidueInPrologue_ = false> + +struct GemmConfig { + // + /// The scalar for A. + typedef ScalarA_ ScalarA; + /// The scalar for B. + typedef ScalarB_ ScalarB; + /// The scalar for C. + typedef ScalarC_ ScalarC; + /// The scalar for D. + typedef ScalarD_ ScalarD; + + /// The tile. + typedef OutputTile_ OutputTile; + /// The functor to do D = A*B + C. + typedef MultiplyAdd_ MultiplyAdd; + /// The shape of the instruction. + typedef typename MultiplyAdd::InstructionShape InstructionShape; + /// The number of accumulators per warp. + typedef typename MultiplyAdd::AccumulatorsPerWarp AccumulatorsPerWarp; + /// The accumulators. + typedef typename MultiplyAdd::Accumulators Accumulators; + + /// The number of warps. + typedef typename ShapeDiv<OutputTile, AccumulatorsPerWarp>::Shape Warps; + /// The default warp size (32 threads per warp). + static int const kWarpSize = cutlass::kWarpSize; + /// The numnber of threads. + static int const kThreads = ShapeCount<Warps>::kCount * kWarpSize; + + /// The number of scalars per LDG/STS/LDS for A. + static int const kScalarsPerLdgA = kScalarsPerLdgA_; + static int const kScalarsPerStsA = kScalarsPerStsA_; + static int const kScalarsPerLdsA = kScalarsPerLdsA_; + + /// The number of scalars per LDG/STS/LDS for B. + static int const kScalarsPerLdgB = kScalarsPerLdgB_; + static int const kScalarsPerStsB = kScalarsPerStsB_; + static int const kScalarsPerLdsB = kScalarsPerLdsB_; + + /// The number of scalars per LDG for C. + static int const kScalarsPerLdgC = kScalarsPerLdgCAndStgD_; + + /// The number of scalars per STS/LDS/STG for D. + static int const kScalarsPerStgD = kScalarsPerLdgCAndStgD_; + static int const kScalarsPerStsD = kScalarsPerStsD_; + static int const kScalarsPerLdsD = kScalarsPerLdsD_; + + /// The number of accumulators that are going to be fed from one LDS A/B. + static int const kAccumulatorsPerLdsA = kScalarsPerLdsA / InstructionShape::kD; + static int const kAccumulatorsPerLdsB = kScalarsPerLdsB / InstructionShape::kD; + + /// The number of stages in shared memory to implement double, triple, more-buffering. + static int const kStages = kStages_; + + /// Do we do the residue in the prologue? + static bool const kResidueInPrologue = kResidueInPrologue_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <enum MatrixLayout::Kind, typename GemmConfig_> +struct GemmTileTraitsHelperA {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_> +struct GemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_> { + /// The layout. + static MatrixLayout::Kind const kLayout = MatrixLayout::kColumnMajor; + + /// The input scalar. + typedef typename GemmConfig_::ScalarA Scalar; + /// The scalar stored in shared memory. + typedef typename GemmConfig_::MultiplyAdd::ScalarA MultiplyAddScalar; + + /// The traits class to build the iterator to load data from global memory for A^N. + typedef GemmGlobalTileTraits< + // That's A. + GemmOperand::kA, + // A is column-major. + MatrixLayout::kColumnMajor, + // The pointer is float const. + Scalar const, + // The tile has size KxM in GEMM's terminology. + Shape<1, GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kW>, + // The threads are distributed as warps x 32 (the traits may reorganize). + Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerLdgA> + GlobalTileTraits; + + /// The traits class to build the iterator to store data to shared memory for A^N. + typedef GemmSharedStoreTileAbTraits< + // The pointer is float. + MultiplyAddScalar, + // The tile has size KxM in GEMM's terminology. + Shape<GemmConfig_::kStages, + GemmConfig_::OutputTile::kD / GemmConfig_::InstructionShape::kD, + GemmConfig_::OutputTile::kW * GemmConfig_::InstructionShape::kD>, + // The threads are distributed as warps x 32 (the traits may reorganize). + typename GlobalTileTraits::Threads, + // The number of scalars per STS (STS.32 or STS.128, etc). + GemmConfig_::kScalarsPerStsA> + SharedStoreTileTraits; + + /// The traits class to build the iterator to load from shared memory for A^N. + typedef GemmSharedLoadTileATraits< + // The pointer is float const. + MultiplyAddScalar const, + // The output tile size. + typename GemmConfig_::OutputTile, + // The number of warps. + typename GemmConfig_::Warps, + // The number of threads per warp. + typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, + // The shape of the FMA instruction. + typename GemmConfig_::InstructionShape, + // The number of stages. + GemmConfig_::kStages, + // The number of scalars per LDS. + GemmConfig_::kScalarsPerLdsA, + // The skew. + 0> + SharedLoadTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_> +struct GemmTileTraitsHelperA<MatrixLayout::kRowMajor, GemmConfig_> { + /// The layout. + static MatrixLayout::Kind const kLayout = MatrixLayout::kRowMajor; + + /// The input scalar. + typedef typename GemmConfig_::ScalarA Scalar; + /// The scalar stored in shared memory. + typedef typename GemmConfig_::MultiplyAdd::ScalarA MultiplyAddScalar; + + /// The traits class to build the iterator to load data from global memory for A^T. + typedef GemmGlobalTileTraits< + // That's A. + GemmOperand::kA, + // A is row-major. + MatrixLayout::kRowMajor, + // The pointer is float const. + Scalar const, + // The tile has size MxK in GEMM's terminology. + Shape<1, GemmConfig_::OutputTile::kW, GemmConfig_::OutputTile::kD>, + // The threads are distributed as (threads / K) x K (the traits may reorganize). + Shape<1, GemmConfig_::kThreads / GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kD>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerLdgA> + GlobalTileTraits; + + /// The number of scalars in 4B. + static int const kScalarsIn4B = sizeof(MultiplyAddScalar) > 4 ? 1 : 4 / sizeof(MultiplyAddScalar); + /// The skew for A. + static int const kSkewA = 128 / sizeof(MultiplyAddScalar) / GemmConfig_::kScalarsPerStsA / + GlobalTileTraits::Threads::kW * kScalarsIn4B; + + /// The traits class to build the iterator to store data to shared memory for A^T. + typedef GemmSharedStoreWithSkewTileAbTraits < + // The pointer is float. + MultiplyAddScalar, + // The tile has size KxM in GEMM's terminology. + Shape<GemmConfig_::kStages, + GemmConfig_::OutputTile::kD / GemmConfig_::InstructionShape::kD, + GemmConfig_::OutputTile::kW * GemmConfig_::InstructionShape::kD>, + // The threads are distributed as (threads / K) x K (the traits may reorganize). + typename GlobalTileTraits::Threads, + // The number of scalars per STS. + GemmConfig_::kScalarsPerStsA, + // The skew to avoid bank conflicts added in the tile W dimension. + kSkewA<GemmConfig_::kScalarsPerLdsA ? GemmConfig_::kScalarsPerLdsA : kSkewA> + SharedStoreTileTraits; + + /// The traits class to build the iterator to load from shared memory for A^T. + typedef GemmSharedLoadTileATraits< + // The pointer is float const. + MultiplyAddScalar const, + // The output tile size. + typename GemmConfig_::OutputTile, + // The number of warps. + typename GemmConfig_::Warps, + // The number of threads per warp. + typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, + // The shape of the FMA instruction. + typename GemmConfig_::InstructionShape, + // The number of stages. + GemmConfig_::kStages, + // The number of scalars per LDS. + GemmConfig_::kScalarsPerLdsA, + // The skew. + SharedStoreTileTraits::kSkew> + SharedLoadTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <enum MatrixLayout::Kind, typename GemmConfig_> +struct GemmTileTraitsHelperB {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_> +struct GemmTileTraitsHelperB<MatrixLayout::kColumnMajor, GemmConfig_> { + /// The layout. + static MatrixLayout::Kind const kLayout = MatrixLayout::kColumnMajor; + + /// The input scalar. + typedef typename GemmConfig_::ScalarB Scalar; + /// The scalar stored in shared memory. + typedef typename GemmConfig_::MultiplyAdd::ScalarB MultiplyAddScalar; + + /// The traits class to build the iterator to load data from global memory for B^N. + typedef GemmGlobalTileTraits< + // That's B. + GemmOperand::kB, + // B is column-major. + MatrixLayout::kColumnMajor, + // The pointer is float const. + Scalar const, + // The tile has size MxK in GEMM's terminology. + Shape<1, GemmConfig_::OutputTile::kH, GemmConfig_::OutputTile::kD>, + // The threads are distributed as (threads / K) x K (the traits may reorganize). + Shape<1, GemmConfig_::kThreads / GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kD>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerLdgB> + GlobalTileTraits; + + /// The number of scalars in 4B. + static int const kScalarsIn4B = sizeof(MultiplyAddScalar) > 4 ? 1 : 4 / sizeof(MultiplyAddScalar); + /// The skew for B. + static int const kSkewB = 128 / sizeof(MultiplyAddScalar) / GemmConfig_::kScalarsPerStsB / + GlobalTileTraits::Threads::kW * kScalarsIn4B; + + /// The traits class to build the iterator to store data to shared memory for B^N. + typedef GemmSharedStoreWithSkewTileAbTraits < + // The pointer is float. + MultiplyAddScalar, + // The tile has size KxN in GEMM's terminology. + Shape<GemmConfig_::kStages, + GemmConfig_::OutputTile::kD / GemmConfig_::InstructionShape::kD, + GemmConfig_::OutputTile::kH * GemmConfig_::InstructionShape::kD>, + // The threads are distributed as (threads / K) x K (the traits may reorganize). + typename GlobalTileTraits::Threads, + // The number of scalars per STS. + GemmConfig_::kScalarsPerStsB, + // The skew to avoid bank conflicts added in the tile W dimension. + kSkewB<GemmConfig_::kScalarsPerLdsB ? GemmConfig_::kScalarsPerLdsB : kSkewB> + SharedStoreTileTraits; + + /// The traits class to build the iterator to load from shared memory for B^N. + typedef GemmSharedLoadTileBTraits< + // The pointer is float const. + MultiplyAddScalar const, + // The output tile size. + typename GemmConfig_::OutputTile, + // The number of warps. + typename GemmConfig_::Warps, + // The number of threads per warp. + typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, + // The shape of the FMA instruction. + typename GemmConfig_::InstructionShape, + // The number of stages. + GemmConfig_::kStages, + // The number of scalars per LDS. + GemmConfig_::kScalarsPerLdsB, + // The skew. + SharedStoreTileTraits::kSkew> + SharedLoadTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_> +struct GemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_> { + /// The layout. + static MatrixLayout::Kind const kLayout = MatrixLayout::kRowMajor; + + /// The input scalar. + typedef typename GemmConfig_::ScalarB Scalar; + /// The scalar stored in shared memory. + typedef typename GemmConfig_::MultiplyAdd::ScalarB MultiplyAddScalar; + + /// The traits class to build the iterator to load data from global memory for B^T. + typedef GemmGlobalTileTraits< + // That's B. + GemmOperand::kB, + // B is row-major. + MatrixLayout::kRowMajor, + // The pointer is float const. + Scalar const, + // The tile has size KxN in GEMM's terminology. + Shape<1, GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kH>, + // The threads are distributed as warps x 32 (the traits may reorganize). + Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerLdgB> + GlobalTileTraits; + + /// The traits class to build the iterator to store data to shared memory for B^T. + typedef GemmSharedStoreTileAbTraits< + // The pointer is float. + MultiplyAddScalar, + // The tile has size KxN in GEMM's terminology. + Shape<GemmConfig_::kStages, + GemmConfig_::OutputTile::kD / GemmConfig_::InstructionShape::kD, + GemmConfig_::OutputTile::kH * GemmConfig_::InstructionShape::kD>, + // The threads are distributed as warps x 32 (the traits may reorganize). + typename GlobalTileTraits::Threads, + // The number of scalars per STS (STS.32 or STS.128, etc). + GemmConfig_::kScalarsPerStsB> + SharedStoreTileTraits; + + /// The traits class to build the iterator to load from shared memory for B^T. + typedef GemmSharedLoadTileBTraits< + // The pointer is float const. + MultiplyAddScalar const, + // The output tile size. + typename GemmConfig_::OutputTile, + // The number of warps. + typename GemmConfig_::Warps, + // The number of threads per warp. + typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, + // The shape of the FMA instruction. + typename GemmConfig_::InstructionShape, + // The number of stages. + GemmConfig_::kStages, + // The number of scalars per LDS. + GemmConfig_::kScalarsPerLdsB, + // The skew. + 0> + SharedLoadTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmTraits_, bool kResidueInPrologue_ = GemmTraits_::kResidueInPrologue> +struct GemmResidue { + /// Move to residue portion. + template <bool kIsPrologue> + static CUTLASS_DEVICE void move_to_residue(typename GemmTraits_::GlobalLoadStreamA& stream_a, + typename GemmTraits_::GlobalLoadStreamB& stream_b, + typename GemmTraits_::Index k) { + // The new code path in CUTLASS 1.0.1: We treat the residue in the prologue so we can have + // complete main loops after that. It helps simplify the logic in the main loop. + if (kIsPrologue) { + stream_a.move_to_residue(k); + stream_b.move_to_residue(k); + } + } + + /// Rollback to beginning of first tile and initialize predicates. + static CUTLASS_DEVICE void rollback(typename GemmTraits_::GlobalLoadStreamA& stream_a, + typename GemmTraits_::GlobalLoadStreamB& stream_b) { + stream_a.rollback(); + stream_b.rollback(); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmTraits_> +struct GemmResidue<GemmTraits_, false> { + /// Move to residue portion. + template <bool kIsPrologue> + static CUTLASS_DEVICE void move_to_residue(typename GemmTraits_::GlobalLoadStreamA& stream_a, + typename GemmTraits_::GlobalLoadStreamB& stream_b, + typename GemmTraits_::Index k) { + // The index. + typedef typename GemmTraits_::Index Index; + // By how much we unroll the main loop. + Index const kUnroll = static_cast<Index>(GemmTraits_::OutputTile::kD); + + // Call the residue code. That's the same path as CUTLASS 1.0.0. + if (kIsPrologue && k < kUnroll) { + stream_a.residue(k, true); + stream_b.residue(k, true); + } else if (k <= kUnroll) { + stream_a.residue(k, false); + stream_b.residue(k, false); + } + } + + /// Rollback to beginning of first tile and initialize predicates. + static CUTLASS_DEVICE void rollback(typename GemmTraits_::GlobalLoadStreamA& stream_a, + typename GemmTraits_::GlobalLoadStreamB& stream_b) {} +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The GEMM configuration. + typename GemmConfig_, + /// The stream to load A from global memory to shared memory. + typename GlobalLoadStreamA_, + /// The stream to load B from global memory to shared memory. + typename GlobalLoadStreamB_, + /// The stream to load A from shared memory. + typename SharedLoadStreamA_, + /// The stream to load B from shared memory. + typename SharedLoadStreamB_, + /// The epilogue. + typename Epilogue_, + /// The block swizzle to reorganize the grid. + typename BlockSwizzle_ = IdentityBlockSwizzle, + /// The index. + typename Index_ = int, + /// The tool used to clear accumulators. + typename ClearAccumulators_ = ClearAccumulators<typename GemmConfig_::Accumulators::Scalar> > + +struct GemmTraits { + /// This class. + typedef GemmTraits<GemmConfig_, + GlobalLoadStreamA_, + GlobalLoadStreamB_, + SharedLoadStreamA_, + SharedLoadStreamB_, + Epilogue_, + BlockSwizzle_, + Index_, + ClearAccumulators_> + This_; + + /// The configuration. + typedef GemmConfig_ GemmConfig; + /// The output tile. + typedef typename GemmConfig::OutputTile OutputTile; + /// Is the residue treated in the prologue? + static bool const kResidueInPrologue = GemmConfig::kResidueInPrologue; + + /// The stream to load A from global memory to shared memory. + typedef GlobalLoadStreamA_ GlobalLoadStreamA; + /// The layout of A. + static MatrixLayout::Kind const kLayoutA = GlobalLoadStreamA::kLayout; + /// The scalar for A. + typedef typename GlobalLoadStreamA_::Scalar ScalarA; + + /// The stream to load B from global memory to shared memory. + typedef GlobalLoadStreamB_ GlobalLoadStreamB; + /// The layout of B. + static MatrixLayout::Kind const kLayoutB = GlobalLoadStreamB::kLayout; + /// The scalar for B. + typedef typename GlobalLoadStreamB_::Scalar ScalarB; + + /// The iterator for A to load from shared memory. + typedef SharedLoadStreamA_ SharedLoadStreamA; + /// The iterator for B to load from shared memory. + typedef SharedLoadStreamB_ SharedLoadStreamB; + + /// The multiply-add functor. + typedef typename GemmConfig::MultiplyAdd MultiplyAdd; + /// The epilogue. + typedef Epilogue_ Epilogue; + /// The scalars in the epilogue. + typedef typename Epilogue::ScalarC ScalarC; + typedef typename Epilogue::ScalarD ScalarD; + + /// The block swizzle to reorganize the grid. + typedef BlockSwizzle_ BlockSwizzle; + /// The index. + typedef Index_ Index; + /// Clear the accumulators. + typedef ClearAccumulators_ ClearAccumulators; + + /// The params. + struct Params { + /// The dimensions of the GEMM. + Index m, n, k; + /// The params for the A stream. + typename GlobalLoadStreamA::Params global_stream_a; + /// The params for the B stream. + typename GlobalLoadStreamB::Params global_stream_b; + /// The params for the A stream from shared memory. + typename SharedLoadStreamA::Params shared_stream_a; + /// The params for the B stream from shared memory. + typename SharedLoadStreamB::Params shared_stream_b; + /// The params for the epilogue. + typename Epilogue::Params epilogue; + + /// Initialize the parameters. + template <typename GemmDesc_> + CUTLASS_HOST_DEVICE int initialize(GemmDesc_ const& desc) { + // Set the problem size. + this->m = desc.m; + this->n = desc.n; + this->k = desc.k; + + // Initialize the iterator for A. + int error_code = + global_stream_a.initialize(desc, reinterpret_cast<ScalarA const*>(desc.d_a), desc.lda); + + if (error_code) { + return error_code; + } + + // Initialize the iterator for B. + error_code = + global_stream_b.initialize(desc, reinterpret_cast<ScalarB const*>(desc.d_b), desc.ldb); + + if (error_code) { + return error_code; + } + + // The epilogue. + return epilogue.initialize(desc); + } + }; + + // The storage for A. + template <typename GlobalLoadStream_, typename SharedLoadStream_> + union StreamSharedStorage { + // The storage needed by the global stream. + typename GlobalLoadStream_::SharedStorage global; + // The storage needed by the shared stream. + typename SharedLoadStream_::SharedStorage shared; + }; + + // The storage for the main loop + prologue. + struct MainLoopSharedStorage { + // The storage to shuffle the A matrix in shared memory. + StreamSharedStorage<GlobalLoadStreamA, SharedLoadStreamA> stream_a; + // The storage to shuffle the B matrix in shared memory. + StreamSharedStorage<GlobalLoadStreamB, SharedLoadStreamB> stream_b; + // The storage to clear the accumulators if needed. + typename ClearAccumulators::SharedStorage clear; + }; + + /// The storage in shared memory. + union SharedStorage { + // The storage for the main loop. + MainLoopSharedStorage main_loop; + // The storage for the epilogue. + typename Epilogue::SharedStorage epilogue; + }; + + /// Assemble the global load streams for A/B. + struct GlobalLoadStream { + /// Ctor. + CUTLASS_DEVICE GlobalLoadStream(Params const& params, + SharedStorage& shared_storage, + dim3 const& block) + : stream_a(params.global_stream_a, + shared_storage.main_loop.stream_a.global, + cutlass::make_Coord(0, params.k, params.m), + cutlass::make_Coord(0, 0, block.x)), + stream_b(params.global_stream_b, + shared_storage.main_loop.stream_b.global, + cutlass::make_Coord(0, params.k, params.n), + make_Coord(0, 0, block.y)) {} + + /// Trigger the copies from shared memory to registers. + CUTLASS_DEVICE void copy() { + stream_a.copy(); + stream_b.copy(); + } + + /// Commit the data. + CUTLASS_DEVICE void commit() { + stream_a.commit(); + stream_b.commit(); + } + + /// Move to residue portion. + template <bool kIsPrologue> + CUTLASS_DEVICE void move_to_residue(Index k) { + GemmResidue<This_>::move_to_residue<kIsPrologue>(stream_a, stream_b, k); + } + + /// Rollback to beginning of first tile and initialize predicates. + CUTLASS_DEVICE void rollback() { GemmResidue<This_>::rollback(stream_a, stream_b); } + + /// The stream for A. + GlobalLoadStreamA stream_a; + /// The stream for B. + GlobalLoadStreamB stream_b; + }; + + /// Assemble the shared load stream for A/B. + struct SharedLoadStream { + /// Ctor. + CUTLASS_DEVICE SharedLoadStream(Params const& params, SharedStorage& shared_storage) { + stream_a.initialize(params.shared_stream_a, shared_storage.main_loop.stream_a.shared); + stream_b.initialize(params.shared_stream_b, shared_storage.main_loop.stream_b.shared); + } + + /// Trigger the copies from shared memory to registers. + CUTLASS_DEVICE void copy(int step) { + stream_a.copy(step, fetched_a[step % 2]); + stream_b.copy(step, fetched_b[step % 2]); + } + + /// Commit the data. + CUTLASS_DEVICE void commit(int step) { + stream_a.commit(fetched_a[step % 2], transformed_a[step % 2]); + stream_b.commit(fetched_b[step % 2], transformed_b[step % 2]); + } + + /// The fragment A. + CUTLASS_DEVICE typename SharedLoadStreamA::Fragment const& fragment_a(int step) const { + return transformed_a[step % 2]; + } + + /// The fragment B. + CUTLASS_DEVICE typename SharedLoadStreamB::Fragment const& fragment_b(int step) const { + return transformed_b[step % 2]; + } + + /// Increment the stage. + CUTLASS_DEVICE void inc_stage() { + stream_a.inc_stage(); + stream_b.inc_stage(); + } + + /// The stream for A. + SharedLoadStreamA stream_a; + /// The fragments to fetch A. + typename SharedLoadStreamA::FetchedFragment fetched_a[2]; + /// The fragments to transform A. + typename SharedLoadStreamA::TransformedFragment transformed_a[2]; + /// The stream for B. + SharedLoadStreamB stream_b; + /// The fragments to fetch B. + typename SharedLoadStreamB::FetchedFragment fetched_b[2]; + /// The fragments to transform B. + typename SharedLoadStreamB::TransformedFragment transformed_b[2]; + }; + + /// The memory fence for shared loads. + static CUTLASS_DEVICE void shared_load_fence(bool in_loop) { + if (SharedLoadStreamA::Iterator::kRequiresLoadFence || + SharedLoadStreamB::Iterator::kRequiresLoadFence) { + __syncthreads(); + } + } + + /// The memory fence for shared stores. + static CUTLASS_DEVICE void shared_store_fence(bool in_loop) { __syncthreads(); } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmTileTraitsHelperA_, typename GemmTileTraitsHelperB_, typename Index_> +struct SimplifiedGemmTraitsHelper { + /// The global iterator to load A from global memory. + typedef GemmGlobalIteratorAb<typename GemmTileTraitsHelperA_::GlobalTileTraits, Index_> + GlobalLoadIteratorA; + /// The data converter for A before storing to shared memory. + typedef Copy<typename GlobalLoadIteratorA::Fragment> GlobalTransformerA; + /// The iterator to store A to shared memory. + typedef TileStoreIterator<typename GemmTileTraitsHelperA_::SharedStoreTileTraits, + typename GemmTileTraitsHelperA_::SharedStoreTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedStoreIteratorA; + /// The stream to load A from global memory to shared memory. + typedef GlobalLoadStream<GlobalLoadIteratorA, SharedStoreIteratorA, GlobalTransformerA> + GlobalLoadStreamA; + + /// The global iterator to load B from global memory. + typedef GemmGlobalIteratorAb<typename GemmTileTraitsHelperB_::GlobalTileTraits, Index_> + GlobalLoadIteratorB; + /// The data converter for B before storing to shared memory. + typedef Copy<typename GlobalLoadIteratorB::Fragment> GlobalTransformerB; + /// The iterator to store B to shared memory. + typedef TileStoreIterator<typename GemmTileTraitsHelperB_::SharedStoreTileTraits, + typename GemmTileTraitsHelperB_::SharedStoreTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedStoreIteratorB; + /// The stream to load B from global memory to shared memory. + typedef GlobalLoadStream<GlobalLoadIteratorB, SharedStoreIteratorB, GlobalTransformerB> + GlobalLoadStreamB; + + /// The iterator to load A from shared memory. + typedef TileLoadIterator<typename GemmTileTraitsHelperA_::SharedLoadTileTraits, + typename GemmTileTraitsHelperA_::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedLoadIteratorA; + /// The stream to load A from shared memory. + typedef SharedLoadStream<SharedLoadIteratorA> SharedLoadStreamA; + /// The iterator to load B from shared memory. + typedef TileLoadIterator<typename GemmTileTraitsHelperB_::SharedLoadTileTraits, + typename GemmTileTraitsHelperB_::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedLoadIteratorB; + /// The stream to load B from shared memory. + typedef SharedLoadStream<SharedLoadIteratorB> SharedLoadStreamB; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The layout for A. + MatrixLayout::Kind kLayoutA_, + /// The layout for B. + MatrixLayout::Kind kLayoutB_, + /// The config for the GEMM. + typename GemmConfig_, + /// The epilogue. + typename Epilogue_, + /// The index. + typename Index_ = int, + // The configuration for the A matrix. + typename GemmTileTraitsHelperA_ = GemmTileTraitsHelperA<kLayoutA_, GemmConfig_>, + // The configuration for the B matrix. + typename GemmTileTraitsHelperB_ = GemmTileTraitsHelperB<kLayoutB_, GemmConfig_>, + // The helper class to create the streams and iterators. + typename Helper_ = + SimplifiedGemmTraitsHelper<GemmTileTraitsHelperA_, GemmTileTraitsHelperB_, Index_> > +struct SimplifiedGemmTraits : public GemmTraits< + // The config. + GemmConfig_, + // The stream to load A from global memory to shared memory. + typename Helper_::GlobalLoadStreamA, + // The stream to load B from global memory to shared memory. + typename Helper_::GlobalLoadStreamB, + // The stream to load A from shared memory. + typename Helper_::SharedLoadStreamA, + // The stream to load B from shared memory. + typename Helper_::SharedLoadStreamB, + // The epilogue. + Epilogue_, + // The block swizzle to reorganize the grid. + IdentityBlockSwizzle, + // The index. + Index_, + // The tool used to clear accumulators. + ClearAccumulators<typename GemmConfig_::Accumulators::Element> > { +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/hgemm_global_tile.h b/cutlass-example/cutlass/gemm/hgemm_global_tile.h new file mode 100644 index 0000000..f14dbb3 --- /dev/null +++ b/cutlass-example/cutlass/gemm/hgemm_global_tile.h @@ -0,0 +1,90 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Tile traits used to construct global tile iterator for HGEMM. This is intended to + partition the thread block-level tile into 2D subtiles loaded by the threads and facilitate + memory accesses larger than 16 bits. +*/ +#pragma once + +#include <cutlass/coord.h> +#include <cutlass/gemm/gemm_global_tile.h> +#include <cutlass/matrix_traits.h> +#include <cutlass/reshape_tile.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <GemmOperand::Kind kOperand_, + MatrixLayout::Kind kLayout_, + typename Scalar_, + typename Tile_, + typename Threads_, + int kAccessSize_> +struct HgemmCrosswiseGlobalTileTraits : public GemmGlobalTileTraits< + // Which GEMM operand? + kOperand_, + // The layout. + kLayout_, + // The scalar. + Scalar_, + // The tile. + Tile_, + // The threads. + Threads_, + // The number of scalars per LDG/STG. + kAccessSize_> { + /// The base class. + typedef GemmGlobalTileTraits<kOperand_, kLayout_, Scalar_, Tile_, Threads_, kAccessSize_> Base; + /// The threads. + typedef typename Base::Threads Threads; + /// The threads strides. + typedef Shape<1, 2, Base::Tile::kC> ThreadsDelta; + /// The strides in each dimension between different loads/stores. + typedef Shape<Base::Threads::kH * 2, 1, Base::Threads::kW, Base::kAccessSize> Delta; + /// The number of iterations needed to load/store the tile. + typedef Shape<Base::Tile::kH / Base::Threads::kH / 2, + 2, + Base::Tile::kW / Base::Threads::kW, + Base::Tile::kC / Base::kAccessSize> + Iterations; + /// Computes the thread offset in (H, W) based on thread ID + struct ThreadOffset { + CUTLASS_HOST_DEVICE + Coord<4> operator()() const { + int thread_offset_h = threadIdx.x / Threads::kW * ThreadsDelta::kH; + int thread_offset_w = threadIdx.x % Threads::kW * ThreadsDelta::kW; + + return make_Coord(0, thread_offset_h, thread_offset_w, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/hgemm_multiply_add.h b/cutlass-example/cutlass/gemm/hgemm_multiply_add.h new file mode 100644 index 0000000..ebbdd06 --- /dev/null +++ b/cutlass-example/cutlass/gemm/hgemm_multiply_add.h @@ -0,0 +1,104 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Specialization implementing multiply-add operation on half-precision floating point + fragments. +*/ +#pragma once + +#include <cutlass/fragment.h> + +#include <cutlass/gemm/thread_multiply_add.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Template performing matrix multiply-add operation within a thread +template <typename AccumulatorsPerThread_, typename ThreadsPerWarp_> +struct ThreadMultiplyAdd<AccumulatorsPerThread_, ThreadsPerWarp_, half, half, half> { + /// The shape of the instruction. + typedef Shape<1, 1, 2, 1> InstructionShape; + /// The number of accumulators per thread. + typedef AccumulatorsPerThread_ AccumulatorsPerThread; + /// The number of threads per warp. + typedef ThreadsPerWarp_ ThreadsPerWarp; + /// The number of accumulators per warp. + typedef typename ShapeMul<AccumulatorsPerThread, ThreadsPerWarp>::Shape AccumulatorsPerWarp; + /// The type for A. + typedef half ScalarA; + /// The fragment for A. + typedef Fragment<ScalarA, AccumulatorsPerThread::kW> FragmentA; + /// The type for B. + typedef half ScalarB; + /// The fragment for B. + typedef Fragment<ScalarB, AccumulatorsPerThread::kH> FragmentB; + /// The type for C and D. + typedef half ScalarC; + /// The accumulators. + typedef Fragment<half, AccumulatorsPerThread::kH * AccumulatorsPerThread::kW> Accumulators; + + /// Make sure there's an even number of elements in both dimensions. + static_assert(AccumulatorsPerThread::kH % 2 == 0, "Invalid size"); + static_assert(AccumulatorsPerThread::kW % 2 == 0, "Invalid size"); + + /// Ctor. + CUTLASS_DEVICE ThreadMultiplyAdd() {} + + /// Multiply : d = a*b + c. + CUTLASS_DEVICE void multiply_add(FragmentA const& a, + FragmentB const& b, + Accumulators const& c, + Accumulators& d) { +#if defined(__CUDACC__) && __CUDA_ARCH__ >= 530 + // The inputs. + __half2 const* a_half2 = reinterpret_cast<__half2 const*>(&a[0]); + __half2 const* b_half2 = reinterpret_cast<__half2 const*>(&b[0]); + __half2 const* c_half2 = reinterpret_cast<__half2 const*>(&c[0]); + + // The output. + __half2* d_half2 = reinterpret_cast<__half2*>(&d[0]); + + for (int j = 0; j < AccumulatorsPerThread::kH / 2; ++j) { + for (int i = 0; i < AccumulatorsPerThread::kW / 2; ++i) { + // The offsets in the output fragment. + int const k0 = (2 * j + 0) * (AccumulatorsPerThread::kW / 2) + i; + int const k1 = (2 * j + 1) * (AccumulatorsPerThread::kW / 2) + i; + + // Compute the product a[i] * b[j].H0_H0. + d_half2[k0] = __hfma2(a_half2[i], __low2half2(b_half2[j]), c_half2[k0]); + // Compute the product a[i] * b[j].H1_H1. + d_half2[k1] = __hfma2(a_half2[i], __high2half2(b_half2[j]), c_half2[k1]); + } + } +#endif + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/hgemm_swizzle.h b/cutlass-example/cutlass/gemm/hgemm_swizzle.h new file mode 100644 index 0000000..ebec0d4 --- /dev/null +++ b/cutlass-example/cutlass/gemm/hgemm_swizzle.h @@ -0,0 +1,94 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Transposes a tile of 16b elements. Used by HGEMM to construct a K-strided layout in + shared memory for multiplicands. +*/ +#pragma once + +#include <cuda_fp16.h> +#include <cutlass/fragment.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GlobalIterator_> +struct HgemmSwizzle { + /// The global iterator. + typedef GlobalIterator_ GlobalIterator; + /// The source fragment. + typedef typename GlobalIterator::Fragment Fragment; + /// The shape of the source fragment. + typedef typename GlobalIterator::FragmentShape FragmentShape; + + /// The input fragment. + typedef Fragment InputFragment; + /// The output fragment. + typedef Fragment OutputFragment; + + /// The src/dst must be half fragments. + static_assert((platform::is_same<typename Fragment::Element, half>::value), "Works on half"); + + /// The number of elements must be a multiple of 2. + static_assert(FragmentShape::kH == 2 && ShapeCount<FragmentShape>::kWc == 2, "Not multiple of 2"); + + /// Ctor. + CUTLASS_DEVICE HgemmSwizzle() {} + + /// Transform a fragment. + CUTLASS_DEVICE void transform(Fragment const& src, Fragment& dst) { + // Expose src/dst as int arrays. + int const* src_int = reinterpret_cast<int const*>(&src[0]); + int* dst_int = reinterpret_cast<int*>(&dst[0]); + + // Transpose the data. + for (int d = 0; d < FragmentShape::kD; ++d) { + // The indices to read two consecutive "rows". + int const i0 = 2 * d + 0; + int const i1 = 2 * d + 1; + + int a0 = src_int[i0]; + int a1 = src_int[i1]; + + int b0, b1; + asm volatile("prmt.b32 %0, %1, %2, 0x5410;" : "=r"(b0) : "r"(a0), "r"(a1)); + asm volatile("prmt.b32 %0, %1, %2, 0x7632;" : "=r"(b1) : "r"(a0), "r"(a1)); + + // The indices to store with "strides". + int const j0 = 0 * (ShapeCount<FragmentShape>::kDhw / 2) + d; + int const j1 = 1 * (ShapeCount<FragmentShape>::kDhw / 2) + d; + + dst_int[j0] = b0; + dst_int[j1] = b1; + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/hgemm_traits.h b/cutlass-example/cutlass/gemm/hgemm_traits.h new file mode 100644 index 0000000..b08645b --- /dev/null +++ b/cutlass-example/cutlass/gemm/hgemm_traits.h @@ -0,0 +1,397 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defies structural properties of half-precision GEMM computation. +*/ +#pragma once + +#include <cutlass/convert.h> +#include <cutlass/reshape_tile.h> + +#include <cutlass/gemm/gemm.h> +#include <cutlass/gemm/gemm_epilogue.h> +#include <cutlass/gemm/gemm_epilogue_traits.h> +#include <cutlass/gemm/gemm_global_tile.h> +#include <cutlass/gemm/gemm_shared_tile.h> +#include <cutlass/gemm/gemm_traits.h> +#include <cutlass/gemm/hgemm_global_tile.h> +#include <cutlass/gemm/hgemm_multiply_add.h> +#include <cutlass/gemm/hgemm_swizzle.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The tile size for the GEMM KxNxM. + typename OutputTile_, + /// The number of accumulators per thread. + typename AccumulatorsPerThread_, + /// The number of scalars per LDG for A. + int kScalarsPerLdgA_ = 2, + /// The number of scalars per LDG for B. + int kScalarsPerLdgB_ = 2> +struct HgemmConfig + : public GemmConfig< + /// The scalar type for A. + half, + /// The scalar type for B. + half, + /// The scalar type for C. + half, + /// The scalar type for D. + half, + /// The tile size for the GEMM KxNxM. + OutputTile_, + /// The functor to do the math in the main loop. + ThreadMultiplyAdd<AccumulatorsPerThread_, Shape<1, 4, 8>, half, half, half>, + /// The number of scalars per LDG for A. + kScalarsPerLdgA_, + /// The number of scalars per STS for A. + kScalarsPerLdgA_, + /// The number of scalars per LDS for A. + 8, + /// The number of scalars per LDG for B. + kScalarsPerLdgB_, + /// The number of scalars per STS for B. + kScalarsPerLdgB_, + /// The number of scalars per LDS for B. + 8, + /// The number of scalars per LDG for C and STG for D. + 2, + /// The number of scalars per STS for D. + 8, + /// The number of scalars per LDS for D. + 2, + /// The number of stages in shared memory. + 2> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <enum MatrixLayout::Kind kLayout_, typename Iterator_> +struct HgemmTransformerA {}; + +template <typename Iterator_> +struct HgemmTransformerA<MatrixLayout::kColumnMajor, Iterator_> { + typedef Convert<typename Iterator_::Fragment, typename Iterator_::Fragment> Transformer; +}; + +template <typename Iterator_> +struct HgemmTransformerA<MatrixLayout::kRowMajor, Iterator_> { + typedef HgemmSwizzle<Iterator_> Transformer; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <enum MatrixLayout::Kind kLayout_, typename Iterator_> +struct HgemmTransformerB {}; + +template <typename Iterator_> +struct HgemmTransformerB<MatrixLayout::kRowMajor, Iterator_> { + typedef Convert<typename Iterator_::Fragment, typename Iterator_::Fragment> Transformer; +}; + +template <typename Iterator_> +struct HgemmTransformerB<MatrixLayout::kColumnMajor, Iterator_> { + typedef HgemmSwizzle<Iterator_> Transformer; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <enum MatrixLayout::Kind kLayout_, typename GemmConfig_> +struct HgemmTileTraitsHelperA : public GemmTileTraitsHelperA<kLayout_, GemmConfig_> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_> +struct HgemmTileTraitsHelperA<MatrixLayout::kRowMajor, GemmConfig_> + : public GemmTileTraitsHelperA<MatrixLayout::kRowMajor, GemmConfig_> { + /// The base config. + typedef GemmTileTraitsHelperA<MatrixLayout::kRowMajor, GemmConfig_> Base; + + /// The traits class to build the iterator to load data from global memory for A^T. + typedef HgemmCrosswiseGlobalTileTraits< + GemmOperand::kA, + // The layout. + MatrixLayout::kRowMajor, + // The pointer. + half const, + // The tile has size MxK in GEMM's terminology. + Shape<1, GemmConfig_::OutputTile::kW, GemmConfig_::OutputTile::kD>, + // The threads are distributed as (threads / K ) x K (the traits may reorganize). + Shape<1, GemmConfig_::kThreads / GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kD>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc) + GemmConfig_::kScalarsPerLdgA> + GlobalTileTraits; + + /// The skew. + static int const kSkewA = 128 / sizeof(half) / GlobalTileTraits::Threads::kW / 2; + + /// The traits class to build the iterator to store data to shared memory for A^T. + typedef GemmSharedStoreWithSkewTileAbTraits < + // The pointer. + half, + // The tile has size KxM in GEMM's terminology. + Shape<GemmConfig_::kStages, + GemmConfig_::OutputTile::kD / GemmConfig_::InstructionShape::kD, + GemmConfig_::OutputTile::kW * GemmConfig_::InstructionShape::kD>, + // The threads are distributed as warps x 32(the traits may reorganize). + typename GlobalTileTraits::Threads, + // The number of scalars per STS (STS.32 or STS.128, etc). + 2, + // The skew to avoid bank conflicts added in the tile W dimension. + kSkewA<GemmConfig_::kScalarsPerLdsA ? GemmConfig_::kScalarsPerLdsA : kSkewA> + SharedStoreTileTraits; + + /// The traits class to build the iterator to load from shared memory for A^T. + typedef GemmSharedLoadTileATraits< + // The pointer. + half const, + // The output tile size. + typename GemmConfig_::OutputTile, + // The number of warps. + typename GemmConfig_::Warps, + // The number of threads per warp. + typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, + // The shape of the FMA instruction. + typename GemmConfig_::InstructionShape, + // The number of stages. + GemmConfig_::kStages, + // The number of scalars per LDS. + 8, + // The skew. + SharedStoreTileTraits::kSkew> + SharedLoadTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <enum MatrixLayout::Kind kLayout_, typename GemmConfig_> +struct HgemmTileTraitsHelperB : public GemmTileTraitsHelperB<kLayout_, GemmConfig_> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_> +struct HgemmTileTraitsHelperB<MatrixLayout::kColumnMajor, GemmConfig_> + : public GemmTileTraitsHelperB<MatrixLayout::kColumnMajor, GemmConfig_> { + /// The base config. + typedef GemmTileTraitsHelperB<MatrixLayout::kColumnMajor, GemmConfig_> Base; + + /// The traits class to build the iterator to load data from global memory for B^N. + typedef HgemmCrosswiseGlobalTileTraits< + GemmOperand::kB, + // The layout. + MatrixLayout::kColumnMajor, + // The pointer. + half const, + // The tile has size KxN in GEMM's terminology. + Shape<1, GemmConfig_::OutputTile::kH, GemmConfig_::OutputTile::kD>, + // The threads are distributed as (threads / K) x K (the traits may reorganize). + Shape<1, GemmConfig_::kThreads / GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kD>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc) + GemmConfig_::kScalarsPerLdgB> + GlobalTileTraits; + + /// The skew for B. + static int const kSkewB = 128 / sizeof(half) / GlobalTileTraits::Threads::kW / 2; + + /// The traits class to build the iterator to store data to shared memory for B^N. + typedef GemmSharedStoreWithSkewTileAbTraits < + // The pointer. + half, + // The tile has size KxN in GEMM's terminology. + Shape<GemmConfig_::kStages, + GemmConfig_::OutputTile::kD / GemmConfig_::InstructionShape::kD, + GemmConfig_::OutputTile::kH * GemmConfig_::InstructionShape::kD>, + // The threads are distributed as (threads / K) x K (the traits may reorganize). + typename GlobalTileTraits::Threads, + // The number of scalars per STS (STS.32 or STS.128, etc). + 2, + // The skew to avoid bank conflicts added in the tile W dimension. + kSkewB<GemmConfig_::kScalarsPerLdsB ? GemmConfig_::kScalarsPerLdsB : kSkewB> + SharedStoreTileTraits; + + /// The traits class to build the iterator to load from shared memory for B^N. + typedef GemmSharedLoadTileBTraits< + // The pointer. + half const, + // The output tile size. + typename GemmConfig_::OutputTile, + // The number of warps. + typename GemmConfig_::Warps, + // The number of threads per warp. + typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, + // The shape of the FMA instruction. + typename GemmConfig_::InstructionShape, + // The number of stages. + GemmConfig_::kStages, + // The number of scalars per LDS. + 8, + // The skew. + SharedStoreTileTraits::kSkew> + SharedLoadTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The layout for A. + MatrixLayout::Kind kLayoutA_, + /// The layout for B. + MatrixLayout::Kind kLayoutB_, + /// The output tile. + typename OutputTile_, + /// The functor to do the math in the epilogue. + typename EpilogueFunctor_, + /// The number of accumulators per thread. + typename AccumulatorsPerThread_ = Shape<8, 8, 16>, + /// The number of halfs loaded in one LDG for A. + int kScalarsPerLdgA_ = 2, + /// The number of halfs loaded in one LDG for B. + int kScalarsPerLdgB_ = 2, + /// The index. + typename Index_ = int> +struct HgemmTraitsHelper { + /// The HGEMM config. + typedef HgemmConfig<OutputTile_, AccumulatorsPerThread_, kScalarsPerLdgA_, kScalarsPerLdgB_> + GemmConfig; + /// The GEMM config for A. + typedef HgemmTileTraitsHelperA<kLayoutA_, GemmConfig> GemmTileTraitsHelperA; + /// The GEMM config for B. + typedef HgemmTileTraitsHelperB<kLayoutB_, GemmConfig> GemmTileTraitsHelperB; + + /// The iterator to load A from global memory. + typedef GemmGlobalIteratorAb<typename GemmTileTraitsHelperA::GlobalTileTraits, Index_> + GlobalLoadIteratorA; + /// The default transformer for A. + typedef typename HgemmTransformerA<GemmTileTraitsHelperA::kLayout, + GlobalLoadIteratorA>::Transformer GlobalTransformerA; + /// The iterator to store A to shared memory. + typedef TileStoreIterator<typename GemmTileTraitsHelperA::SharedStoreTileTraits, + typename GemmTileTraitsHelperA::SharedStoreTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedStoreIteratorA; + /// The stream to load A from global memory to shared memory. + typedef GlobalLoadStream<GlobalLoadIteratorA, SharedStoreIteratorA, GlobalTransformerA> + GlobalLoadStreamA; + + /// The iterator to load B from global memory. + typedef GemmGlobalIteratorAb<typename GemmTileTraitsHelperB::GlobalTileTraits, Index_> + GlobalLoadIteratorB; + // The default transformer for B. + typedef typename HgemmTransformerB<GemmTileTraitsHelperB::kLayout, + GlobalLoadIteratorB>::Transformer GlobalTransformerB; + /// The iterator to store B to shared memory. + typedef TileStoreIterator<typename GemmTileTraitsHelperB::SharedStoreTileTraits, + typename GemmTileTraitsHelperB::SharedStoreTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedStoreIteratorB; + /// The stream to load B from global memory to shared memory. + typedef GlobalLoadStream<GlobalLoadIteratorB, SharedStoreIteratorB, GlobalTransformerB> + GlobalLoadStreamB; + + /// The iterator to load A from shared memory + typedef TileLoadIterator<typename GemmTileTraitsHelperA::SharedLoadTileTraits, + typename GemmTileTraitsHelperA::SharedLoadTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedLoadIteratorA; + /// The stream to load A from shared memory. + typedef SharedLoadStream<SharedLoadIteratorA> SharedLoadStreamA; + /// The iterator to load B from shared memory. + typedef TileLoadIterator<typename GemmTileTraitsHelperB::SharedLoadTileTraits, + typename GemmTileTraitsHelperB::SharedLoadTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedLoadIteratorB; + /// The stream to load B from shared memory. + typedef SharedLoadStream<SharedLoadIteratorB> SharedLoadStreamB; + + /// The functor to do the multiply-add in the main loop. + typedef typename GemmConfig::MultiplyAdd MultiplyAdd; + /// The object to clear accumulators. + typedef ClearAccumulators<typename MultiplyAdd::ScalarC> ClearAccumulators; + + /// The traits class for the epilogue. + typedef SimplifiedGemmEpilogueTraits<GemmConfig, EpilogueFunctor_, Index_> GemmEpilogueTraits; + /// The epilogue. + typedef GemmEpilogue<GemmEpilogueTraits> Epilogue; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The layout for A. + MatrixLayout::Kind kLayoutA_, + /// The layout for B. + MatrixLayout::Kind kLayoutB_, + /// The output tile. + typename OutputTile_ = Shape<8, 128, 128>, + /// The functor to do the math in the epilogue. + typename EpilogueFunctor_ = LinearScaling<half>, + /// The number of accumulators per thread. + typename AccumulatorsPerThread_ = Shape<8, 8, 16>, + /// The number of halfs loaded in one LDG for A. + int kScalarsPerLdgA_ = 2, + /// The number of halfs loaded in one LDG for B. + int kScalarsPerLdgB_ = 2, + /// The index. + typename Index_ = int, + /// The helper class. + typename Helper_ = HgemmTraitsHelper<kLayoutA_, + kLayoutB_, + OutputTile_, + EpilogueFunctor_, + AccumulatorsPerThread_, + kScalarsPerLdgA_, + kScalarsPerLdgB_, + Index_> > +struct HgemmTraits : public GemmTraits< + // The config. + typename Helper_::GemmConfig, + // The stream to load A from global memory to shared memory. + typename Helper_::GlobalLoadStreamA, + // The stream to load B from global memory to shared memory. + typename Helper_::GlobalLoadStreamB, + // The stream to load A from shared memory. + typename Helper_::SharedLoadStreamA, + // The stream to load B from shared memory. + typename Helper_::SharedLoadStreamB, + // The epilogue. + typename Helper_::Epilogue, + // The block swizzle to reorganize the grid. + IdentityBlockSwizzle, + // The index. + Index_, + // The tool used to clear accumulators. + typename Helper_::ClearAccumulators> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/identity_block_swizzle.h b/cutlass-example/cutlass/gemm/identity_block_swizzle.h new file mode 100644 index 0000000..e1bdb2e --- /dev/null +++ b/cutlass-example/cutlass/gemm/identity_block_swizzle.h @@ -0,0 +1,48 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defies functors for mapping blockIdx to partitions of the GEMM computation. + + Currently, we only implement an identity mapping. +*/ +#pragma once + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct IdentityBlockSwizzle { + /// Ctor. + CUTLASS_DEVICE IdentityBlockSwizzle() {} + + /// Swizzle the block index. + CUTLASS_DEVICE dim3 swizzle() { return blockIdx; } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/igemm_epilogue.h b/cutlass-example/cutlass/gemm/igemm_epilogue.h new file mode 100644 index 0000000..0d69980 --- /dev/null +++ b/cutlass-example/cutlass/gemm/igemm_epilogue.h @@ -0,0 +1,320 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines the epilogue phase of the GEMM computation for IGEMM, supporting integer and + floating-point output matrix formats. +*/ +#pragma once + +#include <cutlass/convert.h> +#include <cutlass/fragment.h> +#include <cutlass/gemm/gemm_global_stream.h> +#include <cutlass/gemm/gemm_shared_stream.h> +#include <cutlass/gemm/igemm_global_tile.h> +#include <cutlass/reshape_tile.h> +#include <cutlass/tile_iterator.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <int kElements_> +struct IgemmFloatToInt8Converter { + /// The input fragment. + typedef Fragment<float, kElements_> InputFragment; + /// The output fragment. + typedef Fragment<int8_t, kElements_> OutputFragment; + + // We are packing 4 floats into int32 registers so we need kElements to be multiple of 4. + static_assert(kElements_ % 4 == 0, "kElements must be multiple of 4"); + + /// Ctor. + CUTLASS_DEVICE IgemmFloatToInt8Converter() {} + + /// Transform a fragment. + CUTLASS_DEVICE void transform(InputFragment const& src, OutputFragment& dst) { + transform(src, 0, dst); + } + + /// Transform a fragment. + template <typename Fragment_> + CUTLASS_DEVICE void transform(Fragment_ const& src, int offset, OutputFragment& dst) { + // The inputs. + float4 const* src_f4 = reinterpret_cast<float4 const*>(&src[0]); + // The outputs. + int* dst_int = reinterpret_cast<int*>(&dst[0]); + + // Iterate over the floats and pack them together to produce ints. + for (int i = 0; i < kElements_ / 4; ++i) { + // Read the float4. + float4 f4 = src_f4[i]; + + // Clamp the 4 elements of the floats to the [-128, +127] range. + float x = fmaxf(-128.f, fminf(127.f, f4.x)); + float y = fmaxf(-128.f, fminf(127.f, f4.y)); + float z = fmaxf(-128.f, fminf(127.f, f4.z)); + float w = fmaxf(-128.f, fminf(127.f, f4.w)); + + // Convert to integers. + int ix = (int)x; + int iy = (int)y; + int iz = (int)z; + int iw = (int)w; + + // Extract the lower bytes to build an int32 with 4 int8. + asm volatile("prmt.b32 %0, %0, %1, 0x1140;" : "+r"(ix) : "r"(iy)); + asm volatile("prmt.b32 %0, %0, %1, 0x1140;" : "+r"(iz) : "r"(iw)); + asm volatile("prmt.b32 %0, %0, %1, 0x5410;" : "+r"(ix) : "r"(iz)); + + // Store the int. + dst_int[i] = ix; + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename InputScalar_, typename OutputFragment_> +struct IgemmGlobalStoreTransformer { + typedef Convert<Fragment<InputScalar_, OutputFragment_::kElements>, OutputFragment_> Transformer; +}; + +template <int kElements_> +struct IgemmGlobalStoreTransformer<float, Fragment<int8_t, kElements_> > { + typedef IgemmFloatToInt8Converter<kElements_> Transformer; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <int kElements_> +struct IgemmInt8ToFloatConverter { + /// The input fragment. + typedef Fragment<int8_t, kElements_> InputFragment; + /// The output fragment. + typedef Fragment<float, kElements_> OutputFragment; + + // We are unpacking 4 int8s from int32. + static_assert(kElements_ % 4 == 0, "kElements must be multiple of 4"); + + /// Ctor. + CUTLASS_DEVICE IgemmInt8ToFloatConverter() {} + + /// Transform a fragment. + CUTLASS_DEVICE void transform(InputFragment const& src, OutputFragment& dst) { + transform(src, 0, dst); + } + + /// Transform a fragment. + template <typename Fragment_> + CUTLASS_DEVICE void transform(Fragment_ const& src, int offset, OutputFragment& dst) { + // The inputs. + int const* src_int = reinterpret_cast<int const*>(&src[0]); + // The outputs. + float4* dst_f4 = reinterpret_cast<float4*>(&dst[0]); + + // Iterate over the int8 and unpack them together to produce floats. + for (int i = 0; i < kElements_ / 4; ++i) { + // Read the int. + int ix, iy, iz, iw = src_int[i]; + + // Extract the 4 bytes. + asm volatile("prmt.b32 %0, 0x0, %1, 0x4440;" : "=r"(ix) : "r"(iw)); + asm volatile("prmt.b32 %0, 0x0, %1, 0x4441;" : "=r"(iy) : "r"(iw)); + asm volatile("prmt.b32 %0, 0x0, %1, 0x4442;" : "=r"(iz) : "r"(iw)); + asm volatile("prmt.b32 %0, 0x0, %1, 0x4443;" : "=r"(iw) : "r"(iw)); + + // The floats. + float fx, fy, fz, fw; + + // Convert to floats (make sure we generate I2F.F32.S8). + asm volatile("cvt.rn.f32.s8 %0, %1;" : "=f"(fx) : "r"(ix)); + asm volatile("cvt.rn.f32.s8 %0, %1;" : "=f"(fy) : "r"(iy)); + asm volatile("cvt.rn.f32.s8 %0, %1;" : "=f"(fz) : "r"(iz)); + asm volatile("cvt.rn.f32.s8 %0, %1;" : "=f"(fw) : "r"(iw)); + + // Store the float4. + dst_f4[i] = make_float4(fx, fy, fz, fw); + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename InputFragment_, typename OutputScalar_> +struct IgemmGlobalLoadTransformer { + typedef Convert<InputFragment_, Fragment<OutputScalar_, InputFragment_::kElements> > Transformer; +}; + +template <int kElements_> +struct IgemmGlobalLoadTransformer<Fragment<int8_t, kElements_>, float> { + typedef IgemmInt8ToFloatConverter<kElements_> Transformer; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename InputScalar_, typename OutputFragment_> +struct IgemmSharedStoreTransformer { + typedef Convert<Fragment<InputScalar_, OutputFragment_::kElements>, OutputFragment_> Transformer; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename IgemmConfig_, typename EpilogueFunctor_, typename Index_> +struct IgemmEpilogueTraitsHelper + : public GemmEpilogueTraitsHelper<IgemmConfig_, EpilogueFunctor_, Index_> { + /// The base class. + typedef GemmEpilogueTraitsHelper<IgemmConfig_, EpilogueFunctor_, Index_> Base; + /// The config. + typedef IgemmConfig_ IgemmConfig; + + /// The scalar type of the epilogue. + typedef typename Base::Scalar Scalar; + /// The iterations. + typedef typename Base::Iterations Iterations; + /// The iterations strides. + typedef typename Base::Delta Delta; + + /// The traits class for the iterator. + typedef typename Base::GlobalLoadTileTraits GlobalLoadTileTraits; + /// The iterator to store to shared memory. + typedef GemmGlobalIteratorCd<GlobalLoadTileTraits> GlobalLoadIteratorC; + /// The fragment that needs to be produced by the load iterator. + typedef typename GlobalLoadIteratorC::Fragment GlobalFragmentC; + /// The transformer from loaded data to math fragment. + typedef + typename IgemmGlobalLoadTransformer<GlobalFragmentC, Scalar>::Transformer GlobalTransformerC; + + /// The traits class for the iterator. + typedef typename Base::GlobalStoreTileTraits GlobalStoreTileTraits; + /// The iterator to store to shared memory. + typedef GemmGlobalIteratorCd<GlobalStoreTileTraits> GlobalStoreIteratorD; + /// The fragment that needs to be passed to that store iterator. + typedef typename GlobalStoreIteratorD::Fragment GlobalFragmentD; + /// The transformer from accumulators to shared memory fragments. + typedef + typename IgemmGlobalStoreTransformer<Scalar, GlobalFragmentD>::Transformer GlobalTransformerD; + + /// The traits class for the shared iterator to store D to shared memory. + typedef typename Base::SharedStoreTileTraits SharedStoreTileTraits; + /// The shared iterator to store D to shared memory. + typedef TileStoreIterator<SharedStoreTileTraits, + typename SharedStoreTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kGlobal> + SharedStoreIteratorD; + /// The fragment that needs to be passed to that store iterator. + typedef typename SharedStoreIteratorD::Fragment SharedStoreFragmentD; + /// The transformer from accumulators to shared memory fragments. + typedef typename IgemmSharedStoreTransformer<typename IgemmConfig::Accumulators::Element, + SharedStoreFragmentD>::Transformer + SharedStoreTransformerD; + /// The traits class for the shared iterator to load D from shared memory. + typedef typename Base::SharedLoadTileTraits SharedLoadTileTraits; + /// The shared iterator to load D from shared memory. + typedef TileLoadIterator<SharedLoadTileTraits, + typename SharedLoadTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedLoadIteratorD; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The config. + typename IgemmConfig_, + /// The functor to do the math in the epilogue. + typename EpilogueFunctor_, + /// The index. + typename Index_ = int, + /// The helper class to assemble the traits. + typename Helper_ = IgemmEpilogueTraitsHelper<IgemmConfig_, EpilogueFunctor_, Index_> > +struct IgemmEpilogueTraits : public GemmEpilogueTraits< + // The output tile. + typename IgemmConfig_::OutputTile, + // The accumulators. + typename IgemmConfig_::Accumulators, + // The global iterator for C. + typename Helper_::GlobalLoadIteratorC, + // The transformer for C. + typename Helper_::GlobalTransformerC, + // The transformer for D. + typename Helper_::GlobalTransformerD, + // The global iterator for D. + typename Helper_::GlobalStoreIteratorD, + // The iterator to store D to shared memory. + typename Helper_::SharedStoreIteratorD, + // The shared store transformer for D. + typename Helper_::SharedStoreTransformerD, + // The iterator to load D from shared memory. + typename Helper_::SharedLoadIteratorD, + // The iterations. + typename Helper_::Iterations, + // The strides between iterations. + typename Helper_::Delta, + // The functor to be used in the epilogue. + EpilogueFunctor_, + // The index. + Index_> { + /// Do we output in int8? + static bool const kInt8Output = + platform::is_same<typename IgemmConfig_::ScalarC, int8_t>::value != 0; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmEpilogueTraits_, bool = GemmEpilogueTraits_::kInt8Output> +struct IgemmEpilogue : public GemmEpilogue<GemmEpilogueTraits_> { + /// The base class. + typedef GemmEpilogue<GemmEpilogueTraits_> Base; + + /// Ctor. + CUTLASS_DEVICE IgemmEpilogue(typename Base::Params const& params_, + typename Base::SharedStorage& shared_storage_, + typename Base::Index m_, + typename Base::Index n_) + : Base(params_, shared_storage_, m_, n_) {} +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmEpilogueTraits_> +struct IgemmEpilogue<GemmEpilogueTraits_, true> : public GemmEpilogue<GemmEpilogueTraits_> { + /// The base class. + typedef GemmEpilogue<GemmEpilogueTraits_> Base; + + /// Ctor. + CUTLASS_DEVICE IgemmEpilogue(typename Base::Params const& params_, + typename Base::SharedStorage& shared_storage_, + typename Base::Index m_, + typename Base::Index n_) + : Base(params_, shared_storage_, m_, n_) {} +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/igemm_global_tile.h b/cutlass-example/cutlass/gemm/igemm_global_tile.h new file mode 100644 index 0000000..3f594ac --- /dev/null +++ b/cutlass-example/cutlass/gemm/igemm_global_tile.h @@ -0,0 +1,161 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Implements tile iterators to partition the thread block tile into 2D subtiles and + efficiently load each. Applies permute transformation to construct 'interleaved K-strided' + data layout in which 4-element dot products from the same K index are arranged in consecutive + locations within shared memory. + + Supports efficient loads from shared memory to target the DP4A instruction. +*/ +#pragma once + +#include <cutlass/coord.h> +#include <cutlass/gemm/gemm_global_tile.h> +#include <cutlass/matrix_traits.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <GemmOperand::Kind kOperand_, + MatrixLayout::Kind kLayout_, + typename Scalar_, + typename Tile_, + typename Threads_, + int kAccessSize_> +struct IgemmGlobalTileTraits : public GemmGlobalTileTraits< + // Which GEMM operand? + kOperand_, + // The layout. + kLayout_, + // The scalar. + Scalar_, + // The tile. + Tile_, + // The threads. + Threads_, + // The number of scalars per LDG/STG. + kAccessSize_> { + /// The base class. + typedef GemmGlobalTileTraits<kOperand_, kLayout_, Scalar_, Tile_, Threads_, kAccessSize_> Base; + /// The threads. + typedef typename Base::Threads Threads; + /// The strides in each dimension between different loads/stores. + typedef Shape<Base::Threads::kH * 4, 1, Base::Threads::kW, Base::kAccessSize> Delta; + /// The number of iterations needed to load/store the tile. + typedef Shape<Base::Tile::kH / Base::Threads::kH / 4, + 4, + Base::Tile::kW / Base::Threads::kW, + Base::Tile::kC / Base::kAccessSize> + Iterations; + + /// Computes the thread offset in (H, W) based on thread ID + struct ThreadOffset { + CUTLASS_HOST_DEVICE + Coord<4> operator()() const { + int thread_offset_h = threadIdx.x / Threads::kW * ThreadsDelta::kH; + int thread_offset_w = threadIdx.x % Threads::kW * ThreadsDelta::kW; + + return make_Coord(0, thread_offset_h, thread_offset_w, 0); + } + }; + + public: + /// The threads strides. + typedef Shape<1, 4, Base::Tile::kC> ThreadsDelta; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Deprecated. Please use IgemmGlobalTileTraits instead. + +template <GemmOperand::Kind kOperand_, + MatrixLayout::Kind kLayout_, + typename Scalar_, + typename Tile_, + typename Threads_, + int kAccessSize_> +struct IgemmContiguousGlobalTileTraits + : public IgemmGlobalTileTraits<kOperand_, kLayout_, Scalar_, Tile_, Threads_, kAccessSize_> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename TileTraits_, typename Index_ = int> +struct IgemmGlobalIteratorAb : public GemmGlobalIteratorAb<TileTraits_, Index_> { + /// The base class. + typedef GemmGlobalIteratorAb<TileTraits_, Index_> Base; + /// The functor to compute the thread offset. + typedef typename TileTraits_::ThreadOffset ThreadOffset; + + /// Constructor. + CUTLASS_DEVICE IgemmGlobalIteratorAb(typename Base::Params const& _params, + const Coord<3>& bounds, + const Coord<3>& block, + ThreadOffset thread_offset_func = ThreadOffset()) + : Base(_params, bounds, block, thread_offset_func), in_residue_(false), mask_(0xffffffff) { + // The number of elements read in a single iteration. + int const kBlock = TileTraits_::Tile::kW * TileTraits_::kAccessSize; + // The residue. + int const kResidue = (int)(bounds[1] % kBlock); + + // Compute the number of elements that are valid. + int const left = kResidue - Base::thread_offset[2]; + if (left > 0 && left < 4) { + mask_ = (1u << (8 * left)) - 1u; + } + } + + /// The accessor. + CUTLASS_DEVICE void get(typename Base::AccessType& value, int d, int h, int w, int c) const { + Base::get(value, d, h, w, c); + if (in_residue_) { + reinterpret_cast<uint32_t&>(value) &= mask_; + } + } + + /// Move to residue portion. + CUTLASS_DEVICE void move_to_residue(typename Base::Index k) { + Base::move_to_residue(k); + in_residue_ = true; + } + + /// Move back to the beginning of the first tile. + CUTLASS_DEVICE void rollback() { + Base::rollback(); + in_residue_ = false; + } + + /// Are we in the residue? + bool in_residue_; + /// The mask to clean up the values. + uint32_t mask_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/igemm_multiply_add.h b/cutlass-example/cutlass/gemm/igemm_multiply_add.h new file mode 100644 index 0000000..5a8baec --- /dev/null +++ b/cutlass-example/cutlass/gemm/igemm_multiply_add.h @@ -0,0 +1,89 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Implements matrix multiply accumulate operation of 8-bit integer data using DP4A + instruction. +*/ +#pragma once + +#include <cutlass/fragment.h> + +#include <cutlass/gemm/thread_multiply_add.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Template performing matrix multiply-add operation within a thread +template <typename AccumulatorsPerThread_, typename ThreadsPerWarp_> +struct ThreadMultiplyAdd<AccumulatorsPerThread_, ThreadsPerWarp_, int8_t, int8_t, int> { + /// The shape of the instruction. + typedef Shape<4, 1, 1> InstructionShape; + /// The number of accumulators per thread. + typedef AccumulatorsPerThread_ AccumulatorsPerThread; + /// The number of threads per warp. + typedef ThreadsPerWarp_ ThreadsPerWarp; + /// The number of accumulators per warp. + typedef typename ShapeMul<AccumulatorsPerThread, ThreadsPerWarp>::Shape AccumulatorsPerWarp; + /// The type for A. + typedef int8_t ScalarA; + /// The fragment for A. + typedef Fragment<ScalarA, AccumulatorsPerThread::kW * 4> FragmentA; + /// The type for B. + typedef int8_t ScalarB; + /// The fragment for B. + typedef Fragment<ScalarB, AccumulatorsPerThread::kH * 4> FragmentB; + /// The type for C and D. + typedef int ScalarC; + /// The accumulators. + typedef Fragment<ScalarC, AccumulatorsPerThread::kH * AccumulatorsPerThread::kW> Accumulators; + + /// Ctor. + CUTLASS_DEVICE ThreadMultiplyAdd() {} + + /// Multiply : d = a*b + c. + CUTLASS_DEVICE void multiply_add(FragmentA const& a, + FragmentB const& b, + Accumulators const& c, + Accumulators& d) { + // The inputs. + int const* a_int = reinterpret_cast<int const*>(&a[0]); + int const* b_int = reinterpret_cast<int const*>(&b[0]); + + for (int j = 0; j < AccumulatorsPerThread::kH; ++j) { + for (int i = 0; i < AccumulatorsPerThread::kW; ++i) { + asm volatile("dp4a.s32.s32 %0, %1, %2, %3;" + : "=r"(d[j * AccumulatorsPerThread::kW + i]) + : "r"(a_int[i]), "r"(b_int[j]), "r"(c[j * AccumulatorsPerThread::kW + i])); + } + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/igemm_swizzle.h b/cutlass-example/cutlass/gemm/igemm_swizzle.h new file mode 100644 index 0000000..77cf711 --- /dev/null +++ b/cutlass-example/cutlass/gemm/igemm_swizzle.h @@ -0,0 +1,115 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Transposes a fragment of data containing packed 8-bit integer elements. +*/ +#pragma once + +#include <cutlass/fragment.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GlobalIterator_> +struct IgemmSwizzle { + /// The global iterator. + typedef GlobalIterator_ GlobalIterator; + /// The source fragment. + typedef typename GlobalIterator::Fragment Fragment; + /// The shape of the source fragment. + typedef typename GlobalIterator::FragmentShape FragmentShape; + + /// The source fragment. + typedef Fragment InputFragment; + /// The destination fragment. + typedef Fragment OutputFragment; + + /// The src/dst must be int8 fragments. + static_assert((platform::is_same<typename Fragment::Element, int8_t>::value), "Works on int8"); + + /// The number of elements must be a multiple of 4. + static_assert(FragmentShape::kH % 4 == 0 && ShapeCount<FragmentShape>::kWc % 4 == 0, + "Not multiple of 4"); + + /// Ctor. + CUTLASS_DEVICE IgemmSwizzle() {} + + /// Transform a fragment. + CUTLASS_DEVICE void transform(Fragment const& src, Fragment& dst) { + // Expose src/dst as int arrays. + int const* src_int = reinterpret_cast<int const*>(&src[0]); + int* dst_int = reinterpret_cast<int*>(&dst[0]); + + // Transpose the data. + for (int d = 0; d < FragmentShape::kD; ++d) { + for (int h = 0; h < FragmentShape::kH / 4; ++h) { + for (int w = 0; w < ShapeCount<FragmentShape>::kWc / 4; ++w) { + int const i0 = d * (ShapeCount<FragmentShape>::kHwc / 4) + + (4 * h + 0) * (ShapeCount<FragmentShape>::kWc / 4) + w; + int const i1 = d * (ShapeCount<FragmentShape>::kHwc / 4) + + (4 * h + 1) * (ShapeCount<FragmentShape>::kWc / 4) + w; + int const i2 = d * (ShapeCount<FragmentShape>::kHwc / 4) + + (4 * h + 2) * (ShapeCount<FragmentShape>::kWc / 4) + w; + int const i3 = d * (ShapeCount<FragmentShape>::kHwc / 4) + + (4 * h + 3) * (ShapeCount<FragmentShape>::kWc / 4) + w; + + int a0 = src_int[i0]; + int a1 = src_int[i1]; + int a2 = src_int[i2]; + int a3 = src_int[i3]; + + int b0, b1, b2, b3, c0; + asm volatile("prmt.b32 %0, %1, %2, 0x0040;" : "=r"(b0) : "r"(a0), "r"(a1)); + asm volatile("prmt.b32 %0, %1, %2, 0x0040;" : "=r"(c0) : "r"(a2), "r"(a3)); + asm volatile("prmt.b32 %0, %1, %2, 0x5410;" : "=r"(b0) : "r"(b0), "r"(c0)); + + asm volatile("prmt.b32 %0, %1, %2, 0x0051;" : "=r"(b1) : "r"(a0), "r"(a1)); + asm volatile("prmt.b32 %0, %1, %2, 0x0051;" : "=r"(c0) : "r"(a2), "r"(a3)); + asm volatile("prmt.b32 %0, %1, %2, 0x5410;" : "=r"(b1) : "r"(b1), "r"(c0)); + + asm volatile("prmt.b32 %0, %1, %2, 0x0062;" : "=r"(b2) : "r"(a0), "r"(a1)); + asm volatile("prmt.b32 %0, %1, %2, 0x0062;" : "=r"(c0) : "r"(a2), "r"(a3)); + asm volatile("prmt.b32 %0, %1, %2, 0x5410;" : "=r"(b2) : "r"(b2), "r"(c0)); + + asm volatile("prmt.b32 %0, %1, %2, 0x0073;" : "=r"(b3) : "r"(a0), "r"(a1)); + asm volatile("prmt.b32 %0, %1, %2, 0x0073;" : "=r"(c0) : "r"(a2), "r"(a3)); + asm volatile("prmt.b32 %0, %1, %2, 0x5410;" : "=r"(b3) : "r"(b3), "r"(c0)); + + dst_int[i0] = b0; + dst_int[i1] = b1; + dst_int[i2] = b2; + dst_int[i3] = b3; + } + } + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/igemm_traits.h b/cutlass-example/cutlass/gemm/igemm_traits.h new file mode 100644 index 0000000..82f8de5 --- /dev/null +++ b/cutlass-example/cutlass/gemm/igemm_traits.h @@ -0,0 +1,539 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defies structural properties of mixed-precision integer GEMM. Multiplicands are assumed + to be packed 8bit integers, accumulators are assumed to be 32b signed integers, and output + formats vary. +*/ +#pragma once + +#include <cutlass/convert.h> +#include <cutlass/gemm/gemm.h> +#include <cutlass/gemm/gemm_epilogue.h> +#include <cutlass/gemm/gemm_epilogue_traits.h> +#include <cutlass/gemm/gemm_global_tile.h> +#include <cutlass/gemm/gemm_shared_tile.h> +#include <cutlass/gemm/gemm_traits.h> +#include <cutlass/gemm/igemm_epilogue.h> +#include <cutlass/gemm/igemm_global_tile.h> +#include <cutlass/gemm/igemm_multiply_add.h> +#include <cutlass/gemm/igemm_swizzle.h> +#include <cutlass/reshape_tile.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The tile size for the GEMM KxNxM. + typename OutputTile_, + /// The output type. + typename ScalarD_, + /// The number of accumulators per thread. + typename AccumulatorsPerThread_> +struct IgemmConfig + : public GemmConfig< + /// The scalar type for A. + int8_t, + /// The scalar type for B. + int8_t, + /// The scalar type for C. + ScalarD_, + /// The scalar type for D. + ScalarD_, + /// The tile size for the GEMM KxNxM. + OutputTile_, + /// The functor to do the math in the main loop. + ThreadMultiplyAdd<AccumulatorsPerThread_, Shape<1, 4, 8>, int8_t, int8_t, int>, + /// The number of scalars per LDG for A. + 4, + /// The number of scalars per STS for A. + 4, + /// The number of scalars per LDS for A. + 16, + /// The number of scalars per LDG for B. + 4, + /// The number of scalars per STS for B. + 4, + /// The number of scalars per LDS for B. + 16, + /// The number of scalars per LDG for C and STG for D. + 1, + /// The number of scalars per STS for D. + 4, + /// The number of scalars per LDS for D. + 1, + /// The number of stages in shared memory. + 2, + /// Enable the code path that deals with the residue in epilogue. + true> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename OutputTile_, typename AccumulatorsPerThread_> +struct IgemmConfig<OutputTile_, int8_t, AccumulatorsPerThread_> + : public GemmConfig< + /// The scalar type for A. + int8_t, + /// The scalar type for B. + int8_t, + /// The scalar type for C. + int8_t, + /// The scalar type for D. + int8_t, + /// The tile size for the GEMM KxNxM. + OutputTile_, + /// The functor to do the math in the main loop. + ThreadMultiplyAdd<AccumulatorsPerThread_, Shape<1, 4, 8>, int8_t, int8_t, int>, + /// The number of scalars per LDG for A. + 4, + /// The number of scalars per STS for A. + 4, + /// The number of scalars per LDS for A. + 16, + /// The number of scalars per LDG for B. + 4, + /// The number of scalars per STS for B. + 4, + /// The number of scalars per LDS for B. + 16, + /// The number of scalars per LDG for C and STG for D. + 4, + /// The number of scalars per STS for D. + 4, + /// The number of scalars per LDS for D. + 4, + /// The number of stages in shared memory. + 2, + /// Enable the code path that deals with the residue in epilogue. + true> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <enum MatrixLayout::Kind kLayout_, typename GemmConfig_, typename Index_> +struct IgemmTileTraitsHelperA : public GemmTileTraitsHelperA<kLayout_, GemmConfig_> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_, typename Index_> +struct IgemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_, Index_> + : public GemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_> { + /// The base config. + typedef GemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_> Base; + + /// The number of scalars per LDG/STS/LDS for A. + static int const kScalarsPerStsA = 16; + + /// The traits class to build the iterator to load data from global memory for A^N. + typedef IgemmGlobalTileTraits< + GemmOperand::kA, + // The layout. + MatrixLayout::kColumnMajor, + // The pointer is float const. + int8_t const, + // The tile has size KxM in GEMM's terminology. + Shape<1, GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kW>, + // The threads are distributed as warps x 32 (the traits may reorganize). + Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerLdgA> + GlobalTileTraits; + + // The iterator. + typedef GemmGlobalIteratorAb<GlobalTileTraits, Index_> GlobalLoadIterator; + + /// The traits class to build the iterator to store data to shared memory for A^N. + typedef GemmSharedStoreTileAbTraits< + // The pointer is float. + int8_t, + // The tile has size KxM in GEMM's terminology. + Shape<GemmConfig_::kStages, GemmConfig_::OutputTile::kD / 4, GemmConfig_::OutputTile::kW * 4>, + // The threads are distributed as warps x 32 (the traits may reorganize). + typename GlobalTileTraits::Threads, + // The number of scalars per STS (STS.32 or STS.128, etc). + kScalarsPerStsA> + SharedStoreTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_, typename Index_> +struct IgemmTileTraitsHelperA<MatrixLayout::kRowMajor, GemmConfig_, Index_> { + /// The layout. + static MatrixLayout::Kind const kLayout = MatrixLayout::kRowMajor; + + /// The input scalar. + typedef int8_t Scalar; + /// The scalar stored in shared memory. + typedef int8_t MultiplyAddScalar; + + /// The number of scalars per LDG/STS/LDS for A. + static int const kScalarsPerStsA = 16; + + /// The traits class to build the iterator to load data from global memory for A^T. + typedef IgemmGlobalTileTraits< + GemmOperand::kA, + // The layout. + MatrixLayout::kRowMajor, + // The pointer is float const. + int8_t const, + // The tile has size NxK in GEMM's terminology. + Shape<1, GemmConfig_::OutputTile::kW, GemmConfig_::OutputTile::kD>, + // The threads are distributed as warps x 32 (the traits may reorganize). + Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerLdgA> + GlobalTileTraits; + + // The iterator. + typedef IgemmGlobalIteratorAb<GlobalTileTraits, Index_> GlobalLoadIterator; + + /// The traits class to build the iterator to store data to shared memory for A^N. + typedef GemmSharedStoreWithSkewTileAbTraits< + // The pointer is int8. + int8_t, + // The tile has size KxN in GEMM's terminology. + Shape<GemmConfig_::kStages, GemmConfig_::OutputTile::kD / 4, GemmConfig_::OutputTile::kW * 4>, + // The threads are distributed as (threads / K) x K (the traits may reorganize). + typename GlobalTileTraits::Threads, + // The number of scalars per STS. + kScalarsPerStsA, + // The skew to avoid bank conflicts added in the tile W dimension. + 16> + SharedStoreTileTraits; + + /// The traits class to build the iterator to load from shared memory for A^N. + typedef GemmSharedLoadTileATraits< + // The pointer is float const. + int8_t const, + // The output tile size. + typename GemmConfig_::OutputTile, + // The number of warps. + typename GemmConfig_::Warps, + // The number of threads per warp. + typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, + // The shape of the FMA instruction. + typename GemmConfig_::InstructionShape, + // The number of stages. + GemmConfig_::kStages, + // The number of scalars per LDS. + 16, + // The skew. + SharedStoreTileTraits::kSkew> + SharedLoadTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <enum MatrixLayout::Kind kLayout_, typename GemmConfig_, typename Index_> +struct IgemmTileTraitsHelperB : public GemmTileTraitsHelperB<kLayout_, GemmConfig_> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_, typename Index_> +struct IgemmTileTraitsHelperB<MatrixLayout::kColumnMajor, GemmConfig_, Index_> { + /// The layout. + static MatrixLayout::Kind const kLayout = MatrixLayout::kColumnMajor; + + /// The input scalar. + typedef int8_t Scalar; + /// The scalar stored in shared memory. + typedef int8_t MultiplyAddScalar; + + /// The number of scalars per LDG/STS/LDS for B. + static int const kScalarsPerStsB = 16; + + /// The traits class to build the iterator to load data from global memory for B^T. + typedef IgemmGlobalTileTraits< + GemmOperand::kB, + // The layout. + MatrixLayout::kColumnMajor, + // The pointer is float const. + int8_t const, + // The tile has size NxK in GEMM's terminology. + Shape<1, GemmConfig_::OutputTile::kH, GemmConfig_::OutputTile::kD>, + // The threads are distributed as warps x 32 (the traits may reorganize). + Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerLdgB> + GlobalTileTraits; + + // The iterator. + typedef IgemmGlobalIteratorAb<GlobalTileTraits, Index_> GlobalLoadIterator; + + /// The traits class to build the iterator to store data to shared memory for B^N. + typedef GemmSharedStoreWithSkewTileAbTraits< + // The pointer is int8. + int8_t, + // The tile has size KxN in GEMM's terminology. + Shape<GemmConfig_::kStages, GemmConfig_::OutputTile::kD / 4, GemmConfig_::OutputTile::kH * 4>, + // The threads are distributed as (threads / K) x K (the traits may reorganize). + typename GlobalTileTraits::Threads, + // The number of scalars per STS. + kScalarsPerStsB, + // The skew to avoid bank conflicts added in the tile W dimension. + 16> + SharedStoreTileTraits; + + /// The traits class to build the iterator to load from shared memory for B^N. + typedef GemmSharedLoadTileBTraits< + // The pointer is float const. + int8_t const, + // The output tile size. + typename GemmConfig_::OutputTile, + // The number of warps. + typename GemmConfig_::Warps, + // The number of threads per warp. + typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, + // The shape of the FMA instruction. + typename GemmConfig_::InstructionShape, + // The number of stages. + GemmConfig_::kStages, + // The number of scalars per LDS. + 16, + // The skew. + SharedStoreTileTraits::kSkew> + SharedLoadTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_, typename Index_> +struct IgemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_, Index_> + : public GemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_> { + /// The base config. + typedef GemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_> Base; + + /// The number of scalars per LDG/STS/LDS for B. + static int const kScalarsPerStsB = 16; + + /// The traits class to build the iterator to load data from global memory for B^T. + typedef IgemmGlobalTileTraits< + GemmOperand::kB, + // The layout. + MatrixLayout::kRowMajor, + // The pointer is float const. + int8_t const, + // The tile has size KxM in GEMM's terminology. + Shape<1, GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kH>, + // The threads are distributed as warps x 32 (the traits may reorganize). + Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerLdgB> + GlobalTileTraits; + + // The iterator. + typedef GemmGlobalIteratorAb<GlobalTileTraits, Index_> GlobalLoadIterator; + + /// The traits class to build the iterator to store data to shared memory for B^N. + typedef GemmSharedStoreTileAbTraits< + // The pointer is float. + int8_t, + // The tile has size KxM in GEMM's terminology. + Shape<GemmConfig_::kStages, GemmConfig_::OutputTile::kD / 4, GemmConfig_::OutputTile::kH * 4>, + // The threads are distributed as warps x 32 (the traits may reorganize). + typename GlobalTileTraits::Threads, + // The number of scalars per STS (STS.32 or STS.128, etc). + kScalarsPerStsB> + SharedStoreTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <enum MatrixLayout::Kind kLayout_, typename Iterator_> +struct IgemmTransformerA {}; + +template <typename Iterator_> +struct IgemmTransformerA<MatrixLayout::kRowMajor, Iterator_> { + typedef Copy<typename Iterator_::Fragment> Transformer; +}; + +template <typename Iterator_> +struct IgemmTransformerA<MatrixLayout::kColumnMajor, Iterator_> { + typedef IgemmSwizzle<Iterator_> Transformer; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <enum MatrixLayout::Kind kLayout_, typename Iterator_> +struct IgemmTransformerB {}; + +template <typename Iterator_> +struct IgemmTransformerB<MatrixLayout::kColumnMajor, Iterator_> { + typedef Copy<typename Iterator_::Fragment> Transformer; +}; + +template <typename Iterator_> +struct IgemmTransformerB<MatrixLayout::kRowMajor, Iterator_> { + typedef IgemmSwizzle<Iterator_> Transformer; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The layout for A. + MatrixLayout::Kind kLayoutA_, + /// The layout for B. + MatrixLayout::Kind kLayoutB_, + /// The output tile. + typename OutputTile_, + /// The output type. + typename ScalarD_, + /// The functor to do the math in the epilogue. + typename EpilogueFunctor_, + /// The number of accumulators per thread. + typename AccumulatorsPerThread_ = Shape<32, 8, 8>, + /// The index. + typename Index_ = int> +struct IgemmTraitsHelper { + /// The IGEMM config. + typedef IgemmConfig<OutputTile_, ScalarD_, AccumulatorsPerThread_> GemmConfig; + /// The GEMM config for A. + typedef IgemmTileTraitsHelperA<kLayoutA_, GemmConfig, Index_> GemmTileTraitsHelperA; + /// The GEMM config for B. + typedef IgemmTileTraitsHelperB<kLayoutB_, GemmConfig, Index_> GemmTileTraitsHelperB; + + /// The iterator to load A from global memory. + typedef typename GemmTileTraitsHelperA::GlobalLoadIterator GlobalLoadIteratorA; + + /// The default transformer for A. + typedef typename IgemmTransformerA<GemmTileTraitsHelperA::kLayout, + GlobalLoadIteratorA>::Transformer GlobalTransformerA; + /// The iterator to store A to shared memory. + typedef TileStoreIterator<typename GemmTileTraitsHelperA::SharedStoreTileTraits, + typename GemmTileTraitsHelperA::SharedStoreTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedStoreIteratorA; + /// The stream to load A from global memory to shared memory. + typedef GlobalLoadStream<GlobalLoadIteratorA, SharedStoreIteratorA, GlobalTransformerA> + GlobalLoadStreamA; + + /// The iterator to load B from global memory. + typedef typename GemmTileTraitsHelperB::GlobalLoadIterator GlobalLoadIteratorB; + + // The default transformer for B. + typedef typename IgemmTransformerB<GemmTileTraitsHelperB::kLayout, + GlobalLoadIteratorB>::Transformer GlobalTransformerB; + /// The iterator to store B to shared memory. + typedef TileStoreIterator<typename GemmTileTraitsHelperB::SharedStoreTileTraits, + typename GemmTileTraitsHelperB::SharedStoreTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedStoreIteratorB; + /// The stream to load B from global memory to shared memory. + typedef GlobalLoadStream<GlobalLoadIteratorB, SharedStoreIteratorB, GlobalTransformerB> + GlobalLoadStreamB; + + /// The iterator to load A from shared memory. + typedef TileLoadIterator<typename GemmTileTraitsHelperA::SharedLoadTileTraits, + typename GemmTileTraitsHelperA::SharedLoadTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedLoadIteratorA; + /// The stream to load A from shared memory. + typedef SharedLoadStream<SharedLoadIteratorA, Copy<typename SharedLoadIteratorA::Fragment> > + SharedLoadStreamA; + /// The iterator to load B from shared memory. + typedef TileLoadIterator<typename GemmTileTraitsHelperB::SharedLoadTileTraits, + typename GemmTileTraitsHelperB::SharedLoadTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedLoadIteratorB; + /// The stream to load B from shared memory. + typedef SharedLoadStream<SharedLoadIteratorB, Copy<typename SharedLoadIteratorB::Fragment> > + SharedLoadStreamB; + + /// The multiply-add functor. + typedef typename GemmConfig::MultiplyAdd MultiplyAdd; + /// The object to clear accumulators. + typedef ClearAccumulators<typename MultiplyAdd::ScalarC> ClearAccumulators; + + /// The epilogue. + typedef IgemmEpilogue<IgemmEpilogueTraits<GemmConfig, EpilogueFunctor_> > Epilogue; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename ScalarD_> +struct IgemmEpilogueScalar { + typedef float Scalar; +}; + +template <> +struct IgemmEpilogueScalar<int> { + typedef int Scalar; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The layout for A. + MatrixLayout::Kind kLayoutA_, + /// The layout for B. + MatrixLayout::Kind kLayoutB_, + /// The output tile. + typename OutputTile_ = Shape<32, 128, 128>, + /// The output type. + typename ScalarD_ = int, + /// The functor to do the math in the epilogue. + typename EpilogueFunctor_ = LinearScaling<typename IgemmEpilogueScalar<ScalarD_>::Scalar>, + /// The number of accumulators per thread. + typename AccumulatorsPerThread_ = Shape<32, 8, 8>, + /// The index. + typename Index_ = int, + /// The helper class. + typename Helper_ = IgemmTraitsHelper<kLayoutA_, + kLayoutB_, + OutputTile_, + ScalarD_, + EpilogueFunctor_, + AccumulatorsPerThread_, + Index_> > +struct IgemmTraits : public GemmTraits< + // The config. + typename Helper_::GemmConfig, + // The stream to load A from global memory to shared memory. + typename Helper_::GlobalLoadStreamA, + // The stream to load B from global memory to shared memory. + typename Helper_::GlobalLoadStreamB, + // The stream to load A from shared memory. + typename Helper_::SharedLoadStreamA, + // The stream to load B from shared memory. + typename Helper_::SharedLoadStreamB, + // The epilogue. + typename Helper_::Epilogue, + // The block swizzle to reorganize the grid. + IdentityBlockSwizzle, + // The index. + Index_, + // The tool used to clear accumulators. + typename Helper_::ClearAccumulators> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/linear_scaling.h b/cutlass-example/cutlass/gemm/linear_scaling.h new file mode 100644 index 0000000..979c93f --- /dev/null +++ b/cutlass-example/cutlass/gemm/linear_scaling.h @@ -0,0 +1,85 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Implements the BLAS linear scaling function alpha*AB + beta*C +*/ +#pragma once + +#include <cutlass/fragment_multiply_add.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Functor to compute linear combination of fragments +template <typename Scalar_, typename FragmentMultiplyAdd_ = FragmentMultiplyAdd<Scalar_> > +struct LinearScaling { + // The scalar. + typedef Scalar_ Scalar; + // The adapater. + typedef FragmentMultiplyAdd_ FragmentMultiplyAdd; + + /// The parameters. + struct Params { + /// The alpha/beta scaling params. + Scalar alpha, beta; + + /// Initialize the parameters. + template <typename GemmDesc_> + CUTLASS_HOST_DEVICE int initialize(GemmDesc_ const& desc) { + alpha = desc.alpha; + beta = desc.beta; + return 0; + } + }; + + /// Ctor. + CUTLASS_DEVICE LinearScaling(Params const& params) : alpha(params.alpha), beta(params.beta) {} + + /// Evaluate the functor. + template <typename FragmentA_, typename FragmentB_> + CUTLASS_DEVICE void evaluate(FragmentA_ const& accum, FragmentB_& output) { + FragmentMultiplyAdd mad; + mad.multiply(alpha, accum, output); + } + + /// Evaluate the functor. + template <typename FragmentA_, typename FragmentB_> + CUTLASS_DEVICE void evaluate(FragmentA_ const& accum, FragmentB_ const& old, FragmentB_& output) { + FragmentMultiplyAdd mad; + FragmentB_ tmp; + mad.multiply(beta, old, tmp); + mad.multiply_add(alpha, accum, tmp, output); + } + + /// The alpha/beta scaling factors. + Scalar alpha, beta; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/sgemm_traits.h b/cutlass-example/cutlass/gemm/sgemm_traits.h new file mode 100644 index 0000000..66b7677 --- /dev/null +++ b/cutlass-example/cutlass/gemm/sgemm_traits.h @@ -0,0 +1,127 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defies structural properties of single-precision GEMM. +*/ +#pragma once + +#include <cutlass/gemm/gemm.h> +#include <cutlass/gemm/gemm_epilogue.h> +#include <cutlass/gemm/gemm_epilogue_traits.h> +#include <cutlass/gemm/gemm_global_tile.h> +#include <cutlass/gemm/gemm_shared_tile.h> +#include <cutlass/gemm/gemm_traits.h> +#include <cutlass/gemm/thread_multiply_add.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The tile size for the GEMM KxNxM. + typename OutputTile_, + /// The number of accumulators per thread. + typename AccumulatorsPerThread_, + /// The number of scalars per LDG for A. + int kScalarsPerLdgA_ = 1, + /// The number of scalars per LDG for B. + int kScalarsPerLdgB_ = 1> +struct SgemmConfig + : public GemmConfig< + /// The scalar type for A. + float, + /// The scalar type for B. + float, + /// The scalar type for C. + float, + /// The scalar type for D. + float, + /// The tile size for the GEMM KxNxM. + OutputTile_, + /// The functor to do the math in the main loop. + ThreadMultiplyAdd<AccumulatorsPerThread_, Shape<1, 4, 8>, float, float, float>, + /// The number of scalars per LDG for A. + kScalarsPerLdgA_, + /// The number of scalars per STS for A. + kScalarsPerLdgA_, + /// The number of scalars per LDS for A. + 4, + /// The number of scalars per LDG for B. + kScalarsPerLdgB_, + /// The number of scalars per STS for B. + kScalarsPerLdgB_, + /// The number of scalars per LDS for B. + 4, + /// The number of scalars per LDG for C and STG for D. + 1, + /// The number of scalars per STS for D. + 4, + /// The number of scalars per LDS for D. + 1, + /// The number of stages in shared memory. + 2> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The layout for A. + MatrixLayout::Kind kLayoutA_, + /// The layout for B. + MatrixLayout::Kind kLayoutB_, + /// The output tile. + typename OutputTile_ = Shape<8, 128, 128>, + /// The functor to use in the epilogue. + typename EpilogueFunctor_ = LinearScaling<float>, + /// The number of accumulators per thread. + typename AccumulatorsPerThread_ = Shape<8, 8, 8>, + /// The number of floats loaded in one LDG for A. + int kScalarsPerLdgA_ = 1, + /// The number of floats loaded in one LDG for B. + int kScalarsPerLdgB_ = 1, + /// The index. + typename Index_ = int, + /// The SGEMM config. + typename GemmConfig_ = + SgemmConfig<OutputTile_, AccumulatorsPerThread_, kScalarsPerLdgA_, kScalarsPerLdgB_>, + /// The traits class for the epilogue. + typename GemmEpilogueTraits_ = + SimplifiedGemmEpilogueTraits<GemmConfig_, EpilogueFunctor_, Index_> > +struct SgemmTraits : public SimplifiedGemmTraits< + // The layout for A. + kLayoutA_, + // The layout for B. + kLayoutB_, + // The config. + GemmConfig_, + // The epilogue. + GemmEpilogue<GemmEpilogueTraits_>, + // The index. + Index_> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/thread_multiply_add.h b/cutlass-example/cutlass/gemm/thread_multiply_add.h new file mode 100644 index 0000000..20dca15 --- /dev/null +++ b/cutlass-example/cutlass/gemm/thread_multiply_add.h @@ -0,0 +1,84 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Template implementing matrix multiply-add operations on fragments. +*/ +#pragma once + +#include <cutlass/fragment.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Template performing matrix multiply-add operation within a thread +template <typename AccumulatorsPerThread_, + typename ThreadsPerWarp_, + typename ScalarA_, + typename ScalarB_, + typename ScalarC_> +struct ThreadMultiplyAdd { + /// The shape of the instruction. + typedef Shape<1, 1, 1, 1> InstructionShape; + /// The number of accumulators per thread. + typedef AccumulatorsPerThread_ AccumulatorsPerThread; + /// The number of threads per warp. + typedef ThreadsPerWarp_ ThreadsPerWarp; + /// The number of accumulators per warp. + typedef typename ShapeMul<AccumulatorsPerThread, ThreadsPerWarp>::Shape AccumulatorsPerWarp; + /// The type for A. + typedef ScalarA_ ScalarA; + /// The fragment for A. + typedef Fragment<ScalarA, AccumulatorsPerThread::kW> FragmentA; + /// The type for B. + typedef ScalarB_ ScalarB; + /// The fragment for B. + typedef Fragment<ScalarB, AccumulatorsPerThread::kH> FragmentB; + /// The type for C and D. + typedef ScalarC_ ScalarC; + /// The accumulators. + typedef Fragment<ScalarC, AccumulatorsPerThread::kH * AccumulatorsPerThread::kW, 16> Accumulators; + + /// Ctor. + CUTLASS_DEVICE ThreadMultiplyAdd() {} + + /// Multiply : d = a*b + c. + CUTLASS_DEVICE void multiply_add(FragmentA const& a, + FragmentB const& b, + Accumulators const& c, + Accumulators& d) { + for (int j = 0; j < AccumulatorsPerThread::kH; ++j) { + for (int i = 0; i < AccumulatorsPerThread::kW; ++i) { + d[j * AccumulatorsPerThread::kW + i] = a[i] * b[j] + c[j * AccumulatorsPerThread::kW + i]; + } + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/wmma_gemm_epilogue_traits.h b/cutlass-example/cutlass/gemm/wmma_gemm_epilogue_traits.h new file mode 100644 index 0000000..0fafacf --- /dev/null +++ b/cutlass-example/cutlass/gemm/wmma_gemm_epilogue_traits.h @@ -0,0 +1,161 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines structural properties of WMMA GEMM's epilogue phase. +*/ +#pragma once + +#include <cutlass/wmma_matrix.h> +#ifdef CUTLASS_USE_WMMA_API + +#include <cutlass/convert.h> +#include <cutlass/coord.h> +#include <cutlass/gemm/gemm_global_stream.h> +#include <cutlass/gemm/gemm_shared_stream.h> +#include <cutlass/gemm/linear_scaling.h> +#include <cutlass/gemm/wmma_gemm_global_tile.h> +#include <cutlass/gemm/wmma_gemm_shared_tile.h> +#include <cutlass/reshape_tile.h> +#include <cutlass/tile_iterator.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_, typename EpilogueFunctor_, typename Index_ = int> +struct WmmaGemmEpilogueTraitsHelper { + /// The scalar. + typedef typename EpilogueFunctor_::Scalar Scalar; + /// The output tile. + typedef typename GemmConfig_::OutputTile OutputTile; + + /// The number of WMMAs in the H dimension. + static int const kWmmasPerH = + GemmConfig_::AccumulatorsPerWarp::kH / GemmConfig_::InstructionShape::kH; + /// The number of iterations in the epilogue. That's the number of "horizontal" WMMAs. + typedef Shape<1, 1, kWmmasPerH> Iterations; + // The iteration strides in the H/W dimension. + typedef Shape<0, 0, 0> Delta; + /// The functor to do the math in the epilogue. + typedef EpilogueFunctor_ Functor; + + /// The traits class to build the iterator to store to shared memory for D. + typedef WmmaGemmSharedStoreTileDTraits< + // The output layout. + MatrixLayout::kColumnMajor, + // The pointer is float. + typename Functor::Scalar, + // The output tile size. + typename GemmConfig_::OutputTile, + // The number of warps. + typename GemmConfig_::Warps, + // The shape of the instruction. + typename GemmConfig_::InstructionShape> + SharedStoreTileTraits; + + typedef WmmaMatrix<GemmOperand::kC, + MatrixLayout::kColumnMajor, + Scalar, + typename GemmConfig_::InstructionShape> + WmmaMatrix; + + /// The iterator to store D to shared memory. + typedef TileStoreIterator<SharedStoreTileTraits, + typename SharedStoreTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared, + Index_, + WmmaMatrix, + IteratorFragment::kWmmaMatrix> + SharedStoreIteratorD; + + /// The shared store transformer for D. + typedef Copy<typename SharedStoreIteratorD::Fragment> SharedStoreTransformerD; + + /// The traits class to build the iterator to load from shared memory for D. + typedef WmmaGemmSharedLoadTileDTraits< + // The pointer. + typename Functor::Scalar, + // The tile size. + typename SharedStoreIteratorD::Tile, + // The number of threads. + Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, + // The number of scalars per LDS. + GemmConfig_::kScalarsPerLdsD> + SharedLoadTileTraits; + + /// The iterator to load D from shared memory. + typedef TileLoadIterator<SharedLoadTileTraits, + typename SharedLoadTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedLoadIteratorD; + + /// The traits class to build the iterator to load data from global memory for C^N. + typedef WmmaGemmGlobalIteratorCdTraits< + // The pointer is float const. + typename GemmConfig_::ScalarC const, + // The tile has size (N / Iterations)xM in GEMM's terminology. + Shape<1, + GemmConfig_::OutputTile::kH / ShapeCount<Iterations>::kCount, + GemmConfig_::OutputTile::kW>, + // The threads are distributed as warps x 32 (the traits may reorganize). + Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerLdgC> + GlobalLoadTileTraits; + + /// The iterator to load C. + typedef WmmaGemmGlobalIteratorCd<GlobalLoadTileTraits, Index_> GlobalLoadIteratorC; + /// The transformer for C. + typedef Copy<typename GlobalLoadIteratorC::Fragment> GlobalTransformerC; + + /// The traits class to build the iterator to store data to global memory for D^N. + typedef WmmaGemmGlobalIteratorCdTraits< + // The pointer is float. + typename GemmConfig_::ScalarD, + // The tile has size (N / Iterations)xM in GEMM's terminology. + Shape<1, + GemmConfig_::OutputTile::kH / ShapeCount<Iterations>::kCount, + GemmConfig_::OutputTile::kW>, + // The threads are distributed as warps x 32 (the traits may reorganize). + Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerStgD> + GlobalStoreTileTraits; + + /// The iterator to store D. + typedef WmmaGemmGlobalIteratorCd<GlobalStoreTileTraits, Index_> GlobalStoreIteratorD; + /// The transformer for D. + typedef Copy<typename GlobalStoreIteratorD::Fragment> GlobalTransformerD; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass + +#endif // defined CUTLASS_USE_WMMA_API diff --git a/cutlass-example/cutlass/gemm/wmma_gemm_global_tile.h b/cutlass-example/cutlass/gemm/wmma_gemm_global_tile.h new file mode 100644 index 0000000..dbd57f6 --- /dev/null +++ b/cutlass-example/cutlass/gemm/wmma_gemm_global_tile.h @@ -0,0 +1,211 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines tile iterator traits for loading thread block-level tile from global memory. +*/ +#pragma once + +#include <cutlass/gemm/gemm_global_tile.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, typename Tile_, typename Threads_, int kAccessSize_> +struct WmmaGemmGlobalIteratorCdTraits : public GemmGlobalTileTraits<GemmOperand::kC, + MatrixLayout::kColumnMajor, + Scalar_, + Tile_, + Threads_, + kAccessSize_> { + /// The base class. + typedef GemmGlobalTileTraits<GemmOperand::kC, + MatrixLayout::kColumnMajor, + Scalar_, + Tile_, + Threads_, + kAccessSize_> + Base; + + /// Override the strides in each dimension between different loads/stores. + typedef Shape<0, 0, Base::Delta::kW, Base::Delta::kC> Delta; + + /// Computes the thread offset in (H, W) based on thread ID + struct ThreadOffset { + CUTLASS_HOST_DEVICE + Coord<4> operator()() const { + int thread_offset_h = threadIdx.x / Base::Threads::kW; + int thread_offset_w = threadIdx.x % Base::Threads::kW * Base::ThreadsDelta::kW; + + return make_Coord(0, thread_offset_h, thread_offset_w, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename TileTraits_, typename Index_ = int> +struct WmmaGemmGlobalIteratorCd : public TileIteratorBase<TileTraits_, + typename TileTraits_::Scalar, + IteratorAdvance::kH, + MemorySpace::kGlobal, + Index_> { + /// This class. + typedef WmmaGemmGlobalIteratorCd<TileTraits_, Index_> This_; + /// The traits. + typedef TileTraits_ Traits; + /// The base class. + typedef TileIteratorBase<Traits, + typename TileTraits_::Scalar, + IteratorAdvance::kH, + MemorySpace::kGlobal, + Index_> + Base; + /// Override the strides in each dimension between different loads/stores. + typedef Shape<0, 0, Base::Delta::kW, Base::Delta::kC> ImmediateOffsetStrides; + /// The layout. + static MatrixLayout::Kind const kLayout = TileTraits_::kLayout; + + /// The scalar. + typedef typename TileTraits_::Scalar Scalar; + /// The pointer. + typedef typename TileTraits_::Pointer Pointer; + /// The threads. + typedef typename TileTraits_::Threads Threads; + /// The index. + typedef Index_ Index; + /// The thread offset functor. + typedef typename TileTraits_::ThreadOffset ThreadOffset; + + /// The params. + struct Params { + /// The pointer. + Pointer pointer; + /// The stride in the H dimension to setup the thread in the block. + Index stride_h; + /// The strides to increment the pointer. + Index inc_h, inc_advance; + /// The column offset to compute the predicate for the columns. + Index predicate_offset; + /// The strides to increment the predicate offset. + Index predicate_inc_h, predicate_inc_advance; + + /// Setup the params. + CUTLASS_HOST_DEVICE int initialize( + Pointer pointer, Index ld, Index n, Index epilogue_stride_w, Index epilogue_delta_w) { + // The pointer. + this->pointer = pointer; + // Setup the base stride. One "group of threads" per column. + stride_h = ld; + // Each thread output 1 column per iteration. . + inc_h = ld * TileTraits_::Threads::kH; + inc_advance = inc_h + epilogue_stride_w; + + predicate_offset = n; + predicate_inc_h = TileTraits_::Threads::kH; + predicate_inc_advance = predicate_inc_h + epilogue_delta_w; + + // It worked. + return 0; + } + }; + + Params params; + + Coord<4> thread_offset; + + /// Ctor. + CUTLASS_DEVICE WmmaGemmGlobalIteratorCd() {} + + /// Ctor. + CUTLASS_DEVICE WmmaGemmGlobalIteratorCd(Params const& params, + const Coord<3>& bounds, + const Coord<3>& block, + int const pointer_offset = 0, + int const pred_offset = 0, + ThreadOffset thread_offset_func = ThreadOffset()) + + : params(params) { + thread_offset = thread_offset_func(); + // Each warp works on a different column of the tile. + int const h = thread_offset[1] + block[1]; + // Each lane writes a different element. + int const w = thread_offset[2] + block[2]; + // Setup the pointer. + this->params.pointer += ((h * params.stride_h + w) + pointer_offset); + + // Prepare the vector of predicates. + for (int i = 0; i < Base::Iterations::kW; ++i) { + predicates.set(i, w + i * Base::Delta::kW < bounds[2]); + } + this->params.predicate_offset -= (h + pred_offset); + } + + /// The accessor. + CUTLASS_DEVICE void get(typename Base::AccessType& value, int d, int h, int w, int c) const { + int const imm = + ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(0, 0, w, c); + Load<Scalar, TileTraits_::kAccessSize, MemorySpace::kGlobal>::load(value, params.pointer, imm); + } + + /// Increment the pointer in the C dimension. + CUTLASS_DEVICE void inc_c() {} + /// Increment the pointer in the W dimension. + CUTLASS_DEVICE void inc_w() {} + /// Increment the pointer in the H dimension. + CUTLASS_DEVICE void inc_h() { + params.pointer += params.inc_h; + params.predicate_offset -= params.predicate_inc_h; + } + /// Increment the pointer in the D dimension. + CUTLASS_DEVICE void inc_d() {} + /// Increment the pointer to move to the next iteration. + CUTLASS_DEVICE void inc_advance() { + params.pointer += params.inc_advance; + params.predicate_offset -= params.predicate_inc_advance; + } + + /// The accessor. + CUTLASS_DEVICE void set(typename Base::AccessType const& value, int d, int h, int w, int c) { + int const imm = + ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(d, h, w, 0); + Store<Scalar, TileTraits_::kAccessSize, MemorySpace::kGlobal>::store( + value, params.pointer, imm); + } + + /// Test the predicate. + CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { + return predicates.at(w) && params.predicate_offset > 0; + } + + /// The predicates for the row. + cutlass::PredicateVector<Base::Iterations::kW> predicates; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/wmma_gemm_multiply_add.h b/cutlass-example/cutlass/gemm/wmma_gemm_multiply_add.h new file mode 100644 index 0000000..5968350 --- /dev/null +++ b/cutlass-example/cutlass/gemm/wmma_gemm_multiply_add.h @@ -0,0 +1,108 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Implements warp-level matrix multiply-accumulate operation using CUDA WMMA API. +*/ +#pragma once + +#include <cutlass/wmma_matrix.h> +#ifdef CUTLASS_USE_WMMA_API +#include <cutlass/fragment.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <MatrixLayout::Kind kLayoutA_, + typename ScalarA_, + MatrixLayout::Kind kLayoutB_, + typename ScalarB_, + MatrixLayout::Kind kLayoutC_, + typename ScalarC_, + typename AccumulatorsPerWarp_, + typename InstructionShape_> +struct WmmaGemmMultiplyAdd { + /// The shape of the instruction. + typedef InstructionShape_ InstructionShape; + /// The number of threads per warp. That's a dummy configuration. + typedef Shape<1, InstructionShape_::kH, InstructionShape_::kW> ThreadsPerWarp; + /// The dimensions. + typedef AccumulatorsPerWarp_ AccumulatorsPerWarp; + /// The type for A. + typedef ScalarA_ ScalarA; + /// The type for B. + typedef ScalarB_ ScalarB; + /// The type for C and D. + typedef ScalarC_ ScalarC; + /// The number of iterations. + typedef typename ShapeDiv<AccumulatorsPerWarp, InstructionShape>::Shape Iterations; + + /// The element for A. + typedef WmmaMatrix<GemmOperand::kA, kLayoutA_, ScalarA, InstructionShape> ElementA; + /// The fragment for A. + typedef Fragment<ElementA, Iterations::kW> FragmentA; + + /// The element for B. + typedef WmmaMatrix<GemmOperand::kB, kLayoutB_, ScalarB, InstructionShape> ElementB; + /// The fragment for B. + typedef Fragment<ElementB, Iterations::kH> FragmentB; + + /// The element for C. + typedef WmmaMatrix<GemmOperand::kC, kLayoutC_, ScalarC, InstructionShape> ElementC; + /// The fragment for C. + typedef Fragment<ElementC, Iterations::kH * Iterations::kW> Accumulators; + + /// Ctor. + CUTLASS_DEVICE WmmaGemmMultiplyAdd() {} + + /// Multiply : d = a*b. + CUTLASS_DEVICE void multiply_add(FragmentA const& a, + FragmentB const& b, + Accumulators const& c, + Accumulators& d) { + for (int j = 0; j < Iterations::kH; ++j) { + for (int i = 0; i < Iterations::kW; ++i) { + // The input elements. + ElementA const& elt_a = a[i]; + ElementB const& elt_b = b[j]; + ElementC const& elt_c = c[j * Iterations::kW + i]; + + // The output element. + ElementC& elt_d = d[j * Iterations::kW + i]; + + // The wmma instruction. + nvcuda::wmma::mma_sync(elt_d, elt_a, elt_b, elt_c); + } + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass + +#endif // defined CUTLASS_USE_WMMA_API diff --git a/cutlass-example/cutlass/gemm/wmma_gemm_shared_tile.h b/cutlass-example/cutlass/gemm/wmma_gemm_shared_tile.h new file mode 100644 index 0000000..7d15b26 --- /dev/null +++ b/cutlass-example/cutlass/gemm/wmma_gemm_shared_tile.h @@ -0,0 +1,240 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines iterator traits for efficiently loading and storing fragment to and from shared + memory, specialized for WMMA GEMM. +*/ +#pragma once + +#include <cutlass/wmma_matrix.h> +#ifdef CUTLASS_USE_WMMA_API + +#include <cutlass/gemm/gemm_operand.h> +#include <cutlass/reshape_tile.h> + +namespace cutlass { +namespace gemm { + +template <class> +struct Debug {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <MatrixLayout::Kind kLayout_, + typename Scalar_, + typename Tile_, + typename Warps_, + int kWarpStride_, + typename Iterations_, + typename Delta_, + typename WmmaShape_> +struct WmmaGemmSharedLoadTileATraits { + /// The operand. + static GemmOperand::Kind const kOperand = GemmOperand::kA; + /// The layout. + static MatrixLayout::Kind const kLayout = kLayout_; + /// The scalar. + typedef Scalar_ Scalar; + /// The pointer. + typedef Scalar const* Pointer; + /// The access size + static int const kAccessSize = 1; + /// The tile with skew. + typedef Tile_ Tile; + /// The number of warps. + typedef Warps_ Warps; + /// The warps strides. + static int const kWarpStride = kWarpStride_; + /// The number of iterations. + typedef Iterations_ Iterations; + /// The strides between iterations. + typedef Delta_ Delta; + /// The strides between iterations. + typedef Delta_ ImmediateOffsetStrides; + /// The shape of the WMMA instruction. + typedef WmmaShape_ WmmaShape; + /// The memory space. + static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; + /// ThreadOffset + struct ThreadOffset { + CUTLASS_HOST_DEVICE + Coord<4> operator()() const { + // The warp id. + int const warp = threadIdx.x / kWarpSize; + // The offset. + int const offset = warp % Warps::kW * kWarpStride; + return make_Coord(0, 0, offset, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <MatrixLayout::Kind kLayout_, + typename Scalar_, + typename Tile_, + typename Warps_, + int kWarpStride_, + typename Iterations_, + typename Delta_, + typename WmmaShape_> +struct WmmaGemmSharedLoadTileBTraits { + /// The operand. + static GemmOperand::Kind const kOperand = GemmOperand::kB; + /// The layout. + static MatrixLayout::Kind const kLayout = kLayout_; + /// The scalar. + typedef Scalar_ Scalar; + /// The pointer. + typedef Scalar const* Pointer; + /// The access size + static int const kAccessSize = 1; + /// The tile with skew. + typedef Tile_ Tile; + /// The number of warps. + typedef Warps_ Warps; + /// The warps strides. + static int const kWarpStride = kWarpStride_; + /// The number of iterations. + typedef Iterations_ Iterations; + /// The strides between iterations. + typedef Delta_ Delta; + /// The strides between iterations. + typedef Delta_ ImmediateOffsetStrides; + /// The shape of the WMMA instruction. + typedef WmmaShape_ WmmaShape; + /// The memory space. + static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; + /// ThreadOffset + struct ThreadOffset { + CUTLASS_HOST_DEVICE + Coord<4> operator()() const { + // The warp id. + int const warp = threadIdx.x / kWarpSize; + // The offset. + int const offset = warp / Warps::kW * kWarpStride; + return make_Coord(0, 0, offset, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <MatrixLayout::Kind kLayout_, + typename Scalar_, + typename OutputTile_, + typename Warps_, + typename WmmaShape_, + int kSkew_ = 0> +struct WmmaGemmSharedStoreTileDTraits { + /// The operand. + static GemmOperand::Kind const kOperand = GemmOperand::kC; + /// The layout. + static MatrixLayout::Kind const kLayout = kLayout_; + /// The scalar. + typedef Scalar_ Scalar; + // The access size + static int const kAccessSize = 1; + /// The pointer. + typedef Scalar* Pointer; + /// The number of warps. + typedef Warps_ Warps; + /// The shape of the WMMA instruction. + typedef WmmaShape_ WmmaShape; + /// The skew. + static int const kSkew = kSkew_; + /// The memory space. + static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; + /// The tile with skew. + typedef Shape<1, Warps_::kH * WmmaShape_::kH, OutputTile_::kW + kSkew_> Tile; + /// The number of iterations needed to store the tile. + typedef Shape<1, 1, OutputTile_::kW / Warps::kW / WmmaShape_::kW> Iterations; + /// The strides in each dimension between different loads/stores. + typedef Shape<0, 0, Warps::kW * WmmaShape_::kW, 0> Delta; + /// The strides in each dimension between different loads/stores. + typedef Shape<0, 0, Warps::kW * WmmaShape_::kW, 0> ImmediateOffsetStrides; + + /// ThreadOffset + struct ThreadOffset { + CUTLASS_HOST_DEVICE + Coord<4> operator()() const { + // The warp id. + int const warp = threadIdx.x / kWarpSize; + // The starting column. + int const h = warp / Warps::kW * WmmaShape::kH; + // The w. + int const w = warp % Warps::kW * WmmaShape::kW; + // The offset. + int const offset = h * Tile::kW + w; + return make_Coord(0, 0, offset, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, typename Tile_, typename Threads_, int kScalarsPerLds_> +struct WmmaGemmSharedLoadTileDTraits { + /// The scalar. + typedef Scalar_ Scalar; + /// The pointer. + typedef Scalar const* Pointer; + /// The access size + static int const kAccessSize = kScalarsPerLds_; + /// The tile. + typedef typename ReshapeTile<Tile_, kScalarsPerLds_>::Tile Tile; + /// The threads. + typedef typename ReshapeThreads<Tile, Threads_>::Threads Threads; + /// The threads strides. + typedef Shape<1, Tile::kW * Tile::kC, Tile::kC> ThreadsStrides; + /// The memory space. + static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; + + /// The strides in each dimension between different loads/stores. + typedef Shape<0, Threads::kH * ShapeCount<Tile>::kWc, Threads::kW * kScalarsPerLds_> Delta; + /// The strides in each dimension between different loads/stores. + typedef Shape<0, Threads::kH * ShapeCount<Tile>::kWc, Threads::kW * kScalarsPerLds_> + ImmediateOffsetStrides; + /// The number of iterations needed to load/store the tile. + typedef Shape<1, Tile::kH / Threads::kH, Tile::kW / Threads::kW, Tile::kC / kScalarsPerLds_> + Iterations; + + /// ThreadOffset + struct ThreadOffset { + CUTLASS_HOST_DEVICE + Coord<4> operator()() const { + // The offset. + int const offset = ComputeThreadOffsetFromStrides<Threads, ThreadsStrides>::get(); + return make_Coord(0, 0, offset, 0); + } + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass + +#endif // defined CUTLASS_USE_WMMA_API diff --git a/cutlass-example/cutlass/gemm/wmma_gemm_traits.h b/cutlass-example/cutlass/gemm/wmma_gemm_traits.h new file mode 100644 index 0000000..7901201 --- /dev/null +++ b/cutlass-example/cutlass/gemm/wmma_gemm_traits.h @@ -0,0 +1,574 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defies structural properties of GEMM targeting WMMA API in CUDA. +*/ +#pragma once + +#include <cutlass/wmma_matrix.h> +#ifdef CUTLASS_USE_WMMA_API + +#include <cutlass/convert.h> +#include <cutlass/gemm/gemm.h> +#include <cutlass/gemm/gemm_epilogue.h> +#include <cutlass/gemm/gemm_epilogue_traits.h> +#include <cutlass/gemm/gemm_global_tile.h> +#include <cutlass/gemm/gemm_shared_tile.h> +#include <cutlass/gemm/gemm_traits.h> +#include <cutlass/gemm/wmma_gemm_epilogue_traits.h> +#include <cutlass/gemm/wmma_gemm_global_tile.h> +#include <cutlass/gemm/wmma_gemm_multiply_add.h> + +namespace cutlass { +namespace gemm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The layout for A. + MatrixLayout::Kind kLayoutA_, + /// The layout for B. + MatrixLayout::Kind kLayoutB_, + /// The tile size for the GEMM KxNxM. + typename OutputTile_, + /// The output type. + typename ScalarC_, + /// The accumulator type. + typename Accumulator_, + /// The number of accumulators per warp. + typename AccumulatorsPerWarp_, + /// The shape of the WMMA instruction. + typename InstructionShape_, + /// The number of scalars per LDG for A. + int kScalarsPerLdgA_, + /// The number of scalars per LDG for B. + int kScalarsPerLdgB_> +struct WmmaGemmConfig : public GemmConfig< + /// The scalar type for A. + half, + /// The scalar type for B. + half, + /// The scalar type for C. + ScalarC_, + /// The scalar type for D. + ScalarC_, + /// The tile size for the GEMM KxNxM. + OutputTile_, + /// The functor to do the math in the main loop. + WmmaGemmMultiplyAdd<kLayoutA_, + half, + kLayoutB_, + half, + MatrixLayout::kColumnMajor, + Accumulator_, + AccumulatorsPerWarp_, + InstructionShape_>, + /// The number of scalars per LDG for A. + kScalarsPerLdgA_, + /// The number of scalars per STS for A. + kScalarsPerLdgA_, + /// The number of scalars per LDS for A. + 8, + /// The number of scalars per LDG for B. + kScalarsPerLdgB_, + /// The number of scalars per STS for B. + kScalarsPerLdgB_, + /// The number of scalars per LDS for B. + 8, + /// The number of scalars per LDG for C and STG for D. + 16 / sizeof(ScalarC_), + /// The number of scalars per STS for D. + 16 / sizeof(ScalarC_), + /// The number of scalars per LDS for D. + 16 / sizeof(ScalarC_), + /// The number of stages in shared memory. + 1> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <enum MatrixLayout::Kind kLayout_, typename GemmConfig_> +struct WmmaGemmTileTraitsHelperA {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_> +struct WmmaGemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_> + : public GemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_> { + /// The base config. + typedef GemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_> Base; + + /// The skew. + static int const kSkew = 16 / sizeof(typename Base::MultiplyAddScalar); + /// The shared tile size. + typedef Shape<GemmConfig_::kStages, + GemmConfig_::OutputTile::kD, + GemmConfig_::OutputTile::kW + kSkew> + Tile; + + /// WMMA matrix + typedef WmmaMatrix<GemmOperand::kA, + MatrixLayout::kColumnMajor, + typename Base::MultiplyAddScalar, + typename GemmConfig_::InstructionShape> + WmmaMatrix; + + /// The traits class to build the iterator to store data to shared memory for A^N. + typedef GemmSharedStoreTileAbTraits< + // The pointer. + typename Base::MultiplyAddScalar, + // The tile has size KxM in GEMM's terminology. + Tile, + // The threads are distributed as warps x 32 (the traits may reorganize). + typename Base::GlobalTileTraits::Threads, + // The number of scalars per STS (STS.32 or STS.128, etc). + GemmConfig_::kScalarsPerStsA> + SharedStoreTileTraits; + + /// The number of elements loaded in one LDG. + static int const kScalarsPerW = GemmConfig_::InstructionShape::kW * GemmConfig_::Warps::kW; + /// The number of scalars loaded per iteration. + static int const kScalarsPerIteration = Tile::kW * GemmConfig_::InstructionShape::kD; + /// The traits class to build the iterator to load from shared memory for A. + typedef WmmaGemmSharedLoadTileATraits< + // The layout of the matrix. + MatrixLayout::kColumnMajor, + // The pointer. + typename Base::MultiplyAddScalar, + // The output tile size. + Tile, + // The number of warps. + typename GemmConfig_::Warps, + // The strides between warps. + GemmConfig_::InstructionShape::kW, + // The number of iterations to load the data. + Shape<1, 1, GemmConfig_::OutputTile::kW / kScalarsPerW>, + // The stride between iterations. + Shape<kScalarsPerIteration, 0, kScalarsPerW, 0>, + // The shape of the instruction. + typename GemmConfig_::InstructionShape> + SharedLoadTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_> +struct WmmaGemmTileTraitsHelperA<MatrixLayout::kRowMajor, GemmConfig_> { + /// The layout. + static MatrixLayout::Kind const kLayout = MatrixLayout::kRowMajor; + + /// The input scalar. + typedef typename GemmConfig_::ScalarA Scalar; + /// The scalar stored in shared memory. + typedef typename GemmConfig_::MultiplyAdd::ScalarA MultiplyAddScalar; + + /// WMMA matrix + typedef WmmaMatrix<GemmOperand::kA, + MatrixLayout::kRowMajor, + MultiplyAddScalar, + typename GemmConfig_::InstructionShape> + WmmaMatrix; + + /// The traits class to build the iterator to load data from global memory for A^T. + typedef GemmGlobalTileTraits< + // That's A. + GemmOperand::kA, + // A is row-major. + MatrixLayout::kRowMajor, + // The pointer is float const. + Scalar const, + // The tile has size KxM in GEMM's terminology. + Shape<1, GemmConfig_::OutputTile::kW, GemmConfig_::OutputTile::kD>, + // The threads are distributed as warps x 32 (the traits may reorganize). + Shape<1, GemmConfig_::kThreads / GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kD>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerLdgA> + GlobalTileTraits; + + /// The skew. + static int const kSkew = 16 / sizeof(MultiplyAddScalar); + /// The tile. + typedef Shape<GemmConfig_::kStages, + GemmConfig_::OutputTile::kW, + GemmConfig_::OutputTile::kD + kSkew> + Tile; + + /// The traits class to build the iterator to store data to shared memory for A^N. + typedef GemmSharedStoreTileAbTraits< + // The pointer. + MultiplyAddScalar, + // The tile has size KxM in GEMM's terminology. + Tile, + // The threads are distributed as warps x 32 (the traits may reorganize). + typename GlobalTileTraits::Threads, + // The number of scalars per STS (STS.32 or STS.128, etc). + GemmConfig_::kScalarsPerStsA> + SharedStoreTileTraits; + + /// The number of elements loaded in one LDG. + static int const kScalarsPerW = GemmConfig_::InstructionShape::kW * GemmConfig_::Warps::kW; + /// The traits class to build the iterator to load from shared memory for A. + typedef WmmaGemmSharedLoadTileATraits< + // The layout of the matrix. + MatrixLayout::kRowMajor, + // The pointer. + MultiplyAddScalar, + // The tile in shared memory. + Tile, + // The number of warps. + typename GemmConfig_::Warps, + // The strides between warps. + GemmConfig_::InstructionShape::kW * Tile::kW, + // The number of iterations to load the data. + Shape<1, 1, GemmConfig_::OutputTile::kW / kScalarsPerW>, + // The stride between iterations. + Shape<GemmConfig_::InstructionShape::kD, 0, kScalarsPerW * Tile::kW>, + // The shape of the instruction. + typename GemmConfig_::InstructionShape> + SharedLoadTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <enum MatrixLayout::Kind kLayout_, typename GemmConfig_> +struct WmmaGemmTileTraitsHelperB {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_> +struct WmmaGemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_> + : public GemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_> { + /// The base config. + typedef GemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_> Base; + + /// The skew. + static int const kSkew = 16 / sizeof(typename Base::MultiplyAddScalar); + /// The shared tile size. + typedef Shape<GemmConfig_::kStages, + GemmConfig_::OutputTile::kD, + GemmConfig_::OutputTile::kH + kSkew> + Tile; + + /// WMMA matrix + typedef WmmaMatrix<GemmOperand::kB, + MatrixLayout::kRowMajor, + typename Base::MultiplyAddScalar, + typename GemmConfig_::InstructionShape> + WmmaMatrix; + + /// The traits class to build the iterator to store data to shared memory for B^T. + typedef GemmSharedStoreTileAbTraits< + // The pointer. + typename Base::MultiplyAddScalar, + // The tile has size KxM in GEMM's terminology. + Tile, + // The threads are distributed as warps x 32 (the traits may reorganize). + typename Base::GlobalTileTraits::Threads, + // The number of scalars per STS (STS.32 or STS.128, etc). + GemmConfig_::kScalarsPerStsB> + SharedStoreTileTraits; + + /// The number of elements loaded in one LDG. + static int const kScalarsPerW = GemmConfig_::InstructionShape::kH * GemmConfig_::Warps::kH; + /// The number of scalars loaded per iteration. + static int const kScalarsPerIteration = Tile::kW * GemmConfig_::InstructionShape::kD; + /// The traits class to build the iterator to load from shared memory for B. + typedef WmmaGemmSharedLoadTileBTraits< + // The layout of the matrix. + MatrixLayout::kRowMajor, + // The pointer. + typename Base::MultiplyAddScalar, + // The output tile size. + Tile, + // The number of warps. + typename GemmConfig_::Warps, + // The strides between warps. + GemmConfig_::InstructionShape::kH, + // The number of iterations to load the data. + Shape<1, 1, GemmConfig_::OutputTile::kH / kScalarsPerW>, + // The stride between iterations. + Shape<kScalarsPerIteration, 0, kScalarsPerW, 0>, + // The shape of the instruction. + typename GemmConfig_::InstructionShape> + SharedLoadTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmConfig_> +struct WmmaGemmTileTraitsHelperB<MatrixLayout::kColumnMajor, GemmConfig_> { + /// The layout. + static MatrixLayout::Kind const kLayout = MatrixLayout::kColumnMajor; + + /// The input scalar. + typedef typename GemmConfig_::ScalarB Scalar; + /// The scalar stored in shared memory. + typedef typename GemmConfig_::MultiplyAdd::ScalarB MultiplyAddScalar; + + /// WMMA matrix + typedef WmmaMatrix<GemmOperand::kB, + MatrixLayout::kColumnMajor, + MultiplyAddScalar, + typename GemmConfig_::InstructionShape> + WmmaMatrix; + + /// The traits class to build the iterator to load data from global memory for B^N. + typedef GemmGlobalTileTraits< + // That's B. + GemmOperand::kB, + // A is row-major. + MatrixLayout::kColumnMajor, + // The pointer is float const. + Scalar const, + // The tile has size KxM in GEMM's terminology. + Shape<1, GemmConfig_::OutputTile::kH, GemmConfig_::OutputTile::kD>, + // The threads are distributed as warps x 32 (the traits may reorganize). + Shape<1, GemmConfig_::kThreads / GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kD>, + // The number of scalars per LDG (LDG.32 or LDG.128, etc). + GemmConfig_::kScalarsPerLdgB> + GlobalTileTraits; + + /// The skew. + static int const kSkew = 16 / sizeof(MultiplyAddScalar); + /// The tile. + typedef Shape<GemmConfig_::kStages, + GemmConfig_::OutputTile::kH, + GemmConfig_::OutputTile::kD + kSkew> + Tile; + + /// The traits class to build the iterator to store data to shared memory for B^N. + typedef GemmSharedStoreTileAbTraits< + // The pointer. + MultiplyAddScalar, + // The tile has size KxM in GEMM's terminology. + Tile, + // The threads are distributed as warps x 32 (the traits may reorganize). + typename GlobalTileTraits::Threads, + // The number of scalars per STS (STS.32 or STS.128, etc). + GemmConfig_::kScalarsPerStsB> + SharedStoreTileTraits; + + /// The number of elements loaded in one LDG. + static int const kScalarsPerW = GemmConfig_::InstructionShape::kH * GemmConfig_::Warps::kH; + /// The traits class to build the iterator to load from shared memory for B. + typedef WmmaGemmSharedLoadTileBTraits< + // The layout of the matrix. + MatrixLayout::kColumnMajor, + // The pointer. + MultiplyAddScalar, + // The tile in shared memory. + Tile, + // The number of warps. + typename GemmConfig_::Warps, + // The strides between warps. + GemmConfig_::InstructionShape::kH * Tile::kW, + // The number of iterations to load the data. + Shape<1, 1, GemmConfig_::OutputTile::kH / kScalarsPerW>, + // The stride between iterations. + Shape<GemmConfig_::InstructionShape::kD, 0, kScalarsPerW * Tile::kW>, + // The shape of the instruction. + typename GemmConfig_::InstructionShape> + SharedLoadTileTraits; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The layout for A. + MatrixLayout::Kind kLayoutA_, + /// The layout for B. + MatrixLayout::Kind kLayoutB_, + /// The output tile. + typename OutputTile_, + /// The output type. + typename ScalarC_, + /// The accumulator type. + typename Accumulator_, + /// The functor to do the math in the epilogue. + typename EpilogueFunctor_, + /// The number of accumulators per warp. + typename AccumulatorsPerWarp_, + /// The shape of the WMMA instruction. + typename InstructionShape_, + /// The number of halfs loaded in one LDG for A. + int kScalarsPerLdgA_, + /// The number of halfs loaded in one LDG for B. + int kScalarsPerLdgB_, + /// The index. + typename Index_> +struct WmmaGemmTraitsHelper { + /// The WMMA GEMM config. + typedef WmmaGemmConfig<kLayoutA_, + kLayoutB_, + OutputTile_, + ScalarC_, + Accumulator_, + AccumulatorsPerWarp_, + InstructionShape_, + kScalarsPerLdgA_, + kScalarsPerLdgB_> + GemmConfig; + + /// The GEMM config for A. + typedef WmmaGemmTileTraitsHelperA<kLayoutA_, GemmConfig> GemmTileTraitsHelperA; + /// The GEMM config for B. + typedef WmmaGemmTileTraitsHelperB<kLayoutB_, GemmConfig> GemmTileTraitsHelperB; + + /// The iterator to load A from global memory. + typedef GemmGlobalIteratorAb<typename GemmTileTraitsHelperA::GlobalTileTraits, Index_> + GlobalLoadIteratorA; + /// The default transformer for A. + typedef Copy<typename GlobalLoadIteratorA::Fragment> GlobalTransformerA; + /// The iterator to store A to shared memory. + typedef TileStoreIterator<typename GemmTileTraitsHelperA::SharedStoreTileTraits, + typename GemmTileTraitsHelperA::SharedStoreTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedStoreIteratorA; + /// The stream to load A from global memory to shared memory. + typedef GlobalLoadStream<GlobalLoadIteratorA, SharedStoreIteratorA, GlobalTransformerA> + GlobalLoadStreamA; + + /// The iterator to load B from global memory. + typedef GemmGlobalIteratorAb<typename GemmTileTraitsHelperB::GlobalTileTraits, Index_> + GlobalLoadIteratorB; + // The default transformer for B. + typedef Copy<typename GlobalLoadIteratorB::Fragment> GlobalTransformerB; + /// The iterator to store B to shared memory. + typedef TileStoreIterator<typename GemmTileTraitsHelperB::SharedStoreTileTraits, + typename GemmTileTraitsHelperB::SharedStoreTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared> + SharedStoreIteratorB; + /// The stream to load B from global memory to shared memory. + typedef GlobalLoadStream<GlobalLoadIteratorB, SharedStoreIteratorB, GlobalTransformerB> + GlobalLoadStreamB; + + /// The iterator to load A from shared memory. + typedef TileLoadIterator<typename GemmTileTraitsHelperA::SharedLoadTileTraits, + typename GemmTileTraitsHelperA::SharedLoadTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared, + Index_, + typename GemmTileTraitsHelperA::WmmaMatrix, + IteratorFragment::kWmmaMatrix> + SharedLoadIteratorA; + /// The stream to load A from shared memory. + typedef SharedLoadStream<SharedLoadIteratorA> SharedLoadStreamA; + /// The iterator to load B from shared memory. + typedef TileLoadIterator<typename GemmTileTraitsHelperB::SharedLoadTileTraits, + typename GemmTileTraitsHelperB::SharedLoadTileTraits::Scalar, + IteratorAdvance::kH, + MemorySpace::kShared, + Index_, + typename GemmTileTraitsHelperB::WmmaMatrix, + IteratorFragment::kWmmaMatrix> + SharedLoadIteratorB; + /// The stream to load B from shared memory. + typedef SharedLoadStream<SharedLoadIteratorB> SharedLoadStreamB; + + /// The functor to do the multiply-add in the main loop. + typedef typename GemmConfig::MultiplyAdd MultiplyAdd; + /// The object to clear accumulators. + typedef ClearAccumulators<typename MultiplyAdd::ScalarC> ClearAccumulators; + + /// The helper to create the epilogue traits. + typedef WmmaGemmEpilogueTraitsHelper<GemmConfig, EpilogueFunctor_, Index_> EpilogueTraitsHelper; + /// The traits class for the epilogue. + typedef SimplifiedGemmEpilogueTraits<GemmConfig, EpilogueFunctor_, Index_, EpilogueTraitsHelper> + GemmEpilogueTraits; + /// The epilogue. + typedef GemmEpilogue<GemmEpilogueTraits> Epilogue; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename OutputTile_, typename DefaultShape_ = Shape<64, 32, 64> > +struct WmmaGemmAccumulatorsPerWarp { + typedef typename ShapeMin<OutputTile_, DefaultShape_>::Shape Shape; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template < + /// The layout for A. + MatrixLayout::Kind kLayoutA_, + /// The layout for B. + MatrixLayout::Kind kLayoutB_, + /// The tile size for the GEMM KxNxM. + typename OutputTile_ = Shape<64, 128, 128>, + /// The output type. + typename ScalarC_ = float, + /// The functor to do the math in the epilogue. + typename EpilogueFunctor_ = LinearScaling<ScalarC_>, + /// The accumulator type. + typename Accumulator_ = ScalarC_, + /// The number of accumulators per warp. + typename AccumulatorsPerWarp_ = typename WmmaGemmAccumulatorsPerWarp<OutputTile_>::Shape, + /// The shape of the WMMA instruction. + typename InstructionShape_ = Shape<16, 16, 16>, + /// The number of scalars per LDG for A. + int kScalarsPerLdgA_ = 8, + /// The number of scalars per LDG for B. + int kScalarsPerLdgB_ = 8, + /// The index. + typename Index_ = int, + /// The helper class. + typename Helper_ = WmmaGemmTraitsHelper<kLayoutA_, + kLayoutB_, + OutputTile_, + ScalarC_, + Accumulator_, + EpilogueFunctor_, + AccumulatorsPerWarp_, + InstructionShape_, + kScalarsPerLdgA_, + kScalarsPerLdgB_, + Index_> > +struct WmmaGemmTraits : public GemmTraits< + // The config. + typename Helper_::GemmConfig, + // The stream to load A from global memory to shared memory. + typename Helper_::GlobalLoadStreamA, + // The stream to load B from global memory to shared memory. + typename Helper_::GlobalLoadStreamB, + // The stream to load A from shared memory. + typename Helper_::SharedLoadStreamA, + // The stream to load B from shared memory. + typename Helper_::SharedLoadStreamB, + // The epilogue. + typename Helper_::Epilogue, + // The block swizzle to reorganize the grid. + IdentityBlockSwizzle, + // The index. + Index_, + // The tool used to clear accumulators. + typename Helper_::ClearAccumulators> {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace gemm +} // namespace cutlass + +#endif // defined CUTLASS_USE_WMMA_API diff --git a/cutlass-example/cutlass/iterator_access.h b/cutlass-example/cutlass/iterator_access.h new file mode 100644 index 0000000..e94beb7 --- /dev/null +++ b/cutlass-example/cutlass/iterator_access.h @@ -0,0 +1,318 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Free functions for loading and storing to implementations of tile iteartor concepts. +*/ +#pragma once + +#include <cutlass/fragment_load_store.h> +#include <cutlass/load_store.h> +#include <cutlass/predicate_vector.h> +#include <cutlass/shape.h> + +namespace cutlass { + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Loads a fragment from an input iterator +template <typename InputIterator, typename Fragment> +CUTLASS_HOST_DEVICE void iterator_load(InputIterator &iterator, Fragment &fragment) { + typename InputIterator::FragmentIterator frag_iterator(fragment); + for (int d = 0; d < InputIterator::Iterations::kD; ++d) { + for (int h = 0; h < InputIterator::Iterations::kH; ++h) { + for (int w = 0; w < InputIterator::Iterations::kW; ++w) { + for (int c = 0; c < InputIterator::Iterations::kC; ++c) { + if (iterator.valid(d, h, w, c)) { + iterator.get(reinterpret_cast<typename InputIterator::AccessType &>( + frag_iterator.at(d, h, w, c)), + d, + h, + w, + c); + } + } + if (w < InputIterator::Iterations::kW - 1) { + iterator.inc_w(); + } + } + if (h < InputIterator::Iterations::kH - 1) { + iterator.inc_h(); + } + } + if (d < InputIterator::Iterations::kD - 1) { + iterator.inc_d(); + } + } + iterator.inc_advance(); +} + +/// Loads a fragment from a shared memory input iterator +template <typename InputIterator, typename Fragment> +CUTLASS_DEVICE void shared_iterator_load(InputIterator &iterator, Fragment &fragment) { + typename InputIterator::FragmentIterator frag_iterator(fragment); + for (int d = 0; d < InputIterator::Iterations::kD; ++d) { + for (int h = 0; h < InputIterator::Iterations::kH; ++h) { + for (int w = 0; w < InputIterator::Iterations::kW; ++w) { + for (int c = 0; c < InputIterator::Iterations::kC; ++c) { + int const offset = + ComputeOffsetFromStrides<typename InputIterator::ImmediateOffsetStrides>::get( + d, h, w, c); + + FragmentLoad<InputIterator::kIteratorFragment, + InputIterator::Tile::kC, + typename InputIterator::Scalar, + InputIterator::kMemorySpace, + typename InputIterator::FragmentElement, + InputIterator::Tile::kW>::load(frag_iterator.at(d, h, w, c), + iterator.data(), + offset); + } + } + } + } +} + +/// Loads a fragment from a shared memory input iterator +template <typename InputIterator, typename Fragment> +CUTLASS_DEVICE void shared_iterator_load(InputIterator &iterator, Fragment &fragment, int d) { + typename InputIterator::FragmentIterator frag_iterator(fragment); + for (int h = 0; h < InputIterator::Iterations::kH; ++h) { + for (int w = 0; w < InputIterator::Iterations::kW; ++w) { + for (int c = 0; c < InputIterator::Iterations::kC; ++c) { + int const offset = + ComputeOffsetFromStrides<typename InputIterator::ImmediateOffsetStrides>::get( + d, h, w, c); + + FragmentLoad<InputIterator::kIteratorFragment, + InputIterator::Tile::kC, + typename InputIterator::Scalar, + InputIterator::kMemorySpace, + typename InputIterator::FragmentElement, + InputIterator::Tile::kW>::load(frag_iterator.at(0, h, w, c), + iterator.data(), + offset); + } + } + } +} + +/// Loads a fragment from an input iterator, masked by a predicate iterator +template <typename InputIterator, typename Fragment, typename ConstPredicateAdapter> +CUTLASS_HOST_DEVICE void iterator_load_post_increment(InputIterator &iterator, + Fragment &fragment, + typename InputIterator::Index offset, + ConstPredicateAdapter predicate_adapter) { + for (int d = 0; d < InputIterator::Iterations::kD; ++d, iterator.inc_d()) { + for (int h = 0; h < InputIterator::Iterations::kH; ++h, iterator.inc_h()) { + for (int w = 0; w < InputIterator::Iterations::kW; ++w, iterator.inc_w()) { + if (predicate_adapter.at(d, h, w, 0)) { + int idx = InputIterator::Tile::kC * + (w + InputIterator::Iterations::kW * (h + InputIterator::Iterations::kH * d)); + + Load<typename Fragment::Element, InputIterator::Tile::kC, InputIterator::kMemorySpace>:: + load(reinterpret_cast<typename InputIterator::AccessType &>(fragment[idx]), + iterator.data(), + offset); + } + } + } + } +} + +/// Loads a fragment from an input iterator +template <typename InputIterator, typename Fragment> +CUTLASS_HOST_DEVICE void iterator_load_post_increment(InputIterator &iterator, + Fragment &fragment, + typename InputIterator::Index offset = 0) { + TrivialPredicateTileAdapter pred; + iterator_load_post_increment(iterator, fragment, offset, pred); +} + +/// Loads a fragment from an input iterator +template <typename InputIterator, typename Fragment, typename ConstPredicateAdapter> +CUTLASS_HOST_DEVICE void iterator_load_post_increment(InputIterator &iterator, + Fragment &fragment, + ConstPredicateAdapter pred_it) { + iterator_load_post_increment(iterator, fragment, 0, pred_it); +} + +template <typename InputIterator, typename Fragment, typename ConstPredicateAdapter> +CUTLASS_HOST_DEVICE void iterator_load(InputIterator const &_iterator, + Fragment &fragment, + typename InputIterator::Index offset, + ConstPredicateAdapter predicate_adapter) { + InputIterator iterator(_iterator); + iterator_load_post_increment(iterator, fragment, offset, predicate_adapter); +} + +/// Loads a fragment from an input iterator +template <typename InputIterator, typename Fragment> +CUTLASS_HOST_DEVICE void iterator_load(InputIterator const &iterator, + Fragment &fragment, + typename InputIterator::Index offset = 0) { + TrivialPredicateTileAdapter pred; + iterator_load(iterator, fragment, offset, pred); +} + +/// Loads a fragment from an input iterator +template <typename InputIterator, typename Fragment, typename ConstPredicateAdapter> +CUTLASS_HOST_DEVICE void iterator_load(InputIterator const &iterator, + Fragment &fragment, + ConstPredicateAdapter pred_it) { + iterator_load(iterator, fragment, 0, pred_it); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Stores a fragment to an output iterator +template <typename OutputIterator, typename Fragment> +CUTLASS_HOST_DEVICE void iterator_store(OutputIterator &iterator, Fragment &fragment) { + typename OutputIterator::FragmentIterator frag_iterator(fragment); + for (int d = 0; d < OutputIterator::Iterations::kD; ++d) { + for (int h = 0; h < OutputIterator::Iterations::kH; ++h) { + for (int w = 0; w < OutputIterator::Iterations::kW; ++w) { + if (iterator.valid(d, h, w, 0)) { + iterator.set(reinterpret_cast<typename OutputIterator::AccessType const &>( + frag_iterator.at(d, h, w, 0)), + d, + h, + w, + 0); + } + if (w < OutputIterator::Iterations::kW - 1) { + iterator.inc_w(); + } + } + if (h < OutputIterator::Iterations::kH - 1) { + iterator.inc_h(); + } + } + if (d < OutputIterator::Iterations::kD - 1) { + iterator.inc_d(); + } + } + iterator.inc_advance(); +} + +/// Stores a fragment to a shared memory output iterator +template <typename OutputIterator, typename Fragment> +CUTLASS_DEVICE void shared_iterator_store(OutputIterator &iterator, Fragment const &fragment) { + typename OutputIterator::FragmentConstIterator frag_iterator(fragment); + for (int d = 0; d < OutputIterator::Iterations::kD; ++d) { + for (int h = 0; h < OutputIterator::Iterations::kH; ++h) { + for (int w = 0; w < OutputIterator::Iterations::kW; ++w) { + for (int c = 0; c < OutputIterator::Iterations::kC; ++c) { + int const offset = + ComputeOffsetFromStrides<typename OutputIterator::ImmediateOffsetStrides>::get( + d, h, w, c); + + FragmentStore<OutputIterator::kIteratorFragment, + OutputIterator::Tile::kC, + typename OutputIterator::Scalar, + OutputIterator::kMemorySpace, + typename OutputIterator::FragmentElement, + OutputIterator::Tile::kW>::store(frag_iterator.at(d, h, w, c), + iterator.data(), + offset); + } + } + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Stores a fragment to an output iterator, masked by a predicate iterator +template <typename OutputIterator, typename Fragment, typename ConstPredicateAdapter> +CUTLASS_HOST_DEVICE void iterator_store_post_increment(OutputIterator &iterator, + Fragment const &fragment, + typename OutputIterator::Index offset, + ConstPredicateAdapter predicate_adapter) { + for (int d = 0; d < OutputIterator::Iterations::kD; ++d, iterator.inc_d()) { + for (int h = 0; h < OutputIterator::Iterations::kH; ++h, iterator.inc_h()) { + for (int w = 0; w < OutputIterator::Iterations::kW; ++w, iterator.inc_w()) { + if (predicate_adapter.at(d, h, w, 0)) { + int idx = OutputIterator::Tile::kC * + (w + OutputIterator::Iterations::kW * (h + OutputIterator::Iterations::kH * d)); + + Store<typename Fragment::Element, + OutputIterator::Tile::kC, + OutputIterator::kMemorySpace>:: + store(reinterpret_cast<typename OutputIterator::AccessType const &>(fragment[idx]), + iterator.data(), + offset); + } + } + } + } +} + +/// Stores a fragment to an output iterator +template <typename OutputIterator, typename Fragment> +CUTLASS_HOST_DEVICE void iterator_store_post_increment(OutputIterator &iterator, + Fragment const &fragment, + typename OutputIterator::Index offset = 0) { + TrivialPredicateTileAdapter pred; + iterator_store_post_increment(iterator, fragment, offset, pred); +} + +/// Stores a fragment to an output iterator +template <typename OutputIterator, typename Fragment, typename ConstPredicateAdapter> +CUTLASS_HOST_DEVICE void iterator_store_post_increment(OutputIterator &iterator, + Fragment const &fragment, + ConstPredicateAdapter pred_it) { + iterator_store_post_increment(iterator, fragment, 0, pred_it); +} + +/// Stores a fragment to an output iterator, masked by a predicate iterator +template <typename OutputIterator, typename Fragment, typename ConstPredicateAdapter> +CUTLASS_HOST_DEVICE void iterator_store(OutputIterator const &_iterator, + Fragment const &fragment, + typename OutputIterator::Index offset, + ConstPredicateAdapter predicate_adapter) { + OutputIterator iterator(_iterator); + iterator_store_post_increment(iterator, fragment, offset, predicate_adapter); +} + +/// Stores a fragment to an output iterator +template <typename OutputIterator, typename Fragment> +CUTLASS_HOST_DEVICE void iterator_store(OutputIterator const &iterator, + Fragment const &fragment, + typename OutputIterator::Index offset = 0) { + TrivialPredicateTileAdapter pred; + iterator_store(iterator, fragment, offset, pred); +} + +/// Stores a fragment to an output iterator +template <typename OutputIterator, typename Fragment, typename ConstPredicateAdapter> +CUTLASS_HOST_DEVICE void iterator_store(OutputIterator const &iterator, + Fragment const &fragment, + ConstPredicateAdapter pred_it) { + iterator_store(iterator, fragment, 0, pred_it); +} + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/load_store.h b/cutlass-example/cutlass/load_store.h new file mode 100644 index 0000000..5cb5eb6 --- /dev/null +++ b/cutlass-example/cutlass/load_store.h @@ -0,0 +1,222 @@ +/*************************************************************************************************** + * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines abstractions for efficiently loading and storing vectors to memory. +*/ +#pragma once + +#include <cutlass/vector.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief Enum to specify which memory space data resides in. +*/ +struct MemorySpace { + enum Kind { + kGeneric, // Data accessed through pointer dereferencing + kShared, // Data resides in shared memory + kGlobal // Data resides in global memory + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, + int Lanes_, + MemorySpace::Kind Memory_, + bool = (Lanes_ > 1), + size_t = (sizeof(Scalar_) * Lanes_)> +struct Load { + /// The output type. + typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; + + /// The load function. + static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) { + dst = reinterpret_cast<AccessType const*>(&pointer[offset])[0]; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_> +struct Load<Scalar_, Lanes_, Memory_, true, 4> { + /// The output type. + typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; + + /// The store function. + static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) { + dst.registers[0] = reinterpret_cast<uint32_t const*>(&pointer[offset])[0]; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_> +struct Load<Scalar_, Lanes_, Memory_, true, 8> { + /// The output type. + typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; + + /// The store function. + static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) { + uint2 tmp = reinterpret_cast<uint2 const*>(&pointer[offset])[0]; + dst.registers[0] = tmp.x; + dst.registers[1] = tmp.y; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <MemorySpace::Kind Memory_> +struct Load<double, 2, Memory_, true, 16> { + /// The output type. + typedef typename Vectorize<double, 2>::Type AccessType; + + /// The store function. + static CUTLASS_DEVICE void load(AccessType& dst, double const* pointer, int offset) { + double2 tmp = reinterpret_cast<double2 const*>(&pointer[offset])[0]; + dst[0] = tmp.x; + dst[1] = tmp.y; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#if defined(__CUDACC_VERSION_MAJOR) && __CUDACC_VERSION_MAJOR < 10 +// WAR bug in NVCC where the upper and lower half of the register end up being the same +template <MemorySpace::Kind Memory_> +struct Load<half, 8, Memory_, true, 16> { + /// The output type. + typedef typename Vectorize<half, 8>::Type AccessType; + + /// The store function. + static CUTLASS_DEVICE void load(AccessType& dst, half const* pointer, int offset) { + int2 tmp = reinterpret_cast<int2 const*>(&pointer[offset])[0]; + dst.registers[0] = tmp.x; + dst.registers[1] = tmp.y; + + tmp = reinterpret_cast<int2 const*>(&pointer[offset + 4])[0]; + dst.registers[2] = tmp.x; + dst.registers[3] = tmp.y; + } +}; + +#endif + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_> +struct Load<Scalar_, Lanes_, Memory_, true, 16> { + /// The output type. + typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; + + /// The store function. + static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) { + uint4 tmp = reinterpret_cast<uint4 const*>(&pointer[offset])[0]; + dst.registers[0] = tmp.x; + dst.registers[1] = tmp.y; + dst.registers[2] = tmp.z; + dst.registers[3] = tmp.w; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, + int Lanes_, + MemorySpace::Kind Memory_, + bool = (Lanes_ > 1), + size_t = (sizeof(Scalar_) * Lanes_)> +struct Store { + /// The output type. + typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; + + /// The store function. + static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) { + pointer[offset] = src; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_> +struct Store<Scalar_, Lanes_, Memory_, true, 4> { + /// The output type. + typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; + + /// The store function. + static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) { + uint32_t* addr = reinterpret_cast<uint32_t*>(&pointer[offset]); + addr[0] = src.registers[0]; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_> +struct Store<Scalar_, Lanes_, Memory_, true, 8> { + /// The output type. + typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; + + /// The store function. + static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) { + uint2* addr = reinterpret_cast<uint2*>(&pointer[offset]); + addr[0] = make_uint2(src.registers[0], src.registers[1]); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <MemorySpace::Kind Memory_> +struct Store<double, 2, Memory_, true, 16> { + /// The output type. + typedef typename Vectorize<double, 2>::Type AccessType; + + /// The store function. + static CUTLASS_DEVICE void store(AccessType const& src, double* pointer, int offset) { + double2* addr = reinterpret_cast<double2*>(&pointer[offset]); + addr[0] = make_double2(src[0], src[1]); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_> +struct Store<Scalar_, Lanes_, Memory_, true, 16> { + /// The output type. + typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; + + /// The store function. + static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) { + uint4* addr = reinterpret_cast<uint4*>(&pointer[offset]); + addr[0] = make_uint4(src.registers[0], src.registers[1], src.registers[2], src.registers[3]); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/matrix_traits.h b/cutlass-example/cutlass/matrix_traits.h new file mode 100644 index 0000000..77e8b70 --- /dev/null +++ b/cutlass-example/cutlass/matrix_traits.h @@ -0,0 +1,48 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines properties of matrices used to denote layout and operands to GEMM kernels. +*/ +#pragma once + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Describes layouts of matrices +struct MatrixLayout { + enum Kind { kRowMajor, kColumnMajor }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Gemm operand - D = A * B + C +struct GemmOperand { + enum Kind { kA, kB, kC, kD }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/predicate_vector.h b/cutlass-example/cutlass/predicate_vector.h new file mode 100644 index 0000000..8166857 --- /dev/null +++ b/cutlass-example/cutlass/predicate_vector.h @@ -0,0 +1,493 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines container classes and iterators for managing a statically sized vector + of boolean predicates. +*/ +#pragma once + +#include <stdint.h> + +#include <cutlass/cutlass.h> +#include <cutlass/shape.h> + +#include <cutlass/util/platform.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/*!@defgroup predicate_vector_concept Predicate Vector Concept +@{ + +Implementations of \ref predicate_vector_concept contain an ordered set of boolean predicates which +may be used as conditionals in other device-side operations. Both random access and iterators +offering sequential access are provided. + +@par Predicate Vector + A \ref predicate_vector_concept satisfies the following expressions + - <b>at(int idx)</b> - returns the value of the indexed predicate + - <b>set(int idx, bool value)</b> - sets the value of the indexed predicate + - <b>begin()</b> - returns a \ref predicate_iterator_concept pointing to the first predicate + +@} +*/ + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/*!@defgroup predicate_iterator_concept Predicate Iterator Concept +@{ + +Implementations of \ref predicate_iterator_concept enables accessing and traversing elements of a +bit vector. + +@par Const Predicate Iterator + A const \ref predicate_iterator_concept satisfies the following expressions + - <b>++it</b> increments the iterator to the next predicate + - <b>*it</b> returns the value of the currently pointed-to predicate + +@par Mutable Predicate Iterator + A \ref predicate_iterator_concept that is non-const <b>also</b> satisfies the following expressions + - <b>it.set(bool value)</b> sets the value of the currently pointed-to predicate + +@} +*/ + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/*!@defgroup predicate_tile_adapter Predicate Tile Adapter Concept +@{ + +Implementations of \ref predicate_tile_adapter provide a mapping between a the elements of a \ref +tile_traits_concept and a \ref predicate_vector_concept. + +@par Predicate Tile Adapter + A \ref predicate_tile_adapter satisfies the following expressions + - <b>at(int d, int h, int w, int c)</b> - returns the value of a predicate corresponding to the + access (d, h, w, c) within the tile. + +@} +*/ + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Statically sized array of bits implementing @concept{predicate_vector_concept}. +template < + /// Number of predicates conatined in predicate vector + int kPredicates_, + /// Number of predicates contained in each byte of internal storage + int kPredicatesPerByte_ = 4, + /// Location of first predicate within byte of internal storage + int kPredicateStart_ = 0> +struct PredicateVector { + /// Number of bits stored by the PredicateVector + static int const kPredicates = kPredicates_; + + /// Number of bits stored within each byte of the predicate bit vector + static int const kPredicatesPerByte = kPredicatesPerByte_; + + /// First bit withing each byte containing predicates + static int const kPredicateStart = kPredicateStart_; + + // Make sure no one tries to put more than 8 bits in a byte :) + static_assert(kPredicatesPerByte <= 8, "kPredicatesPerByte must fit within an actual byte"); + // Make sure the "offsetted" bits fit in one byte. + static_assert(kPredicateStart + kPredicatesPerByte < 8, + "The offsetted predicates must fit within an actual byte."); + + /// Storage type of individual elements + typedef uint32_t Storage; + + /// Number of bytes needed + static int const kBytes = (kPredicates + kPredicatesPerByte - 1) / kPredicatesPerByte; + + /// Number of storage elements needed + static int const kWordCount = (kBytes + sizeof(Storage) - 1) / sizeof(Storage); + + private: + // + // Data members + // + + /// Words of bit vector + Storage storageData[kWordCount]; + + // + // Methods + // + + /// Computes the word and bit corresponding to a logical predicate index + CUTLASS_HOST_DEVICE void computeStorageOffset(int &word, int &bit, int idx) const { + CUTLASS_ASSERT(idx < kPredicates); + + int byte = (idx / kPredicatesPerByte); + int bit_offset = (idx % kPredicatesPerByte); + + word = byte / sizeof(Storage); + int byte_offset = (byte % sizeof(Storage)); + + bit = byte_offset * 8 + bit_offset + kPredicateStart; + } + + /// Accesses a given word with optional assertions + CUTLASS_HOST_DEVICE Storage &storage(int word) { + CUTLASS_ASSERT(word < kWordCount); + return storageData[word]; + } + + /// Accesses a given word with optional assertions + CUTLASS_HOST_DEVICE Storage const &storage(int word) const { + CUTLASS_ASSERT(word < kWordCount); + return storageData[word]; + } + + public: + // + // Iterator + // + + /** + * @brief A const iterator implementing \ref predicate_iterator_concept enabling sequential + * read-only access to prediactes. + * @concept{predicate_iterator_concept} + */ + class ConstIterator { + /// Reference to PredicateVector instance + PredicateVector const &vec_; + + /// Index into PredicateVector + int bit_; + + public: + /// Copy constructor + CUTLASS_HOST_DEVICE + ConstIterator(ConstIterator const &it) : vec_(it.vec_), bit_(it.bit_) {} + + /// + CUTLASS_HOST_DEVICE + ConstIterator(PredicateVector const &_vec, int _start = 0) : vec_(_vec), bit_(_start) {} + + /// Pre-increment + CUTLASS_HOST_DEVICE + ConstIterator &operator++() { + ++bit_; + return *this; + } + + /// Pre-decrement + CUTLASS_HOST_DEVICE + ConstIterator &operator--() { + --bit_; + return *this; + } + + /// Post-increment + CUTLASS_HOST_DEVICE + ConstIterator operator++(int) { + ConstIterator ret(*this); + ret.bit_++; + return ret; + } + + /// Post-decrement + CUTLASS_HOST_DEVICE + ConstIterator operator--(int) { + ConstIterator ret(*this); + ret.bit_--; + return ret; + } + + /// Returns true if iterators point to the same bit + CUTLASS_HOST_DEVICE + bool operator==(ConstIterator const &it) const { return bit_ == it.bit_; } + + /// Returns false if iterators point to the same bit + CUTLASS_HOST_DEVICE + bool operator!=(ConstIterator const &it) const { return bit_ != it.bit_; } + + /// Dereferences iterator + CUTLASS_HOST_DEVICE + bool operator*() const { return vec_[bit_]; } + }; + + /** + * @brief An iterator implementing \ref predicate_iterator_concept enabling sequential + * read and write access to predicates. + * @concept{predicate_iterator_concept} + */ + class Iterator { + /// Reference to PredicateVector instance + PredicateVector &vec_; + + /// Index into PredicateVector + int bit_; + + public: + /// Copy constructor + CUTLASS_HOST_DEVICE + Iterator(Iterator const &it) : vec_(it.vec_), bit_(it.bit_) {} + + /// Constructs an iterator from a PredicateVector + CUTLASS_HOST_DEVICE + Iterator(PredicateVector &_vec, int _start = 0) : vec_(_vec), bit_(_start) {} + + /// Pre-increment + CUTLASS_HOST_DEVICE + Iterator &operator++() { + ++bit_; + return *this; + } + + /// Pre-decrement + CUTLASS_HOST_DEVICE + Iterator &operator--() { + --bit_; + return *this; + } + + /// Post-increment + CUTLASS_HOST_DEVICE + Iterator operator++(int) { + Iterator ret(*this); + ret.bit_++; + return ret; + } + + /// Post-decrement + CUTLASS_HOST_DEVICE + Iterator operator--(int) { + Iterator ret(*this); + ret.bit_--; + return ret; + } + + /// Returns true if iterators point to the same bit + CUTLASS_HOST_DEVICE + bool operator==(Iterator const &it) const { return bit_ == it.bit_; } + + /// Returns false if iterators point to the same bit + CUTLASS_HOST_DEVICE + bool operator!=(Iterator const &it) const { return bit_ != it.bit_; } + + /// Gets the bit at the pointed to location + CUTLASS_HOST_DEVICE + bool get() { return vec_[bit_]; } + + /// Dereferences iterator + CUTLASS_HOST_DEVICE + bool operator*() const { return vec_[bit_]; } + + /// Sets the bit at the pointed to location + CUTLASS_HOST_DEVICE + void set(bool value = true) { vec_.set(bit_, value); } + }; + + /// Iterator that always returns true + struct TrivialIterator { + /// Constructor + CUTLASS_HOST_DEVICE + TrivialIterator() {} + + /// Copy constructor + CUTLASS_HOST_DEVICE + TrivialIterator(Iterator const &it) {} + + /// Constructs an iterator from a PredicateVector + CUTLASS_HOST_DEVICE + TrivialIterator(PredicateVector const &_vec) {} + + /// Pre-increment + CUTLASS_HOST_DEVICE + TrivialIterator &operator++() { return *this; } + + /// Post-increment + CUTLASS_HOST_DEVICE + TrivialIterator operator++(int) { return *this; } + + /// Dereferences iterator + CUTLASS_HOST_DEVICE + bool operator*() const { return true; } + }; + + public: + // + // Methods + // + + /// Initialize the predicate vector + CUTLASS_HOST_DEVICE PredicateVector(bool value = true) { fill(value); } + + /// Fills all predicates with a given value + CUTLASS_HOST_DEVICE void fill(bool value = true) { + Storage item = (value ? ~Storage(0) : Storage(0)); + + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < kWordCount; ++i) { + storage(i) = item; + } + } + + /// Accesses a bit within the predicate vector. + CUTLASS_HOST_DEVICE bool operator[](int idx) const { return at(idx); } + + /// Accesses a bit within the predicate vector. + CUTLASS_HOST_DEVICE bool at(int idx) const { + int bit, word; + computeStorageOffset(word, bit, idx); + + return ((storage(word) >> bit) & 1); + } + + /// Set a bit within the predicate vector. + CUTLASS_HOST_DEVICE void set(int idx, bool value = true) { + int bit, word; + computeStorageOffset(word, bit, idx); + + Storage disable_mask = (~(Storage(1) << bit)); + Storage enable_mask = (Storage(value) << bit); + + storage(word) = ((storage(word) & disable_mask) | enable_mask); + } + + /// Computes the intersection of two identical predicate vectors. + CUTLASS_HOST_DEVICE PredicateVector &operator&=(PredicateVector const &predicates) { + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < kWordCount; ++i) { + storage(i) = (storage(i) & predicates.storage(i)); + } + return *this; + } + + /// Computes the union of two identical predicate vectors. + CUTLASS_HOST_DEVICE PredicateVector &operator|=(PredicateVector const &predicates) { + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < kWordCount; ++i) { + storage(i) = (storage(i) | predicates.storage(i)); + } + return *this; + } + + /// Returns true if entire predicate array is zero. + CUTLASS_HOST_DEVICE bool is_zero() const { + Storage mask(0); + for (int byte = 0; byte < sizeof(Storage); ++byte) { + Storage byte_mask = (((1 << kPredicatesPerByte) - 1) << kPredicateStart); + mask |= (byte_mask << (byte * 8)); + } + uint32_t result = 0; + for (int word = 0; word < kWordCount; ++word) { + result |= storage(word); + } + return result == 0; + } + + /// Returns an iterator to the start of the bit vector + CUTLASS_DEVICE + Iterator begin() { return Iterator(*this); } + + /// Returns an iterator + CUTLASS_DEVICE + Iterator end() { return Iterator(*this, kPredicates); } + + /// Returns a ConstIterator + CUTLASS_DEVICE + ConstIterator const_begin() const { return ConstIterator(*this); } + + /// Returns a ConstIterator + CUTLASS_DEVICE + ConstIterator const_end() const { return ConstIterator(*this, kPredicates); } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Always returns true predicate. +struct TrivialPredicateTileAdapter { + /// Ctor. + CUTLASS_HOST_DEVICE TrivialPredicateTileAdapter() {} + + /// The value at location (d, h, w, c). + CUTLASS_HOST_DEVICE bool at(int, int, int, int) const { return true; } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Adapter to enable random access to predicates via logical coordinate within a tile. +template <typename PredicateVector_, typename Iterations_> +struct PredicateTileAdapter { + /// The vector of predicates. + typedef PredicateVector_ PredicateVector; + /// The iterations. + typedef Iterations_ Iterations; + + private: + /// The predicates. + PredicateVector &predicates; + + public: + /// Ctor. + CUTLASS_DEVICE PredicateTileAdapter(PredicateVector &predicates_) : predicates(predicates_) {} + + /// Get the value at location (d, h, w, c). + CUTLASS_DEVICE bool at(int d, int h, int w, int c) const { + int const bit = ComputeOffsetFromShape<Iterations>::get(d, h, w, c); + return predicates.at(bit); + } + + /// Set the value at location (d, h, w, c). + CUTLASS_DEVICE void set(int d, int h, int w, int c, bool value) { + int const bit = ComputeOffsetFromShape<Iterations>::get(d, h, w, c); + predicates.set(bit, value); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Adapter to enable random access to predicates via logical coordinate within a tile. +template <typename PredicateVector_, typename Iterations_> +struct ConstPredicateTileAdapter { + /// The vector of predicates. + typedef PredicateVector_ PredicateVector; + /// The iterations. + typedef Iterations_ Iterations; + + private: + /// The predicates. + PredicateVector const &predicates; + + public: + /// Ctor. + CUTLASS_DEVICE ConstPredicateTileAdapter(PredicateVector const &predicates_) + : predicates(predicates_) {} + + /// Get the value at location (d, h, w, c). + CUTLASS_DEVICE bool at(int d, int h, int w, int c) const { + int const bit = ComputeOffsetFromShape<Iterations>::get(d, h, w, c); + return predicates.at(bit); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/reshape_tile.h b/cutlass-example/cutlass/reshape_tile.h new file mode 100644 index 0000000..55aebfc --- /dev/null +++ b/cutlass-example/cutlass/reshape_tile.h @@ -0,0 +1,58 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines a type for restructuring a tile. +*/ +#pragma once + +#include <cutlass/shape.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// The following functor reshapes a tile of data. The goal is to have at least kAccessSize in +// the inner-most dimension. If the user respects that constraint, there is nothing to be done. If +// that's not the case, this functor will correct that and "extract" the right number of elements +// from the next dimension. + +template <typename Tile_, int kAccessSize_, bool = (Tile_::kC < kAccessSize_)> +struct ReshapeTile { + typedef Tile_ Tile; +}; + +template <typename Tile_, int kAccessSize_> +struct ReshapeTile<Tile_, kAccessSize_, true> { + // Make sure the W dimension of the tile is large enough. + static_assert(Tile_::kW >= kAccessSize_, "The W dimension is too small"); + // Make sure the dimension can be divided by the number of scalars. + static_assert(Tile_::kW % kAccessSize_ == 0, "Not supported"); + // Collapse the W dimension. + typedef Shape<Tile_::kD, Tile_::kH, Tile_::kW / kAccessSize_, kAccessSize_> Tile; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/shape.h b/cutlass-example/cutlass/shape.h new file mode 100644 index 0000000..4f6b222 --- /dev/null +++ b/cutlass-example/cutlass/shape.h @@ -0,0 +1,305 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines Shape implementing the Layout concept for representing a 4D hypercube of objects. +*/ +#pragma once + +#include <cutlass/cutlass.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/*!@defgroup layout_concept Layout Concept +* @{ +* @par Implementations of \ref layout_concept are used to describe a cube with DxHxW elements and C +scalars per element. + A HxW slice of a cube is called an image and a cube consists of D images. +* +* @par Notations +* Let Layout be an implementation of the \ref layout_concept. +* +* @par Valid Expressions +* - <b>Layout::D</b> specifies the depth of a cube +* - <b>Layout::H</b> specifies the height of a cube +* - <b>Layout::W</b> specifies the height of a cube +* - <b>Layout::C</b> specifies the number of channels of each element in a cube +* - <b>Layout::W_c</b> specifies the number of scalars of each row in one image of a cube. +* - <b>Layout::H_w</b> specifies the number of elements in an image slice. +* - <b>Layout::H_w_c</b>_specifies the number of scalars in an image slice. +* - <b>Layout::D_h_w</b> specifies the number of elements in a cube. +* - <b>Layout::D_h_w_c</b> specifies the number of scalars in a cube. +* - <b>Layout::Strides</b> is a \ref layout_concept specifying the strides. +* @} +*/ + +/** +* @brief A Shape implementing \ref layout_concept describing the dimensions of a cube. +* @concept{layout_concept} +*/ +template <int kD_ = 1, int kH_ = 1, int kW_ = 1, int kC_ = 1> +struct Shape { + /// The depth of the cube. + static int const kD = kD_; + /// The height of the cube. + static int const kH = kH_; + /// The width of the cube. + static int const kW = kW_; + /// The number of scalars per element. + static int const kC = kC_; +}; + +/** +* @brief Compute derived counted of a \ref layout_concept based class +*/ +template <typename Shape> +struct ShapeCount { + /// The number of elements per row. + static int const kWc = Shape::kW * Shape::kC; + /// The number of pixels per image. + static int const kHw = Shape::kH * Shape::kW; + /// The number of elements per image. + static int const kHwc = Shape::kH * kWc; + /// The number of pixels per cube. + static int const kDhw = Shape::kD * kHw; + /// The number of elements in the 4D space. + static int const kDhwc = Shape::kD * kHwc; + /// The number of elements in the 4D space. + static int const kCount = kDhwc; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename A_, int kScale_> +struct ShapeScale { + typedef Shape<A_::kD * kScale_, A_::kH * kScale_, A_::kW * kScale_, A_::kC * kScale_> Shape; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename A_, typename B_> +struct ShapeAdd { + typedef Shape<A_::kD + B_::kD, A_::kH + B_::kH, A_::kW + B_::kW, A_::kC + B_::kC> Shape; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename A_, typename B_> +struct ShapeSub { + typedef Shape<A_::kD - B_::kD, A_::kH - B_::kH, A_::kW - B_::kW, A_::kC - B_::kC> Shape; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename A_, typename B_> +struct ShapeMul { + typedef Shape<A_::kD * B_::kD, A_::kH * B_::kH, A_::kW * B_::kW, A_::kC * B_::kC> Shape; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename A_, typename B_> +struct ShapeDiv { + typedef Shape<A_::kD / B_::kD, A_::kH / B_::kH, A_::kW / B_::kW, A_::kC / B_::kC> Shape; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename A_, typename B_> +struct ShapeMax { + typedef Shape<(A_::kD > B_::kD ? A_::kD : B_::kD), + (A_::kH > B_::kH ? A_::kH : B_::kH), + (A_::kW > B_::kW ? A_::kW : B_::kW), + (A_::kC > B_::kC ? A_::kC : B_::kC)> + Shape; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename A_, typename B_> +struct ShapeMin { + typedef Shape<(A_::kD < B_::kD ? A_::kD : B_::kD), + (A_::kH < B_::kH ? A_::kH : B_::kH), + (A_::kW < B_::kW ? A_::kW : B_::kW), + (A_::kC < B_::kC ? A_::kC : B_::kC)> + Shape; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Shape_, int kElementsPerAccess> +struct ShapeStrides { + typedef Shape<Shape_::kH * Shape_::kW * Shape_::kC, + Shape_::kW * Shape_::kC, + Shape_::kC, + kElementsPerAccess> + Shape; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief Compute the offset for the given coordinates in a cube +* @tparam A \ref layout_concept where each dimension of the cube specifies the corresponding stride. +*/ +template <typename Shape_> +struct ComputeOffsetFromShape { + static CUTLASS_DEVICE int get(int d, int h, int w, int c) { + // clang-format off + return d * Shape_::kH * Shape_::kW * Shape_::kC + + h * Shape_::kW * Shape_::kC + + w * Shape_::kC + + c; + // clang-format on + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief Compute the offset for the given coordinates in a cube with a depth of 1 +* @tparam kSh Elements in the H dimension +* @tparam kSw Elements in the W dimension +* @tparam kSc Separation between two elements in "elements" +*/ +template <int kSh_, int kSw_, int kSc_> +struct ComputeOffsetFromShape<Shape<1, kSh_, kSw_, kSc_> > { + static CUTLASS_DEVICE int get(int d, int h, int w, int c) { + return h * kSw_ * kSc_ + w * kSc_ + c; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief Compute the offset for the given coordinates in a cube with one channel and a depth of 1 +* @tparam kSh Elements in the H dimension +* @tparam kSw Elements in the W dimension +*/ +template <int kSh_, int kSw_> +struct ComputeOffsetFromShape<Shape<1, kSh_, kSw_, 1> > { + static CUTLASS_DEVICE int get(int d, int h, int w, int c) { return h * kSw_ + w; } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief Compute the offset for the given coordinates in a cube +* @tparam A \ref layout_concept where each dimension of the cube specifies the corresponding stride. +*/ +template <typename Strides_> +struct ComputeOffsetFromStrides { + static CUTLASS_DEVICE int get(int d, int h, int w, int c) { + return d * Strides_::kD + h * Strides_::kH + w * Strides_::kW + c * Strides_::kC; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief Compute the offset for the given coordinates in a cube with a depth of 1 +* @tparam S_h Stride in the H dimension in scalars +* @tparam S_w Stride in the W dimension in scalars +* @tparam S_c Stride between two scalars. +*/ +template <int S_h_, int S_w_, int S_c_> +struct ComputeOffsetFromStrides<Shape<1, S_h_, S_w_, S_c_> > { + static CUTLASS_DEVICE int get(int d, int h, int w, int c) { + return h * S_h_ + w * S_w_ + c * S_c_; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief Compute the offset for the given coordinates in a cube with one channel and a depth of 1 +* @tparam S_h Stride in the H dimension in scalars +* @tparam S_w Stride in the W dimension in scalars +*/ +template <int S_h_, int S_w_> +struct ComputeOffsetFromStrides<Shape<1, S_h_, S_w_, 1> > { + static CUTLASS_DEVICE int get(int d, int h, int w, int c) { return h * S_h_ + w * S_w_; } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief Decompose threadId.x into coordinate of a cube whose dimensions are specified by Threads_. +* Afterwards compute the offset of those coordinates using Strides_ +* @tparam Threads_ The dimension of the cube the threadIdx.x value is mapped on +* @tparam Strides_ The strides to use when compute the offsets based on the coordinates of the cube. +*/ +template <typename Threads_, typename Strides_> +struct ComputeThreadOffsetFromStrides { + static CUTLASS_DEVICE int get() { + // Decompose the thread index. + int c = threadIdx.x % Threads_::kC; + int w = threadIdx.x / Threads_::kC % Threads_::kW; + int h = threadIdx.x / Threads_::kC / Threads_::kW % Threads_::kH; + int d = threadIdx.x / Threads_::kC / Threads_::kW / Threads_::kH; + + // Compute the offset. + return d * Strides_::kD + h * Strides_::kH + w * Strides_::kW + c * Strides_::kC; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// +/** +*@brief Specialization for D=1 +*/ +template <int T_h_, int T_w_, int T_c_, int S_h_, int S_w_, int S_c_> +struct ComputeThreadOffsetFromStrides<Shape<1, T_h_, T_w_, T_c_>, Shape<1, S_h_, S_w_, S_c_> > { + static CUTLASS_DEVICE int get() { + // Decompose the thread index. + int c = threadIdx.x % T_c_; + int w = threadIdx.x / T_c_ % T_w_; + int h = threadIdx.x / T_c_ / T_w_ % T_h_; + + // Compute the offset. + return h * S_h_ + w * S_w_ + c * S_c_; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +*@brief Specialization for D=1 and C=1 +*/ +template <int T_h_, int T_w_, int S_h_, int S_w_> +struct ComputeThreadOffsetFromStrides<Shape<1, T_h_, T_w_, 1>, Shape<1, S_h_, S_w_, 1> > { + static CUTLASS_DEVICE int get() { + // Decompose the thread index. + int w = threadIdx.x % T_w_; + int h = threadIdx.x / T_w_; + + // Compute the offset. + return h * S_h_ + w * S_w_; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/tensor_ref.h b/cutlass-example/cutlass/tensor_ref.h new file mode 100644 index 0000000..8ef31e3 --- /dev/null +++ b/cutlass-example/cutlass/tensor_ref.h @@ -0,0 +1,151 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines a structure containing strides, bounds, and a pointer to tensor data. +*/ +#pragma once + +#include <typeinfo> + +#include <cutlass/coord.h> +#include <cutlass/cutlass.h> +#include <cutlass/vector.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Structure modeling a pointer and stride into a tensor +template <typename Storage_, int Rank_> +class TensorRef { + public: + /// Data type of individual access + typedef Storage_ Storage; + + /// Rank of tensor + static int const Rank = Rank_; + + private: + // + // Data members + // + + /// Pointer to storage element + Storage* ptr_; + + /// Stride information + Coord<Rank> stride_; + + public: + // + // Methods + // + + /// Default ctor + CUTLASS_HOST_DEVICE + TensorRef() : ptr_(nullptr) {} + + /// Constructs from a pointer, size, and stride + CUTLASS_HOST_DEVICE + TensorRef(Storage* ptr, Coord<Rank> stride) : ptr_(ptr), stride_(stride) {} + + /// Updates the pointer, stride, and location within a TensorRef + CUTLASS_HOST_DEVICE + void reset(Storage* ptr = nullptr, Coord<Rank> stride = Coord<Rank>(0)) { + ptr_ = ptr; + stride_ = stride; + } + + /// Conversion function + template <typename T> + TensorRef<T, Rank> convert() { + Coord<Rank> converted_stride; + for (int i = 0; i < Rank - 1; ++i) { + converted_stride[i] = stride_[i] * Extent<Storage>::kValue / Extent<T>::kValue; + } + converted_stride[Rank - 1] = stride_[Rank - 1]; + + return TensorRef<T, Rank>(reinterpret_cast<T*>(ptr_), converted_stride); + } + + /// Returns true if the TensorRef may be safely accessed + CUTLASS_HOST_DEVICE + bool good() const { return ptr_ != nullptr; } + + /// Returns the pointer to referenced data + CUTLASS_HOST_DEVICE + Storage* data() const { return ptr_; } + + /// Returns the stride of the tensor + CUTLASS_HOST_DEVICE + Coord<Rank> const& stride() const { return stride_; } + + /// Returns the stride of the tensor in the given dimension + CUTLASS_HOST_DEVICE + int const& stride(int dim) const { return stride_.at(dim); } + + /// Returns the maximum stride element as the 'leading dimension' + CUTLASS_HOST_DEVICE + int leading_dim() const { return __NV_STD_MAX(stride_[1], stride_[2]); } + + /// Computes the offset of an index from the origin of the tensor + CUTLASS_HOST_DEVICE + long long offset(Coord<Rank> const& coord) const { + return stride_.template dot<long long>(coord); + } + + /// Returns a reference to the element at a given Coord + CUTLASS_HOST_DEVICE + Storage& at(Coord<Rank> const& coord) const { return ptr_[offset(coord)]; } + + /// Element-wise accessor + Storage& operator[](Coord<Rank> const& coord) const { return at(coord); } + + /// Returns a reference to the element at a given Coord + CUTLASS_HOST_DEVICE + Storage& at(int idx) const { return ptr_[idx]; } + + /// Element-wise accessor + Storage& operator[](int idx) const { return at(idx); } + + /// Adds an offset to the pointer + CUTLASS_HOST_DEVICE + TensorRef& advance(Coord<Rank> const& b) { + ptr_ += offset(b); + return *this; + } + + /// Returns a TensorRef offset by a given amount + CUTLASS_HOST_DEVICE + TensorRef operator+(Coord<Rank> const& b) const { return TensorRef(ptr_ + offset(b), stride_); } + + /// Returns a TensorRef offset by a given amount + CUTLASS_HOST_DEVICE + TensorRef operator-(Coord<Rank> const& b) const { return TensorRef(ptr_ - offset(b), stride_); } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/tensor_view.h b/cutlass-example/cutlass/tensor_view.h new file mode 100644 index 0000000..89c6bd5 --- /dev/null +++ b/cutlass-example/cutlass/tensor_view.h @@ -0,0 +1,172 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines a structure containing strides and a pointer to tensor data. +*/ + +#pragma once + +#include <cmath> + +#include <cutlass/cutlass.h> +#include <cutlass/tensor_ref.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Host-side reference implementation of tensor operations +template <typename T> +class TensorView : public TensorRef<T, 4> { + public: + /// Reference and stride + typedef TensorRef<T, 4> Base; + + /// Reference and stride + typedef Base TensorRef_t; + + /// Reference to constant type + typedef TensorRef<T const, 4> ConstTensorRef_t; + + /// Rank of tensor + static int const Rank = TensorRef_t::Rank; + + /// Type used to compute the offset of an element to the base of a tensor + typedef int Offset_t; + + /// Coordinate into tensor + typedef Coord<Rank> Coord_t; + + private: + // + // Data members + // + + /// Pointer to pitch-linear memory + TensorRef_t ref_; + + /// Dimensions of coordinate (independent of stride) + Coord_t size_; + + public: + // + // Device and Host Methods + // + + /// Default constructor + CUTLASS_HOST_DEVICE + TensorView() {} + + /// Constructs a Tensor_view from a TensorRef and size + CUTLASS_HOST_DEVICE + TensorView(TensorRef_t const& _ref, Coord_t const& _size) : Base(_ref), size_(_size) {} + + /// Returns true if the Tensor_view is bound to some memory + CUTLASS_HOST_DEVICE + bool good() const { return ref().good(); } + + /// Returns a pointer to data + CUTLASS_HOST_DEVICE + T* data() const { return ref().data(); } + + /// Updates the reference and size of a Tensor_view object + CUTLASS_HOST_DEVICE + void reset(TensorRef_t const& _ref = TensorRef_t(0), Coord_t const& _size = Coord_t()) { + Base::operator=(_ref); + size_ = _size; + } + + /// Accesses the tensor reference pointing to data + CUTLASS_HOST_DEVICE + TensorRef_t& ref() { return *this; } + + /// + CUTLASS_HOST_DEVICE + ConstTensorRef_t const_ref() { return ConstTensorRef_t(data(), stride()); } + + /// Accesses the tensor reference pointing to data + CUTLASS_HOST_DEVICE + TensorRef_t const& ref() const { return *this; } + + /// Accesses the size + CUTLASS_HOST_DEVICE + Coord_t const& size() const { return size_; } + + /// Accesses the size + CUTLASS_HOST_DEVICE + int size(int dim) const { return size_.at(dim); } + + /// Accesses the stride + CUTLASS_HOST_DEVICE + Coord_t const& stride() const { return ref().stride(); } + + /// Accesses the stride + CUTLASS_HOST_DEVICE + int const& stride(int dim) const { return ref().stride(dim); } + + /// Assigns the Tensor_view + CUTLASS_HOST_DEVICE + TensorView& operator=(TensorView const& _tensor) { + Base::operator=(_tensor._ref); + size_ = _tensor.size_; + return *this; + } + + /// Returns the index of an element + CUTLASS_HOST_DEVICE + Offset_t offset(Coord_t const& coord) const { return ref().offset(coord); } + + /// Determines whether a location is within a tensor + CUTLASS_HOST_DEVICE + bool contains(Coord_t const& coord) const { + for (int dim = 0; dim < Rank; ++dim) { + if (coord.at(dim) >= size_.at(dim)) { + return false; + } + } + return true; + } + + /// Element-wise accessor + CUTLASS_HOST_DEVICE + T& at(Coord_t const& coord) const { return ref().at(coord); } + + /// Element-wise accessor + T& operator[](Coord<Rank> const& coord) const { return at(coord); } + + /// Element-wise accessor + CUTLASS_HOST_DEVICE + T& at(Offset_t idx) const { return ref().at(idx); } + + /// Returns a Tensor_view given location and size quantities + CUTLASS_HOST_DEVICE + TensorView<T> subview(Coord_t const& location, Coord_t size) const { + return TensorView<T>(ref() + location, size.clamp(size_ - location)); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/tile_iterator.h b/cutlass-example/cutlass/tile_iterator.h new file mode 100644 index 0000000..5d39c4f --- /dev/null +++ b/cutlass-example/cutlass/tile_iterator.h @@ -0,0 +1,899 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines the Tile Traits concept and iterators for loading and storing to tiles + efficiently. +*/ +#pragma once + +#include <cutlass/fragment.h> +#include <cutlass/load_store.h> +#include <cutlass/predicate_vector.h> +#include <cutlass/vector.h> + +namespace cutlass { + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/*!@defgroup tile_traits_concept Tile Traits Concept +@{ + +\ref tile_traits_concept is a type definining the shape of a tile and the distribution of accesses +by individual entities, either threads or other. + +@par Tile Traits Concept + Types satisfying \ref tile_traits_concept define the following members + - <b>Tile</b> - a type satisfying \ref layout_concept describing the dimensions of the tile + - <b>Delta</b> - a type satisfying \ref layout_concept describing the increments between accesses +along each dimension + - <b>Iterations</b> - a type satisfying \ref layout_concept describing the number of accesses +along each dimension + - <b>Offset</b> - the type of a <i>functor</i> computing the offset of each participating entity +as a Coord<4>. +@} +*/ + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Specifies dimension in which post-increment accesses advance +struct IteratorAdvance { + enum Kind { kD, kH, kW }; +}; + +/// Specifies whether iterator storage fragment consists of Scalar values or WMMA matrix +struct IteratorFragment { + enum Kind { kScalar, kWmmaMatrix }; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief A template defining \ref tile_traits_concept +* @concept{tile_traits_concept} +*/ +template <typename Tile_, + typename Delta_, + typename Iterations_, + typename ThreadOffset_, + int kAccessSize> +struct TileTraits { + /// Shape of the tile + typedef Tile_ Tile; + + /// Number of steps between accesses along each dimension + typedef Delta_ Delta; + + /// Number of accesses performed + typedef Iterations_ Iterations; + + /// Functor that returns the logical coordinate of each entity's initial offset in the tile + typedef ThreadOffset_ ThreadOffset; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Iterator for accessing a stripmined tile in memory +template <typename Traits_, + typename Scalar_, + IteratorAdvance::Kind Advance_ = IteratorAdvance::kH, + MemorySpace::Kind MemorySpace = MemorySpace::kGeneric, + typename Index_ = int, + typename FragmentElement_ = Scalar_, + IteratorFragment::Kind IteratorFragment_ = IteratorFragment::kScalar, + typename Skew_ = Shape<0, 0, 0, 0> > +struct TileIteratorBase { + /// concept TileTraits + typedef Traits_ Traits; + + /// Scalar element + typedef Scalar_ Scalar; + + /// Fragment element + typedef FragmentElement_ FragmentElement; + + /// Specifies dimension in which post-increment accesses advance. + static IteratorAdvance::Kind const kAdvance = Advance_; + + /// Specifies iterator storage fragment type (Scalar or WmmaMatrix) + static IteratorFragment::Kind const kIteratorFragment = IteratorFragment_; + + /// Source or destination memory space + static MemorySpace::Kind const kMemorySpace = MemorySpace; + + /// Index type + typedef Index_ Index; + + /// Skew quantity + typedef Skew_ Skew; + + /// Tile shape + typedef typename Traits::Tile Tile; + + /// Distance along each dimension + typedef typename Traits::Delta Delta; + + /// The strides in each dimension between different loads/stores. + typedef typename Traits::ImmediateOffsetStrides ImmediateOffsetStrides; + + /// Iterations + typedef typename Traits::Iterations Iterations; + + /// Thread offset + typedef typename Traits::ThreadOffset ThreadOffset; + + /// The number of scalars accessed per load/store. + static int const kAccessSize = Tile::kC; + + /// The elements loaded/store by one instruction. + typedef typename Vectorize<FragmentElement, kAccessSize>::Type AccessType; + + /// The size of storage needed per fragment + static int const kFragmentSize = + (kIteratorFragment == IteratorFragment::kWmmaMatrix ? 16 : sizeof(AccessType)); + /// The storage. + typedef Fragment<Scalar, ShapeCount<Tile>::kCount, kFragmentSize> Storage; + /// The fragment. + typedef Fragment<FragmentElement, ShapeCount<Iterations>::kCount * kAccessSize> Fragment; + /// The fragment iterator. + typedef FragmentIterator<Fragment, Iterations, AccessType> FragmentIterator; + /// The fragment const iterator. + typedef FragmentConstIterator<Fragment, Iterations, AccessType> FragmentConstIterator; + /// The shape of the fragment. + typedef typename FragmentIterator::FragmentShape FragmentShape; + + /// Default predicate mask type + typedef PredicateVector<ShapeCount<Iterations>::kCount> PredicateVector; + + // + // Params struct + // + + /// Parameters to the iterator + struct Params { + Index stride_d; + Index stride_h; + Index stride_w; + + Index inc_d; + Index inc_h; + Index inc_w; + + Index inc_advance; + + /// Initializes params + CUTLASS_HOST_DEVICE + int initialize(Index _stride_d, + Index _stride_h, + Index _stride_w, + Index _inc_d, + Index _inc_h, + Index _inc_w, + Index _inc_advance) { + stride_d = _stride_d; + stride_h = _stride_h; + stride_w = _stride_w; + + inc_d = _inc_d; + inc_h = _inc_h; + inc_w = _inc_w; + inc_advance = _inc_advance; + + return 0; + } + + CUTLASS_HOST_DEVICE + int initialize(Index _stride_d, Index _stride_h, Index _stride_w) { + stride_d = _stride_d; + stride_h = _stride_h; + stride_w = _stride_w; + + inc_w = stride_w * Delta::kW; + inc_h = stride_h * Delta::kH - stride_w * Delta::kW * (Iterations::kW - 1); + + if (kAdvance == IteratorAdvance::kH) { + // Advance in the H dimension. + inc_d = 0; + } else if (kAdvance == IteratorAdvance::kW) { + // Advance in the W dimension. + inc_d = stride_w * Tile::kW - stride_h * Tile::kH; + } else { + // Advance in the D dimension. + inc_d = stride_d; + } + + inc_advance = 0; + + return 0; + } + + CUTLASS_HOST_DEVICE int initialize() { + stride_d = 0; + stride_h = 0; + stride_w = 1; + + inc_d = inc_h = inc_w = inc_advance = 0; + + return 0; + } + }; + + /// Is the iterator valid? + CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { return true; } + + // + // Static function members + // + + /// Initializes a predicate vector + template <typename PredicateIterator> + CUTLASS_DEVICE static void initialize_predicates(PredicateIterator predicate_it, + Coord<3> const &bounds, + Coord<3> const &offset = make_Coord(0, 0, 0)) { + for (int d = 0; d < Iterations::kD; ++d) { + bool enable_d = (d * Delta::kD + offset[0] < bounds[0]); + for (int h = 0; h < Iterations::kH; ++h) { + bool enable_h = (h * Delta::kH + offset[1] < bounds[1]); + for (int w = 0; w < Iterations::kW; ++w) { + bool enable_w = (w * Tile::kC * Delta::kW + offset[2] < bounds[2]); + predicate_it.set(d, h, w, 0, enable_d && enable_h && enable_w); + } + } + } + } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/*!@defgroup tile_load_iterator_concept Tile Load Iterator Concept +@{ + +\ref tile_load_iterator_concept enables loading a tile from addressable memory into a fragment + +@par Tile Load Iterator Concept + Types satisfying \ref tile_load_iterator_concept define the following members + - <b>PredicateVector</b> - a \ref predicate_vector_concept with sufficient predicate storage for +each access implied by the tile traits + - <b>Fragment</b> - the destination fragment type satisfying \ref fragment_concept + - <b>initialize_predicates(pred_it, bounds, block_offset)</b> - function initializing a predicate +vector according to externally specified bounds + - <b>load_post_increment(fragment, pred_it)</b> - a method that loads a fragment and increments +the iterator to the next tile, guarded by a \ref predicate_iterator_concept + - <b>load_post_increment(fragment)</b> - a method that loads a fragment and increments the +iterator to the next tile + - <b>load(fragment, pred_it)</b> - a const method that loads a fragment, guarded by a \ref +predicate_iterator_concept + - <b>load(fragment)</b> - a method that loads a fragment + +@} +*/ + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief An iterator implementing \ref tile_load_iterator_concept for loading a tile from memory +* @concept{tile_load_iterator_concept} +*/ +template <typename Traits_, + typename Scalar_, + IteratorAdvance::Kind Advance_ = IteratorAdvance::kH, + MemorySpace::Kind MemorySpace = MemorySpace::kGeneric, + typename Index_ = int, + typename FragmentElement_ = Scalar_, + IteratorFragment::Kind IteratorFragment_ = IteratorFragment::kScalar, + typename Skew_ = Shape<0, 0, 0, 0> > +struct TileLoadIterator : public TileIteratorBase<Traits_, + Scalar_, + Advance_, + MemorySpace, + Index_, + FragmentElement_, + IteratorFragment_, + Skew_> { + /// Base class + typedef TileIteratorBase<Traits_, + Scalar_, + Advance_, + MemorySpace, + Index_, + FragmentElement_, + IteratorFragment_, + Skew_> + Base; + + /// concept TileTraits + typedef typename Base::Traits Traits; + + /// Scalar element + typedef typename Base::Scalar Scalar; + + /// Fragment element + typedef typename Base::FragmentElement FragmentElement; + + /// Specifies in which dimension post-increment accesses advance. + static IteratorAdvance::Kind const kAdvance = Base::kAdvance; + + /// Specifies type of iterator fragment storage (Salar or WmmaMatrix) + static IteratorFragment::Kind const kIteratorFragment = Base::kIteratorFragment; + + /// Source or destination memory space + static MemorySpace::Kind const kMemorySpace = Base::kMemorySpace; + + /// Index type + typedef typename Base::Index Index; + + /// Skew quantity + typedef typename Base::Skew Skew; + + /// Tile shape + typedef typename Base::Tile Tile; + + /// Delta + typedef typename Base::Delta Delta; + + /// Iterations + typedef typename Base::Iterations Iterations; + + /// ThreadOffset functor + typedef typename Base::ThreadOffset ThreadOffset; + + /// Fragment type + typedef typename Base::FragmentShape FragmentShape; + + /// Memory access type + typedef typename Base::AccessType AccessType; + + /// Fragment definition + typedef typename Base::Fragment Fragment; + + /// Fragment iterator definition + typedef typename Base::FragmentIterator FragmentIterator; + + /// Fragment const iterator definition + typedef typename Base::FragmentConstIterator FragmentConstIterator; + + /// Default predicate mask type + typedef typename Base::PredicateVector PredicateVector; + + /// Storage object that may be loaded from + typedef typename Base::Storage SharedStorage; + + /// IteratorBase parameters + typedef typename Base::Params BaseParams; + + /// Do we require a fence? + enum { kRequiresLoadFence = Tile::kD == 1 }; + + /// The pointer type + typedef Scalar const *Pointer; + + /// Parameters + struct Params : public BaseParams { + /// Pointer to memory + Scalar const *pointer; + + /// Initialize params to access storage object + CUTLASS_HOST_DEVICE + int initialize(SharedStorage const &storage) { + pointer = &storage[0]; + return 0; + } + + /// Initializes params to access a raw pointer + CUTLASS_HOST_DEVICE + int initialize(Scalar const *ptr, Index stride_d, Index stride_h, Index stride_w) { + Base::Params::initialize(stride_d, stride_h, stride_w); + pointer = ptr; + return 0; + } + + /// Initializes params + CUTLASS_HOST_DEVICE + int initialize(Scalar const *ptr, + Index _stride_d, + Index _stride_h, + Index _stride_w, + Index _inc_d, + Index _inc_h, + Index _inc_w, + Index _inc_advance) { + pointer = ptr; + Base::Params::initialize( + _stride_d, _stride_h, _stride_w, _inc_d, _inc_h, _inc_w, _inc_advance); + return 0; + } + + // Initializes params to default values + CUTLASS_HOST_DEVICE + int initialize() { return Base::Params::initialize(); } + }; + + // + // Data members + // + + /// Parameters structure + Params params; + + /// Offset of an individual lane from the start of the tile + Coord<4> thread_offset; + + /// Stage argument enables wrapping after some number of tiles have been loaded. + int stage; + + // + // Static member functions + // + + /// Initializes a predicate vector + template <typename PredicateIterator> + CUTLASS_HOST_DEVICE void initialize_predicates(PredicateIterator predicate_it, + Coord<3> const &bounds, + Coord<3> const &block_offset = make_Coord(0, + 0, + 0)) { + Base::initialize_predicates( + predicate_it, + bounds, + block_offset + make_Coord(0, thread_offset[1], thread_offset[2] * Tile::kC)); + } + + // + // Methods + // + + /// Default constructor + CUTLASS_HOST_DEVICE + TileLoadIterator() {} + + /// Constructs a tile load iterator + CUTLASS_HOST_DEVICE + TileLoadIterator(Params const &_params, + Coord<3> const &block_offset = make_Coord(0, 0, 0), + ThreadOffset thread_offset_func = ThreadOffset()) + : params(_params), stage(0) { + thread_offset = thread_offset_func(); + + Index block_offset_h = 0; + Index block_offset_w = 0; + if (kAdvance == IteratorAdvance::kH) { + block_offset_h = block_offset[1]; + block_offset_w = block_offset[2]; + } else { + block_offset_h = block_offset[2]; + block_offset_w = block_offset[1]; + } + + params.pointer += block_offset[0] * params.stride_d + + (block_offset_h + thread_offset[1]) * params.stride_h + + (block_offset_w + thread_offset[2] * Tile::kC) / Tile::kC * params.stride_w; + } + + /// Constructs a tile load iterator + CUTLASS_HOST_DEVICE + TileLoadIterator(Params const &, + SharedStorage &shared_storage, + Coord<3> const &block_offset = make_Coord(0, 0, 0), + ThreadOffset thread_offset_func = ThreadOffset()) + : stage(0) { + int const offset = thread_offset_func()[2]; + params.pointer = &shared_storage[offset]; + } + + /// Returns the current pointer + CUTLASS_HOST_DEVICE + Scalar const *data() const { return params.pointer; } + + /// The accessor. + CUTLASS_DEVICE void get(AccessType &value, int d, int h, int w, int c) const { + int const imm = + ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(d, h, w, c); + Load<Scalar, Base::kAccessSize, kMemorySpace>::load(value, params.pointer, imm); + } + + /// Increment in the D dimension + CUTLASS_HOST_DEVICE void inc_d() { params.pointer += params.inc_d; } + + /// Increment in the H dimension + CUTLASS_HOST_DEVICE void inc_h() { params.pointer += params.inc_h; } + + /// Increment in the W dimension + CUTLASS_HOST_DEVICE void inc_w() { params.pointer += params.inc_w; } + + /// Increment in the next dimension + CUTLASS_HOST_DEVICE void inc_advance() { params.pointer += params.inc_advance; } + + /// Increment the stage. + CUTLASS_DEVICE void inc_stage() { + if (Tile::kD > 1) { + int const kStageSize = Tile::kH * Tile::kW * Tile::kC; + if (stage == Tile::kD - 1) { + params.pointer -= (Tile::kD - 1) * kStageSize; + stage = 0; + } else { + params.pointer += kStageSize; + stage = stage + 1; + } + } + } + + public: + /// Loads a fragment and advances the iterator to the next tile. + template <typename Fragment, typename PredicateIterator> + CUTLASS_HOST_DEVICE void load_post_increment(Fragment &fragment, PredicateIterator pred_it) { + FragmentIterator frag_iterator(fragment); + + for (int d = 0; d < Iterations::kD; ++d) { + for (int h = 0; h < Iterations::kH; ++h) { + for (int w = 0; w < Iterations::kW; ++w, ++pred_it) { + if (*pred_it) { + Load<typename Fragment::Element, Tile::kC, kMemorySpace>::load( + reinterpret_cast<AccessType &>(frag_iterator.at(d, h, w, 0)), data(), 0); + } + + if (w < Iterations::kW - 1) { + inc_w(); + } + } + if (h < Iterations::kH - 1) { + inc_h(); + } + } + if (d < Iterations::kD - 1) { + inc_d(); + } + } + inc_advance(); + } + + /// Loads a fragment and advances the iterator to the next tile. + template <typename Fragment> + CUTLASS_HOST_DEVICE void load_post_increment(Fragment &fragment) { + typename PredicateVector::TrivialIterator pred_it; + load_post_increment(fragment, pred_it); + } + + /// Loads a fragment without advancing the iterator.. + template <typename Fragment, typename PredicateIterator> + CUTLASS_HOST_DEVICE void load(Fragment &fragment, PredicateIterator pred_it) const { + TileLoadIterator _load_it(*this); + _load_it.load_post_increment(fragment, pred_it); + } + + /// Loads a fragment without advancing the iterator.. + template <typename Fragment> + CUTLASS_HOST_DEVICE void load(Fragment &fragment) const { + typename PredicateVector::TrivialIterator pred_it; + load(fragment, pred_it); + } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/*!@defgroup tile_store_iterator_concept Tile Store Iterator Concept +@{ + +\ref tile_store_iterator_concept enables storing a tile to addressable memory + +@par Tile Store Iterator Concept + Types satisfying \ref tile_load_iterator_concept define the following members + - <b>PredicateVector</b> - a \ref predicate_vector_concept with sufficient predicate storage for +each access implied by the tile traits + - <b>Fragment</b> - the destination fragment type satisfying \ref fragment_concept + - <b>initialize_predicates(pred_it, bounds, block_offset)</b> - function initializing a predicate +vector according to externally specified bounds + - <b>store_post_increment(fragment, pred_it)</b> - a method that stores a fragment and increments +the iterator to the next tile, guarded by a \ref predicate_iterator_concept + - <b>store_post_increment(fragment)</b> - a method that stores a fragment and increments the +iterator to the next tile + - <b>store(fragment, pred_it)</b> - a const method that stores a fragment, guarded by a \ref +predicate_iterator_concept + - <b>store(fragment)</b> - a method that loads a fragment + +@} +*/ + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/** +* @brief An iterator implementing \ref tile_store_iterator_concept for storing a tile to memory +* @concept{tile_store_iterator_concept} +*/ +template <typename Traits_, + typename Scalar_, + IteratorAdvance::Kind Advance_ = IteratorAdvance::kH, + MemorySpace::Kind MemorySpace = MemorySpace::kGeneric, + typename Index_ = int, + typename FragmentElement_ = Scalar_, + IteratorFragment::Kind IteratorFragment_ = IteratorFragment::kScalar, + typename Skew_ = Shape<0, 0, 0, 0> > +struct TileStoreIterator : public TileIteratorBase<Traits_, + Scalar_, + Advance_, + MemorySpace, + Index_, + FragmentElement_, + IteratorFragment_, + Skew_> { + /// Base class + typedef TileIteratorBase<Traits_, + Scalar_, + Advance_, + MemorySpace, + Index_, + FragmentElement_, + IteratorFragment_, + Skew_> + Base; + + /// concept TileTraits + typedef typename Base::Traits Traits; + + /// Scalar element + typedef typename Base::Scalar Scalar; + + /// Fragment element + typedef typename Base::FragmentElement FragmentElement; + + /// Specifies in which dimension post-increment accesses advance. + static IteratorAdvance::Kind const kAdvance = Base::kAdvance; + + /// Specifies type of iterator fragment storage (Salar or WmmaMatrix) + static IteratorFragment::Kind const kIteratorFragment = Base::kIteratorFragment; + + /// Source or destination memory space + static MemorySpace::Kind const kMemorySpace = Base::kMemorySpace; + + /// Index type + typedef typename Base::Index Index; + + /// Skew quantity + typedef typename Base::Skew Skew; + + /// Tile shape + typedef typename Base::Tile Tile; + + /// Delta + typedef typename Base::Delta Delta; + + /// Iterations + typedef typename Base::Iterations Iterations; + + /// ThreadOffset functor + typedef typename Base::ThreadOffset ThreadOffset; + + /// Fragment type + typedef typename Base::FragmentShape FragmentShape; + + /// Memory access type + typedef typename Base::AccessType AccessType; + + /// Fragment definition + typedef typename Base::Fragment Fragment; + + /// Fragment iterator definition + typedef typename Base::FragmentIterator FragmentIterator; + + /// Fragment const iterator definition + typedef typename Base::FragmentConstIterator FragmentConstIterator; + + /// Default predicate mask type + typedef typename Base::PredicateVector PredicateVector; + + /// Storage object which may be stored to + typedef typename Base::Storage SharedStorage; + + /// IteratorBase parameters + typedef typename Base::Params BaseParams; + + /// Parameters + struct Params : public BaseParams { + /// Pointer to memory + Scalar *pointer; + + /// Initialize params to access storage object + CUTLASS_HOST_DEVICE + int initialize(SharedStorage &storage) { + pointer = &storage[0]; + return 0; + } + + /// Initializes params to access a raw pointer + CUTLASS_HOST_DEVICE + int initialize(Scalar *ptr, Index stride_d, Index stride_h, Index stride_w) { + Base::Params::initialize(stride_d, stride_h, stride_w); + pointer = ptr; + return 0; + } + + /// Initializes params + CUTLASS_HOST_DEVICE + int initialize(Scalar *ptr, + Index _stride_d, + Index _stride_h, + Index _stride_w, + Index _inc_d, + Index _inc_h, + Index _inc_w, + Index _inc_advance) { + pointer = ptr; + Base::Params::initialize( + _stride_d, _stride_h, _stride_w, _inc_d, _inc_h, _inc_w, _inc_advance); + return 0; + } + + /// Initializes params to default values + CUTLASS_HOST_DEVICE + int initialize() { return Base::Params::initialize(); } + }; + + // + // Data members + // + + /// Parameters structure + Params params; + + /// Offset of an individual lane from the start of the tile + Coord<4> thread_offset; + + /// The stage. + int stage; + + // + // Static member functions + // + + /// Initializes a predicate vector + template <typename PredicateIterator> + CUTLASS_HOST_DEVICE void initialize_predicates(PredicateIterator predicate_it, + Coord<3> const &bounds, + Coord<3> const &block_offset = make_Coord(0, + 0, + 0)) { + Base::initialize_predicates( + predicate_it, + bounds, + block_offset + make_Coord(0, thread_offset[1], thread_offset[2] * Tile::kC)); + } + + // + // Methods + // + + /// Default constructor + CUTLASS_HOST_DEVICE + TileStoreIterator() {} + + /// Constructs a tile store iterator + CUTLASS_HOST_DEVICE + TileStoreIterator(Params const &_params, + Coord<3> const &block_offset = make_Coord(0, 0, 0), + ThreadOffset thread_offset_func = ThreadOffset()) + : params(_params), stage(0) { + thread_offset = thread_offset_func(); + + params.pointer += block_offset[0] * params.stride_d + + (block_offset[1] + thread_offset[1]) * params.stride_h + + (block_offset[2] + thread_offset[2] * Tile::kC) / Tile::kC * params.stride_w; + } + + /// Constructs a tile store iterator + CUTLASS_HOST_DEVICE + TileStoreIterator(Params const &, + SharedStorage &shared_storage, + Coord<3> const &block_offset = make_Coord(0, 0, 0), + ThreadOffset thread_offset_func = ThreadOffset()) + : stage(0) { + int const offset = thread_offset_func()[2]; + params.pointer = &shared_storage[offset]; + } + + /// Returns the current pointer + CUTLASS_HOST_DEVICE + Scalar *data() const { return params.pointer; } + + /// Increment in the D dimension + CUTLASS_HOST_DEVICE void inc_d() { params.pointer += params.inc_d; } + + /// Increment in the H dimension + CUTLASS_HOST_DEVICE void inc_h() { params.pointer += params.inc_h; } + + /// Increment in the W dimension + CUTLASS_HOST_DEVICE void inc_w() { params.pointer += params.inc_w; } + + /// Increment in the next dimension + CUTLASS_HOST_DEVICE void inc_advance() {} + + /// Increment the stage. + CUTLASS_DEVICE void inc_stage() { + if (Tile::kD > 1) { + int const kStageSize = Tile::kH * Tile::kW * Tile::kC; + if (stage == Tile::kD - 1) { + params.pointer -= (Tile::kD - 1) * kStageSize; + stage = 0; + } else { + params.pointer += kStageSize; + stage = stage + 1; + } + } + } + + /// The accessor. + CUTLASS_DEVICE void set(AccessType const &value, int d, int h, int w, int c) { + int const imm = + ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(d, h, w, c); + Store<Scalar, Base::kAccessSize, kMemorySpace>::store(value, params.pointer, imm); + } + + public: + /// Stores a fragment and advances to the next tile. + template <typename Fragment, typename PredicateIterator> + CUTLASS_HOST_DEVICE void store_post_increment(Fragment &fragment, PredicateIterator pred_it) { + FragmentIterator frag_iterator(fragment); + + for (int d = 0; d < Iterations::kD; ++d) { + for (int h = 0; h < Iterations::kH; ++h) { + for (int w = 0; w < Iterations::kW; ++w, ++pred_it) { + if (*pred_it) { + Store<typename Fragment::Element, Tile::kC, kMemorySpace>::store( + reinterpret_cast<AccessType &>(frag_iterator.at(d, h, w, 0)), data(), 0); + } + if (w < Iterations::kW - 1) { + inc_w(); + } + } + if (h < Iterations::kH - 1) { + inc_h(); + } + } + if (d < Iterations::kD - 1) { + inc_d(); + } + } + inc_advance(); + } + + /// Stores a fragment and advances to the next tile. + template <typename Fragment> + CUTLASS_HOST_DEVICE void store_post_increment(Fragment &fragment) { + typename PredicateVector::TrivialIterator pred_it; + store_post_increment(fragment, pred_it); + } + + /// Stores a fragment without advancing the iterator. + template <typename Fragment, typename PredicateIterator> + CUTLASS_HOST_DEVICE void store(Fragment &fragment, PredicateIterator pred_it) const { + TileStoreIterator _store_it(*this); + _store_it.store_post_increment(fragment, pred_it); + } + + /// Stores a fragment without advancing the iterator. + template <typename Fragment> + CUTLASS_HOST_DEVICE void store(Fragment &fragment) const { + typename PredicateVector::TrivialIterator pred_it; + store(fragment, pred_it); + } +}; +} diff --git a/cutlass-example/cutlass/tile_traits_standard.h b/cutlass-example/cutlass/tile_traits_standard.h new file mode 100644 index 0000000..14ecd01 --- /dev/null +++ b/cutlass-example/cutlass/tile_traits_standard.h @@ -0,0 +1,238 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines tile traits for several tile partitioning arrangements of threads expected to + achieve efficient streaming performance. +*/ +#pragma once + +#include <cutlass/tile_iterator.h> + +namespace cutlass { + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Basic thread offset function computed from a thread shape +template <typename ThreadShape> +struct TiledThreadOffset { + /// Computes the logical coordinate from thread shape + CUTLASS_HOST_DEVICE + Coord<4> operator()() const { + Coord<4> thread_offset; + + int index = threadIdx.x; + + thread_offset[3] = (index % ThreadShape::kC); + index = (index / ThreadShape::kC); + + thread_offset[2] = (index % ThreadShape::kW); + index = (index / ThreadShape::kW); + + thread_offset[1] = (index % ThreadShape::kH); + index = (index / ThreadShape::kH); + + thread_offset[0] = index; + + return thread_offset; + } +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Tiling in which the number of threads is greater than the +/// contiguous dimension of the tile. +template <typename Tile_, int Threads> +struct TileTraitsStrideMajor { + /// Shape of tile + typedef Tile_ Tile; + + /// Number of participating threads + static int const kThreads = Threads; + + // Static assertions + static_assert(!(ShapeCount<Tile>::kDhw % kThreads), + "Tiling undefined if elements not divisible by threads."); + + static_assert(Tile::kW <= kThreads, + "This specialization assumes there are more threads than the contiguous dimension " + "of the tile."); + + /// Shape of threads + typedef Shape<1, kThreads / Tile::kW, Tile::kW, 1> ThreadShape; + + /// Delta along each dimension + typedef Shape<1, ThreadShape::kH, 1, 1> Delta; + + /// Number of iterations + typedef Shape<1, Tile::kH / ThreadShape::kH, 1, 1> Iterations; + + /// Computes the initial offset + typedef TiledThreadOffset<ThreadShape> ThreadOffset; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Tiling in which the number of threads is fewer than the tile size +/// in the contiguous dimension. +template <typename Tile_, int Threads> +struct TileTraitsContiguousMajor { + /// Shape of tile + typedef Tile_ Tile; + + /// Number of participating threads + static int const kThreads = Threads; + + // Static assertions + static_assert(Tile::kW >= kThreads, + "This specialization assumes there are more threads than the contiguous dimension " + "of the tile."); + + static_assert(!(ShapeCount<Tile>::kDhw % kThreads), + "Tiling undefined if elements not divisible by threads."); + + static_assert(!(Tile::kW % kThreads), + "The contiguous size of the tile must be divisible by the number of threads."); + + /// Thread shape + typedef Shape<1, 1, kThreads> ThreadShape; + + /// Delta between each thread's access + typedef Shape<1, 1, kThreads> Delta; + + /// Number of iterations + typedef Shape<1, Tile::kH, Tile::kW / kThreads> Iterations; + + /// Computes the initial offset + typedef TiledThreadOffset<ThreadShape> ThreadOffset; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Tiling in which warps rake across the contiguous dimension +template <typename Tile_, int Threads> +struct TileTraitsWarpRake { + /// Shape of tile + typedef Tile_ Tile; + + /// Number of participating threads + static int const kThreads = Threads; + + /// Hard-coded warp size + static int const kWarpSize = 32; + + /// Number of participating warps + static int const kWarpCount = kThreads / kWarpSize; + + // Static assertions + static_assert(!(ShapeCount<Tile>::kDhw % kThreads), + "Tiling undefined if elements not divisible by threads."); + + static_assert(!(kThreads % kWarpSize), "Number of threads must be divisible by the warp size."); + + static_assert(!(Tile::kW % kWarpSize), "Contiguous dimension must be divisible by the warp size"); + + /// Warps strip-mined across strided dimension + static int const kWarpsStrided = __NV_STD_MIN(kWarpCount, Tile::kH); + + /// Warps stripmined contiguous dimension + static int const kWarpsContiguous = kWarpCount / kWarpsStrided; + + /// Arrangement of threads + typedef Shape<1, kWarpsStrided, kWarpsContiguous * kWarpSize> ThreadShape; + + /// The same warp rakes along the contiguous dimension + typedef Shape<1, kWarpsStrided, kWarpSize> Delta; + + /// Number of iterations + typedef Shape<1, Tile::kH / Delta::kH, Tile::kW / ThreadShape::kW> Iterations; + + /// Computes the thread offset in (H, W) based on thread ID + struct ThreadOffset { + /// Basic thread offset function computed from a thread shape + CUTLASS_HOST_DEVICE + Coord<4> operator()() const { + int tid = threadIdx.x; + int warp = (tid / kWarpSize); + int lane = (tid % kWarpSize); + + static int const kWarpSpanContiguous = kWarpSize * Iterations::kW; + + int warp_w = (warp % kWarpsContiguous); + int warp_h = (warp / kWarpsContiguous); + + return make_Coord(0, warp_h, lane + kWarpSpanContiguous * warp_w, 0); + } + }; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Chooses 'best' shape to enable warp raking along contiguous dimension if possible. +template <typename Tile_, int Threads> +struct TileTraitsStandard { + /// Shape of tile + typedef Tile_ Tile; + + /// Number of participating threads + static int const kThreads = Threads; + + /// Hard-coded warp size + static int const kWarpSize = 32; + + /// Number of participating warps + static int const kWarpCount = kThreads / kWarpSize; + + // Static assertions + static_assert(!(ShapeCount<Tile>::kDhw % kThreads), + "Tiling undefined if elements not divisible by threads."); + + /// Choose the stride-major contiguous tiling if the contiguous dimension is + /// smaller than the warp size. Otherwise, if it is divisible by the warp size, + /// choose the warp rake arrangement. + typedef typename platform::conditional < + Tile::kW<kWarpSize, + TileTraitsStrideMajor<Tile, Threads>, + typename platform::conditional<!(Tile::kW % kWarpSize), + TileTraitsWarpRake<Tile, Threads>, + TileTraitsContiguousMajor<Tile, Threads> >::type>:: + type Traits; + + /// Delta between accesses + typedef typename Traits::Delta Delta; + + /// Delta between each thread's access + /// TODO MTA this is wrong for sure, but Delta is used for stride computation at the moment + typedef Delta ImmediateOffsetStrides; + + /// Number of accesses + typedef typename Traits::Iterations Iterations; + + /// Thread offset functor + typedef typename Traits::ThreadOffset ThreadOffset; +}; + +/////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/util/cutlass_math.h b/cutlass-example/cutlass/util/cutlass_math.h new file mode 100644 index 0000000..0ecdc43 --- /dev/null +++ b/cutlass-example/cutlass/util/cutlass_math.h @@ -0,0 +1,131 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +#pragma once + +/** + * \file + * \brief Math utilities + */ + +#include <cutlass/util/platform.h> + +namespace cutlass { + +/****************************************************************************** + * Static math utilities + ******************************************************************************/ + +/** + * Statically determine if N is a power-of-two + */ +template <int N> +struct is_pow2 : platform::integral_constant<bool, (N & (N - 1)) == 0> {}; + +/** + * Statically determine log2(N), rounded down + */ +template <int N, int CurrentVal = N, int Count = 0> +struct log2_down { + /// Static logarithm value + enum { value = log2_down<N, (CurrentVal >> 1), Count + 1>::value }; +}; + +// Base case +template <int N, int Count> +struct log2_down<N, 1, Count> { + enum { value = Count }; +}; + +/** + * Statically determine log2(N), rounded up + */ +template <int N, int CurrentVal = N, int Count = 0> +struct log2_up { + /// Static logarithm value + enum { value = log2_up<N, (CurrentVal >> 1), Count + 1>::value }; +}; + +// Base case +template <int N, int Count> +struct log2_up<N, 1, Count> { + enum { value = ((1 << Count) < N) ? Count + 1 : Count }; +}; + +/** + * Statically estimate sqrt(N) to the nearest power-of-two + */ +template <int N> +struct sqrt_est { + enum { value = 1 << (log2_up<N>::value / 2) }; +}; + +/** + * For performing a constant-division with a compile-time assertion that the + * Divisor evenly-divides the Dividend. + */ +template <int Dividend, int Divisor> +struct divide_assert { + enum { value = Dividend / Divisor }; + + static_assert((Dividend % Divisor == 0), "Not an even multiple"); +}; + +/****************************************************************************** + * Rounding + ******************************************************************************/ + +/** + * Round dividend up to the nearest multiple of divisor + */ +template <typename dividend_t, typename divisor_t> +CUTLASS_HOST_DEVICE dividend_t round_nearest(dividend_t dividend, divisor_t divisor) { + return ((dividend + divisor - 1) / divisor) * divisor; +} + +/** + * Greatest common divisor + */ +template <typename value_t> +CUTLASS_HOST_DEVICE value_t gcd(value_t a, value_t b) { + for (;;) { + if (a == 0) return b; + b %= a; + if (b == 0) return a; + a %= b; + } +} + +/** + * Least common multiple + */ +template <typename value_t> +CUTLASS_HOST_DEVICE value_t lcm(value_t a, value_t b) { + value_t temp = gcd(a, b); + + return temp ? (a / temp * b) : 0; +} + +} // namespace cutlass diff --git a/cutlass-example/cutlass/util/debug.h b/cutlass-example/cutlass/util/debug.h new file mode 100644 index 0000000..6055e3f --- /dev/null +++ b/cutlass-example/cutlass/util/debug.h @@ -0,0 +1,122 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +#pragma once + +/** + * \file + * \brief Debugging and logging functionality + */ + +#include <stdio.h> + +namespace cutlass { + +/****************************************************************************** + * Debug and logging macros + ******************************************************************************/ + +/** + * Formats and prints the given message to stdout + */ +#if !defined(CUDA_LOG) +#if !defined(__CUDA_ARCH__) +#define CUDA_LOG(format, ...) printf(format, __VA_ARGS__) +#else +#define CUDA_LOG(format, ...) \ + printf("[block (%d,%d,%d), thread (%d,%d,%d)]: " format, \ + blockIdx.x, \ + blockIdx.y, \ + blockIdx.z, \ + threadIdx.x, \ + threadIdx.y, \ + threadIdx.z, \ + __VA_ARGS__); +#endif +#endif + +/** + * Formats and prints the given message to stdout only if DEBUG is defined + */ +#if !defined(CUDA_LOG_DEBUG) +#ifdef DEBUG +#define CUDA_LOG_DEBUG(format, ...) CUDA_LOG(format, __VA_ARGS__) +#else +#define CUDA_LOG_DEBUG(format, ...) +#endif +#endif + +/** + * \brief The corresponding error message is printed to \p stderr (or \p stdout in device code) + * along with the supplied source context. + * + * \return The CUDA error. + */ +__host__ CUTLASS_DEVICE cudaError_t cuda_perror_impl(cudaError_t error, + const char* filename, + int line) { + (void)filename; + (void)line; + if (error) { +#if !defined(__CUDA_ARCH__) + fprintf( + stderr, "CUDA error %d [%s, %d]: %s\n", error, filename, line, cudaGetErrorString(error)); + fflush(stderr); +#else + printf("CUDA error %d [%s, %d]\n", error, filename, line); +#endif + } + return error; +} + +/** + * \brief Perror macro + */ +#ifndef CUDA_PERROR +#define CUDA_PERROR(e) cuda_perror_impl((cudaError_t)(e), __FILE__, __LINE__) +#endif + +/** + * \brief Perror macro with exit + */ +#ifndef CUDA_PERROR_EXIT +#define CUDA_PERROR_EXIT(e) \ + if (cuda_perror_impl((cudaError_t)(e), __FILE__, __LINE__)) { \ + exit(1); \ + } +#endif + +/** + * \brief Perror macro only if DEBUG is defined + */ +#ifndef CUDA_PERROR_DEBUG +#ifdef DEBUG +#define CUDA_PERROR_DEBUG(e) CUDA_PERROR(e) +#else +#define CUDA_PERROR_DEBUG(e) (e) +#endif +#endif + +} // namespace cutlass diff --git a/cutlass-example/cutlass/util/platform.h b/cutlass-example/cutlass/util/platform.h new file mode 100644 index 0000000..2a44c10 --- /dev/null +++ b/cutlass-example/cutlass/util/platform.h @@ -0,0 +1,801 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +#pragma once + +/** + * \file + * \brief C++ features that may be otherwise unimplemented for CUDA device functions. + * + * This file has three components: + * + * (1) Macros: + * - Empty macro defines for C++ keywords not supported by the current + * version of C++. These simply allow compilation to proceed (but do + * not provide the added semantics). + * - \p noexcept + * - \p constexpr + * - \p nullptr + * - \p static_assert + * + * - Macro functions that we need in constant expressions because the + * C++ equivalents require constexpr compiler support. These are + * prefixed with \p __NV_STD_* + * - \p __NV_STD_MAX + * - \p __NV_STD_MIN + * + * (2) Re-implementations of STL functions and types: + * - C++ features that need the \p __device__ annotation. These are + * placed into the \p platform namespace. + * - \p plus + * - \p less + * - \p greater + * - \p min + * - \p max + * - \p methods on std::pair (==, !=, <, <=, >, >=, and make_pair()) + * + * (3) Stop-gap implementations of unsupported STL functions and types: + * - STL functions and types defined by C++ 11/14/17/etc. that are not + * provided by the current version of C++. These are placed into the + * \p platform namespace + * - \p integral_constant + * - \p nullptr_t + * - \p true_type + * - \p false_type + * - \p bool_constant + * - \p enable_if + * - \p conditional + * - \p is_same + * - \p is_base_of + * - \p remove_const + * - \p remove_volatile + * - \p remove_cv + * - \p is_volatile + * - \p is_pointer + * - \p is_void + * - \p is_integral + * - \p is_floating_point + * - \p is_arithmetic + * - \p is_fundamental + * - \p is_trivially_copyable + * - \p alignment_of + * - \p aligned_storage + * + * (4) Functions and types that are STL-like (but aren't in the STL): + * - \p TODO: min and max functors? + * + * The idea is that, as we drop support for older compilers, we can simply #define + * the \p __NV_STD_XYZ macros and \p platform namespace to alias their C++ + * counterparts (or trivially find-and-replace their occurrences in code text). + */ + +//----------------------------------------------------------------------------- +// Dependencies +//----------------------------------------------------------------------------- + +#include <stdint.h> + +#if !defined(__CUDACC_RTC__) +//----------------------------------------------------------------------------- +// Include STL files that platform provides functionality for +//----------------------------------------------------------------------------- + +#include <algorithm> // Minimum/maximum operations +#include <cstddef> // nullptr_t +#include <functional> // Arithmetic operations +#include <utility> // For methods on std::pair +#if (!defined(_MSC_VER) && (__cplusplus >= 201103L)) || (defined(_MSC_VER) && (_MS_VER >= 1500)) +#include <type_traits> // For integral constants, conditional metaprogramming, and type traits +#endif + +#include <cutlass/cutlass.h> + +#endif +/****************************************************************************** + * Macros + ******************************************************************************/ +//----------------------------------------------------------------------------- +// Keywords +//----------------------------------------------------------------------------- + +/// noexcept, constexpr +#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1900)) +#ifndef noexcept +#define noexcept +#endif +#ifndef constexpr +#define constexpr +#endif +#endif + +/// nullptr +#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1310)) +#ifndef nullptr +#define nullptr 0 +#endif +#endif + +/// static_assert +#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1600)) +#ifndef static_assert +#define __platform_cat_(a, b) a##b +#define __platform_cat(a, b) __platform_cat_(a, b) +#define static_assert(__e, __m) typedef int __platform_cat(AsSeRt, __LINE__)[(__e) ? 1 : -1] +#endif +#endif + +//----------------------------------------------------------------------------- +// Functions +//----------------------------------------------------------------------------- + +/// Select maximum(a, b) +#ifndef __NV_STD_MAX +#define __NV_STD_MAX(a, b) (((b) > (a)) ? (b) : (a)) +#endif + +/// Select minimum(a, b) +#ifndef __NV_STD_MIN +#define __NV_STD_MIN(a, b) (((b) < (a)) ? (b) : (a)) +#endif + +/****************************************************************************** + * Re-implementations + ******************************************************************************/ +namespace cutlass { +namespace platform { + +//----------------------------------------------------------------------------- +// Arithmetic operations, comparisons <functional> +//----------------------------------------------------------------------------- + +/// platform::plus +template <typename T> +struct plus { + CUTLASS_HOST_DEVICE constexpr T operator()(const T& lhs, const T& rhs) const { return lhs + rhs; } +}; + +/// std::less +template <typename T> +struct less { + CUTLASS_HOST_DEVICE constexpr bool operator()(const T& lhs, const T& rhs) const { + return lhs < rhs; + } +}; + +/// std::greater +template <typename T> +struct greater { + CUTLASS_HOST_DEVICE constexpr bool operator()(const T& lhs, const T& rhs) const { + return lhs > rhs; + } +}; + +//----------------------------------------------------------------------------- +// Minimum/maximum operations <algorithm> +//----------------------------------------------------------------------------- + +/// std::min +template <typename T> +CUTLASS_HOST_DEVICE constexpr const T& min(const T& a, const T& b) { + return (b < a) ? b : a; +} + +/// std::max +template <typename T> +CUTLASS_HOST_DEVICE constexpr const T& max(const T& a, const T& b) { + return (a < b) ? b : a; +} + +#if !defined(__CUDACC_RTC__) +//----------------------------------------------------------------------------- +// Methods on std::pair +//----------------------------------------------------------------------------- + +using std::pair; + +template <class T1, class T2> +CUTLASS_HOST_DEVICE constexpr bool operator==(const pair<T1, T2>& lhs, const pair<T1, T2>& rhs) { + return (lhs.first == rhs.first) && (lhs.second == rhs.second); +} + +template <class T1, class T2> +CUTLASS_HOST_DEVICE constexpr bool operator!=(const pair<T1, T2>& lhs, const pair<T1, T2>& rhs) { + return (lhs.first != rhs.first) && (lhs.second != rhs.second); +} + +template <class T1, class T2> +CUTLASS_HOST_DEVICE constexpr bool operator<(const pair<T1, T2>& lhs, const pair<T1, T2>& rhs) { + return (lhs.first < rhs.first) ? true : (rhs.first < lhs.first) ? false + : (lhs.second < rhs.second); +} + +template <class T1, class T2> +CUTLASS_HOST_DEVICE constexpr bool operator<=(const pair<T1, T2>& lhs, const pair<T1, T2>& rhs) { + return !(rhs < lhs); +} + +template <class T1, class T2> +CUTLASS_HOST_DEVICE constexpr bool operator>(const pair<T1, T2>& lhs, const pair<T1, T2>& rhs) { + return (rhs < lhs); +} + +template <class T1, class T2> +CUTLASS_HOST_DEVICE constexpr bool operator>=(const pair<T1, T2>& lhs, const pair<T1, T2>& rhs) { + return !(lhs < rhs); +} + +template <class T1, class T2> +CUTLASS_HOST_DEVICE std::pair<T1, T2> make_pair(T1 t, T2 u) { + std::pair<T1, T2> retval; + retval.first = t; + retval.second = u; + return retval; +} +#endif + +} // namespace platform + +/****************************************************************************** + * Implementations of C++ 11/14/17/... STL features + ******************************************************************************/ + +namespace platform { + +//----------------------------------------------------------------------------- +// Integral constant helper types <type_traits> +//----------------------------------------------------------------------------- + +#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1500)) + +/// std::integral_constant +template <typename value_t, value_t V> +struct integral_constant; + +/// std::integral_constant +template <typename value_t, value_t V> +struct integral_constant { + static const value_t value = V; + + typedef value_t value_type; + typedef integral_constant<value_t, V> type; + + CUTLASS_HOST_DEVICE operator value_type() const { return value; } + + CUTLASS_HOST_DEVICE const value_type operator()() const { return value; } +}; + +#else + +using std::integral_constant; +using std::pair; + +#endif + +/// The type used as a compile-time boolean with true value. +typedef integral_constant<bool, true> true_type; + +/// The type used as a compile-time boolean with false value. +typedef integral_constant<bool, false> false_type; + +#if (!defined(_MSC_VER) && (__cplusplus <= 201402L)) || (defined(_MSC_VER) && (_MSC_VER < 1900)) + +/// std::bool_constant +template <bool V> +struct bool_constant : platform::integral_constant<bool, V> {}; + +#else + +using std::bool_constant; + +#endif + +#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1700)) + +/// std::nullptr_t +struct nullptr_t {}; + +#else + +using std::nullptr_t; + +#endif + +//----------------------------------------------------------------------------- +// Conditional metaprogramming <type_traits> +//----------------------------------------------------------------------------- + +#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1600)) + +/// std::enable_if (true specialization) +template <bool C, typename T = void> +struct enable_if { + typedef T type; +}; + +/// std::enable_if (false specialization) +template <typename T> +struct enable_if<false, T> {}; + +/// std::conditional (true specialization) +template <bool B, class T, class F> +struct conditional { + typedef T type; +}; + +/// std::conditional (false specialization) +template <class T, class F> +struct conditional<false, T, F> { + typedef F type; +}; + +#else + +using std::enable_if; +using std::conditional; + +#endif + +//----------------------------------------------------------------------------- +// Const/volatility specifiers <type_traits> +//----------------------------------------------------------------------------- + +#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1500)) + +/// std::remove_const (non-const specialization) +template <typename T> +struct remove_const { + typedef T type; +}; + +/// std::remove_const (const specialization) +template <typename T> +struct remove_const<const T> { + typedef T type; +}; + +/// std::remove_volatile (non-volatile specialization) +template <typename T> +struct remove_volatile { + typedef T type; +}; + +/// std::remove_volatile (volatile specialization) +template <typename T> +struct remove_volatile<volatile T> { + typedef T type; +}; + +/// std::remove_cv +template <typename T> +struct remove_cv { + typedef typename remove_volatile<typename remove_const<T>::type>::type type; +}; + +#else + +using std::remove_const; +using std::remove_volatile; +using std::remove_cv; + +#endif + +//----------------------------------------------------------------------------- +// Type relationships <type_traits> +//----------------------------------------------------------------------------- + +#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1500)) + +/// std::is_same (false specialization) +template <typename A, typename B> +struct is_same : false_type {}; + +/// std::is_same (true specialization) +template <typename A> +struct is_same<A, A> : true_type {}; + +/// Helper for std::is_base_of +template <typename BaseT, typename DerivedT> +struct is_base_of_helper { + typedef char (&yes)[1]; + typedef char (&no)[2]; + + template <typename B, typename D> + struct dummy { + CUTLASS_HOST_DEVICE operator B*() const; + CUTLASS_HOST_DEVICE operator D*(); + }; + + template <typename T> + CUTLASS_HOST_DEVICE static yes check(DerivedT*, T); + + CUTLASS_HOST_DEVICE static no check(BaseT*, int); + + static const bool value = sizeof(check(dummy<BaseT, DerivedT>(), int())) == sizeof(yes); +}; + +/// std::is_base_of +template <typename BaseT, typename DerivedT> +struct is_base_of + : integral_constant<bool, + (is_base_of_helper<typename remove_cv<BaseT>::type, + typename remove_cv<DerivedT>::type>::value) || + (is_same<typename remove_cv<BaseT>::type, + typename remove_cv<DerivedT>::type>::value)> {}; + +#else + +using std::is_same; +using std::is_base_of; + +#endif + +//----------------------------------------------------------------------------- +// Type properties <type_traits> +//----------------------------------------------------------------------------- + +#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1500)) + +/// std::is_volatile +template <typename T> +struct is_volatile : false_type {}; +template <typename T> +struct is_volatile<volatile T> : true_type {}; + +/// Helper for std::is_pointer (false specialization) +template <typename T> +struct is_pointer_helper : false_type {}; + +/// Helper for std::is_pointer (true specialization) +template <typename T> +struct is_pointer_helper<T*> : true_type {}; + +/// std::is_pointer +template <typename T> +struct is_pointer : is_pointer_helper<typename remove_cv<T>::type> {}; + +/// std::is_void +template <typename T> +struct is_void : is_same<void, typename remove_cv<T>::type> {}; + +/// std::is_integral +template <typename T> +struct is_integral : false_type {}; +template <> +struct is_integral<char> : true_type {}; +template <> +struct is_integral<signed char> : true_type {}; +template <> +struct is_integral<unsigned char> : true_type {}; +template <> +struct is_integral<short> : true_type {}; +template <> +struct is_integral<unsigned short> : true_type {}; +template <> +struct is_integral<int> : true_type {}; +template <> +struct is_integral<unsigned int> : true_type {}; +template <> +struct is_integral<long> : true_type {}; +template <> +struct is_integral<unsigned long> : true_type {}; +template <> +struct is_integral<long long> : true_type {}; +template <> +struct is_integral<unsigned long long> : true_type {}; +template <typename T> +struct is_integral<volatile T> : is_integral<T> {}; +template <typename T> +struct is_integral<const T> : is_integral<T> {}; +template <typename T> +struct is_integral<const volatile T> : is_integral<T> {}; + +/// std::is_floating_point +template <typename T> +struct is_floating_point + : integral_constant<bool, + (is_same<float, typename remove_cv<T>::type>::value || + is_same<double, typename remove_cv<T>::type>::value)> {}; + +/// std::is_arithmetic +template <typename T> +struct is_arithmetic + : integral_constant<bool, (is_integral<T>::value || is_floating_point<T>::value)> {}; + +/// std::is_fundamental +template <typename T> +struct is_fundamental + : integral_constant<bool, + (is_arithmetic<T>::value || is_void<T>::value || + is_same<nullptr_t, typename remove_cv<T>::type>::value)> {}; + +#else + +using std::is_volatile; +using std::is_pointer; +using std::is_void; +using std::is_integral; +using std::is_floating_point; +using std::is_arithmetic; +using std::is_fundamental; + +#endif + +#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1800)) || \ + (defined(__GNUG__) && (__GNUC__ < 5)) + +/** + * std::is_trivially_copyable + * + * This implementation only evaluates true if T is fundamental or pointer + * + * Without help from partial template specializations provided by the user for + * a specific class or struct, this trait will never report that the specified + * class or struct is trivially-copyable ; this is always safe, + * if possibly sub-optimal. + */ +template <typename T> +struct is_trivially_copyable + : integral_constant<bool, (is_fundamental<T>::value || is_pointer<T>::value)> {}; + +#else + +using std::is_trivially_copyable; + +#endif + +//----------------------------------------------------------------------------- +// Alignment and layout utilities +//----------------------------------------------------------------------------- + +#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1500)) + +/// std::alignment_of +template <typename value_t> +struct alignment_of { + struct pad { + value_t val; + char byte; + }; + + enum { value = sizeof(pad) - sizeof(value_t) }; +}; + +#else + +template <typename value_t> +struct alignment_of : std::alignment_of<value_t> {}; + +#endif + +/* 16B specializations where 32-bit Win32 host compiler disagrees with device compiler */ +template <> +struct alignment_of<int4> { + enum { value = 16 }; +}; +template <> +struct alignment_of<uint4> { + enum { value = 16 }; +}; +template <> +struct alignment_of<float4> { + enum { value = 16 }; +}; +template <> +struct alignment_of<long4> { + enum { value = 16 }; +}; +template <> +struct alignment_of<ulong4> { + enum { value = 16 }; +}; +template <> +struct alignment_of<longlong2> { + enum { value = 16 }; +}; +template <> +struct alignment_of<ulonglong2> { + enum { value = 16 }; +}; +template <> +struct alignment_of<double2> { + enum { value = 16 }; +}; +template <> +struct alignment_of<longlong4> { + enum { value = 16 }; +}; +template <> +struct alignment_of<ulonglong4> { + enum { value = 16 }; +}; +template <> +struct alignment_of<double4> { + enum { value = 16 }; +}; + +// Specializations for volatile/const qualified types +template <typename value_t> +struct alignment_of<volatile value_t> : alignment_of<value_t> {}; +template <typename value_t> +struct alignment_of<const value_t> : alignment_of<value_t> {}; +template <typename value_t> +struct alignment_of<const volatile value_t> : alignment_of<value_t> {}; + +#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1800)) + +template <size_t Align> +struct aligned_chunk; +template <> +struct __align__(1) aligned_chunk<1> { + uint8_t buff; +}; +template <> +struct __align__(2) aligned_chunk<2> { + uint16_t buff; +}; +template <> +struct __align__(4) aligned_chunk<4> { + uint32_t buff; +}; +template <> +struct __align__(8) aligned_chunk<8> { + uint32_t buff[2]; +}; +template <> +struct __align__(16) aligned_chunk<16> { + uint32_t buff[4]; +}; +template <> +struct __align__(32) aligned_chunk<32> { + uint32_t buff[8]; +}; +template <> +struct __align__(64) aligned_chunk<64> { + uint32_t buff[16]; +}; +template <> +struct __align__(128) aligned_chunk<128> { + uint32_t buff[32]; +}; +template <> +struct __align__(256) aligned_chunk<256> { + uint32_t buff[64]; +}; +template <> +struct __align__(512) aligned_chunk<512> { + uint32_t buff[128]; +}; +template <> +struct __align__(1024) aligned_chunk<1024> { + uint32_t buff[256]; +}; +template <> +struct __align__(2048) aligned_chunk<2048> { + uint32_t buff[512]; +}; +template <> +struct __align__(4096) aligned_chunk<4096> { + uint32_t buff[1024]; +}; + +/// std::aligned_storage +template <size_t Len, size_t Align> +struct aligned_storage { + typedef aligned_chunk<Align> type[Len / sizeof(aligned_chunk<Align>)]; +}; + +#else + +using std::aligned_storage; + +#endif + +#if !defined(__CUDACC_RTC__) +/// Default deleter +template <typename T> +struct default_delete { + void operator()(T* ptr) const { delete ptr; } +}; + +/// Partial specialization for deleting array types +template <typename T> +struct default_delete<T[]> { + void operator()(T* ptr) const { delete[] ptr; } +}; + +/// std::unique_ptr +template <class T, class Deleter = default_delete<T> > +class unique_ptr { + public: + typedef T* pointer; + typedef T element_type; + typedef Deleter deleter_type; + + private: + /// Pointer to memory + pointer _ptr; + + /// Deleter + deleter_type _deleter; + + public: + unique_ptr() : _ptr(nullptr) {} + unique_ptr(pointer p) : _ptr(p) {} + + ~unique_ptr() { + if (_ptr) { + _deleter(_ptr); + } + } + /// Returns a pointer to the managed object or nullptr if no object is owned. + pointer get() const noexcept { return _ptr; } + + /// Releases ownership of the managed object, if any + pointer release() noexcept { + pointer p(_ptr); + _ptr = nullptr; + return p; + } + + /// Replaces the managed object, deleting the old object. + void reset(pointer p = pointer()) noexcept { + pointer old_ptr = _ptr; + _ptr = p; + if (old_ptr != nullptr) { + get_deleter()(old_ptr); + } + } + + /// Swaps the managed objects with *this and another unique_ptr + void swap(unique_ptr& other) noexcept { std::swap(_ptr, other._ptr); } + + /// Returns the deleter object + Deleter& get_deleter() noexcept { return _deleter; } + + /// Returns the deleter object + Deleter const& get_deleter() const noexcept { return _deleter; } + + /// Checks whether an object is owned + operator bool() const noexcept { return _ptr != nullptr; } + + /// Dereferences the unique_ptr + T& operator*() const { return *_ptr; } + + /// Returns a pointer to the managed object + pointer operator->() const noexcept { return _ptr; } + + /// Array access to managed object + T& operator[](size_t i) const { return _ptr[i]; } +}; + +/// Specializes the swap algorithm +template <typename T, typename Deleter> +void swap(unique_ptr<T, Deleter>& lhs, unique_ptr<T, Deleter>& rhs) noexcept { + lhs.swap(rhs); +} +#endif + +}; // namespace platform +}; // namespace cutlass diff --git a/cutlass-example/cutlass/vector.h b/cutlass-example/cutlass/vector.h new file mode 100644 index 0000000..a66dfde --- /dev/null +++ b/cutlass-example/cutlass/vector.h @@ -0,0 +1,229 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Defines a 1D vector of elements held in the registers of each thread. +*/ +#pragma once + +#if !defined(__CUDACC_RTC__) || defined(CUTLASS_NVRTC_HAS_FP16) +#include <cuda_fp16.h> +#endif + +#include <cutlass/util/platform.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <size_t kAlignment_> +struct AlignedStruct {}; + +template <> +struct __align__(1) AlignedStruct<1>{}; +template <> +struct __align__(2) AlignedStruct<2>{}; +template <> +struct __align__(4) AlignedStruct<4>{}; +template <> +struct __align__(8) AlignedStruct<8>{}; +template <> +struct __align__(16) AlignedStruct<16>{}; +template <> +struct __align__(32) AlignedStruct<32>{}; +template <> +struct __align__(64) AlignedStruct<64>{}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, int kLanes_> +union Vector { + /// The scalar type. + typedef Scalar_ Scalar; + + /// The number of elements in the vector. + enum { kLanes = kLanes_ }; + /// The size of the vector. + enum { kVectorSize = kLanes * (int)sizeof(Scalar) }; + /// The number of registers needed to store the vector. + enum { kRegisters = kVectorSize < 4 ? 1 : kVectorSize / 4 }; + + // Make sure that the vector type makes sense. + static_assert(kVectorSize <= 16, "Vector type is too large"); + + /// The aligned storage to make sure we have good alignment. + AlignedStruct<kVectorSize> aligned_; + /// The associated array of scalars. + Scalar scalars[kLanes]; + /// The data in registers. + uint32_t registers[kRegisters]; + + /// Accessor to the ith lane. + CUTLASS_DEVICE Scalar const& operator[](uint32_t i) const { return scalars[i]; } + /// Accessor to the ith lane. + CUTLASS_DEVICE Scalar& operator[](uint32_t i) { return scalars[i]; } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#if !defined(__CUDACC_RTC__) || defined(CUTLASS_NVRTC_HAS_FP16) + +template <int kLanes_> +union Vector<half, kLanes_> { + /// The scalar type. + typedef half Scalar; + + /// The number of elements in the vector. + enum { kLanes = kLanes_ }; + /// The size of the vector. + enum { kVectorSize = kLanes * (int)sizeof(Scalar) }; + /// The number of registers needed to store the vector. + enum { kRegisters = kVectorSize < 4 ? 1 : kVectorSize / 4 }; + + // Make sure that the vector type makes sense. + static_assert(kVectorSize <= size_t(16), "Vector type is too large"); + + /// The aligned storage to make sure we have good alignment. + AlignedStruct<kVectorSize> aligned_; + /// The associated array of scalars. + uint16_t scalars[kLanes]; + /// The data in registers. + uint32_t registers[kRegisters]; + + /// Accessor to the ith lane. + CUTLASS_DEVICE Scalar const& operator[](uint32_t i) const { + return reinterpret_cast<Scalar const&>(scalars[i]); + } + /// Accessor to the ith lane. + CUTLASS_DEVICE Scalar& operator[](uint32_t i) { return reinterpret_cast<Scalar&>(scalars[i]); } +}; + +#endif + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_> +CUTLASS_DEVICE void make_zero(Scalar_& x) { + x = Scalar_(0); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Element_, int kLanes_ = 1> +struct Vectorize { + typedef Vector<Element_, kLanes_> Type; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Element_> +struct Vectorize<Element_, 1> { + typedef Element_ Type; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename Scalar_, int kLanes_> +CUTLASS_DEVICE void make_zero(Vector<Scalar_, kLanes_>& vec) { + for (int i = 0; i < Vector<Scalar_, kLanes_>::kRegisters; ++i) { + vec.registers[i] = 0; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// cutlass::Extent similar to std::extent but applicable to CUTLASS types +// + +/// Returns the extent of a scalar or vector +template <typename T> +struct Extent { + static size_t const kValue = 1; +}; + +/// Returns the number of lanes of a vector if need be +template <typename T, int Lanes> +struct Extent<Vector<T, Lanes> > { + static size_t const kValue = Lanes; +}; + +/// Returns the number of lanes of a vector if need be +template <typename T, int Lanes> +struct Extent<Vector<T, Lanes> const> { + static size_t const kValue = Lanes; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Traits describing properties of vectors and scalar-as-vectors +template <typename T> +struct VectorTraits { + /// Scalar type + typedef T Scalar; + + /// Number of lanes of vector + static int const kLanes = 1; + + /// True if the type is actually a cutlass::Vector, otherwise false + static bool const IsVector = false; + + /// Type that is always a vector + typedef Vector<T, 1> Vector; +}; + +/// Partial specialization for actual cutlass::Vector +template <typename T, int Lanes> +struct VectorTraits<Vector<T, Lanes> > { + /// Scalar type + typedef T Scalar; + + /// Number of lanes of vector + static int const kLanes = Lanes; + + /// Type is actually a cutlass::Vector + static bool const IsVector = true; + + /// Type that is always a Vector + typedef Vector<T, Lanes> Vector; +}; + +/// Partial specialization for actual cutlass::Vector +template <typename T, int Lanes> +struct VectorTraits<Vector<T, Lanes> const> { + /// Scalar type + typedef T Scalar; + + /// Number of lanes of vector + static int const kLanes = Lanes; + + /// Type is actually a cutlass::Vector + static bool const IsVector = true; + + /// Type that is always a Vector + typedef Vector<T, Lanes> Vector; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/cutlass/wmma_matrix.h b/cutlass-example/cutlass/wmma_matrix.h new file mode 100644 index 0000000..c4d8a0b --- /dev/null +++ b/cutlass-example/cutlass/wmma_matrix.h @@ -0,0 +1,193 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Abstractions for loading and storing matrices using the CUDA WMMA API. +*/ +#pragma once + +#if defined(__CUDACC__) && (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700) + +// Dependent header files should use the following macro to guard all code using +// nvcuda::wmma:: to enable compilation for CUDA Compute Capabilities < sm_70. +// Earlier shader models not support Tensor Cores. +#define CUTLASS_USE_WMMA_API + +#include "stdio.h" + +#include <crt/mma.h> +#include <cutlass/fragment.h> +#include <cutlass/load_store.h> +#include <cutlass/matrix_traits.h> +#include <cutlass/shape.h> +#include <cutlass/vector.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Statically maps cutlass::MatrixLayout => nvcuda::wmma layout tags +template <MatrixLayout::Kind kLayout_> +struct WmmaLayout { + typedef nvcuda::wmma::col_major Layout; +}; + +/// Statically maps cutlass::MatrixLayout => nvcuda::wmma layout tags +template <> +struct WmmaLayout<MatrixLayout::kRowMajor> { + typedef nvcuda::wmma::row_major Layout; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Adapter to nvcuda::wmma fragment load and store operations +template <GemmOperand::Kind kOperand_, + MatrixLayout::Kind kLayout_, + typename Scalar_, + typename WmmaShape_> +struct WmmaMatrix {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Adapter to nvcuda::wmma fragment accessors for A operand +template <MatrixLayout::Kind kLayout_, typename Scalar_, typename WmmaShape_> +struct WmmaMatrix<GemmOperand::kA, kLayout_, Scalar_, WmmaShape_> + : public nvcuda::wmma::fragment< + /// The nvcuda::wmma operand name. + nvcuda::wmma::matrix_a, + /// The dimensions. + WmmaShape_::kW, + WmmaShape_::kH, + WmmaShape_::kD, + /// The scalar. + Scalar_, + /// The layout. + typename WmmaLayout<kLayout_>::Layout> { + /// This type. + typedef WmmaMatrix<GemmOperand::kA, kLayout_, Scalar_, WmmaShape_> This_; + + /// Fill-in the element. + CUTLASS_DEVICE This_& operator=(Scalar_ const& x) { + nvcuda::wmma::fill_fragment(*this, x); + return *this; + } + + /// Load from memory. + CUTLASS_DEVICE void load(Scalar_ const* pointer, int const stride) { + nvcuda::wmma::load_matrix_sync(*this, pointer, stride); + } + + /// Store to memory. + CUTLASS_DEVICE void store(Scalar_* pointer, int const stride) const { + nvcuda::wmma::store_matrix_sync(pointer, *this, stride); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Adapter to nvcuda::wmma fragment accessors for B operand +template <MatrixLayout::Kind kLayout_, typename Scalar_, typename WmmaShape_> +struct WmmaMatrix<GemmOperand::kB, kLayout_, Scalar_, WmmaShape_> + : public nvcuda::wmma::fragment< + /// The nvcuda::wmma operand name. + nvcuda::wmma::matrix_b, + /// The dimensions. + WmmaShape_::kW, + WmmaShape_::kH, + WmmaShape_::kD, + /// The scalar. + Scalar_, + /// The layout. + typename WmmaLayout<kLayout_>::Layout> { + /// This type. + typedef WmmaMatrix<GemmOperand::kB, kLayout_, Scalar_, WmmaShape_> This_; + + /// Fill-in the element. + CUTLASS_DEVICE This_& operator=(Scalar_ const& x) { + nvcuda::wmma::fill_fragment(*this, x); + return *this; + } + + /// Load from memory. + CUTLASS_DEVICE void load(Scalar_ const* pointer, int const stride) { + nvcuda::wmma::load_matrix_sync(*this, pointer, stride); + } + + /// Store to memory. + CUTLASS_DEVICE void store(Scalar_* pointer, int const stride) const { + nvcuda::wmma::store_matrix_sync(pointer, *this, stride); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Adapter to nvcuda::wmma fragment accessors for C operand +template <MatrixLayout::Kind kLayout_, typename Scalar_, typename WmmaShape_> +struct WmmaMatrix<GemmOperand::kC, kLayout_, Scalar_, WmmaShape_> + : public nvcuda::wmma::fragment< + /// The nvcuda::wmma operand name. + nvcuda::wmma::accumulator, + /// The dimensions. + WmmaShape_::kW, + WmmaShape_::kH, + WmmaShape_::kD, + /// The scalar. + Scalar_> { + /// This type. + typedef WmmaMatrix<GemmOperand::kC, kLayout_, Scalar_, WmmaShape_> This_; + /// The layout. + static MatrixLayout::Kind const kLayout = kLayout_; + + /// Fill-in the element. + CUTLASS_DEVICE This_& operator=(Scalar_ const& x) { + nvcuda::wmma::fill_fragment(*this, x); + return *this; + } + + /// Load from memory. + CUTLASS_DEVICE void load(Scalar_ const* pointer, int const stride) { + bool const kIsRowMajor = kLayout == MatrixLayout::kRowMajor; + nvcuda::wmma::load_matrix_sync( + *this, + pointer, + stride, + kIsRowMajor ? nvcuda::wmma::mem_row_major : nvcuda::wmma::mem_col_major); + } + + /// Store to memory. + CUTLASS_DEVICE void store(Scalar_* pointer, int const stride) const { + bool const kIsRowMajor = kLayout == MatrixLayout::kRowMajor; + nvcuda::wmma::store_matrix_sync( + pointer, + *this, + stride, + kIsRowMajor ? nvcuda::wmma::mem_row_major : nvcuda::wmma::mem_col_major); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass + +#endif // defined CUTLASS_USE_WMMA_API diff --git a/cutlass-example/cutlass_example.cu b/cutlass-example/cutlass_example.cu new file mode 100644 index 0000000..d3e8f89 --- /dev/null +++ b/cutlass-example/cutlass_example.cu @@ -0,0 +1,17 @@ +//added by me +#include <cutlass/wmma_matrix.h> +#include <cutlass/gemm/gemm.h> +#include <cutlass/gemm/wmma_gemm_traits.h> +#include <gemm_testbed.h> +#include <gemm.h> + +int main(int argc, char* argv[]) { + typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, + cutlass::MatrixLayout::kRowMajor, + cutlass::Shape<32, 128, 128> > + WmmaGemmTraits; + run_gemm<WmmaGemmTraits>(256, 256, 128); + +} + + diff --git a/cutlass-example/device_memory.h b/cutlass-example/device_memory.h new file mode 100644 index 0000000..ab561d8 --- /dev/null +++ b/cutlass-example/device_memory.h @@ -0,0 +1,178 @@ +/****************************************************************************** + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are not permitted. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +/** + * \file + * \brief C++ interface to CUDA device memory management functions. + */ + +#include <memory> + +#include <cutlass/util/debug.h> +#include <cutlass/util/platform.h> +#include <exceptions.h> + +namespace cutlass { +namespace device_memory { + +/****************************************************************************** + * Allocation lifetime + ******************************************************************************/ + +/// Allocate a buffer of \p count elements of type \p T on the current CUDA device +template <typename T> +T* allocate(size_t count = 1) { + T* ptr = 0; + size_t bytes = sizeof(T) * count; + + cudaError_t cuda_error = CUDA_PERROR(cudaMalloc((void**)&ptr, bytes)); + if (cuda_error != cudaSuccess) { + throw cuda_exception("Failed to allocate memory", cuda_error); + } + + return ptr; +} + +/// Free the buffer pointed to by \p ptr +template <typename T> +void free(T* ptr) { + if (ptr) { + cudaError_t cuda_error = CUDA_PERROR(cudaFree(ptr)); + if (cuda_error != cudaSuccess) { + throw cuda_exception("Failed to free device memory", cuda_error); + } + } +} + +/****************************************************************************** + * Data movement + ******************************************************************************/ + +template <typename T> +void copy(T* dst, T const* src, size_t count, cudaMemcpyKind kind) { + size_t bytes = count * sizeof(T); + + cudaError_t cuda_error = CUDA_PERROR(cudaMemcpy(dst, src, bytes, kind)); + if (cuda_error != cudaSuccess) { + throw cuda_exception("cudaMemcpy() failed", cuda_error); + } +} + +template <typename T> +void copy_to_device(T* dst, T const* src, size_t count = 1) { + copy(dst, src, count, cudaMemcpyHostToDevice); +} + +template <typename T> +void copy_to_host(T* dst, T const* src, size_t count = 1) { + copy(dst, src, count, cudaMemcpyDeviceToHost); +} + +template <typename T> +void copy_device_to_device(T* dst, T const* src, size_t count = 1) { + copy(dst, src, count, cudaMemcpyDeviceToDevice); +} + +/// Copies elements from device memory to host-side range +template <typename OutputIterator, typename T> +void insert_to_host(OutputIterator begin, OutputIterator end, T const* device_begin) { + size_t elements = end - begin; + copy_to_host(&*begin, device_begin, elements); +} + +/// Copies elements to device memory from host-side range +template <typename T, typename InputIterator> +void insert_to_device(T* device_begin, InputIterator begin, InputIterator end) { + size_t elements = end - begin; + copy_to_device(device_begin, &*begin, elements); +} + +/****************************************************************************** + * "Smart" device memory allocation + ******************************************************************************/ + +/// Device allocation abstraction that tracks size and capacity +template <typename T> +struct allocation { + /// Delete functor for CUDA device memory + struct deleter { + void operator()(T* ptr) { + cudaError_t cuda_error = CUDA_PERROR(cudaFree(ptr)); + if (cuda_error != cudaSuccess) { + // noexcept + // throw cuda_exception("cudaFree() failed", cuda_error); + return; + } + } + }; + + /// Number of elements of T allocated on the current CUDA device + size_t capacity; + + /// Smart pointer + platform::unique_ptr<T, deleter> smart_ptr; + + // + // + // + + /// Constructor: allocates no memory + allocation() : capacity(0) {} + + /// Constructor: allocates \p capacity elements on the current CUDA device + allocation(size_t _capacity) : smart_ptr(allocate<T>(_capacity)), capacity(_capacity) {} + + /// Destructor + ~allocation() { reset(); } + + /// Returns a pointer to the managed object + T* get() const { return smart_ptr.get(); } + + /// Releases the ownership of the managed object (without deleting) and resets capacity to zero + T* release() { + capacity = 0; + return smart_ptr.release(); + } + + /// Deletes the managed object and resets capacity to zero + void reset() { + capacity = 0; + smart_ptr.reset(); + } + + /// Deletes managed object, if owned, and replaces its reference with a given pointer and capacity + void reset(T* _ptr, size_t _capacity) { + smart_ptr.reset(_ptr); + capacity = _capacity; + } + + /// Returns a pointer to the object owned by *this + T* operator->() const { return smart_ptr.get(); } + + /// Returns the deleter object which would be used for destruction of the managed object. + deleter& get_deleter() { return smart_ptr.get_deleter(); } + + /// Returns the deleter object which would be used for destruction of the managed object (const) + const deleter& get_deleter() const { return smart_ptr.get_deleter(); } +}; + +} // namespace device_memory +} // namespace cutlass diff --git a/cutlass-example/exceptions.h b/cutlass-example/exceptions.h new file mode 100644 index 0000000..72d99fe --- /dev/null +++ b/cutlass-example/exceptions.h @@ -0,0 +1,62 @@ +/****************************************************************************** + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are not permitted. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +/** + * \file + * \brief C++ exception semantics for CUDA error codes + */ + +#include <cuda_runtime.h> +#include <iosfwd> +#include <stdexcept> + +#include <cutlass/util/platform.h> + +namespace cutlass { + +/// C++ exception wrapper for CUDA \p cudaError_t +class cuda_exception : public std::exception { + public: + /// Constructor + cuda_exception(const char* msg = "", cudaError_t err = cudaErrorUnknown) : msg(msg), err(err) {} + + /// Returns the underlying CUDA \p cudaError_t + cudaError_t cudaError() const { return err; } + + protected: + /// Explanatory string + const char* msg; + + /// Underlying CUDA \p cudaError_t + cudaError_t err; +}; + +/// Writes a cudaError_t to an output stream +inline std::ostream& operator<<(std::ostream& out, cudaError_t result) { + return out << cudaGetErrorString(result); +} + +/// Writes a cuda_exception instance to an output stream +inline std::ostream& operator<<(std::ostream& out, cuda_exception const& e) { + return out << e.what() << ": " << e.cudaError(); +} + +} // namespace cutlass diff --git a/cutlass-example/executionFlow b/cutlass-example/executionFlow new file mode 100644 index 0000000..356afed --- /dev/null +++ b/cutlass-example/executionFlow @@ -0,0 +1,262 @@ +mov.b64%rd33, _ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0; +mov.u64 %rd1, %rd33; +ld.param.u64 %rd34, [_ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0+48]; +cvta.to.global.u64 %rd2, %rd34; +mov.u32 %r301, %ctaid.x; +shl.b32 %r1, %r301, 4; +ld.param.u64 %rd36, [_ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0+136]; +cvta.to.global.u64 %rd4, %rd36; +mov.u32 %r302, %ctaid.y; +shl.b32 %r2, %r302, 4; +mov.u32 %r3, %tid.x; +shr.u32 %r4, %r3, 1; +and.b32 %r303, %r3, 1; +shl.b32 %r5, %r303, 3; +add.s32 %r6, %r5, %r1; +ld.param.u32 %r7, [_ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0+20]; +mad.lo.s32 %r304, %r7, %r4, %r6; +cvt.s64.s32%rd5, %r304; +mul.wide.s32 %rd37, %r304, 2; +add.s64 %rd6, %rd2, %rd37; +ld.param.u32 %r8, [_ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0]; +sub.s32 %r305, %r8, %r6; +ld.param.u32 %r9, [_ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0+8]; +setp.gt.s32%p2, %r9, 0; +setp.gt.s32%p3, %r305, 0; +and.pred %p4, %p3, %p2; +selp.u32%r306, 1, 0, %p4; +setp.gt.s32%p5, %r9, 16; +and.pred %p6, %p3, %p5; +selp.u16%rs1, 1, 0, %p6; +mul.wide.u16 %r10, %rs1, 2; +selp.u32%r307, -1, 0, %p6; +bfi.b32 %r481, %r307, %r306, 1, 1; +bfe.u32 %r308, %r3, 1, 4; +mad.lo.s32 %r12, %r308, 24, %r5; +setp.lt.s32%p7, %r12, 768; +@%p7 bra BB0_2; +ld.param.u32 %r310, [%rd1+4]; +ld.param.u32 %r13, [%rd1+108]; +add.s32 %r16, %r5, %r2; +mad.lo.s32 %r311, %r13, %r4, %r16; +cvt.s64.s32%rd7, %r311; +mul.wide.s32 %rd45, %r311, 2; +add.s64 %rd8, %rd4, %rd45; +sub.s32 %r312, %r310, %r16; +setp.gt.s32%p9, %r312, 0; +and.pred %p11, %p9, %p2; +selp.u32%r313, 1, 0, %p11; +and.pred %p13, %p9, %p5; +selp.u16%rs2, 1, 0, %p13; +mul.wide.u16 %r17, %rs2, 2; +selp.u32%r314, -1, 0, %p13; +bfi.b32 %r480, %r314, %r313, 1, 1; +@%p7 bra BB0_4; +setp.lt.s32%p14, %r9, 32; +@%p14 bra BB0_7; +setp.lt.s32%p18, %r4, %r9; +selp.b32%r321, %r481, %r10, %p18; +add.s32 %r322, %r4, 16; +setp.lt.s32%p19, %r322, %r9; +and.b32 %r323, %r321, 1; +selp.b32%r481, %r321, %r323, %p19; +selp.b32%r324, %r480, %r17, %p18; +and.b32 %r325, %r324, 1; +selp.b32%r480, %r324, %r325, %p19; +bra.uni BB0_8; +and.b32 %r330, %r481, 1; +setp.eq.b32%p20, %r330, 1; +mov.u32 %r571, 0; +mov.u32 %r559, %r571; +mov.u32 %r560, %r571; +mov.u32 %r561, %r571; +mov.u32 %r562, %r571; +@!%p20 bra BB0_10; +bra.uni BB0_9; +ld.global.v4.u32 {%r561, %r562, %r559, %r560}, [%rd6]; +ld.param.u32 %r33, [%rd1+32]; +cvt.s64.s32%rd53, %r33; +add.s64 %rd9, %rd5, %rd53; +shl.b64 %rd54, %rd9, 1; +add.s64 %rd10, %rd2, %rd54; +and.b32 %r339, %r481, 2; +setp.eq.s32%p21, %r339, 0; +mov.u32 %r563, %r571; +mov.u32 %r564, %r571; +mov.u32 %r565, %r571; +mov.u32 %r566, %r571; +@%p21 bra BB0_12; +and.b32 %r348, %r480, 1; +setp.eq.b32%p22, %r348, 1; +mov.u32 %r567, %r571; +mov.u32 %r568, %r571; +mov.u32 %r569, %r571; +mov.u32 %r570, %r571; +@!%p22 bra BB0_14; +bra.uni BB0_13; +ld.global.v4.u32 {%r569, %r570, %r567, %r568}, [%rd8]; +ld.param.u32 %r50, [%rd1+120]; +cvt.s64.s32%rd55, %r50; +add.s64 %rd56, %rd7, %rd55; +shl.b64 %rd57, %rd56, 1; +add.s64 %rd11, %rd4, %rd57; +and.b32 %r357, %r480, 2; +setp.eq.s32%p23, %r357, 0; +mov.u32 %r572, %r571; +mov.u32 %r573, %r571; +mov.u32 %r574, %r571; +@%p23 bra BB0_16; +shl.b32 %r362, %r12, 1; +mov.u32 %r363, _ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE$__cuda_local_var_29658_57_non_const_shared_storage; +add.s32 %r59, %r363, %r362; +st.shared.v4.u32 [%r59], {%r561, %r562, %r559, %r560}; +st.shared.v4.u32 [%r59+768], {%r565, %r566, %r563, %r564}; +st.shared.v4.u32 [%r59+1536], {%r569, %r570, %r567, %r568}; +st.shared.v4.u32 [%r59+2304], {%r573, %r574, %r571, %r572}; +bar.sync 0; +shr.u32 %r60, %r3, 5; +shl.b32 %r61, %r60, 4; +setp.lt.u32%p24, %r61, 768; +@%p24 bra BB0_18; +mul.wide.s32 %rd65, %r61, 2; +cvt.u64.u32 %temp, %r363; +cvta.shared.u64 %rd66, %temp; +add.s64 %rd67, %rd66, %rd65; +add.s64 %rd12, %rd67, 1536; +mov.u32 %r374, 24; +wmma.load.a.sync.col.m16n16k16.f16 {%r590, %r589, %r588, %r587, %r586, %r585, %r584, %r583}, [%rd66], %r374; +wmma.load.b.sync.row.m16n16k16.f16 {%r582, %r581, %r580, %r579, %r578, %r577, %r576, %r575}, [%rd12], %r374; +add.s32 %r599, %r9, -32; +mov.u32 %r609, 0; +setp.lt.s32%p25, %r599, 1; +@%p25 bra BB0_19; +mov.u32 %r610, %r609; +mov.u32 %r611, %r609; +mov.u32 %r612, %r609; +mov.u32 %r613, %r609; +mov.u32 %r614, %r609; +mov.u32 %r615, %r609; +mov.u32 %r616, %r609; +setp.lt.s32%p34, %r599, -31; +@%p34 bra BB0_35; +mov.u32 %r477, %tid.x; +shr.u32 %r476, %r477, 5; +shl.b32 %r475, %r476, 4; +add.s32 %r434, %r475, 384; +mul.wide.s32 %rd96, %r434, 2; +add.s64 %rd97, %rd66, %rd96; +add.s64 %rd26, %rd97, 1536; +add.s64 %rd99, %rd66, 768; +wmma.load.a.sync.col.m16n16k16.f16 {%r257, %r256, %r255, %r254, %r250, %r251, %r253, %r252}, [%rd99], %r374; +wmma.load.b.sync.row.m16n16k16.f16 {%r265, %r264, %r263, %r262, %r258, %r259, %r261, %r260}, [%rd26], %r374; +mov.b32 %f63, %r615; +mov.b32 %f64, %r616; +mov.b32 %f65, %r613; +mov.b32 %f66, %r614; +mov.b32 %f67, %r611; +mov.b32 %f68, %r612; +mov.b32 %f69, %r609; +mov.b32 %f70, %r610; +wmma.mma.sync.col.row.m16n16k16.f32.f32 {%f10, %f11, %f12, %f13, %f14, %f15, %f16, %f17}, {%r590, %r589, %r588, %r587, %r586, %r585, %r584, %r583}, {%r582, %r581, %r580, %r579, %r578, %r577, %r576, %r575}, {%f70, %f69, %f68, %f67, %f66, %f65, %f64, %f63}; +bar.sync 0; +bar.sync 0; +wmma.mma.sync.col.row.m16n16k16.f32.f32 {%f71, %f72, %f73, %f74, %f75, %f76, %f77, %f78}, {%r257, %r256, %r255, %r254, %r250, %r251, %r253, %r252}, {%r265, %r264, %r263, %r262, %r258, %r259, %r261, %r260}, {%f10, %f11, %f12, %f13, %f14, %f15, %f16, %f17}; +mov.b32 %r610, %f71; +mov.b32 %r609, %f72; +mov.b32 %r612, %f73; +mov.b32 %r611, %f74; +mov.b32 %r614, %f75; +mov.b32 %r613, %f76; +mov.b32 %r616, %f77; +mov.b32 %r615, %f78; +add.s32 %r599, %r599, -32; +setp.gt.s32%p35, %r599, -32; +@%p35 bra BB0_34; +mov.b64%rd145, _ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0; +mov.u64 %rd144, %rd145; +mov.u32 %r472, %tid.x; +ld.param.f32 %f145, [%rd144+428]; +ld.param.u32 %r471, [%rd144+324]; +shr.u32 %r470, %r472, 5; +ld.param.u32 %r469, [%rd144+332]; +mov.u32 %r468, %ctaid.x; +shl.b32 %r467, %r468, 4; +mov.u32 %r466, _ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE$__cuda_local_var_29658_57_non_const_shared_storage; +mov.u32 %r465, %ctaid.y; +shl.b32 %r464, %r465, 4; +ld.param.v2.u32 {%r438, %r439}, [%rd144+288]; +ld.param.u32 %r284, [%rd144+300]; +ld.param.u32 %r440, [%rd144+320]; +ld.param.u32 %r285, [%rd144+336]; +ld.param.f32 %f18, [%rd144+424]; +ld.param.u32 %r287, [%rd144+304]; +ld.param.u64 %rd100, [%rd144+312]; +cvta.to.global.u64 %rd101, %rd100; +and.b32 %r441, %r472, 3; +shl.b32 %r288, %r441, 2; +shr.u32 %r289, %r472, 2; +add.s32 %r290, %r289, %r464; +add.s32 %r291, %r288, %r467; +mad.lo.s32 %r442, %r290, %r440, %r291; +cvt.s64.s32%rd102, %r442; +mul.wide.s32 %rd103, %r442, 4; +add.s64 %rd27, %rd101, %rd103; +sub.s32 %r292, %r469, %r290; +shl.b32 %r293, %r470, 8; +shl.b32 %r443, %r289, 4; +add.s32 %r294, %r443, %r288; +shl.b32 %r444, %r294, 2; +add.s32 %r295, %r466, %r444; +cvt.s64.s32%rd104, %r471; +add.s64 %rd105, %rd102, %rd104; +shl.b64 %rd106, %rd105, 2; +add.s64 %rd28, %rd101, %rd106; +setp.eq.f32%p36, %f145, 0f00000000; +@%p36 bra BB0_48; +setp.lt.s32%p51, %r293, 256; +@%p51 bra BB0_50; +setp.lt.s32%p52, %r294, 256; +@%p52 bra BB0_52; +ld.param.u32 %r474, [_ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0]; +mul.wide.s32 %rd142, %r293, 4; +add.s64 %rd32, %rd66, %rd142; +setp.lt.s32%p1, %r291, %r474; +bar.sync 0; +mov.b32 %f121, %r610; +mov.b32 %f122, %r609; +mov.b32 %f123, %r612; +mov.b32 %f124, %r611; +mov.b32 %f125, %r614; +mov.b32 %f126, %r613; +mov.b32 %f127, %r616; +mov.b32 %f128, %r615; +mov.u32 %r458, 16; +wmma.store.d.sync.col.m16n16k16.f32 [%rd32], {%f121, %f122, %f123, %f124, %f125, %f126, %f127, %f128}, %r458; +bar.sync 0; +ld.shared.v4.u32 {%r459, %r460, %r461, %r462}, [%r295+512]; +setp.gt.s32%p53, %r292, 0; +and.pred %p54, %p1, %p53; +@!%p54 bra BB0_54; +bra.uni BB0_53; +ld.shared.v4.f32 {%f129, %f130, %f131, %f132}, [%r295]; +mul.f32 %f137, %f18, %f132; +mul.f32 %f138, %f18, %f131; +mul.f32 %f139, %f18, %f130; +mul.f32 %f140, %f18, %f129; +st.global.v4.f32 [%rd27], {%f140, %f139, %f138, %f137}; +mov.b32 %f141, %r459; +mul.f32 %f43, %f18, %f141; +mov.b32 %f142, %r460; +mul.f32 %f44, %f18, %f142; +mov.b32 %f143, %r461; +mul.f32 %f45, %f18, %f143; +mov.b32 %f144, %r462; +mul.f32 %f46, %f18, %f144; +sub.s32 %r463, %r292, %r285; +setp.gt.s32%p56, %r463, 0; +and.pred %p57, %p1, %p56; +@!%p57 bra BB0_56; +bra.uni BB0_55; +st.global.v4.f32 [%rd28], {%f43, %f44, %f45, %f46}; +bra.uni BB0_56; +ret; diff --git a/cutlass-example/gemm.h b/cutlass-example/gemm.h new file mode 100644 index 0000000..18dfdf6 --- /dev/null +++ b/cutlass-example/gemm.h @@ -0,0 +1,152 @@ +/*************************************************************************************************** +* Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +* +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* * Redistributions of source code must retain the above copyright notice, this list of +* conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright notice, this list of +* conditions and the following disclaimer in the documentation and/or other materials +* provided with the distribution. +* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used +* to endorse or promote products derived from this software without specific prior written +* permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR +* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE +* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +**************************************************************************************************/ + +#include <cutlass/cutlass.h> +#include <gemm_testbed.h> + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmTraits_> +static void run_gemm( + int m, + int n, + int k, + int lda, + int ldb, + int ldc, + typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha = + typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1), + typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta = + typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0)) { + typedef cutlass::gemm::Gemm<GemmTraits_> Gemm; + typename Gemm::Params params; + + printf("run_gemm-2:m=%d\n",m); + printf("run_gemm-2:n=%d\n",n); + printf("run_gemm-2:k=%d\n",k); + printf("run_gemm-2:lda=%d\n",lda); + printf("run_gemm-2:ldb=%d\n",ldb); + printf("run_gemm-2:ldc=%d\n",ldc); + printf("run_gemm-2:alpha=%.2f\n",alpha); + printf("run_gemm-2:beta=%.2f\n",beta); + + test::GemmTestbed< + typename test::GemmTestbedTraits< + typename GemmTraits_::GemmConfig::ScalarA>::host_type, // AType + typename test::GemmTestbedTraits< + typename GemmTraits_::GemmConfig::ScalarB>::host_type, // BType + typename test::GemmTestbedTraits< + typename GemmTraits_::Epilogue::ScalarC>::host_type, // CType + typename test::GemmTestbedTraits< + typename GemmTraits_::Epilogue::Accumulators::Element>::host_type, // Accumulator + typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type // Scalar + > + testbed(m, + n, + k, + lda, + ldb, + ldc, + cutlass::convert(GemmTraits_::kLayoutA), + cutlass::convert(GemmTraits_::kLayoutB), + alpha, + beta); + + testbed.initialize(); + + // if (testbed.has_cublas_support()) { + // EXPECT_TRUE(testbed.verify_host_with_cublas()); + // } + + params.initialize(testbed.M(), + testbed.N(), + testbed.K(), + testbed.alpha, + testbed.ptr_A(), + testbed.lda(), + testbed.ptr_B(), + testbed.ldb(), + testbed.beta, + testbed.ptr_C_initial(), + testbed.ldc(), + testbed.ptr_computed(), + testbed.ldc()); + + printf("SIZE_OF_PARAM=%lu\n",sizeof(params)); + void *ptr =¶ms; + for(int kk=0;kk<108;kk++){ + printf("KERNELPARAM:%d:%08x\n",kk,*((((int *) ptr)+kk))); + } + printf("m=%lu\n",sizeof(params.m)); + printf("n=%lu\n",sizeof(params.n)); + printf("k=%lu\n",sizeof(params.k)); +// printf("alpha=%d\n",sizeof(params.alpha)); +// printf("beta=%d\n",sizeof(params.beta)); +// printf("d_a=%d\n",sizeof(params.d_a)); +// printf("lda=%d\n",sizeof(params.lda)); +// printf("d_b=%d\n",sizeof(params.d_b)); +// printf("ldb=%d\n",sizeof(params.ldb)); +// printf("d_c=%d\n",sizeof(params.d_c)); +// printf("ldc=%d\n",sizeof(params.ldc)); +// printf("d_d=%d\n",sizeof(params.d_d)); +// printf("ldd=%d\n",sizeof(params.ldd)); + Gemm::launch(params); + + cudaError_t result = cudaDeviceSynchronize(); + if(result==cudaSuccess){ + printf("Successfully Launched\n"); + } + int save=1; + int myval=testbed.verify_with_host(save,save); + if(myval==1){ + printf("Result Verified\n"); + } + +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename GemmTraits_> +static void run_gemm( + int m, + int n, + int k, + typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha = + typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1), + typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta = + typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0)) { + int lda = GemmTraits_::kLayoutA == cutlass::MatrixLayout::kColumnMajor ? m : k; + int ldb = GemmTraits_::kLayoutB == cutlass::MatrixLayout::kColumnMajor ? k : n; + printf("run_gemm-1:m=%d\n",m); + printf("run_gemm-1:n=%d\n",n); + printf("run_gemm-1:k=%d\n",k); + printf("run_gemm-1:alpha=%.2f\n",alpha); + printf("run_gemm-1:beta=%.2f\n",beta); + printf("run_gemm-1:lda=%d\n",lda); + printf("run_gemm-1:ldb=%d\n",ldb); + run_gemm<GemmTraits_>(m, n, k, lda, ldb, m, alpha, beta); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/cutlass-example/gemm_testbed.h b/cutlass-example/gemm_testbed.h new file mode 100644 index 0000000..97409b1 --- /dev/null +++ b/cutlass-example/gemm_testbed.h @@ -0,0 +1,462 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Test environment for GEMM +*/ + +#pragma once + +#include <fstream> +#include <iomanip> +#include <sstream> +#include <string> + +#include <cutlass/matrix_traits.h> +#include <cutlass/util/platform.h> + +#include <host_tensor.h> +#include <tensor_view_io.h> +#include <type_traits.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <cutlass::GemmOperand::Kind kOperand_, + cutlass::MatrixLayout::Kind kLayout_, + typename Scalar_, + typename WmmaShape_> +struct WmmaMatrix; +} + +namespace test { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename T> +struct GemmTestbedTraits : public cutlass::TypeTraits<T> {}; + +template <cutlass::GemmOperand::Kind kOperand_, + cutlass::MatrixLayout::Kind kLayout_, + typename Scalar_, + typename WmmaShape_> +struct GemmTestbedTraits<cutlass::WmmaMatrix<kOperand_, kLayout_, Scalar_, WmmaShape_> > { + static cudaDataType_t const cublas_type = cutlass::TypeTraits<Scalar_>::cublas_type; + typedef Scalar_ host_type; + typedef Scalar_ device_type; + static inline double remove_negative_zero(double x) { return x == -0.0 ? 0.0 : x; } + static inline double to_print(double x) { return x; } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// +template <typename AType, typename BType, typename CType, typename Accumulator, typename Scalar> +struct GemmTestbed { + // + // Type definitions + // + + /// Host tensor for operand A + typedef cutlass::HostTensor<AType> HostTensorA; + + /// Host tensor for operand B + typedef cutlass::HostTensor<BType> HostTensorB; + + /// Host tensor for operand C + typedef cutlass::HostTensor<CType> HostTensorC; + + /// Functor to print errors + struct PrintErrors { + /// Equivalently sized integer type + typedef typename GemmTestbedTraits<CType>::integer_type integer_t; + + /// Output stream to write to + std::ostream& out; + + /// Reference tensor view + cutlass::HostTensorView<CType> const& reference; + + /// Computed tensor view + cutlass::HostTensorView<CType> const& experimental; + + /// Errors greater than or this amount result in printing + integer_t ulps_threshold; + + /// + PrintErrors(std::ostream& _out, + cutlass::HostTensorView<CType> const& _reference, + cutlass::HostTensorView<CType> const& _experimental, + integer_t _ulps_threshold = 1) + : out(_out), + reference(_reference), + experimental(_experimental), + ulps_threshold(_ulps_threshold) {} + + /// Compares one element + void operator()(CType const& element, typename HostTensorC::Coord_t coord) { + CType exp = experimental.at(coord); + CType ref = reference.at(coord); + + int64_t int_exp = 0; + int64_t int_ref = 0; + + *reinterpret_cast<CType*>(&int_exp) = exp; + *reinterpret_cast<CType*>(&int_ref) = ref; + + integer_t ulps = integer_t(int_exp - int_ref); + + if (std::abs(ulps) >= ulps_threshold) { + // width in hexadecimal digits of value + int const width = sizeof(integer_t) * 2; + + double relative = double(exp) - double(ref); + if (ref != CType(0)) { + relative /= double(ref); + } + + out << "[" << coord << "] expected: " << GemmTestbedTraits<CType>::to_print(ref) << " (0x" + << std::hex << std::setw(width) << std::setfill('0') << integer_t(int_ref) << std::dec + << ")" + << ", got: " << GemmTestbedTraits<CType>::to_print(exp) << " (0x" << std::hex + << std::setw(width) << std::setfill('0') << integer_t(int_exp) << std::dec << ")" + << " relative error: " << relative << ", ulps: " << ulps << "\n"; + } + } + }; + + /// Generates random elements + template <typename T> + struct RandomGenerator { + RandomGenerator(int seed = -1, bool only_ones_ = false) : only_ones(only_ones_) { srand(seed); } + + T operator()() { + if (only_ones) { + return T(1); + } else { + int val = (rand() % 16) - 8; + return T(val); + } + } + + bool only_ones; + }; + + // + // Data members + // + + /// Status + //cublasStatus_t status; + + /// cuBLAS handle + //cublasHandle_t handle; + + /// cuBLAS GEMM algorithm selector + //cublasGemmAlgo_t algorithm; + + /// A matrix operand + HostTensorA A; + + /// Layout of A matrix + //cublasOperation_t layout_A; + + /// B matrix operand + HostTensorB B; + + /// Layout of B matrix + //cublasOperation_t layout_B; + + /// C matrix operand + HostTensorC C_initial; + + /// Reference result computed on the host + cutlass::HostTensor<CType, false> ref_host; + + + /// Computed result + HostTensorC computed; + + /// Linear scalaring factor + Scalar alpha; + + /// Linear scaling factor + Scalar beta; + + // + // Static helpers + // + //template <typename T, bool DeviceBacked> + //static void resize(cutlass::HostTensor<T, DeviceBacked>& tensor, + // int rows, + // int columns, + // cublasOperation_t layout, + // int ldm = 0) { + // if (!ldm) { + // ldm = (layout == CUBLAS_OP_N ? rows : columns); + // } + + // typedef cutlass::Coord<cutlass::HostTensor<T>::Rank> Coord_t; + + // size_t matrix_stride = layout == CUBLAS_OP_N ? columns * ldm : rows * ldm; + // // TODO: Remove that (int) cast. + // Coord_t stride = cutlass::make_Coord( + // (int)matrix_stride, layout == CUBLAS_OP_N ? 1 : ldm, layout == CUBLAS_OP_N ? ldm : 1, 1); + // Coord_t size = cutlass::make_Coord(1, rows, columns, 1); + // tensor.reset(stride, size); + //} + + /// Helper to resize a matrix with a given size and layout + template <typename T, bool DeviceBacked> + static void resize(cutlass::HostTensor<T, DeviceBacked>& tensor, + int rows, + int columns, + int layout, + int ldm = 0) { + if (!ldm) { + ldm = (layout ? rows : columns); + } + + typedef cutlass::Coord<cutlass::HostTensor<T>::Rank> Coord_t; + size_t matrix_stride = layout ? columns * ldm : rows * ldm; + // TODO: Remove that (int) cast. + Coord_t stride = cutlass::make_Coord( + (int)matrix_stride, layout ? 1 : ldm, layout? ldm : 1, 1); + Coord_t size = cutlass::make_Coord(1, rows, columns, 1); + tensor.reset(stride, size); + } + + // + // Methods + // + + /// Constructs a workspace for verifying GEMM. + GemmTestbed(int M_, + int N_, + int K_, + int lda, + int ldb, + int ldc, + int layout_a, + int layout_b, + Scalar alpha_ = Scalar(1), + Scalar beta_ = Scalar(0)) + //cublasGemmAlgo_t algorithm_ = CUBLAS_GEMM_DEFAULT, + //cublasOperation_t layout_c = CUBLAS_OP_N) + : alpha(alpha_), beta(beta_) { + //status = cublasCreate(&handle); + //if (status != CUBLAS_STATUS_SUCCESS) { + // throw cutlass::cuda_exception("Failed to create CUBLAS handle"); + //} + printf("GemmTestbed:alpha=%f\n",alpha_); + printf("GemmTestbed:beta=%f\n",beta_); + printf("GemmTestbed:lda=%d\n",lda); + printf("GemmTestbed:ldb=%d\n",ldb); + printf("GemmTestbed:ldc=%d\n",ldc); + + resize(A, M_, K_,layout_a, lda); + resize(B, K_, N_,layout_b, ldb); + resize(C_initial, M_, N_,1, ldc); + resize(ref_host, M_, N_,1, ldc); + resize(computed, M_, N_,1, ldc); + } + + /// Returns a pointer to the A operand + typename HostTensorA::DeviceType* ptr_A() const { + return A.device_data(); + } + + /// Stride of A matrix + int lda() const { + printf("lda()=%d\n",std::max(A.stride(HostTensorA::Dim_H), A.stride(HostTensorA::Dim_W))); + return std::max(A.stride(HostTensorA::Dim_H), A.stride(HostTensorA::Dim_W)); + } + + /// Returns a pointer to the B operand + typename HostTensorB::DeviceType* ptr_B() const { + return B.device_data(); + } + + /// Stride of B matrix + int ldb() const { + printf("ldb()=%d\n",std::max(B.stride(HostTensorB::Dim_H), B.stride(HostTensorB::Dim_W))); + return std::max(B.stride(HostTensorB::Dim_H), B.stride(HostTensorB::Dim_W)); + } + + /// Returns a pointer to the initial state of the result tensor in device memory + typename HostTensorC::DeviceType* ptr_C_initial() const { + return C_initial.device_data(); + } + + /// Returns a pointer to the result tensor in device memory + typename HostTensorC::DeviceType* ptr_computed() const { + return computed.device_data(); + } + + /// Stride of C matrix + int ldc() const { + printf("ldc()=%d\n",std::max(C_initial.stride(HostTensorC::Dim_H), C_initial.stride(HostTensorC::Dim_W))); + return std::max(C_initial.stride(HostTensorC::Dim_H), C_initial.stride(HostTensorC::Dim_W)); + } + + /// Returns the number of flops implied by the computation (1 multiply-accumulate = 2 flops) + uint64_t flops() const { return uint64_t(M()) * uint64_t(N()) * uint64_t(K()) * 2ULL; } + + /// Computes the speed of the computation in GFLOPs/s + double GFLOPs_per_sec(double runtime_ms) const { return double(flops()) / runtime_ms / 1.0e6; } + + /// Number of rows of problem + int M() const { + printf("M()=%d\n", C_initial.size(HostTensorC::Dim_H)); + return C_initial.size(HostTensorC::Dim_H); + } + + /// Number of columns of problem + int N() const { + printf("N()=%d\n", C_initial.size(HostTensorC::Dim_W)); + return C_initial.size(HostTensorC::Dim_W); + } + + /// Number of columns of problem + int K() const { + printf("K()=%d\n",A.size(HostTensorA::Dim_W)); + return A.size(HostTensorA::Dim_W); + } + + /// Initializes data, randomly + void initialize(int seed = -1) { + A.fill_random(RandomGenerator<AType>(seed)); + B.fill_random(RandomGenerator<BType>(seed + 11)); + C_initial.fill_random(RandomGenerator<CType>(seed + 13,1)); + } + + /// Computes the matrix product on the host + void compute_host() { + ref_host.fill(C_initial); + + std::string results_name = "host_results_before.txt"; + std::ofstream results(results_name.c_str()); + write(results); + + ref_host.template gemm<AType, BType, Accumulator, Scalar>(A, B, alpha, beta); + results_name = "host_results_after.txt"; + std::ofstream results2(results_name.c_str()); + write(results2); + } + + /// Names a probelm based on data type and problem size + std::string workspace_name() const { + std::stringstream ss; + ss << "gemm_" << "t" + << "t" << "_" << typeid(AType).name() << "_" + << typeid(BType).name() << "_" << typeid(CType).name() << "_" << typeid(Accumulator).name() + << "_" << typeid(Scalar).name() << "_" << M() << "x" << N() << "x" << K(); + + return ss.str(); + } + + /// Writes the workspace to an ostream + std::ostream& write(std::ostream& out) const { + out << "A = " << A << "\nB = " << B << "\nC_initial = " << C_initial + << "\ncomputed = " << computed + << "\nref_host= " << ref_host<< std::endl; + + return out; + } + + /// Outputs each mismatching element + std::ostream& write_errors(std::ostream& out, + cutlass::HostTensorView<CType> const& experimental, + cutlass::HostTensorView<CType> const& ref) const { + PrintErrors printer(out, ref, experimental); + + computed.visit(printer); + + return out; + } + + /// Sync's all input tensors to device + void sync_device() { + A.sync_device(); + B.sync_device(); + C_initial.sync_device(); + + ref_host.fill(C_initial); + computed.fill(C_initial); + + computed.sync_device(); + } + + /// Sync's all output tensors to host + void sync_host() { + computed.sync_host(); + } + + /// Saves the workspace to files + void save_workspace(cutlass::HostTensorView<CType> const& experimental, + cutlass::HostTensorView<CType> const& ref) { + std::string name = workspace_name(); + + std::string results_name = name + "_results.txt"; + std::string errors_name = name + "_errors.txt"; + + std::ofstream results(results_name.c_str()); + std::ofstream errors(errors_name.c_str()); + + write(results); + write_errors(errors, experimental, ref); + } + + /// Verifies the contents of C equal the host-side reference + bool verify_with_host(bool save_on_error = true, bool always_print = false) { + compute_host(); + computed.sync_host(); + + bool passed = computed.bit_equals(ref_host); + + if ((!passed && save_on_error) || always_print) { + save_workspace(computed, ref_host); + } + return passed; + } +}; + +} // namespace test + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass { +inline int convert(cutlass::MatrixLayout::Kind layout) { + switch (layout) { + case cutlass::MatrixLayout::kRowMajor: + return 0;//CUBLAS_OP_T; + case cutlass::MatrixLayout::kColumnMajor: + return 1;//CUBLAS_OP_N; + default: + break; + } + return 1;//CUBLAS_OP_N; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +} diff --git a/cutlass-example/gpgpusim.config b/cutlass-example/gpgpusim.config new file mode 100644 index 0000000..2510d21 --- /dev/null +++ b/cutlass-example/gpgpusim.config @@ -0,0 +1,151 @@ +# This config models the Pascal GP102 (GeForceGTX 1080Ti) + +# functional simulator specification +-gpgpu_ptx_instruction_classification 0 +-gpgpu_ptx_sim_mode 0 +-gpgpu_ptx_force_max_capability 70 + +# SASS execution (only supported with CUDA >= 4.0) +-gpgpu_ptx_convert_to_ptxplus 0 +-gpgpu_ptx_save_converted_ptxplus 0 + +# high level architecture configuration +-gpgpu_n_clusters 80 +-gpgpu_n_cores_per_cluster 1 +-gpgpu_n_mem 11 +-gpgpu_n_sub_partition_per_mchannel 2 + +# Pascal clock domains +#-gpgpu_clock_domains <Core Clock>:<Interconnect Clock>:<L2 Clock>:<DRAM Clock> +# Pascal NVIDIA TITAN X clock domains are adopted from +# https://en.wikipedia.org/wiki/GeForce_10_series +-gpgpu_clock_domains 1481.0:2962.0:1481.0:2750.0 + +# shader core pipeline config +-gpgpu_shader_registers 65536 + +# This implies a maximum of 64 warps/SM +-gpgpu_shader_core_pipeline 2048:32 +-gpgpu_shader_cta 32 +-gpgpu_simd_model 1 + +# Pipeline widths and number of FUs +# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB +## Pascal GP102 has 4 SP SIMD units and 1 SFU unit +## we need to scale the number of pipeline registers to be equal to the number of SP units +-gpgpu_pipeline_widths 4,1,1,1,4,1,1,1,6 +-gpgpu_num_sp_units 4 +-gpgpu_num_sfu_units 1 +# Instruction latencies and initiation intervals +# "ADD,MAX,MUL,MAD,DIV" +# SFU is 32-width in pascal, then dp units initiation is 1 cycle +-ptx_opcode_latency_int 4,13,4,5,145,16,4 +-ptx_opcode_initiation_int 1,2,2,2,8,16,4 +-ptx_opcode_latency_fp 4,13,4,5,39 +-ptx_opcode_initiation_fp 1,2,1,1,4 +-ptx_opcode_latency_dp 8,19,8,8,330 +-ptx_opcode_initiation_dp 1,2,1,1,130 + +# <nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>:<set_index_fn>,<mshr>:<N>:<merge>,<mq>:**<fifo_entry> +# ** Optional parameter - Required when mshr_type==Texture Fifo +# Note: Hashing set index function (H) only applies to a set size of 32 or 64. +# Pascal GP102 has 96KB Shared memory +# Pascal GP102 has 64KB L1 cache +# The default is to disable the L1 cache, unless cache modifieres is used +-gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:128:8,8 +-gpgpu_shmem_size 98304 +-gmem_skip_L1D 1 + +# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache +-gpgpu_cache:dl2 64:128:16,L:B:m:W:L,A:1024:1024,4:0,32 # used to be 128:4 +-gpgpu_cache:dl2_texture_only 0 + +# 4 KB Inst. +-gpgpu_cache:il1 8:128:4,L:R:f:N:L,A:2:48,4 +# 48 KB Tex +-gpgpu_tex_cache:l1 16:128:24,L:R:m:N:L,F:128:4,128:2 +# 12 KB Const +-gpgpu_const_cache:l1 128:64:2,L:R:f:N:L,A:2:64,4 + +# enable operand collector +## larger operand collectors and reg_banks are needed for the 4 warp schedulers and 4 SIMD units +-gpgpu_operand_collector_num_units_sp 20 +-gpgpu_operand_collector_num_units_sfu 4 +#-gpgpu_operand_collector_num_units_tensor_core 24 +-gpgpu_operand_collector_num_units_mem 8 +-gpgpu_operand_collector_num_in_ports_sp 4 +-gpgpu_operand_collector_num_out_ports_sp 4 +-gpgpu_operand_collector_num_in_ports_sfu 1 +-gpgpu_operand_collector_num_out_ports_sfu 1 +#-gpgpu_operand_collector_num_in_ports_tensor_core 1 +#-gpgpu_operand_collector_num_out_ports_tensor_core 1 +-gpgpu_operand_collector_num_in_ports_mem 10 +-gpgpu_operand_collector_num_out_ports_mem 10 +# gpgpu_num_reg_banks should be increased to 32, but it gives an error! +-gpgpu_num_reg_banks 32 + +# shared memory bankconflict detection +-gpgpu_shmem_num_banks 32 +-gpgpu_shmem_limited_broadcast 0 +-gpgpu_shmem_warp_parts 1 + +## In Pascal, a warp scheduler can issue 2 insts per cycle +-gpgpu_max_insn_issue_per_warp 2 + +# interconnection +-network_mode 1 +-inter_config_file config_fermi_islip.icnt + +# memory partition latency config +-rop_latency 120 +-dram_latency 100 + +# dram model config +-gpgpu_dram_scheduler 1 +# The DRAM return queue and the scheduler queue together should provide buffer +# to sustain the memory level parallelism to tolerate DRAM latency +# To allow 100% DRAM utility, there should at least be enough buffer to sustain +# the minimum DRAM latency (100 core cycles). I.e. +# Total buffer space required = 100 x 924MHz / 700MHz = 132 +-gpgpu_frfcfs_dram_sched_queue_size 64 +-gpgpu_dram_return_queue_size 116 + +# for NVIDIA GeForceGTX 1080Ti, bus width is 352bits (11 DRAM chips x 32 bits) +# 11 memory paritions, 4 bytes (1 DRAM chip) per memory partition +# the atom size of GDDR5X (the smallest read request) is 32 bytes +-gpgpu_n_mem_per_ctrlr 1 +-gpgpu_dram_buswidth 4 +-gpgpu_dram_burst_length 8 +-dram_data_command_freq_ratio 4 # GDDR5X is QDR +-gpgpu_mem_address_mask 1 +-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS + +# Use the same GDDR5 timing from hynix H5GQ1H24AFR +# disable bank groups for now, set nbkgrp to 1 and tCCDL and tRTPL to 0 +-gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: + CL=12:WL=4:CDLR=5:WR=12:nbkgrp=1:CCDL=0:RTPL=0" + +# Pascal has four schedulers per core +-gpgpu_num_sched_per_core 2 +# Two Level Scheduler with active and pending pools +#-gpgpu_scheduler two_level_active:6:0:1 +# Loose round robbin scheduler +#-gpgpu_scheduler lrr +# Greedy then oldest scheduler +-gpgpu_scheduler gto + +# stat collection +-gpgpu_memlatency_stat 14 +-gpgpu_runtime_stat 500 +-enable_ptx_file_line_stats 1 +-visualizer_enabled 0 + +# power model configs +-power_simulation_enabled 1 +-gpuwattch_xml_file gpuwattch_gtx1080Ti.xml + +# tracing functionality +#-trace_enabled 1 +#-trace_components WARP_SCHEDULER,SCOREBOARD +#-trace_sampling_core 0 + diff --git a/cutlass-example/gpuwattch_gtx1080Ti.xml b/cutlass-example/gpuwattch_gtx1080Ti.xml new file mode 100755 index 0000000..02619ff --- /dev/null +++ b/cutlass-example/gpuwattch_gtx1080Ti.xml @@ -0,0 +1,538 @@ +<?xml version="1.0" ?> +<component id="root" name="root"> + <component id="system" name="system"> + <!--McPAT will skip the components if number is set to 0 --> + <param name="GPU_Architecture" value="1"/><!-- 0-G80; 1-Fermi; others not supported --> + <param name="number_of_cores" value="28"/> + <param name="architecture" value="1"/> <!-- fermi:1 quadro:2 other: undefined--> + <param name="number_of_L1Directories" value="0"/> + <param name="number_of_L2Directories" value="0"/> + <param name="number_of_L2s" value="1"/> <!-- This number means how many L2 clusters in each cluster there can be multiple banks/ports --> + <param name="number_of_L3s" value="0"/> <!-- This number means how many L3 clusters --> + <param name="number_of_NoCs" value="1"/> + <param name="homogeneous_cores" value="1"/><!--1 means homo --> + <param name="homogeneous_L2s" value="1"/> + <param name="homogeneous_L1Directorys" value="1"/> + <param name="homogeneous_L2Directorys" value="1"/> + <param name="homogeneous_L3s" value="1"/> + <param name="homogeneous_ccs" value="1"/><!--cache coherece hardware --> + <param name="homogeneous_NoCs" value="1"/> + <param name="core_tech_node" value="23"/><!-- nm --> + <param name="target_core_clockrate" value="1481"/><!--MHz --> + <param name="temperature" value="380"/> <!-- Kelvin --> + <param name="number_cache_levels" value="2"/> + <param name="interconnect_projection_type" value="0"/><!--0: agressive wire technology; 1: conservative wire technology --> + <param name="device_type" value="0"/><!--0: HP(High Performance Type); 1: LSTP(Low standby power) 2: LOP (Low Operating Power) --> + <param name="longer_channel_device" value="1"/><!-- 0 no use; 1 use when possible --> + <param name="machine_bits" value="32"/> + <param name="virtual_address_width" value="32"/> + <param name="physical_address_width" value="32"/> + <param name="virtual_memory_page_size" value="4096"/> + <param name="idle_core_power" value="1.59"/><!-- idle core power for GTX479 --> + <!--param name="scaling_coefficients" value="10,0.0884816,10,10,8,10,4.12782,10,2.48832,10,10,10,4.29982,0.387764,0.0714269,0.14302,0.01,0.546811,0.485351,0.806633,0.818073,1.9207,100,100,100,87.9303,100,10,4.3548,10"/--> + <param name="TOT_INST" value="10" /> + <param name="FP_INT" value="10" /> + <param name="IC_H" value="0.001" /> + <param name="IC_M" value="10" /> + <param name="DC_RH" value="1" /> + <param name="DC_RM" value="1" /> + <param name="DC_WH" value="1" /> + <param name="DC_WM" value="1" /> + <param name="TC_H" value="0.001" /> + <param name="TC_M" value="10" /> + <param name="CC_H" value="4.5071" /> + <param name="CC_M" value="10" /> + <param name="SHRD_ACC" value="10" /> + <param name="REG_RD" value="1.6294" /> + <param name="REG_WR" value="0.5031" /> + <param name="NON_REG_OPs" value="0.01" /> + <param name="SP_ACC" value="10" /> + <param name="SFU_ACC" value="0.0082" /> + <param name="FPU_ACC" value="0.4126" /> + <param name="MEM_RD" value="0.1234" /> + <param name="MEM_WR" value="0.001" /> + <param name="MEM_PRE" value="0.001" /> + <param name="L2_RH" value="100" /> + <param name="L2_RM" value="100" /> + <param name="L2_WH" value="100" /> + <param name="L2_WM" value="42.6966" /> + <param name="NOC_A" value="100" /> + <param name="PIPE_A" value="44.8085" /> + <param name="IDLE_CORE_N" value="2.0382"/> + <param name="CONST_DYNAMICN" value="5.0005" /> + <stat name="num_idle_cores" value="0"/><!-- Average Number of idle cores during this period --> + <stat name="total_cycles" value="total_cycles_match_mcpat"/> + <stat name="idle_cycles" value="idle_cycles_match_mcpat"/> + <stat name="busy_cycles" value="busy_cycles_match_mcpat"/> + <!--This page size(B) is complete different from the page size in Main memo secction. this page size is the size of + virtual memory from OS/Archi perspective; the page size in Main memo secction is the actuall physical line in a DRAM bank --> + <!-- *********************** cores ******************* --> + <component id="system.core0" name="core0"> + <!-- Core property --> + <param name="clock_rate" value="1481"/> + <param name="instruction_length" value="32"/> + <param name="opcode_width" value="9"/> + <!-- address width determins the tag_width in Cache, LSQ and buffers in cache controller + default value is machine_bits, if not set --> + <param name="machine_type" value="1"/><!-- 1 inorder; 0 OOO--> + <!-- inorder/OoO --> + <param name="number_hardware_threads" value="32"/> + <!-- number_instruction_fetch_ports(icache ports) is always 1 in single-thread processor, + it only may be more than one in SMT processors. BTB ports always equals to fetch ports since + branch information in consective branch instructions in the same fetch group can be read out from BTB once.--> + <param name="fetch_width" value="1"/> + <!-- fetch_width determins the size of cachelines of L1 cache block --> + <param name="number_instruction_fetch_ports" value="1"/> + <param name="decode_width" value="1"/> + <!-- decode_width determins the number of ports of the + renaming table (both RAM and CAM) scheme --> + <param name="issue_width" value="2"/> + <!-- issue_width determins the number of ports of Issue window and other logic + as in the complexity effective proccessors paper; issue_width==dispatch_width --> + <param name="commit_width" value="2"/> + <!-- commit_width determins the number of ports of register files --> + <param name="fp_issue_width" value="1"/> + <param name="prediction_width" value="0"/> + <!-- number of branch instructions can be predicted simultannouesl--> + <!-- Current version of McPAT does not distinguish int and floating point pipelines + Theses parameters are reserved for future use.--> + <param name="pipelines_per_core" value="1,1"/> + <!--integer_pipeline and floating_pipelines, if the floating_pipelines is 0, then the pipeline is shared--> + <param name="pipeline_depth" value="8,8"/> + <!-- pipeline depth of int and fp, if pipeline is shared, the second number is the average cycles of fp ops --> + <!-- issue and exe unit--> + <param name="ALU_per_core" value="32"/> + <!-- contains an adder, a shifter, and a logical unit --> + <param name="MUL_per_core" value="4"/> + <!-- For MUL and Div --> + <param name="FPU_per_core" value="32"/> + <!-- buffer between IF and ID stage --> + <param name="instruction_buffer_size" value="1"/> + <!-- buffer between ID and sche/exe stage --> + <param name="decoded_stream_buffer_size" value="1"/> + <param name="instruction_window_scheme" value="0"/><!-- 0 PHYREG based, 1 RSBASED--> + <!-- McPAT support 2 types of OoO cores, RS based and physical reg based--> + <param name="instruction_window_size" value="1"/> + <param name="fp_instruction_window_size" value="1"/> + <!-- the instruction issue Q as in Alpha 21264; The RS as in Intel P6 --> + <param name="ROB_size" value="0"/> + <!-- each in-flight instruction has an entry in ROB --> + <!-- registers --> + <!-- SM parameters Added by Syed Gilani --> + <param name="rf_banks" value="32"/> + <param name="simd_width" value="32"/> + <param name="collector_units" value="32"/> + <param name="core_clock_ratio" value="2"/> + <param name="warp_size" value="32"/> + + <param name="archi_Regs_IRF_size" value="65536"/> + <param name="archi_Regs_FRF_size" value="32"/> + <!-- if OoO processor, phy_reg number is needed for renaming logic, + renaming logic is for both integer and floating point insts. --> + <param name="phy_Regs_IRF_size" value="32"/> + <param name="phy_Regs_FRF_size" value="32"/> + <!-- rename logic --> + <param name="rename_scheme" value="0"/> + <!-- can be RAM based(0) or CAM based(1) rename scheme + RAM-based scheme will have free list, status table; + CAM-based scheme have the valid bit in the data field of the CAM + both RAM and CAM need RAM-based checkpoint table, checkpoint_depth=# of in_flight instructions; + Detailed RAT Implementation see TR --> + <param name="register_windows_size" value="0"/> + <!-- how many windows in the windowed register file, sun processors; + no register windowing is used when this number is 0 --> + <!-- In OoO cores, loads and stores can be issued whether inorder(Pentium Pro) or (OoO)out-of-order(Alpha), + They will always try to exeute out-of-order though. --> + <param name="LSU_order" value="inorder"/> + <param name="store_buffer_size" value="32"/> + <!-- By default, in-order cores do not have load buffers --> + <param name="load_buffer_size" value="32"/> + <!-- number of ports refer to sustainable concurrent memory accesses --> + <param name="memory_ports" value="2"/> + <!-- max_allowed_in_flight_memo_instructions determins the # of ports of load and store buffer + as well as the ports of Dcache which is connected to LSU --> + <!-- dual-pumped Dcache can be used to save the extra read/write ports --> + <param name="RAS_size" value="1"/> + <!-- general stats, defines simulation periods;require total, idle, and busy cycles for senity check --> + <!-- please note: if target architecture is X86, then all the instrucions refer to (fused) micro-ops --> + <stat name="total_instructions" value="total_instructions_match_mcpat"/> + <stat name="int_instructions" value="int_instruction_match_mcpat"/> + <stat name="fp_instructions" value="flt_instruction_match_mcpat"/> + <stat name="branch_instructions" value="branch_instruction_match_mcpat"/> + <stat name="branch_mispredictions" value="0"/> + <stat name="load_instructions" value="load_instruction_match_mcpat"/> + <stat name="store_instructions" value="store_instruction_match_mcpat"/> + <stat name="committed_instructions" value="total_instructions_match_mcpat"/> + <stat name="committed_int_instructions" value="int_instruction_match_mcpat"/> + <stat name="committed_fp_instructions" value="flt_instruction_match_mcpat"/> + <stat name="pipeline_duty_cycle" value="0.6"/><!--<=1, runtime_ipc/peak_ipc; averaged for all cores if homogenous --> + <!-- the following cycle stats are used for heterogeneouse cores only, + please ignore them if homogeneouse cores --> + <stat name="total_cycles" value="total_cycles_match_mcpat"/> + <stat name="idle_cycles" value="idle_cycles_match_mcpat"/> + <stat name="busy_cycles" value="busy_cycles_match_mcpat"/> + <!-- instruction buffer stats --> + <!-- ROB stats, both RS and Phy based OoOs have ROB + performance simulator should capture the difference on accesses, + otherwise, McPAT has to guess based on number of commited instructions. --> + <stat name="ROB_reads" value="263886"/> + <stat name="ROB_writes" value="263886"/> + <!-- RAT accesses --> + <stat name="rename_accesses" value="263886"/> + <stat name="fp_rename_accesses" value="263886"/> + <!-- decode and rename stage use this, should be total ic - nop --> + <!-- Inst window stats --> + <stat name="inst_window_reads" value="263886"/> + <stat name="inst_window_writes" value="263886"/> + <stat name="inst_window_wakeup_accesses" value="263886"/> + <stat name="fp_inst_window_reads" value="263886"/> + <stat name="fp_inst_window_writes" value="263886"/> + <stat name="fp_inst_window_wakeup_accesses" value="263886"/> + <!-- RF accesses --> + <stat name="int_regfile_reads" value="int_register_read_access_match_mcpat"/> + <stat name="float_regfile_reads" value="int_register_write_access_match_mcpat"/> + <stat name="int_regfile_writes" value="float_register_read_access_match_mcpat"/> + <stat name="float_regfile_writes" value="float_register_write_access_match_mcpat"/> + + <!-- The following stat is for operand collector power - Added by Syed --> + <stat name="non_rf_operands" value="0"/> + + <!-- accesses to the working reg --> + <stat name="function_calls" value="0"/> + <stat name="context_switches" value="0"/> <!--not used in the McPAT --> + <!-- Number of Windowes switches (number of function calls and returns)--> + <!-- Alu stats by default, the processor has one FPU that includes the divider and + multiplier. The fpu accesses should include accesses to multiplier and divider --> + <stat name="ialu_accesses" value="ialu_accesses_match_mcpat"/> + <stat name="fpu_accesses" value="fpu_accesses_match_mcpat"/> + <stat name="mul_accesses" value="mul_accesses_match_mcpat"/> + <stat name="cdb_alu_accesses" value="0"/> + <stat name="cdb_mul_accesses" value="0"/> + <stat name="cdb_fpu_accesses" value="0"/> + <!-- multiple cycle accesses should be counted multiple times, + otherwise, McPAT can use internal counter for different floating point instructions + to get final accesses. But that needs detailed info for floating point inst mix --> + <!-- currently the performance simulator should + make sure all the numbers are final numbers, + including the explicit read/write accesses, + and the implicite accesses such as replacements and etc. + Future versions of McPAT may be able to reason the implicite access + based on param and stats of last level cache + The same rule applies to all cache access stats too! --> + <!-- following is AF for max power computation. + Do not change them, unless you understand them--> + <stat name="IFU_duty_cycle" value="0.25"/> + <stat name="LSU_duty_cycle" value="0.25"/> + <stat name="MemManU_I_duty_cycle" value="1"/> + <stat name="MemManU_D_duty_cycle" value="0.25"/> + <stat name="ALU_duty_cycle" value="0.9"/> + <stat name="MUL_duty_cycle" value="0.5"/> + <stat name="FPU_duty_cycle" value="1"/><!-- FPU numbers are already average --> + <stat name="ALU_cdb_duty_cycle" value="0.9"/> + <stat name="MUL_cdb_duty_cycle" value="0.5"/> + <stat name="FPU_cdb_duty_cycle" value="15"/> + <component id="system.core0.predictor" name="PBT"> + <!-- branch predictor; tournament predictor see Alpha implementation --> + <param name="local_predictor_size" value="10,3"/> + <param name="local_predictor_entries" value="1024"/> + <param name="global_predictor_entries" value="4096"/> + <param name="global_predictor_bits" value="2"/> + <param name="chooser_predictor_entries" value="4096"/> + <param name="chooser_predictor_bits" value="2"/> + <!-- These parameters can be combined like below in next version + <param name="load_predictor" value="10,3,1024"/> + <param name="global_predictor" value="4096,2"/> + <param name="predictor_chooser" value="4096,2"/> + --> + </component> + <component id="system.core0.itlb" name="itlb"> + <param name="number_entries" value="1"/> + <stat name="total_accesses" value="0"/> + <stat name="total_misses" value="0"/> + <stat name="conflicts" value="0"/> + <!-- there is no write requests to itlb although writes happen to itlb after miss, + which is actually a replacement --> + </component> + <component id="system.core0.icache" name="icache"> + <!-- there is no write requests to itlb although writes happen to it after miss, + which is actually a replacement --> + <param name="icache_config" value="16384,128,4,1,1,3,8,0"/> + <!-- the parameters are capacity,block_width, associativity, bank, throughput w.r.t. core clock, latency w.r.t. core clock,output_width, cache policy --> + <!-- cache_policy;//0 no write or write-though with non-write allocate;1 write-back with write-allocate --> + <param name="buffer_sizes" value="16, 16, 16,0"/> + <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> + <stat name="read_accesses" value="total_instructions_match_mcpat"/> + <stat name="read_misses" value="0"/> + <stat name="conflicts" value="0"/> + </component> + <component id="system.core0.dtlb" name="dtlb"> + <param name="number_entries" value="1"/> + <stat name="total_accesses" value="0"/> + <stat name="total_misses" value="0"/> + <stat name="conflicts" value="0"/> + </component> + <component id="system.core0.ccache" name="ccache"> + <!-- all the buffer related are optional --> + <param name="ccache_config" value="16384,64,2,1,1,3,8,0"/> + <param name="buffer_sizes" value="16, 16, 16, 0"/> + <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> + <stat name="read_accesses" value="ccache_read_accesses_match_mcpat"/> + <stat name="write_accesses" value="0"/> + <stat name="read_misses" value="ccache_read_misses_match_mcpat"/> + <stat name="write_misses" value="0"/> + <stat name="conflicts" value="0"/> + </component> + <component id="system.core0.tcache" name="tcache"> + <!-- all the buffer related are optional --> + <param name="tcache_config" value="49152,128,8,1,1,3,8,0"/> + <param name="buffer_sizes" value="16, 16, 16, 0"/> + <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> + <stat name="read_accesses" value="tcache_read_accesses_match_mcpat"/> + <stat name="write_accesses" value="0"/> + <stat name="read_misses" value="tcache_read_misses_match_mcpat"/> + <stat name="write_misses" value="0"/> + <stat name="conflicts" value="0"/> + </component> + <!--model the shared memory by mimicing dcache--> + <component id="system.core0.sharedmemory" name="sharedmemory"> + <!-- all the buffer related are optional --> + <param name="sharedmemory_config" value="98304,16,1,16,1,3,16,0"/> + <!-- the parameters are capacity,block_width, associativity, bank, throughput w.r.t. core clock, latency w.r.t. core clock,output_width, cache policy --> + <param name="buffer_sizes" value="16, 16, 16, 16"/> + <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> + <stat name="read_accesses" value="sharedmemory_read_access_match_mcpat"/> + <stat name="write_accesses" value="sharedmemory_write_access_match_mcpat"/> + <stat name="read_misses" value="0"/> + <stat name="write_misses" value="0"/> + <stat name="conflicts" value="0"/> + </component> + <component id="system.core0.dcache" name="dcache"> + <!-- all the buffer related are optional --> + <param name="dcache_config" value="16384,32,4,1,1,3,8,0"/> + <param name="buffer_sizes" value="16, 16, 16, 0"/> + <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> + <stat name="read_accesses" value="dcache_read_access_match_mcpat"/> + <stat name="write_accesses" value="dcache_write_access_match_mcpat"/> + <stat name="read_misses" value="dcache_read_miss_match_mcpat"/> + <stat name="write_misses" value="dcache_write_miss_match_mcpat"/> + <stat name="conflicts" value="0"/> + </component> + <component id="system.core0.BTB" name="BTB"> + <!-- all the buffer related are optional --> + <param name="BTB_config" value="8192,4,2,1, 1,3"/> + <!-- the parameters are capacity,block_width,associativity,bank, throughput w.r.t. core clock, latency w.r.t. core clock,--> + </component> + </component> + <component id="system.L1Directory0" name="L1Directory0"> + <param name="Directory_type" value="0"/> + <!--0 cam based shadowed tag. 1 directory cache --> + <param name="Dir_config" value="2048,1,0,1, 4, 4,8"/> + <!-- the parameters are capacity,block_width, associativity,bank, throughput w.r.t. core clock, latency w.r.t. core clock,--> + <param name="buffer_sizes" value="8, 8, 8, 8"/> + <!-- all the buffer related are optional --> + <param name="clockrate" value="1400"/> + <param name="ports" value="1,1,1"/> + <!-- number of r, w, and rw search ports --> + <param name="device_type" value="0"/> + <!-- altough there are multiple access types, + Performance simulator needs to cast them into reads or writes + e.g. the invalidates can be considered as writes --> + <stat name="read_accesses" value="800000"/> + <stat name="write_accesses" value="27276"/> + <stat name="read_misses" value="1632"/> + <stat name="write_misses" value="183"/> + <stat name="conflicts" value="20"/> + <stat name="duty_cycle" value="0.45"/> + </component> + <component id="system.L2Directory0" name="L2Directory0"> + <param name="Directory_type" value="1"/> + <!--0 cam based shadowed tag. 1 directory cache --> + <param name="Dir_config" value="1048576,16,16,1,2, 100"/> + <!-- the parameters are capacity,block_width, associativity,bank, throughput w.r.t. core clock, latency w.r.t. core clock,--> + <param name="buffer_sizes" value="8, 8, 8, 8"/> + <!-- all the buffer related are optional --> + <param name="clockrate" value="1400"/> + <param name="ports" value="1,1,1"/> + <!-- number of r, w, and rw search ports --> + <param name="device_type" value="0"/> + <!-- altough there are multiple access types, + Performance simulator needs to cast them into reads or writes + e.g. the invalidates can be considered as writes --> + <stat name="read_accesses" value="0"/> + <stat name="write_accesses" value="0"/> + <stat name="read_misses" value="0"/> + <stat name="write_misses" value="0"/> + <stat name="conflicts" value="0"/> + <stat name="duty_cycle" value="0.45"/> + </component> + <component id="system.L20" name="L20"> + <!-- all the buffer related are optional --> + <param name="L2_config" value="131072,128,16,1, 4,23, 64, 1"/> + <!-- consider 4-way bank interleaving for Niagara 1 --> + <!-- the parameters are capacity,block_width, associativity, bank, throughput w.r.t. core clock, latency w.r.t. core clock,output_width, cache policy --> + <param name="buffer_sizes" value="16, 16, 16, 16"/> + <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> + <param name="clockrate" value="2962"/> + <param name="ports" value="1,1,1"/> + <!-- number of r, w, and rw ports --> + <param name="device_type" value="0"/> + <stat name="read_accesses" value="200000"/> + <stat name="write_accesses" value="0"/> + <stat name="read_misses" value="0"/> + <stat name="write_misses" value="0"/> + <stat name="conflicts" value="0"/> + <stat name="duty_cycle" value="0.5"/> + </component> + +<!--**********************************************************************--> +<component id="system.L30" name="L30"> + <param name="L3_config" value="1048576,64,16,1, 2,100, 64,1"/> + <!-- the parameters are capacity,block_width, associativity, bank, throughput w.r.t. core clock, latency w.r.t. core clock,output_width, cache policy --> + <param name="clockrate" value="3500"/> + <param name="ports" value="1,1,1"/> + <!-- number of r, w, and rw ports --> + <param name="device_type" value="0"/> + <param name="buffer_sizes" value="16, 16, 16, 16"/> + <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> + <stat name="read_accesses" value="58824"/> + <stat name="write_accesses" value="27276"/> + <stat name="read_misses" value="1632"/> + <stat name="write_misses" value="183"/> + <stat name="conflicts" value="0"/> + <stat name="duty_cycle" value="0.35"/> + </component> + + +<!--**********************************************************************--> + <component id="system.NoC0" name="noc0"> + <param name="clockrate" value="700"/> + <param name="type" value="1"/> + <!-- 1 NoC, O bus --> + <param name="horizontal_nodes" value="2"/> + <param name="vertical_nodes" value="1"/> + <param name="has_global_link" value="0"/> + <!-- 1 has global link, 0 does not have global link --> + <param name="link_throughput" value="1"/><!--w.r.t clock --> + <param name="link_latency" value="1"/><!--w.r.t clock --> + <!-- througput >= latency --> + <!-- Router architecture --> + <param name="input_ports" value="6"/> + <param name="output_ports" value="6"/> + <param name="virtual_channel_per_port" value="1"/> + <!-- input buffer; in classic routers only input ports need buffers --> + <param name="flit_bits" value="32"/> + <param name="input_buffer_entries_per_vc" value="1"/><!--VCs within the same ports share input buffers whose size is propotional to the number of VCs--> + <param name="chip_coverage" value="1"/> + <!-- When multiple NOC present, one NOC will cover part of the whole chip. chip_coverage <=1 --> + <stat name="total_accesses" value="0"/> + <!-- This is the number of total accesses within the whole network not for each router --> + <stat name="duty_cycle" value="0.6"/> + </component> +<!--**********************************************************************--> +<!--**********************************************************************--> + + <component id="system.mem" name="mem"> + <!-- Main memory property --> + <param name="mem_tech_node" value="23"/> + <param name="device_clock" value="200"/><!--MHz, this is clock rate of the actual memory device, not the FSB --> + <param name="peak_transfer_rate" value="3200"/><!--MB/S--> + <param name="internal_prefetch_of_DRAM_chip" value="4"/> + <!-- 2 for DDR, 4 for DDR2, 8 for DDR3...--> + <!-- the device clock, peak_transfer_rate, and the internal prefetch decide the DIMM property --> + <!-- above numbers can be easily found from Wikipedia --> + <param name="capacity_per_channel" value="4096"/> <!-- MB --> + <!-- capacity_per_Dram_chip=capacity_per_channel/number_of_dimms/number_ranks/Dram_chips_per_rank + Current McPAT assumes single DIMMs are used.--> + <param name="number_ranks" value="2"/> + <param name="num_banks_of_DRAM_chip" value="6"/> + <param name="Block_width_of_DRAM_chip" value="64"/> <!-- B --> + <param name="output_width_of_DRAM_chip" value="8"/> + <!--number of Dram_chips_per_rank=" 72/output_width_of_DRAM_chip--> + <!--number of Dram_chips_per_rank=" 72/output_width_of_DRAM_chip--> + <param name="page_size_of_DRAM_chip" value="8"/> <!-- 8 or 16 --> + <param name="burstlength_of_DRAM_chip" value="8"/> + <stat name="memory_accesses" value="1052"/> + <stat name="memory_reads" value="1052"/> + <stat name="memory_writes" value="1052"/> + </component> + <component id="system.mc" name="mc"> + <!-- Memeory controllers are for DDR(2,3...) DIMMs --> + <!-- current version of McPAT uses published values for base parameters of memory controller + improvments on MC will be added in later versions. --> + <param name="type" value="0"/> <!-- 1: low power; 0 high performance --> + <param name="mc_clock" value="1848"/><!--DIMM IO bus clock rate MHz DDR2-400 for Niagara 1--> + <param name="peak_transfer_rate" value="29568"/><!--MB/S Syed: GTX 470 has 177.4GB/s mem transfer rate with 6 MCs --> + <param name="block_size" value="64"/><!--B--> + <param name="number_mcs" value="6"/><!-- 6 GDDR5 memory controllers --> + <!-- current McPAT only supports homogeneous memory controllers --> + <param name="memory_channels_per_mc" value="2"/> + <param name="number_ranks" value="1"/> + <param name="withPHY" value="0"/> + <!-- # of ranks of each channel--> + <param name="req_window_size_per_channel" value="16"/> + <param name="IO_buffer_size_per_channel" value="16"/> + <param name="databus_width" value="32"/> + <param name="addressbus_width" value="32"/> + <param name="PRT_entries" value="32"/> + <!-- # of empirical DRAM model parameter --> + <param name="dram_cmd_coeff" value="0"/> + <param name="dram_act_coeff" value="0"/> + <param name="dram_nop_coeff" value="0"/> + <param name="dram_activity_coeff" value="0"/> + <param name="dram_pre_coeff" value="3.8475e-8f"/> + <param name="dram_rd_coeff" value="7.74707143e-8f"/> + <param name="dram_wr_coeff" value="3.54664286e-8f"/> + <param name="dram_req_coeff" value="0"/> + <param name="dram_const_coeff" value="0"/> + + <!-- McPAT will add the control bus width to the addressbus width automatically --> + <stat name="memory_accesses" value="memory_accesses_match_mcpat"/> + <stat name="memory_reads" value="memory_reads_match_mcpat"/> + <stat name="memory_writes" value="memory_writes_match_mcpat"/> + <!-- McPAT does not track individual mc, instead, it takes the total accesses and calculate + the average power per MC or per channel. This is sufficent for most application. + Further trackdown can be easily added in later versions. --> + </component> +<!--**********************************************************************--> + <component id="system.niu" name="niu"> + <!-- On chip 10Gb Ethernet NIC, including XAUI Phy and MAC controller --> + <!-- For a minimum IP packet size of 84B at 10Gb/s, a new packet arrives every 67.2ns. + the low bound of clock rate of a 10Gb MAC is 150Mhz --> + <param name="type" value="0"/> <!-- 1: low power; 0 high performance --> + <param name="clockrate" value="350"/> + <param name="number_units" value="0"/> <!-- unlike PCIe and memory controllers, each Ethernet controller only have one port --> + <stat name="duty_cycle" value="1.0"/> <!-- achievable max load <= 1.0 --> + <stat name="total_load_perc" value="0.7"/> <!-- ratio of total achived load to total achivable bandwidth --> + <!-- McPAT does not track individual nic, instead, it takes the total accesses and calculate + the average power per nic or per channel. This is sufficent for most application. --> + </component> +<!--**********************************************************************--> + <component id="system.pcie" name="pcie"> + <!-- On chip PCIe controller, including Phy--> + <!-- For a minimum PCIe packet size of 84B at 8Gb/s per lane (PCIe 3.0), a new packet arrives every 84ns. + the low bound of clock rate of a PCIe per lane logic is 120Mhz --> + <param name="type" value="0"/> <!-- 1: low power; 0 high performance --> + <param name="withPHY" value="1"/> + <param name="clockrate" value="350"/> + <param name="number_units" value="0"/> + <param name="num_channels" value="8"/> <!-- 2 ,4 ,8 ,16 ,32 --> + <stat name="duty_cycle" value="1.0"/> <!-- achievable max load <= 1.0 --> + <stat name="total_load_perc" value="0.7"/> <!-- Percentage of total achived load to total achivable bandwidth --> + <!-- McPAT does not track individual pcie controllers, instead, it takes the total accesses and calculate + the average power per pcie controller or per channel. This is sufficent for most application. --> + </component> +<!--**********************************************************************--> + <component id="system.flashc" name="flashc"> + <param name="number_flashcs" value="0"/> + <param name="type" value="1"/> <!-- 1: low power; 0 high performance --> + <param name="withPHY" value="1"/> + <param name="peak_transfer_rate" value="200"/><!--Per controller sustainable reak rate MB/S --> + <stat name="duty_cycle" value="1.0"/> <!-- achievable max load <= 1.0 --> + <stat name="total_load_perc" value="0.7"/> <!-- Percentage of total achived load to total achivable bandwidth --> + <!-- McPAT does not track individual flash controller, instead, it takes the total accesses and calculate + the average power per fc or per channel. This is sufficent for most application --> + </component> +<!--**********************************************************************--> + + </component> +</component> diff --git a/cutlass-example/half.h b/cutlass-example/half.h new file mode 100644 index 0000000..ee536e5 --- /dev/null +++ b/cutlass-example/half.h @@ -0,0 +1,743 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! + \file + \brief Host-side implementation of half-precision float +*/ + +#pragma once + +#include <stdint.h> +#include <cmath> +#include <limits> +#include <utility> +#include <utility> + +#include <iomanip> +#include <istream> +#include <ostream> + +#include <cuda_fp16.h> + +namespace cutlass { + +/// IEEE binary16 floating-point value +class half_t { + public: + half_t(); + half_t(int); /// conversion from integer + half_t(float); /// conversion from fp32 + half_t(double); /// conversion from fp64 + + static half_t bitcast(unsigned short); /// bitcast performs no conversion + + static half_t convert(float const&); /// FP conversion - round toward nearest even + static float convert(unsigned short const&); /// floating point conversion to fp32 + + static half_t zero() { return bitcast(0); } /// +zero + static half_t one() { return bitcast(0x3c00); } /// one + static half_t nan() { return bitcast(0x7fff); } /// canonical not a number + static half_t inf() { return bitcast(0x7c00); } /// +infinity + static half_t ninf() { return bitcast(0xfc00); } /// -infinity + static half_t epsilon() { return bitcast(0x1000); } /// Machine epsilon + + bool signbit() const; /// sign bit - true: negative, false: positive + int exponent() const; /// unbiased exponent + unsigned short mantissa() const; /// mantissa bits + + bool isfinite() const; /// true if neither inf nor nan + bool isinf() const; /// true if value is + or - infinity + bool isnan() const; /// true if value is not a number + bool isnormal() const; /// true if nonzero value is normalized + bool iszero() const; /// true if value is + or - zero + + bool operator==(half_t const&) const; + bool operator!=(half_t const&) const; + bool operator==(float const&) const; + bool operator!=(float const&) const; + + bool operator<(half_t const&) const; + bool operator<=(half_t const&) const; + bool operator>(half_t const&) const; + bool operator>=(half_t const&) const; + + half_t operator+(half_t const&) const; + half_t operator-() const; + half_t operator-(half_t const&) const; + half_t operator*(half_t const&)const; + half_t operator/(half_t const&) const; + + half_t& operator+=(half_t const&); + half_t& operator-=(half_t const&); + half_t& operator*=(half_t const&); + half_t& operator/=(half_t const&); + + half_t& operator++(); + half_t& operator--(); + half_t operator++(int); + half_t operator--(int); + + operator bool() const; /// false if zero + operator int() const; /// conversion to int + operator float() const; /// conversion to fp32 + operator half() const; /// conversion to half + + uint16_t& raw() { return x; } + uint16_t raw() const { return x; } + + public: + /// data + unsigned short x; +}; + +/// Packed pair of half-precision elements +class half2_t { + public: + half2_t(); + half2_t(half_t lo, half_t hi); + half2_t(std::pair<float, float> const&); + explicit half2_t(unsigned data); + + half2_t operator+(half2_t const&) const; + half2_t operator-(half2_t const&) const; + half2_t operator*(half2_t const&)const; + half2_t operator/(half2_t const&) const; + + half2_t& operator+=(half2_t const&); + half2_t& operator-=(half2_t const&); + half2_t& operator*=(half2_t const&); + half2_t& operator/=(half2_t const&); + + float dot(half2_t const&) const; /// dot product with single-precision accumulation + float dot(half2_t const&, float) const; /// dot product with single-precision accumulation + + half_t doth(half2_t const&) const; /// dot product with half_t-precision accumulation + half_t doth(half2_t const&, half_t) const; /// dot product with half_t-precision accumulation + + unsigned packed() const; + + operator std::pair<float, float>() const; + operator unsigned() const; + + public: + half_t lo; + half_t hi; +}; + +template <typename Dest, typename Src> +Dest bitcast(Src const&); +template <> +float bitcast<float, unsigned>(unsigned const&); +template <> +float bitcast<float, int>(int const&); +template <> +unsigned bitcast<unsigned, float>(float const&); +template <> +half_t bitcast<half_t, unsigned short>(unsigned short const&); +template <> +unsigned short bitcast<unsigned short, half_t>(half_t const&); +template <> +half bitcast<half, unsigned short>(unsigned short const&); +} // namespace cutlass + +cutlass::half_t operator+(float, cutlass::half_t const&); +cutlass::half_t operator-(float, cutlass::half_t const&); +cutlass::half_t operator*(float, cutlass::half_t const&); +cutlass::half_t operator/(float, cutlass::half_t const&); + +std::ostream& operator<<(std::ostream&, cutlass::half_t const&); /// writes a half_t +std::istream& operator>>(std::istream&, cutlass::half_t&); /// reads a half_t + +#ifdef BOOST_LEXICAL_CAST_INCLUDED +namespace boost { + +/// lexical cast from string to half_t +template <> +cutlass::half_t lexical_cast<cutlass::half_t>(std::string const& arg); + +/// lexical cast from half_t to string +template <> +std::string lexical_cast<std::string>(cutlass::half_t const& arg); +} // namespace boost +#endif + +#define HLF_MANT_DIG 10 + +namespace std { + +cutlass::half_t abs(cutlass::half_t const&); /// absolute value + +bool isnan(cutlass::half_t const&); /// true if argument is NaN + +bool isfinite(cutlass::half_t const&); /// true if argument is neither NaN nor infinity + +cutlass::half_t nanh(const char* = 0); /// returns a not-a-number + +bool isinf(cutlass::half_t const&); /// returns true if argument is infinitey (+ or -) + +bool isnormal( + cutlass::half_t const&); /// returns true if argument is normal (neither zero nor infinity) + +int fpclassify(cutlass::half_t const&); /// returns a flag classifying floating-point value + +bool signbit(cutlass::half_t const&); /// returns true if negative, false if positive + +cutlass::half_t sqrt(cutlass::half_t const&); /// square root of half_t + +/// Numeric limits +template <> +struct numeric_limits<cutlass::half_t> { + static bool const is_specialized = true; + static bool const is_signed = true; + static bool const is_integer = false; + static bool const is_exact = false; + static bool const has_infinity = true; + static bool const has_quiet_NaN = true; + static bool const has_signaling_NaN = false; + static std::float_denorm_style const has_denorm = std::denorm_present; + static bool const has_denorm_loss = true; + static std::float_round_style const round_style = std::round_to_nearest; + static bool const is_iec559 = false; + static bool const is_bounded = true; + static bool const is_modulo = false; + static int const digits = HLF_MANT_DIG; + + static cutlass::half_t min() { return cutlass::half_t::bitcast(0x0001); } + + static cutlass::half_t lowest() { return cutlass::half_t::bitcast(0xfbff); } + + static cutlass::half_t max() { return cutlass::half_t::bitcast(0x7bff); } + + /// Returns smallest finite value + static cutlass::half_t epsilon() { return cutlass::half_t::epsilon(); } + + /// Returns smallest finite value + static cutlass::half_t round_error() { return cutlass::half_t(0.5f); } + + /// Returns smallest finite value + static cutlass::half_t infinity() { return cutlass::half_t::inf(); } + + /// Returns smallest finite value + static cutlass::half_t quiet_NaN() { return cutlass::half_t::nan(); } + + /// Returns smallest finite value + static cutlass::half_t signaling_NaN() { return cutlass::half_t::nan(); } + + /// Returns smallest finite value + static cutlass::half_t denorm_min() { return cutlass::half_t::bitcast(0x0001); } +}; +} // namespace std + +// +// +// + +inline cutlass::half_t cutlass::half_t::bitcast(unsigned short _x) { + half_t h; + h.x = _x; + return h; +} + +/// FP32 -> FP16 conversion - rounds to nearest even +inline cutlass::half_t cutlass::half_t::convert(float const& flt) { + // software implementation rounds toward nearest even + unsigned const& s = *reinterpret_cast<unsigned const*>(&flt); + uint16_t sign = uint16_t((s >> 16) & 0x8000); + int16_t exp = uint16_t(((s >> 23) & 0xff) - 127); + int mantissa = s & 0x7fffff; + uint16_t u = 0; + + if ((s & 0x7fffffff) == 0) { + // sign-preserving zero + return cutlass::half_t::bitcast(sign); + } + + if (exp > 15) { + if (exp == 128 && mantissa) { + // not a number + u = 0x7fff; + } else { + // overflow to infinity + u = sign | 0x7c00; + } + return cutlass::half_t::bitcast(u); + } + + int sticky_bit = 0; + + if (exp >= -14) { + // normal fp32 to normal fp16 + exp = uint16_t(exp + uint16_t(15)); + u = uint16_t(((exp & 0x1f) << 10)); + u = uint16_t(u | (mantissa >> 13)); + } else { + // normal single-precision to subnormal half_t-precision representation + int rshift = (-14 - exp); + if (rshift < 32) { + mantissa |= (1 << 23); + + sticky_bit = ((mantissa & ((1 << rshift) - 1)) != 0); + + mantissa = (mantissa >> rshift); + u = (uint16_t(mantissa >> 13) & 0x3ff); + } else { + mantissa = 0; + u = 0; + } + } + + // round to nearest even + int round_bit = ((mantissa >> 12) & 1); + sticky_bit |= ((mantissa & ((1 << 12) - 1)) != 0); + + if ((round_bit && sticky_bit) || (round_bit && (u & 1))) { + u = uint16_t(u + 1); + } + + u |= sign; + + return cutlass::half_t::bitcast(u); +} + +inline float cutlass::half_t::convert(unsigned short const& h) { + int sign = ((h >> 15) & 1); + int exp = ((h >> 10) & 0x1f); + int mantissa = (h & 0x3ff); + unsigned f = 0; + + if (exp > 0 && exp < 31) { + // normal + exp += 112; + f = (sign << 31) | (exp << 23) | (mantissa << 13); + } else if (exp == 0) { + if (mantissa) { + // subnormal + exp += 113; + while ((mantissa & (1 << 10)) == 0) { + mantissa <<= 1; + exp--; + } + mantissa &= 0x3ff; + f = (sign << 31) | (exp << 23) | (mantissa << 13); + } else { + // sign-preserving zero + f = (sign << 31); + } + } else if (exp == 31) { + if (mantissa) { + f = 0x7fffffff; // not a number + } else { + f = (0xff << 23) | (sign << 31); // inf + } + } + return *reinterpret_cast<float const*>(&f); +} + +inline cutlass::half_t::half_t() {} + +inline cutlass::half_t::half_t(int i) { x = convert(float(i)).x; } + +inline cutlass::half_t::half_t(float f) { x = convert(f).x; } + +inline cutlass::half_t::half_t(double d) { x = convert(float(d)).x; } + +inline bool cutlass::half_t::signbit() const { return (x >> 15) & 1; } + +inline int cutlass::half_t::exponent() const { return ((x >> 10) & 0x1f) - 15; } + +inline unsigned short cutlass::half_t::mantissa() const { return x & 0x3ff; } + +inline cutlass::half_t::operator bool() const { return (x & 0x7fff) != 0; } + +inline cutlass::half_t::operator int() const { return static_cast<int>(convert(x)); } + +inline cutlass::half_t::operator float() const { return convert(x); } + +inline cutlass::half_t::operator half() const { return cutlass::bitcast<half, unsigned short>(x); } + +inline bool cutlass::half_t::operator==(cutlass::half_t const& h) const { + if (iszero() && h.iszero()) { + return true; + } + return x == h.x; +} + +inline bool cutlass::half_t::operator!=(cutlass::half_t const& h) const { + if (iszero() && h.iszero()) { + return false; + } + return x != h.x; +} + +inline bool cutlass::half_t::operator==(float const& b) const { return x == half_t(b).x; } + +inline bool cutlass::half_t::operator!=(float const& b) const { return x != half_t(b).x; } + +inline bool cutlass::half_t::iszero() const { return (x & 0x7fff) == 0; } + +inline bool cutlass::half_t::isfinite() const { return (exponent() < 16); } + +inline bool cutlass::half_t::isnan() const { + int exp = ((x >> 10) & 0x1f); + if (exp == 0x1f) { + return (x & 0x3ff) != 0; + } + return false; +} + +inline bool cutlass::half_t::isinf() const { + int exp = ((x >> 10) & 0x1f); + if (exp == 0x1f) { + return (x & 0x3ff) == 0; + } + return false; +} + +inline bool cutlass::half_t::isnormal() const { + int exp = exponent(); + return exp > -15 && exp < 16; +} + +inline bool cutlass::half_t::operator<(half_t const& h) const { + int sign = ((x >> 15) & 1); + int h_sign = ((h.x >> 15) & 1); + if (sign == h_sign) { + return (x & 0x7fff) < (h.x & 0x7fff); + } else if (sign) { + return true; + } + return false; +} + +inline bool cutlass::half_t::operator<=(half_t const& h) const { + int sign = ((x >> 15) & 1); + int h_sign = ((h.x >> 15) & 1); + if (sign == h_sign) { + return (x & 0x7fff) <= (h.x & 0x7fff); + } else if (sign) { + return true; + } + return false; +} + +inline bool cutlass::half_t::operator>(half_t const& h) const { + int sign = ((x >> 15) & 1); + int h_sign = ((h.x >> 15) & 1); + if (sign == h_sign) { + return (x & 0x7fff) > (h.x & 0x7fff); + } else if (h_sign) { + return true; + } + return false; +} + +inline bool cutlass::half_t::operator>=(half_t const& h) const { + int sign = ((x >> 15) & 1); + int h_sign = ((h.x >> 15) & 1); + if (sign == h_sign) { + return (x & 0x7fff) >= (h.x & 0x7fff); + } else if (h_sign) { + return true; + } + return false; +} + +inline cutlass::half_t cutlass::half_t::operator+(cutlass::half_t const& b) const { + return cutlass::half_t(float(*this) + float(b)); +} + +inline cutlass::half_t cutlass::half_t::operator-() const { return bitcast(x ^ 0x8000); } + +inline cutlass::half_t cutlass::half_t::operator-(cutlass::half_t const& b) const { + return cutlass::half_t(float(*this) - float(b)); +} + +inline cutlass::half_t cutlass::half_t::operator*(cutlass::half_t const& b) const { + return cutlass::half_t(float(*this) * float(b)); +} + +inline cutlass::half_t cutlass::half_t::operator/(cutlass::half_t const& b) const { + return cutlass::half_t(float(*this) / float(b)); +} + +inline cutlass::half_t& cutlass::half_t::operator+=(cutlass::half_t const& b) { + *this = cutlass::half_t(float(*this) + float(b)); + return *this; +} + +inline cutlass::half_t& cutlass::half_t::operator-=(cutlass::half_t const& b) { + *this = cutlass::half_t(float(*this) - float(b)); + return *this; +} + +inline cutlass::half_t& cutlass::half_t::operator*=(cutlass::half_t const& b) { + *this = cutlass::half_t(float(*this) * float(b)); + return *this; +} + +inline cutlass::half_t& cutlass::half_t::operator/=(cutlass::half_t const& b) { + *this = cutlass::half_t(float(*this) / float(b)); + return *this; +} + +inline cutlass::half_t& cutlass::half_t::operator++() { + *this = cutlass::half_t(float(*this) + 1.0f); + return *this; +} + +inline cutlass::half_t& cutlass::half_t::operator--() { + *this = cutlass::half_t(float(*this) - 1.0f); + return *this; +} + +inline cutlass::half_t cutlass::half_t::operator++(int) { + half_t h = *this; + *this = cutlass::half_t(float(*this) + 1.0f); + return h; +} + +inline cutlass::half_t cutlass::half_t::operator--(int) { + half_t h = *this; + *this = cutlass::half_t(float(*this) - 1.0f); + return h; +} + +inline cutlass::half_t operator+(float a, cutlass::half_t const& b) { + return cutlass::half_t(a + float(b)); +} + +inline cutlass::half_t operator-(float a, cutlass::half_t const& b) { + return cutlass::half_t(a - float(b)); +} + +inline cutlass::half_t operator*(float a, cutlass::half_t const& b) { + return cutlass::half_t(a * float(b)); +} + +inline cutlass::half_t operator/(float a, cutlass::half_t const& b) { + return cutlass::half_t(a / float(b)); +} + +// +// +// + +inline cutlass::half2_t::half2_t() {} + +inline cutlass::half2_t::half2_t(half_t lo, half_t hi) : lo(lo), hi(hi) {} + +inline cutlass::half2_t::half2_t(std::pair<float, float> const& p) : lo(p.first), hi(p.second) {} + +inline cutlass::half2_t::half2_t(unsigned data) + : lo(half_t::bitcast(uint16_t(data & 0x0ffff))), + hi(half_t::bitcast(uint16_t((data >> 16) & 0x0ffff))) {} + +inline cutlass::half2_t cutlass::half2_t::operator+(half2_t const& b) const { + return half2_t(lo + b.lo, hi + b.hi); +} + +inline cutlass::half2_t cutlass::half2_t::operator-(half2_t const& b) const { + return half2_t(lo - b.lo, hi - b.hi); +} + +inline cutlass::half2_t cutlass::half2_t::operator*(half2_t const& b) const { + return half2_t(lo * b.lo, hi * b.hi); +} + +inline cutlass::half2_t cutlass::half2_t::operator/(half2_t const& b) const { + return half2_t(lo / b.lo, hi / b.hi); +} + +inline cutlass::half2_t& cutlass::half2_t::operator+=(half2_t const& b) { + lo += b.lo; + hi += b.hi; + return *this; +} + +inline cutlass::half2_t& cutlass::half2_t::operator-=(half2_t const& b) { + lo -= b.lo; + hi -= b.hi; + return *this; +} + +inline cutlass::half2_t& cutlass::half2_t::operator*=(half2_t const& b) { + lo *= b.lo; + hi *= b.hi; + return *this; +} + +inline cutlass::half2_t& cutlass::half2_t::operator/=(half2_t const& b) { + lo /= b.lo; + hi /= b.hi; + return *this; +} + +inline float cutlass::half2_t::dot(half2_t const& b) const { + return float(lo) * float(b.lo) + float(hi) * float(b.hi); +} + +inline float cutlass::half2_t::dot(half2_t const& b, float c) const { return c + dot(b); } + +inline cutlass::half_t cutlass::half2_t::doth(half2_t const& b) const { + return cutlass::half_t(dot(b)); +} + +inline cutlass::half_t cutlass::half2_t::doth(half2_t const& b, half_t c) const { + return cutlass::half_t(dot(b, float(c))); +} + +inline cutlass::half2_t::operator std::pair<float, float>() const { + return std::pair<float, float>(float(lo), float(hi)); +} + +inline unsigned cutlass::half2_t::packed() const { return (lo.x | (hi.x << 16)); } + +inline cutlass::half2_t::operator unsigned() const { return packed(); } + +// +// +// + +template <> +inline float cutlass::bitcast<float, unsigned>(unsigned const& u) { + return *reinterpret_cast<float const*>(&u); +} + +template <> +inline float cutlass::bitcast<float, int>(int const& i) { + return *reinterpret_cast<float const*>(&i); +} + +template <> +inline unsigned cutlass::bitcast<unsigned, float>(float const& f) { + return *reinterpret_cast<unsigned const*>(&f); +} + +template <> +inline cutlass::half_t cutlass::bitcast<cutlass::half_t, unsigned short>(unsigned short const& s) { + return *reinterpret_cast<cutlass::half_t const*>(&s); +} + +template <> +inline unsigned short cutlass::bitcast<unsigned short, cutlass::half_t>(cutlass::half_t const& h) { + return *reinterpret_cast<unsigned short const*>(&h); +} + +template <> +inline half cutlass::bitcast<half, unsigned short>(unsigned short const& s) { + return *reinterpret_cast<half const*>(&s); +} + +// +// Lexical casts +// + +#ifdef BOOST_LEXICAL_CAST_INCLUDED +namespace boost { +template <> +cutlass::half_t lexical_cast<cutlass::half_t>(std::string const& arg) { + return cutlass::half_t(boost::lexical_cast<float>(arg)); +} + +template <> +std::string lexical_cast<std::string>(cutlass::half_t const& arg) { + return boost::lexical_cast<std::string>(float(arg)); +} +} // namespace boost +#endif + +// +// Standard Library Operations +// + +// std +namespace std { + +inline cutlass::half_t abs(cutlass::half_t const& h) { + return cutlass::half_t::bitcast(h.x & 0x7fff); +} + +inline bool isnan(cutlass::half_t const& h) { return h.isnan(); } + +inline bool isfinite(cutlass::half_t const& h) { return h.isfinite(); } + +inline cutlass::half_t nanh(const char*) { return cutlass::half_t::nan(); } + +inline bool isinf(cutlass::half_t const& h) { return h.isinf(); } + +inline bool isnormal(cutlass::half_t const& h) { return h.isnormal(); } + +inline int fpclassify(cutlass::half_t const& h) { + int exp = h.exponent(); + unsigned short mantissa = h.mantissa(); + if (exp < -14) { + if (mantissa == 0) { + return FP_ZERO; + } else { + return FP_SUBNORMAL; + } + } else if (exp > 15) { + if (mantissa == 0) { + return FP_INFINITE; + } else { + return FP_NAN; + } + } + return FP_NORMAL; +} + +inline bool signbit(cutlass::half_t const& h) { return h.signbit(); } + +inline cutlass::half_t sqrt(cutlass::half_t const& h) { + return cutlass::half_t(std::sqrt(float(h))); +} +} // namespace std + +// +// Stream interactions +// + +/// put to stream - half_t-precision types bitcast as unsigned shorts if base is hexadecimal +inline std::ostream& operator<<(std::ostream& out, cutlass::half_t const& h) { + if (out.flags() & std::ios::hex) { + return out << h.x; + } else { + return out << float(h); + } +} + +/// read from stream - half_t-precision types parsed as unsigned shorts if base is hexadecimal +inline std::istream& operator>>(std::istream& in, cutlass::half_t& h) { + if (in.flags() & std::ios::hex) { + unsigned short u = 0; + in >> u; + h = cutlass::half_t::bitcast(u); + } else { + float f = 0; + in >> f; + h = cutlass::half_t(f); + } + return in; +} diff --git a/cutlass-example/host_tensor.h b/cutlass-example/host_tensor.h new file mode 100644 index 0000000..0936336 --- /dev/null +++ b/cutlass-example/host_tensor.h @@ -0,0 +1,365 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +#pragma once + +/*! \file + \brief Template class to perform computations on tensors and manage memory. +*/ + +#include <cutlass/cutlass.h> +#include <cutlass/matrix_traits.h> +#include <device_memory.h> +#include <host_tensor_view.h> +#include <type_traits.h> +#include <vector> + +namespace cutlass { + +template <typename T, bool DeviceBacked_ = true> +class HostTensor : public HostTensorView<T> { + public: + /// Type used for device-side allocations + typedef typename TypeTraits<T>::device_type DeviceType; + + /// Base class + typedef HostTensorView<T> Base; + + /// If true, allocates device side memory + static bool const DeviceBacked = DeviceBacked_; + + /// Rank of tensor + static int const Rank = Base::Rank; + + /// Type used to compute the offset of an element to the base of a tensor + typedef typename Base::Offset_t Offset_t; + + /// Tensor reference to host memory + typedef typename Base::TensorRef_t TensorRef_t; + + /// Tensor reference to device memory + typedef TensorRef<DeviceType, TensorRef_t::Rank> DeviceTensorRef; + + /// Tensor reference to constant device memory + typedef TensorRef<DeviceType const, TensorRef_t::Rank> ConstDeviceTensorRef; + + /// Coordinate into tensor + typedef typename Base::Coord_t Coord_t; + + private: + /// Host-side memory allocation + std::vector<T> host_; + + /// Device-side memory + cutlass::device_memory::allocation<DeviceType> device_; + + public: + // + // Device and Host Methods + // + + /// Default constructor + HostTensor() {} + + /// Constructs a Tensor_view from stride and size + HostTensor(Coord_t const& _stride, Coord_t const& _size) { reset(_stride, _size); } + + /// Constructs a HostTensor from size - infers strides + HostTensor(Coord_t const& _size) { + Coord_t _stride = make_Coord( + _size.at(2) * _size.at(1) * _size.at(0), _size.at(1) * _size.at(0), _size.at(0), 1); + reset(_stride, _size); + } + + /// Returns the number of elements needed to back vector + size_t capacity() { return Base::capacity(); } + + /// Returns true if the Tensor_view is bound to some memory + bool good() const { return Base::good(); } + + /// Updates the reference and size of a Tensor_view object + void reset(Coord_t const& _stride, Coord_t const& _size) { + size_t _capacity = _size.at(0) * _stride.at(0); + + DeviceType* _device_memory = nullptr; + if (DeviceBacked) { + _device_memory = cutlass::device_memory::allocate<DeviceType>(_capacity); + } + + host_.clear(); + host_.resize(_capacity); + for (size_t i = 0; i < _capacity; ++i) { + host_[i] = T((int)0xdeadbeef); + } + device_.reset(_device_memory, _capacity); + + Base::reset(TensorRef_t(host_.data(), _stride), _size); + } + + /// Initializes the host tensor as a matrix + void resize_matrix(int rows, int columns, MatrixLayout::Kind layout) { + bool col_major = (layout == MatrixLayout::kColumnMajor); + int ldm = (col_major ? rows : columns); + + Coord_t stride = make_Coord(rows * columns, col_major ? 1 : ldm, col_major ? ldm : 1, 1); + + Coord_t size = make_Coord(1, rows, columns, 1); + + reset(stride, size); + } + + /// Simplifies resizing the host tensor + void resize(int elements) { resize_matrix(1, elements, MatrixLayout::kColumnMajor); } + + /// Gets pointer to host data + T const* host_data() const { return &host_[0]; } + + /// Gets pointer to host data + T* host_data() { return &host_[0]; } + + /// Gets pointer to device data + DeviceType* device_data() const { return device_.get(); } + + /// Copies data from device to host + void sync_host() { + if (DeviceBacked) { + device_memory::copy_to_host( + host_.data(), reinterpret_cast<T const*>(device_.get()), host_.size()); + } + } + + /// Copies data from host to device + void sync_device() { + if (DeviceBacked) { + device_memory::copy_to_device( + device_.get(), reinterpret_cast<DeviceType const*>(host_.data()), host_.size()); + } + } + + /// Copy data from a caller-supplied device pointer + void copy_to_host(DeviceType const *ptr_device) { + device_memory::copy_to_host( + host_.data(), reinterpret_cast<T const *>(ptr_device), host_.size()); + } + + /// Copies data to a caller-supplied device pointer + void copy_to_device(DeviceType *ptr_device) { + device_memory::copy_to_device( + ptr_device, reinterpret_cast<DeviceType const *>(host_.data()), host_.size()); + } + + /// Accesses the tensor reference pointing to data + TensorRef_t& host_ref() { return Base::ref(); } + + /// Accesses the tensor reference pointing to data + TensorRef_t const& host_ref() const { return Base::ref(); } + + /// Accesses the tensor reference pointing to data + DeviceTensorRef device_ref() const { return DeviceTensorRef(device_data(), stride()); } + + /// Returns a tensor ref to constant memory on the device + ConstDeviceTensorRef const_device_ref() const { + return ConstDeviceTensorRef(device_data(), stride()); + } + + /// Accesses the size + Coord_t const& size() const { return Base::size(); } + + /// Accesses the size + int size(int dim) const { return Base::size(dim); } + + /// Accesses the size + Coord_t const& stride() const { return Base::stride(); } + + /// Accesses the size + int stride(int dim) const { return Base::stride(dim); } + + /// Returns the index of an element + Offset_t offset(Coord_t const& coord) const { return Base::offset(coord); } + + /// Determines whether a location is within a tensor + bool contains(Coord_t const& coord) const { return Base::contains(coord); } + + /// Element-wise accessor + T& at(Coord_t const& coord) const { return Base::at(coord); } + + /// Element-wise accessor + T& operator[](Coord_t const& coord) { return at(coord); } + + /// Element-wise accessor with basic offset + T& at(int idx) const { return Base::at(idx); } + + /// Returns a Tensor_view given location and size quantities + TensorView<T> subview(Coord_t const& _location, Coord_t _size) const { + return Base::subview(_location, _size); + } + + /// Recurses through all dimensions and applies a unary operation + template <typename F> + void elementwise_in_place(F& op, int dim = 0, Offset_t dst_offset_base = 0) { + Base::elementwise_in_place(op, dim, dst_offset_base); + } + + /// Recurses through all dimensions and applies a unary operator, supplying the logical + /// coordinate within the tensor as an argument + template <typename F> + void elementwise_stream(F& op, int dim = 0, Offset_t dst_offset_base = 0) { + Base::elementwise_stream(op, dim, dst_offset_base); + } + + /// Recurses through all dimensions and applies a unary operator, supplying the logical + /// coordinate within the tensor as an argument + template <typename F> + void elementwise_generate(F& op, + int dim = 0, + Offset_t dst_offset_base = 0, + Coord_t coord = Coord_t(0)) { + Base::elementwise_generate(op, dim, dst_offset_base, coord); + } + + /// Recurses through all dimensions and applies a binary operation + template <typename Src, typename F> + bool elementwise_in_place(F& op, + int dim, + TensorView<Src> const& tensor, + Offset_t dst_offset_base = 0, + Offset_t src_offset_base = 0) { + return Base::elementwise_in_place(op, dim, tensor, dst_offset_base, src_offset_base); + } + + /// Accumulate in place + template <typename Src> + TensorView<T>& operator+=(TensorView<Src> const& tensor) { + Base::operator+=(tensor); + sync_device(); + return *this; + } + + /// Subtract in place + template <typename Src> + TensorView<T>& operator-=(TensorView<Src> const& tensor) { + Base::operator-=(tensor); + sync_device(); + return *this; + } + + /// Multiply in place + template <typename Src> + TensorView<T>& operator*=(TensorView<Src> const& tensor) { + Base::operator*=(tensor); + sync_device(); + return *this; + } + + /// Divide in place + template <typename Src> + TensorView<T>& operator/=(TensorView<Src> const& tensor) { + Base::operator/=(tensor); + sync_device(); + return *this; + } + + /// equality with epsilon tolerance + bool equals(TensorView<T> const& tensor, T epsilon) const { + return Base::equals(tensor, epsilon); + } + + /// equality with ulps tolerance + bool bit_equals(TensorView<T> const& tensor, long long ulps_threshold = 0) { + return Base::bit_equals(tensor, ulps_threshold); + } + + /// Computes general matrix product among select dimensions of a tensor + /// Assumes: + /// D: number of independent GEMMs to compute + /// H: height of matrix + /// W: width of matrix + template < + /// Data type of A matrix elements + typename A, + /// Data type of B matrix elements + typename B, + /// Data type of "compute" type (i.e. accumulator) + typename Ctype, + /// Data type of scale factors + typename Stype> + void gemm(TensorView<A> const& tensor_a, TensorView<B> const& tensor_b, Stype alpha, Stype beta) { + Base::template gemm<A, B, Ctype, Stype>(tensor_a, tensor_b, alpha, beta); + } + + /// Fills with random data + template <typename Gen> + void fill_random(Gen generator) { + Base::fill_random(generator); + sync_device(); + } + + /// Procedurally assigns elements + template <typename Gen> + void generate(Gen generator) { + Base::generate(generator); + sync_device(); + } + + /// Procedurally visits elements + template <typename Gen> + void visit(Gen& generator) const { + Base::visit(generator); + } + + /// initializes with identity + void fill_identity() { + Base::fill_identity(); + sync_device(); + } + + /// computes elements as a linear combination of their coordinates + void fill_linear(Coord_t v, T offset = T(0)) { + Base::fill_linear(v, offset); + sync_device(); + } + + /// computes elements as a linear combination of their coordinates + void fill_sequential(T v = T(1), T offset = T(0)) { + Base::fill_sequential(v, offset); + sync_device(); + } + + /// fills with a value + void fill(T val = T(0)) { + Base::fill(val); + sync_device(); + } + + /// Copies from external data source and performs type conversion + template <typename Src> + void fill(TensorView<Src> const& tensor) { + Base::fill(tensor); + sync_device(); + } + + /// Computes the norm of the matrix in double-precision + double norm() const { return Base::norm(); } +}; +} // namespace cutlass diff --git a/cutlass-example/host_tensor_view.h b/cutlass-example/host_tensor_view.h new file mode 100644 index 0000000..56f02d3 --- /dev/null +++ b/cutlass-example/host_tensor_view.h @@ -0,0 +1,542 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Host-side implementation of useful operations +*/ + +#pragma once + +#include <cutlass/cutlass.h> +#include <cutlass/tensor_view.h> +#include <type_traits.h> + +namespace cutlass { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename SrcType, typename DstType> +struct Cast { + static inline DstType apply(SrcType src) { return static_cast<DstType>(src); }; +}; + +template <> +struct Cast<float, int8_t> { + static inline int8_t apply(float src) { + return static_cast<int8_t>(fmaxf(-128.f, fminf(127.f, src))); + }; +}; + +template <> +struct Cast<float, uint8_t> { + static inline uint8_t apply(float src) { + return static_cast<uint8_t>(fmaxf(0.f, fminf(255.f, src))); + }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template <typename T> +class HostTensorView : public TensorView<T> { + public: + /// Base class + typedef TensorView<T> TensorView_t; + + /// Convention: depth is the first dimension + static int const Dim_D = 0; + + /// Convention: height is the second dimension + static int const Dim_H = 1; + + /// Convention: width is the third dimension + static int const Dim_W = 2; + + /// Convention: channel is the second dimension + static int const Dim_C = 3; + + /// Rank of tensor + static int const Rank = TensorView_t::Rank; + + /// Type used to compute the offset of an element to the base of a tensor + typedef typename TensorView_t::Offset_t Offset_t; + + /// Reference and stride + typedef typename TensorView_t::TensorRef_t TensorRef_t; + + /// Coordinate into tensor + typedef typename TensorView_t::Coord_t Coord_t; + + public: + // + // Device and Host Methods + // + + /// Default constructor + HostTensorView() {} + + /// Constructs a Tensor_view from a TensorRef and size + HostTensorView(TensorRef_t const& _ref, Coord_t const& _size) : TensorView_t(_ref, _size) {} + + /// Accesses the size + Coord_t const& size() const { return TensorView_t::size(); } + + /// Accesses the size of a specified dimension + int size(int dim) const { return size().at(dim); } + + /// Accesses the stride + Coord_t const& stride() const { return TensorView_t::stride(); } + + /// Accesses the stride along a specified dimension + int stride(int dim) const { return stride().at(dim); } + + /// Returns the number of scalar elements needed to store tensor + size_t capacity() const { return size(3) * stride(3) * stride(2) * stride(1) * stride(0); } + + /// Returns true if the Tensor_view is bound to some memory + bool good() const { return TensorView_t::good(); } + + /// Updates the reference and size of a TensorView object + void reset(TensorRef_t const& _ref = TensorRef_t(0), Coord_t const& _size = Coord_t()) { + return TensorView_t::reset(_ref, _size); + } + + /// Accesses the tensor reference pointing to data + TensorRef_t& ref() { return TensorView_t::ref(); } + + /// Accesses the tensor reference pointing to data + TensorRef_t const& ref() const { return TensorView_t::ref(); } + + /// Assigns a tensor view + HostTensorView& operator=(TensorView_t const& _tensor) { + reset(_tensor.ref(), _tensor.size()); + return *this; + } + + /// Returns the index of an element + Offset_t offset(Coord_t const& coord) const { return TensorView_t::offset(coord); } + + /// Determines whether a location is within a tensor + bool contains(Coord_t const& coord) const { return TensorView_t::contains(coord); } + + /// Element-wise accessor + T& at(Coord_t const& coord) const { return TensorView_t::at(coord); } + + /// Element-wise accessor + T& operator[](Coord_t const& coord) const { return at(coord); } + + /// Accesses an element with a raw offset + T& at(int idx) const { return TensorView_t::at(idx); } + + /// Accesses an element with a raw offset + T& operator[](int idx) const { return at(idx); } + + /// Returns a Tensor_view given location and size quantities + TensorView_t subview(Coord_t const& location, Coord_t size) const { + return TensorView_t::subview(location, size); + } + + /// Recurses through all dimensions and applies a unary operation in place + template <typename F> + void elementwise_in_place(F& op, int dim = 0, Offset_t dst_offset_base = 0) { + Offset_t dst_offset = dst_offset_base; + + for (int idx = 0; idx < size(dim); ++idx, dst_offset += stride(dim)) { + if (dim < Rank - 1) { + elementwise_in_place(op, dim + 1, dst_offset); + } else { + op(ref().data()[dst_offset]); + } + } + } + + /// Recurses through all dimensions and applies a unary operator with no arguments + template <typename F> + void elementwise_stream(F& op, int dim = 0, Offset_t dst_offset_base = 0) { + Offset_t dst_offset = dst_offset_base; + + for (int idx = 0; idx < size(dim); ++idx, dst_offset += stride(dim)) { + if (dim < Rank - 1) { + elementwise_stream(op, dim + 1, dst_offset); + } else { + ref().data()[dst_offset] = op(); + } + } + } + + /// Recurses through all dimensions and applies a unary operator, supplying the logical + /// coordinate within the tensor as an argument + template <typename F> + void elementwise_generate(F& op, + int dim = 0, + Offset_t dst_offset_base = 0, + Coord_t coord = Coord_t(0)) { + Offset_t dst_offset = dst_offset_base; + + for (int idx = 0; idx < size(dim); ++idx, dst_offset += stride(dim)) { + coord.at(dim) = idx; + + if (dim < Rank - 1) { + elementwise_generate(op, dim + 1, dst_offset, coord); + } else { + ref().data()[dst_offset] = op(coord); + } + } + } + + /// Recurses through all dimensions and applies a unary operator, supplying the logical + /// coordinate within the tensor as an argument + template <typename F> + void elementwise_visit(F& op, + int dim = 0, + Offset_t dst_offset_base = 0, + Coord_t coord = Coord_t(0)) const { + Offset_t dst_offset = dst_offset_base; + + for (int idx = 0; idx < size(dim); ++idx, dst_offset += stride(dim)) { + coord.at(dim) = idx; + + if (dim < Rank - 1) { + elementwise_visit(op, dim + 1, dst_offset, coord); + } else { + op(ref().data()[dst_offset], coord); + } + } + } + + /// Recurses through all dimensions and applies a binary operation + template <typename Src, typename F> + bool elementwise_in_place(F& op, + TensorView<Src> const& tensor, + int dim = 0, + Offset_t dst_offset_base = 0, + Offset_t src_offset_base = 0) { + Offset_t dst_offset = dst_offset_base; + Offset_t src_offset = src_offset_base; + + if (size().at(dim) != tensor.size().at(dim)) { + return false; + } + + for (int idx = 0; idx < size(dim); + ++idx, dst_offset += stride(dim), src_offset += tensor.stride(dim)) { + if (dim < Rank - 1) { + elementwise_in_place(op, tensor, dim + 1, dst_offset, src_offset); + } else { + op(data()[dst_offset], tensor.data()[src_offset]); + } + } + + return true; + } + + template <typename Src> + struct LambdaBinaryAddition { + void operator()(T& a, Src b) const { a += T(b); } + }; + + template <typename Src> + struct LambdaBinarySubtraction { + void operator()(T& a, Src b) const { a -= T(b); } + }; + + template <typename Src> + struct LambdaBinaryMultiplication { + void operator()(T& a, Src b) const { a *= T(b); } + }; + + template <typename Src> + struct LambdaBinaryDivision { + void operator()(T& a, Src b) const { a /= T(b); } + }; + + /// Accumulate in place + template <typename Src> + TensorView<T>& operator+=(TensorView<Src> const& tensor) { + LambdaBinaryAddition<Src> op; + elementwise_in_place(op, tensor); + + return *this; + } + + /// Subtract in place + template <typename Src> + TensorView<T>& operator-=(TensorView<Src> const& tensor) { + LambdaBinarySubtraction<Src> op; + elementwise_in_place(op, tensor); + + return *this; + } + + /// Multiply in place + template <typename Src> + TensorView<T>& operator*=(TensorView<Src> const& tensor) { + LambdaBinaryMultiplication<Src> op; + elementwise_in_place(op, tensor); + + return *this; + } + + /// Divide in place + template <typename Src> + TensorView<T>& operator/=(TensorView<Src> const& tensor) { + LambdaBinaryDivision<Src> op; + elementwise_in_place(op, tensor); + + return *this; + } + + /// Comparison operator + struct EqualsOperator { + bool equal; + T eps; + + EqualsOperator(T _epsilon) : equal(true), eps(_epsilon) {} + + void operator()(T a, T b) { + if (std::abs(T(a - b)) > eps * std::max(std::abs(a), std::abs(b))) { + equal = false; + } + } + }; + + /// equality with epsilon tolerance + bool equals(TensorView<T> const& tensor, T epsilon) const { + EqualsOperator comparison_op(epsilon); + bool equal_size = elementwise_in_place(comparison_op, tensor); + + return equal_size && comparison_op.equal; + } + + /// Compares two values which are smaller or equal to a long long int + struct BitEqualsOperator { + bool equal; + long long eps; + uint64_t index; + + BitEqualsOperator(long long _ulps_threshold) : equal(true), eps(_ulps_threshold), index(0) {} + + void operator()(T a, T b) { + // convert bits to integers + long long bits_a = 0; + long long bits_b = 0; + + *reinterpret_cast<T*>(&bits_a) = TypeTraits<T>::remove_negative_zero(a); + *reinterpret_cast<T*>(&bits_b) = TypeTraits<T>::remove_negative_zero(b); + + // compute diff + long long ulps = bits_a - bits_b; + if (std::abs(ulps) > eps) { + equal = false; + } + index++; + } + }; + + /// equality with ulps tolerance + bool bit_equals(TensorView<T> const& tensor, long long ulps_threshold = 0) { + BitEqualsOperator comparison_op(ulps_threshold); + bool equal_size = elementwise_in_place(comparison_op, tensor); + + return equal_size && comparison_op.equal; + } + + /// Gets naked pointer to data + T* data() const { return TensorView_t::data(); } + + /// Computes general matrix product among select dimensions of a tensor + /// Assumes: + /// D: number of independent GEMMs to compute + /// H: height of matrix + /// W: width of matrix + /// C: "channels" of each element + template <typename A, typename B, typename Ctype, typename Stype> + void gemm(TensorView<A> const& tensor_a, TensorView<B> const& tensor_b, Stype alpha, Stype beta) { + int const Batch = size(Dim_D); + int const M = size(Dim_H); + int const N = size(Dim_W); + int const K = tensor_a.size(Dim_W); + int const C = tensor_a.size(Dim_C); + + // Sizes must match + if (tensor_a.size(Dim_H) != M || tensor_b.size(Dim_W) != N || tensor_b.size(Dim_C) != C || + tensor_b.size(Dim_H) != K) { + return; + } + + int const Mblock = 32; + int const Nblock = 32; + + for (int batch = 0; batch < Batch; ++batch) { + for (int row_block = 0; row_block < M; row_block += Mblock) { + for (int col_block = 0; col_block < N; col_block += Nblock) { + Ctype accum[Mblock][Nblock]; + + for (int j = 0; j < Nblock; j++) { + for (int i = 0; i < Mblock; i++) { + accum[i][j] = Ctype(0); + } + } + + for (int k_block = 0; k_block < K; ++k_block) { + for (int j = 0; j < Nblock; j++) { + for (int i = 0; i < Mblock; i++) { + int row = row_block + i; + int col = col_block + j; + + if (row < M && col < N) { + for (int channel = 0; channel < C; ++channel) { + Ctype a(tensor_a.at(make_Coord(batch, row, k_block, channel))); + Ctype b(tensor_b.at(make_Coord(batch, k_block, col, channel))); + + accum[i][j] += a * b; + } + } + } + } + } + + for (int j = 0; j < Nblock; j++) { + for (int i = 0; i < Mblock; i++) { + int row = row_block + i; + int col = col_block + j; + + Coord_t coord = make_Coord(batch, row, col, 0); + if (row < M && col < N) { + at(coord) = + Cast<Stype, T>::apply(alpha * Stype(accum[i][j]) + beta * Stype(at(coord))); + } + } + } + } + } + } + } + + /// Fills with random data + template <typename Gen> + void fill_random(Gen generator) { + elementwise_stream(generator); + } + + /// Procedurally assigns elements + template <typename Gen> + void generate(Gen generator) { + elementwise_generate(generator); + } + + /// Procedurally visits elements + template <typename Gen> + void visit(Gen& generator) const { + elementwise_visit(generator); + } + + /// Generator to fill a tensor with the identity matrix + struct LambdaFillIdentity { + T operator()(Coord_t const& coord) { return (coord.at(1) == coord.at(2) ? T(1) : T(0)); } + }; + + /// initializes with identity + void fill_identity() { + LambdaFillIdentity op; + elementwise_generate(op); + } + + /// Lambda for fill_linear() + struct LambdaFillLinear { + Coord_t v_; + T offset_; + + LambdaFillLinear(Coord_t const& _v, T _offset) : v_(_v), offset_(_offset) {} + + T operator()(Coord_t const& coord) { return T(v_.template dot<int>(coord)) + offset_; } + }; + + /// computes elements as a linear combination of their coordinates + void fill_linear(Coord_t v, T offset = T(0)) { + LambdaFillLinear lambda(v, offset); + elementwise_generate(lambda); + } + + /// computes elements as a linear combination of their coordinates + void fill_sequential(T v = T(1), T offset = T(0)) { + int const count = size().count(); + for (int i = 0; i < count; ++i) { + data()[i] = T(i); + } + } + + /// Returns a constant value + struct LambdaFillValue { + T value; + + LambdaFillValue(T _value) : value(_value) {} + + T operator()() { return value; } + }; + + /// fills with a value + void fill(T val = T(0)) { + LambdaFillValue op(val); + elementwise_stream(op); + } + + /// Conversion from Src to T + template <typename Src> + struct LambdaAssign { + void operator()(T& a, Src b) const { a = T(b); } + }; + + /// copies from external data source and performs type conversion + template <typename Src> + void fill(TensorView<Src> const& tensor) { + LambdaAssign<Src> op; + elementwise_in_place(op, tensor); + } + + /// Computes a norm + struct LambdaNorm { + double sum; + + LambdaNorm() : sum(0) {} + + void operator()(T const& element) { + double value(element); + double conj(element); // TODO - conjugates for complex + + sum += value * conj; + } + }; + + /// Computes the norm of the matrix in double-precision + double norm() const { + LambdaNorm op; + elementwise_in_place(op); + + return std::sqrt(op.sum); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass diff --git a/cutlass-example/tensor_view_io.h b/cutlass-example/tensor_view_io.h new file mode 100644 index 0000000..bc2e9b1 --- /dev/null +++ b/cutlass-example/tensor_view_io.h @@ -0,0 +1,61 @@ +/*************************************************************************************************** +* Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +* +* Redistribution and use in source and binary forms, with or without modification, are permitted +* provided that the following conditions are met: +* * Redistributions of source code must retain the above copyright notice, this list of +* conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above copyright notice, this list of +* conditions and the following disclaimer in the documentation and/or other materials +* provided with the distribution. +* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used +* to endorse or promote products derived from this software without specific prior written +* permission. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR +* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE +* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +**************************************************************************************************/ +#pragma once + +#include <cutlass/core_io.h> +#include <cutlass/tensor_view.h> + +template <typename T> +inline std::ostream& tensor_view_output(std::ostream& out, T t) { + out << t; + return out; +} + +template <> +inline std::ostream& tensor_view_output<int8_t>(std::ostream& out, int8_t t) { + out << int(t); + return out; +} + +template <typename T> +inline std::ostream& operator<<(std::ostream& out, cutlass::TensorView<T> const& tensor) { + for (int batch = 0; batch < tensor.size(0); ++batch) { + out << "[\n "; + for (int h = 0; h < tensor.size(1); ++h) { + for (int w = 0; w < tensor.size(2); ++w) { + for (int c = 0; c < tensor.size(3); ++c) { + out << ((c | w) ? " " : ""); + tensor_view_output(out, tensor.at(cutlass::make_Coord(batch, h, w, c))); + } + } + if (h + 1 < tensor.size(1)) { + out << " ;\n "; + } + } + out << " ]"; + } + + return out; +} diff --git a/cutlass-example/type_traits.h b/cutlass-example/type_traits.h new file mode 100644 index 0000000..eabd67e --- /dev/null +++ b/cutlass-example/type_traits.h @@ -0,0 +1,160 @@ +/*************************************************************************************************** + * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted + * provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this list of + * conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, this list of + * conditions and the following disclaimer in the documentation and/or other materials + * provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used + * to endorse or promote products derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! \file + \brief Type traits for common CUDA types +*/ + +#pragma once + +#include <cuda_fp16.h> +#include <stdint.h> + +#include "half.h" + +namespace cutlass { +struct half_t; + +template <typename T> +struct TypeTraits; + +template <> +struct TypeTraits<int8_t> { +// static cudaDataType_t const cublas_type = CUDA_R_8I; + typedef int8_t host_type; + typedef int8_t device_type; + typedef int8_t integer_type; + typedef uint8_t unsigned_type; + static inline int8_t remove_negative_zero(int8_t x) { return x; } + static inline int to_print(int8_t x) { return (int)x; } +}; + +template <> +struct TypeTraits<uint8_t> { +// static cudaDataType_t const cublas_type = CUDA_R_8I; + typedef uint8_t host_type; + typedef uint8_t device_type; + typedef uint8_t integer_type; + typedef uint8_t unsigned_type; + static inline uint8_t remove_negative_zero(uint8_t x) { return x; } + static inline uint32_t to_print(uint8_t x) { return (uint32_t)x; } +}; + +template <> +struct TypeTraits<int> { +// static cudaDataType_t const cublas_type = CUDA_R_32I; + typedef int host_type; + typedef int device_type; + typedef int32_t integer_type; + typedef uint32_t unsigned_type; + static inline int32_t remove_negative_zero(int32_t x) { return x; } + static inline int to_print(int x) { return x; } +}; + +template <> +struct TypeTraits<unsigned> { +// static cudaDataType_t const cublas_type = CUDA_R_32I; + typedef unsigned host_type; + typedef unsigned device_type; + typedef uint32_t integer_type; + typedef uint32_t unsigned_type; + static inline uint32_t remove_negative_zero(uint32_t x) { return x; } + static inline uint32_t to_print(uint32_t x) { return x; } +}; + +template <> +struct TypeTraits<half> { +// static cudaDataType_t const cublas_type = CUDA_R_16F; + typedef half_t host_type; + typedef half device_type; + typedef int16_t integer_type; + typedef uint16_t unsigned_type; + static inline half remove_negative_zero(half x) { + integer_type h_int = reinterpret_cast<integer_type const&>(x); + if (h_int == 0x8000) { + h_int = 0; + } + x = reinterpret_cast<half const&>(h_int); + return x; + } + static inline half to_print(half x) { return x; } +}; + +template <> +struct TypeTraits<int64_t> { +// static cudaDataType_t const cublas_type = CUDA_R_8I; + typedef int64_t host_type; + typedef int64_t device_type; + typedef int64_t integer_type; + typedef uint64_t unsigned_type; + static inline int64_t remove_negative_zero(int64_t x) { return x; } + static inline int64_t to_print(int64_t x) { return x; } +}; + +template <> +struct TypeTraits<uint64_t> { +// static cudaDataType_t const cublas_type = CUDA_R_8I; + typedef uint64_t host_type; + typedef uint64_t device_type; + typedef uint64_t integer_type; + typedef uint64_t unsigned_type; + static inline uint64_t remove_negative_zero(uint64_t x) { return x; } + static inline uint64_t to_print(uint64_t x) { return x; } +}; + +template <> +struct TypeTraits<cutlass::half_t> { +// static cudaDataType_t const cublas_type = CUDA_R_16F; + typedef half_t host_type; + typedef half device_type; + typedef int16_t integer_type; + typedef uint16_t unsigned_type; + static inline half_t remove_negative_zero(half_t x) { + return (x.raw() == 0x8000 ? half_t::bitcast(0) : x); + } + static inline half_t to_print(half_t x) { return x; } +}; + +template <> +struct TypeTraits<float> { +// static cudaDataType_t const cublas_type = CUDA_R_32F; + typedef float host_type; + typedef float device_type; + typedef int32_t integer_type; + typedef uint32_t unsigned_type; + static inline float remove_negative_zero(float x) { return x == -0.f ? 0.f : x; } + static inline float to_print(float x) { return x; } +}; + +template <> +struct TypeTraits<double> { +// static cudaDataType_t const cublas_type = CUDA_R_64F; + typedef double host_type; + typedef double device_type; + typedef int64_t integer_type; + typedef uint64_t unsigned_type; + static inline double remove_negative_zero(double x) { return x == -0.0 ? 0.0 : x; } + static inline double to_print(double x) { return x; } +}; +} // namespace cutlass |