diff options
Diffstat (limited to 'cutlass-example')
68 files changed, 0 insertions, 16832 deletions
diff --git a/cutlass-example/Makefile b/cutlass-example/Makefile deleted file mode 100644 index f72b732..0000000 --- a/cutlass-example/Makefile +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 1993-2017, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -CC = nvcc -src = $(wildcard *.cu) -obj = $(src:.c=.o) -INC_DIR = ./ - - -#LDFLAGS = -g -G --gpu-architecture=compute_70 --gpu-code=compute_70 -lcudart -I$(INC_DIR) -#LDFLAGS = -O0 -Xcicc -O0 -Xptxas -O0 --gpu-architecture=compute_70 --gpu-code=compute_70 -lcudart -I$(INC_DIR) -LDFLAGS = --gpu-architecture=compute_70 --gpu-code=compute_70 -lcudart -I$(INC_DIR) - -myprog: $(obj) - $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) - -clean: - rm -rf gemm_tt_* - rm -rf host_results_* - rm -rf gpgpusim_power_report__* - rm -rf _cuobjdump* - rm -rf gpgpu_inst_stats.txt - rm -rf myprog - echo Clean done diff --git a/cutlass-example/command_line.h b/cutlass-example/command_line.h deleted file mode 100644 index 8f2b17a..0000000 --- a/cutlass-example/command_line.h +++ /dev/null @@ -1,254 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are not permitted. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#pragma once - -/** - * \file - * Utility for parsing command line arguments - */ - -#include <iostream> -#include <limits> -#include <sstream> -#include <string> -#include <vector> - -#include <cuda_runtime.h> - -namespace cutlass { - -/****************************************************************************** - * command_line - ******************************************************************************/ - -/** - * Utility for parsing command line arguments - */ -struct CommandLine { - std::vector<std::string> keys; - std::vector<std::string> values; - std::vector<std::string> args; - - /** - * Constructor - */ - CommandLine(int argc, const char** argv) : keys(10), values(10) { - using namespace std; - - for (int i = 1; i < argc; i++) { - string arg = argv[i]; - - if ((arg[0] != '-') || (arg[1] != '-')) { - args.push_back(arg); - continue; - } - - string::size_type pos; - string key, val; - if ((pos = arg.find('=')) == string::npos) { - key = string(arg, 2, arg.length() - 2); - val = ""; - } else { - key = string(arg, 2, pos - 2); - val = string(arg, pos + 1, arg.length() - 1); - } - - keys.push_back(key); - values.push_back(val); - } - } - - /** - * Checks whether a flag "--<flag>" is present in the commandline - */ - bool check_cmd_line_flag(const char* arg_name) const { - using namespace std; - - for (int i = 0; i < int(keys.size()); ++i) { - if (keys[i] == string(arg_name)) return true; - } - return false; - } - - /** - * Returns number of naked (non-flag and non-key-value) commandline parameters - */ - template <typename value_t> - int num_naked_args() const { - return args.size(); - } - - /** - * Returns the commandline parameter for a given index (not including flags) - */ - template <typename value_t> - void get_cmd_line_argument(int index, value_t& val) const { - using namespace std; - if (index < args.size()) { - istringstream str_stream(args[index]); - str_stream >> val; - } - } - - /** - * Returns the commandline parameter for a given index (not including flags) - */ - void get_cmd_line_argument(const char* arg_name, bool& val, bool _default = true) const { - val = _default; - if (check_cmd_line_flag(arg_name)) { - std::string value; - get_cmd_line_argument(arg_name, value); - - val = !(value == "0" || value == "false"); - } - } - - /** - * Returns the value specified for a given commandline parameter --<flag>=<value> - */ - template <typename value_t> - void get_cmd_line_argument(const char* arg_name, - value_t& val, - value_t const& _default = value_t()) const { - using namespace std; - - val = _default; - - for (int i = 0; i < int(keys.size()); ++i) { - if (keys[i] == string(arg_name)) { - istringstream str_stream(values[i]); - str_stream >> val; - } - } - } - - /** - * Returns the values specified for a given commandline parameter --<flag>=<value>,<value>* - */ - template <typename value_t> - void get_cmd_line_arguments(const char* arg_name, - std::vector<value_t>& vals, - char sep = ',') const { - using namespace std; - - if (check_cmd_line_flag(arg_name)) { - // Clear any default values - vals.clear(); - - // Recover from multi-value string - for (int i = 0; i < keys.size(); ++i) { - if (keys[i] == string(arg_name)) { - string val_string(values[i]); - istringstream str_stream(val_string); - string::size_type old_pos = 0; - string::size_type new_pos = 0; - - // Iterate <sep>-delimited values - value_t val; - while ((new_pos = val_string.find(sep, old_pos)) != string::npos) { - if (new_pos != old_pos) { - str_stream.width(new_pos - old_pos); - str_stream >> val; - vals.push_back(val); - } - - // skip over delimiter - str_stream.ignore(1); - old_pos = new_pos + 1; - } - - // Read last value - str_stream >> val; - vals.push_back(val); - } - } - } - } - - /** - * Returns the values specified for a given commandline parameter - * --<flag>=<key:value>,<key:value>* - */ - void get_cmd_line_argument_pairs(const char* arg_name, - std::vector<std::pair<std::string, std::string> >& tokens, - char delim = ',', - char sep = ':') const { - if (check_cmd_line_flag(arg_name)) { - std::string value; - get_cmd_line_argument(arg_name, value); - - tokenize(tokens, value, delim, sep); - } - } - - /** - * The number of pairs parsed - */ - int parsed_argc() const { return (int)keys.size(); } - - //------------------------------------------------------------------------- - // Utility functions - //------------------------------------------------------------------------- - - /// Tokenizes a comma-delimited list of string pairs delimited by ':' - static void tokenize(std::vector<std::pair<std::string, std::string> >& tokens, - std::string const& str, - char delim = ',', - char sep = ':') { - // Home-built to avoid Boost dependency - size_t s_idx = 0; - size_t d_idx = std::string::npos; - while (s_idx < str.size()) { - d_idx = str.find_first_of(delim, s_idx); - - size_t end_idx = (d_idx != std::string::npos ? d_idx : str.size()); - size_t sep_idx = str.find_first_of(sep, s_idx); - size_t offset = 1; - if (sep_idx == std::string::npos || sep_idx >= end_idx) { - sep_idx = end_idx; - offset = 0; - } - - std::pair<std::string, std::string> item( - str.substr(s_idx, sep_idx - s_idx), - str.substr(sep_idx + offset, end_idx - sep_idx - offset)); - - tokens.push_back(item); - s_idx = end_idx + 1; - } - } - - /// Tokenizes a comma-delimited list of string pairs delimited by ':' - static void tokenize(std::vector<std::string>& tokens, - std::string const& str, - char delim = ',', - char sep = ':') { - typedef std::vector<std::pair<std::string, std::string> > TokenVector; - typedef TokenVector::const_iterator token_iterator; - - std::vector<std::pair<std::string, std::string> > token_pairs; - tokenize(token_pairs, str, delim, sep); - for (token_iterator tok = token_pairs.begin(); tok != token_pairs.end(); ++tok) { - tokens.push_back(tok->first); - } - } -}; - -} // namespace cutlass diff --git a/cutlass-example/config_fermi_islip.icnt b/cutlass-example/config_fermi_islip.icnt deleted file mode 100644 index 3b8b496..0000000 --- a/cutlass-example/config_fermi_islip.icnt +++ /dev/null @@ -1,70 +0,0 @@ -//21*1 fly with 32 flits per packet under gpgpusim injection mode -use_map = 0; -flit_size = 32; - -// currently we do not use this, see subnets below -network_count = 2; - -// Topology -topology = fly; -k = 102; -n = 1; - -// Routing - -routing_function = dest_tag; - -// Flow control - -num_vcs = 1; -vc_buf_size = 8; - -wait_for_tail_credit = 0; - -// Router architecture - -vc_allocator = islip; //separable_input_first; -sw_allocator = islip; //separable_input_first; -alloc_iters = 1; - -credit_delay = 0; -routing_delay = 0; -vc_alloc_delay = 1; -sw_alloc_delay = 1; - -input_speedup = 2; -output_speedup = 1; -internal_speedup = 1.0; - -// Traffic, GPGPU-Sim does not use this - -traffic = uniform; -packet_size ={{1,2,3,4},{10,20}}; -packet_size_rate={{1,1,1,1},{2,1}}; - -// Simulation - Don't change - -sim_type = gpgpusim; -//sim_type = latency; -injection_rate = 0.1; - -subnets = 2; - -// Always use read and write no matter following line -//use_read_write = 1; - - -read_request_subnet = 0; -read_reply_subnet = 1; -write_request_subnet = 0; -write_reply_subnet = 1; - -read_request_begin_vc = 0; -read_request_end_vc = 0; -write_request_begin_vc = 0; -write_request_end_vc = 0; -read_reply_begin_vc = 0; -read_reply_end_vc = 0; -write_reply_begin_vc = 0; -write_reply_end_vc = 0; - diff --git a/cutlass-example/cutlass/convert.h b/cutlass-example/cutlass/convert.h deleted file mode 100644 index 933d68a..0000000 --- a/cutlass-example/cutlass/convert.h +++ /dev/null @@ -1,102 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! - \file - \brief Defines conversion operations among Fragments of different base type. -*/ -#pragma once - -#include <cutlass/fragment.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename InputFragment_, typename OutputFragment_> -struct Convert {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename InputScalar_, typename OutputScalar_, int kScalars_> -struct Convert<Fragment<InputScalar_, kScalars_>, Fragment<OutputScalar_, kScalars_> > { - /// The input fragment. - typedef Fragment<InputScalar_, kScalars_> InputFragment; - /// The output fragment. - typedef Fragment<OutputScalar_, kScalars_> OutputFragment; - - /// Ctor. - CUTLASS_DEVICE Convert() {} - - /// Transform a fragment. - CUTLASS_DEVICE void transform(InputFragment const& src, OutputFragment& dst) { - transform(src, 0, dst); - } - - /// Transform a fragment. - template <typename Fragment_> - CUTLASS_DEVICE void transform(Fragment_ const& src, int offset, OutputFragment& dst) { - for (int i = 0; i < kScalars_; ++i) { - dst[i] = static_cast<OutputScalar_>(src[i + offset]); - } - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Fragment_> -struct Copy { - /// The input fragment. - typedef Fragment_ InputFragment; - /// The output fragment. - typedef Fragment_ OutputFragment; - - /// Ctor. - CUTLASS_DEVICE Copy() {} - - /// Transform a fragment. - CUTLASS_DEVICE void transform(Fragment_ const& src, Fragment_& dst) { transform(src, 0, dst); } - - /// Transform a fragment. - template <typename InputFragment_> - CUTLASS_DEVICE void transform(InputFragment_ const& src, int offset, Fragment_& dst) { - if (sizeof(typename Fragment_::Element) == 8) { - uint64_t const* src_ptr = reinterpret_cast<uint64_t const*>(&src[offset]); - uint64_t* dst_ptr = reinterpret_cast<uint64_t*>(&dst[0]); - for (int i = 0; i < sizeof(Fragment_) / 8; ++i) { - dst_ptr[i] = src_ptr[i]; - } - } else { - uint32_t const* src_ptr = reinterpret_cast<uint32_t const*>(&src[offset]); - uint32_t* dst_ptr = reinterpret_cast<uint32_t*>(&dst[0]); - for (int i = 0; i < sizeof(Fragment_) / 4; ++i) { - dst_ptr[i] = src_ptr[i]; - } - } - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/coord.h b/cutlass-example/cutlass/coord.h deleted file mode 100644 index 431c9bf..0000000 --- a/cutlass-example/cutlass/coord.h +++ /dev/null @@ -1,287 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief A Coord is a coordinate of arbitrary rank into a tensor or matrix -*/ - -#pragma once - -#include <cutlass/cutlass.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Describes identity elements -struct Identity { - /// Enumeration describing identity elements. Value assignments are significant. - /// Feel free to add or multiply by these, respectively. - enum Kind { Additive = 0, Multiplicative = 1 }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Statically-sized array specifying Coords within a tensor -template <int N_> -struct Coord { - // - // Type and constant definitions - // - - static int const N = N_; - - // - // Data members - // - - /// Indices - int idx[N]; - - // - // Methods - // - - /// Default ctor initializes uniformly - CUTLASS_HOST_DEVICE - Coord(int value = 0) { - for (int i = 0; i < N; ++i) { - idx[i] = value; - } - } - - /// Constructs from an array of integers - CUTLASS_HOST_DEVICE - Coord(int _idx[]) { - for (int i = 0; i < N; ++i) { - idx[i] = _idx[i]; - } - } - - /// Element-wise addition - CUTLASS_HOST_DEVICE - Coord operator+(Coord const& b) const { - Coord c; - for (int i = 0; i < N; ++i) { - c.idx[i] = idx[i] + b.idx[i]; - } - return c; - } - - /// Element-wise subtraction - CUTLASS_HOST_DEVICE - Coord operator-(Coord const& b) const { - Coord c; - for (int i = 0; i < N; ++i) { - c.idx[i] = idx[i] - b.idx[i]; - } - return c; - } - - /// Element-wise multiplication - CUTLASS_HOST_DEVICE - Coord operator*(Coord const& b) const { - Coord c; - for (int i = 0; i < N; ++i) { - c.idx[i] = idx[i] * b.idx[i]; - } - return c; - } - - /// Element-wise division - CUTLASS_HOST_DEVICE - Coord operator/(Coord const& b) const { - Coord c; - for (int i = 0; i < N; ++i) { - c.idx[i] = idx[i] / b.idx[i]; - } - return c; - } - - /// In-place addition - CUTLASS_HOST_DEVICE - Coord& operator+=(Coord const& b) { - for (int i = 0; i < N; ++i) { - idx[i] += b.idx[i]; - } - return *this; - } - - /// In-place subtraction - CUTLASS_HOST_DEVICE - Coord& operator-=(Coord const& b) { - for (int i = 0; i < N; ++i) { - idx[i] -= b.idx[i]; - } - return *this; - } - - /// In-place multiplication - CUTLASS_HOST_DEVICE - Coord& operator*=(Coord const& b) { - for (int i = 0; i < N; ++i) { - idx[i] *= b.idx[i]; - } - return *this; - } - - /// In-place division - CUTLASS_HOST_DEVICE - Coord& operator/=(Coord const& b) { - for (int i = 0; i < N; ++i) { - idx[i] /= b.idx[i]; - } - return *this; - } - - /// Member access operator - CUTLASS_HOST_DEVICE int& operator[](int dim) { return idx[dim]; } - - /// Member access operator - CUTLASS_HOST_DEVICE int const& operator[](int dim) const { return idx[dim]; } - - /// Computes the dot product of two Coord instances - template <typename T> - CUTLASS_HOST_DEVICE T dot(Coord const& b, T sum) const { - for (int i = 0; i < N; ++i) { - sum += idx[i] * b.idx[i]; - } - return sum; - } - - /// Computes the dot product of two Coord instances - template <typename T> - CUTLASS_HOST_DEVICE T dot(Coord const& b) const { - T sum = T(0); - for (int i = 0; i < N; ++i) { - sum += idx[i] * b.idx[i]; - } - return sum; - } - - /// Gets the index of a given Coord element - template <int Dim> - CUTLASS_HOST_DEVICE int& at() { - return idx[Dim]; - } - - /// Access via index; may limit unrolling potential - CUTLASS_HOST_DEVICE - int& at(int dim) { return idx[dim]; } - - /// Gets the index of a given Coord element - template <int Dim> - CUTLASS_HOST_DEVICE int const& at() const { - return idx[Dim]; - } - - /// Access via index; may limit unrolling potential - CUTLASS_HOST_DEVICE - int const& at(int dim) const { return idx[dim]; } - - /// Determines if two Coord<> objects are equal - CUTLASS_HOST_DEVICE - bool operator==(Coord<N> const& b) const { - bool equal = true; - for (int i = 0; equal && i < N; ++i) { - equal = (idx[i] == b.idx[i]); - } - return equal; - } - - /// Not equal - CUTLASS_HOST_DEVICE - bool operator!=(Coord<N> const& b) const { return !(*this == b); } - - /// Clamps a coordinate to a range specified by maximum and minimum values - CUTLASS_HOST_DEVICE - Coord& clamp(Coord<N> const& max, Coord<N> const& min = Coord<N>()) { - for (int i = 0; i < N; ++i) { - idx[i] = __NV_STD_MAX(__NV_STD_MIN(idx[i], max.idx[i]), min.idx[i]); - } - return *this; - } - - /// Returns the product of all elements - CUTLASS_HOST_DEVICE - int count() const { - int product = idx[0]; - for (int i = 1; i < N; ++i) { - product *= idx[i]; - } - return product; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Helper to make a 2-element coordinate -CUTLASS_HOST_DEVICE -Coord<1> make_Coord(int _0) { - int values[1] = {_0}; - return Coord<1>(values); -} - -/// Helper to make a 2-element coordinate -CUTLASS_HOST_DEVICE -Coord<2> make_Coord(int _0, int _1) { - int values[2] = {_0, _1}; - return Coord<2>(values); -} - -/// Helper to make a 3-element coordinate -CUTLASS_HOST_DEVICE -Coord<3> make_Coord(int _0, int _1, int _2) { - int values[3] = {_0, _1, _2}; - return Coord<3>(values); -} - -/// Helper to make a 4-element coordinate -CUTLASS_HOST_DEVICE -Coord<4> make_Coord(int _0, int _1, int _2, int _3) { - int values[4] = {_0, _1, _2, _3}; - return Coord<4>(values); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Getter -CUTLASS_HOST_DEVICE -Coord<2> get_Coord_hw(Coord<3> const& coord) { return make_Coord(coord[1], coord[2]); } - -/// Getter -CUTLASS_HOST_DEVICE -Coord<2> get_Coord_hw(Coord<4> const& coord) { return make_Coord(coord[1], coord[2]); } - -/// Getter -CUTLASS_HOST_DEVICE -Coord<3> get_Coord_hwc(Coord<4> const& coord) { return make_Coord(coord[1], coord[2], coord[3]); } - -/// Getter -CUTLASS_HOST_DEVICE -Coord<3> get_Coord_dhw(Coord<4> const& coord) { return make_Coord(coord[0], coord[1], coord[2]); } - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/core_io.h b/cutlass-example/cutlass/core_io.h deleted file mode 100644 index cceea4c..0000000 --- a/cutlass-example/cutlass/core_io.h +++ /dev/null @@ -1,44 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -#pragma once - -/*! \file - \brief Helpers for printing cutlass/core objects -*/ - -#pragma once - -#include <iosfwd> -#include <typeinfo> - -#include <cutlass/coord.h> - -template <int Rank> -std::ostream& operator<<(std::ostream& out, cutlass::Coord<Rank> const& coord) { - for (int i = 0; i < Rank; ++i) { - out << (i ? ", " : "") << coord.idx[i]; - } - return out; -} diff --git a/cutlass-example/cutlass/cutlass.h b/cutlass-example/cutlass/cutlass.h deleted file mode 100644 index 19600ec..0000000 --- a/cutlass-example/cutlass/cutlass.h +++ /dev/null @@ -1,73 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ - -/*! \file - \brief Basic include for CUTLASS macros -*/ - -#pragma once - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -#define CUTLASS_MAJOR 1 -#define CUTLASS_MINOR 0 -#define CUTLASS_PATCH 1 -#define CUTLASS_VERSION ((CUTLASS_MAJOR)*100 + (CUTLASS_MINOR)*10 + CUTLASS_PATCH) - -#ifdef __NVCC__ -#define CUTLASS_HOST_DEVICE __forceinline__ __device__ __host__ -#define CUTLASS_DEVICE __forceinline__ __device__ -#elif defined(__CUDACC_RTC__) -#define CUTLASS_HOST_DEVICE __forceinline__ __device__ -#define CUTLASS_DEVICE __forceinline__ __device__ -#else -#define CUTLASS_HOST_DEVICE -// CUTLASS_DEVICE is an error if not compiling device code -#endif - -// CUTLASS_PRAGMA_UNROLL inserts a CUTLASS_PRAGMA_UNROLL if supported by the compiler -#if defined(__CUDA_ARCH__) -#if defined(_MSC_VER) -#define CUTLASS_PRAGMA_UNROLL __pragma("unroll") -#define CUTLASS_PRAGMA_NO_UNROLL __pragma("unroll 1") -#else -#define CUTLASS_PRAGMA_UNROLL _Pragma("unroll") -#define CUTLASS_PRAGMA_NO_UNROLL _Pragma("unroll 1") -#endif -#else -#define CUTLASS_PRAGMA_UNROLL -#define CUTLASS_PRAGMA_NO_UNROLL -#endif - -#define CUTLASS_ASSERT(x) assert(x) - -namespace cutlass { - -/// NVIDIA GPU Warp size -static const int kWarpSize = 32; - -} // namespace cutlass - -//////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/cutlass-example/cutlass/fragment.h b/cutlass-example/cutlass/fragment.h deleted file mode 100644 index 886b114..0000000 --- a/cutlass-example/cutlass/fragment.h +++ /dev/null @@ -1,278 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines Fragment, a statically-sized array for storing parts of matrices within a - thread's registers. -*/ -#pragma once - -#include <assert.h> -#include <cutlass/shape.h> -#include <cutlass/util/cutlass_math.h> -#include <cutlass/vector.h> - -namespace cutlass { - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/*!@defgroup fragment_concept Fragment Concept -@{ - -\ref fragment_concept is a statically sized array for storing parts of tiles held by individual CUDA -threads. - -@par \ref fragment_concept - Types satisfying \ref fragment_concept define the following members - - <b>Element</b> - type of each access held within the fragment - - <b>kElements</b> - number of elements stored by the fragment - - <b>clear()</b> - overwrites the fragment storage with zeros - - <b>Element & operator[](int i)</b> - by-reference access of the ith element - - <b>Element const & operator[](int i) const</b> - const by-reference access of the ith element -@} -*/ - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/*!@defgroup fragment_iterator_concept Fragment Iterator Concept -@{ - -\ref fragment_iterator_concept provides structured access to the elements within a fragment with an -optional bitcast to the desired access type - -@par \ref fragment_iterator_concept - Types satisfying \ref fragment_iterator_concept define the following members - - <b>AccessType& operator[](int i)</b> - provides access to the ith element of the fragment - - <b>AccessType& at(int d, int h, int w, int c)</b> - applies \ref layout_concept to fragment and -provides access to element at (d, h, w, c) - -@} -*/ - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <int kAlignment_> -struct StorageType { - typedef uint64_t Type; -}; -template <> -struct StorageType<4> { - typedef uint32_t Type; -}; -template <> -struct StorageType<2> { - typedef uint16_t Type; -}; -template <> -struct StorageType<1> { - typedef uint8_t Type; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief A template defining \ref fragment_concept -* @concept{fragment_concept} -*/ -template <typename Element_, int kElements_, size_t kAlignment_ = 16> -struct Fragment : public AlignedStruct<kAlignment_> { - /// Make sure the alignment makes sense wrt the size of elements. - static_assert(kAlignment_ == 16 || kAlignment_ >= sizeof(Element_), "Alignment is too small"); - /// Alignment must be a power of two - static_assert(is_pow2<kAlignment_>::value, "Alignment must be a power of two"); - - /// This class. - typedef Fragment<Element_, kElements_> This_; - /// The element. - typedef Element_ Element; - /// The number of elements. - static int const kElements = kElements_; - - /// Clear a fragment. - CUTLASS_DEVICE void clear() { - // Avoid element-wise access for sub 32b element type - if (kAlignment_ >= 8 && (kElements * sizeof(Element)) % 8 == 0) { - uint64_t* ptr = reinterpret_cast<uint64_t*>(storage); - for (int i = 0; i < (kElements * sizeof(Element)) / 8; ++i) { - ptr[i] = uint64_t(0); - } - } else if (kAlignment_ >= 4 && (kElements * sizeof(Element)) % 4 == 0) { - uint32_t* ptr = reinterpret_cast<uint32_t*>(storage); - for (int i = 0; i < (kElements * sizeof(Element)) / 4; ++i) { - ptr[i] = uint32_t(0); - } - } else if (kAlignment_ >= 2 && (kElements * sizeof(Element)) % 2 == 0) { - uint16_t* ptr = reinterpret_cast<uint16_t*>(storage); - for (int i = 0; i < (kElements * sizeof(Element)) / 2; ++i) { - ptr[i] = uint16_t(0); - } - } else { - for (int i = 0; i < kElements; ++i) { - storage[i] = 0; - } - } - } - - /// The accessor. - CUTLASS_DEVICE Element& operator[](int i) { - assert(i < kElements_); - return reinterpret_cast<Element*>(storage)[i]; - } - - /// The accessor. - CUTLASS_DEVICE Element const& operator[](int i) const { - assert(i < kElements_); - return reinterpret_cast<Element const*>(storage)[i]; - } - - private: - /// Storage type to use for Elements - typedef typename StorageType<kAlignment_>::Type StorageType; - - /// Number of elements in the storage - static int const kStorageCount = - (sizeof(Element_) * kElements_ + sizeof(StorageType) - 1) / sizeof(StorageType); - /// The storage. - StorageType storage[kStorageCount]; - - /// Ensure that there's enough storage for all elements - static_assert(sizeof(StorageType) <= kAlignment_, "StorageType is too big for given alignment"); -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief A template defining \ref fragment_iterator_concept -* @concept{fragment_iterator_concept} -*/ -template <typename Fragment_, typename Iterations_, typename AccessType_> -struct FragmentIterator { - /// This class. - typedef FragmentIterator<Fragment_, Iterations_, AccessType_> This_; - /// The fragment. - typedef Fragment_ Fragment; - /// The number of iterations. - typedef Iterations_ Iterations; - /// The access type. - typedef AccessType_ AccessType; - - /// The element. - typedef typename Fragment::Element Element; - /// The number of elements per access. - static int const kElementsPerAccess = (int)(sizeof(AccessType) / sizeof(Element)); - /// The shape of the the fragment. - typedef typename ShapeMul<Iterations, Shape<1, 1, 1, kElementsPerAccess> >::Shape FragmentShape; - /// The linear strides for iterations. - typedef typename ShapeStrides<FragmentShape, kElementsPerAccess>::Shape Strides; - - /// Ctor. - template <typename OtherFragment_> - CUTLASS_DEVICE FragmentIterator(OtherFragment_& fragment, int offset = 0) - : pointer(reinterpret_cast<Element*>(&fragment[offset])) { - static_assert(OtherFragment_::kElements >= Fragment::kElements, ""); - } - - /// The accessor. - CUTLASS_DEVICE AccessType const& at(int d, int h, int w, int c = 0) const { - int const imm = ComputeOffsetFromStrides<Strides>::get(d, h, w, c); - return reinterpret_cast<AccessType const&>(pointer[imm]); - } - - /// The accessor. - CUTLASS_DEVICE AccessType& at(int d, int h, int w, int c = 0) { - int const imm = ComputeOffsetFromStrides<Strides>::get(d, h, w, c); - return reinterpret_cast<AccessType&>(pointer[imm]); - } - - /// The accessor. - CUTLASS_DEVICE AccessType const& operator[](int i) const { - return reinterpret_cast<AccessType const&>(pointer[i * kElementsPerAccess]); - } - - /// The accessor. - CUTLASS_DEVICE AccessType& operator[](int i) { - return reinterpret_cast<AccessType&>(pointer[i * kElementsPerAccess]); - } - - /// Is the iterator valid? - CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { return true; } - - /// The pointer. - Element* pointer; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Fragment_, typename Iterations_, typename AccessType_> -struct FragmentConstIterator { - /// This class. - typedef FragmentIterator<Fragment_, Iterations_, AccessType_> This_; - /// The fragment. - typedef Fragment_ Fragment; - /// The number of iterations. - typedef Iterations_ Iterations; - /// The access type. - typedef AccessType_ AccessType; - - /// The element. - typedef typename Fragment::Element Element; - /// The number of elements per access. - static int const kElementsPerAccess = (int)(sizeof(AccessType) / sizeof(Element)); - /// The shape of the the fragment. - typedef typename ShapeMul<Iterations, Shape<1, 1, 1, kElementsPerAccess> >::Shape FragmentShape; - /// The linear strides for iterations. - typedef typename ShapeStrides<FragmentShape, kElementsPerAccess>::Shape IterationsStrides; - - /// Ctor. - template <typename OtherFragment_> - CUTLASS_DEVICE FragmentConstIterator(OtherFragment_& fragment, int offset = 0) - : pointer(reinterpret_cast<Element const*>(&fragment[offset])) { - static_assert(OtherFragment_::kElements >= Fragment::kElements, ""); - } - /// Create from non-constant FragmentIterator - CUTLASS_DEVICE FragmentConstIterator( - FragmentIterator<Fragment_, Iterations_, AccessType_> const& rhs_) - : pointer(reinterpret_cast<Element const*>(rhs_.offset)) {} - - /// The accessor. - CUTLASS_DEVICE AccessType const& at(int d, int h, int w, int c = 0) const { - int const imm = ComputeOffsetFromStrides<IterationsStrides>::get(d, h, w, c); - return reinterpret_cast<AccessType const&>(pointer[imm]); - } - - /// The accessor. - CUTLASS_DEVICE AccessType const& operator[](int i) const { - return reinterpret_cast<AccessType const&>(pointer[i * kElementsPerAccess]); - } - - /// Is the iterator valid? - CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { return true; } - - /// The pointer. - Element const* pointer; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/fragment_load_store.h b/cutlass-example/cutlass/fragment_load_store.h deleted file mode 100644 index a7d272e..0000000 --- a/cutlass-example/cutlass/fragment_load_store.h +++ /dev/null @@ -1,135 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines accessors for loading and storing fragments to memory efficiently. -*/ -#pragma once - -#include <cutlass/load_store.h> -#include <cutlass/vector.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <IteratorFragment::Kind kIteratorFragment, - int kAccessSize, - typename Scalar_, - MemorySpace::Kind Memory_, - typename FragmentElement_, - int kStride> -struct FragmentLoad {}; - -template <int kAccessSize, - typename Scalar_, - MemorySpace::Kind Memory_, - typename FragmentElement_, - int kStride> -struct FragmentLoad<IteratorFragment::kWmmaMatrix, - kAccessSize, - Scalar_, - Memory_, - FragmentElement_, - kStride> { - /// The output type. - typedef FragmentElement_ AccessType; - - /// The load function. - static CUTLASS_DEVICE void load(AccessType& value, Scalar_ const* pointer, int offset) { - value.load(&pointer[offset], kStride); - } -}; - -template <int kAccessSize, - typename Scalar_, - MemorySpace::Kind Memory_, - typename FragmentElement_, - int kStride> -struct FragmentLoad<IteratorFragment::kScalar, - kAccessSize, - Scalar_, - Memory_, - FragmentElement_, - kStride> { - /// The output type. - typedef typename Vectorize<Scalar_, kAccessSize>::Type AccessType; - - /// The load function. - static CUTLASS_DEVICE void load(AccessType& value, Scalar_ const* pointer, int offset) { - Load<Scalar_, kAccessSize, Memory_>::load(value, pointer, offset); - } -}; - -template <IteratorFragment::Kind kIteratorFragment, - int kAccessSize, - typename Scalar_, - MemorySpace::Kind Memory_, - typename FragmentElement_, - int kStride> -struct FragmentStore {}; - -template <int kAccessSize, - typename Scalar_, - MemorySpace::Kind Memory_, - typename FragmentElement_, - int kStride> -struct FragmentStore<IteratorFragment::kWmmaMatrix, - kAccessSize, - Scalar_, - Memory_, - FragmentElement_, - kStride> { - /// The input type. - typedef FragmentElement_ AccessType; - - /// The store function. - static CUTLASS_DEVICE void store(AccessType const& value, Scalar_* pointer, int offset) { - value.store(&pointer[offset], kStride); - } -}; - -template <int kAccessSize, - typename Scalar_, - MemorySpace::Kind Memory_, - typename FragmentElement_, - int kStride> -struct FragmentStore<IteratorFragment::kScalar, - kAccessSize, - Scalar_, - Memory_, - FragmentElement_, - kStride> { - /// The input type. - typedef typename Vectorize<Scalar_, kAccessSize>::Type AccessType; - - /// The store function. - static CUTLASS_DEVICE void store(AccessType const& value, Scalar_* pointer, int offset) { - Store<Scalar_, kAccessSize, Memory_>::store(value, pointer, offset); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} /// namespace cutlass diff --git a/cutlass-example/cutlass/fragment_multiply_add.h b/cutlass-example/cutlass/fragment_multiply_add.h deleted file mode 100644 index 36a4d6f..0000000 --- a/cutlass-example/cutlass/fragment_multiply_add.h +++ /dev/null @@ -1,149 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines multiply-add operations on fragments within a thread. -*/ -#pragma once - -#include <cutlass/fragment.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_> -struct FragmentMultiplyAdd { - /// The shape of the instruction. - typedef Shape<1, 1, 1, 1> InstructionShape; - /// The type for A. - typedef Scalar_ ScalarA; - /// The type for B. - typedef Scalar_ ScalarB; - /// The type for C and D. - typedef Scalar_ ScalarC; - - /// Ctor. - CUTLASS_DEVICE FragmentMultiplyAdd() {} - - /// Multiply : d = a*b. - template <typename FragmentB_, typename FragmentCd_> - CUTLASS_DEVICE void multiply(Scalar_ a, FragmentB_ const& b, FragmentCd_& d) { - int const kReduction = FragmentB_::kElements / FragmentCd_::kElements; - for (int j = 0; j < FragmentCd_::kElements; ++j) { - d[j] = a * b[j * kReduction + 0]; - for (int k = 1; k < kReduction; ++k) { - d[j] += a * b[j * kReduction + k]; - } - } - } - - /// Multiply : d = a*b + c. - template <typename FragmentB_, typename FragmentCd_> - CUTLASS_DEVICE void multiply_add(Scalar_ a, - FragmentB_ const& b, - FragmentCd_ const& c, - FragmentCd_& d) { - int const kReduction = FragmentB_::kElements / FragmentCd_::kElements; - for (int j = 0; j < FragmentCd_::kElements; ++j) { - d[j] = a * b[j * kReduction + 0] + c[j]; - for (int k = 1; k < kReduction; ++k) { - d[j] += a * b[j * kReduction + k]; - } - } - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -#if !defined(__CUDACC_RTC__) || defined(CUTLASS_NVRTC_HAS_FP16) -template <> -struct FragmentMultiplyAdd<half> { - /// The shape of the instruction. - typedef Shape<1, 1, 2, 1> InstructionShape; - /// The type for A. - typedef half ScalarA; - /// The type for B. - typedef half ScalarB; - /// The type for C and D. - typedef half ScalarC; - - /// Ctor. - CUTLASS_DEVICE FragmentMultiplyAdd() {} - - /// Multiply : d = a*b. - template <typename FragmentB_, typename FragmentCd_> - CUTLASS_DEVICE void multiply(half a, FragmentB_ const& b, FragmentCd_& d) { -#if defined(__CUDACC__) && __CUDA_ARCH__ >= 530 - - // Assemble a half2 from a. - __half2 const a_half2 = __half2half2(a); - // The input. - __half2 const* b_half2 = reinterpret_cast<__half2 const*>(&b[0]); - // The output. - __half2* d_half2 = reinterpret_cast<__half2*>(&d[0]); - - int const kReduction = FragmentB_::kElements / FragmentCd_::kElements; - for (int j = 0; j < FragmentCd_::kElements / 2; ++j) { - d_half2[j] = __hmul2(a_half2, b_half2[j * kReduction + 0]); - for (int k = 1; k < kReduction; ++k) { - d_half2[j] = __hfma2(a_half2, b_half2[j * kReduction + k], d_half2[j]); - } - } -#endif - } - - /// Multiply : d = a*b + c. - template <typename FragmentB_, typename FragmentCd_> - CUTLASS_DEVICE void multiply_add(half a, - FragmentB_ const& b, - FragmentCd_ const& c, - FragmentCd_& d) { -#if defined(__CUDACC__) && __CUDA_ARCH__ >= 530 - // Assemble a half2 from a. - __half2 const a_half2 = __half2half2(a); - // The inputs. - __half2 const* b_half2 = reinterpret_cast<__half2 const*>(&b[0]); - __half2 const* c_half2 = reinterpret_cast<__half2 const*>(&c[0]); - // The output. - __half2* d_half2 = reinterpret_cast<__half2*>(&d[0]); - - int const kReduction = (FragmentB_::kElements / FragmentCd_::kElements); - for (int j = 0; j < FragmentCd_::kElements / 2; ++j) { - d_half2[j] = __hfma2(a_half2, b_half2[j * kReduction + 0], c_half2[j]); - for (int k = 1; k < kReduction; ++k) { - d_half2[j] = __hfma2(a_half2, b_half2[j * kReduction + k], d_half2[j]); - } - } -#endif - } -}; - -#endif - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/clear_accumulators.h b/cutlass-example/cutlass/gemm/clear_accumulators.h deleted file mode 100644 index 441370f..0000000 --- a/cutlass-example/cutlass/gemm/clear_accumulators.h +++ /dev/null @@ -1,57 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines abstractions for efficiently clearing accumulator tiles. -*/ -#pragma once - -#include <cutlass/vector.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, int kLanes_ = 1> -struct ClearAccumulators { - /// The shared storage. - struct SharedStorage {}; - - /// Ctor. - CUTLASS_DEVICE ClearAccumulators() {} - /// Ctor. - CUTLASS_DEVICE ClearAccumulators(SharedStorage& shared_storage) {} - - /// Clear the fragment. - template <typename Fragment_> - CUTLASS_DEVICE void clear(Fragment_& fragment) { - fragment.clear(); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/dgemm_traits.h b/cutlass-example/cutlass/gemm/dgemm_traits.h deleted file mode 100644 index 0bbc221..0000000 --- a/cutlass-example/cutlass/gemm/dgemm_traits.h +++ /dev/null @@ -1,127 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines structural traits of double-precision GEMM. -*/ -#pragma once - -#include <cutlass/gemm/gemm.h> -#include <cutlass/gemm/gemm_epilogue.h> -#include <cutlass/gemm/gemm_epilogue_traits.h> -#include <cutlass/gemm/gemm_global_tile.h> -#include <cutlass/gemm/gemm_shared_tile.h> -#include <cutlass/gemm/gemm_traits.h> -#include <cutlass/gemm/thread_multiply_add.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The tile size for the GEMM KxNxM. - typename OutputTile_, - /// The number of accumulators per thread. - typename AccumulatorsPerThread_, - /// The number of scalars per LDG for A. - int kScalarsPerLdgA_ = 1, - /// The number of scalars per LDG for B. - int kScalarsPerLdgB_ = 1> -struct DgemmConfig - : public GemmConfig< - /// The scalar type for A. - double, - /// The scalar type for B. - double, - /// The scalar type for C. - double, - /// The scalar type for D. - double, - /// The tile size for the GEMM KxNxM. - OutputTile_, - /// The functor to do the math in the main loop. - ThreadMultiplyAdd<AccumulatorsPerThread_, Shape<1, 4, 8>, double, double, double>, - /// The number of scalars per LDG for A. - kScalarsPerLdgA_, - /// The number of scalars per STS for A. - kScalarsPerLdgA_, - /// The number of scalars per LDS for A. - 2, - /// The number of scalars per LDG for B. - kScalarsPerLdgB_, - /// The number of scalars per STS for B. - kScalarsPerLdgB_, - /// The number of scalars per LDS for B. - 2, - /// The number of scalars per LDG for C and STG for D. - 1, - /// The number of scalars per STS for D. - 2, - /// The number of scalars per LDS for D. - 1, - /// The number of stages in shared memory. - 2> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The layout for A. - MatrixLayout::Kind kLayoutA_, - /// The layout for B. - MatrixLayout::Kind kLayoutB_, - /// The output tile. - typename OutputTile_ = Shape<8, 64, 128>, - /// The functor to use in the epilogue. - typename EpilogueFunctor_ = LinearScaling<double>, - /// The number of accumulators per thread. - typename AccumulatorsPerThread_ = Shape<8, 8, 8>, - /// The number of doubles loaded in one LDG for A. - int kScalarsPerLdgA_ = 1, - /// The number of doubles loaded in one LDG for B. - int kScalarsPerLdgB_ = 1, - /// The index. - typename Index_ = int, - /// The DGEMM config. - typename GemmConfig_ = - DgemmConfig<OutputTile_, AccumulatorsPerThread_, kScalarsPerLdgA_, kScalarsPerLdgB_>, - /// The traits class for the epilogue. - typename GemmEpilogueTraits_ = - SimplifiedGemmEpilogueTraits<GemmConfig_, EpilogueFunctor_, Index_> > -struct DgemmTraits : public SimplifiedGemmTraits< - // The layout for A. - kLayoutA_, - // The layout for B. - kLayoutB_, - // The config. - GemmConfig_, - // The epilogue. - GemmEpilogue<GemmEpilogueTraits_>, - // The index. - Index_> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm.h b/cutlass-example/cutlass/gemm/gemm.h deleted file mode 100644 index c50a3f0..0000000 --- a/cutlass-example/cutlass/gemm/gemm.h +++ /dev/null @@ -1,344 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Implements a software-pipelined efficient GEMM. -*/ -#pragma once - -#if !defined(__CUDACC_RTC__) -#include <cuda.h> -#endif - -#include <cutlass/coord.h> -#include <cutlass/util/platform.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Gemm_> -__global__ /*__launch_bounds__(Gemm_::kThreads)*/ void gemm_kernel(typename Gemm_::Params params) { - // Declare shared memory. - __shared__ typename Gemm_::SharedStorage shared_storage; - - // Construct the GEMM object. - Gemm_ gemm(params, shared_storage); - // Run GEMM. - gemm.multiply_add(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, typename Index_ = int> -struct GemmDesc { - /// The dimensions of the GEMM. - Index_ m, n, k; - /// The alpha/beta scaling values. - Scalar_ alpha, beta; - /// The source matrix A. - void const* d_a; - /// The stride for A. - Index_ lda; - /// The source matrix B. - void const* d_b; - /// The stride for B. - Index_ ldb; - /// The source matrix C. - void const* d_c; - /// The stride for C. - Index_ ldc; - /// The destination matrix D. - void* d_d; - /// The stride for D. - Index_ ldd; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmTraits_> -struct Gemm { - /// This class. - typedef Gemm<GemmTraits_> This_; - /// The traits. - typedef GemmTraits_ Traits; - /// The shared storage. - typedef typename Traits::SharedStorage SharedStorage; - - /// The scalar for A. - typedef typename Traits::ScalarA ScalarA; - /// The scalar for B. - typedef typename Traits::ScalarB ScalarB; - /// The scalar in the epilogue. - typedef typename Traits::Epilogue::Scalar ScalarEpilogue; - /// The scalar for C. - typedef typename Traits::Epilogue::ScalarC ScalarC; - /// The scalar for D. - typedef typename Traits::Epilogue::ScalarD ScalarD; - /// The index. - typedef typename Traits::Index Index; - - /// The number of threads. - static int const kThreads = Traits::GemmConfig::kThreads; - - /// The params. - struct Params : public Traits::Params { - CUTLASS_HOST_DEVICE int initialize(Index m, - Index n, - Index k, - ScalarEpilogue alpha, - ScalarA const* d_a, - Index lda, - ScalarB const* d_b, - Index ldb, - ScalarEpilogue beta, - ScalarC const* d_c, - Index ldc, - ScalarD* d_d, - Index ldd) { - GemmDesc<ScalarEpilogue, Index> desc; - desc.m = m; - desc.n = n; - desc.k = k; - desc.alpha = alpha; - desc.beta = beta; - desc.d_a = reinterpret_cast<void const*>(d_a); - desc.lda = lda; - desc.d_b = reinterpret_cast<void const*>(d_b); - desc.ldb = ldb; - desc.d_c = reinterpret_cast<void const*>(d_c); - desc.ldc = ldc; - desc.d_d = reinterpret_cast<void*>(d_d); - desc.ldd = ldd; - return Traits::Params::initialize(desc); - } - }; - -#if !defined(__CUDACC_RTC__) - /// Launch the kernel. - static __host__ cudaError_t launch(Params const& params, - cudaStream_t stream = cudaStreamDefault) { - // Setup the grid. - dim3 grid; - grid.x = (params.m + Traits::OutputTile::kW - 1) / Traits::OutputTile::kW; - grid.y = (params.n + Traits::OutputTile::kH - 1) / Traits::OutputTile::kH; - - // The number of threads. - dim3 block; - block.x = kThreads; - - // Launch the kernel. - void const* params_ = reinterpret_cast<void const*>(¶ms); - - return cudaLaunchKernel(reinterpret_cast<void*>(&gemm_kernel<This_>), - grid, - block, - const_cast<void**>(¶ms_), - 0, - stream); - } - - /// Launch the kernel. - static __host__ cudaError_t launch(CUfunction kernel, - Params const& params, - CUstream stream = CU_STREAM_LEGACY) { - // Setup the grid. - dim3 grid; - grid.x = (params.m + Traits::OutputTile::kW - 1) / Traits::OutputTile::kW; - grid.y = (params.n + Traits::OutputTile::kH - 1) / Traits::OutputTile::kH; - - // The number of threads. - dim3 block; - block.x = kThreads; - - // Launch the kernel. - void* params_[] = {const_cast<void*>(reinterpret_cast<void const*>(¶ms))}; - - // return cudaLaunchKernel(reinterpret_cast<void*>(&gemm_kernel<This_>), grid, block, - // const_cast<void**>(¶ms_), 0, stream); - CUresult result = cuLaunchKernel( - kernel, grid.x, grid.y, grid.z, block.x, block.y, block.z, 0, stream, params_, 0); - - if (result != CUDA_SUCCESS) { - return cudaErrorLaunchFailure; - } - return cudaSuccess; - } - -#endif - - /// Ctor. - CUTLASS_DEVICE Gemm(Params const& params_, SharedStorage& shared_storage_) - : params(params_), shared_storage(shared_storage_) {} - - /// Consume a single iteration of the loop. - template <bool kIsLastIteration> - CUTLASS_DEVICE void consume_tile(typename Traits::GlobalLoadStream& global_stream, - typename Traits::SharedLoadStream& shared_load_stream, - typename Traits::MultiplyAdd::Accumulators& accumulators, - Index outer_k) { - // If that's the last "load iteration" update the predicates. - if (!kIsLastIteration) { - global_stream.move_to_residue<false>(outer_k); - } - - // Load data for the next iteration of the main loop. - if (!kIsLastIteration) { - global_stream.copy(); - } - - // The unrolling steps for the main loop. - int const kUnrollingSteps = - Traits::MultiplyAdd::AccumulatorsPerWarp::kD / Traits::MultiplyAdd::InstructionShape::kD; - - CUTLASS_PRAGMA_UNROLL - for (int step = 0; step < kUnrollingSteps - 1; ++step) { - // Trigger the copy from shared memory for the next A/B values. - shared_load_stream.copy(step + 1); - // Make sure the values are available for the current iteration to do the multiply-add. - shared_load_stream.commit(step); - - // Do the math on the fragments of the current iteration. - typename Traits::MultiplyAdd multiply_add; - multiply_add.multiply_add(shared_load_stream.fragment_a(step), - shared_load_stream.fragment_b(step), - accumulators, - accumulators); - } - - // Make sure the data from shared memory has been entirely consumed. - Traits::shared_load_fence(true); - - // Commit the data in shared memory for A/B. - if (!kIsLastIteration) { - global_stream.commit(); - } - - // Make sure the data is in shared memory. - Traits::shared_store_fence(true); - - // Trigger the loads for the next iteration (if needed). - if (!kIsLastIteration) { - // Move to the next stage for the load (if it makes sense). - shared_load_stream.inc_stage(); - // Trigger the copy from shared memory for the next loop iteration. - shared_load_stream.copy(0); - } - - // Make sure the values are available for the current iteration to do the multiply-add. - shared_load_stream.commit(kUnrollingSteps - 1); - - // Do the math on the fragments of the current iteration. - typename Traits::MultiplyAdd multiply_add; - multiply_add.multiply_add(shared_load_stream.fragment_a(kUnrollingSteps - 1), - shared_load_stream.fragment_b(kUnrollingSteps - 1), - accumulators, - accumulators); - } - - /// Do the GEMM. - CUTLASS_DEVICE void multiply_add() { - // Swizzle the IDs of the block (to enable better cache behavior). - typename Traits::BlockSwizzle block_swizzle; - dim3 block = block_swizzle.swizzle(); - - // Scale the id. - block.x *= Traits::OutputTile::kW; - block.y *= Traits::OutputTile::kH; - - // We may want to use shared memory to clear the registers. - typedef typename Traits::ClearAccumulators ClearAccumulators; - - // The streams to read A/B from global memory to shared memory. - typename Traits::GlobalLoadStream global_stream(params, shared_storage, block); - - // Create the accumulator clear. - ClearAccumulators clear(shared_storage.main_loop.clear); - - // By how much we unroll the main loop. - Index const kUnroll = static_cast<Index>(Traits::OutputTile::kD); - - // If we do not have enough steps in the main loop, trigger the residue code. - global_stream.move_to_residue<true>(params.k); - - // Fetch the fragments for A and B from global memory. - global_stream.copy(); - - // Copy the elements to shared memory (after transformation if needed). - global_stream.commit(); - - // Make sure the data is in shared memory. - Traits::shared_store_fence(false); - - // Rollback to the beginning of the GEMM-K dimension. It may have no impact. - global_stream.rollback(); - - // The unrolling steps for the main loop. - int const kUnrollingSteps = - Traits::MultiplyAdd::AccumulatorsPerWarp::kD / Traits::MultiplyAdd::InstructionShape::kD; - - // Make sure we have at least 2 unrolling steps or our pipeling is not going to work. - static_assert(kUnrollingSteps >= 2, "The pipelining assumes at least two steps"); - - // The stream of data from shared memory to fragments. - typename Traits::SharedLoadStream shared_load_stream(params, shared_storage); - - // Trigger the copy from shared memory for the 1st stream. - shared_load_stream.copy(0); - - // Allocate the accumulators. - typename Traits::MultiplyAdd::Accumulators accumulators; - // Clear the accumulators. - clear.clear(accumulators); - - // The loop index. - Index outer_k = params.k - kUnroll; - - // Enter the main loop and iterate. - for (; outer_k > 0; outer_k -= kUnroll) { - consume_tile<false>(global_stream, shared_load_stream, accumulators, outer_k); - } - - // Residual loop. - for (; outer_k > -kUnroll; outer_k -= kUnroll) { - consume_tile<true>(global_stream, shared_load_stream, accumulators, outer_k); - } - - // Epilogue. - typedef typename Traits::Epilogue Epilogue; - Epilogue epilogue(params.epilogue, shared_storage.epilogue, params.m, params.n); - epilogue.epilogue(cutlass::make_Coord(0, block.y, block.x), accumulators); - } - - /// The params. - Params const& params; - /// The shared storage. - SharedStorage& shared_storage; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_epilogue.h b/cutlass-example/cutlass/gemm/gemm_epilogue.h deleted file mode 100644 index bc25307..0000000 --- a/cutlass-example/cutlass/gemm/gemm_epilogue.h +++ /dev/null @@ -1,231 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Implements the epilogue phase of the GEMM kernel that efficiently updates global memory - with - the computed matrix product. -*/ -#pragma once - -#include <cutlass/convert.h> -#include <cutlass/coord.h> -#include <cutlass/fragment.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename T> -CUTLASS_DEVICE bool is_zero(T x) { - return x == T(0); -} - -#if !defined(__CUDACC_RTC__) || defined(CUTLASS_NVRTC_HAS_FP16) -CUTLASS_DEVICE bool is_zero(half x) { return reinterpret_cast<int16_t&>(x) == int16_t(0); } -#endif - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmEpilogueTraits_> -struct GemmEpilogue { - /// The traits class. - typedef GemmEpilogueTraits_ Traits; - /// The params. - typedef typename Traits::Params Params; - /// The shared storage. - typedef typename Traits::SharedStorage SharedStorage; - - /// The output tile. - typedef typename Traits::OutputTile OutputTile; - /// The number of iterations. - typedef typename Traits::Iterations Iterations; - /// The accumulators. - typedef typename Traits::Accumulators Accumulators; - /// The scalar. - typedef typename Traits::Scalar Scalar; - /// The functor in charge of the math. - typedef typename Traits::Functor Functor; - - /// We do not support 3D or 4D shapes. - static_assert(Iterations::kD == 1 && Iterations::kC == 1, "Unsupported 3D/4D shapes"); - - /// The iterator for C in global memory. - typedef typename Traits::GlobalLoadIteratorC GlobalLoadIteratorC; - /// The transformer for C. - typedef typename Traits::GlobalTransformerC GlobalTransformerC; - /// The transformer for D. - typedef typename Traits::GlobalTransformerD GlobalTransformerD; - /// The iterator for D in global memory. - typedef typename Traits::GlobalStoreIteratorD GlobalStoreIteratorD; - /// The iterator to store D in shared memory. - typedef typename Traits::SharedStoreIteratorD SharedStoreIteratorD; - /// The shared store transformer for D. - typedef typename Traits::SharedStoreTransformerD SharedStoreTransformerD; - /// The iterator to load D in shared memory. - typedef typename Traits::SharedLoadIteratorD SharedLoadIteratorD; - /// The shared load transformer for D. - typedef Copy<typename SharedLoadIteratorD::Fragment> SharedLoadTransformerD; - - /// The index. - typedef typename Traits::Index Index; - - /// The scalar for C. - typedef typename GlobalLoadIteratorC::Scalar ScalarC; - /// The scalar for D. - typedef typename GlobalStoreIteratorD::Scalar ScalarD; - - /// Ctor. - CUTLASS_DEVICE GemmEpilogue(Params const& params_, - SharedStorage& shared_storage_, - Index m_, - Index n_) - : params(params_), shared_storage(shared_storage_), m(m_), n(n_) {} - - /// Execute the epilogue. - CUTLASS_DEVICE void epilogue(Coord<3> const& block, Accumulators& accumulators) { - if (is_zero(params.functor.beta)) { - epilogue_with_or_without_beta<true>(block, accumulators); - } else { - epilogue_with_or_without_beta<false>(block, accumulators); - } - } - - template <bool kBetaIsZero_> - CUTLASS_DEVICE void epilogue_with_or_without_beta(Coord<3> const& block, - Accumulators& accumulators) { - - // The problem size. - Coord<3> const bounds = cutlass::make_Coord(0, n, m); - - // The functor. - Functor functor(params.functor); - // The C fragment. - typename GlobalLoadIteratorC::Fragment fragment_c; - // The transformed C fragment. - typename GlobalTransformerC::OutputFragment transformed_c; - - CUTLASS_PRAGMA_UNROLL - for (int h = 0; h < Iterations::kH; ++h) { - // Compute pointer and predicate offsets for C and D global iterators. - int const pointer_offset = - ((params.iterator_d.inc_h * (GlobalStoreIteratorD::Iterations::kH - 1) + - params.iterator_d.inc_advance) * - Iterations::kW + - params.stride_h) * - h; - int const predicate_offset = - ((params.iterator_d.predicate_inc_h * (GlobalStoreIteratorD::Iterations::kH - 1) + - params.iterator_d.predicate_inc_advance) * - Iterations::kW + - Traits::Delta::kH) * - h; - - // The iterator to load the elements of the C matrix. - GlobalLoadIteratorC global_load_iterator( - params.iterator_c, bounds, block, pointer_offset, predicate_offset); - // The transformer for C. - GlobalTransformerC transformer_c; - // The transformer for D. - GlobalTransformerD transformer_d; - // The iterator to store into the D matrix. - GlobalStoreIteratorD global_store_iterator( - params.iterator_d, bounds, block, pointer_offset, predicate_offset); - - // The transformer to transform before storing to shared memory. - SharedStoreTransformerD shared_store_transformer; - typename SharedStoreTransformerD::OutputFragment shared_store_transformed_d; - - // The iterator to store to shared memory. - SharedStoreIteratorD shared_store_iterator(params.shared_store_iterator_d, - shared_storage.shared_stream.store); - - // The iterator to load from shared memory. TODO: Use a stream. - SharedLoadIteratorD shared_load_iterator(params.shared_load_iterator_d, - shared_storage.shared_stream.load); - - CUTLASS_PRAGMA_UNROLL - for (int w = 0; w < Iterations::kW; ++w) { - // Load the C matrix into fragment. - if (!kBetaIsZero_) { - iterator_load(global_load_iterator, fragment_c); - } - - // Make sure we can write to shared memory. - shared_load_fence(); - - // Copy the accumulators to shared memory. - int const offset = (h * Iterations::kW + w) * SharedStoreIteratorD::Fragment::kElements; - - shared_store_transformer.transform(accumulators, offset, shared_store_transformed_d); - shared_iterator_store(shared_store_iterator, shared_store_transformed_d); - - // Make sure the data is in shared memory. - shared_store_fence(); - - // Copy the accumulators back to registers from shared memory. - typename SharedLoadIteratorD::Fragment fetched_d; - shared_iterator_load(shared_load_iterator, fetched_d); - - // Do the math. - typename GlobalTransformerD::InputFragment fragment_d; - - if (kBetaIsZero_) { - functor.evaluate(fetched_d, fragment_d); - } else { - // Transform C fragment. - transformer_c.transform(fragment_c, transformed_c); - // Do the math. - functor.evaluate(fetched_d, transformed_c, fragment_d); - } - - // Transform D fragment. - typename GlobalTransformerD::OutputFragment transformed_d; - transformer_d.transform(fragment_d, transformed_d); - - // Copy the results to global memory. - iterator_store(global_store_iterator, transformed_d); - } - } - } - - /// The memory fence for shared loads. - CUTLASS_DEVICE void shared_load_fence() { __syncthreads(); } - - /// The memory fence for shared stores. - CUTLASS_DEVICE void shared_store_fence() { __syncthreads(); } - - /// The params. - Params const& params; - /// The shared storage. - SharedStorage& shared_storage; - /// The dimensions of the GEMM. - Index m, n; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_epilogue_traits.h b/cutlass-example/cutlass/gemm/gemm_epilogue_traits.h deleted file mode 100644 index c06fc25..0000000 --- a/cutlass-example/cutlass/gemm/gemm_epilogue_traits.h +++ /dev/null @@ -1,331 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines structural properties of the GEMM epilogue. -*/ -#pragma once - -#include <cutlass/convert.h> -#include <cutlass/coord.h> -#include <cutlass/gemm/gemm_global_stream.h> -#include <cutlass/gemm/gemm_shared_stream.h> -#include <cutlass/gemm/linear_scaling.h> -#include <cutlass/reshape_tile.h> -#include <cutlass/tile_iterator.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The output tile. - typename OutputTile_, - /// The accumulators. - typename Accumulators_, - /// The iterator to load C from global memory. - typename GlobalLoadIteratorC_, - /// The transformer for C. - typename GlobalTransformerC_, - /// The transformer for D. - typename GlobalTransformerD_, - /// The iterator to store D to global memory. - typename GlobalStoreIteratorD_, - /// The iterator to store D to shared memory. - typename SharedStoreIteratorD_, - /// The shared store transformer for D. - typename SharedStoreTransformerD_, - /// The iterator to load D from shared memory. - typename SharedLoadIteratorD_, - /// The number of iterations in the epilogue. - typename Iterations_, - /// The iterations strides. - typename Delta_, - /// The functor to be used in the epilogue. - typename Functor_, - /// The index. - typename Index_ = int> -struct GemmEpilogueTraits { - // - /// The output tile. - typedef OutputTile_ OutputTile; - /// The number of iterations. - /// The accumulators. - typedef Accumulators_ Accumulators; - /// The iterator for C in global memory. - typedef GlobalLoadIteratorC_ GlobalLoadIteratorC; - /// The transformer for C. - typedef GlobalTransformerC_ GlobalTransformerC; - /// The transformer for D. - typedef GlobalTransformerD_ GlobalTransformerD; - /// The iterator for D in global memory. - typedef GlobalStoreIteratorD_ GlobalStoreIteratorD; - /// The iterator to store D in shared memory. - typedef SharedStoreIteratorD_ SharedStoreIteratorD; - /// The shared store transformer for D. - typedef SharedStoreTransformerD_ SharedStoreTransformerD; - /// The iterator to store D in shared memory. - typedef SharedLoadIteratorD_ SharedLoadIteratorD; - /// typedef typename GemmConfig::EpilogueIterations Iterations; - typedef Iterations_ Iterations; - /// The iterations strides. - typedef Delta_ Delta; - - /// The functor in charge of the math. - typedef Functor_ Functor; - /// The index. - typedef Index_ Index; - - /// We do not support 3D or 4D shapes. - static_assert(Iterations::kD == 1 && Iterations::kC == 1, "Unsupported 3D/4D shapes"); - - /// The scalar. - typedef typename Functor::Scalar Scalar; - /// The scalar for C. - typedef typename GlobalLoadIteratorC::Scalar ScalarC; - /// The scalar for D. - typedef typename GlobalStoreIteratorD::Scalar ScalarD; - - /// The params. - struct Params { - /// The strides for H and W in the different iterations of the epilogue. - Index stride_h, stride_w; - /// The params for the C iterator. - typename GlobalLoadIteratorC::Params iterator_c; - /// The params for the D global iterator. - typename GlobalStoreIteratorD::Params iterator_d; - /// The params for the D shared store iterator. - typename SharedStoreIteratorD::Params shared_store_iterator_d; - /// The params for the D shared load iterator. - typename SharedLoadIteratorD::Params shared_load_iterator_d; - /// The functor params. - typename Functor::Params functor; - - /// Setup the params. - template <typename GemmDesc_> - CUTLASS_HOST_DEVICE int initialize(GemmDesc_ const& desc) { - // The parameters for the functor. - int error_code = functor.initialize(desc); - if (error_code) { - return error_code; - } - - // At the end of the H iteration, we jump over a number of columns. - this->stride_h = desc.ldd * Delta::kH; - // Nothing to do here. - this->stride_w = 0; - - // Setup the params for the global memory iterator for C. - error_code = iterator_c.initialize( - reinterpret_cast<ScalarC const*>(desc.d_c), desc.ldc, desc.n, stride_w, Delta::kW); - if (error_code) { - return error_code; - } - - // Setup the params for the global memory iterator for D. - return iterator_d.initialize( - reinterpret_cast<ScalarD*>(desc.d_d), desc.ldd, desc.n, stride_w, Delta::kW); - } - }; - - /// The shared memory storage to exchange data. - union StreamSharedStorage { - // The storage for the store iterator. - typename SharedStoreIteratorD::SharedStorage store; - // The storage for the store iterator. - typename SharedLoadIteratorD::SharedStorage load; - }; - - /// The shared memory to swizzle the data in the epilogue. - struct SharedStorage { - // The storage for the shared stream D. - StreamSharedStorage shared_stream; - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_, typename EpilogueFunctor_, typename Index_ = int> -struct GemmEpilogueTraitsHelper { - /// The scalar. - typedef typename EpilogueFunctor_::Scalar Scalar; - /// The output tile. - typedef typename GemmConfig_::OutputTile OutputTile; - - /// The number of iterations in the epilogue. - typedef Shape<1, - GemmConfig_::MultiplyAdd::AccumulatorsPerThread::kH / - GemmConfig_::kAccumulatorsPerLdsB, - GemmConfig_::kAccumulatorsPerLdsB> - Iterations; - // The iteration strides in the H/W dimension. - typedef Shape<0, - GemmConfig_::kAccumulatorsPerLdsB*( - GemmConfig_::Warps::kH* GemmConfig_::MultiplyAdd::ThreadsPerWarp::kH - 1), - 0> - Delta; - /// The functor to do the math in the epilogue. - typedef EpilogueFunctor_ Functor; - - /// The traits class to build the iterator to store to shared memory for D. - typedef GemmSharedStoreTileDTraits< - // The pointer is float. - typename Functor::Scalar, - // The output tile size. - typename GemmConfig_::OutputTile, - // The number of warps. - typename GemmConfig_::Warps, - // The number of threads per warp. - typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, - // The number of scalars per STS. - GemmConfig_::kScalarsPerStsD, - // The skew -- 128 / sizeof(ScalarD) / kScalarsPerStsD is the number of threads involved in - // a single STS. We divide by 2 as our objective is to add a skew to the odd threads to - // avoid bank conflicts between odd and even threads. - 128 / sizeof(typename GemmConfig_::ScalarD) / GemmConfig_::kScalarsPerStsD / 2 * - GemmConfig_::kScalarsPerStsD> - SharedStoreTileTraits; - - /// The iterator to store D to shared memory. - typedef TileStoreIterator<SharedStoreTileTraits, - typename SharedStoreTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedStoreIteratorD; - - /// The shared store transformer for D. - typedef Copy<typename SharedStoreIteratorD::Fragment> SharedStoreTransformerD; - - /// The traits class to build the iterator to load from shared memory for D. - typedef GemmSharedLoadTileDTraits< - // The pointer is float. - typename Functor::Scalar, - // The output tile size. - typename GemmConfig_::OutputTile, - // The number of warps. - typename GemmConfig_::Warps, - // The number of threads per warp. - typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, - // The number of columns of the output tile written by iteration. - GemmConfig_::OutputTile::kH / ShapeCount<Iterations>::kCount, - // The number of scalars per LDS. - GemmConfig_::kScalarsPerLdsD, - // The skew. - SharedStoreTileTraits::kSkew> - SharedLoadTileTraits; - - /// The iterator to load D from shared memory. - typedef TileLoadIterator<SharedLoadTileTraits, - typename SharedLoadTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedLoadIteratorD; - - /// The traits class to build the iterator to load data from global memory for C^N. - typedef GemmGlobalTileCdTraits< - // The pointer is float const. - typename GemmConfig_::ScalarC const, - // The tile has size (N / Iterations)xM in GEMM's terminology. - Shape<1, - GemmConfig_::OutputTile::kH / ShapeCount<Iterations>::kCount, - GemmConfig_::OutputTile::kW>, - // The threads are distributed as warps x 32 (the traits may reorganize). - Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, - // How many elements do we jump over at each iteration? - Iterations::kW, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerLdgC> - GlobalLoadTileTraits; - - /// The iterator to load C. - typedef GemmGlobalIteratorCd<GlobalLoadTileTraits, Index_> GlobalLoadIteratorC; - /// The transformer for C. - typedef Copy<typename GlobalLoadIteratorC::Fragment> GlobalTransformerC; - - /// The traits class to build the iterator to store data to global memory for D^N. - typedef GemmGlobalTileCdTraits< - // The pointer is float. - typename GemmConfig_::ScalarD, - // The tile has size (N / Iterations)xM in GEMM's terminology. - Shape<1, - GemmConfig_::OutputTile::kH / ShapeCount<Iterations>::kCount, - GemmConfig_::OutputTile::kW>, - // The threads are distributed as warps x 32 (the traits may reorganize). - Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, - // How many elements do we jump over at each iteration? - Iterations::kW, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerStgD> - GlobalStoreTileTraits; - - /// The iterator to store D. - typedef GemmGlobalIteratorCd<GlobalStoreTileTraits, Index_> GlobalStoreIteratorD; - /// The transformer for D. - typedef Copy<typename GlobalStoreIteratorD::Fragment> GlobalTransformerD; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The GEMM config. - typename GemmConfig_, - /// The epilogue functor to do the math in the epilogue. - typename EpilogueFunctor_, - /// The index. - typename Index_ = int, - /// The helper to create the traits class. - typename Helper_ = GemmEpilogueTraitsHelper<GemmConfig_, EpilogueFunctor_, Index_> > -struct SimplifiedGemmEpilogueTraits : public GemmEpilogueTraits< - // The output tile. - typename GemmConfig_::OutputTile, - // The accumulators. - typename GemmConfig_::Accumulators, - // The global iterator for C. - typename Helper_::GlobalLoadIteratorC, - // The transformer for C. - typename Helper_::GlobalTransformerC, - // The transformer for D. - typename Helper_::GlobalTransformerD, - // The global iterator for D. - typename Helper_::GlobalStoreIteratorD, - // The iterator to store D to shared memory. - typename Helper_::SharedStoreIteratorD, - // The shared store transformer for D. - typename Helper_::SharedStoreTransformerD, - // The iterator to load D from shared memory. - typename Helper_::SharedLoadIteratorD, - // The number of iterations. - typename Helper_::Iterations, - // The strides between iterations. - typename Helper_::Delta, - // The functor to be used in the epilogue. - EpilogueFunctor_, - // The index. - Index_> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_global_stream.h b/cutlass-example/cutlass/gemm/gemm_global_stream.h deleted file mode 100644 index ec675a3..0000000 --- a/cutlass-example/cutlass/gemm/gemm_global_stream.h +++ /dev/null @@ -1,182 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Implements efficient loading of the thread block-level tile from global memory and - storing - to shared memory. -*/ -#pragma once - -#include <cutlass/convert.h> -#include <cutlass/gemm/gemm_global_tile.h> -#include <cutlass/iterator_access.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The load iterator. - typename LoadIterator_, - /// The store iterator to copy to shared memory. - typename StoreIterator_, - /// The transformer to be applied after the data has been copied from global memory. - typename Transformer_> - -struct GlobalLoadStreamBase { - /// The load iterator. - typedef LoadIterator_ LoadIterator; - /// The transformer. - typedef Transformer_ Transformer; - /// The store iterator to write to shared memory. - typedef StoreIterator_ StoreIterator; - - /// The fragment that is copied from shared memory. - typedef typename LoadIterator::Fragment FetchedFragment; - /// The fragment that is obtained after the transformation by the transformer. - typedef typename Transformer::OutputFragment TransformedFragment; - /// Make sure the fragments match. - static_assert((platform::is_same<FetchedFragment, typename Transformer::InputFragment>::value), - ""); - /// The output fragment. - typedef TransformedFragment Fragment; - /// Make sure the transformed fragment is the same as the store fragment. - static_assert((platform::is_same<TransformedFragment, typename StoreIterator::Fragment>::value), - ""); - - /// The layout. - static MatrixLayout::Kind const kLayout = LoadIterator::kLayout; - /// The scalar type of the iterator. - typedef typename LoadIterator::Scalar Scalar; - /// The pointer. - typedef typename LoadIterator::Pointer Pointer; - /// The index. - typedef typename LoadIterator::Index Index; - - /// The params. - struct Params { - // The load iterator. - typename LoadIterator::Params load_iterator; - // The store iterator. - typename StoreIterator::Params store_iterator; - - /// Setup the params. - template <typename GemmDesc_> - CUTLASS_HOST_DEVICE int initialize(GemmDesc_ const& desc, Pointer pointer, Index ld) { - int error_code = load_iterator.initialize(desc, pointer, ld); - if (error_code) { - return error_code; - } - - return store_iterator.initialize(); - } - }; - - /// The amount of storage in shared memory needed to store the tile. - typedef typename StoreIterator::SharedStorage SharedStoreStorage; - - /// The storage in shared memory needed by that stream. - union SharedStorage { - // The load iterator. - typename LoadIterator::SharedStorage load_iterator; - // The store iterator. - SharedStoreStorage store_iterator; - }; - - /// Ctor. - CUTLASS_DEVICE GlobalLoadStreamBase(Params const& params, - SharedStorage& shared_storage, - Coord<3> const bounds, - Coord<3> const& block) - : load_iterator(params.load_iterator, bounds, block), - transformer(), - store_iterator(params.store_iterator, shared_storage.store_iterator) - - { - fetched_fragment.clear(); - } - - /// Load the data from shared memory to the fetch fragment. - CUTLASS_DEVICE void copy() { iterator_load(load_iterator, fetched_fragment); } - - /// Commit the data. - CUTLASS_DEVICE void commit() { - transformer.transform(fetched_fragment, transformed_fragment); - iterator_store(store_iterator, transformed_fragment); - store_iterator.inc_stage(); - } - - /// Move to the beginning of the residue code. That's a new code path in CUTLASS 1.0.1. - CUTLASS_DEVICE void move_to_residue(Index k) { load_iterator.move_to_residue(k); } - - /// Execute the residue code. - CUTLASS_DEVICE void residue(Index k, bool skip_clear = false) { - load_iterator.residue(k); - if (!skip_clear) { - fetched_fragment.clear(); - } - } - - /// Rollback to the beginning of the GEMM-k dimension. - CUTLASS_DEVICE void rollback() { load_iterator.rollback(); } - - /// The iterator. - LoadIterator load_iterator; - /// The fragment to fetch from shared memory. - FetchedFragment fetched_fragment; - /// The transformer. - Transformer transformer; - /// The fragment to convert the data after it has been fetched from shared memory. - TransformedFragment transformed_fragment; - /// The store iterator. - StoreIterator store_iterator; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The load iterator. - typename LoadIterator_, - /// The store iterator to copy to shared memory. - typename StoreIterator_, - /// The transformer to be applied after the data has been copied from global memory. - typename Transformer_ = Copy<typename LoadIterator_::Fragment> > - -struct GlobalLoadStream : public GlobalLoadStreamBase<LoadIterator_, StoreIterator_, Transformer_> { - /// The base class. - typedef GlobalLoadStreamBase<LoadIterator_, StoreIterator_, Transformer_> Base; - - /// Ctor. - CUTLASS_DEVICE GlobalLoadStream(typename Base::Params const& params, - typename Base::SharedStorage& shared_storage, - Coord<3> const& bounds, - Coord<3> const& block) - : Base(params, shared_storage, bounds, block) {} -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_global_tile.h b/cutlass-example/cutlass/gemm/gemm_global_tile.h deleted file mode 100644 index 1cc3b33..0000000 --- a/cutlass-example/cutlass/gemm/gemm_global_tile.h +++ /dev/null @@ -1,541 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines iterators for efficiently loading and storing to global memory. -*/ -#pragma once - -#include <cutlass/coord.h> -#include <cutlass/util/platform.h> - -#include <cutlass/gemm/gemm_operand.h> -#include <cutlass/matrix_traits.h> -#include <cutlass/predicate_vector.h> -#include <cutlass/reshape_tile.h> -#include <cutlass/tile_iterator.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -// The following functor reshapes a tile of threads to match a tile of data. The idea is that when -// the user wants to build the iterator traits, he/she may want to specify the tile independently -// from the number of scalars loaded/stored per instruction. For example, in the row-major version -// with a tile of size 128x8 - the user may want to that the iterator works with 32x8 threads if -// each thread loads 1 scalar per LDG. If the user changes to 4 scalars per LDG, then the tile of -// threads has to change. The code below detects that and correct the code automatically - it is -// a helper when the user does not specify the right configuration. - -template <typename Tile_, typename Threads_, bool = (Tile_::kW < Threads_::kW)> -struct ReshapeThreads { - typedef Threads_ Threads; -}; - -template <typename Tile_, typename Threads_> -struct ReshapeThreads<Tile_, Threads_, true> { - typedef Shape<Threads_::kD, Threads_::kH * Threads_::kW / Tile_::kW, Tile_::kW, 1> Threads; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <GemmOperand::Kind kOperand_, - MatrixLayout::Kind kLayout_, - typename Scalar_, - typename Tile_, - typename Threads_, - int kAccessSize_> -struct GemmGlobalTileTraits { - /// Identity of the operand - static GemmOperand::Kind const kOperand = kOperand_; - /// The layout. - static MatrixLayout::Kind const kLayout = kLayout_; - /// The scalar. - typedef typename platform::remove_const<Scalar_>::type Scalar; - /// The pointer. - typedef Scalar_* Pointer; - /// The number of scalars per LDG/STG. - static int const kAccessSize = kAccessSize_; - /// The memory space. - static MemorySpace::Kind const kMemorySpace = MemorySpace::kGlobal; - - /// The tile shape - typedef typename ReshapeTile<Tile_, kAccessSize_>::Tile Tile; - /// The threads shape - typedef typename ReshapeThreads<Tile, Threads_>::Threads Threads; - /// The relative offset between two elements in the H/W dimension in adjacent threads. - typedef Shape<1, 1, Tile::kC> ThreadsDelta; - - /// The strides in each dimension between different loads/stores. - typedef Shape<0, Threads::kH, Threads::kW * kAccessSize> Delta; - /// Strides for immediate offset computation - typedef Shape<0, 0, Threads::kW * ThreadsDelta::kW, kAccessSize> ImmediateOffsetStrides; - /// The number of iterations needed to load/store the tile. - typedef Shape<1, Tile::kH / Threads::kH, Tile::kW / Threads::kW, Tile::kC / kAccessSize> - Iterations; - - typedef GemmMultiplicandTraits<Tile, kOperand, kLayout> MultiplicandTraits; - - /// Computes the thread offset in (H, W) based on thread ID - struct ThreadOffset { - CUTLASS_HOST_DEVICE - Coord<4> operator()() const { - int thread_offset_h = threadIdx.x / Threads::kW * ThreadsDelta::kH; - int thread_offset_w = threadIdx.x % Threads::kW * ThreadsDelta::kW; - - return make_Coord(0, thread_offset_h, thread_offset_w, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, typename Tile_, typename Threads_, int kStrideH_, int kAccessSize_> -struct GemmGlobalTileCdTraits : public GemmGlobalTileTraits<GemmOperand::kC, - MatrixLayout::kColumnMajor, - Scalar_, - Tile_, - Threads_, - kAccessSize_> { - /// The base class. - typedef GemmGlobalTileTraits<GemmOperand::kC, - MatrixLayout::kColumnMajor, - Scalar_, - Tile_, - Threads_, - kAccessSize_> - Base; - - /// The stride in the H dimension. - static int const kStrideH = kStrideH_; - /// Override the strides in each dimension between different loads/stores. - typedef Shape<0, 0, Base::Delta::kW, Base::Delta::kC> Delta; - - typedef typename Base::Iterations Iterations; - - typedef typename Base::Threads Threads; - - typedef typename Base::ThreadsDelta ThreadsDelta; - - typedef typename Base::ImmediateOffsetStrides ImmediateOffsetStrides; - - /// Computes the thread offset in (H, W) based on thread ID - struct ThreadOffset { - CUTLASS_HOST_DEVICE - Coord<4> operator()() const { - int thread_offset_h = threadIdx.x / Threads::kW * kStrideH * Iterations::kH; - int thread_offset_w = threadIdx.x % Threads::kW * ThreadsDelta::kW; - - return make_Coord(0, thread_offset_h, thread_offset_w, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename TileTraits_, typename Index_ = int> -struct GemmGlobalIteratorAb - : public TileLoadIterator<TileTraits_, - typename TileTraits_::Scalar, - TileTraits_::MultiplicandTraits::kKstrided ? IteratorAdvance::kH - : IteratorAdvance::kW, - MemorySpace::kGlobal, - Index_> { - /// This class. - typedef GemmGlobalIteratorAb<TileTraits_, Index_> This_; /// The base class. - - typedef TileLoadIterator<TileTraits_, - typename TileTraits_::Scalar, - TileTraits_::MultiplicandTraits::kKstrided ? IteratorAdvance::kH - : IteratorAdvance::kW, - MemorySpace::kGlobal, - Index_> - Base; - /// The layout. - static MatrixLayout::Kind const kLayout = TileTraits_::kLayout; - /// Fragment type loaded by the iterator - typedef typename Base::Fragment Fragment; - /// The scalar. - typedef typename TileTraits_::Scalar Scalar; - /// The threads. - typedef typename TileTraits_::Threads Threads; - /// The index. - typedef Index_ Index; - /// The thread offset - typedef typename TileTraits_::ThreadOffset ThreadOffset; - /// Specifies in which dimension post-increment accesses advance. - static IteratorAdvance::Kind const kAdvance = Base::kAdvance; - - typedef cutlass::PredicateVector<ShapeCount<typename Base::Iterations>::kCount> PredicateVector; - - /// Iterator parameters type - typedef typename Base::Params BaseParams; - - struct Params : public BaseParams { - /// Initializes params to load a strip-mined tile, given pointer and stride_h. - template <typename GemmDesc_> - CUTLASS_HOST_DEVICE int initialize(GemmDesc_ const& desc, Scalar const* ptr, Index stride_h) { - Index inc_d = 0; - Index inc_advance = 0; - // Move by some columns for each iteration in the H dimension. - Index inc_h = Base::Delta::kH * stride_h; - - // Move by some more columns in the number of iterations if the D dimension is > 1. - if (Base::Delta::kD > 0) { - inc_d = Base::Delta::kD * stride_h - (Base::Iterations::kH - 1) * inc_h; - } - - // Move to the beginning of the next iteration. - if (kAdvance == IteratorAdvance::kH && Base::Delta::kD > 0) { - inc_advance = inc_d; - } else if (kAdvance == IteratorAdvance::kH) { - inc_advance = inc_h; - } else if (Base::Delta::kD > 0) { - inc_advance = (Base::Iterations::kW + 0) * ShapeCount<typename Base::Delta>::kWc - - (Base::Iterations::kH - 1) * inc_h - - (Base::Iterations::kD - 1) * Base::Delta::kD * stride_h; - } else { - inc_advance = (Base::Iterations::kW + 0) * ShapeCount<typename Base::Delta>::kWc - - (Base::Iterations::kH - 1) * inc_h; - } - - // The dimensions of the tile. - int const kH = TileTraits_::Tile::kH; - int const kW = TileTraits_::Tile::kW * TileTraits_::kAccessSize; - - // Move to the residue. - Index const kBlock = kAdvance == IteratorAdvance::kH ? kH : kW; - // The jump in the gemm-k dimension. - Index const stride = kAdvance == IteratorAdvance::kH ? stride_h : 1; - - // Compute the offset to the residue and how to "come" back. - Index const kResidue = desc.k % kBlock; - if (kResidue > 0) { - move_to_residue_offset = (desc.k - kResidue) * stride; - } else { - move_to_residue_offset = (desc.k - kBlock) * stride; - } - - Base::Params::initialize(ptr, 0, stride_h, 1, inc_d, inc_h, 0, inc_advance); - return 0; - } - - // The extra offset to control moving to the residue. - Index move_to_residue_offset; - }; - - /// Ctor. - CUTLASS_DEVICE GemmGlobalIteratorAb(Params const& _params, - const Coord<3>& bounds, - const Coord<3>& block, - ThreadOffset thread_offset_func = ThreadOffset()) - : params(_params) { - thread_offset = thread_offset_func(); - // The column. - Index block_h = thread_offset[1]; - // The contiguous dimension. - Index block_w = thread_offset[2]; - - // Add the blocks indices. - if (kAdvance == IteratorAdvance::kH) { - block_h += block[1]; - block_w += block[2]; - - } else { - block_h += block[2]; - block_w += block[1]; - } - - // Setup the pointer. - params.pointer += (block_h * params.stride_h + block_w); - - // Initialize predicates - initialize_predicates(bounds, make_Coord(0, block_h, block_w)); - } - - /// The accessor. - CUTLASS_DEVICE void get(typename Base::AccessType& value, int d, int h, int w, int c) const { - int const imm = - ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(0, 0, w, c); - Load<Scalar, TileTraits_::kAccessSize, MemorySpace::kGlobal>::load(value, params.pointer, imm); - } - - /// Increment the pointer in the H dimension. - CUTLASS_DEVICE void inc_h() { params.pointer += params.inc_h; } - /// Increment the pointer in the D dimension. - CUTLASS_DEVICE void inc_d() { params.pointer += params.inc_d; } - /// Increment the pointer to move to the next iteration. - CUTLASS_DEVICE void inc_advance() { params.pointer += params.inc_advance; } - - /// Initialize the predicates. - CUTLASS_DEVICE void initialize_predicates(const Coord<3>& bounds, const Coord<3>& block) { - // Setup the masks to control loads. - predicates.fill(0); - - int bounds_h, bounds_w; - if (kAdvance == IteratorAdvance::kH) { - bounds_w = bounds[2] - block[2]; - bounds_h = bounds[1]; - - } else { - bounds_w = bounds[1]; - bounds_h = bounds[2] - block[1]; - } - - // Fill in the bits of the predicate vector. - for (int d = 0; d < Base::Iterations::kD; ++d) { - for (int h = 0; h < Base::Iterations::kH; ++h) { - for (int w = 0; w < Base::Iterations::kW; ++w) { - for (int c = 0; c < Base::Iterations::kC; ++c) { - bool flag = w * Base::Delta::kW < bounds_w; - if (kAdvance == IteratorAdvance::kH) { - flag = flag && (h * Base::Delta::kH + d * Base::Delta::kD) < bounds_h; - } else { - flag = flag && (h * Base::Delta::kH) < bounds_h; - } - int const bit = ComputeOffsetFromShape<typename Base::Iterations>::get(d, h, w, c); - predicates.set(bit, flag); - } - } - } - } - } - - /// Move to residue portion. - CUTLASS_DEVICE void move_to_residue(Index k) { - // Store the pointer and the predicates. - stored_pointer = params.pointer; - stored_predicates = predicates; - - // Move the pointer to the residue. - params.pointer += params.move_to_residue_offset; - - // The dimensions of the tile. - int const kH = TileTraits_::Tile::kH; - int const kW = TileTraits_::Tile::kW * TileTraits_::kAccessSize; - - // The unrolling factor. - int const kUnroll = kAdvance == IteratorAdvance::kH ? kH : kW; - - // Clear the predicates for the residue. TODO: We can do something smarter. - int const kResidue = (int)(k % (Index)kUnroll); - if (kResidue > 0) { - residue(kResidue); - } - } - - /// That's the residue! Update the predicates. - CUTLASS_DEVICE void residue(Index k) { - // The coordinates of the thread. - Index block_h = thread_offset[1]; - // The contiguous dimension. - Index block_w = thread_offset[2]; - - // Update the predicate vector. - for (int d = 0; d < Base::Iterations::kD; ++d) { - for (int h = 0; h < Base::Iterations::kH; ++h) { - for (int w = 0; w < Base::Iterations::kW; ++w) { - for (int c = 0; c < Base::Iterations::kC; ++c) { - Index offset = 0; - if (kAdvance == IteratorAdvance::kH) { - offset += block_h + h * Base::Delta::kH + d * Base::Delta::kD; - } else { - offset += block_w + w * Base::Delta::kW; - } - - int const bit = ComputeOffsetFromShape<typename Base::Iterations>::get(d, h, w, c); - if (offset >= k) { - predicates.set(bit, false); - } - } - } - } - } - } - - /// Rollback to beginning of first tile and initialize predicates. - CUTLASS_DEVICE void rollback() { - params.pointer = stored_pointer; - predicates = stored_predicates; - } - - /// Is the iterator valid? - CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { - int const bit = ComputeOffsetFromShape<typename Base::Iterations>::get(d, h, w, c); - return predicates[bit]; - } - - /// Offset of an individual lane from the start of the tile - Coord<4> thread_offset; - /// The parameters - Params params; - /// The pointer. - typename Base::Scalar const* stored_pointer; - /// The predicates. - PredicateVector predicates, stored_predicates; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename TileTraits_, typename Index_ = int> -struct GemmGlobalIteratorCd : public TileIteratorBase<TileTraits_, - typename TileTraits_::Scalar, - IteratorAdvance::kH, - MemorySpace::kGlobal, - Index_> { - /// This class. - typedef GemmGlobalIteratorCd<TileTraits_, Index_> This_; - /// The base class. - typedef TileIteratorBase<TileTraits_, - typename TileTraits_::Scalar, - IteratorAdvance::kH, - MemorySpace::kGlobal, - Index_> - Base; - - /// The layout. - static MatrixLayout::Kind const kLayout = TileTraits_::kLayout; - - /// The scalar. - typedef typename TileTraits_::Scalar Scalar; - /// The pointer. - typedef typename TileTraits_::Pointer Pointer; - /// The threads. - typedef typename TileTraits_::Threads Threads; - /// The index. - typedef Index_ Index; - /// The thread offset - typedef typename TileTraits_::ThreadOffset ThreadOffset; - - /// The params. - struct Params { - /// The pointer. - Pointer pointer; - /// The stride in the H dimension to setup the thread in the block. - Index stride_h; - /// The strides to increment the pointer. - Index inc_advance, inc_h; - /// The strides to increment the predicate offset - Index predicate_inc_advance, predicate_inc_h; - /// The column offset to compute the predicate for the columns. - Index predicate_offset; - - /// Setup the params. - CUTLASS_HOST_DEVICE int initialize( - Pointer pointer, Index ld, Index bound, Index epilogue_stride_w, Index epilogue_delta_w) { - // The pointer. - this->pointer = pointer; - // Each column of the matrix. - stride_h = TileTraits_::ThreadsDelta::kH * ld; - // Each thread output 1 column per iteration. The stride between columns is given by the - // number of scalars that are loaded per LDS for B. - inc_h = ld * TileTraits_::kStrideH; - inc_advance = - (ld - ld * TileTraits_::kStrideH * (Base::Iterations::kH - 1)) + epilogue_stride_w; - - predicate_offset = bound; - predicate_inc_h = TileTraits_::kStrideH; - predicate_inc_advance = - -((TileTraits_::kStrideH * (Base::Iterations::kH - 1) - 1) + epilogue_delta_w); - - return 0; - } - }; - - Params params; - /// Offset of an individual lane from the start of the tile - Coord<4> thread_offset; - - /// Ctor. - CUTLASS_DEVICE GemmGlobalIteratorCd() {} - - /// Ctor. - CUTLASS_DEVICE GemmGlobalIteratorCd(Params const& params, - const Coord<3>& bounds, - const Coord<3>& block, - int offset = 0, - int pred_offset = 0, - ThreadOffset thread_offset_func = ThreadOffset()) - : params(params) { - thread_offset = thread_offset_func(); - // Each warp works on a different column of the tile. - int const h = thread_offset[1] + block[1]; - // Each lane writes a different element. - int const w = thread_offset[2] + block[2]; - // Setup the pointer. - this->params.pointer += ((h * params.stride_h + w) + offset); - - // Prepare the vector of predicates. - for (int i = 0; i < Base::Iterations::kW; ++i) { - predicates.set(i, w + i * Base::Delta::kW < bounds[2]); - } - this->params.predicate_offset -= (h + pred_offset); - } - - /// The accessor. - CUTLASS_DEVICE void get(typename Base::AccessType& value, int d, int h, int w, int c) const { - int const imm = - ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(0, 0, w, c); - Load<Scalar, TileTraits_::kAccessSize, MemorySpace::kGlobal>::load(value, params.pointer, imm); - } - - /// Increment the pointer in the C dimension. - CUTLASS_DEVICE void inc_c() {} - /// Increment the pointer in the W dimension. - CUTLASS_DEVICE void inc_w() {} - /// Increment the pointer in the H dimension. - CUTLASS_DEVICE void inc_h() { - params.pointer += params.inc_h; - params.predicate_offset -= params.predicate_inc_h; - } - /// Increment the pointer in the D dimension. - CUTLASS_DEVICE void inc_d() {} - /// Increment the pointer to move to the next iteration. - CUTLASS_DEVICE void inc_advance() { - params.pointer += params.inc_advance; - this->params.predicate_offset -= params.predicate_inc_advance; - } - - /// The accessor. - CUTLASS_DEVICE void set(typename Base::AccessType const& value, int d, int h, int w, int c) { - int const imm = - ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(0, 0, w, c); - Store<Scalar, TileTraits_::kAccessSize, MemorySpace::kGlobal>::store( - value, params.pointer, imm); - } - - /// Test the validity of the iterator. - CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { - return predicates.at(w) && params.predicate_offset > 0; - } - - /// The predicates for the row. - cutlass::PredicateVector<Base::Iterations::kW> predicates; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_operand.h b/cutlass-example/cutlass/gemm/gemm_operand.h deleted file mode 100644 index 737f993..0000000 --- a/cutlass-example/cutlass/gemm/gemm_operand.h +++ /dev/null @@ -1,141 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines constant expressions for mapping GEMM problem size and strides onto pitch-linear - memory. -*/ -#pragma once - -#include <cutlass/matrix_traits.h> -#include <cutlass/reshape_tile.h> -#include <cutlass/util/platform.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Helper to describe attributes of GEMM matrix operands -template <GemmOperand::Kind kOperand_, MatrixLayout::Kind kLayout_> -struct GemmOperandTraitsAb { - static const bool Congruous = - (kOperand_ == GemmOperand::kA ^ kLayout_ == MatrixLayout::kRowMajor); -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmOperand::Kind kOperand_, typename Tile_> -struct GetExtent; - -template <typename Tile_> -struct GetExtent<GemmOperand::kA, Tile_> { - static const int kExtent = Tile_::kW; -}; - -template <typename Tile_> -struct GetExtent<GemmOperand::kB, Tile_> { - static const int kExtent = Tile_::kH; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Determines the shape of a multiplicand tile in terms of strided (H) and contiguous (W) -/// dimensions -template <typename ThreadBlockTile_, GemmOperand::Kind Usage, MatrixLayout::Kind Layout> -struct GemmMultiplicandTraits { - // Only defined for A or B - static_assert(Usage == GemmOperand::kA || Usage == GemmOperand::kB, - "MultiplicandTileShape defined only for A or B operands."); - - /// Shape of GEMM thread block tile (K, N, M) - typedef ThreadBlockTile_ ThreadBlockTile; - - /// Identifies multiplicand - static GemmOperand::Kind const kUsage = Usage; - - /// Layout of tile - static MatrixLayout::Kind const kLayout = Layout; - - // True if K is the strided dimension - static bool const kKstrided = (kUsage == GemmOperand::kA ^ kLayout == MatrixLayout::kRowMajor); - - /// Map the ThreadBlockShape onto (kH, kW) dimensions for A and B operand - typedef typename platform::conditional< - kKstrided, - Shape<1, ThreadBlockTile::kD, GetExtent<Usage, ThreadBlockTile>::kExtent>, - Shape<1, GetExtent<Usage, ThreadBlockTile>::kExtent, ThreadBlockTile::kD> >::type Shape; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Project's a coordinate (K, N, M) onto inner and outer dimensions defined for each -/// operand. -template <GemmOperand::Kind operand, bool Kstrided = true> -struct ProjectOperand; - -/// Project A operand - (0, K, M) -template <bool Kstrided> -struct ProjectOperand<GemmOperand::kA, Kstrided> { - CUTLASS_HOST_DEVICE - static Coord<3> project(Coord<3> const &coord) { - if (Kstrided) { - return make_Coord(0, coord[0], coord[2]); - } else { - return make_Coord(0, coord[2], coord[0]); - } - } -}; - -/// Project B operand - (0, K, N) -template <bool Kstrided> -struct ProjectOperand<GemmOperand::kB, Kstrided> { - CUTLASS_HOST_DEVICE - static Coord<3> project(Coord<3> const &coord) { - if (Kstrided) { - return make_Coord(0, coord[0], coord[1]); - } else { - return make_Coord(0, coord[1], coord[0]); - } - } -}; - -/// Project C operand - (0, N, M) -template <> -struct ProjectOperand<GemmOperand::kC, true> { - CUTLASS_HOST_DEVICE - static Coord<3> project(Coord<3> const &coord) { return make_Coord(0, coord[1], coord[2]); } -}; - -/// Project D operand - (0, N, M) -template <> -struct ProjectOperand<GemmOperand::kD, true> { - CUTLASS_HOST_DEVICE - static Coord<3> project(Coord<3> const &coord) { return make_Coord(0, coord[1], coord[2]); } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_shared_stream.h b/cutlass-example/cutlass/gemm/gemm_shared_stream.h deleted file mode 100644 index c6ff7bd..0000000 --- a/cutlass-example/cutlass/gemm/gemm_shared_stream.h +++ /dev/null @@ -1,113 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines abstractions for managing loading and storing fragments to shared memory in the - efficient GEMM pipeline. -*/ -#pragma once - -#include <cutlass/gemm/gemm_shared_tile.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The load iterator. - typename Iterator_, - /// The transformer to be applied after the data has been copied from shared memory. - typename Transformer_ = Copy<typename Iterator_::Fragment> > - -struct SharedLoadStream { - /// The load iterator. - typedef Iterator_ Iterator; - /// The transformer. - typedef Transformer_ Transformer; - - /// The fragment that is copied from shared memory. - typedef typename Iterator::Fragment FetchedFragment; - /// The fragment that is obtained after the transformation by the transformer. - typedef typename Transformer::OutputFragment TransformedFragment; - /// Make sure the fragments match. - static_assert((platform::is_same<FetchedFragment, typename Transformer::InputFragment>::value), - ""); - /// The output fragment. - typedef TransformedFragment Fragment; - - /// The params. - struct Params { - /// The iterator params. - typename Iterator::Params iterator; - - /// Setup the params. - CUTLASS_HOST_DEVICE int initialize() { return iterator.initialize(); } - }; - - /// The storage in shared memory needed by that stream. - typedef typename Iterator::Storage SharedStorage; - - /// Ctor. - CUTLASS_DEVICE SharedLoadStream() {} - - /// Ctor. - CUTLASS_DEVICE SharedLoadStream(Params const ¶ms, SharedStorage &shared_storage) { - this->initialize(params, shared_storage); - } - - /// Initialize the stream. - CUTLASS_DEVICE void initialize(Params const ¶ms, SharedStorage &shared_storage) { - // The iterator. - iterator = Iterator(params.iterator, shared_storage); - // The transformer. - transformer = Transformer(); - } - - /// Load the data from shared memory to the fetch fragment. - CUTLASS_DEVICE void copy(FetchedFragment &fetched) { shared_iterator_load(iterator, fetched); } - - /// Load the data from shared memory to the fetch fragment. - CUTLASS_DEVICE void copy(int d, FetchedFragment &fetched) { - shared_iterator_load(iterator, fetched, d); - } - - /// Commit the data. - CUTLASS_DEVICE void commit(FetchedFragment &fetched, TransformedFragment &transformed) { - transformer.transform(fetched, transformed); - } - - /// Increment the stage. - CUTLASS_DEVICE void inc_stage() { iterator.inc_stage(); } - - /// The iterator. - Iterator iterator; - /// The transformer. - Transformer transformer; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_shared_tile.h b/cutlass-example/cutlass/gemm/gemm_shared_tile.h deleted file mode 100644 index 7c61e02..0000000 --- a/cutlass-example/cutlass/gemm/gemm_shared_tile.h +++ /dev/null @@ -1,417 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines iterators for efficiently loading and storing tiles to and from shared memory. -*/ -#pragma once - -#include <cutlass/gemm/gemm_operand.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, typename Tile_, typename Threads_, int kScalarsPerSts_> -struct GemmSharedStoreTileAbTraits { - /// The scalar. - typedef typename platform::remove_const<Scalar_>::type Scalar; - /// The pointer. - typedef Scalar_* Pointer; - /// The tile. - typedef typename ReshapeTile<Tile_, kScalarsPerSts_>::Tile Tile; - /// The threads. - typedef Threads_ Threads; - /// The strides to compute the base position of the thread. - typedef Shape<0, ShapeCount<Tile>::kWc, Tile::kC, kScalarsPerSts_> ThreadsStrides; - /// The skew. - static int const kSkew = 0; - /// The number of scalars per LDG/STG. - static int const kAccessSize = kScalarsPerSts_; - /// The memory space. - static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; - - /// The number of iterations needed to load/store the tile. - typedef Shape<1, - Tile::kH / Threads::kH, - Tile::kW / Threads::kW, - Tile::kC / Threads::kC / kAccessSize> - Iterations; - /// The strides in each dimension between different loads/stores. - typedef Shape<0, Threads::kH * ShapeCount<Tile>::kWc, Threads::kW * kAccessSize> Delta; - /// The strides in each dimension between different loads/stores. - typedef Shape<0, Threads::kH * ShapeCount<Tile>::kWc, Threads::kW * kAccessSize> - ImmediateOffsetStrides; - - struct ThreadOffset { - CUTLASS_HOST_DEVICE - Coord<4> operator()() const { - int offset = ComputeThreadOffsetFromStrides<Threads, ThreadsStrides>::get(); - return make_Coord(0, 0, offset, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, typename Tile_, typename Threads_, int kScalarsPerSts_, int kSkew_> -struct GemmSharedStoreWithSkewTileAbTraits { - /// The scalar. - typedef typename platform::remove_const<Scalar_>::type Scalar; - /// The pointer. - typedef Scalar_* Pointer; - /// The tile without skews. - typedef typename ReshapeTile<Tile_, kScalarsPerSts_>::Tile TileWithoutSkew; - /// The tile. - typedef typename ReshapeTile<Shape<Tile_::kD, Tile_::kH, Tile_::kW + kSkew_>, - kScalarsPerSts_>::Tile Tile; - /// The threads. - typedef Threads_ Threads; - /// The skew. - static int const kSkew = kSkew_; - /// The number of scalars per STS. - static int const kAccessSize = kScalarsPerSts_; - /// The memory space. - static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; - - /// The number of iterations needed to load/store the tile. - typedef Shape<1, TileWithoutSkew::kH / Threads::kW, TileWithoutSkew::kW / Threads::kH> Iterations; - /// The strides in each dimension between different loads/stores. - typedef Shape<0, ShapeCount<Tile>::kWc, Threads::kH * kAccessSize> Delta; - /// The strides in each dimension between different loads/stores. - typedef Shape<0, ShapeCount<Tile>::kWc, Threads::kH * kAccessSize> ImmediateOffsetStrides; - - struct ThreadOffset { - CUTLASS_HOST_DEVICE Coord<4> operator()() const { - int offset = ComputeThreadOffsetFromStrides<Threads, ThreadsStrides>::get(); - return make_Coord(0, 0, offset, 0); - } - }; - - protected: - /// The strides to compute the base position of the thread. - typedef Shape<0, kScalarsPerSts_, ShapeCount<Tile>::kHwc / Threads::kW> ThreadsStrides; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, - typename OutputTile_, - typename Warps_, - typename ThreadsPerWarp_, - typename InstructionShape_, - int kStages_, - int kScalarsPerLds_, - int kSkew_ = 0> -struct GemmSharedLoadTileATraits { - static GemmOperand::Kind const kOperand = GemmOperand::kA; - /// The scalar. - typedef typename platform::remove_const<Scalar_>::type Scalar; - /// The pointer. - typedef Scalar_* Pointer; - /// The tile without skew. - typedef Shape<kStages_, - OutputTile_::kD / InstructionShape_::kD, - GetExtent<kOperand, OutputTile_>::kExtent * InstructionShape_::kD> - TileWithoutSkew_; - /// The tile with skew. - typedef Shape<kStages_, TileWithoutSkew_::kH, TileWithoutSkew_::kW + kSkew_> TileWithSkew; - /// The tile without skew after reshaping. - typedef typename ReshapeTile<TileWithoutSkew_, kScalarsPerLds_>::Tile TileWithoutSkew; - /// The tile. - typedef typename ReshapeTile<TileWithSkew, kScalarsPerLds_>::Tile Tile; - /// The number of warps. - typedef Warps_ Warps; - /// The threads in a warp. - typedef ThreadsPerWarp_ ThreadsPerWarp; - /// The number of scalars per LDG/STG. - // static int const kScalarsPerLds = kScalarsPerLds_; - static int const kAccessSize = kScalarsPerLds_; - /// The skew. - static int const kSkew = kSkew_; - /// The memory space. - static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; - - /// The number of warps. - static int const kWarps = GetExtent<kOperand, Warps>::kExtent; - /// The number of threads in one dimension of the warp. - static int const kThreadsPerWarp = GetExtent<kOperand, ThreadsPerWarp>::kExtent; - - /// The number of iterations needed to load/store the tile. - typedef Shape<1, 1, TileWithoutSkew::kW / kWarps / kThreadsPerWarp /* / kScalarsPerLds*/> - Iterations; - /// The strides in each dimension between different loads/stores. - typedef Shape<TileWithSkew::kW * Warps::kD, 0, kWarps * kThreadsPerWarp * kAccessSize, 0> - ImmediateOffsetStrides; - typedef Shape<TileWithSkew::kW * Warps::kD, 0, kWarps * kThreadsPerWarp * kAccessSize, 0> Delta; - - /// Computes the thread offset in (H, W) based on thread ID - struct ThreadOffset { - CUTLASS_HOST_DEVICE Coord<4> operator()() const { - // Extract the warp. - int const warp = threadIdx.x / kWarpSize; - // Extract the slice. - int const slice = warp / (Warps::kH * Warps::kW); - // Compute the row offset for each warp. - int const warp_row = warp % Warps::kW; - // Compute the row offset for each thread. - int const lane_row = (threadIdx.x & 0x0e) / 2; - // The offset. - int const offset = - slice * Tile::kW * Tile::kC + (warp_row * ThreadsPerWarp::kW + lane_row) * kAccessSize; - // Embed the offset in a 4D coordinate vector. - return make_Coord(0, 0, offset, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, - typename OutputTile_, - typename Warps_, - typename ThreadsPerWarp_, - typename InstructionShape_, - int kStages_, - int kScalarsPerLds_, - int kSkew_ = 0> -struct GemmSharedLoadTileBTraits { - static GemmOperand::Kind const kOperand = GemmOperand::kB; - /// The scalar. - typedef typename platform::remove_const<Scalar_>::type Scalar; - /// The pointer. - typedef Scalar_* Pointer; - /// The tile without skew. - typedef Shape<kStages_, - OutputTile_::kD / InstructionShape_::kD, - GetExtent<kOperand, OutputTile_>::kExtent * InstructionShape_::kD> - TileWithoutSkew_; - /// The tile with skew. - typedef Shape<kStages_, TileWithoutSkew_::kH, TileWithoutSkew_::kW + kSkew_> TileWithSkew; - /// The tile without skew after reshaping. - typedef typename ReshapeTile<TileWithoutSkew_, kScalarsPerLds_>::Tile TileWithoutSkew; - /// The tile. - typedef typename ReshapeTile<TileWithSkew, kScalarsPerLds_>::Tile Tile; - /// The number of warps. - typedef Warps_ Warps; - /// The threads in a warp. - typedef ThreadsPerWarp_ ThreadsPerWarp; - /// The number of scalars per LDG/STG. - static int const kAccessSize = kScalarsPerLds_; - /// The skew. - static int const kSkew = kSkew_; - /// The memory space. - static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; - - /// The number of warps. - static int const kWarps = GetExtent<kOperand, Warps>::kExtent; - /// The number of threads in one dimension of the warp. - static int const kThreadsPerWarp = GetExtent<kOperand, ThreadsPerWarp>::kExtent; - - /// The number of iterations needed to load/store the tile. - typedef Shape<1, 1, TileWithoutSkew::kW / kWarps / kThreadsPerWarp /* / kAccessSize*/> Iterations; - /// The strides in each dimension between different loads/stores. - typedef Shape<TileWithSkew::kW * Warps::kD, 0, kWarps * kThreadsPerWarp * kAccessSize, 0> - ImmediateOffsetStrides; - typedef Shape<TileWithSkew::kW * Warps::kD, 0, kWarps * kThreadsPerWarp * kAccessSize, 0> Delta; - - /// Computes the thread offset in (H, W) based on thread ID - struct ThreadOffset { - CUTLASS_HOST_DEVICE Coord<4> operator()() const { - // Extract the warp. - int const warp = threadIdx.x / kWarpSize; - // Extract the slice. - int const slice = warp / (Warps::kH * Warps::kW); - // The warp in the slice. - int const warp_in_slice = warp % (Warps::kH * Warps::kW); - // Compute the row offset for each warp. - int const warp_col = warp_in_slice / Warps::kW; - // Compute the row offset for each thread. - int const lane_col = (threadIdx.x & 0x10) / 8 + (threadIdx.x & 0x01); - // The offset. - int const offset = - slice * Tile::kW * Tile::kC + (warp_col * ThreadsPerWarp::kH + lane_col) * kAccessSize; - // Embed the offset in a 4D coordinate. - return make_Coord(0, 0, offset, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, - typename OutputTile_, - typename Warps_, - typename ThreadsPerWarp_, - int kScalarsPerSts_, - int kSkew_ = 0> -struct GemmSharedStoreTileDTraits { - /// The scalar. - typedef typename platform::remove_const<Scalar_>::type Scalar; - /// The pointer. - typedef Scalar_* Pointer; - /// The dimension of the output tile. - typedef OutputTile_ OutputTile; - /// The warps in the tile. - typedef Warps_ Warps; - /// The threads in the warps. - typedef ThreadsPerWarp_ ThreadsPerWarp; - /// The number of scalars per LDG/STG. - static int const kAccessSize = kScalarsPerSts_; - /// The skew. - static int const kSkew = kSkew_; - /// The memory space. - static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; - - /// The number of scalars per thread. - static int const kScalarsPerThread = OutputTile_::kW / Warps::kW / ThreadsPerWarp::kW; - /// The number of threads. - static int const kThreads = ShapeCount<Warps>::kCount * kWarpSize; - /// The number of scalars per row. We build a tile with 2 rows (to avoid bank conflicts). - static int const kScalarsPerRow = kThreads / 2 * kScalarsPerThread + kSkew; - - /// The tile. - typedef Shape<1, 2, kScalarsPerRow / kAccessSize, kAccessSize> Tile; - /// The number of iterations needed to store the tile. - typedef Shape<1, 1, kScalarsPerThread / kAccessSize> Iterations; - /// The strides in each dimension between different loads/stores. - typedef Shape<0, 0, Warps::kW * ThreadsPerWarp::kW * kAccessSize> Delta; - /// The strides in each dimension between different loads/stores. - typedef Shape<0, 0, Warps::kW * ThreadsPerWarp::kW * kAccessSize> ImmediateOffsetStrides; - - /// Computes the thread offset in (H, W) based on thread ID - struct ThreadOffset { - CUTLASS_HOST_DEVICE Coord<4> operator()() const { - // The warp. - int const warp = threadIdx.x / kWarpSize; - - // The position of the warp in the 2D tile. - int const warp_row = warp % Warps::kW; - int const warp_col = warp / Warps::kW; - - // We assume that the elements are distributed in a warps as 4 columns of 8 elements. The - // columns are stored in threads col0=[0, 2, 4, 6, 8, 10, 12, 14], col1=[1, 3, 5, 7, .., 15], - // col2=[16, 18, 20, ..., 30] and col3=[17, 19, ..., 31]. - int hi_halfwarp_offset = ((threadIdx.x >> 4) & 0x1) * OutputTile::kW; - int lo_halfwarp_offset = ((threadIdx.x >> 1) & 0x7) + ThreadsPerWarp::kW * warp_row; - - // Odd threads go to the second half of shared memory. - int const row = threadIdx.x & 0x01; - int col = warp_col * (ThreadsPerWarp::kH / 2) * OutputTile::kW + - lo_halfwarp_offset * kAccessSize + hi_halfwarp_offset; - // Embed the offset in a 4D coords. - return make_Coord(0, 0, row * kScalarsPerRow + col, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, - typename OutputTile_, - typename Warps_, - typename ThreadsPerWarp_, - int kTileH_, - int kScalarsPerLds_, - int kSkew_ = 0> -struct GemmSharedLoadTileDTraits { - /// The scalar. - typedef typename platform::remove_const<Scalar_>::type Scalar; - /// The pointer. - typedef Scalar_* Pointer; - /// The dimension of the output tile. - typedef OutputTile_ OutputTile; - /// The warps in the tile. - typedef Warps_ Warps; - /// The threads in the warps. - typedef ThreadsPerWarp_ ThreadsPerWarp; - /// The number of scalars per LDG/STG. - static int const kAccessSize = kScalarsPerLds_; - /// The skew. - static int const kSkew = kSkew_; - /// The memory space. - static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; - - /// The number of scalars per thread. - static int const kScalarsPerThread = OutputTile_::kW / Warps::kW / ThreadsPerWarp::kW; - /// The number of threads. - static int const kThreads = ShapeCount<Warps>::kCount * kWarpSize; - /// The number of scalars per row. We build a tile with 2 rows (to avoid bank conflicts). - static int const kScalarsPerRow = kThreads / 2 * kScalarsPerThread + kSkew; - - /// The tile. We have 2 rows of scalars. We use those two rows to make sure we do not have bank - /// conflicts in the epilogue. - typedef Shape<1, 2, kScalarsPerRow / kAccessSize, kAccessSize> Tile; - - // Compute the number of iterations per warp in the Tile::kH dimension. - static int const kIterationsInHPerWarp = kTileH_ / ShapeCount<Warps>::kCount; - - // As explained above, the shared memory tile is composed of 2 rows and each rows is made of - // kScalarsPerRow. A warp is expected to read from the 1st row, then move to the 2nd row and go - // back to the 1st row. To model that scheme we define the Iterations shape as Shape<X, 2, ...>. - // However, in some cases, we have only 1 iteration per warp. In that case, we must define the - // shape as Shape<1, 1, ...>. The following code does that except that we hijack the kH dimension - // to keep the number of elements to reduce for split-K. - static int const kIterationsH = kIterationsInHPerWarp == 1 ? 1 : 2; - // As soon as we know kIterationsH, it is trivial to compute kIterationsD: - static int const kIterationsD = kIterationsInHPerWarp / kIterationsH; - - // If we have split-K enabled, we have to jump over the elements from the "odd/even" column of - // threads to grab the other elements. - static int const kSplitK = OutputTile::kW * ThreadsPerWarp::kH / 2 * Warps::kH; - - /// The number of iterations needed to store the tile. - typedef Shape<kIterationsD, kIterationsH, OutputTile::kW / kWarpSize / kAccessSize, Warps::kD> - Iterations; - /// The strides in each dimension between different loads/stores. - typedef Shape<OutputTile::kW, kScalarsPerRow, kWarpSize * kAccessSize, kSplitK> - ImmediateOffsetStrides; - /// The strides in each dimension between different loads/stores. - typedef Shape<OutputTile::kW, kScalarsPerRow, kWarpSize * kAccessSize, kSplitK> Delta; - - /// Computes the thread offset in (H, W) based on thread ID - struct ThreadOffset { - CUTLASS_HOST_DEVICE Coord<4> operator()() const { - // Each warp works on a different column. - int const h = threadIdx.x / kWarpSize; - // Compute the row. - int const w = (threadIdx.x & (kWarpSize - 1)) * kAccessSize; - int offset = 0; - if (Iterations::kH == 1) { - int const row = h & 0x1; - int const col = h / 2; - offset = row * ShapeCount<Tile>::kWc + col * OutputTile::kW * Iterations::kD + w; - } else { - offset = h * OutputTile::kW * Iterations::kD + w; - } - return make_Coord(0, 0, offset, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/gemm_traits.h b/cutlass-example/cutlass/gemm/gemm_traits.h deleted file mode 100644 index cb57c4d..0000000 --- a/cutlass-example/cutlass/gemm/gemm_traits.h +++ /dev/null @@ -1,818 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines structural properties of complete GEMM computation. -*/ -#pragma once - -#include <cutlass/convert.h> -#include <cutlass/gemm/clear_accumulators.h> -#include <cutlass/gemm/gemm_global_stream.h> -#include <cutlass/gemm/gemm_operand.h> -#include <cutlass/gemm/gemm_shared_stream.h> -#include <cutlass/gemm/identity_block_swizzle.h> -#include <cutlass/matrix_traits.h> -#include <cutlass/reshape_tile.h> -#include <cutlass/tile_iterator.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The scalar type for A. - typename ScalarA_, - /// The scalar type for B. - typename ScalarB_, - /// The scalar type for C. - typename ScalarC_, - /// The scalar type for D. - typename ScalarD_, - /// The output tile size for the GEMM KxNxM. - typename OutputTile_, - /// The functor to do the math. - typename MultiplyAdd_, - /// The number of scalars per LDG for A. - int kScalarsPerLdgA_, - /// The number of scalars per STS for A. - int kScalarsPerStsA_, - /// The number of scalars per LDG for A. - int kScalarsPerLdsA_, - /// The number of scalars per LDG for B. - int kScalarsPerLdgB_, - /// The number of scalars per STS for B. - int kScalarsPerStsB_, - /// The number of scalars per LDS for B. - int kScalarsPerLdsB_, - /// The number of scalars per LDG for C and STG for D. - int kScalarsPerLdgCAndStgD_, - /// The number of scalars per STS for D. - int kScalarsPerStsD_, - /// The number of scalars per LDS for D. - int kScalarsPerLdsD_, - /// The number of stages in shared memory to do single/double/triple-buffering. - int kStages_, - /// Do we do the residue in the prologue? - bool kResidueInPrologue_ = false> - -struct GemmConfig { - // - /// The scalar for A. - typedef ScalarA_ ScalarA; - /// The scalar for B. - typedef ScalarB_ ScalarB; - /// The scalar for C. - typedef ScalarC_ ScalarC; - /// The scalar for D. - typedef ScalarD_ ScalarD; - - /// The tile. - typedef OutputTile_ OutputTile; - /// The functor to do D = A*B + C. - typedef MultiplyAdd_ MultiplyAdd; - /// The shape of the instruction. - typedef typename MultiplyAdd::InstructionShape InstructionShape; - /// The number of accumulators per warp. - typedef typename MultiplyAdd::AccumulatorsPerWarp AccumulatorsPerWarp; - /// The accumulators. - typedef typename MultiplyAdd::Accumulators Accumulators; - - /// The number of warps. - typedef typename ShapeDiv<OutputTile, AccumulatorsPerWarp>::Shape Warps; - /// The default warp size (32 threads per warp). - static int const kWarpSize = cutlass::kWarpSize; - /// The numnber of threads. - static int const kThreads = ShapeCount<Warps>::kCount * kWarpSize; - - /// The number of scalars per LDG/STS/LDS for A. - static int const kScalarsPerLdgA = kScalarsPerLdgA_; - static int const kScalarsPerStsA = kScalarsPerStsA_; - static int const kScalarsPerLdsA = kScalarsPerLdsA_; - - /// The number of scalars per LDG/STS/LDS for B. - static int const kScalarsPerLdgB = kScalarsPerLdgB_; - static int const kScalarsPerStsB = kScalarsPerStsB_; - static int const kScalarsPerLdsB = kScalarsPerLdsB_; - - /// The number of scalars per LDG for C. - static int const kScalarsPerLdgC = kScalarsPerLdgCAndStgD_; - - /// The number of scalars per STS/LDS/STG for D. - static int const kScalarsPerStgD = kScalarsPerLdgCAndStgD_; - static int const kScalarsPerStsD = kScalarsPerStsD_; - static int const kScalarsPerLdsD = kScalarsPerLdsD_; - - /// The number of accumulators that are going to be fed from one LDS A/B. - static int const kAccumulatorsPerLdsA = kScalarsPerLdsA / InstructionShape::kD; - static int const kAccumulatorsPerLdsB = kScalarsPerLdsB / InstructionShape::kD; - - /// The number of stages in shared memory to implement double, triple, more-buffering. - static int const kStages = kStages_; - - /// Do we do the residue in the prologue? - static bool const kResidueInPrologue = kResidueInPrologue_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <enum MatrixLayout::Kind, typename GemmConfig_> -struct GemmTileTraitsHelperA {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_> -struct GemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_> { - /// The layout. - static MatrixLayout::Kind const kLayout = MatrixLayout::kColumnMajor; - - /// The input scalar. - typedef typename GemmConfig_::ScalarA Scalar; - /// The scalar stored in shared memory. - typedef typename GemmConfig_::MultiplyAdd::ScalarA MultiplyAddScalar; - - /// The traits class to build the iterator to load data from global memory for A^N. - typedef GemmGlobalTileTraits< - // That's A. - GemmOperand::kA, - // A is column-major. - MatrixLayout::kColumnMajor, - // The pointer is float const. - Scalar const, - // The tile has size KxM in GEMM's terminology. - Shape<1, GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kW>, - // The threads are distributed as warps x 32 (the traits may reorganize). - Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerLdgA> - GlobalTileTraits; - - /// The traits class to build the iterator to store data to shared memory for A^N. - typedef GemmSharedStoreTileAbTraits< - // The pointer is float. - MultiplyAddScalar, - // The tile has size KxM in GEMM's terminology. - Shape<GemmConfig_::kStages, - GemmConfig_::OutputTile::kD / GemmConfig_::InstructionShape::kD, - GemmConfig_::OutputTile::kW * GemmConfig_::InstructionShape::kD>, - // The threads are distributed as warps x 32 (the traits may reorganize). - typename GlobalTileTraits::Threads, - // The number of scalars per STS (STS.32 or STS.128, etc). - GemmConfig_::kScalarsPerStsA> - SharedStoreTileTraits; - - /// The traits class to build the iterator to load from shared memory for A^N. - typedef GemmSharedLoadTileATraits< - // The pointer is float const. - MultiplyAddScalar const, - // The output tile size. - typename GemmConfig_::OutputTile, - // The number of warps. - typename GemmConfig_::Warps, - // The number of threads per warp. - typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, - // The shape of the FMA instruction. - typename GemmConfig_::InstructionShape, - // The number of stages. - GemmConfig_::kStages, - // The number of scalars per LDS. - GemmConfig_::kScalarsPerLdsA, - // The skew. - 0> - SharedLoadTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_> -struct GemmTileTraitsHelperA<MatrixLayout::kRowMajor, GemmConfig_> { - /// The layout. - static MatrixLayout::Kind const kLayout = MatrixLayout::kRowMajor; - - /// The input scalar. - typedef typename GemmConfig_::ScalarA Scalar; - /// The scalar stored in shared memory. - typedef typename GemmConfig_::MultiplyAdd::ScalarA MultiplyAddScalar; - - /// The traits class to build the iterator to load data from global memory for A^T. - typedef GemmGlobalTileTraits< - // That's A. - GemmOperand::kA, - // A is row-major. - MatrixLayout::kRowMajor, - // The pointer is float const. - Scalar const, - // The tile has size MxK in GEMM's terminology. - Shape<1, GemmConfig_::OutputTile::kW, GemmConfig_::OutputTile::kD>, - // The threads are distributed as (threads / K) x K (the traits may reorganize). - Shape<1, GemmConfig_::kThreads / GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kD>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerLdgA> - GlobalTileTraits; - - /// The number of scalars in 4B. - static int const kScalarsIn4B = sizeof(MultiplyAddScalar) > 4 ? 1 : 4 / sizeof(MultiplyAddScalar); - /// The skew for A. - static int const kSkewA = 128 / sizeof(MultiplyAddScalar) / GemmConfig_::kScalarsPerStsA / - GlobalTileTraits::Threads::kW * kScalarsIn4B; - - /// The traits class to build the iterator to store data to shared memory for A^T. - typedef GemmSharedStoreWithSkewTileAbTraits < - // The pointer is float. - MultiplyAddScalar, - // The tile has size KxM in GEMM's terminology. - Shape<GemmConfig_::kStages, - GemmConfig_::OutputTile::kD / GemmConfig_::InstructionShape::kD, - GemmConfig_::OutputTile::kW * GemmConfig_::InstructionShape::kD>, - // The threads are distributed as (threads / K) x K (the traits may reorganize). - typename GlobalTileTraits::Threads, - // The number of scalars per STS. - GemmConfig_::kScalarsPerStsA, - // The skew to avoid bank conflicts added in the tile W dimension. - kSkewA<GemmConfig_::kScalarsPerLdsA ? GemmConfig_::kScalarsPerLdsA : kSkewA> - SharedStoreTileTraits; - - /// The traits class to build the iterator to load from shared memory for A^T. - typedef GemmSharedLoadTileATraits< - // The pointer is float const. - MultiplyAddScalar const, - // The output tile size. - typename GemmConfig_::OutputTile, - // The number of warps. - typename GemmConfig_::Warps, - // The number of threads per warp. - typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, - // The shape of the FMA instruction. - typename GemmConfig_::InstructionShape, - // The number of stages. - GemmConfig_::kStages, - // The number of scalars per LDS. - GemmConfig_::kScalarsPerLdsA, - // The skew. - SharedStoreTileTraits::kSkew> - SharedLoadTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <enum MatrixLayout::Kind, typename GemmConfig_> -struct GemmTileTraitsHelperB {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_> -struct GemmTileTraitsHelperB<MatrixLayout::kColumnMajor, GemmConfig_> { - /// The layout. - static MatrixLayout::Kind const kLayout = MatrixLayout::kColumnMajor; - - /// The input scalar. - typedef typename GemmConfig_::ScalarB Scalar; - /// The scalar stored in shared memory. - typedef typename GemmConfig_::MultiplyAdd::ScalarB MultiplyAddScalar; - - /// The traits class to build the iterator to load data from global memory for B^N. - typedef GemmGlobalTileTraits< - // That's B. - GemmOperand::kB, - // B is column-major. - MatrixLayout::kColumnMajor, - // The pointer is float const. - Scalar const, - // The tile has size MxK in GEMM's terminology. - Shape<1, GemmConfig_::OutputTile::kH, GemmConfig_::OutputTile::kD>, - // The threads are distributed as (threads / K) x K (the traits may reorganize). - Shape<1, GemmConfig_::kThreads / GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kD>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerLdgB> - GlobalTileTraits; - - /// The number of scalars in 4B. - static int const kScalarsIn4B = sizeof(MultiplyAddScalar) > 4 ? 1 : 4 / sizeof(MultiplyAddScalar); - /// The skew for B. - static int const kSkewB = 128 / sizeof(MultiplyAddScalar) / GemmConfig_::kScalarsPerStsB / - GlobalTileTraits::Threads::kW * kScalarsIn4B; - - /// The traits class to build the iterator to store data to shared memory for B^N. - typedef GemmSharedStoreWithSkewTileAbTraits < - // The pointer is float. - MultiplyAddScalar, - // The tile has size KxN in GEMM's terminology. - Shape<GemmConfig_::kStages, - GemmConfig_::OutputTile::kD / GemmConfig_::InstructionShape::kD, - GemmConfig_::OutputTile::kH * GemmConfig_::InstructionShape::kD>, - // The threads are distributed as (threads / K) x K (the traits may reorganize). - typename GlobalTileTraits::Threads, - // The number of scalars per STS. - GemmConfig_::kScalarsPerStsB, - // The skew to avoid bank conflicts added in the tile W dimension. - kSkewB<GemmConfig_::kScalarsPerLdsB ? GemmConfig_::kScalarsPerLdsB : kSkewB> - SharedStoreTileTraits; - - /// The traits class to build the iterator to load from shared memory for B^N. - typedef GemmSharedLoadTileBTraits< - // The pointer is float const. - MultiplyAddScalar const, - // The output tile size. - typename GemmConfig_::OutputTile, - // The number of warps. - typename GemmConfig_::Warps, - // The number of threads per warp. - typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, - // The shape of the FMA instruction. - typename GemmConfig_::InstructionShape, - // The number of stages. - GemmConfig_::kStages, - // The number of scalars per LDS. - GemmConfig_::kScalarsPerLdsB, - // The skew. - SharedStoreTileTraits::kSkew> - SharedLoadTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_> -struct GemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_> { - /// The layout. - static MatrixLayout::Kind const kLayout = MatrixLayout::kRowMajor; - - /// The input scalar. - typedef typename GemmConfig_::ScalarB Scalar; - /// The scalar stored in shared memory. - typedef typename GemmConfig_::MultiplyAdd::ScalarB MultiplyAddScalar; - - /// The traits class to build the iterator to load data from global memory for B^T. - typedef GemmGlobalTileTraits< - // That's B. - GemmOperand::kB, - // B is row-major. - MatrixLayout::kRowMajor, - // The pointer is float const. - Scalar const, - // The tile has size KxN in GEMM's terminology. - Shape<1, GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kH>, - // The threads are distributed as warps x 32 (the traits may reorganize). - Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerLdgB> - GlobalTileTraits; - - /// The traits class to build the iterator to store data to shared memory for B^T. - typedef GemmSharedStoreTileAbTraits< - // The pointer is float. - MultiplyAddScalar, - // The tile has size KxN in GEMM's terminology. - Shape<GemmConfig_::kStages, - GemmConfig_::OutputTile::kD / GemmConfig_::InstructionShape::kD, - GemmConfig_::OutputTile::kH * GemmConfig_::InstructionShape::kD>, - // The threads are distributed as warps x 32 (the traits may reorganize). - typename GlobalTileTraits::Threads, - // The number of scalars per STS (STS.32 or STS.128, etc). - GemmConfig_::kScalarsPerStsB> - SharedStoreTileTraits; - - /// The traits class to build the iterator to load from shared memory for B^T. - typedef GemmSharedLoadTileBTraits< - // The pointer is float const. - MultiplyAddScalar const, - // The output tile size. - typename GemmConfig_::OutputTile, - // The number of warps. - typename GemmConfig_::Warps, - // The number of threads per warp. - typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, - // The shape of the FMA instruction. - typename GemmConfig_::InstructionShape, - // The number of stages. - GemmConfig_::kStages, - // The number of scalars per LDS. - GemmConfig_::kScalarsPerLdsB, - // The skew. - 0> - SharedLoadTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmTraits_, bool kResidueInPrologue_ = GemmTraits_::kResidueInPrologue> -struct GemmResidue { - /// Move to residue portion. - template <bool kIsPrologue> - static CUTLASS_DEVICE void move_to_residue(typename GemmTraits_::GlobalLoadStreamA& stream_a, - typename GemmTraits_::GlobalLoadStreamB& stream_b, - typename GemmTraits_::Index k) { - // The new code path in CUTLASS 1.0.1: We treat the residue in the prologue so we can have - // complete main loops after that. It helps simplify the logic in the main loop. - if (kIsPrologue) { - stream_a.move_to_residue(k); - stream_b.move_to_residue(k); - } - } - - /// Rollback to beginning of first tile and initialize predicates. - static CUTLASS_DEVICE void rollback(typename GemmTraits_::GlobalLoadStreamA& stream_a, - typename GemmTraits_::GlobalLoadStreamB& stream_b) { - stream_a.rollback(); - stream_b.rollback(); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmTraits_> -struct GemmResidue<GemmTraits_, false> { - /// Move to residue portion. - template <bool kIsPrologue> - static CUTLASS_DEVICE void move_to_residue(typename GemmTraits_::GlobalLoadStreamA& stream_a, - typename GemmTraits_::GlobalLoadStreamB& stream_b, - typename GemmTraits_::Index k) { - // The index. - typedef typename GemmTraits_::Index Index; - // By how much we unroll the main loop. - Index const kUnroll = static_cast<Index>(GemmTraits_::OutputTile::kD); - - // Call the residue code. That's the same path as CUTLASS 1.0.0. - if (kIsPrologue && k < kUnroll) { - stream_a.residue(k, true); - stream_b.residue(k, true); - } else if (k <= kUnroll) { - stream_a.residue(k, false); - stream_b.residue(k, false); - } - } - - /// Rollback to beginning of first tile and initialize predicates. - static CUTLASS_DEVICE void rollback(typename GemmTraits_::GlobalLoadStreamA& stream_a, - typename GemmTraits_::GlobalLoadStreamB& stream_b) {} -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The GEMM configuration. - typename GemmConfig_, - /// The stream to load A from global memory to shared memory. - typename GlobalLoadStreamA_, - /// The stream to load B from global memory to shared memory. - typename GlobalLoadStreamB_, - /// The stream to load A from shared memory. - typename SharedLoadStreamA_, - /// The stream to load B from shared memory. - typename SharedLoadStreamB_, - /// The epilogue. - typename Epilogue_, - /// The block swizzle to reorganize the grid. - typename BlockSwizzle_ = IdentityBlockSwizzle, - /// The index. - typename Index_ = int, - /// The tool used to clear accumulators. - typename ClearAccumulators_ = ClearAccumulators<typename GemmConfig_::Accumulators::Scalar> > - -struct GemmTraits { - /// This class. - typedef GemmTraits<GemmConfig_, - GlobalLoadStreamA_, - GlobalLoadStreamB_, - SharedLoadStreamA_, - SharedLoadStreamB_, - Epilogue_, - BlockSwizzle_, - Index_, - ClearAccumulators_> - This_; - - /// The configuration. - typedef GemmConfig_ GemmConfig; - /// The output tile. - typedef typename GemmConfig::OutputTile OutputTile; - /// Is the residue treated in the prologue? - static bool const kResidueInPrologue = GemmConfig::kResidueInPrologue; - - /// The stream to load A from global memory to shared memory. - typedef GlobalLoadStreamA_ GlobalLoadStreamA; - /// The layout of A. - static MatrixLayout::Kind const kLayoutA = GlobalLoadStreamA::kLayout; - /// The scalar for A. - typedef typename GlobalLoadStreamA_::Scalar ScalarA; - - /// The stream to load B from global memory to shared memory. - typedef GlobalLoadStreamB_ GlobalLoadStreamB; - /// The layout of B. - static MatrixLayout::Kind const kLayoutB = GlobalLoadStreamB::kLayout; - /// The scalar for B. - typedef typename GlobalLoadStreamB_::Scalar ScalarB; - - /// The iterator for A to load from shared memory. - typedef SharedLoadStreamA_ SharedLoadStreamA; - /// The iterator for B to load from shared memory. - typedef SharedLoadStreamB_ SharedLoadStreamB; - - /// The multiply-add functor. - typedef typename GemmConfig::MultiplyAdd MultiplyAdd; - /// The epilogue. - typedef Epilogue_ Epilogue; - /// The scalars in the epilogue. - typedef typename Epilogue::ScalarC ScalarC; - typedef typename Epilogue::ScalarD ScalarD; - - /// The block swizzle to reorganize the grid. - typedef BlockSwizzle_ BlockSwizzle; - /// The index. - typedef Index_ Index; - /// Clear the accumulators. - typedef ClearAccumulators_ ClearAccumulators; - - /// The params. - struct Params { - /// The dimensions of the GEMM. - Index m, n, k; - /// The params for the A stream. - typename GlobalLoadStreamA::Params global_stream_a; - /// The params for the B stream. - typename GlobalLoadStreamB::Params global_stream_b; - /// The params for the A stream from shared memory. - typename SharedLoadStreamA::Params shared_stream_a; - /// The params for the B stream from shared memory. - typename SharedLoadStreamB::Params shared_stream_b; - /// The params for the epilogue. - typename Epilogue::Params epilogue; - - /// Initialize the parameters. - template <typename GemmDesc_> - CUTLASS_HOST_DEVICE int initialize(GemmDesc_ const& desc) { - // Set the problem size. - this->m = desc.m; - this->n = desc.n; - this->k = desc.k; - - // Initialize the iterator for A. - int error_code = - global_stream_a.initialize(desc, reinterpret_cast<ScalarA const*>(desc.d_a), desc.lda); - - if (error_code) { - return error_code; - } - - // Initialize the iterator for B. - error_code = - global_stream_b.initialize(desc, reinterpret_cast<ScalarB const*>(desc.d_b), desc.ldb); - - if (error_code) { - return error_code; - } - - // The epilogue. - return epilogue.initialize(desc); - } - }; - - // The storage for A. - template <typename GlobalLoadStream_, typename SharedLoadStream_> - union StreamSharedStorage { - // The storage needed by the global stream. - typename GlobalLoadStream_::SharedStorage global; - // The storage needed by the shared stream. - typename SharedLoadStream_::SharedStorage shared; - }; - - // The storage for the main loop + prologue. - struct MainLoopSharedStorage { - // The storage to shuffle the A matrix in shared memory. - StreamSharedStorage<GlobalLoadStreamA, SharedLoadStreamA> stream_a; - // The storage to shuffle the B matrix in shared memory. - StreamSharedStorage<GlobalLoadStreamB, SharedLoadStreamB> stream_b; - // The storage to clear the accumulators if needed. - typename ClearAccumulators::SharedStorage clear; - }; - - /// The storage in shared memory. - union SharedStorage { - // The storage for the main loop. - MainLoopSharedStorage main_loop; - // The storage for the epilogue. - typename Epilogue::SharedStorage epilogue; - }; - - /// Assemble the global load streams for A/B. - struct GlobalLoadStream { - /// Ctor. - CUTLASS_DEVICE GlobalLoadStream(Params const& params, - SharedStorage& shared_storage, - dim3 const& block) - : stream_a(params.global_stream_a, - shared_storage.main_loop.stream_a.global, - cutlass::make_Coord(0, params.k, params.m), - cutlass::make_Coord(0, 0, block.x)), - stream_b(params.global_stream_b, - shared_storage.main_loop.stream_b.global, - cutlass::make_Coord(0, params.k, params.n), - make_Coord(0, 0, block.y)) {} - - /// Trigger the copies from shared memory to registers. - CUTLASS_DEVICE void copy() { - stream_a.copy(); - stream_b.copy(); - } - - /// Commit the data. - CUTLASS_DEVICE void commit() { - stream_a.commit(); - stream_b.commit(); - } - - /// Move to residue portion. - template <bool kIsPrologue> - CUTLASS_DEVICE void move_to_residue(Index k) { - GemmResidue<This_>::move_to_residue<kIsPrologue>(stream_a, stream_b, k); - } - - /// Rollback to beginning of first tile and initialize predicates. - CUTLASS_DEVICE void rollback() { GemmResidue<This_>::rollback(stream_a, stream_b); } - - /// The stream for A. - GlobalLoadStreamA stream_a; - /// The stream for B. - GlobalLoadStreamB stream_b; - }; - - /// Assemble the shared load stream for A/B. - struct SharedLoadStream { - /// Ctor. - CUTLASS_DEVICE SharedLoadStream(Params const& params, SharedStorage& shared_storage) { - stream_a.initialize(params.shared_stream_a, shared_storage.main_loop.stream_a.shared); - stream_b.initialize(params.shared_stream_b, shared_storage.main_loop.stream_b.shared); - } - - /// Trigger the copies from shared memory to registers. - CUTLASS_DEVICE void copy(int step) { - stream_a.copy(step, fetched_a[step % 2]); - stream_b.copy(step, fetched_b[step % 2]); - } - - /// Commit the data. - CUTLASS_DEVICE void commit(int step) { - stream_a.commit(fetched_a[step % 2], transformed_a[step % 2]); - stream_b.commit(fetched_b[step % 2], transformed_b[step % 2]); - } - - /// The fragment A. - CUTLASS_DEVICE typename SharedLoadStreamA::Fragment const& fragment_a(int step) const { - return transformed_a[step % 2]; - } - - /// The fragment B. - CUTLASS_DEVICE typename SharedLoadStreamB::Fragment const& fragment_b(int step) const { - return transformed_b[step % 2]; - } - - /// Increment the stage. - CUTLASS_DEVICE void inc_stage() { - stream_a.inc_stage(); - stream_b.inc_stage(); - } - - /// The stream for A. - SharedLoadStreamA stream_a; - /// The fragments to fetch A. - typename SharedLoadStreamA::FetchedFragment fetched_a[2]; - /// The fragments to transform A. - typename SharedLoadStreamA::TransformedFragment transformed_a[2]; - /// The stream for B. - SharedLoadStreamB stream_b; - /// The fragments to fetch B. - typename SharedLoadStreamB::FetchedFragment fetched_b[2]; - /// The fragments to transform B. - typename SharedLoadStreamB::TransformedFragment transformed_b[2]; - }; - - /// The memory fence for shared loads. - static CUTLASS_DEVICE void shared_load_fence(bool in_loop) { - if (SharedLoadStreamA::Iterator::kRequiresLoadFence || - SharedLoadStreamB::Iterator::kRequiresLoadFence) { - __syncthreads(); - } - } - - /// The memory fence for shared stores. - static CUTLASS_DEVICE void shared_store_fence(bool in_loop) { __syncthreads(); } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmTileTraitsHelperA_, typename GemmTileTraitsHelperB_, typename Index_> -struct SimplifiedGemmTraitsHelper { - /// The global iterator to load A from global memory. - typedef GemmGlobalIteratorAb<typename GemmTileTraitsHelperA_::GlobalTileTraits, Index_> - GlobalLoadIteratorA; - /// The data converter for A before storing to shared memory. - typedef Copy<typename GlobalLoadIteratorA::Fragment> GlobalTransformerA; - /// The iterator to store A to shared memory. - typedef TileStoreIterator<typename GemmTileTraitsHelperA_::SharedStoreTileTraits, - typename GemmTileTraitsHelperA_::SharedStoreTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedStoreIteratorA; - /// The stream to load A from global memory to shared memory. - typedef GlobalLoadStream<GlobalLoadIteratorA, SharedStoreIteratorA, GlobalTransformerA> - GlobalLoadStreamA; - - /// The global iterator to load B from global memory. - typedef GemmGlobalIteratorAb<typename GemmTileTraitsHelperB_::GlobalTileTraits, Index_> - GlobalLoadIteratorB; - /// The data converter for B before storing to shared memory. - typedef Copy<typename GlobalLoadIteratorB::Fragment> GlobalTransformerB; - /// The iterator to store B to shared memory. - typedef TileStoreIterator<typename GemmTileTraitsHelperB_::SharedStoreTileTraits, - typename GemmTileTraitsHelperB_::SharedStoreTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedStoreIteratorB; - /// The stream to load B from global memory to shared memory. - typedef GlobalLoadStream<GlobalLoadIteratorB, SharedStoreIteratorB, GlobalTransformerB> - GlobalLoadStreamB; - - /// The iterator to load A from shared memory. - typedef TileLoadIterator<typename GemmTileTraitsHelperA_::SharedLoadTileTraits, - typename GemmTileTraitsHelperA_::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedLoadIteratorA; - /// The stream to load A from shared memory. - typedef SharedLoadStream<SharedLoadIteratorA> SharedLoadStreamA; - /// The iterator to load B from shared memory. - typedef TileLoadIterator<typename GemmTileTraitsHelperB_::SharedLoadTileTraits, - typename GemmTileTraitsHelperB_::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedLoadIteratorB; - /// The stream to load B from shared memory. - typedef SharedLoadStream<SharedLoadIteratorB> SharedLoadStreamB; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The layout for A. - MatrixLayout::Kind kLayoutA_, - /// The layout for B. - MatrixLayout::Kind kLayoutB_, - /// The config for the GEMM. - typename GemmConfig_, - /// The epilogue. - typename Epilogue_, - /// The index. - typename Index_ = int, - // The configuration for the A matrix. - typename GemmTileTraitsHelperA_ = GemmTileTraitsHelperA<kLayoutA_, GemmConfig_>, - // The configuration for the B matrix. - typename GemmTileTraitsHelperB_ = GemmTileTraitsHelperB<kLayoutB_, GemmConfig_>, - // The helper class to create the streams and iterators. - typename Helper_ = - SimplifiedGemmTraitsHelper<GemmTileTraitsHelperA_, GemmTileTraitsHelperB_, Index_> > -struct SimplifiedGemmTraits : public GemmTraits< - // The config. - GemmConfig_, - // The stream to load A from global memory to shared memory. - typename Helper_::GlobalLoadStreamA, - // The stream to load B from global memory to shared memory. - typename Helper_::GlobalLoadStreamB, - // The stream to load A from shared memory. - typename Helper_::SharedLoadStreamA, - // The stream to load B from shared memory. - typename Helper_::SharedLoadStreamB, - // The epilogue. - Epilogue_, - // The block swizzle to reorganize the grid. - IdentityBlockSwizzle, - // The index. - Index_, - // The tool used to clear accumulators. - ClearAccumulators<typename GemmConfig_::Accumulators::Element> > { -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/hgemm_global_tile.h b/cutlass-example/cutlass/gemm/hgemm_global_tile.h deleted file mode 100644 index f14dbb3..0000000 --- a/cutlass-example/cutlass/gemm/hgemm_global_tile.h +++ /dev/null @@ -1,90 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Tile traits used to construct global tile iterator for HGEMM. This is intended to - partition the thread block-level tile into 2D subtiles loaded by the threads and facilitate - memory accesses larger than 16 bits. -*/ -#pragma once - -#include <cutlass/coord.h> -#include <cutlass/gemm/gemm_global_tile.h> -#include <cutlass/matrix_traits.h> -#include <cutlass/reshape_tile.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <GemmOperand::Kind kOperand_, - MatrixLayout::Kind kLayout_, - typename Scalar_, - typename Tile_, - typename Threads_, - int kAccessSize_> -struct HgemmCrosswiseGlobalTileTraits : public GemmGlobalTileTraits< - // Which GEMM operand? - kOperand_, - // The layout. - kLayout_, - // The scalar. - Scalar_, - // The tile. - Tile_, - // The threads. - Threads_, - // The number of scalars per LDG/STG. - kAccessSize_> { - /// The base class. - typedef GemmGlobalTileTraits<kOperand_, kLayout_, Scalar_, Tile_, Threads_, kAccessSize_> Base; - /// The threads. - typedef typename Base::Threads Threads; - /// The threads strides. - typedef Shape<1, 2, Base::Tile::kC> ThreadsDelta; - /// The strides in each dimension between different loads/stores. - typedef Shape<Base::Threads::kH * 2, 1, Base::Threads::kW, Base::kAccessSize> Delta; - /// The number of iterations needed to load/store the tile. - typedef Shape<Base::Tile::kH / Base::Threads::kH / 2, - 2, - Base::Tile::kW / Base::Threads::kW, - Base::Tile::kC / Base::kAccessSize> - Iterations; - /// Computes the thread offset in (H, W) based on thread ID - struct ThreadOffset { - CUTLASS_HOST_DEVICE - Coord<4> operator()() const { - int thread_offset_h = threadIdx.x / Threads::kW * ThreadsDelta::kH; - int thread_offset_w = threadIdx.x % Threads::kW * ThreadsDelta::kW; - - return make_Coord(0, thread_offset_h, thread_offset_w, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/hgemm_multiply_add.h b/cutlass-example/cutlass/gemm/hgemm_multiply_add.h deleted file mode 100644 index ebbdd06..0000000 --- a/cutlass-example/cutlass/gemm/hgemm_multiply_add.h +++ /dev/null @@ -1,104 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Specialization implementing multiply-add operation on half-precision floating point - fragments. -*/ -#pragma once - -#include <cutlass/fragment.h> - -#include <cutlass/gemm/thread_multiply_add.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Template performing matrix multiply-add operation within a thread -template <typename AccumulatorsPerThread_, typename ThreadsPerWarp_> -struct ThreadMultiplyAdd<AccumulatorsPerThread_, ThreadsPerWarp_, half, half, half> { - /// The shape of the instruction. - typedef Shape<1, 1, 2, 1> InstructionShape; - /// The number of accumulators per thread. - typedef AccumulatorsPerThread_ AccumulatorsPerThread; - /// The number of threads per warp. - typedef ThreadsPerWarp_ ThreadsPerWarp; - /// The number of accumulators per warp. - typedef typename ShapeMul<AccumulatorsPerThread, ThreadsPerWarp>::Shape AccumulatorsPerWarp; - /// The type for A. - typedef half ScalarA; - /// The fragment for A. - typedef Fragment<ScalarA, AccumulatorsPerThread::kW> FragmentA; - /// The type for B. - typedef half ScalarB; - /// The fragment for B. - typedef Fragment<ScalarB, AccumulatorsPerThread::kH> FragmentB; - /// The type for C and D. - typedef half ScalarC; - /// The accumulators. - typedef Fragment<half, AccumulatorsPerThread::kH * AccumulatorsPerThread::kW> Accumulators; - - /// Make sure there's an even number of elements in both dimensions. - static_assert(AccumulatorsPerThread::kH % 2 == 0, "Invalid size"); - static_assert(AccumulatorsPerThread::kW % 2 == 0, "Invalid size"); - - /// Ctor. - CUTLASS_DEVICE ThreadMultiplyAdd() {} - - /// Multiply : d = a*b + c. - CUTLASS_DEVICE void multiply_add(FragmentA const& a, - FragmentB const& b, - Accumulators const& c, - Accumulators& d) { -#if defined(__CUDACC__) && __CUDA_ARCH__ >= 530 - // The inputs. - __half2 const* a_half2 = reinterpret_cast<__half2 const*>(&a[0]); - __half2 const* b_half2 = reinterpret_cast<__half2 const*>(&b[0]); - __half2 const* c_half2 = reinterpret_cast<__half2 const*>(&c[0]); - - // The output. - __half2* d_half2 = reinterpret_cast<__half2*>(&d[0]); - - for (int j = 0; j < AccumulatorsPerThread::kH / 2; ++j) { - for (int i = 0; i < AccumulatorsPerThread::kW / 2; ++i) { - // The offsets in the output fragment. - int const k0 = (2 * j + 0) * (AccumulatorsPerThread::kW / 2) + i; - int const k1 = (2 * j + 1) * (AccumulatorsPerThread::kW / 2) + i; - - // Compute the product a[i] * b[j].H0_H0. - d_half2[k0] = __hfma2(a_half2[i], __low2half2(b_half2[j]), c_half2[k0]); - // Compute the product a[i] * b[j].H1_H1. - d_half2[k1] = __hfma2(a_half2[i], __high2half2(b_half2[j]), c_half2[k1]); - } - } -#endif - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/hgemm_swizzle.h b/cutlass-example/cutlass/gemm/hgemm_swizzle.h deleted file mode 100644 index ebec0d4..0000000 --- a/cutlass-example/cutlass/gemm/hgemm_swizzle.h +++ /dev/null @@ -1,94 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Transposes a tile of 16b elements. Used by HGEMM to construct a K-strided layout in - shared memory for multiplicands. -*/ -#pragma once - -#include <cuda_fp16.h> -#include <cutlass/fragment.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GlobalIterator_> -struct HgemmSwizzle { - /// The global iterator. - typedef GlobalIterator_ GlobalIterator; - /// The source fragment. - typedef typename GlobalIterator::Fragment Fragment; - /// The shape of the source fragment. - typedef typename GlobalIterator::FragmentShape FragmentShape; - - /// The input fragment. - typedef Fragment InputFragment; - /// The output fragment. - typedef Fragment OutputFragment; - - /// The src/dst must be half fragments. - static_assert((platform::is_same<typename Fragment::Element, half>::value), "Works on half"); - - /// The number of elements must be a multiple of 2. - static_assert(FragmentShape::kH == 2 && ShapeCount<FragmentShape>::kWc == 2, "Not multiple of 2"); - - /// Ctor. - CUTLASS_DEVICE HgemmSwizzle() {} - - /// Transform a fragment. - CUTLASS_DEVICE void transform(Fragment const& src, Fragment& dst) { - // Expose src/dst as int arrays. - int const* src_int = reinterpret_cast<int const*>(&src[0]); - int* dst_int = reinterpret_cast<int*>(&dst[0]); - - // Transpose the data. - for (int d = 0; d < FragmentShape::kD; ++d) { - // The indices to read two consecutive "rows". - int const i0 = 2 * d + 0; - int const i1 = 2 * d + 1; - - int a0 = src_int[i0]; - int a1 = src_int[i1]; - - int b0, b1; - asm volatile("prmt.b32 %0, %1, %2, 0x5410;" : "=r"(b0) : "r"(a0), "r"(a1)); - asm volatile("prmt.b32 %0, %1, %2, 0x7632;" : "=r"(b1) : "r"(a0), "r"(a1)); - - // The indices to store with "strides". - int const j0 = 0 * (ShapeCount<FragmentShape>::kDhw / 2) + d; - int const j1 = 1 * (ShapeCount<FragmentShape>::kDhw / 2) + d; - - dst_int[j0] = b0; - dst_int[j1] = b1; - } - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/hgemm_traits.h b/cutlass-example/cutlass/gemm/hgemm_traits.h deleted file mode 100644 index b08645b..0000000 --- a/cutlass-example/cutlass/gemm/hgemm_traits.h +++ /dev/null @@ -1,397 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defies structural properties of half-precision GEMM computation. -*/ -#pragma once - -#include <cutlass/convert.h> -#include <cutlass/reshape_tile.h> - -#include <cutlass/gemm/gemm.h> -#include <cutlass/gemm/gemm_epilogue.h> -#include <cutlass/gemm/gemm_epilogue_traits.h> -#include <cutlass/gemm/gemm_global_tile.h> -#include <cutlass/gemm/gemm_shared_tile.h> -#include <cutlass/gemm/gemm_traits.h> -#include <cutlass/gemm/hgemm_global_tile.h> -#include <cutlass/gemm/hgemm_multiply_add.h> -#include <cutlass/gemm/hgemm_swizzle.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The tile size for the GEMM KxNxM. - typename OutputTile_, - /// The number of accumulators per thread. - typename AccumulatorsPerThread_, - /// The number of scalars per LDG for A. - int kScalarsPerLdgA_ = 2, - /// The number of scalars per LDG for B. - int kScalarsPerLdgB_ = 2> -struct HgemmConfig - : public GemmConfig< - /// The scalar type for A. - half, - /// The scalar type for B. - half, - /// The scalar type for C. - half, - /// The scalar type for D. - half, - /// The tile size for the GEMM KxNxM. - OutputTile_, - /// The functor to do the math in the main loop. - ThreadMultiplyAdd<AccumulatorsPerThread_, Shape<1, 4, 8>, half, half, half>, - /// The number of scalars per LDG for A. - kScalarsPerLdgA_, - /// The number of scalars per STS for A. - kScalarsPerLdgA_, - /// The number of scalars per LDS for A. - 8, - /// The number of scalars per LDG for B. - kScalarsPerLdgB_, - /// The number of scalars per STS for B. - kScalarsPerLdgB_, - /// The number of scalars per LDS for B. - 8, - /// The number of scalars per LDG for C and STG for D. - 2, - /// The number of scalars per STS for D. - 8, - /// The number of scalars per LDS for D. - 2, - /// The number of stages in shared memory. - 2> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <enum MatrixLayout::Kind kLayout_, typename Iterator_> -struct HgemmTransformerA {}; - -template <typename Iterator_> -struct HgemmTransformerA<MatrixLayout::kColumnMajor, Iterator_> { - typedef Convert<typename Iterator_::Fragment, typename Iterator_::Fragment> Transformer; -}; - -template <typename Iterator_> -struct HgemmTransformerA<MatrixLayout::kRowMajor, Iterator_> { - typedef HgemmSwizzle<Iterator_> Transformer; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <enum MatrixLayout::Kind kLayout_, typename Iterator_> -struct HgemmTransformerB {}; - -template <typename Iterator_> -struct HgemmTransformerB<MatrixLayout::kRowMajor, Iterator_> { - typedef Convert<typename Iterator_::Fragment, typename Iterator_::Fragment> Transformer; -}; - -template <typename Iterator_> -struct HgemmTransformerB<MatrixLayout::kColumnMajor, Iterator_> { - typedef HgemmSwizzle<Iterator_> Transformer; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <enum MatrixLayout::Kind kLayout_, typename GemmConfig_> -struct HgemmTileTraitsHelperA : public GemmTileTraitsHelperA<kLayout_, GemmConfig_> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_> -struct HgemmTileTraitsHelperA<MatrixLayout::kRowMajor, GemmConfig_> - : public GemmTileTraitsHelperA<MatrixLayout::kRowMajor, GemmConfig_> { - /// The base config. - typedef GemmTileTraitsHelperA<MatrixLayout::kRowMajor, GemmConfig_> Base; - - /// The traits class to build the iterator to load data from global memory for A^T. - typedef HgemmCrosswiseGlobalTileTraits< - GemmOperand::kA, - // The layout. - MatrixLayout::kRowMajor, - // The pointer. - half const, - // The tile has size MxK in GEMM's terminology. - Shape<1, GemmConfig_::OutputTile::kW, GemmConfig_::OutputTile::kD>, - // The threads are distributed as (threads / K ) x K (the traits may reorganize). - Shape<1, GemmConfig_::kThreads / GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kD>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc) - GemmConfig_::kScalarsPerLdgA> - GlobalTileTraits; - - /// The skew. - static int const kSkewA = 128 / sizeof(half) / GlobalTileTraits::Threads::kW / 2; - - /// The traits class to build the iterator to store data to shared memory for A^T. - typedef GemmSharedStoreWithSkewTileAbTraits < - // The pointer. - half, - // The tile has size KxM in GEMM's terminology. - Shape<GemmConfig_::kStages, - GemmConfig_::OutputTile::kD / GemmConfig_::InstructionShape::kD, - GemmConfig_::OutputTile::kW * GemmConfig_::InstructionShape::kD>, - // The threads are distributed as warps x 32(the traits may reorganize). - typename GlobalTileTraits::Threads, - // The number of scalars per STS (STS.32 or STS.128, etc). - 2, - // The skew to avoid bank conflicts added in the tile W dimension. - kSkewA<GemmConfig_::kScalarsPerLdsA ? GemmConfig_::kScalarsPerLdsA : kSkewA> - SharedStoreTileTraits; - - /// The traits class to build the iterator to load from shared memory for A^T. - typedef GemmSharedLoadTileATraits< - // The pointer. - half const, - // The output tile size. - typename GemmConfig_::OutputTile, - // The number of warps. - typename GemmConfig_::Warps, - // The number of threads per warp. - typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, - // The shape of the FMA instruction. - typename GemmConfig_::InstructionShape, - // The number of stages. - GemmConfig_::kStages, - // The number of scalars per LDS. - 8, - // The skew. - SharedStoreTileTraits::kSkew> - SharedLoadTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <enum MatrixLayout::Kind kLayout_, typename GemmConfig_> -struct HgemmTileTraitsHelperB : public GemmTileTraitsHelperB<kLayout_, GemmConfig_> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_> -struct HgemmTileTraitsHelperB<MatrixLayout::kColumnMajor, GemmConfig_> - : public GemmTileTraitsHelperB<MatrixLayout::kColumnMajor, GemmConfig_> { - /// The base config. - typedef GemmTileTraitsHelperB<MatrixLayout::kColumnMajor, GemmConfig_> Base; - - /// The traits class to build the iterator to load data from global memory for B^N. - typedef HgemmCrosswiseGlobalTileTraits< - GemmOperand::kB, - // The layout. - MatrixLayout::kColumnMajor, - // The pointer. - half const, - // The tile has size KxN in GEMM's terminology. - Shape<1, GemmConfig_::OutputTile::kH, GemmConfig_::OutputTile::kD>, - // The threads are distributed as (threads / K) x K (the traits may reorganize). - Shape<1, GemmConfig_::kThreads / GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kD>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc) - GemmConfig_::kScalarsPerLdgB> - GlobalTileTraits; - - /// The skew for B. - static int const kSkewB = 128 / sizeof(half) / GlobalTileTraits::Threads::kW / 2; - - /// The traits class to build the iterator to store data to shared memory for B^N. - typedef GemmSharedStoreWithSkewTileAbTraits < - // The pointer. - half, - // The tile has size KxN in GEMM's terminology. - Shape<GemmConfig_::kStages, - GemmConfig_::OutputTile::kD / GemmConfig_::InstructionShape::kD, - GemmConfig_::OutputTile::kH * GemmConfig_::InstructionShape::kD>, - // The threads are distributed as (threads / K) x K (the traits may reorganize). - typename GlobalTileTraits::Threads, - // The number of scalars per STS (STS.32 or STS.128, etc). - 2, - // The skew to avoid bank conflicts added in the tile W dimension. - kSkewB<GemmConfig_::kScalarsPerLdsB ? GemmConfig_::kScalarsPerLdsB : kSkewB> - SharedStoreTileTraits; - - /// The traits class to build the iterator to load from shared memory for B^N. - typedef GemmSharedLoadTileBTraits< - // The pointer. - half const, - // The output tile size. - typename GemmConfig_::OutputTile, - // The number of warps. - typename GemmConfig_::Warps, - // The number of threads per warp. - typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, - // The shape of the FMA instruction. - typename GemmConfig_::InstructionShape, - // The number of stages. - GemmConfig_::kStages, - // The number of scalars per LDS. - 8, - // The skew. - SharedStoreTileTraits::kSkew> - SharedLoadTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The layout for A. - MatrixLayout::Kind kLayoutA_, - /// The layout for B. - MatrixLayout::Kind kLayoutB_, - /// The output tile. - typename OutputTile_, - /// The functor to do the math in the epilogue. - typename EpilogueFunctor_, - /// The number of accumulators per thread. - typename AccumulatorsPerThread_ = Shape<8, 8, 16>, - /// The number of halfs loaded in one LDG for A. - int kScalarsPerLdgA_ = 2, - /// The number of halfs loaded in one LDG for B. - int kScalarsPerLdgB_ = 2, - /// The index. - typename Index_ = int> -struct HgemmTraitsHelper { - /// The HGEMM config. - typedef HgemmConfig<OutputTile_, AccumulatorsPerThread_, kScalarsPerLdgA_, kScalarsPerLdgB_> - GemmConfig; - /// The GEMM config for A. - typedef HgemmTileTraitsHelperA<kLayoutA_, GemmConfig> GemmTileTraitsHelperA; - /// The GEMM config for B. - typedef HgemmTileTraitsHelperB<kLayoutB_, GemmConfig> GemmTileTraitsHelperB; - - /// The iterator to load A from global memory. - typedef GemmGlobalIteratorAb<typename GemmTileTraitsHelperA::GlobalTileTraits, Index_> - GlobalLoadIteratorA; - /// The default transformer for A. - typedef typename HgemmTransformerA<GemmTileTraitsHelperA::kLayout, - GlobalLoadIteratorA>::Transformer GlobalTransformerA; - /// The iterator to store A to shared memory. - typedef TileStoreIterator<typename GemmTileTraitsHelperA::SharedStoreTileTraits, - typename GemmTileTraitsHelperA::SharedStoreTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedStoreIteratorA; - /// The stream to load A from global memory to shared memory. - typedef GlobalLoadStream<GlobalLoadIteratorA, SharedStoreIteratorA, GlobalTransformerA> - GlobalLoadStreamA; - - /// The iterator to load B from global memory. - typedef GemmGlobalIteratorAb<typename GemmTileTraitsHelperB::GlobalTileTraits, Index_> - GlobalLoadIteratorB; - // The default transformer for B. - typedef typename HgemmTransformerB<GemmTileTraitsHelperB::kLayout, - GlobalLoadIteratorB>::Transformer GlobalTransformerB; - /// The iterator to store B to shared memory. - typedef TileStoreIterator<typename GemmTileTraitsHelperB::SharedStoreTileTraits, - typename GemmTileTraitsHelperB::SharedStoreTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedStoreIteratorB; - /// The stream to load B from global memory to shared memory. - typedef GlobalLoadStream<GlobalLoadIteratorB, SharedStoreIteratorB, GlobalTransformerB> - GlobalLoadStreamB; - - /// The iterator to load A from shared memory - typedef TileLoadIterator<typename GemmTileTraitsHelperA::SharedLoadTileTraits, - typename GemmTileTraitsHelperA::SharedLoadTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedLoadIteratorA; - /// The stream to load A from shared memory. - typedef SharedLoadStream<SharedLoadIteratorA> SharedLoadStreamA; - /// The iterator to load B from shared memory. - typedef TileLoadIterator<typename GemmTileTraitsHelperB::SharedLoadTileTraits, - typename GemmTileTraitsHelperB::SharedLoadTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedLoadIteratorB; - /// The stream to load B from shared memory. - typedef SharedLoadStream<SharedLoadIteratorB> SharedLoadStreamB; - - /// The functor to do the multiply-add in the main loop. - typedef typename GemmConfig::MultiplyAdd MultiplyAdd; - /// The object to clear accumulators. - typedef ClearAccumulators<typename MultiplyAdd::ScalarC> ClearAccumulators; - - /// The traits class for the epilogue. - typedef SimplifiedGemmEpilogueTraits<GemmConfig, EpilogueFunctor_, Index_> GemmEpilogueTraits; - /// The epilogue. - typedef GemmEpilogue<GemmEpilogueTraits> Epilogue; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The layout for A. - MatrixLayout::Kind kLayoutA_, - /// The layout for B. - MatrixLayout::Kind kLayoutB_, - /// The output tile. - typename OutputTile_ = Shape<8, 128, 128>, - /// The functor to do the math in the epilogue. - typename EpilogueFunctor_ = LinearScaling<half>, - /// The number of accumulators per thread. - typename AccumulatorsPerThread_ = Shape<8, 8, 16>, - /// The number of halfs loaded in one LDG for A. - int kScalarsPerLdgA_ = 2, - /// The number of halfs loaded in one LDG for B. - int kScalarsPerLdgB_ = 2, - /// The index. - typename Index_ = int, - /// The helper class. - typename Helper_ = HgemmTraitsHelper<kLayoutA_, - kLayoutB_, - OutputTile_, - EpilogueFunctor_, - AccumulatorsPerThread_, - kScalarsPerLdgA_, - kScalarsPerLdgB_, - Index_> > -struct HgemmTraits : public GemmTraits< - // The config. - typename Helper_::GemmConfig, - // The stream to load A from global memory to shared memory. - typename Helper_::GlobalLoadStreamA, - // The stream to load B from global memory to shared memory. - typename Helper_::GlobalLoadStreamB, - // The stream to load A from shared memory. - typename Helper_::SharedLoadStreamA, - // The stream to load B from shared memory. - typename Helper_::SharedLoadStreamB, - // The epilogue. - typename Helper_::Epilogue, - // The block swizzle to reorganize the grid. - IdentityBlockSwizzle, - // The index. - Index_, - // The tool used to clear accumulators. - typename Helper_::ClearAccumulators> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/identity_block_swizzle.h b/cutlass-example/cutlass/gemm/identity_block_swizzle.h deleted file mode 100644 index e1bdb2e..0000000 --- a/cutlass-example/cutlass/gemm/identity_block_swizzle.h +++ /dev/null @@ -1,48 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defies functors for mapping blockIdx to partitions of the GEMM computation. - - Currently, we only implement an identity mapping. -*/ -#pragma once - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -struct IdentityBlockSwizzle { - /// Ctor. - CUTLASS_DEVICE IdentityBlockSwizzle() {} - - /// Swizzle the block index. - CUTLASS_DEVICE dim3 swizzle() { return blockIdx; } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/igemm_epilogue.h b/cutlass-example/cutlass/gemm/igemm_epilogue.h deleted file mode 100644 index 0d69980..0000000 --- a/cutlass-example/cutlass/gemm/igemm_epilogue.h +++ /dev/null @@ -1,320 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines the epilogue phase of the GEMM computation for IGEMM, supporting integer and - floating-point output matrix formats. -*/ -#pragma once - -#include <cutlass/convert.h> -#include <cutlass/fragment.h> -#include <cutlass/gemm/gemm_global_stream.h> -#include <cutlass/gemm/gemm_shared_stream.h> -#include <cutlass/gemm/igemm_global_tile.h> -#include <cutlass/reshape_tile.h> -#include <cutlass/tile_iterator.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <int kElements_> -struct IgemmFloatToInt8Converter { - /// The input fragment. - typedef Fragment<float, kElements_> InputFragment; - /// The output fragment. - typedef Fragment<int8_t, kElements_> OutputFragment; - - // We are packing 4 floats into int32 registers so we need kElements to be multiple of 4. - static_assert(kElements_ % 4 == 0, "kElements must be multiple of 4"); - - /// Ctor. - CUTLASS_DEVICE IgemmFloatToInt8Converter() {} - - /// Transform a fragment. - CUTLASS_DEVICE void transform(InputFragment const& src, OutputFragment& dst) { - transform(src, 0, dst); - } - - /// Transform a fragment. - template <typename Fragment_> - CUTLASS_DEVICE void transform(Fragment_ const& src, int offset, OutputFragment& dst) { - // The inputs. - float4 const* src_f4 = reinterpret_cast<float4 const*>(&src[0]); - // The outputs. - int* dst_int = reinterpret_cast<int*>(&dst[0]); - - // Iterate over the floats and pack them together to produce ints. - for (int i = 0; i < kElements_ / 4; ++i) { - // Read the float4. - float4 f4 = src_f4[i]; - - // Clamp the 4 elements of the floats to the [-128, +127] range. - float x = fmaxf(-128.f, fminf(127.f, f4.x)); - float y = fmaxf(-128.f, fminf(127.f, f4.y)); - float z = fmaxf(-128.f, fminf(127.f, f4.z)); - float w = fmaxf(-128.f, fminf(127.f, f4.w)); - - // Convert to integers. - int ix = (int)x; - int iy = (int)y; - int iz = (int)z; - int iw = (int)w; - - // Extract the lower bytes to build an int32 with 4 int8. - asm volatile("prmt.b32 %0, %0, %1, 0x1140;" : "+r"(ix) : "r"(iy)); - asm volatile("prmt.b32 %0, %0, %1, 0x1140;" : "+r"(iz) : "r"(iw)); - asm volatile("prmt.b32 %0, %0, %1, 0x5410;" : "+r"(ix) : "r"(iz)); - - // Store the int. - dst_int[i] = ix; - } - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename InputScalar_, typename OutputFragment_> -struct IgemmGlobalStoreTransformer { - typedef Convert<Fragment<InputScalar_, OutputFragment_::kElements>, OutputFragment_> Transformer; -}; - -template <int kElements_> -struct IgemmGlobalStoreTransformer<float, Fragment<int8_t, kElements_> > { - typedef IgemmFloatToInt8Converter<kElements_> Transformer; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <int kElements_> -struct IgemmInt8ToFloatConverter { - /// The input fragment. - typedef Fragment<int8_t, kElements_> InputFragment; - /// The output fragment. - typedef Fragment<float, kElements_> OutputFragment; - - // We are unpacking 4 int8s from int32. - static_assert(kElements_ % 4 == 0, "kElements must be multiple of 4"); - - /// Ctor. - CUTLASS_DEVICE IgemmInt8ToFloatConverter() {} - - /// Transform a fragment. - CUTLASS_DEVICE void transform(InputFragment const& src, OutputFragment& dst) { - transform(src, 0, dst); - } - - /// Transform a fragment. - template <typename Fragment_> - CUTLASS_DEVICE void transform(Fragment_ const& src, int offset, OutputFragment& dst) { - // The inputs. - int const* src_int = reinterpret_cast<int const*>(&src[0]); - // The outputs. - float4* dst_f4 = reinterpret_cast<float4*>(&dst[0]); - - // Iterate over the int8 and unpack them together to produce floats. - for (int i = 0; i < kElements_ / 4; ++i) { - // Read the int. - int ix, iy, iz, iw = src_int[i]; - - // Extract the 4 bytes. - asm volatile("prmt.b32 %0, 0x0, %1, 0x4440;" : "=r"(ix) : "r"(iw)); - asm volatile("prmt.b32 %0, 0x0, %1, 0x4441;" : "=r"(iy) : "r"(iw)); - asm volatile("prmt.b32 %0, 0x0, %1, 0x4442;" : "=r"(iz) : "r"(iw)); - asm volatile("prmt.b32 %0, 0x0, %1, 0x4443;" : "=r"(iw) : "r"(iw)); - - // The floats. - float fx, fy, fz, fw; - - // Convert to floats (make sure we generate I2F.F32.S8). - asm volatile("cvt.rn.f32.s8 %0, %1;" : "=f"(fx) : "r"(ix)); - asm volatile("cvt.rn.f32.s8 %0, %1;" : "=f"(fy) : "r"(iy)); - asm volatile("cvt.rn.f32.s8 %0, %1;" : "=f"(fz) : "r"(iz)); - asm volatile("cvt.rn.f32.s8 %0, %1;" : "=f"(fw) : "r"(iw)); - - // Store the float4. - dst_f4[i] = make_float4(fx, fy, fz, fw); - } - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename InputFragment_, typename OutputScalar_> -struct IgemmGlobalLoadTransformer { - typedef Convert<InputFragment_, Fragment<OutputScalar_, InputFragment_::kElements> > Transformer; -}; - -template <int kElements_> -struct IgemmGlobalLoadTransformer<Fragment<int8_t, kElements_>, float> { - typedef IgemmInt8ToFloatConverter<kElements_> Transformer; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename InputScalar_, typename OutputFragment_> -struct IgemmSharedStoreTransformer { - typedef Convert<Fragment<InputScalar_, OutputFragment_::kElements>, OutputFragment_> Transformer; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename IgemmConfig_, typename EpilogueFunctor_, typename Index_> -struct IgemmEpilogueTraitsHelper - : public GemmEpilogueTraitsHelper<IgemmConfig_, EpilogueFunctor_, Index_> { - /// The base class. - typedef GemmEpilogueTraitsHelper<IgemmConfig_, EpilogueFunctor_, Index_> Base; - /// The config. - typedef IgemmConfig_ IgemmConfig; - - /// The scalar type of the epilogue. - typedef typename Base::Scalar Scalar; - /// The iterations. - typedef typename Base::Iterations Iterations; - /// The iterations strides. - typedef typename Base::Delta Delta; - - /// The traits class for the iterator. - typedef typename Base::GlobalLoadTileTraits GlobalLoadTileTraits; - /// The iterator to store to shared memory. - typedef GemmGlobalIteratorCd<GlobalLoadTileTraits> GlobalLoadIteratorC; - /// The fragment that needs to be produced by the load iterator. - typedef typename GlobalLoadIteratorC::Fragment GlobalFragmentC; - /// The transformer from loaded data to math fragment. - typedef - typename IgemmGlobalLoadTransformer<GlobalFragmentC, Scalar>::Transformer GlobalTransformerC; - - /// The traits class for the iterator. - typedef typename Base::GlobalStoreTileTraits GlobalStoreTileTraits; - /// The iterator to store to shared memory. - typedef GemmGlobalIteratorCd<GlobalStoreTileTraits> GlobalStoreIteratorD; - /// The fragment that needs to be passed to that store iterator. - typedef typename GlobalStoreIteratorD::Fragment GlobalFragmentD; - /// The transformer from accumulators to shared memory fragments. - typedef - typename IgemmGlobalStoreTransformer<Scalar, GlobalFragmentD>::Transformer GlobalTransformerD; - - /// The traits class for the shared iterator to store D to shared memory. - typedef typename Base::SharedStoreTileTraits SharedStoreTileTraits; - /// The shared iterator to store D to shared memory. - typedef TileStoreIterator<SharedStoreTileTraits, - typename SharedStoreTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kGlobal> - SharedStoreIteratorD; - /// The fragment that needs to be passed to that store iterator. - typedef typename SharedStoreIteratorD::Fragment SharedStoreFragmentD; - /// The transformer from accumulators to shared memory fragments. - typedef typename IgemmSharedStoreTransformer<typename IgemmConfig::Accumulators::Element, - SharedStoreFragmentD>::Transformer - SharedStoreTransformerD; - /// The traits class for the shared iterator to load D from shared memory. - typedef typename Base::SharedLoadTileTraits SharedLoadTileTraits; - /// The shared iterator to load D from shared memory. - typedef TileLoadIterator<SharedLoadTileTraits, - typename SharedLoadTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedLoadIteratorD; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The config. - typename IgemmConfig_, - /// The functor to do the math in the epilogue. - typename EpilogueFunctor_, - /// The index. - typename Index_ = int, - /// The helper class to assemble the traits. - typename Helper_ = IgemmEpilogueTraitsHelper<IgemmConfig_, EpilogueFunctor_, Index_> > -struct IgemmEpilogueTraits : public GemmEpilogueTraits< - // The output tile. - typename IgemmConfig_::OutputTile, - // The accumulators. - typename IgemmConfig_::Accumulators, - // The global iterator for C. - typename Helper_::GlobalLoadIteratorC, - // The transformer for C. - typename Helper_::GlobalTransformerC, - // The transformer for D. - typename Helper_::GlobalTransformerD, - // The global iterator for D. - typename Helper_::GlobalStoreIteratorD, - // The iterator to store D to shared memory. - typename Helper_::SharedStoreIteratorD, - // The shared store transformer for D. - typename Helper_::SharedStoreTransformerD, - // The iterator to load D from shared memory. - typename Helper_::SharedLoadIteratorD, - // The iterations. - typename Helper_::Iterations, - // The strides between iterations. - typename Helper_::Delta, - // The functor to be used in the epilogue. - EpilogueFunctor_, - // The index. - Index_> { - /// Do we output in int8? - static bool const kInt8Output = - platform::is_same<typename IgemmConfig_::ScalarC, int8_t>::value != 0; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmEpilogueTraits_, bool = GemmEpilogueTraits_::kInt8Output> -struct IgemmEpilogue : public GemmEpilogue<GemmEpilogueTraits_> { - /// The base class. - typedef GemmEpilogue<GemmEpilogueTraits_> Base; - - /// Ctor. - CUTLASS_DEVICE IgemmEpilogue(typename Base::Params const& params_, - typename Base::SharedStorage& shared_storage_, - typename Base::Index m_, - typename Base::Index n_) - : Base(params_, shared_storage_, m_, n_) {} -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmEpilogueTraits_> -struct IgemmEpilogue<GemmEpilogueTraits_, true> : public GemmEpilogue<GemmEpilogueTraits_> { - /// The base class. - typedef GemmEpilogue<GemmEpilogueTraits_> Base; - - /// Ctor. - CUTLASS_DEVICE IgemmEpilogue(typename Base::Params const& params_, - typename Base::SharedStorage& shared_storage_, - typename Base::Index m_, - typename Base::Index n_) - : Base(params_, shared_storage_, m_, n_) {} -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/igemm_global_tile.h b/cutlass-example/cutlass/gemm/igemm_global_tile.h deleted file mode 100644 index 3f594ac..0000000 --- a/cutlass-example/cutlass/gemm/igemm_global_tile.h +++ /dev/null @@ -1,161 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Implements tile iterators to partition the thread block tile into 2D subtiles and - efficiently load each. Applies permute transformation to construct 'interleaved K-strided' - data layout in which 4-element dot products from the same K index are arranged in consecutive - locations within shared memory. - - Supports efficient loads from shared memory to target the DP4A instruction. -*/ -#pragma once - -#include <cutlass/coord.h> -#include <cutlass/gemm/gemm_global_tile.h> -#include <cutlass/matrix_traits.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <GemmOperand::Kind kOperand_, - MatrixLayout::Kind kLayout_, - typename Scalar_, - typename Tile_, - typename Threads_, - int kAccessSize_> -struct IgemmGlobalTileTraits : public GemmGlobalTileTraits< - // Which GEMM operand? - kOperand_, - // The layout. - kLayout_, - // The scalar. - Scalar_, - // The tile. - Tile_, - // The threads. - Threads_, - // The number of scalars per LDG/STG. - kAccessSize_> { - /// The base class. - typedef GemmGlobalTileTraits<kOperand_, kLayout_, Scalar_, Tile_, Threads_, kAccessSize_> Base; - /// The threads. - typedef typename Base::Threads Threads; - /// The strides in each dimension between different loads/stores. - typedef Shape<Base::Threads::kH * 4, 1, Base::Threads::kW, Base::kAccessSize> Delta; - /// The number of iterations needed to load/store the tile. - typedef Shape<Base::Tile::kH / Base::Threads::kH / 4, - 4, - Base::Tile::kW / Base::Threads::kW, - Base::Tile::kC / Base::kAccessSize> - Iterations; - - /// Computes the thread offset in (H, W) based on thread ID - struct ThreadOffset { - CUTLASS_HOST_DEVICE - Coord<4> operator()() const { - int thread_offset_h = threadIdx.x / Threads::kW * ThreadsDelta::kH; - int thread_offset_w = threadIdx.x % Threads::kW * ThreadsDelta::kW; - - return make_Coord(0, thread_offset_h, thread_offset_w, 0); - } - }; - - public: - /// The threads strides. - typedef Shape<1, 4, Base::Tile::kC> ThreadsDelta; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Deprecated. Please use IgemmGlobalTileTraits instead. - -template <GemmOperand::Kind kOperand_, - MatrixLayout::Kind kLayout_, - typename Scalar_, - typename Tile_, - typename Threads_, - int kAccessSize_> -struct IgemmContiguousGlobalTileTraits - : public IgemmGlobalTileTraits<kOperand_, kLayout_, Scalar_, Tile_, Threads_, kAccessSize_> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename TileTraits_, typename Index_ = int> -struct IgemmGlobalIteratorAb : public GemmGlobalIteratorAb<TileTraits_, Index_> { - /// The base class. - typedef GemmGlobalIteratorAb<TileTraits_, Index_> Base; - /// The functor to compute the thread offset. - typedef typename TileTraits_::ThreadOffset ThreadOffset; - - /// Constructor. - CUTLASS_DEVICE IgemmGlobalIteratorAb(typename Base::Params const& _params, - const Coord<3>& bounds, - const Coord<3>& block, - ThreadOffset thread_offset_func = ThreadOffset()) - : Base(_params, bounds, block, thread_offset_func), in_residue_(false), mask_(0xffffffff) { - // The number of elements read in a single iteration. - int const kBlock = TileTraits_::Tile::kW * TileTraits_::kAccessSize; - // The residue. - int const kResidue = (int)(bounds[1] % kBlock); - - // Compute the number of elements that are valid. - int const left = kResidue - Base::thread_offset[2]; - if (left > 0 && left < 4) { - mask_ = (1u << (8 * left)) - 1u; - } - } - - /// The accessor. - CUTLASS_DEVICE void get(typename Base::AccessType& value, int d, int h, int w, int c) const { - Base::get(value, d, h, w, c); - if (in_residue_) { - reinterpret_cast<uint32_t&>(value) &= mask_; - } - } - - /// Move to residue portion. - CUTLASS_DEVICE void move_to_residue(typename Base::Index k) { - Base::move_to_residue(k); - in_residue_ = true; - } - - /// Move back to the beginning of the first tile. - CUTLASS_DEVICE void rollback() { - Base::rollback(); - in_residue_ = false; - } - - /// Are we in the residue? - bool in_residue_; - /// The mask to clean up the values. - uint32_t mask_; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/igemm_multiply_add.h b/cutlass-example/cutlass/gemm/igemm_multiply_add.h deleted file mode 100644 index 5a8baec..0000000 --- a/cutlass-example/cutlass/gemm/igemm_multiply_add.h +++ /dev/null @@ -1,89 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Implements matrix multiply accumulate operation of 8-bit integer data using DP4A - instruction. -*/ -#pragma once - -#include <cutlass/fragment.h> - -#include <cutlass/gemm/thread_multiply_add.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Template performing matrix multiply-add operation within a thread -template <typename AccumulatorsPerThread_, typename ThreadsPerWarp_> -struct ThreadMultiplyAdd<AccumulatorsPerThread_, ThreadsPerWarp_, int8_t, int8_t, int> { - /// The shape of the instruction. - typedef Shape<4, 1, 1> InstructionShape; - /// The number of accumulators per thread. - typedef AccumulatorsPerThread_ AccumulatorsPerThread; - /// The number of threads per warp. - typedef ThreadsPerWarp_ ThreadsPerWarp; - /// The number of accumulators per warp. - typedef typename ShapeMul<AccumulatorsPerThread, ThreadsPerWarp>::Shape AccumulatorsPerWarp; - /// The type for A. - typedef int8_t ScalarA; - /// The fragment for A. - typedef Fragment<ScalarA, AccumulatorsPerThread::kW * 4> FragmentA; - /// The type for B. - typedef int8_t ScalarB; - /// The fragment for B. - typedef Fragment<ScalarB, AccumulatorsPerThread::kH * 4> FragmentB; - /// The type for C and D. - typedef int ScalarC; - /// The accumulators. - typedef Fragment<ScalarC, AccumulatorsPerThread::kH * AccumulatorsPerThread::kW> Accumulators; - - /// Ctor. - CUTLASS_DEVICE ThreadMultiplyAdd() {} - - /// Multiply : d = a*b + c. - CUTLASS_DEVICE void multiply_add(FragmentA const& a, - FragmentB const& b, - Accumulators const& c, - Accumulators& d) { - // The inputs. - int const* a_int = reinterpret_cast<int const*>(&a[0]); - int const* b_int = reinterpret_cast<int const*>(&b[0]); - - for (int j = 0; j < AccumulatorsPerThread::kH; ++j) { - for (int i = 0; i < AccumulatorsPerThread::kW; ++i) { - asm volatile("dp4a.s32.s32 %0, %1, %2, %3;" - : "=r"(d[j * AccumulatorsPerThread::kW + i]) - : "r"(a_int[i]), "r"(b_int[j]), "r"(c[j * AccumulatorsPerThread::kW + i])); - } - } - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/igemm_swizzle.h b/cutlass-example/cutlass/gemm/igemm_swizzle.h deleted file mode 100644 index 77cf711..0000000 --- a/cutlass-example/cutlass/gemm/igemm_swizzle.h +++ /dev/null @@ -1,115 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Transposes a fragment of data containing packed 8-bit integer elements. -*/ -#pragma once - -#include <cutlass/fragment.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GlobalIterator_> -struct IgemmSwizzle { - /// The global iterator. - typedef GlobalIterator_ GlobalIterator; - /// The source fragment. - typedef typename GlobalIterator::Fragment Fragment; - /// The shape of the source fragment. - typedef typename GlobalIterator::FragmentShape FragmentShape; - - /// The source fragment. - typedef Fragment InputFragment; - /// The destination fragment. - typedef Fragment OutputFragment; - - /// The src/dst must be int8 fragments. - static_assert((platform::is_same<typename Fragment::Element, int8_t>::value), "Works on int8"); - - /// The number of elements must be a multiple of 4. - static_assert(FragmentShape::kH % 4 == 0 && ShapeCount<FragmentShape>::kWc % 4 == 0, - "Not multiple of 4"); - - /// Ctor. - CUTLASS_DEVICE IgemmSwizzle() {} - - /// Transform a fragment. - CUTLASS_DEVICE void transform(Fragment const& src, Fragment& dst) { - // Expose src/dst as int arrays. - int const* src_int = reinterpret_cast<int const*>(&src[0]); - int* dst_int = reinterpret_cast<int*>(&dst[0]); - - // Transpose the data. - for (int d = 0; d < FragmentShape::kD; ++d) { - for (int h = 0; h < FragmentShape::kH / 4; ++h) { - for (int w = 0; w < ShapeCount<FragmentShape>::kWc / 4; ++w) { - int const i0 = d * (ShapeCount<FragmentShape>::kHwc / 4) + - (4 * h + 0) * (ShapeCount<FragmentShape>::kWc / 4) + w; - int const i1 = d * (ShapeCount<FragmentShape>::kHwc / 4) + - (4 * h + 1) * (ShapeCount<FragmentShape>::kWc / 4) + w; - int const i2 = d * (ShapeCount<FragmentShape>::kHwc / 4) + - (4 * h + 2) * (ShapeCount<FragmentShape>::kWc / 4) + w; - int const i3 = d * (ShapeCount<FragmentShape>::kHwc / 4) + - (4 * h + 3) * (ShapeCount<FragmentShape>::kWc / 4) + w; - - int a0 = src_int[i0]; - int a1 = src_int[i1]; - int a2 = src_int[i2]; - int a3 = src_int[i3]; - - int b0, b1, b2, b3, c0; - asm volatile("prmt.b32 %0, %1, %2, 0x0040;" : "=r"(b0) : "r"(a0), "r"(a1)); - asm volatile("prmt.b32 %0, %1, %2, 0x0040;" : "=r"(c0) : "r"(a2), "r"(a3)); - asm volatile("prmt.b32 %0, %1, %2, 0x5410;" : "=r"(b0) : "r"(b0), "r"(c0)); - - asm volatile("prmt.b32 %0, %1, %2, 0x0051;" : "=r"(b1) : "r"(a0), "r"(a1)); - asm volatile("prmt.b32 %0, %1, %2, 0x0051;" : "=r"(c0) : "r"(a2), "r"(a3)); - asm volatile("prmt.b32 %0, %1, %2, 0x5410;" : "=r"(b1) : "r"(b1), "r"(c0)); - - asm volatile("prmt.b32 %0, %1, %2, 0x0062;" : "=r"(b2) : "r"(a0), "r"(a1)); - asm volatile("prmt.b32 %0, %1, %2, 0x0062;" : "=r"(c0) : "r"(a2), "r"(a3)); - asm volatile("prmt.b32 %0, %1, %2, 0x5410;" : "=r"(b2) : "r"(b2), "r"(c0)); - - asm volatile("prmt.b32 %0, %1, %2, 0x0073;" : "=r"(b3) : "r"(a0), "r"(a1)); - asm volatile("prmt.b32 %0, %1, %2, 0x0073;" : "=r"(c0) : "r"(a2), "r"(a3)); - asm volatile("prmt.b32 %0, %1, %2, 0x5410;" : "=r"(b3) : "r"(b3), "r"(c0)); - - dst_int[i0] = b0; - dst_int[i1] = b1; - dst_int[i2] = b2; - dst_int[i3] = b3; - } - } - } - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/igemm_traits.h b/cutlass-example/cutlass/gemm/igemm_traits.h deleted file mode 100644 index 82f8de5..0000000 --- a/cutlass-example/cutlass/gemm/igemm_traits.h +++ /dev/null @@ -1,539 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defies structural properties of mixed-precision integer GEMM. Multiplicands are assumed - to be packed 8bit integers, accumulators are assumed to be 32b signed integers, and output - formats vary. -*/ -#pragma once - -#include <cutlass/convert.h> -#include <cutlass/gemm/gemm.h> -#include <cutlass/gemm/gemm_epilogue.h> -#include <cutlass/gemm/gemm_epilogue_traits.h> -#include <cutlass/gemm/gemm_global_tile.h> -#include <cutlass/gemm/gemm_shared_tile.h> -#include <cutlass/gemm/gemm_traits.h> -#include <cutlass/gemm/igemm_epilogue.h> -#include <cutlass/gemm/igemm_global_tile.h> -#include <cutlass/gemm/igemm_multiply_add.h> -#include <cutlass/gemm/igemm_swizzle.h> -#include <cutlass/reshape_tile.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The tile size for the GEMM KxNxM. - typename OutputTile_, - /// The output type. - typename ScalarD_, - /// The number of accumulators per thread. - typename AccumulatorsPerThread_> -struct IgemmConfig - : public GemmConfig< - /// The scalar type for A. - int8_t, - /// The scalar type for B. - int8_t, - /// The scalar type for C. - ScalarD_, - /// The scalar type for D. - ScalarD_, - /// The tile size for the GEMM KxNxM. - OutputTile_, - /// The functor to do the math in the main loop. - ThreadMultiplyAdd<AccumulatorsPerThread_, Shape<1, 4, 8>, int8_t, int8_t, int>, - /// The number of scalars per LDG for A. - 4, - /// The number of scalars per STS for A. - 4, - /// The number of scalars per LDS for A. - 16, - /// The number of scalars per LDG for B. - 4, - /// The number of scalars per STS for B. - 4, - /// The number of scalars per LDS for B. - 16, - /// The number of scalars per LDG for C and STG for D. - 1, - /// The number of scalars per STS for D. - 4, - /// The number of scalars per LDS for D. - 1, - /// The number of stages in shared memory. - 2, - /// Enable the code path that deals with the residue in epilogue. - true> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename OutputTile_, typename AccumulatorsPerThread_> -struct IgemmConfig<OutputTile_, int8_t, AccumulatorsPerThread_> - : public GemmConfig< - /// The scalar type for A. - int8_t, - /// The scalar type for B. - int8_t, - /// The scalar type for C. - int8_t, - /// The scalar type for D. - int8_t, - /// The tile size for the GEMM KxNxM. - OutputTile_, - /// The functor to do the math in the main loop. - ThreadMultiplyAdd<AccumulatorsPerThread_, Shape<1, 4, 8>, int8_t, int8_t, int>, - /// The number of scalars per LDG for A. - 4, - /// The number of scalars per STS for A. - 4, - /// The number of scalars per LDS for A. - 16, - /// The number of scalars per LDG for B. - 4, - /// The number of scalars per STS for B. - 4, - /// The number of scalars per LDS for B. - 16, - /// The number of scalars per LDG for C and STG for D. - 4, - /// The number of scalars per STS for D. - 4, - /// The number of scalars per LDS for D. - 4, - /// The number of stages in shared memory. - 2, - /// Enable the code path that deals with the residue in epilogue. - true> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <enum MatrixLayout::Kind kLayout_, typename GemmConfig_, typename Index_> -struct IgemmTileTraitsHelperA : public GemmTileTraitsHelperA<kLayout_, GemmConfig_> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_, typename Index_> -struct IgemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_, Index_> - : public GemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_> { - /// The base config. - typedef GemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_> Base; - - /// The number of scalars per LDG/STS/LDS for A. - static int const kScalarsPerStsA = 16; - - /// The traits class to build the iterator to load data from global memory for A^N. - typedef IgemmGlobalTileTraits< - GemmOperand::kA, - // The layout. - MatrixLayout::kColumnMajor, - // The pointer is float const. - int8_t const, - // The tile has size KxM in GEMM's terminology. - Shape<1, GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kW>, - // The threads are distributed as warps x 32 (the traits may reorganize). - Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerLdgA> - GlobalTileTraits; - - // The iterator. - typedef GemmGlobalIteratorAb<GlobalTileTraits, Index_> GlobalLoadIterator; - - /// The traits class to build the iterator to store data to shared memory for A^N. - typedef GemmSharedStoreTileAbTraits< - // The pointer is float. - int8_t, - // The tile has size KxM in GEMM's terminology. - Shape<GemmConfig_::kStages, GemmConfig_::OutputTile::kD / 4, GemmConfig_::OutputTile::kW * 4>, - // The threads are distributed as warps x 32 (the traits may reorganize). - typename GlobalTileTraits::Threads, - // The number of scalars per STS (STS.32 or STS.128, etc). - kScalarsPerStsA> - SharedStoreTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_, typename Index_> -struct IgemmTileTraitsHelperA<MatrixLayout::kRowMajor, GemmConfig_, Index_> { - /// The layout. - static MatrixLayout::Kind const kLayout = MatrixLayout::kRowMajor; - - /// The input scalar. - typedef int8_t Scalar; - /// The scalar stored in shared memory. - typedef int8_t MultiplyAddScalar; - - /// The number of scalars per LDG/STS/LDS for A. - static int const kScalarsPerStsA = 16; - - /// The traits class to build the iterator to load data from global memory for A^T. - typedef IgemmGlobalTileTraits< - GemmOperand::kA, - // The layout. - MatrixLayout::kRowMajor, - // The pointer is float const. - int8_t const, - // The tile has size NxK in GEMM's terminology. - Shape<1, GemmConfig_::OutputTile::kW, GemmConfig_::OutputTile::kD>, - // The threads are distributed as warps x 32 (the traits may reorganize). - Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerLdgA> - GlobalTileTraits; - - // The iterator. - typedef IgemmGlobalIteratorAb<GlobalTileTraits, Index_> GlobalLoadIterator; - - /// The traits class to build the iterator to store data to shared memory for A^N. - typedef GemmSharedStoreWithSkewTileAbTraits< - // The pointer is int8. - int8_t, - // The tile has size KxN in GEMM's terminology. - Shape<GemmConfig_::kStages, GemmConfig_::OutputTile::kD / 4, GemmConfig_::OutputTile::kW * 4>, - // The threads are distributed as (threads / K) x K (the traits may reorganize). - typename GlobalTileTraits::Threads, - // The number of scalars per STS. - kScalarsPerStsA, - // The skew to avoid bank conflicts added in the tile W dimension. - 16> - SharedStoreTileTraits; - - /// The traits class to build the iterator to load from shared memory for A^N. - typedef GemmSharedLoadTileATraits< - // The pointer is float const. - int8_t const, - // The output tile size. - typename GemmConfig_::OutputTile, - // The number of warps. - typename GemmConfig_::Warps, - // The number of threads per warp. - typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, - // The shape of the FMA instruction. - typename GemmConfig_::InstructionShape, - // The number of stages. - GemmConfig_::kStages, - // The number of scalars per LDS. - 16, - // The skew. - SharedStoreTileTraits::kSkew> - SharedLoadTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <enum MatrixLayout::Kind kLayout_, typename GemmConfig_, typename Index_> -struct IgemmTileTraitsHelperB : public GemmTileTraitsHelperB<kLayout_, GemmConfig_> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_, typename Index_> -struct IgemmTileTraitsHelperB<MatrixLayout::kColumnMajor, GemmConfig_, Index_> { - /// The layout. - static MatrixLayout::Kind const kLayout = MatrixLayout::kColumnMajor; - - /// The input scalar. - typedef int8_t Scalar; - /// The scalar stored in shared memory. - typedef int8_t MultiplyAddScalar; - - /// The number of scalars per LDG/STS/LDS for B. - static int const kScalarsPerStsB = 16; - - /// The traits class to build the iterator to load data from global memory for B^T. - typedef IgemmGlobalTileTraits< - GemmOperand::kB, - // The layout. - MatrixLayout::kColumnMajor, - // The pointer is float const. - int8_t const, - // The tile has size NxK in GEMM's terminology. - Shape<1, GemmConfig_::OutputTile::kH, GemmConfig_::OutputTile::kD>, - // The threads are distributed as warps x 32 (the traits may reorganize). - Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerLdgB> - GlobalTileTraits; - - // The iterator. - typedef IgemmGlobalIteratorAb<GlobalTileTraits, Index_> GlobalLoadIterator; - - /// The traits class to build the iterator to store data to shared memory for B^N. - typedef GemmSharedStoreWithSkewTileAbTraits< - // The pointer is int8. - int8_t, - // The tile has size KxN in GEMM's terminology. - Shape<GemmConfig_::kStages, GemmConfig_::OutputTile::kD / 4, GemmConfig_::OutputTile::kH * 4>, - // The threads are distributed as (threads / K) x K (the traits may reorganize). - typename GlobalTileTraits::Threads, - // The number of scalars per STS. - kScalarsPerStsB, - // The skew to avoid bank conflicts added in the tile W dimension. - 16> - SharedStoreTileTraits; - - /// The traits class to build the iterator to load from shared memory for B^N. - typedef GemmSharedLoadTileBTraits< - // The pointer is float const. - int8_t const, - // The output tile size. - typename GemmConfig_::OutputTile, - // The number of warps. - typename GemmConfig_::Warps, - // The number of threads per warp. - typename GemmConfig_::MultiplyAdd::ThreadsPerWarp, - // The shape of the FMA instruction. - typename GemmConfig_::InstructionShape, - // The number of stages. - GemmConfig_::kStages, - // The number of scalars per LDS. - 16, - // The skew. - SharedStoreTileTraits::kSkew> - SharedLoadTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_, typename Index_> -struct IgemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_, Index_> - : public GemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_> { - /// The base config. - typedef GemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_> Base; - - /// The number of scalars per LDG/STS/LDS for B. - static int const kScalarsPerStsB = 16; - - /// The traits class to build the iterator to load data from global memory for B^T. - typedef IgemmGlobalTileTraits< - GemmOperand::kB, - // The layout. - MatrixLayout::kRowMajor, - // The pointer is float const. - int8_t const, - // The tile has size KxM in GEMM's terminology. - Shape<1, GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kH>, - // The threads are distributed as warps x 32 (the traits may reorganize). - Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerLdgB> - GlobalTileTraits; - - // The iterator. - typedef GemmGlobalIteratorAb<GlobalTileTraits, Index_> GlobalLoadIterator; - - /// The traits class to build the iterator to store data to shared memory for B^N. - typedef GemmSharedStoreTileAbTraits< - // The pointer is float. - int8_t, - // The tile has size KxM in GEMM's terminology. - Shape<GemmConfig_::kStages, GemmConfig_::OutputTile::kD / 4, GemmConfig_::OutputTile::kH * 4>, - // The threads are distributed as warps x 32 (the traits may reorganize). - typename GlobalTileTraits::Threads, - // The number of scalars per STS (STS.32 or STS.128, etc). - kScalarsPerStsB> - SharedStoreTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <enum MatrixLayout::Kind kLayout_, typename Iterator_> -struct IgemmTransformerA {}; - -template <typename Iterator_> -struct IgemmTransformerA<MatrixLayout::kRowMajor, Iterator_> { - typedef Copy<typename Iterator_::Fragment> Transformer; -}; - -template <typename Iterator_> -struct IgemmTransformerA<MatrixLayout::kColumnMajor, Iterator_> { - typedef IgemmSwizzle<Iterator_> Transformer; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <enum MatrixLayout::Kind kLayout_, typename Iterator_> -struct IgemmTransformerB {}; - -template <typename Iterator_> -struct IgemmTransformerB<MatrixLayout::kColumnMajor, Iterator_> { - typedef Copy<typename Iterator_::Fragment> Transformer; -}; - -template <typename Iterator_> -struct IgemmTransformerB<MatrixLayout::kRowMajor, Iterator_> { - typedef IgemmSwizzle<Iterator_> Transformer; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The layout for A. - MatrixLayout::Kind kLayoutA_, - /// The layout for B. - MatrixLayout::Kind kLayoutB_, - /// The output tile. - typename OutputTile_, - /// The output type. - typename ScalarD_, - /// The functor to do the math in the epilogue. - typename EpilogueFunctor_, - /// The number of accumulators per thread. - typename AccumulatorsPerThread_ = Shape<32, 8, 8>, - /// The index. - typename Index_ = int> -struct IgemmTraitsHelper { - /// The IGEMM config. - typedef IgemmConfig<OutputTile_, ScalarD_, AccumulatorsPerThread_> GemmConfig; - /// The GEMM config for A. - typedef IgemmTileTraitsHelperA<kLayoutA_, GemmConfig, Index_> GemmTileTraitsHelperA; - /// The GEMM config for B. - typedef IgemmTileTraitsHelperB<kLayoutB_, GemmConfig, Index_> GemmTileTraitsHelperB; - - /// The iterator to load A from global memory. - typedef typename GemmTileTraitsHelperA::GlobalLoadIterator GlobalLoadIteratorA; - - /// The default transformer for A. - typedef typename IgemmTransformerA<GemmTileTraitsHelperA::kLayout, - GlobalLoadIteratorA>::Transformer GlobalTransformerA; - /// The iterator to store A to shared memory. - typedef TileStoreIterator<typename GemmTileTraitsHelperA::SharedStoreTileTraits, - typename GemmTileTraitsHelperA::SharedStoreTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedStoreIteratorA; - /// The stream to load A from global memory to shared memory. - typedef GlobalLoadStream<GlobalLoadIteratorA, SharedStoreIteratorA, GlobalTransformerA> - GlobalLoadStreamA; - - /// The iterator to load B from global memory. - typedef typename GemmTileTraitsHelperB::GlobalLoadIterator GlobalLoadIteratorB; - - // The default transformer for B. - typedef typename IgemmTransformerB<GemmTileTraitsHelperB::kLayout, - GlobalLoadIteratorB>::Transformer GlobalTransformerB; - /// The iterator to store B to shared memory. - typedef TileStoreIterator<typename GemmTileTraitsHelperB::SharedStoreTileTraits, - typename GemmTileTraitsHelperB::SharedStoreTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedStoreIteratorB; - /// The stream to load B from global memory to shared memory. - typedef GlobalLoadStream<GlobalLoadIteratorB, SharedStoreIteratorB, GlobalTransformerB> - GlobalLoadStreamB; - - /// The iterator to load A from shared memory. - typedef TileLoadIterator<typename GemmTileTraitsHelperA::SharedLoadTileTraits, - typename GemmTileTraitsHelperA::SharedLoadTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedLoadIteratorA; - /// The stream to load A from shared memory. - typedef SharedLoadStream<SharedLoadIteratorA, Copy<typename SharedLoadIteratorA::Fragment> > - SharedLoadStreamA; - /// The iterator to load B from shared memory. - typedef TileLoadIterator<typename GemmTileTraitsHelperB::SharedLoadTileTraits, - typename GemmTileTraitsHelperB::SharedLoadTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedLoadIteratorB; - /// The stream to load B from shared memory. - typedef SharedLoadStream<SharedLoadIteratorB, Copy<typename SharedLoadIteratorB::Fragment> > - SharedLoadStreamB; - - /// The multiply-add functor. - typedef typename GemmConfig::MultiplyAdd MultiplyAdd; - /// The object to clear accumulators. - typedef ClearAccumulators<typename MultiplyAdd::ScalarC> ClearAccumulators; - - /// The epilogue. - typedef IgemmEpilogue<IgemmEpilogueTraits<GemmConfig, EpilogueFunctor_> > Epilogue; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename ScalarD_> -struct IgemmEpilogueScalar { - typedef float Scalar; -}; - -template <> -struct IgemmEpilogueScalar<int> { - typedef int Scalar; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The layout for A. - MatrixLayout::Kind kLayoutA_, - /// The layout for B. - MatrixLayout::Kind kLayoutB_, - /// The output tile. - typename OutputTile_ = Shape<32, 128, 128>, - /// The output type. - typename ScalarD_ = int, - /// The functor to do the math in the epilogue. - typename EpilogueFunctor_ = LinearScaling<typename IgemmEpilogueScalar<ScalarD_>::Scalar>, - /// The number of accumulators per thread. - typename AccumulatorsPerThread_ = Shape<32, 8, 8>, - /// The index. - typename Index_ = int, - /// The helper class. - typename Helper_ = IgemmTraitsHelper<kLayoutA_, - kLayoutB_, - OutputTile_, - ScalarD_, - EpilogueFunctor_, - AccumulatorsPerThread_, - Index_> > -struct IgemmTraits : public GemmTraits< - // The config. - typename Helper_::GemmConfig, - // The stream to load A from global memory to shared memory. - typename Helper_::GlobalLoadStreamA, - // The stream to load B from global memory to shared memory. - typename Helper_::GlobalLoadStreamB, - // The stream to load A from shared memory. - typename Helper_::SharedLoadStreamA, - // The stream to load B from shared memory. - typename Helper_::SharedLoadStreamB, - // The epilogue. - typename Helper_::Epilogue, - // The block swizzle to reorganize the grid. - IdentityBlockSwizzle, - // The index. - Index_, - // The tool used to clear accumulators. - typename Helper_::ClearAccumulators> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/linear_scaling.h b/cutlass-example/cutlass/gemm/linear_scaling.h deleted file mode 100644 index 979c93f..0000000 --- a/cutlass-example/cutlass/gemm/linear_scaling.h +++ /dev/null @@ -1,85 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Implements the BLAS linear scaling function alpha*AB + beta*C -*/ -#pragma once - -#include <cutlass/fragment_multiply_add.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Functor to compute linear combination of fragments -template <typename Scalar_, typename FragmentMultiplyAdd_ = FragmentMultiplyAdd<Scalar_> > -struct LinearScaling { - // The scalar. - typedef Scalar_ Scalar; - // The adapater. - typedef FragmentMultiplyAdd_ FragmentMultiplyAdd; - - /// The parameters. - struct Params { - /// The alpha/beta scaling params. - Scalar alpha, beta; - - /// Initialize the parameters. - template <typename GemmDesc_> - CUTLASS_HOST_DEVICE int initialize(GemmDesc_ const& desc) { - alpha = desc.alpha; - beta = desc.beta; - return 0; - } - }; - - /// Ctor. - CUTLASS_DEVICE LinearScaling(Params const& params) : alpha(params.alpha), beta(params.beta) {} - - /// Evaluate the functor. - template <typename FragmentA_, typename FragmentB_> - CUTLASS_DEVICE void evaluate(FragmentA_ const& accum, FragmentB_& output) { - FragmentMultiplyAdd mad; - mad.multiply(alpha, accum, output); - } - - /// Evaluate the functor. - template <typename FragmentA_, typename FragmentB_> - CUTLASS_DEVICE void evaluate(FragmentA_ const& accum, FragmentB_ const& old, FragmentB_& output) { - FragmentMultiplyAdd mad; - FragmentB_ tmp; - mad.multiply(beta, old, tmp); - mad.multiply_add(alpha, accum, tmp, output); - } - - /// The alpha/beta scaling factors. - Scalar alpha, beta; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/sgemm_traits.h b/cutlass-example/cutlass/gemm/sgemm_traits.h deleted file mode 100644 index 66b7677..0000000 --- a/cutlass-example/cutlass/gemm/sgemm_traits.h +++ /dev/null @@ -1,127 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defies structural properties of single-precision GEMM. -*/ -#pragma once - -#include <cutlass/gemm/gemm.h> -#include <cutlass/gemm/gemm_epilogue.h> -#include <cutlass/gemm/gemm_epilogue_traits.h> -#include <cutlass/gemm/gemm_global_tile.h> -#include <cutlass/gemm/gemm_shared_tile.h> -#include <cutlass/gemm/gemm_traits.h> -#include <cutlass/gemm/thread_multiply_add.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The tile size for the GEMM KxNxM. - typename OutputTile_, - /// The number of accumulators per thread. - typename AccumulatorsPerThread_, - /// The number of scalars per LDG for A. - int kScalarsPerLdgA_ = 1, - /// The number of scalars per LDG for B. - int kScalarsPerLdgB_ = 1> -struct SgemmConfig - : public GemmConfig< - /// The scalar type for A. - float, - /// The scalar type for B. - float, - /// The scalar type for C. - float, - /// The scalar type for D. - float, - /// The tile size for the GEMM KxNxM. - OutputTile_, - /// The functor to do the math in the main loop. - ThreadMultiplyAdd<AccumulatorsPerThread_, Shape<1, 4, 8>, float, float, float>, - /// The number of scalars per LDG for A. - kScalarsPerLdgA_, - /// The number of scalars per STS for A. - kScalarsPerLdgA_, - /// The number of scalars per LDS for A. - 4, - /// The number of scalars per LDG for B. - kScalarsPerLdgB_, - /// The number of scalars per STS for B. - kScalarsPerLdgB_, - /// The number of scalars per LDS for B. - 4, - /// The number of scalars per LDG for C and STG for D. - 1, - /// The number of scalars per STS for D. - 4, - /// The number of scalars per LDS for D. - 1, - /// The number of stages in shared memory. - 2> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The layout for A. - MatrixLayout::Kind kLayoutA_, - /// The layout for B. - MatrixLayout::Kind kLayoutB_, - /// The output tile. - typename OutputTile_ = Shape<8, 128, 128>, - /// The functor to use in the epilogue. - typename EpilogueFunctor_ = LinearScaling<float>, - /// The number of accumulators per thread. - typename AccumulatorsPerThread_ = Shape<8, 8, 8>, - /// The number of floats loaded in one LDG for A. - int kScalarsPerLdgA_ = 1, - /// The number of floats loaded in one LDG for B. - int kScalarsPerLdgB_ = 1, - /// The index. - typename Index_ = int, - /// The SGEMM config. - typename GemmConfig_ = - SgemmConfig<OutputTile_, AccumulatorsPerThread_, kScalarsPerLdgA_, kScalarsPerLdgB_>, - /// The traits class for the epilogue. - typename GemmEpilogueTraits_ = - SimplifiedGemmEpilogueTraits<GemmConfig_, EpilogueFunctor_, Index_> > -struct SgemmTraits : public SimplifiedGemmTraits< - // The layout for A. - kLayoutA_, - // The layout for B. - kLayoutB_, - // The config. - GemmConfig_, - // The epilogue. - GemmEpilogue<GemmEpilogueTraits_>, - // The index. - Index_> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/thread_multiply_add.h b/cutlass-example/cutlass/gemm/thread_multiply_add.h deleted file mode 100644 index 20dca15..0000000 --- a/cutlass-example/cutlass/gemm/thread_multiply_add.h +++ /dev/null @@ -1,84 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Template implementing matrix multiply-add operations on fragments. -*/ -#pragma once - -#include <cutlass/fragment.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Template performing matrix multiply-add operation within a thread -template <typename AccumulatorsPerThread_, - typename ThreadsPerWarp_, - typename ScalarA_, - typename ScalarB_, - typename ScalarC_> -struct ThreadMultiplyAdd { - /// The shape of the instruction. - typedef Shape<1, 1, 1, 1> InstructionShape; - /// The number of accumulators per thread. - typedef AccumulatorsPerThread_ AccumulatorsPerThread; - /// The number of threads per warp. - typedef ThreadsPerWarp_ ThreadsPerWarp; - /// The number of accumulators per warp. - typedef typename ShapeMul<AccumulatorsPerThread, ThreadsPerWarp>::Shape AccumulatorsPerWarp; - /// The type for A. - typedef ScalarA_ ScalarA; - /// The fragment for A. - typedef Fragment<ScalarA, AccumulatorsPerThread::kW> FragmentA; - /// The type for B. - typedef ScalarB_ ScalarB; - /// The fragment for B. - typedef Fragment<ScalarB, AccumulatorsPerThread::kH> FragmentB; - /// The type for C and D. - typedef ScalarC_ ScalarC; - /// The accumulators. - typedef Fragment<ScalarC, AccumulatorsPerThread::kH * AccumulatorsPerThread::kW, 16> Accumulators; - - /// Ctor. - CUTLASS_DEVICE ThreadMultiplyAdd() {} - - /// Multiply : d = a*b + c. - CUTLASS_DEVICE void multiply_add(FragmentA const& a, - FragmentB const& b, - Accumulators const& c, - Accumulators& d) { - for (int j = 0; j < AccumulatorsPerThread::kH; ++j) { - for (int i = 0; i < AccumulatorsPerThread::kW; ++i) { - d[j * AccumulatorsPerThread::kW + i] = a[i] * b[j] + c[j * AccumulatorsPerThread::kW + i]; - } - } - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/wmma_gemm_epilogue_traits.h b/cutlass-example/cutlass/gemm/wmma_gemm_epilogue_traits.h deleted file mode 100644 index 0fafacf..0000000 --- a/cutlass-example/cutlass/gemm/wmma_gemm_epilogue_traits.h +++ /dev/null @@ -1,161 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines structural properties of WMMA GEMM's epilogue phase. -*/ -#pragma once - -#include <cutlass/wmma_matrix.h> -#ifdef CUTLASS_USE_WMMA_API - -#include <cutlass/convert.h> -#include <cutlass/coord.h> -#include <cutlass/gemm/gemm_global_stream.h> -#include <cutlass/gemm/gemm_shared_stream.h> -#include <cutlass/gemm/linear_scaling.h> -#include <cutlass/gemm/wmma_gemm_global_tile.h> -#include <cutlass/gemm/wmma_gemm_shared_tile.h> -#include <cutlass/reshape_tile.h> -#include <cutlass/tile_iterator.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_, typename EpilogueFunctor_, typename Index_ = int> -struct WmmaGemmEpilogueTraitsHelper { - /// The scalar. - typedef typename EpilogueFunctor_::Scalar Scalar; - /// The output tile. - typedef typename GemmConfig_::OutputTile OutputTile; - - /// The number of WMMAs in the H dimension. - static int const kWmmasPerH = - GemmConfig_::AccumulatorsPerWarp::kH / GemmConfig_::InstructionShape::kH; - /// The number of iterations in the epilogue. That's the number of "horizontal" WMMAs. - typedef Shape<1, 1, kWmmasPerH> Iterations; - // The iteration strides in the H/W dimension. - typedef Shape<0, 0, 0> Delta; - /// The functor to do the math in the epilogue. - typedef EpilogueFunctor_ Functor; - - /// The traits class to build the iterator to store to shared memory for D. - typedef WmmaGemmSharedStoreTileDTraits< - // The output layout. - MatrixLayout::kColumnMajor, - // The pointer is float. - typename Functor::Scalar, - // The output tile size. - typename GemmConfig_::OutputTile, - // The number of warps. - typename GemmConfig_::Warps, - // The shape of the instruction. - typename GemmConfig_::InstructionShape> - SharedStoreTileTraits; - - typedef WmmaMatrix<GemmOperand::kC, - MatrixLayout::kColumnMajor, - Scalar, - typename GemmConfig_::InstructionShape> - WmmaMatrix; - - /// The iterator to store D to shared memory. - typedef TileStoreIterator<SharedStoreTileTraits, - typename SharedStoreTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared, - Index_, - WmmaMatrix, - IteratorFragment::kWmmaMatrix> - SharedStoreIteratorD; - - /// The shared store transformer for D. - typedef Copy<typename SharedStoreIteratorD::Fragment> SharedStoreTransformerD; - - /// The traits class to build the iterator to load from shared memory for D. - typedef WmmaGemmSharedLoadTileDTraits< - // The pointer. - typename Functor::Scalar, - // The tile size. - typename SharedStoreIteratorD::Tile, - // The number of threads. - Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, - // The number of scalars per LDS. - GemmConfig_::kScalarsPerLdsD> - SharedLoadTileTraits; - - /// The iterator to load D from shared memory. - typedef TileLoadIterator<SharedLoadTileTraits, - typename SharedLoadTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedLoadIteratorD; - - /// The traits class to build the iterator to load data from global memory for C^N. - typedef WmmaGemmGlobalIteratorCdTraits< - // The pointer is float const. - typename GemmConfig_::ScalarC const, - // The tile has size (N / Iterations)xM in GEMM's terminology. - Shape<1, - GemmConfig_::OutputTile::kH / ShapeCount<Iterations>::kCount, - GemmConfig_::OutputTile::kW>, - // The threads are distributed as warps x 32 (the traits may reorganize). - Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerLdgC> - GlobalLoadTileTraits; - - /// The iterator to load C. - typedef WmmaGemmGlobalIteratorCd<GlobalLoadTileTraits, Index_> GlobalLoadIteratorC; - /// The transformer for C. - typedef Copy<typename GlobalLoadIteratorC::Fragment> GlobalTransformerC; - - /// The traits class to build the iterator to store data to global memory for D^N. - typedef WmmaGemmGlobalIteratorCdTraits< - // The pointer is float. - typename GemmConfig_::ScalarD, - // The tile has size (N / Iterations)xM in GEMM's terminology. - Shape<1, - GemmConfig_::OutputTile::kH / ShapeCount<Iterations>::kCount, - GemmConfig_::OutputTile::kW>, - // The threads are distributed as warps x 32 (the traits may reorganize). - Shape<1, ShapeCount<typename GemmConfig_::Warps>::kCount, GemmConfig_::kWarpSize>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerStgD> - GlobalStoreTileTraits; - - /// The iterator to store D. - typedef WmmaGemmGlobalIteratorCd<GlobalStoreTileTraits, Index_> GlobalStoreIteratorD; - /// The transformer for D. - typedef Copy<typename GlobalStoreIteratorD::Fragment> GlobalTransformerD; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass - -#endif // defined CUTLASS_USE_WMMA_API diff --git a/cutlass-example/cutlass/gemm/wmma_gemm_global_tile.h b/cutlass-example/cutlass/gemm/wmma_gemm_global_tile.h deleted file mode 100644 index dbd57f6..0000000 --- a/cutlass-example/cutlass/gemm/wmma_gemm_global_tile.h +++ /dev/null @@ -1,211 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines tile iterator traits for loading thread block-level tile from global memory. -*/ -#pragma once - -#include <cutlass/gemm/gemm_global_tile.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, typename Tile_, typename Threads_, int kAccessSize_> -struct WmmaGemmGlobalIteratorCdTraits : public GemmGlobalTileTraits<GemmOperand::kC, - MatrixLayout::kColumnMajor, - Scalar_, - Tile_, - Threads_, - kAccessSize_> { - /// The base class. - typedef GemmGlobalTileTraits<GemmOperand::kC, - MatrixLayout::kColumnMajor, - Scalar_, - Tile_, - Threads_, - kAccessSize_> - Base; - - /// Override the strides in each dimension between different loads/stores. - typedef Shape<0, 0, Base::Delta::kW, Base::Delta::kC> Delta; - - /// Computes the thread offset in (H, W) based on thread ID - struct ThreadOffset { - CUTLASS_HOST_DEVICE - Coord<4> operator()() const { - int thread_offset_h = threadIdx.x / Base::Threads::kW; - int thread_offset_w = threadIdx.x % Base::Threads::kW * Base::ThreadsDelta::kW; - - return make_Coord(0, thread_offset_h, thread_offset_w, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename TileTraits_, typename Index_ = int> -struct WmmaGemmGlobalIteratorCd : public TileIteratorBase<TileTraits_, - typename TileTraits_::Scalar, - IteratorAdvance::kH, - MemorySpace::kGlobal, - Index_> { - /// This class. - typedef WmmaGemmGlobalIteratorCd<TileTraits_, Index_> This_; - /// The traits. - typedef TileTraits_ Traits; - /// The base class. - typedef TileIteratorBase<Traits, - typename TileTraits_::Scalar, - IteratorAdvance::kH, - MemorySpace::kGlobal, - Index_> - Base; - /// Override the strides in each dimension between different loads/stores. - typedef Shape<0, 0, Base::Delta::kW, Base::Delta::kC> ImmediateOffsetStrides; - /// The layout. - static MatrixLayout::Kind const kLayout = TileTraits_::kLayout; - - /// The scalar. - typedef typename TileTraits_::Scalar Scalar; - /// The pointer. - typedef typename TileTraits_::Pointer Pointer; - /// The threads. - typedef typename TileTraits_::Threads Threads; - /// The index. - typedef Index_ Index; - /// The thread offset functor. - typedef typename TileTraits_::ThreadOffset ThreadOffset; - - /// The params. - struct Params { - /// The pointer. - Pointer pointer; - /// The stride in the H dimension to setup the thread in the block. - Index stride_h; - /// The strides to increment the pointer. - Index inc_h, inc_advance; - /// The column offset to compute the predicate for the columns. - Index predicate_offset; - /// The strides to increment the predicate offset. - Index predicate_inc_h, predicate_inc_advance; - - /// Setup the params. - CUTLASS_HOST_DEVICE int initialize( - Pointer pointer, Index ld, Index n, Index epilogue_stride_w, Index epilogue_delta_w) { - // The pointer. - this->pointer = pointer; - // Setup the base stride. One "group of threads" per column. - stride_h = ld; - // Each thread output 1 column per iteration. . - inc_h = ld * TileTraits_::Threads::kH; - inc_advance = inc_h + epilogue_stride_w; - - predicate_offset = n; - predicate_inc_h = TileTraits_::Threads::kH; - predicate_inc_advance = predicate_inc_h + epilogue_delta_w; - - // It worked. - return 0; - } - }; - - Params params; - - Coord<4> thread_offset; - - /// Ctor. - CUTLASS_DEVICE WmmaGemmGlobalIteratorCd() {} - - /// Ctor. - CUTLASS_DEVICE WmmaGemmGlobalIteratorCd(Params const& params, - const Coord<3>& bounds, - const Coord<3>& block, - int const pointer_offset = 0, - int const pred_offset = 0, - ThreadOffset thread_offset_func = ThreadOffset()) - - : params(params) { - thread_offset = thread_offset_func(); - // Each warp works on a different column of the tile. - int const h = thread_offset[1] + block[1]; - // Each lane writes a different element. - int const w = thread_offset[2] + block[2]; - // Setup the pointer. - this->params.pointer += ((h * params.stride_h + w) + pointer_offset); - - // Prepare the vector of predicates. - for (int i = 0; i < Base::Iterations::kW; ++i) { - predicates.set(i, w + i * Base::Delta::kW < bounds[2]); - } - this->params.predicate_offset -= (h + pred_offset); - } - - /// The accessor. - CUTLASS_DEVICE void get(typename Base::AccessType& value, int d, int h, int w, int c) const { - int const imm = - ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(0, 0, w, c); - Load<Scalar, TileTraits_::kAccessSize, MemorySpace::kGlobal>::load(value, params.pointer, imm); - } - - /// Increment the pointer in the C dimension. - CUTLASS_DEVICE void inc_c() {} - /// Increment the pointer in the W dimension. - CUTLASS_DEVICE void inc_w() {} - /// Increment the pointer in the H dimension. - CUTLASS_DEVICE void inc_h() { - params.pointer += params.inc_h; - params.predicate_offset -= params.predicate_inc_h; - } - /// Increment the pointer in the D dimension. - CUTLASS_DEVICE void inc_d() {} - /// Increment the pointer to move to the next iteration. - CUTLASS_DEVICE void inc_advance() { - params.pointer += params.inc_advance; - params.predicate_offset -= params.predicate_inc_advance; - } - - /// The accessor. - CUTLASS_DEVICE void set(typename Base::AccessType const& value, int d, int h, int w, int c) { - int const imm = - ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(d, h, w, 0); - Store<Scalar, TileTraits_::kAccessSize, MemorySpace::kGlobal>::store( - value, params.pointer, imm); - } - - /// Test the predicate. - CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { - return predicates.at(w) && params.predicate_offset > 0; - } - - /// The predicates for the row. - cutlass::PredicateVector<Base::Iterations::kW> predicates; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass diff --git a/cutlass-example/cutlass/gemm/wmma_gemm_multiply_add.h b/cutlass-example/cutlass/gemm/wmma_gemm_multiply_add.h deleted file mode 100644 index 5968350..0000000 --- a/cutlass-example/cutlass/gemm/wmma_gemm_multiply_add.h +++ /dev/null @@ -1,108 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Implements warp-level matrix multiply-accumulate operation using CUDA WMMA API. -*/ -#pragma once - -#include <cutlass/wmma_matrix.h> -#ifdef CUTLASS_USE_WMMA_API -#include <cutlass/fragment.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <MatrixLayout::Kind kLayoutA_, - typename ScalarA_, - MatrixLayout::Kind kLayoutB_, - typename ScalarB_, - MatrixLayout::Kind kLayoutC_, - typename ScalarC_, - typename AccumulatorsPerWarp_, - typename InstructionShape_> -struct WmmaGemmMultiplyAdd { - /// The shape of the instruction. - typedef InstructionShape_ InstructionShape; - /// The number of threads per warp. That's a dummy configuration. - typedef Shape<1, InstructionShape_::kH, InstructionShape_::kW> ThreadsPerWarp; - /// The dimensions. - typedef AccumulatorsPerWarp_ AccumulatorsPerWarp; - /// The type for A. - typedef ScalarA_ ScalarA; - /// The type for B. - typedef ScalarB_ ScalarB; - /// The type for C and D. - typedef ScalarC_ ScalarC; - /// The number of iterations. - typedef typename ShapeDiv<AccumulatorsPerWarp, InstructionShape>::Shape Iterations; - - /// The element for A. - typedef WmmaMatrix<GemmOperand::kA, kLayoutA_, ScalarA, InstructionShape> ElementA; - /// The fragment for A. - typedef Fragment<ElementA, Iterations::kW> FragmentA; - - /// The element for B. - typedef WmmaMatrix<GemmOperand::kB, kLayoutB_, ScalarB, InstructionShape> ElementB; - /// The fragment for B. - typedef Fragment<ElementB, Iterations::kH> FragmentB; - - /// The element for C. - typedef WmmaMatrix<GemmOperand::kC, kLayoutC_, ScalarC, InstructionShape> ElementC; - /// The fragment for C. - typedef Fragment<ElementC, Iterations::kH * Iterations::kW> Accumulators; - - /// Ctor. - CUTLASS_DEVICE WmmaGemmMultiplyAdd() {} - - /// Multiply : d = a*b. - CUTLASS_DEVICE void multiply_add(FragmentA const& a, - FragmentB const& b, - Accumulators const& c, - Accumulators& d) { - for (int j = 0; j < Iterations::kH; ++j) { - for (int i = 0; i < Iterations::kW; ++i) { - // The input elements. - ElementA const& elt_a = a[i]; - ElementB const& elt_b = b[j]; - ElementC const& elt_c = c[j * Iterations::kW + i]; - - // The output element. - ElementC& elt_d = d[j * Iterations::kW + i]; - - // The wmma instruction. - nvcuda::wmma::mma_sync(elt_d, elt_a, elt_b, elt_c); - } - } - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass - -#endif // defined CUTLASS_USE_WMMA_API diff --git a/cutlass-example/cutlass/gemm/wmma_gemm_shared_tile.h b/cutlass-example/cutlass/gemm/wmma_gemm_shared_tile.h deleted file mode 100644 index 7d15b26..0000000 --- a/cutlass-example/cutlass/gemm/wmma_gemm_shared_tile.h +++ /dev/null @@ -1,240 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines iterator traits for efficiently loading and storing fragment to and from shared - memory, specialized for WMMA GEMM. -*/ -#pragma once - -#include <cutlass/wmma_matrix.h> -#ifdef CUTLASS_USE_WMMA_API - -#include <cutlass/gemm/gemm_operand.h> -#include <cutlass/reshape_tile.h> - -namespace cutlass { -namespace gemm { - -template <class> -struct Debug {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <MatrixLayout::Kind kLayout_, - typename Scalar_, - typename Tile_, - typename Warps_, - int kWarpStride_, - typename Iterations_, - typename Delta_, - typename WmmaShape_> -struct WmmaGemmSharedLoadTileATraits { - /// The operand. - static GemmOperand::Kind const kOperand = GemmOperand::kA; - /// The layout. - static MatrixLayout::Kind const kLayout = kLayout_; - /// The scalar. - typedef Scalar_ Scalar; - /// The pointer. - typedef Scalar const* Pointer; - /// The access size - static int const kAccessSize = 1; - /// The tile with skew. - typedef Tile_ Tile; - /// The number of warps. - typedef Warps_ Warps; - /// The warps strides. - static int const kWarpStride = kWarpStride_; - /// The number of iterations. - typedef Iterations_ Iterations; - /// The strides between iterations. - typedef Delta_ Delta; - /// The strides between iterations. - typedef Delta_ ImmediateOffsetStrides; - /// The shape of the WMMA instruction. - typedef WmmaShape_ WmmaShape; - /// The memory space. - static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; - /// ThreadOffset - struct ThreadOffset { - CUTLASS_HOST_DEVICE - Coord<4> operator()() const { - // The warp id. - int const warp = threadIdx.x / kWarpSize; - // The offset. - int const offset = warp % Warps::kW * kWarpStride; - return make_Coord(0, 0, offset, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <MatrixLayout::Kind kLayout_, - typename Scalar_, - typename Tile_, - typename Warps_, - int kWarpStride_, - typename Iterations_, - typename Delta_, - typename WmmaShape_> -struct WmmaGemmSharedLoadTileBTraits { - /// The operand. - static GemmOperand::Kind const kOperand = GemmOperand::kB; - /// The layout. - static MatrixLayout::Kind const kLayout = kLayout_; - /// The scalar. - typedef Scalar_ Scalar; - /// The pointer. - typedef Scalar const* Pointer; - /// The access size - static int const kAccessSize = 1; - /// The tile with skew. - typedef Tile_ Tile; - /// The number of warps. - typedef Warps_ Warps; - /// The warps strides. - static int const kWarpStride = kWarpStride_; - /// The number of iterations. - typedef Iterations_ Iterations; - /// The strides between iterations. - typedef Delta_ Delta; - /// The strides between iterations. - typedef Delta_ ImmediateOffsetStrides; - /// The shape of the WMMA instruction. - typedef WmmaShape_ WmmaShape; - /// The memory space. - static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; - /// ThreadOffset - struct ThreadOffset { - CUTLASS_HOST_DEVICE - Coord<4> operator()() const { - // The warp id. - int const warp = threadIdx.x / kWarpSize; - // The offset. - int const offset = warp / Warps::kW * kWarpStride; - return make_Coord(0, 0, offset, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <MatrixLayout::Kind kLayout_, - typename Scalar_, - typename OutputTile_, - typename Warps_, - typename WmmaShape_, - int kSkew_ = 0> -struct WmmaGemmSharedStoreTileDTraits { - /// The operand. - static GemmOperand::Kind const kOperand = GemmOperand::kC; - /// The layout. - static MatrixLayout::Kind const kLayout = kLayout_; - /// The scalar. - typedef Scalar_ Scalar; - // The access size - static int const kAccessSize = 1; - /// The pointer. - typedef Scalar* Pointer; - /// The number of warps. - typedef Warps_ Warps; - /// The shape of the WMMA instruction. - typedef WmmaShape_ WmmaShape; - /// The skew. - static int const kSkew = kSkew_; - /// The memory space. - static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; - /// The tile with skew. - typedef Shape<1, Warps_::kH * WmmaShape_::kH, OutputTile_::kW + kSkew_> Tile; - /// The number of iterations needed to store the tile. - typedef Shape<1, 1, OutputTile_::kW / Warps::kW / WmmaShape_::kW> Iterations; - /// The strides in each dimension between different loads/stores. - typedef Shape<0, 0, Warps::kW * WmmaShape_::kW, 0> Delta; - /// The strides in each dimension between different loads/stores. - typedef Shape<0, 0, Warps::kW * WmmaShape_::kW, 0> ImmediateOffsetStrides; - - /// ThreadOffset - struct ThreadOffset { - CUTLASS_HOST_DEVICE - Coord<4> operator()() const { - // The warp id. - int const warp = threadIdx.x / kWarpSize; - // The starting column. - int const h = warp / Warps::kW * WmmaShape::kH; - // The w. - int const w = warp % Warps::kW * WmmaShape::kW; - // The offset. - int const offset = h * Tile::kW + w; - return make_Coord(0, 0, offset, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, typename Tile_, typename Threads_, int kScalarsPerLds_> -struct WmmaGemmSharedLoadTileDTraits { - /// The scalar. - typedef Scalar_ Scalar; - /// The pointer. - typedef Scalar const* Pointer; - /// The access size - static int const kAccessSize = kScalarsPerLds_; - /// The tile. - typedef typename ReshapeTile<Tile_, kScalarsPerLds_>::Tile Tile; - /// The threads. - typedef typename ReshapeThreads<Tile, Threads_>::Threads Threads; - /// The threads strides. - typedef Shape<1, Tile::kW * Tile::kC, Tile::kC> ThreadsStrides; - /// The memory space. - static MemorySpace::Kind const kMemorySpace = MemorySpace::kShared; - - /// The strides in each dimension between different loads/stores. - typedef Shape<0, Threads::kH * ShapeCount<Tile>::kWc, Threads::kW * kScalarsPerLds_> Delta; - /// The strides in each dimension between different loads/stores. - typedef Shape<0, Threads::kH * ShapeCount<Tile>::kWc, Threads::kW * kScalarsPerLds_> - ImmediateOffsetStrides; - /// The number of iterations needed to load/store the tile. - typedef Shape<1, Tile::kH / Threads::kH, Tile::kW / Threads::kW, Tile::kC / kScalarsPerLds_> - Iterations; - - /// ThreadOffset - struct ThreadOffset { - CUTLASS_HOST_DEVICE - Coord<4> operator()() const { - // The offset. - int const offset = ComputeThreadOffsetFromStrides<Threads, ThreadsStrides>::get(); - return make_Coord(0, 0, offset, 0); - } - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass - -#endif // defined CUTLASS_USE_WMMA_API diff --git a/cutlass-example/cutlass/gemm/wmma_gemm_traits.h b/cutlass-example/cutlass/gemm/wmma_gemm_traits.h deleted file mode 100644 index 7901201..0000000 --- a/cutlass-example/cutlass/gemm/wmma_gemm_traits.h +++ /dev/null @@ -1,574 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defies structural properties of GEMM targeting WMMA API in CUDA. -*/ -#pragma once - -#include <cutlass/wmma_matrix.h> -#ifdef CUTLASS_USE_WMMA_API - -#include <cutlass/convert.h> -#include <cutlass/gemm/gemm.h> -#include <cutlass/gemm/gemm_epilogue.h> -#include <cutlass/gemm/gemm_epilogue_traits.h> -#include <cutlass/gemm/gemm_global_tile.h> -#include <cutlass/gemm/gemm_shared_tile.h> -#include <cutlass/gemm/gemm_traits.h> -#include <cutlass/gemm/wmma_gemm_epilogue_traits.h> -#include <cutlass/gemm/wmma_gemm_global_tile.h> -#include <cutlass/gemm/wmma_gemm_multiply_add.h> - -namespace cutlass { -namespace gemm { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The layout for A. - MatrixLayout::Kind kLayoutA_, - /// The layout for B. - MatrixLayout::Kind kLayoutB_, - /// The tile size for the GEMM KxNxM. - typename OutputTile_, - /// The output type. - typename ScalarC_, - /// The accumulator type. - typename Accumulator_, - /// The number of accumulators per warp. - typename AccumulatorsPerWarp_, - /// The shape of the WMMA instruction. - typename InstructionShape_, - /// The number of scalars per LDG for A. - int kScalarsPerLdgA_, - /// The number of scalars per LDG for B. - int kScalarsPerLdgB_> -struct WmmaGemmConfig : public GemmConfig< - /// The scalar type for A. - half, - /// The scalar type for B. - half, - /// The scalar type for C. - ScalarC_, - /// The scalar type for D. - ScalarC_, - /// The tile size for the GEMM KxNxM. - OutputTile_, - /// The functor to do the math in the main loop. - WmmaGemmMultiplyAdd<kLayoutA_, - half, - kLayoutB_, - half, - MatrixLayout::kColumnMajor, - Accumulator_, - AccumulatorsPerWarp_, - InstructionShape_>, - /// The number of scalars per LDG for A. - kScalarsPerLdgA_, - /// The number of scalars per STS for A. - kScalarsPerLdgA_, - /// The number of scalars per LDS for A. - 8, - /// The number of scalars per LDG for B. - kScalarsPerLdgB_, - /// The number of scalars per STS for B. - kScalarsPerLdgB_, - /// The number of scalars per LDS for B. - 8, - /// The number of scalars per LDG for C and STG for D. - 16 / sizeof(ScalarC_), - /// The number of scalars per STS for D. - 16 / sizeof(ScalarC_), - /// The number of scalars per LDS for D. - 16 / sizeof(ScalarC_), - /// The number of stages in shared memory. - 1> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <enum MatrixLayout::Kind kLayout_, typename GemmConfig_> -struct WmmaGemmTileTraitsHelperA {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_> -struct WmmaGemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_> - : public GemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_> { - /// The base config. - typedef GemmTileTraitsHelperA<MatrixLayout::kColumnMajor, GemmConfig_> Base; - - /// The skew. - static int const kSkew = 16 / sizeof(typename Base::MultiplyAddScalar); - /// The shared tile size. - typedef Shape<GemmConfig_::kStages, - GemmConfig_::OutputTile::kD, - GemmConfig_::OutputTile::kW + kSkew> - Tile; - - /// WMMA matrix - typedef WmmaMatrix<GemmOperand::kA, - MatrixLayout::kColumnMajor, - typename Base::MultiplyAddScalar, - typename GemmConfig_::InstructionShape> - WmmaMatrix; - - /// The traits class to build the iterator to store data to shared memory for A^N. - typedef GemmSharedStoreTileAbTraits< - // The pointer. - typename Base::MultiplyAddScalar, - // The tile has size KxM in GEMM's terminology. - Tile, - // The threads are distributed as warps x 32 (the traits may reorganize). - typename Base::GlobalTileTraits::Threads, - // The number of scalars per STS (STS.32 or STS.128, etc). - GemmConfig_::kScalarsPerStsA> - SharedStoreTileTraits; - - /// The number of elements loaded in one LDG. - static int const kScalarsPerW = GemmConfig_::InstructionShape::kW * GemmConfig_::Warps::kW; - /// The number of scalars loaded per iteration. - static int const kScalarsPerIteration = Tile::kW * GemmConfig_::InstructionShape::kD; - /// The traits class to build the iterator to load from shared memory for A. - typedef WmmaGemmSharedLoadTileATraits< - // The layout of the matrix. - MatrixLayout::kColumnMajor, - // The pointer. - typename Base::MultiplyAddScalar, - // The output tile size. - Tile, - // The number of warps. - typename GemmConfig_::Warps, - // The strides between warps. - GemmConfig_::InstructionShape::kW, - // The number of iterations to load the data. - Shape<1, 1, GemmConfig_::OutputTile::kW / kScalarsPerW>, - // The stride between iterations. - Shape<kScalarsPerIteration, 0, kScalarsPerW, 0>, - // The shape of the instruction. - typename GemmConfig_::InstructionShape> - SharedLoadTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_> -struct WmmaGemmTileTraitsHelperA<MatrixLayout::kRowMajor, GemmConfig_> { - /// The layout. - static MatrixLayout::Kind const kLayout = MatrixLayout::kRowMajor; - - /// The input scalar. - typedef typename GemmConfig_::ScalarA Scalar; - /// The scalar stored in shared memory. - typedef typename GemmConfig_::MultiplyAdd::ScalarA MultiplyAddScalar; - - /// WMMA matrix - typedef WmmaMatrix<GemmOperand::kA, - MatrixLayout::kRowMajor, - MultiplyAddScalar, - typename GemmConfig_::InstructionShape> - WmmaMatrix; - - /// The traits class to build the iterator to load data from global memory for A^T. - typedef GemmGlobalTileTraits< - // That's A. - GemmOperand::kA, - // A is row-major. - MatrixLayout::kRowMajor, - // The pointer is float const. - Scalar const, - // The tile has size KxM in GEMM's terminology. - Shape<1, GemmConfig_::OutputTile::kW, GemmConfig_::OutputTile::kD>, - // The threads are distributed as warps x 32 (the traits may reorganize). - Shape<1, GemmConfig_::kThreads / GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kD>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerLdgA> - GlobalTileTraits; - - /// The skew. - static int const kSkew = 16 / sizeof(MultiplyAddScalar); - /// The tile. - typedef Shape<GemmConfig_::kStages, - GemmConfig_::OutputTile::kW, - GemmConfig_::OutputTile::kD + kSkew> - Tile; - - /// The traits class to build the iterator to store data to shared memory for A^N. - typedef GemmSharedStoreTileAbTraits< - // The pointer. - MultiplyAddScalar, - // The tile has size KxM in GEMM's terminology. - Tile, - // The threads are distributed as warps x 32 (the traits may reorganize). - typename GlobalTileTraits::Threads, - // The number of scalars per STS (STS.32 or STS.128, etc). - GemmConfig_::kScalarsPerStsA> - SharedStoreTileTraits; - - /// The number of elements loaded in one LDG. - static int const kScalarsPerW = GemmConfig_::InstructionShape::kW * GemmConfig_::Warps::kW; - /// The traits class to build the iterator to load from shared memory for A. - typedef WmmaGemmSharedLoadTileATraits< - // The layout of the matrix. - MatrixLayout::kRowMajor, - // The pointer. - MultiplyAddScalar, - // The tile in shared memory. - Tile, - // The number of warps. - typename GemmConfig_::Warps, - // The strides between warps. - GemmConfig_::InstructionShape::kW * Tile::kW, - // The number of iterations to load the data. - Shape<1, 1, GemmConfig_::OutputTile::kW / kScalarsPerW>, - // The stride between iterations. - Shape<GemmConfig_::InstructionShape::kD, 0, kScalarsPerW * Tile::kW>, - // The shape of the instruction. - typename GemmConfig_::InstructionShape> - SharedLoadTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <enum MatrixLayout::Kind kLayout_, typename GemmConfig_> -struct WmmaGemmTileTraitsHelperB {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_> -struct WmmaGemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_> - : public GemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_> { - /// The base config. - typedef GemmTileTraitsHelperB<MatrixLayout::kRowMajor, GemmConfig_> Base; - - /// The skew. - static int const kSkew = 16 / sizeof(typename Base::MultiplyAddScalar); - /// The shared tile size. - typedef Shape<GemmConfig_::kStages, - GemmConfig_::OutputTile::kD, - GemmConfig_::OutputTile::kH + kSkew> - Tile; - - /// WMMA matrix - typedef WmmaMatrix<GemmOperand::kB, - MatrixLayout::kRowMajor, - typename Base::MultiplyAddScalar, - typename GemmConfig_::InstructionShape> - WmmaMatrix; - - /// The traits class to build the iterator to store data to shared memory for B^T. - typedef GemmSharedStoreTileAbTraits< - // The pointer. - typename Base::MultiplyAddScalar, - // The tile has size KxM in GEMM's terminology. - Tile, - // The threads are distributed as warps x 32 (the traits may reorganize). - typename Base::GlobalTileTraits::Threads, - // The number of scalars per STS (STS.32 or STS.128, etc). - GemmConfig_::kScalarsPerStsB> - SharedStoreTileTraits; - - /// The number of elements loaded in one LDG. - static int const kScalarsPerW = GemmConfig_::InstructionShape::kH * GemmConfig_::Warps::kH; - /// The number of scalars loaded per iteration. - static int const kScalarsPerIteration = Tile::kW * GemmConfig_::InstructionShape::kD; - /// The traits class to build the iterator to load from shared memory for B. - typedef WmmaGemmSharedLoadTileBTraits< - // The layout of the matrix. - MatrixLayout::kRowMajor, - // The pointer. - typename Base::MultiplyAddScalar, - // The output tile size. - Tile, - // The number of warps. - typename GemmConfig_::Warps, - // The strides between warps. - GemmConfig_::InstructionShape::kH, - // The number of iterations to load the data. - Shape<1, 1, GemmConfig_::OutputTile::kH / kScalarsPerW>, - // The stride between iterations. - Shape<kScalarsPerIteration, 0, kScalarsPerW, 0>, - // The shape of the instruction. - typename GemmConfig_::InstructionShape> - SharedLoadTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmConfig_> -struct WmmaGemmTileTraitsHelperB<MatrixLayout::kColumnMajor, GemmConfig_> { - /// The layout. - static MatrixLayout::Kind const kLayout = MatrixLayout::kColumnMajor; - - /// The input scalar. - typedef typename GemmConfig_::ScalarB Scalar; - /// The scalar stored in shared memory. - typedef typename GemmConfig_::MultiplyAdd::ScalarB MultiplyAddScalar; - - /// WMMA matrix - typedef WmmaMatrix<GemmOperand::kB, - MatrixLayout::kColumnMajor, - MultiplyAddScalar, - typename GemmConfig_::InstructionShape> - WmmaMatrix; - - /// The traits class to build the iterator to load data from global memory for B^N. - typedef GemmGlobalTileTraits< - // That's B. - GemmOperand::kB, - // A is row-major. - MatrixLayout::kColumnMajor, - // The pointer is float const. - Scalar const, - // The tile has size KxM in GEMM's terminology. - Shape<1, GemmConfig_::OutputTile::kH, GemmConfig_::OutputTile::kD>, - // The threads are distributed as warps x 32 (the traits may reorganize). - Shape<1, GemmConfig_::kThreads / GemmConfig_::OutputTile::kD, GemmConfig_::OutputTile::kD>, - // The number of scalars per LDG (LDG.32 or LDG.128, etc). - GemmConfig_::kScalarsPerLdgB> - GlobalTileTraits; - - /// The skew. - static int const kSkew = 16 / sizeof(MultiplyAddScalar); - /// The tile. - typedef Shape<GemmConfig_::kStages, - GemmConfig_::OutputTile::kH, - GemmConfig_::OutputTile::kD + kSkew> - Tile; - - /// The traits class to build the iterator to store data to shared memory for B^N. - typedef GemmSharedStoreTileAbTraits< - // The pointer. - MultiplyAddScalar, - // The tile has size KxM in GEMM's terminology. - Tile, - // The threads are distributed as warps x 32 (the traits may reorganize). - typename GlobalTileTraits::Threads, - // The number of scalars per STS (STS.32 or STS.128, etc). - GemmConfig_::kScalarsPerStsB> - SharedStoreTileTraits; - - /// The number of elements loaded in one LDG. - static int const kScalarsPerW = GemmConfig_::InstructionShape::kH * GemmConfig_::Warps::kH; - /// The traits class to build the iterator to load from shared memory for B. - typedef WmmaGemmSharedLoadTileBTraits< - // The layout of the matrix. - MatrixLayout::kColumnMajor, - // The pointer. - MultiplyAddScalar, - // The tile in shared memory. - Tile, - // The number of warps. - typename GemmConfig_::Warps, - // The strides between warps. - GemmConfig_::InstructionShape::kH * Tile::kW, - // The number of iterations to load the data. - Shape<1, 1, GemmConfig_::OutputTile::kH / kScalarsPerW>, - // The stride between iterations. - Shape<GemmConfig_::InstructionShape::kD, 0, kScalarsPerW * Tile::kW>, - // The shape of the instruction. - typename GemmConfig_::InstructionShape> - SharedLoadTileTraits; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The layout for A. - MatrixLayout::Kind kLayoutA_, - /// The layout for B. - MatrixLayout::Kind kLayoutB_, - /// The output tile. - typename OutputTile_, - /// The output type. - typename ScalarC_, - /// The accumulator type. - typename Accumulator_, - /// The functor to do the math in the epilogue. - typename EpilogueFunctor_, - /// The number of accumulators per warp. - typename AccumulatorsPerWarp_, - /// The shape of the WMMA instruction. - typename InstructionShape_, - /// The number of halfs loaded in one LDG for A. - int kScalarsPerLdgA_, - /// The number of halfs loaded in one LDG for B. - int kScalarsPerLdgB_, - /// The index. - typename Index_> -struct WmmaGemmTraitsHelper { - /// The WMMA GEMM config. - typedef WmmaGemmConfig<kLayoutA_, - kLayoutB_, - OutputTile_, - ScalarC_, - Accumulator_, - AccumulatorsPerWarp_, - InstructionShape_, - kScalarsPerLdgA_, - kScalarsPerLdgB_> - GemmConfig; - - /// The GEMM config for A. - typedef WmmaGemmTileTraitsHelperA<kLayoutA_, GemmConfig> GemmTileTraitsHelperA; - /// The GEMM config for B. - typedef WmmaGemmTileTraitsHelperB<kLayoutB_, GemmConfig> GemmTileTraitsHelperB; - - /// The iterator to load A from global memory. - typedef GemmGlobalIteratorAb<typename GemmTileTraitsHelperA::GlobalTileTraits, Index_> - GlobalLoadIteratorA; - /// The default transformer for A. - typedef Copy<typename GlobalLoadIteratorA::Fragment> GlobalTransformerA; - /// The iterator to store A to shared memory. - typedef TileStoreIterator<typename GemmTileTraitsHelperA::SharedStoreTileTraits, - typename GemmTileTraitsHelperA::SharedStoreTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedStoreIteratorA; - /// The stream to load A from global memory to shared memory. - typedef GlobalLoadStream<GlobalLoadIteratorA, SharedStoreIteratorA, GlobalTransformerA> - GlobalLoadStreamA; - - /// The iterator to load B from global memory. - typedef GemmGlobalIteratorAb<typename GemmTileTraitsHelperB::GlobalTileTraits, Index_> - GlobalLoadIteratorB; - // The default transformer for B. - typedef Copy<typename GlobalLoadIteratorB::Fragment> GlobalTransformerB; - /// The iterator to store B to shared memory. - typedef TileStoreIterator<typename GemmTileTraitsHelperB::SharedStoreTileTraits, - typename GemmTileTraitsHelperB::SharedStoreTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared> - SharedStoreIteratorB; - /// The stream to load B from global memory to shared memory. - typedef GlobalLoadStream<GlobalLoadIteratorB, SharedStoreIteratorB, GlobalTransformerB> - GlobalLoadStreamB; - - /// The iterator to load A from shared memory. - typedef TileLoadIterator<typename GemmTileTraitsHelperA::SharedLoadTileTraits, - typename GemmTileTraitsHelperA::SharedLoadTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared, - Index_, - typename GemmTileTraitsHelperA::WmmaMatrix, - IteratorFragment::kWmmaMatrix> - SharedLoadIteratorA; - /// The stream to load A from shared memory. - typedef SharedLoadStream<SharedLoadIteratorA> SharedLoadStreamA; - /// The iterator to load B from shared memory. - typedef TileLoadIterator<typename GemmTileTraitsHelperB::SharedLoadTileTraits, - typename GemmTileTraitsHelperB::SharedLoadTileTraits::Scalar, - IteratorAdvance::kH, - MemorySpace::kShared, - Index_, - typename GemmTileTraitsHelperB::WmmaMatrix, - IteratorFragment::kWmmaMatrix> - SharedLoadIteratorB; - /// The stream to load B from shared memory. - typedef SharedLoadStream<SharedLoadIteratorB> SharedLoadStreamB; - - /// The functor to do the multiply-add in the main loop. - typedef typename GemmConfig::MultiplyAdd MultiplyAdd; - /// The object to clear accumulators. - typedef ClearAccumulators<typename MultiplyAdd::ScalarC> ClearAccumulators; - - /// The helper to create the epilogue traits. - typedef WmmaGemmEpilogueTraitsHelper<GemmConfig, EpilogueFunctor_, Index_> EpilogueTraitsHelper; - /// The traits class for the epilogue. - typedef SimplifiedGemmEpilogueTraits<GemmConfig, EpilogueFunctor_, Index_, EpilogueTraitsHelper> - GemmEpilogueTraits; - /// The epilogue. - typedef GemmEpilogue<GemmEpilogueTraits> Epilogue; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename OutputTile_, typename DefaultShape_ = Shape<64, 32, 64> > -struct WmmaGemmAccumulatorsPerWarp { - typedef typename ShapeMin<OutputTile_, DefaultShape_>::Shape Shape; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template < - /// The layout for A. - MatrixLayout::Kind kLayoutA_, - /// The layout for B. - MatrixLayout::Kind kLayoutB_, - /// The tile size for the GEMM KxNxM. - typename OutputTile_ = Shape<64, 128, 128>, - /// The output type. - typename ScalarC_ = float, - /// The functor to do the math in the epilogue. - typename EpilogueFunctor_ = LinearScaling<ScalarC_>, - /// The accumulator type. - typename Accumulator_ = ScalarC_, - /// The number of accumulators per warp. - typename AccumulatorsPerWarp_ = typename WmmaGemmAccumulatorsPerWarp<OutputTile_>::Shape, - /// The shape of the WMMA instruction. - typename InstructionShape_ = Shape<16, 16, 16>, - /// The number of scalars per LDG for A. - int kScalarsPerLdgA_ = 8, - /// The number of scalars per LDG for B. - int kScalarsPerLdgB_ = 8, - /// The index. - typename Index_ = int, - /// The helper class. - typename Helper_ = WmmaGemmTraitsHelper<kLayoutA_, - kLayoutB_, - OutputTile_, - ScalarC_, - Accumulator_, - EpilogueFunctor_, - AccumulatorsPerWarp_, - InstructionShape_, - kScalarsPerLdgA_, - kScalarsPerLdgB_, - Index_> > -struct WmmaGemmTraits : public GemmTraits< - // The config. - typename Helper_::GemmConfig, - // The stream to load A from global memory to shared memory. - typename Helper_::GlobalLoadStreamA, - // The stream to load B from global memory to shared memory. - typename Helper_::GlobalLoadStreamB, - // The stream to load A from shared memory. - typename Helper_::SharedLoadStreamA, - // The stream to load B from shared memory. - typename Helper_::SharedLoadStreamB, - // The epilogue. - typename Helper_::Epilogue, - // The block swizzle to reorganize the grid. - IdentityBlockSwizzle, - // The index. - Index_, - // The tool used to clear accumulators. - typename Helper_::ClearAccumulators> {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace gemm -} // namespace cutlass - -#endif // defined CUTLASS_USE_WMMA_API diff --git a/cutlass-example/cutlass/iterator_access.h b/cutlass-example/cutlass/iterator_access.h deleted file mode 100644 index e94beb7..0000000 --- a/cutlass-example/cutlass/iterator_access.h +++ /dev/null @@ -1,318 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Free functions for loading and storing to implementations of tile iteartor concepts. -*/ -#pragma once - -#include <cutlass/fragment_load_store.h> -#include <cutlass/load_store.h> -#include <cutlass/predicate_vector.h> -#include <cutlass/shape.h> - -namespace cutlass { - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Loads a fragment from an input iterator -template <typename InputIterator, typename Fragment> -CUTLASS_HOST_DEVICE void iterator_load(InputIterator &iterator, Fragment &fragment) { - typename InputIterator::FragmentIterator frag_iterator(fragment); - for (int d = 0; d < InputIterator::Iterations::kD; ++d) { - for (int h = 0; h < InputIterator::Iterations::kH; ++h) { - for (int w = 0; w < InputIterator::Iterations::kW; ++w) { - for (int c = 0; c < InputIterator::Iterations::kC; ++c) { - if (iterator.valid(d, h, w, c)) { - iterator.get(reinterpret_cast<typename InputIterator::AccessType &>( - frag_iterator.at(d, h, w, c)), - d, - h, - w, - c); - } - } - if (w < InputIterator::Iterations::kW - 1) { - iterator.inc_w(); - } - } - if (h < InputIterator::Iterations::kH - 1) { - iterator.inc_h(); - } - } - if (d < InputIterator::Iterations::kD - 1) { - iterator.inc_d(); - } - } - iterator.inc_advance(); -} - -/// Loads a fragment from a shared memory input iterator -template <typename InputIterator, typename Fragment> -CUTLASS_DEVICE void shared_iterator_load(InputIterator &iterator, Fragment &fragment) { - typename InputIterator::FragmentIterator frag_iterator(fragment); - for (int d = 0; d < InputIterator::Iterations::kD; ++d) { - for (int h = 0; h < InputIterator::Iterations::kH; ++h) { - for (int w = 0; w < InputIterator::Iterations::kW; ++w) { - for (int c = 0; c < InputIterator::Iterations::kC; ++c) { - int const offset = - ComputeOffsetFromStrides<typename InputIterator::ImmediateOffsetStrides>::get( - d, h, w, c); - - FragmentLoad<InputIterator::kIteratorFragment, - InputIterator::Tile::kC, - typename InputIterator::Scalar, - InputIterator::kMemorySpace, - typename InputIterator::FragmentElement, - InputIterator::Tile::kW>::load(frag_iterator.at(d, h, w, c), - iterator.data(), - offset); - } - } - } - } -} - -/// Loads a fragment from a shared memory input iterator -template <typename InputIterator, typename Fragment> -CUTLASS_DEVICE void shared_iterator_load(InputIterator &iterator, Fragment &fragment, int d) { - typename InputIterator::FragmentIterator frag_iterator(fragment); - for (int h = 0; h < InputIterator::Iterations::kH; ++h) { - for (int w = 0; w < InputIterator::Iterations::kW; ++w) { - for (int c = 0; c < InputIterator::Iterations::kC; ++c) { - int const offset = - ComputeOffsetFromStrides<typename InputIterator::ImmediateOffsetStrides>::get( - d, h, w, c); - - FragmentLoad<InputIterator::kIteratorFragment, - InputIterator::Tile::kC, - typename InputIterator::Scalar, - InputIterator::kMemorySpace, - typename InputIterator::FragmentElement, - InputIterator::Tile::kW>::load(frag_iterator.at(0, h, w, c), - iterator.data(), - offset); - } - } - } -} - -/// Loads a fragment from an input iterator, masked by a predicate iterator -template <typename InputIterator, typename Fragment, typename ConstPredicateAdapter> -CUTLASS_HOST_DEVICE void iterator_load_post_increment(InputIterator &iterator, - Fragment &fragment, - typename InputIterator::Index offset, - ConstPredicateAdapter predicate_adapter) { - for (int d = 0; d < InputIterator::Iterations::kD; ++d, iterator.inc_d()) { - for (int h = 0; h < InputIterator::Iterations::kH; ++h, iterator.inc_h()) { - for (int w = 0; w < InputIterator::Iterations::kW; ++w, iterator.inc_w()) { - if (predicate_adapter.at(d, h, w, 0)) { - int idx = InputIterator::Tile::kC * - (w + InputIterator::Iterations::kW * (h + InputIterator::Iterations::kH * d)); - - Load<typename Fragment::Element, InputIterator::Tile::kC, InputIterator::kMemorySpace>:: - load(reinterpret_cast<typename InputIterator::AccessType &>(fragment[idx]), - iterator.data(), - offset); - } - } - } - } -} - -/// Loads a fragment from an input iterator -template <typename InputIterator, typename Fragment> -CUTLASS_HOST_DEVICE void iterator_load_post_increment(InputIterator &iterator, - Fragment &fragment, - typename InputIterator::Index offset = 0) { - TrivialPredicateTileAdapter pred; - iterator_load_post_increment(iterator, fragment, offset, pred); -} - -/// Loads a fragment from an input iterator -template <typename InputIterator, typename Fragment, typename ConstPredicateAdapter> -CUTLASS_HOST_DEVICE void iterator_load_post_increment(InputIterator &iterator, - Fragment &fragment, - ConstPredicateAdapter pred_it) { - iterator_load_post_increment(iterator, fragment, 0, pred_it); -} - -template <typename InputIterator, typename Fragment, typename ConstPredicateAdapter> -CUTLASS_HOST_DEVICE void iterator_load(InputIterator const &_iterator, - Fragment &fragment, - typename InputIterator::Index offset, - ConstPredicateAdapter predicate_adapter) { - InputIterator iterator(_iterator); - iterator_load_post_increment(iterator, fragment, offset, predicate_adapter); -} - -/// Loads a fragment from an input iterator -template <typename InputIterator, typename Fragment> -CUTLASS_HOST_DEVICE void iterator_load(InputIterator const &iterator, - Fragment &fragment, - typename InputIterator::Index offset = 0) { - TrivialPredicateTileAdapter pred; - iterator_load(iterator, fragment, offset, pred); -} - -/// Loads a fragment from an input iterator -template <typename InputIterator, typename Fragment, typename ConstPredicateAdapter> -CUTLASS_HOST_DEVICE void iterator_load(InputIterator const &iterator, - Fragment &fragment, - ConstPredicateAdapter pred_it) { - iterator_load(iterator, fragment, 0, pred_it); -} - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Stores a fragment to an output iterator -template <typename OutputIterator, typename Fragment> -CUTLASS_HOST_DEVICE void iterator_store(OutputIterator &iterator, Fragment &fragment) { - typename OutputIterator::FragmentIterator frag_iterator(fragment); - for (int d = 0; d < OutputIterator::Iterations::kD; ++d) { - for (int h = 0; h < OutputIterator::Iterations::kH; ++h) { - for (int w = 0; w < OutputIterator::Iterations::kW; ++w) { - if (iterator.valid(d, h, w, 0)) { - iterator.set(reinterpret_cast<typename OutputIterator::AccessType const &>( - frag_iterator.at(d, h, w, 0)), - d, - h, - w, - 0); - } - if (w < OutputIterator::Iterations::kW - 1) { - iterator.inc_w(); - } - } - if (h < OutputIterator::Iterations::kH - 1) { - iterator.inc_h(); - } - } - if (d < OutputIterator::Iterations::kD - 1) { - iterator.inc_d(); - } - } - iterator.inc_advance(); -} - -/// Stores a fragment to a shared memory output iterator -template <typename OutputIterator, typename Fragment> -CUTLASS_DEVICE void shared_iterator_store(OutputIterator &iterator, Fragment const &fragment) { - typename OutputIterator::FragmentConstIterator frag_iterator(fragment); - for (int d = 0; d < OutputIterator::Iterations::kD; ++d) { - for (int h = 0; h < OutputIterator::Iterations::kH; ++h) { - for (int w = 0; w < OutputIterator::Iterations::kW; ++w) { - for (int c = 0; c < OutputIterator::Iterations::kC; ++c) { - int const offset = - ComputeOffsetFromStrides<typename OutputIterator::ImmediateOffsetStrides>::get( - d, h, w, c); - - FragmentStore<OutputIterator::kIteratorFragment, - OutputIterator::Tile::kC, - typename OutputIterator::Scalar, - OutputIterator::kMemorySpace, - typename OutputIterator::FragmentElement, - OutputIterator::Tile::kW>::store(frag_iterator.at(d, h, w, c), - iterator.data(), - offset); - } - } - } - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Stores a fragment to an output iterator, masked by a predicate iterator -template <typename OutputIterator, typename Fragment, typename ConstPredicateAdapter> -CUTLASS_HOST_DEVICE void iterator_store_post_increment(OutputIterator &iterator, - Fragment const &fragment, - typename OutputIterator::Index offset, - ConstPredicateAdapter predicate_adapter) { - for (int d = 0; d < OutputIterator::Iterations::kD; ++d, iterator.inc_d()) { - for (int h = 0; h < OutputIterator::Iterations::kH; ++h, iterator.inc_h()) { - for (int w = 0; w < OutputIterator::Iterations::kW; ++w, iterator.inc_w()) { - if (predicate_adapter.at(d, h, w, 0)) { - int idx = OutputIterator::Tile::kC * - (w + OutputIterator::Iterations::kW * (h + OutputIterator::Iterations::kH * d)); - - Store<typename Fragment::Element, - OutputIterator::Tile::kC, - OutputIterator::kMemorySpace>:: - store(reinterpret_cast<typename OutputIterator::AccessType const &>(fragment[idx]), - iterator.data(), - offset); - } - } - } - } -} - -/// Stores a fragment to an output iterator -template <typename OutputIterator, typename Fragment> -CUTLASS_HOST_DEVICE void iterator_store_post_increment(OutputIterator &iterator, - Fragment const &fragment, - typename OutputIterator::Index offset = 0) { - TrivialPredicateTileAdapter pred; - iterator_store_post_increment(iterator, fragment, offset, pred); -} - -/// Stores a fragment to an output iterator -template <typename OutputIterator, typename Fragment, typename ConstPredicateAdapter> -CUTLASS_HOST_DEVICE void iterator_store_post_increment(OutputIterator &iterator, - Fragment const &fragment, - ConstPredicateAdapter pred_it) { - iterator_store_post_increment(iterator, fragment, 0, pred_it); -} - -/// Stores a fragment to an output iterator, masked by a predicate iterator -template <typename OutputIterator, typename Fragment, typename ConstPredicateAdapter> -CUTLASS_HOST_DEVICE void iterator_store(OutputIterator const &_iterator, - Fragment const &fragment, - typename OutputIterator::Index offset, - ConstPredicateAdapter predicate_adapter) { - OutputIterator iterator(_iterator); - iterator_store_post_increment(iterator, fragment, offset, predicate_adapter); -} - -/// Stores a fragment to an output iterator -template <typename OutputIterator, typename Fragment> -CUTLASS_HOST_DEVICE void iterator_store(OutputIterator const &iterator, - Fragment const &fragment, - typename OutputIterator::Index offset = 0) { - TrivialPredicateTileAdapter pred; - iterator_store(iterator, fragment, offset, pred); -} - -/// Stores a fragment to an output iterator -template <typename OutputIterator, typename Fragment, typename ConstPredicateAdapter> -CUTLASS_HOST_DEVICE void iterator_store(OutputIterator const &iterator, - Fragment const &fragment, - ConstPredicateAdapter pred_it) { - iterator_store(iterator, fragment, 0, pred_it); -} - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/load_store.h b/cutlass-example/cutlass/load_store.h deleted file mode 100644 index 5cb5eb6..0000000 --- a/cutlass-example/cutlass/load_store.h +++ /dev/null @@ -1,222 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines abstractions for efficiently loading and storing vectors to memory. -*/ -#pragma once - -#include <cutlass/vector.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief Enum to specify which memory space data resides in. -*/ -struct MemorySpace { - enum Kind { - kGeneric, // Data accessed through pointer dereferencing - kShared, // Data resides in shared memory - kGlobal // Data resides in global memory - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, - int Lanes_, - MemorySpace::Kind Memory_, - bool = (Lanes_ > 1), - size_t = (sizeof(Scalar_) * Lanes_)> -struct Load { - /// The output type. - typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; - - /// The load function. - static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) { - dst = reinterpret_cast<AccessType const*>(&pointer[offset])[0]; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_> -struct Load<Scalar_, Lanes_, Memory_, true, 4> { - /// The output type. - typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; - - /// The store function. - static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) { - dst.registers[0] = reinterpret_cast<uint32_t const*>(&pointer[offset])[0]; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_> -struct Load<Scalar_, Lanes_, Memory_, true, 8> { - /// The output type. - typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; - - /// The store function. - static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) { - uint2 tmp = reinterpret_cast<uint2 const*>(&pointer[offset])[0]; - dst.registers[0] = tmp.x; - dst.registers[1] = tmp.y; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <MemorySpace::Kind Memory_> -struct Load<double, 2, Memory_, true, 16> { - /// The output type. - typedef typename Vectorize<double, 2>::Type AccessType; - - /// The store function. - static CUTLASS_DEVICE void load(AccessType& dst, double const* pointer, int offset) { - double2 tmp = reinterpret_cast<double2 const*>(&pointer[offset])[0]; - dst[0] = tmp.x; - dst[1] = tmp.y; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -#if defined(__CUDACC_VERSION_MAJOR) && __CUDACC_VERSION_MAJOR < 10 -// WAR bug in NVCC where the upper and lower half of the register end up being the same -template <MemorySpace::Kind Memory_> -struct Load<half, 8, Memory_, true, 16> { - /// The output type. - typedef typename Vectorize<half, 8>::Type AccessType; - - /// The store function. - static CUTLASS_DEVICE void load(AccessType& dst, half const* pointer, int offset) { - int2 tmp = reinterpret_cast<int2 const*>(&pointer[offset])[0]; - dst.registers[0] = tmp.x; - dst.registers[1] = tmp.y; - - tmp = reinterpret_cast<int2 const*>(&pointer[offset + 4])[0]; - dst.registers[2] = tmp.x; - dst.registers[3] = tmp.y; - } -}; - -#endif - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_> -struct Load<Scalar_, Lanes_, Memory_, true, 16> { - /// The output type. - typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; - - /// The store function. - static CUTLASS_DEVICE void load(AccessType& dst, Scalar_ const* pointer, int offset) { - uint4 tmp = reinterpret_cast<uint4 const*>(&pointer[offset])[0]; - dst.registers[0] = tmp.x; - dst.registers[1] = tmp.y; - dst.registers[2] = tmp.z; - dst.registers[3] = tmp.w; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, - int Lanes_, - MemorySpace::Kind Memory_, - bool = (Lanes_ > 1), - size_t = (sizeof(Scalar_) * Lanes_)> -struct Store { - /// The output type. - typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; - - /// The store function. - static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) { - pointer[offset] = src; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_> -struct Store<Scalar_, Lanes_, Memory_, true, 4> { - /// The output type. - typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; - - /// The store function. - static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) { - uint32_t* addr = reinterpret_cast<uint32_t*>(&pointer[offset]); - addr[0] = src.registers[0]; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_> -struct Store<Scalar_, Lanes_, Memory_, true, 8> { - /// The output type. - typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; - - /// The store function. - static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) { - uint2* addr = reinterpret_cast<uint2*>(&pointer[offset]); - addr[0] = make_uint2(src.registers[0], src.registers[1]); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <MemorySpace::Kind Memory_> -struct Store<double, 2, Memory_, true, 16> { - /// The output type. - typedef typename Vectorize<double, 2>::Type AccessType; - - /// The store function. - static CUTLASS_DEVICE void store(AccessType const& src, double* pointer, int offset) { - double2* addr = reinterpret_cast<double2*>(&pointer[offset]); - addr[0] = make_double2(src[0], src[1]); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, int Lanes_, MemorySpace::Kind Memory_> -struct Store<Scalar_, Lanes_, Memory_, true, 16> { - /// The output type. - typedef typename Vectorize<Scalar_, Lanes_>::Type AccessType; - - /// The store function. - static CUTLASS_DEVICE void store(AccessType const& src, Scalar_* pointer, int offset) { - uint4* addr = reinterpret_cast<uint4*>(&pointer[offset]); - addr[0] = make_uint4(src.registers[0], src.registers[1], src.registers[2], src.registers[3]); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/matrix_traits.h b/cutlass-example/cutlass/matrix_traits.h deleted file mode 100644 index 77e8b70..0000000 --- a/cutlass-example/cutlass/matrix_traits.h +++ /dev/null @@ -1,48 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines properties of matrices used to denote layout and operands to GEMM kernels. -*/ -#pragma once - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Describes layouts of matrices -struct MatrixLayout { - enum Kind { kRowMajor, kColumnMajor }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Gemm operand - D = A * B + C -struct GemmOperand { - enum Kind { kA, kB, kC, kD }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/predicate_vector.h b/cutlass-example/cutlass/predicate_vector.h deleted file mode 100644 index 8166857..0000000 --- a/cutlass-example/cutlass/predicate_vector.h +++ /dev/null @@ -1,493 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines container classes and iterators for managing a statically sized vector - of boolean predicates. -*/ -#pragma once - -#include <stdint.h> - -#include <cutlass/cutlass.h> -#include <cutlass/shape.h> - -#include <cutlass/util/platform.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/*!@defgroup predicate_vector_concept Predicate Vector Concept -@{ - -Implementations of \ref predicate_vector_concept contain an ordered set of boolean predicates which -may be used as conditionals in other device-side operations. Both random access and iterators -offering sequential access are provided. - -@par Predicate Vector - A \ref predicate_vector_concept satisfies the following expressions - - <b>at(int idx)</b> - returns the value of the indexed predicate - - <b>set(int idx, bool value)</b> - sets the value of the indexed predicate - - <b>begin()</b> - returns a \ref predicate_iterator_concept pointing to the first predicate - -@} -*/ - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/*!@defgroup predicate_iterator_concept Predicate Iterator Concept -@{ - -Implementations of \ref predicate_iterator_concept enables accessing and traversing elements of a -bit vector. - -@par Const Predicate Iterator - A const \ref predicate_iterator_concept satisfies the following expressions - - <b>++it</b> increments the iterator to the next predicate - - <b>*it</b> returns the value of the currently pointed-to predicate - -@par Mutable Predicate Iterator - A \ref predicate_iterator_concept that is non-const <b>also</b> satisfies the following expressions - - <b>it.set(bool value)</b> sets the value of the currently pointed-to predicate - -@} -*/ - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/*!@defgroup predicate_tile_adapter Predicate Tile Adapter Concept -@{ - -Implementations of \ref predicate_tile_adapter provide a mapping between a the elements of a \ref -tile_traits_concept and a \ref predicate_vector_concept. - -@par Predicate Tile Adapter - A \ref predicate_tile_adapter satisfies the following expressions - - <b>at(int d, int h, int w, int c)</b> - returns the value of a predicate corresponding to the - access (d, h, w, c) within the tile. - -@} -*/ - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Statically sized array of bits implementing @concept{predicate_vector_concept}. -template < - /// Number of predicates conatined in predicate vector - int kPredicates_, - /// Number of predicates contained in each byte of internal storage - int kPredicatesPerByte_ = 4, - /// Location of first predicate within byte of internal storage - int kPredicateStart_ = 0> -struct PredicateVector { - /// Number of bits stored by the PredicateVector - static int const kPredicates = kPredicates_; - - /// Number of bits stored within each byte of the predicate bit vector - static int const kPredicatesPerByte = kPredicatesPerByte_; - - /// First bit withing each byte containing predicates - static int const kPredicateStart = kPredicateStart_; - - // Make sure no one tries to put more than 8 bits in a byte :) - static_assert(kPredicatesPerByte <= 8, "kPredicatesPerByte must fit within an actual byte"); - // Make sure the "offsetted" bits fit in one byte. - static_assert(kPredicateStart + kPredicatesPerByte < 8, - "The offsetted predicates must fit within an actual byte."); - - /// Storage type of individual elements - typedef uint32_t Storage; - - /// Number of bytes needed - static int const kBytes = (kPredicates + kPredicatesPerByte - 1) / kPredicatesPerByte; - - /// Number of storage elements needed - static int const kWordCount = (kBytes + sizeof(Storage) - 1) / sizeof(Storage); - - private: - // - // Data members - // - - /// Words of bit vector - Storage storageData[kWordCount]; - - // - // Methods - // - - /// Computes the word and bit corresponding to a logical predicate index - CUTLASS_HOST_DEVICE void computeStorageOffset(int &word, int &bit, int idx) const { - CUTLASS_ASSERT(idx < kPredicates); - - int byte = (idx / kPredicatesPerByte); - int bit_offset = (idx % kPredicatesPerByte); - - word = byte / sizeof(Storage); - int byte_offset = (byte % sizeof(Storage)); - - bit = byte_offset * 8 + bit_offset + kPredicateStart; - } - - /// Accesses a given word with optional assertions - CUTLASS_HOST_DEVICE Storage &storage(int word) { - CUTLASS_ASSERT(word < kWordCount); - return storageData[word]; - } - - /// Accesses a given word with optional assertions - CUTLASS_HOST_DEVICE Storage const &storage(int word) const { - CUTLASS_ASSERT(word < kWordCount); - return storageData[word]; - } - - public: - // - // Iterator - // - - /** - * @brief A const iterator implementing \ref predicate_iterator_concept enabling sequential - * read-only access to prediactes. - * @concept{predicate_iterator_concept} - */ - class ConstIterator { - /// Reference to PredicateVector instance - PredicateVector const &vec_; - - /// Index into PredicateVector - int bit_; - - public: - /// Copy constructor - CUTLASS_HOST_DEVICE - ConstIterator(ConstIterator const &it) : vec_(it.vec_), bit_(it.bit_) {} - - /// - CUTLASS_HOST_DEVICE - ConstIterator(PredicateVector const &_vec, int _start = 0) : vec_(_vec), bit_(_start) {} - - /// Pre-increment - CUTLASS_HOST_DEVICE - ConstIterator &operator++() { - ++bit_; - return *this; - } - - /// Pre-decrement - CUTLASS_HOST_DEVICE - ConstIterator &operator--() { - --bit_; - return *this; - } - - /// Post-increment - CUTLASS_HOST_DEVICE - ConstIterator operator++(int) { - ConstIterator ret(*this); - ret.bit_++; - return ret; - } - - /// Post-decrement - CUTLASS_HOST_DEVICE - ConstIterator operator--(int) { - ConstIterator ret(*this); - ret.bit_--; - return ret; - } - - /// Returns true if iterators point to the same bit - CUTLASS_HOST_DEVICE - bool operator==(ConstIterator const &it) const { return bit_ == it.bit_; } - - /// Returns false if iterators point to the same bit - CUTLASS_HOST_DEVICE - bool operator!=(ConstIterator const &it) const { return bit_ != it.bit_; } - - /// Dereferences iterator - CUTLASS_HOST_DEVICE - bool operator*() const { return vec_[bit_]; } - }; - - /** - * @brief An iterator implementing \ref predicate_iterator_concept enabling sequential - * read and write access to predicates. - * @concept{predicate_iterator_concept} - */ - class Iterator { - /// Reference to PredicateVector instance - PredicateVector &vec_; - - /// Index into PredicateVector - int bit_; - - public: - /// Copy constructor - CUTLASS_HOST_DEVICE - Iterator(Iterator const &it) : vec_(it.vec_), bit_(it.bit_) {} - - /// Constructs an iterator from a PredicateVector - CUTLASS_HOST_DEVICE - Iterator(PredicateVector &_vec, int _start = 0) : vec_(_vec), bit_(_start) {} - - /// Pre-increment - CUTLASS_HOST_DEVICE - Iterator &operator++() { - ++bit_; - return *this; - } - - /// Pre-decrement - CUTLASS_HOST_DEVICE - Iterator &operator--() { - --bit_; - return *this; - } - - /// Post-increment - CUTLASS_HOST_DEVICE - Iterator operator++(int) { - Iterator ret(*this); - ret.bit_++; - return ret; - } - - /// Post-decrement - CUTLASS_HOST_DEVICE - Iterator operator--(int) { - Iterator ret(*this); - ret.bit_--; - return ret; - } - - /// Returns true if iterators point to the same bit - CUTLASS_HOST_DEVICE - bool operator==(Iterator const &it) const { return bit_ == it.bit_; } - - /// Returns false if iterators point to the same bit - CUTLASS_HOST_DEVICE - bool operator!=(Iterator const &it) const { return bit_ != it.bit_; } - - /// Gets the bit at the pointed to location - CUTLASS_HOST_DEVICE - bool get() { return vec_[bit_]; } - - /// Dereferences iterator - CUTLASS_HOST_DEVICE - bool operator*() const { return vec_[bit_]; } - - /// Sets the bit at the pointed to location - CUTLASS_HOST_DEVICE - void set(bool value = true) { vec_.set(bit_, value); } - }; - - /// Iterator that always returns true - struct TrivialIterator { - /// Constructor - CUTLASS_HOST_DEVICE - TrivialIterator() {} - - /// Copy constructor - CUTLASS_HOST_DEVICE - TrivialIterator(Iterator const &it) {} - - /// Constructs an iterator from a PredicateVector - CUTLASS_HOST_DEVICE - TrivialIterator(PredicateVector const &_vec) {} - - /// Pre-increment - CUTLASS_HOST_DEVICE - TrivialIterator &operator++() { return *this; } - - /// Post-increment - CUTLASS_HOST_DEVICE - TrivialIterator operator++(int) { return *this; } - - /// Dereferences iterator - CUTLASS_HOST_DEVICE - bool operator*() const { return true; } - }; - - public: - // - // Methods - // - - /// Initialize the predicate vector - CUTLASS_HOST_DEVICE PredicateVector(bool value = true) { fill(value); } - - /// Fills all predicates with a given value - CUTLASS_HOST_DEVICE void fill(bool value = true) { - Storage item = (value ? ~Storage(0) : Storage(0)); - - CUTLASS_PRAGMA_UNROLL - for (int i = 0; i < kWordCount; ++i) { - storage(i) = item; - } - } - - /// Accesses a bit within the predicate vector. - CUTLASS_HOST_DEVICE bool operator[](int idx) const { return at(idx); } - - /// Accesses a bit within the predicate vector. - CUTLASS_HOST_DEVICE bool at(int idx) const { - int bit, word; - computeStorageOffset(word, bit, idx); - - return ((storage(word) >> bit) & 1); - } - - /// Set a bit within the predicate vector. - CUTLASS_HOST_DEVICE void set(int idx, bool value = true) { - int bit, word; - computeStorageOffset(word, bit, idx); - - Storage disable_mask = (~(Storage(1) << bit)); - Storage enable_mask = (Storage(value) << bit); - - storage(word) = ((storage(word) & disable_mask) | enable_mask); - } - - /// Computes the intersection of two identical predicate vectors. - CUTLASS_HOST_DEVICE PredicateVector &operator&=(PredicateVector const &predicates) { - CUTLASS_PRAGMA_UNROLL - for (int i = 0; i < kWordCount; ++i) { - storage(i) = (storage(i) & predicates.storage(i)); - } - return *this; - } - - /// Computes the union of two identical predicate vectors. - CUTLASS_HOST_DEVICE PredicateVector &operator|=(PredicateVector const &predicates) { - CUTLASS_PRAGMA_UNROLL - for (int i = 0; i < kWordCount; ++i) { - storage(i) = (storage(i) | predicates.storage(i)); - } - return *this; - } - - /// Returns true if entire predicate array is zero. - CUTLASS_HOST_DEVICE bool is_zero() const { - Storage mask(0); - for (int byte = 0; byte < sizeof(Storage); ++byte) { - Storage byte_mask = (((1 << kPredicatesPerByte) - 1) << kPredicateStart); - mask |= (byte_mask << (byte * 8)); - } - uint32_t result = 0; - for (int word = 0; word < kWordCount; ++word) { - result |= storage(word); - } - return result == 0; - } - - /// Returns an iterator to the start of the bit vector - CUTLASS_DEVICE - Iterator begin() { return Iterator(*this); } - - /// Returns an iterator - CUTLASS_DEVICE - Iterator end() { return Iterator(*this, kPredicates); } - - /// Returns a ConstIterator - CUTLASS_DEVICE - ConstIterator const_begin() const { return ConstIterator(*this); } - - /// Returns a ConstIterator - CUTLASS_DEVICE - ConstIterator const_end() const { return ConstIterator(*this, kPredicates); } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Always returns true predicate. -struct TrivialPredicateTileAdapter { - /// Ctor. - CUTLASS_HOST_DEVICE TrivialPredicateTileAdapter() {} - - /// The value at location (d, h, w, c). - CUTLASS_HOST_DEVICE bool at(int, int, int, int) const { return true; } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Adapter to enable random access to predicates via logical coordinate within a tile. -template <typename PredicateVector_, typename Iterations_> -struct PredicateTileAdapter { - /// The vector of predicates. - typedef PredicateVector_ PredicateVector; - /// The iterations. - typedef Iterations_ Iterations; - - private: - /// The predicates. - PredicateVector &predicates; - - public: - /// Ctor. - CUTLASS_DEVICE PredicateTileAdapter(PredicateVector &predicates_) : predicates(predicates_) {} - - /// Get the value at location (d, h, w, c). - CUTLASS_DEVICE bool at(int d, int h, int w, int c) const { - int const bit = ComputeOffsetFromShape<Iterations>::get(d, h, w, c); - return predicates.at(bit); - } - - /// Set the value at location (d, h, w, c). - CUTLASS_DEVICE void set(int d, int h, int w, int c, bool value) { - int const bit = ComputeOffsetFromShape<Iterations>::get(d, h, w, c); - predicates.set(bit, value); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Adapter to enable random access to predicates via logical coordinate within a tile. -template <typename PredicateVector_, typename Iterations_> -struct ConstPredicateTileAdapter { - /// The vector of predicates. - typedef PredicateVector_ PredicateVector; - /// The iterations. - typedef Iterations_ Iterations; - - private: - /// The predicates. - PredicateVector const &predicates; - - public: - /// Ctor. - CUTLASS_DEVICE ConstPredicateTileAdapter(PredicateVector const &predicates_) - : predicates(predicates_) {} - - /// Get the value at location (d, h, w, c). - CUTLASS_DEVICE bool at(int d, int h, int w, int c) const { - int const bit = ComputeOffsetFromShape<Iterations>::get(d, h, w, c); - return predicates.at(bit); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/reshape_tile.h b/cutlass-example/cutlass/reshape_tile.h deleted file mode 100644 index 55aebfc..0000000 --- a/cutlass-example/cutlass/reshape_tile.h +++ /dev/null @@ -1,58 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines a type for restructuring a tile. -*/ -#pragma once - -#include <cutlass/shape.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -// The following functor reshapes a tile of data. The goal is to have at least kAccessSize in -// the inner-most dimension. If the user respects that constraint, there is nothing to be done. If -// that's not the case, this functor will correct that and "extract" the right number of elements -// from the next dimension. - -template <typename Tile_, int kAccessSize_, bool = (Tile_::kC < kAccessSize_)> -struct ReshapeTile { - typedef Tile_ Tile; -}; - -template <typename Tile_, int kAccessSize_> -struct ReshapeTile<Tile_, kAccessSize_, true> { - // Make sure the W dimension of the tile is large enough. - static_assert(Tile_::kW >= kAccessSize_, "The W dimension is too small"); - // Make sure the dimension can be divided by the number of scalars. - static_assert(Tile_::kW % kAccessSize_ == 0, "Not supported"); - // Collapse the W dimension. - typedef Shape<Tile_::kD, Tile_::kH, Tile_::kW / kAccessSize_, kAccessSize_> Tile; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/shape.h b/cutlass-example/cutlass/shape.h deleted file mode 100644 index 4f6b222..0000000 --- a/cutlass-example/cutlass/shape.h +++ /dev/null @@ -1,305 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines Shape implementing the Layout concept for representing a 4D hypercube of objects. -*/ -#pragma once - -#include <cutlass/cutlass.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/*!@defgroup layout_concept Layout Concept -* @{ -* @par Implementations of \ref layout_concept are used to describe a cube with DxHxW elements and C -scalars per element. - A HxW slice of a cube is called an image and a cube consists of D images. -* -* @par Notations -* Let Layout be an implementation of the \ref layout_concept. -* -* @par Valid Expressions -* - <b>Layout::D</b> specifies the depth of a cube -* - <b>Layout::H</b> specifies the height of a cube -* - <b>Layout::W</b> specifies the height of a cube -* - <b>Layout::C</b> specifies the number of channels of each element in a cube -* - <b>Layout::W_c</b> specifies the number of scalars of each row in one image of a cube. -* - <b>Layout::H_w</b> specifies the number of elements in an image slice. -* - <b>Layout::H_w_c</b>_specifies the number of scalars in an image slice. -* - <b>Layout::D_h_w</b> specifies the number of elements in a cube. -* - <b>Layout::D_h_w_c</b> specifies the number of scalars in a cube. -* - <b>Layout::Strides</b> is a \ref layout_concept specifying the strides. -* @} -*/ - -/** -* @brief A Shape implementing \ref layout_concept describing the dimensions of a cube. -* @concept{layout_concept} -*/ -template <int kD_ = 1, int kH_ = 1, int kW_ = 1, int kC_ = 1> -struct Shape { - /// The depth of the cube. - static int const kD = kD_; - /// The height of the cube. - static int const kH = kH_; - /// The width of the cube. - static int const kW = kW_; - /// The number of scalars per element. - static int const kC = kC_; -}; - -/** -* @brief Compute derived counted of a \ref layout_concept based class -*/ -template <typename Shape> -struct ShapeCount { - /// The number of elements per row. - static int const kWc = Shape::kW * Shape::kC; - /// The number of pixels per image. - static int const kHw = Shape::kH * Shape::kW; - /// The number of elements per image. - static int const kHwc = Shape::kH * kWc; - /// The number of pixels per cube. - static int const kDhw = Shape::kD * kHw; - /// The number of elements in the 4D space. - static int const kDhwc = Shape::kD * kHwc; - /// The number of elements in the 4D space. - static int const kCount = kDhwc; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename A_, int kScale_> -struct ShapeScale { - typedef Shape<A_::kD * kScale_, A_::kH * kScale_, A_::kW * kScale_, A_::kC * kScale_> Shape; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename A_, typename B_> -struct ShapeAdd { - typedef Shape<A_::kD + B_::kD, A_::kH + B_::kH, A_::kW + B_::kW, A_::kC + B_::kC> Shape; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename A_, typename B_> -struct ShapeSub { - typedef Shape<A_::kD - B_::kD, A_::kH - B_::kH, A_::kW - B_::kW, A_::kC - B_::kC> Shape; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename A_, typename B_> -struct ShapeMul { - typedef Shape<A_::kD * B_::kD, A_::kH * B_::kH, A_::kW * B_::kW, A_::kC * B_::kC> Shape; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename A_, typename B_> -struct ShapeDiv { - typedef Shape<A_::kD / B_::kD, A_::kH / B_::kH, A_::kW / B_::kW, A_::kC / B_::kC> Shape; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename A_, typename B_> -struct ShapeMax { - typedef Shape<(A_::kD > B_::kD ? A_::kD : B_::kD), - (A_::kH > B_::kH ? A_::kH : B_::kH), - (A_::kW > B_::kW ? A_::kW : B_::kW), - (A_::kC > B_::kC ? A_::kC : B_::kC)> - Shape; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename A_, typename B_> -struct ShapeMin { - typedef Shape<(A_::kD < B_::kD ? A_::kD : B_::kD), - (A_::kH < B_::kH ? A_::kH : B_::kH), - (A_::kW < B_::kW ? A_::kW : B_::kW), - (A_::kC < B_::kC ? A_::kC : B_::kC)> - Shape; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Shape_, int kElementsPerAccess> -struct ShapeStrides { - typedef Shape<Shape_::kH * Shape_::kW * Shape_::kC, - Shape_::kW * Shape_::kC, - Shape_::kC, - kElementsPerAccess> - Shape; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief Compute the offset for the given coordinates in a cube -* @tparam A \ref layout_concept where each dimension of the cube specifies the corresponding stride. -*/ -template <typename Shape_> -struct ComputeOffsetFromShape { - static CUTLASS_DEVICE int get(int d, int h, int w, int c) { - // clang-format off - return d * Shape_::kH * Shape_::kW * Shape_::kC + - h * Shape_::kW * Shape_::kC + - w * Shape_::kC + - c; - // clang-format on - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief Compute the offset for the given coordinates in a cube with a depth of 1 -* @tparam kSh Elements in the H dimension -* @tparam kSw Elements in the W dimension -* @tparam kSc Separation between two elements in "elements" -*/ -template <int kSh_, int kSw_, int kSc_> -struct ComputeOffsetFromShape<Shape<1, kSh_, kSw_, kSc_> > { - static CUTLASS_DEVICE int get(int d, int h, int w, int c) { - return h * kSw_ * kSc_ + w * kSc_ + c; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief Compute the offset for the given coordinates in a cube with one channel and a depth of 1 -* @tparam kSh Elements in the H dimension -* @tparam kSw Elements in the W dimension -*/ -template <int kSh_, int kSw_> -struct ComputeOffsetFromShape<Shape<1, kSh_, kSw_, 1> > { - static CUTLASS_DEVICE int get(int d, int h, int w, int c) { return h * kSw_ + w; } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief Compute the offset for the given coordinates in a cube -* @tparam A \ref layout_concept where each dimension of the cube specifies the corresponding stride. -*/ -template <typename Strides_> -struct ComputeOffsetFromStrides { - static CUTLASS_DEVICE int get(int d, int h, int w, int c) { - return d * Strides_::kD + h * Strides_::kH + w * Strides_::kW + c * Strides_::kC; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief Compute the offset for the given coordinates in a cube with a depth of 1 -* @tparam S_h Stride in the H dimension in scalars -* @tparam S_w Stride in the W dimension in scalars -* @tparam S_c Stride between two scalars. -*/ -template <int S_h_, int S_w_, int S_c_> -struct ComputeOffsetFromStrides<Shape<1, S_h_, S_w_, S_c_> > { - static CUTLASS_DEVICE int get(int d, int h, int w, int c) { - return h * S_h_ + w * S_w_ + c * S_c_; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief Compute the offset for the given coordinates in a cube with one channel and a depth of 1 -* @tparam S_h Stride in the H dimension in scalars -* @tparam S_w Stride in the W dimension in scalars -*/ -template <int S_h_, int S_w_> -struct ComputeOffsetFromStrides<Shape<1, S_h_, S_w_, 1> > { - static CUTLASS_DEVICE int get(int d, int h, int w, int c) { return h * S_h_ + w * S_w_; } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief Decompose threadId.x into coordinate of a cube whose dimensions are specified by Threads_. -* Afterwards compute the offset of those coordinates using Strides_ -* @tparam Threads_ The dimension of the cube the threadIdx.x value is mapped on -* @tparam Strides_ The strides to use when compute the offsets based on the coordinates of the cube. -*/ -template <typename Threads_, typename Strides_> -struct ComputeThreadOffsetFromStrides { - static CUTLASS_DEVICE int get() { - // Decompose the thread index. - int c = threadIdx.x % Threads_::kC; - int w = threadIdx.x / Threads_::kC % Threads_::kW; - int h = threadIdx.x / Threads_::kC / Threads_::kW % Threads_::kH; - int d = threadIdx.x / Threads_::kC / Threads_::kW / Threads_::kH; - - // Compute the offset. - return d * Strides_::kD + h * Strides_::kH + w * Strides_::kW + c * Strides_::kC; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// -/** -*@brief Specialization for D=1 -*/ -template <int T_h_, int T_w_, int T_c_, int S_h_, int S_w_, int S_c_> -struct ComputeThreadOffsetFromStrides<Shape<1, T_h_, T_w_, T_c_>, Shape<1, S_h_, S_w_, S_c_> > { - static CUTLASS_DEVICE int get() { - // Decompose the thread index. - int c = threadIdx.x % T_c_; - int w = threadIdx.x / T_c_ % T_w_; - int h = threadIdx.x / T_c_ / T_w_ % T_h_; - - // Compute the offset. - return h * S_h_ + w * S_w_ + c * S_c_; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -*@brief Specialization for D=1 and C=1 -*/ -template <int T_h_, int T_w_, int S_h_, int S_w_> -struct ComputeThreadOffsetFromStrides<Shape<1, T_h_, T_w_, 1>, Shape<1, S_h_, S_w_, 1> > { - static CUTLASS_DEVICE int get() { - // Decompose the thread index. - int w = threadIdx.x % T_w_; - int h = threadIdx.x / T_w_; - - // Compute the offset. - return h * S_h_ + w * S_w_; - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/tensor_ref.h b/cutlass-example/cutlass/tensor_ref.h deleted file mode 100644 index 8ef31e3..0000000 --- a/cutlass-example/cutlass/tensor_ref.h +++ /dev/null @@ -1,151 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines a structure containing strides, bounds, and a pointer to tensor data. -*/ -#pragma once - -#include <typeinfo> - -#include <cutlass/coord.h> -#include <cutlass/cutlass.h> -#include <cutlass/vector.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Structure modeling a pointer and stride into a tensor -template <typename Storage_, int Rank_> -class TensorRef { - public: - /// Data type of individual access - typedef Storage_ Storage; - - /// Rank of tensor - static int const Rank = Rank_; - - private: - // - // Data members - // - - /// Pointer to storage element - Storage* ptr_; - - /// Stride information - Coord<Rank> stride_; - - public: - // - // Methods - // - - /// Default ctor - CUTLASS_HOST_DEVICE - TensorRef() : ptr_(nullptr) {} - - /// Constructs from a pointer, size, and stride - CUTLASS_HOST_DEVICE - TensorRef(Storage* ptr, Coord<Rank> stride) : ptr_(ptr), stride_(stride) {} - - /// Updates the pointer, stride, and location within a TensorRef - CUTLASS_HOST_DEVICE - void reset(Storage* ptr = nullptr, Coord<Rank> stride = Coord<Rank>(0)) { - ptr_ = ptr; - stride_ = stride; - } - - /// Conversion function - template <typename T> - TensorRef<T, Rank> convert() { - Coord<Rank> converted_stride; - for (int i = 0; i < Rank - 1; ++i) { - converted_stride[i] = stride_[i] * Extent<Storage>::kValue / Extent<T>::kValue; - } - converted_stride[Rank - 1] = stride_[Rank - 1]; - - return TensorRef<T, Rank>(reinterpret_cast<T*>(ptr_), converted_stride); - } - - /// Returns true if the TensorRef may be safely accessed - CUTLASS_HOST_DEVICE - bool good() const { return ptr_ != nullptr; } - - /// Returns the pointer to referenced data - CUTLASS_HOST_DEVICE - Storage* data() const { return ptr_; } - - /// Returns the stride of the tensor - CUTLASS_HOST_DEVICE - Coord<Rank> const& stride() const { return stride_; } - - /// Returns the stride of the tensor in the given dimension - CUTLASS_HOST_DEVICE - int const& stride(int dim) const { return stride_.at(dim); } - - /// Returns the maximum stride element as the 'leading dimension' - CUTLASS_HOST_DEVICE - int leading_dim() const { return __NV_STD_MAX(stride_[1], stride_[2]); } - - /// Computes the offset of an index from the origin of the tensor - CUTLASS_HOST_DEVICE - long long offset(Coord<Rank> const& coord) const { - return stride_.template dot<long long>(coord); - } - - /// Returns a reference to the element at a given Coord - CUTLASS_HOST_DEVICE - Storage& at(Coord<Rank> const& coord) const { return ptr_[offset(coord)]; } - - /// Element-wise accessor - Storage& operator[](Coord<Rank> const& coord) const { return at(coord); } - - /// Returns a reference to the element at a given Coord - CUTLASS_HOST_DEVICE - Storage& at(int idx) const { return ptr_[idx]; } - - /// Element-wise accessor - Storage& operator[](int idx) const { return at(idx); } - - /// Adds an offset to the pointer - CUTLASS_HOST_DEVICE - TensorRef& advance(Coord<Rank> const& b) { - ptr_ += offset(b); - return *this; - } - - /// Returns a TensorRef offset by a given amount - CUTLASS_HOST_DEVICE - TensorRef operator+(Coord<Rank> const& b) const { return TensorRef(ptr_ + offset(b), stride_); } - - /// Returns a TensorRef offset by a given amount - CUTLASS_HOST_DEVICE - TensorRef operator-(Coord<Rank> const& b) const { return TensorRef(ptr_ - offset(b), stride_); } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/tensor_view.h b/cutlass-example/cutlass/tensor_view.h deleted file mode 100644 index 89c6bd5..0000000 --- a/cutlass-example/cutlass/tensor_view.h +++ /dev/null @@ -1,172 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines a structure containing strides and a pointer to tensor data. -*/ - -#pragma once - -#include <cmath> - -#include <cutlass/cutlass.h> -#include <cutlass/tensor_ref.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Host-side reference implementation of tensor operations -template <typename T> -class TensorView : public TensorRef<T, 4> { - public: - /// Reference and stride - typedef TensorRef<T, 4> Base; - - /// Reference and stride - typedef Base TensorRef_t; - - /// Reference to constant type - typedef TensorRef<T const, 4> ConstTensorRef_t; - - /// Rank of tensor - static int const Rank = TensorRef_t::Rank; - - /// Type used to compute the offset of an element to the base of a tensor - typedef int Offset_t; - - /// Coordinate into tensor - typedef Coord<Rank> Coord_t; - - private: - // - // Data members - // - - /// Pointer to pitch-linear memory - TensorRef_t ref_; - - /// Dimensions of coordinate (independent of stride) - Coord_t size_; - - public: - // - // Device and Host Methods - // - - /// Default constructor - CUTLASS_HOST_DEVICE - TensorView() {} - - /// Constructs a Tensor_view from a TensorRef and size - CUTLASS_HOST_DEVICE - TensorView(TensorRef_t const& _ref, Coord_t const& _size) : Base(_ref), size_(_size) {} - - /// Returns true if the Tensor_view is bound to some memory - CUTLASS_HOST_DEVICE - bool good() const { return ref().good(); } - - /// Returns a pointer to data - CUTLASS_HOST_DEVICE - T* data() const { return ref().data(); } - - /// Updates the reference and size of a Tensor_view object - CUTLASS_HOST_DEVICE - void reset(TensorRef_t const& _ref = TensorRef_t(0), Coord_t const& _size = Coord_t()) { - Base::operator=(_ref); - size_ = _size; - } - - /// Accesses the tensor reference pointing to data - CUTLASS_HOST_DEVICE - TensorRef_t& ref() { return *this; } - - /// - CUTLASS_HOST_DEVICE - ConstTensorRef_t const_ref() { return ConstTensorRef_t(data(), stride()); } - - /// Accesses the tensor reference pointing to data - CUTLASS_HOST_DEVICE - TensorRef_t const& ref() const { return *this; } - - /// Accesses the size - CUTLASS_HOST_DEVICE - Coord_t const& size() const { return size_; } - - /// Accesses the size - CUTLASS_HOST_DEVICE - int size(int dim) const { return size_.at(dim); } - - /// Accesses the stride - CUTLASS_HOST_DEVICE - Coord_t const& stride() const { return ref().stride(); } - - /// Accesses the stride - CUTLASS_HOST_DEVICE - int const& stride(int dim) const { return ref().stride(dim); } - - /// Assigns the Tensor_view - CUTLASS_HOST_DEVICE - TensorView& operator=(TensorView const& _tensor) { - Base::operator=(_tensor._ref); - size_ = _tensor.size_; - return *this; - } - - /// Returns the index of an element - CUTLASS_HOST_DEVICE - Offset_t offset(Coord_t const& coord) const { return ref().offset(coord); } - - /// Determines whether a location is within a tensor - CUTLASS_HOST_DEVICE - bool contains(Coord_t const& coord) const { - for (int dim = 0; dim < Rank; ++dim) { - if (coord.at(dim) >= size_.at(dim)) { - return false; - } - } - return true; - } - - /// Element-wise accessor - CUTLASS_HOST_DEVICE - T& at(Coord_t const& coord) const { return ref().at(coord); } - - /// Element-wise accessor - T& operator[](Coord<Rank> const& coord) const { return at(coord); } - - /// Element-wise accessor - CUTLASS_HOST_DEVICE - T& at(Offset_t idx) const { return ref().at(idx); } - - /// Returns a Tensor_view given location and size quantities - CUTLASS_HOST_DEVICE - TensorView<T> subview(Coord_t const& location, Coord_t size) const { - return TensorView<T>(ref() + location, size.clamp(size_ - location)); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/tile_iterator.h b/cutlass-example/cutlass/tile_iterator.h deleted file mode 100644 index 5d39c4f..0000000 --- a/cutlass-example/cutlass/tile_iterator.h +++ /dev/null @@ -1,899 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines the Tile Traits concept and iterators for loading and storing to tiles - efficiently. -*/ -#pragma once - -#include <cutlass/fragment.h> -#include <cutlass/load_store.h> -#include <cutlass/predicate_vector.h> -#include <cutlass/vector.h> - -namespace cutlass { - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/*!@defgroup tile_traits_concept Tile Traits Concept -@{ - -\ref tile_traits_concept is a type definining the shape of a tile and the distribution of accesses -by individual entities, either threads or other. - -@par Tile Traits Concept - Types satisfying \ref tile_traits_concept define the following members - - <b>Tile</b> - a type satisfying \ref layout_concept describing the dimensions of the tile - - <b>Delta</b> - a type satisfying \ref layout_concept describing the increments between accesses -along each dimension - - <b>Iterations</b> - a type satisfying \ref layout_concept describing the number of accesses -along each dimension - - <b>Offset</b> - the type of a <i>functor</i> computing the offset of each participating entity -as a Coord<4>. -@} -*/ - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Specifies dimension in which post-increment accesses advance -struct IteratorAdvance { - enum Kind { kD, kH, kW }; -}; - -/// Specifies whether iterator storage fragment consists of Scalar values or WMMA matrix -struct IteratorFragment { - enum Kind { kScalar, kWmmaMatrix }; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief A template defining \ref tile_traits_concept -* @concept{tile_traits_concept} -*/ -template <typename Tile_, - typename Delta_, - typename Iterations_, - typename ThreadOffset_, - int kAccessSize> -struct TileTraits { - /// Shape of the tile - typedef Tile_ Tile; - - /// Number of steps between accesses along each dimension - typedef Delta_ Delta; - - /// Number of accesses performed - typedef Iterations_ Iterations; - - /// Functor that returns the logical coordinate of each entity's initial offset in the tile - typedef ThreadOffset_ ThreadOffset; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Iterator for accessing a stripmined tile in memory -template <typename Traits_, - typename Scalar_, - IteratorAdvance::Kind Advance_ = IteratorAdvance::kH, - MemorySpace::Kind MemorySpace = MemorySpace::kGeneric, - typename Index_ = int, - typename FragmentElement_ = Scalar_, - IteratorFragment::Kind IteratorFragment_ = IteratorFragment::kScalar, - typename Skew_ = Shape<0, 0, 0, 0> > -struct TileIteratorBase { - /// concept TileTraits - typedef Traits_ Traits; - - /// Scalar element - typedef Scalar_ Scalar; - - /// Fragment element - typedef FragmentElement_ FragmentElement; - - /// Specifies dimension in which post-increment accesses advance. - static IteratorAdvance::Kind const kAdvance = Advance_; - - /// Specifies iterator storage fragment type (Scalar or WmmaMatrix) - static IteratorFragment::Kind const kIteratorFragment = IteratorFragment_; - - /// Source or destination memory space - static MemorySpace::Kind const kMemorySpace = MemorySpace; - - /// Index type - typedef Index_ Index; - - /// Skew quantity - typedef Skew_ Skew; - - /// Tile shape - typedef typename Traits::Tile Tile; - - /// Distance along each dimension - typedef typename Traits::Delta Delta; - - /// The strides in each dimension between different loads/stores. - typedef typename Traits::ImmediateOffsetStrides ImmediateOffsetStrides; - - /// Iterations - typedef typename Traits::Iterations Iterations; - - /// Thread offset - typedef typename Traits::ThreadOffset ThreadOffset; - - /// The number of scalars accessed per load/store. - static int const kAccessSize = Tile::kC; - - /// The elements loaded/store by one instruction. - typedef typename Vectorize<FragmentElement, kAccessSize>::Type AccessType; - - /// The size of storage needed per fragment - static int const kFragmentSize = - (kIteratorFragment == IteratorFragment::kWmmaMatrix ? 16 : sizeof(AccessType)); - /// The storage. - typedef Fragment<Scalar, ShapeCount<Tile>::kCount, kFragmentSize> Storage; - /// The fragment. - typedef Fragment<FragmentElement, ShapeCount<Iterations>::kCount * kAccessSize> Fragment; - /// The fragment iterator. - typedef FragmentIterator<Fragment, Iterations, AccessType> FragmentIterator; - /// The fragment const iterator. - typedef FragmentConstIterator<Fragment, Iterations, AccessType> FragmentConstIterator; - /// The shape of the fragment. - typedef typename FragmentIterator::FragmentShape FragmentShape; - - /// Default predicate mask type - typedef PredicateVector<ShapeCount<Iterations>::kCount> PredicateVector; - - // - // Params struct - // - - /// Parameters to the iterator - struct Params { - Index stride_d; - Index stride_h; - Index stride_w; - - Index inc_d; - Index inc_h; - Index inc_w; - - Index inc_advance; - - /// Initializes params - CUTLASS_HOST_DEVICE - int initialize(Index _stride_d, - Index _stride_h, - Index _stride_w, - Index _inc_d, - Index _inc_h, - Index _inc_w, - Index _inc_advance) { - stride_d = _stride_d; - stride_h = _stride_h; - stride_w = _stride_w; - - inc_d = _inc_d; - inc_h = _inc_h; - inc_w = _inc_w; - inc_advance = _inc_advance; - - return 0; - } - - CUTLASS_HOST_DEVICE - int initialize(Index _stride_d, Index _stride_h, Index _stride_w) { - stride_d = _stride_d; - stride_h = _stride_h; - stride_w = _stride_w; - - inc_w = stride_w * Delta::kW; - inc_h = stride_h * Delta::kH - stride_w * Delta::kW * (Iterations::kW - 1); - - if (kAdvance == IteratorAdvance::kH) { - // Advance in the H dimension. - inc_d = 0; - } else if (kAdvance == IteratorAdvance::kW) { - // Advance in the W dimension. - inc_d = stride_w * Tile::kW - stride_h * Tile::kH; - } else { - // Advance in the D dimension. - inc_d = stride_d; - } - - inc_advance = 0; - - return 0; - } - - CUTLASS_HOST_DEVICE int initialize() { - stride_d = 0; - stride_h = 0; - stride_w = 1; - - inc_d = inc_h = inc_w = inc_advance = 0; - - return 0; - } - }; - - /// Is the iterator valid? - CUTLASS_DEVICE bool valid(int d, int h, int w, int c) const { return true; } - - // - // Static function members - // - - /// Initializes a predicate vector - template <typename PredicateIterator> - CUTLASS_DEVICE static void initialize_predicates(PredicateIterator predicate_it, - Coord<3> const &bounds, - Coord<3> const &offset = make_Coord(0, 0, 0)) { - for (int d = 0; d < Iterations::kD; ++d) { - bool enable_d = (d * Delta::kD + offset[0] < bounds[0]); - for (int h = 0; h < Iterations::kH; ++h) { - bool enable_h = (h * Delta::kH + offset[1] < bounds[1]); - for (int w = 0; w < Iterations::kW; ++w) { - bool enable_w = (w * Tile::kC * Delta::kW + offset[2] < bounds[2]); - predicate_it.set(d, h, w, 0, enable_d && enable_h && enable_w); - } - } - } - } -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/*!@defgroup tile_load_iterator_concept Tile Load Iterator Concept -@{ - -\ref tile_load_iterator_concept enables loading a tile from addressable memory into a fragment - -@par Tile Load Iterator Concept - Types satisfying \ref tile_load_iterator_concept define the following members - - <b>PredicateVector</b> - a \ref predicate_vector_concept with sufficient predicate storage for -each access implied by the tile traits - - <b>Fragment</b> - the destination fragment type satisfying \ref fragment_concept - - <b>initialize_predicates(pred_it, bounds, block_offset)</b> - function initializing a predicate -vector according to externally specified bounds - - <b>load_post_increment(fragment, pred_it)</b> - a method that loads a fragment and increments -the iterator to the next tile, guarded by a \ref predicate_iterator_concept - - <b>load_post_increment(fragment)</b> - a method that loads a fragment and increments the -iterator to the next tile - - <b>load(fragment, pred_it)</b> - a const method that loads a fragment, guarded by a \ref -predicate_iterator_concept - - <b>load(fragment)</b> - a method that loads a fragment - -@} -*/ - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief An iterator implementing \ref tile_load_iterator_concept for loading a tile from memory -* @concept{tile_load_iterator_concept} -*/ -template <typename Traits_, - typename Scalar_, - IteratorAdvance::Kind Advance_ = IteratorAdvance::kH, - MemorySpace::Kind MemorySpace = MemorySpace::kGeneric, - typename Index_ = int, - typename FragmentElement_ = Scalar_, - IteratorFragment::Kind IteratorFragment_ = IteratorFragment::kScalar, - typename Skew_ = Shape<0, 0, 0, 0> > -struct TileLoadIterator : public TileIteratorBase<Traits_, - Scalar_, - Advance_, - MemorySpace, - Index_, - FragmentElement_, - IteratorFragment_, - Skew_> { - /// Base class - typedef TileIteratorBase<Traits_, - Scalar_, - Advance_, - MemorySpace, - Index_, - FragmentElement_, - IteratorFragment_, - Skew_> - Base; - - /// concept TileTraits - typedef typename Base::Traits Traits; - - /// Scalar element - typedef typename Base::Scalar Scalar; - - /// Fragment element - typedef typename Base::FragmentElement FragmentElement; - - /// Specifies in which dimension post-increment accesses advance. - static IteratorAdvance::Kind const kAdvance = Base::kAdvance; - - /// Specifies type of iterator fragment storage (Salar or WmmaMatrix) - static IteratorFragment::Kind const kIteratorFragment = Base::kIteratorFragment; - - /// Source or destination memory space - static MemorySpace::Kind const kMemorySpace = Base::kMemorySpace; - - /// Index type - typedef typename Base::Index Index; - - /// Skew quantity - typedef typename Base::Skew Skew; - - /// Tile shape - typedef typename Base::Tile Tile; - - /// Delta - typedef typename Base::Delta Delta; - - /// Iterations - typedef typename Base::Iterations Iterations; - - /// ThreadOffset functor - typedef typename Base::ThreadOffset ThreadOffset; - - /// Fragment type - typedef typename Base::FragmentShape FragmentShape; - - /// Memory access type - typedef typename Base::AccessType AccessType; - - /// Fragment definition - typedef typename Base::Fragment Fragment; - - /// Fragment iterator definition - typedef typename Base::FragmentIterator FragmentIterator; - - /// Fragment const iterator definition - typedef typename Base::FragmentConstIterator FragmentConstIterator; - - /// Default predicate mask type - typedef typename Base::PredicateVector PredicateVector; - - /// Storage object that may be loaded from - typedef typename Base::Storage SharedStorage; - - /// IteratorBase parameters - typedef typename Base::Params BaseParams; - - /// Do we require a fence? - enum { kRequiresLoadFence = Tile::kD == 1 }; - - /// The pointer type - typedef Scalar const *Pointer; - - /// Parameters - struct Params : public BaseParams { - /// Pointer to memory - Scalar const *pointer; - - /// Initialize params to access storage object - CUTLASS_HOST_DEVICE - int initialize(SharedStorage const &storage) { - pointer = &storage[0]; - return 0; - } - - /// Initializes params to access a raw pointer - CUTLASS_HOST_DEVICE - int initialize(Scalar const *ptr, Index stride_d, Index stride_h, Index stride_w) { - Base::Params::initialize(stride_d, stride_h, stride_w); - pointer = ptr; - return 0; - } - - /// Initializes params - CUTLASS_HOST_DEVICE - int initialize(Scalar const *ptr, - Index _stride_d, - Index _stride_h, - Index _stride_w, - Index _inc_d, - Index _inc_h, - Index _inc_w, - Index _inc_advance) { - pointer = ptr; - Base::Params::initialize( - _stride_d, _stride_h, _stride_w, _inc_d, _inc_h, _inc_w, _inc_advance); - return 0; - } - - // Initializes params to default values - CUTLASS_HOST_DEVICE - int initialize() { return Base::Params::initialize(); } - }; - - // - // Data members - // - - /// Parameters structure - Params params; - - /// Offset of an individual lane from the start of the tile - Coord<4> thread_offset; - - /// Stage argument enables wrapping after some number of tiles have been loaded. - int stage; - - // - // Static member functions - // - - /// Initializes a predicate vector - template <typename PredicateIterator> - CUTLASS_HOST_DEVICE void initialize_predicates(PredicateIterator predicate_it, - Coord<3> const &bounds, - Coord<3> const &block_offset = make_Coord(0, - 0, - 0)) { - Base::initialize_predicates( - predicate_it, - bounds, - block_offset + make_Coord(0, thread_offset[1], thread_offset[2] * Tile::kC)); - } - - // - // Methods - // - - /// Default constructor - CUTLASS_HOST_DEVICE - TileLoadIterator() {} - - /// Constructs a tile load iterator - CUTLASS_HOST_DEVICE - TileLoadIterator(Params const &_params, - Coord<3> const &block_offset = make_Coord(0, 0, 0), - ThreadOffset thread_offset_func = ThreadOffset()) - : params(_params), stage(0) { - thread_offset = thread_offset_func(); - - Index block_offset_h = 0; - Index block_offset_w = 0; - if (kAdvance == IteratorAdvance::kH) { - block_offset_h = block_offset[1]; - block_offset_w = block_offset[2]; - } else { - block_offset_h = block_offset[2]; - block_offset_w = block_offset[1]; - } - - params.pointer += block_offset[0] * params.stride_d + - (block_offset_h + thread_offset[1]) * params.stride_h + - (block_offset_w + thread_offset[2] * Tile::kC) / Tile::kC * params.stride_w; - } - - /// Constructs a tile load iterator - CUTLASS_HOST_DEVICE - TileLoadIterator(Params const &, - SharedStorage &shared_storage, - Coord<3> const &block_offset = make_Coord(0, 0, 0), - ThreadOffset thread_offset_func = ThreadOffset()) - : stage(0) { - int const offset = thread_offset_func()[2]; - params.pointer = &shared_storage[offset]; - } - - /// Returns the current pointer - CUTLASS_HOST_DEVICE - Scalar const *data() const { return params.pointer; } - - /// The accessor. - CUTLASS_DEVICE void get(AccessType &value, int d, int h, int w, int c) const { - int const imm = - ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(d, h, w, c); - Load<Scalar, Base::kAccessSize, kMemorySpace>::load(value, params.pointer, imm); - } - - /// Increment in the D dimension - CUTLASS_HOST_DEVICE void inc_d() { params.pointer += params.inc_d; } - - /// Increment in the H dimension - CUTLASS_HOST_DEVICE void inc_h() { params.pointer += params.inc_h; } - - /// Increment in the W dimension - CUTLASS_HOST_DEVICE void inc_w() { params.pointer += params.inc_w; } - - /// Increment in the next dimension - CUTLASS_HOST_DEVICE void inc_advance() { params.pointer += params.inc_advance; } - - /// Increment the stage. - CUTLASS_DEVICE void inc_stage() { - if (Tile::kD > 1) { - int const kStageSize = Tile::kH * Tile::kW * Tile::kC; - if (stage == Tile::kD - 1) { - params.pointer -= (Tile::kD - 1) * kStageSize; - stage = 0; - } else { - params.pointer += kStageSize; - stage = stage + 1; - } - } - } - - public: - /// Loads a fragment and advances the iterator to the next tile. - template <typename Fragment, typename PredicateIterator> - CUTLASS_HOST_DEVICE void load_post_increment(Fragment &fragment, PredicateIterator pred_it) { - FragmentIterator frag_iterator(fragment); - - for (int d = 0; d < Iterations::kD; ++d) { - for (int h = 0; h < Iterations::kH; ++h) { - for (int w = 0; w < Iterations::kW; ++w, ++pred_it) { - if (*pred_it) { - Load<typename Fragment::Element, Tile::kC, kMemorySpace>::load( - reinterpret_cast<AccessType &>(frag_iterator.at(d, h, w, 0)), data(), 0); - } - - if (w < Iterations::kW - 1) { - inc_w(); - } - } - if (h < Iterations::kH - 1) { - inc_h(); - } - } - if (d < Iterations::kD - 1) { - inc_d(); - } - } - inc_advance(); - } - - /// Loads a fragment and advances the iterator to the next tile. - template <typename Fragment> - CUTLASS_HOST_DEVICE void load_post_increment(Fragment &fragment) { - typename PredicateVector::TrivialIterator pred_it; - load_post_increment(fragment, pred_it); - } - - /// Loads a fragment without advancing the iterator.. - template <typename Fragment, typename PredicateIterator> - CUTLASS_HOST_DEVICE void load(Fragment &fragment, PredicateIterator pred_it) const { - TileLoadIterator _load_it(*this); - _load_it.load_post_increment(fragment, pred_it); - } - - /// Loads a fragment without advancing the iterator.. - template <typename Fragment> - CUTLASS_HOST_DEVICE void load(Fragment &fragment) const { - typename PredicateVector::TrivialIterator pred_it; - load(fragment, pred_it); - } -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/*!@defgroup tile_store_iterator_concept Tile Store Iterator Concept -@{ - -\ref tile_store_iterator_concept enables storing a tile to addressable memory - -@par Tile Store Iterator Concept - Types satisfying \ref tile_load_iterator_concept define the following members - - <b>PredicateVector</b> - a \ref predicate_vector_concept with sufficient predicate storage for -each access implied by the tile traits - - <b>Fragment</b> - the destination fragment type satisfying \ref fragment_concept - - <b>initialize_predicates(pred_it, bounds, block_offset)</b> - function initializing a predicate -vector according to externally specified bounds - - <b>store_post_increment(fragment, pred_it)</b> - a method that stores a fragment and increments -the iterator to the next tile, guarded by a \ref predicate_iterator_concept - - <b>store_post_increment(fragment)</b> - a method that stores a fragment and increments the -iterator to the next tile - - <b>store(fragment, pred_it)</b> - a const method that stores a fragment, guarded by a \ref -predicate_iterator_concept - - <b>store(fragment)</b> - a method that loads a fragment - -@} -*/ - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/** -* @brief An iterator implementing \ref tile_store_iterator_concept for storing a tile to memory -* @concept{tile_store_iterator_concept} -*/ -template <typename Traits_, - typename Scalar_, - IteratorAdvance::Kind Advance_ = IteratorAdvance::kH, - MemorySpace::Kind MemorySpace = MemorySpace::kGeneric, - typename Index_ = int, - typename FragmentElement_ = Scalar_, - IteratorFragment::Kind IteratorFragment_ = IteratorFragment::kScalar, - typename Skew_ = Shape<0, 0, 0, 0> > -struct TileStoreIterator : public TileIteratorBase<Traits_, - Scalar_, - Advance_, - MemorySpace, - Index_, - FragmentElement_, - IteratorFragment_, - Skew_> { - /// Base class - typedef TileIteratorBase<Traits_, - Scalar_, - Advance_, - MemorySpace, - Index_, - FragmentElement_, - IteratorFragment_, - Skew_> - Base; - - /// concept TileTraits - typedef typename Base::Traits Traits; - - /// Scalar element - typedef typename Base::Scalar Scalar; - - /// Fragment element - typedef typename Base::FragmentElement FragmentElement; - - /// Specifies in which dimension post-increment accesses advance. - static IteratorAdvance::Kind const kAdvance = Base::kAdvance; - - /// Specifies type of iterator fragment storage (Salar or WmmaMatrix) - static IteratorFragment::Kind const kIteratorFragment = Base::kIteratorFragment; - - /// Source or destination memory space - static MemorySpace::Kind const kMemorySpace = Base::kMemorySpace; - - /// Index type - typedef typename Base::Index Index; - - /// Skew quantity - typedef typename Base::Skew Skew; - - /// Tile shape - typedef typename Base::Tile Tile; - - /// Delta - typedef typename Base::Delta Delta; - - /// Iterations - typedef typename Base::Iterations Iterations; - - /// ThreadOffset functor - typedef typename Base::ThreadOffset ThreadOffset; - - /// Fragment type - typedef typename Base::FragmentShape FragmentShape; - - /// Memory access type - typedef typename Base::AccessType AccessType; - - /// Fragment definition - typedef typename Base::Fragment Fragment; - - /// Fragment iterator definition - typedef typename Base::FragmentIterator FragmentIterator; - - /// Fragment const iterator definition - typedef typename Base::FragmentConstIterator FragmentConstIterator; - - /// Default predicate mask type - typedef typename Base::PredicateVector PredicateVector; - - /// Storage object which may be stored to - typedef typename Base::Storage SharedStorage; - - /// IteratorBase parameters - typedef typename Base::Params BaseParams; - - /// Parameters - struct Params : public BaseParams { - /// Pointer to memory - Scalar *pointer; - - /// Initialize params to access storage object - CUTLASS_HOST_DEVICE - int initialize(SharedStorage &storage) { - pointer = &storage[0]; - return 0; - } - - /// Initializes params to access a raw pointer - CUTLASS_HOST_DEVICE - int initialize(Scalar *ptr, Index stride_d, Index stride_h, Index stride_w) { - Base::Params::initialize(stride_d, stride_h, stride_w); - pointer = ptr; - return 0; - } - - /// Initializes params - CUTLASS_HOST_DEVICE - int initialize(Scalar *ptr, - Index _stride_d, - Index _stride_h, - Index _stride_w, - Index _inc_d, - Index _inc_h, - Index _inc_w, - Index _inc_advance) { - pointer = ptr; - Base::Params::initialize( - _stride_d, _stride_h, _stride_w, _inc_d, _inc_h, _inc_w, _inc_advance); - return 0; - } - - /// Initializes params to default values - CUTLASS_HOST_DEVICE - int initialize() { return Base::Params::initialize(); } - }; - - // - // Data members - // - - /// Parameters structure - Params params; - - /// Offset of an individual lane from the start of the tile - Coord<4> thread_offset; - - /// The stage. - int stage; - - // - // Static member functions - // - - /// Initializes a predicate vector - template <typename PredicateIterator> - CUTLASS_HOST_DEVICE void initialize_predicates(PredicateIterator predicate_it, - Coord<3> const &bounds, - Coord<3> const &block_offset = make_Coord(0, - 0, - 0)) { - Base::initialize_predicates( - predicate_it, - bounds, - block_offset + make_Coord(0, thread_offset[1], thread_offset[2] * Tile::kC)); - } - - // - // Methods - // - - /// Default constructor - CUTLASS_HOST_DEVICE - TileStoreIterator() {} - - /// Constructs a tile store iterator - CUTLASS_HOST_DEVICE - TileStoreIterator(Params const &_params, - Coord<3> const &block_offset = make_Coord(0, 0, 0), - ThreadOffset thread_offset_func = ThreadOffset()) - : params(_params), stage(0) { - thread_offset = thread_offset_func(); - - params.pointer += block_offset[0] * params.stride_d + - (block_offset[1] + thread_offset[1]) * params.stride_h + - (block_offset[2] + thread_offset[2] * Tile::kC) / Tile::kC * params.stride_w; - } - - /// Constructs a tile store iterator - CUTLASS_HOST_DEVICE - TileStoreIterator(Params const &, - SharedStorage &shared_storage, - Coord<3> const &block_offset = make_Coord(0, 0, 0), - ThreadOffset thread_offset_func = ThreadOffset()) - : stage(0) { - int const offset = thread_offset_func()[2]; - params.pointer = &shared_storage[offset]; - } - - /// Returns the current pointer - CUTLASS_HOST_DEVICE - Scalar *data() const { return params.pointer; } - - /// Increment in the D dimension - CUTLASS_HOST_DEVICE void inc_d() { params.pointer += params.inc_d; } - - /// Increment in the H dimension - CUTLASS_HOST_DEVICE void inc_h() { params.pointer += params.inc_h; } - - /// Increment in the W dimension - CUTLASS_HOST_DEVICE void inc_w() { params.pointer += params.inc_w; } - - /// Increment in the next dimension - CUTLASS_HOST_DEVICE void inc_advance() {} - - /// Increment the stage. - CUTLASS_DEVICE void inc_stage() { - if (Tile::kD > 1) { - int const kStageSize = Tile::kH * Tile::kW * Tile::kC; - if (stage == Tile::kD - 1) { - params.pointer -= (Tile::kD - 1) * kStageSize; - stage = 0; - } else { - params.pointer += kStageSize; - stage = stage + 1; - } - } - } - - /// The accessor. - CUTLASS_DEVICE void set(AccessType const &value, int d, int h, int w, int c) { - int const imm = - ComputeOffsetFromStrides<typename Base::ImmediateOffsetStrides>::get(d, h, w, c); - Store<Scalar, Base::kAccessSize, kMemorySpace>::store(value, params.pointer, imm); - } - - public: - /// Stores a fragment and advances to the next tile. - template <typename Fragment, typename PredicateIterator> - CUTLASS_HOST_DEVICE void store_post_increment(Fragment &fragment, PredicateIterator pred_it) { - FragmentIterator frag_iterator(fragment); - - for (int d = 0; d < Iterations::kD; ++d) { - for (int h = 0; h < Iterations::kH; ++h) { - for (int w = 0; w < Iterations::kW; ++w, ++pred_it) { - if (*pred_it) { - Store<typename Fragment::Element, Tile::kC, kMemorySpace>::store( - reinterpret_cast<AccessType &>(frag_iterator.at(d, h, w, 0)), data(), 0); - } - if (w < Iterations::kW - 1) { - inc_w(); - } - } - if (h < Iterations::kH - 1) { - inc_h(); - } - } - if (d < Iterations::kD - 1) { - inc_d(); - } - } - inc_advance(); - } - - /// Stores a fragment and advances to the next tile. - template <typename Fragment> - CUTLASS_HOST_DEVICE void store_post_increment(Fragment &fragment) { - typename PredicateVector::TrivialIterator pred_it; - store_post_increment(fragment, pred_it); - } - - /// Stores a fragment without advancing the iterator. - template <typename Fragment, typename PredicateIterator> - CUTLASS_HOST_DEVICE void store(Fragment &fragment, PredicateIterator pred_it) const { - TileStoreIterator _store_it(*this); - _store_it.store_post_increment(fragment, pred_it); - } - - /// Stores a fragment without advancing the iterator. - template <typename Fragment> - CUTLASS_HOST_DEVICE void store(Fragment &fragment) const { - typename PredicateVector::TrivialIterator pred_it; - store(fragment, pred_it); - } -}; -} diff --git a/cutlass-example/cutlass/tile_traits_standard.h b/cutlass-example/cutlass/tile_traits_standard.h deleted file mode 100644 index 14ecd01..0000000 --- a/cutlass-example/cutlass/tile_traits_standard.h +++ /dev/null @@ -1,238 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines tile traits for several tile partitioning arrangements of threads expected to - achieve efficient streaming performance. -*/ -#pragma once - -#include <cutlass/tile_iterator.h> - -namespace cutlass { - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Basic thread offset function computed from a thread shape -template <typename ThreadShape> -struct TiledThreadOffset { - /// Computes the logical coordinate from thread shape - CUTLASS_HOST_DEVICE - Coord<4> operator()() const { - Coord<4> thread_offset; - - int index = threadIdx.x; - - thread_offset[3] = (index % ThreadShape::kC); - index = (index / ThreadShape::kC); - - thread_offset[2] = (index % ThreadShape::kW); - index = (index / ThreadShape::kW); - - thread_offset[1] = (index % ThreadShape::kH); - index = (index / ThreadShape::kH); - - thread_offset[0] = index; - - return thread_offset; - } -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Tiling in which the number of threads is greater than the -/// contiguous dimension of the tile. -template <typename Tile_, int Threads> -struct TileTraitsStrideMajor { - /// Shape of tile - typedef Tile_ Tile; - - /// Number of participating threads - static int const kThreads = Threads; - - // Static assertions - static_assert(!(ShapeCount<Tile>::kDhw % kThreads), - "Tiling undefined if elements not divisible by threads."); - - static_assert(Tile::kW <= kThreads, - "This specialization assumes there are more threads than the contiguous dimension " - "of the tile."); - - /// Shape of threads - typedef Shape<1, kThreads / Tile::kW, Tile::kW, 1> ThreadShape; - - /// Delta along each dimension - typedef Shape<1, ThreadShape::kH, 1, 1> Delta; - - /// Number of iterations - typedef Shape<1, Tile::kH / ThreadShape::kH, 1, 1> Iterations; - - /// Computes the initial offset - typedef TiledThreadOffset<ThreadShape> ThreadOffset; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Tiling in which the number of threads is fewer than the tile size -/// in the contiguous dimension. -template <typename Tile_, int Threads> -struct TileTraitsContiguousMajor { - /// Shape of tile - typedef Tile_ Tile; - - /// Number of participating threads - static int const kThreads = Threads; - - // Static assertions - static_assert(Tile::kW >= kThreads, - "This specialization assumes there are more threads than the contiguous dimension " - "of the tile."); - - static_assert(!(ShapeCount<Tile>::kDhw % kThreads), - "Tiling undefined if elements not divisible by threads."); - - static_assert(!(Tile::kW % kThreads), - "The contiguous size of the tile must be divisible by the number of threads."); - - /// Thread shape - typedef Shape<1, 1, kThreads> ThreadShape; - - /// Delta between each thread's access - typedef Shape<1, 1, kThreads> Delta; - - /// Number of iterations - typedef Shape<1, Tile::kH, Tile::kW / kThreads> Iterations; - - /// Computes the initial offset - typedef TiledThreadOffset<ThreadShape> ThreadOffset; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Tiling in which warps rake across the contiguous dimension -template <typename Tile_, int Threads> -struct TileTraitsWarpRake { - /// Shape of tile - typedef Tile_ Tile; - - /// Number of participating threads - static int const kThreads = Threads; - - /// Hard-coded warp size - static int const kWarpSize = 32; - - /// Number of participating warps - static int const kWarpCount = kThreads / kWarpSize; - - // Static assertions - static_assert(!(ShapeCount<Tile>::kDhw % kThreads), - "Tiling undefined if elements not divisible by threads."); - - static_assert(!(kThreads % kWarpSize), "Number of threads must be divisible by the warp size."); - - static_assert(!(Tile::kW % kWarpSize), "Contiguous dimension must be divisible by the warp size"); - - /// Warps strip-mined across strided dimension - static int const kWarpsStrided = __NV_STD_MIN(kWarpCount, Tile::kH); - - /// Warps stripmined contiguous dimension - static int const kWarpsContiguous = kWarpCount / kWarpsStrided; - - /// Arrangement of threads - typedef Shape<1, kWarpsStrided, kWarpsContiguous * kWarpSize> ThreadShape; - - /// The same warp rakes along the contiguous dimension - typedef Shape<1, kWarpsStrided, kWarpSize> Delta; - - /// Number of iterations - typedef Shape<1, Tile::kH / Delta::kH, Tile::kW / ThreadShape::kW> Iterations; - - /// Computes the thread offset in (H, W) based on thread ID - struct ThreadOffset { - /// Basic thread offset function computed from a thread shape - CUTLASS_HOST_DEVICE - Coord<4> operator()() const { - int tid = threadIdx.x; - int warp = (tid / kWarpSize); - int lane = (tid % kWarpSize); - - static int const kWarpSpanContiguous = kWarpSize * Iterations::kW; - - int warp_w = (warp % kWarpsContiguous); - int warp_h = (warp / kWarpsContiguous); - - return make_Coord(0, warp_h, lane + kWarpSpanContiguous * warp_w, 0); - } - }; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Chooses 'best' shape to enable warp raking along contiguous dimension if possible. -template <typename Tile_, int Threads> -struct TileTraitsStandard { - /// Shape of tile - typedef Tile_ Tile; - - /// Number of participating threads - static int const kThreads = Threads; - - /// Hard-coded warp size - static int const kWarpSize = 32; - - /// Number of participating warps - static int const kWarpCount = kThreads / kWarpSize; - - // Static assertions - static_assert(!(ShapeCount<Tile>::kDhw % kThreads), - "Tiling undefined if elements not divisible by threads."); - - /// Choose the stride-major contiguous tiling if the contiguous dimension is - /// smaller than the warp size. Otherwise, if it is divisible by the warp size, - /// choose the warp rake arrangement. - typedef typename platform::conditional < - Tile::kW<kWarpSize, - TileTraitsStrideMajor<Tile, Threads>, - typename platform::conditional<!(Tile::kW % kWarpSize), - TileTraitsWarpRake<Tile, Threads>, - TileTraitsContiguousMajor<Tile, Threads> >::type>:: - type Traits; - - /// Delta between accesses - typedef typename Traits::Delta Delta; - - /// Delta between each thread's access - /// TODO MTA this is wrong for sure, but Delta is used for stride computation at the moment - typedef Delta ImmediateOffsetStrides; - - /// Number of accesses - typedef typename Traits::Iterations Iterations; - - /// Thread offset functor - typedef typename Traits::ThreadOffset ThreadOffset; -}; - -/////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/util/cutlass_math.h b/cutlass-example/cutlass/util/cutlass_math.h deleted file mode 100644 index 0ecdc43..0000000 --- a/cutlass-example/cutlass/util/cutlass_math.h +++ /dev/null @@ -1,131 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ - -#pragma once - -/** - * \file - * \brief Math utilities - */ - -#include <cutlass/util/platform.h> - -namespace cutlass { - -/****************************************************************************** - * Static math utilities - ******************************************************************************/ - -/** - * Statically determine if N is a power-of-two - */ -template <int N> -struct is_pow2 : platform::integral_constant<bool, (N & (N - 1)) == 0> {}; - -/** - * Statically determine log2(N), rounded down - */ -template <int N, int CurrentVal = N, int Count = 0> -struct log2_down { - /// Static logarithm value - enum { value = log2_down<N, (CurrentVal >> 1), Count + 1>::value }; -}; - -// Base case -template <int N, int Count> -struct log2_down<N, 1, Count> { - enum { value = Count }; -}; - -/** - * Statically determine log2(N), rounded up - */ -template <int N, int CurrentVal = N, int Count = 0> -struct log2_up { - /// Static logarithm value - enum { value = log2_up<N, (CurrentVal >> 1), Count + 1>::value }; -}; - -// Base case -template <int N, int Count> -struct log2_up<N, 1, Count> { - enum { value = ((1 << Count) < N) ? Count + 1 : Count }; -}; - -/** - * Statically estimate sqrt(N) to the nearest power-of-two - */ -template <int N> -struct sqrt_est { - enum { value = 1 << (log2_up<N>::value / 2) }; -}; - -/** - * For performing a constant-division with a compile-time assertion that the - * Divisor evenly-divides the Dividend. - */ -template <int Dividend, int Divisor> -struct divide_assert { - enum { value = Dividend / Divisor }; - - static_assert((Dividend % Divisor == 0), "Not an even multiple"); -}; - -/****************************************************************************** - * Rounding - ******************************************************************************/ - -/** - * Round dividend up to the nearest multiple of divisor - */ -template <typename dividend_t, typename divisor_t> -CUTLASS_HOST_DEVICE dividend_t round_nearest(dividend_t dividend, divisor_t divisor) { - return ((dividend + divisor - 1) / divisor) * divisor; -} - -/** - * Greatest common divisor - */ -template <typename value_t> -CUTLASS_HOST_DEVICE value_t gcd(value_t a, value_t b) { - for (;;) { - if (a == 0) return b; - b %= a; - if (b == 0) return a; - a %= b; - } -} - -/** - * Least common multiple - */ -template <typename value_t> -CUTLASS_HOST_DEVICE value_t lcm(value_t a, value_t b) { - value_t temp = gcd(a, b); - - return temp ? (a / temp * b) : 0; -} - -} // namespace cutlass diff --git a/cutlass-example/cutlass/util/debug.h b/cutlass-example/cutlass/util/debug.h deleted file mode 100644 index 6055e3f..0000000 --- a/cutlass-example/cutlass/util/debug.h +++ /dev/null @@ -1,122 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ - -#pragma once - -/** - * \file - * \brief Debugging and logging functionality - */ - -#include <stdio.h> - -namespace cutlass { - -/****************************************************************************** - * Debug and logging macros - ******************************************************************************/ - -/** - * Formats and prints the given message to stdout - */ -#if !defined(CUDA_LOG) -#if !defined(__CUDA_ARCH__) -#define CUDA_LOG(format, ...) printf(format, __VA_ARGS__) -#else -#define CUDA_LOG(format, ...) \ - printf("[block (%d,%d,%d), thread (%d,%d,%d)]: " format, \ - blockIdx.x, \ - blockIdx.y, \ - blockIdx.z, \ - threadIdx.x, \ - threadIdx.y, \ - threadIdx.z, \ - __VA_ARGS__); -#endif -#endif - -/** - * Formats and prints the given message to stdout only if DEBUG is defined - */ -#if !defined(CUDA_LOG_DEBUG) -#ifdef DEBUG -#define CUDA_LOG_DEBUG(format, ...) CUDA_LOG(format, __VA_ARGS__) -#else -#define CUDA_LOG_DEBUG(format, ...) -#endif -#endif - -/** - * \brief The corresponding error message is printed to \p stderr (or \p stdout in device code) - * along with the supplied source context. - * - * \return The CUDA error. - */ -__host__ CUTLASS_DEVICE cudaError_t cuda_perror_impl(cudaError_t error, - const char* filename, - int line) { - (void)filename; - (void)line; - if (error) { -#if !defined(__CUDA_ARCH__) - fprintf( - stderr, "CUDA error %d [%s, %d]: %s\n", error, filename, line, cudaGetErrorString(error)); - fflush(stderr); -#else - printf("CUDA error %d [%s, %d]\n", error, filename, line); -#endif - } - return error; -} - -/** - * \brief Perror macro - */ -#ifndef CUDA_PERROR -#define CUDA_PERROR(e) cuda_perror_impl((cudaError_t)(e), __FILE__, __LINE__) -#endif - -/** - * \brief Perror macro with exit - */ -#ifndef CUDA_PERROR_EXIT -#define CUDA_PERROR_EXIT(e) \ - if (cuda_perror_impl((cudaError_t)(e), __FILE__, __LINE__)) { \ - exit(1); \ - } -#endif - -/** - * \brief Perror macro only if DEBUG is defined - */ -#ifndef CUDA_PERROR_DEBUG -#ifdef DEBUG -#define CUDA_PERROR_DEBUG(e) CUDA_PERROR(e) -#else -#define CUDA_PERROR_DEBUG(e) (e) -#endif -#endif - -} // namespace cutlass diff --git a/cutlass-example/cutlass/util/platform.h b/cutlass-example/cutlass/util/platform.h deleted file mode 100644 index 2a44c10..0000000 --- a/cutlass-example/cutlass/util/platform.h +++ /dev/null @@ -1,801 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ - -#pragma once - -/** - * \file - * \brief C++ features that may be otherwise unimplemented for CUDA device functions. - * - * This file has three components: - * - * (1) Macros: - * - Empty macro defines for C++ keywords not supported by the current - * version of C++. These simply allow compilation to proceed (but do - * not provide the added semantics). - * - \p noexcept - * - \p constexpr - * - \p nullptr - * - \p static_assert - * - * - Macro functions that we need in constant expressions because the - * C++ equivalents require constexpr compiler support. These are - * prefixed with \p __NV_STD_* - * - \p __NV_STD_MAX - * - \p __NV_STD_MIN - * - * (2) Re-implementations of STL functions and types: - * - C++ features that need the \p __device__ annotation. These are - * placed into the \p platform namespace. - * - \p plus - * - \p less - * - \p greater - * - \p min - * - \p max - * - \p methods on std::pair (==, !=, <, <=, >, >=, and make_pair()) - * - * (3) Stop-gap implementations of unsupported STL functions and types: - * - STL functions and types defined by C++ 11/14/17/etc. that are not - * provided by the current version of C++. These are placed into the - * \p platform namespace - * - \p integral_constant - * - \p nullptr_t - * - \p true_type - * - \p false_type - * - \p bool_constant - * - \p enable_if - * - \p conditional - * - \p is_same - * - \p is_base_of - * - \p remove_const - * - \p remove_volatile - * - \p remove_cv - * - \p is_volatile - * - \p is_pointer - * - \p is_void - * - \p is_integral - * - \p is_floating_point - * - \p is_arithmetic - * - \p is_fundamental - * - \p is_trivially_copyable - * - \p alignment_of - * - \p aligned_storage - * - * (4) Functions and types that are STL-like (but aren't in the STL): - * - \p TODO: min and max functors? - * - * The idea is that, as we drop support for older compilers, we can simply #define - * the \p __NV_STD_XYZ macros and \p platform namespace to alias their C++ - * counterparts (or trivially find-and-replace their occurrences in code text). - */ - -//----------------------------------------------------------------------------- -// Dependencies -//----------------------------------------------------------------------------- - -#include <stdint.h> - -#if !defined(__CUDACC_RTC__) -//----------------------------------------------------------------------------- -// Include STL files that platform provides functionality for -//----------------------------------------------------------------------------- - -#include <algorithm> // Minimum/maximum operations -#include <cstddef> // nullptr_t -#include <functional> // Arithmetic operations -#include <utility> // For methods on std::pair -#if (!defined(_MSC_VER) && (__cplusplus >= 201103L)) || (defined(_MSC_VER) && (_MS_VER >= 1500)) -#include <type_traits> // For integral constants, conditional metaprogramming, and type traits -#endif - -#include <cutlass/cutlass.h> - -#endif -/****************************************************************************** - * Macros - ******************************************************************************/ -//----------------------------------------------------------------------------- -// Keywords -//----------------------------------------------------------------------------- - -/// noexcept, constexpr -#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1900)) -#ifndef noexcept -#define noexcept -#endif -#ifndef constexpr -#define constexpr -#endif -#endif - -/// nullptr -#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1310)) -#ifndef nullptr -#define nullptr 0 -#endif -#endif - -/// static_assert -#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1600)) -#ifndef static_assert -#define __platform_cat_(a, b) a##b -#define __platform_cat(a, b) __platform_cat_(a, b) -#define static_assert(__e, __m) typedef int __platform_cat(AsSeRt, __LINE__)[(__e) ? 1 : -1] -#endif -#endif - -//----------------------------------------------------------------------------- -// Functions -//----------------------------------------------------------------------------- - -/// Select maximum(a, b) -#ifndef __NV_STD_MAX -#define __NV_STD_MAX(a, b) (((b) > (a)) ? (b) : (a)) -#endif - -/// Select minimum(a, b) -#ifndef __NV_STD_MIN -#define __NV_STD_MIN(a, b) (((b) < (a)) ? (b) : (a)) -#endif - -/****************************************************************************** - * Re-implementations - ******************************************************************************/ -namespace cutlass { -namespace platform { - -//----------------------------------------------------------------------------- -// Arithmetic operations, comparisons <functional> -//----------------------------------------------------------------------------- - -/// platform::plus -template <typename T> -struct plus { - CUTLASS_HOST_DEVICE constexpr T operator()(const T& lhs, const T& rhs) const { return lhs + rhs; } -}; - -/// std::less -template <typename T> -struct less { - CUTLASS_HOST_DEVICE constexpr bool operator()(const T& lhs, const T& rhs) const { - return lhs < rhs; - } -}; - -/// std::greater -template <typename T> -struct greater { - CUTLASS_HOST_DEVICE constexpr bool operator()(const T& lhs, const T& rhs) const { - return lhs > rhs; - } -}; - -//----------------------------------------------------------------------------- -// Minimum/maximum operations <algorithm> -//----------------------------------------------------------------------------- - -/// std::min -template <typename T> -CUTLASS_HOST_DEVICE constexpr const T& min(const T& a, const T& b) { - return (b < a) ? b : a; -} - -/// std::max -template <typename T> -CUTLASS_HOST_DEVICE constexpr const T& max(const T& a, const T& b) { - return (a < b) ? b : a; -} - -#if !defined(__CUDACC_RTC__) -//----------------------------------------------------------------------------- -// Methods on std::pair -//----------------------------------------------------------------------------- - -using std::pair; - -template <class T1, class T2> -CUTLASS_HOST_DEVICE constexpr bool operator==(const pair<T1, T2>& lhs, const pair<T1, T2>& rhs) { - return (lhs.first == rhs.first) && (lhs.second == rhs.second); -} - -template <class T1, class T2> -CUTLASS_HOST_DEVICE constexpr bool operator!=(const pair<T1, T2>& lhs, const pair<T1, T2>& rhs) { - return (lhs.first != rhs.first) && (lhs.second != rhs.second); -} - -template <class T1, class T2> -CUTLASS_HOST_DEVICE constexpr bool operator<(const pair<T1, T2>& lhs, const pair<T1, T2>& rhs) { - return (lhs.first < rhs.first) ? true : (rhs.first < lhs.first) ? false - : (lhs.second < rhs.second); -} - -template <class T1, class T2> -CUTLASS_HOST_DEVICE constexpr bool operator<=(const pair<T1, T2>& lhs, const pair<T1, T2>& rhs) { - return !(rhs < lhs); -} - -template <class T1, class T2> -CUTLASS_HOST_DEVICE constexpr bool operator>(const pair<T1, T2>& lhs, const pair<T1, T2>& rhs) { - return (rhs < lhs); -} - -template <class T1, class T2> -CUTLASS_HOST_DEVICE constexpr bool operator>=(const pair<T1, T2>& lhs, const pair<T1, T2>& rhs) { - return !(lhs < rhs); -} - -template <class T1, class T2> -CUTLASS_HOST_DEVICE std::pair<T1, T2> make_pair(T1 t, T2 u) { - std::pair<T1, T2> retval; - retval.first = t; - retval.second = u; - return retval; -} -#endif - -} // namespace platform - -/****************************************************************************** - * Implementations of C++ 11/14/17/... STL features - ******************************************************************************/ - -namespace platform { - -//----------------------------------------------------------------------------- -// Integral constant helper types <type_traits> -//----------------------------------------------------------------------------- - -#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1500)) - -/// std::integral_constant -template <typename value_t, value_t V> -struct integral_constant; - -/// std::integral_constant -template <typename value_t, value_t V> -struct integral_constant { - static const value_t value = V; - - typedef value_t value_type; - typedef integral_constant<value_t, V> type; - - CUTLASS_HOST_DEVICE operator value_type() const { return value; } - - CUTLASS_HOST_DEVICE const value_type operator()() const { return value; } -}; - -#else - -using std::integral_constant; -using std::pair; - -#endif - -/// The type used as a compile-time boolean with true value. -typedef integral_constant<bool, true> true_type; - -/// The type used as a compile-time boolean with false value. -typedef integral_constant<bool, false> false_type; - -#if (!defined(_MSC_VER) && (__cplusplus <= 201402L)) || (defined(_MSC_VER) && (_MSC_VER < 1900)) - -/// std::bool_constant -template <bool V> -struct bool_constant : platform::integral_constant<bool, V> {}; - -#else - -using std::bool_constant; - -#endif - -#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1700)) - -/// std::nullptr_t -struct nullptr_t {}; - -#else - -using std::nullptr_t; - -#endif - -//----------------------------------------------------------------------------- -// Conditional metaprogramming <type_traits> -//----------------------------------------------------------------------------- - -#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1600)) - -/// std::enable_if (true specialization) -template <bool C, typename T = void> -struct enable_if { - typedef T type; -}; - -/// std::enable_if (false specialization) -template <typename T> -struct enable_if<false, T> {}; - -/// std::conditional (true specialization) -template <bool B, class T, class F> -struct conditional { - typedef T type; -}; - -/// std::conditional (false specialization) -template <class T, class F> -struct conditional<false, T, F> { - typedef F type; -}; - -#else - -using std::enable_if; -using std::conditional; - -#endif - -//----------------------------------------------------------------------------- -// Const/volatility specifiers <type_traits> -//----------------------------------------------------------------------------- - -#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1500)) - -/// std::remove_const (non-const specialization) -template <typename T> -struct remove_const { - typedef T type; -}; - -/// std::remove_const (const specialization) -template <typename T> -struct remove_const<const T> { - typedef T type; -}; - -/// std::remove_volatile (non-volatile specialization) -template <typename T> -struct remove_volatile { - typedef T type; -}; - -/// std::remove_volatile (volatile specialization) -template <typename T> -struct remove_volatile<volatile T> { - typedef T type; -}; - -/// std::remove_cv -template <typename T> -struct remove_cv { - typedef typename remove_volatile<typename remove_const<T>::type>::type type; -}; - -#else - -using std::remove_const; -using std::remove_volatile; -using std::remove_cv; - -#endif - -//----------------------------------------------------------------------------- -// Type relationships <type_traits> -//----------------------------------------------------------------------------- - -#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1500)) - -/// std::is_same (false specialization) -template <typename A, typename B> -struct is_same : false_type {}; - -/// std::is_same (true specialization) -template <typename A> -struct is_same<A, A> : true_type {}; - -/// Helper for std::is_base_of -template <typename BaseT, typename DerivedT> -struct is_base_of_helper { - typedef char (&yes)[1]; - typedef char (&no)[2]; - - template <typename B, typename D> - struct dummy { - CUTLASS_HOST_DEVICE operator B*() const; - CUTLASS_HOST_DEVICE operator D*(); - }; - - template <typename T> - CUTLASS_HOST_DEVICE static yes check(DerivedT*, T); - - CUTLASS_HOST_DEVICE static no check(BaseT*, int); - - static const bool value = sizeof(check(dummy<BaseT, DerivedT>(), int())) == sizeof(yes); -}; - -/// std::is_base_of -template <typename BaseT, typename DerivedT> -struct is_base_of - : integral_constant<bool, - (is_base_of_helper<typename remove_cv<BaseT>::type, - typename remove_cv<DerivedT>::type>::value) || - (is_same<typename remove_cv<BaseT>::type, - typename remove_cv<DerivedT>::type>::value)> {}; - -#else - -using std::is_same; -using std::is_base_of; - -#endif - -//----------------------------------------------------------------------------- -// Type properties <type_traits> -//----------------------------------------------------------------------------- - -#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1500)) - -/// std::is_volatile -template <typename T> -struct is_volatile : false_type {}; -template <typename T> -struct is_volatile<volatile T> : true_type {}; - -/// Helper for std::is_pointer (false specialization) -template <typename T> -struct is_pointer_helper : false_type {}; - -/// Helper for std::is_pointer (true specialization) -template <typename T> -struct is_pointer_helper<T*> : true_type {}; - -/// std::is_pointer -template <typename T> -struct is_pointer : is_pointer_helper<typename remove_cv<T>::type> {}; - -/// std::is_void -template <typename T> -struct is_void : is_same<void, typename remove_cv<T>::type> {}; - -/// std::is_integral -template <typename T> -struct is_integral : false_type {}; -template <> -struct is_integral<char> : true_type {}; -template <> -struct is_integral<signed char> : true_type {}; -template <> -struct is_integral<unsigned char> : true_type {}; -template <> -struct is_integral<short> : true_type {}; -template <> -struct is_integral<unsigned short> : true_type {}; -template <> -struct is_integral<int> : true_type {}; -template <> -struct is_integral<unsigned int> : true_type {}; -template <> -struct is_integral<long> : true_type {}; -template <> -struct is_integral<unsigned long> : true_type {}; -template <> -struct is_integral<long long> : true_type {}; -template <> -struct is_integral<unsigned long long> : true_type {}; -template <typename T> -struct is_integral<volatile T> : is_integral<T> {}; -template <typename T> -struct is_integral<const T> : is_integral<T> {}; -template <typename T> -struct is_integral<const volatile T> : is_integral<T> {}; - -/// std::is_floating_point -template <typename T> -struct is_floating_point - : integral_constant<bool, - (is_same<float, typename remove_cv<T>::type>::value || - is_same<double, typename remove_cv<T>::type>::value)> {}; - -/// std::is_arithmetic -template <typename T> -struct is_arithmetic - : integral_constant<bool, (is_integral<T>::value || is_floating_point<T>::value)> {}; - -/// std::is_fundamental -template <typename T> -struct is_fundamental - : integral_constant<bool, - (is_arithmetic<T>::value || is_void<T>::value || - is_same<nullptr_t, typename remove_cv<T>::type>::value)> {}; - -#else - -using std::is_volatile; -using std::is_pointer; -using std::is_void; -using std::is_integral; -using std::is_floating_point; -using std::is_arithmetic; -using std::is_fundamental; - -#endif - -#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1800)) || \ - (defined(__GNUG__) && (__GNUC__ < 5)) - -/** - * std::is_trivially_copyable - * - * This implementation only evaluates true if T is fundamental or pointer - * - * Without help from partial template specializations provided by the user for - * a specific class or struct, this trait will never report that the specified - * class or struct is trivially-copyable ; this is always safe, - * if possibly sub-optimal. - */ -template <typename T> -struct is_trivially_copyable - : integral_constant<bool, (is_fundamental<T>::value || is_pointer<T>::value)> {}; - -#else - -using std::is_trivially_copyable; - -#endif - -//----------------------------------------------------------------------------- -// Alignment and layout utilities -//----------------------------------------------------------------------------- - -#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1500)) - -/// std::alignment_of -template <typename value_t> -struct alignment_of { - struct pad { - value_t val; - char byte; - }; - - enum { value = sizeof(pad) - sizeof(value_t) }; -}; - -#else - -template <typename value_t> -struct alignment_of : std::alignment_of<value_t> {}; - -#endif - -/* 16B specializations where 32-bit Win32 host compiler disagrees with device compiler */ -template <> -struct alignment_of<int4> { - enum { value = 16 }; -}; -template <> -struct alignment_of<uint4> { - enum { value = 16 }; -}; -template <> -struct alignment_of<float4> { - enum { value = 16 }; -}; -template <> -struct alignment_of<long4> { - enum { value = 16 }; -}; -template <> -struct alignment_of<ulong4> { - enum { value = 16 }; -}; -template <> -struct alignment_of<longlong2> { - enum { value = 16 }; -}; -template <> -struct alignment_of<ulonglong2> { - enum { value = 16 }; -}; -template <> -struct alignment_of<double2> { - enum { value = 16 }; -}; -template <> -struct alignment_of<longlong4> { - enum { value = 16 }; -}; -template <> -struct alignment_of<ulonglong4> { - enum { value = 16 }; -}; -template <> -struct alignment_of<double4> { - enum { value = 16 }; -}; - -// Specializations for volatile/const qualified types -template <typename value_t> -struct alignment_of<volatile value_t> : alignment_of<value_t> {}; -template <typename value_t> -struct alignment_of<const value_t> : alignment_of<value_t> {}; -template <typename value_t> -struct alignment_of<const volatile value_t> : alignment_of<value_t> {}; - -#if (!defined(_MSC_VER) && (__cplusplus < 201103L)) || (defined(_MSC_VER) && (_MSC_VER < 1800)) - -template <size_t Align> -struct aligned_chunk; -template <> -struct __align__(1) aligned_chunk<1> { - uint8_t buff; -}; -template <> -struct __align__(2) aligned_chunk<2> { - uint16_t buff; -}; -template <> -struct __align__(4) aligned_chunk<4> { - uint32_t buff; -}; -template <> -struct __align__(8) aligned_chunk<8> { - uint32_t buff[2]; -}; -template <> -struct __align__(16) aligned_chunk<16> { - uint32_t buff[4]; -}; -template <> -struct __align__(32) aligned_chunk<32> { - uint32_t buff[8]; -}; -template <> -struct __align__(64) aligned_chunk<64> { - uint32_t buff[16]; -}; -template <> -struct __align__(128) aligned_chunk<128> { - uint32_t buff[32]; -}; -template <> -struct __align__(256) aligned_chunk<256> { - uint32_t buff[64]; -}; -template <> -struct __align__(512) aligned_chunk<512> { - uint32_t buff[128]; -}; -template <> -struct __align__(1024) aligned_chunk<1024> { - uint32_t buff[256]; -}; -template <> -struct __align__(2048) aligned_chunk<2048> { - uint32_t buff[512]; -}; -template <> -struct __align__(4096) aligned_chunk<4096> { - uint32_t buff[1024]; -}; - -/// std::aligned_storage -template <size_t Len, size_t Align> -struct aligned_storage { - typedef aligned_chunk<Align> type[Len / sizeof(aligned_chunk<Align>)]; -}; - -#else - -using std::aligned_storage; - -#endif - -#if !defined(__CUDACC_RTC__) -/// Default deleter -template <typename T> -struct default_delete { - void operator()(T* ptr) const { delete ptr; } -}; - -/// Partial specialization for deleting array types -template <typename T> -struct default_delete<T[]> { - void operator()(T* ptr) const { delete[] ptr; } -}; - -/// std::unique_ptr -template <class T, class Deleter = default_delete<T> > -class unique_ptr { - public: - typedef T* pointer; - typedef T element_type; - typedef Deleter deleter_type; - - private: - /// Pointer to memory - pointer _ptr; - - /// Deleter - deleter_type _deleter; - - public: - unique_ptr() : _ptr(nullptr) {} - unique_ptr(pointer p) : _ptr(p) {} - - ~unique_ptr() { - if (_ptr) { - _deleter(_ptr); - } - } - /// Returns a pointer to the managed object or nullptr if no object is owned. - pointer get() const noexcept { return _ptr; } - - /// Releases ownership of the managed object, if any - pointer release() noexcept { - pointer p(_ptr); - _ptr = nullptr; - return p; - } - - /// Replaces the managed object, deleting the old object. - void reset(pointer p = pointer()) noexcept { - pointer old_ptr = _ptr; - _ptr = p; - if (old_ptr != nullptr) { - get_deleter()(old_ptr); - } - } - - /// Swaps the managed objects with *this and another unique_ptr - void swap(unique_ptr& other) noexcept { std::swap(_ptr, other._ptr); } - - /// Returns the deleter object - Deleter& get_deleter() noexcept { return _deleter; } - - /// Returns the deleter object - Deleter const& get_deleter() const noexcept { return _deleter; } - - /// Checks whether an object is owned - operator bool() const noexcept { return _ptr != nullptr; } - - /// Dereferences the unique_ptr - T& operator*() const { return *_ptr; } - - /// Returns a pointer to the managed object - pointer operator->() const noexcept { return _ptr; } - - /// Array access to managed object - T& operator[](size_t i) const { return _ptr[i]; } -}; - -/// Specializes the swap algorithm -template <typename T, typename Deleter> -void swap(unique_ptr<T, Deleter>& lhs, unique_ptr<T, Deleter>& rhs) noexcept { - lhs.swap(rhs); -} -#endif - -}; // namespace platform -}; // namespace cutlass diff --git a/cutlass-example/cutlass/vector.h b/cutlass-example/cutlass/vector.h deleted file mode 100644 index a66dfde..0000000 --- a/cutlass-example/cutlass/vector.h +++ /dev/null @@ -1,229 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Defines a 1D vector of elements held in the registers of each thread. -*/ -#pragma once - -#if !defined(__CUDACC_RTC__) || defined(CUTLASS_NVRTC_HAS_FP16) -#include <cuda_fp16.h> -#endif - -#include <cutlass/util/platform.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <size_t kAlignment_> -struct AlignedStruct {}; - -template <> -struct __align__(1) AlignedStruct<1>{}; -template <> -struct __align__(2) AlignedStruct<2>{}; -template <> -struct __align__(4) AlignedStruct<4>{}; -template <> -struct __align__(8) AlignedStruct<8>{}; -template <> -struct __align__(16) AlignedStruct<16>{}; -template <> -struct __align__(32) AlignedStruct<32>{}; -template <> -struct __align__(64) AlignedStruct<64>{}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, int kLanes_> -union Vector { - /// The scalar type. - typedef Scalar_ Scalar; - - /// The number of elements in the vector. - enum { kLanes = kLanes_ }; - /// The size of the vector. - enum { kVectorSize = kLanes * (int)sizeof(Scalar) }; - /// The number of registers needed to store the vector. - enum { kRegisters = kVectorSize < 4 ? 1 : kVectorSize / 4 }; - - // Make sure that the vector type makes sense. - static_assert(kVectorSize <= 16, "Vector type is too large"); - - /// The aligned storage to make sure we have good alignment. - AlignedStruct<kVectorSize> aligned_; - /// The associated array of scalars. - Scalar scalars[kLanes]; - /// The data in registers. - uint32_t registers[kRegisters]; - - /// Accessor to the ith lane. - CUTLASS_DEVICE Scalar const& operator[](uint32_t i) const { return scalars[i]; } - /// Accessor to the ith lane. - CUTLASS_DEVICE Scalar& operator[](uint32_t i) { return scalars[i]; } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -#if !defined(__CUDACC_RTC__) || defined(CUTLASS_NVRTC_HAS_FP16) - -template <int kLanes_> -union Vector<half, kLanes_> { - /// The scalar type. - typedef half Scalar; - - /// The number of elements in the vector. - enum { kLanes = kLanes_ }; - /// The size of the vector. - enum { kVectorSize = kLanes * (int)sizeof(Scalar) }; - /// The number of registers needed to store the vector. - enum { kRegisters = kVectorSize < 4 ? 1 : kVectorSize / 4 }; - - // Make sure that the vector type makes sense. - static_assert(kVectorSize <= size_t(16), "Vector type is too large"); - - /// The aligned storage to make sure we have good alignment. - AlignedStruct<kVectorSize> aligned_; - /// The associated array of scalars. - uint16_t scalars[kLanes]; - /// The data in registers. - uint32_t registers[kRegisters]; - - /// Accessor to the ith lane. - CUTLASS_DEVICE Scalar const& operator[](uint32_t i) const { - return reinterpret_cast<Scalar const&>(scalars[i]); - } - /// Accessor to the ith lane. - CUTLASS_DEVICE Scalar& operator[](uint32_t i) { return reinterpret_cast<Scalar&>(scalars[i]); } -}; - -#endif - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_> -CUTLASS_DEVICE void make_zero(Scalar_& x) { - x = Scalar_(0); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Element_, int kLanes_ = 1> -struct Vectorize { - typedef Vector<Element_, kLanes_> Type; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Element_> -struct Vectorize<Element_, 1> { - typedef Element_ Type; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename Scalar_, int kLanes_> -CUTLASS_DEVICE void make_zero(Vector<Scalar_, kLanes_>& vec) { - for (int i = 0; i < Vector<Scalar_, kLanes_>::kRegisters; ++i) { - vec.registers[i] = 0; - } -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// -// -// cutlass::Extent similar to std::extent but applicable to CUTLASS types -// - -/// Returns the extent of a scalar or vector -template <typename T> -struct Extent { - static size_t const kValue = 1; -}; - -/// Returns the number of lanes of a vector if need be -template <typename T, int Lanes> -struct Extent<Vector<T, Lanes> > { - static size_t const kValue = Lanes; -}; - -/// Returns the number of lanes of a vector if need be -template <typename T, int Lanes> -struct Extent<Vector<T, Lanes> const> { - static size_t const kValue = Lanes; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Traits describing properties of vectors and scalar-as-vectors -template <typename T> -struct VectorTraits { - /// Scalar type - typedef T Scalar; - - /// Number of lanes of vector - static int const kLanes = 1; - - /// True if the type is actually a cutlass::Vector, otherwise false - static bool const IsVector = false; - - /// Type that is always a vector - typedef Vector<T, 1> Vector; -}; - -/// Partial specialization for actual cutlass::Vector -template <typename T, int Lanes> -struct VectorTraits<Vector<T, Lanes> > { - /// Scalar type - typedef T Scalar; - - /// Number of lanes of vector - static int const kLanes = Lanes; - - /// Type is actually a cutlass::Vector - static bool const IsVector = true; - - /// Type that is always a Vector - typedef Vector<T, Lanes> Vector; -}; - -/// Partial specialization for actual cutlass::Vector -template <typename T, int Lanes> -struct VectorTraits<Vector<T, Lanes> const> { - /// Scalar type - typedef T Scalar; - - /// Number of lanes of vector - static int const kLanes = Lanes; - - /// Type is actually a cutlass::Vector - static bool const IsVector = true; - - /// Type that is always a Vector - typedef Vector<T, Lanes> Vector; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/cutlass/wmma_matrix.h b/cutlass-example/cutlass/wmma_matrix.h deleted file mode 100644 index c4d8a0b..0000000 --- a/cutlass-example/cutlass/wmma_matrix.h +++ /dev/null @@ -1,193 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Abstractions for loading and storing matrices using the CUDA WMMA API. -*/ -#pragma once - -#if defined(__CUDACC__) && (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700) - -// Dependent header files should use the following macro to guard all code using -// nvcuda::wmma:: to enable compilation for CUDA Compute Capabilities < sm_70. -// Earlier shader models not support Tensor Cores. -#define CUTLASS_USE_WMMA_API - -#include "stdio.h" - -#include <crt/mma.h> -#include <cutlass/fragment.h> -#include <cutlass/load_store.h> -#include <cutlass/matrix_traits.h> -#include <cutlass/shape.h> -#include <cutlass/vector.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Statically maps cutlass::MatrixLayout => nvcuda::wmma layout tags -template <MatrixLayout::Kind kLayout_> -struct WmmaLayout { - typedef nvcuda::wmma::col_major Layout; -}; - -/// Statically maps cutlass::MatrixLayout => nvcuda::wmma layout tags -template <> -struct WmmaLayout<MatrixLayout::kRowMajor> { - typedef nvcuda::wmma::row_major Layout; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Adapter to nvcuda::wmma fragment load and store operations -template <GemmOperand::Kind kOperand_, - MatrixLayout::Kind kLayout_, - typename Scalar_, - typename WmmaShape_> -struct WmmaMatrix {}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Adapter to nvcuda::wmma fragment accessors for A operand -template <MatrixLayout::Kind kLayout_, typename Scalar_, typename WmmaShape_> -struct WmmaMatrix<GemmOperand::kA, kLayout_, Scalar_, WmmaShape_> - : public nvcuda::wmma::fragment< - /// The nvcuda::wmma operand name. - nvcuda::wmma::matrix_a, - /// The dimensions. - WmmaShape_::kW, - WmmaShape_::kH, - WmmaShape_::kD, - /// The scalar. - Scalar_, - /// The layout. - typename WmmaLayout<kLayout_>::Layout> { - /// This type. - typedef WmmaMatrix<GemmOperand::kA, kLayout_, Scalar_, WmmaShape_> This_; - - /// Fill-in the element. - CUTLASS_DEVICE This_& operator=(Scalar_ const& x) { - nvcuda::wmma::fill_fragment(*this, x); - return *this; - } - - /// Load from memory. - CUTLASS_DEVICE void load(Scalar_ const* pointer, int const stride) { - nvcuda::wmma::load_matrix_sync(*this, pointer, stride); - } - - /// Store to memory. - CUTLASS_DEVICE void store(Scalar_* pointer, int const stride) const { - nvcuda::wmma::store_matrix_sync(pointer, *this, stride); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Adapter to nvcuda::wmma fragment accessors for B operand -template <MatrixLayout::Kind kLayout_, typename Scalar_, typename WmmaShape_> -struct WmmaMatrix<GemmOperand::kB, kLayout_, Scalar_, WmmaShape_> - : public nvcuda::wmma::fragment< - /// The nvcuda::wmma operand name. - nvcuda::wmma::matrix_b, - /// The dimensions. - WmmaShape_::kW, - WmmaShape_::kH, - WmmaShape_::kD, - /// The scalar. - Scalar_, - /// The layout. - typename WmmaLayout<kLayout_>::Layout> { - /// This type. - typedef WmmaMatrix<GemmOperand::kB, kLayout_, Scalar_, WmmaShape_> This_; - - /// Fill-in the element. - CUTLASS_DEVICE This_& operator=(Scalar_ const& x) { - nvcuda::wmma::fill_fragment(*this, x); - return *this; - } - - /// Load from memory. - CUTLASS_DEVICE void load(Scalar_ const* pointer, int const stride) { - nvcuda::wmma::load_matrix_sync(*this, pointer, stride); - } - - /// Store to memory. - CUTLASS_DEVICE void store(Scalar_* pointer, int const stride) const { - nvcuda::wmma::store_matrix_sync(pointer, *this, stride); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/// Adapter to nvcuda::wmma fragment accessors for C operand -template <MatrixLayout::Kind kLayout_, typename Scalar_, typename WmmaShape_> -struct WmmaMatrix<GemmOperand::kC, kLayout_, Scalar_, WmmaShape_> - : public nvcuda::wmma::fragment< - /// The nvcuda::wmma operand name. - nvcuda::wmma::accumulator, - /// The dimensions. - WmmaShape_::kW, - WmmaShape_::kH, - WmmaShape_::kD, - /// The scalar. - Scalar_> { - /// This type. - typedef WmmaMatrix<GemmOperand::kC, kLayout_, Scalar_, WmmaShape_> This_; - /// The layout. - static MatrixLayout::Kind const kLayout = kLayout_; - - /// Fill-in the element. - CUTLASS_DEVICE This_& operator=(Scalar_ const& x) { - nvcuda::wmma::fill_fragment(*this, x); - return *this; - } - - /// Load from memory. - CUTLASS_DEVICE void load(Scalar_ const* pointer, int const stride) { - bool const kIsRowMajor = kLayout == MatrixLayout::kRowMajor; - nvcuda::wmma::load_matrix_sync( - *this, - pointer, - stride, - kIsRowMajor ? nvcuda::wmma::mem_row_major : nvcuda::wmma::mem_col_major); - } - - /// Store to memory. - CUTLASS_DEVICE void store(Scalar_* pointer, int const stride) const { - bool const kIsRowMajor = kLayout == MatrixLayout::kRowMajor; - nvcuda::wmma::store_matrix_sync( - pointer, - *this, - stride, - kIsRowMajor ? nvcuda::wmma::mem_row_major : nvcuda::wmma::mem_col_major); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass - -#endif // defined CUTLASS_USE_WMMA_API diff --git a/cutlass-example/cutlass_example.cu b/cutlass-example/cutlass_example.cu deleted file mode 100644 index 0e5722c..0000000 --- a/cutlass-example/cutlass_example.cu +++ /dev/null @@ -1,355 +0,0 @@ -//added by me -#include <cutlass/wmma_matrix.h> -#include <cutlass/gemm/gemm.h> -#include <cutlass/gemm/wmma_gemm_traits.h> -#include <gemm_testbed.h> -#include <gemm.h> - -int main(int argc, char* argv[]) { - -#ifdef WMMA_GEMM_16x16x16_NT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(16, 16, 16); -#endif - -#ifdef WMMA_GEMM_16x16x32_NT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(16, 16, 32); -#endif - -#ifdef WMMA_GEMM_16x16x16_NN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(16, 16, 16); -#endif - - -#ifdef WMMA_GEMM_16x16x32_NN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(16, 16, 32); -#endif - -#ifdef WMMA_GEMM_16x16x16_TT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(16, 16, 16); -#endif - -#ifdef WMMA_GEMM_16x16x32_TT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(16, 16, 32); -#endif - -#ifdef WMMA_GEMM_16x16x16_TN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(16, 16, 16); -#endif - -#ifdef WMMA_GEMM_16x16x32_TN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(16, 16, 32); -#endif - -#ifdef WMMA_GEMM_32x32x32_NT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(32, 32, 32); -#endif -#ifdef WMMA_GEMM_32x32x32_NN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(32, 32, 32); -#endif -#ifdef WMMA_GEMM_32x32x32_TT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(32, 32, 32); -#endif -#ifdef WMMA_GEMM_32x32x32_TN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(32, 32, 32); -#endif - -#ifdef WMMA_GEMM_128x128x128_NT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(128, 128, 128); -#endif -#ifdef WMMA_GEMM_128x128x128_NN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(128, 128, 128); -#endif -#ifdef WMMA_GEMM_128x128x128_TT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(128, 128, 128); -#endif -#ifdef WMMA_GEMM_128x128x128_TN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<32, 16, 16> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(128, 128, 128); -#endif - -#ifdef WMMA_16x16x16_GEMM_256x256x128_NT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<32, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(256, 256, 128); -#endif - -#ifdef WMMA_16x16x16_GEMM_256x256x128_NN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<32, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(256, 256, 128); -#endif - -#ifdef WMMA_16x16x16_GEMM_256x256x128_TT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<32, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(256, 256, 128); -#endif - -#ifdef WMMA_16x16x16_GEMM_256x256x128_TN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<32, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(256, 256, 128); -#endif - -#ifdef WMMA_GEMM_256x256x256_NT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(256, 256, 256); -#endif - -#ifdef WMMA_GEMM_256x256x256_NN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(256, 256, 256); -#endif - -#ifdef WMMA_GEMM_256x256x256_TT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(256, 256, 256); -#endif - -#ifdef WMMA_GEMM_256x256x256_TN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(256, 256, 256); -#endif -#ifdef WMMA_GEMM_512x512x512_NT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(512, 512, 512); -#endif - -#ifdef WMMA_GEMM_512x512x512_NN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(512, 512, 512); -#endif - -#ifdef WMMA_GEMM_512x512x512_TT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(512, 512, 512); -#endif - -#ifdef WMMA_GEMM_512x512x512_TN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(512, 512, 512); -#endif - -#ifdef WMMA_GEMM_768x768x768_NT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(768, 768, 768); -#endif -#ifdef WMMA_GEMM_768x768x768_NN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(768, 768, 768); -#endif - -#ifdef WMMA_GEMM_768x768x768_TT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(768, 768, 768); -#endif - -#ifdef WMMA_GEMM_768x768x768_TN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(768, 768, 768); -#endif - - -#ifdef WMMA_GEMM_1024x1024x1024_NT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(1024, 1024, 1024); -#endif - -#ifdef WMMA_GEMM_1024x1024x1024_NN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(1024, 1024, 1024); -#endif - -#ifdef WMMA_GEMM_1024x1024x1024_TT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(1024, 1024, 1024); -#endif - -#ifdef WMMA_GEMM_1024x1024x1024_TN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(1024, 1024, 1024); -#endif -#ifdef WMMA_GEMM_2048x2048x2048_NT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(2048, 2048, 2048); -#endif - -#ifdef WMMA_GEMM_2048x2048x2048_NN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(2048, 2048, 2048); -#endif - -#ifdef WMMA_GEMM_2048x2048x2048_TT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(2048, 2048, 2048); -#endif - -#ifdef WMMA_GEMM_2048x2048x2048_TN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(2048, 2048, 2048); -#endif -#ifdef WMMA_GEMM_4096x4096x4096_NT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(4096, 4096, 4096); -#endif - -#ifdef WMMA_GEMM_4096x4096x4096_NN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(4096, 4096, 4096); -#endif - -#ifdef WMMA_GEMM_4096x4096x4096_TT - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kRowMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(4096, 4096, 4096); -#endif - -#ifdef WMMA_GEMM_4096x4096x4096_TN - typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor, - cutlass::MatrixLayout::kColumnMajor, - cutlass::Shape<64, 128, 128> > - WmmaGemmTraits; - run_gemm<WmmaGemmTraits>(4096, 4096, 4096); -#endif -} - - diff --git a/cutlass-example/device_memory.h b/cutlass-example/device_memory.h deleted file mode 100644 index ab561d8..0000000 --- a/cutlass-example/device_memory.h +++ /dev/null @@ -1,178 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are not permitted. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#pragma once - -/** - * \file - * \brief C++ interface to CUDA device memory management functions. - */ - -#include <memory> - -#include <cutlass/util/debug.h> -#include <cutlass/util/platform.h> -#include <exceptions.h> - -namespace cutlass { -namespace device_memory { - -/****************************************************************************** - * Allocation lifetime - ******************************************************************************/ - -/// Allocate a buffer of \p count elements of type \p T on the current CUDA device -template <typename T> -T* allocate(size_t count = 1) { - T* ptr = 0; - size_t bytes = sizeof(T) * count; - - cudaError_t cuda_error = CUDA_PERROR(cudaMalloc((void**)&ptr, bytes)); - if (cuda_error != cudaSuccess) { - throw cuda_exception("Failed to allocate memory", cuda_error); - } - - return ptr; -} - -/// Free the buffer pointed to by \p ptr -template <typename T> -void free(T* ptr) { - if (ptr) { - cudaError_t cuda_error = CUDA_PERROR(cudaFree(ptr)); - if (cuda_error != cudaSuccess) { - throw cuda_exception("Failed to free device memory", cuda_error); - } - } -} - -/****************************************************************************** - * Data movement - ******************************************************************************/ - -template <typename T> -void copy(T* dst, T const* src, size_t count, cudaMemcpyKind kind) { - size_t bytes = count * sizeof(T); - - cudaError_t cuda_error = CUDA_PERROR(cudaMemcpy(dst, src, bytes, kind)); - if (cuda_error != cudaSuccess) { - throw cuda_exception("cudaMemcpy() failed", cuda_error); - } -} - -template <typename T> -void copy_to_device(T* dst, T const* src, size_t count = 1) { - copy(dst, src, count, cudaMemcpyHostToDevice); -} - -template <typename T> -void copy_to_host(T* dst, T const* src, size_t count = 1) { - copy(dst, src, count, cudaMemcpyDeviceToHost); -} - -template <typename T> -void copy_device_to_device(T* dst, T const* src, size_t count = 1) { - copy(dst, src, count, cudaMemcpyDeviceToDevice); -} - -/// Copies elements from device memory to host-side range -template <typename OutputIterator, typename T> -void insert_to_host(OutputIterator begin, OutputIterator end, T const* device_begin) { - size_t elements = end - begin; - copy_to_host(&*begin, device_begin, elements); -} - -/// Copies elements to device memory from host-side range -template <typename T, typename InputIterator> -void insert_to_device(T* device_begin, InputIterator begin, InputIterator end) { - size_t elements = end - begin; - copy_to_device(device_begin, &*begin, elements); -} - -/****************************************************************************** - * "Smart" device memory allocation - ******************************************************************************/ - -/// Device allocation abstraction that tracks size and capacity -template <typename T> -struct allocation { - /// Delete functor for CUDA device memory - struct deleter { - void operator()(T* ptr) { - cudaError_t cuda_error = CUDA_PERROR(cudaFree(ptr)); - if (cuda_error != cudaSuccess) { - // noexcept - // throw cuda_exception("cudaFree() failed", cuda_error); - return; - } - } - }; - - /// Number of elements of T allocated on the current CUDA device - size_t capacity; - - /// Smart pointer - platform::unique_ptr<T, deleter> smart_ptr; - - // - // - // - - /// Constructor: allocates no memory - allocation() : capacity(0) {} - - /// Constructor: allocates \p capacity elements on the current CUDA device - allocation(size_t _capacity) : smart_ptr(allocate<T>(_capacity)), capacity(_capacity) {} - - /// Destructor - ~allocation() { reset(); } - - /// Returns a pointer to the managed object - T* get() const { return smart_ptr.get(); } - - /// Releases the ownership of the managed object (without deleting) and resets capacity to zero - T* release() { - capacity = 0; - return smart_ptr.release(); - } - - /// Deletes the managed object and resets capacity to zero - void reset() { - capacity = 0; - smart_ptr.reset(); - } - - /// Deletes managed object, if owned, and replaces its reference with a given pointer and capacity - void reset(T* _ptr, size_t _capacity) { - smart_ptr.reset(_ptr); - capacity = _capacity; - } - - /// Returns a pointer to the object owned by *this - T* operator->() const { return smart_ptr.get(); } - - /// Returns the deleter object which would be used for destruction of the managed object. - deleter& get_deleter() { return smart_ptr.get_deleter(); } - - /// Returns the deleter object which would be used for destruction of the managed object (const) - const deleter& get_deleter() const { return smart_ptr.get_deleter(); } -}; - -} // namespace device_memory -} // namespace cutlass diff --git a/cutlass-example/exceptions.h b/cutlass-example/exceptions.h deleted file mode 100644 index 72d99fe..0000000 --- a/cutlass-example/exceptions.h +++ /dev/null @@ -1,62 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are not permitted. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#pragma once - -/** - * \file - * \brief C++ exception semantics for CUDA error codes - */ - -#include <cuda_runtime.h> -#include <iosfwd> -#include <stdexcept> - -#include <cutlass/util/platform.h> - -namespace cutlass { - -/// C++ exception wrapper for CUDA \p cudaError_t -class cuda_exception : public std::exception { - public: - /// Constructor - cuda_exception(const char* msg = "", cudaError_t err = cudaErrorUnknown) : msg(msg), err(err) {} - - /// Returns the underlying CUDA \p cudaError_t - cudaError_t cudaError() const { return err; } - - protected: - /// Explanatory string - const char* msg; - - /// Underlying CUDA \p cudaError_t - cudaError_t err; -}; - -/// Writes a cudaError_t to an output stream -inline std::ostream& operator<<(std::ostream& out, cudaError_t result) { - return out << cudaGetErrorString(result); -} - -/// Writes a cuda_exception instance to an output stream -inline std::ostream& operator<<(std::ostream& out, cuda_exception const& e) { - return out << e.what() << ": " << e.cudaError(); -} - -} // namespace cutlass diff --git a/cutlass-example/executionFlow b/cutlass-example/executionFlow deleted file mode 100644 index 356afed..0000000 --- a/cutlass-example/executionFlow +++ /dev/null @@ -1,262 +0,0 @@ -mov.b64%rd33, _ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0; -mov.u64 %rd1, %rd33; -ld.param.u64 %rd34, [_ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0+48]; -cvta.to.global.u64 %rd2, %rd34; -mov.u32 %r301, %ctaid.x; -shl.b32 %r1, %r301, 4; -ld.param.u64 %rd36, [_ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0+136]; -cvta.to.global.u64 %rd4, %rd36; -mov.u32 %r302, %ctaid.y; -shl.b32 %r2, %r302, 4; -mov.u32 %r3, %tid.x; -shr.u32 %r4, %r3, 1; -and.b32 %r303, %r3, 1; -shl.b32 %r5, %r303, 3; -add.s32 %r6, %r5, %r1; -ld.param.u32 %r7, [_ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0+20]; -mad.lo.s32 %r304, %r7, %r4, %r6; -cvt.s64.s32%rd5, %r304; -mul.wide.s32 %rd37, %r304, 2; -add.s64 %rd6, %rd2, %rd37; -ld.param.u32 %r8, [_ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0]; -sub.s32 %r305, %r8, %r6; -ld.param.u32 %r9, [_ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0+8]; -setp.gt.s32%p2, %r9, 0; -setp.gt.s32%p3, %r305, 0; -and.pred %p4, %p3, %p2; -selp.u32%r306, 1, 0, %p4; -setp.gt.s32%p5, %r9, 16; -and.pred %p6, %p3, %p5; -selp.u16%rs1, 1, 0, %p6; -mul.wide.u16 %r10, %rs1, 2; -selp.u32%r307, -1, 0, %p6; -bfi.b32 %r481, %r307, %r306, 1, 1; -bfe.u32 %r308, %r3, 1, 4; -mad.lo.s32 %r12, %r308, 24, %r5; -setp.lt.s32%p7, %r12, 768; -@%p7 bra BB0_2; -ld.param.u32 %r310, [%rd1+4]; -ld.param.u32 %r13, [%rd1+108]; -add.s32 %r16, %r5, %r2; -mad.lo.s32 %r311, %r13, %r4, %r16; -cvt.s64.s32%rd7, %r311; -mul.wide.s32 %rd45, %r311, 2; -add.s64 %rd8, %rd4, %rd45; -sub.s32 %r312, %r310, %r16; -setp.gt.s32%p9, %r312, 0; -and.pred %p11, %p9, %p2; -selp.u32%r313, 1, 0, %p11; -and.pred %p13, %p9, %p5; -selp.u16%rs2, 1, 0, %p13; -mul.wide.u16 %r17, %rs2, 2; -selp.u32%r314, -1, 0, %p13; -bfi.b32 %r480, %r314, %r313, 1, 1; -@%p7 bra BB0_4; -setp.lt.s32%p14, %r9, 32; -@%p14 bra BB0_7; -setp.lt.s32%p18, %r4, %r9; -selp.b32%r321, %r481, %r10, %p18; -add.s32 %r322, %r4, 16; -setp.lt.s32%p19, %r322, %r9; -and.b32 %r323, %r321, 1; -selp.b32%r481, %r321, %r323, %p19; -selp.b32%r324, %r480, %r17, %p18; -and.b32 %r325, %r324, 1; -selp.b32%r480, %r324, %r325, %p19; -bra.uni BB0_8; -and.b32 %r330, %r481, 1; -setp.eq.b32%p20, %r330, 1; -mov.u32 %r571, 0; -mov.u32 %r559, %r571; -mov.u32 %r560, %r571; -mov.u32 %r561, %r571; -mov.u32 %r562, %r571; -@!%p20 bra BB0_10; -bra.uni BB0_9; -ld.global.v4.u32 {%r561, %r562, %r559, %r560}, [%rd6]; -ld.param.u32 %r33, [%rd1+32]; -cvt.s64.s32%rd53, %r33; -add.s64 %rd9, %rd5, %rd53; -shl.b64 %rd54, %rd9, 1; -add.s64 %rd10, %rd2, %rd54; -and.b32 %r339, %r481, 2; -setp.eq.s32%p21, %r339, 0; -mov.u32 %r563, %r571; -mov.u32 %r564, %r571; -mov.u32 %r565, %r571; -mov.u32 %r566, %r571; -@%p21 bra BB0_12; -and.b32 %r348, %r480, 1; -setp.eq.b32%p22, %r348, 1; -mov.u32 %r567, %r571; -mov.u32 %r568, %r571; -mov.u32 %r569, %r571; -mov.u32 %r570, %r571; -@!%p22 bra BB0_14; -bra.uni BB0_13; -ld.global.v4.u32 {%r569, %r570, %r567, %r568}, [%rd8]; -ld.param.u32 %r50, [%rd1+120]; -cvt.s64.s32%rd55, %r50; -add.s64 %rd56, %rd7, %rd55; -shl.b64 %rd57, %rd56, 1; -add.s64 %rd11, %rd4, %rd57; -and.b32 %r357, %r480, 2; -setp.eq.s32%p23, %r357, 0; -mov.u32 %r572, %r571; -mov.u32 %r573, %r571; -mov.u32 %r574, %r571; -@%p23 bra BB0_16; -shl.b32 %r362, %r12, 1; -mov.u32 %r363, _ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE$__cuda_local_var_29658_57_non_const_shared_storage; -add.s32 %r59, %r363, %r362; -st.shared.v4.u32 [%r59], {%r561, %r562, %r559, %r560}; -st.shared.v4.u32 [%r59+768], {%r565, %r566, %r563, %r564}; -st.shared.v4.u32 [%r59+1536], {%r569, %r570, %r567, %r568}; -st.shared.v4.u32 [%r59+2304], {%r573, %r574, %r571, %r572}; -bar.sync 0; -shr.u32 %r60, %r3, 5; -shl.b32 %r61, %r60, 4; -setp.lt.u32%p24, %r61, 768; -@%p24 bra BB0_18; -mul.wide.s32 %rd65, %r61, 2; -cvt.u64.u32 %temp, %r363; -cvta.shared.u64 %rd66, %temp; -add.s64 %rd67, %rd66, %rd65; -add.s64 %rd12, %rd67, 1536; -mov.u32 %r374, 24; -wmma.load.a.sync.col.m16n16k16.f16 {%r590, %r589, %r588, %r587, %r586, %r585, %r584, %r583}, [%rd66], %r374; -wmma.load.b.sync.row.m16n16k16.f16 {%r582, %r581, %r580, %r579, %r578, %r577, %r576, %r575}, [%rd12], %r374; -add.s32 %r599, %r9, -32; -mov.u32 %r609, 0; -setp.lt.s32%p25, %r599, 1; -@%p25 bra BB0_19; -mov.u32 %r610, %r609; -mov.u32 %r611, %r609; -mov.u32 %r612, %r609; -mov.u32 %r613, %r609; -mov.u32 %r614, %r609; -mov.u32 %r615, %r609; -mov.u32 %r616, %r609; -setp.lt.s32%p34, %r599, -31; -@%p34 bra BB0_35; -mov.u32 %r477, %tid.x; -shr.u32 %r476, %r477, 5; -shl.b32 %r475, %r476, 4; -add.s32 %r434, %r475, 384; -mul.wide.s32 %rd96, %r434, 2; -add.s64 %rd97, %rd66, %rd96; -add.s64 %rd26, %rd97, 1536; -add.s64 %rd99, %rd66, 768; -wmma.load.a.sync.col.m16n16k16.f16 {%r257, %r256, %r255, %r254, %r250, %r251, %r253, %r252}, [%rd99], %r374; -wmma.load.b.sync.row.m16n16k16.f16 {%r265, %r264, %r263, %r262, %r258, %r259, %r261, %r260}, [%rd26], %r374; -mov.b32 %f63, %r615; -mov.b32 %f64, %r616; -mov.b32 %f65, %r613; -mov.b32 %f66, %r614; -mov.b32 %f67, %r611; -mov.b32 %f68, %r612; -mov.b32 %f69, %r609; -mov.b32 %f70, %r610; -wmma.mma.sync.col.row.m16n16k16.f32.f32 {%f10, %f11, %f12, %f13, %f14, %f15, %f16, %f17}, {%r590, %r589, %r588, %r587, %r586, %r585, %r584, %r583}, {%r582, %r581, %r580, %r579, %r578, %r577, %r576, %r575}, {%f70, %f69, %f68, %f67, %f66, %f65, %f64, %f63}; -bar.sync 0; -bar.sync 0; -wmma.mma.sync.col.row.m16n16k16.f32.f32 {%f71, %f72, %f73, %f74, %f75, %f76, %f77, %f78}, {%r257, %r256, %r255, %r254, %r250, %r251, %r253, %r252}, {%r265, %r264, %r263, %r262, %r258, %r259, %r261, %r260}, {%f10, %f11, %f12, %f13, %f14, %f15, %f16, %f17}; -mov.b32 %r610, %f71; -mov.b32 %r609, %f72; -mov.b32 %r612, %f73; -mov.b32 %r611, %f74; -mov.b32 %r614, %f75; -mov.b32 %r613, %f76; -mov.b32 %r616, %f77; -mov.b32 %r615, %f78; -add.s32 %r599, %r599, -32; -setp.gt.s32%p35, %r599, -32; -@%p35 bra BB0_34; -mov.b64%rd145, _ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0; -mov.u64 %rd144, %rd145; -mov.u32 %r472, %tid.x; -ld.param.f32 %f145, [%rd144+428]; -ld.param.u32 %r471, [%rd144+324]; -shr.u32 %r470, %r472, 5; -ld.param.u32 %r469, [%rd144+332]; -mov.u32 %r468, %ctaid.x; -shl.b32 %r467, %r468, 4; -mov.u32 %r466, _ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE$__cuda_local_var_29658_57_non_const_shared_storage; -mov.u32 %r465, %ctaid.y; -shl.b32 %r464, %r465, 4; -ld.param.v2.u32 {%r438, %r439}, [%rd144+288]; -ld.param.u32 %r284, [%rd144+300]; -ld.param.u32 %r440, [%rd144+320]; -ld.param.u32 %r285, [%rd144+336]; -ld.param.f32 %f18, [%rd144+424]; -ld.param.u32 %r287, [%rd144+304]; -ld.param.u64 %rd100, [%rd144+312]; -cvta.to.global.u64 %rd101, %rd100; -and.b32 %r441, %r472, 3; -shl.b32 %r288, %r441, 2; -shr.u32 %r289, %r472, 2; -add.s32 %r290, %r289, %r464; -add.s32 %r291, %r288, %r467; -mad.lo.s32 %r442, %r290, %r440, %r291; -cvt.s64.s32%rd102, %r442; -mul.wide.s32 %rd103, %r442, 4; -add.s64 %rd27, %rd101, %rd103; -sub.s32 %r292, %r469, %r290; -shl.b32 %r293, %r470, 8; -shl.b32 %r443, %r289, 4; -add.s32 %r294, %r443, %r288; -shl.b32 %r444, %r294, 2; -add.s32 %r295, %r466, %r444; -cvt.s64.s32%rd104, %r471; -add.s64 %rd105, %rd102, %rd104; -shl.b64 %rd106, %rd105, 2; -add.s64 %rd28, %rd101, %rd106; -setp.eq.f32%p36, %f145, 0f00000000; -@%p36 bra BB0_48; -setp.lt.s32%p51, %r293, 256; -@%p51 bra BB0_50; -setp.lt.s32%p52, %r294, 256; -@%p52 bra BB0_52; -ld.param.u32 %r474, [_ZN7cutlass4gemm11gemm_kernelINS0_4GemmINS0_14WmmaGemmTraitsILNS_12MatrixLayout4KindE1ELS5_0ENS_5ShapeILi32ELi16ELi16ELi1EEEfNS0_13LinearScalingIfNS0_19FragmentMultiplyAddIfEEEEfS7_NS6_ILi16ELi16ELi16ELi1EEELi8ELi8EiNS0_20WmmaGemmTraitsHelperILS5_1ELS5_0ES7_ffSB_S7_SC_Li8ELi8EiEEEEEEEEvNT_6ParamsE_param_0]; -mul.wide.s32 %rd142, %r293, 4; -add.s64 %rd32, %rd66, %rd142; -setp.lt.s32%p1, %r291, %r474; -bar.sync 0; -mov.b32 %f121, %r610; -mov.b32 %f122, %r609; -mov.b32 %f123, %r612; -mov.b32 %f124, %r611; -mov.b32 %f125, %r614; -mov.b32 %f126, %r613; -mov.b32 %f127, %r616; -mov.b32 %f128, %r615; -mov.u32 %r458, 16; -wmma.store.d.sync.col.m16n16k16.f32 [%rd32], {%f121, %f122, %f123, %f124, %f125, %f126, %f127, %f128}, %r458; -bar.sync 0; -ld.shared.v4.u32 {%r459, %r460, %r461, %r462}, [%r295+512]; -setp.gt.s32%p53, %r292, 0; -and.pred %p54, %p1, %p53; -@!%p54 bra BB0_54; -bra.uni BB0_53; -ld.shared.v4.f32 {%f129, %f130, %f131, %f132}, [%r295]; -mul.f32 %f137, %f18, %f132; -mul.f32 %f138, %f18, %f131; -mul.f32 %f139, %f18, %f130; -mul.f32 %f140, %f18, %f129; -st.global.v4.f32 [%rd27], {%f140, %f139, %f138, %f137}; -mov.b32 %f141, %r459; -mul.f32 %f43, %f18, %f141; -mov.b32 %f142, %r460; -mul.f32 %f44, %f18, %f142; -mov.b32 %f143, %r461; -mul.f32 %f45, %f18, %f143; -mov.b32 %f144, %r462; -mul.f32 %f46, %f18, %f144; -sub.s32 %r463, %r292, %r285; -setp.gt.s32%p56, %r463, 0; -and.pred %p57, %p1, %p56; -@!%p57 bra BB0_56; -bra.uni BB0_55; -st.global.v4.f32 [%rd28], {%f43, %f44, %f45, %f46}; -bra.uni BB0_56; -ret; diff --git a/cutlass-example/gemm.h b/cutlass-example/gemm.h deleted file mode 100644 index 18dfdf6..0000000 --- a/cutlass-example/gemm.h +++ /dev/null @@ -1,152 +0,0 @@ -/*************************************************************************************************** -* Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. -* -* Redistribution and use in source and binary forms, with or without modification, are permitted -* provided that the following conditions are met: -* * Redistributions of source code must retain the above copyright notice, this list of -* conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright notice, this list of -* conditions and the following disclaimer in the documentation and/or other materials -* provided with the distribution. -* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used -* to endorse or promote products derived from this software without specific prior written -* permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR -* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE -* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -* -**************************************************************************************************/ - -#include <cutlass/cutlass.h> -#include <gemm_testbed.h> - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmTraits_> -static void run_gemm( - int m, - int n, - int k, - int lda, - int ldb, - int ldc, - typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha = - typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1), - typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta = - typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0)) { - typedef cutlass::gemm::Gemm<GemmTraits_> Gemm; - typename Gemm::Params params; - - printf("run_gemm-2:m=%d\n",m); - printf("run_gemm-2:n=%d\n",n); - printf("run_gemm-2:k=%d\n",k); - printf("run_gemm-2:lda=%d\n",lda); - printf("run_gemm-2:ldb=%d\n",ldb); - printf("run_gemm-2:ldc=%d\n",ldc); - printf("run_gemm-2:alpha=%.2f\n",alpha); - printf("run_gemm-2:beta=%.2f\n",beta); - - test::GemmTestbed< - typename test::GemmTestbedTraits< - typename GemmTraits_::GemmConfig::ScalarA>::host_type, // AType - typename test::GemmTestbedTraits< - typename GemmTraits_::GemmConfig::ScalarB>::host_type, // BType - typename test::GemmTestbedTraits< - typename GemmTraits_::Epilogue::ScalarC>::host_type, // CType - typename test::GemmTestbedTraits< - typename GemmTraits_::Epilogue::Accumulators::Element>::host_type, // Accumulator - typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type // Scalar - > - testbed(m, - n, - k, - lda, - ldb, - ldc, - cutlass::convert(GemmTraits_::kLayoutA), - cutlass::convert(GemmTraits_::kLayoutB), - alpha, - beta); - - testbed.initialize(); - - // if (testbed.has_cublas_support()) { - // EXPECT_TRUE(testbed.verify_host_with_cublas()); - // } - - params.initialize(testbed.M(), - testbed.N(), - testbed.K(), - testbed.alpha, - testbed.ptr_A(), - testbed.lda(), - testbed.ptr_B(), - testbed.ldb(), - testbed.beta, - testbed.ptr_C_initial(), - testbed.ldc(), - testbed.ptr_computed(), - testbed.ldc()); - - printf("SIZE_OF_PARAM=%lu\n",sizeof(params)); - void *ptr =¶ms; - for(int kk=0;kk<108;kk++){ - printf("KERNELPARAM:%d:%08x\n",kk,*((((int *) ptr)+kk))); - } - printf("m=%lu\n",sizeof(params.m)); - printf("n=%lu\n",sizeof(params.n)); - printf("k=%lu\n",sizeof(params.k)); -// printf("alpha=%d\n",sizeof(params.alpha)); -// printf("beta=%d\n",sizeof(params.beta)); -// printf("d_a=%d\n",sizeof(params.d_a)); -// printf("lda=%d\n",sizeof(params.lda)); -// printf("d_b=%d\n",sizeof(params.d_b)); -// printf("ldb=%d\n",sizeof(params.ldb)); -// printf("d_c=%d\n",sizeof(params.d_c)); -// printf("ldc=%d\n",sizeof(params.ldc)); -// printf("d_d=%d\n",sizeof(params.d_d)); -// printf("ldd=%d\n",sizeof(params.ldd)); - Gemm::launch(params); - - cudaError_t result = cudaDeviceSynchronize(); - if(result==cudaSuccess){ - printf("Successfully Launched\n"); - } - int save=1; - int myval=testbed.verify_with_host(save,save); - if(myval==1){ - printf("Result Verified\n"); - } - -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename GemmTraits_> -static void run_gemm( - int m, - int n, - int k, - typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha = - typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1), - typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta = - typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0)) { - int lda = GemmTraits_::kLayoutA == cutlass::MatrixLayout::kColumnMajor ? m : k; - int ldb = GemmTraits_::kLayoutB == cutlass::MatrixLayout::kColumnMajor ? k : n; - printf("run_gemm-1:m=%d\n",m); - printf("run_gemm-1:n=%d\n",n); - printf("run_gemm-1:k=%d\n",k); - printf("run_gemm-1:alpha=%.2f\n",alpha); - printf("run_gemm-1:beta=%.2f\n",beta); - printf("run_gemm-1:lda=%d\n",lda); - printf("run_gemm-1:ldb=%d\n",ldb); - run_gemm<GemmTraits_>(m, n, k, lda, ldb, m, alpha, beta); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/cutlass-example/gemm_testbed.h b/cutlass-example/gemm_testbed.h deleted file mode 100644 index 97409b1..0000000 --- a/cutlass-example/gemm_testbed.h +++ /dev/null @@ -1,462 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Test environment for GEMM -*/ - -#pragma once - -#include <fstream> -#include <iomanip> -#include <sstream> -#include <string> - -#include <cutlass/matrix_traits.h> -#include <cutlass/util/platform.h> - -#include <host_tensor.h> -#include <tensor_view_io.h> -#include <type_traits.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <cutlass::GemmOperand::Kind kOperand_, - cutlass::MatrixLayout::Kind kLayout_, - typename Scalar_, - typename WmmaShape_> -struct WmmaMatrix; -} - -namespace test { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename T> -struct GemmTestbedTraits : public cutlass::TypeTraits<T> {}; - -template <cutlass::GemmOperand::Kind kOperand_, - cutlass::MatrixLayout::Kind kLayout_, - typename Scalar_, - typename WmmaShape_> -struct GemmTestbedTraits<cutlass::WmmaMatrix<kOperand_, kLayout_, Scalar_, WmmaShape_> > { - static cudaDataType_t const cublas_type = cutlass::TypeTraits<Scalar_>::cublas_type; - typedef Scalar_ host_type; - typedef Scalar_ device_type; - static inline double remove_negative_zero(double x) { return x == -0.0 ? 0.0 : x; } - static inline double to_print(double x) { return x; } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// -template <typename AType, typename BType, typename CType, typename Accumulator, typename Scalar> -struct GemmTestbed { - // - // Type definitions - // - - /// Host tensor for operand A - typedef cutlass::HostTensor<AType> HostTensorA; - - /// Host tensor for operand B - typedef cutlass::HostTensor<BType> HostTensorB; - - /// Host tensor for operand C - typedef cutlass::HostTensor<CType> HostTensorC; - - /// Functor to print errors - struct PrintErrors { - /// Equivalently sized integer type - typedef typename GemmTestbedTraits<CType>::integer_type integer_t; - - /// Output stream to write to - std::ostream& out; - - /// Reference tensor view - cutlass::HostTensorView<CType> const& reference; - - /// Computed tensor view - cutlass::HostTensorView<CType> const& experimental; - - /// Errors greater than or this amount result in printing - integer_t ulps_threshold; - - /// - PrintErrors(std::ostream& _out, - cutlass::HostTensorView<CType> const& _reference, - cutlass::HostTensorView<CType> const& _experimental, - integer_t _ulps_threshold = 1) - : out(_out), - reference(_reference), - experimental(_experimental), - ulps_threshold(_ulps_threshold) {} - - /// Compares one element - void operator()(CType const& element, typename HostTensorC::Coord_t coord) { - CType exp = experimental.at(coord); - CType ref = reference.at(coord); - - int64_t int_exp = 0; - int64_t int_ref = 0; - - *reinterpret_cast<CType*>(&int_exp) = exp; - *reinterpret_cast<CType*>(&int_ref) = ref; - - integer_t ulps = integer_t(int_exp - int_ref); - - if (std::abs(ulps) >= ulps_threshold) { - // width in hexadecimal digits of value - int const width = sizeof(integer_t) * 2; - - double relative = double(exp) - double(ref); - if (ref != CType(0)) { - relative /= double(ref); - } - - out << "[" << coord << "] expected: " << GemmTestbedTraits<CType>::to_print(ref) << " (0x" - << std::hex << std::setw(width) << std::setfill('0') << integer_t(int_ref) << std::dec - << ")" - << ", got: " << GemmTestbedTraits<CType>::to_print(exp) << " (0x" << std::hex - << std::setw(width) << std::setfill('0') << integer_t(int_exp) << std::dec << ")" - << " relative error: " << relative << ", ulps: " << ulps << "\n"; - } - } - }; - - /// Generates random elements - template <typename T> - struct RandomGenerator { - RandomGenerator(int seed = -1, bool only_ones_ = false) : only_ones(only_ones_) { srand(seed); } - - T operator()() { - if (only_ones) { - return T(1); - } else { - int val = (rand() % 16) - 8; - return T(val); - } - } - - bool only_ones; - }; - - // - // Data members - // - - /// Status - //cublasStatus_t status; - - /// cuBLAS handle - //cublasHandle_t handle; - - /// cuBLAS GEMM algorithm selector - //cublasGemmAlgo_t algorithm; - - /// A matrix operand - HostTensorA A; - - /// Layout of A matrix - //cublasOperation_t layout_A; - - /// B matrix operand - HostTensorB B; - - /// Layout of B matrix - //cublasOperation_t layout_B; - - /// C matrix operand - HostTensorC C_initial; - - /// Reference result computed on the host - cutlass::HostTensor<CType, false> ref_host; - - - /// Computed result - HostTensorC computed; - - /// Linear scalaring factor - Scalar alpha; - - /// Linear scaling factor - Scalar beta; - - // - // Static helpers - // - //template <typename T, bool DeviceBacked> - //static void resize(cutlass::HostTensor<T, DeviceBacked>& tensor, - // int rows, - // int columns, - // cublasOperation_t layout, - // int ldm = 0) { - // if (!ldm) { - // ldm = (layout == CUBLAS_OP_N ? rows : columns); - // } - - // typedef cutlass::Coord<cutlass::HostTensor<T>::Rank> Coord_t; - - // size_t matrix_stride = layout == CUBLAS_OP_N ? columns * ldm : rows * ldm; - // // TODO: Remove that (int) cast. - // Coord_t stride = cutlass::make_Coord( - // (int)matrix_stride, layout == CUBLAS_OP_N ? 1 : ldm, layout == CUBLAS_OP_N ? ldm : 1, 1); - // Coord_t size = cutlass::make_Coord(1, rows, columns, 1); - // tensor.reset(stride, size); - //} - - /// Helper to resize a matrix with a given size and layout - template <typename T, bool DeviceBacked> - static void resize(cutlass::HostTensor<T, DeviceBacked>& tensor, - int rows, - int columns, - int layout, - int ldm = 0) { - if (!ldm) { - ldm = (layout ? rows : columns); - } - - typedef cutlass::Coord<cutlass::HostTensor<T>::Rank> Coord_t; - size_t matrix_stride = layout ? columns * ldm : rows * ldm; - // TODO: Remove that (int) cast. - Coord_t stride = cutlass::make_Coord( - (int)matrix_stride, layout ? 1 : ldm, layout? ldm : 1, 1); - Coord_t size = cutlass::make_Coord(1, rows, columns, 1); - tensor.reset(stride, size); - } - - // - // Methods - // - - /// Constructs a workspace for verifying GEMM. - GemmTestbed(int M_, - int N_, - int K_, - int lda, - int ldb, - int ldc, - int layout_a, - int layout_b, - Scalar alpha_ = Scalar(1), - Scalar beta_ = Scalar(0)) - //cublasGemmAlgo_t algorithm_ = CUBLAS_GEMM_DEFAULT, - //cublasOperation_t layout_c = CUBLAS_OP_N) - : alpha(alpha_), beta(beta_) { - //status = cublasCreate(&handle); - //if (status != CUBLAS_STATUS_SUCCESS) { - // throw cutlass::cuda_exception("Failed to create CUBLAS handle"); - //} - printf("GemmTestbed:alpha=%f\n",alpha_); - printf("GemmTestbed:beta=%f\n",beta_); - printf("GemmTestbed:lda=%d\n",lda); - printf("GemmTestbed:ldb=%d\n",ldb); - printf("GemmTestbed:ldc=%d\n",ldc); - - resize(A, M_, K_,layout_a, lda); - resize(B, K_, N_,layout_b, ldb); - resize(C_initial, M_, N_,1, ldc); - resize(ref_host, M_, N_,1, ldc); - resize(computed, M_, N_,1, ldc); - } - - /// Returns a pointer to the A operand - typename HostTensorA::DeviceType* ptr_A() const { - return A.device_data(); - } - - /// Stride of A matrix - int lda() const { - printf("lda()=%d\n",std::max(A.stride(HostTensorA::Dim_H), A.stride(HostTensorA::Dim_W))); - return std::max(A.stride(HostTensorA::Dim_H), A.stride(HostTensorA::Dim_W)); - } - - /// Returns a pointer to the B operand - typename HostTensorB::DeviceType* ptr_B() const { - return B.device_data(); - } - - /// Stride of B matrix - int ldb() const { - printf("ldb()=%d\n",std::max(B.stride(HostTensorB::Dim_H), B.stride(HostTensorB::Dim_W))); - return std::max(B.stride(HostTensorB::Dim_H), B.stride(HostTensorB::Dim_W)); - } - - /// Returns a pointer to the initial state of the result tensor in device memory - typename HostTensorC::DeviceType* ptr_C_initial() const { - return C_initial.device_data(); - } - - /// Returns a pointer to the result tensor in device memory - typename HostTensorC::DeviceType* ptr_computed() const { - return computed.device_data(); - } - - /// Stride of C matrix - int ldc() const { - printf("ldc()=%d\n",std::max(C_initial.stride(HostTensorC::Dim_H), C_initial.stride(HostTensorC::Dim_W))); - return std::max(C_initial.stride(HostTensorC::Dim_H), C_initial.stride(HostTensorC::Dim_W)); - } - - /// Returns the number of flops implied by the computation (1 multiply-accumulate = 2 flops) - uint64_t flops() const { return uint64_t(M()) * uint64_t(N()) * uint64_t(K()) * 2ULL; } - - /// Computes the speed of the computation in GFLOPs/s - double GFLOPs_per_sec(double runtime_ms) const { return double(flops()) / runtime_ms / 1.0e6; } - - /// Number of rows of problem - int M() const { - printf("M()=%d\n", C_initial.size(HostTensorC::Dim_H)); - return C_initial.size(HostTensorC::Dim_H); - } - - /// Number of columns of problem - int N() const { - printf("N()=%d\n", C_initial.size(HostTensorC::Dim_W)); - return C_initial.size(HostTensorC::Dim_W); - } - - /// Number of columns of problem - int K() const { - printf("K()=%d\n",A.size(HostTensorA::Dim_W)); - return A.size(HostTensorA::Dim_W); - } - - /// Initializes data, randomly - void initialize(int seed = -1) { - A.fill_random(RandomGenerator<AType>(seed)); - B.fill_random(RandomGenerator<BType>(seed + 11)); - C_initial.fill_random(RandomGenerator<CType>(seed + 13,1)); - } - - /// Computes the matrix product on the host - void compute_host() { - ref_host.fill(C_initial); - - std::string results_name = "host_results_before.txt"; - std::ofstream results(results_name.c_str()); - write(results); - - ref_host.template gemm<AType, BType, Accumulator, Scalar>(A, B, alpha, beta); - results_name = "host_results_after.txt"; - std::ofstream results2(results_name.c_str()); - write(results2); - } - - /// Names a probelm based on data type and problem size - std::string workspace_name() const { - std::stringstream ss; - ss << "gemm_" << "t" - << "t" << "_" << typeid(AType).name() << "_" - << typeid(BType).name() << "_" << typeid(CType).name() << "_" << typeid(Accumulator).name() - << "_" << typeid(Scalar).name() << "_" << M() << "x" << N() << "x" << K(); - - return ss.str(); - } - - /// Writes the workspace to an ostream - std::ostream& write(std::ostream& out) const { - out << "A = " << A << "\nB = " << B << "\nC_initial = " << C_initial - << "\ncomputed = " << computed - << "\nref_host= " << ref_host<< std::endl; - - return out; - } - - /// Outputs each mismatching element - std::ostream& write_errors(std::ostream& out, - cutlass::HostTensorView<CType> const& experimental, - cutlass::HostTensorView<CType> const& ref) const { - PrintErrors printer(out, ref, experimental); - - computed.visit(printer); - - return out; - } - - /// Sync's all input tensors to device - void sync_device() { - A.sync_device(); - B.sync_device(); - C_initial.sync_device(); - - ref_host.fill(C_initial); - computed.fill(C_initial); - - computed.sync_device(); - } - - /// Sync's all output tensors to host - void sync_host() { - computed.sync_host(); - } - - /// Saves the workspace to files - void save_workspace(cutlass::HostTensorView<CType> const& experimental, - cutlass::HostTensorView<CType> const& ref) { - std::string name = workspace_name(); - - std::string results_name = name + "_results.txt"; - std::string errors_name = name + "_errors.txt"; - - std::ofstream results(results_name.c_str()); - std::ofstream errors(errors_name.c_str()); - - write(results); - write_errors(errors, experimental, ref); - } - - /// Verifies the contents of C equal the host-side reference - bool verify_with_host(bool save_on_error = true, bool always_print = false) { - compute_host(); - computed.sync_host(); - - bool passed = computed.bit_equals(ref_host); - - if ((!passed && save_on_error) || always_print) { - save_workspace(computed, ref_host); - } - return passed; - } -}; - -} // namespace test - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -namespace cutlass { -inline int convert(cutlass::MatrixLayout::Kind layout) { - switch (layout) { - case cutlass::MatrixLayout::kRowMajor: - return 0;//CUBLAS_OP_T; - case cutlass::MatrixLayout::kColumnMajor: - return 1;//CUBLAS_OP_N; - default: - break; - } - return 1;//CUBLAS_OP_N; -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// -} diff --git a/cutlass-example/gpgpusim.config b/cutlass-example/gpgpusim.config deleted file mode 100644 index 3daa539..0000000 --- a/cutlass-example/gpgpusim.config +++ /dev/null @@ -1,151 +0,0 @@ -# This config models the Pascal GP102 (GeForceGTX 1080Ti) - -# functional simulator specification --gpgpu_ptx_instruction_classification 0 --gpgpu_ptx_sim_mode 0 --gpgpu_ptx_force_max_capability 70 - -# SASS execution (only supported with CUDA >= 4.0) --gpgpu_ptx_convert_to_ptxplus 0 --gpgpu_ptx_save_converted_ptxplus 0 - -# high level architecture configuration --gpgpu_n_clusters 80 --gpgpu_n_cores_per_cluster 1 --gpgpu_n_mem 11 --gpgpu_n_sub_partition_per_mchannel 2 - -# Pascal clock domains -#-gpgpu_clock_domains <Core Clock>:<Interconnect Clock>:<L2 Clock>:<DRAM Clock> -# Pascal NVIDIA TITAN X clock domains are adopted from -# https://en.wikipedia.org/wiki/GeForce_10_series --gpgpu_clock_domains 1481.0:2962.0:1481.0:2750.0 - -# shader core pipeline config --gpgpu_shader_registers 65536 - -# This implies a maximum of 64 warps/SM --gpgpu_shader_core_pipeline 2048:32 --gpgpu_shader_cta 32 --gpgpu_simd_model 1 - -# Pipeline widths and number of FUs -# ID_OC_SP,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_SFU,OC_EX_MEM,EX_WB -## Pascal GP102 has 4 SP SIMD units and 1 SFU unit -## we need to scale the number of pipeline registers to be equal to the number of SP units --gpgpu_pipeline_widths 4,1,1,1,1,4,1,1,1,1,6 --gpgpu_num_sp_units 4 --gpgpu_num_sfu_units 1 -# Instruction latencies and initiation intervals -# "ADD,MAX,MUL,MAD,DIV" -# SFU is 32-width in pascal, then dp units initiation is 1 cycle --ptx_opcode_latency_int 4,13,4,5,145,16,4 --ptx_opcode_initiation_int 1,2,2,2,8,16,4 --ptx_opcode_latency_fp 4,13,4,5,39 --ptx_opcode_initiation_fp 1,2,1,1,4 --ptx_opcode_latency_dp 8,19,8,8,330 --ptx_opcode_initiation_dp 1,2,1,1,130 - -# <nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>:<set_index_fn>,<mshr>:<N>:<merge>,<mq>:**<fifo_entry> -# ** Optional parameter - Required when mshr_type==Texture Fifo -# Note: Hashing set index function (H) only applies to a set size of 32 or 64. -# Pascal GP102 has 96KB Shared memory -# Pascal GP102 has 64KB L1 cache -# The default is to disable the L1 cache, unless cache modifieres is used --gpgpu_cache:dl1 64:128:6,L:L:m:N:H,A:128:8,8 --gpgpu_shmem_size 98304 --gmem_skip_L1D 1 - -# 64 sets, each 128 bytes 16-way for each memory sub partition (128 KB per memory sub partition). This gives 3MB L2 cache --gpgpu_cache:dl2 64:128:16,L:B:m:W:L,A:1024:1024,4:0,32 # used to be 128:4 --gpgpu_cache:dl2_texture_only 0 - -# 4 KB Inst. --gpgpu_cache:il1 8:128:4,L:R:f:N:L,A:2:48,4 -# 48 KB Tex --gpgpu_tex_cache:l1 16:128:24,L:R:m:N:L,F:128:4,128:2 -# 12 KB Const --gpgpu_const_cache:l1 128:64:2,L:R:f:N:L,A:2:64,4 - -# enable operand collector -## larger operand collectors and reg_banks are needed for the 4 warp schedulers and 4 SIMD units --gpgpu_operand_collector_num_units_sp 20 --gpgpu_operand_collector_num_units_sfu 4 -#-gpgpu_operand_collector_num_units_tensor_core 24 --gpgpu_operand_collector_num_units_mem 8 --gpgpu_operand_collector_num_in_ports_sp 4 --gpgpu_operand_collector_num_out_ports_sp 4 --gpgpu_operand_collector_num_in_ports_sfu 1 --gpgpu_operand_collector_num_out_ports_sfu 1 -#-gpgpu_operand_collector_num_in_ports_tensor_core 1 -#-gpgpu_operand_collector_num_out_ports_tensor_core 1 --gpgpu_operand_collector_num_in_ports_mem 10 --gpgpu_operand_collector_num_out_ports_mem 10 -# gpgpu_num_reg_banks should be increased to 32, but it gives an error! --gpgpu_num_reg_banks 32 - -# shared memory bankconflict detection --gpgpu_shmem_num_banks 32 --gpgpu_shmem_limited_broadcast 0 --gpgpu_shmem_warp_parts 1 - -## In Pascal, a warp scheduler can issue 2 insts per cycle --gpgpu_max_insn_issue_per_warp 2 - -# interconnection --network_mode 1 --inter_config_file config_fermi_islip.icnt - -# memory partition latency config --rop_latency 120 --dram_latency 100 - -# dram model config --gpgpu_dram_scheduler 1 -# The DRAM return queue and the scheduler queue together should provide buffer -# to sustain the memory level parallelism to tolerate DRAM latency -# To allow 100% DRAM utility, there should at least be enough buffer to sustain -# the minimum DRAM latency (100 core cycles). I.e. -# Total buffer space required = 100 x 924MHz / 700MHz = 132 --gpgpu_frfcfs_dram_sched_queue_size 64 --gpgpu_dram_return_queue_size 116 - -# for NVIDIA GeForceGTX 1080Ti, bus width is 352bits (11 DRAM chips x 32 bits) -# 11 memory paritions, 4 bytes (1 DRAM chip) per memory partition -# the atom size of GDDR5X (the smallest read request) is 32 bytes --gpgpu_n_mem_per_ctrlr 1 --gpgpu_dram_buswidth 4 --gpgpu_dram_burst_length 8 --dram_data_command_freq_ratio 4 # GDDR5X is QDR --gpgpu_mem_address_mask 1 --gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS - -# Use the same GDDR5 timing from hynix H5GQ1H24AFR -# disable bank groups for now, set nbkgrp to 1 and tCCDL and tRTPL to 0 --gpgpu_dram_timing_opt "nbk=16:CCD=2:RRD=6:RCD=12:RAS=28:RP=12:RC=40: - CL=12:WL=4:CDLR=5:WR=12:nbkgrp=1:CCDL=0:RTPL=0" - -# Pascal has four schedulers per core --gpgpu_num_sched_per_core 2 -# Two Level Scheduler with active and pending pools -#-gpgpu_scheduler two_level_active:6:0:1 -# Loose round robbin scheduler -#-gpgpu_scheduler lrr -# Greedy then oldest scheduler --gpgpu_scheduler gto - -# stat collection --gpgpu_memlatency_stat 14 --gpgpu_runtime_stat 500 --enable_ptx_file_line_stats 1 --visualizer_enabled 0 - -# power model configs --power_simulation_enabled 1 --gpuwattch_xml_file gpuwattch_gtx1080Ti.xml - -# tracing functionality -#-trace_enabled 1 -#-trace_components WARP_SCHEDULER,SCOREBOARD -#-trace_sampling_core 0 - diff --git a/cutlass-example/gpuwattch_gtx1080Ti.xml b/cutlass-example/gpuwattch_gtx1080Ti.xml deleted file mode 100755 index 02619ff..0000000 --- a/cutlass-example/gpuwattch_gtx1080Ti.xml +++ /dev/null @@ -1,538 +0,0 @@ -<?xml version="1.0" ?> -<component id="root" name="root"> - <component id="system" name="system"> - <!--McPAT will skip the components if number is set to 0 --> - <param name="GPU_Architecture" value="1"/><!-- 0-G80; 1-Fermi; others not supported --> - <param name="number_of_cores" value="28"/> - <param name="architecture" value="1"/> <!-- fermi:1 quadro:2 other: undefined--> - <param name="number_of_L1Directories" value="0"/> - <param name="number_of_L2Directories" value="0"/> - <param name="number_of_L2s" value="1"/> <!-- This number means how many L2 clusters in each cluster there can be multiple banks/ports --> - <param name="number_of_L3s" value="0"/> <!-- This number means how many L3 clusters --> - <param name="number_of_NoCs" value="1"/> - <param name="homogeneous_cores" value="1"/><!--1 means homo --> - <param name="homogeneous_L2s" value="1"/> - <param name="homogeneous_L1Directorys" value="1"/> - <param name="homogeneous_L2Directorys" value="1"/> - <param name="homogeneous_L3s" value="1"/> - <param name="homogeneous_ccs" value="1"/><!--cache coherece hardware --> - <param name="homogeneous_NoCs" value="1"/> - <param name="core_tech_node" value="23"/><!-- nm --> - <param name="target_core_clockrate" value="1481"/><!--MHz --> - <param name="temperature" value="380"/> <!-- Kelvin --> - <param name="number_cache_levels" value="2"/> - <param name="interconnect_projection_type" value="0"/><!--0: agressive wire technology; 1: conservative wire technology --> - <param name="device_type" value="0"/><!--0: HP(High Performance Type); 1: LSTP(Low standby power) 2: LOP (Low Operating Power) --> - <param name="longer_channel_device" value="1"/><!-- 0 no use; 1 use when possible --> - <param name="machine_bits" value="32"/> - <param name="virtual_address_width" value="32"/> - <param name="physical_address_width" value="32"/> - <param name="virtual_memory_page_size" value="4096"/> - <param name="idle_core_power" value="1.59"/><!-- idle core power for GTX479 --> - <!--param name="scaling_coefficients" value="10,0.0884816,10,10,8,10,4.12782,10,2.48832,10,10,10,4.29982,0.387764,0.0714269,0.14302,0.01,0.546811,0.485351,0.806633,0.818073,1.9207,100,100,100,87.9303,100,10,4.3548,10"/--> - <param name="TOT_INST" value="10" /> - <param name="FP_INT" value="10" /> - <param name="IC_H" value="0.001" /> - <param name="IC_M" value="10" /> - <param name="DC_RH" value="1" /> - <param name="DC_RM" value="1" /> - <param name="DC_WH" value="1" /> - <param name="DC_WM" value="1" /> - <param name="TC_H" value="0.001" /> - <param name="TC_M" value="10" /> - <param name="CC_H" value="4.5071" /> - <param name="CC_M" value="10" /> - <param name="SHRD_ACC" value="10" /> - <param name="REG_RD" value="1.6294" /> - <param name="REG_WR" value="0.5031" /> - <param name="NON_REG_OPs" value="0.01" /> - <param name="SP_ACC" value="10" /> - <param name="SFU_ACC" value="0.0082" /> - <param name="FPU_ACC" value="0.4126" /> - <param name="MEM_RD" value="0.1234" /> - <param name="MEM_WR" value="0.001" /> - <param name="MEM_PRE" value="0.001" /> - <param name="L2_RH" value="100" /> - <param name="L2_RM" value="100" /> - <param name="L2_WH" value="100" /> - <param name="L2_WM" value="42.6966" /> - <param name="NOC_A" value="100" /> - <param name="PIPE_A" value="44.8085" /> - <param name="IDLE_CORE_N" value="2.0382"/> - <param name="CONST_DYNAMICN" value="5.0005" /> - <stat name="num_idle_cores" value="0"/><!-- Average Number of idle cores during this period --> - <stat name="total_cycles" value="total_cycles_match_mcpat"/> - <stat name="idle_cycles" value="idle_cycles_match_mcpat"/> - <stat name="busy_cycles" value="busy_cycles_match_mcpat"/> - <!--This page size(B) is complete different from the page size in Main memo secction. this page size is the size of - virtual memory from OS/Archi perspective; the page size in Main memo secction is the actuall physical line in a DRAM bank --> - <!-- *********************** cores ******************* --> - <component id="system.core0" name="core0"> - <!-- Core property --> - <param name="clock_rate" value="1481"/> - <param name="instruction_length" value="32"/> - <param name="opcode_width" value="9"/> - <!-- address width determins the tag_width in Cache, LSQ and buffers in cache controller - default value is machine_bits, if not set --> - <param name="machine_type" value="1"/><!-- 1 inorder; 0 OOO--> - <!-- inorder/OoO --> - <param name="number_hardware_threads" value="32"/> - <!-- number_instruction_fetch_ports(icache ports) is always 1 in single-thread processor, - it only may be more than one in SMT processors. BTB ports always equals to fetch ports since - branch information in consective branch instructions in the same fetch group can be read out from BTB once.--> - <param name="fetch_width" value="1"/> - <!-- fetch_width determins the size of cachelines of L1 cache block --> - <param name="number_instruction_fetch_ports" value="1"/> - <param name="decode_width" value="1"/> - <!-- decode_width determins the number of ports of the - renaming table (both RAM and CAM) scheme --> - <param name="issue_width" value="2"/> - <!-- issue_width determins the number of ports of Issue window and other logic - as in the complexity effective proccessors paper; issue_width==dispatch_width --> - <param name="commit_width" value="2"/> - <!-- commit_width determins the number of ports of register files --> - <param name="fp_issue_width" value="1"/> - <param name="prediction_width" value="0"/> - <!-- number of branch instructions can be predicted simultannouesl--> - <!-- Current version of McPAT does not distinguish int and floating point pipelines - Theses parameters are reserved for future use.--> - <param name="pipelines_per_core" value="1,1"/> - <!--integer_pipeline and floating_pipelines, if the floating_pipelines is 0, then the pipeline is shared--> - <param name="pipeline_depth" value="8,8"/> - <!-- pipeline depth of int and fp, if pipeline is shared, the second number is the average cycles of fp ops --> - <!-- issue and exe unit--> - <param name="ALU_per_core" value="32"/> - <!-- contains an adder, a shifter, and a logical unit --> - <param name="MUL_per_core" value="4"/> - <!-- For MUL and Div --> - <param name="FPU_per_core" value="32"/> - <!-- buffer between IF and ID stage --> - <param name="instruction_buffer_size" value="1"/> - <!-- buffer between ID and sche/exe stage --> - <param name="decoded_stream_buffer_size" value="1"/> - <param name="instruction_window_scheme" value="0"/><!-- 0 PHYREG based, 1 RSBASED--> - <!-- McPAT support 2 types of OoO cores, RS based and physical reg based--> - <param name="instruction_window_size" value="1"/> - <param name="fp_instruction_window_size" value="1"/> - <!-- the instruction issue Q as in Alpha 21264; The RS as in Intel P6 --> - <param name="ROB_size" value="0"/> - <!-- each in-flight instruction has an entry in ROB --> - <!-- registers --> - <!-- SM parameters Added by Syed Gilani --> - <param name="rf_banks" value="32"/> - <param name="simd_width" value="32"/> - <param name="collector_units" value="32"/> - <param name="core_clock_ratio" value="2"/> - <param name="warp_size" value="32"/> - - <param name="archi_Regs_IRF_size" value="65536"/> - <param name="archi_Regs_FRF_size" value="32"/> - <!-- if OoO processor, phy_reg number is needed for renaming logic, - renaming logic is for both integer and floating point insts. --> - <param name="phy_Regs_IRF_size" value="32"/> - <param name="phy_Regs_FRF_size" value="32"/> - <!-- rename logic --> - <param name="rename_scheme" value="0"/> - <!-- can be RAM based(0) or CAM based(1) rename scheme - RAM-based scheme will have free list, status table; - CAM-based scheme have the valid bit in the data field of the CAM - both RAM and CAM need RAM-based checkpoint table, checkpoint_depth=# of in_flight instructions; - Detailed RAT Implementation see TR --> - <param name="register_windows_size" value="0"/> - <!-- how many windows in the windowed register file, sun processors; - no register windowing is used when this number is 0 --> - <!-- In OoO cores, loads and stores can be issued whether inorder(Pentium Pro) or (OoO)out-of-order(Alpha), - They will always try to exeute out-of-order though. --> - <param name="LSU_order" value="inorder"/> - <param name="store_buffer_size" value="32"/> - <!-- By default, in-order cores do not have load buffers --> - <param name="load_buffer_size" value="32"/> - <!-- number of ports refer to sustainable concurrent memory accesses --> - <param name="memory_ports" value="2"/> - <!-- max_allowed_in_flight_memo_instructions determins the # of ports of load and store buffer - as well as the ports of Dcache which is connected to LSU --> - <!-- dual-pumped Dcache can be used to save the extra read/write ports --> - <param name="RAS_size" value="1"/> - <!-- general stats, defines simulation periods;require total, idle, and busy cycles for senity check --> - <!-- please note: if target architecture is X86, then all the instrucions refer to (fused) micro-ops --> - <stat name="total_instructions" value="total_instructions_match_mcpat"/> - <stat name="int_instructions" value="int_instruction_match_mcpat"/> - <stat name="fp_instructions" value="flt_instruction_match_mcpat"/> - <stat name="branch_instructions" value="branch_instruction_match_mcpat"/> - <stat name="branch_mispredictions" value="0"/> - <stat name="load_instructions" value="load_instruction_match_mcpat"/> - <stat name="store_instructions" value="store_instruction_match_mcpat"/> - <stat name="committed_instructions" value="total_instructions_match_mcpat"/> - <stat name="committed_int_instructions" value="int_instruction_match_mcpat"/> - <stat name="committed_fp_instructions" value="flt_instruction_match_mcpat"/> - <stat name="pipeline_duty_cycle" value="0.6"/><!--<=1, runtime_ipc/peak_ipc; averaged for all cores if homogenous --> - <!-- the following cycle stats are used for heterogeneouse cores only, - please ignore them if homogeneouse cores --> - <stat name="total_cycles" value="total_cycles_match_mcpat"/> - <stat name="idle_cycles" value="idle_cycles_match_mcpat"/> - <stat name="busy_cycles" value="busy_cycles_match_mcpat"/> - <!-- instruction buffer stats --> - <!-- ROB stats, both RS and Phy based OoOs have ROB - performance simulator should capture the difference on accesses, - otherwise, McPAT has to guess based on number of commited instructions. --> - <stat name="ROB_reads" value="263886"/> - <stat name="ROB_writes" value="263886"/> - <!-- RAT accesses --> - <stat name="rename_accesses" value="263886"/> - <stat name="fp_rename_accesses" value="263886"/> - <!-- decode and rename stage use this, should be total ic - nop --> - <!-- Inst window stats --> - <stat name="inst_window_reads" value="263886"/> - <stat name="inst_window_writes" value="263886"/> - <stat name="inst_window_wakeup_accesses" value="263886"/> - <stat name="fp_inst_window_reads" value="263886"/> - <stat name="fp_inst_window_writes" value="263886"/> - <stat name="fp_inst_window_wakeup_accesses" value="263886"/> - <!-- RF accesses --> - <stat name="int_regfile_reads" value="int_register_read_access_match_mcpat"/> - <stat name="float_regfile_reads" value="int_register_write_access_match_mcpat"/> - <stat name="int_regfile_writes" value="float_register_read_access_match_mcpat"/> - <stat name="float_regfile_writes" value="float_register_write_access_match_mcpat"/> - - <!-- The following stat is for operand collector power - Added by Syed --> - <stat name="non_rf_operands" value="0"/> - - <!-- accesses to the working reg --> - <stat name="function_calls" value="0"/> - <stat name="context_switches" value="0"/> <!--not used in the McPAT --> - <!-- Number of Windowes switches (number of function calls and returns)--> - <!-- Alu stats by default, the processor has one FPU that includes the divider and - multiplier. The fpu accesses should include accesses to multiplier and divider --> - <stat name="ialu_accesses" value="ialu_accesses_match_mcpat"/> - <stat name="fpu_accesses" value="fpu_accesses_match_mcpat"/> - <stat name="mul_accesses" value="mul_accesses_match_mcpat"/> - <stat name="cdb_alu_accesses" value="0"/> - <stat name="cdb_mul_accesses" value="0"/> - <stat name="cdb_fpu_accesses" value="0"/> - <!-- multiple cycle accesses should be counted multiple times, - otherwise, McPAT can use internal counter for different floating point instructions - to get final accesses. But that needs detailed info for floating point inst mix --> - <!-- currently the performance simulator should - make sure all the numbers are final numbers, - including the explicit read/write accesses, - and the implicite accesses such as replacements and etc. - Future versions of McPAT may be able to reason the implicite access - based on param and stats of last level cache - The same rule applies to all cache access stats too! --> - <!-- following is AF for max power computation. - Do not change them, unless you understand them--> - <stat name="IFU_duty_cycle" value="0.25"/> - <stat name="LSU_duty_cycle" value="0.25"/> - <stat name="MemManU_I_duty_cycle" value="1"/> - <stat name="MemManU_D_duty_cycle" value="0.25"/> - <stat name="ALU_duty_cycle" value="0.9"/> - <stat name="MUL_duty_cycle" value="0.5"/> - <stat name="FPU_duty_cycle" value="1"/><!-- FPU numbers are already average --> - <stat name="ALU_cdb_duty_cycle" value="0.9"/> - <stat name="MUL_cdb_duty_cycle" value="0.5"/> - <stat name="FPU_cdb_duty_cycle" value="15"/> - <component id="system.core0.predictor" name="PBT"> - <!-- branch predictor; tournament predictor see Alpha implementation --> - <param name="local_predictor_size" value="10,3"/> - <param name="local_predictor_entries" value="1024"/> - <param name="global_predictor_entries" value="4096"/> - <param name="global_predictor_bits" value="2"/> - <param name="chooser_predictor_entries" value="4096"/> - <param name="chooser_predictor_bits" value="2"/> - <!-- These parameters can be combined like below in next version - <param name="load_predictor" value="10,3,1024"/> - <param name="global_predictor" value="4096,2"/> - <param name="predictor_chooser" value="4096,2"/> - --> - </component> - <component id="system.core0.itlb" name="itlb"> - <param name="number_entries" value="1"/> - <stat name="total_accesses" value="0"/> - <stat name="total_misses" value="0"/> - <stat name="conflicts" value="0"/> - <!-- there is no write requests to itlb although writes happen to itlb after miss, - which is actually a replacement --> - </component> - <component id="system.core0.icache" name="icache"> - <!-- there is no write requests to itlb although writes happen to it after miss, - which is actually a replacement --> - <param name="icache_config" value="16384,128,4,1,1,3,8,0"/> - <!-- the parameters are capacity,block_width, associativity, bank, throughput w.r.t. core clock, latency w.r.t. core clock,output_width, cache policy --> - <!-- cache_policy;//0 no write or write-though with non-write allocate;1 write-back with write-allocate --> - <param name="buffer_sizes" value="16, 16, 16,0"/> - <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> - <stat name="read_accesses" value="total_instructions_match_mcpat"/> - <stat name="read_misses" value="0"/> - <stat name="conflicts" value="0"/> - </component> - <component id="system.core0.dtlb" name="dtlb"> - <param name="number_entries" value="1"/> - <stat name="total_accesses" value="0"/> - <stat name="total_misses" value="0"/> - <stat name="conflicts" value="0"/> - </component> - <component id="system.core0.ccache" name="ccache"> - <!-- all the buffer related are optional --> - <param name="ccache_config" value="16384,64,2,1,1,3,8,0"/> - <param name="buffer_sizes" value="16, 16, 16, 0"/> - <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> - <stat name="read_accesses" value="ccache_read_accesses_match_mcpat"/> - <stat name="write_accesses" value="0"/> - <stat name="read_misses" value="ccache_read_misses_match_mcpat"/> - <stat name="write_misses" value="0"/> - <stat name="conflicts" value="0"/> - </component> - <component id="system.core0.tcache" name="tcache"> - <!-- all the buffer related are optional --> - <param name="tcache_config" value="49152,128,8,1,1,3,8,0"/> - <param name="buffer_sizes" value="16, 16, 16, 0"/> - <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> - <stat name="read_accesses" value="tcache_read_accesses_match_mcpat"/> - <stat name="write_accesses" value="0"/> - <stat name="read_misses" value="tcache_read_misses_match_mcpat"/> - <stat name="write_misses" value="0"/> - <stat name="conflicts" value="0"/> - </component> - <!--model the shared memory by mimicing dcache--> - <component id="system.core0.sharedmemory" name="sharedmemory"> - <!-- all the buffer related are optional --> - <param name="sharedmemory_config" value="98304,16,1,16,1,3,16,0"/> - <!-- the parameters are capacity,block_width, associativity, bank, throughput w.r.t. core clock, latency w.r.t. core clock,output_width, cache policy --> - <param name="buffer_sizes" value="16, 16, 16, 16"/> - <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> - <stat name="read_accesses" value="sharedmemory_read_access_match_mcpat"/> - <stat name="write_accesses" value="sharedmemory_write_access_match_mcpat"/> - <stat name="read_misses" value="0"/> - <stat name="write_misses" value="0"/> - <stat name="conflicts" value="0"/> - </component> - <component id="system.core0.dcache" name="dcache"> - <!-- all the buffer related are optional --> - <param name="dcache_config" value="16384,32,4,1,1,3,8,0"/> - <param name="buffer_sizes" value="16, 16, 16, 0"/> - <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> - <stat name="read_accesses" value="dcache_read_access_match_mcpat"/> - <stat name="write_accesses" value="dcache_write_access_match_mcpat"/> - <stat name="read_misses" value="dcache_read_miss_match_mcpat"/> - <stat name="write_misses" value="dcache_write_miss_match_mcpat"/> - <stat name="conflicts" value="0"/> - </component> - <component id="system.core0.BTB" name="BTB"> - <!-- all the buffer related are optional --> - <param name="BTB_config" value="8192,4,2,1, 1,3"/> - <!-- the parameters are capacity,block_width,associativity,bank, throughput w.r.t. core clock, latency w.r.t. core clock,--> - </component> - </component> - <component id="system.L1Directory0" name="L1Directory0"> - <param name="Directory_type" value="0"/> - <!--0 cam based shadowed tag. 1 directory cache --> - <param name="Dir_config" value="2048,1,0,1, 4, 4,8"/> - <!-- the parameters are capacity,block_width, associativity,bank, throughput w.r.t. core clock, latency w.r.t. core clock,--> - <param name="buffer_sizes" value="8, 8, 8, 8"/> - <!-- all the buffer related are optional --> - <param name="clockrate" value="1400"/> - <param name="ports" value="1,1,1"/> - <!-- number of r, w, and rw search ports --> - <param name="device_type" value="0"/> - <!-- altough there are multiple access types, - Performance simulator needs to cast them into reads or writes - e.g. the invalidates can be considered as writes --> - <stat name="read_accesses" value="800000"/> - <stat name="write_accesses" value="27276"/> - <stat name="read_misses" value="1632"/> - <stat name="write_misses" value="183"/> - <stat name="conflicts" value="20"/> - <stat name="duty_cycle" value="0.45"/> - </component> - <component id="system.L2Directory0" name="L2Directory0"> - <param name="Directory_type" value="1"/> - <!--0 cam based shadowed tag. 1 directory cache --> - <param name="Dir_config" value="1048576,16,16,1,2, 100"/> - <!-- the parameters are capacity,block_width, associativity,bank, throughput w.r.t. core clock, latency w.r.t. core clock,--> - <param name="buffer_sizes" value="8, 8, 8, 8"/> - <!-- all the buffer related are optional --> - <param name="clockrate" value="1400"/> - <param name="ports" value="1,1,1"/> - <!-- number of r, w, and rw search ports --> - <param name="device_type" value="0"/> - <!-- altough there are multiple access types, - Performance simulator needs to cast them into reads or writes - e.g. the invalidates can be considered as writes --> - <stat name="read_accesses" value="0"/> - <stat name="write_accesses" value="0"/> - <stat name="read_misses" value="0"/> - <stat name="write_misses" value="0"/> - <stat name="conflicts" value="0"/> - <stat name="duty_cycle" value="0.45"/> - </component> - <component id="system.L20" name="L20"> - <!-- all the buffer related are optional --> - <param name="L2_config" value="131072,128,16,1, 4,23, 64, 1"/> - <!-- consider 4-way bank interleaving for Niagara 1 --> - <!-- the parameters are capacity,block_width, associativity, bank, throughput w.r.t. core clock, latency w.r.t. core clock,output_width, cache policy --> - <param name="buffer_sizes" value="16, 16, 16, 16"/> - <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> - <param name="clockrate" value="2962"/> - <param name="ports" value="1,1,1"/> - <!-- number of r, w, and rw ports --> - <param name="device_type" value="0"/> - <stat name="read_accesses" value="200000"/> - <stat name="write_accesses" value="0"/> - <stat name="read_misses" value="0"/> - <stat name="write_misses" value="0"/> - <stat name="conflicts" value="0"/> - <stat name="duty_cycle" value="0.5"/> - </component> - -<!--**********************************************************************--> -<component id="system.L30" name="L30"> - <param name="L3_config" value="1048576,64,16,1, 2,100, 64,1"/> - <!-- the parameters are capacity,block_width, associativity, bank, throughput w.r.t. core clock, latency w.r.t. core clock,output_width, cache policy --> - <param name="clockrate" value="3500"/> - <param name="ports" value="1,1,1"/> - <!-- number of r, w, and rw ports --> - <param name="device_type" value="0"/> - <param name="buffer_sizes" value="16, 16, 16, 16"/> - <!-- cache controller buffer sizes: miss_buffer_size(MSHR),fill_buffer_size,prefetch_buffer_size,wb_buffer_size--> - <stat name="read_accesses" value="58824"/> - <stat name="write_accesses" value="27276"/> - <stat name="read_misses" value="1632"/> - <stat name="write_misses" value="183"/> - <stat name="conflicts" value="0"/> - <stat name="duty_cycle" value="0.35"/> - </component> - - -<!--**********************************************************************--> - <component id="system.NoC0" name="noc0"> - <param name="clockrate" value="700"/> - <param name="type" value="1"/> - <!-- 1 NoC, O bus --> - <param name="horizontal_nodes" value="2"/> - <param name="vertical_nodes" value="1"/> - <param name="has_global_link" value="0"/> - <!-- 1 has global link, 0 does not have global link --> - <param name="link_throughput" value="1"/><!--w.r.t clock --> - <param name="link_latency" value="1"/><!--w.r.t clock --> - <!-- througput >= latency --> - <!-- Router architecture --> - <param name="input_ports" value="6"/> - <param name="output_ports" value="6"/> - <param name="virtual_channel_per_port" value="1"/> - <!-- input buffer; in classic routers only input ports need buffers --> - <param name="flit_bits" value="32"/> - <param name="input_buffer_entries_per_vc" value="1"/><!--VCs within the same ports share input buffers whose size is propotional to the number of VCs--> - <param name="chip_coverage" value="1"/> - <!-- When multiple NOC present, one NOC will cover part of the whole chip. chip_coverage <=1 --> - <stat name="total_accesses" value="0"/> - <!-- This is the number of total accesses within the whole network not for each router --> - <stat name="duty_cycle" value="0.6"/> - </component> -<!--**********************************************************************--> -<!--**********************************************************************--> - - <component id="system.mem" name="mem"> - <!-- Main memory property --> - <param name="mem_tech_node" value="23"/> - <param name="device_clock" value="200"/><!--MHz, this is clock rate of the actual memory device, not the FSB --> - <param name="peak_transfer_rate" value="3200"/><!--MB/S--> - <param name="internal_prefetch_of_DRAM_chip" value="4"/> - <!-- 2 for DDR, 4 for DDR2, 8 for DDR3...--> - <!-- the device clock, peak_transfer_rate, and the internal prefetch decide the DIMM property --> - <!-- above numbers can be easily found from Wikipedia --> - <param name="capacity_per_channel" value="4096"/> <!-- MB --> - <!-- capacity_per_Dram_chip=capacity_per_channel/number_of_dimms/number_ranks/Dram_chips_per_rank - Current McPAT assumes single DIMMs are used.--> - <param name="number_ranks" value="2"/> - <param name="num_banks_of_DRAM_chip" value="6"/> - <param name="Block_width_of_DRAM_chip" value="64"/> <!-- B --> - <param name="output_width_of_DRAM_chip" value="8"/> - <!--number of Dram_chips_per_rank=" 72/output_width_of_DRAM_chip--> - <!--number of Dram_chips_per_rank=" 72/output_width_of_DRAM_chip--> - <param name="page_size_of_DRAM_chip" value="8"/> <!-- 8 or 16 --> - <param name="burstlength_of_DRAM_chip" value="8"/> - <stat name="memory_accesses" value="1052"/> - <stat name="memory_reads" value="1052"/> - <stat name="memory_writes" value="1052"/> - </component> - <component id="system.mc" name="mc"> - <!-- Memeory controllers are for DDR(2,3...) DIMMs --> - <!-- current version of McPAT uses published values for base parameters of memory controller - improvments on MC will be added in later versions. --> - <param name="type" value="0"/> <!-- 1: low power; 0 high performance --> - <param name="mc_clock" value="1848"/><!--DIMM IO bus clock rate MHz DDR2-400 for Niagara 1--> - <param name="peak_transfer_rate" value="29568"/><!--MB/S Syed: GTX 470 has 177.4GB/s mem transfer rate with 6 MCs --> - <param name="block_size" value="64"/><!--B--> - <param name="number_mcs" value="6"/><!-- 6 GDDR5 memory controllers --> - <!-- current McPAT only supports homogeneous memory controllers --> - <param name="memory_channels_per_mc" value="2"/> - <param name="number_ranks" value="1"/> - <param name="withPHY" value="0"/> - <!-- # of ranks of each channel--> - <param name="req_window_size_per_channel" value="16"/> - <param name="IO_buffer_size_per_channel" value="16"/> - <param name="databus_width" value="32"/> - <param name="addressbus_width" value="32"/> - <param name="PRT_entries" value="32"/> - <!-- # of empirical DRAM model parameter --> - <param name="dram_cmd_coeff" value="0"/> - <param name="dram_act_coeff" value="0"/> - <param name="dram_nop_coeff" value="0"/> - <param name="dram_activity_coeff" value="0"/> - <param name="dram_pre_coeff" value="3.8475e-8f"/> - <param name="dram_rd_coeff" value="7.74707143e-8f"/> - <param name="dram_wr_coeff" value="3.54664286e-8f"/> - <param name="dram_req_coeff" value="0"/> - <param name="dram_const_coeff" value="0"/> - - <!-- McPAT will add the control bus width to the addressbus width automatically --> - <stat name="memory_accesses" value="memory_accesses_match_mcpat"/> - <stat name="memory_reads" value="memory_reads_match_mcpat"/> - <stat name="memory_writes" value="memory_writes_match_mcpat"/> - <!-- McPAT does not track individual mc, instead, it takes the total accesses and calculate - the average power per MC or per channel. This is sufficent for most application. - Further trackdown can be easily added in later versions. --> - </component> -<!--**********************************************************************--> - <component id="system.niu" name="niu"> - <!-- On chip 10Gb Ethernet NIC, including XAUI Phy and MAC controller --> - <!-- For a minimum IP packet size of 84B at 10Gb/s, a new packet arrives every 67.2ns. - the low bound of clock rate of a 10Gb MAC is 150Mhz --> - <param name="type" value="0"/> <!-- 1: low power; 0 high performance --> - <param name="clockrate" value="350"/> - <param name="number_units" value="0"/> <!-- unlike PCIe and memory controllers, each Ethernet controller only have one port --> - <stat name="duty_cycle" value="1.0"/> <!-- achievable max load <= 1.0 --> - <stat name="total_load_perc" value="0.7"/> <!-- ratio of total achived load to total achivable bandwidth --> - <!-- McPAT does not track individual nic, instead, it takes the total accesses and calculate - the average power per nic or per channel. This is sufficent for most application. --> - </component> -<!--**********************************************************************--> - <component id="system.pcie" name="pcie"> - <!-- On chip PCIe controller, including Phy--> - <!-- For a minimum PCIe packet size of 84B at 8Gb/s per lane (PCIe 3.0), a new packet arrives every 84ns. - the low bound of clock rate of a PCIe per lane logic is 120Mhz --> - <param name="type" value="0"/> <!-- 1: low power; 0 high performance --> - <param name="withPHY" value="1"/> - <param name="clockrate" value="350"/> - <param name="number_units" value="0"/> - <param name="num_channels" value="8"/> <!-- 2 ,4 ,8 ,16 ,32 --> - <stat name="duty_cycle" value="1.0"/> <!-- achievable max load <= 1.0 --> - <stat name="total_load_perc" value="0.7"/> <!-- Percentage of total achived load to total achivable bandwidth --> - <!-- McPAT does not track individual pcie controllers, instead, it takes the total accesses and calculate - the average power per pcie controller or per channel. This is sufficent for most application. --> - </component> -<!--**********************************************************************--> - <component id="system.flashc" name="flashc"> - <param name="number_flashcs" value="0"/> - <param name="type" value="1"/> <!-- 1: low power; 0 high performance --> - <param name="withPHY" value="1"/> - <param name="peak_transfer_rate" value="200"/><!--Per controller sustainable reak rate MB/S --> - <stat name="duty_cycle" value="1.0"/> <!-- achievable max load <= 1.0 --> - <stat name="total_load_perc" value="0.7"/> <!-- Percentage of total achived load to total achivable bandwidth --> - <!-- McPAT does not track individual flash controller, instead, it takes the total accesses and calculate - the average power per fc or per channel. This is sufficent for most application --> - </component> -<!--**********************************************************************--> - - </component> -</component> diff --git a/cutlass-example/half.h b/cutlass-example/half.h deleted file mode 100644 index ee536e5..0000000 --- a/cutlass-example/half.h +++ /dev/null @@ -1,743 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! - \file - \brief Host-side implementation of half-precision float -*/ - -#pragma once - -#include <stdint.h> -#include <cmath> -#include <limits> -#include <utility> -#include <utility> - -#include <iomanip> -#include <istream> -#include <ostream> - -#include <cuda_fp16.h> - -namespace cutlass { - -/// IEEE binary16 floating-point value -class half_t { - public: - half_t(); - half_t(int); /// conversion from integer - half_t(float); /// conversion from fp32 - half_t(double); /// conversion from fp64 - - static half_t bitcast(unsigned short); /// bitcast performs no conversion - - static half_t convert(float const&); /// FP conversion - round toward nearest even - static float convert(unsigned short const&); /// floating point conversion to fp32 - - static half_t zero() { return bitcast(0); } /// +zero - static half_t one() { return bitcast(0x3c00); } /// one - static half_t nan() { return bitcast(0x7fff); } /// canonical not a number - static half_t inf() { return bitcast(0x7c00); } /// +infinity - static half_t ninf() { return bitcast(0xfc00); } /// -infinity - static half_t epsilon() { return bitcast(0x1000); } /// Machine epsilon - - bool signbit() const; /// sign bit - true: negative, false: positive - int exponent() const; /// unbiased exponent - unsigned short mantissa() const; /// mantissa bits - - bool isfinite() const; /// true if neither inf nor nan - bool isinf() const; /// true if value is + or - infinity - bool isnan() const; /// true if value is not a number - bool isnormal() const; /// true if nonzero value is normalized - bool iszero() const; /// true if value is + or - zero - - bool operator==(half_t const&) const; - bool operator!=(half_t const&) const; - bool operator==(float const&) const; - bool operator!=(float const&) const; - - bool operator<(half_t const&) const; - bool operator<=(half_t const&) const; - bool operator>(half_t const&) const; - bool operator>=(half_t const&) const; - - half_t operator+(half_t const&) const; - half_t operator-() const; - half_t operator-(half_t const&) const; - half_t operator*(half_t const&)const; - half_t operator/(half_t const&) const; - - half_t& operator+=(half_t const&); - half_t& operator-=(half_t const&); - half_t& operator*=(half_t const&); - half_t& operator/=(half_t const&); - - half_t& operator++(); - half_t& operator--(); - half_t operator++(int); - half_t operator--(int); - - operator bool() const; /// false if zero - operator int() const; /// conversion to int - operator float() const; /// conversion to fp32 - operator half() const; /// conversion to half - - uint16_t& raw() { return x; } - uint16_t raw() const { return x; } - - public: - /// data - unsigned short x; -}; - -/// Packed pair of half-precision elements -class half2_t { - public: - half2_t(); - half2_t(half_t lo, half_t hi); - half2_t(std::pair<float, float> const&); - explicit half2_t(unsigned data); - - half2_t operator+(half2_t const&) const; - half2_t operator-(half2_t const&) const; - half2_t operator*(half2_t const&)const; - half2_t operator/(half2_t const&) const; - - half2_t& operator+=(half2_t const&); - half2_t& operator-=(half2_t const&); - half2_t& operator*=(half2_t const&); - half2_t& operator/=(half2_t const&); - - float dot(half2_t const&) const; /// dot product with single-precision accumulation - float dot(half2_t const&, float) const; /// dot product with single-precision accumulation - - half_t doth(half2_t const&) const; /// dot product with half_t-precision accumulation - half_t doth(half2_t const&, half_t) const; /// dot product with half_t-precision accumulation - - unsigned packed() const; - - operator std::pair<float, float>() const; - operator unsigned() const; - - public: - half_t lo; - half_t hi; -}; - -template <typename Dest, typename Src> -Dest bitcast(Src const&); -template <> -float bitcast<float, unsigned>(unsigned const&); -template <> -float bitcast<float, int>(int const&); -template <> -unsigned bitcast<unsigned, float>(float const&); -template <> -half_t bitcast<half_t, unsigned short>(unsigned short const&); -template <> -unsigned short bitcast<unsigned short, half_t>(half_t const&); -template <> -half bitcast<half, unsigned short>(unsigned short const&); -} // namespace cutlass - -cutlass::half_t operator+(float, cutlass::half_t const&); -cutlass::half_t operator-(float, cutlass::half_t const&); -cutlass::half_t operator*(float, cutlass::half_t const&); -cutlass::half_t operator/(float, cutlass::half_t const&); - -std::ostream& operator<<(std::ostream&, cutlass::half_t const&); /// writes a half_t -std::istream& operator>>(std::istream&, cutlass::half_t&); /// reads a half_t - -#ifdef BOOST_LEXICAL_CAST_INCLUDED -namespace boost { - -/// lexical cast from string to half_t -template <> -cutlass::half_t lexical_cast<cutlass::half_t>(std::string const& arg); - -/// lexical cast from half_t to string -template <> -std::string lexical_cast<std::string>(cutlass::half_t const& arg); -} // namespace boost -#endif - -#define HLF_MANT_DIG 10 - -namespace std { - -cutlass::half_t abs(cutlass::half_t const&); /// absolute value - -bool isnan(cutlass::half_t const&); /// true if argument is NaN - -bool isfinite(cutlass::half_t const&); /// true if argument is neither NaN nor infinity - -cutlass::half_t nanh(const char* = 0); /// returns a not-a-number - -bool isinf(cutlass::half_t const&); /// returns true if argument is infinitey (+ or -) - -bool isnormal( - cutlass::half_t const&); /// returns true if argument is normal (neither zero nor infinity) - -int fpclassify(cutlass::half_t const&); /// returns a flag classifying floating-point value - -bool signbit(cutlass::half_t const&); /// returns true if negative, false if positive - -cutlass::half_t sqrt(cutlass::half_t const&); /// square root of half_t - -/// Numeric limits -template <> -struct numeric_limits<cutlass::half_t> { - static bool const is_specialized = true; - static bool const is_signed = true; - static bool const is_integer = false; - static bool const is_exact = false; - static bool const has_infinity = true; - static bool const has_quiet_NaN = true; - static bool const has_signaling_NaN = false; - static std::float_denorm_style const has_denorm = std::denorm_present; - static bool const has_denorm_loss = true; - static std::float_round_style const round_style = std::round_to_nearest; - static bool const is_iec559 = false; - static bool const is_bounded = true; - static bool const is_modulo = false; - static int const digits = HLF_MANT_DIG; - - static cutlass::half_t min() { return cutlass::half_t::bitcast(0x0001); } - - static cutlass::half_t lowest() { return cutlass::half_t::bitcast(0xfbff); } - - static cutlass::half_t max() { return cutlass::half_t::bitcast(0x7bff); } - - /// Returns smallest finite value - static cutlass::half_t epsilon() { return cutlass::half_t::epsilon(); } - - /// Returns smallest finite value - static cutlass::half_t round_error() { return cutlass::half_t(0.5f); } - - /// Returns smallest finite value - static cutlass::half_t infinity() { return cutlass::half_t::inf(); } - - /// Returns smallest finite value - static cutlass::half_t quiet_NaN() { return cutlass::half_t::nan(); } - - /// Returns smallest finite value - static cutlass::half_t signaling_NaN() { return cutlass::half_t::nan(); } - - /// Returns smallest finite value - static cutlass::half_t denorm_min() { return cutlass::half_t::bitcast(0x0001); } -}; -} // namespace std - -// -// -// - -inline cutlass::half_t cutlass::half_t::bitcast(unsigned short _x) { - half_t h; - h.x = _x; - return h; -} - -/// FP32 -> FP16 conversion - rounds to nearest even -inline cutlass::half_t cutlass::half_t::convert(float const& flt) { - // software implementation rounds toward nearest even - unsigned const& s = *reinterpret_cast<unsigned const*>(&flt); - uint16_t sign = uint16_t((s >> 16) & 0x8000); - int16_t exp = uint16_t(((s >> 23) & 0xff) - 127); - int mantissa = s & 0x7fffff; - uint16_t u = 0; - - if ((s & 0x7fffffff) == 0) { - // sign-preserving zero - return cutlass::half_t::bitcast(sign); - } - - if (exp > 15) { - if (exp == 128 && mantissa) { - // not a number - u = 0x7fff; - } else { - // overflow to infinity - u = sign | 0x7c00; - } - return cutlass::half_t::bitcast(u); - } - - int sticky_bit = 0; - - if (exp >= -14) { - // normal fp32 to normal fp16 - exp = uint16_t(exp + uint16_t(15)); - u = uint16_t(((exp & 0x1f) << 10)); - u = uint16_t(u | (mantissa >> 13)); - } else { - // normal single-precision to subnormal half_t-precision representation - int rshift = (-14 - exp); - if (rshift < 32) { - mantissa |= (1 << 23); - - sticky_bit = ((mantissa & ((1 << rshift) - 1)) != 0); - - mantissa = (mantissa >> rshift); - u = (uint16_t(mantissa >> 13) & 0x3ff); - } else { - mantissa = 0; - u = 0; - } - } - - // round to nearest even - int round_bit = ((mantissa >> 12) & 1); - sticky_bit |= ((mantissa & ((1 << 12) - 1)) != 0); - - if ((round_bit && sticky_bit) || (round_bit && (u & 1))) { - u = uint16_t(u + 1); - } - - u |= sign; - - return cutlass::half_t::bitcast(u); -} - -inline float cutlass::half_t::convert(unsigned short const& h) { - int sign = ((h >> 15) & 1); - int exp = ((h >> 10) & 0x1f); - int mantissa = (h & 0x3ff); - unsigned f = 0; - - if (exp > 0 && exp < 31) { - // normal - exp += 112; - f = (sign << 31) | (exp << 23) | (mantissa << 13); - } else if (exp == 0) { - if (mantissa) { - // subnormal - exp += 113; - while ((mantissa & (1 << 10)) == 0) { - mantissa <<= 1; - exp--; - } - mantissa &= 0x3ff; - f = (sign << 31) | (exp << 23) | (mantissa << 13); - } else { - // sign-preserving zero - f = (sign << 31); - } - } else if (exp == 31) { - if (mantissa) { - f = 0x7fffffff; // not a number - } else { - f = (0xff << 23) | (sign << 31); // inf - } - } - return *reinterpret_cast<float const*>(&f); -} - -inline cutlass::half_t::half_t() {} - -inline cutlass::half_t::half_t(int i) { x = convert(float(i)).x; } - -inline cutlass::half_t::half_t(float f) { x = convert(f).x; } - -inline cutlass::half_t::half_t(double d) { x = convert(float(d)).x; } - -inline bool cutlass::half_t::signbit() const { return (x >> 15) & 1; } - -inline int cutlass::half_t::exponent() const { return ((x >> 10) & 0x1f) - 15; } - -inline unsigned short cutlass::half_t::mantissa() const { return x & 0x3ff; } - -inline cutlass::half_t::operator bool() const { return (x & 0x7fff) != 0; } - -inline cutlass::half_t::operator int() const { return static_cast<int>(convert(x)); } - -inline cutlass::half_t::operator float() const { return convert(x); } - -inline cutlass::half_t::operator half() const { return cutlass::bitcast<half, unsigned short>(x); } - -inline bool cutlass::half_t::operator==(cutlass::half_t const& h) const { - if (iszero() && h.iszero()) { - return true; - } - return x == h.x; -} - -inline bool cutlass::half_t::operator!=(cutlass::half_t const& h) const { - if (iszero() && h.iszero()) { - return false; - } - return x != h.x; -} - -inline bool cutlass::half_t::operator==(float const& b) const { return x == half_t(b).x; } - -inline bool cutlass::half_t::operator!=(float const& b) const { return x != half_t(b).x; } - -inline bool cutlass::half_t::iszero() const { return (x & 0x7fff) == 0; } - -inline bool cutlass::half_t::isfinite() const { return (exponent() < 16); } - -inline bool cutlass::half_t::isnan() const { - int exp = ((x >> 10) & 0x1f); - if (exp == 0x1f) { - return (x & 0x3ff) != 0; - } - return false; -} - -inline bool cutlass::half_t::isinf() const { - int exp = ((x >> 10) & 0x1f); - if (exp == 0x1f) { - return (x & 0x3ff) == 0; - } - return false; -} - -inline bool cutlass::half_t::isnormal() const { - int exp = exponent(); - return exp > -15 && exp < 16; -} - -inline bool cutlass::half_t::operator<(half_t const& h) const { - int sign = ((x >> 15) & 1); - int h_sign = ((h.x >> 15) & 1); - if (sign == h_sign) { - return (x & 0x7fff) < (h.x & 0x7fff); - } else if (sign) { - return true; - } - return false; -} - -inline bool cutlass::half_t::operator<=(half_t const& h) const { - int sign = ((x >> 15) & 1); - int h_sign = ((h.x >> 15) & 1); - if (sign == h_sign) { - return (x & 0x7fff) <= (h.x & 0x7fff); - } else if (sign) { - return true; - } - return false; -} - -inline bool cutlass::half_t::operator>(half_t const& h) const { - int sign = ((x >> 15) & 1); - int h_sign = ((h.x >> 15) & 1); - if (sign == h_sign) { - return (x & 0x7fff) > (h.x & 0x7fff); - } else if (h_sign) { - return true; - } - return false; -} - -inline bool cutlass::half_t::operator>=(half_t const& h) const { - int sign = ((x >> 15) & 1); - int h_sign = ((h.x >> 15) & 1); - if (sign == h_sign) { - return (x & 0x7fff) >= (h.x & 0x7fff); - } else if (h_sign) { - return true; - } - return false; -} - -inline cutlass::half_t cutlass::half_t::operator+(cutlass::half_t const& b) const { - return cutlass::half_t(float(*this) + float(b)); -} - -inline cutlass::half_t cutlass::half_t::operator-() const { return bitcast(x ^ 0x8000); } - -inline cutlass::half_t cutlass::half_t::operator-(cutlass::half_t const& b) const { - return cutlass::half_t(float(*this) - float(b)); -} - -inline cutlass::half_t cutlass::half_t::operator*(cutlass::half_t const& b) const { - return cutlass::half_t(float(*this) * float(b)); -} - -inline cutlass::half_t cutlass::half_t::operator/(cutlass::half_t const& b) const { - return cutlass::half_t(float(*this) / float(b)); -} - -inline cutlass::half_t& cutlass::half_t::operator+=(cutlass::half_t const& b) { - *this = cutlass::half_t(float(*this) + float(b)); - return *this; -} - -inline cutlass::half_t& cutlass::half_t::operator-=(cutlass::half_t const& b) { - *this = cutlass::half_t(float(*this) - float(b)); - return *this; -} - -inline cutlass::half_t& cutlass::half_t::operator*=(cutlass::half_t const& b) { - *this = cutlass::half_t(float(*this) * float(b)); - return *this; -} - -inline cutlass::half_t& cutlass::half_t::operator/=(cutlass::half_t const& b) { - *this = cutlass::half_t(float(*this) / float(b)); - return *this; -} - -inline cutlass::half_t& cutlass::half_t::operator++() { - *this = cutlass::half_t(float(*this) + 1.0f); - return *this; -} - -inline cutlass::half_t& cutlass::half_t::operator--() { - *this = cutlass::half_t(float(*this) - 1.0f); - return *this; -} - -inline cutlass::half_t cutlass::half_t::operator++(int) { - half_t h = *this; - *this = cutlass::half_t(float(*this) + 1.0f); - return h; -} - -inline cutlass::half_t cutlass::half_t::operator--(int) { - half_t h = *this; - *this = cutlass::half_t(float(*this) - 1.0f); - return h; -} - -inline cutlass::half_t operator+(float a, cutlass::half_t const& b) { - return cutlass::half_t(a + float(b)); -} - -inline cutlass::half_t operator-(float a, cutlass::half_t const& b) { - return cutlass::half_t(a - float(b)); -} - -inline cutlass::half_t operator*(float a, cutlass::half_t const& b) { - return cutlass::half_t(a * float(b)); -} - -inline cutlass::half_t operator/(float a, cutlass::half_t const& b) { - return cutlass::half_t(a / float(b)); -} - -// -// -// - -inline cutlass::half2_t::half2_t() {} - -inline cutlass::half2_t::half2_t(half_t lo, half_t hi) : lo(lo), hi(hi) {} - -inline cutlass::half2_t::half2_t(std::pair<float, float> const& p) : lo(p.first), hi(p.second) {} - -inline cutlass::half2_t::half2_t(unsigned data) - : lo(half_t::bitcast(uint16_t(data & 0x0ffff))), - hi(half_t::bitcast(uint16_t((data >> 16) & 0x0ffff))) {} - -inline cutlass::half2_t cutlass::half2_t::operator+(half2_t const& b) const { - return half2_t(lo + b.lo, hi + b.hi); -} - -inline cutlass::half2_t cutlass::half2_t::operator-(half2_t const& b) const { - return half2_t(lo - b.lo, hi - b.hi); -} - -inline cutlass::half2_t cutlass::half2_t::operator*(half2_t const& b) const { - return half2_t(lo * b.lo, hi * b.hi); -} - -inline cutlass::half2_t cutlass::half2_t::operator/(half2_t const& b) const { - return half2_t(lo / b.lo, hi / b.hi); -} - -inline cutlass::half2_t& cutlass::half2_t::operator+=(half2_t const& b) { - lo += b.lo; - hi += b.hi; - return *this; -} - -inline cutlass::half2_t& cutlass::half2_t::operator-=(half2_t const& b) { - lo -= b.lo; - hi -= b.hi; - return *this; -} - -inline cutlass::half2_t& cutlass::half2_t::operator*=(half2_t const& b) { - lo *= b.lo; - hi *= b.hi; - return *this; -} - -inline cutlass::half2_t& cutlass::half2_t::operator/=(half2_t const& b) { - lo /= b.lo; - hi /= b.hi; - return *this; -} - -inline float cutlass::half2_t::dot(half2_t const& b) const { - return float(lo) * float(b.lo) + float(hi) * float(b.hi); -} - -inline float cutlass::half2_t::dot(half2_t const& b, float c) const { return c + dot(b); } - -inline cutlass::half_t cutlass::half2_t::doth(half2_t const& b) const { - return cutlass::half_t(dot(b)); -} - -inline cutlass::half_t cutlass::half2_t::doth(half2_t const& b, half_t c) const { - return cutlass::half_t(dot(b, float(c))); -} - -inline cutlass::half2_t::operator std::pair<float, float>() const { - return std::pair<float, float>(float(lo), float(hi)); -} - -inline unsigned cutlass::half2_t::packed() const { return (lo.x | (hi.x << 16)); } - -inline cutlass::half2_t::operator unsigned() const { return packed(); } - -// -// -// - -template <> -inline float cutlass::bitcast<float, unsigned>(unsigned const& u) { - return *reinterpret_cast<float const*>(&u); -} - -template <> -inline float cutlass::bitcast<float, int>(int const& i) { - return *reinterpret_cast<float const*>(&i); -} - -template <> -inline unsigned cutlass::bitcast<unsigned, float>(float const& f) { - return *reinterpret_cast<unsigned const*>(&f); -} - -template <> -inline cutlass::half_t cutlass::bitcast<cutlass::half_t, unsigned short>(unsigned short const& s) { - return *reinterpret_cast<cutlass::half_t const*>(&s); -} - -template <> -inline unsigned short cutlass::bitcast<unsigned short, cutlass::half_t>(cutlass::half_t const& h) { - return *reinterpret_cast<unsigned short const*>(&h); -} - -template <> -inline half cutlass::bitcast<half, unsigned short>(unsigned short const& s) { - return *reinterpret_cast<half const*>(&s); -} - -// -// Lexical casts -// - -#ifdef BOOST_LEXICAL_CAST_INCLUDED -namespace boost { -template <> -cutlass::half_t lexical_cast<cutlass::half_t>(std::string const& arg) { - return cutlass::half_t(boost::lexical_cast<float>(arg)); -} - -template <> -std::string lexical_cast<std::string>(cutlass::half_t const& arg) { - return boost::lexical_cast<std::string>(float(arg)); -} -} // namespace boost -#endif - -// -// Standard Library Operations -// - -// std -namespace std { - -inline cutlass::half_t abs(cutlass::half_t const& h) { - return cutlass::half_t::bitcast(h.x & 0x7fff); -} - -inline bool isnan(cutlass::half_t const& h) { return h.isnan(); } - -inline bool isfinite(cutlass::half_t const& h) { return h.isfinite(); } - -inline cutlass::half_t nanh(const char*) { return cutlass::half_t::nan(); } - -inline bool isinf(cutlass::half_t const& h) { return h.isinf(); } - -inline bool isnormal(cutlass::half_t const& h) { return h.isnormal(); } - -inline int fpclassify(cutlass::half_t const& h) { - int exp = h.exponent(); - unsigned short mantissa = h.mantissa(); - if (exp < -14) { - if (mantissa == 0) { - return FP_ZERO; - } else { - return FP_SUBNORMAL; - } - } else if (exp > 15) { - if (mantissa == 0) { - return FP_INFINITE; - } else { - return FP_NAN; - } - } - return FP_NORMAL; -} - -inline bool signbit(cutlass::half_t const& h) { return h.signbit(); } - -inline cutlass::half_t sqrt(cutlass::half_t const& h) { - return cutlass::half_t(std::sqrt(float(h))); -} -} // namespace std - -// -// Stream interactions -// - -/// put to stream - half_t-precision types bitcast as unsigned shorts if base is hexadecimal -inline std::ostream& operator<<(std::ostream& out, cutlass::half_t const& h) { - if (out.flags() & std::ios::hex) { - return out << h.x; - } else { - return out << float(h); - } -} - -/// read from stream - half_t-precision types parsed as unsigned shorts if base is hexadecimal -inline std::istream& operator>>(std::istream& in, cutlass::half_t& h) { - if (in.flags() & std::ios::hex) { - unsigned short u = 0; - in >> u; - h = cutlass::half_t::bitcast(u); - } else { - float f = 0; - in >> f; - h = cutlass::half_t(f); - } - return in; -} diff --git a/cutlass-example/host_tensor.h b/cutlass-example/host_tensor.h deleted file mode 100644 index 0936336..0000000 --- a/cutlass-example/host_tensor.h +++ /dev/null @@ -1,365 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -#pragma once - -/*! \file - \brief Template class to perform computations on tensors and manage memory. -*/ - -#include <cutlass/cutlass.h> -#include <cutlass/matrix_traits.h> -#include <device_memory.h> -#include <host_tensor_view.h> -#include <type_traits.h> -#include <vector> - -namespace cutlass { - -template <typename T, bool DeviceBacked_ = true> -class HostTensor : public HostTensorView<T> { - public: - /// Type used for device-side allocations - typedef typename TypeTraits<T>::device_type DeviceType; - - /// Base class - typedef HostTensorView<T> Base; - - /// If true, allocates device side memory - static bool const DeviceBacked = DeviceBacked_; - - /// Rank of tensor - static int const Rank = Base::Rank; - - /// Type used to compute the offset of an element to the base of a tensor - typedef typename Base::Offset_t Offset_t; - - /// Tensor reference to host memory - typedef typename Base::TensorRef_t TensorRef_t; - - /// Tensor reference to device memory - typedef TensorRef<DeviceType, TensorRef_t::Rank> DeviceTensorRef; - - /// Tensor reference to constant device memory - typedef TensorRef<DeviceType const, TensorRef_t::Rank> ConstDeviceTensorRef; - - /// Coordinate into tensor - typedef typename Base::Coord_t Coord_t; - - private: - /// Host-side memory allocation - std::vector<T> host_; - - /// Device-side memory - cutlass::device_memory::allocation<DeviceType> device_; - - public: - // - // Device and Host Methods - // - - /// Default constructor - HostTensor() {} - - /// Constructs a Tensor_view from stride and size - HostTensor(Coord_t const& _stride, Coord_t const& _size) { reset(_stride, _size); } - - /// Constructs a HostTensor from size - infers strides - HostTensor(Coord_t const& _size) { - Coord_t _stride = make_Coord( - _size.at(2) * _size.at(1) * _size.at(0), _size.at(1) * _size.at(0), _size.at(0), 1); - reset(_stride, _size); - } - - /// Returns the number of elements needed to back vector - size_t capacity() { return Base::capacity(); } - - /// Returns true if the Tensor_view is bound to some memory - bool good() const { return Base::good(); } - - /// Updates the reference and size of a Tensor_view object - void reset(Coord_t const& _stride, Coord_t const& _size) { - size_t _capacity = _size.at(0) * _stride.at(0); - - DeviceType* _device_memory = nullptr; - if (DeviceBacked) { - _device_memory = cutlass::device_memory::allocate<DeviceType>(_capacity); - } - - host_.clear(); - host_.resize(_capacity); - for (size_t i = 0; i < _capacity; ++i) { - host_[i] = T((int)0xdeadbeef); - } - device_.reset(_device_memory, _capacity); - - Base::reset(TensorRef_t(host_.data(), _stride), _size); - } - - /// Initializes the host tensor as a matrix - void resize_matrix(int rows, int columns, MatrixLayout::Kind layout) { - bool col_major = (layout == MatrixLayout::kColumnMajor); - int ldm = (col_major ? rows : columns); - - Coord_t stride = make_Coord(rows * columns, col_major ? 1 : ldm, col_major ? ldm : 1, 1); - - Coord_t size = make_Coord(1, rows, columns, 1); - - reset(stride, size); - } - - /// Simplifies resizing the host tensor - void resize(int elements) { resize_matrix(1, elements, MatrixLayout::kColumnMajor); } - - /// Gets pointer to host data - T const* host_data() const { return &host_[0]; } - - /// Gets pointer to host data - T* host_data() { return &host_[0]; } - - /// Gets pointer to device data - DeviceType* device_data() const { return device_.get(); } - - /// Copies data from device to host - void sync_host() { - if (DeviceBacked) { - device_memory::copy_to_host( - host_.data(), reinterpret_cast<T const*>(device_.get()), host_.size()); - } - } - - /// Copies data from host to device - void sync_device() { - if (DeviceBacked) { - device_memory::copy_to_device( - device_.get(), reinterpret_cast<DeviceType const*>(host_.data()), host_.size()); - } - } - - /// Copy data from a caller-supplied device pointer - void copy_to_host(DeviceType const *ptr_device) { - device_memory::copy_to_host( - host_.data(), reinterpret_cast<T const *>(ptr_device), host_.size()); - } - - /// Copies data to a caller-supplied device pointer - void copy_to_device(DeviceType *ptr_device) { - device_memory::copy_to_device( - ptr_device, reinterpret_cast<DeviceType const *>(host_.data()), host_.size()); - } - - /// Accesses the tensor reference pointing to data - TensorRef_t& host_ref() { return Base::ref(); } - - /// Accesses the tensor reference pointing to data - TensorRef_t const& host_ref() const { return Base::ref(); } - - /// Accesses the tensor reference pointing to data - DeviceTensorRef device_ref() const { return DeviceTensorRef(device_data(), stride()); } - - /// Returns a tensor ref to constant memory on the device - ConstDeviceTensorRef const_device_ref() const { - return ConstDeviceTensorRef(device_data(), stride()); - } - - /// Accesses the size - Coord_t const& size() const { return Base::size(); } - - /// Accesses the size - int size(int dim) const { return Base::size(dim); } - - /// Accesses the size - Coord_t const& stride() const { return Base::stride(); } - - /// Accesses the size - int stride(int dim) const { return Base::stride(dim); } - - /// Returns the index of an element - Offset_t offset(Coord_t const& coord) const { return Base::offset(coord); } - - /// Determines whether a location is within a tensor - bool contains(Coord_t const& coord) const { return Base::contains(coord); } - - /// Element-wise accessor - T& at(Coord_t const& coord) const { return Base::at(coord); } - - /// Element-wise accessor - T& operator[](Coord_t const& coord) { return at(coord); } - - /// Element-wise accessor with basic offset - T& at(int idx) const { return Base::at(idx); } - - /// Returns a Tensor_view given location and size quantities - TensorView<T> subview(Coord_t const& _location, Coord_t _size) const { - return Base::subview(_location, _size); - } - - /// Recurses through all dimensions and applies a unary operation - template <typename F> - void elementwise_in_place(F& op, int dim = 0, Offset_t dst_offset_base = 0) { - Base::elementwise_in_place(op, dim, dst_offset_base); - } - - /// Recurses through all dimensions and applies a unary operator, supplying the logical - /// coordinate within the tensor as an argument - template <typename F> - void elementwise_stream(F& op, int dim = 0, Offset_t dst_offset_base = 0) { - Base::elementwise_stream(op, dim, dst_offset_base); - } - - /// Recurses through all dimensions and applies a unary operator, supplying the logical - /// coordinate within the tensor as an argument - template <typename F> - void elementwise_generate(F& op, - int dim = 0, - Offset_t dst_offset_base = 0, - Coord_t coord = Coord_t(0)) { - Base::elementwise_generate(op, dim, dst_offset_base, coord); - } - - /// Recurses through all dimensions and applies a binary operation - template <typename Src, typename F> - bool elementwise_in_place(F& op, - int dim, - TensorView<Src> const& tensor, - Offset_t dst_offset_base = 0, - Offset_t src_offset_base = 0) { - return Base::elementwise_in_place(op, dim, tensor, dst_offset_base, src_offset_base); - } - - /// Accumulate in place - template <typename Src> - TensorView<T>& operator+=(TensorView<Src> const& tensor) { - Base::operator+=(tensor); - sync_device(); - return *this; - } - - /// Subtract in place - template <typename Src> - TensorView<T>& operator-=(TensorView<Src> const& tensor) { - Base::operator-=(tensor); - sync_device(); - return *this; - } - - /// Multiply in place - template <typename Src> - TensorView<T>& operator*=(TensorView<Src> const& tensor) { - Base::operator*=(tensor); - sync_device(); - return *this; - } - - /// Divide in place - template <typename Src> - TensorView<T>& operator/=(TensorView<Src> const& tensor) { - Base::operator/=(tensor); - sync_device(); - return *this; - } - - /// equality with epsilon tolerance - bool equals(TensorView<T> const& tensor, T epsilon) const { - return Base::equals(tensor, epsilon); - } - - /// equality with ulps tolerance - bool bit_equals(TensorView<T> const& tensor, long long ulps_threshold = 0) { - return Base::bit_equals(tensor, ulps_threshold); - } - - /// Computes general matrix product among select dimensions of a tensor - /// Assumes: - /// D: number of independent GEMMs to compute - /// H: height of matrix - /// W: width of matrix - template < - /// Data type of A matrix elements - typename A, - /// Data type of B matrix elements - typename B, - /// Data type of "compute" type (i.e. accumulator) - typename Ctype, - /// Data type of scale factors - typename Stype> - void gemm(TensorView<A> const& tensor_a, TensorView<B> const& tensor_b, Stype alpha, Stype beta) { - Base::template gemm<A, B, Ctype, Stype>(tensor_a, tensor_b, alpha, beta); - } - - /// Fills with random data - template <typename Gen> - void fill_random(Gen generator) { - Base::fill_random(generator); - sync_device(); - } - - /// Procedurally assigns elements - template <typename Gen> - void generate(Gen generator) { - Base::generate(generator); - sync_device(); - } - - /// Procedurally visits elements - template <typename Gen> - void visit(Gen& generator) const { - Base::visit(generator); - } - - /// initializes with identity - void fill_identity() { - Base::fill_identity(); - sync_device(); - } - - /// computes elements as a linear combination of their coordinates - void fill_linear(Coord_t v, T offset = T(0)) { - Base::fill_linear(v, offset); - sync_device(); - } - - /// computes elements as a linear combination of their coordinates - void fill_sequential(T v = T(1), T offset = T(0)) { - Base::fill_sequential(v, offset); - sync_device(); - } - - /// fills with a value - void fill(T val = T(0)) { - Base::fill(val); - sync_device(); - } - - /// Copies from external data source and performs type conversion - template <typename Src> - void fill(TensorView<Src> const& tensor) { - Base::fill(tensor); - sync_device(); - } - - /// Computes the norm of the matrix in double-precision - double norm() const { return Base::norm(); } -}; -} // namespace cutlass diff --git a/cutlass-example/host_tensor_view.h b/cutlass-example/host_tensor_view.h deleted file mode 100644 index 56f02d3..0000000 --- a/cutlass-example/host_tensor_view.h +++ /dev/null @@ -1,542 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Host-side implementation of useful operations -*/ - -#pragma once - -#include <cutlass/cutlass.h> -#include <cutlass/tensor_view.h> -#include <type_traits.h> - -namespace cutlass { - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename SrcType, typename DstType> -struct Cast { - static inline DstType apply(SrcType src) { return static_cast<DstType>(src); }; -}; - -template <> -struct Cast<float, int8_t> { - static inline int8_t apply(float src) { - return static_cast<int8_t>(fmaxf(-128.f, fminf(127.f, src))); - }; -}; - -template <> -struct Cast<float, uint8_t> { - static inline uint8_t apply(float src) { - return static_cast<uint8_t>(fmaxf(0.f, fminf(255.f, src))); - }; -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -template <typename T> -class HostTensorView : public TensorView<T> { - public: - /// Base class - typedef TensorView<T> TensorView_t; - - /// Convention: depth is the first dimension - static int const Dim_D = 0; - - /// Convention: height is the second dimension - static int const Dim_H = 1; - - /// Convention: width is the third dimension - static int const Dim_W = 2; - - /// Convention: channel is the second dimension - static int const Dim_C = 3; - - /// Rank of tensor - static int const Rank = TensorView_t::Rank; - - /// Type used to compute the offset of an element to the base of a tensor - typedef typename TensorView_t::Offset_t Offset_t; - - /// Reference and stride - typedef typename TensorView_t::TensorRef_t TensorRef_t; - - /// Coordinate into tensor - typedef typename TensorView_t::Coord_t Coord_t; - - public: - // - // Device and Host Methods - // - - /// Default constructor - HostTensorView() {} - - /// Constructs a Tensor_view from a TensorRef and size - HostTensorView(TensorRef_t const& _ref, Coord_t const& _size) : TensorView_t(_ref, _size) {} - - /// Accesses the size - Coord_t const& size() const { return TensorView_t::size(); } - - /// Accesses the size of a specified dimension - int size(int dim) const { return size().at(dim); } - - /// Accesses the stride - Coord_t const& stride() const { return TensorView_t::stride(); } - - /// Accesses the stride along a specified dimension - int stride(int dim) const { return stride().at(dim); } - - /// Returns the number of scalar elements needed to store tensor - size_t capacity() const { return size(3) * stride(3) * stride(2) * stride(1) * stride(0); } - - /// Returns true if the Tensor_view is bound to some memory - bool good() const { return TensorView_t::good(); } - - /// Updates the reference and size of a TensorView object - void reset(TensorRef_t const& _ref = TensorRef_t(0), Coord_t const& _size = Coord_t()) { - return TensorView_t::reset(_ref, _size); - } - - /// Accesses the tensor reference pointing to data - TensorRef_t& ref() { return TensorView_t::ref(); } - - /// Accesses the tensor reference pointing to data - TensorRef_t const& ref() const { return TensorView_t::ref(); } - - /// Assigns a tensor view - HostTensorView& operator=(TensorView_t const& _tensor) { - reset(_tensor.ref(), _tensor.size()); - return *this; - } - - /// Returns the index of an element - Offset_t offset(Coord_t const& coord) const { return TensorView_t::offset(coord); } - - /// Determines whether a location is within a tensor - bool contains(Coord_t const& coord) const { return TensorView_t::contains(coord); } - - /// Element-wise accessor - T& at(Coord_t const& coord) const { return TensorView_t::at(coord); } - - /// Element-wise accessor - T& operator[](Coord_t const& coord) const { return at(coord); } - - /// Accesses an element with a raw offset - T& at(int idx) const { return TensorView_t::at(idx); } - - /// Accesses an element with a raw offset - T& operator[](int idx) const { return at(idx); } - - /// Returns a Tensor_view given location and size quantities - TensorView_t subview(Coord_t const& location, Coord_t size) const { - return TensorView_t::subview(location, size); - } - - /// Recurses through all dimensions and applies a unary operation in place - template <typename F> - void elementwise_in_place(F& op, int dim = 0, Offset_t dst_offset_base = 0) { - Offset_t dst_offset = dst_offset_base; - - for (int idx = 0; idx < size(dim); ++idx, dst_offset += stride(dim)) { - if (dim < Rank - 1) { - elementwise_in_place(op, dim + 1, dst_offset); - } else { - op(ref().data()[dst_offset]); - } - } - } - - /// Recurses through all dimensions and applies a unary operator with no arguments - template <typename F> - void elementwise_stream(F& op, int dim = 0, Offset_t dst_offset_base = 0) { - Offset_t dst_offset = dst_offset_base; - - for (int idx = 0; idx < size(dim); ++idx, dst_offset += stride(dim)) { - if (dim < Rank - 1) { - elementwise_stream(op, dim + 1, dst_offset); - } else { - ref().data()[dst_offset] = op(); - } - } - } - - /// Recurses through all dimensions and applies a unary operator, supplying the logical - /// coordinate within the tensor as an argument - template <typename F> - void elementwise_generate(F& op, - int dim = 0, - Offset_t dst_offset_base = 0, - Coord_t coord = Coord_t(0)) { - Offset_t dst_offset = dst_offset_base; - - for (int idx = 0; idx < size(dim); ++idx, dst_offset += stride(dim)) { - coord.at(dim) = idx; - - if (dim < Rank - 1) { - elementwise_generate(op, dim + 1, dst_offset, coord); - } else { - ref().data()[dst_offset] = op(coord); - } - } - } - - /// Recurses through all dimensions and applies a unary operator, supplying the logical - /// coordinate within the tensor as an argument - template <typename F> - void elementwise_visit(F& op, - int dim = 0, - Offset_t dst_offset_base = 0, - Coord_t coord = Coord_t(0)) const { - Offset_t dst_offset = dst_offset_base; - - for (int idx = 0; idx < size(dim); ++idx, dst_offset += stride(dim)) { - coord.at(dim) = idx; - - if (dim < Rank - 1) { - elementwise_visit(op, dim + 1, dst_offset, coord); - } else { - op(ref().data()[dst_offset], coord); - } - } - } - - /// Recurses through all dimensions and applies a binary operation - template <typename Src, typename F> - bool elementwise_in_place(F& op, - TensorView<Src> const& tensor, - int dim = 0, - Offset_t dst_offset_base = 0, - Offset_t src_offset_base = 0) { - Offset_t dst_offset = dst_offset_base; - Offset_t src_offset = src_offset_base; - - if (size().at(dim) != tensor.size().at(dim)) { - return false; - } - - for (int idx = 0; idx < size(dim); - ++idx, dst_offset += stride(dim), src_offset += tensor.stride(dim)) { - if (dim < Rank - 1) { - elementwise_in_place(op, tensor, dim + 1, dst_offset, src_offset); - } else { - op(data()[dst_offset], tensor.data()[src_offset]); - } - } - - return true; - } - - template <typename Src> - struct LambdaBinaryAddition { - void operator()(T& a, Src b) const { a += T(b); } - }; - - template <typename Src> - struct LambdaBinarySubtraction { - void operator()(T& a, Src b) const { a -= T(b); } - }; - - template <typename Src> - struct LambdaBinaryMultiplication { - void operator()(T& a, Src b) const { a *= T(b); } - }; - - template <typename Src> - struct LambdaBinaryDivision { - void operator()(T& a, Src b) const { a /= T(b); } - }; - - /// Accumulate in place - template <typename Src> - TensorView<T>& operator+=(TensorView<Src> const& tensor) { - LambdaBinaryAddition<Src> op; - elementwise_in_place(op, tensor); - - return *this; - } - - /// Subtract in place - template <typename Src> - TensorView<T>& operator-=(TensorView<Src> const& tensor) { - LambdaBinarySubtraction<Src> op; - elementwise_in_place(op, tensor); - - return *this; - } - - /// Multiply in place - template <typename Src> - TensorView<T>& operator*=(TensorView<Src> const& tensor) { - LambdaBinaryMultiplication<Src> op; - elementwise_in_place(op, tensor); - - return *this; - } - - /// Divide in place - template <typename Src> - TensorView<T>& operator/=(TensorView<Src> const& tensor) { - LambdaBinaryDivision<Src> op; - elementwise_in_place(op, tensor); - - return *this; - } - - /// Comparison operator - struct EqualsOperator { - bool equal; - T eps; - - EqualsOperator(T _epsilon) : equal(true), eps(_epsilon) {} - - void operator()(T a, T b) { - if (std::abs(T(a - b)) > eps * std::max(std::abs(a), std::abs(b))) { - equal = false; - } - } - }; - - /// equality with epsilon tolerance - bool equals(TensorView<T> const& tensor, T epsilon) const { - EqualsOperator comparison_op(epsilon); - bool equal_size = elementwise_in_place(comparison_op, tensor); - - return equal_size && comparison_op.equal; - } - - /// Compares two values which are smaller or equal to a long long int - struct BitEqualsOperator { - bool equal; - long long eps; - uint64_t index; - - BitEqualsOperator(long long _ulps_threshold) : equal(true), eps(_ulps_threshold), index(0) {} - - void operator()(T a, T b) { - // convert bits to integers - long long bits_a = 0; - long long bits_b = 0; - - *reinterpret_cast<T*>(&bits_a) = TypeTraits<T>::remove_negative_zero(a); - *reinterpret_cast<T*>(&bits_b) = TypeTraits<T>::remove_negative_zero(b); - - // compute diff - long long ulps = bits_a - bits_b; - if (std::abs(ulps) > eps) { - equal = false; - } - index++; - } - }; - - /// equality with ulps tolerance - bool bit_equals(TensorView<T> const& tensor, long long ulps_threshold = 0) { - BitEqualsOperator comparison_op(ulps_threshold); - bool equal_size = elementwise_in_place(comparison_op, tensor); - - return equal_size && comparison_op.equal; - } - - /// Gets naked pointer to data - T* data() const { return TensorView_t::data(); } - - /// Computes general matrix product among select dimensions of a tensor - /// Assumes: - /// D: number of independent GEMMs to compute - /// H: height of matrix - /// W: width of matrix - /// C: "channels" of each element - template <typename A, typename B, typename Ctype, typename Stype> - void gemm(TensorView<A> const& tensor_a, TensorView<B> const& tensor_b, Stype alpha, Stype beta) { - int const Batch = size(Dim_D); - int const M = size(Dim_H); - int const N = size(Dim_W); - int const K = tensor_a.size(Dim_W); - int const C = tensor_a.size(Dim_C); - - // Sizes must match - if (tensor_a.size(Dim_H) != M || tensor_b.size(Dim_W) != N || tensor_b.size(Dim_C) != C || - tensor_b.size(Dim_H) != K) { - return; - } - - int const Mblock = 32; - int const Nblock = 32; - - for (int batch = 0; batch < Batch; ++batch) { - for (int row_block = 0; row_block < M; row_block += Mblock) { - for (int col_block = 0; col_block < N; col_block += Nblock) { - Ctype accum[Mblock][Nblock]; - - for (int j = 0; j < Nblock; j++) { - for (int i = 0; i < Mblock; i++) { - accum[i][j] = Ctype(0); - } - } - - for (int k_block = 0; k_block < K; ++k_block) { - for (int j = 0; j < Nblock; j++) { - for (int i = 0; i < Mblock; i++) { - int row = row_block + i; - int col = col_block + j; - - if (row < M && col < N) { - for (int channel = 0; channel < C; ++channel) { - Ctype a(tensor_a.at(make_Coord(batch, row, k_block, channel))); - Ctype b(tensor_b.at(make_Coord(batch, k_block, col, channel))); - - accum[i][j] += a * b; - } - } - } - } - } - - for (int j = 0; j < Nblock; j++) { - for (int i = 0; i < Mblock; i++) { - int row = row_block + i; - int col = col_block + j; - - Coord_t coord = make_Coord(batch, row, col, 0); - if (row < M && col < N) { - at(coord) = - Cast<Stype, T>::apply(alpha * Stype(accum[i][j]) + beta * Stype(at(coord))); - } - } - } - } - } - } - } - - /// Fills with random data - template <typename Gen> - void fill_random(Gen generator) { - elementwise_stream(generator); - } - - /// Procedurally assigns elements - template <typename Gen> - void generate(Gen generator) { - elementwise_generate(generator); - } - - /// Procedurally visits elements - template <typename Gen> - void visit(Gen& generator) const { - elementwise_visit(generator); - } - - /// Generator to fill a tensor with the identity matrix - struct LambdaFillIdentity { - T operator()(Coord_t const& coord) { return (coord.at(1) == coord.at(2) ? T(1) : T(0)); } - }; - - /// initializes with identity - void fill_identity() { - LambdaFillIdentity op; - elementwise_generate(op); - } - - /// Lambda for fill_linear() - struct LambdaFillLinear { - Coord_t v_; - T offset_; - - LambdaFillLinear(Coord_t const& _v, T _offset) : v_(_v), offset_(_offset) {} - - T operator()(Coord_t const& coord) { return T(v_.template dot<int>(coord)) + offset_; } - }; - - /// computes elements as a linear combination of their coordinates - void fill_linear(Coord_t v, T offset = T(0)) { - LambdaFillLinear lambda(v, offset); - elementwise_generate(lambda); - } - - /// computes elements as a linear combination of their coordinates - void fill_sequential(T v = T(1), T offset = T(0)) { - int const count = size().count(); - for (int i = 0; i < count; ++i) { - data()[i] = T(i); - } - } - - /// Returns a constant value - struct LambdaFillValue { - T value; - - LambdaFillValue(T _value) : value(_value) {} - - T operator()() { return value; } - }; - - /// fills with a value - void fill(T val = T(0)) { - LambdaFillValue op(val); - elementwise_stream(op); - } - - /// Conversion from Src to T - template <typename Src> - struct LambdaAssign { - void operator()(T& a, Src b) const { a = T(b); } - }; - - /// copies from external data source and performs type conversion - template <typename Src> - void fill(TensorView<Src> const& tensor) { - LambdaAssign<Src> op; - elementwise_in_place(op, tensor); - } - - /// Computes a norm - struct LambdaNorm { - double sum; - - LambdaNorm() : sum(0) {} - - void operator()(T const& element) { - double value(element); - double conj(element); // TODO - conjugates for complex - - sum += value * conj; - } - }; - - /// Computes the norm of the matrix in double-precision - double norm() const { - LambdaNorm op; - elementwise_in_place(op); - - return std::sqrt(op.sum); - } -}; - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -} // namespace cutlass diff --git a/cutlass-example/launch.sh b/cutlass-example/launch.sh deleted file mode 100755 index e48f53e..0000000 --- a/cutlass-example/launch.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -make clean -make CFLAGS=-DWMMA_GEMM_16x16x16_NT -./myprog > LOG_WMMA_GEMM_16x16x16_NT -make clean -make CFLAGS=-DWMMA_GEMM_16x16x32_NT -./myprog > LOG_WMMA_GEMM_16x16x32_NT -make clean -make CFLAGS=-DWMMA_GEMM_16x16x16_NN -./myprog > LOG_WMMA_GEMM_16x16x16_NN -make clean -make CFLAGS=-DWMMA_GEMM_16x16x32_NN -./myprog > LOG_WMMA_GEMM_16x16x32_NN -make clean -make CFLAGS=-DWMMA_GEMM_16x16x16_TT -./myprog > LOG_WMMA_GEMM_16x16x16_TT -make clean -make CFLAGS=-DWMMA_GEMM_16x16x32_TT -./myprog > LOG_WMMA_GEMM_16x16x32_TT -make clean -make CFLAGS=-DWMMA_GEMM_16x16x16_TN -./myprog > LOG_WMMA_GEMM_16x16x16_TN -make clean -make CFLAGS=-DWMMA_GEMM_16x16x32_TN -./myprog > LOG_WMMA_GEMM_16x16x32_TN -make clean -make CFLAGS=-DWMMA_16x16x16_GEMM_256x256x128_NT -./myprog > LOG_WMMA_16x16x16_GEMM_256x256x128_NT -make clean -make CFLAGS=-DWMMA_16x16x16_GEMM_256x256x128_NN -./myprog > LOG_WMMA_16x16x16_GEMM_256x256x128_NN -make clean -make CFLAGS=-DWMMA_16x16x16_GEMM_256x256x128_TT -./myprog > LOG_WMMA_16x16x16_GEMM_256x256x128_TT -make clean -make CFLAGS=-DWMMA_16x16x16_GEMM_256x256x128_TN -./myprog > LOG_WMMA_16x16x16_GEMM_256x256x128_TN -make clean -make CFLAGS=-DWMMA_GEMM_256x256x256_NT -./myprog > LOG_WMMA_GEMM_256x256x256_NT -make clean -make CFLAGS=-DWMMA_GEMM_256x256x256_NN -./myprog > LOG_WMMA_GEMM_256x256x256_NN -make clean -make CFLAGS=-DWMMA_GEMM_256x256x256_TT -./myprog > LOG_WMMA_GEMM_256x256x256_TT -make clean -make CFLAGS=-DWMMA_GEMM_256x256x256_TN -./myprog > LOG_WMMA_GEMM_256x256x256_TN -make clean -make CFLAGS=-DWMMA_GEMM_512x512x512_NT -./myprog > LOG_WMMA_GEMM_512x512x512_NT -make clean -make CFLAGS=-DWMMA_GEMM_512x512x512_NN -./myprog > LOG_WMMA_GEMM_512x512x512_NN -make clean -make CFLAGS=-DWMMA_GEMM_512x512x512_TT -./myprog > LOG_WMMA_GEMM_512x512x512_TT -make clean -make CFLAGS=-DWMMA_GEMM_512x512x512_TN -./myprog > LOG_WMMA_GEMM_512x512x512_TN -make clean -make CFLAGS=-DWMMA_GEMM_1024x1024x1024_NT -./myprog > LOG_WMMA_GEMM_1024x1024x1024_NT -make clean -make CFLAGS=-DWMMA_GEMM_1024x1024x1024_NN -./myprog > LOG_WMMA_GEMM_1024x1024x1024_NN -make clean -make CFLAGS=-DWMMA_GEMM_1024x1024x1024_TT -./myprog > LOG_WMMA_GEMM_1024x1024x1024_TT -make clean -make CFLAGS=-DWMMA_GEMM_1024x1024x1024_TN -./myprog > LOG_WMMA_GEMM_1024x1024x1024_TN -make clean -make CFLAGS=-DWMMA_GEMM_2048x2048x2048_NT -./myprog > LOG_WMMA_GEMM_2048x2048x2048_NT -make clean -make CFLAGS=-DWMMA_GEMM_2048x2048x2048_NN -./myprog > LOG_WMMA_GEMM_2048x2048x2048_NN -make clean -make CFLAGS=-DWMMA_GEMM_2048x2048x2048_TT -./myprog > LOG_WMMA_GEMM_2048x2048x2048_TT -make clean -make CFLAGS=-DWMMA_GEMM_2048x2048x2048_TN -./myprog > LOG_WMMA_GEMM_2048x2048x2048_TN -make clean -make CFLAGS=-DWMMA_GEMM_4096x4096x4096_NT -./myprog > LOG_WMMA_GEMM_4096x4096x4096_NT -make clean -make CFLAGS=-DWMMA_GEMM_4096x4096x4096_NN -./myprog > LOG_WMMA_GEMM_4096x4096x4096_NN -make clean -make CFLAGS=-DWMMA_GEMM_4096x4096x4096_TT -./myprog > LOG_WMMA_GEMM_4096x4096x4096_TT -make clean -make CFLAGS=-DWMMA_GEMM_4096x4096x4096_TN -./myprog > LOG_WMMA_GEMM_4096x4096x4096_TN -make clean -make CFLAGS=-DWMMA_GEMM_32x32x32_NT -./myprog > LOG_WMMA_GEMM_32x32x32_NT -make clean -make CFLAGS=-DWMMA_GEMM_32x32x32_NN -./myprog > LOG_WMMA_GEMM_32x32x32_NN -make clean -make CFLAGS=-DWMMA_GEMM_32x32x32_TT -./myprog > LOG_WMMA_GEMM_32x32x32_TT -make clean -make CFLAGS=-DWMMA_GEMM_32x32x32_TN -./myprog > LOG_WMMA_GEMM_32x32x32_TN -make clean -make CFLAGS=-DWMMA_GEMM_128x128x128_NT -./myprog > LOG_WMMA_GEMM_128x128x128_NT -make clean -make CFLAGS=-DWMMA_GEMM_128x128x128_NN -./myprog > LOG_WMMA_GEMM_128x128x128_NN -make clean -make CFLAGS=-DWMMA_GEMM_128x128x128_TT -./myprog > LOG_WMMA_GEMM_128x128x128_TT -make clean -make CFLAGS=-DWMMA_GEMM_128x128x128_TN -./myprog > LOG_WMMA_GEMM_128x128x128_TN -make clean -make CFLAGS=-DWMMA_GEMM_768x768x768_NT -./myprog > LOG_WMMA_GEMM_768x768x768_NT -make clean -make CFLAGS=-DWMMA_GEMM_768x768x768_NN -./myprog > LOG_WMMA_GEMM_768x768x768_NN -make clean -make CFLAGS=-DWMMA_GEMM_768x768x768_TT -./myprog > LOG_WMMA_GEMM_768x768x768_TT -make clean -make CFLAGS=-DWMMA_GEMM_768x768x768_TN -./myprog > LOG_WMMA_GEMM_768x768x768_TN -make clean diff --git a/cutlass-example/tensor_view_io.h b/cutlass-example/tensor_view_io.h deleted file mode 100644 index bc2e9b1..0000000 --- a/cutlass-example/tensor_view_io.h +++ /dev/null @@ -1,61 +0,0 @@ -/*************************************************************************************************** -* Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. -* -* Redistribution and use in source and binary forms, with or without modification, are permitted -* provided that the following conditions are met: -* * Redistributions of source code must retain the above copyright notice, this list of -* conditions and the following disclaimer. -* * Redistributions in binary form must reproduce the above copyright notice, this list of -* conditions and the following disclaimer in the documentation and/or other materials -* provided with the distribution. -* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used -* to endorse or promote products derived from this software without specific prior written -* permission. -* -* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR -* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE -* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -* -**************************************************************************************************/ -#pragma once - -#include <cutlass/core_io.h> -#include <cutlass/tensor_view.h> - -template <typename T> -inline std::ostream& tensor_view_output(std::ostream& out, T t) { - out << t; - return out; -} - -template <> -inline std::ostream& tensor_view_output<int8_t>(std::ostream& out, int8_t t) { - out << int(t); - return out; -} - -template <typename T> -inline std::ostream& operator<<(std::ostream& out, cutlass::TensorView<T> const& tensor) { - for (int batch = 0; batch < tensor.size(0); ++batch) { - out << "[\n "; - for (int h = 0; h < tensor.size(1); ++h) { - for (int w = 0; w < tensor.size(2); ++w) { - for (int c = 0; c < tensor.size(3); ++c) { - out << ((c | w) ? " " : ""); - tensor_view_output(out, tensor.at(cutlass::make_Coord(batch, h, w, c))); - } - } - if (h + 1 < tensor.size(1)) { - out << " ;\n "; - } - } - out << " ]"; - } - - return out; -} diff --git a/cutlass-example/type_traits.h b/cutlass-example/type_traits.h deleted file mode 100644 index eabd67e..0000000 --- a/cutlass-example/type_traits.h +++ /dev/null @@ -1,160 +0,0 @@ -/*************************************************************************************************** - * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without modification, are permitted - * provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright notice, this list of - * conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright notice, this list of - * conditions and the following disclaimer in the documentation and/or other materials - * provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used - * to endorse or promote products derived from this software without specific prior written - * permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND - * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -/*! \file - \brief Type traits for common CUDA types -*/ - -#pragma once - -#include <cuda_fp16.h> -#include <stdint.h> - -#include "half.h" - -namespace cutlass { -struct half_t; - -template <typename T> -struct TypeTraits; - -template <> -struct TypeTraits<int8_t> { -// static cudaDataType_t const cublas_type = CUDA_R_8I; - typedef int8_t host_type; - typedef int8_t device_type; - typedef int8_t integer_type; - typedef uint8_t unsigned_type; - static inline int8_t remove_negative_zero(int8_t x) { return x; } - static inline int to_print(int8_t x) { return (int)x; } -}; - -template <> -struct TypeTraits<uint8_t> { -// static cudaDataType_t const cublas_type = CUDA_R_8I; - typedef uint8_t host_type; - typedef uint8_t device_type; - typedef uint8_t integer_type; - typedef uint8_t unsigned_type; - static inline uint8_t remove_negative_zero(uint8_t x) { return x; } - static inline uint32_t to_print(uint8_t x) { return (uint32_t)x; } -}; - -template <> -struct TypeTraits<int> { -// static cudaDataType_t const cublas_type = CUDA_R_32I; - typedef int host_type; - typedef int device_type; - typedef int32_t integer_type; - typedef uint32_t unsigned_type; - static inline int32_t remove_negative_zero(int32_t x) { return x; } - static inline int to_print(int x) { return x; } -}; - -template <> -struct TypeTraits<unsigned> { -// static cudaDataType_t const cublas_type = CUDA_R_32I; - typedef unsigned host_type; - typedef unsigned device_type; - typedef uint32_t integer_type; - typedef uint32_t unsigned_type; - static inline uint32_t remove_negative_zero(uint32_t x) { return x; } - static inline uint32_t to_print(uint32_t x) { return x; } -}; - -template <> -struct TypeTraits<half> { -// static cudaDataType_t const cublas_type = CUDA_R_16F; - typedef half_t host_type; - typedef half device_type; - typedef int16_t integer_type; - typedef uint16_t unsigned_type; - static inline half remove_negative_zero(half x) { - integer_type h_int = reinterpret_cast<integer_type const&>(x); - if (h_int == 0x8000) { - h_int = 0; - } - x = reinterpret_cast<half const&>(h_int); - return x; - } - static inline half to_print(half x) { return x; } -}; - -template <> -struct TypeTraits<int64_t> { -// static cudaDataType_t const cublas_type = CUDA_R_8I; - typedef int64_t host_type; - typedef int64_t device_type; - typedef int64_t integer_type; - typedef uint64_t unsigned_type; - static inline int64_t remove_negative_zero(int64_t x) { return x; } - static inline int64_t to_print(int64_t x) { return x; } -}; - -template <> -struct TypeTraits<uint64_t> { -// static cudaDataType_t const cublas_type = CUDA_R_8I; - typedef uint64_t host_type; - typedef uint64_t device_type; - typedef uint64_t integer_type; - typedef uint64_t unsigned_type; - static inline uint64_t remove_negative_zero(uint64_t x) { return x; } - static inline uint64_t to_print(uint64_t x) { return x; } -}; - -template <> -struct TypeTraits<cutlass::half_t> { -// static cudaDataType_t const cublas_type = CUDA_R_16F; - typedef half_t host_type; - typedef half device_type; - typedef int16_t integer_type; - typedef uint16_t unsigned_type; - static inline half_t remove_negative_zero(half_t x) { - return (x.raw() == 0x8000 ? half_t::bitcast(0) : x); - } - static inline half_t to_print(half_t x) { return x; } -}; - -template <> -struct TypeTraits<float> { -// static cudaDataType_t const cublas_type = CUDA_R_32F; - typedef float host_type; - typedef float device_type; - typedef int32_t integer_type; - typedef uint32_t unsigned_type; - static inline float remove_negative_zero(float x) { return x == -0.f ? 0.f : x; } - static inline float to_print(float x) { return x; } -}; - -template <> -struct TypeTraits<double> { -// static cudaDataType_t const cublas_type = CUDA_R_64F; - typedef double host_type; - typedef double device_type; - typedef int64_t integer_type; - typedef uint64_t unsigned_type; - static inline double remove_negative_zero(double x) { return x == -0.0 ? 0.0 : x; } - static inline double to_print(double x) { return x; } -}; -} // namespace cutlass |