From a34a4baad45a40840308167307c79371b4024a9d Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 13 Sep 2019 07:54:00 -0400 Subject: Big reformat change using clang-format-6.0 --- src/cuda-sim/cuda-math.h | 483 +- src/cuda-sim/cuda-sim.cc | 4923 +++++++-------- src/cuda-sim/cuda-sim.h | 314 +- src/cuda-sim/cuda_device_printf.cc | 192 +- src/cuda-sim/cuda_device_printf.h | 37 +- src/cuda-sim/cuda_device_runtime.cc | 568 +- src/cuda-sim/cuda_device_runtime.h | 91 +- src/cuda-sim/half.h | 6543 +++++++++++--------- src/cuda-sim/instructions.cc | 10867 ++++++++++++++++++---------------- src/cuda-sim/memory.cc | 363 +- src/cuda-sim/memory.h | 185 +- src/cuda-sim/opcodes.h | 104 +- src/cuda-sim/ptx-stats.cc | 403 +- src/cuda-sim/ptx-stats.h | 81 +- src/cuda-sim/ptx_ir.cc | 2692 +++++---- src/cuda-sim/ptx_ir.h | 2973 +++++----- src/cuda-sim/ptx_loader.cc | 978 +-- src/cuda-sim/ptx_loader.h | 65 +- src/cuda-sim/ptx_parser.cc | 1812 +++--- src/cuda-sim/ptx_parser.h | 329 +- src/cuda-sim/ptx_sim.cc | 1088 ++-- src/cuda-sim/ptx_sim.h | 939 ++- 22 files changed, 18965 insertions(+), 17065 deletions(-) (limited to 'src/cuda-sim') diff --git a/src/cuda-sim/cuda-math.h b/src/cuda-sim/cuda-math.h index 9a5468c..67539d4 100644 --- a/src/cuda-sim/cuda-math.h +++ b/src/cuda-sim/cuda-math.h @@ -1,6 +1,6 @@ // This file created from vector_types.h distributed with CUDA 1.1 // (see original copyright notice below) -// +// // Changes Copyright (c) 2009-2011, Tor M. Aamodt, Wilson W.L. Fung // The University of British Columbia // All rights reserved. @@ -10,61 +10,60 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. /* * Copyright 1993-2007 NVIDIA Corporation. All rights reserved. * - * NOTICE TO USER: + * NOTICE TO USER: * - * This source code is subject to NVIDIA ownership rights under U.S. and - * international Copyright laws. Users and possessors of this source code - * are hereby granted a nonexclusive, royalty-free license to use this code + * This source code is subject to NVIDIA ownership rights under U.S. and + * international Copyright laws. Users and possessors of this source code + * are hereby granted a nonexclusive, royalty-free license to use this code * in individual and commercial software. * - * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE - * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR - * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH - * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF + * NVIDIA MAKES NO REPRESENTATION ABOUT THE SUITABILITY OF THIS SOURCE + * CODE FOR ANY PURPOSE. IT IS PROVIDED "AS IS" WITHOUT EXPRESS OR + * IMPLIED WARRANTY OF ANY KIND. NVIDIA DISCLAIMS ALL WARRANTIES WITH + * REGARD TO THIS SOURCE CODE, INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY, NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. - * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, - * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS - * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE - * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE - * OR PERFORMANCE OF THIS SOURCE CODE. + * IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY SPECIAL, INDIRECT, INCIDENTAL, + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE + * OR PERFORMANCE OF THIS SOURCE CODE. * - * U.S. Government End Users. This source code is a "commercial item" as - * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of - * "commercial computer software" and "commercial computer software - * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) - * and is provided to the U.S. Government only as a commercial end item. - * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through - * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the - * source code with only those rights set forth herein. + * U.S. Government End Users. This source code is a "commercial item" as + * that term is defined at 48 C.F.R. 2.101 (OCT 1995), consisting of + * "commercial computer software" and "commercial computer software + * documentation" as such terms are used in 48 C.F.R. 12.212 (SEPT 1995) + * and is provided to the U.S. Government only as a commercial end item. + * Consistent with 48 C.F.R.12.212 and 48 C.F.R. 227.7202-1 through + * 227.7202-4 (JUNE 1995), all U.S. Government End Users acquire the + * source code with only those rights set forth herein. * - * Any use of this source code in individual and commercial software must + * Any use of this source code in individual and commercial software must * include, in the user documentation and internal comments to the code, * the above Disclaimer and U.S. Government End Users Notice. */ - #ifndef CUDA_MATH #define CUDA_MATH @@ -74,32 +73,31 @@ #undef max #undef min namespace cuda_math { -#define __attribute__(a) // to remove warnings inside math_functions.h +#define __attribute__(a) // to remove warnings inside math_functions.h #undef INT_MAX #if CUDART_VERSION < 3000 // DEVICE_BUILTIN - struct int4 { - int x, y, z, w; - }; - struct uint4 { - unsigned int x, y, z, w; - }; - struct float4 { - float x, y, z, w; - }; - struct float2 { - float x, y; - }; - +struct int4 { + int x, y, z, w; +}; +struct uint4 { + unsigned int x, y, z, w; +}; +struct float4 { + float x, y, z, w; +}; +struct float2 { + float x, y; +}; // DEVICE_BUILTIN - typedef struct int4 int4; - typedef struct uint4 uint4; - typedef struct float4 float4; - typedef struct float2 float2; +typedef struct int4 int4; +typedef struct uint4 uint4; +typedef struct float4 float4; +typedef struct float2 float2; -extern float rsqrtf(float); // CUDA 2.3 beta +extern float rsqrtf(float); // CUDA 2.3 beta #define CUDA_FLOAT_MATH_FUNCTIONS #include @@ -108,38 +106,36 @@ extern float rsqrtf(float); // CUDA 2.3 beta #undef __CUDA_INTERNAL_COMPILATION__ #undef __attribute__ -// float to integer conversion -int float2int(float a, enum cudaRoundMode mode) -{ - return __internal_float2uint(a, mode); +// float to integer conversion +int float2int(float a, enum cudaRoundMode mode) { + return __internal_float2uint(a, mode); } -// float to unsigned integer conversion -unsigned int float2uint(float a, enum cudaRoundMode mode) -{ - return __internal_float2uint(a, mode); +// float to unsigned integer conversion +unsigned int float2uint(float a, enum cudaRoundMode mode) { + return __internal_float2uint(a, mode); } float __ll2float_rz(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_ru(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rd(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } #else @@ -147,205 +143,211 @@ float __ll2float_rd(long long int a) { #define CUDA_FLOAT_MATH_FUNCTIONS #define __CUDACC__ -// implementing int to float intrinsics with different rounding modes +// implementing int to float intrinsics with different rounding modes #include #include - // 32-bit integer to float float __int2float_rn(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_rz(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_ru(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __int2float_rd(int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } // 32-bit unsigned integer to float float __uint2float_rn(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_rz(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_ru(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __uint2float_rd(unsigned int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } // 64-bit integer to float float __ll2float_rn(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rz(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_ru(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ll2float_rd(long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } -// 64-bit unsigned integer to float +// 64-bit unsigned integer to float float __ull2float_rn(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TONEAREST); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TONEAREST); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_rz(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_TOWARDZERO); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_TOWARDZERO); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_ru(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_UPWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; + int orig_rnd_mode = fegetround(); + fesetround(FE_UPWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; } float __ull2float_rd(unsigned long long int a) { - int orig_rnd_mode = fegetround(); - fesetround(FE_DOWNWARD); - float b = a; - fesetround(orig_rnd_mode); - return b; -} - -// float to integer conversion -int float2int(float a, enum cudaRoundMode mode) -{ - int tmp; - switch (mode) { - case cudaRoundZero: tmp = truncf(a); break; - case cudaRoundNearest: tmp = nearbyintf(a); break; - case cudaRoundMinInf: tmp = floorf(a); break; - case cudaRoundPosInf: tmp = ceilf(a); break; - default: abort(); - } - return tmp; -} - -int __internal_float2int(float a, enum cudaRoundMode mode) -{ - return float2int(a, mode); -} - -// float to unsigned integer conversion -unsigned int float2uint(float a, enum cudaRoundMode mode) -{ - unsigned int tmp; - switch (mode) { - case cudaRoundZero: tmp = truncf(a); break; - case cudaRoundNearest: tmp = nearbyintf(a); break; - case cudaRoundMinInf: tmp = floorf(a); break; - case cudaRoundPosInf: tmp = ceilf(a); break; - default: abort(); - } - return tmp; -} - -unsigned int __internal_float2uint(float a, enum cudaRoundMode mode) -{ - return float2uint(a, mode); -} - -// intrinsic for division -float fdividef(float a, float b) -{ - return (a / b); -} - -float __internal_accurate_fdividef(float a, float b) -{ - return fdividef(a, b); -} + int orig_rnd_mode = fegetround(); + fesetround(FE_DOWNWARD); + float b = a; + fesetround(orig_rnd_mode); + return b; +} + +// float to integer conversion +int float2int(float a, enum cudaRoundMode mode) { + int tmp; + switch (mode) { + case cudaRoundZero: + tmp = truncf(a); + break; + case cudaRoundNearest: + tmp = nearbyintf(a); + break; + case cudaRoundMinInf: + tmp = floorf(a); + break; + case cudaRoundPosInf: + tmp = ceilf(a); + break; + default: + abort(); + } + return tmp; +} + +int __internal_float2int(float a, enum cudaRoundMode mode) { + return float2int(a, mode); +} + +// float to unsigned integer conversion +unsigned int float2uint(float a, enum cudaRoundMode mode) { + unsigned int tmp; + switch (mode) { + case cudaRoundZero: + tmp = truncf(a); + break; + case cudaRoundNearest: + tmp = nearbyintf(a); + break; + case cudaRoundMinInf: + tmp = floorf(a); + break; + case cudaRoundPosInf: + tmp = ceilf(a); + break; + default: + abort(); + } + return tmp; +} + +unsigned int __internal_float2uint(float a, enum cudaRoundMode mode) { + return float2uint(a, mode); +} + +// intrinsic for division +float fdividef(float a, float b) { return (a / b); } + +float __internal_accurate_fdividef(float a, float b) { return fdividef(a, b); } // intrinsic for saturate (clamp values beyond 0 and 1) -float __saturatef(float a) -{ - float b; - if (std::isnan(a)) b = 0.0f; - else if (a >= 1.0f) b = 1.0f; - else if (a <= 0.0f) b = 0.0f; - else b = a; - return b; -} - -// intrinsic for power -float __powf(float a, float b) -{ - return powf(a, b); -} +float __saturatef(float a) { + float b; + if (std::isnan(a)) + b = 0.0f; + else if (a >= 1.0f) + b = 1.0f; + else if (a <= 0.0f) + b = 0.0f; + else + b = a; + return b; +} + +// intrinsic for power +float __powf(float a, float b) { return powf(a, b); } // math functions missing in Mac OSX GCC #ifdef __APPLE__ -int __signbitd(double d) -{ - unsigned long long int u = *((unsigned long long int*)&d); - return ((u & 0x8000000000000000ULL) != 0); +int __signbitd(double d) { + unsigned long long int u = *((unsigned long long int*)&d); + return ((u & 0x8000000000000000ULL) != 0); } -#endif +#endif #undef __CUDACC__ #define __CUDA_INTERNAL_COMPILATION__ @@ -355,14 +357,11 @@ int __signbitd(double d) #endif -} +} // namespace cuda_math // math functions missing in Mac OSX GCC #ifdef __APPLE__ -int isnanf(float a) -{ - return (std::isnan(a)); -} -#endif +int isnanf(float a) { return (std::isnan(a)); } +#endif #endif diff --git a/src/cuda-sim/cuda-sim.cc b/src/cuda-sim/cuda-sim.cc index 7a130ea..39e2b7e 100644 --- a/src/cuda-sim/cuda-sim.cc +++ b/src/cuda-sim/cuda-sim.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, -// George L. Yuan, Jimmy Kwa +// George L. Yuan, Jimmy Kwa // The University of British Columbia // All rights reserved. // @@ -8,283 +8,340 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "cuda-sim.h" #include "instructions.h" #include "ptx_ir.h" class ptx_recognizer; -typedef void * yyscan_t; -#include "ptx.tab.h" -#include "ptx_sim.h" +typedef void *yyscan_t; #include -#include -#include "opcodes.h" -#include "../statwrapper.h" -#include #include +#include +#include +#include "../../libcuda/gpgpu_context.h" #include "../abstract_hardware_model.h" +#include "../gpgpu-sim/gpu-sim.h" +#include "../gpgpusim_entrypoint.h" +#include "../statwrapper.h" +#include "../stream_manager.h" +#include "cuda_device_runtime.h" +#include "decuda_pred_table/decuda_pred_table.h" #include "memory.h" +#include "opcodes.h" #include "ptx-stats.h" +#include "ptx.tab.h" #include "ptx_loader.h" #include "ptx_parser.h" -#include "../gpgpu-sim/gpu-sim.h" #include "ptx_sim.h" -#include "../gpgpusim_entrypoint.h" -#include "decuda_pred_table/decuda_pred_table.h" -#include "../stream_manager.h" -#include "cuda_device_runtime.h" -#include "../../libcuda/gpgpu_context.h" int g_debug_execution = 0; // Output debug information to file options - -void cuda_sim::ptx_opcocde_latency_options (option_parser_t opp) { - option_parser_register(opp, "-ptx_opcode_latency_int", OPT_CSTR, &opcode_latency_int, - "Opcode latencies for integers " - "Default 1,1,19,25,145", - "1,1,19,25,145"); - option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, &opcode_latency_fp, - "Opcode latencies for single precision floating points " - "Default 1,1,1,1,30", - "1,1,1,1,30"); - option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, &opcode_latency_dp, - "Opcode latencies for double precision floating points " - "Default 8,8,8,8,335", - "8,8,8,8,335"); - option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, &opcode_latency_sfu, - "Opcode latencies for SFU instructions" - "Default 8", - "8"); - option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, &opcode_latency_tensor, - "Opcode latencies for Tensor instructions" - "Default 64", - "64"); - option_parser_register(opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, - "Opcode initiation intervals for integers " - "Default 1,1,4,4,32", - "1,1,4,4,32"); - option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, &opcode_initiation_fp, - "Opcode initiation intervals for single precision floating points " - "Default 1,1,1,1,5", - "1,1,1,1,5"); - option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, &opcode_initiation_dp, - "Opcode initiation intervals for double precision floating points " - "Default 8,8,8,8,130", - "8,8,8,8,130"); - option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, &opcode_initiation_sfu, - "Opcode initiation intervals for sfu instructions" - "Default 8", - "8"); - option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, &opcode_initiation_tensor, - "Opcode initiation intervals for tensor instructions" - "Default 64", - "64"); - option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, - "CDP API latency " + "Default 1,1,19,25,145", + "1,1,19,25,145"); + option_parser_register(opp, "-ptx_opcode_latency_fp", OPT_CSTR, + &opcode_latency_fp, + "Opcode latencies for single precision floating " + "points " + "Default 1,1,1,1,30", + "1,1,1,1,30"); + option_parser_register(opp, "-ptx_opcode_latency_dp", OPT_CSTR, + &opcode_latency_dp, + "Opcode latencies for double precision floating " + "points " + "Default 8,8,8,8,335", + "8,8,8,8,335"); + option_parser_register(opp, "-ptx_opcode_latency_sfu", OPT_CSTR, + &opcode_latency_sfu, + "Opcode latencies for SFU instructions" + "Default 8", + "8"); + option_parser_register(opp, "-ptx_opcode_latency_tesnor", OPT_CSTR, + &opcode_latency_tensor, + "Opcode latencies for Tensor instructions" + "Default 64", + "64"); + option_parser_register( + opp, "-ptx_opcode_initiation_int", OPT_CSTR, &opcode_initiation_int, + "Opcode initiation intervals for integers " + "Default 1,1,4,4,32", + "1,1,4,4,32"); + option_parser_register(opp, "-ptx_opcode_initiation_fp", OPT_CSTR, + &opcode_initiation_fp, + "Opcode initiation intervals for single precision " + "floating points " + "Default 1,1,1,1,5", + "1,1,1,1,5"); + option_parser_register(opp, "-ptx_opcode_initiation_dp", OPT_CSTR, + &opcode_initiation_dp, + "Opcode initiation intervals for double precision " + "floating points " + "Default 8,8,8,8,130", + "8,8,8,8,130"); + option_parser_register(opp, "-ptx_opcode_initiation_sfu", OPT_CSTR, + &opcode_initiation_sfu, + "Opcode initiation intervals for sfu instructions" + "Default 8", + "8"); + option_parser_register(opp, "-ptx_opcode_initiation_tensor", OPT_CSTR, + &opcode_initiation_tensor, + "Opcode initiation intervals for tensor instructions" + "Default 64", + "64"); + option_parser_register(opp, "-cdp_latency", OPT_CSTR, &cdp_latency_str, + "CDP API latency " - "Default 7200,8000,100,12000,1600", - "7200,8000,100,12000,1600"); -} - -void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture(const char* name, const struct textureReference* texref, int dim, int readmode, int ext) -{ - std::string texname(name); - if (m_NameToTextureRef.find(texname)==m_NameToTextureRef.end()){ - m_NameToTextureRef[texname] = std::set(); - }else{ - const struct textureReference* tr = *m_NameToTextureRef[texname].begin(); - assert(tr!=NULL); - //asserts that all texrefs in set have same fields - assert(tr->normalized==texref->normalized&& - tr->filterMode==texref->filterMode&& - tr->addressMode[0]==texref->addressMode[0]&& - tr->addressMode[1]==texref->addressMode[1]&& - tr->addressMode[2]==texref->addressMode[2]&& - tr->channelDesc.x==texref->channelDesc.x&& - tr->channelDesc.y==texref->channelDesc.y&& - tr->channelDesc.z==texref->channelDesc.z&& - tr->channelDesc.w==texref->channelDesc.w&& - tr->channelDesc.f==texref->channelDesc.f - ); - } - m_NameToTextureRef[texname].insert(texref); - m_TextureRefToName[texref] = texname; - const textureReferenceAttr *texAttr = new textureReferenceAttr(texref, dim, (enum cudaTextureReadMode)readmode, ext); - m_NameToAttribute[texname] = texAttr; -} - -const char* gpgpu_t::gpgpu_ptx_sim_findNamefromTexture(const struct textureReference* texref) -{ - std::map::const_iterator t=m_TextureRefToName.find(texref); - assert( t != m_TextureRefToName.end() ); - return t->second.c_str(); -} - -unsigned int intLOGB2( unsigned int v ) { - unsigned int shift; - unsigned int r; - - r = 0; - - shift = (( v & 0xFFFF0000) != 0 ) << 4; v >>= shift; r |= shift; - shift = (( v & 0xFF00 ) != 0 ) << 3; v >>= shift; r |= shift; - shift = (( v & 0xF0 ) != 0 ) << 2; v >>= shift; r |= shift; - shift = (( v & 0xC ) != 0 ) << 1; v >>= shift; r |= shift; - shift = (( v & 0x2 ) != 0 ) << 0; v >>= shift; r |= shift; - - return r; -} - -void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray(const struct textureReference* texref, const struct cudaArray* array) -{ - std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); - - std::map::const_iterator t=m_NameToCudaArray.find(texname); - //check that there's nothing there first - if(t != m_NameToCudaArray.end()){ - printf("GPGPU-Sim PTX: Warning: binding to texref associated with %s, which was previously bound.\nImplicitly unbinding texref associated to %s first\n", texname.c_str(), texname.c_str()); - } - m_NameToCudaArray[texname] = array; - unsigned int texel_size_bits = array->desc.w + array->desc.x + array->desc.y + array->desc.z; - unsigned int texel_size = texel_size_bits/8; - unsigned int Tx, Ty; - int r; - - printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); - printf("GPGPU-Sim PTX: texture cache linesize = %d\n", m_function_model_config.get_texcache_linesize()); - //first determine base Tx size for given linesize - switch (m_function_model_config.get_texcache_linesize()) { - case 16: Tx = 4; break; - case 32: Tx = 8; break; - case 64: Tx = 8; break; - case 128: Tx = 16; break; - case 256: Tx = 16; break; - default: - printf("GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", m_function_model_config.get_texcache_linesize()); + "Default 7200,8000,100,12000,1600", + "7200,8000,100,12000,1600"); +} + +void gpgpu_t::gpgpu_ptx_sim_bindNameToTexture( + const char *name, const struct textureReference *texref, int dim, + int readmode, int ext) { + std::string texname(name); + if (m_NameToTextureRef.find(texname) == m_NameToTextureRef.end()) { + m_NameToTextureRef[texname] = std::set(); + } else { + const struct textureReference *tr = *m_NameToTextureRef[texname].begin(); + assert(tr != NULL); + // asserts that all texrefs in set have same fields + assert(tr->normalized == texref->normalized && + tr->filterMode == texref->filterMode && + tr->addressMode[0] == texref->addressMode[0] && + tr->addressMode[1] == texref->addressMode[1] && + tr->addressMode[2] == texref->addressMode[2] && + tr->channelDesc.x == texref->channelDesc.x && + tr->channelDesc.y == texref->channelDesc.y && + tr->channelDesc.z == texref->channelDesc.z && + tr->channelDesc.w == texref->channelDesc.w && + tr->channelDesc.f == texref->channelDesc.f); + } + m_NameToTextureRef[texname].insert(texref); + m_TextureRefToName[texref] = texname; + const textureReferenceAttr *texAttr = new textureReferenceAttr( + texref, dim, (enum cudaTextureReadMode)readmode, ext); + m_NameToAttribute[texname] = texAttr; +} + +const char *gpgpu_t::gpgpu_ptx_sim_findNamefromTexture( + const struct textureReference *texref) { + std::map::const_iterator t = + m_TextureRefToName.find(texref); + assert(t != m_TextureRefToName.end()); + return t->second.c_str(); +} + +unsigned int intLOGB2(unsigned int v) { + unsigned int shift; + unsigned int r; + + r = 0; + + shift = ((v & 0xFFFF0000) != 0) << 4; + v >>= shift; + r |= shift; + shift = ((v & 0xFF00) != 0) << 3; + v >>= shift; + r |= shift; + shift = ((v & 0xF0) != 0) << 2; + v >>= shift; + r |= shift; + shift = ((v & 0xC) != 0) << 1; + v >>= shift; + r |= shift; + shift = ((v & 0x2) != 0) << 0; + v >>= shift; + r |= shift; + + return r; +} + +void gpgpu_t::gpgpu_ptx_sim_bindTextureToArray( + const struct textureReference *texref, const struct cudaArray *array) { + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + + std::map::const_iterator t = + m_NameToCudaArray.find(texname); + // check that there's nothing there first + if (t != m_NameToCudaArray.end()) { + printf( + "GPGPU-Sim PTX: Warning: binding to texref associated with %s, which " + "was previously bound.\nImplicitly unbinding texref associated to %s " + "first\n", + texname.c_str(), texname.c_str()); + } + m_NameToCudaArray[texname] = array; + unsigned int texel_size_bits = + array->desc.w + array->desc.x + array->desc.y + array->desc.z; + unsigned int texel_size = texel_size_bits / 8; + unsigned int Tx, Ty; + int r; + + printf("GPGPU-Sim PTX: texel size = %d\n", texel_size); + printf("GPGPU-Sim PTX: texture cache linesize = %d\n", + m_function_model_config.get_texcache_linesize()); + // first determine base Tx size for given linesize + switch (m_function_model_config.get_texcache_linesize()) { + case 16: + Tx = 4; + break; + case 32: + Tx = 8; + break; + case 64: + Tx = 8; + break; + case 128: + Tx = 16; + break; + case 256: + Tx = 16; + break; + default: + printf( + "GPGPU-Sim PTX: Line size of %d bytes currently not supported.\n", + m_function_model_config.get_texcache_linesize()); assert(0); break; - } - r = texel_size >> 2; - //modify base Tx size to take into account size of each texel in bytes - while (r != 0) { - Tx = Tx >> 1; - r = r >> 2; - } - //by now, got the correct Tx size, calculate correct Ty size - Ty = m_function_model_config.get_texcache_linesize()/(Tx*texel_size); - - printf("GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); - printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", texel_size, intLOGB2(texel_size)); - printf("GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", array->devPtr32); - printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); - struct textureInfo* texInfo = (struct textureInfo*) malloc(sizeof(struct textureInfo)); - texInfo->Tx = Tx; - texInfo->Ty = Ty; - texInfo->Tx_numbits = intLOGB2(Tx); - texInfo->Ty_numbits = intLOGB2(Ty); - texInfo->texel_size = texel_size; - texInfo->texel_size_numbits = intLOGB2(texel_size); - m_NameToTextureInfo[texname] = texInfo; -} - -void gpgpu_t::gpgpu_ptx_sim_unbindTexture(const struct textureReference* texref) -{ - //assumes bind-use-unbind-bind-use-unbind pattern - std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); - m_NameToCudaArray.erase(texname); - m_NameToTextureInfo.erase(texname); + } + r = texel_size >> 2; + // modify base Tx size to take into account size of each texel in bytes + while (r != 0) { + Tx = Tx >> 1; + r = r >> 2; + } + // by now, got the correct Tx size, calculate correct Ty size + Ty = m_function_model_config.get_texcache_linesize() / (Tx * texel_size); + + printf( + "GPGPU-Sim PTX: Tx = %d; Ty = %d, Tx_numbits = %d, Ty_numbits = %d\n", + Tx, Ty, intLOGB2(Tx), intLOGB2(Ty)); + printf("GPGPU-Sim PTX: Texel size = %d bytes; texel_size_numbits = %d\n", + texel_size, intLOGB2(texel_size)); + printf( + "GPGPU-Sim PTX: Binding texture to array starting at devPtr32 = 0x%x\n", + array->devPtr32); + printf("GPGPU-Sim PTX: Texel size = %d bytes\n", texel_size); + struct textureInfo *texInfo = + (struct textureInfo *)malloc(sizeof(struct textureInfo)); + texInfo->Tx = Tx; + texInfo->Ty = Ty; + texInfo->Tx_numbits = intLOGB2(Tx); + texInfo->Ty_numbits = intLOGB2(Ty); + texInfo->texel_size = texel_size; + texInfo->texel_size_numbits = intLOGB2(texel_size); + m_NameToTextureInfo[texname] = texInfo; +} + +void gpgpu_t::gpgpu_ptx_sim_unbindTexture( + const struct textureReference *texref) { + // assumes bind-use-unbind-bind-use-unbind pattern + std::string texname = gpgpu_ptx_sim_findNamefromTexture(texref); + m_NameToCudaArray.erase(texname); + m_NameToTextureInfo.erase(texname); } #define MAX_INST_SIZE 8 /*bytes*/ -void function_info::ptx_assemble() -{ - if( m_assembled ) { - return; - } - - // get the instructions into instruction memory... - unsigned num_inst = m_instructions.size(); - m_instr_mem_size = MAX_INST_SIZE*(num_inst+1); - m_instr_mem = new ptx_instruction*[ m_instr_mem_size ]; - - printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", m_name.c_str() ); - fflush(stdout); - std::list::iterator i; - - addr_t PC = gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique address (across functions) - // start function on an aligned address - for( unsigned i=0; i < (PC%MAX_INST_SIZE); i++ ) - gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); - PC += PC%MAX_INST_SIZE; - m_start_PC = PC; - - addr_t n=0; // offset in m_instr_mem - //Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? reserve is cumulative. - //s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + MAX_INST_SIZE*m_instructions.size()); - gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE*m_instructions.size()); - for ( i=m_instructions.begin(); i != m_instructions.end(); i++ ) { - ptx_instruction *pI = *i; - if ( pI->is_label() ) { - const symbol *l = pI->get_label(); - labels[l->name()] = n; - } else { - gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; - m_instr_mem[n] = pI; - gpgpu_ctx->s_g_pc_to_insn.push_back(pI); - assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); - pI->set_m_instr_mem_index(n); - pI->set_PC(PC); - assert( pI->inst_size() <= MAX_INST_SIZE ); - for( unsigned i=1; i < pI->inst_size(); i++ ) { - gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction*)NULL); - m_instr_mem[n+i]=NULL; - } - n += pI->inst_size(); - PC += pI->inst_size(); +void function_info::ptx_assemble() { + if (m_assembled) { + return; + } + + // get the instructions into instruction memory... + unsigned num_inst = m_instructions.size(); + m_instr_mem_size = MAX_INST_SIZE * (num_inst + 1); + m_instr_mem = new ptx_instruction *[m_instr_mem_size]; + + printf("GPGPU-Sim PTX: instruction assembly for function \'%s\'... ", + m_name.c_str()); + fflush(stdout); + std::list::iterator i; + + addr_t PC = + gpgpu_ctx->func_sim->g_assemble_code_next_pc; // globally unique address + // (across functions) + // start function on an aligned address + for (unsigned i = 0; i < (PC % MAX_INST_SIZE); i++) + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction *)NULL); + PC += PC % MAX_INST_SIZE; + m_start_PC = PC; + + addr_t n = 0; // offset in m_instr_mem + // Why s_g_pc_to_insn.size() is needed to reserve additional memory for insts? + // reserve is cumulative. s_g_pc_to_insn.reserve(s_g_pc_to_insn.size() + + // MAX_INST_SIZE*m_instructions.size()); + gpgpu_ctx->s_g_pc_to_insn.reserve(MAX_INST_SIZE * m_instructions.size()); + for (i = m_instructions.begin(); i != m_instructions.end(); i++) { + ptx_instruction *pI = *i; + if (pI->is_label()) { + const symbol *l = pI->get_label(); + labels[l->name()] = n; + } else { + gpgpu_ctx->func_sim->g_pc_to_finfo[PC] = this; + m_instr_mem[n] = pI; + gpgpu_ctx->s_g_pc_to_insn.push_back(pI); + assert(pI == gpgpu_ctx->s_g_pc_to_insn[PC]); + pI->set_m_instr_mem_index(n); + pI->set_PC(PC); + assert(pI->inst_size() <= MAX_INST_SIZE); + for (unsigned i = 1; i < pI->inst_size(); i++) { + gpgpu_ctx->s_g_pc_to_insn.push_back((ptx_instruction *)NULL); + m_instr_mem[n + i] = NULL; } - } - gpgpu_ctx->func_sim->g_assemble_code_next_pc=PC; - for ( unsigned ii=0; ii < n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions - ptx_instruction *pI = m_instr_mem[ii]; - if ( pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || pI->get_opcode() == CALLP_OP) { - operand_info &target = pI->dst(); //get operand, e.g. target name - if ( labels.find(target.name()) == labels.end() ) { - printf("GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not appear in assembly code.", - pI->source_file(),pI->source_line(), target.name().c_str() ); - abort(); - } - unsigned index = labels[ target.name() ]; //determine address from name - unsigned PC = m_instr_mem[index]->get_PC(); - m_symtab->set_label_address( target.get_symbol(), PC ); - target.set_type(label_t); + n += pI->inst_size(); + PC += pI->inst_size(); + } + } + gpgpu_ctx->func_sim->g_assemble_code_next_pc = PC; + for (unsigned ii = 0; ii < n; + ii += m_instr_mem[ii]->inst_size()) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + if (pI->get_opcode() == BRA_OP || pI->get_opcode() == BREAKADDR_OP || + pI->get_opcode() == CALLP_OP) { + operand_info &target = pI->dst(); // get operand, e.g. target name + if (labels.find(target.name()) == labels.end()) { + printf( + "GPGPU-Sim PTX: Loader error (%s:%u): Branch label \"%s\" does not " + "appear in assembly code.", + pI->source_file(), pI->source_line(), target.name().c_str()); + abort(); } - } - m_n = n; - printf(" done.\n"); - fflush(stdout); + unsigned index = labels[target.name()]; // determine address from name + unsigned PC = m_instr_mem[index]->get_PC(); + m_symtab->set_label_address(target.get_symbol(), PC); + target.set_type(label_t); + } + } + m_n = n; + printf(" done.\n"); + fflush(stdout); - //disable pdom analysis here and do it at runtime + // disable pdom analysis here and do it at runtime #if 0 printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", m_name.c_str() ); create_basic_blocks(); @@ -323,2247 +380,2445 @@ void function_info::ptx_assemble() #endif } -addr_t shared_to_generic( unsigned smid, addr_t addr ) -{ - assert( addr < SHARED_MEM_SIZE_MAX ); - return SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX + addr; -} - -addr_t global_to_generic( addr_t addr ) -{ - return addr; -} - -bool isspace_shared( unsigned smid, addr_t addr ) -{ - addr_t start = SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX; - addr_t end = SHARED_GENERIC_START + (smid+1)*SHARED_MEM_SIZE_MAX; - if( (addr >= end) || (addr < start) ) - return false; - return true; -} - -bool isspace_global( addr_t addr ) -{ - return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); -} - -memory_space_t whichspace( addr_t addr ) -{ - if( (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT) ) { - return global_space; - } else if( addr >= SHARED_GENERIC_START ) { - return shared_space; - } else { - return local_space; - } -} - -addr_t generic_to_shared( unsigned smid, addr_t addr ) -{ - assert(isspace_shared(smid,addr)); - return addr - (SHARED_GENERIC_START + smid*SHARED_MEM_SIZE_MAX); -} - -addr_t local_to_generic( unsigned smid, unsigned hwtid, addr_t addr ) -{ - assert(addr < LOCAL_MEM_SIZE_MAX); - return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid) + addr; -} - -bool isspace_local( unsigned smid, unsigned hwtid, addr_t addr ) -{ - addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid); - addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * (hwtid+1)); - if( (addr >= end) || (addr < start) ) - return false; - return true; -} - -addr_t generic_to_local( unsigned smid, unsigned hwtid, addr_t addr ) -{ - assert(isspace_local(smid,hwtid,addr)); - return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + (LOCAL_MEM_SIZE_MAX * hwtid)); -} - -addr_t generic_to_global( addr_t addr ) -{ - return addr; -} - - -void* gpgpu_t::gpu_malloc( size_t size ) -{ - unsigned long long result = m_dev_malloc; - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); - fflush(stdout); - } - m_dev_malloc += size; - if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries - return(void*) result; -} - -void* gpgpu_t::gpu_mallocarray( size_t size ) -{ - unsigned long long result = m_dev_malloc; - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address 0x%Lx\n", size, m_dev_malloc ); - fflush(stdout); - } - m_dev_malloc += size; - if (size%256) m_dev_malloc += (256 - size%256); //align to 256 byte boundaries - return(void*) result; -} - - -void gpgpu_t::memcpy_to_gpu( size_t dst_start_addr, const void *src, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", count, (unsigned long long) src, (unsigned long long) dst_start_addr ); - fflush(stdout); - } - char *src_data = (char*)src; - for (unsigned n=0; n < count; n ++ ) - m_global_mem->write(dst_start_addr+n,1, src_data+n,NULL,NULL); - - // Copy into the performance model. - //extern gpgpu_sim* g_the_gpu; - gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } -} - -void gpgpu_t::memcpy_from_gpu( void *dst, size_t src_start_addr, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", count, (unsigned long long) src_start_addr, (unsigned long long) dst ); - fflush(stdout); - } - unsigned char *dst_data = (unsigned char*)dst; - for (unsigned n=0; n < count; n ++ ) - m_global_mem->read(src_start_addr+n,1,dst_data+n); - - // Copy into the performance model. - //extern gpgpu_sim* g_the_gpu; - gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } -} - -void gpgpu_t::memcpy_gpu_to_gpu( size_t dst, size_t src, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", count, - (unsigned long long) src, (unsigned long long) dst ); - fflush(stdout); - } - for (unsigned n=0; n < count; n ++ ) { - unsigned char tmp; - m_global_mem->read(src+n,1,&tmp); - m_global_mem->write(dst+n,1, &tmp,NULL,NULL); - } - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } -} - -void gpgpu_t::gpu_memset( size_t dst_start_addr, int c, size_t count ) -{ - if(g_debug_execution >= 3) { - printf("GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at 0x%Lx... ", - count, (unsigned char) c, (unsigned long long) dst_start_addr ); - fflush(stdout); - } - unsigned char c_value = (unsigned char)c; - for (unsigned n=0; n < count; n ++ ) - m_global_mem->write(dst_start_addr+n,1,&c_value,NULL,NULL); - if(g_debug_execution >= 3) { - printf( " done.\n"); - fflush(stdout); - } -} +addr_t shared_to_generic(unsigned smid, addr_t addr) { + assert(addr < SHARED_MEM_SIZE_MAX); + return SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX + addr; +} + +addr_t global_to_generic(addr_t addr) { return addr; } -void cuda_sim::ptx_print_insn( address_type pc, FILE *fp ) -{ - std::map::iterator f = g_pc_to_finfo.find(pc); - if( f == g_pc_to_finfo.end() ) { - fprintf(fp,"", pc ); - return; - } - function_info *finfo = f->second; - assert( finfo ); - finfo->print_insn(pc,fp); -} - -std::string cuda_sim::ptx_get_insn_str( address_type pc ) -{ - std::map::iterator f = g_pc_to_finfo.find(pc); - if( f == g_pc_to_finfo.end() ) { - #define STR_SIZE 255 - char buff[STR_SIZE]; - buff[STR_SIZE - 1] = '\0'; - snprintf(buff, STR_SIZE,"", pc ); - return std::string(buff); - } - function_info *finfo = f->second; - assert( finfo ); - return finfo->get_insn_str(pc); -} - -void ptx_instruction::set_fp_or_int_archop(){ - oprnd_type=UN_OP; - if((m_opcode == MEMBAR_OP)||(m_opcode == SSY_OP )||(m_opcode == BRA_OP) || (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || (m_opcode == CALL_OP)){ - // do nothing - }else if((m_opcode == CVT_OP || m_opcode == SET_OP || m_opcode == SLCT_OP)){ - if(get_type2()==F16_TYPE || get_type2()==F32_TYPE || get_type2() == F64_TYPE || get_type2() == FF64_TYPE){ - oprnd_type= FP_OP; - }else oprnd_type=INT_OP; - - }else{ - if(get_type()==F16_TYPE || get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ - oprnd_type= FP_OP; - }else oprnd_type=INT_OP; - } -} -void ptx_instruction::set_mul_div_or_other_archop(){ - sp_op=OTHER_OP; - if((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && (m_opcode != CALL_OP)){ - if(get_type()==F32_TYPE || get_type() == F64_TYPE || get_type() == FF64_TYPE){ - switch(get_opcode()){ - case MUL_OP: - case MAD_OP: - sp_op=FP_MUL_OP; - break; - case DIV_OP: - sp_op=FP_DIV_OP; - break; - case LG2_OP: - sp_op=FP_LG_OP; - break; - case RSQRT_OP: - case SQRT_OP: - sp_op=FP_SQRT_OP; - break; - case RCP_OP: - sp_op=FP_DIV_OP; - break; - case SIN_OP: - case COS_OP: - sp_op=FP_SIN_OP; - break; - case EX2_OP: - sp_op=FP_EXP_OP; - break; - default: - if((op==ALU_OP)||(op==TENSOR_CORE_OP)) - sp_op=FP__OP; - break; - - } - }else { - switch(get_opcode()){ - case MUL24_OP: - case MAD24_OP: - sp_op=INT_MUL24_OP; - break; - case MUL_OP: - case MAD_OP: - if(get_type()==U32_TYPE || get_type()==S32_TYPE || get_type()==B32_TYPE) - sp_op=INT_MUL32_OP; - else - sp_op=INT_MUL_OP; - break; - case DIV_OP: - sp_op=INT_DIV_OP; - break; - default: - if((op==ALU_OP)) - sp_op=INT__OP; - break; - } - } - } - -} - - - -void ptx_instruction::set_bar_type() -{ - if(m_opcode==BAR_OP) { - switch(m_barrier_op){ - case SYNC_OPTION: - bar_type = SYNC; - break; - case ARRIVE_OPTION: - bar_type = ARRIVE; - break; - case RED_OPTION: - bar_type = RED; - switch(m_atomic_spec){ - case ATOMIC_POPC: - red_type = POPC_RED; - break; - case ATOMIC_AND: - red_type = AND_RED; - break; - case ATOMIC_OR: - red_type = OR_RED; - break; - } - break; - default: - abort(); - } - } - else if(m_opcode==SST_OP) { - bar_type = SYNC; - } -} - - -void ptx_instruction::set_opcode_and_latency() -{ - unsigned int_latency[5]; - unsigned fp_latency[5]; - unsigned dp_latency[5]; - unsigned sfu_latency; - unsigned tensor_latency; - unsigned int_init[5]; - unsigned fp_init[5]; - unsigned dp_init[5]; - unsigned sfu_init; - unsigned tensor_init; - /* - * [0] ADD,SUB - * [1] MAX,Min - * [2] MUL - * [3] MAD - * [4] DIV - */ - sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", - &int_latency[0],&int_latency[1],&int_latency[2], - &int_latency[3],&int_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", - &fp_latency[0],&fp_latency[1],&fp_latency[2], - &fp_latency[3],&fp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", - &dp_latency[0],&dp_latency[1],&dp_latency[2], - &dp_latency[3],&dp_latency[4]); - sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", - &sfu_latency); - sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", - &tensor_latency); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_int, "%u,%u,%u,%u,%u", - &int_init[0],&int_init[1],&int_init[2], - &int_init[3],&int_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_fp, "%u,%u,%u,%u,%u", - &fp_init[0],&fp_init[1],&fp_init[2], - &fp_init[3],&fp_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_dp, "%u,%u,%u,%u,%u", - &dp_init[0],&dp_init[1],&dp_init[2], - &dp_init[3],&dp_init[4]); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_sfu, "%u", - &sfu_init); - sscanf(gpgpu_ctx->func_sim->opcode_initiation_tensor, "%u", - &tensor_init); - sscanf(gpgpu_ctx->func_sim->cdp_latency_str, "%u,%u,%u,%u,%u", - &gpgpu_ctx->func_sim->cdp_latency[0], - &gpgpu_ctx->func_sim->cdp_latency[1], - &gpgpu_ctx->func_sim->cdp_latency[2], - &gpgpu_ctx->func_sim->cdp_latency[3], - &gpgpu_ctx->func_sim->cdp_latency[4]); - - if(!m_operands.empty()){ - std::vector::iterator it; - for(it=++m_operands.begin();it!=m_operands.end();it++){ - num_operands++; - if((it->is_reg() || it->is_vector())){ - num_regs++; - } - } - } - op = ALU_OP; - mem_op= NOT_TEX; - initiation_interval = latency = 1; - switch( m_opcode ) { - case MOV_OP: - assert( !(has_memory_read() && has_memory_write()) ); - if ( has_memory_read() ) op = LOAD_OP; - if ( has_memory_write() ) op = STORE_OP; - break; - case LD_OP: op = LOAD_OP; break; - case MMA_LD_OP: op = TENSOR_CORE_LOAD_OP; break; - case LDU_OP: op = LOAD_OP; break; - case ST_OP: op = STORE_OP; break; - case MMA_ST_OP: op = TENSOR_CORE_STORE_OP; break; - case BRA_OP: op = BRANCH_OP; break; - case BREAKADDR_OP: op = BRANCH_OP; break; - case TEX_OP: op = LOAD_OP; mem_op=TEX; break; - case ATOM_OP: op = LOAD_OP; break; - case BAR_OP: op = BARRIER_OP; break; - case SST_OP: op = BARRIER_OP; break; - case MEMBAR_OP: op = MEMORY_BARRIER_OP; break; - case CALL_OP: - { - if(m_is_printf || m_is_cdp) { - op = ALU_OP; - } - else - op = CALL_OPS; - break; - } - case CALLP_OP: - { - if(m_is_printf || m_is_cdp) { - op = ALU_OP; - } - else - op = CALL_OPS; - break; - } - case RET_OP: case RETP_OP: op = RET_OPS;break; - case ADD_OP: case ADDP_OP: case ADDC_OP: case SUB_OP: case SUBC_OP: - //ADD,SUB latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[0]; - initiation_interval = fp_init[0]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[0]; - initiation_interval = dp_init[0]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[0]; - initiation_interval = int_init[0]; - op = INTP_OP; - break; - } - break; - case MAX_OP: case MIN_OP: - //MAX,MIN latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[1]; - initiation_interval = fp_init[1]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[1]; - initiation_interval = dp_init[1]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[1]; - initiation_interval = int_init[1]; - op = INTP_OP; - break; - } - break; - case MUL_OP: - //MUL latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[2]; - initiation_interval = fp_init[2]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[2]; - initiation_interval = dp_init[2]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[2]; - initiation_interval = int_init[2]; - op = INTP_OP; - break; - } - break; - case MAD_OP: case MADC_OP: case MADP_OP: - //MAD latency - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[3]; - initiation_interval = fp_init[3]; - op = SP_OP; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[3]; - initiation_interval = dp_init[3]; - op = DP_OP; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[3]; - initiation_interval = int_init[3]; - op = INTP_OP; - break; - } - break; - case DIV_OP: - // Floating point only - op = SFU_OP; - switch(get_type()){ - case F32_TYPE: - latency = fp_latency[4]; - initiation_interval = fp_init[4]; - break; - case F64_TYPE: - case FF64_TYPE: - latency = dp_latency[4]; - initiation_interval = dp_init[4]; - break; - case B32_TYPE: - case U32_TYPE: - case S32_TYPE: - default: //Use int settings for default - latency = int_latency[4]; - initiation_interval = int_init[4]; - break; - } - break; - case SQRT_OP: case SIN_OP: case COS_OP: case EX2_OP: case LG2_OP: case RSQRT_OP: case RCP_OP: - latency = sfu_latency; - initiation_interval = sfu_init; +bool isspace_shared(unsigned smid, addr_t addr) { + addr_t start = SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX; + addr_t end = SHARED_GENERIC_START + (smid + 1) * SHARED_MEM_SIZE_MAX; + if ((addr >= end) || (addr < start)) return false; + return true; +} + +bool isspace_global(addr_t addr) { + return (addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT); +} + +memory_space_t whichspace(addr_t addr) { + if ((addr >= GLOBAL_HEAP_START) || (addr < STATIC_ALLOC_LIMIT)) { + return global_space; + } else if (addr >= SHARED_GENERIC_START) { + return shared_space; + } else { + return local_space; + } +} + +addr_t generic_to_shared(unsigned smid, addr_t addr) { + assert(isspace_shared(smid, addr)); + return addr - (SHARED_GENERIC_START + smid * SHARED_MEM_SIZE_MAX); +} + +addr_t local_to_generic(unsigned smid, unsigned hwtid, addr_t addr) { + assert(addr < LOCAL_MEM_SIZE_MAX); + return LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * hwtid) + addr; +} + +bool isspace_local(unsigned smid, unsigned hwtid, addr_t addr) { + addr_t start = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * hwtid); + addr_t end = LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * (hwtid + 1)); + if ((addr >= end) || (addr < start)) return false; + return true; +} + +addr_t generic_to_local(unsigned smid, unsigned hwtid, addr_t addr) { + assert(isspace_local(smid, hwtid, addr)); + return addr - (LOCAL_GENERIC_START + (TOTAL_LOCAL_MEM_PER_SM * smid) + + (LOCAL_MEM_SIZE_MAX * hwtid)); +} + +addr_t generic_to_global(addr_t addr) { return addr; } + +void *gpgpu_t::gpu_malloc(size_t size) { + unsigned long long result = m_dev_malloc; + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address " + "0x%Lx\n", + size, m_dev_malloc); + fflush(stdout); + } + m_dev_malloc += size; + if (size % 256) + m_dev_malloc += (256 - size % 256); // align to 256 byte boundaries + return (void *)result; +} + +void *gpgpu_t::gpu_mallocarray(size_t size) { + unsigned long long result = m_dev_malloc; + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: allocating %zu bytes on GPU starting at address " + "0x%Lx\n", + size, m_dev_malloc); + fflush(stdout); + } + m_dev_malloc += size; + if (size % 256) + m_dev_malloc += (256 - size % 256); // align to 256 byte boundaries + return (void *)result; +} + +void gpgpu_t::memcpy_to_gpu(size_t dst_start_addr, const void *src, + size_t count) { + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: copying %zu bytes from CPU[0x%Lx] to GPU[0x%Lx] ... ", + count, (unsigned long long)src, (unsigned long long)dst_start_addr); + fflush(stdout); + } + char *src_data = (char *)src; + for (unsigned n = 0; n < count; n++) + m_global_mem->write(dst_start_addr + n, 1, src_data + n, NULL, NULL); + + // Copy into the performance model. + // extern gpgpu_sim* g_the_gpu; + gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(dst_start_addr, count); + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::memcpy_from_gpu(void *dst, size_t src_start_addr, size_t count) { + if (g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to CPU[0x%Lx] ...", + count, (unsigned long long)src_start_addr, (unsigned long long)dst); + fflush(stdout); + } + unsigned char *dst_data = (unsigned char *)dst; + for (unsigned n = 0; n < count; n++) + m_global_mem->read(src_start_addr + n, 1, dst_data + n); + + // Copy into the performance model. + // extern gpgpu_sim* g_the_gpu; + gpgpu_ctx->the_gpgpusim->g_the_gpu->perf_memcpy_to_gpu(src_start_addr, count); + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::memcpy_gpu_to_gpu(size_t dst, size_t src, size_t count) { + if (g_debug_execution >= 3) { + printf("GPGPU-Sim PTX: copying %zu bytes from GPU[0x%Lx] to GPU[0x%Lx] ...", + count, (unsigned long long)src, (unsigned long long)dst); + fflush(stdout); + } + for (unsigned n = 0; n < count; n++) { + unsigned char tmp; + m_global_mem->read(src + n, 1, &tmp); + m_global_mem->write(dst + n, 1, &tmp, NULL, NULL); + } + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } +} + +void gpgpu_t::gpu_memset(size_t dst_start_addr, int c, size_t count) { + if (g_debug_execution >= 3) { + printf( + "GPGPU-Sim PTX: setting %zu bytes of memory to 0x%x starting at " + "0x%Lx... ", + count, (unsigned char)c, (unsigned long long)dst_start_addr); + fflush(stdout); + } + unsigned char c_value = (unsigned char)c; + for (unsigned n = 0; n < count; n++) + m_global_mem->write(dst_start_addr + n, 1, &c_value, NULL, NULL); + if (g_debug_execution >= 3) { + printf(" done.\n"); + fflush(stdout); + } +} + +void cuda_sim::ptx_print_insn(address_type pc, FILE *fp) { + std::map::iterator f = g_pc_to_finfo.find(pc); + if (f == g_pc_to_finfo.end()) { + fprintf(fp, "", pc); + return; + } + function_info *finfo = f->second; + assert(finfo); + finfo->print_insn(pc, fp); +} + +std::string cuda_sim::ptx_get_insn_str(address_type pc) { + std::map::iterator f = g_pc_to_finfo.find(pc); + if (f == g_pc_to_finfo.end()) { +#define STR_SIZE 255 + char buff[STR_SIZE]; + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE, "", pc); + return std::string(buff); + } + function_info *finfo = f->second; + assert(finfo); + return finfo->get_insn_str(pc); +} + +void ptx_instruction::set_fp_or_int_archop() { + oprnd_type = UN_OP; + if ((m_opcode == MEMBAR_OP) || (m_opcode == SSY_OP) || (m_opcode == BRA_OP) || + (m_opcode == BAR_OP) || (m_opcode == RET_OP) || (m_opcode == RETP_OP) || + (m_opcode == NOP_OP) || (m_opcode == EXIT_OP) || (m_opcode == CALLP_OP) || + (m_opcode == CALL_OP)) { + // do nothing + } else if ((m_opcode == CVT_OP || m_opcode == SET_OP || + m_opcode == SLCT_OP)) { + if (get_type2() == F16_TYPE || get_type2() == F32_TYPE || + get_type2() == F64_TYPE || get_type2() == FF64_TYPE) { + oprnd_type = FP_OP; + } else + oprnd_type = INT_OP; + + } else { + if (get_type() == F16_TYPE || get_type() == F32_TYPE || + get_type() == F64_TYPE || get_type() == FF64_TYPE) { + oprnd_type = FP_OP; + } else + oprnd_type = INT_OP; + } +} +void ptx_instruction::set_mul_div_or_other_archop() { + sp_op = OTHER_OP; + if ((m_opcode != MEMBAR_OP) && (m_opcode != SSY_OP) && (m_opcode != BRA_OP) && + (m_opcode != BAR_OP) && (m_opcode != EXIT_OP) && (m_opcode != NOP_OP) && + (m_opcode != RETP_OP) && (m_opcode != RET_OP) && (m_opcode != CALLP_OP) && + (m_opcode != CALL_OP)) { + if (get_type() == F32_TYPE || get_type() == F64_TYPE || + get_type() == FF64_TYPE) { + switch (get_opcode()) { + case MUL_OP: + case MAD_OP: + sp_op = FP_MUL_OP; + break; + case DIV_OP: + sp_op = FP_DIV_OP; + break; + case LG2_OP: + sp_op = FP_LG_OP; + break; + case RSQRT_OP: + case SQRT_OP: + sp_op = FP_SQRT_OP; + break; + case RCP_OP: + sp_op = FP_DIV_OP; + break; + case SIN_OP: + case COS_OP: + sp_op = FP_SIN_OP; + break; + case EX2_OP: + sp_op = FP_EXP_OP; + break; + default: + if ((op == ALU_OP) || (op == TENSOR_CORE_OP)) sp_op = FP__OP; + break; + } + } else { + switch (get_opcode()) { + case MUL24_OP: + case MAD24_OP: + sp_op = INT_MUL24_OP; + break; + case MUL_OP: + case MAD_OP: + if (get_type() == U32_TYPE || get_type() == S32_TYPE || + get_type() == B32_TYPE) + sp_op = INT_MUL32_OP; + else + sp_op = INT_MUL_OP; + break; + case DIV_OP: + sp_op = INT_DIV_OP; + break; + default: + if ((op == ALU_OP)) sp_op = INT__OP; + break; + } + } + } +} + +void ptx_instruction::set_bar_type() { + if (m_opcode == BAR_OP) { + switch (m_barrier_op) { + case SYNC_OPTION: + bar_type = SYNC; + break; + case ARRIVE_OPTION: + bar_type = ARRIVE; + break; + case RED_OPTION: + bar_type = RED; + switch (m_atomic_spec) { + case ATOMIC_POPC: + red_type = POPC_RED; + break; + case ATOMIC_AND: + red_type = AND_RED; + break; + case ATOMIC_OR: + red_type = OR_RED; + break; + } + break; + default: + abort(); + } + } else if (m_opcode == SST_OP) { + bar_type = SYNC; + } +} + +void ptx_instruction::set_opcode_and_latency() { + unsigned int_latency[5]; + unsigned fp_latency[5]; + unsigned dp_latency[5]; + unsigned sfu_latency; + unsigned tensor_latency; + unsigned int_init[5]; + unsigned fp_init[5]; + unsigned dp_init[5]; + unsigned sfu_init; + unsigned tensor_init; + /* + * [0] ADD,SUB + * [1] MAX,Min + * [2] MUL + * [3] MAD + * [4] DIV + */ + sscanf(gpgpu_ctx->func_sim->opcode_latency_int, "%u,%u,%u,%u,%u", + &int_latency[0], &int_latency[1], &int_latency[2], &int_latency[3], + &int_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_fp, "%u,%u,%u,%u,%u", + &fp_latency[0], &fp_latency[1], &fp_latency[2], &fp_latency[3], + &fp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_dp, "%u,%u,%u,%u,%u", + &dp_latency[0], &dp_latency[1], &dp_latency[2], &dp_latency[3], + &dp_latency[4]); + sscanf(gpgpu_ctx->func_sim->opcode_latency_sfu, "%u", &sfu_latency); + sscanf(gpgpu_ctx->func_sim->opcode_latency_tensor, "%u", &tensor_latency); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_int, "%u,%u,%u,%u,%u", + &int_init[0], &int_init[1], &int_init[2], &int_init[3], &int_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_fp, "%u,%u,%u,%u,%u", + &fp_init[0], &fp_init[1], &fp_init[2], &fp_init[3], &fp_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_dp, "%u,%u,%u,%u,%u", + &dp_init[0], &dp_init[1], &dp_init[2], &dp_init[3], &dp_init[4]); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_sfu, "%u", &sfu_init); + sscanf(gpgpu_ctx->func_sim->opcode_initiation_tensor, "%u", &tensor_init); + sscanf(gpgpu_ctx->func_sim->cdp_latency_str, "%u,%u,%u,%u,%u", + &gpgpu_ctx->func_sim->cdp_latency[0], + &gpgpu_ctx->func_sim->cdp_latency[1], + &gpgpu_ctx->func_sim->cdp_latency[2], + &gpgpu_ctx->func_sim->cdp_latency[3], + &gpgpu_ctx->func_sim->cdp_latency[4]); + + if (!m_operands.empty()) { + std::vector::iterator it; + for (it = ++m_operands.begin(); it != m_operands.end(); it++) { + num_operands++; + if ((it->is_reg() || it->is_vector())) { + num_regs++; + } + } + } + op = ALU_OP; + mem_op = NOT_TEX; + initiation_interval = latency = 1; + switch (m_opcode) { + case MOV_OP: + assert(!(has_memory_read() && has_memory_write())); + if (has_memory_read()) op = LOAD_OP; + if (has_memory_write()) op = STORE_OP; + break; + case LD_OP: + op = LOAD_OP; + break; + case MMA_LD_OP: + op = TENSOR_CORE_LOAD_OP; + break; + case LDU_OP: + op = LOAD_OP; + break; + case ST_OP: + op = STORE_OP; + break; + case MMA_ST_OP: + op = TENSOR_CORE_STORE_OP; + break; + case BRA_OP: + op = BRANCH_OP; + break; + case BREAKADDR_OP: + op = BRANCH_OP; + break; + case TEX_OP: + op = LOAD_OP; + mem_op = TEX; + break; + case ATOM_OP: + op = LOAD_OP; + break; + case BAR_OP: + op = BARRIER_OP; + break; + case SST_OP: + op = BARRIER_OP; + break; + case MEMBAR_OP: + op = MEMORY_BARRIER_OP; + break; + case CALL_OP: { + if (m_is_printf || m_is_cdp) { + op = ALU_OP; + } else + op = CALL_OPS; + break; + } + case CALLP_OP: { + if (m_is_printf || m_is_cdp) { + op = ALU_OP; + } else + op = CALL_OPS; + break; + } + case RET_OP: + case RETP_OP: + op = RET_OPS; + break; + case ADD_OP: + case ADDP_OP: + case ADDC_OP: + case SUB_OP: + case SUBC_OP: + // ADD,SUB latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[0]; + initiation_interval = fp_init[0]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[0]; + initiation_interval = dp_init[0]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[0]; + initiation_interval = int_init[0]; + op = INTP_OP; + break; + } + break; + case MAX_OP: + case MIN_OP: + // MAX,MIN latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[1]; + initiation_interval = fp_init[1]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[1]; + initiation_interval = dp_init[1]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[1]; + initiation_interval = int_init[1]; + op = INTP_OP; + break; + } + break; + case MUL_OP: + // MUL latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[2]; + initiation_interval = fp_init[2]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[2]; + initiation_interval = dp_init[2]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[2]; + initiation_interval = int_init[2]; + op = INTP_OP; + break; + } + break; + case MAD_OP: + case MADC_OP: + case MADP_OP: + // MAD latency + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[3]; + initiation_interval = fp_init[3]; + op = SP_OP; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[3]; + initiation_interval = dp_init[3]; + op = DP_OP; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[3]; + initiation_interval = int_init[3]; + op = INTP_OP; + break; + } + break; + case DIV_OP: + // Floating point only op = SFU_OP; + switch (get_type()) { + case F32_TYPE: + latency = fp_latency[4]; + initiation_interval = fp_init[4]; + break; + case F64_TYPE: + case FF64_TYPE: + latency = dp_latency[4]; + initiation_interval = dp_init[4]; + break; + case B32_TYPE: + case U32_TYPE: + case S32_TYPE: + default: // Use int settings for default + latency = int_latency[4]; + initiation_interval = int_init[4]; + break; + } break; - case MMA_OP: - latency = tensor_latency; - initiation_interval = tensor_init; - op=TENSOR_CORE_OP; - break; - case SHFL_OP: - latency = 4; - initiation_interval = 4; - break; - default: - break; - } - set_fp_or_int_archop(); - set_mul_div_or_other_archop(); - -} - -void ptx_thread_info::ptx_fetch_inst( inst_t &inst ) const -{ - addr_t pc = get_pc(); - const ptx_instruction *pI = m_func_info->get_instruction(pc); - inst = (const inst_t&)*pI; - assert( inst.valid() ); -} - -static unsigned datatype2size( unsigned data_type ) -{ - unsigned data_size; - switch ( data_type ) { - case B8_TYPE: - case S8_TYPE: - case U8_TYPE: - data_size = 1; break; - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: - case F16_TYPE: - data_size = 2; break; - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: - case F32_TYPE: - data_size = 4; break; - case B64_TYPE: - case BB64_TYPE: - case S64_TYPE: - case U64_TYPE: - case F64_TYPE: - case FF64_TYPE: - data_size = 8; break; - case BB128_TYPE: - data_size = 16; break; - default: assert(0); break; - } - return data_size; + case SQRT_OP: + case SIN_OP: + case COS_OP: + case EX2_OP: + case LG2_OP: + case RSQRT_OP: + case RCP_OP: + latency = sfu_latency; + initiation_interval = sfu_init; + op = SFU_OP; + break; + case MMA_OP: + latency = tensor_latency; + initiation_interval = tensor_init; + op = TENSOR_CORE_OP; + break; + case SHFL_OP: + latency = 4; + initiation_interval = 4; + break; + default: + break; + } + set_fp_or_int_archop(); + set_mul_div_or_other_archop(); } -void ptx_instruction::pre_decode() -{ - pc = m_PC; - isize = m_inst_size; - for(unsigned i=0; iget_instruction(pc); + inst = (const inst_t &)*pI; + assert(inst.valid()); +} - switch ( get_opcode() ) { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: has_dst = (DST!=0); break; +static unsigned datatype2size(unsigned data_type) { + unsigned data_size; + switch (data_type) { + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + data_size = 1; + break; + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + data_size = 2; + break; + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + data_size = 4; + break; + case B64_TYPE: + case BB64_TYPE: + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case FF64_TYPE: + data_size = 8; + break; + case BB128_TYPE: + data_size = 16; + break; + default: + assert(0); + break; + } + return data_size; +} + +void ptx_instruction::pre_decode() { + pc = m_PC; + isize = m_inst_size; + for (unsigned i = 0; i < MAX_OUTPUT_VALUES; i++) { + out[i] = 0; + } + for (unsigned i = 0; i < MAX_INPUT_VALUES; i++) { + in[i] = 0; + } + incount = 0; + outcount = 0; + is_vectorin = 0; + is_vectorout = 0; + std::fill_n(arch_reg.src, MAX_REG_OPERANDS, -1); + std::fill_n(arch_reg.dst, MAX_REG_OPERANDS, -1); + pred = 0; + ar1 = 0; + ar2 = 0; + space = m_space_spec; + memory_op = no_memory_op; + data_size = 0; + if (has_memory_read() || has_memory_write()) { + unsigned to_type = get_type(); + data_size = datatype2size(to_type); + memory_op = has_memory_read() ? memory_load : memory_store; + } + + bool has_dst = false; + + switch (get_opcode()) { +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + has_dst = (DST != 0); \ + break; +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + has_dst = (DST != 0); \ + break; #include "opcodes.def" #undef OP_DEF #undef OP_W_DEF - default: - printf( "Execution error: Invalid opcode (0x%x)\n", get_opcode() ); + default: + printf("Execution error: Invalid opcode (0x%x)\n", get_opcode()); break; - } + } - switch( m_cache_option ) { - case CA_OPTION: cache_op = CACHE_ALL; break; - case NC_OPTION: cache_op = CACHE_L1; break; - case CG_OPTION: cache_op = CACHE_GLOBAL; break; - case CS_OPTION: cache_op = CACHE_STREAMING; break; - case LU_OPTION: cache_op = CACHE_LAST_USE; break; - case CV_OPTION: cache_op = CACHE_VOLATILE; break; - case WB_OPTION: cache_op = CACHE_WRITE_BACK; break; - case WT_OPTION: cache_op = CACHE_WRITE_THROUGH; break; - default: - //if( m_opcode == LD_OP || m_opcode == LDU_OP ) - if( m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP ) - cache_op = CACHE_ALL; - //else if( m_opcode == ST_OP ) - else if( m_opcode == MMA_ST_OP || m_opcode == ST_OP ) - cache_op = CACHE_WRITE_BACK; - else if( m_opcode == ATOM_OP ) - cache_op = CACHE_GLOBAL; + switch (m_cache_option) { + case CA_OPTION: + cache_op = CACHE_ALL; break; - } - - set_opcode_and_latency(); - set_bar_type(); - // Get register operands - int n=0,m=0; - ptx_instruction::const_iterator opr=op_iter_begin(); - for ( ; opr != op_iter_end(); opr++, n++ ) { //process operands - const operand_info &o = *opr; - if ( has_dst && n==0 ) { - // Do not set the null register "_" as an architectural register - if ( o.is_reg() && !o.is_non_arch_reg() ) { - out[0] = o.reg_num(); - arch_reg.dst[0] = o.arch_reg_num(); - } else if ( o.is_vector() ) { - is_vectorin = 1; - unsigned num_elem = o.get_vect_nelem(); - if( num_elem >= 1 ) out[0] = o.reg1_num(); - if( num_elem >= 2 ) out[1] = o.reg2_num(); - if( num_elem >= 3 ) out[2] = o.reg3_num(); - if( num_elem >= 4 ) out[3] = o.reg4_num(); - if( num_elem >= 5 ) out[4] = o.reg5_num(); - if( num_elem >= 6 ) out[5] = o.reg6_num(); - if( num_elem >= 7 ) out[6] = o.reg7_num(); - if( num_elem >= 8 ) out[7] = o.reg8_num(); - for (int i = 0; i < num_elem; i++) - arch_reg.dst[i] = o.arch_reg_num(i); - } - } else { - if ( o.is_reg() && !o.is_non_arch_reg() ) { - int reg_num = o.reg_num(); - arch_reg.src[m] = o.arch_reg_num(); - switch ( m ) { - case 0: in[0] = reg_num; break; - case 1: in[1] = reg_num; break; - case 2: in[2] = reg_num; break; - default: break; - } - m++; - } else if ( o.is_vector() ) { - //assert(m == 0); //only support 1 vector operand (for textures) right now - is_vectorout = 1; - unsigned num_elem = o.get_vect_nelem(); - if( num_elem >= 1 ) in[m+0] = o.reg1_num(); - if( num_elem >= 2 ) in[m+1] = o.reg2_num(); - if( num_elem >= 3 ) in[m+2] = o.reg3_num(); - if( num_elem >= 4 ) in[m+3] = o.reg4_num(); - if( num_elem >= 5 ) in[m+4] = o.reg5_num(); - if( num_elem >= 6 ) in[m+5] = o.reg6_num(); - if( num_elem >= 7 ) in[m+6] = o.reg7_num(); - if( num_elem >= 8 ) in[m+7] = o.reg8_num(); - for (int i = 0; i < num_elem; i++) - arch_reg.src[m+i] = o.arch_reg_num(i); - m+=num_elem; - } + case NC_OPTION: + cache_op = CACHE_L1; + break; + case CG_OPTION: + cache_op = CACHE_GLOBAL; + break; + case CS_OPTION: + cache_op = CACHE_STREAMING; + break; + case LU_OPTION: + cache_op = CACHE_LAST_USE; + break; + case CV_OPTION: + cache_op = CACHE_VOLATILE; + break; + case WB_OPTION: + cache_op = CACHE_WRITE_BACK; + break; + case WT_OPTION: + cache_op = CACHE_WRITE_THROUGH; + break; + default: + // if( m_opcode == LD_OP || m_opcode == LDU_OP ) + if (m_opcode == MMA_LD_OP || m_opcode == LD_OP || m_opcode == LDU_OP) + cache_op = CACHE_ALL; + // else if( m_opcode == ST_OP ) + else if (m_opcode == MMA_ST_OP || m_opcode == ST_OP) + cache_op = CACHE_WRITE_BACK; + else if (m_opcode == ATOM_OP) + cache_op = CACHE_GLOBAL; + break; + } + + set_opcode_and_latency(); + set_bar_type(); + // Get register operands + int n = 0, m = 0; + ptx_instruction::const_iterator opr = op_iter_begin(); + for (; opr != op_iter_end(); opr++, n++) { // process operands + const operand_info &o = *opr; + if (has_dst && n == 0) { + // Do not set the null register "_" as an architectural register + if (o.is_reg() && !o.is_non_arch_reg()) { + out[0] = o.reg_num(); + arch_reg.dst[0] = o.arch_reg_num(); + } else if (o.is_vector()) { + is_vectorin = 1; + unsigned num_elem = o.get_vect_nelem(); + if (num_elem >= 1) out[0] = o.reg1_num(); + if (num_elem >= 2) out[1] = o.reg2_num(); + if (num_elem >= 3) out[2] = o.reg3_num(); + if (num_elem >= 4) out[3] = o.reg4_num(); + if (num_elem >= 5) out[4] = o.reg5_num(); + if (num_elem >= 6) out[5] = o.reg6_num(); + if (num_elem >= 7) out[6] = o.reg7_num(); + if (num_elem >= 8) out[7] = o.reg8_num(); + for (int i = 0; i < num_elem; i++) arch_reg.dst[i] = o.arch_reg_num(i); } - } - - //Setting number of input and output operands which is required for scoreboard check - for(int i=0;i0) - outcount++; - - for(int i=0;i0) - incount++; - - // Get predicate - if(has_pred()) { - const operand_info &p = get_pred(); - pred = p.reg_num(); - } - - // Get address registers inside memory operands. - // Assuming only one memory operand per instruction, - // and maximum of two address registers for one memory operand. - if( has_memory_read() || has_memory_write() ) { - ptx_instruction::const_iterator op=op_iter_begin(); - for ( ; op != op_iter_end(); op++, n++ ) { //process operands - const operand_info &o = *op; - - if(o.is_memory_operand()) { - // We do not support the null register as a memory operand - assert( !o.is_non_arch_reg() ); - - // Check PTXPlus-type operand - // memory operand with addressing (ex. s[0x4] or g[$r1]) - if(o.is_memory_operand2()) { - - // memory operand with one address register (ex. g[$r1+0x4] or s[$r2+=0x4]) - if(o.get_double_operand_type() == 0 || o.get_double_operand_type() == 3){ - ar1 = o.reg_num(); - arch_reg.src[4] = o.arch_reg_num(); - // TODO: address register in $r2+=0x4 should be an output register as well - } - // memory operand with two address register (ex. s[$r1+$r1] or g[$r1+=$r2]) - else if(o.get_double_operand_type() == 1 || o.get_double_operand_type() == 2) { - ar1 = o.reg1_num(); - arch_reg.src[4] = o.arch_reg_num(); - ar2 = o.reg2_num(); - arch_reg.src[5] = o.arch_reg_num(); - // TODO: first address register in $r1+=$r2 should be an output register as well - } - } - else if(o.is_immediate_address()){ - - } - // Regular PTX operand - else if (o.get_symbol()->type()->get_key().is_reg()) { // Memory operand contains a register - ar1 = o.reg_num(); - arch_reg.src[4] = o.arch_reg_num(); - } - - } + } else { + if (o.is_reg() && !o.is_non_arch_reg()) { + int reg_num = o.reg_num(); + arch_reg.src[m] = o.arch_reg_num(); + switch (m) { + case 0: + in[0] = reg_num; + break; + case 1: + in[1] = reg_num; + break; + case 2: + in[2] = reg_num; + break; + default: + break; + } + m++; + } else if (o.is_vector()) { + // assert(m == 0); //only support 1 vector operand (for textures) right + // now + is_vectorout = 1; + unsigned num_elem = o.get_vect_nelem(); + if (num_elem >= 1) in[m + 0] = o.reg1_num(); + if (num_elem >= 2) in[m + 1] = o.reg2_num(); + if (num_elem >= 3) in[m + 2] = o.reg3_num(); + if (num_elem >= 4) in[m + 3] = o.reg4_num(); + if (num_elem >= 5) in[m + 4] = o.reg5_num(); + if (num_elem >= 6) in[m + 5] = o.reg6_num(); + if (num_elem >= 7) in[m + 6] = o.reg7_num(); + if (num_elem >= 8) in[m + 7] = o.reg8_num(); + for (int i = 0; i < num_elem; i++) + arch_reg.src[m + i] = o.arch_reg_num(i); + m += num_elem; } - } - - // get reconvergence pc - reconvergence_pc = gpgpu_ctx->func_sim->get_converge_point(pc); - - m_decoded=true; -} - -void function_info::add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ) -{ - unsigned parsed_index; - char buffer[2048]; - snprintf(buffer,2048,"%s_param_%%u", m_name.c_str() ); - int ntokens = sscanf(name.c_str(),buffer,&parsed_index); - if( ntokens == 1 ) { - assert( m_ptx_kernel_param_info.find(parsed_index) == m_ptx_kernel_param_info.end() ); - m_ptx_kernel_param_info[parsed_index] = param_info(name, type, size, ptr, space); - } else { - assert( m_ptx_kernel_param_info.find(index) == m_ptx_kernel_param_info.end() ); - m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); - } -} - -void function_info::add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ) -{ - const void *data = args->m_start; - - bool scratchpad_memory_param = false; // Is this parameter in CUDA shared memory or OpenCL local memory - - std::map::iterator i=m_ptx_kernel_param_info.find(argn); - if( i != m_ptx_kernel_param_info.end() ) { - if (i->second.is_ptr_shared()) { - assert(args->m_start == NULL && "OpenCL parameter pointer to local memory must have NULL as value"); - scratchpad_memory_param = true; - } else { - param_t tmp; - tmp.pdata = args->m_start; - tmp.size = args->m_nbytes; - tmp.offset = args->m_offset; - tmp.type = 0; - i->second.add_data(tmp); - i->second.add_offset((unsigned) args->m_offset); + } + } + + // Setting number of input and output operands which is required for + // scoreboard check + for (int i = 0; i < MAX_OUTPUT_VALUES; i++) + if (out[i] > 0) outcount++; + + for (int i = 0; i < MAX_INPUT_VALUES; i++) + if (in[i] > 0) incount++; + + // Get predicate + if (has_pred()) { + const operand_info &p = get_pred(); + pred = p.reg_num(); + } + + // Get address registers inside memory operands. + // Assuming only one memory operand per instruction, + // and maximum of two address registers for one memory operand. + if (has_memory_read() || has_memory_write()) { + ptx_instruction::const_iterator op = op_iter_begin(); + for (; op != op_iter_end(); op++, n++) { // process operands + const operand_info &o = *op; + + if (o.is_memory_operand()) { + // We do not support the null register as a memory operand + assert(!o.is_non_arch_reg()); + + // Check PTXPlus-type operand + // memory operand with addressing (ex. s[0x4] or g[$r1]) + if (o.is_memory_operand2()) { + // memory operand with one address register (ex. g[$r1+0x4] or + // s[$r2+=0x4]) + if (o.get_double_operand_type() == 0 || + o.get_double_operand_type() == 3) { + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + // TODO: address register in $r2+=0x4 should be an output register + // as well + } + // memory operand with two address register (ex. s[$r1+$r1] or + // g[$r1+=$r2]) + else if (o.get_double_operand_type() == 1 || + o.get_double_operand_type() == 2) { + ar1 = o.reg1_num(); + arch_reg.src[4] = o.arch_reg_num(); + ar2 = o.reg2_num(); + arch_reg.src[5] = o.arch_reg_num(); + // TODO: first address register in $r1+=$r2 should be an output + // register as well + } + } else if (o.is_immediate_address()) { + } + // Regular PTX operand + else if (o.get_symbol() + ->type() + ->get_key() + .is_reg()) { // Memory operand contains a register + ar1 = o.reg_num(); + arch_reg.src[4] = o.arch_reg_num(); + } } - } else { - scratchpad_memory_param = true; - } - - if (scratchpad_memory_param) { - // This should only happen for OpenCL: - // - // The LLVM PTX compiler in NVIDIA's driver (version 190.29) - // does not generate an argument in the function declaration - // for __constant arguments. - // - // The associated constant memory space can be allocated in two - // ways. It can be explicitly initialized in the .ptx file where - // it is declared. Or, it can be allocated using the clCreateBuffer - // on the host. In this later case, the .ptx file will contain - // a global declaration of the parameter, but it will have an unknown - // array size. Thus, the symbol's address will not be set and we need - // to set it here before executing the PTX. - - char buffer[2048]; - snprintf(buffer,2048,"%s_param_%u",m_name.c_str(),argn); - - symbol *p = m_symtab->lookup(buffer); - if( p == NULL ) { - printf("GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot bind buffer\n", buffer); - abort(); + } + } + + // get reconvergence pc + reconvergence_pc = gpgpu_ctx->func_sim->get_converge_point(pc); + + m_decoded = true; +} + +void function_info::add_param_name_type_size(unsigned index, std::string name, + int type, size_t size, bool ptr, + memory_space_t space) { + unsigned parsed_index; + char buffer[2048]; + snprintf(buffer, 2048, "%s_param_%%u", m_name.c_str()); + int ntokens = sscanf(name.c_str(), buffer, &parsed_index); + if (ntokens == 1) { + assert(m_ptx_kernel_param_info.find(parsed_index) == + m_ptx_kernel_param_info.end()); + m_ptx_kernel_param_info[parsed_index] = + param_info(name, type, size, ptr, space); + } else { + assert(m_ptx_kernel_param_info.find(index) == + m_ptx_kernel_param_info.end()); + m_ptx_kernel_param_info[index] = param_info(name, type, size, ptr, space); + } +} + +void function_info::add_param_data(unsigned argn, + struct gpgpu_ptx_sim_arg *args) { + const void *data = args->m_start; + + bool scratchpad_memory_param = + false; // Is this parameter in CUDA shared memory or OpenCL local memory + + std::map::iterator i = + m_ptx_kernel_param_info.find(argn); + if (i != m_ptx_kernel_param_info.end()) { + if (i->second.is_ptr_shared()) { + assert( + args->m_start == NULL && + "OpenCL parameter pointer to local memory must have NULL as value"); + scratchpad_memory_param = true; + } else { + param_t tmp; + tmp.pdata = args->m_start; + tmp.size = args->m_nbytes; + tmp.offset = args->m_offset; + tmp.type = 0; + i->second.add_data(tmp); + i->second.add_offset((unsigned)args->m_offset); + } + } else { + scratchpad_memory_param = true; + } + + if (scratchpad_memory_param) { + // This should only happen for OpenCL: + // + // The LLVM PTX compiler in NVIDIA's driver (version 190.29) + // does not generate an argument in the function declaration + // for __constant arguments. + // + // The associated constant memory space can be allocated in two + // ways. It can be explicitly initialized in the .ptx file where + // it is declared. Or, it can be allocated using the clCreateBuffer + // on the host. In this later case, the .ptx file will contain + // a global declaration of the parameter, but it will have an unknown + // array size. Thus, the symbol's address will not be set and we need + // to set it here before executing the PTX. + + char buffer[2048]; + snprintf(buffer, 2048, "%s_param_%u", m_name.c_str(), argn); + + symbol *p = m_symtab->lookup(buffer); + if (p == NULL) { + printf( + "GPGPU-Sim PTX: ERROR ** could not locate symbol for \'%s\' : cannot " + "bind buffer\n", + buffer); + abort(); + } + if (data) + p->set_address((addr_t) * (size_t *)data); + else { + // clSetKernelArg was passed NULL pointer for data... + // this is used for dynamically sized shared memory on NVIDIA platforms + bool is_ptr_shared = false; + if (i != m_ptx_kernel_param_info.end()) { + is_ptr_shared = i->second.is_ptr_shared(); } - if( data ) - p->set_address((addr_t)*(size_t*)data); - else { - // clSetKernelArg was passed NULL pointer for data... - // this is used for dynamically sized shared memory on NVIDIA platforms - bool is_ptr_shared = false; - if( i != m_ptx_kernel_param_info.end() ) { - is_ptr_shared = i->second.is_ptr_shared(); - } - - if( !is_ptr_shared and !p->is_shared() ) { - printf("GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not shared memory\n"); - abort(); - } - unsigned num_bits = 8*args->m_nbytes; - printf("GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from 0x%x to 0x%x (shared memory space)\n", - p->name().c_str(), - m_symtab->get_shared_next(), - m_symtab->get_shared_next() + num_bits/8 ); - fflush(stdout); - assert( (num_bits%8) == 0 ); - addr_t addr = m_symtab->get_shared_next(); - addr_t addr_pad = num_bits ? (((num_bits/8) - (addr % (num_bits/8))) % (num_bits/8)) : 0; - p->set_address( addr+addr_pad ); - m_symtab->alloc_shared( num_bits/8 + addr_pad ); + + if (!is_ptr_shared and !p->is_shared()) { + printf( + "GPGPU-Sim PTX: ERROR ** clSetKernelArg passed NULL but arg not " + "shared memory\n"); + abort(); } - } + unsigned num_bits = 8 * args->m_nbytes; + printf( + "GPGPU-Sim PTX: deferred allocation of shared region for \"%s\" from " + "0x%x to 0x%x (shared memory space)\n", + p->name().c_str(), m_symtab->get_shared_next(), + m_symtab->get_shared_next() + num_bits / 8); + fflush(stdout); + assert((num_bits % 8) == 0); + addr_t addr = m_symtab->get_shared_next(); + addr_t addr_pad = + num_bits + ? (((num_bits / 8) - (addr % (num_bits / 8))) % (num_bits / 8)) + : 0; + p->set_address(addr + addr_pad); + m_symtab->alloc_shared(num_bits / 8 + addr_pad); + } + } } unsigned function_info::get_args_aligned_size() { - - if(m_args_aligned_size >= 0) - return m_args_aligned_size; - - unsigned param_address = 0; - unsigned int total_size = 0; - for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - - size_t arg_size = p.get_size() / 8; // size of param in bytes - total_size = (total_size + arg_size - 1) / arg_size * arg_size; //aligned - p.add_offset(total_size); - param->set_address(param_address + total_size); - total_size += arg_size; - } - - m_args_aligned_size = (total_size + 3) / 4 * 4; //final size aligned to word - - return m_args_aligned_size; - -} - - -void function_info::finalize( memory_space *param_mem ) -{ - unsigned param_address = 0; - for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? - std::string name = p.get_name(); - int type = p.get_type(); - param_t param_value = p.get_value(); - param_value.type = type; - symbol *param = m_symtab->lookup(name.c_str()); - unsigned xtype = param->type()->get_key().scalar_type(); - assert(xtype==(unsigned)type); - size_t size; - size = param_value.size; // size of param in bytes - // assert(param_value.offset == param_address); - if( size != p.get_size() / 8) { - printf("GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. formal size = %zu (using smaller of two)\n", - size, p.get_size()/8); - size = (size<(p.get_size()/8))?size:(p.get_size()/8); - } - // copy the parameter over word-by-word so that parameter that crosses a memory page can be copied over - //Jin: copy parameter using aligned rules - const type_info *paramtype = param->type(); - int align_amount = paramtype->get_key().get_alignment_spec(); - align_amount = (align_amount == -1) ? size : align_amount; - param_address = (param_address + align_amount - 1) / align_amount * align_amount; //aligned - - const size_t word_size = 4; - //param_address = (param_address + size - 1) / size * size; //aligned with size - for (size_t idx = 0; idx < size; idx += word_size) { - const char *pdata = reinterpret_cast(param_value.pdata) + idx; // cast to char * for ptr arithmetic - param_mem->write(param_address + idx, word_size, pdata,NULL,NULL); + if (m_args_aligned_size >= 0) return m_args_aligned_size; + + unsigned param_address = 0; + unsigned int total_size = 0; + for (std::map::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + + size_t arg_size = p.get_size() / 8; // size of param in bytes + total_size = (total_size + arg_size - 1) / arg_size * arg_size; // aligned + p.add_offset(total_size); + param->set_address(param_address + total_size); + total_size += arg_size; + } + + m_args_aligned_size = (total_size + 3) / 4 * 4; // final size aligned to word + + return m_args_aligned_size; +} + +void function_info::finalize(memory_space *param_mem) { + unsigned param_address = 0; + for (std::map::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + if (p.is_ptr_shared()) + continue; // Pointer to local memory: Should we pass the allocated shared + // memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t param_value = p.get_value(); + param_value.type = type; + symbol *param = m_symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype == (unsigned)type); + size_t size; + size = param_value.size; // size of param in bytes + // assert(param_value.offset == param_address); + if (size != p.get_size() / 8) { + printf( + "GPGPU-Sim PTX: WARNING actual kernel paramter size = %zu bytes vs. " + "formal size = %zu (using smaller of two)\n", + size, p.get_size() / 8); + size = (size < (p.get_size() / 8)) ? size : (p.get_size() / 8); + } + // copy the parameter over word-by-word so that parameter that crosses a + // memory page can be copied over + // Jin: copy parameter using aligned rules + const type_info *paramtype = param->type(); + int align_amount = paramtype->get_key().get_alignment_spec(); + align_amount = (align_amount == -1) ? size : align_amount; + param_address = (param_address + align_amount - 1) / align_amount * + align_amount; // aligned + + const size_t word_size = 4; + // param_address = (param_address + size - 1) / size * size; //aligned with + // size + for (size_t idx = 0; idx < size; idx += word_size) { + const char *pdata = reinterpret_cast(param_value.pdata) + + idx; // cast to char * for ptr arithmetic + param_mem->write(param_address + idx, word_size, pdata, NULL, NULL); + } + unsigned offset = p.get_offset(); + assert(offset == param_address); + param->set_address(param_address); + param_address += size; + } +} + +void function_info::param_to_shared(memory_space *shared_mem, + symbol_table *symtab) { + // TODO: call this only for PTXPlus with GT200 models + // extern gpgpu_sim* g_the_gpu; + if (not gpgpu_ctx->the_gpgpusim->g_the_gpu->get_config().convert_to_ptxplus()) + return; + + // copies parameters into simulated shared memory + for (std::map::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + if (p.is_ptr_shared()) + continue; // Pointer to local memory: Should we pass the allocated shared + // memory address to the param memory space? + std::string name = p.get_name(); + int type = p.get_type(); + param_t value = p.get_value(); + value.type = type; + symbol *param = symtab->lookup(name.c_str()); + unsigned xtype = param->type()->get_key().scalar_type(); + assert(xtype == (unsigned)type); + + int tmp; + size_t size; + unsigned offset = p.get_offset(); + type_info_key::type_decode(xtype, size, tmp); + + // Write to shared memory - offset + 0x10 + shared_mem->write(offset + 0x10, size / 8, value.pdata, NULL, NULL); + } +} + +void function_info::list_param(FILE *fout) const { + for (std::map::const_iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + const param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); + } + fflush(fout); +} + +void function_info::ptx_jit_config( + std::map mallocPtr_Size, + memory_space *param_mem, gpgpu_t *gpu, dim3 gridDim, dim3 blockDim) { + static unsigned long long counter = 0; + std::vector > param_data; + std::vector offsets; + std::vector paramIsPointer; + + char *gpgpusim_path = getenv("GPGPUSIM_ROOT"); + assert(gpgpusim_path != NULL); + char *wys_exec_path = getenv("WYS_EXEC_PATH"); + assert(wys_exec_path != NULL); + std::string command = + std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; + std::string filename(std::string(gpgpusim_path) + + "/debug_tools/WatchYourStep/data/params.config" + + std::to_string(counter)); + + // initialize paramList + char buff[1024]; + std::string filename_c(filename + "_c"); + snprintf(buff, 1024, "c++filt %s > %s", get_name().c_str(), + filename_c.c_str()); + assert(system(buff) != NULL); + FILE *fp = fopen(filename_c.c_str(), "r"); + fgets(buff, 1024, fp); + fclose(fp); + std::string fn(buff); + size_t pos1, pos2; + pos1 = fn.find_last_of("("); + pos2 = fn.find(")", pos1); + assert(pos2 > pos1 && pos1 > 0); + strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); + char *tok; + tok = strtok(buff, ","); + std::string tmp; + while (tok != NULL) { + std::string param(tok); + if (param.find("<") != std::string::npos) { + assert(param.find(">") == std::string::npos); + assert(param.find("*") == std::string::npos); + tmp = param; + } else { + if (tmp.length() > 0) { + tmp = ""; + assert(param.find(">") != std::string::npos); + assert(param.find("<") == std::string::npos); + assert(param.find("*") == std::string::npos); + } + printf("%s\n", param.c_str()); + if (param.find("*") != std::string::npos) { + paramIsPointer.push_back(true); + } else { + paramIsPointer.push_back(false); } - unsigned offset = p.get_offset(); - assert(offset == param_address); - param->set_address(param_address); - param_address += size; - } -} - -void function_info::param_to_shared( memory_space *shared_mem, symbol_table *symtab ) -{ - // TODO: call this only for PTXPlus with GT200 models - //extern gpgpu_sim* g_the_gpu; - if (not gpgpu_ctx->the_gpgpusim->g_the_gpu->get_config().convert_to_ptxplus()) return; - - // copies parameters into simulated shared memory - for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - if (p.is_ptr_shared()) continue; // Pointer to local memory: Should we pass the allocated shared memory address to the param memory space? - std::string name = p.get_name(); - int type = p.get_type(); - param_t value = p.get_value(); - value.type = type; - symbol *param = symtab->lookup(name.c_str()); - unsigned xtype = param->type()->get_key().scalar_type(); - assert(xtype==(unsigned)type); - - int tmp; - size_t size; - unsigned offset = p.get_offset(); - type_info_key::type_decode(xtype,size,tmp); - - // Write to shared memory - offset + 0x10 - shared_mem->write(offset+0x10,size/8,value.pdata,NULL,NULL); - } -} - - -void function_info::list_param( FILE *fout ) const -{ - for( std::map::const_iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - const param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - addr_t param_addr = param->get_address(); - fprintf(fout, "%s: %#08x\n", name.c_str(), param_addr); - } - fflush(fout); -} - -void function_info::ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) -{ - static unsigned long long counter = 0; - std::vector< std::pair > param_data; - std::vector offsets; - std::vector paramIsPointer; - - char * gpgpusim_path = getenv("GPGPUSIM_ROOT"); - assert(gpgpusim_path!=NULL); - char * wys_exec_path = getenv("WYS_EXEC_PATH"); - assert(wys_exec_path!=NULL); - std::string command = std::string("mkdir ") + gpgpusim_path + "/debug_tools/WatchYourStep/data"; - std::string filename(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/params.config" + std::to_string(counter)); - - //initialize paramList - char buff[1024]; - std::string filename_c(filename+"_c"); - snprintf(buff,1024,"c++filt %s > %s", get_name().c_str(), filename_c.c_str()); - assert(system(buff) != NULL); - FILE *fp = fopen(filename_c.c_str(), "r"); - fgets(buff, 1024, fp); - fclose(fp); - std::string fn(buff); - size_t pos1, pos2; - pos1 = fn.find_last_of("("); - pos2 = fn.find(")", pos1); - assert(pos2>pos1&&pos1>0); - strcpy(buff, fn.substr(pos1 + 1, pos2 - pos1 - 1).c_str()); - char *tok; - tok = strtok(buff, ","); - std::string tmp; - while(tok!=NULL){ - std::string param(tok); - if(param.find("<")!=std::string::npos){ - assert(param.find(">")==std::string::npos); - assert(param.find("*")==std::string::npos); - tmp = param; - } else { - if (tmp.length()>0){ - tmp = ""; - assert(param.find(">")!=std::string::npos); - assert(param.find("<")==std::string::npos); - assert(param.find("*")==std::string::npos); - } - printf("%s\n", param.c_str()); - if(param.find("*")!=std::string::npos){ - paramIsPointer.push_back(true); - }else{ - paramIsPointer.push_back(false); - } - } - tok = strtok(NULL, ","); } - - - for( std::map::iterator i=m_ptx_kernel_param_info.begin(); i!=m_ptx_kernel_param_info.end(); i++ ) { - param_info &p = i->second; - std::string name = p.get_name(); - symbol *param = m_symtab->lookup(name.c_str()); - addr_t param_addr = param->get_address(); - param_t param_value = p.get_value(); - offsets.push_back((unsigned)p.get_offset()); - - if (paramIsPointer[i->first] && (*(unsigned long long*)param_value.pdata != 0)){ - //is pointer - assert(param_value.size==sizeof(void*)&&"MisID'd this param as pointer"); - size_t array_size = 0; - unsigned long long param_pointer = *(unsigned long long*)param_value.pdata; - if(mallocPtr_Size.find(param_pointer)!=mallocPtr_Size.end()){ - array_size = mallocPtr_Size[param_pointer]; - }else{ - for( std::map::iterator j=mallocPtr_Size.begin(); j!=mallocPtr_Size.end(); j++ ) { - if(param_pointer>j->first&¶m_pointerfirst + j->second){ - array_size = j->first + j->second - param_pointer; - break; - } - } - assert(array_size>0&&"pointer was not previously malloc'd"); - } - - unsigned char* val = (unsigned char*) malloc(param_value.size); - param_mem->read(param_addr,param_value.size,(void*)val); - unsigned char* array_val = (unsigned char*) malloc(array_size); - gpu->get_global_memory()->read(*(unsigned*)((void*)val),array_size,(void*)array_val); - param_data.push_back(std::pair(array_size,array_val)); - paramIsPointer.push_back(true); - }else{ - unsigned char* val = (unsigned char*) malloc(param_value.size); - param_mem->read(param_addr,param_value.size,(void*)val); - param_data.push_back(std::pair(param_value.size,val)); - paramIsPointer.push_back(false); + tok = strtok(NULL, ","); + } + + for (std::map::iterator i = + m_ptx_kernel_param_info.begin(); + i != m_ptx_kernel_param_info.end(); i++) { + param_info &p = i->second; + std::string name = p.get_name(); + symbol *param = m_symtab->lookup(name.c_str()); + addr_t param_addr = param->get_address(); + param_t param_value = p.get_value(); + offsets.push_back((unsigned)p.get_offset()); + + if (paramIsPointer[i->first] && + (*(unsigned long long *)param_value.pdata != 0)) { + // is pointer + assert(param_value.size == sizeof(void *) && + "MisID'd this param as pointer"); + size_t array_size = 0; + unsigned long long param_pointer = + *(unsigned long long *)param_value.pdata; + if (mallocPtr_Size.find(param_pointer) != mallocPtr_Size.end()) { + array_size = mallocPtr_Size[param_pointer]; + } else { + for (std::map::iterator j = + mallocPtr_Size.begin(); + j != mallocPtr_Size.end(); j++) { + if (param_pointer > j->first && + param_pointer < j->first + j->second) { + array_size = j->first + j->second - param_pointer; + break; + } } - } + assert(array_size > 0 && "pointer was not previously malloc'd"); + } - FILE *fout = fopen (filename.c_str(), "w"); - printf("Writing data to %s ...\n", filename.c_str()); - fprintf(fout, "%s\n", get_name().c_str()); - fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, blockDim.x, blockDim.y, blockDim.z); - size_t index = 0; - for( std::vector< std::pair >::const_iterator i=param_data.begin(); i!=param_data.end(); i++ ) { - if (paramIsPointer[index]){ - fprintf(fout, "*"); - } - fprintf(fout, "%lu :", i->first); - for (size_t j = 0; jfirst; j++){ - fprintf(fout, " %u", i->second[j]); - } - fprintf(fout, " : %u", offsets[index]); - free (i->second); - fprintf(fout, "\n"); - index++; + unsigned char *val = (unsigned char *)malloc(param_value.size); + param_mem->read(param_addr, param_value.size, (void *)val); + unsigned char *array_val = (unsigned char *)malloc(array_size); + gpu->get_global_memory()->read(*(unsigned *)((void *)val), array_size, + (void *)array_val); + param_data.push_back( + std::pair(array_size, array_val)); + paramIsPointer.push_back(true); + } else { + unsigned char *val = (unsigned char *)malloc(param_value.size); + param_mem->read(param_addr, param_value.size, (void *)val); + param_data.push_back( + std::pair(param_value.size, val)); + paramIsPointer.push_back(false); } - fflush(fout); - fclose(fout); - - //ptx config - std::string ptx_config_fn(std::string(gpgpusim_path) + "/debug_tools/WatchYourStep/data/ptx.config" + std::to_string(counter)); - snprintf(buff, 1024, "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); - if (system(buff)!=0){ - printf("WARNING: Failed to execute grep to find ptx source \n"); - printf("Problematic call: %s", buff); - abort(); + } + + FILE *fout = fopen(filename.c_str(), "w"); + printf("Writing data to %s ...\n", filename.c_str()); + fprintf(fout, "%s\n", get_name().c_str()); + fprintf(fout, "%u,%u,%u %u,%u,%u\n", gridDim.x, gridDim.y, gridDim.z, + blockDim.x, blockDim.y, blockDim.z); + size_t index = 0; + for (std::vector >::const_iterator i = + param_data.begin(); + i != param_data.end(); i++) { + if (paramIsPointer[index]) { + fprintf(fout, "*"); } - FILE *fin = fopen(ptx_config_fn.c_str(), "r"); - char ptx_source[256]; - unsigned line_number; - int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); - assert(numscanned == 2); - fclose(fin); - snprintf(buff, 1024, "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk \"NR>={}&&NR<={}+2\" %s > %s", ptx_source, ptx_source, ptx_config_fn.c_str()); - if (system(buff)!=0){ - printf("WARNING: Failed to execute grep to find ptx header \n"); - printf("Problematic call: %s", buff); - abort(); + fprintf(fout, "%lu :", i->first); + for (size_t j = 0; j < i->first; j++) { + fprintf(fout, " %u", i->second[j]); } - fin = fopen(ptx_source, "r"); - assert(fin!=NULL); - printf("Writing data to %s ...\n", ptx_config_fn.c_str()); - fout = fopen(ptx_config_fn.c_str(), "a"); - assert(fout!=NULL); - for (unsigned i = 0; isecond); + fprintf(fout, "\n"); + index++; + } + fflush(fout); + fclose(fout); + + // ptx config + std::string ptx_config_fn(std::string(gpgpusim_path) + + "/debug_tools/WatchYourStep/data/ptx.config" + + std::to_string(counter)); + snprintf(buff, 1024, + "grep -rn \".entry %s\" %s/*.ptx | cut -d \":\" -f 1-2 > %s", + get_name().c_str(), wys_exec_path, ptx_config_fn.c_str()); + if (system(buff) != 0) { + printf("WARNING: Failed to execute grep to find ptx source \n"); + printf("Problematic call: %s", buff); + abort(); + } + FILE *fin = fopen(ptx_config_fn.c_str(), "r"); + char ptx_source[256]; + unsigned line_number; + int numscanned = fscanf(fin, "%[^:]:%u", ptx_source, &line_number); + assert(numscanned == 2); + fclose(fin); + snprintf(buff, 1024, + "grep -rn \".version\" %s | cut -d \":\" -f 1 | xargs -I \"{}\" awk " + "\"NR>={}&&NR<={}+2\" %s > %s", + ptx_source, ptx_source, ptx_config_fn.c_str()); + if (system(buff) != 0) { + printf("WARNING: Failed to execute grep to find ptx header \n"); + printf("Problematic call: %s", buff); + abort(); + } + fin = fopen(ptx_source, "r"); + assert(fin != NULL); + printf("Writing data to %s ...\n", ptx_config_fn.c_str()); + fout = fopen(ptx_config_fn.c_str(), "a"); + assert(fout != NULL); + for (unsigned i = 0; i < line_number; i++) { + assert(fgets(buff, 1024, fin) != NULL); + assert(!feof(fin)); + } + fprintf(fout, "\n\n"); + do { + fprintf(fout, "%s", buff); + assert(fgets(buff, 1024, fin) != NULL); + if (feof(fin)) { + break; } - fprintf(fout, "\n\n"); - do{ - fprintf(fout, "%s", buff); - assert(fgets(buff, 1024, fin) != NULL); - if(feof(fin)){ - break; - } - } while(strstr(buff, "entry")==NULL); + } while (strstr(buff, "entry") == NULL); - fclose(fin); - fflush(fout); - fclose(fout); - counter++; + fclose(fin); + fflush(fout); + fclose(fout); + counter++; } -template -bool cuda_sim::ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) -{ - if (g_debug_execution >= activate_level) { - // check each type of debug dump constraint to filter out dumps - if ( (g_debug_thread_uid != 0) && (thd_uid != (unsigned)g_debug_thread_uid) ) { - return false; - } - if ( (g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc) ) { - return false; - } +template +bool cuda_sim::ptx_debug_exec_dump_cond(int thd_uid, addr_t pc) { + if (g_debug_execution >= activate_level) { + // check each type of debug dump constraint to filter out dumps + if ((g_debug_thread_uid != 0) && + (thd_uid != (unsigned)g_debug_thread_uid)) { + return false; + } + if ((g_debug_pc != 0xBEEF1518) && (pc != g_debug_pc)) { + return false; + } - return true; - } - - return false; -} - -void cuda_sim::init_inst_classification_stat() -{ - static std::set init; - if( init.find(g_ptx_kernel_count) != init.end() ) - return; - init.insert(g_ptx_kernel_count); - - #define MAX_CLASS_KER 1024 - char kernelname[MAX_CLASS_KER] =""; - if (!g_inst_classification_stat) g_inst_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); - snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n",g_ptx_kernel_count ); - assert( g_ptx_kernel_count < MAX_CLASS_KER ) ; // a static limit on number of kernels increase it if it fails! - g_inst_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,20); - if (!g_inst_op_classification_stat) g_inst_op_classification_stat = (void**)calloc(MAX_CLASS_KER, sizeof(void*)); - snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n",g_ptx_kernel_count ); - g_inst_op_classification_stat[g_ptx_kernel_count] = StatCreate(kernelname,1,100); -} - -static unsigned get_tex_datasize( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &src1 = pI->src1(); //the name of the texture - std::string texname = src1.name(); - - /* - For programs with many streams, textures can be bound and unbound - asynchronously. This means we need to use the kernel's "snapshot" of - the state of the texture mappings when it was launched (so that we - don't try to access the incorrect texture mapping if it's been updated, - or that we don't access a mapping that has been unbound). - */ - kernel_info_t& k = thread->get_kernel(); - const struct textureInfo* texInfo = k.get_texinfo(texname); - - unsigned data_size = texInfo->texel_size; - return data_size; -} - -int tensorcore_op(int inst_opcode){ - - if((inst_opcode==MMA_OP)||(inst_opcode==MMA_LD_OP)||(inst_opcode==MMA_ST_OP)) - return 1; - else - return 0; -} -void ptx_thread_info::ptx_exec_inst( warp_inst_t &inst, unsigned lane_id) -{ - - bool skip = false; - int op_classification = 0; - addr_t pc = next_instr(); - assert( pc == inst.pc ); // make sure timing model and functional model are in sync - const ptx_instruction *pI = m_func_info->get_instruction(pc); - - set_npc( pc + pI->inst_size() ); - - - try { - - clearRPC(); - m_last_set_operand_value.u64 = 0; - - if(is_done()) - { - printf("attempted to execute instruction on a thread that is already done.\n"); + return true; + } + + return false; +} + +void cuda_sim::init_inst_classification_stat() { + static std::set init; + if (init.find(g_ptx_kernel_count) != init.end()) return; + init.insert(g_ptx_kernel_count); + +#define MAX_CLASS_KER 1024 + char kernelname[MAX_CLASS_KER] = ""; + if (!g_inst_classification_stat) + g_inst_classification_stat = (void **)calloc(MAX_CLASS_KER, sizeof(void *)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d Classification\n", + g_ptx_kernel_count); + assert(g_ptx_kernel_count < + MAX_CLASS_KER); // a static limit on number of kernels increase it if + // it fails! + g_inst_classification_stat[g_ptx_kernel_count] = + StatCreate(kernelname, 1, 20); + if (!g_inst_op_classification_stat) + g_inst_op_classification_stat = + (void **)calloc(MAX_CLASS_KER, sizeof(void *)); + snprintf(kernelname, MAX_CLASS_KER, "Kernel %d OP Classification\n", + g_ptx_kernel_count); + g_inst_op_classification_stat[g_ptx_kernel_count] = + StatCreate(kernelname, 1, 100); +} + +static unsigned get_tex_datasize(const ptx_instruction *pI, + ptx_thread_info *thread) { + const operand_info &src1 = pI->src1(); // the name of the texture + std::string texname = src1.name(); + + /* + For programs with many streams, textures can be bound and unbound + asynchronously. This means we need to use the kernel's "snapshot" of + the state of the texture mappings when it was launched (so that we + don't try to access the incorrect texture mapping if it's been updated, + or that we don't access a mapping that has been unbound). + */ + kernel_info_t &k = thread->get_kernel(); + const struct textureInfo *texInfo = k.get_texinfo(texname); + + unsigned data_size = texInfo->texel_size; + return data_size; +} + +int tensorcore_op(int inst_opcode) { + if ((inst_opcode == MMA_OP) || (inst_opcode == MMA_LD_OP) || + (inst_opcode == MMA_ST_OP)) + return 1; + else + return 0; +} +void ptx_thread_info::ptx_exec_inst(warp_inst_t &inst, unsigned lane_id) { + bool skip = false; + int op_classification = 0; + addr_t pc = next_instr(); + assert(pc == + inst.pc); // make sure timing model and functional model are in sync + const ptx_instruction *pI = m_func_info->get_instruction(pc); + + set_npc(pc + pI->inst_size()); + + try { + clearRPC(); + m_last_set_operand_value.u64 = 0; + + if (is_done()) { + printf( + "attempted to execute instruction on a thread that is already " + "done.\n"); assert(0); - } - - if ( g_debug_execution >= 6 || m_gpu->get_config().get_ptx_inst_debug_to_file()) { - if ( (m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid==0) - || (get_uid() == (unsigned)(m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid)) ) { - - clear_modifiedregs(); - enable_debug_trace(); + } + + if (g_debug_execution >= 6 || + m_gpu->get_config().get_ptx_inst_debug_to_file()) { + if ((m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid == 0) || + (get_uid() == + (unsigned)(m_gpu->gpgpu_ctx->func_sim->g_debug_thread_uid))) { + clear_modifiedregs(); + enable_debug_trace(); } - } - - - if( pI->has_pred() ) { + } + + if (pI->has_pred()) { const operand_info &pred = pI->get_pred(); ptx_reg_t pred_value = get_operand_value(pred, pred, PRED_TYPE, this, 0); - if(pI->get_pred_mod() == -1) { - skip = (pred_value.pred & 0x0001) ^ pI->get_pred_neg(); //ptxplus inverts the zero flag + if (pI->get_pred_mod() == -1) { + skip = (pred_value.pred & 0x0001) ^ + pI->get_pred_neg(); // ptxplus inverts the zero flag } else { - skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); + skip = !pred_lookup(pI->get_pred_mod(), pred_value.pred & 0x000F); } - } - int inst_opcode=pI->get_opcode(); - - if( skip ) { + } + int inst_opcode = pI->get_opcode(); + + if (skip) { inst.set_not_active(lane_id); - } else { + } else { const ptx_instruction *pI_saved = pI; ptx_instruction *pJ = NULL; - if( pI->get_opcode() == VOTE_OP ) { - pJ = new ptx_instruction(*pI); - *((warp_inst_t*)pJ) = inst; // copy active mask information - pI = pJ; + if (pI->get_opcode() == VOTE_OP) { + pJ = new ptx_instruction(*pI); + *((warp_inst_t *)pJ) = inst; // copy active mask information + pI = pJ; } - - if(((inst_opcode==MMA_OP||inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))){ - if(inst.active_count()!=MAX_WARP_SIZE) - { - printf("Tensor Core operation are warp synchronous operation. All the threads needs to be active."); - assert(0); - } + + if (((inst_opcode == MMA_OP || inst_opcode == MMA_LD_OP || + inst_opcode == MMA_ST_OP))) { + if (inst.active_count() != MAX_WARP_SIZE) { + printf( + "Tensor Core operation are warp synchronous operation. All the " + "threads needs to be active."); + assert(0); + } } - - //Tensorcore is warp synchronous operation. So these instructions needs to be executed only once. To make the simulation faster removing the redundant tensorcore operation - if(!tensorcore_op(inst_opcode)||((tensorcore_op(inst_opcode))&&(lane_id==0))){ - switch ( inst_opcode ) { - #define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,this); op_classification = CLASSIFICATION; break; - #define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) case OP: FUNC(pI,get_core(),inst); op_classification = CLASSIFICATION; break; - #include "opcodes.def" - #undef OP_DEF - #undef OP_W_DEF - default: printf( "Execution error: Invalid opcode (0x%x)\n", pI->get_opcode() ); break; - } + + // Tensorcore is warp synchronous operation. So these instructions needs + // to be executed only once. To make the simulation faster removing the + // redundant tensorcore operation + if (!tensorcore_op(inst_opcode) || + ((tensorcore_op(inst_opcode)) && (lane_id == 0))) { + switch (inst_opcode) { +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + FUNC(pI, this); \ + op_classification = CLASSIFICATION; \ + break; +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) \ + case OP: \ + FUNC(pI, get_core(), inst); \ + op_classification = CLASSIFICATION; \ + break; +#include "opcodes.def" +#undef OP_DEF +#undef OP_W_DEF + default: + printf("Execution error: Invalid opcode (0x%x)\n", + pI->get_opcode()); + break; + } } delete pJ; pI = pI_saved; - + // Run exit instruction if exit option included - if(pI->is_exit()) - exit_impl(pI,this); - } - - - - const gpgpu_functional_sim_config &config = m_gpu->get_config(); - - // Output instruction information to file and stdout - if( config.get_ptx_inst_debug_to_file() != 0 && - (config.get_ptx_inst_debug_thread_uid() == 0 || config.get_ptx_inst_debug_thread_uid() == get_uid()) ) { - fprintf(m_gpu->get_ptx_inst_debug_file(), - "[thd=%u] : (%s:%u - %s)\n", - get_uid(), - pI->source_file(), pI->source_line(), pI->get_source() ); - //fprintf(ptx_inst_debug_file, "has memory read=%d, has memory write=%d\n", pI->has_memory_read(), pI->has_memory_write()); + if (pI->is_exit()) exit_impl(pI, this); + } + + const gpgpu_functional_sim_config &config = m_gpu->get_config(); + + // Output instruction information to file and stdout + if (config.get_ptx_inst_debug_to_file() != 0 && + (config.get_ptx_inst_debug_thread_uid() == 0 || + config.get_ptx_inst_debug_thread_uid() == get_uid())) { + fprintf(m_gpu->get_ptx_inst_debug_file(), "[thd=%u] : (%s:%u - %s)\n", + get_uid(), pI->source_file(), pI->source_line(), + pI->get_source()); + // fprintf(ptx_inst_debug_file, "has memory read=%d, has memory + // write=%d\n", pI->has_memory_read(), pI->has_memory_write()); fflush(m_gpu->get_ptx_inst_debug_file()); - } + } - if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<5>(get_uid(), pc) ) { + if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<5>(get_uid(), + pc)) { dim3 ctaid = get_ctaid(); dim3 tid = get_tid(); - printf("%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u [pc=%u] (%s:%u - %s) [0x%llx]\n", - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, - get_uid(), - pI->uid(), ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z, - get_icount(), - pc, pI->source_file(), pI->source_line(), pI->get_source(), - m_last_set_operand_value.u64 ); + printf( + "%u [thd=%u][i=%u] : ctaid=(%u,%u,%u) tid=(%u,%u,%u) icount=%u " + "[pc=%u] (%s:%u - %s) [0x%llx]\n", + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, get_uid(), pI->uid(), + ctaid.x, ctaid.y, ctaid.z, tid.x, tid.y, tid.z, get_icount(), pc, + pI->source_file(), pI->source_line(), pI->get_source(), + m_last_set_operand_value.u64); fflush(stdout); - } - - addr_t insn_memaddr = 0xFEEBDAED; - memory_space_t insn_space = undefined_space; - _memory_op_t insn_memory_op = no_memory_op; - unsigned insn_data_size = 0; - if ( (pI->has_memory_read() || pI->has_memory_write()) ) { - if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) - { + } + + addr_t insn_memaddr = 0xFEEBDAED; + memory_space_t insn_space = undefined_space; + _memory_op_t insn_memory_op = no_memory_op; + unsigned insn_data_size = 0; + if ((pI->has_memory_read() || pI->has_memory_write())) { + if (!((inst_opcode == MMA_LD_OP || inst_opcode == MMA_ST_OP))) { insn_memaddr = last_eaddr(); insn_space = last_space(); unsigned to_type = pI->get_type(); insn_data_size = datatype2size(to_type); insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - } - } - - if ( pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { - inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,false /*not atomic*/); - } + } + } + + if (pI->get_opcode() == BAR_OP && pI->barrier_op() == RED_OPTION) { + inst.add_callback(lane_id, last_callback().function, + last_callback().instruction, this, + false /*not atomic*/); + } - if ( pI->get_opcode() == ATOM_OP ) { + if (pI->get_opcode() == ATOM_OP) { insn_memaddr = last_eaddr(); insn_space = last_space(); - inst.add_callback( lane_id, last_callback().function, last_callback().instruction, this,true /*atomic*/); + inst.add_callback(lane_id, last_callback().function, + last_callback().instruction, this, true /*atomic*/); unsigned to_type = pI->get_type(); insn_data_size = datatype2size(to_type); - } + } - if (pI->get_opcode() == TEX_OP) { - inst.set_addr(lane_id, last_eaddr() ); - assert( inst.space == last_space() ); - insn_data_size = get_tex_datasize(pI, this); // texture obtain its data granularity from the texture info - } + if (pI->get_opcode() == TEX_OP) { + inst.set_addr(lane_id, last_eaddr()); + assert(inst.space == last_space()); + insn_data_size = get_tex_datasize( + pI, + this); // texture obtain its data granularity from the texture info + } - // Output register information to file and stdout - if( config.get_ptx_inst_debug_to_file()!=0 && - (config.get_ptx_inst_debug_thread_uid()==0||config.get_ptx_inst_debug_thread_uid()==get_uid()) ) { + // Output register information to file and stdout + if (config.get_ptx_inst_debug_to_file() != 0 && + (config.get_ptx_inst_debug_thread_uid() == 0 || + config.get_ptx_inst_debug_thread_uid() == get_uid())) { dump_modifiedregs(m_gpu->get_ptx_inst_debug_file()); dump_regs(m_gpu->get_ptx_inst_debug_file()); - } + } - if ( g_debug_execution >= 6 ) { - if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<6>(get_uid(), pc) ) - dump_modifiedregs(stdout); - } - if ( g_debug_execution >= 10 ) { - if ( m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<10>(get_uid(), pc) ) - dump_regs(stdout); - } - update_pc(); - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn++; - - //not using it with functional simulation mode - if(!(this->m_functionalSimulationMode)) - ptx_file_line_stats_add_exec_count(pI); - - if ( m_gpu->gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification ) { + if (g_debug_execution >= 6) { + if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<6>(get_uid(), + pc)) + dump_modifiedregs(stdout); + } + if (g_debug_execution >= 10) { + if (m_gpu->gpgpu_ctx->func_sim->ptx_debug_exec_dump_cond<10>(get_uid(), + pc)) + dump_regs(stdout); + } + update_pc(); + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn++; + + // not using it with functional simulation mode + if (!(this->m_functionalSimulationMode)) + ptx_file_line_stats_add_exec_count(pI); + + if (m_gpu->gpgpu_ctx->func_sim->gpgpu_ptx_instruction_classification) { m_gpu->gpgpu_ctx->func_sim->init_inst_classification_stat(); - unsigned space_type=0; - switch ( pI->get_space().get_type() ) { - case global_space: space_type = 10; break; - case local_space: space_type = 11; break; - case tex_space: space_type = 12; break; - case surf_space: space_type = 13; break; - case param_space_kernel: - case param_space_local: - space_type = 14; break; - case shared_space: space_type = 15; break; - case const_space: space_type = 16; break; - default: - space_type = 0 ; - break; + unsigned space_type = 0; + switch (pI->get_space().get_type()) { + case global_space: + space_type = 10; + break; + case local_space: + space_type = 11; + break; + case tex_space: + space_type = 12; + break; + case surf_space: + space_type = 13; + break; + case param_space_kernel: + case param_space_local: + space_type = 14; + break; + case shared_space: + space_type = 15; + break; + case const_space: + space_type = 16; + break; + default: + space_type = 0; + break; } - StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], op_classification); - if (space_type) StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], ( int )space_type); - StatAddSample( m_gpu->gpgpu_ctx->func_sim->g_inst_op_classification_stat[m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], (int) pI->get_opcode() ); - } - if ( (m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn % 100000) == 0 ) { + StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat + [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], + op_classification); + if (space_type) + StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_classification_stat + [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], + (int)space_type); + StatAddSample(m_gpu->gpgpu_ctx->func_sim->g_inst_op_classification_stat + [m_gpu->gpgpu_ctx->func_sim->g_ptx_kernel_count], + (int)pI->get_opcode()); + } + if ((m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn % 100000) == 0) { dim3 ctaid = get_ctaid(); dim3 tid = get_tid(); - DPRINTF(LIVENESS, "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) tid=(%u,%u,%u)\n", - m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, ctaid.x,ctaid.y,ctaid.z,tid.x,tid.y,tid.z ); + DPRINTF(LIVENESS, + "GPGPU-Sim PTX: %u instructions simulated : ctaid=(%u,%u,%u) " + "tid=(%u,%u,%u)\n", + m_gpu->gpgpu_ctx->func_sim->g_ptx_sim_num_insn, ctaid.x, ctaid.y, + ctaid.z, tid.x, tid.y, tid.z); fflush(stdout); - } - - // "Return values" - if(!skip) { - if(!((inst_opcode==MMA_LD_OP||inst_opcode==MMA_ST_OP))) - { - inst.space = insn_space; - inst.set_addr(lane_id, insn_memaddr); - inst.data_size = insn_data_size; // simpleAtomicIntrinsics - assert( inst.memory_op == insn_memory_op ); - } - } - - } catch ( int x ) { - printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, pI->source_file(), pI->source_line() ); - printf("GPGPU-Sim PTX: '%s'\n", pI->get_source() ); - abort(); - } - -} - -void cuda_sim::set_param_gpgpu_num_shaders(int num_shaders) -{ - gpgpu_param_num_shaders = num_shaders; -} - -const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const function_info *kernel) -{ - return kernel->get_kernel_info(); -} - -const warp_inst_t *gpgpu_context::ptx_fetch_inst( address_type pc ) -{ - return pc_to_instruction(pc); -} - -unsigned ptx_sim_init_thread( kernel_info_t &kernel, - ptx_thread_info** thread_info, - int sid, - unsigned tid, - unsigned threads_left, - unsigned num_threads, - core_t *core, - unsigned hw_cta_id, - unsigned hw_warp_id, - gpgpu_t *gpu, - bool isInFunctionalSimulationMode) -{ - std::list &active_threads = kernel.active_threads(); - - static std::map shared_memory_lookup; - static std::map sstarr_memory_lookup; - static std::map ptx_cta_lookup; - static std::map ptx_warp_lookup; - static std::map > local_memory_lookup; - - if ( *thread_info != NULL ) { - ptx_thread_info *thd = *thread_info; - assert( thd->is_done() ); - if ( g_debug_execution==-1 ) { - dim3 ctaid = thd->get_ctaid(); - dim3 t = thd->get_tid(); - printf("GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) tid=(%u,%u,%u) uid=%u\n", - ctaid.x,ctaid.y,ctaid.z,t.x,t.y,t.z, thd->get_uid() ); - fflush(stdout); - } - thd->m_cta_info->register_deleted_thread(thd); - delete thd; - *thread_info = NULL; - } - - if ( !active_threads.empty() ) { - assert( active_threads.size() <= threads_left ); - ptx_thread_info *thd = active_threads.front(); - active_threads.pop_front(); - *thread_info = thd; - thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, isInFunctionalSimulationMode ); - return 1; - } - - if ( kernel.no_more_ctas_to_run() ) { - return 0; //finished! - } - - if ( threads_left < kernel.threads_per_cta() ) { - return 0; - } - - if ( g_debug_execution==-1 ) { - printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); - fflush(stdout); - } - - //initializing new CTA - ptx_cta_info *cta_info = NULL; - memory_space *shared_mem = NULL; - memory_space *sstarr_mem = NULL; - - unsigned cta_size = kernel.threads_per_cta(); - unsigned max_cta_per_sm = num_threads/cta_size; // e.g., 256 / 48 = 5 - assert( max_cta_per_sm > 0 ); - - //unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; - unsigned sm_idx = hw_cta_id*gpu->gpgpu_ctx->func_sim->gpgpu_param_num_shaders + sid; - - if ( shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end() ) { - if ( g_debug_execution >= 1 ) { - printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", - sm_idx, sid, max_cta_per_sm ); - } - char buf[512]; - snprintf(buf,512,"shared_%u", sid); - shared_mem = new memory_space_impl<16*1024>(buf,4); - shared_memory_lookup[sm_idx] = shared_mem; - snprintf(buf,512,"sstarr_%u", sid); - sstarr_mem = new memory_space_impl<16*1024>(buf,4); - sstarr_memory_lookup[sm_idx] = sstarr_mem; - cta_info = new ptx_cta_info(sm_idx, gpu->gpgpu_ctx); - ptx_cta_lookup[sm_idx] = cta_info; - } else { - if ( g_debug_execution >= 1 ) { - printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", - sm_idx, sid, max_cta_per_sm ); - } - shared_mem = shared_memory_lookup[sm_idx]; - sstarr_mem = sstarr_memory_lookup[sm_idx]; - cta_info = ptx_cta_lookup[sm_idx]; - cta_info->check_cta_thread_status_and_reset(); - } - - std::map &local_mem_lookup = local_memory_lookup[sid]; - while( kernel.more_threads_in_cta() ) { - dim3 ctaid3d = kernel.get_next_cta_id(); - unsigned new_tid = kernel.get_next_thread_id(); - dim3 tid3d = kernel.get_next_thread_id_3d(); - kernel.increment_thread_id(); - new_tid += tid; - ptx_thread_info *thd = new ptx_thread_info(kernel); - ptx_warp_info *warp_info = NULL; - if ( ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end() ) { - warp_info = new ptx_warp_info(); - ptx_warp_lookup[hw_warp_id] = warp_info; - } else { - warp_info = ptx_warp_lookup[hw_warp_id]; - } - thd->m_warp_info = warp_info; + } - memory_space *local_mem = NULL; - std::map::iterator l = local_mem_lookup.find(new_tid); - if ( l != local_mem_lookup.end() ) { - local_mem = l->second; - } else { - char buf[512]; - snprintf(buf,512,"local_%u_%u", sid, new_tid); - local_mem = new memory_space_impl<32>(buf,32); - local_mem_lookup[new_tid] = local_mem; + // "Return values" + if (!skip) { + if (!((inst_opcode == MMA_LD_OP || inst_opcode == MMA_ST_OP))) { + inst.space = insn_space; + inst.set_addr(lane_id, insn_memaddr); + inst.data_size = insn_data_size; // simpleAtomicIntrinsics + assert(inst.memory_op == insn_memory_op); } - thd->set_info(kernel.entry()); - thd->set_nctaid(kernel.get_grid_dim()); - thd->set_ntid(kernel.get_cta_dim()); - thd->set_ctaid(ctaid3d); - thd->set_tid(tid3d); - if( kernel.entry()->get_ptx_version().extensions() ) - thd->cpy_tid_to_reg(tid3d); - thd->set_valid(); - thd->m_shared_mem = shared_mem; - thd->m_sstarr_mem = sstarr_mem; - function_info *finfo = thd->func_info(); - symbol_table *st = finfo->get_symtab(); - thd->func_info()->param_to_shared(thd->m_shared_mem,st); - thd->func_info()->param_to_shared(thd->m_sstarr_mem,st); - thd->m_cta_info = cta_info; - cta_info->add_thread(thd); - thd->m_local_mem = local_mem; - if ( g_debug_execution==-1 ) { - printf("GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) tid=(%u,%u,%u) @ 0x%Lx\n", - ctaid3d.x,ctaid3d.y,ctaid3d.z,tid3d.x,tid3d.y,tid3d.z, (unsigned long long)thd ); - fflush(stdout); - } - active_threads.push_back(thd); - } - if ( g_debug_execution==-1 ) { - printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); - fflush(stdout); - } - - kernel.increment_cta_id(); + } - assert( active_threads.size() <= threads_left ); - *thread_info = active_threads.front(); - (*thread_info)->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid,isInFunctionalSimulationMode ); - active_threads.pop_front(); - return 1; + } catch (int x) { + printf("GPGPU-Sim PTX: ERROR (%d) executing intruction (%s:%u)\n", x, + pI->source_file(), pI->source_line()); + printf("GPGPU-Sim PTX: '%s'\n", pI->get_source()); + abort(); + } } -size_t get_kernel_code_size( class function_info *entry ) -{ - return entry->get_function_size(); +void cuda_sim::set_param_gpgpu_num_shaders(int num_shaders) { + gpgpu_param_num_shaders = num_shaders; } +const struct gpgpu_ptx_sim_info *ptx_sim_kernel_info( + const function_info *kernel) { + return kernel->get_kernel_info(); +} -kernel_info_t *cuda_sim::gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, - gpgpu_ptx_sim_arg_list_t args, - struct dim3 gridDim, - struct dim3 blockDim, - gpgpu_t *gpu ) -{ - kernel_info_t *result = new kernel_info_t(gridDim,blockDim,entry,gpu->getNameArrayMapping(),gpu->getNameInfoMapping()); - unsigned argcount=args.size(); - unsigned argn=1; - for( gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); a++ ) { - entry->add_param_data(argcount-argn,&(*a)); - argn++; - } - entry->finalize(result->get_param_memory()); - g_ptx_kernel_count++; - fflush(stdout); - - return result; +const warp_inst_t *gpgpu_context::ptx_fetch_inst(address_type pc) { + return pc_to_instruction(pc); } -#include "../../version" -#include "detailed_version" +unsigned ptx_sim_init_thread(kernel_info_t &kernel, + ptx_thread_info **thread_info, int sid, + unsigned tid, unsigned threads_left, + unsigned num_threads, core_t *core, + unsigned hw_cta_id, unsigned hw_warp_id, + gpgpu_t *gpu, bool isInFunctionalSimulationMode) { + std::list &active_threads = kernel.active_threads(); -void print_splash() -{ - static int splash_printed=0; - if ( !splash_printed ) { - fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", g_gpgpusim_version_string, g_gpgpusim_build_string ); - splash_printed=1; - } -} + static std::map shared_memory_lookup; + static std::map sstarr_memory_lookup; + static std::map ptx_cta_lookup; + static std::map ptx_warp_lookup; + static std::map > + local_memory_lookup; -void cuda_sim::gpgpu_ptx_sim_register_const_variable(void *hostVar, const char *deviceName, size_t size ) -{ - printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", deviceName, size ); - g_const_name_lookup[hostVar] = deviceName; -} + if (*thread_info != NULL) { + ptx_thread_info *thd = *thread_info; + assert(thd->is_done()); + if (g_debug_execution == -1) { + dim3 ctaid = thd->get_ctaid(); + dim3 t = thd->get_tid(); + printf( + "GPGPU-Sim PTX simulator: thread exiting ctaid=(%u,%u,%u) " + "tid=(%u,%u,%u) uid=%u\n", + ctaid.x, ctaid.y, ctaid.z, t.x, t.y, t.z, thd->get_uid()); + fflush(stdout); + } + thd->m_cta_info->register_deleted_thread(thd); + delete thd; + *thread_info = NULL; + } + + if (!active_threads.empty()) { + assert(active_threads.size() <= threads_left); + ptx_thread_info *thd = active_threads.front(); + active_threads.pop_front(); + *thread_info = thd; + thd->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, + isInFunctionalSimulationMode); + return 1; + } + + if (kernel.no_more_ctas_to_run()) { + return 0; // finished! + } + + if (threads_left < kernel.threads_per_cta()) { + return 0; + } + + if (g_debug_execution == -1) { + printf("GPGPU-Sim PTX simulator: STARTING THREAD ALLOCATION --> \n"); + fflush(stdout); + } + + // initializing new CTA + ptx_cta_info *cta_info = NULL; + memory_space *shared_mem = NULL; + memory_space *sstarr_mem = NULL; + + unsigned cta_size = kernel.threads_per_cta(); + unsigned max_cta_per_sm = num_threads / cta_size; // e.g., 256 / 48 = 5 + assert(max_cta_per_sm > 0); + + // unsigned sm_idx = (tid/cta_size)*gpgpu_param_num_shaders + sid; + unsigned sm_idx = + hw_cta_id * gpu->gpgpu_ctx->func_sim->gpgpu_param_num_shaders + sid; + + if (shared_memory_lookup.find(sm_idx) == shared_memory_lookup.end()) { + if (g_debug_execution >= 1) { + printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", sm_idx, + sid, max_cta_per_sm); + } + char buf[512]; + snprintf(buf, 512, "shared_%u", sid); + shared_mem = new memory_space_impl<16 * 1024>(buf, 4); + shared_memory_lookup[sm_idx] = shared_mem; + snprintf(buf, 512, "sstarr_%u", sid); + sstarr_mem = new memory_space_impl<16 * 1024>(buf, 4); + sstarr_memory_lookup[sm_idx] = sstarr_mem; + cta_info = new ptx_cta_info(sm_idx, gpu->gpgpu_ctx); + ptx_cta_lookup[sm_idx] = cta_info; + } else { + if (g_debug_execution >= 1) { + printf(" : sm_idx=%u sid=%u max_cta_per_sm=%u\n", sm_idx, + sid, max_cta_per_sm); + } + shared_mem = shared_memory_lookup[sm_idx]; + sstarr_mem = sstarr_memory_lookup[sm_idx]; + cta_info = ptx_cta_lookup[sm_idx]; + cta_info->check_cta_thread_status_and_reset(); + } + + std::map &local_mem_lookup = + local_memory_lookup[sid]; + while (kernel.more_threads_in_cta()) { + dim3 ctaid3d = kernel.get_next_cta_id(); + unsigned new_tid = kernel.get_next_thread_id(); + dim3 tid3d = kernel.get_next_thread_id_3d(); + kernel.increment_thread_id(); + new_tid += tid; + ptx_thread_info *thd = new ptx_thread_info(kernel); + ptx_warp_info *warp_info = NULL; + if (ptx_warp_lookup.find(hw_warp_id) == ptx_warp_lookup.end()) { + warp_info = new ptx_warp_info(); + ptx_warp_lookup[hw_warp_id] = warp_info; + } else { + warp_info = ptx_warp_lookup[hw_warp_id]; + } + thd->m_warp_info = warp_info; -void cuda_sim::gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ) -{ - printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", deviceName ); - g_global_name_lookup[hostVar] = deviceName; + memory_space *local_mem = NULL; + std::map::iterator l = + local_mem_lookup.find(new_tid); + if (l != local_mem_lookup.end()) { + local_mem = l->second; + } else { + char buf[512]; + snprintf(buf, 512, "local_%u_%u", sid, new_tid); + local_mem = new memory_space_impl<32>(buf, 32); + local_mem_lookup[new_tid] = local_mem; + } + thd->set_info(kernel.entry()); + thd->set_nctaid(kernel.get_grid_dim()); + thd->set_ntid(kernel.get_cta_dim()); + thd->set_ctaid(ctaid3d); + thd->set_tid(tid3d); + if (kernel.entry()->get_ptx_version().extensions()) + thd->cpy_tid_to_reg(tid3d); + thd->set_valid(); + thd->m_shared_mem = shared_mem; + thd->m_sstarr_mem = sstarr_mem; + function_info *finfo = thd->func_info(); + symbol_table *st = finfo->get_symtab(); + thd->func_info()->param_to_shared(thd->m_shared_mem, st); + thd->func_info()->param_to_shared(thd->m_sstarr_mem, st); + thd->m_cta_info = cta_info; + cta_info->add_thread(thd); + thd->m_local_mem = local_mem; + if (g_debug_execution == -1) { + printf( + "GPGPU-Sim PTX simulator: allocating thread ctaid=(%u,%u,%u) " + "tid=(%u,%u,%u) @ 0x%Lx\n", + ctaid3d.x, ctaid3d.y, ctaid3d.z, tid3d.x, tid3d.y, tid3d.z, + (unsigned long long)thd); + fflush(stdout); + } + active_threads.push_back(thd); + } + if (g_debug_execution == -1) { + printf("GPGPU-Sim PTX simulator: <-- FINISHING THREAD ALLOCATION\n"); + fflush(stdout); + } + + kernel.increment_cta_id(); + + assert(active_threads.size() <= threads_left); + *thread_info = active_threads.front(); + (*thread_info) + ->init(gpu, core, sid, hw_cta_id, hw_warp_id, tid, + isInFunctionalSimulationMode); + active_threads.pop_front(); + return 1; +} + +size_t get_kernel_code_size(class function_info *entry) { + return entry->get_function_size(); +} + +kernel_info_t *cuda_sim::gpgpu_opencl_ptx_sim_init_grid( + class function_info *entry, gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, struct dim3 blockDim, gpgpu_t *gpu) { + kernel_info_t *result = + new kernel_info_t(gridDim, blockDim, entry, gpu->getNameArrayMapping(), + gpu->getNameInfoMapping()); + unsigned argcount = args.size(); + unsigned argn = 1; + for (gpgpu_ptx_sim_arg_list_t::iterator a = args.begin(); a != args.end(); + a++) { + entry->add_param_data(argcount - argn, &(*a)); + argn++; + } + entry->finalize(result->get_param_memory()); + g_ptx_kernel_count++; + fflush(stdout); + + return result; } -void cuda_sim::gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ) -{ - printf("GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", hostVar); - bool found_sym = false; - memory_space_t mem_region = undefined_space; - std::string sym_name; - - std::map::iterator c=gpu->gpgpu_ctx->func_sim->g_const_name_lookup.find(hostVar); - if ( c!=gpu->gpgpu_ctx->func_sim->g_const_name_lookup.end() ) { - found_sym = true; - sym_name = c->second; - mem_region = const_space; - } - std::map::iterator g=gpu->gpgpu_ctx->func_sim->g_global_name_lookup.find(hostVar); - if ( g!=gpu->gpgpu_ctx->func_sim->g_global_name_lookup.end() ) { - if ( found_sym ) { - printf("Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared both const and global?\n", - sym_name.c_str(), (unsigned long long)hostVar ); - abort(); - } - found_sym = true; - sym_name = g->second; - mem_region = global_space; - } - if( g_globals.find(hostVar) != g_globals.end() ) { - found_sym = true; - sym_name = hostVar; - mem_region = global_space; - } - if( g_constants.find(hostVar) != g_constants.end() ) { - found_sym = true; - sym_name = hostVar; - mem_region = const_space; - } +#include "../../version" +#include "detailed_version" - if ( !found_sym ) { - printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); +void print_splash() { + static int splash_printed = 0; + if (!splash_printed) { + fprintf(stdout, "\n\n *** %s [build %s] ***\n\n\n", + g_gpgpusim_version_string, g_gpgpusim_build_string); + splash_printed = 1; + } +} + +void cuda_sim::gpgpu_ptx_sim_register_const_variable(void *hostVar, + const char *deviceName, + size_t size) { + printf("GPGPU-Sim PTX registering constant %s (%zu bytes) to name mapping\n", + deviceName, size); + g_const_name_lookup[hostVar] = deviceName; +} + +void cuda_sim::gpgpu_ptx_sim_register_global_variable(void *hostVar, + const char *deviceName, + size_t size) { + printf("GPGPU-Sim PTX registering global %s hostVar to name mapping\n", + deviceName); + g_global_name_lookup[hostVar] = deviceName; +} + +void cuda_sim::gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, + size_t count, size_t offset, int to, + gpgpu_t *gpu) { + printf( + "GPGPU-Sim PTX: starting gpgpu_ptx_sim_memcpy_symbol with hostVar 0x%p\n", + hostVar); + bool found_sym = false; + memory_space_t mem_region = undefined_space; + std::string sym_name; + + std::map::iterator c = + gpu->gpgpu_ctx->func_sim->g_const_name_lookup.find(hostVar); + if (c != gpu->gpgpu_ctx->func_sim->g_const_name_lookup.end()) { + found_sym = true; + sym_name = c->second; + mem_region = const_space; + } + std::map::iterator g = + gpu->gpgpu_ctx->func_sim->g_global_name_lookup.find(hostVar); + if (g != gpu->gpgpu_ctx->func_sim->g_global_name_lookup.end()) { + if (found_sym) { + printf( + "Execution error: PTX symbol \"%s\" w/ hostVar=0x%Lx is declared " + "both const and global?\n", + sym_name.c_str(), (unsigned long long)hostVar); abort(); - } else printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ hostVar=0x%Lx\n", (unsigned long long)hostVar ); - const char *mem_name = NULL; - memory_space *mem = NULL; - - std::map::iterator st = gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.find(sym_name.c_str()); - assert( st != gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.end() ); - symbol_table *symtab = st->second; - - symbol *sym = symtab->lookup(sym_name.c_str()); - assert(sym); - unsigned dst = sym->get_address() + offset; - switch (mem_region.get_type()) { - case const_space: + } + found_sym = true; + sym_name = g->second; + mem_region = global_space; + } + if (g_globals.find(hostVar) != g_globals.end()) { + found_sym = true; + sym_name = hostVar; + mem_region = global_space; + } + if (g_constants.find(hostVar) != g_constants.end()) { + found_sym = true; + sym_name = hostVar; + mem_region = const_space; + } + + if (!found_sym) { + printf("Execution error: No information for PTX symbol w/ hostVar=0x%Lx\n", + (unsigned long long)hostVar); + abort(); + } else + printf( + "GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: Found PTX symbol w/ " + "hostVar=0x%Lx\n", + (unsigned long long)hostVar); + const char *mem_name = NULL; + memory_space *mem = NULL; + + std::map::iterator st = + gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.find(sym_name.c_str()); + assert(st != gpgpu_ctx->ptx_parser->g_sym_name_to_symbol_table.end()); + symbol_table *symtab = st->second; + + symbol *sym = symtab->lookup(sym_name.c_str()); + assert(sym); + unsigned dst = sym->get_address() + offset; + switch (mem_region.get_type()) { + case const_space: mem = gpu->get_global_memory(); mem_name = "const"; break; - case global_space: + case global_space: mem = gpu->get_global_memory(); mem_name = "global"; break; - default: + default: abort(); - } - printf("GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes %s symbol %s+%zu @0x%x ...\n", - mem_name, count, (to?" to ":"from"), sym_name.c_str(), offset, dst ); - for ( unsigned n=0; n < count; n++ ) { - if( to ) mem->write(dst+n,1,((char*)src)+n,NULL,NULL); - else mem->read(dst+n,1,((char*)src)+n); - } - fflush(stdout); + } + printf( + "GPGPU-Sim PTX: gpgpu_ptx_sim_memcpy_symbol: copying %s memory %zu bytes " + "%s symbol %s+%zu @0x%x ...\n", + mem_name, count, (to ? " to " : "from"), sym_name.c_str(), offset, dst); + for (unsigned n = 0; n < count; n++) { + if (to) + mem->write(dst + n, 1, ((char *)src) + n, NULL, NULL); + else + mem->read(dst + n, 1, ((char *)src) + n); + } + fflush(stdout); } extern int ptx_debug; -void cuda_sim::read_sim_environment_variables() -{ - ptx_debug = 0; - g_debug_execution = 0; - g_interactive_debugger_enabled = false; - - char *mode = getenv("PTX_SIM_MODE_FUNC"); - if ( mode ) - sscanf(mode,"%u", &g_ptx_sim_mode); - printf("GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC environment variable:\n", g_ptx_sim_mode); - printf(" 1=functional simulation only, 0=detailed performance simulator)\n"); - char *dbg_inter = getenv("GPGPUSIM_DEBUG"); - if ( dbg_inter && strlen(dbg_inter) ) { - printf("GPGPU-Sim PTX: enabling interactive debugger\n"); - fflush(stdout); - g_interactive_debugger_enabled = true; - } - char *dbg_level = getenv("PTX_SIM_DEBUG"); - if ( dbg_level && strlen(dbg_level) ) { - printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level ); - fflush(stdout); - sscanf(dbg_level,"%d", &g_debug_execution); - } - char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); - if ( dbg_thread && strlen(dbg_thread) ) { - printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", dbg_thread ); - fflush(stdout); - sscanf(dbg_thread,"%d", &g_debug_thread_uid); - } - char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); - if ( dbg_pc && strlen(dbg_pc) ) { - printf("GPGPU-Sim PTX: printing debug information for instruction with PC = %s\n", dbg_pc ); - fflush(stdout); - sscanf(dbg_pc,"%d", &g_debug_pc); - } +void cuda_sim::read_sim_environment_variables() { + ptx_debug = 0; + g_debug_execution = 0; + g_interactive_debugger_enabled = false; + + char *mode = getenv("PTX_SIM_MODE_FUNC"); + if (mode) sscanf(mode, "%u", &g_ptx_sim_mode); + printf( + "GPGPU-Sim PTX: simulation mode %d (can change with PTX_SIM_MODE_FUNC " + "environment variable:\n", + g_ptx_sim_mode); + printf( + " 1=functional simulation only, 0=detailed performance " + "simulator)\n"); + char *dbg_inter = getenv("GPGPUSIM_DEBUG"); + if (dbg_inter && strlen(dbg_inter)) { + printf("GPGPU-Sim PTX: enabling interactive debugger\n"); + fflush(stdout); + g_interactive_debugger_enabled = true; + } + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if (dbg_level && strlen(dbg_level)) { + printf("GPGPU-Sim PTX: setting debug level to %s\n", dbg_level); + fflush(stdout); + sscanf(dbg_level, "%d", &g_debug_execution); + } + char *dbg_thread = getenv("PTX_SIM_DEBUG_THREAD_UID"); + if (dbg_thread && strlen(dbg_thread)) { + printf("GPGPU-Sim PTX: printing debug information for thread uid %s\n", + dbg_thread); + fflush(stdout); + sscanf(dbg_thread, "%d", &g_debug_thread_uid); + } + char *dbg_pc = getenv("PTX_SIM_DEBUG_PC"); + if (dbg_pc && strlen(dbg_pc)) { + printf( + "GPGPU-Sim PTX: printing debug information for instruction with PC = " + "%s\n", + dbg_pc); + fflush(stdout); + sscanf(dbg_pc, "%d", &g_debug_pc); + } #if CUDART_VERSION > 1010 - g_override_embedded_ptx = false; - char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); - if (usefile && strlen(usefile)) { - printf("GPGPU-Sim PTX: overriding embedded ptx with ptx file (PTX_SIM_USE_PTX_FILE is set)\n"); - fflush(stdout); - g_override_embedded_ptx = true; - } - char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); - if( blocking && !strcmp(blocking,"1") ) { - g_cuda_launch_blocking = true; - } + g_override_embedded_ptx = false; + char *usefile = getenv("PTX_SIM_USE_PTX_FILE"); + if (usefile && strlen(usefile)) { + printf( + "GPGPU-Sim PTX: overriding embedded ptx with ptx file " + "(PTX_SIM_USE_PTX_FILE is set)\n"); + fflush(stdout); + g_override_embedded_ptx = true; + } + char *blocking = getenv("CUDA_LAUNCH_BLOCKING"); + if (blocking && !strcmp(blocking, "1")) { + g_cuda_launch_blocking = true; + } #else - g_cuda_launch_blocking = true; - g_override_embedded_ptx = true; + g_cuda_launch_blocking = true; + g_override_embedded_ptx = true; #endif - if ( g_debug_execution >= 40 ) { - ptx_debug = 1; - } -} - -#define MAX(a,b) (((a)>(b))?(a):(b)) - -unsigned max_cta (const struct gpgpu_ptx_sim_info *kernel_info, unsigned threads_per_cta, unsigned int warp_size, unsigned int n_thread_per_shader, unsigned int gpgpu_shmem_size, unsigned int gpgpu_shader_registers, unsigned int max_cta_per_core) -{ - - unsigned int padded_cta_size = threads_per_cta; - if (padded_cta_size%warp_size) - padded_cta_size = ((padded_cta_size/warp_size)+1)*(warp_size); - unsigned int result_thread = n_thread_per_shader / padded_cta_size; - - unsigned int result_shmem = (unsigned)-1; - if (kernel_info->smem > 0) - result_shmem = gpgpu_shmem_size / kernel_info->smem; - unsigned int result_regs = (unsigned)-1; - if (kernel_info->regs > 0) - result_regs = gpgpu_shader_registers / (padded_cta_size * ((kernel_info->regs+3)&~3)); - printf("padded cta size is %d and %d and %d",padded_cta_size, kernel_info->regs, ((kernel_info->regs+3)&~3) ); - //Limit by CTA - unsigned int result_cta = max_cta_per_core; - - unsigned result = result_thread; - result = gs_min2(result, result_shmem); - result = gs_min2(result, result_regs); - result = gs_min2(result, result_cta); - - printf ("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); - if (result == result_thread) printf (" threads"); - if (result == result_shmem) printf (" shmem"); - if (result == result_regs) printf (" regs"); - if (result == result_cta) printf (" cta_limit"); - printf ("\n"); - - return result; + if (g_debug_execution >= 40) { + ptx_debug = 1; + } +} + +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) + +unsigned max_cta(const struct gpgpu_ptx_sim_info *kernel_info, + unsigned threads_per_cta, unsigned int warp_size, + unsigned int n_thread_per_shader, + unsigned int gpgpu_shmem_size, + unsigned int gpgpu_shader_registers, + unsigned int max_cta_per_core) { + unsigned int padded_cta_size = threads_per_cta; + if (padded_cta_size % warp_size) + padded_cta_size = ((padded_cta_size / warp_size) + 1) * (warp_size); + unsigned int result_thread = n_thread_per_shader / padded_cta_size; + + unsigned int result_shmem = (unsigned)-1; + if (kernel_info->smem > 0) + result_shmem = gpgpu_shmem_size / kernel_info->smem; + unsigned int result_regs = (unsigned)-1; + if (kernel_info->regs > 0) + result_regs = gpgpu_shader_registers / + (padded_cta_size * ((kernel_info->regs + 3) & ~3)); + printf("padded cta size is %d and %d and %d", padded_cta_size, + kernel_info->regs, ((kernel_info->regs + 3) & ~3)); + // Limit by CTA + unsigned int result_cta = max_cta_per_core; + + unsigned result = result_thread; + result = gs_min2(result, result_shmem); + result = gs_min2(result, result_regs); + result = gs_min2(result, result_cta); + + printf("GPGPU-Sim uArch: CTA/core = %u, limited by:", result); + if (result == result_thread) printf(" threads"); + if (result == result_shmem) printf(" shmem"); + if (result == result_regs) printf(" regs"); + if (result == result_cta) printf(" cta_limit"); + printf("\n"); + + return result; } /*! -This function simulates the CUDA code functionally, it takes a kernel_info_t parameter -which holds the data for the CUDA kernel to be executed +This function simulates the CUDA code functionally, it takes a kernel_info_t +parameter which holds the data for the CUDA kernel to be executed !*/ -void cuda_sim::gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL ) -{ - printf("GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n",kernel.name().c_str()); - - //using a shader core object for book keeping, it is not needed but as most function built for performance simulation need it we use it here - //extern gpgpu_sim *g_the_gpu; - //before we execute, we should do PDOM analysis for functional simulation scenario. - function_info *kernel_func_info = kernel.entry(); - const struct gpgpu_ptx_sim_info *kernel_info = ptx_sim_kernel_info(kernel_func_info); - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); - - if (kernel_func_info->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", kernel.name().c_str() ); +void cuda_sim::gpgpu_cuda_ptx_sim_main_func(kernel_info_t &kernel, + bool openCL) { + printf( + "GPGPU-Sim: Performing Functional Simulation, executing kernel %s...\n", + kernel.name().c_str()); + + // using a shader core object for book keeping, it is not needed but as most + // function built for performance simulation need it we use it here + // extern gpgpu_sim *g_the_gpu; + // before we execute, we should do PDOM analysis for functional simulation + // scenario. + function_info *kernel_func_info = kernel.entry(); + const struct gpgpu_ptx_sim_info *kernel_info = + ptx_sim_kernel_info(kernel_func_info); + checkpoint *g_checkpoint; + g_checkpoint = new checkpoint(); + + if (kernel_func_info->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", + kernel.name().c_str()); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", + kernel.name().c_str()); + kernel_func_info->do_pdom(); + kernel_func_info->set_pdom(); + } + + unsigned max_cta_tot = max_cta( + kernel_info, kernel.threads_per_cta(), + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->n_thread_per_shader, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->gpgpu_shmem_size, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->gpgpu_shader_registers, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig() + ->max_cta_per_core); + printf("Max CTA : %d\n", max_cta_tot); + + int cp_op = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_option; + int cp_kernel = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_kernel; + cp_count = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_insn_Y; + cp_cta_resume = gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_CTA_t; + int cta_launched = 0; + + // we excute the kernel one CTA (Block) at the time, as synchronization + // functions work block wise + while (!kernel.no_more_ctas_to_run()) { + unsigned temp = kernel.get_next_cta_id_single(); + + if (cp_op == 0 || + (cp_op == 1 && cta_launched < cp_cta_resume && + kernel.get_uid() == cp_kernel) || + kernel.get_uid() < cp_kernel) // just fro testing + { + functionalCoreSim cta( + &kernel, gpgpu_ctx->the_gpgpusim->g_the_gpu, + gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size); + cta.execute(cp_count, temp); + +#if (CUDART_VERSION >= 5000) + gpgpu_ctx->device_runtime->launch_all_device_kernels(); +#endif } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", kernel.name().c_str() ); - kernel_func_info->do_pdom(); - kernel_func_info->set_pdom(); + kernel.increment_cta_id(); } - - unsigned max_cta_tot = max_cta(kernel_info,kernel.threads_per_cta(), gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->n_thread_per_shader, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->gpgpu_shmem_size, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->gpgpu_shader_registers, gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->max_cta_per_core); - printf("Max CTA : %d\n",max_cta_tot); - - int cp_op= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_option; - int cp_kernel= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_kernel; - cp_count= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_insn_Y; - cp_cta_resume= gpgpu_ctx->the_gpgpusim->g_the_gpu->checkpoint_CTA_t; - int cta_launched =0; - - //we excute the kernel one CTA (Block) at the time, as synchronization functions work block wise - while(!kernel.no_more_ctas_to_run()){ - unsigned temp=kernel.get_next_cta_id_single(); - - - if(cp_op==0 || (cp_op==1 && cta_launchedthe_gpgpusim->g_the_gpu, - gpgpu_ctx->the_gpgpusim->g_the_gpu->getShaderCoreConfig()->warp_size - ); - cta.execute(cp_count,temp); - - #if (CUDART_VERSION >= 5000) - gpgpu_ctx->device_runtime->launch_all_device_kernels(); - #endif - } - else - { - kernel.increment_cta_id(); - } cta_launched++; + } + + if (cp_op == 1) { + char f1name[2048]; + snprintf(f1name, 2048, "checkpoint_files/global_mem_%d.txt", + kernel.get_uid()); + g_checkpoint->store_global_mem( + gpgpu_ctx->the_gpgpusim->g_the_gpu->get_global_memory(), f1name, + (char *)"%08x"); + } + + // registering this kernel as done + + // openCL kernel simulation calls don't register the kernel so we don't + // register its exit + if (!openCL) { + // extern stream_manager *g_stream_manager; + gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel( + kernel.get_uid()); + } + + //******PRINTING******* + printf("GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", + g_ptx_sim_num_insn); + if (gpgpu_ptx_instruction_classification) { + StatDisp(g_inst_classification_stat[g_ptx_kernel_count]); + StatDisp(g_inst_op_classification_stat[g_ptx_kernel_count]); + } + + // time_t variables used to calculate the total simulation time + // the start time of simulation is hold by the global variable + // g_simulation_starttime g_simulation_starttime is initilized by + // gpgpu_ptx_sim_init_perf() in gpgpusim_entrypoint.cc upon starting gpgpu-sim + time_t end_time, elapsed_time, days, hrs, minutes, sec; + end_time = time((time_t *)NULL); + elapsed_time = + MAX(end_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); + + // calculating and printing simulation time in terms of days, hours, minutes + // and seconds + days = elapsed_time / (3600 * 24); + hrs = elapsed_time / 3600 - 24 * days; + minutes = elapsed_time / 60 - 60 * (hrs + 24 * days); + sec = elapsed_time - 60 * (minutes + 60 * (hrs + 24 * days)); + + fflush(stderr); + printf( + "\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", + (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, + (unsigned)elapsed_time); + printf("gpgpu_simulation_rate = %u (inst/sec)\n", + (unsigned)(g_ptx_sim_num_insn / elapsed_time)); + fflush(stdout); +} + +void functionalCoreSim::initializeCTA(unsigned ctaid_cp) { + int ctaLiveThreads = 0; + symbol_table *symtab = m_kernel->entry()->get_symtab(); + + for (int i = 0; i < m_warp_count; i++) { + m_warpAtBarrier[i] = false; + m_liveThreadCount[i] = 0; + } + for (int i = 0; i < m_warp_count * m_warp_size; i++) m_thread[i] = NULL; + + // get threads for a cta + for (unsigned i = 0; i < m_kernel->threads_per_cta(); i++) { + ptx_sim_init_thread(*m_kernel, &m_thread[i], 0, i, + m_kernel->threads_per_cta() - i, + m_kernel->threads_per_cta(), this, 0, i / m_warp_size, + (gpgpu_t *)m_gpu, true); + assert(m_thread[i] != NULL && !m_thread[i]->is_done()); + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/thread_%d_0_reg.txt", i); + if (m_gpu->gpgpu_ctx->func_sim->cp_cta_resume == 1) + m_thread[i]->resume_reg_thread(fname, symtab); + ctaLiveThreads++; + } + + for (int k = 0; k < m_warp_count; k++) createWarp(k); +} + +void functionalCoreSim::createWarp(unsigned warpId) { + simt_mask_t initialMask; + unsigned liveThreadsCount = 0; + initialMask.set(); + for (int i = warpId * m_warp_size; i < warpId * m_warp_size + m_warp_size; + i++) { + if (m_thread[i] == NULL) + initialMask.reset(i - warpId * m_warp_size); + else + liveThreadsCount++; + } + + assert(m_thread[warpId * m_warp_size] != NULL); + m_simt_stack[warpId]->launch(m_thread[warpId * m_warp_size]->get_pc(), + initialMask); + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/warp_%d_0_simt.txt", warpId); + + if (m_gpu->gpgpu_ctx->func_sim->cp_cta_resume == 1) { + unsigned pc, rpc; + m_simt_stack[warpId]->resume(fname); + m_simt_stack[warpId]->get_pdom_stack_top_info(&pc, &rpc); + for (int i = warpId * m_warp_size; i < warpId * m_warp_size + m_warp_size; + i++) { + m_thread[i]->set_npc(pc); + m_thread[i]->update_pc(); + } + } + m_liveThreadCount[warpId] = liveThreadsCount; +} + +void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) { + m_gpu->gpgpu_ctx->func_sim->cp_count = m_gpu->checkpoint_insn_Y; + m_gpu->gpgpu_ctx->func_sim->cp_cta_resume = m_gpu->checkpoint_CTA_t; + initializeCTA(ctaid_cp); + + int count = 0; + while (true) { + bool someOneLive = false; + bool allAtBarrier = true; + for (unsigned i = 0; i < m_warp_count; i++) { + executeWarp(i, allAtBarrier, someOneLive); + count++; } - - - if(cp_op==1) - { - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/global_mem_%d.txt", kernel.get_uid() ); - g_checkpoint->store_global_mem(gpgpu_ctx->the_gpgpusim->g_the_gpu->get_global_memory(), f1name , (char *)"%08x"); - } - - - - - //registering this kernel as done - - //openCL kernel simulation calls don't register the kernel so we don't register its exit - if(!openCL) { - //extern stream_manager *g_stream_manager; - gpgpu_ctx->the_gpgpusim->g_stream_manager->register_finished_kernel(kernel.get_uid()); - } - - //******PRINTING******* - printf( "GPGPU-Sim: Done functional simulation (%u instructions simulated).\n", g_ptx_sim_num_insn ); - if ( gpgpu_ptx_instruction_classification ) { - StatDisp( g_inst_classification_stat[g_ptx_kernel_count]); - StatDisp ( g_inst_op_classification_stat[g_ptx_kernel_count]); - } - - //time_t variables used to calculate the total simulation time - //the start time of simulation is hold by the global variable g_simulation_starttime - //g_simulation_starttime is initilized by gpgpu_ptx_sim_init_perf() in gpgpusim_entrypoint.cc upon starting gpgpu-sim - time_t end_time, elapsed_time, days, hrs, minutes, sec; - end_time = time((time_t *)NULL); - elapsed_time = MAX(end_time - gpgpu_ctx->the_gpgpusim->g_simulation_starttime, 1); - - - //calculating and printing simulation time in terms of days, hours, minutes and seconds - days = elapsed_time/(3600*24); - hrs = elapsed_time/3600 - 24*days; - minutes = elapsed_time/60 - 60*(hrs + 24*days); - sec = elapsed_time - 60*(minutes + 60*(hrs + 24*days)); - - fflush(stderr); - printf("\n\ngpgpu_simulation_time = %u days, %u hrs, %u min, %u sec (%u sec)\n", - (unsigned)days, (unsigned)hrs, (unsigned)minutes, (unsigned)sec, (unsigned)elapsed_time ); - printf("gpgpu_simulation_rate = %u (inst/sec)\n", (unsigned)(g_ptx_sim_num_insn / elapsed_time) ); - fflush(stdout); -} - -void functionalCoreSim::initializeCTA(unsigned ctaid_cp) -{ - int ctaLiveThreads=0; - symbol_table * symtab= m_kernel->entry()->get_symtab(); - - for(int i=0; i< m_warp_count; i++){ - m_warpAtBarrier[i]=false; - m_liveThreadCount[i]=0; + if (inst_count > 0 && count > inst_count && + (m_kernel->get_uid() == m_gpu->checkpoint_kernel) && + (ctaid_cp >= m_gpu->checkpoint_CTA) && + (ctaid_cp < m_gpu->checkpoint_CTA_t) && m_gpu->checkpoint_option == 1) { + someOneLive = false; + break; } - for(int i=0; i< m_warp_count*m_warp_size;i++) - m_thread[i]=NULL; - - //get threads for a cta - for(unsigned i=0; ithreads_per_cta();i++) { - ptx_sim_init_thread(*m_kernel,&m_thread[i],0,i,m_kernel->threads_per_cta()-i,m_kernel->threads_per_cta(),this,0,i/m_warp_size,(gpgpu_t*)m_gpu, true); - assert(m_thread[i]!=NULL && !m_thread[i]->is_done()); - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_0_reg.txt",i ); - if(m_gpu->gpgpu_ctx->func_sim->cp_cta_resume==1) - m_thread[i]->resume_reg_thread(fname,symtab); - ctaLiveThreads++; + if (!someOneLive) break; + if (allAtBarrier) { + for (unsigned i = 0; i < m_warp_count; i++) m_warpAtBarrier[i] = false; } - - for(int k=0;klaunch(m_thread[warpId*m_warp_size]->get_pc(),initialMask); - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/warp_%d_0_simt.txt",warpId ); - - if(m_gpu->gpgpu_ctx->func_sim->cp_cta_resume==1) - { - unsigned pc,rpc; - m_simt_stack[warpId]->resume(fname); - m_simt_stack[warpId]->get_pdom_stack_top_info(&pc,&rpc); - for(int i=warpId*m_warp_size; iset_npc(pc); - m_thread[i]->update_pc(); - } - - } - m_liveThreadCount[warpId]= liveThreadsCount; -} - -void functionalCoreSim::execute(int inst_count, unsigned ctaid_cp) - { - m_gpu->gpgpu_ctx->func_sim->cp_count= m_gpu->checkpoint_insn_Y; - m_gpu->gpgpu_ctx->func_sim->cp_cta_resume= m_gpu->checkpoint_CTA_t; - initializeCTA(ctaid_cp); - - int count=0; - while(true){ - bool someOneLive= false; - bool allAtBarrier = true; - for(unsigned i=0;i0 && count>inst_count && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t) && m_gpu->checkpoint_option==1) - { - someOneLive=false; - break; - } - if(!someOneLive) break; - if(allAtBarrier){ - for(unsigned i=0;iget_next_cta_id_single(); + if (m_gpu->checkpoint_option == 1 && + (m_kernel->get_uid() == m_gpu->checkpoint_kernel) && + (ctaid_cp >= m_gpu->checkpoint_CTA) && + (ctaid_cp < m_gpu->checkpoint_CTA_t)) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/shared_mem_%d.txt", ctaid - 1); + g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname, + (char *)"%08x"); + for (int i = 0; i < 32 * m_warp_count; i++) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/thread_%d_%d_reg.txt", i, + ctaid - 1); + m_thread[i]->print_reg_thread(fname); + char f1name[2048]; + snprintf(f1name, 2048, "checkpoint_files/local_mem_thread_%d_%d_reg.txt", + i, ctaid - 1); + g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name, + (char *)"%08x"); + m_thread[i]->set_done(); + m_thread[i]->exitCore(); + m_thread[i]->registerExit(); } - checkpoint *g_checkpoint; - g_checkpoint = new checkpoint(); - - ptx_reg_t regval; - regval.u64= 123; - - unsigned ctaid =m_kernel->get_next_cta_id_single(); - if(m_gpu->checkpoint_option==1 && (m_kernel->get_uid()==m_gpu->checkpoint_kernel) && (ctaid_cp>=m_gpu->checkpoint_CTA) && (ctaid_cpcheckpoint_CTA_t)) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/shared_mem_%d.txt",ctaid-1 ); - g_checkpoint->store_global_mem(m_thread[0]->m_shared_mem, fname , (char *)"%08x"); - for(int i=0; i<32*m_warp_count;i++) - { - char fname[2048]; - snprintf(fname,2048,"checkpoint_files/thread_%d_%d_reg.txt",i,ctaid-1 ); - m_thread[i]->print_reg_thread(fname); - char f1name[2048]; - snprintf(f1name,2048,"checkpoint_files/local_mem_thread_%d_%d_reg.txt",i,ctaid-1 ); - g_checkpoint->store_global_mem(m_thread[i]->m_local_mem, f1name , (char *)"%08x"); - m_thread[i]->set_done(); - m_thread[i]->exitCore(); - m_thread[i]->registerExit(); - } - - for(int i=0;iprint_checkpoint(fp); - fclose(fp); - } - } - + for (int i = 0; i < m_warp_count; i++) { + char fname[2048]; + snprintf(fname, 2048, "checkpoint_files/warp_%d_%d_simt.txt", i, + ctaid - 1); + FILE *fp = fopen(fname, "w"); + assert(fp != NULL); + m_simt_stack[i]->print_checkpoint(fp); + fclose(fp); + } + } } -void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, bool & someOneLive) -{ - if(!m_warpAtBarrier[i] && m_liveThreadCount[i]!=0){ - warp_inst_t inst =getExecuteWarp(i); - execute_warp_inst_t(inst,i); - if(inst.isatomic()) inst.do_atomic(true); - if(inst.op==BARRIER_OP || inst.op==MEMORY_BARRIER_OP ) m_warpAtBarrier[i]=true; - updateSIMTStack( i, &inst ); - } - if(m_liveThreadCount[i]>0) someOneLive=true; - if(!m_warpAtBarrier[i]&& m_liveThreadCount[i]>0) allAtBarrier = false; +void functionalCoreSim::executeWarp(unsigned i, bool &allAtBarrier, + bool &someOneLive) { + if (!m_warpAtBarrier[i] && m_liveThreadCount[i] != 0) { + warp_inst_t inst = getExecuteWarp(i); + execute_warp_inst_t(inst, i); + if (inst.isatomic()) inst.do_atomic(true); + if (inst.op == BARRIER_OP || inst.op == MEMORY_BARRIER_OP) + m_warpAtBarrier[i] = true; + updateSIMTStack(i, &inst); + } + if (m_liveThreadCount[i] > 0) someOneLive = true; + if (!m_warpAtBarrier[i] && m_liveThreadCount[i] > 0) allAtBarrier = false; } -unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) -{ - // this function assumes that the kernel fits inside a single PTX file - // function_info *pFunc = g_func_info; // assume that the current kernel is the one in query - const ptx_instruction *pInsn = pc_to_instruction(pc); - unsigned ptx_line_number = pInsn->source_line(); +unsigned gpgpu_context::translate_pc_to_ptxlineno(unsigned pc) { + // this function assumes that the kernel fits inside a single PTX file + // function_info *pFunc = g_func_info; // assume that the current kernel is + // the one in query + const ptx_instruction *pInsn = pc_to_instruction(pc); + unsigned ptx_line_number = pInsn->source_line(); - return ptx_line_number; + return ptx_line_number; } // ptxinfo parser -extern std::map get_duplicate(); +extern std::map get_duplicate(); static char *g_ptxinfo_kname = NULL; static struct gpgpu_ptx_sim_info g_ptxinfo; -static std::map g_duplicate; +static std::map g_duplicate; static const char *g_last_dup_type; -const char *get_ptxinfo_kname() -{ - return g_ptxinfo_kname; -} +const char *get_ptxinfo_kname() { return g_ptxinfo_kname; } -void print_ptxinfo() -{ - if(! get_ptxinfo_kname()){ - printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", - g_ptxinfo.gmem, - g_ptxinfo.cmem); - } - if(get_ptxinfo_kname()){ - printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", - get_ptxinfo_kname(), - g_ptxinfo.regs, - g_ptxinfo.lmem, - g_ptxinfo.smem, - g_ptxinfo.cmem ); - } +void print_ptxinfo() { + if (!get_ptxinfo_kname()) { + printf("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", g_ptxinfo.gmem, + g_ptxinfo.cmem); + } + if (get_ptxinfo_kname()) { + printf( + "GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + get_ptxinfo_kname(), g_ptxinfo.regs, g_ptxinfo.lmem, g_ptxinfo.smem, + g_ptxinfo.cmem); + } } - -struct gpgpu_ptx_sim_info get_ptxinfo() -{ - return g_ptxinfo; +struct gpgpu_ptx_sim_info get_ptxinfo() { + return g_ptxinfo; } -std::map get_duplicate() -{ - return g_duplicate; -} +std::map get_duplicate() { return g_duplicate; } -void ptxinfo_linenum( unsigned linenum ) -{ - g_duplicate[linenum] = g_last_dup_type; +void ptxinfo_linenum(unsigned linenum) { + g_duplicate[linenum] = g_last_dup_type; } -void ptxinfo_dup_type( const char *dup_type ) -{ - g_last_dup_type = dup_type; -} +void ptxinfo_dup_type(const char *dup_type) { g_last_dup_type = dup_type; } -void ptxinfo_function(const char *fname ) -{ - clear_ptxinfo(); - g_ptxinfo_kname = strdup(fname); +void ptxinfo_function(const char *fname) { + clear_ptxinfo(); + g_ptxinfo_kname = strdup(fname); } -void ptxinfo_regs( unsigned nregs ) -{ - g_ptxinfo.regs=nregs; -} +void ptxinfo_regs(unsigned nregs) { g_ptxinfo.regs = nregs; } -void ptxinfo_lmem( unsigned declared, unsigned system ) -{ - g_ptxinfo.lmem=declared+system; +void ptxinfo_lmem(unsigned declared, unsigned system) { + g_ptxinfo.lmem = declared + system; } -void ptxinfo_gmem( unsigned declared, unsigned system ) -{ - g_ptxinfo.gmem=declared+system; +void ptxinfo_gmem(unsigned declared, unsigned system) { + g_ptxinfo.gmem = declared + system; } -void ptxinfo_smem( unsigned declared, unsigned system ) -{ - g_ptxinfo.smem=declared+system; +void ptxinfo_smem(unsigned declared, unsigned system) { + g_ptxinfo.smem = declared + system; } -void ptxinfo_cmem( unsigned nbytes, unsigned bank ) -{ - g_ptxinfo.cmem+=nbytes; -} +void ptxinfo_cmem(unsigned nbytes, unsigned bank) { g_ptxinfo.cmem += nbytes; } -void clear_ptxinfo() -{ - free(g_ptxinfo_kname); - g_ptxinfo_kname=NULL; - g_ptxinfo.regs=0; - g_ptxinfo.lmem=0; - g_ptxinfo.smem=0; - g_ptxinfo.cmem=0; - g_ptxinfo.gmem=0; - g_ptxinfo.ptx_version=0; - g_ptxinfo.sm_target=0; +void clear_ptxinfo() { + free(g_ptxinfo_kname); + g_ptxinfo_kname = NULL; + g_ptxinfo.regs = 0; + g_ptxinfo.lmem = 0; + g_ptxinfo.smem = 0; + g_ptxinfo.cmem = 0; + g_ptxinfo.gmem = 0; + g_ptxinfo.ptx_version = 0; + g_ptxinfo.sm_target = 0; } +void ptxinfo_opencl_addinfo(std::map &kernels) { + if (!g_ptxinfo_kname) { + printf("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", g_ptxinfo.gmem, + g_ptxinfo.cmem); + clear_ptxinfo(); + return; + } -void ptxinfo_opencl_addinfo( std::map &kernels ) -{ - - if(! g_ptxinfo_kname) { - printf ("GPGPU-Sim PTX: Binary info : gmem=%u, cmem=%u\n", - g_ptxinfo.gmem, - g_ptxinfo.cmem); - clear_ptxinfo(); - return; - } - - if( !strcmp("__cuda_dummy_entry__",g_ptxinfo_kname) ) { - // this string produced by ptxas for empty ptx files (e.g., bandwidth test) - clear_ptxinfo(); - return; - } - std::map::iterator k=kernels.find(g_ptxinfo_kname); - if( k==kernels.end() ) { - printf ("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", g_ptxinfo_kname ); - abort(); - } else { - printf ("GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", - g_ptxinfo_kname, - g_ptxinfo.regs, - g_ptxinfo.lmem, - g_ptxinfo.smem, - g_ptxinfo.cmem ); - function_info *finfo = k->second; - assert(finfo!=NULL); - finfo->set_kernel_info( g_ptxinfo ); - } - clear_ptxinfo(); -} - -struct rec_pts cuda_sim::find_reconvergence_points( function_info *finfo ) -{ - rec_pts tmp; - std::map::iterator r=g_rpts.find(finfo); - - if( r==g_rpts.end() ) { - int num_recon = finfo->get_num_reconvergence_pairs(); - - gpgpu_recon_t *kernel_recon_points = (struct gpgpu_recon_t*) calloc(num_recon, sizeof(struct gpgpu_recon_t)); - finfo->get_reconvergence_pairs(kernel_recon_points); - printf("GPGPU-Sim PTX: reconvergence points for %s...\n", finfo->get_name().c_str() ); - for (int i=0;iprint_insn(); - printf("\n"); - printf("GPGPU-Sim PTX: immediate post dominator @ " ); - if( kernel_recon_points[i].target_inst ) - kernel_recon_points[i].target_inst->print_insn(); - printf("\n"); - } - printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", finfo->get_name().c_str() ); - - tmp.s_kernel_recon_points = kernel_recon_points; - tmp.s_num_recon = num_recon; - g_rpts[finfo] = tmp; - } else { - tmp = r->second; - } - return tmp; -} - -address_type get_return_pc( void *thd ) -{ - // function call return - ptx_thread_info *the_thread = (ptx_thread_info*)thd; - assert( the_thread != NULL ); - return the_thread->get_return_PC(); -} - -address_type cuda_sim::get_converge_point( address_type pc ) -{ - // the branch could encode the reconvergence point and/or a bit that indicates the - // reconvergence point is the return PC on the call stack in the case the branch has - // no immediate postdominator in the function (i.e., due to multiple return points). - - std::map::iterator f=g_pc_to_finfo.find(pc); - assert( f != g_pc_to_finfo.end() ); - function_info *finfo = f->second; - rec_pts tmp = find_reconvergence_points(finfo); - - int i=0; - for (; i < tmp.s_num_recon; ++i) { - if (tmp.s_kernel_recon_points[i].source_pc == pc) { - if( tmp.s_kernel_recon_points[i].target_pc == (unsigned) -2 ) { - return RECONVERGE_RETURN_PC; - } else { - return tmp.s_kernel_recon_points[i].target_pc; - } + if (!strcmp("__cuda_dummy_entry__", g_ptxinfo_kname)) { + // this string produced by ptxas for empty ptx files (e.g., bandwidth test) + clear_ptxinfo(); + return; + } + std::map::iterator k = + kernels.find(g_ptxinfo_kname); + if (k == kernels.end()) { + printf("GPGPU-Sim PTX: ERROR ** implementation for '%s' not found.\n", + g_ptxinfo_kname); + abort(); + } else { + printf( + "GPGPU-Sim PTX: Kernel \'%s\' : regs=%u, lmem=%u, smem=%u, cmem=%u\n", + g_ptxinfo_kname, g_ptxinfo.regs, g_ptxinfo.lmem, g_ptxinfo.smem, + g_ptxinfo.cmem); + function_info *finfo = k->second; + assert(finfo != NULL); + finfo->set_kernel_info(g_ptxinfo); + } + clear_ptxinfo(); +} + +struct rec_pts cuda_sim::find_reconvergence_points(function_info *finfo) { + rec_pts tmp; + std::map::iterator r = g_rpts.find(finfo); + + if (r == g_rpts.end()) { + int num_recon = finfo->get_num_reconvergence_pairs(); + + gpgpu_recon_t *kernel_recon_points = + (struct gpgpu_recon_t *)calloc(num_recon, sizeof(struct gpgpu_recon_t)); + finfo->get_reconvergence_pairs(kernel_recon_points); + printf("GPGPU-Sim PTX: reconvergence points for %s...\n", + finfo->get_name().c_str()); + for (int i = 0; i < num_recon; i++) { + printf("GPGPU-Sim PTX: %2u (potential) branch divergence @ ", i + 1); + kernel_recon_points[i].source_inst->print_insn(); + printf("\n"); + printf("GPGPU-Sim PTX: immediate post dominator @ "); + if (kernel_recon_points[i].target_inst) + kernel_recon_points[i].target_inst->print_insn(); + printf("\n"); + } + printf("GPGPU-Sim PTX: ... end of reconvergence points for %s\n", + finfo->get_name().c_str()); + + tmp.s_kernel_recon_points = kernel_recon_points; + tmp.s_num_recon = num_recon; + g_rpts[finfo] = tmp; + } else { + tmp = r->second; + } + return tmp; +} + +address_type get_return_pc(void *thd) { + // function call return + ptx_thread_info *the_thread = (ptx_thread_info *)thd; + assert(the_thread != NULL); + return the_thread->get_return_PC(); +} + +address_type cuda_sim::get_converge_point(address_type pc) { + // the branch could encode the reconvergence point and/or a bit that indicates + // the reconvergence point is the return PC on the call stack in the case the + // branch has no immediate postdominator in the function (i.e., due to + // multiple return points). + + std::map::iterator f = g_pc_to_finfo.find(pc); + assert(f != g_pc_to_finfo.end()); + function_info *finfo = f->second; + rec_pts tmp = find_reconvergence_points(finfo); + + int i = 0; + for (; i < tmp.s_num_recon; ++i) { + if (tmp.s_kernel_recon_points[i].source_pc == pc) { + if (tmp.s_kernel_recon_points[i].target_pc == (unsigned)-2) { + return RECONVERGE_RETURN_PC; + } else { + return tmp.s_kernel_recon_points[i].target_pc; } - } - return NO_BRANCH_DIVERGENCE; + } + } + return NO_BRANCH_DIVERGENCE; } -void functionalCoreSim::warp_exit( unsigned warp_id ) -{ - for(int i=0;im_cta_info->register_deleted_thread(m_thread[i]); - delete m_thread[i]; - } +void functionalCoreSim::warp_exit(unsigned warp_id) { + for (int i = 0; i < m_warp_count * m_warp_size; i++) { + if (m_thread[i] != NULL) { + m_thread[i]->m_cta_info->register_deleted_thread(m_thread[i]); + delete m_thread[i]; } + } } diff --git a/src/cuda-sim/cuda-sim.h b/src/cuda-sim/cuda-sim.h index 1be3d19..21e1ca0 100644 --- a/src/cuda-sim/cuda-sim.h +++ b/src/cuda-sim/cuda-sim.h @@ -7,34 +7,35 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef CUDASIM_H_INCLUDED #define CUDASIM_H_INCLUDED -#include "../abstract_hardware_model.h" -#include"../gpgpu-sim/shader.h" #include #include -#include #include -#include"ptx_sim.h" +#include +#include "../abstract_hardware_model.h" +#include "../gpgpu-sim/shader.h" +#include "ptx_sim.h" class gpgpu_context; class memory_space; @@ -44,69 +45,65 @@ class symbol_table; extern const char *g_gpgpusim_version_string; extern int g_debug_execution; -extern void print_splash(); +extern void print_splash(); -extern void ptxinfo_opencl_addinfo( std::map &kernels ); -unsigned ptx_sim_init_thread( kernel_info_t &kernel, - class ptx_thread_info** thread_info, - int sid, - unsigned tid, - unsigned threads_left, - unsigned num_threads, - class core_t *core, - unsigned hw_cta_id, - unsigned hw_warp_id, - gpgpu_t *gpu, - bool functionalSimulationMode = false); -const struct gpgpu_ptx_sim_info* ptx_sim_kernel_info(const class function_info *kernel); +extern void ptxinfo_opencl_addinfo( + std::map &kernels); +unsigned ptx_sim_init_thread(kernel_info_t &kernel, + class ptx_thread_info **thread_info, int sid, + unsigned tid, unsigned threads_left, + unsigned num_threads, class core_t *core, + unsigned hw_cta_id, unsigned hw_warp_id, + gpgpu_t *gpu, + bool functionalSimulationMode = false); +const struct gpgpu_ptx_sim_info *ptx_sim_kernel_info( + const class function_info *kernel); /*! - * This class functionally executes a kernel. It uses the basic data structures and procedures in core_t + * This class functionally executes a kernel. It uses the basic data structures + * and procedures in core_t */ -class functionalCoreSim: public core_t -{ -public: - functionalCoreSim(kernel_info_t * kernel, gpgpu_sim *g, unsigned warp_size) - : core_t( g, kernel, warp_size, kernel->threads_per_cta() ) - { - m_warpAtBarrier = new bool [m_warp_count]; - m_liveThreadCount = new unsigned [m_warp_count]; - } - virtual ~functionalCoreSim(){ - warp_exit(0); - delete[] m_liveThreadCount; - delete[] m_warpAtBarrier; - } - //! executes all warps till completion - void execute(int inst_count, unsigned ctaid_cp); - virtual void warp_exit( unsigned warp_id ); - virtual bool warp_waiting_at_barrier( unsigned warp_id ) const - { - return (m_warpAtBarrier[warp_id] || !(m_liveThreadCount[warp_id]>0)); - } - -private: - void executeWarp(unsigned, bool &, bool &); - //initializes threads in the CTA block which we are executing - void initializeCTA(unsigned ctaid_cp); - virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, unsigned tid) - { - if(m_thread[tid]==NULL || m_thread[tid]->is_done()){ - m_liveThreadCount[tid/m_warp_size]--; - } +class functionalCoreSim : public core_t { + public: + functionalCoreSim(kernel_info_t *kernel, gpgpu_sim *g, unsigned warp_size) + : core_t(g, kernel, warp_size, kernel->threads_per_cta()) { + m_warpAtBarrier = new bool[m_warp_count]; + m_liveThreadCount = new unsigned[m_warp_count]; + } + virtual ~functionalCoreSim() { + warp_exit(0); + delete[] m_liveThreadCount; + delete[] m_warpAtBarrier; + } + //! executes all warps till completion + void execute(int inst_count, unsigned ctaid_cp); + virtual void warp_exit(unsigned warp_id); + virtual bool warp_waiting_at_barrier(unsigned warp_id) const { + return (m_warpAtBarrier[warp_id] || !(m_liveThreadCount[warp_id] > 0)); + } + + private: + void executeWarp(unsigned, bool &, bool &); + // initializes threads in the CTA block which we are executing + void initializeCTA(unsigned ctaid_cp); + virtual void checkExecutionStatusAndUpdate(warp_inst_t &inst, unsigned t, + unsigned tid) { + if (m_thread[tid] == NULL || m_thread[tid]->is_done()) { + m_liveThreadCount[tid / m_warp_size]--; } - - // lunches the stack and set the threads count - void createWarp(unsigned warpId); - - //each warp live thread count and barrier indicator - unsigned * m_liveThreadCount; - bool* m_warpAtBarrier; + } + + // lunches the stack and set the threads count + void createWarp(unsigned warpId); + + // each warp live thread count and barrier indicator + unsigned *m_liveThreadCount; + bool *m_warpAtBarrier; }; #define RECONVERGE_RETURN_PC ((address_type)-2) #define NO_BRANCH_DIVERGENCE ((address_type)-1) -address_type get_return_pc( void *thd ); +address_type get_return_pc(void *thd); const char *get_ptxinfo_kname(); void print_ptxinfo(); void clear_ptxinfo(); @@ -114,89 +111,98 @@ struct gpgpu_ptx_sim_info get_ptxinfo(); class gpgpu_recon_t; struct rec_pts { - gpgpu_recon_t *s_kernel_recon_points; - int s_num_recon; + gpgpu_recon_t *s_kernel_recon_points; + int s_num_recon; }; - class cuda_sim { - public: - cuda_sim( gpgpu_context* ctx ) { - g_ptx_sim_num_insn = 0; - g_ptx_kernel_count = -1; // used for classification stat collection purposes - gpgpu_param_num_shaders = 0; - g_cuda_launch_blocking = false; - g_inst_classification_stat = NULL; - g_inst_op_classification_stat= NULL; - g_assemble_code_next_pc=0; - g_debug_thread_uid = 0; - g_override_embedded_ptx = false; - ptx_tex_regs = NULL; - g_ptx_thread_info_delete_count=0; - g_ptx_thread_info_uid_next=1; - g_debug_pc = 0xBEEF1518; - gpgpu_ctx = ctx; - } - //global variables - char *opcode_latency_int; - char *opcode_latency_fp; - char *opcode_latency_dp; - char *opcode_latency_sfu; - char *opcode_latency_tensor; - char *opcode_initiation_int; - char *opcode_initiation_fp; - char *opcode_initiation_dp; - char *opcode_initiation_sfu; - char *opcode_initiation_tensor; - int cp_count; - int cp_cta_resume; - int g_ptxinfo_error_detected; - unsigned g_ptx_sim_num_insn; - char *cdp_latency_str; - int g_ptx_kernel_count; // used for classification stat collection purposes - std::map g_global_name_lookup; // indexed by hostVar - std::map g_const_name_lookup; // indexed by hostVar - int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no notion of a clock cycle) - unsigned gpgpu_param_num_shaders; - class std::map g_rpts; - bool g_cuda_launch_blocking; - void ** g_inst_classification_stat; - void ** g_inst_op_classification_stat; - std::set g_globals; - std::set g_constants; - std::map g_pc_to_finfo; - int gpgpu_ptx_instruction_classification; - unsigned cdp_latency[5]; - unsigned g_assemble_code_next_pc; - int g_debug_thread_uid; - bool g_override_embedded_ptx; - std::set g_ptx_cta_info_sm_idx_used; - ptx_reg_t* ptx_tex_regs; - unsigned g_ptx_thread_info_delete_count; - unsigned g_ptx_thread_info_uid_next; - addr_t g_debug_pc; - // backward pointer - class gpgpu_context* gpgpu_ctx; - //global functions - void ptx_opcocde_latency_options (option_parser_t opp); - void gpgpu_cuda_ptx_sim_main_func( kernel_info_t &kernel, bool openCL = false ); - int gpgpu_opencl_ptx_sim_main_func( kernel_info_t *grid ); - void init_inst_classification_stat(); - kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, - gpgpu_ptx_sim_arg_list_t args, - struct dim3 gridDim, - struct dim3 blockDim, - gpgpu_t *gpu ); - void gpgpu_ptx_sim_register_global_variable(void *hostVar, const char *deviceName, size_t size ); - void gpgpu_ptx_sim_register_const_variable(void*, const char *deviceName, size_t size ); - void read_sim_environment_variables(); - void set_param_gpgpu_num_shaders(int num_shaders); - struct rec_pts find_reconvergence_points( function_info *finfo ); - address_type get_converge_point( address_type pc ); - void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, size_t count, size_t offset, int to, gpgpu_t *gpu ); - void ptx_print_insn( address_type pc, FILE *fp ); - std::string ptx_get_insn_str( address_type pc ); - template bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc); + public: + cuda_sim(gpgpu_context *ctx) { + g_ptx_sim_num_insn = 0; + g_ptx_kernel_count = + -1; // used for classification stat collection purposes + gpgpu_param_num_shaders = 0; + g_cuda_launch_blocking = false; + g_inst_classification_stat = NULL; + g_inst_op_classification_stat = NULL; + g_assemble_code_next_pc = 0; + g_debug_thread_uid = 0; + g_override_embedded_ptx = false; + ptx_tex_regs = NULL; + g_ptx_thread_info_delete_count = 0; + g_ptx_thread_info_uid_next = 1; + g_debug_pc = 0xBEEF1518; + gpgpu_ctx = ctx; + } + // global variables + char *opcode_latency_int; + char *opcode_latency_fp; + char *opcode_latency_dp; + char *opcode_latency_sfu; + char *opcode_latency_tensor; + char *opcode_initiation_int; + char *opcode_initiation_fp; + char *opcode_initiation_dp; + char *opcode_initiation_sfu; + char *opcode_initiation_tensor; + int cp_count; + int cp_cta_resume; + int g_ptxinfo_error_detected; + unsigned g_ptx_sim_num_insn; + char *cdp_latency_str; + int g_ptx_kernel_count; // used for classification stat collection purposes + std::map + g_global_name_lookup; // indexed by hostVar + std::map + g_const_name_lookup; // indexed by hostVar + int g_ptx_sim_mode; // if non-zero run functional simulation only (i.e., no + // notion of a clock cycle) + unsigned gpgpu_param_num_shaders; + class std::map g_rpts; + bool g_cuda_launch_blocking; + void **g_inst_classification_stat; + void **g_inst_op_classification_stat; + std::set g_globals; + std::set g_constants; + std::map g_pc_to_finfo; + int gpgpu_ptx_instruction_classification; + unsigned cdp_latency[5]; + unsigned g_assemble_code_next_pc; + int g_debug_thread_uid; + bool g_override_embedded_ptx; + std::set g_ptx_cta_info_sm_idx_used; + ptx_reg_t *ptx_tex_regs; + unsigned g_ptx_thread_info_delete_count; + unsigned g_ptx_thread_info_uid_next; + addr_t g_debug_pc; + // backward pointer + class gpgpu_context *gpgpu_ctx; + // global functions + void ptx_opcocde_latency_options(option_parser_t opp); + void gpgpu_cuda_ptx_sim_main_func(kernel_info_t &kernel, bool openCL = false); + int gpgpu_opencl_ptx_sim_main_func(kernel_info_t *grid); + void init_inst_classification_stat(); + kernel_info_t *gpgpu_opencl_ptx_sim_init_grid(class function_info *entry, + gpgpu_ptx_sim_arg_list_t args, + struct dim3 gridDim, + struct dim3 blockDim, + gpgpu_t *gpu); + void gpgpu_ptx_sim_register_global_variable(void *hostVar, + const char *deviceName, + size_t size); + void gpgpu_ptx_sim_register_const_variable(void *, const char *deviceName, + size_t size); + void read_sim_environment_variables(); + void set_param_gpgpu_num_shaders(int num_shaders); + struct rec_pts find_reconvergence_points(function_info *finfo); + address_type get_converge_point(address_type pc); + void gpgpu_ptx_sim_memcpy_symbol(const char *hostVar, const void *src, + size_t count, size_t offset, int to, + gpgpu_t *gpu); + void ptx_print_insn(address_type pc, FILE *fp); + std::string ptx_get_insn_str(address_type pc); + template + bool ptx_debug_exec_dump_cond(int thd_uid, addr_t pc); }; #endif diff --git a/src/cuda-sim/cuda_device_printf.cc b/src/cuda-sim/cuda_device_printf.cc index 9ac0727..1da6b85 100644 --- a/src/cuda-sim/cuda_device_printf.cc +++ b/src/cuda-sim/cuda_device_printf.cc @@ -7,108 +7,112 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "cuda_device_printf.h" #include "ptx_ir.h" -void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr); +void decode_space(memory_space_t &space, ptx_thread_info *thread, + const operand_info &op, memory_space *&mem, addr_t &addr); -void my_cuda_printf(const char *fmtstr,const char *arg_list) -{ - FILE *fp = stdout; - unsigned i=0,j=0; - unsigned arg_offset=0; - char buf[64]; - bool in_fmt=false; - while( fmtstr[i] ) { - char c = fmtstr[i++]; - if( !in_fmt ) { - if( c != '%' ) { - fprintf(fp,"%c",c); - } else { - in_fmt=true; - buf[0] = c; - j=1; - } +void my_cuda_printf(const char *fmtstr, const char *arg_list) { + FILE *fp = stdout; + unsigned i = 0, j = 0; + unsigned arg_offset = 0; + char buf[64]; + bool in_fmt = false; + while (fmtstr[i]) { + char c = fmtstr[i++]; + if (!in_fmt) { + if (c != '%') { + fprintf(fp, "%c", c); } else { - if(!( c == 'u' || c == 'f' || c == 'd' )) { - printf("GPGPU-Sim PTX: ERROR ** printf parsing support is limited to %%u, %%f, %%d at present"); - abort(); - } - buf[j] = c; - buf[j+1] = 0; - void* ptr = (void*)&arg_list[arg_offset]; - //unsigned long long value = ((unsigned long long*)arg_list)[arg_offset]; - if( c == 'u' || c == 'd' ) { - fprintf(fp,buf,*((unsigned long long*)ptr)); - } else if( c == 'f' ) { - double tmp = *((double*)ptr); - fprintf(fp,buf,tmp); - } - arg_offset++; - in_fmt=false; + in_fmt = true; + buf[0] = c; + j = 1; } - } + } else { + if (!(c == 'u' || c == 'f' || c == 'd')) { + printf( + "GPGPU-Sim PTX: ERROR ** printf parsing support is limited to %%u, " + "%%f, %%d at present"); + abort(); + } + buf[j] = c; + buf[j + 1] = 0; + void *ptr = (void *)&arg_list[arg_offset]; + // unsigned long long value = ((unsigned long long*)arg_list)[arg_offset]; + if (c == 'u' || c == 'd') { + fprintf(fp, buf, *((unsigned long long *)ptr)); + } else if (c == 'f') { + double tmp = *((double *)ptr); + fprintf(fp, buf, tmp); + } + arg_offset++; + in_fmt = false; + } + } } -void gpgpusim_cuda_vprintf(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func ) -{ - char *fmtstr = NULL; - char *arg_list = NULL; - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - assert( n_args == 2 ); - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); - const symbol *formal_param = target_func->get_arg(arg); - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - unsigned long long buffer[1024]; - assert(size<1024*sizeof(unsigned long long)); - thread->m_local_mem->read(from_addr,size,buffer); - addr_t addr = (addr_t)buffer[0]; // should be pointer to generic memory location - memory_space *mem=NULL; - memory_space_t space = generic_space; - decode_space(space,thread,actual_param_op,mem,addr); // figure out which space - if( arg == 0 ) { - unsigned len = 0; - char b = 0; - do { // figure out length - mem->read(addr+len,1,&b); - len++; - } while(b); - fmtstr = (char*)malloc(len+64); - for( int i=0; i < len; i++ ) - mem->read(addr+i,1,fmtstr+i); - //mem->read(addr,len,fmtstr); - } else { - unsigned len = thread->get_finfo()->local_mem_framesize(); - arg_list = (char*)malloc(len+64); - for( int i=0; i < len; i++ ) - mem->read(addr+i,1,arg_list+i); - //mem->read(addr,len,arg_list); - } - } - my_cuda_printf(fmtstr,arg_list); - free(fmtstr); - free(arg_list); +void gpgpusim_cuda_vprintf(const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + char *fmtstr = NULL; + char *arg_list = NULL; + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + assert(n_args == 2); + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); + const symbol *formal_param = target_func->get_arg(arg); + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + unsigned long long buffer[1024]; + assert(size < 1024 * sizeof(unsigned long long)); + thread->m_local_mem->read(from_addr, size, buffer); + addr_t addr = + (addr_t)buffer[0]; // should be pointer to generic memory location + memory_space *mem = NULL; + memory_space_t space = generic_space; + decode_space(space, thread, actual_param_op, mem, + addr); // figure out which space + if (arg == 0) { + unsigned len = 0; + char b = 0; + do { // figure out length + mem->read(addr + len, 1, &b); + len++; + } while (b); + fmtstr = (char *)malloc(len + 64); + for (int i = 0; i < len; i++) mem->read(addr + i, 1, fmtstr + i); + // mem->read(addr,len,fmtstr); + } else { + unsigned len = thread->get_finfo()->local_mem_framesize(); + arg_list = (char *)malloc(len + 64); + for (int i = 0; i < len; i++) mem->read(addr + i, 1, arg_list + i); + // mem->read(addr,len,arg_list); + } + } + my_cuda_printf(fmtstr, arg_list); + free(fmtstr); + free(arg_list); } diff --git a/src/cuda-sim/cuda_device_printf.h b/src/cuda-sim/cuda_device_printf.h index 4e9baaa..c4ee75d 100644 --- a/src/cuda-sim/cuda_device_printf.h +++ b/src/cuda-sim/cuda_device_printf.h @@ -7,27 +7,30 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef CUDA_DEVICE_PRINTF_INCLUDED #define CUDA_DEVICE_PRINTF_INCLUDED -void gpgpusim_cuda_vprintf(const class ptx_instruction * pI, class ptx_thread_info * thread, const class function_info * target_func ); +void gpgpusim_cuda_vprintf(const class ptx_instruction* pI, + class ptx_thread_info* thread, + const class function_info* target_func); #endif diff --git a/src/cuda-sim/cuda_device_runtime.cc b/src/cuda-sim/cuda_device_runtime.cc index 4baced5..4a99c1c 100644 --- a/src/cuda-sim/cuda_device_runtime.cc +++ b/src/cuda-sim/cuda_device_runtime.cc @@ -1,297 +1,327 @@ -//Jin: cuda_device_runtime.cc -//Defines CUDA device runtime APIs for CDP support - +// Jin: cuda_device_runtime.cc +// Defines CUDA device runtime APIs for CDP support #include #include - - #if (CUDART_VERSION >= 5000) #define __CUDA_RUNTIME_API_H__ #include #include +#include "../../libcuda/gpgpu_context.h" #include "../gpgpu-sim/gpu-sim.h" -#include "cuda-sim.h" -#include "ptx_ir.h" -#include "../stream_manager.h" #include "../gpgpusim_entrypoint.h" +#include "../stream_manager.h" +#include "cuda-sim.h" #include "cuda_device_runtime.h" -#include "../../libcuda/gpgpu_context.h" +#include "ptx_ir.h" -#define DEV_RUNTIME_REPORT(a) \ - if( g_debug_execution ) { \ - std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ - std::cout.flush(); \ - } - - - -//Handling device runtime api: -//void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 blockDimension, unsigned int sharedMemSize) -void cuda_device_runtime::gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) -{ - DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert( n_args == 4 ); - - function_info * child_kernel_entry; - struct dim3 grid_dim, block_dim; - unsigned int shared_mem; - - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# - const symbol *formal_param = target_func->get_arg(arg); //cudaGetParameterBufferV2_param_# - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - - if(arg == 0) {//function_info* for the child kernel - unsigned long long buf; - assert(size == sizeof(function_info *)); - thread->m_local_mem->read(from_addr, size, &buf); - child_kernel_entry = (function_info *)buf; - assert(child_kernel_entry); - DEV_RUNTIME_REPORT("child kernel name " << child_kernel_entry->get_name()); - } - else if(arg == 1) { //dim3 grid_dim for the child kernel - assert(size == sizeof(struct dim3)); - thread->m_local_mem->read(from_addr, size, & grid_dim); - DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " << grid_dim.z << ")"); - } - else if(arg == 2) { //dim3 block_dim for the child kernel - assert(size == sizeof(struct dim3)); - thread->m_local_mem->read(from_addr, size, & block_dim); - DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " << block_dim.z << ")"); - } - else if(arg == 3) { //unsigned int shared_mem - assert(size == sizeof(unsigned int)); - thread->m_local_mem->read(from_addr, size, & shared_mem); - DEV_RUNTIME_REPORT("shared memory " << shared_mem); - } +#define DEV_RUNTIME_REPORT(a) \ + if (g_debug_execution) { \ + std::cout << __FILE__ << ", " << __LINE__ << ": " << a << "\n"; \ + std::cout.flush(); \ + } + +// Handling device runtime api: +// void * cudaGetParameterBufferV2(void *func, dim3 gridDimension, dim3 +// blockDimension, unsigned int sharedMemSize) +void cuda_device_runtime::gpgpusim_cuda_getParameterBufferV2( + const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + DEV_RUNTIME_REPORT("Calling cudaGetParameterBufferV2"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert(n_args == 4); + + function_info *child_kernel_entry; + struct dim3 grid_dim, block_dim; + unsigned int shared_mem; + + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); // param# + const symbol *formal_param = + target_func->get_arg(arg); // cudaGetParameterBufferV2_param_# + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if (arg == 0) { // function_info* for the child kernel + unsigned long long buf; + assert(size == sizeof(function_info *)); + thread->m_local_mem->read(from_addr, size, &buf); + child_kernel_entry = (function_info *)buf; + assert(child_kernel_entry); + DEV_RUNTIME_REPORT("child kernel name " + << child_kernel_entry->get_name()); + } else if (arg == 1) { // dim3 grid_dim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, &grid_dim); + DEV_RUNTIME_REPORT("grid (" << grid_dim.x << ", " << grid_dim.y << ", " + << grid_dim.z << ")"); + } else if (arg == 2) { // dim3 block_dim for the child kernel + assert(size == sizeof(struct dim3)); + thread->m_local_mem->read(from_addr, size, &block_dim); + DEV_RUNTIME_REPORT("block (" << block_dim.x << ", " << block_dim.y << ", " + << block_dim.z << ")"); + } else if (arg == 3) { // unsigned int shared_mem + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, &shared_mem); + DEV_RUNTIME_REPORT("shared memory " << shared_mem); } - - //get total child kernel argument size and malloc buffer in global memory - unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); - void * param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); - g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256); - DEV_RUNTIME_REPORT("child kernel arg size total " << child_kernel_arg_size << ", parameter buffer allocated at " << param_buffer); - if(g_total_param_size > g_max_total_param_size) - g_max_total_param_size = g_total_param_size; - - //store param buffer address and launch config - device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, child_kernel_entry); - assert(g_cuda_device_launch_param_map.find(param_buffer) == g_cuda_device_launch_param_map.end()); - g_cuda_device_launch_param_map[param_buffer] = device_launch_config; - - //copy the buffer address to retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 - const symbol *formal_return = target_func->get_return_var(); //void * - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && return_size == sizeof(void *)); - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, NULL); - + } + + // get total child kernel argument size and malloc buffer in global memory + unsigned child_kernel_arg_size = child_kernel_entry->get_args_aligned_size(); + void *param_buffer = thread->get_gpu()->gpu_malloc(child_kernel_arg_size); + g_total_param_size += ((child_kernel_arg_size + 255) / 256 * 256); + DEV_RUNTIME_REPORT("child kernel arg size total " + << child_kernel_arg_size + << ", parameter buffer allocated at " << param_buffer); + if (g_total_param_size > g_max_total_param_size) + g_max_total_param_size = g_total_param_size; + + // store param buffer address and launch config + device_launch_config_t device_launch_config(grid_dim, block_dim, shared_mem, + child_kernel_entry); + assert(g_cuda_device_launch_param_map.find(param_buffer) == + g_cuda_device_launch_param_map.end()); + g_cuda_device_launch_param_map[param_buffer] = device_launch_config; + + // copy the buffer address to retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 + const symbol *formal_return = target_func->get_return_var(); // void * + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaGetParameterBufferV2 return value has size of " + << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && + return_size == sizeof(void *)); + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, ¶m_buffer, NULL, + NULL); } -//Handling device runtime api: -//cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) -void cuda_device_runtime::gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { - DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert( n_args == 2 ); - - kernel_info_t * device_grid = NULL; - function_info * device_kernel_entry = NULL; - void * parameter_buffer; - struct CUstream_st * child_stream; - device_launch_config_t config; - device_launch_operation_t device_launch_op; - - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# - const symbol *formal_param = target_func->get_arg(arg); //cudaLaunchDeviceV2_param_# - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - - if(arg == 0) {//paramter buffer for child kernel (in global memory) - //get parameter_buffer from the cudaLaunchDeviceV2_param0 - assert(size == sizeof(void *)); - thread->m_local_mem->read(from_addr, size, ¶meter_buffer); - assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); - DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " << parameter_buffer); - - //get child grid info through parameter_buffer address - assert(g_cuda_device_launch_param_map.find(parameter_buffer) != g_cuda_device_launch_param_map.end()); - config = g_cuda_device_launch_param_map[parameter_buffer]; - //device_grid = op.grid; - device_kernel_entry = config.entry; - DEV_RUNTIME_REPORT("find device kernel " << device_kernel_entry->get_name()); - - //PDOM analysis is done for Parent kernel but not for child kernel. - if (device_kernel_entry->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", device_kernel_entry->get_name().c_str() ); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", device_kernel_entry->get_name().c_str() ); - /* - * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. - * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. - */ - if (device_kernel_entry->get_function_size() >0) - device_kernel_entry->do_pdom(); - device_kernel_entry->set_pdom(); - } - - //copy data in parameter_buffer to device kernel param memory - unsigned device_kernel_arg_size = device_kernel_entry->get_args_aligned_size(); - DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size); - memory_space *device_kernel_param_mem; - - //create child kernel_info_t and index it with parameter_buffer address - gpgpu_t* gpu=thread->get_gpu(); - device_grid = new kernel_info_t(config.grid_dim, config.block_dim, device_kernel_entry, gpu->getNameArrayMapping(), gpu->getNameInfoMapping()); - device_grid->launch_cycle = gpu->gpu_sim_cycle + gpu->gpu_tot_sim_cycle; - kernel_info_t & parent_grid = thread->get_kernel(); - DEV_RUNTIME_REPORT("child kernel launched by " << parent_grid.name() << ", cta (" << - thread->get_ctaid().x << ", " << thread->get_ctaid().y << ", " << thread->get_ctaid().z << - "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y << ", " << thread->get_tid().z << - ")"); - device_grid->set_parent(&parent_grid, thread->get_ctaid(), thread->get_tid()); - device_launch_op = device_launch_operation_t(device_grid, NULL); - device_kernel_param_mem = device_grid->get_param_memory(); //kernel param - size_t param_start_address = 0; - //copy in word - for(unsigned n = 0; n < device_kernel_arg_size; n += 4) { - unsigned int oneword; - thread->get_gpu()->get_global_memory()->read((size_t)parameter_buffer + n, 4, &oneword); - device_kernel_param_mem->write(param_start_address + n, 4, &oneword, NULL, NULL); - } - } - else if(arg == 1) { //cudaStream for the child kernel - - assert(size == sizeof(cudaStream_t)); - thread->m_local_mem->read(from_addr, size, &child_stream); - - kernel_info_t & parent_kernel = thread->get_kernel(); - if(child_stream == 0) { //default stream on device for current CTA - child_stream = parent_kernel.get_default_stream_cta(thread->get_ctaid()); - DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << - " to default stream of the cta " << child_stream->get_uid() << ": " << child_stream); - } - else { - assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); - DEV_RUNTIME_REPORT("launching child kernel " << device_grid->get_uid() << - " to stream " << child_stream->get_uid() << ": " << child_stream); - } - - device_launch_op.stream = child_stream; - } - +// Handling device runtime api: +// cudaError_t cudaLaunchDeviceV2(void *parameterBuffer, cudaStream_t stream) +void cuda_device_runtime::gpgpusim_cuda_launchDeviceV2( + const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + DEV_RUNTIME_REPORT("Calling cudaLaunchDeviceV2"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert(n_args == 2); + + kernel_info_t *device_grid = NULL; + function_info *device_kernel_entry = NULL; + void *parameter_buffer; + struct CUstream_st *child_stream; + device_launch_config_t config; + device_launch_operation_t device_launch_op; + + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); // param# + const symbol *formal_param = + target_func->get_arg(arg); // cudaLaunchDeviceV2_param_# + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if (arg == 0) { // paramter buffer for child kernel (in global memory) + // get parameter_buffer from the cudaLaunchDeviceV2_param0 + assert(size == sizeof(void *)); + thread->m_local_mem->read(from_addr, size, ¶meter_buffer); + assert((size_t)parameter_buffer >= GLOBAL_HEAP_START); + DEV_RUNTIME_REPORT("Parameter buffer locating at global memory " + << parameter_buffer); + + // get child grid info through parameter_buffer address + assert(g_cuda_device_launch_param_map.find(parameter_buffer) != + g_cuda_device_launch_param_map.end()); + config = g_cuda_device_launch_param_map[parameter_buffer]; + // device_grid = op.grid; + device_kernel_entry = config.entry; + DEV_RUNTIME_REPORT("find device kernel " + << device_kernel_entry->get_name()); + + // PDOM analysis is done for Parent kernel but not for child kernel. + if (device_kernel_entry->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", + device_kernel_entry->get_name().c_str()); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", + device_kernel_entry->get_name().c_str()); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong + * impression that it is a function call. As printf() doesnt have a body + * like functions do, doing pdom analysis for printf() causes a crash. + */ + if (device_kernel_entry->get_function_size() > 0) + device_kernel_entry->do_pdom(); + device_kernel_entry->set_pdom(); + } + + // copy data in parameter_buffer to device kernel param memory + unsigned device_kernel_arg_size = + device_kernel_entry->get_args_aligned_size(); + DEV_RUNTIME_REPORT("device_kernel_arg_size " << device_kernel_arg_size); + memory_space *device_kernel_param_mem; + + // create child kernel_info_t and index it with parameter_buffer address + gpgpu_t *gpu = thread->get_gpu(); + device_grid = new kernel_info_t( + config.grid_dim, config.block_dim, device_kernel_entry, + gpu->getNameArrayMapping(), gpu->getNameInfoMapping()); + device_grid->launch_cycle = gpu->gpu_sim_cycle + gpu->gpu_tot_sim_cycle; + kernel_info_t &parent_grid = thread->get_kernel(); + DEV_RUNTIME_REPORT( + "child kernel launched by " + << parent_grid.name() << ", cta (" << thread->get_ctaid().x << ", " + << thread->get_ctaid().y << ", " << thread->get_ctaid().z + << "), thread (" << thread->get_tid().x << ", " << thread->get_tid().y + << ", " << thread->get_tid().z << ")"); + device_grid->set_parent(&parent_grid, thread->get_ctaid(), + thread->get_tid()); + device_launch_op = device_launch_operation_t(device_grid, NULL); + device_kernel_param_mem = device_grid->get_param_memory(); // kernel + // param + size_t param_start_address = 0; + // copy in word + for (unsigned n = 0; n < device_kernel_arg_size; n += 4) { + unsigned int oneword; + thread->get_gpu()->get_global_memory()->read( + (size_t)parameter_buffer + n, 4, &oneword); + device_kernel_param_mem->write(param_start_address + n, 4, &oneword, + NULL, NULL); + } + } else if (arg == 1) { // cudaStream for the child kernel + + assert(size == sizeof(cudaStream_t)); + thread->m_local_mem->read(from_addr, size, &child_stream); + + kernel_info_t &parent_kernel = thread->get_kernel(); + if (child_stream == 0) { // default stream on device for current CTA + child_stream = + parent_kernel.get_default_stream_cta(thread->get_ctaid()); + DEV_RUNTIME_REPORT("launching child kernel " + << device_grid->get_uid() + << " to default stream of the cta " + << child_stream->get_uid() << ": " << child_stream); + } else { + assert(parent_kernel.cta_has_stream(thread->get_ctaid(), child_stream)); + DEV_RUNTIME_REPORT("launching child kernel " + << device_grid->get_uid() << " to stream " + << child_stream->get_uid() << ": " << child_stream); + } + + device_launch_op.stream = child_stream; } - - - //launch child kernel - g_cuda_device_launch_op.push_back(device_launch_op); - g_cuda_device_launch_param_map.erase(parameter_buffer); - - //set retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 - const symbol *formal_return = target_func->get_return_var(); //cudaError_t - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size - && return_size == sizeof(cudaError_t)); - cudaError_t error = cudaSuccess; - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); - + } + + // launch child kernel + g_cuda_device_launch_op.push_back(device_launch_op); + g_cuda_device_launch_param_map.erase(parameter_buffer); + + // set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 + const symbol *formal_return = target_func->get_return_var(); // cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaLaunchDeviceV2 return value has size of " + << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && + return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); } - -//Handling device runtime api: -//cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int flags) -//flags can only be cudaStreamNonBlocking -void cuda_device_runtime::gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func) { - DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags"); - - unsigned n_return = target_func->has_return(); - assert(n_return); - unsigned n_args = target_func->num_args(); - assert( n_args == 2 ); - - size_t generic_pStream_addr; - addr_t pStream_addr; - unsigned int flags; - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); //param# - const symbol *formal_param = target_func->get_arg(arg); //cudaStreamCreateWithFlags_param_# - unsigned size=formal_param->get_size_in_bytes(); - assert( formal_param->is_param_local() ); - assert( actual_param_op.is_param_local() ); - addr_t from_addr = actual_param_op.get_symbol()->get_address(); - - if(arg == 0) {//cudaStream_t * pStream, address of cudaStream_t - assert(size == sizeof(cudaStream_t *)); - thread->m_local_mem->read(from_addr, size, &generic_pStream_addr); - - //pStream should be non-zero address in local memory - pStream_addr = generic_to_local(thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr); - - DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr); - } - else if(arg == 1) { //unsigned int flags, should be cudaStreamNonBlocking - assert(size == sizeof(unsigned int)); - thread->m_local_mem->read(from_addr, size, &flags); - assert(flags == cudaStreamNonBlocking); - } +// Handling device runtime api: +// cudaError_t cudaStreamCreateWithFlags ( cudaStream_t* pStream, unsigned int +// flags) flags can only be cudaStreamNonBlocking +void cuda_device_runtime::gpgpusim_cuda_streamCreateWithFlags( + const ptx_instruction *pI, ptx_thread_info *thread, + const function_info *target_func) { + DEV_RUNTIME_REPORT("Calling cudaStreamCreateWithFlags"); + + unsigned n_return = target_func->has_return(); + assert(n_return); + unsigned n_args = target_func->num_args(); + assert(n_args == 2); + + size_t generic_pStream_addr; + addr_t pStream_addr; + unsigned int flags; + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); // param# + const symbol *formal_param = + target_func->get_arg(arg); // cudaStreamCreateWithFlags_param_# + unsigned size = formal_param->get_size_in_bytes(); + assert(formal_param->is_param_local()); + assert(actual_param_op.is_param_local()); + addr_t from_addr = actual_param_op.get_symbol()->get_address(); + + if (arg == 0) { // cudaStream_t * pStream, address of cudaStream_t + assert(size == sizeof(cudaStream_t *)); + thread->m_local_mem->read(from_addr, size, &generic_pStream_addr); + + // pStream should be non-zero address in local memory + pStream_addr = generic_to_local( + thread->get_hw_sid(), thread->get_hw_tid(), generic_pStream_addr); + + DEV_RUNTIME_REPORT("pStream locating at local memory " << pStream_addr); + } else if (arg == + 1) { // unsigned int flags, should be cudaStreamNonBlocking + assert(size == sizeof(unsigned int)); + thread->m_local_mem->read(from_addr, size, &flags); + assert(flags == cudaStreamNonBlocking); } - - //create stream and write back to param0 - CUstream_st * stream = thread->get_kernel().create_stream_cta(thread->get_ctaid()); - DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream); - thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, NULL); - - //set retval0 - const operand_info &actual_return_op = pI->operand_lookup(0); //retval0 - const symbol *formal_return = target_func->get_return_var(); //cudaError_t - unsigned int return_size = formal_return->get_size_in_bytes(); - DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " << return_size); - assert(actual_return_op.is_param_local()); - assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size - && return_size == sizeof(cudaError_t)); - cudaError_t error = cudaSuccess; - addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); - thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); - + } + + // create stream and write back to param0 + CUstream_st *stream = + thread->get_kernel().create_stream_cta(thread->get_ctaid()); + DEV_RUNTIME_REPORT("Create stream " << stream->get_uid() << ": " << stream); + thread->m_local_mem->write(pStream_addr, sizeof(cudaStream_t), &stream, NULL, + NULL); + + // set retval0 + const operand_info &actual_return_op = pI->operand_lookup(0); // retval0 + const symbol *formal_return = target_func->get_return_var(); // cudaError_t + unsigned int return_size = formal_return->get_size_in_bytes(); + DEV_RUNTIME_REPORT("cudaStreamCreateWithFlags return value has size of " + << return_size); + assert(actual_return_op.is_param_local()); + assert(actual_return_op.get_symbol()->get_size_in_bytes() == return_size && + return_size == sizeof(cudaError_t)); + cudaError_t error = cudaSuccess; + addr_t ret_param_addr = actual_return_op.get_symbol()->get_address(); + thread->m_local_mem->write(ret_param_addr, return_size, &error, NULL, NULL); } - void cuda_device_runtime::launch_one_device_kernel() { - if(!g_cuda_device_launch_op.empty()) { - device_launch_operation_t &op = g_cuda_device_launch_op.front(); - - stream_operation stream_op = stream_operation(op.grid, gpgpu_ctx->func_sim->g_ptx_sim_mode, op.stream); - gpgpu_ctx->the_gpgpusim->g_stream_manager->push(stream_op); - g_cuda_device_launch_op.pop_front(); - } + if (!g_cuda_device_launch_op.empty()) { + device_launch_operation_t &op = g_cuda_device_launch_op.front(); + + stream_operation stream_op = stream_operation( + op.grid, gpgpu_ctx->func_sim->g_ptx_sim_mode, op.stream); + gpgpu_ctx->the_gpgpusim->g_stream_manager->push(stream_op); + g_cuda_device_launch_op.pop_front(); + } } void cuda_device_runtime::launch_all_device_kernels() { - while(!g_cuda_device_launch_op.empty()) { - launch_one_device_kernel(); - } + while (!g_cuda_device_launch_op.empty()) { + launch_one_device_kernel(); + } } #endif diff --git a/src/cuda-sim/cuda_device_runtime.h b/src/cuda-sim/cuda_device_runtime.h index 7f7a0ca..1d661b2 100644 --- a/src/cuda-sim/cuda_device_runtime.h +++ b/src/cuda-sim/cuda_device_runtime.h @@ -1,67 +1,66 @@ #ifndef __cuda_device_runtime_h__ #define __cuda_device_runtime_h__ -//Jin: cuda_device_runtime.h -//Defines CUDA device runtime APIs for CDP support +// Jin: cuda_device_runtime.h +// Defines CUDA device runtime APIs for CDP support class device_launch_config_t { + public: + device_launch_config_t() {} -public: - device_launch_config_t() {} - - device_launch_config_t(dim3 _grid_dim, - dim3 _block_dim, - unsigned int _shared_mem, - function_info * _entry): - grid_dim(_grid_dim), - block_dim(_block_dim), - shared_mem(_shared_mem), - entry(_entry) {} - - dim3 grid_dim; - dim3 block_dim; - unsigned int shared_mem; - function_info * entry; + device_launch_config_t(dim3 _grid_dim, dim3 _block_dim, + unsigned int _shared_mem, function_info* _entry) + : grid_dim(_grid_dim), + block_dim(_block_dim), + shared_mem(_shared_mem), + entry(_entry) {} + dim3 grid_dim; + dim3 block_dim; + unsigned int shared_mem; + function_info* entry; }; class device_launch_operation_t { + public: + device_launch_operation_t() {} + device_launch_operation_t(kernel_info_t* _grid, CUstream_st* _stream) + : grid(_grid), stream(_stream) {} -public: - device_launch_operation_t() {} - device_launch_operation_t(kernel_info_t *_grid, - CUstream_st * _stream) : - grid(_grid), stream(_stream) {} - - kernel_info_t * grid; //a new child grid - - CUstream_st * stream; + kernel_info_t* grid; // a new child grid + CUstream_st* stream; }; class gpgpu_context; class cuda_device_runtime { - public: - cuda_device_runtime( gpgpu_context* ctx ) { - g_total_param_size = 0; - g_max_total_param_size = 0; - gpgpu_ctx = ctx; - } - unsigned long long g_total_param_size; - std::map g_cuda_device_launch_param_map; - std::list g_cuda_device_launch_op; - unsigned g_kernel_launch_latency; - unsigned long long g_max_total_param_size; - bool g_cdp_enabled; + public: + cuda_device_runtime(gpgpu_context* ctx) { + g_total_param_size = 0; + g_max_total_param_size = 0; + gpgpu_ctx = ctx; + } + unsigned long long g_total_param_size; + std::map g_cuda_device_launch_param_map; + std::list g_cuda_device_launch_op; + unsigned g_kernel_launch_latency; + unsigned long long g_max_total_param_size; + bool g_cdp_enabled; - // backward pointer - class gpgpu_context* gpgpu_ctx; + // backward pointer + class gpgpu_context* gpgpu_ctx; #if (CUDART_VERSION >= 5000) #pragma once - void gpgpusim_cuda_launchDeviceV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); - void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); - void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction * pI, ptx_thread_info * thread, const function_info * target_func); - void launch_all_device_kernels(); - void launch_one_device_kernel(); + void gpgpusim_cuda_launchDeviceV2(const ptx_instruction* pI, + ptx_thread_info* thread, + const function_info* target_func); + void gpgpusim_cuda_streamCreateWithFlags(const ptx_instruction* pI, + ptx_thread_info* thread, + const function_info* target_func); + void gpgpusim_cuda_getParameterBufferV2(const ptx_instruction* pI, + ptx_thread_info* thread, + const function_info* target_func); + void launch_all_device_kernels(); + void launch_one_device_kernel(); #endif }; diff --git a/src/cuda-sim/half.h b/src/cuda-sim/half.h index 9f74bb7..54bedc2 100644 --- a/src/cuda-sim/half.h +++ b/src/cuda-sim/half.h @@ -2,17 +2,23 @@ // // Copyright (c) 2012-2017 Christian Rau // -// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation -// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, -// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: // -// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. // -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. // Version 1.12.0 @@ -23,180 +29,182 @@ #define HALF_HALF_HPP /// Combined gcc version number. -#define HALF_GNUC_VERSION (__GNUC__*100+__GNUC_MINOR__) - -//check C++11 language features -#if defined(__clang__) //clang - #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR) - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT) - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS) - #define HALF_ENABLE_CPP11_USER_LITERALS 1 - #endif - #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG) - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif -/*#elif defined(__INTEL_COMPILER) //Intel C++ - #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ???????? - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) ???????? - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if __INTEL_COMPILER >= 1300 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) ???????? - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if __INTEL_COMPILER >= 1100 && !defined(HALF_ENABLE_CPP11_LONG_LONG) ???????? - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif*/ -#elif defined(__GNUC__) //gcc - #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L - #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) - #define HALF_ENABLE_CPP11_USER_LITERALS 1 - #endif - #if !defined(HALF_ENABLE_CPP11_LONG_LONG) - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif - #endif -#elif defined(_MSC_VER) //Visual C++ - #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) - #define HALF_ENABLE_CPP11_CONSTEXPR 1 - #endif - #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) - #define HALF_ENABLE_CPP11_NOEXCEPT 1 - #endif - #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) - #define HALF_ENABLE_CPP11_USER_LITERALS 1 - #endif - #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) - #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 - #endif - #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG) - #define HALF_ENABLE_CPP11_LONG_LONG 1 - #endif - #define HALF_POP_WARNINGS 1 - #pragma warning(push) - #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned -#endif - -//check C++11 library features +#define HALF_GNUC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +// check C++11 language features +#if defined(__clang__) // clang +#if __has_feature(cxx_static_assert) && \ + !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) +#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 +#endif +#if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR) +#define HALF_ENABLE_CPP11_CONSTEXPR 1 +#endif +#if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT) +#define HALF_ENABLE_CPP11_NOEXCEPT 1 +#endif +#if __has_feature(cxx_user_literals) && \ + !defined(HALF_ENABLE_CPP11_USER_LITERALS) +#define HALF_ENABLE_CPP11_USER_LITERALS 1 +#endif +#if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && \ + !defined(HALF_ENABLE_CPP11_LONG_LONG) +#define HALF_ENABLE_CPP11_LONG_LONG 1 +#endif +/*#elif defined(__INTEL_COMPILER) + //Intel C++ #if __INTEL_COMPILER >= 1100 && + !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) ???????? #define + HALF_ENABLE_CPP11_STATIC_ASSERT 1 #endif #if __INTEL_COMPILER >= 1300 && + !defined(HALF_ENABLE_CPP11_CONSTEXPR) ???????? #define + HALF_ENABLE_CPP11_CONSTEXPR 1 #endif #if __INTEL_COMPILER >= 1300 && + !defined(HALF_ENABLE_CPP11_NOEXCEPT) ???????? #define + HALF_ENABLE_CPP11_NOEXCEPT 1 #endif #if __INTEL_COMPILER >= 1100 && + !defined(HALF_ENABLE_CPP11_LONG_LONG) ???????? #define + HALF_ENABLE_CPP11_LONG_LONG 1 #endif*/ +#elif defined(__GNUC__) // gcc +#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L +#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) +#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 +#endif +#if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) +#define HALF_ENABLE_CPP11_CONSTEXPR 1 +#endif +#if HALF_GNUC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) +#define HALF_ENABLE_CPP11_NOEXCEPT 1 +#endif +#if HALF_GNUC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) +#define HALF_ENABLE_CPP11_USER_LITERALS 1 +#endif +#if !defined(HALF_ENABLE_CPP11_LONG_LONG) +#define HALF_ENABLE_CPP11_LONG_LONG 1 +#endif +#endif +#elif defined(_MSC_VER) // Visual C++ +#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) +#define HALF_ENABLE_CPP11_CONSTEXPR 1 +#endif +#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) +#define HALF_ENABLE_CPP11_NOEXCEPT 1 +#endif +#if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) +#define HALF_ENABLE_CPP11_USER_LITERALS 1 +#endif +#if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) +#define HALF_ENABLE_CPP11_STATIC_ASSERT 1 +#endif +#if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG) +#define HALF_ENABLE_CPP11_LONG_LONG 1 +#endif +#define HALF_POP_WARNINGS 1 +#pragma warning(push) +#pragma warning(disable : 4099 4127 4146) // struct vs class, constant in if, + // negative unsigned +#endif + +// check C++11 library features #include -#if defined(_LIBCPP_VERSION) //libc++ - #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 - #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS - #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 - #endif - #ifndef HALF_ENABLE_CPP11_CSTDINT - #define HALF_ENABLE_CPP11_CSTDINT 1 - #endif - #ifndef HALF_ENABLE_CPP11_CMATH - #define HALF_ENABLE_CPP11_CMATH 1 - #endif - #ifndef HALF_ENABLE_CPP11_HASH - #define HALF_ENABLE_CPP11_HASH 1 - #endif - #endif -#elif defined(__GLIBCXX__) //libstdc++ - #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 - #ifdef __clang__ - #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) - #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 - #endif - #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT) - #define HALF_ENABLE_CPP11_CSTDINT 1 - #endif - #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH) - #define HALF_ENABLE_CPP11_CMATH 1 - #endif - #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH) - #define HALF_ENABLE_CPP11_HASH 1 - #endif - #else - #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT) - #define HALF_ENABLE_CPP11_CSTDINT 1 - #endif - #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH) - #define HALF_ENABLE_CPP11_CMATH 1 - #endif - #if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH) - #define HALF_ENABLE_CPP11_HASH 1 - #endif - #endif - #endif -#elif defined(_CPPLIB_VER) //Dinkumware/Visual C++ - #if _CPPLIB_VER >= 520 - #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS - #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 - #endif - #ifndef HALF_ENABLE_CPP11_CSTDINT - #define HALF_ENABLE_CPP11_CSTDINT 1 - #endif - #ifndef HALF_ENABLE_CPP11_HASH - #define HALF_ENABLE_CPP11_HASH 1 - #endif - #endif - #if _CPPLIB_VER >= 610 - #ifndef HALF_ENABLE_CPP11_CMATH - #define HALF_ENABLE_CPP11_CMATH 1 - #endif - #endif +#if defined(_LIBCPP_VERSION) // libc++ +#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 +#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS +#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 +#endif +#ifndef HALF_ENABLE_CPP11_CSTDINT +#define HALF_ENABLE_CPP11_CSTDINT 1 +#endif +#ifndef HALF_ENABLE_CPP11_CMATH +#define HALF_ENABLE_CPP11_CMATH 1 +#endif +#ifndef HALF_ENABLE_CPP11_HASH +#define HALF_ENABLE_CPP11_HASH 1 +#endif +#endif +#elif defined(__GLIBCXX__) // libstdc++ +#if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 +#ifdef __clang__ +#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) +#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 +#endif +#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT) +#define HALF_ENABLE_CPP11_CSTDINT 1 +#endif +#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH) +#define HALF_ENABLE_CPP11_CMATH 1 +#endif +#if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH) +#define HALF_ENABLE_CPP11_HASH 1 +#endif +#else +#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT) +#define HALF_ENABLE_CPP11_CSTDINT 1 +#endif +#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH) +#define HALF_ENABLE_CPP11_CMATH 1 +#endif +#if HALF_GNUC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH) +#define HALF_ENABLE_CPP11_HASH 1 +#endif +#endif +#endif +#elif defined(_CPPLIB_VER) // Dinkumware/Visual C++ +#if _CPPLIB_VER >= 520 +#ifndef HALF_ENABLE_CPP11_TYPE_TRAITS +#define HALF_ENABLE_CPP11_TYPE_TRAITS 1 +#endif +#ifndef HALF_ENABLE_CPP11_CSTDINT +#define HALF_ENABLE_CPP11_CSTDINT 1 +#endif +#ifndef HALF_ENABLE_CPP11_HASH +#define HALF_ENABLE_CPP11_HASH 1 +#endif +#endif +#if _CPPLIB_VER >= 610 +#ifndef HALF_ENABLE_CPP11_CMATH +#define HALF_ENABLE_CPP11_CMATH 1 +#endif +#endif #endif #undef HALF_GNUC_VERSION -//support constexpr +// support constexpr #if HALF_ENABLE_CPP11_CONSTEXPR - #define HALF_CONSTEXPR constexpr - #define HALF_CONSTEXPR_CONST constexpr +#define HALF_CONSTEXPR constexpr +#define HALF_CONSTEXPR_CONST constexpr #else - #define HALF_CONSTEXPR - #define HALF_CONSTEXPR_CONST const +#define HALF_CONSTEXPR +#define HALF_CONSTEXPR_CONST const #endif -//support noexcept +// support noexcept #if HALF_ENABLE_CPP11_NOEXCEPT - #define HALF_NOEXCEPT noexcept - #define HALF_NOTHROW noexcept +#define HALF_NOEXCEPT noexcept +#define HALF_NOTHROW noexcept #else - #define HALF_NOEXCEPT - #define HALF_NOTHROW throw() +#define HALF_NOEXCEPT +#define HALF_NOTHROW throw() #endif #include -#include -#include #include #include #include +#include +#include #if HALF_ENABLE_CPP11_TYPE_TRAITS - #include +#include #endif #if HALF_ENABLE_CPP11_CSTDINT - #include +#include #endif #if HALF_ENABLE_CPP11_HASH - #include +#include #endif - /// Default rounding mode. -/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and `float`s as well as -/// for the half_cast() if not specifying a rounding mode explicitly. It can be redefined (before including half.hpp) to one -/// of the standard rounding modes using their respective constants or the equivalent values of `std::float_round_style`: +/// This specifies the rounding mode used for all conversions between +/// [half](\ref half_float::half)s and `float`s as well as for the half_cast() +/// if not specifying a rounding mode explicitly. It can be redefined (before +/// including half.hpp) to one of the standard rounding modes using their +/// respective constants or the equivalent values of `std::float_round_style`: /// /// `std::float_round_style` | value | rounding /// ---------------------------------|-------|------------------------- @@ -206,2862 +214,3509 @@ /// `std::round_toward_infinity` | 2 | toward positive infinity /// `std::round_toward_neg_infinity` | 3 | toward negative infinity /// -/// By default this is set to `-1` (`std::round_indeterminate`), which uses truncation (round toward zero, but with overflows -/// set to infinity) and is the fastest rounding mode possible. It can even be set to `std::numeric_limits::round_style` -/// to synchronize the rounding mode with that of the underlying single-precision implementation. +/// By default this is set to `-1` (`std::round_indeterminate`), which uses +/// truncation (round toward zero, but with overflows set to infinity) and is +/// the fastest rounding mode possible. It can even be set to +/// `std::numeric_limits::round_style` to synchronize the rounding mode +/// with that of the underlying single-precision implementation. #ifndef HALF_ROUND_STYLE - #define HALF_ROUND_STYLE -1 // = std::round_indeterminate +#define HALF_ROUND_STYLE -1 // = std::round_indeterminate #endif /// Tie-breaking behaviour for round to nearest. -/// This specifies if ties in round to nearest should be resolved by rounding to the nearest even value. By default this is -/// defined to `0` resulting in the faster but slightly more biased behaviour of rounding away from zero in half-way cases (and -/// thus equal to the round() function), but can be redefined to `1` (before including half.hpp) if more IEEE-conformant +/// This specifies if ties in round to nearest should be resolved by rounding to +/// the nearest even value. By default this is defined to `0` resulting in the +/// faster but slightly more biased behaviour of rounding away from zero in +/// half-way cases (and thus equal to the round() function), but can be +/// redefined to `1` (before including half.hpp) if more IEEE-conformant /// behaviour is needed. #ifndef HALF_ROUND_TIES_TO_EVEN - #define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero +#define HALF_ROUND_TIES_TO_EVEN 0 // ties away from zero #endif /// Value signaling overflow. -/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to a positive value signaling the overflow of an -/// operation, in particular it just evaluates to positive infinity. -#define HUGE_VALH std::numeric_limits::infinity() +/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to +/// a positive value signaling the overflow of an operation, in particular it +/// just evaluates to positive infinity. +#define HUGE_VALH std::numeric_limits::infinity() /// Fast half-precision fma function. -/// This symbol is only defined if the fma() function generally executes as fast as, or faster than, a separate -/// half-precision multiplication followed by an addition. Due to the internal single-precision implementation of all +/// This symbol is only defined if the fma() function generally executes as fast +/// as, or faster than, a separate half-precision multiplication followed by an +/// addition. Due to the internal single-precision implementation of all /// arithmetic operations, this is in fact always the case. -#define FP_FAST_FMAH 1 +#define FP_FAST_FMAH 1 #ifndef FP_ILOGB0 - #define FP_ILOGB0 INT_MIN +#define FP_ILOGB0 INT_MIN #endif #ifndef FP_ILOGBNAN - #define FP_ILOGBNAN INT_MAX +#define FP_ILOGBNAN INT_MAX #endif #ifndef FP_SUBNORMAL - #define FP_SUBNORMAL 0 +#define FP_SUBNORMAL 0 #endif #ifndef FP_ZERO - #define FP_ZERO 1 +#define FP_ZERO 1 #endif #ifndef FP_NAN - #define FP_NAN 2 +#define FP_NAN 2 #endif #ifndef FP_INFINITE - #define FP_INFINITE 3 +#define FP_INFINITE 3 #endif #ifndef FP_NORMAL - #define FP_NORMAL 4 +#define FP_NORMAL 4 #endif - /// Main namespace for half precision functionality. /// This namespace contains all the functionality provided by the library. -namespace half_float -{ - class half; +namespace half_float { +class half; #if HALF_ENABLE_CPP11_USER_LITERALS - /// Library-defined half-precision literals. - /// Import this namespace to enable half-precision floating point literals: - /// ~~~~{.cpp} - /// using namespace half_float::literal; - /// half_float::half = 4.2_h; - /// ~~~~ - namespace literal - { - half operator"" _h(long double); - } -#endif - - /// \internal - /// \brief Implementation details. - namespace detail - { - #if HALF_ENABLE_CPP11_TYPE_TRAITS - /// Conditional type. - template struct conditional : std::conditional {}; - - /// Helper for tag dispatching. - template struct bool_type : std::integral_constant {}; - using std::true_type; - using std::false_type; - - /// Type traits for floating point types. - template struct is_float : std::is_floating_point {}; - #else - /// Conditional type. - template struct conditional { typedef T type; }; - template struct conditional { typedef F type; }; - - /// Helper for tag dispatching. - template struct bool_type {}; - typedef bool_type true_type; - typedef bool_type false_type; - - /// Type traits for floating point types. - template struct is_float : false_type {}; - template struct is_float : is_float {}; - template struct is_float : is_float {}; - template struct is_float : is_float {}; - template<> struct is_float : true_type {}; - template<> struct is_float : true_type {}; - template<> struct is_float : true_type {}; - #endif - - /// Type traits for floating point bits. - template struct bits { typedef unsigned char type; }; - template struct bits : bits {}; - template struct bits : bits {}; - template struct bits : bits {}; - - #if HALF_ENABLE_CPP11_CSTDINT - /// Unsigned integer of (at least) 16 bits width. - typedef std::uint_least16_t uint16; - - /// Unsigned integer of (at least) 32 bits width. - template<> struct bits { typedef std::uint_least32_t type; }; - - /// Unsigned integer of (at least) 64 bits width. - template<> struct bits { typedef std::uint_least64_t type; }; - #else - /// Unsigned integer of (at least) 16 bits width. - typedef unsigned short uint16; - - /// Unsigned integer of (at least) 32 bits width. - template<> struct bits : conditional::digits>=32,unsigned int,unsigned long> {}; - - #if HALF_ENABLE_CPP11_LONG_LONG - /// Unsigned integer of (at least) 64 bits width. - template<> struct bits : conditional::digits>=64,unsigned long,unsigned long long> {}; - #else - /// Unsigned integer of (at least) 64 bits width. - template<> struct bits { typedef unsigned long type; }; - #endif - #endif - - /// Tag type for binary construction. - struct binary_t {}; - - /// Tag for binary construction. - HALF_CONSTEXPR_CONST binary_t binary = binary_t(); - - /// Temporary half-precision expression. - /// This class represents a half-precision expression which just stores a single-precision value internally. - struct expr - { - /// Conversion constructor. - /// \param f single-precision value to convert - explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {} - - /// Conversion to single-precision. - /// \return single precision value representing expression value - HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; } - - private: - /// Internal expression value stored in single-precision. - float value_; - }; - - /// SFINAE helper for generic half-precision functions. - /// This class template has to be specialized for each valid combination of argument types to provide a corresponding - /// `type` member equivalent to \a T. - /// \tparam T type to return - template struct enable {}; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - template struct enable { typedef T type; }; - - /// Return type for specialized generic 2-argument half-precision functions. - /// This class template has to be specialized for each valid combination of argument types to provide a corresponding - /// `type` member denoting the appropriate return type. - /// \tparam T first argument type - /// \tparam U first argument type - template struct result : enable {}; - template<> struct result { typedef half type; }; - - /// \name Classification helpers - /// \{ - - /// Check for infinity. - /// \tparam T argument type (builtin floating point type) - /// \param arg value to query - /// \retval true if infinity - /// \retval false else - template bool builtin_isinf(T arg) - { - #if HALF_ENABLE_CPP11_CMATH - return std::isinf(arg); - #elif defined(_MSC_VER) - return !::_finite(static_cast(arg)) && !::_isnan(static_cast(arg)); - #else - return arg == std::numeric_limits::infinity() || arg == -std::numeric_limits::infinity(); - #endif - } - - /// Check for NaN. - /// \tparam T argument type (builtin floating point type) - /// \param arg value to query - /// \retval true if not a number - /// \retval false else - template bool builtin_isnan(T arg) - { - #if HALF_ENABLE_CPP11_CMATH - return std::isnan(arg); - #elif defined(_MSC_VER) - return ::_isnan(static_cast(arg)) != 0; - #else - return arg != arg; - #endif - } - - /// Check sign. - /// \tparam T argument type (builtin floating point type) - /// \param arg value to query - /// \retval true if signbit set - /// \retval false else - template bool builtin_signbit(T arg) - { - #if HALF_ENABLE_CPP11_CMATH - return std::signbit(arg); - #else - return arg < T() || (arg == T() && T(1)/arg < T()); - #endif - } - - /// \} - /// \name Conversion - /// \{ - - /// Convert IEEE single-precision to half-precision. - /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \param value single-precision value - /// \return binary representation of half-precision value - template uint16 float2half_impl(float value, true_type) - { - typedef bits::type uint32; - uint32 bits;// = *reinterpret_cast(&value); //violating strict aliasing! - std::memcpy(&bits, &value, sizeof(float)); -/* uint16 hbits = (bits>>16) & 0x8000; - bits &= 0x7FFFFFFF; - int exp = bits >> 23; - if(exp == 255) - return hbits | 0x7C00 | (0x3FF&-static_cast((bits&0x7FFFFF)!=0)); - if(exp > 142) - { - if(R == std::round_toward_infinity) - return hbits | 0x7C00 - (hbits>>15); - if(R == std::round_toward_neg_infinity) - return hbits | 0x7BFF + (hbits>>15); - return hbits | 0x7BFF + (R!=std::round_toward_zero); - } - int g, s; - if(exp > 112) - { - g = (bits>>12) & 1; - s = (bits&0xFFF) != 0; - hbits |= ((exp-112)<<10) | ((bits>>13)&0x3FF); - } - else if(exp > 101) - { - int i = 125 - exp; - bits = (bits&0x7FFFFF) | 0x800000; - g = (bits>>i) & 1; - s = (bits&((1L<> (i+1); - } - else - { - g = 0; - s = bits != 0; - } - if(R == std::round_to_nearest) - #if HALF_ROUND_TIES_TO_EVEN - hbits += g & (s|hbits); - #else - hbits += g; - #endif - else if(R == std::round_toward_infinity) - hbits += ~(hbits>>15) & (s|g); - else if(R == std::round_toward_neg_infinity) - hbits += (hbits>>15) & (g|s); -*/ static const uint16 base_table[512] = { - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, - 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100, - 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00, - 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, - 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, - 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, - 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00 }; - static const unsigned char shift_table[512] = { - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, - 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 }; - uint16 hbits = base_table[bits>>23] + static_cast((bits&0x7FFFFF)>>shift_table[bits>>23]); - if(R == std::round_to_nearest) - hbits += (((bits&0x7FFFFF)>>(shift_table[bits>>23]-1))|(((bits>>23)&0xFF)==102)) & ((hbits&0x7C00)!=0x7C00) - #if HALF_ROUND_TIES_TO_EVEN - & (((((static_cast(1)<<(shift_table[bits>>23]-1))-1)&bits)!=0)|hbits) - #endif - ; - else if(R == std::round_toward_zero) - hbits -= ((hbits&0x7FFF)==0x7C00) & ~shift_table[bits>>23]; - else if(R == std::round_toward_infinity) - hbits += ((((bits&0x7FFFFF&((static_cast(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=102)& - ((bits>>23)!=0)))&(hbits<0x7C00)) - ((hbits==0xFC00)&((bits>>23)!=511)); - else if(R == std::round_toward_neg_infinity) - hbits += ((((bits&0x7FFFFF&((static_cast(1)<<(shift_table[bits>>23]))-1))!=0)|(((bits>>23)<=358)& - ((bits>>23)!=256)))&(hbits<0xFC00)&(hbits>>15)) - ((hbits==0x7C00)&((bits>>23)!=255)); - return hbits; - } - - /// Convert IEEE double-precision to half-precision. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \param value double-precision value - /// \return binary representation of half-precision value - template uint16 float2half_impl(double value, true_type) - { - typedef bits::type uint32; - typedef bits::type uint64; - uint64 bits;// = *reinterpret_cast(&value); //violating strict aliasing! - std::memcpy(&bits, &value, sizeof(double)); - uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF; - uint16 hbits = (hi>>16) & 0x8000; - hi &= 0x7FFFFFFF; - int exp = hi >> 20; - if(exp == 2047) - return hbits | 0x7C00 | (0x3FF&-static_cast((bits&0xFFFFFFFFFFFFF)!=0)); - if(exp > 1038) - { - if(R == std::round_toward_infinity) - return hbits | 0x7C00 - (hbits>>15); - if(R == std::round_toward_neg_infinity) - return hbits | 0x7BFF + (hbits>>15); - return hbits | 0x7BFF + (R!=std::round_toward_zero); - } - int g, s = lo != 0; - if(exp > 1008) - { - g = (hi>>9) & 1; - s |= (hi&0x1FF) != 0; - hbits |= ((exp-1008)<<10) | ((hi>>10)&0x3FF); - } - else if(exp > 997) - { - int i = 1018 - exp; - hi = (hi&0xFFFFF) | 0x100000; - g = (hi>>i) & 1; - s |= (hi&((1L<> (i+1); - } - else - { - g = 0; - s |= hi != 0; - } - if(R == std::round_to_nearest) - #if HALF_ROUND_TIES_TO_EVEN - hbits += g & (s|hbits); - #else - hbits += g; - #endif - else if(R == std::round_toward_infinity) - hbits += ~(hbits>>15) & (s|g); - else if(R == std::round_toward_neg_infinity) - hbits += (hbits>>15) & (g|s); - return hbits; - } - - /// Convert non-IEEE floating point to half-precision. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam T source type (builtin floating point type) - /// \param value floating point value - /// \return binary representation of half-precision value - template uint16 float2half_impl(T value, ...) - { - uint16 hbits = static_cast(builtin_signbit(value)) << 15; - if(value == T()) - return hbits; - if(builtin_isnan(value)) - return hbits | 0x7FFF; - if(builtin_isinf(value)) - return hbits | 0x7C00; - int exp; - std::frexp(value, &exp); - if(exp > 16) - { - if(R == std::round_toward_infinity) - return hbits | (0x7C00 - (hbits>>15)); - else if(R == std::round_toward_neg_infinity) - return hbits | (0x7BFF + (hbits>>15)); - return hbits | (0x7BFF + (R!=std::round_toward_zero)); - } - if(exp < -13) - value = std::ldexp(value, 24); - else - { - value = std::ldexp(value, 11-exp); - hbits |= ((exp+13)<<10); - } - T ival, frac = std::modf(value, &ival); - hbits += static_cast(std::abs(static_cast(ival))); - if(R == std::round_to_nearest) - { - frac = std::abs(frac); - #if HALF_ROUND_TIES_TO_EVEN - hbits += (frac>T(0.5)) | ((frac==T(0.5))&hbits); - #else - hbits += frac >= T(0.5); - #endif - } - else if(R == std::round_toward_infinity) - hbits += frac > T(); - else if(R == std::round_toward_neg_infinity) - hbits += frac < T(); - return hbits; - } - - /// Convert floating point to half-precision. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam T source type (builtin floating point type) - /// \param value floating point value - /// \return binary representation of half-precision value - template uint16 float2half(T value) - { - return float2half_impl(value, bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); - } - - /// Convert integer to half-precision floating point. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam S `true` if value negative, `false` else - /// \tparam T type to convert (builtin integer type) - /// \param value non-negative integral value - /// \return binary representation of half-precision value - template uint16 int2half_impl(T value) - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_integral::value, "int to half conversion only supports builtin integer types"); - #endif - if(S) - value = -value; - uint16 bits = S << 15; - if(value > 0xFFFF) - { - if(R == std::round_toward_infinity) - bits |= 0x7C00 - S; - else if(R == std::round_toward_neg_infinity) - bits |= 0x7BFF + S; - else - bits |= 0x7BFF + (R!=std::round_toward_zero); - } - else if(value) - { - unsigned int m = value, exp = 24; - for(; m<0x400; m<<=1,--exp) ; - for(; m>0x7FF; m>>=1,++exp) ; - bits |= (exp<<10) + m; - if(exp > 24) - { - if(R == std::round_to_nearest) - bits += (value>>(exp-25)) & 1 - #if HALF_ROUND_TIES_TO_EVEN - & (((((1<<(exp-25))-1)&value)!=0)|bits) - #endif - ; - else if(R == std::round_toward_infinity) - bits += ((value&((1<<(exp-24))-1))!=0) & !S; - else if(R == std::round_toward_neg_infinity) - bits += ((value&((1<<(exp-24))-1))!=0) & S; - } - } - return bits; - } - - /// Convert integer to half-precision floating point. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam T type to convert (builtin integer type) - /// \param value integral value - /// \return binary representation of half-precision value - template uint16 int2half(T value) - { - return (value<0) ? int2half_impl(value) : int2half_impl(value); - } - - /// Convert half-precision to IEEE single-precision. - /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). - /// \param value binary representation of half-precision value - /// \return single-precision value - inline float half2float_impl(uint16 value, float, true_type) - { - typedef bits::type uint32; -/* uint32 bits = static_cast(value&0x8000) << 16; - int abs = value & 0x7FFF; - if(abs) - { - bits |= 0x38000000 << static_cast(abs>=0x7C00); - for(; abs<0x400; abs<<=1,bits-=0x800000) ; - bits += static_cast(abs) << 13; - } -*/ static const uint32 mantissa_table[2048] = { - 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000, - 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000, - 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000, - 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000, - 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000, - 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000, - 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000, - 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000, - 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, - 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, - 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000, - 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000, - 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000, - 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000, - 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000, - 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000, - 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000, - 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000, - 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000, - 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000, - 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000, - 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000, - 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000, - 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, - 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, - 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000, - 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000, - 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000, - 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000, - 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, - 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000, - 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000, - 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000, - 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000, - 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000, - 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000, - 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000, - 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000, - 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, - 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, - 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000, - 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000, - 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000, - 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000, - 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, - 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000, - 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000, - 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000, - 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000, - 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000, - 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000, - 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000, - 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000, - 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000, - 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, - 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000, - 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000, - 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000, - 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000, - 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000, - 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000, - 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000, - 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000, - 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, - 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000, - 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000, - 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000, - 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000, - 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, - 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, - 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000, - 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000, - 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000, - 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000, - 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000, - 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000, - 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000, - 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000, - 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, - 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000, - 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000, - 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000, - 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000, - 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, - 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, - 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000, - 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000, - 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000, - 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000, - 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, - 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000, - 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000, - 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000, - 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, - 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000, - 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000, - 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000, - 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000, - 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, - 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, - 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000, - 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000, - 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000, - 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000, - 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, - 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000, - 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000, - 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000, - 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, - 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000, - 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000, - 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000, - 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000, - 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, - 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, - 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000, - 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000, - 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000, - 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000, - 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000, - 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000, - 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000, - 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000, - 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, - 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000, - 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000, - 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000, - 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 }; - static const uint32 exponent_table[64] = { - 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000, - 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000, - 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, - 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 }; - static const unsigned short offset_table[64] = { - 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, - 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 }; - uint32 bits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10]; -// return *reinterpret_cast(&bits); //violating strict aliasing! - float out; - std::memcpy(&out, &bits, sizeof(float)); - return out; - } - - /// Convert half-precision to IEEE double-precision. - /// \param value binary representation of half-precision value - /// \return double-precision value - inline double half2float_impl(uint16 value, double, true_type) - { - typedef bits::type uint32; - typedef bits::type uint64; - uint32 hi = static_cast(value&0x8000) << 16; - int abs = value & 0x7FFF; - if(abs) - { - hi |= 0x3F000000 << static_cast(abs>=0x7C00); - for(; abs<0x400; abs<<=1,hi-=0x100000) ; - hi += static_cast(abs) << 10; - } - uint64 bits = static_cast(hi) << 32; -// return *reinterpret_cast(&bits); //violating strict aliasing! - double out; - std::memcpy(&out, &bits, sizeof(double)); - return out; - } - - /// Convert half-precision to non-IEEE floating point. - /// \tparam T type to convert to (builtin integer type) - /// \param value binary representation of half-precision value - /// \return floating point value - template T half2float_impl(uint16 value, T, ...) - { - T out; - int abs = value & 0x7FFF; - if(abs > 0x7C00) - out = std::numeric_limits::has_quiet_NaN ? std::numeric_limits::quiet_NaN() : T(); - else if(abs == 0x7C00) - out = std::numeric_limits::has_infinity ? std::numeric_limits::infinity() : std::numeric_limits::max(); - else if(abs > 0x3FF) - out = std::ldexp(static_cast((abs&0x3FF)|0x400), (abs>>10)-25); - else - out = std::ldexp(static_cast(abs), -24); - return (value&0x8000) ? -out : out; - } - - /// Convert half-precision to floating point. - /// \tparam T type to convert to (builtin integer type) - /// \param value binary representation of half-precision value - /// \return floating point value - template T half2float(uint16 value) - { - return half2float_impl(value, T(), bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); - } - - /// Convert half-precision floating point to integer. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam E `true` for round to even, `false` for round away from zero - /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) - /// \param value binary representation of half-precision value - /// \return integral value - template T half2int_impl(uint16 value) - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_integral::value, "half to int conversion only supports builtin integer types"); - #endif - unsigned int e = value & 0x7FFF; - if(e >= 0x7C00) - return (value&0x8000) ? std::numeric_limits::min() : std::numeric_limits::max(); - if(e < 0x3800) - { - if(R == std::round_toward_infinity) - return T(~(value>>15)&(e!=0)); - else if(R == std::round_toward_neg_infinity) - return -T(value>0x8000); - return T(); - } - unsigned int m = (value&0x3FF) | 0x400; - e >>= 10; - if(e < 25) - { - if(R == std::round_to_nearest) - m += (1<<(24-e)) - (~(m>>(25-e))&E); - else if(R == std::round_toward_infinity) - m += ((value>>15)-1) & ((1<<(25-e))-1U); - else if(R == std::round_toward_neg_infinity) - m += -(value>>15) & ((1<<(25-e))-1U); - m >>= 25 - e; - } - else - m <<= e - 25; - return (value&0x8000) ? -static_cast(m) : static_cast(m); - } - - /// Convert half-precision floating point to integer. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) - /// \param value binary representation of half-precision value - /// \return integral value - template T half2int(uint16 value) { return half2int_impl(value); } - - /// Convert half-precision floating point to integer using round-to-nearest-away-from-zero. - /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) - /// \param value binary representation of half-precision value - /// \return integral value - template T half2int_up(uint16 value) { return half2int_impl(value); } - - /// Round half-precision number to nearest integer value. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \tparam E `true` for round to even, `false` for round away from zero - /// \param value binary representation of half-precision value - /// \return half-precision bits for nearest integral value - template uint16 round_half_impl(uint16 value) - { - unsigned int e = value & 0x7FFF; - uint16 result = value; - if(e < 0x3C00) - { - result &= 0x8000; - if(R == std::round_to_nearest) - result |= 0x3C00U & -(e>=(0x3800+E)); - else if(R == std::round_toward_infinity) - result |= 0x3C00U & -(~(value>>15)&(e!=0)); - else if(R == std::round_toward_neg_infinity) - result |= 0x3C00U & -(value>0x8000); - } - else if(e < 0x6400) - { - e = 25 - (e>>10); - unsigned int mask = (1<>e)&E); - else if(R == std::round_toward_infinity) - result += mask & ((value>>15)-1); - else if(R == std::round_toward_neg_infinity) - result += mask & -(value>>15); - result &= ~mask; - } - return result; - } - - /// Round half-precision number to nearest integer value. - /// \tparam R rounding mode to use, `std::round_indeterminate` for fastest rounding - /// \param value binary representation of half-precision value - /// \return half-precision bits for nearest integral value - template uint16 round_half(uint16 value) { return round_half_impl(value); } - - /// Round half-precision number to nearest integer value using round-to-nearest-away-from-zero. - /// \param value binary representation of half-precision value - /// \return half-precision bits for nearest integral value - inline uint16 round_half_up(uint16 value) { return round_half_impl(value); } - /// \} - - struct functions; - template struct unary_specialized; - template struct binary_specialized; - template struct half_caster; - } - - /// Half-precision floating point type. - /// This class implements an IEEE-conformant half-precision floating point type with the usual arithmetic operators and - /// conversions. It is implicitly convertible to single-precision floating point, which makes artihmetic expressions and - /// functions with mixed-type operands to be of the most precise operand type. Additionally all arithmetic operations - /// (and many mathematical functions) are carried out in single-precision internally. All conversions from single- to - /// half-precision are done using the library's default rounding mode, but temporary results inside chained arithmetic - /// expressions are kept in single-precision as long as possible (while of course still maintaining a strong half-precision type). - /// - /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and - /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which - /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the - /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of - /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most - /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit - /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if - /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on - /// nearly any reasonable platform. - /// - /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable - /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation. - class half - { - friend struct detail::functions; - friend struct detail::unary_specialized; - friend struct detail::binary_specialized; - template friend struct detail::half_caster; - friend class std::numeric_limits; - #if HALF_ENABLE_CPP11_HASH - friend struct std::hash; - #endif - #if HALF_ENABLE_CPP11_USER_LITERALS - friend half literal::operator"" _h(long double); - #endif - - public: - /// Default constructor. - /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics - /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics. - HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {} - - /// Copy constructor. - /// \tparam T type of concrete half expression - /// \param rhs half expression to copy from - half(detail::expr rhs) : data_(detail::float2half(static_cast(rhs))) {} - - /// Conversion constructor. - /// \param rhs float to convert - explicit half(float rhs) : data_(detail::float2half(rhs)) {} - - /// Conversion to single-precision. - /// \return single precision value representing expression value - operator float() const { return detail::half2float(data_); } - - /// Assignment operator. - /// \tparam T type of concrete half expression - /// \param rhs half expression to copy from - /// \return reference to this half - half& operator=(detail::expr rhs) { return *this = static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to add - /// \return reference to this half - template typename detail::enable::type operator+=(T rhs) { return *this += static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to subtract - /// \return reference to this half - template typename detail::enable::type operator-=(T rhs) { return *this -= static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to multiply with - /// \return reference to this half - template typename detail::enable::type operator*=(T rhs) { return *this *= static_cast(rhs); } - - /// Arithmetic assignment. - /// \tparam T type of concrete half expression - /// \param rhs half expression to divide by - /// \return reference to this half - template typename detail::enable::type operator/=(T rhs) { return *this /= static_cast(rhs); } - - /// Assignment operator. - /// \param rhs single-precision value to copy from - /// \return reference to this half - half& operator=(float rhs) { data_ = detail::float2half(rhs); return *this; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to add - /// \return reference to this half - half& operator+=(float rhs) { data_ = detail::float2half(detail::half2float(data_)+rhs); return *this; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to subtract - /// \return reference to this half - half& operator-=(float rhs) { data_ = detail::float2half(detail::half2float(data_)-rhs); return *this; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to multiply with - /// \return reference to this half - half& operator*=(float rhs) { data_ = detail::float2half(detail::half2float(data_)*rhs); return *this; } - - /// Arithmetic assignment. - /// \param rhs single-precision value to divide by - /// \return reference to this half - half& operator/=(float rhs) { data_ = detail::float2half(detail::half2float(data_)/rhs); return *this; } - - /// Prefix increment. - /// \return incremented half value - half& operator++() { return *this += 1.0f; } - - /// Prefix decrement. - /// \return decremented half value - half& operator--() { return *this -= 1.0f; } - - /// Postfix increment. - /// \return non-incremented half value - half operator++(int) { half out(*this); ++*this; return out; } - - /// Postfix decrement. - /// \return non-decremented half value - half operator--(int) { half out(*this); --*this; return out; } - - private: - /// Rounding mode to use - static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE); - - /// Constructor. - /// \param bits binary representation to set half to - HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT : data_(bits) {} - - /// Internal binary representation - detail::uint16 data_; - }; +/// Library-defined half-precision literals. +/// Import this namespace to enable half-precision floating point literals: +/// ~~~~{.cpp} +/// using namespace half_float::literal; +/// half_float::half = 4.2_h; +/// ~~~~ +namespace literal { +half operator"" _h(long double); +} +#endif + +/// \internal +/// \brief Implementation details. +namespace detail { +#if HALF_ENABLE_CPP11_TYPE_TRAITS +/// Conditional type. +template +struct conditional : std::conditional {}; + +/// Helper for tag dispatching. +template +struct bool_type : std::integral_constant {}; +using std::false_type; +using std::true_type; + +/// Type traits for floating point types. +template +struct is_float : std::is_floating_point {}; +#else +/// Conditional type. +template +struct conditional { + typedef T type; +}; +template +struct conditional { + typedef F type; +}; + +/// Helper for tag dispatching. +template +struct bool_type {}; +typedef bool_type true_type; +typedef bool_type false_type; + +/// Type traits for floating point types. +template +struct is_float : false_type {}; +template +struct is_float : is_float {}; +template +struct is_float : is_float {}; +template +struct is_float : is_float {}; +template <> +struct is_float : true_type {}; +template <> +struct is_float : true_type {}; +template <> +struct is_float : true_type {}; +#endif + +/// Type traits for floating point bits. +template +struct bits { + typedef unsigned char type; +}; +template +struct bits : bits {}; +template +struct bits : bits {}; +template +struct bits : bits {}; + +#if HALF_ENABLE_CPP11_CSTDINT +/// Unsigned integer of (at least) 16 bits width. +typedef std::uint_least16_t uint16; + +/// Unsigned integer of (at least) 32 bits width. +template <> +struct bits { + typedef std::uint_least32_t type; +}; + +/// Unsigned integer of (at least) 64 bits width. +template <> +struct bits { + typedef std::uint_least64_t type; +}; +#else +/// Unsigned integer of (at least) 16 bits width. +typedef unsigned short uint16; + +/// Unsigned integer of (at least) 32 bits width. +template <> +struct bits + : conditional::digits >= 32, unsigned int, + unsigned long> {}; -#if HALF_ENABLE_CPP11_USER_LITERALS - namespace literal - { - /// Half literal. - /// While this returns an actual half-precision value, half literals can unfortunately not be constant expressions due - /// to rather involved conversions. - /// \param value literal value - /// \return half with given value (if representable) - inline half operator"" _h(long double value) { return half(detail::binary, detail::float2half(value)); } - } -#endif - - namespace detail - { - /// Wrapper implementing unspecialized half-precision functions. - struct functions - { - /// Addition implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision sum stored in single-precision - static expr plus(float x, float y) { return expr(x+y); } - - /// Subtraction implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision difference stored in single-precision - static expr minus(float x, float y) { return expr(x-y); } - - /// Multiplication implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision product stored in single-precision - static expr multiplies(float x, float y) { return expr(x*y); } - - /// Division implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision quotient stored in single-precision - static expr divides(float x, float y) { return expr(x/y); } - - /// Output implementation. - /// \param out stream to write to - /// \param arg value to write - /// \return reference to stream - template static std::basic_ostream& write(std::basic_ostream &out, float arg) { return out << arg; } - - /// Input implementation. - /// \param in stream to read from - /// \param arg half to read into - /// \return reference to stream - template static std::basic_istream& read(std::basic_istream &in, half &arg) - { - float f; - if(in >> f) - arg = f; - return in; - } - - /// Modulo implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision division remainder stored in single-precision - static expr fmod(float x, float y) { return expr(std::fmod(x, y)); } - - /// Remainder implementation. - /// \param x first operand - /// \param y second operand - /// \return Half-precision division remainder stored in single-precision - static expr remainder(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::remainder(x, y)); - #else - if(builtin_isnan(x) || builtin_isnan(y)) - return expr(std::numeric_limits::quiet_NaN()); - float ax = std::fabs(x), ay = std::fabs(y); - if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) - return expr(std::numeric_limits::quiet_NaN()); - if(ay >= 65536.0f) - return expr(x); - if(ax == ay) - return expr(builtin_signbit(x) ? -0.0f : 0.0f); - ax = std::fmod(ax, ay+ay); - float y2 = 0.5f * ay; - if(ax > y2) - { - ax -= ay; - if(ax >= y2) - ax -= ay; - } - return expr(builtin_signbit(x) ? -ax : ax); - #endif - } - - /// Remainder implementation. - /// \param x first operand - /// \param y second operand - /// \param quo address to store quotient bits at - /// \return Half-precision division remainder stored in single-precision - static expr remquo(float x, float y, int *quo) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::remquo(x, y, quo)); - #else - if(builtin_isnan(x) || builtin_isnan(y)) - return expr(std::numeric_limits::quiet_NaN()); - bool sign = builtin_signbit(x), qsign = static_cast(sign^builtin_signbit(y)); - float ax = std::fabs(x), ay = std::fabs(y); - if(ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) - return expr(std::numeric_limits::quiet_NaN()); - if(ay >= 65536.0f) - return expr(x); - if(ax == ay) - return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f); - ax = std::fmod(ax, 8.0f*ay); - int cquo = 0; - if(ax >= 4.0f * ay) - { - ax -= 4.0f * ay; - cquo += 4; - } - if(ax >= 2.0f * ay) - { - ax -= 2.0f * ay; - cquo += 2; - } - float y2 = 0.5f * ay; - if(ax > y2) - { - ax -= ay; - ++cquo; - if(ax >= y2) - { - ax -= ay; - ++cquo; - } - } - return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax); - #endif - } - - /// Positive difference implementation. - /// \param x first operand - /// \param y second operand - /// \return Positive difference stored in single-precision - static expr fdim(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::fdim(x, y)); - #else - return expr((x<=y) ? 0.0f : (x-y)); - #endif - } - - /// Fused multiply-add implementation. - /// \param x first operand - /// \param y second operand - /// \param z third operand - /// \return \a x * \a y + \a z stored in single-precision - static expr fma(float x, float y, float z) - { - #if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF) - return expr(std::fma(x, y, z)); - #else - return expr(x*y+z); - #endif - } - - /// Get NaN. - /// \return Half-precision quiet NaN - static half nanh() { return half(binary, 0x7FFF); } - - /// Exponential implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr exp(float arg) { return expr(std::exp(arg)); } - - /// Exponential implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr expm1(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::expm1(arg)); - #else - return expr(static_cast(std::exp(static_cast(arg))-1.0)); - #endif - } - - /// Binary exponential implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr exp2(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::exp2(arg)); - #else - return expr(static_cast(std::exp(arg*0.69314718055994530941723212145818))); - #endif - } - - /// Logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log(float arg) { return expr(std::log(arg)); } - - /// Common logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log10(float arg) { return expr(std::log10(arg)); } - - /// Logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log1p(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::log1p(arg)); - #else - return expr(static_cast(std::log(1.0+arg))); - #endif - } - - /// Binary logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr log2(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::log2(arg)); - #else - return expr(static_cast(std::log(static_cast(arg))*1.4426950408889634073599246810019)); - #endif - } - - /// Square root implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr sqrt(float arg) { return expr(std::sqrt(arg)); } - - /// Cubic root implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr cbrt(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::cbrt(arg)); - #else - if(builtin_isnan(arg) || builtin_isinf(arg)) - return expr(arg); - return expr(builtin_signbit(arg) ? -static_cast(std::pow(-static_cast(arg), 1.0/3.0)) : - static_cast(std::pow(static_cast(arg), 1.0/3.0))); - #endif - } - - /// Hypotenuse implementation. - /// \param x first argument - /// \param y second argument - /// \return function value stored in single-preicision - static expr hypot(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::hypot(x, y)); - #else - return expr((builtin_isinf(x) || builtin_isinf(y)) ? std::numeric_limits::infinity() : - static_cast(std::sqrt(static_cast(x)*x+static_cast(y)*y))); - #endif - } - - /// Power implementation. - /// \param base value to exponentiate - /// \param exp power to expontiate to - /// \return function value stored in single-preicision - static expr pow(float base, float exp) { return expr(std::pow(base, exp)); } - - /// Sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr sin(float arg) { return expr(std::sin(arg)); } - - /// Cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr cos(float arg) { return expr(std::cos(arg)); } - - /// Tan implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr tan(float arg) { return expr(std::tan(arg)); } - - /// Arc sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr asin(float arg) { return expr(std::asin(arg)); } - - /// Arc cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr acos(float arg) { return expr(std::acos(arg)); } - - /// Arc tangent implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr atan(float arg) { return expr(std::atan(arg)); } - - /// Arc tangent implementation. - /// \param x first argument - /// \param y second argument - /// \return function value stored in single-preicision - static expr atan2(float x, float y) { return expr(std::atan2(x, y)); } - - /// Hyperbolic sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr sinh(float arg) { return expr(std::sinh(arg)); } - - /// Hyperbolic cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr cosh(float arg) { return expr(std::cosh(arg)); } - - /// Hyperbolic tangent implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr tanh(float arg) { return expr(std::tanh(arg)); } - - /// Hyperbolic area sine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr asinh(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::asinh(arg)); - #else - return expr((arg==-std::numeric_limits::infinity()) ? arg : static_cast(std::log(arg+std::sqrt(arg*arg+1.0)))); - #endif - } - - /// Hyperbolic area cosine implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr acosh(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::acosh(arg)); - #else - return expr((arg<-1.0f) ? std::numeric_limits::quiet_NaN() : static_cast(std::log(arg+std::sqrt(arg*arg-1.0)))); - #endif - } - - /// Hyperbolic area tangent implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr atanh(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::atanh(arg)); - #else - return expr(static_cast(0.5*std::log((1.0+arg)/(1.0-arg)))); - #endif - } - - /// Error function implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr erf(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::erf(arg)); - #else - return expr(static_cast(erf(static_cast(arg)))); - #endif - } - - /// Complementary implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr erfc(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::erfc(arg)); - #else - return expr(static_cast(1.0-erf(static_cast(arg)))); - #endif - } - - /// Gamma logarithm implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr lgamma(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::lgamma(arg)); - #else - if(builtin_isinf(arg)) - return expr(std::numeric_limits::infinity()); - if(arg < 0.0f) - { - float i, f = std::modf(-arg, &i); - if(f == 0.0f) - return expr(std::numeric_limits::infinity()); - return expr(static_cast(1.1447298858494001741434273513531- - std::log(std::abs(std::sin(3.1415926535897932384626433832795*f)))-lgamma(1.0-arg))); - } - return expr(static_cast(lgamma(static_cast(arg)))); - #endif - } - - /// Gamma implementation. - /// \param arg function argument - /// \return function value stored in single-preicision - static expr tgamma(float arg) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::tgamma(arg)); - #else - if(arg == 0.0f) - return builtin_signbit(arg) ? expr(-std::numeric_limits::infinity()) : expr(std::numeric_limits::infinity()); - if(arg < 0.0f) - { - float i, f = std::modf(-arg, &i); - if(f == 0.0f) - return expr(std::numeric_limits::quiet_NaN()); - double value = 3.1415926535897932384626433832795 / (std::sin(3.1415926535897932384626433832795*f)*std::exp(lgamma(1.0-arg))); - return expr(static_cast((std::fmod(i, 2.0f)==0.0f) ? -value : value)); - } - if(builtin_isinf(arg)) - return expr(arg); - return expr(static_cast(std::exp(lgamma(static_cast(arg))))); - #endif - } - - /// Floor implementation. - /// \param arg value to round - /// \return rounded value - static half floor(half arg) { return half(binary, round_half(arg.data_)); } - - /// Ceiling implementation. - /// \param arg value to round - /// \return rounded value - static half ceil(half arg) { return half(binary, round_half(arg.data_)); } - - /// Truncation implementation. - /// \param arg value to round - /// \return rounded value - static half trunc(half arg) { return half(binary, round_half(arg.data_)); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static half round(half arg) { return half(binary, round_half_up(arg.data_)); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long lround(half arg) { return detail::half2int_up(arg.data_); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static half rint(half arg) { return half(binary, round_half(arg.data_)); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long lrint(half arg) { return detail::half2int(arg.data_); } - - #if HALF_ENABLE_CPP11_LONG_LONG - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long long llround(half arg) { return detail::half2int_up(arg.data_); } - - /// Nearest integer implementation. - /// \param arg value to round - /// \return rounded value - static long long llrint(half arg) { return detail::half2int(arg.data_); } - #endif - - /// Decompression implementation. - /// \param arg number to decompress - /// \param exp address to store exponent at - /// \return normalized significant - static half frexp(half arg, int *exp) - { - int m = arg.data_ & 0x7FFF, e = -14; - if(m >= 0x7C00 || !m) - return *exp = 0, arg; - for(; m<0x400; m<<=1,--e) ; - return *exp = e+(m>>10), half(binary, (arg.data_&0x8000)|0x3800|(m&0x3FF)); - } - - /// Decompression implementation. - /// \param arg number to decompress - /// \param iptr address to store integer part at - /// \return fractional part - static half modf(half arg, half *iptr) - { - unsigned int e = arg.data_ & 0x7FFF; - if(e >= 0x6400) - return *iptr = arg, half(binary, arg.data_&(0x8000U|-(e>0x7C00))); - if(e < 0x3C00) - return iptr->data_ = arg.data_ & 0x8000, arg; - e >>= 10; - unsigned int mask = (1<<(25-e)) - 1, m = arg.data_ & mask; - iptr->data_ = arg.data_ & ~mask; - if(!m) - return half(binary, arg.data_&0x8000); - for(; m<0x400; m<<=1,--e) ; - return half(binary, static_cast((arg.data_&0x8000)|(e<<10)|(m&0x3FF))); - } - - /// Scaling implementation. - /// \param arg number to scale - /// \param exp power of two to scale by - /// \return scaled number - static half scalbln(half arg, long exp) - { - unsigned int m = arg.data_ & 0x7FFF; - if(m >= 0x7C00 || !m) - return arg; - for(; m<0x400; m<<=1,--exp) ; - exp += m >> 10; - uint16 value = arg.data_ & 0x8000; - if(exp > 30) - { - if(half::round_style == std::round_toward_zero) - value |= 0x7BFF; - else if(half::round_style == std::round_toward_infinity) - value |= 0x7C00 - (value>>15); - else if(half::round_style == std::round_toward_neg_infinity) - value |= 0x7BFF + (value>>15); - else - value |= 0x7C00; - } - else if(exp > 0) - value |= (exp<<10) | (m&0x3FF); - else if(exp > -11) - { - m = (m&0x3FF) | 0x400; - if(half::round_style == std::round_to_nearest) - { - m += 1 << -exp; - #if HALF_ROUND_TIES_TO_EVEN - m -= (m>>(1-exp)) & 1; - #endif - } - else if(half::round_style == std::round_toward_infinity) - m += ((value>>15)-1) & ((1<<(1-exp))-1U); - else if(half::round_style == std::round_toward_neg_infinity) - m += -(value>>15) & ((1<<(1-exp))-1U); - value |= m >> (1-exp); - } - else if(half::round_style == std::round_toward_infinity) - value -= (value>>15) - 1; - else if(half::round_style == std::round_toward_neg_infinity) - value += value >> 15; - return half(binary, value); - } - - /// Exponent implementation. - /// \param arg number to query - /// \return floating point exponent - static int ilogb(half arg) - { - int abs = arg.data_ & 0x7FFF; - if(!abs) - return FP_ILOGB0; - if(abs < 0x7C00) - { - int exp = (abs>>10) - 15; - if(abs < 0x400) - for(; abs<0x200; abs<<=1,--exp) ; - return exp; - } - if(abs > 0x7C00) - return FP_ILOGBNAN; - return INT_MAX; - } - - /// Exponent implementation. - /// \param arg number to query - /// \return floating point exponent - static half logb(half arg) - { - int abs = arg.data_ & 0x7FFF; - if(!abs) - return half(binary, 0xFC00); - if(abs < 0x7C00) - { - int exp = (abs>>10) - 15; - if(abs < 0x400) - for(; abs<0x200; abs<<=1,--exp) ; - uint16 bits = (exp<0) << 15; - if(exp) - { - unsigned int m = std::abs(exp) << 6, e = 18; - for(; m<0x400; m<<=1,--e) ; - bits |= (e<<10) + m; - } - return half(binary, bits); - } - if(abs > 0x7C00) - return arg; - return half(binary, 0x7C00); - } - - /// Enumeration implementation. - /// \param from number to increase/decrease - /// \param to direction to enumerate into - /// \return next representable number - static half nextafter(half from, half to) - { - uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; - if(fabs > 0x7C00) - return from; - if(tabs > 0x7C00 || from.data_ == to.data_ || !(fabs|tabs)) - return to; - if(!fabs) - return half(binary, (to.data_&0x8000)+1); - bool lt = ((fabs==from.data_) ? static_cast(fabs) : -static_cast(fabs)) < - ((tabs==to.data_) ? static_cast(tabs) : -static_cast(tabs)); - return half(binary, from.data_+(((from.data_>>15)^static_cast(lt))<<1)-1); - } - - /// Enumeration implementation. - /// \param from number to increase/decrease - /// \param to direction to enumerate into - /// \return next representable number - static half nexttoward(half from, long double to) - { - if(isnan(from)) - return from; - long double lfrom = static_cast(from); - if(builtin_isnan(to) || lfrom == to) - return half(static_cast(to)); - if(!(from.data_&0x7FFF)) - return half(binary, (static_cast(builtin_signbit(to))<<15)+1); - return half(binary, from.data_+(((from.data_>>15)^static_cast(lfrom0x3FF) ? ((abs>=0x7C00) ? ((abs>0x7C00) ? FP_NAN : FP_INFINITE) : FP_NORMAL) :FP_SUBNORMAL) : FP_ZERO; - } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if finite number - /// \retval false else - static bool isfinite(half arg) { return (arg.data_&0x7C00) != 0x7C00; } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if infinite number - /// \retval false else - static bool isinf(half arg) { return (arg.data_&0x7FFF) == 0x7C00; } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if not a number - /// \retval false else - static bool isnan(half arg) { return (arg.data_&0x7FFF) > 0x7C00; } - - /// Classification implementation. - /// \param arg value to classify - /// \retval true if normal number - /// \retval false else - static bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); } - - /// Sign bit implementation. - /// \param arg value to check - /// \retval true if signed - /// \retval false if unsigned - static bool signbit(half arg) { return (arg.data_&0x8000) != 0; } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if operands equal - /// \retval false else - static bool isequal(half x, half y) { return (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)) && !isnan(x); } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if operands not equal - /// \retval false else - static bool isnotequal(half x, half y) { return (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)) || isnan(x); } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x > \a y - /// \retval false else - static bool isgreater(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x >= \a y - /// \retval false else - static bool isgreaterequal(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) >= ((yabs==y.data_) ? yabs : -yabs)); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x < \a y - /// \retval false else - static bool isless(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x <= \a y - /// \retval false else - static bool islessequal(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - return xabs<=0x7C00 && yabs<=0x7C00 && (((xabs==x.data_) ? xabs : -xabs) <= ((yabs==y.data_) ? yabs : -yabs)); - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if either \a x > \a y nor \a x < \a y - /// \retval false else - static bool islessgreater(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - if(xabs > 0x7C00 || yabs > 0x7C00) - return false; - int a = (xabs==x.data_) ? xabs : -xabs, b = (yabs==y.data_) ? yabs : -yabs; - return a < b || a > b; - } - - /// Comparison implementation. - /// \param x first operand - /// \param y second operand - /// \retval true if operand unordered - /// \retval false else - static bool isunordered(half x, half y) { return isnan(x) || isnan(y); } - - private: - static double erf(double arg) - { - if(builtin_isinf(arg)) - return (arg<0.0) ? -1.0 : 1.0; - double x2 = arg * arg, ax2 = 0.147 * x2, value = std::sqrt(1.0-std::exp(-x2*(1.2732395447351626861510701069801+ax2)/(1.0+ax2))); - return builtin_signbit(arg) ? -value : value; - } - - static double lgamma(double arg) - { - double v = 1.0; - for(; arg<8.0; ++arg) v *= arg; - double w = 1.0 / (arg*arg); - return (((((((-0.02955065359477124183006535947712*w+0.00641025641025641025641025641026)*w+ - -0.00191752691752691752691752691753)*w+8.4175084175084175084175084175084e-4)*w+ - -5.952380952380952380952380952381e-4)*w+7.9365079365079365079365079365079e-4)*w+ - -0.00277777777777777777777777777778)*w+0.08333333333333333333333333333333)/arg + - 0.91893853320467274178032973640562 - std::log(v) - arg + (arg-0.5) * std::log(arg); - } - }; - - /// Wrapper for unary half-precision functions needing specialization for individual argument types. - /// \tparam T argument type - template struct unary_specialized - { - /// Negation implementation. - /// \param arg value to negate - /// \return negated value - static HALF_CONSTEXPR half negate(half arg) { return half(binary, arg.data_^0x8000); } - - /// Absolute value implementation. - /// \param arg function argument - /// \return absolute value - static half fabs(half arg) { return half(binary, arg.data_&0x7FFF); } - }; - template<> struct unary_specialized - { - static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); } - static expr fabs(float arg) { return expr(std::fabs(arg)); } - }; - - /// Wrapper for binary half-precision functions needing specialization for individual argument types. - /// \tparam T first argument type - /// \tparam U first argument type - template struct binary_specialized - { - /// Minimum implementation. - /// \param x first operand - /// \param y second operand - /// \return minimum value - static expr fmin(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::fmin(x, y)); - #else - if(builtin_isnan(x)) - return expr(y); - if(builtin_isnan(y)) - return expr(x); - return expr(std::min(x, y)); - #endif - } - - /// Maximum implementation. - /// \param x first operand - /// \param y second operand - /// \return maximum value - static expr fmax(float x, float y) - { - #if HALF_ENABLE_CPP11_CMATH - return expr(std::fmax(x, y)); - #else - if(builtin_isnan(x)) - return expr(y); - if(builtin_isnan(y)) - return expr(x); - return expr(std::max(x, y)); - #endif - } - }; - template<> struct binary_specialized - { - static half fmin(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - if(xabs > 0x7C00) - return y; - if(yabs > 0x7C00) - return x; - return (((xabs==x.data_) ? xabs : -xabs) > ((yabs==y.data_) ? yabs : -yabs)) ? y : x; - } - static half fmax(half x, half y) - { - int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; - if(xabs > 0x7C00) - return y; - if(yabs > 0x7C00) - return x; - return (((xabs==x.data_) ? xabs : -xabs) < ((yabs==y.data_) ? yabs : -yabs)) ? y : x; - } - }; - - /// Helper class for half casts. - /// This class template has to be specialized for all valid cast argument to define an appropriate static `cast` member - /// function and a corresponding `type` member denoting its return type. - /// \tparam T destination type - /// \tparam U source type - /// \tparam R rounding mode to use - template struct half_caster {}; - template struct half_caster - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, "half_cast from non-arithmetic type unsupported"); - #endif - - static half cast(U arg) { return cast_impl(arg, is_float()); }; - - private: - static half cast_impl(U arg, true_type) { return half(binary, float2half(arg)); } - static half cast_impl(U arg, false_type) { return half(binary, int2half(arg)); } - }; - template struct half_caster - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); - #endif - - static T cast(half arg) { return cast_impl(arg, is_float()); } - - private: - static T cast_impl(half arg, true_type) { return half2float(arg.data_); } - static T cast_impl(half arg, false_type) { return half2int(arg.data_); } - }; - template struct half_caster - { - #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS - static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); - #endif - - static T cast(expr arg) { return cast_impl(arg, is_float()); } - - private: - static T cast_impl(float arg, true_type) { return static_cast(arg); } - static T cast_impl(half arg, false_type) { return half2int(arg.data_); } - }; - template struct half_caster - { - static half cast(half arg) { return arg; } - }; - template struct half_caster : half_caster {}; - - /// \name Comparison operators - /// \{ - - /// Comparison for equality. - /// \param x first operand - /// \param y second operand - /// \retval true if operands equal - /// \retval false else - template typename enable::type operator==(T x, U y) { return functions::isequal(x, y); } - - /// Comparison for inequality. - /// \param x first operand - /// \param y second operand - /// \retval true if operands not equal - /// \retval false else - template typename enable::type operator!=(T x, U y) { return functions::isnotequal(x, y); } - - /// Comparison for less than. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x less than \a y - /// \retval false else - template typename enable::type operator<(T x, U y) { return functions::isless(x, y); } - - /// Comparison for greater than. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x greater than \a y - /// \retval false else - template typename enable::type operator>(T x, U y) { return functions::isgreater(x, y); } - - /// Comparison for less equal. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x less equal \a y - /// \retval false else - template typename enable::type operator<=(T x, U y) { return functions::islessequal(x, y); } - - /// Comparison for greater equal. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x greater equal \a y - /// \retval false else - template typename enable::type operator>=(T x, U y) { return functions::isgreaterequal(x, y); } - - /// \} - /// \name Arithmetic operators - /// \{ - - /// Add halfs. - /// \param x left operand - /// \param y right operand - /// \return sum of half expressions - template typename enable::type operator+(T x, U y) { return functions::plus(x, y); } - - /// Subtract halfs. - /// \param x left operand - /// \param y right operand - /// \return difference of half expressions - template typename enable::type operator-(T x, U y) { return functions::minus(x, y); } - - /// Multiply halfs. - /// \param x left operand - /// \param y right operand - /// \return product of half expressions - template typename enable::type operator*(T x, U y) { return functions::multiplies(x, y); } - - /// Divide halfs. - /// \param x left operand - /// \param y right operand - /// \return quotient of half expressions - template typename enable::type operator/(T x, U y) { return functions::divides(x, y); } - - /// Identity. - /// \param arg operand - /// \return uncahnged operand - template HALF_CONSTEXPR typename enable::type operator+(T arg) { return arg; } - - /// Negation. - /// \param arg operand - /// \return negated operand - template HALF_CONSTEXPR typename enable::type operator-(T arg) { return unary_specialized::negate(arg); } - - /// \} - /// \name Input and output - /// \{ - - /// Output operator. - /// \param out output stream to write into - /// \param arg half expression to write - /// \return reference to output stream - template typename enable&,T>::type - operator<<(std::basic_ostream &out, T arg) { return functions::write(out, arg); } - - /// Input operator. - /// \param in input stream to read from - /// \param arg half to read into - /// \return reference to input stream - template std::basic_istream& - operator>>(std::basic_istream &in, half &arg) { return functions::read(in, arg); } - - /// \} - /// \name Basic mathematical operations - /// \{ - - /// Absolute value. - /// \param arg operand - /// \return absolute value of \a arg -// template typename enable::type abs(T arg) { return unary_specialized::fabs(arg); } - inline half abs(half arg) { return unary_specialized::fabs(arg); } - inline expr abs(expr arg) { return unary_specialized::fabs(arg); } - - /// Absolute value. - /// \param arg operand - /// \return absolute value of \a arg -// template typename enable::type fabs(T arg) { return unary_specialized::fabs(arg); } - inline half fabs(half arg) { return unary_specialized::fabs(arg); } - inline expr fabs(expr arg) { return unary_specialized::fabs(arg); } - - /// Remainder of division. - /// \param x first operand - /// \param y second operand - /// \return remainder of floating point division. -// template typename enable::type fmod(T x, U y) { return functions::fmod(x, y); } - inline expr fmod(half x, half y) { return functions::fmod(x, y); } - inline expr fmod(half x, expr y) { return functions::fmod(x, y); } - inline expr fmod(expr x, half y) { return functions::fmod(x, y); } - inline expr fmod(expr x, expr y) { return functions::fmod(x, y); } - - /// Remainder of division. - /// \param x first operand - /// \param y second operand - /// \return remainder of floating point division. -// template typename enable::type remainder(T x, U y) { return functions::remainder(x, y); } - inline expr remainder(half x, half y) { return functions::remainder(x, y); } - inline expr remainder(half x, expr y) { return functions::remainder(x, y); } - inline expr remainder(expr x, half y) { return functions::remainder(x, y); } - inline expr remainder(expr x, expr y) { return functions::remainder(x, y); } - - /// Remainder of division. - /// \param x first operand - /// \param y second operand - /// \param quo address to store some bits of quotient at - /// \return remainder of floating point division. -// template typename enable::type remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); } - inline expr remquo(half x, half y, int *quo) { return functions::remquo(x, y, quo); } - inline expr remquo(half x, expr y, int *quo) { return functions::remquo(x, y, quo); } - inline expr remquo(expr x, half y, int *quo) { return functions::remquo(x, y, quo); } - inline expr remquo(expr x, expr y, int *quo) { return functions::remquo(x, y, quo); } - - /// Fused multiply add. - /// \param x first operand - /// \param y second operand - /// \param z third operand - /// \return ( \a x * \a y ) + \a z rounded as one operation. -// template typename enable::type fma(T x, U y, V z) { return functions::fma(x, y, z); } - inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); } - inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); } - inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); } - inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); } - inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); } - inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); } - inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); } - inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); } - - /// Maximum of half expressions. - /// \param x first operand - /// \param y second operand - /// \return maximum of operands -// template typename result::type fmax(T x, U y) { return binary_specialized::fmax(x, y); } - inline half fmax(half x, half y) { return binary_specialized::fmax(x, y); } - inline expr fmax(half x, expr y) { return binary_specialized::fmax(x, y); } - inline expr fmax(expr x, half y) { return binary_specialized::fmax(x, y); } - inline expr fmax(expr x, expr y) { return binary_specialized::fmax(x, y); } - - /// Minimum of half expressions. - /// \param x first operand - /// \param y second operand - /// \return minimum of operands -// template typename result::type fmin(T x, U y) { return binary_specialized::fmin(x, y); } - inline half fmin(half x, half y) { return binary_specialized::fmin(x, y); } - inline expr fmin(half x, expr y) { return binary_specialized::fmin(x, y); } - inline expr fmin(expr x, half y) { return binary_specialized::fmin(x, y); } - inline expr fmin(expr x, expr y) { return binary_specialized::fmin(x, y); } - - /// Positive difference. - /// \param x first operand - /// \param y second operand - /// \return \a x - \a y or 0 if difference negative -// template typename enable::type fdim(T x, U y) { return functions::fdim(x, y); } - inline expr fdim(half x, half y) { return functions::fdim(x, y); } - inline expr fdim(half x, expr y) { return functions::fdim(x, y); } - inline expr fdim(expr x, half y) { return functions::fdim(x, y); } - inline expr fdim(expr x, expr y) { return functions::fdim(x, y); } - - /// Get NaN value. - /// \return quiet NaN - inline half nanh(const char*) { return functions::nanh(); } - - /// \} - /// \name Exponential functions - /// \{ - - /// Exponential function. - /// \param arg function argument - /// \return e raised to \a arg -// template typename enable::type exp(T arg) { return functions::exp(arg); } - inline expr exp(half arg) { return functions::exp(arg); } - inline expr exp(expr arg) { return functions::exp(arg); } - - /// Exponential minus one. - /// \param arg function argument - /// \return e raised to \a arg subtracted by 1 -// template typename enable::type expm1(T arg) { return functions::expm1(arg); } - inline expr expm1(half arg) { return functions::expm1(arg); } - inline expr expm1(expr arg) { return functions::expm1(arg); } - - /// Binary exponential. - /// \param arg function argument - /// \return 2 raised to \a arg -// template typename enable::type exp2(T arg) { return functions::exp2(arg); } - inline expr exp2(half arg) { return functions::exp2(arg); } - inline expr exp2(expr arg) { return functions::exp2(arg); } - - /// Natural logorithm. - /// \param arg function argument - /// \return logarithm of \a arg to base e -// template typename enable::type log(T arg) { return functions::log(arg); } - inline expr log(half arg) { return functions::log(arg); } - inline expr log(expr arg) { return functions::log(arg); } - - /// Common logorithm. - /// \param arg function argument - /// \return logarithm of \a arg to base 10 -// template typename enable::type log10(T arg) { return functions::log10(arg); } - inline expr log10(half arg) { return functions::log10(arg); } - inline expr log10(expr arg) { return functions::log10(arg); } - - /// Natural logorithm. - /// \param arg function argument - /// \return logarithm of \a arg plus 1 to base e -// template typename enable::type log1p(T arg) { return functions::log1p(arg); } - inline expr log1p(half arg) { return functions::log1p(arg); } - inline expr log1p(expr arg) { return functions::log1p(arg); } - - /// Binary logorithm. - /// \param arg function argument - /// \return logarithm of \a arg to base 2 -// template typename enable::type log2(T arg) { return functions::log2(arg); } - inline expr log2(half arg) { return functions::log2(arg); } - inline expr log2(expr arg) { return functions::log2(arg); } - - /// \} - /// \name Power functions - /// \{ - - /// Square root. - /// \param arg function argument - /// \return square root of \a arg -// template typename enable::type sqrt(T arg) { return functions::sqrt(arg); } - inline expr sqrt(half arg) { return functions::sqrt(arg); } - inline expr sqrt(expr arg) { return functions::sqrt(arg); } - - /// Cubic root. - /// \param arg function argument - /// \return cubic root of \a arg -// template typename enable::type cbrt(T arg) { return functions::cbrt(arg); } - inline expr cbrt(half arg) { return functions::cbrt(arg); } - inline expr cbrt(expr arg) { return functions::cbrt(arg); } - - /// Hypotenuse function. - /// \param x first argument - /// \param y second argument - /// \return square root of sum of squares without internal over- or underflows -// template typename enable::type hypot(T x, U y) { return functions::hypot(x, y); } - inline expr hypot(half x, half y) { return functions::hypot(x, y); } - inline expr hypot(half x, expr y) { return functions::hypot(x, y); } - inline expr hypot(expr x, half y) { return functions::hypot(x, y); } - inline expr hypot(expr x, expr y) { return functions::hypot(x, y); } - - /// Power function. - /// \param base first argument - /// \param exp second argument - /// \return \a base raised to \a exp -// template typename enable::type pow(T base, U exp) { return functions::pow(base, exp); } - inline expr pow(half base, half exp) { return functions::pow(base, exp); } - inline expr pow(half base, expr exp) { return functions::pow(base, exp); } - inline expr pow(expr base, half exp) { return functions::pow(base, exp); } - inline expr pow(expr base, expr exp) { return functions::pow(base, exp); } - - /// \} - /// \name Trigonometric functions - /// \{ - - /// Sine function. - /// \param arg function argument - /// \return sine value of \a arg -// template typename enable::type sin(T arg) { return functions::sin(arg); } - inline expr sin(half arg) { return functions::sin(arg); } - inline expr sin(expr arg) { return functions::sin(arg); } - - /// Cosine function. - /// \param arg function argument - /// \return cosine value of \a arg -// template typename enable::type cos(T arg) { return functions::cos(arg); } - inline expr cos(half arg) { return functions::cos(arg); } - inline expr cos(expr arg) { return functions::cos(arg); } - - /// Tangent function. - /// \param arg function argument - /// \return tangent value of \a arg -// template typename enable::type tan(T arg) { return functions::tan(arg); } - inline expr tan(half arg) { return functions::tan(arg); } - inline expr tan(expr arg) { return functions::tan(arg); } - - /// Arc sine. - /// \param arg function argument - /// \return arc sine value of \a arg -// template typename enable::type asin(T arg) { return functions::asin(arg); } - inline expr asin(half arg) { return functions::asin(arg); } - inline expr asin(expr arg) { return functions::asin(arg); } - - /// Arc cosine function. - /// \param arg function argument - /// \return arc cosine value of \a arg -// template typename enable::type acos(T arg) { return functions::acos(arg); } - inline expr acos(half arg) { return functions::acos(arg); } - inline expr acos(expr arg) { return functions::acos(arg); } - - /// Arc tangent function. - /// \param arg function argument - /// \return arc tangent value of \a arg -// template typename enable::type atan(T arg) { return functions::atan(arg); } - inline expr atan(half arg) { return functions::atan(arg); } - inline expr atan(expr arg) { return functions::atan(arg); } - - /// Arc tangent function. - /// \param x first argument - /// \param y second argument - /// \return arc tangent value -// template typename enable::type atan2(T x, U y) { return functions::atan2(x, y); } - inline expr atan2(half x, half y) { return functions::atan2(x, y); } - inline expr atan2(half x, expr y) { return functions::atan2(x, y); } - inline expr atan2(expr x, half y) { return functions::atan2(x, y); } - inline expr atan2(expr x, expr y) { return functions::atan2(x, y); } - - /// \} - /// \name Hyperbolic functions - /// \{ - - /// Hyperbolic sine. - /// \param arg function argument - /// \return hyperbolic sine value of \a arg -// template typename enable::type sinh(T arg) { return functions::sinh(arg); } - inline expr sinh(half arg) { return functions::sinh(arg); } - inline expr sinh(expr arg) { return functions::sinh(arg); } - - /// Hyperbolic cosine. - /// \param arg function argument - /// \return hyperbolic cosine value of \a arg -// template typename enable::type cosh(T arg) { return functions::cosh(arg); } - inline expr cosh(half arg) { return functions::cosh(arg); } - inline expr cosh(expr arg) { return functions::cosh(arg); } - - /// Hyperbolic tangent. - /// \param arg function argument - /// \return hyperbolic tangent value of \a arg -// template typename enable::type tanh(T arg) { return functions::tanh(arg); } - inline expr tanh(half arg) { return functions::tanh(arg); } - inline expr tanh(expr arg) { return functions::tanh(arg); } - - /// Hyperbolic area sine. - /// \param arg function argument - /// \return area sine value of \a arg -// template typename enable::type asinh(T arg) { return functions::asinh(arg); } - inline expr asinh(half arg) { return functions::asinh(arg); } - inline expr asinh(expr arg) { return functions::asinh(arg); } - - /// Hyperbolic area cosine. - /// \param arg function argument - /// \return area cosine value of \a arg -// template typename enable::type acosh(T arg) { return functions::acosh(arg); } - inline expr acosh(half arg) { return functions::acosh(arg); } - inline expr acosh(expr arg) { return functions::acosh(arg); } - - /// Hyperbolic area tangent. - /// \param arg function argument - /// \return area tangent value of \a arg -// template typename enable::type atanh(T arg) { return functions::atanh(arg); } - inline expr atanh(half arg) { return functions::atanh(arg); } - inline expr atanh(expr arg) { return functions::atanh(arg); } - - /// \} - /// \name Error and gamma functions - /// \{ - - /// Error function. - /// \param arg function argument - /// \return error function value of \a arg -// template typename enable::type erf(T arg) { return functions::erf(arg); } - inline expr erf(half arg) { return functions::erf(arg); } - inline expr erf(expr arg) { return functions::erf(arg); } - - /// Complementary error function. - /// \param arg function argument - /// \return 1 minus error function value of \a arg -// template typename enable::type erfc(T arg) { return functions::erfc(arg); } - inline expr erfc(half arg) { return functions::erfc(arg); } - inline expr erfc(expr arg) { return functions::erfc(arg); } - - /// Natural logarithm of gamma function. - /// \param arg function argument - /// \return natural logarith of gamma function for \a arg -// template typename enable::type lgamma(T arg) { return functions::lgamma(arg); } - inline expr lgamma(half arg) { return functions::lgamma(arg); } - inline expr lgamma(expr arg) { return functions::lgamma(arg); } - - /// Gamma function. - /// \param arg function argument - /// \return gamma function value of \a arg -// template typename enable::type tgamma(T arg) { return functions::tgamma(arg); } - inline expr tgamma(half arg) { return functions::tgamma(arg); } - inline expr tgamma(expr arg) { return functions::tgamma(arg); } - - /// \} - /// \name Rounding - /// \{ - - /// Nearest integer not less than half value. - /// \param arg half to round - /// \return nearest integer not less than \a arg -// template typename enable::type ceil(T arg) { return functions::ceil(arg); } - inline half ceil(half arg) { return functions::ceil(arg); } - inline half ceil(expr arg) { return functions::ceil(arg); } - - /// Nearest integer not greater than half value. - /// \param arg half to round - /// \return nearest integer not greater than \a arg -// template typename enable::type floor(T arg) { return functions::floor(arg); } - inline half floor(half arg) { return functions::floor(arg); } - inline half floor(expr arg) { return functions::floor(arg); } - - /// Nearest integer not greater in magnitude than half value. - /// \param arg half to round - /// \return nearest integer not greater in magnitude than \a arg -// template typename enable::type trunc(T arg) { return functions::trunc(arg); } - inline half trunc(half arg) { return functions::trunc(arg); } - inline half trunc(expr arg) { return functions::trunc(arg); } - - /// Nearest integer. - /// \param arg half to round - /// \return nearest integer, rounded away from zero in half-way cases -// template typename enable::type round(T arg) { return functions::round(arg); } - inline half round(half arg) { return functions::round(arg); } - inline half round(expr arg) { return functions::round(arg); } - - /// Nearest integer. - /// \param arg half to round - /// \return nearest integer, rounded away from zero in half-way cases -// template typename enable::type lround(T arg) { return functions::lround(arg); } - inline long lround(half arg) { return functions::lround(arg); } - inline long lround(expr arg) { return functions::lround(arg); } - - /// Nearest integer using half's internal rounding mode. - /// \param arg half expression to round - /// \return nearest integer using default rounding mode -// template typename enable::type nearbyint(T arg) { return functions::nearbyint(arg); } - inline half nearbyint(half arg) { return functions::rint(arg); } - inline half nearbyint(expr arg) { return functions::rint(arg); } - - /// Nearest integer using half's internal rounding mode. - /// \param arg half expression to round - /// \return nearest integer using default rounding mode -// template typename enable::type rint(T arg) { return functions::rint(arg); } - inline half rint(half arg) { return functions::rint(arg); } - inline half rint(expr arg) { return functions::rint(arg); } - - /// Nearest integer using half's internal rounding mode. - /// \param arg half expression to round - /// \return nearest integer using default rounding mode -// template typename enable::type lrint(T arg) { return functions::lrint(arg); } - inline long lrint(half arg) { return functions::lrint(arg); } - inline long lrint(expr arg) { return functions::lrint(arg); } - #if HALF_ENABLE_CPP11_LONG_LONG - /// Nearest integer. - /// \param arg half to round - /// \return nearest integer, rounded away from zero in half-way cases -// template typename enable::type llround(T arg) { return functions::llround(arg); } - inline long long llround(half arg) { return functions::llround(arg); } - inline long long llround(expr arg) { return functions::llround(arg); } - - /// Nearest integer using half's internal rounding mode. - /// \param arg half expression to round - /// \return nearest integer using default rounding mode -// template typename enable::type llrint(T arg) { return functions::llrint(arg); } - inline long long llrint(half arg) { return functions::llrint(arg); } - inline long long llrint(expr arg) { return functions::llrint(arg); } - #endif - - /// \} - /// \name Floating point manipulation - /// \{ - - /// Decompress floating point number. - /// \param arg number to decompress - /// \param exp address to store exponent at - /// \return significant in range [0.5, 1) -// template typename enable::type frexp(T arg, int *exp) { return functions::frexp(arg, exp); } - inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); } - inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); } - - /// Multiply by power of two. - /// \param arg number to modify - /// \param exp power of two to multiply with - /// \return \a arg multplied by 2 raised to \a exp -// template typename enable::type ldexp(T arg, int exp) { return functions::scalbln(arg, exp); } - inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); } - inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); } - - /// Extract integer and fractional parts. - /// \param arg number to decompress - /// \param iptr address to store integer part at - /// \return fractional part -// template typename enable::type modf(T arg, half *iptr) { return functions::modf(arg, iptr); } - inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); } - inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); } - - /// Multiply by power of two. - /// \param arg number to modify - /// \param exp power of two to multiply with - /// \return \a arg multplied by 2 raised to \a exp -// template typename enable::type scalbn(T arg, int exp) { return functions::scalbln(arg, exp); } - inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); } - inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); } - - /// Multiply by power of two. - /// \param arg number to modify - /// \param exp power of two to multiply with - /// \return \a arg multplied by 2 raised to \a exp -// template typename enable::type scalbln(T arg, long exp) { return functions::scalbln(arg, exp); } - inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); } - inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); } - - /// Extract exponent. - /// \param arg number to query - /// \return floating point exponent - /// \retval FP_ILOGB0 for zero - /// \retval FP_ILOGBNAN for NaN - /// \retval MAX_INT for infinity -// template typename enable::type ilogb(T arg) { return functions::ilogb(arg); } - inline int ilogb(half arg) { return functions::ilogb(arg); } - inline int ilogb(expr arg) { return functions::ilogb(arg); } - - /// Extract exponent. - /// \param arg number to query - /// \return floating point exponent -// template typename enable::type logb(T arg) { return functions::logb(arg); } - inline half logb(half arg) { return functions::logb(arg); } - inline half logb(expr arg) { return functions::logb(arg); } - - /// Next representable value. - /// \param from value to compute next representable value for - /// \param to direction towards which to compute next value - /// \return next representable value after \a from in direction towards \a to -// template typename enable::type nextafter(T from, U to) { return functions::nextafter(from, to); } - inline half nextafter(half from, half to) { return functions::nextafter(from, to); } - inline half nextafter(half from, expr to) { return functions::nextafter(from, to); } - inline half nextafter(expr from, half to) { return functions::nextafter(from, to); } - inline half nextafter(expr from, expr to) { return functions::nextafter(from, to); } - - /// Next representable value. - /// \param from value to compute next representable value for - /// \param to direction towards which to compute next value - /// \return next representable value after \a from in direction towards \a to -// template typename enable::type nexttoward(T from, long double to) { return functions::nexttoward(from, to); } - inline half nexttoward(half from, long double to) { return functions::nexttoward(from, to); } - inline half nexttoward(expr from, long double to) { return functions::nexttoward(from, to); } - - /// Take sign. - /// \param x value to change sign for - /// \param y value to take sign from - /// \return value equal to \a x in magnitude and to \a y in sign -// template typename enable::type copysign(T x, U y) { return functions::copysign(x, y); } - inline half copysign(half x, half y) { return functions::copysign(x, y); } - inline half copysign(half x, expr y) { return functions::copysign(x, y); } - inline half copysign(expr x, half y) { return functions::copysign(x, y); } - inline half copysign(expr x, expr y) { return functions::copysign(x, y); } - - /// \} - /// \name Floating point classification - /// \{ - - - /// Classify floating point value. - /// \param arg number to classify - /// \retval FP_ZERO for positive and negative zero - /// \retval FP_SUBNORMAL for subnormal numbers - /// \retval FP_INFINITY for positive and negative infinity - /// \retval FP_NAN for NaNs - /// \retval FP_NORMAL for all other (normal) values -// template typename enable::type fpclassify(T arg) { return functions::fpclassify(arg); } - inline int fpclassify(half arg) { return functions::fpclassify(arg); } - inline int fpclassify(expr arg) { return functions::fpclassify(arg); } - - /// Check if finite number. - /// \param arg number to check - /// \retval true if neither infinity nor NaN - /// \retval false else -// template typename enable::type isfinite(T arg) { return functions::isfinite(arg); } - inline bool isfinite(half arg) { return functions::isfinite(arg); } - inline bool isfinite(expr arg) { return functions::isfinite(arg); } - - /// Check for infinity. - /// \param arg number to check - /// \retval true for positive or negative infinity - /// \retval false else -// template typename enable::type isinf(T arg) { return functions::isinf(arg); } - inline bool isinf(half arg) { return functions::isinf(arg); } - inline bool isinf(expr arg) { return functions::isinf(arg); } - - /// Check for NaN. - /// \param arg number to check - /// \retval true for NaNs - /// \retval false else -// template typename enable::type isnan(T arg) { return functions::isnan(arg); } - inline bool isnan(half arg) { return functions::isnan(arg); } - inline bool isnan(expr arg) { return functions::isnan(arg); } - - /// Check if normal number. - /// \param arg number to check - /// \retval true if normal number - /// \retval false if either subnormal, zero, infinity or NaN -// template typename enable::type isnormal(T arg) { return functions::isnormal(arg); } - inline bool isnormal(half arg) { return functions::isnormal(arg); } - inline bool isnormal(expr arg) { return functions::isnormal(arg); } - - /// Check sign. - /// \param arg number to check - /// \retval true for negative number - /// \retval false for positive number -// template typename enable::type signbit(T arg) { return functions::signbit(arg); } - inline bool signbit(half arg) { return functions::signbit(arg); } - inline bool signbit(expr arg) { return functions::signbit(arg); } - - /// \} - /// \name Comparison - /// \{ - - /// Comparison for greater than. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x greater than \a y - /// \retval false else -// template typename enable::type isgreater(T x, U y) { return functions::isgreater(x, y); } - inline bool isgreater(half x, half y) { return functions::isgreater(x, y); } - inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); } - inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); } - inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); } - - /// Comparison for greater equal. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x greater equal \a y - /// \retval false else -// template typename enable::type isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); } - inline bool isgreaterequal(half x, half y) { return functions::isgreaterequal(x, y); } - inline bool isgreaterequal(half x, expr y) { return functions::isgreaterequal(x, y); } - inline bool isgreaterequal(expr x, half y) { return functions::isgreaterequal(x, y); } - inline bool isgreaterequal(expr x, expr y) { return functions::isgreaterequal(x, y); } - - /// Comparison for less than. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x less than \a y - /// \retval false else -// template typename enable::type isless(T x, U y) { return functions::isless(x, y); } - inline bool isless(half x, half y) { return functions::isless(x, y); } - inline bool isless(half x, expr y) { return functions::isless(x, y); } - inline bool isless(expr x, half y) { return functions::isless(x, y); } - inline bool isless(expr x, expr y) { return functions::isless(x, y); } - - /// Comparison for less equal. - /// \param x first operand - /// \param y second operand - /// \retval true if \a x less equal \a y - /// \retval false else -// template typename enable::type islessequal(T x, U y) { return functions::islessequal(x, y); } - inline bool islessequal(half x, half y) { return functions::islessequal(x, y); } - inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); } - inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); } - inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); } - - /// Comarison for less or greater. - /// \param x first operand - /// \param y second operand - /// \retval true if either less or greater - /// \retval false else -// template typename enable::type islessgreater(T x, U y) { return functions::islessgreater(x, y); } - inline bool islessgreater(half x, half y) { return functions::islessgreater(x, y); } - inline bool islessgreater(half x, expr y) { return functions::islessgreater(x, y); } - inline bool islessgreater(expr x, half y) { return functions::islessgreater(x, y); } - inline bool islessgreater(expr x, expr y) { return functions::islessgreater(x, y); } - - /// Check if unordered. - /// \param x first operand - /// \param y second operand - /// \retval true if unordered (one or two NaN operands) - /// \retval false else -// template typename enable::type isunordered(T x, U y) { return functions::isunordered(x, y); } - inline bool isunordered(half x, half y) { return functions::isunordered(x, y); } - inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); } - inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); } - inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); } - - /// \name Casting - /// \{ - - /// Cast to or from half-precision floating point number. - /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted - /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. - /// It uses the default rounding mode. - /// - /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types - /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler - /// error and casting between [half](\ref half_float::half)s is just a no-op. - /// \tparam T destination type (half or built-in arithmetic type) - /// \tparam U source type (half or built-in arithmetic type) - /// \param arg value to cast - /// \return \a arg converted to destination type - template T half_cast(U arg) { return half_caster::cast(arg); } - - /// Cast to or from half-precision floating point number. - /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted - /// directly using the given rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. - /// - /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types - /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler - /// error and casting between [half](\ref half_float::half)s is just a no-op. - /// \tparam T destination type (half or built-in arithmetic type) - /// \tparam R rounding mode to use. - /// \tparam U source type (half or built-in arithmetic type) - /// \param arg value to cast - /// \return \a arg converted to destination type - template T half_cast(U arg) { return half_caster::cast(arg); } - /// \} - } - - using detail::operator==; - using detail::operator!=; - using detail::operator<; - using detail::operator>; - using detail::operator<=; - using detail::operator>=; - using detail::operator+; - using detail::operator-; - using detail::operator*; - using detail::operator/; - using detail::operator<<; - using detail::operator>>; - - using detail::abs; - using detail::fabs; - using detail::fmod; - using detail::remainder; - using detail::remquo; - using detail::fma; - using detail::fmax; - using detail::fmin; - using detail::fdim; - using detail::nanh; - using detail::exp; - using detail::expm1; - using detail::exp2; - using detail::log; - using detail::log10; - using detail::log1p; - using detail::log2; - using detail::sqrt; - using detail::cbrt; - using detail::hypot; - using detail::pow; - using detail::sin; - using detail::cos; - using detail::tan; - using detail::asin; - using detail::acos; - using detail::atan; - using detail::atan2; - using detail::sinh; - using detail::cosh; - using detail::tanh; - using detail::asinh; - using detail::acosh; - using detail::atanh; - using detail::erf; - using detail::erfc; - using detail::lgamma; - using detail::tgamma; - using detail::ceil; - using detail::floor; - using detail::trunc; - using detail::round; - using detail::lround; - using detail::nearbyint; - using detail::rint; - using detail::lrint; #if HALF_ENABLE_CPP11_LONG_LONG - using detail::llround; - using detail::llrint; -#endif - using detail::frexp; - using detail::ldexp; - using detail::modf; - using detail::scalbn; - using detail::scalbln; - using detail::ilogb; - using detail::logb; - using detail::nextafter; - using detail::nexttoward; - using detail::copysign; - using detail::fpclassify; - using detail::isfinite; - using detail::isinf; - using detail::isnan; - using detail::isnormal; - using detail::signbit; - using detail::isgreater; - using detail::isgreaterequal; - using detail::isless; - using detail::islessequal; - using detail::islessgreater; - using detail::isunordered; - - using detail::half_cast; +/// Unsigned integer of (at least) 64 bits width. +template <> +struct bits + : conditional::digits >= 64, + unsigned long, unsigned long long> {}; +#else +/// Unsigned integer of (at least) 64 bits width. +template <> +struct bits { + typedef unsigned long type; +}; +#endif +#endif + +/// Tag type for binary construction. +struct binary_t {}; + +/// Tag for binary construction. +HALF_CONSTEXPR_CONST binary_t binary = binary_t(); + +/// Temporary half-precision expression. +/// This class represents a half-precision expression which just stores a +/// single-precision value internally. +struct expr { + /// Conversion constructor. + /// \param f single-precision value to convert + explicit HALF_CONSTEXPR expr(float f) HALF_NOEXCEPT : value_(f) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + HALF_CONSTEXPR operator float() const HALF_NOEXCEPT { return value_; } + + private: + /// Internal expression value stored in single-precision. + float value_; +}; + +/// SFINAE helper for generic half-precision functions. +/// This class template has to be specialized for each valid combination of +/// argument types to provide a corresponding `type` member equivalent to \a T. +/// \tparam T type to return +template +struct enable {}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; +template +struct enable { + typedef T type; +}; + +/// Return type for specialized generic 2-argument half-precision functions. +/// This class template has to be specialized for each valid combination of +/// argument types to provide a corresponding `type` member denoting the +/// appropriate return type. \tparam T first argument type \tparam U first +/// argument type +template +struct result : enable {}; +template <> +struct result { + typedef half type; +}; + +/// \name Classification helpers +/// \{ + +/// Check for infinity. +/// \tparam T argument type (builtin floating point type) +/// \param arg value to query +/// \retval true if infinity +/// \retval false else +template +bool builtin_isinf(T arg) { +#if HALF_ENABLE_CPP11_CMATH + return std::isinf(arg); +#elif defined(_MSC_VER) + return !::_finite(static_cast(arg)) && + !::_isnan(static_cast(arg)); +#else + return arg == std::numeric_limits::infinity() || + arg == -std::numeric_limits::infinity(); +#endif } +/// Check for NaN. +/// \tparam T argument type (builtin floating point type) +/// \param arg value to query +/// \retval true if not a number +/// \retval false else +template +bool builtin_isnan(T arg) { +#if HALF_ENABLE_CPP11_CMATH + return std::isnan(arg); +#elif defined(_MSC_VER) + return ::_isnan(static_cast(arg)) != 0; +#else + return arg != arg; +#endif +} -/// Extensions to the C++ standard library. -namespace std -{ - /// Numeric limits for half-precision floats. - /// Because of the underlying single-precision implementation of many operations, it inherits some properties from - /// `std::numeric_limits`. - template<> class numeric_limits : public numeric_limits - { - public: - /// Supports signed values. - static HALF_CONSTEXPR_CONST bool is_signed = true; +/// Check sign. +/// \tparam T argument type (builtin floating point type) +/// \param arg value to query +/// \retval true if signbit set +/// \retval false else +template +bool builtin_signbit(T arg) { +#if HALF_ENABLE_CPP11_CMATH + return std::signbit(arg); +#else + return arg < T() || (arg == T() && T(1) / arg < T()); +#endif +} + +/// \} +/// \name Conversion +/// \{ + +/// Convert IEEE single-precision to half-precision. +/// Credit for this goes to [Jeroen van der +/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). \tparam R +/// rounding mode to use, `std::round_indeterminate` for fastest rounding \param +/// value single-precision value \return binary representation of half-precision +/// value +template +uint16 float2half_impl(float value, true_type) { + typedef bits::type uint32; + uint32 bits; // = *reinterpret_cast(&value); //violating + // strict aliasing! + std::memcpy(&bits, &value, sizeof(float)); + /* uint16 hbits = (bits>>16) & 0x8000; + bits &= 0x7FFFFFFF; + int exp = bits >> 23; + if(exp == 255) + return hbits | 0x7C00 | + (0x3FF&-static_cast((bits&0x7FFFFF)!=0)); if(exp > 142) + { + if(R == std::round_toward_infinity) + return hbits | 0x7C00 - (hbits>>15); + if(R == std::round_toward_neg_infinity) + return hbits | 0x7BFF + (hbits>>15); + return hbits | 0x7BFF + + (R!=std::round_toward_zero); + } + int g, s; + if(exp > 112) + { + g = (bits>>12) & 1; + s = (bits&0xFFF) != 0; + hbits |= ((exp-112)<<10) | ((bits>>13)&0x3FF); + } + else if(exp > 101) + { + int i = 125 - exp; + bits = (bits&0x7FFFFF) | 0x800000; + g = (bits>>i) & 1; + s = (bits&((1L<> (i+1); + } + else + { + g = 0; + s = bits != 0; + } + if(R == std::round_to_nearest) + #if HALF_ROUND_TIES_TO_EVEN + hbits += g & (s|hbits); + #else + hbits += g; + #endif + else if(R == std::round_toward_infinity) + hbits += ~(hbits>>15) & (s|g); + else if(R == std::round_toward_neg_infinity) + hbits += (hbits>>15) & (g|s); + */ + static const uint16 base_table[512] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, + 0x0020, 0x0040, 0x0080, 0x0100, 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, + 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, + 0x3800, 0x3C00, 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, + 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x7C00, + 0x7C00, 0x7C00, 0x7C00, 0x7C00, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, + 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, 0x8200, + 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, + 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, 0xC000, 0xC400, 0xC800, + 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, + 0xF000, 0xF400, 0xF800, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, + 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00, 0xFC00}; + static const unsigned char shift_table[512] = { + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, + 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, + 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 13}; + uint16 hbits = + base_table[bits >> 23] + + static_cast((bits & 0x7FFFFF) >> shift_table[bits >> 23]); + if (R == std::round_to_nearest) + hbits += + (((bits & 0x7FFFFF) >> (shift_table[bits >> 23] - 1)) | + (((bits >> 23) & 0xFF) == 102)) & + ((hbits & 0x7C00) != 0x7C00) +#if HALF_ROUND_TIES_TO_EVEN + & (((((static_cast(1) << (shift_table[bits >> 23] - 1)) - 1) & + bits) != 0) | + hbits) +#endif + ; + else if (R == std::round_toward_zero) + hbits -= ((hbits & 0x7FFF) == 0x7C00) & ~shift_table[bits >> 23]; + else if (R == std::round_toward_infinity) + hbits += + ((((bits & 0x7FFFFF & + ((static_cast(1) << (shift_table[bits >> 23])) - 1)) != 0) | + (((bits >> 23) <= 102) & ((bits >> 23) != 0))) & + (hbits < 0x7C00)) - + ((hbits == 0xFC00) & ((bits >> 23) != 511)); + else if (R == std::round_toward_neg_infinity) + hbits += + ((((bits & 0x7FFFFF & + ((static_cast(1) << (shift_table[bits >> 23])) - 1)) != 0) | + (((bits >> 23) <= 358) & ((bits >> 23) != 256))) & + (hbits < 0xFC00) & (hbits >> 15)) - + ((hbits == 0x7C00) & ((bits >> 23) != 255)); + return hbits; +} + +/// Convert IEEE double-precision to half-precision. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \param value double-precision value \return binary representation +/// of half-precision value +template +uint16 float2half_impl(double value, true_type) { + typedef bits::type uint32; + typedef bits::type uint64; + uint64 bits; // = *reinterpret_cast(&value); //violating + // strict aliasing! + std::memcpy(&bits, &value, sizeof(double)); + uint32 hi = bits >> 32, lo = bits & 0xFFFFFFFF; + uint16 hbits = (hi >> 16) & 0x8000; + hi &= 0x7FFFFFFF; + int exp = hi >> 20; + if (exp == 2047) + return hbits | 0x7C00 | + (0x3FF & -static_cast((bits & 0xFFFFFFFFFFFFF) != 0)); + if (exp > 1038) { + if (R == std::round_toward_infinity) return hbits | 0x7C00 - (hbits >> 15); + if (R == std::round_toward_neg_infinity) + return hbits | 0x7BFF + (hbits >> 15); + return hbits | 0x7BFF + (R != std::round_toward_zero); + } + int g, s = lo != 0; + if (exp > 1008) { + g = (hi >> 9) & 1; + s |= (hi & 0x1FF) != 0; + hbits |= ((exp - 1008) << 10) | ((hi >> 10) & 0x3FF); + } else if (exp > 997) { + int i = 1018 - exp; + hi = (hi & 0xFFFFF) | 0x100000; + g = (hi >> i) & 1; + s |= (hi & ((1L << i) - 1)) != 0; + hbits |= hi >> (i + 1); + } else { + g = 0; + s |= hi != 0; + } + if (R == std::round_to_nearest) +#if HALF_ROUND_TIES_TO_EVEN + hbits += g & (s | hbits); +#else + hbits += g; +#endif + else if (R == std::round_toward_infinity) + hbits += ~(hbits >> 15) & (s | g); + else if (R == std::round_toward_neg_infinity) + hbits += (hbits >> 15) & (g | s); + return hbits; +} + +/// Convert non-IEEE floating point to half-precision. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam T source type (builtin floating point type) \param value +/// floating point value \return binary representation of half-precision value +template +uint16 float2half_impl(T value, ...) { + uint16 hbits = static_cast(builtin_signbit(value)) << 15; + if (value == T()) return hbits; + if (builtin_isnan(value)) return hbits | 0x7FFF; + if (builtin_isinf(value)) return hbits | 0x7C00; + int exp; + std::frexp(value, &exp); + if (exp > 16) { + if (R == std::round_toward_infinity) + return hbits | (0x7C00 - (hbits >> 15)); + else if (R == std::round_toward_neg_infinity) + return hbits | (0x7BFF + (hbits >> 15)); + return hbits | (0x7BFF + (R != std::round_toward_zero)); + } + if (exp < -13) + value = std::ldexp(value, 24); + else { + value = std::ldexp(value, 11 - exp); + hbits |= ((exp + 13) << 10); + } + T ival, frac = std::modf(value, &ival); + hbits += static_cast(std::abs(static_cast(ival))); + if (R == std::round_to_nearest) { + frac = std::abs(frac); +#if HALF_ROUND_TIES_TO_EVEN + hbits += (frac > T(0.5)) | ((frac == T(0.5)) & hbits); +#else + hbits += frac >= T(0.5); +#endif + } else if (R == std::round_toward_infinity) + hbits += frac > T(); + else if (R == std::round_toward_neg_infinity) + hbits += frac < T(); + return hbits; +} + +/// Convert floating point to half-precision. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam T source type (builtin floating point type) \param value +/// floating point value \return binary representation of half-precision value +template +uint16 float2half(T value) { + return float2half_impl( + value, bool_type < std::numeric_limits::is_iec559 && + sizeof(typename bits::type) == sizeof(T) > ()); +} + +/// Convert integer to half-precision floating point. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam S `true` if value negative, `false` else \tparam T type to +/// convert (builtin integer type) \param value non-negative integral value +/// \return binary representation of half-precision value +template +uint16 int2half_impl(T value) { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_integral::value, + "int to half conversion only supports builtin integer types"); +#endif + if (S) value = -value; + uint16 bits = S << 15; + if (value > 0xFFFF) { + if (R == std::round_toward_infinity) + bits |= 0x7C00 - S; + else if (R == std::round_toward_neg_infinity) + bits |= 0x7BFF + S; + else + bits |= 0x7BFF + (R != std::round_toward_zero); + } else if (value) { + unsigned int m = value, exp = 24; + for (; m < 0x400; m <<= 1, --exp) + ; + for (; m > 0x7FF; m >>= 1, ++exp) + ; + bits |= (exp << 10) + m; + if (exp > 24) { + if (R == std::round_to_nearest) + bits += (value >> (exp - 25)) & 1 +#if HALF_ROUND_TIES_TO_EVEN + & (((((1 << (exp - 25)) - 1) & value) != 0) | bits) +#endif + ; + else if (R == std::round_toward_infinity) + bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & !S; + else if (R == std::round_toward_neg_infinity) + bits += ((value & ((1 << (exp - 24)) - 1)) != 0) & S; + } + } + return bits; +} + +/// Convert integer to half-precision floating point. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam T type to convert (builtin integer type) \param value +/// integral value \return binary representation of half-precision value +template +uint16 int2half(T value) { + return (value < 0) ? int2half_impl(value) + : int2half_impl(value); +} + +/// Convert half-precision to IEEE single-precision. +/// Credit for this goes to [Jeroen van der +/// Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). \param +/// value binary representation of half-precision value \return single-precision +/// value +inline float half2float_impl(uint16 value, float, true_type) { + typedef bits::type uint32; + /* uint32 bits = static_cast(value&0x8000) << 16; + int abs = value & 0x7FFF; + if(abs) + { + bits |= 0x38000000 << + static_cast(abs>=0x7C00); for(; abs<0x400; + abs<<=1,bits-=0x800000) ; bits += static_cast(abs) << 13; + } + */ + static const uint32 mantissa_table[2048] = { + 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, + 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, + 0x35400000, 0x35500000, 0x35600000, 0x35700000, 0x35800000, 0x35880000, + 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, + 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, + 0x35F00000, 0x35F80000, 0x36000000, 0x36040000, 0x36080000, 0x360C0000, + 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, + 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000, + 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, + 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, + 0x36700000, 0x36740000, 0x36780000, 0x367C0000, 0x36800000, 0x36820000, + 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, + 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, + 0x369C0000, 0x369E0000, 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, + 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, + 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000, + 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, + 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, + 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000, 0x36E00000, 0x36E20000, + 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, + 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, + 0x36FC0000, 0x36FE0000, 0x37000000, 0x37010000, 0x37020000, 0x37030000, + 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, + 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, + 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, + 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, + 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, 0x37200000, 0x37210000, + 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, + 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, + 0x372E0000, 0x372F0000, 0x37300000, 0x37310000, 0x37320000, 0x37330000, + 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, + 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000, + 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, + 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, + 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000, 0x37500000, 0x37510000, + 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, + 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, + 0x375E0000, 0x375F0000, 0x37600000, 0x37610000, 0x37620000, 0x37630000, + 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, + 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000, + 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, + 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, + 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000, 0x37800000, 0x37808000, + 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, + 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, + 0x37870000, 0x37878000, 0x37880000, 0x37888000, 0x37890000, 0x37898000, + 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, + 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000, + 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, + 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, + 0x37960000, 0x37968000, 0x37970000, 0x37978000, 0x37980000, 0x37988000, + 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, + 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, + 0x379F0000, 0x379F8000, 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, + 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, + 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000, + 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, + 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, + 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000, 0x37B00000, 0x37B08000, + 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, + 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, + 0x37B70000, 0x37B78000, 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, + 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, + 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, + 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, + 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, + 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, 0x37C80000, 0x37C88000, + 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, + 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, + 0x37CF0000, 0x37CF8000, 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, + 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, + 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000, + 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, + 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, + 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000, 0x37E00000, 0x37E08000, + 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, + 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, + 0x37E70000, 0x37E78000, 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, + 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, + 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, + 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, + 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, + 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000, 0x37F80000, 0x37F88000, + 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, + 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, + 0x37FF0000, 0x37FF8000, 0x38000000, 0x38004000, 0x38008000, 0x3800C000, + 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, + 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000, + 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, + 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, + 0x38070000, 0x38074000, 0x38078000, 0x3807C000, 0x38080000, 0x38084000, + 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, + 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, + 0x380B8000, 0x380BC000, 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, + 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, + 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000, + 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, + 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, + 0x38130000, 0x38134000, 0x38138000, 0x3813C000, 0x38140000, 0x38144000, + 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, + 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, + 0x38178000, 0x3817C000, 0x38180000, 0x38184000, 0x38188000, 0x3818C000, + 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, + 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, + 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, + 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, + 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, 0x38200000, 0x38204000, + 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, + 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, + 0x38238000, 0x3823C000, 0x38240000, 0x38244000, 0x38248000, 0x3824C000, + 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, + 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000, + 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, + 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, + 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000, 0x382C0000, 0x382C4000, + 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, + 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, + 0x382F8000, 0x382FC000, 0x38300000, 0x38304000, 0x38308000, 0x3830C000, + 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, + 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, + 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, + 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, + 0x38370000, 0x38374000, 0x38378000, 0x3837C000, 0x38380000, 0x38384000, + 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, + 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, + 0x383B8000, 0x383BC000, 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, + 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, + 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000, + 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, + 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, + 0x38430000, 0x38434000, 0x38438000, 0x3843C000, 0x38440000, 0x38444000, + 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, + 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, + 0x38478000, 0x3847C000, 0x38480000, 0x38484000, 0x38488000, 0x3848C000, + 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, + 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000, + 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, + 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, + 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000, 0x38500000, 0x38504000, + 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, + 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, + 0x38538000, 0x3853C000, 0x38540000, 0x38544000, 0x38548000, 0x3854C000, + 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, + 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000, + 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, + 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, + 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, 0x385C0000, 0x385C4000, + 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, + 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, + 0x385F8000, 0x385FC000, 0x38600000, 0x38604000, 0x38608000, 0x3860C000, + 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, + 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000, + 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, + 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, + 0x38670000, 0x38674000, 0x38678000, 0x3867C000, 0x38680000, 0x38684000, + 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, + 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, + 0x386B8000, 0x386BC000, 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, + 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, + 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000, + 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, + 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, + 0x38730000, 0x38734000, 0x38738000, 0x3873C000, 0x38740000, 0x38744000, + 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, + 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, + 0x38778000, 0x3877C000, 0x38780000, 0x38784000, 0x38788000, 0x3878C000, + 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, + 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000, + 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, + 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, + 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, 0x38000000, 0x38002000, + 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, + 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, + 0x3801C000, 0x3801E000, 0x38020000, 0x38022000, 0x38024000, 0x38026000, + 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, + 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000, + 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, + 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, + 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000, 0x38060000, 0x38062000, + 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, + 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, + 0x3807C000, 0x3807E000, 0x38080000, 0x38082000, 0x38084000, 0x38086000, + 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, + 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, + 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, + 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, + 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, 0x380C0000, 0x380C2000, + 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, + 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, + 0x380DC000, 0x380DE000, 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, + 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, + 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000, + 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, + 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, + 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000, 0x38120000, 0x38122000, + 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, + 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, + 0x3813C000, 0x3813E000, 0x38140000, 0x38142000, 0x38144000, 0x38146000, + 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, + 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000, + 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, + 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, + 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000, 0x38180000, 0x38182000, + 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, + 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, + 0x3819C000, 0x3819E000, 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, + 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, + 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000, + 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, + 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, + 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, 0x381E0000, 0x381E2000, + 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, + 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, + 0x381FC000, 0x381FE000, 0x38200000, 0x38202000, 0x38204000, 0x38206000, + 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, + 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000, + 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, + 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, + 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000, 0x38240000, 0x38242000, + 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, + 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, + 0x3825C000, 0x3825E000, 0x38260000, 0x38262000, 0x38264000, 0x38266000, + 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, + 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, + 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, + 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, + 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, 0x382A0000, 0x382A2000, + 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, + 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, + 0x382BC000, 0x382BE000, 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, + 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, + 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000, + 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, + 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, + 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000, 0x38300000, 0x38302000, + 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, + 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, + 0x3831C000, 0x3831E000, 0x38320000, 0x38322000, 0x38324000, 0x38326000, + 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, + 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, + 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, + 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, + 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000, 0x38360000, 0x38362000, + 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, + 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, + 0x3837C000, 0x3837E000, 0x38380000, 0x38382000, 0x38384000, 0x38386000, + 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, + 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000, + 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, + 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, + 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, 0x383C0000, 0x383C2000, + 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, + 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, + 0x383DC000, 0x383DE000, 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, + 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, + 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000, + 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, + 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, + 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000, 0x38420000, 0x38422000, + 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, + 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, + 0x3843C000, 0x3843E000, 0x38440000, 0x38442000, 0x38444000, 0x38446000, + 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, + 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, + 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, + 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, + 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, 0x38480000, 0x38482000, + 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, + 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, + 0x3849C000, 0x3849E000, 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, + 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, + 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000, + 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, + 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, + 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000, 0x384E0000, 0x384E2000, + 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, + 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, + 0x384FC000, 0x384FE000, 0x38500000, 0x38502000, 0x38504000, 0x38506000, + 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, + 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, + 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, + 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, + 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000, 0x38540000, 0x38542000, + 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, + 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, + 0x3855C000, 0x3855E000, 0x38560000, 0x38562000, 0x38564000, 0x38566000, + 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, + 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000, + 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, + 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, + 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, 0x385A0000, 0x385A2000, + 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, + 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, + 0x385BC000, 0x385BE000, 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, + 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, + 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000, + 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, + 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, + 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000, 0x38600000, 0x38602000, + 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, + 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, + 0x3861C000, 0x3861E000, 0x38620000, 0x38622000, 0x38624000, 0x38626000, + 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, + 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, + 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, + 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, + 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, 0x38660000, 0x38662000, + 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, + 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, + 0x3867C000, 0x3867E000, 0x38680000, 0x38682000, 0x38684000, 0x38686000, + 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, + 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000, + 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, + 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, + 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000, 0x386C0000, 0x386C2000, + 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, + 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, + 0x386DC000, 0x386DE000, 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, + 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, + 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000, + 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, + 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, + 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000, 0x38720000, 0x38722000, + 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, + 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, + 0x3873C000, 0x3873E000, 0x38740000, 0x38742000, 0x38744000, 0x38746000, + 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, + 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000, + 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, + 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, + 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, 0x38780000, 0x38782000, + 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, + 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, + 0x3879C000, 0x3879E000, 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, + 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, + 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000, + 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, + 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, + 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000, 0x387E0000, 0x387E2000, + 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, + 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, + 0x387FC000, 0x387FE000}; + static const uint32 exponent_table[64] = { + 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, + 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, + 0x06000000, 0x06800000, 0x07000000, 0x07800000, 0x08000000, 0x08800000, + 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, + 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, + 0x0F000000, 0x47800000, 0x80000000, 0x80800000, 0x81000000, 0x81800000, + 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, + 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, + 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, + 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, + 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000}; + static const unsigned short offset_table[64] = { + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 0, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024}; + uint32 bits = mantissa_table[offset_table[value >> 10] + (value & 0x3FF)] + + exponent_table[value >> 10]; + // return *reinterpret_cast(&bits); //violating + //strict aliasing! + float out; + std::memcpy(&out, &bits, sizeof(float)); + return out; +} + +/// Convert half-precision to IEEE double-precision. +/// \param value binary representation of half-precision value +/// \return double-precision value +inline double half2float_impl(uint16 value, double, true_type) { + typedef bits::type uint32; + typedef bits::type uint64; + uint32 hi = static_cast(value & 0x8000) << 16; + int abs = value & 0x7FFF; + if (abs) { + hi |= 0x3F000000 << static_cast(abs >= 0x7C00); + for (; abs < 0x400; abs <<= 1, hi -= 0x100000) + ; + hi += static_cast(abs) << 10; + } + uint64 bits = static_cast(hi) << 32; + // return *reinterpret_cast(&bits); //violating + //strict aliasing! + double out; + std::memcpy(&out, &bits, sizeof(double)); + return out; +} + +/// Convert half-precision to non-IEEE floating point. +/// \tparam T type to convert to (builtin integer type) +/// \param value binary representation of half-precision value +/// \return floating point value +template +T half2float_impl(uint16 value, T, ...) { + T out; + int abs = value & 0x7FFF; + if (abs > 0x7C00) + out = std::numeric_limits::has_quiet_NaN + ? std::numeric_limits::quiet_NaN() + : T(); + else if (abs == 0x7C00) + out = std::numeric_limits::has_infinity + ? std::numeric_limits::infinity() + : std::numeric_limits::max(); + else if (abs > 0x3FF) + out = std::ldexp(static_cast((abs & 0x3FF) | 0x400), (abs >> 10) - 25); + else + out = std::ldexp(static_cast(abs), -24); + return (value & 0x8000) ? -out : out; +} + +/// Convert half-precision to floating point. +/// \tparam T type to convert to (builtin integer type) +/// \param value binary representation of half-precision value +/// \return floating point value +template +T half2float(uint16 value) { + return half2float_impl(value, T(), + bool_type < std::numeric_limits::is_iec559 && + sizeof(typename bits::type) == sizeof(T) > ()); +} + +/// Convert half-precision floating point to integer. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam E `true` for round to even, `false` for round away from +/// zero \tparam T type to convert to (buitlin integer type with at least 16 +/// bits precision, excluding any implicit sign bits) \param value binary +/// representation of half-precision value \return integral value +template +T half2int_impl(uint16 value) { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_integral::value, + "half to int conversion only supports builtin integer types"); +#endif + unsigned int e = value & 0x7FFF; + if (e >= 0x7C00) + return (value & 0x8000) ? std::numeric_limits::min() + : std::numeric_limits::max(); + if (e < 0x3800) { + if (R == std::round_toward_infinity) + return T(~(value >> 15) & (e != 0)); + else if (R == std::round_toward_neg_infinity) + return -T(value > 0x8000); + return T(); + } + unsigned int m = (value & 0x3FF) | 0x400; + e >>= 10; + if (e < 25) { + if (R == std::round_to_nearest) + m += (1 << (24 - e)) - (~(m >> (25 - e)) & E); + else if (R == std::round_toward_infinity) + m += ((value >> 15) - 1) & ((1 << (25 - e)) - 1U); + else if (R == std::round_toward_neg_infinity) + m += -(value >> 15) & ((1 << (25 - e)) - 1U); + m >>= 25 - e; + } else + m <<= e - 25; + return (value & 0x8000) ? -static_cast(m) : static_cast(m); +} + +/// Convert half-precision floating point to integer. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam T type to convert to (buitlin integer type with at least 16 +/// bits precision, excluding any implicit sign bits) \param value binary +/// representation of half-precision value \return integral value +template +T half2int(uint16 value) { + return half2int_impl(value); +} + +/// Convert half-precision floating point to integer using +/// round-to-nearest-away-from-zero. \tparam T type to convert to (buitlin +/// integer type with at least 16 bits precision, excluding any implicit sign +/// bits) \param value binary representation of half-precision value \return +/// integral value +template +T half2int_up(uint16 value) { + return half2int_impl(value); +} + +/// Round half-precision number to nearest integer value. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \tparam E `true` for round to even, `false` for round away from +/// zero \param value binary representation of half-precision value \return +/// half-precision bits for nearest integral value +template +uint16 round_half_impl(uint16 value) { + unsigned int e = value & 0x7FFF; + uint16 result = value; + if (e < 0x3C00) { + result &= 0x8000; + if (R == std::round_to_nearest) + result |= 0x3C00U & -(e >= (0x3800 + E)); + else if (R == std::round_toward_infinity) + result |= 0x3C00U & -(~(value >> 15) & (e != 0)); + else if (R == std::round_toward_neg_infinity) + result |= 0x3C00U & -(value > 0x8000); + } else if (e < 0x6400) { + e = 25 - (e >> 10); + unsigned int mask = (1 << e) - 1; + if (R == std::round_to_nearest) + result += (1 << (e - 1)) - (~(result >> e) & E); + else if (R == std::round_toward_infinity) + result += mask & ((value >> 15) - 1); + else if (R == std::round_toward_neg_infinity) + result += mask & -(value >> 15); + result &= ~mask; + } + return result; +} + +/// Round half-precision number to nearest integer value. +/// \tparam R rounding mode to use, `std::round_indeterminate` for fastest +/// rounding \param value binary representation of half-precision value \return +/// half-precision bits for nearest integral value +template +uint16 round_half(uint16 value) { + return round_half_impl(value); +} + +/// Round half-precision number to nearest integer value using +/// round-to-nearest-away-from-zero. \param value binary representation of +/// half-precision value \return half-precision bits for nearest integral value +inline uint16 round_half_up(uint16 value) { + return round_half_impl(value); +} +/// \} + +struct functions; +template +struct unary_specialized; +template +struct binary_specialized; +template +struct half_caster; +} // namespace detail + +/// Half-precision floating point type. +/// This class implements an IEEE-conformant half-precision floating point type +/// with the usual arithmetic operators and conversions. It is implicitly +/// convertible to single-precision floating point, which makes artihmetic +/// expressions and functions with mixed-type operands to be of the most precise +/// operand type. Additionally all arithmetic operations (and many mathematical +/// functions) are carried out in single-precision internally. All conversions +/// from single- to half-precision are done using the library's default rounding +/// mode, but temporary results inside chained arithmetic expressions are kept +/// in single-precision as long as possible (while of course still maintaining a +/// strong half-precision type). +/// +/// According to the C++98/03 definition, the half type is not a POD type. But +/// according to C++11's less strict and extended definitions it is both a +/// standard layout type and a trivially copyable type (even if not a POD type), +/// which means it can be standard-conformantly copied using raw binary copies. +/// But in this context some more words about the actual size of the type. +/// Although the half is representing an IEEE 16-bit type, it does not +/// neccessarily have to be of exactly 16-bits size. But on any reasonable +/// implementation the actual binary representation of this type will most +/// probably not ivolve any additional "magic" or padding beyond the simple +/// binary representation of the underlying 16-bit IEEE number, even if not +/// strictly guaranteed by the standard. But even then it only has an actual +/// size of 16 bits if your C++ implementation supports an unsigned integer type +/// of exactly 16 bits width. But this should be the case on nearly any +/// reasonable platform. +/// +/// So if your C++ implementation is not totally exotic or imposes special +/// alignment requirements, it is a reasonable assumption that the data of a +/// half is just comprised of the 2 bytes of the underlying IEEE representation. +class half { + friend struct detail::functions; + friend struct detail::unary_specialized; + friend struct detail::binary_specialized; + template + friend struct detail::half_caster; + friend class std::numeric_limits; +#if HALF_ENABLE_CPP11_HASH + friend struct std::hash; +#endif +#if HALF_ENABLE_CPP11_USER_LITERALS + friend half literal::operator"" _h(long double); +#endif - /// Is not exact. - static HALF_CONSTEXPR_CONST bool is_exact = false; + public: + /// Default constructor. + /// This initializes the half to 0. Although this does not match the builtin + /// types' default-initialization semantics and may be less efficient than no + /// initialization, it is needed to provide proper value-initialization + /// semantics. + HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {} + + /// Copy constructor. + /// \tparam T type of concrete half expression + /// \param rhs half expression to copy from + half(detail::expr rhs) + : data_(detail::float2half(static_cast(rhs))) {} + + /// Conversion constructor. + /// \param rhs float to convert + explicit half(float rhs) : data_(detail::float2half(rhs)) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + operator float() const { return detail::half2float(data_); } + + /// Assignment operator. + /// \tparam T type of concrete half expression + /// \param rhs half expression to copy from + /// \return reference to this half + half &operator=(detail::expr rhs) { return *this = static_cast(rhs); } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to add + /// \return reference to this half + template + typename detail::enable::type operator+=(T rhs) { + return *this += static_cast(rhs); + } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to subtract + /// \return reference to this half + template + typename detail::enable::type operator-=(T rhs) { + return *this -= static_cast(rhs); + } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to multiply with + /// \return reference to this half + template + typename detail::enable::type operator*=(T rhs) { + return *this *= static_cast(rhs); + } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to divide by + /// \return reference to this half + template + typename detail::enable::type operator/=(T rhs) { + return *this /= static_cast(rhs); + } + + /// Assignment operator. + /// \param rhs single-precision value to copy from + /// \return reference to this half + half &operator=(float rhs) { + data_ = detail::float2half(rhs); + return *this; + } + + /// Arithmetic assignment. + /// \param rhs single-precision value to add + /// \return reference to this half + half &operator+=(float rhs) { + data_ = + detail::float2half(detail::half2float(data_) + rhs); + return *this; + } + + /// Arithmetic assignment. + /// \param rhs single-precision value to subtract + /// \return reference to this half + half &operator-=(float rhs) { + data_ = + detail::float2half(detail::half2float(data_) - rhs); + return *this; + } + + /// Arithmetic assignment. + /// \param rhs single-precision value to multiply with + /// \return reference to this half + half &operator*=(float rhs) { + data_ = + detail::float2half(detail::half2float(data_) * rhs); + return *this; + } + + /// Arithmetic assignment. + /// \param rhs single-precision value to divide by + /// \return reference to this half + half &operator/=(float rhs) { + data_ = + detail::float2half(detail::half2float(data_) / rhs); + return *this; + } + + /// Prefix increment. + /// \return incremented half value + half &operator++() { return *this += 1.0f; } + + /// Prefix decrement. + /// \return decremented half value + half &operator--() { return *this -= 1.0f; } + + /// Postfix increment. + /// \return non-incremented half value + half operator++(int) { + half out(*this); + ++*this; + return out; + } + + /// Postfix decrement. + /// \return non-decremented half value + half operator--(int) { + half out(*this); + --*this; + return out; + } + + private: + /// Rounding mode to use + static const std::float_round_style round_style = + (std::float_round_style)(HALF_ROUND_STYLE); + + /// Constructor. + /// \param bits binary representation to set half to + HALF_CONSTEXPR half(detail::binary_t, detail::uint16 bits) HALF_NOEXCEPT + : data_(bits) {} + + /// Internal binary representation + detail::uint16 data_; +}; - /// Doesn't provide modulo arithmetic. - static HALF_CONSTEXPR_CONST bool is_modulo = false; +#if HALF_ENABLE_CPP11_USER_LITERALS +namespace literal { +/// Half literal. +/// While this returns an actual half-precision value, half literals can +/// unfortunately not be constant expressions due to rather involved +/// conversions. \param value literal value \return half with given value (if +/// representable) +inline half operator"" _h(long double value) { + return half(detail::binary, detail::float2half(value)); +} +} // namespace literal +#endif - /// IEEE conformant. - static HALF_CONSTEXPR_CONST bool is_iec559 = true; +namespace detail { +/// Wrapper implementing unspecialized half-precision functions. +struct functions { + /// Addition implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision sum stored in single-precision + static expr plus(float x, float y) { return expr(x + y); } + + /// Subtraction implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision difference stored in single-precision + static expr minus(float x, float y) { return expr(x - y); } + + /// Multiplication implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision product stored in single-precision + static expr multiplies(float x, float y) { return expr(x * y); } + + /// Division implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision quotient stored in single-precision + static expr divides(float x, float y) { return expr(x / y); } + + /// Output implementation. + /// \param out stream to write to + /// \param arg value to write + /// \return reference to stream + template + static std::basic_ostream &write( + std::basic_ostream &out, float arg) { + return out << arg; + } + + /// Input implementation. + /// \param in stream to read from + /// \param arg half to read into + /// \return reference to stream + template + static std::basic_istream &read( + std::basic_istream &in, half &arg) { + float f; + if (in >> f) arg = f; + return in; + } + + /// Modulo implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision division remainder stored in single-precision + static expr fmod(float x, float y) { return expr(std::fmod(x, y)); } + + /// Remainder implementation. + /// \param x first operand + /// \param y second operand + /// \return Half-precision division remainder stored in single-precision + static expr remainder(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::remainder(x, y)); +#else + if (builtin_isnan(x) || builtin_isnan(y)) + return expr(std::numeric_limits::quiet_NaN()); + float ax = std::fabs(x), ay = std::fabs(y); + if (ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) + return expr(std::numeric_limits::quiet_NaN()); + if (ay >= 65536.0f) return expr(x); + if (ax == ay) return expr(builtin_signbit(x) ? -0.0f : 0.0f); + ax = std::fmod(ax, ay + ay); + float y2 = 0.5f * ay; + if (ax > y2) { + ax -= ay; + if (ax >= y2) ax -= ay; + } + return expr(builtin_signbit(x) ? -ax : ax); +#endif + } + + /// Remainder implementation. + /// \param x first operand + /// \param y second operand + /// \param quo address to store quotient bits at + /// \return Half-precision division remainder stored in single-precision + static expr remquo(float x, float y, int *quo) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::remquo(x, y, quo)); +#else + if (builtin_isnan(x) || builtin_isnan(y)) + return expr(std::numeric_limits::quiet_NaN()); + bool sign = builtin_signbit(x), + qsign = static_cast(sign ^ builtin_signbit(y)); + float ax = std::fabs(x), ay = std::fabs(y); + if (ax >= 65536.0f || ay < std::ldexp(1.0f, -24)) + return expr(std::numeric_limits::quiet_NaN()); + if (ay >= 65536.0f) return expr(x); + if (ax == ay) return *quo = qsign ? -1 : 1, expr(sign ? -0.0f : 0.0f); + ax = std::fmod(ax, 8.0f * ay); + int cquo = 0; + if (ax >= 4.0f * ay) { + ax -= 4.0f * ay; + cquo += 4; + } + if (ax >= 2.0f * ay) { + ax -= 2.0f * ay; + cquo += 2; + } + float y2 = 0.5f * ay; + if (ax > y2) { + ax -= ay; + ++cquo; + if (ax >= y2) { + ax -= ay; + ++cquo; + } + } + return *quo = qsign ? -cquo : cquo, expr(sign ? -ax : ax); +#endif + } + + /// Positive difference implementation. + /// \param x first operand + /// \param y second operand + /// \return Positive difference stored in single-precision + static expr fdim(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::fdim(x, y)); +#else + return expr((x <= y) ? 0.0f : (x - y)); +#endif + } + + /// Fused multiply-add implementation. + /// \param x first operand + /// \param y second operand + /// \param z third operand + /// \return \a x * \a y + \a z stored in single-precision + static expr fma(float x, float y, float z) { +#if HALF_ENABLE_CPP11_CMATH && defined(FP_FAST_FMAF) + return expr(std::fma(x, y, z)); +#else + return expr(x * y + z); +#endif + } + + /// Get NaN. + /// \return Half-precision quiet NaN + static half nanh() { return half(binary, 0x7FFF); } + + /// Exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr exp(float arg) { return expr(std::exp(arg)); } + + /// Exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr expm1(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::expm1(arg)); +#else + return expr(static_cast(std::exp(static_cast(arg)) - 1.0)); +#endif + } + + /// Binary exponential implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr exp2(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::exp2(arg)); +#else + return expr( + static_cast(std::exp(arg * 0.69314718055994530941723212145818))); +#endif + } + + /// Logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log(float arg) { return expr(std::log(arg)); } + + /// Common logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log10(float arg) { return expr(std::log10(arg)); } + + /// Logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log1p(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::log1p(arg)); +#else + return expr(static_cast(std::log(1.0 + arg))); +#endif + } + + /// Binary logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr log2(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::log2(arg)); +#else + return expr(static_cast(std::log(static_cast(arg)) * + 1.4426950408889634073599246810019)); +#endif + } + + /// Square root implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sqrt(float arg) { return expr(std::sqrt(arg)); } + + /// Cubic root implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cbrt(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::cbrt(arg)); +#else + if (builtin_isnan(arg) || builtin_isinf(arg)) return expr(arg); + return expr(builtin_signbit(arg) + ? -static_cast( + std::pow(-static_cast(arg), 1.0 / 3.0)) + : static_cast( + std::pow(static_cast(arg), 1.0 / 3.0))); +#endif + } + + /// Hypotenuse implementation. + /// \param x first argument + /// \param y second argument + /// \return function value stored in single-preicision + static expr hypot(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::hypot(x, y)); +#else + return expr( + (builtin_isinf(x) || builtin_isinf(y)) + ? std::numeric_limits::infinity() + : static_cast(std::sqrt(static_cast(x) * x + + static_cast(y) * y))); +#endif + } + + /// Power implementation. + /// \param base value to exponentiate + /// \param exp power to expontiate to + /// \return function value stored in single-preicision + static expr pow(float base, float exp) { return expr(std::pow(base, exp)); } + + /// Sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sin(float arg) { return expr(std::sin(arg)); } + + /// Cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cos(float arg) { return expr(std::cos(arg)); } + + /// Tan implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tan(float arg) { return expr(std::tan(arg)); } + + /// Arc sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr asin(float arg) { return expr(std::asin(arg)); } + + /// Arc cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr acos(float arg) { return expr(std::acos(arg)); } + + /// Arc tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr atan(float arg) { return expr(std::atan(arg)); } + + /// Arc tangent implementation. + /// \param x first argument + /// \param y second argument + /// \return function value stored in single-preicision + static expr atan2(float x, float y) { return expr(std::atan2(x, y)); } + + /// Hyperbolic sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr sinh(float arg) { return expr(std::sinh(arg)); } + + /// Hyperbolic cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr cosh(float arg) { return expr(std::cosh(arg)); } + + /// Hyperbolic tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tanh(float arg) { return expr(std::tanh(arg)); } + + /// Hyperbolic area sine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr asinh(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::asinh(arg)); +#else + return expr( + (arg == -std::numeric_limits::infinity()) + ? arg + : static_cast(std::log(arg + std::sqrt(arg * arg + 1.0)))); +#endif + } + + /// Hyperbolic area cosine implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr acosh(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::acosh(arg)); +#else + return expr((arg < -1.0f) ? std::numeric_limits::quiet_NaN() + : static_cast(std::log( + arg + std::sqrt(arg * arg - 1.0)))); +#endif + } + + /// Hyperbolic area tangent implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr atanh(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::atanh(arg)); +#else + return expr(static_cast(0.5 * std::log((1.0 + arg) / (1.0 - arg)))); +#endif + } + + /// Error function implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr erf(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::erf(arg)); +#else + return expr(static_cast(erf(static_cast(arg)))); +#endif + } + + /// Complementary implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr erfc(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::erfc(arg)); +#else + return expr(static_cast(1.0 - erf(static_cast(arg)))); +#endif + } + + /// Gamma logarithm implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr lgamma(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::lgamma(arg)); +#else + if (builtin_isinf(arg)) return expr(std::numeric_limits::infinity()); + if (arg < 0.0f) { + float i, f = std::modf(-arg, &i); + if (f == 0.0f) return expr(std::numeric_limits::infinity()); + return expr(static_cast( + 1.1447298858494001741434273513531 - + std::log(std::abs(std::sin(3.1415926535897932384626433832795 * f))) - + lgamma(1.0 - arg))); + } + return expr(static_cast(lgamma(static_cast(arg)))); +#endif + } + + /// Gamma implementation. + /// \param arg function argument + /// \return function value stored in single-preicision + static expr tgamma(float arg) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::tgamma(arg)); +#else + if (arg == 0.0f) + return builtin_signbit(arg) + ? expr(-std::numeric_limits::infinity()) + : expr(std::numeric_limits::infinity()); + if (arg < 0.0f) { + float i, f = std::modf(-arg, &i); + if (f == 0.0f) return expr(std::numeric_limits::quiet_NaN()); + double value = 3.1415926535897932384626433832795 / + (std::sin(3.1415926535897932384626433832795 * f) * + std::exp(lgamma(1.0 - arg))); + return expr( + static_cast((std::fmod(i, 2.0f) == 0.0f) ? -value : value)); + } + if (builtin_isinf(arg)) return expr(arg); + return expr(static_cast(std::exp(lgamma(static_cast(arg))))); +#endif + } + + /// Floor implementation. + /// \param arg value to round + /// \return rounded value + static half floor(half arg) { + return half(binary, round_half(arg.data_)); + } + + /// Ceiling implementation. + /// \param arg value to round + /// \return rounded value + static half ceil(half arg) { + return half(binary, round_half(arg.data_)); + } + + /// Truncation implementation. + /// \param arg value to round + /// \return rounded value + static half trunc(half arg) { + return half(binary, round_half(arg.data_)); + } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static half round(half arg) { return half(binary, round_half_up(arg.data_)); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long lround(half arg) { return detail::half2int_up(arg.data_); } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static half rint(half arg) { + return half(binary, round_half(arg.data_)); + } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long lrint(half arg) { + return detail::half2int(arg.data_); + } - /// Supports infinity. - static HALF_CONSTEXPR_CONST bool has_infinity = true; +#if HALF_ENABLE_CPP11_LONG_LONG + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long long llround(half arg) { + return detail::half2int_up(arg.data_); + } + + /// Nearest integer implementation. + /// \param arg value to round + /// \return rounded value + static long long llrint(half arg) { + return detail::half2int(arg.data_); + } +#endif - /// Supports quiet NaNs. - static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true; + /// Decompression implementation. + /// \param arg number to decompress + /// \param exp address to store exponent at + /// \return normalized significant + static half frexp(half arg, int *exp) { + int m = arg.data_ & 0x7FFF, e = -14; + if (m >= 0x7C00 || !m) return *exp = 0, arg; + for (; m < 0x400; m <<= 1, --e) + ; + return *exp = e + (m >> 10), + half(binary, (arg.data_ & 0x8000) | 0x3800 | (m & 0x3FF)); + } + + /// Decompression implementation. + /// \param arg number to decompress + /// \param iptr address to store integer part at + /// \return fractional part + static half modf(half arg, half *iptr) { + unsigned int e = arg.data_ & 0x7FFF; + if (e >= 0x6400) + return *iptr = arg, half(binary, arg.data_ & (0x8000U | -(e > 0x7C00))); + if (e < 0x3C00) return iptr->data_ = arg.data_ & 0x8000, arg; + e >>= 10; + unsigned int mask = (1 << (25 - e)) - 1, m = arg.data_ & mask; + iptr->data_ = arg.data_ & ~mask; + if (!m) return half(binary, arg.data_ & 0x8000); + for (; m < 0x400; m <<= 1, --e) + ; + return half(binary, static_cast((arg.data_ & 0x8000) | (e << 10) | + (m & 0x3FF))); + } + + /// Scaling implementation. + /// \param arg number to scale + /// \param exp power of two to scale by + /// \return scaled number + static half scalbln(half arg, long exp) { + unsigned int m = arg.data_ & 0x7FFF; + if (m >= 0x7C00 || !m) return arg; + for (; m < 0x400; m <<= 1, --exp) + ; + exp += m >> 10; + uint16 value = arg.data_ & 0x8000; + if (exp > 30) { + if (half::round_style == std::round_toward_zero) + value |= 0x7BFF; + else if (half::round_style == std::round_toward_infinity) + value |= 0x7C00 - (value >> 15); + else if (half::round_style == std::round_toward_neg_infinity) + value |= 0x7BFF + (value >> 15); + else + value |= 0x7C00; + } else if (exp > 0) + value |= (exp << 10) | (m & 0x3FF); + else if (exp > -11) { + m = (m & 0x3FF) | 0x400; + if (half::round_style == std::round_to_nearest) { + m += 1 << -exp; +#if HALF_ROUND_TIES_TO_EVEN + m -= (m >> (1 - exp)) & 1; +#endif + } else if (half::round_style == std::round_toward_infinity) + m += ((value >> 15) - 1) & ((1 << (1 - exp)) - 1U); + else if (half::round_style == std::round_toward_neg_infinity) + m += -(value >> 15) & ((1 << (1 - exp)) - 1U); + value |= m >> (1 - exp); + } else if (half::round_style == std::round_toward_infinity) + value -= (value >> 15) - 1; + else if (half::round_style == std::round_toward_neg_infinity) + value += value >> 15; + return half(binary, value); + } + + /// Exponent implementation. + /// \param arg number to query + /// \return floating point exponent + static int ilogb(half arg) { + int abs = arg.data_ & 0x7FFF; + if (!abs) return FP_ILOGB0; + if (abs < 0x7C00) { + int exp = (abs >> 10) - 15; + if (abs < 0x400) + for (; abs < 0x200; abs <<= 1, --exp) + ; + return exp; + } + if (abs > 0x7C00) return FP_ILOGBNAN; + return INT_MAX; + } + + /// Exponent implementation. + /// \param arg number to query + /// \return floating point exponent + static half logb(half arg) { + int abs = arg.data_ & 0x7FFF; + if (!abs) return half(binary, 0xFC00); + if (abs < 0x7C00) { + int exp = (abs >> 10) - 15; + if (abs < 0x400) + for (; abs < 0x200; abs <<= 1, --exp) + ; + uint16 bits = (exp < 0) << 15; + if (exp) { + unsigned int m = std::abs(exp) << 6, e = 18; + for (; m < 0x400; m <<= 1, --e) + ; + bits |= (e << 10) + m; + } + return half(binary, bits); + } + if (abs > 0x7C00) return arg; + return half(binary, 0x7C00); + } + + /// Enumeration implementation. + /// \param from number to increase/decrease + /// \param to direction to enumerate into + /// \return next representable number + static half nextafter(half from, half to) { + uint16 fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; + if (fabs > 0x7C00) return from; + if (tabs > 0x7C00 || from.data_ == to.data_ || !(fabs | tabs)) return to; + if (!fabs) return half(binary, (to.data_ & 0x8000) + 1); + bool lt = + ((fabs == from.data_) ? static_cast(fabs) + : -static_cast(fabs)) < + ((tabs == to.data_) ? static_cast(tabs) : -static_cast(tabs)); + return half(binary, + from.data_ + + (((from.data_ >> 15) ^ static_cast(lt)) << 1) - + 1); + } + + /// Enumeration implementation. + /// \param from number to increase/decrease + /// \param to direction to enumerate into + /// \return next representable number + static half nexttoward(half from, long double to) { + if (isnan(from)) return from; + long double lfrom = static_cast(from); + if (builtin_isnan(to) || lfrom == to) return half(static_cast(to)); + if (!(from.data_ & 0x7FFF)) + return half(binary, + (static_cast(builtin_signbit(to)) << 15) + 1); + return half( + binary, + from.data_ + + (((from.data_ >> 15) ^ static_cast(lfrom < to)) << 1) - + 1); + } + + /// Sign implementation + /// \param x first operand + /// \param y second operand + /// \return composed value + static half copysign(half x, half y) { + return half(binary, x.data_ ^ ((x.data_ ^ y.data_) & 0x8000)); + } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if infinite number + /// \retval false else + static int fpclassify(half arg) { + unsigned int abs = arg.data_ & 0x7FFF; + return abs ? ((abs > 0x3FF) ? ((abs >= 0x7C00) + ? ((abs > 0x7C00) ? FP_NAN : FP_INFINITE) + : FP_NORMAL) + : FP_SUBNORMAL) + : FP_ZERO; + } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if finite number + /// \retval false else + static bool isfinite(half arg) { return (arg.data_ & 0x7C00) != 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if infinite number + /// \retval false else + static bool isinf(half arg) { return (arg.data_ & 0x7FFF) == 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if not a number + /// \retval false else + static bool isnan(half arg) { return (arg.data_ & 0x7FFF) > 0x7C00; } + + /// Classification implementation. + /// \param arg value to classify + /// \retval true if normal number + /// \retval false else + static bool isnormal(half arg) { + return ((arg.data_ & 0x7C00) != 0) & ((arg.data_ & 0x7C00) != 0x7C00); + } + + /// Sign bit implementation. + /// \param arg value to check + /// \retval true if signed + /// \retval false if unsigned + static bool signbit(half arg) { return (arg.data_ & 0x8000) != 0; } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operands equal + /// \retval false else + static bool isequal(half x, half y) { + return (x.data_ == y.data_ || !((x.data_ | y.data_) & 0x7FFF)) && !isnan(x); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operands not equal + /// \retval false else + static bool isnotequal(half x, half y) { + return (x.data_ != y.data_ && ((x.data_ | y.data_) & 0x7FFF)) || isnan(x); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x > \a y + /// \retval false else + static bool isgreater(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs <= 0x7C00 && yabs <= 0x7C00 && + (((xabs == x.data_) ? xabs : -xabs) > + ((yabs == y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x >= \a y + /// \retval false else + static bool isgreaterequal(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs <= 0x7C00 && yabs <= 0x7C00 && + (((xabs == x.data_) ? xabs : -xabs) >= + ((yabs == y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x < \a y + /// \retval false else + static bool isless(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs <= 0x7C00 && yabs <= 0x7C00 && + (((xabs == x.data_) ? xabs : -xabs) < + ((yabs == y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x <= \a y + /// \retval false else + static bool islessequal(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + return xabs <= 0x7C00 && yabs <= 0x7C00 && + (((xabs == x.data_) ? xabs : -xabs) <= + ((yabs == y.data_) ? yabs : -yabs)); + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if either \a x > \a y nor \a x < \a y + /// \retval false else + static bool islessgreater(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if (xabs > 0x7C00 || yabs > 0x7C00) return false; + int a = (xabs == x.data_) ? xabs : -xabs, + b = (yabs == y.data_) ? yabs : -yabs; + return a < b || a > b; + } + + /// Comparison implementation. + /// \param x first operand + /// \param y second operand + /// \retval true if operand unordered + /// \retval false else + static bool isunordered(half x, half y) { return isnan(x) || isnan(y); } + + private: + static double erf(double arg) { + if (builtin_isinf(arg)) return (arg < 0.0) ? -1.0 : 1.0; + double x2 = arg * arg, ax2 = 0.147 * x2, + value = std::sqrt( + 1.0 - std::exp(-x2 * (1.2732395447351626861510701069801 + ax2) / + (1.0 + ax2))); + return builtin_signbit(arg) ? -value : value; + } + + static double lgamma(double arg) { + double v = 1.0; + for (; arg < 8.0; ++arg) v *= arg; + double w = 1.0 / (arg * arg); + return (((((((-0.02955065359477124183006535947712 * w + + 0.00641025641025641025641025641026) * + w + + -0.00191752691752691752691752691753) * + w + + 8.4175084175084175084175084175084e-4) * + w + + -5.952380952380952380952380952381e-4) * + w + + 7.9365079365079365079365079365079e-4) * + w + + -0.00277777777777777777777777777778) * + w + + 0.08333333333333333333333333333333) / + arg + + 0.91893853320467274178032973640562 - std::log(v) - arg + + (arg - 0.5) * std::log(arg); + } +}; + +/// Wrapper for unary half-precision functions needing specialization for +/// individual argument types. \tparam T argument type +template +struct unary_specialized { + /// Negation implementation. + /// \param arg value to negate + /// \return negated value + static HALF_CONSTEXPR half negate(half arg) { + return half(binary, arg.data_ ^ 0x8000); + } + + /// Absolute value implementation. + /// \param arg function argument + /// \return absolute value + static half fabs(half arg) { return half(binary, arg.data_ & 0x7FFF); } +}; +template <> +struct unary_specialized { + static HALF_CONSTEXPR expr negate(float arg) { return expr(-arg); } + static expr fabs(float arg) { return expr(std::fabs(arg)); } +}; + +/// Wrapper for binary half-precision functions needing specialization for +/// individual argument types. \tparam T first argument type \tparam U first +/// argument type +template +struct binary_specialized { + /// Minimum implementation. + /// \param x first operand + /// \param y second operand + /// \return minimum value + static expr fmin(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::fmin(x, y)); +#else + if (builtin_isnan(x)) return expr(y); + if (builtin_isnan(y)) return expr(x); + return expr(std::min(x, y)); +#endif + } + + /// Maximum implementation. + /// \param x first operand + /// \param y second operand + /// \return maximum value + static expr fmax(float x, float y) { +#if HALF_ENABLE_CPP11_CMATH + return expr(std::fmax(x, y)); +#else + if (builtin_isnan(x)) return expr(y); + if (builtin_isnan(y)) return expr(x); + return expr(std::max(x, y)); +#endif + } +}; +template <> +struct binary_specialized { + static half fmin(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if (xabs > 0x7C00) return y; + if (yabs > 0x7C00) return x; + return (((xabs == x.data_) ? xabs : -xabs) > + ((yabs == y.data_) ? yabs : -yabs)) + ? y + : x; + } + static half fmax(half x, half y) { + int xabs = x.data_ & 0x7FFF, yabs = y.data_ & 0x7FFF; + if (xabs > 0x7C00) return y; + if (yabs > 0x7C00) return x; + return (((xabs == x.data_) ? xabs : -xabs) < + ((yabs == y.data_) ? yabs : -yabs)) + ? y + : x; + } +}; + +/// Helper class for half casts. +/// This class template has to be specialized for all valid cast argument to +/// define an appropriate static `cast` member function and a corresponding +/// `type` member denoting its return type. \tparam T destination type \tparam U +/// source type \tparam R rounding mode to use +template +struct half_caster {}; +template +struct half_caster { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, + "half_cast from non-arithmetic type unsupported"); +#endif - /// Supports subnormal values. - static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present; + static half cast(U arg) { return cast_impl(arg, is_float()); }; + + private: + static half cast_impl(U arg, true_type) { + return half(binary, float2half(arg)); + } + static half cast_impl(U arg, false_type) { + return half(binary, int2half(arg)); + } +}; +template +struct half_caster { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, + "half_cast to non-arithmetic type unsupported"); +#endif - /// Rounding mode. - /// Due to the mix of internal single-precision computations (using the rounding mode of the underlying - /// single-precision implementation) with the rounding mode of the single-to-half conversions, the actual rounding - /// mode might be `std::round_indeterminate` if the default half-precision rounding mode doesn't match the - /// single-precision rounding mode. - static HALF_CONSTEXPR_CONST float_round_style round_style = (std::numeric_limits::round_style== - half_float::half::round_style) ? half_float::half::round_style : round_indeterminate; + static T cast(half arg) { return cast_impl(arg, is_float()); } + + private: + static T cast_impl(half arg, true_type) { return half2float(arg.data_); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } +}; +template +struct half_caster { +#if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, + "half_cast to non-arithmetic type unsupported"); +#endif - /// Significant digits. - static HALF_CONSTEXPR_CONST int digits = 11; + static T cast(expr arg) { return cast_impl(arg, is_float()); } + + private: + static T cast_impl(float arg, true_type) { return static_cast(arg); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } +}; +template +struct half_caster { + static half cast(half arg) { return arg; } +}; +template +struct half_caster : half_caster {}; + +/// \name Comparison operators +/// \{ + +/// Comparison for equality. +/// \param x first operand +/// \param y second operand +/// \retval true if operands equal +/// \retval false else +template +typename enable::type operator==(T x, U y) { + return functions::isequal(x, y); +} - /// Significant decimal digits. - static HALF_CONSTEXPR_CONST int digits10 = 3; +/// Comparison for inequality. +/// \param x first operand +/// \param y second operand +/// \retval true if operands not equal +/// \retval false else +template +typename enable::type operator!=(T x, U y) { + return functions::isnotequal(x, y); +} - /// Required decimal digits to represent all possible values. - static HALF_CONSTEXPR_CONST int max_digits10 = 5; +/// Comparison for less than. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x less than \a y +/// \retval false else +template +typename enable::type operator<(T x, U y) { + return functions::isless(x, y); +} - /// Number base. - static HALF_CONSTEXPR_CONST int radix = 2; +/// Comparison for greater than. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x greater than \a y +/// \retval false else +template +typename enable::type operator>(T x, U y) { + return functions::isgreater(x, y); +} - /// One more than smallest exponent. - static HALF_CONSTEXPR_CONST int min_exponent = -13; +/// Comparison for less equal. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x less equal \a y +/// \retval false else +template +typename enable::type operator<=(T x, U y) { + return functions::islessequal(x, y); +} - /// Smallest normalized representable power of 10. - static HALF_CONSTEXPR_CONST int min_exponent10 = -4; +/// Comparison for greater equal. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x greater equal \a y +/// \retval false else +template +typename enable::type operator>=(T x, U y) { + return functions::isgreaterequal(x, y); +} - /// One more than largest exponent - static HALF_CONSTEXPR_CONST int max_exponent = 16; +/// \} +/// \name Arithmetic operators +/// \{ + +/// Add halfs. +/// \param x left operand +/// \param y right operand +/// \return sum of half expressions +template +typename enable::type operator+(T x, U y) { + return functions::plus(x, y); +} - /// Largest finitely representable power of 10. - static HALF_CONSTEXPR_CONST int max_exponent10 = 4; +/// Subtract halfs. +/// \param x left operand +/// \param y right operand +/// \return difference of half expressions +template +typename enable::type operator-(T x, U y) { + return functions::minus(x, y); +} - /// Smallest positive normal value. - static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); } +/// Multiply halfs. +/// \param x left operand +/// \param y right operand +/// \return product of half expressions +template +typename enable::type operator*(T x, U y) { + return functions::multiplies(x, y); +} - /// Smallest finite value. - static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); } +/// Divide halfs. +/// \param x left operand +/// \param y right operand +/// \return quotient of half expressions +template +typename enable::type operator/(T x, U y) { + return functions::divides(x, y); +} - /// Largest finite value. - static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); } +/// Identity. +/// \param arg operand +/// \return uncahnged operand +template +HALF_CONSTEXPR typename enable::type operator+(T arg) { + return arg; +} - /// Difference between one and next representable value. - static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); } +/// Negation. +/// \param arg operand +/// \return negated operand +template +HALF_CONSTEXPR typename enable::type operator-(T arg) { + return unary_specialized::negate(arg); +} - /// Maximum rounding error. - static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW - { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); } +/// \} +/// \name Input and output +/// \{ + +/// Output operator. +/// \param out output stream to write into +/// \param arg half expression to write +/// \return reference to output stream +template +typename enable &, T>::type operator<<( + std::basic_ostream &out, T arg) { + return functions::write(out, arg); +} - /// Positive infinity. - static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); } +/// Input operator. +/// \param in input stream to read from +/// \param arg half to read into +/// \return reference to input stream +template +std::basic_istream &operator>>( + std::basic_istream &in, half &arg) { + return functions::read(in, arg); +} - /// Quiet NaN. - static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); } +/// \} +/// \name Basic mathematical operations +/// \{ + +/// Absolute value. +/// \param arg operand +/// \return absolute value of \a arg +// template typename enable::type abs(T arg) { +//return unary_specialized::fabs(arg); } +inline half abs(half arg) { return unary_specialized::fabs(arg); } +inline expr abs(expr arg) { return unary_specialized::fabs(arg); } + +/// Absolute value. +/// \param arg operand +/// \return absolute value of \a arg +// template typename enable::type fabs(T arg) { +//return unary_specialized::fabs(arg); } +inline half fabs(half arg) { return unary_specialized::fabs(arg); } +inline expr fabs(expr arg) { return unary_specialized::fabs(arg); } + +/// Remainder of division. +/// \param x first operand +/// \param y second operand +/// \return remainder of floating point division. +// template typename enable::type +//fmod(T x, U y) { return functions::fmod(x, y); } +inline expr fmod(half x, half y) { return functions::fmod(x, y); } +inline expr fmod(half x, expr y) { return functions::fmod(x, y); } +inline expr fmod(expr x, half y) { return functions::fmod(x, y); } +inline expr fmod(expr x, expr y) { return functions::fmod(x, y); } + +/// Remainder of division. +/// \param x first operand +/// \param y second operand +/// \return remainder of floating point division. +// template typename enable::type +//remainder(T x, U y) { return functions::remainder(x, y); } +inline expr remainder(half x, half y) { return functions::remainder(x, y); } +inline expr remainder(half x, expr y) { return functions::remainder(x, y); } +inline expr remainder(expr x, half y) { return functions::remainder(x, y); } +inline expr remainder(expr x, expr y) { return functions::remainder(x, y); } + +/// Remainder of division. +/// \param x first operand +/// \param y second operand +/// \param quo address to store some bits of quotient at +/// \return remainder of floating point division. +// template typename enable::type +//remquo(T x, U y, int *quo) { return functions::remquo(x, y, quo); } +inline expr remquo(half x, half y, int *quo) { + return functions::remquo(x, y, quo); +} +inline expr remquo(half x, expr y, int *quo) { + return functions::remquo(x, y, quo); +} +inline expr remquo(expr x, half y, int *quo) { + return functions::remquo(x, y, quo); +} +inline expr remquo(expr x, expr y, int *quo) { + return functions::remquo(x, y, quo); +} - /// Signalling NaN. - static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); } +/// Fused multiply add. +/// \param x first operand +/// \param y second operand +/// \param z third operand +/// \return ( \a x * \a y ) + \a z rounded as one operation. +// template typename +//enable::type fma(T x, U y, V z) { return functions::fma(x, y, z); +//} +inline expr fma(half x, half y, half z) { return functions::fma(x, y, z); } +inline expr fma(half x, half y, expr z) { return functions::fma(x, y, z); } +inline expr fma(half x, expr y, half z) { return functions::fma(x, y, z); } +inline expr fma(half x, expr y, expr z) { return functions::fma(x, y, z); } +inline expr fma(expr x, half y, half z) { return functions::fma(x, y, z); } +inline expr fma(expr x, half y, expr z) { return functions::fma(x, y, z); } +inline expr fma(expr x, expr y, half z) { return functions::fma(x, y, z); } +inline expr fma(expr x, expr y, expr z) { return functions::fma(x, y, z); } + +/// Maximum of half expressions. +/// \param x first operand +/// \param y second operand +/// \return maximum of operands +// template typename result::type fmax(T +//x, U y) { return binary_specialized::fmax(x, y); } +inline half fmax(half x, half y) { + return binary_specialized::fmax(x, y); +} +inline expr fmax(half x, expr y) { + return binary_specialized::fmax(x, y); +} +inline expr fmax(expr x, half y) { + return binary_specialized::fmax(x, y); +} +inline expr fmax(expr x, expr y) { + return binary_specialized::fmax(x, y); +} - /// Smallest positive subnormal value. - static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); } - }; +/// Minimum of half expressions. +/// \param x first operand +/// \param y second operand +/// \return minimum of operands +// template typename result::type fmin(T +//x, U y) { return binary_specialized::fmin(x, y); } +inline half fmin(half x, half y) { + return binary_specialized::fmin(x, y); +} +inline expr fmin(half x, expr y) { + return binary_specialized::fmin(x, y); +} +inline expr fmin(expr x, half y) { + return binary_specialized::fmin(x, y); +} +inline expr fmin(expr x, expr y) { + return binary_specialized::fmin(x, y); +} -#if HALF_ENABLE_CPP11_HASH - /// Hash function for half-precision floats. - /// This is only defined if C++11 `std::hash` is supported and enabled. - template<> struct hash //: unary_function - { - /// Type of function argument. - typedef half_float::half argument_type; - - /// Function return type. - typedef size_t result_type; - - /// Compute hash function. - /// \param arg half to hash - /// \return hash value - result_type operator()(argument_type arg) const - { return hash()(static_cast(arg.data_)&-(arg.data_!=0x8000)); } - }; +/// Positive difference. +/// \param x first operand +/// \param y second operand +/// \return \a x - \a y or 0 if difference negative +// template typename enable::type +//fdim(T x, U y) { return functions::fdim(x, y); } +inline expr fdim(half x, half y) { return functions::fdim(x, y); } +inline expr fdim(half x, expr y) { return functions::fdim(x, y); } +inline expr fdim(expr x, half y) { return functions::fdim(x, y); } +inline expr fdim(expr x, expr y) { return functions::fdim(x, y); } + +/// Get NaN value. +/// \return quiet NaN +inline half nanh(const char *) { return functions::nanh(); } + +/// \} +/// \name Exponential functions +/// \{ + +/// Exponential function. +/// \param arg function argument +/// \return e raised to \a arg +// template typename enable::type exp(T arg) { +//return functions::exp(arg); } +inline expr exp(half arg) { return functions::exp(arg); } +inline expr exp(expr arg) { return functions::exp(arg); } + +/// Exponential minus one. +/// \param arg function argument +/// \return e raised to \a arg subtracted by 1 +// template typename enable::type expm1(T arg) { +//return functions::expm1(arg); } +inline expr expm1(half arg) { return functions::expm1(arg); } +inline expr expm1(expr arg) { return functions::expm1(arg); } + +/// Binary exponential. +/// \param arg function argument +/// \return 2 raised to \a arg +// template typename enable::type exp2(T arg) { +//return functions::exp2(arg); } +inline expr exp2(half arg) { return functions::exp2(arg); } +inline expr exp2(expr arg) { return functions::exp2(arg); } + +/// Natural logorithm. +/// \param arg function argument +/// \return logarithm of \a arg to base e +// template typename enable::type log(T arg) { +//return functions::log(arg); } +inline expr log(half arg) { return functions::log(arg); } +inline expr log(expr arg) { return functions::log(arg); } + +/// Common logorithm. +/// \param arg function argument +/// \return logarithm of \a arg to base 10 +// template typename enable::type log10(T arg) { +//return functions::log10(arg); } +inline expr log10(half arg) { return functions::log10(arg); } +inline expr log10(expr arg) { return functions::log10(arg); } + +/// Natural logorithm. +/// \param arg function argument +/// \return logarithm of \a arg plus 1 to base e +// template typename enable::type log1p(T arg) { +//return functions::log1p(arg); } +inline expr log1p(half arg) { return functions::log1p(arg); } +inline expr log1p(expr arg) { return functions::log1p(arg); } + +/// Binary logorithm. +/// \param arg function argument +/// \return logarithm of \a arg to base 2 +// template typename enable::type log2(T arg) { +//return functions::log2(arg); } +inline expr log2(half arg) { return functions::log2(arg); } +inline expr log2(expr arg) { return functions::log2(arg); } + +/// \} +/// \name Power functions +/// \{ + +/// Square root. +/// \param arg function argument +/// \return square root of \a arg +// template typename enable::type sqrt(T arg) { +//return functions::sqrt(arg); } +inline expr sqrt(half arg) { return functions::sqrt(arg); } +inline expr sqrt(expr arg) { return functions::sqrt(arg); } + +/// Cubic root. +/// \param arg function argument +/// \return cubic root of \a arg +// template typename enable::type cbrt(T arg) { +//return functions::cbrt(arg); } +inline expr cbrt(half arg) { return functions::cbrt(arg); } +inline expr cbrt(expr arg) { return functions::cbrt(arg); } + +/// Hypotenuse function. +/// \param x first argument +/// \param y second argument +/// \return square root of sum of squares without internal over- or underflows +// template typename enable::type +//hypot(T x, U y) { return functions::hypot(x, y); } +inline expr hypot(half x, half y) { return functions::hypot(x, y); } +inline expr hypot(half x, expr y) { return functions::hypot(x, y); } +inline expr hypot(expr x, half y) { return functions::hypot(x, y); } +inline expr hypot(expr x, expr y) { return functions::hypot(x, y); } + +/// Power function. +/// \param base first argument +/// \param exp second argument +/// \return \a base raised to \a exp +// template typename enable::type +//pow(T base, U exp) { return functions::pow(base, exp); } +inline expr pow(half base, half exp) { return functions::pow(base, exp); } +inline expr pow(half base, expr exp) { return functions::pow(base, exp); } +inline expr pow(expr base, half exp) { return functions::pow(base, exp); } +inline expr pow(expr base, expr exp) { return functions::pow(base, exp); } + +/// \} +/// \name Trigonometric functions +/// \{ + +/// Sine function. +/// \param arg function argument +/// \return sine value of \a arg +// template typename enable::type sin(T arg) { +//return functions::sin(arg); } +inline expr sin(half arg) { return functions::sin(arg); } +inline expr sin(expr arg) { return functions::sin(arg); } + +/// Cosine function. +/// \param arg function argument +/// \return cosine value of \a arg +// template typename enable::type cos(T arg) { +//return functions::cos(arg); } +inline expr cos(half arg) { return functions::cos(arg); } +inline expr cos(expr arg) { return functions::cos(arg); } + +/// Tangent function. +/// \param arg function argument +/// \return tangent value of \a arg +// template typename enable::type tan(T arg) { +//return functions::tan(arg); } +inline expr tan(half arg) { return functions::tan(arg); } +inline expr tan(expr arg) { return functions::tan(arg); } + +/// Arc sine. +/// \param arg function argument +/// \return arc sine value of \a arg +// template typename enable::type asin(T arg) { +//return functions::asin(arg); } +inline expr asin(half arg) { return functions::asin(arg); } +inline expr asin(expr arg) { return functions::asin(arg); } + +/// Arc cosine function. +/// \param arg function argument +/// \return arc cosine value of \a arg +// template typename enable::type acos(T arg) { +//return functions::acos(arg); } +inline expr acos(half arg) { return functions::acos(arg); } +inline expr acos(expr arg) { return functions::acos(arg); } + +/// Arc tangent function. +/// \param arg function argument +/// \return arc tangent value of \a arg +// template typename enable::type atan(T arg) { +//return functions::atan(arg); } +inline expr atan(half arg) { return functions::atan(arg); } +inline expr atan(expr arg) { return functions::atan(arg); } + +/// Arc tangent function. +/// \param x first argument +/// \param y second argument +/// \return arc tangent value +// template typename enable::type +//atan2(T x, U y) { return functions::atan2(x, y); } +inline expr atan2(half x, half y) { return functions::atan2(x, y); } +inline expr atan2(half x, expr y) { return functions::atan2(x, y); } +inline expr atan2(expr x, half y) { return functions::atan2(x, y); } +inline expr atan2(expr x, expr y) { return functions::atan2(x, y); } + +/// \} +/// \name Hyperbolic functions +/// \{ + +/// Hyperbolic sine. +/// \param arg function argument +/// \return hyperbolic sine value of \a arg +// template typename enable::type sinh(T arg) { +//return functions::sinh(arg); } +inline expr sinh(half arg) { return functions::sinh(arg); } +inline expr sinh(expr arg) { return functions::sinh(arg); } + +/// Hyperbolic cosine. +/// \param arg function argument +/// \return hyperbolic cosine value of \a arg +// template typename enable::type cosh(T arg) { +//return functions::cosh(arg); } +inline expr cosh(half arg) { return functions::cosh(arg); } +inline expr cosh(expr arg) { return functions::cosh(arg); } + +/// Hyperbolic tangent. +/// \param arg function argument +/// \return hyperbolic tangent value of \a arg +// template typename enable::type tanh(T arg) { +//return functions::tanh(arg); } +inline expr tanh(half arg) { return functions::tanh(arg); } +inline expr tanh(expr arg) { return functions::tanh(arg); } + +/// Hyperbolic area sine. +/// \param arg function argument +/// \return area sine value of \a arg +// template typename enable::type asinh(T arg) { +//return functions::asinh(arg); } +inline expr asinh(half arg) { return functions::asinh(arg); } +inline expr asinh(expr arg) { return functions::asinh(arg); } + +/// Hyperbolic area cosine. +/// \param arg function argument +/// \return area cosine value of \a arg +// template typename enable::type acosh(T arg) { +//return functions::acosh(arg); } +inline expr acosh(half arg) { return functions::acosh(arg); } +inline expr acosh(expr arg) { return functions::acosh(arg); } + +/// Hyperbolic area tangent. +/// \param arg function argument +/// \return area tangent value of \a arg +// template typename enable::type atanh(T arg) { +//return functions::atanh(arg); } +inline expr atanh(half arg) { return functions::atanh(arg); } +inline expr atanh(expr arg) { return functions::atanh(arg); } + +/// \} +/// \name Error and gamma functions +/// \{ + +/// Error function. +/// \param arg function argument +/// \return error function value of \a arg +// template typename enable::type erf(T arg) { +//return functions::erf(arg); } +inline expr erf(half arg) { return functions::erf(arg); } +inline expr erf(expr arg) { return functions::erf(arg); } + +/// Complementary error function. +/// \param arg function argument +/// \return 1 minus error function value of \a arg +// template typename enable::type erfc(T arg) { +//return functions::erfc(arg); } +inline expr erfc(half arg) { return functions::erfc(arg); } +inline expr erfc(expr arg) { return functions::erfc(arg); } + +/// Natural logarithm of gamma function. +/// \param arg function argument +/// \return natural logarith of gamma function for \a arg +// template typename enable::type lgamma(T arg) { +//return functions::lgamma(arg); } +inline expr lgamma(half arg) { return functions::lgamma(arg); } +inline expr lgamma(expr arg) { return functions::lgamma(arg); } + +/// Gamma function. +/// \param arg function argument +/// \return gamma function value of \a arg +// template typename enable::type tgamma(T arg) { +//return functions::tgamma(arg); } +inline expr tgamma(half arg) { return functions::tgamma(arg); } +inline expr tgamma(expr arg) { return functions::tgamma(arg); } + +/// \} +/// \name Rounding +/// \{ + +/// Nearest integer not less than half value. +/// \param arg half to round +/// \return nearest integer not less than \a arg +// template typename enable::type ceil(T arg) { +//return functions::ceil(arg); } +inline half ceil(half arg) { return functions::ceil(arg); } +inline half ceil(expr arg) { return functions::ceil(arg); } + +/// Nearest integer not greater than half value. +/// \param arg half to round +/// \return nearest integer not greater than \a arg +// template typename enable::type floor(T arg) { +//return functions::floor(arg); } +inline half floor(half arg) { return functions::floor(arg); } +inline half floor(expr arg) { return functions::floor(arg); } + +/// Nearest integer not greater in magnitude than half value. +/// \param arg half to round +/// \return nearest integer not greater in magnitude than \a arg +// template typename enable::type trunc(T arg) { +//return functions::trunc(arg); } +inline half trunc(half arg) { return functions::trunc(arg); } +inline half trunc(expr arg) { return functions::trunc(arg); } + +/// Nearest integer. +/// \param arg half to round +/// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type round(T arg) { +//return functions::round(arg); } +inline half round(half arg) { return functions::round(arg); } +inline half round(expr arg) { return functions::round(arg); } + +/// Nearest integer. +/// \param arg half to round +/// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type lround(T arg) { +//return functions::lround(arg); } +inline long lround(half arg) { return functions::lround(arg); } +inline long lround(expr arg) { return functions::lround(arg); } + +/// Nearest integer using half's internal rounding mode. +/// \param arg half expression to round +/// \return nearest integer using default rounding mode +// template typename enable::type nearbyint(T +//arg) { return functions::nearbyint(arg); } +inline half nearbyint(half arg) { return functions::rint(arg); } +inline half nearbyint(expr arg) { return functions::rint(arg); } + +/// Nearest integer using half's internal rounding mode. +/// \param arg half expression to round +/// \return nearest integer using default rounding mode +// template typename enable::type rint(T arg) { +//return functions::rint(arg); } +inline half rint(half arg) { return functions::rint(arg); } +inline half rint(expr arg) { return functions::rint(arg); } + +/// Nearest integer using half's internal rounding mode. +/// \param arg half expression to round +/// \return nearest integer using default rounding mode +// template typename enable::type lrint(T arg) { +//return functions::lrint(arg); } +inline long lrint(half arg) { return functions::lrint(arg); } +inline long lrint(expr arg) { return functions::lrint(arg); } +#if HALF_ENABLE_CPP11_LONG_LONG +/// Nearest integer. +/// \param arg half to round +/// \return nearest integer, rounded away from zero in half-way cases +// template typename enable::type llround(T +//arg) { return functions::llround(arg); } +inline long long llround(half arg) { return functions::llround(arg); } +inline long long llround(expr arg) { return functions::llround(arg); } + +/// Nearest integer using half's internal rounding mode. +/// \param arg half expression to round +/// \return nearest integer using default rounding mode +// template typename enable::type llrint(T +//arg) { return functions::llrint(arg); } +inline long long llrint(half arg) { return functions::llrint(arg); } +inline long long llrint(expr arg) { return functions::llrint(arg); } #endif + +/// \} +/// \name Floating point manipulation +/// \{ + +/// Decompress floating point number. +/// \param arg number to decompress +/// \param exp address to store exponent at +/// \return significant in range [0.5, 1) +// template typename enable::type frexp(T arg, +//int *exp) { return functions::frexp(arg, exp); } +inline half frexp(half arg, int *exp) { return functions::frexp(arg, exp); } +inline half frexp(expr arg, int *exp) { return functions::frexp(arg, exp); } + +/// Multiply by power of two. +/// \param arg number to modify +/// \param exp power of two to multiply with +/// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type ldexp(T arg, +//int exp) { return functions::scalbln(arg, exp); } +inline half ldexp(half arg, int exp) { return functions::scalbln(arg, exp); } +inline half ldexp(expr arg, int exp) { return functions::scalbln(arg, exp); } + +/// Extract integer and fractional parts. +/// \param arg number to decompress +/// \param iptr address to store integer part at +/// \return fractional part +// template typename enable::type modf(T arg, +//half *iptr) { return functions::modf(arg, iptr); } +inline half modf(half arg, half *iptr) { return functions::modf(arg, iptr); } +inline half modf(expr arg, half *iptr) { return functions::modf(arg, iptr); } + +/// Multiply by power of two. +/// \param arg number to modify +/// \param exp power of two to multiply with +/// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type scalbn(T arg, +//int exp) { return functions::scalbln(arg, exp); } +inline half scalbn(half arg, int exp) { return functions::scalbln(arg, exp); } +inline half scalbn(expr arg, int exp) { return functions::scalbln(arg, exp); } + +/// Multiply by power of two. +/// \param arg number to modify +/// \param exp power of two to multiply with +/// \return \a arg multplied by 2 raised to \a exp +// template typename enable::type scalbln(T arg, +//long exp) { return functions::scalbln(arg, exp); } +inline half scalbln(half arg, long exp) { return functions::scalbln(arg, exp); } +inline half scalbln(expr arg, long exp) { return functions::scalbln(arg, exp); } + +/// Extract exponent. +/// \param arg number to query +/// \return floating point exponent +/// \retval FP_ILOGB0 for zero +/// \retval FP_ILOGBNAN for NaN +/// \retval MAX_INT for infinity +// template typename enable::type ilogb(T arg) { +//return functions::ilogb(arg); } +inline int ilogb(half arg) { return functions::ilogb(arg); } +inline int ilogb(expr arg) { return functions::ilogb(arg); } + +/// Extract exponent. +/// \param arg number to query +/// \return floating point exponent +// template typename enable::type logb(T arg) { +//return functions::logb(arg); } +inline half logb(half arg) { return functions::logb(arg); } +inline half logb(expr arg) { return functions::logb(arg); } + +/// Next representable value. +/// \param from value to compute next representable value for +/// \param to direction towards which to compute next value +/// \return next representable value after \a from in direction towards \a to +// template typename enable::type +//nextafter(T from, U to) { return functions::nextafter(from, to); } +inline half nextafter(half from, half to) { + return functions::nextafter(from, to); +} +inline half nextafter(half from, expr to) { + return functions::nextafter(from, to); } +inline half nextafter(expr from, half to) { + return functions::nextafter(from, to); +} +inline half nextafter(expr from, expr to) { + return functions::nextafter(from, to); +} + +/// Next representable value. +/// \param from value to compute next representable value for +/// \param to direction towards which to compute next value +/// \return next representable value after \a from in direction towards \a to +// template typename enable::type nexttoward(T +//from, long double to) { return functions::nexttoward(from, to); } +inline half nexttoward(half from, long double to) { + return functions::nexttoward(from, to); +} +inline half nexttoward(expr from, long double to) { + return functions::nexttoward(from, to); +} + +/// Take sign. +/// \param x value to change sign for +/// \param y value to take sign from +/// \return value equal to \a x in magnitude and to \a y in sign +// template typename enable::type +//copysign(T x, U y) { return functions::copysign(x, y); } +inline half copysign(half x, half y) { return functions::copysign(x, y); } +inline half copysign(half x, expr y) { return functions::copysign(x, y); } +inline half copysign(expr x, half y) { return functions::copysign(x, y); } +inline half copysign(expr x, expr y) { return functions::copysign(x, y); } + +/// \} +/// \name Floating point classification +/// \{ + +/// Classify floating point value. +/// \param arg number to classify +/// \retval FP_ZERO for positive and negative zero +/// \retval FP_SUBNORMAL for subnormal numbers +/// \retval FP_INFINITY for positive and negative infinity +/// \retval FP_NAN for NaNs +/// \retval FP_NORMAL for all other (normal) values +// template typename enable::type fpclassify(T +//arg) { return functions::fpclassify(arg); } +inline int fpclassify(half arg) { return functions::fpclassify(arg); } +inline int fpclassify(expr arg) { return functions::fpclassify(arg); } + +/// Check if finite number. +/// \param arg number to check +/// \retval true if neither infinity nor NaN +/// \retval false else +// template typename enable::type isfinite(T arg) +//{ return functions::isfinite(arg); } +inline bool isfinite(half arg) { return functions::isfinite(arg); } +inline bool isfinite(expr arg) { return functions::isfinite(arg); } + +/// Check for infinity. +/// \param arg number to check +/// \retval true for positive or negative infinity +/// \retval false else +// template typename enable::type isinf(T arg) { +//return functions::isinf(arg); } +inline bool isinf(half arg) { return functions::isinf(arg); } +inline bool isinf(expr arg) { return functions::isinf(arg); } + +/// Check for NaN. +/// \param arg number to check +/// \retval true for NaNs +/// \retval false else +// template typename enable::type isnan(T arg) { +//return functions::isnan(arg); } +inline bool isnan(half arg) { return functions::isnan(arg); } +inline bool isnan(expr arg) { return functions::isnan(arg); } + +/// Check if normal number. +/// \param arg number to check +/// \retval true if normal number +/// \retval false if either subnormal, zero, infinity or NaN +// template typename enable::type isnormal(T arg) +//{ return functions::isnormal(arg); } +inline bool isnormal(half arg) { return functions::isnormal(arg); } +inline bool isnormal(expr arg) { return functions::isnormal(arg); } + +/// Check sign. +/// \param arg number to check +/// \retval true for negative number +/// \retval false for positive number +// template typename enable::type signbit(T arg) +//{ return functions::signbit(arg); } +inline bool signbit(half arg) { return functions::signbit(arg); } +inline bool signbit(expr arg) { return functions::signbit(arg); } + +/// \} +/// \name Comparison +/// \{ + +/// Comparison for greater than. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x greater than \a y +/// \retval false else +// template typename enable::type +//isgreater(T x, U y) { return functions::isgreater(x, y); } +inline bool isgreater(half x, half y) { return functions::isgreater(x, y); } +inline bool isgreater(half x, expr y) { return functions::isgreater(x, y); } +inline bool isgreater(expr x, half y) { return functions::isgreater(x, y); } +inline bool isgreater(expr x, expr y) { return functions::isgreater(x, y); } + +/// Comparison for greater equal. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x greater equal \a y +/// \retval false else +// template typename enable::type +//isgreaterequal(T x, U y) { return functions::isgreaterequal(x, y); } +inline bool isgreaterequal(half x, half y) { + return functions::isgreaterequal(x, y); +} +inline bool isgreaterequal(half x, expr y) { + return functions::isgreaterequal(x, y); +} +inline bool isgreaterequal(expr x, half y) { + return functions::isgreaterequal(x, y); +} +inline bool isgreaterequal(expr x, expr y) { + return functions::isgreaterequal(x, y); +} + +/// Comparison for less than. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x less than \a y +/// \retval false else +// template typename enable::type +//isless(T x, U y) { return functions::isless(x, y); } +inline bool isless(half x, half y) { return functions::isless(x, y); } +inline bool isless(half x, expr y) { return functions::isless(x, y); } +inline bool isless(expr x, half y) { return functions::isless(x, y); } +inline bool isless(expr x, expr y) { return functions::isless(x, y); } + +/// Comparison for less equal. +/// \param x first operand +/// \param y second operand +/// \retval true if \a x less equal \a y +/// \retval false else +// template typename enable::type +//islessequal(T x, U y) { return functions::islessequal(x, y); } +inline bool islessequal(half x, half y) { return functions::islessequal(x, y); } +inline bool islessequal(half x, expr y) { return functions::islessequal(x, y); } +inline bool islessequal(expr x, half y) { return functions::islessequal(x, y); } +inline bool islessequal(expr x, expr y) { return functions::islessequal(x, y); } + +/// Comarison for less or greater. +/// \param x first operand +/// \param y second operand +/// \retval true if either less or greater +/// \retval false else +// template typename enable::type +//islessgreater(T x, U y) { return functions::islessgreater(x, y); } +inline bool islessgreater(half x, half y) { + return functions::islessgreater(x, y); +} +inline bool islessgreater(half x, expr y) { + return functions::islessgreater(x, y); +} +inline bool islessgreater(expr x, half y) { + return functions::islessgreater(x, y); +} +inline bool islessgreater(expr x, expr y) { + return functions::islessgreater(x, y); +} + +/// Check if unordered. +/// \param x first operand +/// \param y second operand +/// \retval true if unordered (one or two NaN operands) +/// \retval false else +// template typename enable::type +//isunordered(T x, U y) { return functions::isunordered(x, y); } +inline bool isunordered(half x, half y) { return functions::isunordered(x, y); } +inline bool isunordered(half x, expr y) { return functions::isunordered(x, y); } +inline bool isunordered(expr x, half y) { return functions::isunordered(x, y); } +inline bool isunordered(expr x, expr y) { return functions::isunordered(x, y); } + +/// \name Casting +/// \{ + +/// Cast to or from half-precision floating point number. +/// This casts between [half](\ref half_float::half) and any built-in arithmetic +/// type. The values are converted directly using the given rounding mode, +/// without any roundtrip over `float` that a `static_cast` would otherwise do. +/// It uses the default rounding mode. +/// +/// Using this cast with neither of the two types being a [half](\ref +/// half_float::half) or with any of the two types not being a built-in +/// arithmetic type (apart from [half](\ref half_float::half), of course) +/// results in a compiler error and casting between [half](\ref +/// half_float::half)s is just a no-op. \tparam T destination type (half or +/// built-in arithmetic type) \tparam U source type (half or built-in arithmetic +/// type) \param arg value to cast \return \a arg converted to destination type +template +T half_cast(U arg) { + return half_caster::cast(arg); +} + +/// Cast to or from half-precision floating point number. +/// This casts between [half](\ref half_float::half) and any built-in arithmetic +/// type. The values are converted directly using the given rounding mode, +/// without any roundtrip over `float` that a `static_cast` would otherwise do. +/// +/// Using this cast with neither of the two types being a [half](\ref +/// half_float::half) or with any of the two types not being a built-in +/// arithmetic type (apart from [half](\ref half_float::half), of course) +/// results in a compiler error and casting between [half](\ref +/// half_float::half)s is just a no-op. \tparam T destination type (half or +/// built-in arithmetic type) \tparam R rounding mode to use. \tparam U source +/// type (half or built-in arithmetic type) \param arg value to cast \return \a +/// arg converted to destination type +template +T half_cast(U arg) { + return half_caster::cast(arg); +} +/// \} +} // namespace detail + +using detail::operator==; +using detail::operator!=; +using detail::operator<; +using detail::operator>; +using detail::operator<=; +using detail::operator>=; +using detail::operator+; +using detail::operator-; +using detail::operator*; +using detail::operator/; +using detail::operator<<; +using detail::operator>>; + +using detail::abs; +using detail::acos; +using detail::acosh; +using detail::asin; +using detail::asinh; +using detail::atan; +using detail::atan2; +using detail::atanh; +using detail::cbrt; +using detail::ceil; +using detail::cos; +using detail::cosh; +using detail::erf; +using detail::erfc; +using detail::exp; +using detail::exp2; +using detail::expm1; +using detail::fabs; +using detail::fdim; +using detail::floor; +using detail::fma; +using detail::fmax; +using detail::fmin; +using detail::fmod; +using detail::hypot; +using detail::lgamma; +using detail::log; +using detail::log10; +using detail::log1p; +using detail::log2; +using detail::lrint; +using detail::lround; +using detail::nanh; +using detail::nearbyint; +using detail::pow; +using detail::remainder; +using detail::remquo; +using detail::rint; +using detail::round; +using detail::sin; +using detail::sinh; +using detail::sqrt; +using detail::tan; +using detail::tanh; +using detail::tgamma; +using detail::trunc; +#if HALF_ENABLE_CPP11_LONG_LONG +using detail::llrint; +using detail::llround; +#endif +using detail::copysign; +using detail::fpclassify; +using detail::frexp; +using detail::ilogb; +using detail::isfinite; +using detail::isgreater; +using detail::isgreaterequal; +using detail::isinf; +using detail::isless; +using detail::islessequal; +using detail::islessgreater; +using detail::isnan; +using detail::isnormal; +using detail::isunordered; +using detail::ldexp; +using detail::logb; +using detail::modf; +using detail::nextafter; +using detail::nexttoward; +using detail::scalbln; +using detail::scalbn; +using detail::signbit; + +using detail::half_cast; +} // namespace half_float + +/// Extensions to the C++ standard library. +namespace std { +/// Numeric limits for half-precision floats. +/// Because of the underlying single-precision implementation of many +/// operations, it inherits some properties from `std::numeric_limits`. +template <> +class numeric_limits : public numeric_limits { + public: + /// Supports signed values. + static HALF_CONSTEXPR_CONST bool is_signed = true; + + /// Is not exact. + static HALF_CONSTEXPR_CONST bool is_exact = false; + + /// Doesn't provide modulo arithmetic. + static HALF_CONSTEXPR_CONST bool is_modulo = false; + + /// IEEE conformant. + static HALF_CONSTEXPR_CONST bool is_iec559 = true; + + /// Supports infinity. + static HALF_CONSTEXPR_CONST bool has_infinity = true; + + /// Supports quiet NaNs. + static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true; + + /// Supports subnormal values. + static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present; + + /// Rounding mode. + /// Due to the mix of internal single-precision computations (using the + /// rounding mode of the underlying single-precision implementation) with the + /// rounding mode of the single-to-half conversions, the actual rounding mode + /// might be `std::round_indeterminate` if the default half-precision rounding + /// mode doesn't match the single-precision rounding mode. + static HALF_CONSTEXPR_CONST float_round_style round_style = + (std::numeric_limits::round_style == half_float::half::round_style) + ? half_float::half::round_style + : round_indeterminate; + + /// Significant digits. + static HALF_CONSTEXPR_CONST int digits = 11; + + /// Significant decimal digits. + static HALF_CONSTEXPR_CONST int digits10 = 3; + + /// Required decimal digits to represent all possible values. + static HALF_CONSTEXPR_CONST int max_digits10 = 5; + + /// Number base. + static HALF_CONSTEXPR_CONST int radix = 2; + + /// One more than smallest exponent. + static HALF_CONSTEXPR_CONST int min_exponent = -13; + + /// Smallest normalized representable power of 10. + static HALF_CONSTEXPR_CONST int min_exponent10 = -4; + + /// One more than largest exponent + static HALF_CONSTEXPR_CONST int max_exponent = 16; + + /// Largest finitely representable power of 10. + static HALF_CONSTEXPR_CONST int max_exponent10 = 4; + + /// Smallest positive normal value. + static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x0400); + } + + /// Smallest finite value. + static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0xFBFF); + } + + /// Largest finite value. + static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x7BFF); + } + + /// Difference between one and next representable value. + static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x1400); + } + + /// Maximum rounding error. + static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW { + return half_float::half( + half_float::detail::binary, + (round_style == std::round_to_nearest) ? 0x3800 : 0x3C00); + } + + /// Positive infinity. + static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x7C00); + } + + /// Quiet NaN. + static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x7FFF); + } + + /// Signalling NaN. + static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x7DFF); + } + + /// Smallest positive subnormal value. + static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { + return half_float::half(half_float::detail::binary, 0x0001); + } +}; +#if HALF_ENABLE_CPP11_HASH +/// Hash function for half-precision floats. +/// This is only defined if C++11 `std::hash` is supported and enabled. +template <> +struct hash //: unary_function +{ + /// Type of function argument. + typedef half_float::half argument_type; + + /// Function return type. + typedef size_t result_type; + + /// Compute hash function. + /// \param arg half to hash + /// \return hash value + result_type operator()(argument_type arg) const { + return hash()(static_cast(arg.data_) & + -(arg.data_ != 0x8000)); + } +}; +#endif +} // namespace std #undef HALF_CONSTEXPR #undef HALF_CONSTEXPR_CONST #undef HALF_NOEXCEPT #undef HALF_NOTHROW #ifdef HALF_POP_WARNINGS - #pragma warning(pop) - #undef HALF_POP_WARNINGS +#pragma warning(pop) +#undef HALF_POP_WARNINGS #endif #endif diff --git a/src/cuda-sim/instructions.cc b/src/cuda-sim/instructions.cc index 014e588..0d56558 100644 --- a/src/cuda-sim/instructions.cc +++ b/src/cuda-sim/instructions.cc @@ -8,51 +8,51 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +#include "instructions.h" #include "half.h" #include "half.hpp" -#include "instructions.h" -#include "ptx_ir.h" #include "opcodes.h" +#include "ptx_ir.h" #include "ptx_sim.h" -typedef void * yyscan_t; +typedef void *yyscan_t; class ptx_recognizer; -#include "ptx.tab.h" -#include +#include +#include #include +#include +#include +#include #include -#include -#include "cuda-math.h" +#include +#include +#include #include "../abstract_hardware_model.h" -#include "ptx_loader.h" -#include "cuda_device_printf.h" #include "../gpgpu-sim/gpu-sim.h" #include "../gpgpu-sim/shader.h" -#include -#include -#include -#include -#include -#include -#include +#include "cuda-math.h" +#include "cuda_device_printf.h" +#include "ptx.tab.h" +#include "ptx_loader.h" -//Jin: include device runtime for CDP +// Jin: include device runtime for CDP #include "cuda_device_runtime.h" #include @@ -60,5629 +60,6420 @@ class ptx_recognizer; using half_float::half; - - const char *g_opcode_string[NUM_OPCODES] = { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) STR, +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) STR, +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) STR, #include "opcodes.def" #undef OP_DEF #undef OP_W_DEF }; -//Using profiled information::check the TensorCoreMatrixArrangement.xls for details -unsigned thread_group_offset(int thread,unsigned wmma_type,unsigned wmma_layout,unsigned type,int stride){ - - unsigned offset; - unsigned load_a_row[8]={0,128,0,128,64,192,64,192}; - unsigned load_a_col[8]={0,8,0,8,4,12,4,12}; - unsigned load_b_row[8]={0,8,0,8,4,12,4,12}; - unsigned load_b_col[8]={0,128,0,128,64,192,64,192}; - unsigned load_c_float_row[8]={0,128,8,136,64,192,72,200}; - unsigned load_c_float_col[8]={0,8,128,136,4,12,132,140}; - unsigned load_c_half_row[8]={0,128,8,136,64,192,72,200}; - unsigned load_c_half_col[8]={0,8,128,136,4,12,132,140}; - unsigned thread_group = thread/4; - unsigned in_tg_index = thread%4; - - switch(wmma_type){ - case LOAD_A: - if(wmma_layout==ROW) - offset=load_a_row[thread_group]+16*in_tg_index; - else - offset=load_a_col[thread_group]+16*in_tg_index; - break; - - - case LOAD_B: - if(wmma_layout==ROW) - offset=load_b_row[thread_group]+16*in_tg_index; - else - offset=load_b_col[thread_group]+16*in_tg_index; - break; - - case LOAD_C: - case STORE_D: - if(type==F16_TYPE){ - if(wmma_layout==ROW) - offset=load_c_half_row[thread_group]+16*in_tg_index; - else - offset=load_c_half_col[thread_group]+in_tg_index; - } - else{ - if(wmma_layout==ROW) - offset=load_c_float_row[thread_group]; - else - offset=load_c_float_col[thread_group]; - - switch(in_tg_index){ - case 0: - break; - case 1: - if(wmma_layout==ROW) - offset+=16; - else - offset+=1; - break; - case 2: - if(wmma_layout==ROW) - offset+=2; - else - offset+=32; - break; - case 3: - if(wmma_layout==ROW) - offset+=18; - else - offset+=33; - break; - default: - abort(); - } - } - break; - - default: - abort(); - - } - offset = (offset/16)*stride+offset%16; - return offset; -} - -int acc_float_offset(int index,int wmma_layout,int stride){ - - int c_row_offset[]={0,1,32,33,4,5,36,37}; - int c_col_offset[]={0,16,2,18,64,80,66,82}; - int offset; - - - if(wmma_layout==ROW) - offset=c_row_offset[index]; - else if(wmma_layout==COL) - offset=c_col_offset[index]; - else{ - printf("wrong layout"); - abort(); - } - offset = (offset/16)*stride+offset%16; - return offset; -} - -void inst_not_implemented( const ptx_instruction * pI ) ; -ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread); - -void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ); - -void ptx_thread_info::set_reg( const symbol *reg, const ptx_reg_t &value ) -{ - assert( reg != NULL ); - if( reg->name() == "_" ) return; - assert( !m_regs.empty() ); - assert( reg->uid() > 0 ); - m_regs.back()[ reg ] = value; - if (m_enable_debug_trace ) - m_debug_trace_regs_modified.back()[ reg ] = value; - m_last_set_operand_value = value; -} - -void ptx_thread_info::print_reg_thread(char * fname) -{ - - FILE *fp= fopen(fname,"w"); - assert(fp!=NULL); +// Using profiled information::check the TensorCoreMatrixArrangement.xls for +// details +unsigned thread_group_offset(int thread, unsigned wmma_type, + unsigned wmma_layout, unsigned type, int stride) { + unsigned offset; + unsigned load_a_row[8] = {0, 128, 0, 128, 64, 192, 64, 192}; + unsigned load_a_col[8] = {0, 8, 0, 8, 4, 12, 4, 12}; + unsigned load_b_row[8] = {0, 8, 0, 8, 4, 12, 4, 12}; + unsigned load_b_col[8] = {0, 128, 0, 128, 64, 192, 64, 192}; + unsigned load_c_float_row[8] = {0, 128, 8, 136, 64, 192, 72, 200}; + unsigned load_c_float_col[8] = {0, 8, 128, 136, 4, 12, 132, 140}; + unsigned load_c_half_row[8] = {0, 128, 8, 136, 64, 192, 72, 200}; + unsigned load_c_half_col[8] = {0, 8, 128, 136, 4, 12, 132, 140}; + unsigned thread_group = thread / 4; + unsigned in_tg_index = thread % 4; + + switch (wmma_type) { + case LOAD_A: + if (wmma_layout == ROW) + offset = load_a_row[thread_group] + 16 * in_tg_index; + else + offset = load_a_col[thread_group] + 16 * in_tg_index; + break; - int size = m_regs.size(); - - if(size>0) - { - reg_map_t reg = m_regs.back(); - - reg_map_t::const_iterator it; - for (it = reg.begin(); it != reg.end(); ++it) - { - const std::string &name = it->first->name(); - const std::string &dec= it->first->decl_location(); - unsigned size = it->first->get_size_in_bytes(); - fprintf(fp,"%s %llu %s %d\n", name.c_str(), it->second, dec.c_str(), size); - + case LOAD_B: + if (wmma_layout == ROW) + offset = load_b_row[thread_group] + 16 * in_tg_index; + else + offset = load_b_col[thread_group] + 16 * in_tg_index; + break; + + case LOAD_C: + case STORE_D: + if (type == F16_TYPE) { + if (wmma_layout == ROW) + offset = load_c_half_row[thread_group] + 16 * in_tg_index; + else + offset = load_c_half_col[thread_group] + in_tg_index; + } else { + if (wmma_layout == ROW) + offset = load_c_float_row[thread_group]; + else + offset = load_c_float_col[thread_group]; + + switch (in_tg_index) { + case 0: + break; + case 1: + if (wmma_layout == ROW) + offset += 16; + else + offset += 1; + break; + case 2: + if (wmma_layout == ROW) + offset += 2; + else + offset += 32; + break; + case 3: + if (wmma_layout == ROW) + offset += 18; + else + offset += 33; + break; + default: + abort(); } - //m_regs.pop_back(); + } + break; + + default: + abort(); + } + offset = (offset / 16) * stride + offset % 16; + return offset; +} + +int acc_float_offset(int index, int wmma_layout, int stride) { + int c_row_offset[] = {0, 1, 32, 33, 4, 5, 36, 37}; + int c_col_offset[] = {0, 16, 2, 18, 64, 80, 66, 82}; + int offset; + + if (wmma_layout == ROW) + offset = c_row_offset[index]; + else if (wmma_layout == COL) + offset = c_col_offset[index]; + else { + printf("wrong layout"); + abort(); + } + offset = (offset / 16) * stride + offset % 16; + return offset; +} + +void inst_not_implemented(const ptx_instruction *pI); +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, + operand_info dstInfo, unsigned type, + ptx_thread_info *thread); + +void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst); + +void ptx_thread_info::set_reg(const symbol *reg, const ptx_reg_t &value) { + assert(reg != NULL); + if (reg->name() == "_") return; + assert(!m_regs.empty()); + assert(reg->uid() > 0); + m_regs.back()[reg] = value; + if (m_enable_debug_trace) m_debug_trace_regs_modified.back()[reg] = value; + m_last_set_operand_value = value; +} + +void ptx_thread_info::print_reg_thread(char *fname) { + FILE *fp = fopen(fname, "w"); + assert(fp != NULL); + + int size = m_regs.size(); + + if (size > 0) { + reg_map_t reg = m_regs.back(); + + reg_map_t::const_iterator it; + for (it = reg.begin(); it != reg.end(); ++it) { + const std::string &name = it->first->name(); + const std::string &dec = it->first->decl_location(); + unsigned size = it->first->get_size_in_bytes(); + fprintf(fp, "%s %llu %s %d\n", name.c_str(), it->second, dec.c_str(), + size); + } + // m_regs.pop_back(); } fclose(fp); +} +void ptx_thread_info::resume_reg_thread(char *fname, symbol_table *symtab) { + FILE *fp2 = fopen(fname, "r"); + assert(fp2 != NULL); + // m_regs.push_back( reg_map_t() ); + char line[200]; + while (fgets(line, sizeof line, fp2) != NULL) { + symbol *reg; + char *pch; + pch = strtok(line, " "); + char *name = pch; + reg = symtab->lookup(name); + ptx_reg_t data; + pch = strtok(NULL, " "); + data = atoi(pch); + pch = strtok(NULL, " "); + pch = strtok(NULL, " "); + m_regs.back()[reg] = data; } + fclose(fp2); +} -void ptx_thread_info::resume_reg_thread(char * fname, symbol_table * symtab) -{ - - - FILE * fp2 = fopen(fname, "r"); - assert(fp2!=NULL); - //m_regs.push_back( reg_map_t() ); - char line [ 200 ]; - while ( fgets ( line, sizeof line, fp2 ) != NULL ) - { - symbol *reg; - char * pch; - pch = strtok (line," "); - char * name =pch; - reg= symtab->lookup(name); - ptx_reg_t data; - pch = strtok (NULL," "); - data = atoi(pch); - pch = strtok (NULL," "); - pch = strtok (NULL," "); - m_regs.back()[reg] = data; - } - fclose ( fp2 ); -} - - -ptx_reg_t ptx_thread_info::get_reg( const symbol *reg ) -{ - static bool unfound_register_warned = false; - assert( reg != NULL ); - assert( !m_regs.empty() ); - reg_map_t::iterator regs_iter = m_regs.back().find(reg); - if (regs_iter == m_regs.back().end()) { - assert( reg->type()->get_key().is_reg() ); - const std::string &name = reg->name(); - unsigned call_uid = m_callstack.back().m_call_uid; - ptx_reg_t uninit_reg; - uninit_reg.u32 = 0x0; - set_reg(reg, uninit_reg); // give it a value since we are going to warn the user anyway - std::string file_loc = get_location(); - if( !unfound_register_warned ) { - printf("GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' (cuid:%u). Setting to 0X00000000. This is okay if you are simulating the native ISA" - "\n", - file_loc.c_str(), name.c_str(), call_uid ); - unfound_register_warned = true; - } - regs_iter = m_regs.back().find(reg); - } - if (m_enable_debug_trace ) - m_debug_trace_regs_read.back()[ reg ] = regs_iter->second; - return regs_iter->second; -} - -ptx_reg_t ptx_thread_info::get_operand_value( const operand_info &op, operand_info dstInfo, unsigned opType, ptx_thread_info *thread, int derefFlag ) -{ - ptx_reg_t result, tmp; - - - if(op.get_double_operand_type() == 0) { - if(((opType != BB128_TYPE) && (opType != BB64_TYPE) && (opType != FF64_TYPE)) || (op.get_addr_space() != undefined_space)) { - if ( op.is_reg() ) { - result = get_reg( op.get_symbol() ); - } else if ( op.is_builtin()) { - result.u32 = get_builtin( op.get_int(), op.get_addr_offset() ); - } else if(op.is_immediate_address()){ - result.u64 = op.get_addr_offset(); - } else if ( op.is_memory_operand() ) { - // a few options here... - const symbol *sym = op.get_symbol(); - const type_info *type = sym->type(); - const type_info_key &info = type->get_key(); - - if ( info.is_reg() ) { - const symbol *name = op.get_symbol(); - result.u64 = get_reg(name).u64 + op.get_addr_offset(); - } else if ( info.is_param_kernel() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( info.is_param_local() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( info.is_global() ) { - assert( op.get_addr_offset() == 0 ); - result.u64 = sym->get_address(); - } else if ( info.is_local() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( info.is_const() ) { - result.u64 = sym->get_address() + op.get_addr_offset(); - } else if ( op.is_shared() ) { - result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); - } else if ( op.is_sstarr() ) { - result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); - } else { - const char *name = op.name().c_str(); - printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory operand type for %s\n", name ); - abort(); - } +ptx_reg_t ptx_thread_info::get_reg(const symbol *reg) { + static bool unfound_register_warned = false; + assert(reg != NULL); + assert(!m_regs.empty()); + reg_map_t::iterator regs_iter = m_regs.back().find(reg); + if (regs_iter == m_regs.back().end()) { + assert(reg->type()->get_key().is_reg()); + const std::string &name = reg->name(); + unsigned call_uid = m_callstack.back().m_call_uid; + ptx_reg_t uninit_reg; + uninit_reg.u32 = 0x0; + set_reg(reg, uninit_reg); // give it a value since we are going to warn the + // user anyway + std::string file_loc = get_location(); + if (!unfound_register_warned) { + printf( + "GPGPU-Sim PTX: WARNING (%s) ** reading undefined register \'%s\' " + "(cuid:%u). Setting to 0X00000000. This is okay if you are " + "simulating the native ISA" + "\n", + file_loc.c_str(), name.c_str(), call_uid); + unfound_register_warned = true; + } + regs_iter = m_regs.back().find(reg); + } + if (m_enable_debug_trace) + m_debug_trace_regs_read.back()[reg] = regs_iter->second; + return regs_iter->second; +} - } else if ( op.is_literal() ) { - result = op.get_literal_value(); - } else if ( op.is_label() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_shared() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_sstarr() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_const() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_global() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_local() ) { - result.u64 = op.get_symbol()->get_address(); - } else if ( op.is_function_address() ) { - result.u64 = (size_t)op.get_symbol()->get_pc(); - } else if ( op.is_param_kernel()) { - result.u64 = op.get_symbol()->get_address(); - }else { - const char *name = op.name().c_str(); - const symbol *sym2 = op.get_symbol(); - const type_info *type2 = sym2->type(); - const type_info_key &info2 = type2->get_key(); - if ( info2.is_param_kernel() ) { - result.u64 = sym2->get_address()+ op.get_addr_offset(); - } - else{ - printf("GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand type for %s\n", name ); - assert(0); - } - } - - if(op.get_operand_lohi() == 1) - result.u64 = result.u64 & 0xFFFF; - else if(op.get_operand_lohi() == 2) - result.u64 = (result.u64>>16) & 0xFFFF; - } else if (opType == BB128_TYPE) { - // b128 - result.u128.lowest = get_reg( op.vec_symbol(0) ).u32; - result.u128.low = get_reg( op.vec_symbol(1) ).u32; - result.u128.high = get_reg( op.vec_symbol(2) ).u32; - result.u128.highest = get_reg( op.vec_symbol(3) ).u32; +ptx_reg_t ptx_thread_info::get_operand_value(const operand_info &op, + operand_info dstInfo, + unsigned opType, + ptx_thread_info *thread, + int derefFlag) { + ptx_reg_t result, tmp; + + if (op.get_double_operand_type() == 0) { + if (((opType != BB128_TYPE) && (opType != BB64_TYPE) && + (opType != FF64_TYPE)) || + (op.get_addr_space() != undefined_space)) { + if (op.is_reg()) { + result = get_reg(op.get_symbol()); + } else if (op.is_builtin()) { + result.u32 = get_builtin(op.get_int(), op.get_addr_offset()); + } else if (op.is_immediate_address()) { + result.u64 = op.get_addr_offset(); + } else if (op.is_memory_operand()) { + // a few options here... + const symbol *sym = op.get_symbol(); + const type_info *type = sym->type(); + const type_info_key &info = type->get_key(); + + if (info.is_reg()) { + const symbol *name = op.get_symbol(); + result.u64 = get_reg(name).u64 + op.get_addr_offset(); + } else if (info.is_param_kernel()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (info.is_param_local()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (info.is_global()) { + assert(op.get_addr_offset() == 0); + result.u64 = sym->get_address(); + } else if (info.is_local()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (info.is_const()) { + result.u64 = sym->get_address() + op.get_addr_offset(); + } else if (op.is_shared()) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else if (op.is_sstarr()) { + result.u64 = op.get_symbol()->get_address() + op.get_addr_offset(); + } else { + const char *name = op.name().c_str(); + printf( + "GPGPU-Sim PTX: ERROR ** get_operand_value : unknown memory " + "operand type for %s\n", + name); + abort(); + } + + } else if (op.is_literal()) { + result = op.get_literal_value(); + } else if (op.is_label()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_shared()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_sstarr()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_const()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_global()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_local()) { + result.u64 = op.get_symbol()->get_address(); + } else if (op.is_function_address()) { + result.u64 = (size_t)op.get_symbol()->get_pc(); + } else if (op.is_param_kernel()) { + result.u64 = op.get_symbol()->get_address(); } else { - // bb64 or ff64 - result.bits.ls = get_reg( op.vec_symbol(0) ).u32; - result.bits.ms = get_reg( op.vec_symbol(1) ).u32; + const char *name = op.name().c_str(); + const symbol *sym2 = op.get_symbol(); + const type_info *type2 = sym2->type(); + const type_info_key &info2 = type2->get_key(); + if (info2.is_param_kernel()) { + result.u64 = sym2->get_address() + op.get_addr_offset(); + } else { + printf( + "GPGPU-Sim PTX: ERROR ** get_operand_value : unknown operand " + "type for %s\n", + name); + assert(0); + } } - } else if (op.get_double_operand_type() == 1) { - ptx_reg_t firstHalf, secondHalf; - firstHalf.u64 = get_reg( op.vec_symbol(0) ).u64; - secondHalf.u64 = get_reg( op.vec_symbol(1) ).u64; - if(op.get_operand_lohi() == 1) - secondHalf.u64 = secondHalf.u64 & 0xFFFF; - else if(op.get_operand_lohi() == 2) - secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; - result.u64 = firstHalf.u64 + secondHalf.u64; - } else if (op.get_double_operand_type() == 2) { - // s[reg1 += reg2] - // reg1 is incremented after value is returned: the value returned is s[reg1] - ptx_reg_t firstHalf, secondHalf; - firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; - secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; - if(op.get_operand_lohi() == 1) - secondHalf.u64 = secondHalf.u64 & 0xFFFF; - else if(op.get_operand_lohi() == 2) - secondHalf.u64 = (secondHalf.u64>>16) & 0xFFFF; - result.u64 = firstHalf.u64; - firstHalf.u64 = firstHalf.u64 + secondHalf.u64; - set_reg(op.vec_symbol(0),firstHalf); - } else if (op.get_double_operand_type() == 3) { - // s[reg += immediate] - // reg is incremented after value is returned: the value returned is s[reg] - ptx_reg_t firstHalf; - firstHalf.u64 = get_reg(op.get_symbol()).u64; - result.u64 = firstHalf.u64; - firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); - set_reg(op.get_symbol(),firstHalf); - } - - ptx_reg_t finalResult; - memory_space *mem = NULL; - size_t size=0; - int t=0; - finalResult.u64=0; - - //complete other cases for reading from memory, such as reading from other const memory - if((op.get_addr_space() == global_space)&&(derefFlag)) { - // global memory - g[4], g[$r0] - mem = thread->get_global_memory(); - type_info_key::type_decode(opType,size,t); - mem->read(result.u32,size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = global_space; - - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else if((op.get_addr_space() == shared_space)&&(derefFlag)) { - // shared memory - s[4], s[$r0] - mem = thread->m_shared_mem; - type_info_key::type_decode(opType,size,t); - mem->read(result.u32,size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = shared_space; - - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else if((op.get_addr_space() == const_space)&&(derefFlag)) { - // const memory - ce0c1[4], ce0c1[$r0] - mem = thread->get_global_memory(); - type_info_key::type_decode(opType,size,t); - mem->read((result.u32 + op.get_const_mem_offset()),size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = const_space; - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else if((op.get_addr_space() == local_space)&&(derefFlag)) { - // local memory - l0[4], l0[$r0] - mem = thread->m_local_mem; - type_info_key::type_decode(opType,size,t); - mem->read(result.u32,size/8,&finalResult.u128); - thread->m_last_effective_address = result.u32; - thread->m_last_memory_space = local_space; - if( opType == S16_TYPE || opType == S32_TYPE ) - sign_extend(finalResult,size,dstInfo); - } else { - finalResult = result; - } - - if((op.get_operand_neg() == true)&&(derefFlag)) { - switch( opType ) { + + if (op.get_operand_lohi() == 1) + result.u64 = result.u64 & 0xFFFF; + else if (op.get_operand_lohi() == 2) + result.u64 = (result.u64 >> 16) & 0xFFFF; + } else if (opType == BB128_TYPE) { + // b128 + result.u128.lowest = get_reg(op.vec_symbol(0)).u32; + result.u128.low = get_reg(op.vec_symbol(1)).u32; + result.u128.high = get_reg(op.vec_symbol(2)).u32; + result.u128.highest = get_reg(op.vec_symbol(3)).u32; + } else { + // bb64 or ff64 + result.bits.ls = get_reg(op.vec_symbol(0)).u32; + result.bits.ms = get_reg(op.vec_symbol(1)).u32; + } + } else if (op.get_double_operand_type() == 1) { + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; + secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; + if (op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if (op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64 >> 16) & 0xFFFF; + result.u64 = firstHalf.u64 + secondHalf.u64; + } else if (op.get_double_operand_type() == 2) { + // s[reg1 += reg2] + // reg1 is incremented after value is returned: the value returned is + // s[reg1] + ptx_reg_t firstHalf, secondHalf; + firstHalf.u64 = get_reg(op.vec_symbol(0)).u64; + secondHalf.u64 = get_reg(op.vec_symbol(1)).u64; + if (op.get_operand_lohi() == 1) + secondHalf.u64 = secondHalf.u64 & 0xFFFF; + else if (op.get_operand_lohi() == 2) + secondHalf.u64 = (secondHalf.u64 >> 16) & 0xFFFF; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + secondHalf.u64; + set_reg(op.vec_symbol(0), firstHalf); + } else if (op.get_double_operand_type() == 3) { + // s[reg += immediate] + // reg is incremented after value is returned: the value returned is s[reg] + ptx_reg_t firstHalf; + firstHalf.u64 = get_reg(op.get_symbol()).u64; + result.u64 = firstHalf.u64; + firstHalf.u64 = firstHalf.u64 + op.get_addr_offset(); + set_reg(op.get_symbol(), firstHalf); + } + + ptx_reg_t finalResult; + memory_space *mem = NULL; + size_t size = 0; + int t = 0; + finalResult.u64 = 0; + + // complete other cases for reading from memory, such as reading from other + // const memory + if ((op.get_addr_space() == global_space) && (derefFlag)) { + // global memory - g[4], g[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType, size, t); + mem->read(result.u32, size / 8, &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = global_space; + + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else if ((op.get_addr_space() == shared_space) && (derefFlag)) { + // shared memory - s[4], s[$r0] + mem = thread->m_shared_mem; + type_info_key::type_decode(opType, size, t); + mem->read(result.u32, size / 8, &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = shared_space; + + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else if ((op.get_addr_space() == const_space) && (derefFlag)) { + // const memory - ce0c1[4], ce0c1[$r0] + mem = thread->get_global_memory(); + type_info_key::type_decode(opType, size, t); + mem->read((result.u32 + op.get_const_mem_offset()), size / 8, + &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = const_space; + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else if ((op.get_addr_space() == local_space) && (derefFlag)) { + // local memory - l0[4], l0[$r0] + mem = thread->m_local_mem; + type_info_key::type_decode(opType, size, t); + mem->read(result.u32, size / 8, &finalResult.u128); + thread->m_last_effective_address = result.u32; + thread->m_last_memory_space = local_space; + if (opType == S16_TYPE || opType == S32_TYPE) + sign_extend(finalResult, size, dstInfo); + } else { + finalResult = result; + } + + if ((op.get_operand_neg() == true) && (derefFlag)) { + switch (opType) { // Default to f32 for now, need to add support for others case S8_TYPE: case U8_TYPE: case B8_TYPE: - finalResult.s8 = -finalResult.s8; - break; + finalResult.s8 = -finalResult.s8; + break; case S16_TYPE: case U16_TYPE: case B16_TYPE: - finalResult.s16 = -finalResult.s16; - break; + finalResult.s16 = -finalResult.s16; + break; case S32_TYPE: case U32_TYPE: case B32_TYPE: - finalResult.s32 = -finalResult.s32; - break; + finalResult.s32 = -finalResult.s32; + break; case S64_TYPE: case U64_TYPE: case B64_TYPE: - finalResult.s64 = -finalResult.s64; - break; + finalResult.s64 = -finalResult.s64; + break; case F16_TYPE: - finalResult.f16 = -finalResult.f16; - break; + finalResult.f16 = -finalResult.f16; + break; case F32_TYPE: - finalResult.f32 = -finalResult.f32; - break; + finalResult.f32 = -finalResult.f32; + break; case F64_TYPE: case FF64_TYPE: - finalResult.f64 = -finalResult.f64; - break; + finalResult.f64 = -finalResult.f64; + break; default: - assert(0); - } + assert(0); + } + } + + return finalResult; +} - } - - return finalResult; - -} - -unsigned get_operand_nbits( const operand_info &op ) -{ - if ( op.is_reg() ) { - const symbol *sym = op.get_symbol(); - const type_info *typ = sym->type(); - type_info_key t = typ->get_key(); - switch( t.scalar_type() ) { - case PRED_TYPE: - return 1; - case B8_TYPE: case S8_TYPE: case U8_TYPE: - return 8; - case S16_TYPE: case U16_TYPE: case F16_TYPE: case B16_TYPE: - return 16; - case S32_TYPE: case U32_TYPE: case F32_TYPE: case B32_TYPE: - return 32; - case S64_TYPE: case U64_TYPE: case F64_TYPE: case B64_TYPE: - return 64; +unsigned get_operand_nbits(const operand_info &op) { + if (op.is_reg()) { + const symbol *sym = op.get_symbol(); + const type_info *typ = sym->type(); + type_info_key t = typ->get_key(); + switch (t.scalar_type()) { + case PRED_TYPE: + return 1; + case B8_TYPE: + case S8_TYPE: + case U8_TYPE: + return 8; + case S16_TYPE: + case U16_TYPE: + case F16_TYPE: + case B16_TYPE: + return 16; + case S32_TYPE: + case U32_TYPE: + case F32_TYPE: + case B32_TYPE: + return 32; + case S64_TYPE: + case U64_TYPE: + case F64_TYPE: + case B64_TYPE: + return 64; default: - printf("ERROR: unknown register type\n"); - fflush(stdout); - abort(); - } - } else { - printf("ERROR: Need to implement get_operand_nbits() for currently unsupported operand_info type\n"); - fflush(stdout); - abort(); - } + printf("ERROR: unknown register type\n"); + fflush(stdout); + abort(); + } + } else { + printf( + "ERROR: Need to implement get_operand_nbits() for currently " + "unsupported operand_info type\n"); + fflush(stdout); + abort(); + } - return 0; + return 0; } -void ptx_thread_info::get_vector_operand_values( const operand_info &op, ptx_reg_t* ptx_regs, unsigned num_elements ) -{ - assert( op.is_vector() ); - assert( num_elements <= 8 ); +void ptx_thread_info::get_vector_operand_values(const operand_info &op, + ptx_reg_t *ptx_regs, + unsigned num_elements) { + assert(op.is_vector()); + assert(num_elements <= 8); + + for (int idx = num_elements - 1; idx >= 0; --idx) { + const symbol *sym = NULL; + sym = op.vec_symbol(idx); + if (strcmp(sym->name().c_str(), "_") != 0) { + reg_map_t::iterator reg_iter = m_regs.back().find(sym); + assert(reg_iter != m_regs.back().end()); + ptx_regs[idx] = reg_iter->second; + } + } +} - for (int idx = num_elements - 1; idx >= 0; --idx) { - const symbol *sym = NULL; - sym = op.vec_symbol(idx); - if( strcmp(sym->name().c_str(),"_") != 0) { - reg_map_t::iterator reg_iter = m_regs.back().find(sym); - assert( reg_iter != m_regs.back().end() ); - ptx_regs[idx] = reg_iter->second; - } - } +void sign_extend(ptx_reg_t &data, unsigned src_size, const operand_info &dst) { + if (!dst.is_reg()) return; + unsigned dst_size = get_operand_nbits(dst); + if (src_size >= dst_size) return; + // src_size < dst_size + unsigned long long mask = 1; + mask <<= (src_size - 1); + if ((mask & data.u64) == 0) { + // no need to sign extend + return; + } + // need to sign extend + mask = 1; + mask <<= dst_size - src_size; + mask -= 1; + mask <<= src_size; + data.u64 |= mask; } -void sign_extend( ptx_reg_t &data, unsigned src_size, const operand_info &dst ) -{ - if( !dst.is_reg() ) - return; - unsigned dst_size = get_operand_nbits( dst ); - if( src_size >= dst_size ) - return; - // src_size < dst_size - unsigned long long mask = 1; - mask <<= (src_size-1); - if( (mask & data.u64) == 0 ) { - // no need to sign extend - return; - } - // need to sign extend - mask = 1; - mask <<= dst_size-src_size; - mask -= 1; - mask <<= src_size; - data.u64 |= mask; +void ptx_thread_info::set_operand_value(const operand_info &dst, + const ptx_reg_t &data, unsigned type, + ptx_thread_info *thread, + const ptx_instruction *pI, int overflow, + int carry) { + thread->set_operand_value(dst, data, type, thread, pI); + + if (dst.get_double_operand_type() == -2) { + ptx_reg_t predValue; + + const symbol *sym = dst.vec_symbol(0); + predValue.u64 = (m_regs.back()[sym].u64) & ~(0x0C); + predValue.u64 |= ((overflow & 0x01) << 3); + predValue.u64 |= ((carry & 0x01) << 2); + + set_reg(sym, predValue); + } else if (dst.get_double_operand_type() == 0) { + // intentionally do nothing + } else { + printf("Unexpected double destination\n"); + assert(0); + } } -void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI, int overflow, int carry ) -{ - thread->set_operand_value( dst, data, type, thread, pI ); +void ptx_thread_info::set_operand_value(const operand_info &dst, + const ptx_reg_t &data, unsigned type, + ptx_thread_info *thread, + const ptx_instruction *pI) { + ptx_reg_t dstData; + memory_space *mem = NULL; + size_t size; + int t; + + type_info_key::type_decode(type, size, t); + + /*complete this section for other cases*/ + if (dst.get_addr_space() == undefined_space) { + ptx_reg_t setValue; + setValue.u64 = data.u64; + + // Double destination in set instruction ($p0|$p1) - second is negation of + // first + if (dst.get_double_operand_type() == -1) { + ptx_reg_t setValue2; + const symbol *name1 = dst.vec_symbol(0); + const symbol *name2 = dst.vec_symbol(1); + + if ((type == F16_TYPE) || (type == F32_TYPE) || (type == F64_TYPE) || + (type == FF64_TYPE)) { + setValue2.f32 = (setValue.u64 == 0) ? 1.0f : 0.0f; + } else { + setValue2.u32 = (setValue.u64 == 0) ? 0xFFFFFFFF : 0; + } - if (dst.get_double_operand_type() == -2) - { - ptx_reg_t predValue; - - const symbol *sym = dst.vec_symbol(0); - predValue.u64 = (m_regs.back()[ sym ].u64) & ~(0x0C); - predValue.u64 |= ((overflow & 0x01)<<3); - predValue.u64 |= ((carry & 0x01)<<2); - - set_reg(sym,predValue); - } - else if (dst.get_double_operand_type() == 0) - { - //intentionally do nothing + set_reg(name1, setValue); + set_reg(name2, setValue2); } - else - { - printf("Unexpected double destination\n"); - assert(0); + + // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second + // register operand receives data, first predicate operand is set as + // $p0=($r4!=0) Also for Double destination in set instruction ($p0/$r1) + else if ((dst.get_double_operand_type() == -2) || + (dst.get_double_operand_type() == -3)) { + ptx_reg_t predValue; + const symbol *predName = dst.vec_symbol(0); + const symbol *regName = dst.vec_symbol(1); + predValue.u64 = 0; + + switch (type) { + case S8_TYPE: + if ((setValue.s8 & 0x7F) == 0) predValue.u64 |= 1; + break; + case S16_TYPE: + if ((setValue.s16 & 0x7FFF) == 0) predValue.u64 |= 1; + break; + case S32_TYPE: + if ((setValue.s32 & 0x7FFFFFFF) == 0) predValue.u64 |= 1; + break; + case S64_TYPE: + if ((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) predValue.u64 |= 1; + break; + case U8_TYPE: + case B8_TYPE: + if (setValue.u8 == 0) predValue.u64 |= 1; + break; + case U16_TYPE: + case B16_TYPE: + if (setValue.u16 == 0) predValue.u64 |= 1; + break; + case U32_TYPE: + case B32_TYPE: + if (setValue.u32 == 0) predValue.u64 |= 1; + break; + case U64_TYPE: + case B64_TYPE: + if (setValue.u64 == 0) predValue.u64 |= 1; + break; + case F16_TYPE: + if (setValue.f16 == 0) predValue.u64 |= 1; + break; + case F32_TYPE: + if (setValue.f32 == 0) predValue.u64 |= 1; + break; + case F64_TYPE: + case FF64_TYPE: + if (setValue.f64 == 0) predValue.u64 |= 1; + break; + default: + assert(0); + break; + } + + if ((type == S8_TYPE) || (type == S16_TYPE) || (type == S32_TYPE) || + (type == S64_TYPE) || (type == U8_TYPE) || (type == U16_TYPE) || + (type == U32_TYPE) || (type == U64_TYPE) || (type == B8_TYPE) || + (type == B16_TYPE) || (type == B32_TYPE) || (type == B64_TYPE)) { + if ((setValue.u32 & (1 << (size - 1))) != 0) predValue.u64 |= 1 << 1; + } + if (type == F32_TYPE) { + if (setValue.f32 < 0) predValue.u64 |= 1 << 1; + } + + if (dst.get_operand_lohi() == 1) { + setValue.u64 = + ((m_regs.back()[regName].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); + } else if (dst.get_operand_lohi() == 2) { + setValue.u64 = ((m_regs.back()[regName].u64) & (~(0xFFFF0000))) + + ((data.u64 << 16) & 0xFFFF0000); + } + + set_reg(predName, predValue); + set_reg(regName, setValue); + } else if (type == BB128_TYPE) { + // b128 stuff here. + ptx_reg_t setValue2, setValue3, setValue4; + setValue.u64 = 0; + setValue2.u64 = 0; + setValue3.u64 = 0; + setValue4.u64 = 0; + setValue.u32 = data.u128.lowest; + setValue2.u32 = data.u128.low; + setValue3.u32 = data.u128.high; + setValue4.u32 = data.u128.highest; + + const symbol *name1, *name2, *name3, *name4 = NULL; + + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); + name3 = dst.vec_symbol(2); + name4 = dst.vec_symbol(3); + + set_reg(name1, setValue); + set_reg(name2, setValue2); + set_reg(name3, setValue3); + set_reg(name4, setValue4); + } else if (type == BB64_TYPE || type == FF64_TYPE) { + // ptxplus version of storing 64 bit values to registers stores to two + // adjacent registers + ptx_reg_t setValue2; + setValue.u32 = 0; + setValue2.u32 = 0; + + setValue.u32 = data.bits.ls; + setValue2.u32 = data.bits.ms; + + const symbol *name1, *name2 = NULL; + + name1 = dst.vec_symbol(0); + name2 = dst.vec_symbol(1); + + set_reg(name1, setValue); + set_reg(name2, setValue2); + } else { + if (dst.get_operand_lohi() == 1) { + setValue.u64 = ((m_regs.back()[dst.get_symbol()].u64) & (~(0xFFFF))) + + (data.u64 & 0xFFFF); + } else if (dst.get_operand_lohi() == 2) { + setValue.u64 = + ((m_regs.back()[dst.get_symbol()].u64) & (~(0xFFFF0000))) + + ((data.u64 << 16) & 0xFFFF0000); + } + set_reg(dst.get_symbol(), setValue); } + } -} + // global memory - g[4], g[$r0] + else if (dst.get_addr_space() == global_space) { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->get_global_memory(); + type_info_key::type_decode(type, size, t); -void ptx_thread_info::set_operand_value( const operand_info &dst, const ptx_reg_t &data, unsigned type, ptx_thread_info *thread, const ptx_instruction *pI ) -{ - ptx_reg_t dstData; - memory_space *mem = NULL; - size_t size; - int t; + mem->write(dstData.u32, size / 8, &data.u128, thread, pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = global_space; + } - type_info_key::type_decode(type,size,t); + // shared memory - s[4], s[$r0] + else if (dst.get_addr_space() == shared_space) { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_shared_mem; + type_info_key::type_decode(type, size, t); - /*complete this section for other cases*/ - if(dst.get_addr_space() == undefined_space) - { - ptx_reg_t setValue; - setValue.u64 = data.u64; + mem->write(dstData.u32, size / 8, &data.u128, thread, pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = shared_space; + } - // Double destination in set instruction ($p0|$p1) - second is negation of first - if (dst.get_double_operand_type() == -1) - { - ptx_reg_t setValue2; - const symbol *name1 = dst.vec_symbol(0); - const symbol *name2 = dst.vec_symbol(1); + // local memory - l0[4], l0[$r0] + else if (dst.get_addr_space() == local_space) { + dstData = thread->get_operand_value(dst, dst, type, thread, 0); + mem = thread->m_local_mem; + type_info_key::type_decode(type, size, t); - if ( (type==F16_TYPE)||(type==F32_TYPE)||(type==F64_TYPE)||(type==FF64_TYPE) ) { - setValue2.f32 = (setValue.u64==0)?1.0f:0.0f; - } else { - setValue2.u32 = (setValue.u64==0)?0xFFFFFFFF:0; - } + mem->write(dstData.u32, size / 8, &data.u128, thread, pI); + thread->m_last_effective_address = dstData.u32; + thread->m_last_memory_space = local_space; + } + + else { + printf("Destination stores to unknown location."); + assert(0); + } +} - set_reg(name1,setValue); - set_reg(name2,setValue2); +void ptx_thread_info::set_vector_operand_values(const operand_info &dst, + const ptx_reg_t &data1, + const ptx_reg_t &data2, + const ptx_reg_t &data3, + const ptx_reg_t &data4) { + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements > 0) { + set_reg(dst.vec_symbol(0), data1); + if (num_elements > 1) { + set_reg(dst.vec_symbol(1), data2); + if (num_elements > 2) { + set_reg(dst.vec_symbol(2), data3); + if (num_elements > 3) { + set_reg(dst.vec_symbol(3), data4); + } } + } + } - // Double destination in cvt,shr,mul,etc. instruction ($p0|$r4) - second register operand receives data, first predicate operand - // is set as $p0=($r4!=0) - // Also for Double destination in set instruction ($p0/$r1) - else if ((dst.get_double_operand_type() == -2)||(dst.get_double_operand_type() == -3)) - { - ptx_reg_t predValue; - const symbol *predName = dst.vec_symbol(0); - const symbol *regName = dst.vec_symbol(1); - predValue.u64 = 0; - - switch ( type ) { - case S8_TYPE: - if((setValue.s8 & 0x7F) == 0) - predValue.u64 |= 1; - break; - case S16_TYPE: - if((setValue.s16 & 0x7FFF) == 0) - predValue.u64 |= 1; - break; - case S32_TYPE: - if((setValue.s32 & 0x7FFFFFFF) == 0) - predValue.u64 |= 1; - break; - case S64_TYPE: - if((setValue.s64 & 0x7FFFFFFFFFFFFFFF) == 0) - predValue.u64 |= 1; - break; - case U8_TYPE: - case B8_TYPE: - if(setValue.u8 == 0) - predValue.u64 |= 1; - break; - case U16_TYPE: - case B16_TYPE: - if(setValue.u16 == 0) - predValue.u64 |= 1; - break; - case U32_TYPE: - case B32_TYPE: - if(setValue.u32 == 0) - predValue.u64 |= 1; - break; - case U64_TYPE: - case B64_TYPE: - if(setValue.u64 == 0) - predValue.u64 |= 1; - break; - case F16_TYPE: - if(setValue.f16 == 0) - predValue.u64 |= 1; - break; - case F32_TYPE: - if(setValue.f32 == 0) - predValue.u64 |= 1; - break; - case F64_TYPE: - case FF64_TYPE: - if(setValue.f64 == 0) - predValue.u64 |= 1; - break; - default: assert(0); break; - } + m_last_set_operand_value = data1; +} +void ptx_thread_info::set_wmma_vector_operand_values( + const operand_info &dst, const ptx_reg_t &data1, const ptx_reg_t &data2, + const ptx_reg_t &data3, const ptx_reg_t &data4, const ptx_reg_t &data5, + const ptx_reg_t &data6, const ptx_reg_t &data7, const ptx_reg_t &data8) { + unsigned num_elements = dst.get_vect_nelem(); + if (num_elements == 8) { + set_reg(dst.vec_symbol(0), data1); + set_reg(dst.vec_symbol(1), data2); + set_reg(dst.vec_symbol(2), data3); + set_reg(dst.vec_symbol(3), data4); + set_reg(dst.vec_symbol(4), data5); + set_reg(dst.vec_symbol(5), data6); + set_reg(dst.vec_symbol(6), data7); + set_reg(dst.vec_symbol(7), data8); + } else { + printf("error:set_wmma_vector_operands"); + } + m_last_set_operand_value = data8; +} - if ( (type==S8_TYPE)||(type==S16_TYPE)||(type==S32_TYPE)||(type==S64_TYPE)|| - (type==U8_TYPE)||(type==U16_TYPE)||(type==U32_TYPE)||(type==U64_TYPE)|| - (type==B8_TYPE)||(type==B16_TYPE)||(type==B32_TYPE)||(type==B64_TYPE)) { - if((setValue.u32 & (1<<(size-1))) != 0) - predValue.u64 |= 1<<1; - } - if ( type==F32_TYPE ) { - if(setValue.f32 < 0) - predValue.u64 |= 1<<1; - } +#define my_abs(a) (((a) < 0) ? (-a) : (a)) - if(dst.get_operand_lohi() == 1) - { - setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); - } - else if(dst.get_operand_lohi() == 2) - { - setValue.u64 = ((m_regs.back()[ regName ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); - } +#define MY_MAX_I(a, b) (a > b) ? a : b +#define MY_MAX_F(a, b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b - set_reg(predName,predValue); - set_reg(regName,setValue); - } - else if (type == BB128_TYPE) - { - //b128 stuff here. - ptx_reg_t setValue2, setValue3, setValue4; - setValue.u64 = 0; - setValue2.u64 = 0; - setValue3.u64 = 0; - setValue4.u64 = 0; - setValue.u32 = data.u128.lowest; - setValue2.u32 = data.u128.low; - setValue3.u32 = data.u128.high; - setValue4.u32 = data.u128.highest; - - const symbol *name1, *name2, *name3, *name4 = NULL; - - name1 = dst.vec_symbol(0); - name2 = dst.vec_symbol(1); - name3 = dst.vec_symbol(2); - name4 = dst.vec_symbol(3); - - set_reg(name1,setValue); - set_reg(name2,setValue2); - set_reg(name3,setValue3); - set_reg(name4,setValue4); - } - else if (type == BB64_TYPE || type == FF64_TYPE) - { - //ptxplus version of storing 64 bit values to registers stores to two adjacent registers - ptx_reg_t setValue2; - setValue.u32 = 0; - setValue2.u32 = 0; +#define MY_MIN_I(a, b) (a < b) ? a : b +#define MY_MIN_F(a, b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b - setValue.u32 = data.bits.ls; - setValue2.u32 = data.bits.ms; +#define MY_INC_I(a, b) (a >= b) ? 0 : a + 1 +#define MY_DEC_I(a, b) ((a == 0) || (a > b)) ? b : a - 1 - const symbol *name1, *name2 = NULL; +#define MY_CAS_I(a, b, c) (a == b) ? c : a - name1 = dst.vec_symbol(0); - name2 = dst.vec_symbol(1); +#define MY_EXCH(a, b) b - set_reg(name1,setValue); - set_reg(name2,setValue2); - } - else - { - if(dst.get_operand_lohi() == 1) - { - setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF))) + (data.u64 & 0xFFFF); - } - else if(dst.get_operand_lohi() == 2) - { - setValue.u64 = ((m_regs.back()[ dst.get_symbol() ].u64) & (~(0xFFFF0000))) + ((data.u64<<16) & 0xFFFF0000); - } - set_reg(dst.get_symbol(),setValue); - } - } - - // global memory - g[4], g[$r0] - else if(dst.get_addr_space() == global_space) - { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->get_global_memory(); - type_info_key::type_decode(type,size,t); - - mem->write(dstData.u32,size/8,&data.u128,thread,pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = global_space; - } - - // shared memory - s[4], s[$r0] - else if(dst.get_addr_space() == shared_space) - { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->m_shared_mem; - type_info_key::type_decode(type,size,t); - - mem->write(dstData.u32,size/8,&data.u128,thread,pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = shared_space; - } - - // local memory - l0[4], l0[$r0] - else if(dst.get_addr_space() == local_space) - { - dstData = thread->get_operand_value(dst, dst, type, thread, 0); - mem = thread->m_local_mem; - type_info_key::type_decode(type,size,t); - - mem->write(dstData.u32,size/8,&data.u128,thread,pI); - thread->m_last_effective_address = dstData.u32; - thread->m_last_memory_space = local_space; - } - - else - { - printf("Destination stores to unknown location."); - assert(0); - } - - -} - -void ptx_thread_info::set_vector_operand_values( const operand_info &dst, - const ptx_reg_t &data1, - const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4 ) -{ - unsigned num_elements = dst.get_vect_nelem(); - if (num_elements > 0) { - set_reg(dst.vec_symbol(0), data1); - if (num_elements > 1) { - set_reg(dst.vec_symbol(1), data2); - if (num_elements > 2) { - set_reg(dst.vec_symbol(2), data3); - if (num_elements > 3) { - set_reg(dst.vec_symbol(3), data4); - } - } - } - } - - m_last_set_operand_value = data1; -} -void ptx_thread_info::set_wmma_vector_operand_values( const operand_info &dst, - const ptx_reg_t &data1, - const ptx_reg_t &data2, - const ptx_reg_t &data3, - const ptx_reg_t &data4, - const ptx_reg_t &data5, - const ptx_reg_t &data6, - const ptx_reg_t &data7, - const ptx_reg_t &data8 ) -{ - unsigned num_elements = dst.get_vect_nelem(); - if (num_elements == 8) { - set_reg(dst.vec_symbol(0), data1); - set_reg(dst.vec_symbol(1), data2); - set_reg(dst.vec_symbol(2), data3); - set_reg(dst.vec_symbol(3), data4); - set_reg(dst.vec_symbol(4), data5); - set_reg(dst.vec_symbol(5), data6); - set_reg(dst.vec_symbol(6), data7); - set_reg(dst.vec_symbol(7), data8); - } - else{ - printf("error:set_wmma_vector_operands"); - } - - m_last_set_operand_value = data8; -} - -#define my_abs(a) (((a)<0)?(-a):(a)) - -#define MY_MAX_I(a,b) (a > b) ? a : b -#define MY_MAX_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a > b) ? a : b - -#define MY_MIN_I(a,b) (a < b) ? a : b -#define MY_MIN_F(a,b) isNaN(a) ? b : isNaN(b) ? a : (a < b) ? a : b - -#define MY_INC_I(a,b) (a >= b) ? 0 : a+1 -#define MY_DEC_I(a,b) ((a == 0) || (a > b)) ? b : a-1 - -#define MY_CAS_I(a,b,c) (a == b) ? c : a - -#define MY_EXCH(a,b) b - -void abs_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case S16_TYPE: d.s16 = my_abs(a.s16); break; - case S32_TYPE: d.s32 = my_abs(a.s32); break; - case S64_TYPE: d.s64 = my_abs(a.s64); break; - case U16_TYPE: d.s16 = my_abs(a.u16); break; - case U32_TYPE: d.s32 = my_abs(a.u32); break; - case U64_TYPE: d.s64 = my_abs(a.u64); break; - case F32_TYPE: d.f32 = my_abs(a.f32); break; - case F64_TYPE: case FF64_TYPE: d.f64 = my_abs(a.f64); break; - default: +void abs_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case S16_TYPE: + d.s16 = my_abs(a.s16); + break; + case S32_TYPE: + d.s32 = my_abs(a.s32); + break; + case S64_TYPE: + d.s64 = my_abs(a.s64); + break; + case U16_TYPE: + d.s16 = my_abs(a.u16); + break; + case U32_TYPE: + d.s32 = my_abs(a.u32); + break; + case U64_TYPE: + d.s64 = my_abs(a.u64); + break; + case F32_TYPE: + d.f32 = my_abs(a.f32); + break; + case F64_TYPE: + case FF64_TYPE: + d.f64 = my_abs(a.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -void addp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - //PTXPlus add instruction with carry (carry is kept in a predicate) register - ptx_reg_t src1_data, src2_data, src3_data, data; - int overflow = 0; - int carry = 0; - - const operand_info &dst = pI->dst(); //get operand info of sources and destination - const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - - unsigned rounding_mode = pI->rounding_mode(); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - - //performs addition. Sets carry and overflow if needed. - //src3_data.pred&0x4 is the carry flag - switch ( i_type ) { - case S8_TYPE: - data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + (src3_data.pred & 0x4); - if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } - carry = (data.u64 & 0x000000100)>>8; - break; - case S16_TYPE: - data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + (src3_data.pred & 0x4); - if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } - carry = (data.u64 & 0x000010000)>>16; - break; - case S32_TYPE: - data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + (src3_data.pred & 0x4); - if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } - carry = (data.u64 & 0x100000000)>>32; - break; - case S64_TYPE: + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void addp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // PTXPlus add instruction with carry (carry is kept in a predicate) register + ptx_reg_t src1_data, src2_data, src3_data, data; + int overflow = 0; + int carry = 0; + + const operand_info &dst = + pI->dst(); // get operand info of sources and destination + const operand_info &src1 = + pI->src1(); // use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + + // performs addition. Sets carry and overflow if needed. + // src3_data.pred&0x4 is the carry flag + switch (i_type) { + case S8_TYPE: + data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF) + + (src3_data.pred & 0x4); + if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) == 0) { + overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000000100) >> 8; + break; + case S16_TYPE: + data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF) + + (src3_data.pred & 0x4); + if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) == 0) { + overflow = + ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000010000) >> 16; + break; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF) + + (src3_data.pred & 0x4); + if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) == 0) { + overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 + ? 0 + : 1; + } + carry = (data.u64 & 0x100000000) >> 32; + break; + case S64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case U8_TYPE: - data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + (src3_data.pred & 0x4); - carry = (data.u64 & 0x100)>>8; + case U8_TYPE: + data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF) + + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100) >> 8; break; - case U16_TYPE: - data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + (src3_data.pred & 0x4); - carry = (data.u64 & 0x10000)>>16; + case U16_TYPE: + data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF) + + (src3_data.pred & 0x4); + carry = (data.u64 & 0x10000) >> 16; break; - case U32_TYPE: - data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + (src3_data.pred & 0x4); - carry = (data.u64 & 0x100000000)>>32; + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF) + + (src3_data.pred & 0x4); + carry = (data.u64 & 0x100000000) >> 32; break; - case U64_TYPE: + case U64_TYPE: data.s64 = src1_data.s64 + src2_data.s64 + (src3_data.pred & 0x4); break; - case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; - default: assert(0); break; - } - fesetround( orig_rm ); - - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); -} - -void add_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - int overflow = 0; - int carry = 0; - - const operand_info &dst = pI->dst(); //get operand info of sources and destination - const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - unsigned rounding_mode = pI->rounding_mode(); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - - //performs addition. Sets carry and overflow if needed. - switch ( i_type ) { - case S8_TYPE: + case F16_TYPE: + data.f16 = src1_data.f16 + src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 + src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 + src2_data.f64; + break; + default: + assert(0); + break; + } + fesetround(orig_rm); + + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); +} + +void add_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + int overflow = 0; + int carry = 0; + + const operand_info &dst = + pI->dst(); // get operand info of sources and destination + const operand_info &src1 = + pI->src1(); // use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + unsigned rounding_mode = pI->rounding_mode(); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + + // performs addition. Sets carry and overflow if needed. + switch (i_type) { + case S8_TYPE: data.s64 = (src1_data.s64 & 0x0000000FF) + (src2_data.s64 & 0x0000000FF); - if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) == 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } - carry = (data.u64 & 0x000000100)>>8; + if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) == 0) { + overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000000100) >> 8; break; - case S16_TYPE: + case S16_TYPE: data.s64 = (src1_data.s64 & 0x00000FFFF) + (src2_data.s64 & 0x00000FFFF); - if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) == 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } - carry = (data.u64 & 0x000010000)>>16; + if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) == 0) { + overflow = + ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; + } + carry = (data.u64 & 0x000010000) >> 16; break; - case S32_TYPE: + case S32_TYPE: data.s64 = (src1_data.s64 & 0x0FFFFFFFF) + (src2_data.s64 & 0x0FFFFFFFF); - if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) == 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } - carry = (data.u64 & 0x100000000)>>32; + if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) == 0) { + overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 + ? 0 + : 1; + } + carry = (data.u64 & 0x100000000) >> 32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 + src2_data.s64; break; - case U8_TYPE: + case U8_TYPE: data.u64 = (src1_data.u64 & 0xFF) + (src2_data.u64 & 0xFF); - carry = (data.u64 & 0x100)>>8; + carry = (data.u64 & 0x100) >> 8; break; - case U16_TYPE: + case U16_TYPE: data.u64 = (src1_data.u64 & 0xFFFF) + (src2_data.u64 & 0xFFFF); - carry = (data.u64 & 0x10000)>>16; + carry = (data.u64 & 0x10000) >> 16; break; - case U32_TYPE: + case U32_TYPE: data.u64 = (src1_data.u64 & 0xFFFFFFFF) + (src2_data.u64 & 0xFFFFFFFF); - carry = (data.u64 & 0x100000000)>>32; + carry = (data.u64 & 0x100000000) >> 32; break; - case U64_TYPE: + case U64_TYPE: data.u64 = src1_data.u64 + src2_data.u64; break; - case F16_TYPE: data.f16=src1_data.f16+src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 + src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 + src2_data.f64; break; - default: assert(0); break; - } - fesetround( orig_rm ); + case F16_TYPE: + data.f16 = src1_data.f16 + src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 + src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 + src2_data.f64; + break; + default: + assert(0); + break; + } + fesetround(orig_rm); - thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry ); + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); } -void addc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - -void and_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void addc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void and_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); - else - data.u64 = src1_data.u64 & src2_data.u64; + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) & ~(src2_data.pred)); + else + data.u64 = src1_data.u64 & src2_data.u64; - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void andn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void andn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case B16_TYPE: src2_data.u16 = ~src2_data.u16; break; - case B32_TYPE: src2_data.u32 = ~src2_data.u32; break; - case B64_TYPE: src2_data.u64 = ~src2_data.u64; break; - default: + switch (i_type) { + case B16_TYPE: + src2_data.u16 = ~src2_data.u16; + break; + case B32_TYPE: + src2_data.u32 = ~src2_data.u32; + break; + case B64_TYPE: + src2_data.u64 = ~src2_data.u64; + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - data.u64 = src1_data.u64 & src2_data.u64; - - thread->set_operand_value(dst,data, i_type, thread, pI); -} - -void bar_callback( const inst_t* inst, ptx_thread_info* thread) -{ - unsigned ctaid = thread->get_cta_uid(); - unsigned barid = inst->bar_id; - unsigned value = thread->get_reduction_value(ctaid,barid); - const ptx_instruction *pI = dynamic_cast(inst); - const operand_info &dst = pI->dst(); - ptx_reg_t data; - data.u32 = value; - thread->set_operand_value(dst,value, U32_TYPE, thread, pI); -} - -void atom_callback( const inst_t* inst, ptx_thread_info* thread) -{ - const ptx_instruction *pI = dynamic_cast(inst); - - // "Decode" the output type - unsigned to_type = pI->get_type(); - size_t size; - int t; - type_info_key::type_decode(to_type, size, t); - - // Set up operand variables - ptx_reg_t data; // d - ptx_reg_t src1_data; // a - ptx_reg_t src2_data; // b - ptx_reg_t op_result; // temp variable to hold operation result - - bool data_ready = false; - - // Get operand info of sources and destination - const operand_info &dst = pI->dst(); // d - const operand_info &src1 = pI->src1(); // a - const operand_info &src2 = pI->src2(); // b - - // Get operand values - src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a - if (dst.get_symbol()->type()){ - src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b - } else { - //This is the case whent he first argument (dest) is '_' - src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b - } - - // Check state space - addr_t effective_address = src1_data.u64; - memory_space_t space = pI->get_space(); - if (space == undefined_space) { - // generic space - determine space via address - if( whichspace(effective_address) == global_space ) { - effective_address = generic_to_global(effective_address); - space = global_space; - } else if( whichspace(effective_address) == shared_space ) { - unsigned smid = thread->get_hw_sid(); - effective_address = generic_to_shared(smid,effective_address); - space = shared_space; - } else { - abort(); - } - } - assert( space == global_space || space == shared_space ); - - memory_space *mem = NULL; - if(space == global_space) - mem = thread->get_global_memory(); - else if(space == shared_space) - mem = thread->m_shared_mem; - else - abort(); - - // Copy value pointed to in operand 'a' into register 'd' - // (i.e. copy src1_data to dst) - mem->read(effective_address,size/8,&data.s64); - if (dst.get_symbol()->type()){ - thread->set_operand_value(dst, data, to_type, thread, pI); // Write value into register 'd' - } - - // Get the atomic operation to be performed - unsigned m_atomic_spec = pI->get_atomic(); - - switch ( m_atomic_spec ) { - // AND - case ATOMIC_AND: - { - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 & src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 & src2_data.s32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.AND only accepts b32\n", to_type); - assert(0); - break; - } + assert(0); + break; + } - break; - } - // OR - case ATOMIC_OR: - { - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 | src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 | src2_data.s32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.OR only accepts b32\n", to_type); - assert(0); - break; - } + data.u64 = src1_data.u64 & src2_data.u64; - break; - } - // XOR - case ATOMIC_XOR: - { - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = data.u32 ^ src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 ^ src2_data.s32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.XOR only accepts b32\n", to_type); - assert(0); - break; - } + thread->set_operand_value(dst, data, i_type, thread, pI); +} - break; - } - // CAS - case ATOMIC_CAS: - { - - ptx_reg_t src3_data; - const operand_info &src3 = pI->src3(); - src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); - - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); - data_ready = true; - break; - case B64_TYPE: - case U64_TYPE: - op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.CAS only accepts b32 and b64\n", to_type); - assert(0); - break; - } +void bar_callback(const inst_t *inst, ptx_thread_info *thread) { + unsigned ctaid = thread->get_cta_uid(); + unsigned barid = inst->bar_id; + unsigned value = thread->get_reduction_value(ctaid, barid); + const ptx_instruction *pI = dynamic_cast(inst); + const operand_info &dst = pI->dst(); + ptx_reg_t data; + data.u32 = value; + thread->set_operand_value(dst, value, U32_TYPE, thread, pI); +} - break; - } - // EXCH - case ATOMIC_EXCH: - { - switch ( to_type ) { - case B32_TYPE: - case U32_TYPE: - op_result.u32 = MY_EXCH(data.u32, src2_data.u32); - data_ready = true; - break; - case B64_TYPE: - case U64_TYPE: - op_result.u64 = MY_EXCH(data.u64, src2_data.u64); - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = MY_EXCH(data.s32, src2_data.s32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch (%x) with instruction\natom.EXCH only accepts b32\n", to_type); - assert(0); - break; - } +void atom_callback(const inst_t *inst, ptx_thread_info *thread) { + const ptx_instruction *pI = dynamic_cast(inst); + + // "Decode" the output type + unsigned to_type = pI->get_type(); + size_t size; + int t; + type_info_key::type_decode(to_type, size, t); + + // Set up operand variables + ptx_reg_t data; // d + ptx_reg_t src1_data; // a + ptx_reg_t src2_data; // b + ptx_reg_t op_result; // temp variable to hold operation result + + bool data_ready = false; + + // Get operand info of sources and destination + const operand_info &dst = pI->dst(); // d + const operand_info &src1 = pI->src1(); // a + const operand_info &src2 = pI->src2(); // b + + // Get operand values + src1_data = thread->get_operand_value(src1, src1, to_type, thread, 1); // a + if (dst.get_symbol()->type()) { + src2_data = thread->get_operand_value(src2, dst, to_type, thread, 1); // b + } else { + // This is the case whent he first argument (dest) is '_' + src2_data = thread->get_operand_value(src2, src1, to_type, thread, 1); // b + } + + // Check state space + addr_t effective_address = src1_data.u64; + memory_space_t space = pI->get_space(); + if (space == undefined_space) { + // generic space - determine space via address + if (whichspace(effective_address) == global_space) { + effective_address = generic_to_global(effective_address); + space = global_space; + } else if (whichspace(effective_address) == shared_space) { + unsigned smid = thread->get_hw_sid(); + effective_address = generic_to_shared(smid, effective_address); + space = shared_space; + } else { + abort(); + } + } + assert(space == global_space || space == shared_space); + + memory_space *mem = NULL; + if (space == global_space) + mem = thread->get_global_memory(); + else if (space == shared_space) + mem = thread->m_shared_mem; + else + abort(); + + // Copy value pointed to in operand 'a' into register 'd' + // (i.e. copy src1_data to dst) + mem->read(effective_address, size / 8, &data.s64); + if (dst.get_symbol()->type()) { + thread->set_operand_value(dst, data, to_type, thread, + pI); // Write value into register 'd' + } - break; + // Get the atomic operation to be performed + unsigned m_atomic_spec = pI->get_atomic(); + + switch (m_atomic_spec) { + // AND + case ATOMIC_AND: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 & src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 & src2_data.s32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.AND " + "only accepts b32\n", + to_type); + assert(0); + break; } - // ADD - case ATOMIC_ADD: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = data.u32 + src2_data.u32; - data_ready = true; - break; - case S32_TYPE: - op_result.s32 = data.s32 + src2_data.s32; - data_ready = true; - break; - case U64_TYPE: - op_result.u64 = data.u64 + src2_data.u64; - data_ready = true; - break; - case F32_TYPE: - op_result.f32 = data.f32 + src2_data.f32; - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.ADD only accepts u32, s32, u64, and f32\n"); - assert(0); - break; - } + break; + } + // OR + case ATOMIC_OR: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 | src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 | src2_data.s32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.OR " + "only accepts b32\n", + to_type); + assert(0); + break; + } - break; + break; + } + // XOR + case ATOMIC_XOR: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = data.u32 ^ src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 ^ src2_data.s32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.XOR " + "only accepts b32\n", + to_type); + assert(0); + break; } - // INC - case ATOMIC_INC: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_INC_I(data.u32, src2_data.u32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.INC only accepts u32 and s32\n"); - assert(0); - break; - } - break; + break; + } + // CAS + case ATOMIC_CAS: { + ptx_reg_t src3_data; + const operand_info &src3 = pI->src3(); + src3_data = thread->get_operand_value(src3, dst, to_type, thread, 1); + + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_CAS_I(data.u32, src2_data.u32, src3_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_CAS_I(data.u64, src2_data.u64, src3_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_CAS_I(data.s32, src2_data.s32, src3_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.CAS " + "only accepts b32 and b64\n", + to_type); + assert(0); + break; + } + + break; + } + // EXCH + case ATOMIC_EXCH: { + switch (to_type) { + case B32_TYPE: + case U32_TYPE: + op_result.u32 = MY_EXCH(data.u32, src2_data.u32); + data_ready = true; + break; + case B64_TYPE: + case U64_TYPE: + op_result.u64 = MY_EXCH(data.u64, src2_data.u64); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_EXCH(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch (%x) with instruction\natom.EXCH " + "only accepts b32\n", + to_type); + assert(0); + break; + } + + break; + } + // ADD + case ATOMIC_ADD: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = data.u32 + src2_data.u32; + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = data.s32 + src2_data.s32; + data_ready = true; + break; + case U64_TYPE: + op_result.u64 = data.u64 + src2_data.u64; + data_ready = true; + break; + case F32_TYPE: + op_result.f32 = data.f32 + src2_data.f32; + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.ADD only " + "accepts u32, s32, u64, and f32\n"); + assert(0); + break; + } + + break; + } + // INC + case ATOMIC_INC: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_INC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.INC only " + "accepts u32 and s32\n"); + assert(0); + break; } - // DEC - case ATOMIC_DEC: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); - data_ready = true; - break; - default: - printf("Execution error: type mismatch with instruction\natom.DEC only accepts u32 and s32\n"); - assert(0); - break; - } - break; + break; + } + // DEC + case ATOMIC_DEC: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_DEC_I(data.u32, src2_data.u32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.DEC only " + "accepts u32 and s32\n"); + assert(0); + break; } + + break; + } // MIN - case ATOMIC_MIN: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); - data_ready = true; + case ATOMIC_MIN: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_MIN_I(data.u32, src2_data.u32); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.MIN only " + "accepts u32 and s32\n"); + assert(0); + break; + } + + break; + } + // MAX + case ATOMIC_MAX: { + switch (to_type) { + case U32_TYPE: + op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); + data_ready = true; + break; + case S32_TYPE: + op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); + data_ready = true; + break; + default: + printf( + "Execution error: type mismatch with instruction\natom.MAX only " + "accepts u32 and s32\n"); + assert(0); + break; + } + + break; + } + // DEFAULT + default: { + assert(0); + break; + } + } + + // Write operation result into memory + // (i.e. copy src1_data to dst) + if (data_ready) { + mem->write(effective_address, size / 8, &op_result.s64, thread, pI); + } else { + printf("Execution error: data_ready not set\n"); + assert(0); + } +} + +// atom_impl will now result in a callback being called in mem_ctrl_pop +// (gpu-sim.c) +void atom_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // SYNTAX + // atom.space.operation.type d, a, b[, c]; (now read in callback) + + // obtain memory space of the operation + memory_space_t space = pI->get_space(); + + // get the memory address + const operand_info &src1 = pI->src1(); + // const operand_info &dst = pI->dst(); // not needed for effective address + // calculation + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data; + src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); + addr_t effective_address = src1_data.u64; + + addr_t effective_address_final; + + // handle generic memory space by converting it to global + if (space == undefined_space) { + if (whichspace(effective_address) == global_space) { + effective_address_final = generic_to_global(effective_address); + space = global_space; + } else if (whichspace(effective_address) == shared_space) { + unsigned smid = thread->get_hw_sid(); + effective_address_final = generic_to_shared(smid, effective_address); + space = shared_space; + } else { + abort(); + } + } else { + assert(space == global_space || space == shared_space); + effective_address_final = effective_address; + } + + // Check state space + assert(space == global_space || space == shared_space); + + thread->m_last_effective_address = effective_address_final; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = atom_callback; + thread->m_last_dram_callback.instruction = pI; +} + +void bar_impl(const ptx_instruction *pIin, ptx_thread_info *thread) { + ptx_instruction *pI = const_cast(pIin); + unsigned bar_op = pI->barrier_op(); + unsigned red_op = pI->get_atomic(); + unsigned ctaid = thread->get_cta_uid(); + + switch (bar_op) { + case SYNC_OPTION: { + if (pI->get_num_operands() > 1) { + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + } else { + const operand_info &op0 = pI->dst(); + ptx_reg_t op0_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + } + break; + } + case ARRIVE_OPTION: { + const operand_info &op0 = pI->dst(); + const operand_info &op1 = pI->src1(); + ptx_reg_t op0_data; + ptx_reg_t op1_data; + op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + pI->set_bar_id(op0_data.u32); + pI->set_bar_count(op1_data.u32); + break; + } + case RED_OPTION: { + if (pI->get_num_operands() > 3) { + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + const operand_info &op3 = pI->src3(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + ptx_reg_t op3_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); + op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); + op3_data.u32 = !(op3_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(op2_data.u32); + switch (red_op) { + case ATOMIC_POPC: + thread->popc_reduction(ctaid, op1_data.u32, op3_data.u32); break; - case S32_TYPE: - op_result.s32 = MY_MIN_I(data.s32, src2_data.s32); - data_ready = true; + case ATOMIC_AND: + thread->and_reduction(ctaid, op1_data.u32, op3_data.u32); break; - default: - printf("Execution error: type mismatch with instruction\natom.MIN only accepts u32 and s32\n"); - assert(0); + case ATOMIC_OR: + thread->or_reduction(ctaid, op1_data.u32, op3_data.u32); break; - } - - break; - } - // MAX - case ATOMIC_MAX: - { - switch ( to_type ) { - case U32_TYPE: - op_result.u32 = MY_MAX_I(data.u32, src2_data.u32); - data_ready = true; + default: + abort(); break; - case S32_TYPE: - op_result.s32 = MY_MAX_I(data.s32, src2_data.s32); - data_ready = true; + } + } else { + const operand_info &op1 = pI->src1(); + const operand_info &op2 = pI->src2(); + ptx_reg_t op1_data; + ptx_reg_t op2_data; + op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); + op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); + op2_data.u32 = !(op2_data.pred & 0x0001); + pI->set_bar_id(op1_data.u32); + pI->set_bar_count(thread->get_ntid().x * thread->get_ntid().y * + thread->get_ntid().z); + switch (red_op) { + case ATOMIC_POPC: + thread->popc_reduction(ctaid, op1_data.u32, op2_data.u32); break; - default: - printf("Execution error: type mismatch with instruction\natom.MAX only accepts u32 and s32\n"); - assert(0); + case ATOMIC_AND: + thread->and_reduction(ctaid, op1_data.u32, op2_data.u32); break; - } - - break; + case ATOMIC_OR: + thread->or_reduction(ctaid, op1_data.u32, op2_data.u32); + break; + default: + abort(); + break; + } } - // DEFAULT - default: - { - assert(0); - break; + break; + } + default: + abort(); + break; + } + + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = pIin; +} + +void bfe_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + unsigned i_type = pI->get_type(); + unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + ptx_reg_t data; + unsigned pos = b.u32 & 0xFF; + unsigned len = c.u32 & 0xFF; + switch (i_type) { + case U32_TYPE: { + unsigned mask; + data.u32 = src.u32 >> pos; + mask = 0xFFFFFFFF >> (32 - len); + data.u32 &= mask; + break; + } + case U64_TYPE: { + unsigned long mask; + data.u64 = src.u64 >> pos; + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + data.u64 &= mask; + break; + } + case S32_TYPE: { + unsigned mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; + data.s32 = src.s32 >> pos; + if (sbit > 0) { + mask = 0xFFFFFFFF << len; + data.s32 |= mask; + } else { + mask = 0xFFFFFFFF >> (32 - len); + data.s32 &= mask; } - } - - // Write operation result into memory - // (i.e. copy src1_data to dst) - if ( data_ready ) { - mem->write(effective_address,size/8,&op_result.s64,thread,pI); - } else { - printf("Execution error: data_ready not set\n"); - assert(0); - } -} - -// atom_impl will now result in a callback being called in mem_ctrl_pop (gpu-sim.c) -void atom_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - // SYNTAX - // atom.space.operation.type d, a, b[, c]; (now read in callback) - - // obtain memory space of the operation - memory_space_t space = pI->get_space(); - - // get the memory address - const operand_info &src1 = pI->src1(); - // const operand_info &dst = pI->dst(); // not needed for effective address calculation - unsigned i_type = pI->get_type(); - ptx_reg_t src1_data; - src1_data = thread->get_operand_value(src1, src1, i_type, thread, 1); - addr_t effective_address = src1_data.u64; - - addr_t effective_address_final; - - // handle generic memory space by converting it to global - if ( space == undefined_space ) { - if( whichspace(effective_address) == global_space ) { - effective_address_final = generic_to_global(effective_address); - space = global_space; - } else if( whichspace(effective_address) == shared_space ) { - unsigned smid = thread->get_hw_sid(); - effective_address_final = generic_to_shared(smid,effective_address); - space = shared_space; + break; + } + case S64_TYPE: { + unsigned long mask; + unsigned min = MY_MIN_I(pos + len - 1, msb); + unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; + data.s64 = src.s64 >> pos; + if (sbit > 0) { + mask = 0xFFFFFFFFFFFFFFFF << len; + data.s64 |= mask; } else { - abort(); + mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); + data.s64 &= mask; } - } else { - assert( space == global_space || space == shared_space ); - effective_address_final = effective_address; - } - - // Check state space - assert( space == global_space || space == shared_space ); - - thread->m_last_effective_address = effective_address_final; - thread->m_last_memory_space = space; - thread->m_last_dram_callback.function = atom_callback; - thread->m_last_dram_callback.instruction = pI; -} - -void bar_impl( const ptx_instruction *pIin, ptx_thread_info *thread ) -{ - ptx_instruction * pI = const_cast(pIin); - unsigned bar_op = pI->barrier_op(); - unsigned red_op = pI->get_atomic(); - unsigned ctaid = thread->get_cta_uid(); - - switch(bar_op){ - case SYNC_OPTION: - { - if(pI->get_num_operands()>1){ - const operand_info &op0 = pI->dst(); - const operand_info &op1 = pI->src1(); - ptx_reg_t op0_data; - ptx_reg_t op1_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - pI->set_bar_count(op1_data.u32); - }else{ - const operand_info &op0 = pI->dst(); - ptx_reg_t op0_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - } - break; - } - case ARRIVE_OPTION: - { - const operand_info &op0 = pI->dst(); - const operand_info &op1 = pI->src1(); - ptx_reg_t op0_data; - ptx_reg_t op1_data; - op0_data = thread->get_operand_value(op0, op0, U32_TYPE, thread, 1); - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - pI->set_bar_id(op0_data.u32); - pI->set_bar_count(op1_data.u32); - break; - } - case RED_OPTION: - { - if(pI->get_num_operands()>3){ - const operand_info &op1 = pI->src1(); - const operand_info &op2 = pI->src2(); - const operand_info &op3 = pI->src3(); - ptx_reg_t op1_data; - ptx_reg_t op2_data; - ptx_reg_t op3_data; - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - op2_data = thread->get_operand_value(op2, op2, U32_TYPE, thread, 1); - op3_data = thread->get_operand_value(op3, op3, PRED_TYPE, thread, 1); - op3_data.u32=!(op3_data.pred & 0x0001); - pI->set_bar_id(op1_data.u32); - pI->set_bar_count(op2_data.u32); - switch(red_op){ - case ATOMIC_POPC: - thread->popc_reduction(ctaid,op1_data.u32,op3_data.u32); - break; - case ATOMIC_AND: - thread->and_reduction(ctaid,op1_data.u32,op3_data.u32); - break; - case ATOMIC_OR: - thread->or_reduction(ctaid,op1_data.u32,op3_data.u32); - break; - default: - abort(); - break; - } - }else{ - const operand_info &op1 = pI->src1(); - const operand_info &op2 = pI->src2(); - ptx_reg_t op1_data; - ptx_reg_t op2_data; - op1_data = thread->get_operand_value(op1, op1, U32_TYPE, thread, 1); - op2_data = thread->get_operand_value(op2, op2, PRED_TYPE, thread, 1); - op2_data.u32=!(op2_data.pred & 0x0001); - pI->set_bar_id(op1_data.u32); - pI->set_bar_count(thread->get_ntid().x * thread->get_ntid().y * thread->get_ntid().z); - switch(red_op){ - case ATOMIC_POPC: - thread->popc_reduction(ctaid,op1_data.u32,op2_data.u32); - break; - case ATOMIC_AND: - thread->and_reduction(ctaid,op1_data.u32,op2_data.u32); - break; - case ATOMIC_OR: - thread->or_reduction(ctaid,op1_data.u32,op2_data.u32); - break; - default: - abort(); - break; - } - } - break; - } - default: - abort(); - break; - } - - thread->m_last_dram_callback.function = bar_callback; - thread->m_last_dram_callback.instruction = pIin; -} - -void bfe_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - unsigned i_type = pI->get_type(); - unsigned msb = (i_type == U32_TYPE || i_type == S32_TYPE) ? 31 : 63; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t src = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - ptx_reg_t data; - unsigned pos = b.u32 & 0xFF; - unsigned len = c.u32 & 0xFF; - switch (i_type) - { - case U32_TYPE: - { - unsigned mask; - data.u32 = src.u32 >> pos; - mask = 0xFFFFFFFF >> (32 - len); - data.u32 &= mask; - break; - } - case U64_TYPE: - { - unsigned long mask; - data.u64 = src.u64 >> pos; - mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - data.u64 &= mask; - break; - } - case S32_TYPE: - { - unsigned mask; - unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (src.s32 >> min) & 0x1; - data.s32 = src.s32 >> pos; - if (sbit > 0) - { - mask = 0xFFFFFFFF << len; - data.s32 |= mask; - } - else - { - mask = 0xFFFFFFFF >> (32 - len); - data.s32 &= mask; - } - break; - } - case S64_TYPE: - { - unsigned long mask; - unsigned min = MY_MIN_I(pos + len - 1, msb); - unsigned sbit = len == 0 ? 0 : (src.s64 >> min) & 0x1; - data.s64 = src.s64 >> pos; - if (sbit > 0) - { - mask = 0xFFFFFFFFFFFFFFFF << len; - data.s64 |= mask; - } - else - { - mask = 0xFFFFFFFFFFFFFFFF >> (64 - len); - data.s64 &= mask; - } - break; - } - default: - printf("Operand type not supported for BFE instruction.\n"); - abort(); - return; - } - thread->set_operand_value(dst, data, i_type, thread, pI); + break; + } + default: + printf("Operand type not supported for BFE instruction.\n"); + abort(); + return; + } + thread->set_operand_value(dst, data, i_type, thread, pI); } -void bfi_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { - int i,max; - ptx_reg_t src1_data, src2_data; - ptx_reg_t src3_data, src4_data, data; - - const operand_info &dst = pI->dst(); //get operand info of sources and destination - const operand_info &src1 = pI->src1(); //use them to determine that they are of type 'register' - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - const operand_info &src4 = pI->src4(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); - - switch ( i_type ) { - case B32_TYPE: +void bfi_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + int i, max; + ptx_reg_t src1_data, src2_data; + ptx_reg_t src3_data, src4_data, data; + + const operand_info &dst = + pI->dst(); // get operand info of sources and destination + const operand_info &src1 = + pI->src1(); // use them to determine that they are of type 'register' + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + const operand_info &src4 = pI->src4(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + src4_data = thread->get_operand_value(src4, dst, i_type, thread, 1); + + switch (i_type) { + case B32_TYPE: max = 32; break; - case B64_TYPE: + case B64_TYPE: max = 64; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - data=src2_data; - unsigned pos = src3_data.u32 & 0xFF; - unsigned len = src4_data.u32 & 0xFF; - for(i=0;iset_operand_value(dst, data, i_type, thread, pI); + } + data = src2_data; + unsigned pos = src3_data.u32 & 0xFF; + unsigned len = src4_data.u32 & 0xFF; + for (i = 0; i < len && pos + i < max; i++) { + data.u32 = (~((0x00000001) << (pos + i))) & data.u32; + data.u32 = data.u32 | ((src1_data.u32 & ((0x00000001) << (i))) << (pos)); + } + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void bfind_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); } -void bfind_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void bra_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void bra_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void brx_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void brx_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void break_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = thread->pop_breakaddr(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); +void break_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = thread->pop_breakaddr(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - thread->m_branch_taken = true; - thread->set_npc(target_pc); + thread->m_branch_taken = true; + thread->set_npc(target_pc); } -void breakaddr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &target = pI->dst(); - thread->push_breakaddr(target); - assert(pI->has_pred() == false); // pdom analysis cannot handle if this instruction is predicated +void breakaddr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &target = pI->dst(); + thread->push_breakaddr(target); + assert( + pI->has_pred() == + false); // pdom analysis cannot handle if this instruction is predicated } -void brev_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); +void brev_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + unsigned msb; + switch (i_type) { + case B32_TYPE: + msb = 31; + for (unsigned i = 0; i <= msb; i++) { + if ((src1_data.u32 & (1 << i))) data.u32 |= 1 << (msb - i); + } + break; + case B64_TYPE: + msb = 63; + for (unsigned i = 0; i <= msb; i++) { + if ((src1_data.u64 & (1 << i))) data.u64 |= 1 << (msb - i); + } + break; + default: + assert(0); + } + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void brkpt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} - unsigned msb; - switch(i_type){ - case B32_TYPE: - msb = 31; - for (unsigned i=0; i<=msb; i++) { - if((src1_data.u32 & (1 << i))) - data.u32 |= 1 << (msb - i); - } - break; - case B64_TYPE: - msb = 63; - for (unsigned i=0; i<=msb; i++) { - if((src1_data.u64 & (1 << i))) - data.u64 |= 1 << (msb - i); +unsigned trunc(unsigned num, unsigned precision) { + int mask = 1, latest_one = -1; + unsigned data = num; + for (unsigned j = 0; j < sizeof(unsigned) * 8; j++) { + int bit = data & mask; + if (bit == 1) latest_one = j; + data >>= 1; + } + if (latest_one >= precision) { + // round_up is 1 if the most significant truncated digit is a 1, otherwise + // it is 0 + // int round_up = (num & (1 << (latest_one-precision))) >> + // (latest_one-precision); unsigned shifted_output = num >> + // (latest_one-precision+1); + // if shifted_output is a number like 1111, don't round up + // if (shifted_output == (pow(2,precision)-1)) round_up = 0; + // num = shifted_output + round_up; + num >>= (latest_one - precision + 1); + } + return num; +} +void mapping(int thread, int wmma_type, int wmma_layout, int type, int index, + int stride, int &row, int &col, int &assg_offset) { + int offset; + int c_row_offset[] = {0, 8, 0, 8, 4, 12, 4, 12}; + int c_col_offset[] = {0, 0, 8, 8, 0, 0, 8, 8}; + int c_tg_inside_row_offset[] = {0, 1, 0, 1}; + int c_tg_inside_col_offset[] = {0, 0, 2, 2}; + int c_inside_row_offset[] = {0, 0, 2, 2, 0, 0, 2, 2}; + int c_inside_col_offset[] = {0, 1, 0, 1, 4, 5, 4, 5}; + + offset = thread_group_offset(thread, wmma_type, wmma_layout, type, stride); + + if (wmma_type == LOAD_A) { + if (wmma_layout == ROW) { + offset += index + 8 * ((thread % 16) / 8); + } else { + offset += 64 * (index / 4) + index % 4 + 128 * ((thread % 16) / 8); + } + offset = (offset / 16) * stride + offset % 16; + assg_offset = index + 8 * ((thread % 16) / 8); + } else if (wmma_type == LOAD_B) { + if (wmma_layout == ROW) { + offset += 64 * (index / 4) + index % 4 + 128 * ((thread % 16) / 8); + } else { + offset += index + 8 * ((thread % 16) / 8); + } + offset = (offset / 16) * stride + offset % 16; + assg_offset = index + 8 * ((thread % 16) / 8); + } else if (wmma_type == LOAD_C) { + if (type == F16_TYPE) { + row = c_row_offset[thread / 4] + thread % 4; + col = c_col_offset[thread / 4] + index; + } else { + row = c_row_offset[thread / 4] + c_tg_inside_row_offset[thread % 4] + + c_inside_row_offset[index]; + col = c_col_offset[thread / 4] + c_tg_inside_col_offset[thread % 4] + + c_inside_col_offset[index]; + } + assg_offset = index; + } + + if (wmma_type == LOAD_A || wmma_type == LOAD_B) { + if (wmma_layout == ROW) { + row = offset / 16; + col = offset % 16; + } else { + col = offset / 16; + row = offset % 16; + } + } +} + +void mma_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst) { + int i, j, k, thrd; + int row, col, offset; + ptx_reg_t matrix_a[16][16]; + ptx_reg_t matrix_b[16][16]; + ptx_reg_t matrix_c[16][16]; + ptx_reg_t matrix_d[16][16]; + ptx_reg_t src_data; + ptx_thread_info *thread; + + unsigned a_layout = pI->get_wmma_layout(0); + unsigned b_layout = pI->get_wmma_layout(1); + unsigned type = pI->get_type(); + unsigned type2 = pI->get_type2(); + int tid; + const operand_info &dst = pI->operand_lookup(0); + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + float temp; + half temp2; + + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid + thrd]; + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("THREAD=%d\n:", thrd); + for (int operand_num = 1; operand_num <= 3; operand_num++) { + const operand_info &src_a = pI->operand_lookup(operand_num); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t v[8]; + thread->get_vector_operand_values(src_a, v, nelem); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("Thread%d_Iteration=%d\n:", thrd, operand_num); + for (k = 0; k < nelem; k++) { + printf("%llx ", v[k].u64); + } + printf("\n"); + } + ptx_reg_t nw_v[16]; + int hex_val; + + if (!((operand_num == 3) && (type2 == F32_TYPE))) { + for (k = 0; k < 2 * nelem; k++) { + if (k % 2 == 1) + hex_val = (v[k / 2].s64 & 0xffff); + else + hex_val = ((v[k / 2].s64 & 0xffff0000) >> 16); + nw_v[k].f16 = *((half *)&hex_val); + } + } + if (!((operand_num == 3) && (type2 == F32_TYPE))) { + for (k = 0; k < 2 * nelem; k++) { + temp = nw_v[k].f16; + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("%.2f ", temp); + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) printf("\n"); + } else { + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + for (k = 0; k < 8; k++) { + printf("%.2f ", v[k].f32); + } + printf("\n"); + } + } + switch (operand_num) { + case 1: // operand 1 + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_A, a_layout, F16_TYPE, k, 16, row, col, offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("A:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, + offset); + matrix_a[row][col] = nw_v[offset]; + } + break; + case 2: // operand 2 + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_B, b_layout, F16_TYPE, k, 16, row, col, offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("B:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, + offset); + matrix_b[row][col] = nw_v[offset]; + } + break; + case 3: // operand 3 + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_C, ROW, type2, k, 16, row, col, offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("C:thread=%d,row=%d,col=%d,offset=%d\n", thrd, row, col, + offset); + if (type2 != F16_TYPE) { + matrix_c[row][col] = v[offset]; + } else { + matrix_c[row][col] = nw_v[offset]; } - break; - default: assert(0); + } + break; + default: + printf("Invalid Operand Index\n"); + } + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) printf("\n"); + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("MATRIX_A\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + temp = matrix_a[i][j].f16; + printf("%.2f ", temp); + } + printf("\n"); + } + printf("MATRIX_B\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + temp = matrix_b[i][j].f16; + printf("%.2f ", temp); + } + printf("\n"); + } + printf("MATRIX_C\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + if (type2 == F16_TYPE) { + temp = matrix_c[i][j].f16; + printf("%.2f ", temp); + } else + printf("%.2f ", matrix_c[i][j].f32); + } + printf("\n"); + } + } + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + matrix_d[i][j].f16 = 0; + } + } + + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + for (k = 0; k < 16; k++) { + matrix_d[i][j].f16 = + matrix_d[i][j].f16 + matrix_a[i][k].f16 * matrix_b[k][j].f16; + } + if ((type == F16_TYPE) && (type2 == F16_TYPE)) + matrix_d[i][j].f16 += matrix_c[i][j].f16; + else if ((type == F32_TYPE) && (type2 == F16_TYPE)) { + temp2 = matrix_d[i][j].f16 + matrix_c[i][j].f16; + temp = temp2; + matrix_d[i][j].f32 = temp; + } else if ((type == F16_TYPE) && (type2 == F32_TYPE)) { + temp = matrix_d[i][j].f16; + temp += matrix_c[i][j].f32; + matrix_d[i][j].f16 = half(temp); + } else { + temp = matrix_d[i][j].f16; + temp += matrix_c[i][j].f32; + matrix_d[i][j].f32 = temp; + } + } + } + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("MATRIX_D\n"); + for (i = 0; i < 16; i++) { + for (j = 0; j < 16; j++) { + if (type == F16_TYPE) { + temp = matrix_d[i][j].f16; + printf("%.2f ", temp); + } else + printf("%.2f ", matrix_d[i][j].f32); + } + printf("\n"); + } + } + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + int row_t[8]; + int col_t[8]; + for (k = 0; k < 8; k++) { + mapping(thrd, LOAD_C, ROW, type, k, 16, row_t[k], col_t[k], offset); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma:store:row:%d,col%d\n", row_t[k], col_t[k]); + } + thread = core->get_thread_info()[tid + thrd]; + + if (type == F32_TYPE) { + thread->set_wmma_vector_operand_values( + dst, matrix_d[row_t[0]][col_t[0]], matrix_d[row_t[1]][col_t[1]], + matrix_d[row_t[2]][col_t[2]], matrix_d[row_t[3]][col_t[3]], + matrix_d[row_t[4]][col_t[4]], matrix_d[row_t[5]][col_t[5]], + matrix_d[row_t[6]][col_t[6]], matrix_d[row_t[7]][col_t[7]]); + + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("thread%d:", thrd); + for (k = 0; k < 8; k++) { + printf("%.2f ", matrix_d[row_t[k]][col_t[k]].f32); + } + printf("\n"); + } + } else if (type == F16_TYPE) { + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf("thread%d:", thrd); + for (k = 0; k < 8; k++) { + temp = matrix_d[row_t[k]][col_t[k]].f16; + printf("%.2f ", temp); + } + printf("\n"); + + printf("thread%d:", thrd); + for (k = 0; k < 8; k++) { + printf("%x ", (unsigned int)matrix_d[row_t[k]][col_t[k]].f16); + } + printf("\n"); + } + ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; + nw_data1.s64 = ((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff)) | + ((matrix_d[row_t[1]][col_t[1]].s64 & 0xffff) << 16); + nw_data2.s64 = ((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff)) | + ((matrix_d[row_t[3]][col_t[3]].s64 & 0xffff) << 16); + nw_data3.s64 = ((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff)) | + ((matrix_d[row_t[5]][col_t[5]].s64 & 0xffff) << 16); + nw_data4.s64 = ((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff)) | + ((matrix_d[row_t[7]][col_t[7]].s64 & 0xffff) << 16); + thread->set_vector_operand_values(dst, nw_data1, nw_data2, nw_data3, + nw_data4); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("thread%d=%llx,%llx,%llx,%llx", thrd, nw_data1.s64, nw_data2.s64, + nw_data3.s64, nw_data4.s64); + + } else { + printf("wmma:mma:wrong type\n"); + abort(); } - thread->set_operand_value(dst,data, i_type, thread, pI); + } } -void brkpt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -unsigned trunc(unsigned num, unsigned precision) { - int mask = 1, latest_one = -1; - unsigned data = num; - for (unsigned j = 0; j < sizeof(unsigned)*8; j++) { - int bit = data & mask; - if (bit == 1) latest_one = j; - data >>= 1; - } - if (latest_one >= precision) { - // round_up is 1 if the most significant truncated digit is a 1, otherwise it is 0 - //int round_up = (num & (1 << (latest_one-precision))) >> (latest_one-precision); - //unsigned shifted_output = num >> (latest_one-precision+1); - // if shifted_output is a number like 1111, don't round up - //if (shifted_output == (pow(2,precision)-1)) round_up = 0; - //num = shifted_output + round_up; - num >>= (latest_one-precision+1); - } - return num; -} -void mapping(int thread,int wmma_type,int wmma_layout,int type,int index,int stride,int &row,int &col,int &assg_offset){ - int offset; - int c_row_offset[]={0,8,0,8,4,12,4,12}; - int c_col_offset[]={0,0,8,8,0,0,8,8}; - int c_tg_inside_row_offset[]={0,1,0,1}; - int c_tg_inside_col_offset[]={0,0,2,2}; - int c_inside_row_offset[]={0,0,2,2,0,0,2,2}; - int c_inside_col_offset[]={0,1,0,1,4,5,4,5}; - - offset=thread_group_offset(thread,wmma_type,wmma_layout,type,stride); - - if(wmma_type==LOAD_A){ - if(wmma_layout==ROW){ - offset+=index+8*((thread%16)/8); - } - else{ - offset+=64*(index/4)+index%4+128*((thread%16)/8); - } - offset=(offset/16)*stride+offset%16; - assg_offset=index+8*((thread%16)/8); - } - else if(wmma_type==LOAD_B){ - if(wmma_layout==ROW){ - offset+=64*(index/4)+index%4+128*((thread%16)/8); - } - else{ - offset+=index+8*((thread%16)/8); - } - offset=(offset/16)*stride+offset%16; - assg_offset=index+8*((thread%16)/8); - } - else if( wmma_type==LOAD_C){ - if(type==F16_TYPE){ - row=c_row_offset[thread/4]+thread%4; - col=c_col_offset[thread/4]+index; - } - else{ - row=c_row_offset[thread/4]+c_tg_inside_row_offset[thread%4]+c_inside_row_offset[index]; - col=c_col_offset[thread/4]+c_tg_inside_col_offset[thread%4]+c_inside_col_offset[index]; - } - assg_offset=index; - } - - if(wmma_type==LOAD_A||wmma_type==LOAD_B){ - if(wmma_layout==ROW){ - row=offset/16; - col=offset%16; - } - else{ - col=offset/16; - row=offset%16; - } - } -} - -void mma_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) -{ - int i,j,k,thrd; - int row,col,offset; - ptx_reg_t matrix_a[16][16]; - ptx_reg_t matrix_b[16][16]; - ptx_reg_t matrix_c[16][16]; - ptx_reg_t matrix_d[16][16]; - ptx_reg_t src_data; - ptx_thread_info *thread; - - unsigned a_layout = pI->get_wmma_layout(0); - unsigned b_layout = pI->get_wmma_layout(1); - unsigned type = pI->get_type(); - unsigned type2 = pI->get_type2(); - int tid ; - const operand_info &dst = pI->operand_lookup(0); - - if(core->get_gpu()->is_functional_sim()) - tid= inst.warp_id_func()*core->get_warp_size(); - else - tid= inst.warp_id()*core->get_warp_size(); - float temp; - half temp2; - - for (thrd=0; thrd < core->get_warp_size(); thrd++){ - thread = core->get_thread_info()[tid+thrd]; - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("THREAD=%d\n:",thrd); - for(int operand_num=1;operand_num<=3;operand_num++){ - const operand_info &src_a= pI->operand_lookup(operand_num); - unsigned nelem = src_a.get_vect_nelem(); - ptx_reg_t v[8]; - thread->get_vector_operand_values( src_a, v, nelem ); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("Thread%d_Iteration=%d\n:", thrd, operand_num); - for(k = 0; k < nelem; k++){ - printf("%llx ",v[k].u64); - } - printf("\n"); - } - ptx_reg_t nw_v[16]; - int hex_val; - - if(!((operand_num==3)&&(type2==F32_TYPE))){ - for(k=0;k<2*nelem;k++){ - if(k%2==1) - hex_val=(v[k/2].s64&0xffff); - else - hex_val=((v[k/2].s64&0xffff0000)>>16); - nw_v[k].f16 =*((half *)&hex_val); - } - } - if(!((operand_num==3)&&(type2==F32_TYPE))){ - for(k=0;k<2*nelem;k++){ - temp=nw_v[k].f16; - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("%.2f ",temp); - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("\n"); - } - else{ - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - for(k=0;k<8;k++){ - printf("%.2f ",v[k].f32); - } - printf("\n"); - } - } - switch(operand_num) { - case 1 ://operand 1 - for(k=0;k<8;k++){ - mapping(thrd,LOAD_A,a_layout,F16_TYPE,k,16,row,col,offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("A:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); - matrix_a[row][col]=nw_v[offset]; - } - break; - case 2 ://operand 2 - for(k=0;k<8;k++){ - mapping(thrd,LOAD_B,b_layout,F16_TYPE,k,16,row,col,offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("B:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); - matrix_b[row][col]=nw_v[offset]; - } - break; - case 3 ://operand 3 - for(k=0;k<8;k++){ - mapping(thrd,LOAD_C,ROW,type2,k,16,row,col,offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("C:thread=%d,row=%d,col=%d,offset=%d\n",thrd,row,col,offset); - if(type2!=F16_TYPE){ - matrix_c[row][col]=v[offset]; - } - else { - matrix_c[row][col]=nw_v[offset]; - } - } - break; - default : - printf("Invalid Operand Index\n" ); - } - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("\n"); - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("MATRIX_A\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - temp=matrix_a[i][j].f16; - printf("%.2f ",temp); - } - printf("\n"); - } - printf("MATRIX_B\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - temp=matrix_b[i][j].f16; - printf("%.2f ",temp); - } - printf("\n"); - } - printf("MATRIX_C\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - if(type2==F16_TYPE){ - temp=matrix_c[i][j].f16; - printf("%.2f ",temp); - } - else - printf("%.2f ",matrix_c[i][j].f32); - } - printf("\n"); - } - } - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - matrix_d[i][j].f16=0; - } - } - - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - for(k=0;k<16;k++){ - matrix_d[i][j].f16=matrix_d[i][j].f16+matrix_a[i][k].f16*matrix_b[k][j].f16; - } - if((type==F16_TYPE)&&(type2==F16_TYPE)) - matrix_d[i][j].f16+=matrix_c[i][j].f16; - else if((type==F32_TYPE)&&(type2==F16_TYPE)){ - temp2=matrix_d[i][j].f16+matrix_c[i][j].f16; - temp=temp2; - matrix_d[i][j].f32=temp; - } - else if((type==F16_TYPE)&&(type2==F32_TYPE)){ - temp=matrix_d[i][j].f16; - temp+=matrix_c[i][j].f32; - matrix_d[i][j].f16=half(temp); - } - else{ - temp=matrix_d[i][j].f16; - temp+=matrix_c[i][j].f32; - matrix_d[i][j].f32=temp; - } - } - } - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("MATRIX_D\n"); - for (i=0;i<16;i++){ - for(j=0;j<16;j++){ - if(type==F16_TYPE){ - temp=matrix_d[i][j].f16; - printf("%.2f ",temp); - } - else - printf("%.2f ",matrix_d[i][j].f32); - } - printf("\n"); - } - } - for (thrd=0; thrd < core->get_warp_size(); thrd++){ - int row_t[8]; - int col_t[8]; - for(k=0;k<8;k++){ - mapping(thrd,LOAD_C,ROW,type,k,16,row_t[k],col_t[k],offset); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma:store:row:%d,col%d\n",row_t[k],col_t[k]); - } - thread = core->get_thread_info()[tid+thrd]; - - - if(type==F32_TYPE){ - thread->set_wmma_vector_operand_values(dst,matrix_d[row_t[0]][col_t[0]],matrix_d[row_t[1]][col_t[1]],matrix_d[row_t[2]][col_t[2]],matrix_d[row_t[3]][col_t[3]],matrix_d[row_t[4]][col_t[4]],matrix_d[row_t[5]][col_t[5]],matrix_d[row_t[6]][col_t[6]],matrix_d[row_t[7]][col_t[7]]); - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - { - printf("thread%d:",thrd); - for(k=0;k<8;k++){ - printf("%.2f ",matrix_d[row_t[k]][col_t[k]].f32); - } - printf("\n"); - } - } - else if(type==F16_TYPE){ - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("thread%d:",thrd); - for(k=0;k<8;k++){ - temp=matrix_d[row_t[k]][col_t[k]].f16; - printf("%.2f ",temp); - } - printf("\n"); - - printf("thread%d:",thrd); - for(k=0;k<8;k++){ - printf("%x ", (unsigned int)matrix_d[row_t[k]][col_t[k]].f16); - } - printf("\n"); - } - ptx_reg_t nw_data1, nw_data2, nw_data3, nw_data4; - nw_data1.s64=((matrix_d[row_t[0]][col_t[0]].s64 & 0xffff))|((matrix_d[row_t[1]][col_t[1]].s64&0xffff)<<16); - nw_data2.s64=((matrix_d[row_t[2]][col_t[2]].s64 & 0xffff))|((matrix_d[row_t[3]][col_t[3]].s64&0xffff)<<16); - nw_data3.s64=((matrix_d[row_t[4]][col_t[4]].s64 & 0xffff))|((matrix_d[row_t[5]][col_t[5]].s64&0xffff)<<16); - nw_data4.s64=((matrix_d[row_t[6]][col_t[6]].s64 & 0xffff))|((matrix_d[row_t[7]][col_t[7]].s64&0xffff)<<16); - thread->set_vector_operand_values(dst,nw_data1,nw_data2,nw_data3,nw_data4); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("thread%d=%llx,%llx,%llx,%llx", thrd, nw_data1.s64, nw_data2.s64, nw_data3.s64, nw_data4.s64); - - } - else{ - printf("wmma:mma:wrong type\n"); - abort(); - } - } -} - -void call_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - static unsigned call_uid_next = 1; - - const operand_info &target = pI->func_addr(); - assert( target.is_function_address() ); - const symbol *func_addr = target.get_symbol(); - function_info *target_func = func_addr->get_pc(); - if (target_func->is_pdom_set()) { - printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", target_func->get_name().c_str() ); - } else { - printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", target_func->get_name().c_str() ); - /* - * Some of the instructions like printf() gives the gpgpusim the wrong impression that it is a function call. - * As printf() doesnt have a body like functions do, doing pdom analysis for printf() causes a crash. - */ - if (target_func->get_function_size() >0) - target_func->do_pdom(); - target_func->set_pdom(); - } - - // check that number of args and return match function requirements - if( pI->has_return() ^ target_func->has_return() ) { - printf("GPGPU-Sim PTX: Execution error - mismatch in number of return values between\n" - " call instruction and function declaration\n"); - abort(); - } - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - unsigned n_operands = pI->get_num_operands(); - - if( n_operands != (n_return+1+n_args) ) { - printf("GPGPU-Sim PTX: Execution error - mismatch in number of arguements between\n" - " call instruction and function declaration\n"); - abort(); - } - - // handle intrinsic functions - std::string fname = target_func->get_name(); - if( fname == "vprintf" ) { - gpgpusim_cuda_vprintf(pI, thread, target_func); - return; - } +void call_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + static unsigned call_uid_next = 1; + + const operand_info &target = pI->func_addr(); + assert(target.is_function_address()); + const symbol *func_addr = target.get_symbol(); + function_info *target_func = func_addr->get_pc(); + if (target_func->is_pdom_set()) { + printf("GPGPU-Sim PTX: PDOM analysis already done for %s \n", + target_func->get_name().c_str()); + } else { + printf("GPGPU-Sim PTX: finding reconvergence points for \'%s\'...\n", + target_func->get_name().c_str()); + /* + * Some of the instructions like printf() gives the gpgpusim the wrong + * impression that it is a function call. As printf() doesnt have a body + * like functions do, doing pdom analysis for printf() causes a crash. + */ + if (target_func->get_function_size() > 0) target_func->do_pdom(); + target_func->set_pdom(); + } + + // check that number of args and return match function requirements + if (pI->has_return() ^ target_func->has_return()) { + printf( + "GPGPU-Sim PTX: Execution error - mismatch in number of return values " + "between\n" + " call instruction and function declaration\n"); + abort(); + } + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + unsigned n_operands = pI->get_num_operands(); + + if (n_operands != (n_return + 1 + n_args)) { + printf( + "GPGPU-Sim PTX: Execution error - mismatch in number of arguements " + "between\n" + " call instruction and function declaration\n"); + abort(); + } + // handle intrinsic functions + std::string fname = target_func->get_name(); + if (fname == "vprintf") { + gpgpusim_cuda_vprintf(pI, thread, target_func); + return; + } #if (CUDART_VERSION >= 5000) - //Jin: handle device runtime apis for CDP - else if(fname == "cudaGetParameterBufferV2") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_getParameterBufferV2(pI, thread, target_func); - return; - } - else if(fname == "cudaLaunchDeviceV2") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_launchDeviceV2(pI, thread, target_func); - return; - } - else if(fname == "cudaStreamCreateWithFlags") { - target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_streamCreateWithFlags(pI, thread, target_func); - return; - } + // Jin: handle device runtime apis for CDP + else if (fname == "cudaGetParameterBufferV2") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_getParameterBufferV2( + pI, thread, target_func); + return; + } else if (fname == "cudaLaunchDeviceV2") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_launchDeviceV2( + pI, thread, target_func); + return; + } else if (fname == "cudaStreamCreateWithFlags") { + target_func->gpgpu_ctx->device_runtime->gpgpusim_cuda_streamCreateWithFlags( + pI, thread, target_func); + return; + } #endif - // read source arguements into register specified in declaration of function - arg_buffer_list_t arg_values; - copy_args_into_buffer_list(pI, thread, target_func, arg_values); + // read source arguements into register specified in declaration of function + arg_buffer_list_t arg_values; + copy_args_into_buffer_list(pI, thread, target_func, arg_values); - // record local for return value (we only support a single return value) - const symbol *return_var_src = NULL; - const symbol *return_var_dst = NULL; - if( target_func->has_return() ) { - return_var_dst = pI->dst().get_symbol(); - return_var_src = target_func->get_return_var(); - } + // record local for return value (we only support a single return value) + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; + if (target_func->has_return()) { + return_var_dst = pI->dst().get_symbol(); + return_var_src = target_func->get_return_var(); + } - gpgpu_sim *gpu = thread->get_gpu(); - unsigned callee_pc=0, callee_rpc=0; - if( gpu->simd_model() == POST_DOMINATOR ) { - thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); - assert( callee_pc == thread->get_pc() ); - } + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc = 0, callee_rpc = 0; + if (gpu->simd_model() == POST_DOMINATOR) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(), + &callee_pc, &callee_rpc); + assert(callee_pc == thread->get_pc()); + } - thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); + thread->callstack_push(callee_pc + pI->inst_size(), callee_rpc, + return_var_src, return_var_dst, call_uid_next++); - copy_buffer_list_into_frame(thread, arg_values); + copy_buffer_list_into_frame(thread, arg_values); - thread->set_npc(target_func); + thread->set_npc(target_func); } -//Ptxplus version of call instruction. Jumps to a label not a different Kernel. -void callp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - - static unsigned call_uid_next = 1; +// Ptxplus version of call instruction. Jumps to a label not a different Kernel. +void callp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + static unsigned call_uid_next = 1; - const operand_info &target = pI->dst(); - ptx_reg_t target_pc = thread->get_operand_value(target, target, U32_TYPE, thread, 1); + const operand_info &target = pI->dst(); + ptx_reg_t target_pc = + thread->get_operand_value(target, target, U32_TYPE, thread, 1); - const symbol *return_var_src = NULL; - const symbol *return_var_dst = NULL; + const symbol *return_var_src = NULL; + const symbol *return_var_dst = NULL; - gpgpu_sim *gpu = thread->get_gpu(); - unsigned callee_pc=0, callee_rpc=0; - if( gpu->simd_model() == POST_DOMINATOR ) { - thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(),&callee_pc,&callee_rpc); - assert( callee_pc == thread->get_pc() ); - } + gpgpu_sim *gpu = thread->get_gpu(); + unsigned callee_pc = 0, callee_rpc = 0; + if (gpu->simd_model() == POST_DOMINATOR) { + thread->get_core()->get_pdom_stack_top_info(thread->get_hw_wid(), + &callee_pc, &callee_rpc); + assert(callee_pc == thread->get_pc()); + } - thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, return_var_src, return_var_dst, call_uid_next++); - thread->set_npc(target_pc); + thread->callstack_push_plus(callee_pc + pI->inst_size(), callee_rpc, + return_var_src, return_var_dst, call_uid_next++); + thread->set_npc(target_pc); } -void clz_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void clz_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - int max; - unsigned long long mask; - d.u64 = 0; + int max; + unsigned long long mask; + d.u64 = 0; - switch ( i_type ) { - case B32_TYPE: + switch (i_type) { + case B32_TYPE: max = 32; mask = 0x80000000; break; - case B64_TYPE: + case B64_TYPE: max = 64; mask = 0x8000000000000000; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - while ((d.u32 < max) && ((a.u64&mask) == 0) ) { - d.u32++; - a.u64 = a.u64 << 1; - } + while ((d.u32 < max) && ((a.u64 & mask) == 0)) { + d.u32++; + a.u64 = a.u64 << 1; + } - thread->set_operand_value(dst,d, B32_TYPE, thread, pI); + thread->set_operand_value(dst, d, B32_TYPE, thread, pI); } -void cnot_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void cnot_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case PRED_TYPE: d.pred = ((a.pred & 0x0001) == 0)?1:0; break; - case B16_TYPE: d.u16 = (a.u16 == 0)?1:0; break; - case B32_TYPE: d.u32 = (a.u32 == 0)?1:0; break; - case B64_TYPE: d.u64 = (a.u64 == 0)?1:0; break; - default: + switch (i_type) { + case PRED_TYPE: + d.pred = ((a.pred & 0x0001) == 0) ? 1 : 0; + break; + case B16_TYPE: + d.u16 = (a.u16 == 0) ? 1 : 0; + break; + case B32_TYPE: + d.u32 = (a.u32 == 0) ? 1 : 0; + break; + case B64_TYPE: + d.u64 = (a.u64 == 0) ? 1 : 0; + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void cos_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); +void cos_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: d.f32 = cos(a.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -ptx_reg_t chop( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - switch ( to_width ) { - case 8: x.mask_and(0,0xFF); break; - case 16: x.mask_and(0,0xFFFF); break; - case 32: x.mask_and(0,0xFFFFFFFF); break; - case 64: break; - default: assert(0); - } - return x; -} - -ptx_reg_t sext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - x=chop(x,0,from_width,0,rounding_mode,saturation_mode); - switch ( from_width ) { - case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; - case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; - case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; - case 64: break; - default: assert(0); - } - return x; -} - -// sign extend depending on the destination register size - hack to get SobelFilter working in CUDA 4.2 -ptx_reg_t sexd( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - x=chop(x,0,from_width,0,rounding_mode,saturation_mode); - switch ( to_width ) { - case 8: if ( x.get_bit(7) ) x.mask_or(0xFFFFFFFF,0xFFFFFF00);break; - case 16:if ( x.get_bit(15) ) x.mask_or(0xFFFFFFFF,0xFFFF0000);break; - case 32: if ( x.get_bit(31) ) x.mask_or(0xFFFFFFFF,0x00000000);break; - case 64: break; - default: assert(0); - } - return x; -} - -ptx_reg_t zext( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - return chop(x,0,from_width,0,rounding_mode,saturation_mode); -} - -int saturatei(int a, int max, int min) -{ - if (a > max) a = max; - else if (a < min) a = min; - return a; -} - -unsigned int saturatei(unsigned int a, unsigned int max) -{ - if (a > max) a = max; - return a; -} - -ptx_reg_t f2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - half mytemp; - half_float::half tmp_h; - //assert( from_width == 32); - - enum cudaRoundMode mode = cudaRoundZero; - switch (rounding_mode) { - case RZI_OPTION: mode = cudaRoundZero; break; - case RNI_OPTION: mode = cudaRoundNearest; break; - case RMI_OPTION: mode = cudaRoundMinInf; break; - case RPI_OPTION: mode = cudaRoundPosInf; break; - default: break; - } - - ptx_reg_t y; - if ( to_sign == 1 ) { // convert to 64-bit number first? - int tmp = cuda_math::float2int(x.f32, mode); - if ((x.u32 & 0x7f800000) == 0) - tmp = 0; // round denorm. FP to 0 - if (saturation_mode && to_width < 32) { - tmp = saturatei(tmp, (1<::round_style>(x.f32);//mytemp; - break; - case 32: - y.f32=float(x.f16); - break; // handled by f2f - case 64: - y.f64 = x.f32; - break; - default: assert(0); break; - } - } - return y; -} - -double saturated2i (double a, double max, double min) { - if (a > max) a = max; - else if (a < min) a = min; - return a; -} - -ptx_reg_t d2x( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - assert( from_width == 64); - - double tmp; - switch (rounding_mode) { - case RZI_OPTION: tmp = trunc(x.f64); break; - case RNI_OPTION: tmp = nearbyint(x.f64); break; - case RMI_OPTION: tmp = floor(x.f64); break; - case RPI_OPTION: tmp = ceil(x.f64); break; - default: tmp = x.f64; break; - } - - ptx_reg_t y; - if ( to_sign == 1 ) { - tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), (1<<(to_width - 1)) ); - switch ( to_width ) { - case 8: y.s8 = (char)tmp; break; - case 16: y.s16 = (short)tmp; break; - case 32: y.s32 = (int)tmp; break; - case 64: y.s64 = (long long)tmp; break; - default: assert(0); break; - } - } else if ( to_sign == 0 ) { - tmp = saturated2i(tmp, ((1<<(to_width - 1)) - 1), 0); - switch ( to_width ) { - case 8: y.u8 = (unsigned char)tmp; break; - case 16: y.u16 = (unsigned short)tmp; break; - case 32: y.u32 = (unsigned int)tmp; break; - case 64: y.u64 = (unsigned long long)tmp; break; - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: assert(0); break; - case 32: - y.f32 = x.f64; - break; - case 64: - y.f64 = x.f64; // should be handled by d2d - break; - default: assert(0); break; - } - } - return y; -} - -ptx_reg_t s2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - - if (from_width < 64) { // 32-bit conversion - y = sext(x,from_width,32,0,rounding_mode,saturation_mode); - - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__int2float_rz(y.s32); break; - case RN_OPTION: y.f32 = cuda_math::__int2float_rn(y.s32); break; - case RM_OPTION: y.f32 = cuda_math::__int2float_rd(y.s32); break; - case RP_OPTION: y.f32 = cuda_math::__int2float_ru(y.s32); break; - default: break; - } - break; - case 64: y.f64 = y.s32; break; // no rounding needed - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__ll2float_rz(y.s64); break; - case RN_OPTION: y.f32 = cuda_math::__ll2float_rn(y.s64); break; - case RM_OPTION: y.f32 = cuda_math::__ll2float_rd(y.s64); break; - case RP_OPTION: y.f32 = cuda_math::__ll2float_ru(y.s64); break; - default: break; - } - break; - case 64: y.f64 = y.s64; break; // no internal implementation found - default: assert(0); break; - } - } - - // saturating an integer to 1 or 0? - return y; -} - -ptx_reg_t u2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - - if (from_width < 64) { // 32-bit conversion - y = zext(x,from_width,32,0,rounding_mode,saturation_mode); - - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__uint2float_rz(y.u32); break; - case RN_OPTION: y.f32 = cuda_math::__uint2float_rn(y.u32); break; - case RM_OPTION: y.f32 = cuda_math::__uint2float_rd(y.u32); break; - case RP_OPTION: y.f32 = cuda_math::__uint2float_ru(y.u32); break; - default: break; - } - break; - case 64: y.f64 = y.u32; break; // no rounding needed - default: assert(0); break; - } - } else { - switch ( to_width ) { - case 16: assert(0); break; - case 32: - switch (rounding_mode) { - case RZ_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - case RN_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - case RM_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - case RP_OPTION: y.f32 = cuda_math::__ull2float_rn(y.u64); break; - default: break; - } - break; - case 64: y.f64 = y.u64; break; // no internal implementation found - default: assert(0); break; - } - } - - // saturating an integer to 1 or 0? - return y; -} - -ptx_reg_t f2f( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - if (from_width == 16){ - half_float::detail::uint16 val = x.u16; - y.f32 = half_float::detail::half2float(val); - }else{ - switch ( rounding_mode ) { - case RZI_OPTION: - y.f32 = truncf(x.f32); - break; - case RNI_OPTION: - #if CUDART_VERSION >= 3000 - y.f32 = nearbyintf(x.f32); - #else - y.f32 = cuda_math::__internal_nearbyintf(x.f32); - #endif - break; - case RMI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = floorf(x.f32); - } - break; - case RPI_OPTION: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = ceilf(x.f32); - } - break; - default: - if ((x.u32 & 0x7f800000) == 0) { - y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign - } else { - y.f32 = x.f32; - } - break; - } - #if CUDART_VERSION >= 3000 - if (isnanf(y.f32)) - #else - if (cuda_math::__cuda___isnanf(y.f32)) - #endif - { - y.u32 = 0x7fffffff; - } else if (saturation_mode) { - y.f32 = cuda_math::__saturatef(y.f32); - } - } - - return y; -} - -ptx_reg_t d2d( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, int rounding_mode, int saturation_mode ) -{ - ptx_reg_t y; - switch ( rounding_mode ) { - case RZI_OPTION: - y.f64 = trunc(x.f64); - break; - case RNI_OPTION: -#if CUDART_VERSION >= 3000 - y.f64 = nearbyint(x.f64); -#else - y.f64 = cuda_math::__internal_nearbyintf(x.f64); -#endif - break; - case RMI_OPTION: - y.f64 = floor(x.f64); - break; - case RPI_OPTION: - y.f64 = ceil(x.f64); - break; - default: - y.f64 = x.f64; - break; - } - if (std::isnan(y.f64)) { - y.u64 = 0xfff8000000000000ull; - } else if (saturation_mode) { - y.f64 = cuda_math::__saturatef(y.f64); - } - return y; -} - -ptx_reg_t (*g_cvt_fn[11][11])( ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, - int rounding_mode, int saturation_mode ) = { - { NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, - { chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, - { chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, - { chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, - { NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, - { chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, - { chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, - { chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, - { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , NULL,f2f, f2x}, - { f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x , f2x, f2f, f2x}, - { d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x , d2x, d2x, d2d} -}; + assert(0); + break; + } -void ptx_round(ptx_reg_t& data, int rounding_mode, int type) -{ - if (rounding_mode == RN_OPTION) { - return; - } - switch ( rounding_mode ) { - case RZI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: data.f16=truncf(data.f16);break;//assert(0); break; - case F32_TYPE: - data.f32 = truncf(data.f32); - break; - case F64_TYPE: - case FF64_TYPE: - if (data.f64 < 0) data.f64 = ceil(data.f64); //negative - else data.f64 = floor(data.f64); //positive - break; - default: assert(0); break; + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +ptx_reg_t chop(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + switch (to_width) { + case 8: + x.mask_and(0, 0xFF); + break; + case 16: + x.mask_and(0, 0xFFFF); + break; + case 32: + x.mask_and(0, 0xFFFFFFFF); + break; + case 64: + break; + default: + assert(0); + } + return x; +} + +ptx_reg_t sext(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + x = chop(x, 0, from_width, 0, rounding_mode, saturation_mode); + switch (from_width) { + case 8: + if (x.get_bit(7)) x.mask_or(0xFFFFFFFF, 0xFFFFFF00); + break; + case 16: + if (x.get_bit(15)) x.mask_or(0xFFFFFFFF, 0xFFFF0000); + break; + case 32: + if (x.get_bit(31)) x.mask_or(0xFFFFFFFF, 0x00000000); + break; + case 64: + break; + default: + assert(0); + } + return x; +} + +// sign extend depending on the destination register size - hack to get +// SobelFilter working in CUDA 4.2 +ptx_reg_t sexd(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + x = chop(x, 0, from_width, 0, rounding_mode, saturation_mode); + switch (to_width) { + case 8: + if (x.get_bit(7)) x.mask_or(0xFFFFFFFF, 0xFFFFFF00); + break; + case 16: + if (x.get_bit(15)) x.mask_or(0xFFFFFFFF, 0xFFFF0000); + break; + case 32: + if (x.get_bit(31)) x.mask_or(0xFFFFFFFF, 0x00000000); + break; + case 64: + break; + default: + assert(0); + } + return x; +} + +ptx_reg_t zext(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + return chop(x, 0, from_width, 0, rounding_mode, saturation_mode); +} + +int saturatei(int a, int max, int min) { + if (a > max) + a = max; + else if (a < min) + a = min; + return a; +} + +unsigned int saturatei(unsigned int a, unsigned int max) { + if (a > max) a = max; + return a; +} + +ptx_reg_t f2x(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + half mytemp; + half_float::half tmp_h; + // assert( from_width == 32); + + enum cudaRoundMode mode = cudaRoundZero; + switch (rounding_mode) { + case RZI_OPTION: + mode = cudaRoundZero; + break; + case RNI_OPTION: + mode = cudaRoundNearest; + break; + case RMI_OPTION: + mode = cudaRoundMinInf; + break; + case RPI_OPTION: + mode = cudaRoundPosInf; + break; + default: + break; + } + + ptx_reg_t y; + if (to_sign == 1) { // convert to 64-bit number first? + int tmp = cuda_math::float2int(x.f32, mode); + if ((x.u32 & 0x7f800000) == 0) tmp = 0; // round denorm. FP to 0 + if (saturation_mode && to_width < 32) { + tmp = saturatei(tmp, (1 << to_width) - 1, -(1 << to_width)); + } + switch (to_width) { + case 8: + y.s8 = (char)tmp; + break; + case 16: + y.s16 = (short)tmp; + break; + case 32: + y.s32 = (int)tmp; + break; + case 64: + y.s64 = (long long)tmp; + break; + default: + assert(0); + break; + } + } else if (to_sign == 0) { + unsigned int tmp = cuda_math::float2uint(x.f32, mode); + if ((x.u32 & 0x7f800000) == 0) tmp = 0; // round denorm. FP to 0 + if (saturation_mode && to_width < 32) { + tmp = saturatei(tmp, (1 << to_width) - 1); + } + switch (to_width) { + case 8: + y.u8 = (unsigned char)tmp; + break; + case 16: + y.u16 = (unsigned short)tmp; + break; + case 32: + y.u32 = (unsigned int)tmp; + break; + case 64: + y.u64 = (unsigned long long)tmp; + break; + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + y.f16 = half_float::half_cast::round_style>( + x.f32); // mytemp; + break; + case 32: + y.f32 = float(x.f16); + break; // handled by f2f + case 64: + y.f64 = x.f32; + break; + default: + assert(0); + break; + } + } + return y; +} + +double saturated2i(double a, double max, double min) { + if (a > max) + a = max; + else if (a < min) + a = min; + return a; +} + +ptx_reg_t d2x(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + assert(from_width == 64); + + double tmp; + switch (rounding_mode) { + case RZI_OPTION: + tmp = trunc(x.f64); + break; + case RNI_OPTION: + tmp = nearbyint(x.f64); + break; + case RMI_OPTION: + tmp = floor(x.f64); + break; + case RPI_OPTION: + tmp = ceil(x.f64); + break; + default: + tmp = x.f64; + break; + } + + ptx_reg_t y; + if (to_sign == 1) { + tmp = saturated2i(tmp, ((1 << (to_width - 1)) - 1), (1 << (to_width - 1))); + switch (to_width) { + case 8: + y.s8 = (char)tmp; + break; + case 16: + y.s16 = (short)tmp; + break; + case 32: + y.s32 = (int)tmp; + break; + case 64: + y.s64 = (long long)tmp; + break; + default: + assert(0); + break; + } + } else if (to_sign == 0) { + tmp = saturated2i(tmp, ((1 << (to_width - 1)) - 1), 0); + switch (to_width) { + case 8: + y.u8 = (unsigned char)tmp; + break; + case 16: + y.u16 = (unsigned short)tmp; + break; + case 32: + y.u32 = (unsigned int)tmp; + break; + case 64: + y.u64 = (unsigned long long)tmp; + break; + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + assert(0); + break; + case 32: + y.f32 = x.f64; + break; + case 64: + y.f64 = x.f64; // should be handled by d2d + break; + default: + assert(0); + break; + } + } + return y; +} + +ptx_reg_t s2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + + if (from_width < 64) { // 32-bit conversion + y = sext(x, from_width, 32, 0, rounding_mode, saturation_mode); + + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__int2float_rz(y.s32); + break; + case RN_OPTION: + y.f32 = cuda_math::__int2float_rn(y.s32); + break; + case RM_OPTION: + y.f32 = cuda_math::__int2float_rd(y.s32); + break; + case RP_OPTION: + y.f32 = cuda_math::__int2float_ru(y.s32); + break; + default: + break; + } + break; + case 64: + y.f64 = y.s32; + break; // no rounding needed + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__ll2float_rz(y.s64); + break; + case RN_OPTION: + y.f32 = cuda_math::__ll2float_rn(y.s64); + break; + case RM_OPTION: + y.f32 = cuda_math::__ll2float_rd(y.s64); + break; + case RP_OPTION: + y.f32 = cuda_math::__ll2float_ru(y.s64); + break; + default: + break; + } + break; + case 64: + y.f64 = y.s64; + break; // no internal implementation found + default: + assert(0); + break; + } + } + + // saturating an integer to 1 or 0? + return y; +} + +ptx_reg_t u2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + + if (from_width < 64) { // 32-bit conversion + y = zext(x, from_width, 32, 0, rounding_mode, saturation_mode); + + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__uint2float_rz(y.u32); + break; + case RN_OPTION: + y.f32 = cuda_math::__uint2float_rn(y.u32); + break; + case RM_OPTION: + y.f32 = cuda_math::__uint2float_rd(y.u32); + break; + case RP_OPTION: + y.f32 = cuda_math::__uint2float_ru(y.u32); + break; + default: + break; + } + break; + case 64: + y.f64 = y.u32; + break; // no rounding needed + default: + assert(0); + break; + } + } else { + switch (to_width) { + case 16: + assert(0); + break; + case 32: + switch (rounding_mode) { + case RZ_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + case RN_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + case RM_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + case RP_OPTION: + y.f32 = cuda_math::__ull2float_rn(y.u64); + break; + default: + break; + } + break; + case 64: + y.f64 = y.u64; + break; // no internal implementation found + default: + assert(0); + break; + } + } + + // saturating an integer to 1 or 0? + return y; +} + +ptx_reg_t f2f(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + if (from_width == 16) { + half_float::detail::uint16 val = x.u16; + y.f32 = half_float::detail::half2float(val); + } else { + switch (rounding_mode) { + case RZI_OPTION: + y.f32 = truncf(x.f32); + break; + case RNI_OPTION: +#if CUDART_VERSION >= 3000 + y.f32 = nearbyintf(x.f32); +#else + y.f32 = cuda_math::__internal_nearbyintf(x.f32); +#endif + break; + case RMI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = floorf(x.f32); + } + break; + case RPI_OPTION: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = ceilf(x.f32); + } + break; + default: + if ((x.u32 & 0x7f800000) == 0) { + y.u32 = x.u32 & 0x80000000; // round denorm. FP to 0, keeping sign + } else { + y.f32 = x.f32; + } + break; + } +#if CUDART_VERSION >= 3000 + if (isnanf(y.f32)) +#else + if (cuda_math::__cuda___isnanf(y.f32)) +#endif + { + y.u32 = 0x7fffffff; + } else if (saturation_mode) { + y.f32 = cuda_math::__saturatef(y.f32); + } + } + + return y; +} + +ptx_reg_t d2d(ptx_reg_t x, unsigned from_width, unsigned to_width, int to_sign, + int rounding_mode, int saturation_mode) { + ptx_reg_t y; + switch (rounding_mode) { + case RZI_OPTION: + y.f64 = trunc(x.f64); + break; + case RNI_OPTION: +#if CUDART_VERSION >= 3000 + y.f64 = nearbyint(x.f64); +#else + y.f64 = cuda_math::__internal_nearbyintf(x.f64); +#endif + break; + case RMI_OPTION: + y.f64 = floor(x.f64); + break; + case RPI_OPTION: + y.f64 = ceil(x.f64); + break; + default: + y.f64 = x.f64; + break; + } + if (std::isnan(y.f64)) { + y.u64 = 0xfff8000000000000ull; + } else if (saturation_mode) { + y.f64 = cuda_math::__saturatef(y.f64); + } + return y; +} + +ptx_reg_t (*g_cvt_fn[11][11])(ptx_reg_t x, unsigned from_width, + unsigned to_width, int to_sign, int rounding_mode, + int saturation_mode) = { + {NULL, sext, sext, sext, NULL, sext, sext, sext, s2f, s2f, s2f}, + {chop, NULL, sext, sext, chop, NULL, sext, sext, s2f, s2f, s2f}, + {chop, sexd, NULL, sext, chop, chop, NULL, sext, s2f, s2f, s2f}, + {chop, chop, chop, NULL, chop, chop, chop, NULL, s2f, s2f, s2f}, + {NULL, zext, zext, zext, NULL, zext, zext, zext, u2f, u2f, u2f}, + {chop, NULL, zext, zext, chop, NULL, zext, zext, u2f, u2f, u2f}, + {chop, chop, NULL, zext, chop, chop, NULL, zext, u2f, u2f, u2f}, + {chop, chop, chop, NULL, chop, chop, chop, NULL, u2f, u2f, u2f}, + {f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, NULL, f2f, f2x}, + {f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2x, f2f, f2x}, + {d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2x, d2d}}; + +void ptx_round(ptx_reg_t &data, int rounding_mode, int type) { + if (rounding_mode == RN_OPTION) { + return; + } + switch (rounding_mode) { + case RZI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: + data.f16 = truncf(data.f16); + break; // assert(0); break; + case F32_TYPE: + data.f32 = truncf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 < 0) + data.f64 = ceil(data.f64); // negative + else + data.f64 = floor(data.f64); // positive + break; + default: + assert(0); + break; } break; - case RNI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE:// assert(0); break; + case RNI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: // assert(0); break; #if CUDART_VERSION >= 3000 - data.f16 = nearbyintf(data.f16); + data.f16 = nearbyintf(data.f16); #else - data.f16 = cuda_math::__cuda_nearbyintf(data.f16); + data.f16 = cuda_math::__cuda_nearbyintf(data.f16); #endif - break; - case F32_TYPE: + break; + case F32_TYPE: #if CUDART_VERSION >= 3000 - data.f32 = nearbyintf(data.f32); + data.f32 = nearbyintf(data.f32); #else - data.f32 = cuda_math::__cuda_nearbyintf(data.f32); + data.f32 = cuda_math::__cuda_nearbyintf(data.f32); #endif - break; - case F64_TYPE: case FF64_TYPE: data.f64 = round(data.f64); break; - default: assert(0); break; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = round(data.f64); + break; + default: + assert(0); + break; } break; - case RMI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: data.f16=floorf(data.f16);break;//assert(0); break; - case F32_TYPE: - data.f32 = floorf(data.f32); - break; - case F64_TYPE: case FF64_TYPE: data.f64 = floor(data.f64); break; - default: assert(0); break; + case RMI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: + data.f16 = floorf(data.f16); + break; // assert(0); break; + case F32_TYPE: + data.f32 = floorf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = floor(data.f64); + break; + default: + assert(0); + break; } break; - case RPI_OPTION: - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to round an integer??\n"); assert(0); break; - case F16_TYPE: data.f16 = ceilf(data.f16); break; //assert(0); break; - case F32_TYPE: data.f32 = ceilf(data.f32); break; - case F64_TYPE: case FF64_TYPE: data.f64 = ceil(data.f64); break; - default: assert(0); break; + case RPI_OPTION: + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to round an integer??\n"); + assert(0); + break; + case F16_TYPE: + data.f16 = ceilf(data.f16); + break; // assert(0); break; + case F32_TYPE: + data.f32 = ceilf(data.f32); + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = ceil(data.f64); + break; + default: + assert(0); + break; } break; - default: break; - } + default: + break; + } - if (type == F32_TYPE) { + if (type == F32_TYPE) { #if CUDART_VERSION >= 3000 - if (isnanf(data.f32)) + if (isnanf(data.f32)) #else - if (cuda_math::__cuda___isnanf(data.f32)) + if (cuda_math::__cuda___isnanf(data.f32)) #endif - { - data.u32 = 0x7fffffff; - } - } - if ((type == F64_TYPE)||(type == FF64_TYPE)) { - if (std::isnan(data.f64)) { - data.u64 = 0xfff8000000000000ull; - } - } + { + data.u32 = 0x7fffffff; + } + } + if ((type == F64_TYPE) || (type == FF64_TYPE)) { + if (std::isnan(data.f64)) { + data.u64 = 0xfff8000000000000ull; + } + } } -void ptx_saturate(ptx_reg_t& data, int saturation_mode, int type) -{ - if (!saturation_mode) { - return; - } - switch ( type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - printf("Trying to clamp an integer to 1??\n"); assert(0); break; - case F16_TYPE: //assert(0); break; - if (data.f16 > 1.0f) data.f16 = 1.0f; //negative - if (data.f16 < 0.0f) data.f16 = 0.0f; //positive - break; - case F32_TYPE: - if (data.f32 > 1.0f) data.f32 = 1.0f; //negative - if (data.f32 < 0.0f) data.f32 = 0.0f; //positive - break; - case F64_TYPE: - case FF64_TYPE: - if (data.f64 > 1.0f) data.f64 = 1.0f; //negative - if (data.f64 < 0.0f) data.f64 = 0.0f; //positive - break; - default: assert(0); break; - } - -} - -void cvt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned to_type = pI->get_type(); - unsigned from_type = pI->get_type2(); - unsigned rounding_mode = pI->rounding_mode(); - unsigned saturation_mode = pI->saturation_mode(); - -// if ( to_type == F16_TYPE || from_type == F16_TYPE ) -// abort(); - - int to_sign, from_sign; - size_t from_width, to_width; - unsigned src_fmt = type_info_key::type_decode(from_type, from_width, from_sign); - unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); - - ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); - - if(pI->is_neg()){ - - switch( from_type ) { +void ptx_saturate(ptx_reg_t &data, int saturation_mode, int type) { + if (!saturation_mode) { + return; + } + switch (type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + printf("Trying to clamp an integer to 1??\n"); + assert(0); + break; + case F16_TYPE: // assert(0); break; + if (data.f16 > 1.0f) data.f16 = 1.0f; // negative + if (data.f16 < 0.0f) data.f16 = 0.0f; // positive + break; + case F32_TYPE: + if (data.f32 > 1.0f) data.f32 = 1.0f; // negative + if (data.f32 < 0.0f) data.f32 = 0.0f; // positive + break; + case F64_TYPE: + case FF64_TYPE: + if (data.f64 > 1.0f) data.f64 = 1.0f; // negative + if (data.f64 < 0.0f) data.f64 = 0.0f; // positive + break; + default: + assert(0); + break; + } +} + +void cvt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned to_type = pI->get_type(); + unsigned from_type = pI->get_type2(); + unsigned rounding_mode = pI->rounding_mode(); + unsigned saturation_mode = pI->saturation_mode(); + + // if ( to_type == F16_TYPE || from_type == F16_TYPE ) + // abort(); + + int to_sign, from_sign; + size_t from_width, to_width; + unsigned src_fmt = + type_info_key::type_decode(from_type, from_width, from_sign); + unsigned dst_fmt = type_info_key::type_decode(to_type, to_width, to_sign); + + ptx_reg_t data = thread->get_operand_value(src1, dst, from_type, thread, 1); + + if (pI->is_neg()) { + switch (from_type) { // Default to f32 for now, need to add support for others case S8_TYPE: case U8_TYPE: case B8_TYPE: - data.s8 = -data.s8; - break; + data.s8 = -data.s8; + break; case S16_TYPE: case U16_TYPE: case B16_TYPE: - data.s16 = -data.s16; - break; + data.s16 = -data.s16; + break; case S32_TYPE: case U32_TYPE: case B32_TYPE: - data.s32 = -data.s32; - break; + data.s32 = -data.s32; + break; case S64_TYPE: case U64_TYPE: case B64_TYPE: - data.s64 = -data.s64; - break; + data.s64 = -data.s64; + break; case F16_TYPE: - data.f16 = -data.f16; - break; + data.f16 = -data.f16; + break; case F32_TYPE: - data.f32 = -data.f32; - break; + data.f32 = -data.f32; + break; case F64_TYPE: case FF64_TYPE: - data.f64 = -data.f64; - break; + data.f64 = -data.f64; + break; default: - assert(0); + assert(0); + } + } + + if (g_cvt_fn[src_fmt][dst_fmt] != NULL) { + ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt]( + data, from_width, to_width, to_sign, rounding_mode, saturation_mode); + data = result; + } + + thread->set_operand_value(dst, data, to_type, thread, pI); +} + +void cvta_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); + bool to_non_generic = pI->is_to(); + + unsigned i_type = pI->get_type(); + ptx_reg_t from_addr = thread->get_operand_value(src1, dst, i_type, thread, 1); + addr_t from_addr_hw = (addr_t)from_addr.u64; + addr_t to_addr_hw = 0; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); + + if (to_non_generic) { + switch (space.get_type()) { + case shared_space: + to_addr_hw = generic_to_shared(smid, from_addr_hw); + break; + case local_space: + to_addr_hw = generic_to_local(smid, hwtid, from_addr_hw); + break; + case global_space: + to_addr_hw = generic_to_global(from_addr_hw); + break; + default: + abort(); + } + } else { + switch (space.get_type()) { + case shared_space: + to_addr_hw = shared_to_generic(smid, from_addr_hw); + break; + case local_space: + to_addr_hw = local_to_generic(smid, hwtid, from_addr_hw) + + thread->get_local_mem_stack_pointer(); + break; // add stack ptr here so that it can be passed as a pointer at + // function call + case global_space: + to_addr_hw = global_to_generic(from_addr_hw); + break; + default: + abort(); + } + } + + ptx_reg_t to_addr; + to_addr.u64 = to_addr_hw; + thread->set_reg(dst.get_symbol(), to_addr); +} + +void div_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + switch (i_type) { + case S8_TYPE: + data.s8 = src1_data.s8 / src2_data.s8; + break; + case S16_TYPE: + data.s16 = src1_data.s16 / src2_data.s16; + break; + case S32_TYPE: + data.s32 = src1_data.s32 / src2_data.s32; + break; + case S64_TYPE: + data.s64 = src1_data.s64 / src2_data.s64; + break; + case U8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8; + break; + case U16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; + break; + case U32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; + break; + case U64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; + break; + case B8_TYPE: + data.u8 = src1_data.u8 / src2_data.u8; + break; + case B16_TYPE: + data.u16 = src1_data.u16 / src2_data.u16; + break; + case B32_TYPE: + data.u32 = src1_data.u32 / src2_data.u32; + break; + case B64_TYPE: + data.u64 = src1_data.u64 / src2_data.u64; + break; + case F16_TYPE: + data.f16 = src1_data.f16 / src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 / src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 / src2_data.f64; + break; + default: + assert(0); + break; + } + thread->set_operand_value(dst, data, i_type, thread, pI); +} + +void dp4a_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + printf("DP4A instruction not implemented yet"); + assert(0); +} + +void ex2_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + data.f32 = cuda_math::__powf(2.0, src1_data.f32); + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst, data, i_type, thread, pI); +} + +void exit_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); +} + +void mad_def(const ptx_instruction *pI, ptx_thread_info *thread, + bool use_carry = false); + +void fma_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread); +} + +void isspacep_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a; + bool t = false; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + memory_space_t space = pI->get_space(); + + a = thread->get_reg(src1.get_symbol()); + addr_t addr = (addr_t)a.u64; + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); + + switch (space.get_type()) { + case shared_space: + t = isspace_shared(smid, addr); + case local_space: + t = isspace_local(smid, hwtid, addr); + case global_space: + t = isspace_global(addr); + default: + abort(); + } + + ptx_reg_t p; + p.pred = t ? 1 : 0; + + thread->set_reg(dst.get_symbol(), p); +} + +void decode_space(memory_space_t &space, ptx_thread_info *thread, + const operand_info &op, memory_space *&mem, addr_t &addr) { + unsigned smid = thread->get_hw_sid(); + unsigned hwtid = thread->get_hw_tid(); + + if (space == param_space_unclassified) { + // need to op to determine whether it refers to a kernel param or local + // param + const symbol *s = op.get_symbol(); + const type_info *t = s->type(); + type_info_key ti = t->get_key(); + if (ti.is_param_kernel()) + space = param_space_kernel; + else if (ti.is_param_local()) { + space = param_space_local; + } + // mov r1, param-label + else if (ti.is_reg()) { + space = param_space_kernel; + } else { + printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", + s->name().c_str()); + abort(); + } + } + switch (space.get_type()) { + case global_space: + mem = thread->get_global_memory(); + break; + case param_space_local: + case local_space: + mem = thread->m_local_mem; + addr += thread->get_local_mem_stack_pointer(); + break; + case tex_space: + mem = thread->get_tex_memory(); + break; + case surf_space: + mem = thread->get_surf_memory(); + break; + case param_space_kernel: + mem = thread->get_param_memory(); + break; + case shared_space: + mem = thread->m_shared_mem; + break; + case sstarr_space: + mem = thread->m_sstarr_mem; + break; + case const_space: + mem = thread->get_global_memory(); + break; + case generic_space: + if (thread->get_ptx_version().ver() >= 2.0) { + // convert generic address to memory space address + space = whichspace(addr); + switch (space.get_type()) { + case global_space: + mem = thread->get_global_memory(); + addr = generic_to_global(addr); + break; + case local_space: + mem = thread->m_local_mem; + addr = generic_to_local(smid, hwtid, addr); + break; + case shared_space: + mem = thread->m_shared_mem; + addr = generic_to_shared(smid, addr); + break; + default: + abort(); + } + } else { + abort(); } + break; + case param_space_unclassified: + case undefined_space: + default: + abort(); + } +} + +void ld_exec(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned type = pI->get_type(); + + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); + + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + + decode_space(space, thread, src1, mem, addr); + + size_t size; + int t; + data.u64 = 0; + type_info_key::type_decode(type, size, t); + if (!vector_spec) { + mem->read(addr, size / 8, &data.s64); + if (type == S16_TYPE || type == S32_TYPE) sign_extend(data, size, dst); + thread->set_operand_value(dst, data, type, thread, pI); + } else { + ptx_reg_t data1, data2, data3, data4; + mem->read(addr, size / 8, &data1.s64); + mem->read(addr + size / 8, size / 8, &data2.s64); + if (vector_spec != V2_TYPE) { // either V3 or V4 + mem->read(addr + 2 * size / 8, size / 8, &data3.s64); + if (vector_spec != V3_TYPE) { // v4 + mem->read(addr + 3 * size / 8, size / 8, &data4.s64); + thread->set_vector_operand_values(dst, data1, data2, data3, data4); + } else // v3 + thread->set_vector_operand_values(dst, data1, data2, data3, data3); + } else // v2 + thread->set_vector_operand_values(dst, data1, data2, data2, data2); + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; +} - } +void ld_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ld_exec(pI, thread); +} +void ldu_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ld_exec(pI, thread); +} +void mma_st_impl(const ptx_instruction *pI, core_t *core, warp_inst_t &inst) { + size_t size; + unsigned smid; + int t; + int thrd, k; + ptx_thread_info *thread; + + const operand_info &src = pI->operand_lookup(1); + const operand_info &src1 = pI->operand_lookup(0); + const operand_info &src2 = pI->operand_lookup(2); + int tid; + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int stride; + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + + _memory_op_t insn_memory_op = + pI->has_memory_read() ? memory_load : memory_store; + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid + thrd]; + ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); + const operand_info &src_a = pI->operand_lookup(1); + unsigned nelem = src_a.get_vect_nelem(); + ptx_reg_t *v = new ptx_reg_t[8]; + thread->get_vector_operand_values(src_a, v, nelem); + stride = src2_data.u32; - if ( g_cvt_fn[src_fmt][dst_fmt] != NULL ) { - ptx_reg_t result = g_cvt_fn[src_fmt][dst_fmt](data,from_width,to_width,to_sign, rounding_mode, saturation_mode); - data = result; - } + memory_space_t space = pI->get_space(); - thread->set_operand_value(dst, data, to_type, thread, pI ); -} + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; -void cvta_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn = 0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - memory_space_t space = pI->get_space(); - bool to_non_generic = pI->is_to(); + smid = thread->get_hw_sid(); + if (whichspace(addr) == shared_space) { + addr = generic_to_shared(smid, addr); + space = shared_space; + } + decode_space(space, thread, src1, mem, addr); + + type_info_key::type_decode(type, size, t); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma_st: thrd=%d, addr=%x, fp(size=%zu), stride=%d\n", thrd, + addr_reg.u32, size, src2_data.u32); + addr_t new_addr = + addr + thread_group_offset(thrd, wmma_type, wmma_layout, type, stride) * + size / 8; + addr_t push_addr; + + ptx_reg_t nw_v[8]; + for (k = 0; k < 8; k++) { + if (k % 2 == 0) + nw_v[k].s64 = (v[k / 2].s64 & 0xffff); + else + nw_v[k].s64 = ((v[k / 2].s64 & 0xffff0000) >> 16); + } - unsigned i_type = pI->get_type(); - ptx_reg_t from_addr = thread->get_operand_value(src1,dst,i_type,thread,1); - addr_t from_addr_hw = (addr_t)from_addr.u64; - addr_t to_addr_hw = 0; - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); + for (k = 0; k < 8; k++) { + if (type == F32_TYPE) { + // mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); + push_addr = new_addr + 4 * acc_float_offset(k, wmma_layout, stride); + mem->write(push_addr, size / 8, &v[k].s64, thread, pI); + mem_txn_addr[num_mem_txn++] = push_addr; + + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf( + "wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n", + thrd, v[0].s64, v[1].s64, v[2].s64, v[3].s64, v[4].s64, v[5].s64, + v[6].s64, v[7].s64); + float temp; + int l; + printf("thread=%d:", thrd); + for (l = 0; l < 8; l++) { + temp = v[l].f32; + printf("%.2f", temp); + } + printf("\n"); + } + } else if (type == F16_TYPE) { + if (wmma_layout == ROW) { + // mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); + push_addr = new_addr + k * 2; + mem->write(push_addr, size / 8, &nw_v[k].s64, thread, pI); + if (k % 2 == 0) mem_txn_addr[num_mem_txn++] = push_addr; + } else if (wmma_layout == COL) { + // mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); + push_addr = new_addr + k * 2 * stride; + mem->write(push_addr, size / 8, &nw_v[k].s64, thread, pI); + mem_txn_addr[num_mem_txn++] = push_addr; + } - if( to_non_generic ) { - switch( space.get_type() ) { - case shared_space: to_addr_hw = generic_to_shared( smid, from_addr_hw ); break; - case local_space: to_addr_hw = generic_to_local( smid, hwtid, from_addr_hw ); break; - case global_space: to_addr_hw = generic_to_global(from_addr_hw ); break; - default: abort(); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf( + "wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n", + thrd, nw_v[0].s64, nw_v[1].s64, nw_v[2].s64, nw_v[3].s64, + nw_v[4].s64, nw_v[5].s64, nw_v[6].s64, nw_v[7].s64); } - } else { - switch( space.get_type() ) { - case shared_space: to_addr_hw = shared_to_generic( smid, from_addr_hw ); break; - case local_space: to_addr_hw = local_to_generic( smid, hwtid, from_addr_hw ) - + thread->get_local_mem_stack_pointer(); break; // add stack ptr here so that it can be passed as a pointer at function call - case global_space: to_addr_hw = global_to_generic( from_addr_hw ); break; - default: abort(); - } - } - - ptx_reg_t to_addr; - to_addr.u64 = to_addr_hw; - thread->set_reg(dst.get_symbol(),to_addr); -} - -void div_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - - switch ( i_type ) { - case S8_TYPE: - data.s8 = src1_data.s8 / src2_data.s8 ; break; - case S16_TYPE: - data.s16 = src1_data.s16 / src2_data.s16; break; - case S32_TYPE: - data.s32 = src1_data.s32 / src2_data.s32; break; - case S64_TYPE: - data.s64 = src1_data.s64 / src2_data.s64; break; - case U8_TYPE: - data.u8 = src1_data.u8 / src2_data.u8 ; break; - case U16_TYPE: - data.u16 = src1_data.u16 / src2_data.u16; break; - case U32_TYPE: - data.u32 = src1_data.u32 / src2_data.u32; break; - case U64_TYPE: - data.u64 = src1_data.u64 / src2_data.u64; break; - case B8_TYPE: - data.u8 = src1_data.u8 / src2_data.u8 ; break; - case B16_TYPE: - data.u16 = src1_data.u16 / src2_data.u16; break; - case B32_TYPE: - data.u32 = src1_data.u32 / src2_data.u32; break; - case B64_TYPE: - data.u64 = src1_data.u64 / src2_data.u64; break; - case F16_TYPE: data.f16 = src1_data.f16 / src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 / src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 / src2_data.f64; break; - default: assert(0); break; - } - thread->set_operand_value(dst,data, i_type, thread,pI); -} - -void dp4a_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - printf("DP4A instruction not implemented yet"); - assert(0); - -} - -void ex2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: - data.f32 = cuda_math::__powf(2.0, src1_data.f32); - break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst,data, i_type, thread,pI); -} + } -void exit_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - thread->set_done(); - thread->exitCore(); - thread->registerExit(); -} + delete[] v; + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr, num_mem_txn); -void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry = false ); + if ((type == F16_TYPE) && + (wmma_layout == COL)) // check the profiling xls for details + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction -void fma_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI,thread); + assert(inst.memory_op == insn_memory_op); + // thread->m_last_effective_address = addr; + // thread->m_last_memory_space = space; + } } -void isspacep_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a; - bool t=false; +void mma_ld_impl(const ptx_instruction *pI, core_t *core, warp_inst_t &inst) { + size_t size; + int t, i; + unsigned smid; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned type = pI->get_type(); + unsigned wmma_type = pI->get_wmma_type(); + unsigned wmma_layout = pI->get_wmma_layout(0); + int tid; + int thrd, stride; + ptx_thread_info *thread; + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + + _memory_op_t insn_memory_op = + pI->has_memory_read() ? memory_load : memory_store; + + for (thrd = 0; thrd < core->get_warp_size(); thrd++) { + thread = core->get_thread_info()[tid + thrd]; + ptx_reg_t src1_data = + thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); + ptx_reg_t src2_data = + thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); + stride = src2_data.u32; + memory_space_t space = pI->get_space(); + + memory_space *mem = NULL; + addr_t addr = src1_data.u32; + smid = thread->get_hw_sid(); + if (whichspace(addr) == shared_space) { + addr = generic_to_shared(smid, addr); + space = shared_space; + } + + decode_space(space, thread, src1, mem, addr); + type_info_key::type_decode(type, size, t); + + ptx_reg_t data[16]; + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) + printf("mma_ld: thrd=%d,addr=%x, fpsize=%zu, stride=%d\n", thrd, + src1_data.u32, size, src2_data.u32); + + addr_t new_addr = + addr + thread_group_offset(thrd, wmma_type, wmma_layout, type, stride) * + size / 8; + addr_t fetch_addr; + new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; + int num_mem_txn = 0; + + if (wmma_type == LOAD_A) { + for (i = 0; i < 16; i++) { + if (wmma_layout == ROW) { + // mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + } else if (wmma_layout == COL) { + // mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr = new_addr + 2 * (i % 4) + 2 * stride * 4 * (i / 4); + mem->read(fetch_addr, size / 8, &data[i].s64); + } else { + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; + } + } else if (wmma_type == LOAD_B) { + for (i = 0; i < 16; i++) { + if (wmma_layout == COL) { + // mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + } else if (wmma_layout == ROW) { + // mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); + fetch_addr = new_addr + 2 * (i % 4) + 2 * stride * 4 * (i / 4); + mem->read(fetch_addr, size / 8, &data[i].s64); + } else { + printf("mma_ld:wrong_layout_type\n"); + abort(); + } + if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; + } + } else if (wmma_type == LOAD_C) { + for (i = 0; i < 8; i++) { + if (type == F16_TYPE) { + if (wmma_layout == ROW) { + // mem->read(new_addr+2*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + if (i % 2 == 0) mem_txn_addr[num_mem_txn++] = fetch_addr; + } else if (wmma_layout == COL) { + // mem->read(new_addr+2*stride*i,size/8,&data[i].s64); + fetch_addr = new_addr + 2 * stride * i; + mem->read(fetch_addr, size / 8, &data[i].s64); + mem_txn_addr[num_mem_txn++] = fetch_addr; + } else { + printf("mma_ld:wrong_type\n"); + abort(); + } + } else if (type == F32_TYPE) { + // mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); + fetch_addr = new_addr + 4 * acc_float_offset(i, wmma_layout, stride); + mem->read(fetch_addr, size / 8, &data[i].s64); + mem_txn_addr[num_mem_txn++] = fetch_addr; + } else { + printf("wrong type"); + abort(); + } + } + } else { + printf("wrong wmma type\n"); + ; + abort(); + } + // generate timing memory request + inst.space = space; + inst.set_addr(thrd, (new_addr_type *)mem_txn_addr, num_mem_txn); + + if ((wmma_type == LOAD_C) && (type == F16_TYPE) && + (wmma_layout == COL)) // memory address is scattered, check the + // profiling xls for more detail. + inst.data_size = 2; // 2 byte transaction + else + inst.data_size = 4; // 4 byte transaction + assert(inst.memory_op == insn_memory_op); + + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + if (type == F16_TYPE) { + printf("\nmma_ld:thread%d= ", thrd); + for (i = 0; i < 16; i++) { + printf("%llx ", data[i].u64); + } + printf("\n"); + + printf("\nmma_ld:thread%d= ", thrd); + float temp; + for (i = 0; i < 16; i++) { + temp = data[i].f16; + printf("%.2f ", temp); + } + printf("\n"); + } else { + printf("\nmma_ld:thread%d= ", thrd); + for (i = 0; i < 8; i++) { + printf("%.2f ", data[i].f32); + } + printf("\n"); + printf("\nmma_ld:thread%d= ", thrd); + for (i = 0; i < 8; i++) { + printf("%llx ", data[i].u64); + } + printf("\n"); + } + } - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - memory_space_t space = pI->get_space(); + if ((wmma_type == LOAD_C) && (type == F32_TYPE)) { + thread->set_wmma_vector_operand_values(dst, data[0], data[1], data[2], + data[3], data[4], data[5], data[6], + data[7]); + } else { + ptx_reg_t nw_data[8]; + int num_reg; - a = thread->get_reg(src1.get_symbol()); - addr_t addr = (addr_t)a.u64; - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); + if (wmma_type == LOAD_C) + num_reg = 4; + else + num_reg = 8; - switch( space.get_type() ) { - case shared_space: t = isspace_shared( smid, addr ); - case local_space: t = isspace_local( smid, hwtid, addr ); - case global_space: t = isspace_global( addr ); - default: abort(); - } + for (i = 0; i < num_reg; i++) { + nw_data[i].s64 = ((data[2 * i].s64 & 0xffff) << 16) | + ((data[2 * i + 1].s64 & 0xffff)); + } - ptx_reg_t p; - p.pred = t?1:0; + if (wmma_type == LOAD_C) + thread->set_vector_operand_values(dst, nw_data[0], nw_data[1], + nw_data[2], nw_data[3]); + else + thread->set_wmma_vector_operand_values( + dst, nw_data[0], nw_data[1], nw_data[2], nw_data[3], nw_data[4], + nw_data[5], nw_data[6], nw_data[7]); + if (core->get_gpu()->gpgpu_ctx->debug_tensorcore) { + printf( + "mma_ld:data[0].s64=%llx,data[1].s64=%llx,new_data[0].s64=%llx\n", + data[0].u64, data[1].u64, nw_data[0].u64); + printf( + "mma_ld:data[2].s64=%llx,data[3].s64=%llx,new_data[1].s64=%llx\n", + data[2].u64, data[3].u64, nw_data[1].u64); + printf( + "mma_ld:data[4].s64=%llx,data[5].s64=%llx,new_data[2].s64=%llx\n", + data[4].u64, data[5].u64, nw_data[2].u64); + printf( + "mma_ld:data[6].s64=%llx,data[7].s64=%llx,new_data[3].s64=%llx\n", + data[6].u64, data[7].u64, nw_data[3].u64); + if (wmma_type != LOAD_C) { + printf( + "mma_ld:data[8].s64=%llx,data[9].s64=%llx,new_data[4].s64=%llx\n", + data[8].u64, data[9].u64, nw_data[4].s64); + printf( + "mma_ld:data[10].s64=%llx,data[11].s64=%llx,new_data[5].s64=%" + "llx\n", + data[10].u64, data[11].u64, nw_data[5].u64); + printf( + "mma_ld:data[12].s64=%llx,data[13].s64=%llx,new_data[6].s64=%" + "llx\n", + data[12].u64, data[13].u64, nw_data[6].u64); + printf( + "mma_ld:data[14].s64=%llx,data[15].s64=%llx,new_data[7].s64=%" + "llx\n", + data[14].u64, data[15].u64, nw_data[3].u64); + } + } + } - thread->set_reg(dst.get_symbol(),p); + // thread->m_last_effective_address = addr; + // thread->m_last_memory_space = space; + } } -void decode_space( memory_space_t &space, ptx_thread_info *thread, const operand_info &op, memory_space *&mem, addr_t &addr) -{ - unsigned smid = thread->get_hw_sid(); - unsigned hwtid = thread->get_hw_tid(); +void lg2_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - if( space == param_space_unclassified ) { - // need to op to determine whether it refers to a kernel param or local param - const symbol *s = op.get_symbol(); - const type_info *t = s->type(); - type_info_key ti = t->get_key(); - if( ti.is_param_kernel() ) - space = param_space_kernel; - else if( ti.is_param_local() ) { - space = param_space_local; - } - //mov r1, param-label - else if (ti.is_reg() ){ - space = param_space_kernel; - } - else { - printf("GPGPU-Sim PTX: ERROR ** cannot resolve .param space for '%s'\n", s->name().c_str() ); - abort(); - } - } - switch ( space.get_type() ) { - case global_space: mem = thread->get_global_memory(); break; - case param_space_local: - case local_space: - mem = thread->m_local_mem; - addr += thread->get_local_mem_stack_pointer(); - break; - case tex_space: mem = thread->get_tex_memory(); break; - case surf_space: mem = thread->get_surf_memory(); break; - case param_space_kernel: mem = thread->get_param_memory(); break; - case shared_space: mem = thread->m_shared_mem; break; - case sstarr_space: mem = thread->m_sstarr_mem; break; - case const_space: mem = thread->get_global_memory(); break; - case generic_space: - if( thread->get_ptx_version().ver() >= 2.0 ) { - // convert generic address to memory space address - space = whichspace(addr); - switch ( space.get_type() ) { - case global_space: mem = thread->get_global_memory(); addr = generic_to_global(addr); break; - case local_space: mem = thread->m_local_mem; addr = generic_to_local(smid,hwtid,addr); break; - case shared_space: mem = thread->m_shared_mem; addr = generic_to_shared(smid,addr); break; - default: abort(); - } - } else { - abort(); - } + unsigned i_type = pI->get_type(); + + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + d.f32 = log(a.f32) / log(2); break; - case param_space_unclassified: - case undefined_space: - default: - abort(); - } -} - -void ld_exec( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned type = pI->get_type(); - - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, type, thread, 1); - ptx_reg_t data; - memory_space_t space = pI->get_space(); - unsigned vector_spec = pI->get_vector(); - - memory_space *mem = NULL; - addr_t addr = src1_data.u32; - - decode_space(space,thread,src1,mem,addr); - - size_t size; - int t; - data.u64=0; - type_info_key::type_decode(type,size,t); - if (!vector_spec) { - mem->read(addr,size/8,&data.s64); - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(data,size,dst); - thread->set_operand_value(dst,data, type, thread, pI); - } else { - ptx_reg_t data1, data2, data3, data4; - mem->read(addr,size/8,&data1.s64); - mem->read(addr+size/8,size/8,&data2.s64); - if (vector_spec != V2_TYPE) { //either V3 or V4 - mem->read(addr+2*size/8,size/8,&data3.s64); - if (vector_spec != V3_TYPE) { //v4 - mem->read(addr+3*size/8,size/8,&data4.s64); - thread->set_vector_operand_values(dst,data1,data2,data3,data4); - } else //v3 - thread->set_vector_operand_values(dst,data1,data2,data3,data3); - } else //v2 - thread->set_vector_operand_values(dst,data1,data2,data2,data2); - } - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; -} - -void ld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ld_exec(pI,thread); -} -void ldu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ld_exec(pI,thread); -} - -void mma_st_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) -{ - size_t size; - unsigned smid; - int t; - int thrd, k; - ptx_thread_info *thread; - - const operand_info &src = pI->operand_lookup(1); - const operand_info &src1 = pI->operand_lookup(0); - const operand_info &src2 = pI->operand_lookup(2); - int tid ; - unsigned type = pI->get_type(); - unsigned wmma_type = pI->get_wmma_type(); - unsigned wmma_layout = pI->get_wmma_layout(0); - int stride; - - if(core->get_gpu()->is_functional_sim()) - tid= inst.warp_id_func()*core->get_warp_size(); - else - tid= inst.warp_id()*core->get_warp_size(); - - _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - for (thrd=0; thrd < core->get_warp_size(); thrd++) { - thread = core->get_thread_info()[tid+thrd]; - ptx_reg_t addr_reg = thread->get_operand_value(src1, src, type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, src, type, thread, 1); - const operand_info &src_a= pI->operand_lookup(1); - unsigned nelem = src_a.get_vect_nelem(); - ptx_reg_t* v= new ptx_reg_t[8]; - thread->get_vector_operand_values( src_a, v, nelem ); - stride = src2_data.u32; - - memory_space_t space = pI->get_space(); - - memory_space *mem = NULL; - addr_t addr = addr_reg.u32; - - new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; - int num_mem_txn=0; - - smid = thread->get_hw_sid(); - if( whichspace(addr) == shared_space ) { - addr= generic_to_shared(smid,addr); - space = shared_space; - } - decode_space(space,thread,src1,mem,addr); - - type_info_key::type_decode(type, size, t); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma_st: thrd=%d, addr=%x, fp(size=%zu), stride=%d\n", thrd, addr_reg.u32, size, src2_data.u32); - addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; - addr_t push_addr; - - ptx_reg_t nw_v[8]; - for(k=0;k<8;k++){ - if(k%2==0) - nw_v[k].s64=(v[k/2].s64&0xffff); - else - nw_v[k].s64=((v[k/2].s64&0xffff0000)>>16); - } - - for(k=0;k<8;k++){ - if(type==F32_TYPE){ - //mem->write(new_addr+4*acc_float_offset(k,wmma_layout,stride),size/8,&v[k].s64,thread,pI); - push_addr=new_addr+4*acc_float_offset(k,wmma_layout,stride); - mem->write(push_addr,size/8,&v[k].s64,thread,pI); - mem_txn_addr[num_mem_txn++]=push_addr; - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n",thrd,v[0].s64,v[1].s64,v[2].s64,v[3].s64,v[4].s64,v[5].s64,v[6].s64,v[7].s64); - float temp; - int l; - printf("thread=%d:",thrd); - for(l=0;l<8;l++){ - temp=v[l].f32; - printf("%.2f",temp); - } - printf("\n"); - } - } - else if(type==F16_TYPE){ - if(wmma_layout==ROW){ - //mem->write(new_addr+k*2,size/8,&nw_v[k].s64,thread,pI); - push_addr=new_addr+k*2; - mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); - if(k%2==0) - mem_txn_addr[num_mem_txn++]=push_addr; - } - else if(wmma_layout==COL){ - //mem->write(new_addr+k*2*stride,size/8,&nw_v[k].s64,thread,pI); - push_addr=new_addr+k*2*stride; - mem->write(push_addr,size/8,&nw_v[k].s64,thread,pI); - mem_txn_addr[num_mem_txn++]=push_addr; - } - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("wmma:store:thread%d=%llx,%llx,%llx,%llx,%llx,%llx,%llx,%llx\n",thrd,nw_v[0].s64,nw_v[1].s64,nw_v[2].s64,nw_v[3].s64,nw_v[4].s64,nw_v[5].s64,nw_v[6].s64,nw_v[7].s64); - } - } - - delete [] v; - inst.space = space; - inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); - - if((type==F16_TYPE)&&(wmma_layout==COL))//check the profiling xls for details - inst.data_size = 2; // 2 byte transaction - else - inst.data_size = 4; // 4 byte transaction - - assert( inst.memory_op == insn_memory_op ); - //thread->m_last_effective_address = addr; - //thread->m_last_memory_space = space; - } -} - -void mma_ld_impl( const ptx_instruction *pI, core_t *core, warp_inst_t &inst ) -{ - size_t size; - int t,i; - unsigned smid; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned type = pI->get_type(); - unsigned wmma_type = pI->get_wmma_type(); - unsigned wmma_layout = pI->get_wmma_layout(0); - int tid; - int thrd,stride; - ptx_thread_info *thread; - - - if(core->get_gpu()->is_functional_sim()) - tid= inst.warp_id_func()*core->get_warp_size(); - else - tid= inst.warp_id()*core->get_warp_size(); - - _memory_op_t insn_memory_op = pI->has_memory_read() ? memory_load : memory_store; - - for (thrd=0; thrd < core->get_warp_size(); thrd++){ - thread = core->get_thread_info()[tid+thrd]; - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, U32_TYPE, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, U32_TYPE, thread, 1); - stride=src2_data.u32; - memory_space_t space = pI->get_space(); - - memory_space *mem = NULL; - addr_t addr = src1_data.u32; - smid = thread->get_hw_sid(); - if( whichspace(addr) == shared_space ) { - addr= generic_to_shared(smid,addr); - space = shared_space; - } - - decode_space(space,thread,src1,mem,addr); - type_info_key::type_decode(type, size, t); - - ptx_reg_t data[16]; - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore) - printf("mma_ld: thrd=%d,addr=%x, fpsize=%zu, stride=%d\n", thrd, src1_data.u32, size, src2_data.u32); - - addr_t new_addr = addr+thread_group_offset(thrd,wmma_type,wmma_layout,type,stride)*size/8; - addr_t fetch_addr; - new_addr_type mem_txn_addr[MAX_ACCESSES_PER_INSN_PER_THREAD]; - int num_mem_txn=0; - - if(wmma_type==LOAD_A){ - for(i=0;i<16;i++){ - if(wmma_layout==ROW){ - //mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*i; - mem->read(fetch_addr,size/8,&data[i].s64); - } - else if(wmma_layout==COL){ - //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); - fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); - mem->read(fetch_addr,size/8,&data[i].s64); - } - else{ - printf("mma_ld:wrong_layout_type\n"); - abort(); - - } - if(i%2==0) - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - } - else if(wmma_type==LOAD_B){ - for(i=0;i<16;i++){ - if(wmma_layout==COL){ - //mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*i; - mem->read(fetch_addr,size/8,&data[i].s64); - } - else if(wmma_layout==ROW){ - //mem->read(new_addr+2*(i%4)+2*stride*4*(i/4),size/8,&data[i].s64); - fetch_addr=new_addr+2*(i%4)+2*stride*4*(i/4); - mem->read(fetch_addr,size/8,&data[i].s64); - } - else{ - printf("mma_ld:wrong_layout_type\n"); - abort(); - } - if(i%2==0) - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - } - else if(wmma_type==LOAD_C){ - for(i=0;i<8;i++){ - if(type==F16_TYPE){ - if(wmma_layout==ROW){ - //mem->read(new_addr+2*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*i; - mem->read(fetch_addr,size/8,&data[i].s64); - if(i%2==0) - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - else if(wmma_layout==COL){ - //mem->read(new_addr+2*stride*i,size/8,&data[i].s64); - fetch_addr=new_addr+2*stride*i; - mem->read(fetch_addr,size/8,&data[i].s64); - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - else{ - printf("mma_ld:wrong_type\n"); - abort(); - } - } - else if(type==F32_TYPE){ - //mem->read(new_addr+4*acc_float_offset(i,wmma_layout,stride),size/8,&data[i].s64); - fetch_addr=new_addr+4*acc_float_offset(i,wmma_layout,stride); - mem->read(fetch_addr,size/8,&data[i].s64); - mem_txn_addr[num_mem_txn++]=fetch_addr; - } - else{ - printf("wrong type"); - abort(); - } - } - } - else{ - printf("wrong wmma type\n");; - abort(); - } - //generate timing memory request - inst.space = space; - inst.set_addr(thrd, (new_addr_type *)mem_txn_addr , num_mem_txn); - - if((wmma_type==LOAD_C)&&(type==F16_TYPE)&&(wmma_layout==COL))//memory address is scattered, check the profiling xls for more detail. - inst.data_size = 2; // 2 byte transaction - else - inst.data_size = 4; // 4 byte transaction - assert( inst.memory_op == insn_memory_op ); - - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - if(type==F16_TYPE){ - printf("\nmma_ld:thread%d= ",thrd); - for(i=0;i<16;i++){ - printf("%llx ",data[i].u64); - } - printf("\n"); - - printf("\nmma_ld:thread%d= ",thrd); - float temp; - for(i=0;i<16;i++){ - temp=data[i].f16; - printf("%.2f ",temp); - } - printf("\n"); - } - else{ - printf("\nmma_ld:thread%d= ",thrd); - for(i=0;i<8;i++){ - printf("%.2f ",data[i].f32); - } - printf("\n"); - printf("\nmma_ld:thread%d= ",thrd); - for(i=0;i<8;i++){ - printf("%llx ",data[i].u64); - } - printf("\n"); - } - } - - if((wmma_type==LOAD_C)&&(type==F32_TYPE)){ - thread->set_wmma_vector_operand_values(dst,data[0],data[1],data[2],data[3],data[4],data[5],data[6],data[7]); - } - else{ - ptx_reg_t nw_data[8]; - int num_reg; - - if(wmma_type==LOAD_C) - num_reg=4; - else - num_reg=8; - - for(i=0;iset_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3]); - else - thread->set_wmma_vector_operand_values(dst,nw_data[0],nw_data[1],nw_data[2],nw_data[3],nw_data[4],nw_data[5],nw_data[6],nw_data[7]); - if(core->get_gpu()->gpgpu_ctx->debug_tensorcore){ - printf("mma_ld:data[0].s64=%llx,data[1].s64=%llx,new_data[0].s64=%llx\n",data[0].u64,data[1].u64,nw_data[0].u64); - printf("mma_ld:data[2].s64=%llx,data[3].s64=%llx,new_data[1].s64=%llx\n",data[2].u64,data[3].u64,nw_data[1].u64); - printf("mma_ld:data[4].s64=%llx,data[5].s64=%llx,new_data[2].s64=%llx\n",data[4].u64,data[5].u64,nw_data[2].u64); - printf("mma_ld:data[6].s64=%llx,data[7].s64=%llx,new_data[3].s64=%llx\n",data[6].u64,data[7].u64,nw_data[3].u64); - if(wmma_type!=LOAD_C){ - printf("mma_ld:data[8].s64=%llx,data[9].s64=%llx,new_data[4].s64=%llx\n",data[8].u64,data[9].u64,nw_data[4].s64); - printf("mma_ld:data[10].s64=%llx,data[11].s64=%llx,new_data[5].s64=%llx\n",data[10].u64,data[11].u64,nw_data[5].u64); - printf("mma_ld:data[12].s64=%llx,data[13].s64=%llx,new_data[6].s64=%llx\n",data[12].u64,data[13].u64,nw_data[6].u64); - printf("mma_ld:data[14].s64=%llx,data[15].s64=%llx,new_data[7].s64=%llx\n",data[14].u64,data[15].u64,nw_data[3].u64); - } - } - } - - //thread->m_last_effective_address = addr; - //thread->m_last_memory_space = space; - } -} - -void lg2_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: - d.f32 = log(a.f32)/log(2); - break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mad24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t d, t; +void mad24_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - unsigned sat_mode = pI->saturation_mode(); + unsigned sat_mode = pI->saturation_mode(); - assert( !pI->is_wide() ); + assert(!pI->is_wide()); - switch ( i_type ) { - case S32_TYPE: + switch (i_type) { + case S32_TYPE: t.s64 = a.s32 * b.s32; - if ( pI->is_hi() ) { - d.s64 = (t.s64>>16) + c.s32; - if ( sat_mode ) { - if ( d.s64 > (int)0x7FFFFFFF ) - d.s64 = (int)0x7FFFFFFF; - else if ( d.s64 < (int)0x80000000 ) - d.s64 = (int)0x80000000; - } - } else if ( pI->is_lo() ) d.s64 = t.s32 + c.s32; - else assert(0); - break; - case U32_TYPE: + if (pI->is_hi()) { + d.s64 = (t.s64 >> 16) + c.s32; + if (sat_mode) { + if (d.s64 > (int)0x7FFFFFFF) + d.s64 = (int)0x7FFFFFFF; + else if (d.s64 < (int)0x80000000) + d.s64 = (int)0x80000000; + } + } else if (pI->is_lo()) + d.s64 = t.s32 + c.s32; + else + assert(0); + break; + case U32_TYPE: t.u64 = a.u32 * b.u32; - if ( pI->is_hi() ) d.u64 = (t.u64>>16) + c.u32; - else if ( pI->is_lo() ) d.u64 = t.u32 + c.u32; - else assert(0); + if (pI->is_hi()) + d.u64 = (t.u64 >> 16) + c.u32; + else if (pI->is_lo()) + d.u64 = t.u32 + c.u32; + else + assert(0); break; - default: + default: assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI, thread, false); +void mad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread, false); } -void madp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI, thread, true); +void madp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread, true); } -void madc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - mad_def(pI, thread, true); +void madc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + mad_def(pI, thread, true); } -void mad_def( const ptx_instruction *pI, ptx_thread_info *thread, bool use_carry ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - ptx_reg_t d, t; - - int carry=0; - int overflow=0; - - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); - - // take the carry bit, it should be the 4th operand - ptx_reg_t carry_bit; - carry_bit.u64 = 0; - if (use_carry) { - const operand_info &carry = pI->operand_lookup(4); - carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); - carry_bit.pred &= 0x4; - carry_bit.pred >>=2; - } +void mad_def(const ptx_instruction *pI, ptx_thread_info *thread, + bool use_carry) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + ptx_reg_t d, t; + + int carry = 0; + int overflow = 0; + + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + ptx_reg_t c = thread->get_operand_value(src3, dst, i_type, thread, 1); + + // take the carry bit, it should be the 4th operand + ptx_reg_t carry_bit; + carry_bit.u64 = 0; + if (use_carry) { + const operand_info &carry = pI->operand_lookup(4); + carry_bit = thread->get_operand_value(carry, dst, PRED_TYPE, thread, 0); + carry_bit.pred &= 0x4; + carry_bit.pred >>= 2; + } - unsigned rounding_mode = pI->rounding_mode(); + unsigned rounding_mode = pI->rounding_mode(); - switch ( i_type ) { - case S16_TYPE: + switch (i_type) { + case S16_TYPE: t.s32 = a.s16 * b.s16; - if ( pI->is_wide() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; - else if ( pI->is_hi() ) d.s16 = (t.s32>>16) + c.s16 + carry_bit.pred; - else if ( pI->is_lo() ) d.s16 = t.s16 + c.s16 + carry_bit.pred; - else assert(0); - carry = ((long long int)(t.s32 + c.s32 + carry_bit.pred)&0x100000000)>>32; + if (pI->is_wide()) + d.s32 = t.s32 + c.s32 + carry_bit.pred; + else if (pI->is_hi()) + d.s16 = (t.s32 >> 16) + c.s16 + carry_bit.pred; + else if (pI->is_lo()) + d.s16 = t.s16 + c.s16 + carry_bit.pred; + else + assert(0); + carry = + ((long long int)(t.s32 + c.s32 + carry_bit.pred) & 0x100000000) >> 32; break; - case S32_TYPE: + case S32_TYPE: t.s64 = a.s32 * b.s32; - if ( pI->is_wide() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; - else if ( pI->is_hi() ) d.s32 = (t.s64>>32) + c.s32 + carry_bit.pred; - else if ( pI->is_lo() ) d.s32 = t.s32 + c.s32 + carry_bit.pred; - else assert(0); + if (pI->is_wide()) + d.s64 = t.s64 + c.s64 + carry_bit.pred; + else if (pI->is_hi()) + d.s32 = (t.s64 >> 32) + c.s32 + carry_bit.pred; + else if (pI->is_lo()) + d.s32 = t.s32 + c.s32 + carry_bit.pred; + else + assert(0); break; - case S64_TYPE: + case S64_TYPE: t.s64 = a.s64 * b.s64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - assert( use_carry == false); - if ( pI->is_lo() ) d.s64 = t.s64 + c.s64 + carry_bit.pred; - else assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + assert(use_carry == false); + if (pI->is_lo()) + d.s64 = t.s64 + c.s64 + carry_bit.pred; + else + assert(0); break; - case U16_TYPE: + case U16_TYPE: t.u32 = a.u16 * b.u16; - if ( pI->is_wide() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; - else if ( pI->is_hi() ) d.u16 = (t.u32 + c.u16 + carry_bit.pred)>>16; - else if ( pI->is_lo() ) d.u16 = t.u16 + c.u16 + carry_bit.pred; - else assert(0); - carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred)&0x100000000)>>32; + if (pI->is_wide()) + d.u32 = t.u32 + c.u32 + carry_bit.pred; + else if (pI->is_hi()) + d.u16 = (t.u32 + c.u16 + carry_bit.pred) >> 16; + else if (pI->is_lo()) + d.u16 = t.u16 + c.u16 + carry_bit.pred; + else + assert(0); + carry = ((long long int)((long long int)t.u32 + c.u32 + carry_bit.pred) & + 0x100000000) >> + 32; break; - case U32_TYPE: + case U32_TYPE: t.u64 = a.u32 * b.u32; - if ( pI->is_wide() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; - else if ( pI->is_hi() ) d.u32 = (t.u64 + c.u32 + carry_bit.pred)>>32; - else if ( pI->is_lo() ) d.u32 = t.u32 + c.u32 + carry_bit.pred; - else assert(0); + if (pI->is_wide()) + d.u64 = t.u64 + c.u64 + carry_bit.pred; + else if (pI->is_hi()) + d.u32 = (t.u64 + c.u32 + carry_bit.pred) >> 32; + else if (pI->is_lo()) + d.u32 = t.u32 + c.u32 + carry_bit.pred; + else + assert(0); break; - case U64_TYPE: + case U64_TYPE: t.u64 = a.u64 * b.u64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - assert( use_carry == false); - if ( pI->is_lo() ) d.u64 = t.u64 + c.u64 + carry_bit.pred; - else assert(0); - break; - case F16_TYPE:{ - // assert(0); - // break; - assert( use_carry == false); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f16 = a.f16 * b.f16 + c.f16; - if ( pI->saturation_mode() ) { - if ( d.f16 < 0 ) d.f16 = 0; - else if ( d.f16 > 1.0f ) d.f16 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F32_TYPE: { - assert( use_carry == false); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f32 = a.f32 * b.f32 + c.f32; - if ( pI->saturation_mode() ) { - if ( d.f32 < 0 ) d.f32 = 0; - else if ( d.f32 > 1.0f ) d.f32 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F64_TYPE: case FF64_TYPE: { - assert( use_carry == false); - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f64 = a.f64 * b.f64 + c.f64; - if ( pI->saturation_mode() ) { - if ( d.f64 < 0 ) d.f64 = 0; - else if ( d.f64 > 1.0f ) d.f64 = 1.0; - } - fesetround( orig_rm ); - break; + assert(!pI->is_wide()); + assert(!pI->is_hi()); + assert(use_carry == false); + if (pI->is_lo()) + d.u64 = t.u64 + c.u64 + carry_bit.pred; + else + assert(0); + break; + case F16_TYPE: { + // assert(0); + // break; + assert(use_carry == false); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f16 = a.f16 * b.f16 + c.f16; + if (pI->saturation_mode()) { + if (d.f16 < 0) + d.f16 = 0; + else if (d.f16 > 1.0f) + d.f16 = 1.0f; + } + fesetround(orig_rm); + break; + } + case F32_TYPE: { + assert(use_carry == false); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f32 = a.f32 * b.f32 + c.f32; + if (pI->saturation_mode()) { + if (d.f32 < 0) + d.f32 = 0; + else if (d.f32 > 1.0f) + d.f32 = 1.0f; } - default: + fesetround(orig_rm); + break; + } + case F64_TYPE: + case FF64_TYPE: { + assert(use_carry == false); + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f64 = a.f64 * b.f64 + c.f64; + if (pI->saturation_mode()) { + if (d.f64 < 0) + d.f64 = 0; + else if (d.f64 > 1.0f) + d.f64 = 1.0; + } + fesetround(orig_rm); + break; + } + default: assert(0); break; - } - thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); -} - -bool isNaN(float x) -{ - return std::isnan(x); + } + thread->set_operand_value(dst, d, i_type, thread, pI, overflow, carry); } -bool isNaN(double x) -{ - return std::isnan(x); -} +bool isNaN(float x) { return std::isnan(x); } -void max_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +bool isNaN(double x) { return std::isnan(x); } - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); +void max_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case U16_TYPE: d.u16 = MY_MAX_I(a.u16,b.u16); break; - case U32_TYPE: d.u32 = MY_MAX_I(a.u32,b.u32); break; - case U64_TYPE: d.u64 = MY_MAX_I(a.u64,b.u64); break; - case S16_TYPE: d.s16 = MY_MAX_I(a.s16,b.s16); break; - case S32_TYPE: d.s32 = MY_MAX_I(a.s32,b.s32); break; - case S64_TYPE: d.s64 = MY_MAX_I(a.s64,b.s64); break; - case F32_TYPE: d.f32 = MY_MAX_F(a.f32,b.f32); break; - case F64_TYPE: case FF64_TYPE: d.f64 = MY_MAX_F(a.f64,b.f64); break; - default: + switch (i_type) { + case U16_TYPE: + d.u16 = MY_MAX_I(a.u16, b.u16); + break; + case U32_TYPE: + d.u32 = MY_MAX_I(a.u32, b.u32); + break; + case U64_TYPE: + d.u64 = MY_MAX_I(a.u64, b.u64); + break; + case S16_TYPE: + d.s16 = MY_MAX_I(a.s16, b.s16); + break; + case S32_TYPE: + d.s32 = MY_MAX_I(a.s32, b.s32); + break; + case S64_TYPE: + d.s64 = MY_MAX_I(a.s64, b.s64); + break; + case F32_TYPE: + d.f32 = MY_MAX_F(a.f32, b.f32); + break; + case F64_TYPE: + case FF64_TYPE: + d.f64 = MY_MAX_F(a.f64, b.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void membar_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - // handled by timing simulator +void membar_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // handled by timing simulator } -void min_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void min_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); - - switch ( i_type ) { - case U16_TYPE: d.u16 = MY_MIN_I(a.u16,b.u16); break; - case U32_TYPE: d.u32 = MY_MIN_I(a.u32,b.u32); break; - case U64_TYPE: d.u64 = MY_MIN_I(a.u64,b.u64); break; - case S16_TYPE: d.s16 = MY_MIN_I(a.s16,b.s16); break; - case S32_TYPE: d.s32 = MY_MIN_I(a.s32,b.s32); break; - case S64_TYPE: d.s64 = MY_MIN_I(a.s64,b.s64); break; - case F32_TYPE: d.f32 = MY_MIN_F(a.f32,b.f32); break; - case F64_TYPE: case FF64_TYPE: d.f64 = MY_MIN_F(a.f64,b.f64); break; - default: + switch (i_type) { + case U16_TYPE: + d.u16 = MY_MIN_I(a.u16, b.u16); + break; + case U32_TYPE: + d.u32 = MY_MIN_I(a.u32, b.u32); + break; + case U64_TYPE: + d.u64 = MY_MIN_I(a.u64, b.u64); + break; + case S16_TYPE: + d.s16 = MY_MIN_I(a.s16, b.s16); + break; + case S32_TYPE: + d.s32 = MY_MIN_I(a.s32, b.s32); + break; + case S64_TYPE: + d.s64 = MY_MIN_I(a.s64, b.s64); + break; + case F32_TYPE: + d.f32 = MY_MIN_F(a.f32, b.f32); + break; + case F64_TYPE: + case FF64_TYPE: + d.f64 = MY_MIN_F(a.f64, b.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void mov_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; +void mov_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - assert( src1.is_param_local() == 0 ); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + assert(src1.is_param_local() == 0); - if( (src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && (i_type != BB128_TYPE) && (i_type != FF64_TYPE) ) { - // pack or unpack operation - unsigned nbits_to_move; - ptx_reg_t tmp_bits; + if ((src1.is_vector() || dst.is_vector()) && (i_type != BB64_TYPE) && + (i_type != BB128_TYPE) && (i_type != FF64_TYPE)) { + // pack or unpack operation + unsigned nbits_to_move; + ptx_reg_t tmp_bits; - switch( pI->get_type() ) { - case B16_TYPE: nbits_to_move = 16; break; - case B32_TYPE: nbits_to_move = 32; break; - case B64_TYPE: nbits_to_move = 64; break; - default: printf("Execution error: mov pack/unpack with unsupported type qualifier\n"); assert(0); break; - } + switch (pI->get_type()) { + case B16_TYPE: + nbits_to_move = 16; + break; + case B32_TYPE: + nbits_to_move = 32; + break; + case B64_TYPE: + nbits_to_move = 64; + break; + default: + printf( + "Execution error: mov pack/unpack with unsupported type " + "qualifier\n"); + assert(0); + break; + } - if( src1.is_vector() ) { - unsigned nelem = src1.get_vect_nelem(); - ptx_reg_t v[4]; - thread->get_vector_operand_values(src1, v, nelem ); - - unsigned bits_per_src_elem = nbits_to_move / nelem; - for( unsigned i=0; i < nelem; i++ ) { - switch(bits_per_src_elem) { - case 8: tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8*i)); break; - case 16: tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16*i)); break; - case 32: tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32*i)); break; - default: printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (src)\n"); assert(0); break; - } - } - } else { - data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - switch( pI->get_type() ) { - case B16_TYPE: tmp_bits.u16 = data.u16; break; - case B32_TYPE: tmp_bits.u32 = data.u32; break; - case B64_TYPE: tmp_bits.u64 = data.u64; break; - default: assert(0); break; - } - } + if (src1.is_vector()) { + unsigned nelem = src1.get_vect_nelem(); + ptx_reg_t v[4]; + thread->get_vector_operand_values(src1, v, nelem); - if( dst.is_vector() ) { - unsigned nelem = dst.get_vect_nelem(); - ptx_reg_t v[4]; - unsigned bits_per_dst_elem = nbits_to_move / nelem; - for( unsigned i=0; i < nelem; i++ ) { - switch(bits_per_dst_elem) { - case 8: v[i].u8 = (tmp_bits.u64 >> (8*i)) & ((unsigned long long) 0xFF); break; - case 16: v[i].u16 = (tmp_bits.u64 >> (16*i)) & ((unsigned long long) 0xFFFF); break; - case 32: v[i].u32 = (tmp_bits.u64 >> (32*i)) & ((unsigned long long) 0xFFFFFFFF); break; - default: - printf("Execution error: mov pack/unpack with unsupported source/dst size ratio (dst)\n"); - assert(0); - break; - } - } - thread->set_vector_operand_values(dst,v[0],v[1],v[2],v[3]); - } else { - thread->set_operand_value(dst,tmp_bits, i_type, thread, pI); + unsigned bits_per_src_elem = nbits_to_move / nelem; + for (unsigned i = 0; i < nelem; i++) { + switch (bits_per_src_elem) { + case 8: + tmp_bits.u64 |= ((unsigned long long)(v[i].u8) << (8 * i)); + break; + case 16: + tmp_bits.u64 |= ((unsigned long long)(v[i].u16) << (16 * i)); + break; + case 32: + tmp_bits.u64 |= ((unsigned long long)(v[i].u32) << (32 * i)); + break; + default: + printf( + "Execution error: mov pack/unpack with unsupported source/dst " + "size ratio (src)\n"); + assert(0); + break; + } } - } else if (i_type == PRED_TYPE and src1.is_literal() == true) { - // in ptx, literal input translate to predicate as 0 = false and 1 = true - // we have adopted the opposite to simplify implementation of zero flags in ptxplus + } else { data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t finaldata; - finaldata.pred = (data.u32 == 0)? 1 : 0; // setting zero-flag in predicate - thread->set_operand_value(dst, finaldata, i_type, thread, pI); - } else { - - data = thread->get_operand_value(src1, dst, i_type, thread, 1); + switch (pI->get_type()) { + case B16_TYPE: + tmp_bits.u16 = data.u16; + break; + case B32_TYPE: + tmp_bits.u32 = data.u32; + break; + case B64_TYPE: + tmp_bits.u64 = data.u64; + break; + default: + assert(0); + break; + } + } - thread->set_operand_value(dst, data, i_type, thread, pI); + if (dst.is_vector()) { + unsigned nelem = dst.get_vect_nelem(); + ptx_reg_t v[4]; + unsigned bits_per_dst_elem = nbits_to_move / nelem; + for (unsigned i = 0; i < nelem; i++) { + switch (bits_per_dst_elem) { + case 8: + v[i].u8 = (tmp_bits.u64 >> (8 * i)) & ((unsigned long long)0xFF); + break; + case 16: + v[i].u16 = + (tmp_bits.u64 >> (16 * i)) & ((unsigned long long)0xFFFF); + break; + case 32: + v[i].u32 = + (tmp_bits.u64 >> (32 * i)) & ((unsigned long long)0xFFFFFFFF); + break; + default: + printf( + "Execution error: mov pack/unpack with unsupported source/dst " + "size ratio (dst)\n"); + assert(0); + break; + } + } + thread->set_vector_operand_values(dst, v[0], v[1], v[2], v[3]); + } else { + thread->set_operand_value(dst, tmp_bits, i_type, thread, pI); + } + } else if (i_type == PRED_TYPE and src1.is_literal() == true) { + // in ptx, literal input translate to predicate as 0 = false and 1 = true + // we have adopted the opposite to simplify implementation of zero flags in + // ptxplus + data = thread->get_operand_value(src1, dst, i_type, thread, 1); + + ptx_reg_t finaldata; + finaldata.pred = (data.u32 == 0) ? 1 : 0; // setting zero-flag in predicate + thread->set_operand_value(dst, finaldata, i_type, thread, pI); + } else { + data = thread->get_operand_value(src1, dst, i_type, thread, 1); - } + thread->set_operand_value(dst, data, i_type, thread, pI); + } } -void mul24_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void mul24_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + // src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); + // src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); - //src1_data = srcOperandModifiers(src1_data, src1, dst, i_type, thread); - //src2_data = srcOperandModifiers(src2_data, src2, dst, i_type, thread); + src1_data.mask_and(0, 0x00FFFFFF); + src2_data.mask_and(0, 0x00FFFFFF); - src1_data.mask_and(0,0x00FFFFFF); - src2_data.mask_and(0,0x00FFFFFF); - - switch ( i_type ) { - case S32_TYPE: - if( src1_data.get_bit(23) ) - src1_data.mask_or(0xFFFFFFFF,0xFF000000); - if( src2_data.get_bit(23) ) - src2_data.mask_or(0xFFFFFFFF,0xFF000000); + switch (i_type) { + case S32_TYPE: + if (src1_data.get_bit(23)) src1_data.mask_or(0xFFFFFFFF, 0xFF000000); + if (src2_data.get_bit(23)) src2_data.mask_or(0xFFFFFFFF, 0xFF000000); data.s64 = src1_data.s64 * src2_data.s64; break; - case U32_TYPE: + case U32_TYPE: data.u64 = src1_data.u64 * src2_data.u64; break; - default: - printf("GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); + default: + printf( + "GPGPU-Sim PTX: Execution error - type mismatch with instruction\n"); assert(0); break; - } + } - if ( pI->is_hi() ) { - data.u64 = data.u64 >> 16; - data.mask_and(0,0xFFFFFFFF); - } else if (pI->is_lo()) { - data.mask_and(0,0xFFFFFFFF); - } + if (pI->is_hi()) { + data.u64 = data.u64 >> 16; + data.mask_and(0, 0xFFFFFFFF); + } else if (pI->is_lo()) { + data.mask_and(0, 0xFFFFFFFF); + } - thread->set_operand_value(dst, data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void mul_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; +void mul_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - ptx_reg_t d, t; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + ptx_reg_t d, t; - unsigned i_type = pI->get_type(); - ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t a = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t b = thread->get_operand_value(src2, dst, i_type, thread, 1); - unsigned rounding_mode = pI->rounding_mode(); + unsigned rounding_mode = pI->rounding_mode(); - switch ( i_type ) { - case S16_TYPE: + switch (i_type) { + case S16_TYPE: t.s32 = ((int)a.s16) * ((int)b.s16); - if ( pI->is_wide() ) d.s32 = t.s32; - else if ( pI->is_hi() ) d.s16 = (t.s32>>16); - else if ( pI->is_lo() ) d.s16 = t.s16; - else assert(0); + if (pI->is_wide()) + d.s32 = t.s32; + else if (pI->is_hi()) + d.s16 = (t.s32 >> 16); + else if (pI->is_lo()) + d.s16 = t.s16; + else + assert(0); break; - case S32_TYPE: + case S32_TYPE: t.s64 = ((long long)a.s32) * ((long long)b.s32); - if ( pI->is_wide() ) d.s64 = t.s64; - else if ( pI->is_hi() ) d.s32 = (t.s64>>32); - else if ( pI->is_lo() ) d.s32 = t.s32; - else assert(0); + if (pI->is_wide()) + d.s64 = t.s64; + else if (pI->is_hi()) + d.s32 = (t.s64 >> 32); + else if (pI->is_lo()) + d.s32 = t.s32; + else + assert(0); break; - case S64_TYPE: + case S64_TYPE: t.s64 = a.s64 * b.s64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - if ( pI->is_lo() ) d.s64 = t.s64; - else assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + if (pI->is_lo()) + d.s64 = t.s64; + else + assert(0); break; - case U16_TYPE: + case U16_TYPE: t.u32 = ((unsigned)a.u16) * ((unsigned)b.u16); - if ( pI->is_wide() ) d.u32 = t.u32; - else if ( pI->is_lo() ) d.u16 = t.u16; - else if ( pI->is_hi() ) d.u16 = (t.u32>>16); - else assert(0); + if (pI->is_wide()) + d.u32 = t.u32; + else if (pI->is_lo()) + d.u16 = t.u16; + else if (pI->is_hi()) + d.u16 = (t.u32 >> 16); + else + assert(0); break; - case U32_TYPE: + case U32_TYPE: t.u64 = ((unsigned long long)a.u32) * ((unsigned long long)b.u32); - if ( pI->is_wide() ) d.u64 = t.u64; - else if ( pI->is_lo() ) d.u32 = t.u32; - else if ( pI->is_hi() ) d.u32 = (t.u64>>32); - else assert(0); + if (pI->is_wide()) + d.u64 = t.u64; + else if (pI->is_lo()) + d.u32 = t.u32; + else if (pI->is_hi()) + d.u32 = (t.u64 >> 32); + else + assert(0); break; - case U64_TYPE: + case U64_TYPE: t.u64 = a.u64 * b.u64; - assert( !pI->is_wide() ); - assert( !pI->is_hi() ); - if ( pI->is_lo() ) d.u64 = t.u64; - else assert(0); - break; - case F16_TYPE:{ - //assert(0); - //break; - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - - d.f16 = a.f16 * b.f16; - - if ( pI->saturation_mode() ) { - if ( d.f16 < 0 ) d.f16 = 0; - else if ( d.f16 > 1.0f ) d.f16 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F32_TYPE: { - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - - d.f32 = a.f32 * b.f32; - - if ( pI->saturation_mode() ) { - if ( d.f32 < 0 ) d.f32 = 0; - else if ( d.f32 > 1.0f ) d.f32 = 1.0f; - } - fesetround( orig_rm ); - break; - } - case F64_TYPE: case FF64_TYPE:{ - int orig_rm = fegetround(); - switch ( rounding_mode ) { - case RN_OPTION: break; - case RZ_OPTION: fesetround( FE_TOWARDZERO ); break; - default: assert(0); break; - } - d.f64 = a.f64 * b.f64; - if ( pI->saturation_mode() ) { - if ( d.f64 < 0 ) d.f64 = 0; - else if ( d.f64 > 1.0f ) d.f64 = 1.0; - } - fesetround( orig_rm ); - break; - } - default: - assert(0); + assert(!pI->is_wide()); + assert(!pI->is_hi()); + if (pI->is_lo()) + d.u64 = t.u64; + else + assert(0); break; - } + case F16_TYPE: { + // assert(0); + // break; + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } - thread->set_operand_value(dst, d, i_type, thread, pI); -} + d.f16 = a.f16 * b.f16; -void neg_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; + if (pI->saturation_mode()) { + if (d.f16 < 0) + d.f16 = 0; + else if (d.f16 > 1.0f) + d.f16 = 1.0f; + } + fesetround(orig_rm); + break; + } + case F32_TYPE: { + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); + d.f32 = a.f32 * b.f32; - unsigned to_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); + if (pI->saturation_mode()) { + if (d.f32 < 0) + d.f32 = 0; + else if (d.f32 > 1.0f) + d.f32 = 1.0f; + } + fesetround(orig_rm); + break; + } + case F64_TYPE: + case FF64_TYPE: { + int orig_rm = fegetround(); + switch (rounding_mode) { + case RN_OPTION: + break; + case RZ_OPTION: + fesetround(FE_TOWARDZERO); + break; + default: + assert(0); + break; + } + d.f64 = a.f64 * b.f64; + if (pI->saturation_mode()) { + if (d.f64 < 0) + d.f64 = 0; + else if (d.f64 > 1.0f) + d.f64 = 1.0; + } + fesetround(orig_rm); + break; + } + default: + assert(0); + break; + } + thread->set_operand_value(dst, d, i_type, thread, pI); +} - switch ( to_type ) { - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: - case S64_TYPE: - data.s64 = 0 - src1_data.s64; break; // seems buggy, but not (just ignore higher bits) - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case U64_TYPE: - assert(0); break; - case F16_TYPE: data.f16 =0.0f - src1_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = 0.0f - src1_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = 0.0f - src1_data.f64; break; - default: assert(0); break; - } +void neg_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned to_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, to_type, thread, 1); + + switch (to_type) { + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: + case S64_TYPE: + data.s64 = 0 - src1_data.s64; + break; // seems buggy, but not (just ignore higher bits) + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case U64_TYPE: + assert(0); + break; + case F16_TYPE: + data.f16 = 0.0f - src1_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = 0.0f - src1_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = 0.0f - src1_data.f64; + break; + default: + assert(0); + break; + } - thread->set_operand_value(dst,data, to_type, thread, pI); + thread->set_operand_value(dst, data, to_type, thread, pI); } -//nandn bitwise negates second operand then bitwise nands with the first operand -void nandn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +// nandn bitwise negates second operand then bitwise nands with the first +// operand +void nandn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = (~src1_data.pred & src2_data.pred); + else + data.u64 = ~(src1_data.u64 & ~src2_data.u64); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = (~src1_data.pred & src2_data.pred); - else - data.u64 = ~(src1_data.u64 & ~src2_data.u64); + thread->set_operand_value(dst, data, i_type, thread, pI); +} - thread->set_operand_value(dst,data, i_type, thread, pI); +// norn bitwise negates first operand then bitwise ands with the second operand +void norn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; -} + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); -//norn bitwise negates first operand then bitwise ands with the second operand -void norn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(src1_data.pred & ~(src2_data.pred)); + else + data.u64 = ~(src1_data.u64) & src2_data.u64; - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void not_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(src1_data.pred & ~(src2_data.pred)); - else - data.u64 = ~(src1_data.u64) & src2_data.u64; + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - thread->set_operand_value(dst,data, i_type, thread, pI); + switch (i_type) { + case PRED_TYPE: + d.pred = (~(a.pred) & 0x000F); + break; + case B16_TYPE: + d.u16 = ~a.u16; + break; + case B32_TYPE: + d.u32 = ~a.u32; + break; + case B64_TYPE: + d.u64 = ~a.u64; + break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + thread->set_operand_value(dst, d, i_type, thread, pI); } -void not_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); +void or_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case PRED_TYPE: d.pred = (~(a.pred) & 0x000F); break; - case B16_TYPE: d.u16 = ~a.u16; break; - case B32_TYPE: d.u32 = ~a.u32; break; - case B64_TYPE: d.u64 = ~a.u64; break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); + else + data.u64 = src1_data.u64 | src2_data.u64; - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void or_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void orn_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) | ~(src2_data.pred)); - else - data.u64 = src1_data.u64 | src2_data.u64; + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) | (src2_data.pred)); + else + data.u64 = src1_data.u64 | ~src2_data.u64; - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void orn_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); +void pmevent_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void popc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src = pI->src1(); + + unsigned i_type = pI->get_type(); + src_data = thread->get_operand_value(src, dst, i_type, thread, 1); + + switch (i_type) { + case B32_TYPE: { + std::bitset<32> mask(src_data.u32); + data.u32 = mask.count(); + } break; + case B64_TYPE: { + std::bitset<64> mask(src_data.u64); + data.u32 = mask.count(); + } break; + default: + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + + thread->set_operand_value(dst, data, i_type, thread, pI); +} +void prefetch_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void prefetchu_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); +int prmt_mode_present(int mode) { + int returnval = 0; + switch (mode) { + case PRMT_F4E_MODE: + case PRMT_B4E_MODE: + case PRMT_RC8_MODE: + case PRMT_RC16_MODE: + case PRMT_ECL_MODE: + case PRMT_ECR_MODE: + returnval = 1; + break; + default: + break; + } + return returnval; +} +int read_byte(int mode, int control, int d_sel_index, signed long long value) { + int returnval = 0; + int prmt_f4e_mode[4][4] = { + {0, 1, 2, 3}, {1, 2, 3, 4}, {2, 3, 4, 5}, {3, 4, 5, 6}}; + int prmt_b4e_mode[4][4] = { + {0, 7, 6, 5}, {1, 0, 7, 6}, {2, 1, 0, 7}, {3, 2, 1, 0}}; + int prmt_rc8_mode[4][4] = { + {0, 0, 0, 0}, {1, 1, 1, 1}, {2, 2, 2, 2}, {3, 3, 3, 3}}; + int prmt_ecl_mode[4][4] = { + {0, 1, 2, 3}, {1, 1, 2, 3}, {2, 2, 2, 3}, {3, 3, 3, 3}}; + int prmt_ecr_mode[4][4] = { + {0, 0, 0, 0}, {0, 1, 1, 1}, {0, 1, 2, 2}, {0, 1, 2, 3}}; + int prmt_rc16_mode[4][4] = { + {0, 1, 0, 1}, {2, 3, 2, 3}, {0, 1, 0, 1}, {2, 3, 2, 3}}; + + if (!prmt_mode_present(mode)) { + if (control & 0x8) { + returnval = 0xff; + } else { + returnval = (value >> (8 * control)) & 0xff; + } + } else { + switch (mode) { + case PRMT_F4E_MODE: + returnval = prmt_f4e_mode[control][d_sel_index]; + break; + case PRMT_B4E_MODE: + returnval = prmt_b4e_mode[control][d_sel_index]; + break; + case PRMT_RC8_MODE: + returnval = prmt_rc8_mode[control][d_sel_index]; + break; + case PRMT_ECL_MODE: + returnval = prmt_ecl_mode[control][d_sel_index]; + break; + case PRMT_ECR_MODE: + returnval = prmt_ecr_mode[control][d_sel_index]; + break; + case PRMT_RC16_MODE: + returnval = prmt_rc16_mode[control][d_sel_index]; + break; + // Change the default from printing "ERROR" to just asserting + default: + assert(false); + } + } + return (returnval << 8 * d_sel_index); +} - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) | (src2_data.pred)); - else - data.u64 = src1_data.u64 | ~src2_data.u64; +void prmt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, src3_data, tmpdata, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + unsigned mode = pI->prmt_op(); + unsigned i_type = pI->get_type(); + + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); + + tmpdata.s64 = src1_data.s32 | (src2_data.s64 << 32); + int ctl[4]; + + if (!prmt_mode_present(mode)) { + ctl[0] = (src3_data.s32 >> 0) & 0xf; + ctl[1] = (src3_data.s32 >> 4) & 0xf; + ctl[2] = (src3_data.s32 >> 8) & 0xf; + ctl[3] = (src3_data.s32 >> 12) & 0xf; + } else { + ctl[0] = ctl[1] = ctl[2] = ctl[3] = (src3_data.s32 >> 0) & 0x3; + } - thread->set_operand_value(dst,data, i_type, thread, pI); + data.s32 = 0; + data.s32 = data.s32 | read_byte(mode, ctl[0], 0, tmpdata.s64); // First + // byte-0 + data.s32 = + data.s32 | read_byte(mode, ctl[1], 1, tmpdata.s64); // Second byte-1 + data.s32 = data.s32 | read_byte(mode, ctl[2], 2, tmpdata.s64); // Third + // byte-2 + data.s32 = + data.s32 | read_byte(mode, ctl[3], 3, tmpdata.s64); // Fourth byte-3 + + thread->set_operand_value(dst, data, i_type, thread, pI); } -void pmevent_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void popc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src = pI->src1(); +void rcp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); - unsigned i_type = pI->get_type(); - src_data = thread->get_operand_value(src, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case B32_TYPE: { - std::bitset<32> mask(src_data.u32); - data.u32 = mask.count(); - } break; - case B64_TYPE: { - std::bitset<64> mask(src_data.u64); - data.u32 = mask.count(); - } break; - default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst,data, i_type, thread, pI); -} -void prefetch_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void prefetchu_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - -int prmt_mode_present(int mode) -{ - int returnval=0; - switch(mode){ - case PRMT_F4E_MODE: - case PRMT_B4E_MODE: - case PRMT_RC8_MODE: - case PRMT_RC16_MODE: - case PRMT_ECL_MODE: - case PRMT_ECR_MODE: - returnval=1; - break; - default: - break; - } - return returnval; -} -int read_byte(int mode, int control, int d_sel_index, signed long long value){ - - int returnval = 0; - int prmt_f4e_mode[4][4]={{0,1,2,3},{1,2,3,4},{2,3,4,5},{3,4,5,6}}; - int prmt_b4e_mode[4][4]={{0,7,6,5},{1,0,7,6},{2,1,0,7},{3,2,1,0}}; - int prmt_rc8_mode[4][4]={{0,0,0,0},{1,1,1,1},{2,2,2,2},{3,3,3,3}}; - int prmt_ecl_mode[4][4]={{0,1,2,3},{1,1,2,3},{2,2,2,3},{3,3,3,3}}; - int prmt_ecr_mode[4][4]={{0,0,0,0},{0,1,1,1},{0,1,2,2},{0,1,2,3}}; - int prmt_rc16_mode[4][4]={{0,1,0,1},{2,3,2,3},{0,1,0,1},{2,3,2,3}}; - - if(!prmt_mode_present(mode)){ - if(control&0x8){ - returnval=0xff; - } - else{ - returnval= (value>>(8*control)) & 0xff; - } - } - else{ - switch(mode){ - case PRMT_F4E_MODE: returnval=prmt_f4e_mode[control][d_sel_index];break; - case PRMT_B4E_MODE: returnval=prmt_b4e_mode[control][d_sel_index];break; - case PRMT_RC8_MODE: returnval=prmt_rc8_mode[control][d_sel_index];break; - case PRMT_ECL_MODE: returnval=prmt_ecl_mode[control][d_sel_index];break; - case PRMT_ECR_MODE: returnval=prmt_ecr_mode[control][d_sel_index];break; - case PRMT_RC16_MODE: returnval=prmt_rc16_mode[control][d_sel_index];break; - // Change the default from printing "ERROR" to just asserting - default: assert(false); - } - } - return (returnval << 8 * d_sel_index); -} - -void prmt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { - - ptx_reg_t src1_data, src2_data, src3_data,tmpdata,data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - - unsigned mode = pI->prmt_op(); - unsigned i_type = pI->get_type(); - - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - src3_data = thread->get_operand_value(src3, dst, i_type, thread, 1); - - tmpdata.s64=src1_data.s32|(src2_data.s64<<32); - int ctl[4]; - - if(!prmt_mode_present(mode)){ - ctl[0]=(src3_data.s32>>0)&0xf; - ctl[1]=(src3_data.s32>>4)&0xf; - ctl[2]=(src3_data.s32>>8)&0xf; - ctl[3]=(src3_data.s32>>12)&0xf; - } - else{ - ctl[0]=ctl[1]=ctl[2]=ctl[3]=(src3_data.s32>>0)&0x3; - } - - data.s32=0; - data.s32=data.s32|read_byte(mode,ctl[0],0,tmpdata.s64); //First byte-0 - data.s32=data.s32|read_byte(mode,ctl[1],1,tmpdata.s64); //Second byte-1 - data.s32=data.s32|read_byte(mode,ctl[2],2,tmpdata.s64); //Third byte-2 - data.s32=data.s32|read_byte(mode,ctl[3],3,tmpdata.s64); //Fourth byte-3 - - thread->set_operand_value(dst,data, i_type, thread, pI); - - -} - -void rcp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: data.f32 = 1.0f / src1_data.f32; break; - case F64_TYPE: - case FF64_TYPE: + case F64_TYPE: + case FF64_TYPE: data.f64 = 1.0f / src1_data.f64; break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void red_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } +void red_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} -void rem_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; +void rem_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - switch ( i_type ) { - case S32_TYPE: + switch (i_type) { + case S32_TYPE: data.s32 = src1_data.s32 % src2_data.s32; break; - case S64_TYPE: + case S64_TYPE: data.s64 = src1_data.s64 % src2_data.s64; break; - case U32_TYPE: + case U32_TYPE: data.u32 = src1_data.u32 % src2_data.u32; break; - case U64_TYPE: + case U64_TYPE: data.u64 = src1_data.u64 % src2_data.u64; break; - default: assert(0); break; - } + default: + assert(0); + break; + } - thread->set_operand_value(dst,data, i_type, thread, pI); + thread->set_operand_value(dst, data, i_type, thread, pI); } -void ret_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - bool empty = thread->callstack_pop(); - if( empty ) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); - } +void ret_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + bool empty = thread->callstack_pop(); + if (empty) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } } -//Ptxplus version of ret instruction. -void retp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - bool empty = thread->callstack_pop_plus(); - if( empty ) { - thread->set_done(); - thread->exitCore(); - thread->registerExit(); - } +// Ptxplus version of ret instruction. +void retp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + bool empty = thread->callstack_pop_plus(); + if (empty) { + thread->set_done(); + thread->exitCore(); + thread->registerExit(); + } } -void rsqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: - if ( a.f32 < 0 ) { - d.u64 = 0; - d.u64 = 0x7fc00000; // NaN - } else if ( a.f32 == 0 ) { - d.u64 = 0; - d.u32 = 0x7f800000; // Inf +void rsqrt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + if (a.f32 < 0) { + d.u64 = 0; + d.u64 = 0x7fc00000; // NaN + } else if (a.f32 == 0) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf } else - d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); - break; - case F64_TYPE: - case FF64_TYPE: - if ( a.f32 < 0 ) { - d.u64 = 0; - d.u32 = 0x7fc00000; // NaN - float x = d.f32; - d.f64 = (double)x; - } else if ( a.f32 == 0 ) { - d.u64 = 0; - d.u32 = 0x7f800000; // Inf - float x = d.f32; - d.f64 = (double)x; + d.f32 = cuda_math::__internal_accurate_fdividef(1.0f, sqrtf(a.f32)); + break; + case F64_TYPE: + case FF64_TYPE: + if (a.f32 < 0) { + d.u64 = 0; + d.u32 = 0x7fc00000; // NaN + float x = d.f32; + d.f64 = (double)x; + } else if (a.f32 == 0) { + d.u64 = 0; + d.u32 = 0x7f800000; // Inf + float x = d.f32; + d.f64 = (double)x; } else - d.f64 = 1.0 / sqrt(a.f64); + d.f64 = 1.0 / sqrt(a.f64); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -#define SAD(d,a,b,c) d = c + ((adst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +#define SAD(d, a, b, c) d = c + ((a < b) ? (b - a) : (a - b)) - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, i_type, thread, 1); +void sad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, c, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); - switch ( i_type ) { - case U16_TYPE: SAD(d.u16,a.u16,b.u16,c.u16); break; - case U32_TYPE: SAD(d.u32,a.u32,b.u32,c.u32); break; - case U64_TYPE: SAD(d.u64,a.u64,b.u64,c.u64); break; - case S16_TYPE: SAD(d.s16,a.s16,b.s16,c.s16); break; - case S32_TYPE: SAD(d.s32,a.s32,b.s32,c.s32); break; - case S64_TYPE: SAD(d.s64,a.s64,b.s64,c.s64); break; - case F32_TYPE: SAD(d.f32,a.f32,b.f32,c.f32); break; - case F64_TYPE: case FF64_TYPE: SAD(d.f64,a.f64,b.f64,c.f64); break; - default: + switch (i_type) { + case U16_TYPE: + SAD(d.u16, a.u16, b.u16, c.u16); + break; + case U32_TYPE: + SAD(d.u32, a.u32, b.u32, c.u32); + break; + case U64_TYPE: + SAD(d.u64, a.u64, b.u64, c.u64); + break; + case S16_TYPE: + SAD(d.s16, a.s16, b.s16, c.s16); + break; + case S32_TYPE: + SAD(d.s32, a.s32, b.s32, c.s32); + break; + case S64_TYPE: + SAD(d.s64, a.s64, b.s64, c.s64); + break; + case F32_TYPE: + SAD(d.f32, a.f32, b.f32, c.f32); + break; + case F64_TYPE: + case FF64_TYPE: + SAD(d.f64, a.f64, b.f64, c.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void selp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); +void selp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); - ptx_reg_t a, b, c, d; + ptx_reg_t a, b, c, d; - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, i_type, thread, 1); - //predicate value was changed so the lowest bit being set means the zero flag is set. - //As a result, the value of c.pred must be inverted to get proper behavior - d = (!(c.pred & 0x0001))?a:b; + // predicate value was changed so the lowest bit being set means the zero flag + // is set. As a result, the value of c.pred must be inverted to get proper + // behavior + d = (!(c.pred & 0x0001)) ? a : b; - thread->set_operand_value(dst,d, PRED_TYPE, thread, pI); + thread->set_operand_value(dst, d, PRED_TYPE, thread, pI); } -bool isFloat(int type) -{ - switch ( type ) { - case F16_TYPE: - case F32_TYPE: - case F64_TYPE: - case FF64_TYPE: +bool isFloat(int type) { + switch (type) { + case F16_TYPE: + case F32_TYPE: + case F64_TYPE: + case FF64_TYPE: return true; - default: + default: return false; - } + } } -bool CmpOp( int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop ) -{ - bool t = false; +bool CmpOp(int type, ptx_reg_t a, ptx_reg_t b, unsigned cmpop) { + bool t = false; - switch ( type ) { - case B16_TYPE: + switch (type) { + case B16_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u16 == b.u16); break; - case NE_OPTION: t = (a.u16 != b.u16); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u16 == b.u16); + break; + case NE_OPTION: + t = (a.u16 != b.u16); + break; + default: + assert(0); } - case B32_TYPE: + case B32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u32 == b.u32); break; - case NE_OPTION: t = (a.u32 != b.u32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u32 == b.u32); + break; + case NE_OPTION: + t = (a.u32 != b.u32); + break; + default: + assert(0); } - case B64_TYPE: + case B64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u64 == b.u64); break; - case NE_OPTION: t = (a.u64 != b.u64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u64 == b.u64); + break; + case NE_OPTION: + t = (a.u64 != b.u64); + break; + default: + assert(0); } break; - case S8_TYPE: - case S16_TYPE: + case S8_TYPE: + case S16_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.s16 == b.s16); break; - case NE_OPTION: t = (a.s16 != b.s16); break; - case LT_OPTION: t = (a.s16 < b.s16); break; - case LE_OPTION: t = (a.s16 <= b.s16); break; - case GT_OPTION: t = (a.s16 > b.s16); break; - case GE_OPTION: t = (a.s16 >= b.s16); break; - default: - assert(0); + case EQ_OPTION: + t = (a.s16 == b.s16); + break; + case NE_OPTION: + t = (a.s16 != b.s16); + break; + case LT_OPTION: + t = (a.s16 < b.s16); + break; + case LE_OPTION: + t = (a.s16 <= b.s16); + break; + case GT_OPTION: + t = (a.s16 > b.s16); + break; + case GE_OPTION: + t = (a.s16 >= b.s16); + break; + default: + assert(0); } break; - case S32_TYPE: + case S32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.s32 == b.s32); break; - case NE_OPTION: t = (a.s32 != b.s32); break; - case LT_OPTION: t = (a.s32 < b.s32); break; - case LE_OPTION: t = (a.s32 <= b.s32); break; - case GT_OPTION: t = (a.s32 > b.s32); break; - case GE_OPTION: t = (a.s32 >= b.s32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.s32 == b.s32); + break; + case NE_OPTION: + t = (a.s32 != b.s32); + break; + case LT_OPTION: + t = (a.s32 < b.s32); + break; + case LE_OPTION: + t = (a.s32 <= b.s32); + break; + case GT_OPTION: + t = (a.s32 > b.s32); + break; + case GE_OPTION: + t = (a.s32 >= b.s32); + break; + default: + assert(0); } break; - case S64_TYPE: + case S64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.s64 == b.s64); break; - case NE_OPTION: t = (a.s64 != b.s64); break; - case LT_OPTION: t = (a.s64 < b.s64); break; - case LE_OPTION: t = (a.s64 <= b.s64); break; - case GT_OPTION: t = (a.s64 > b.s64); break; - case GE_OPTION: t = (a.s64 >= b.s64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.s64 == b.s64); + break; + case NE_OPTION: + t = (a.s64 != b.s64); + break; + case LT_OPTION: + t = (a.s64 < b.s64); + break; + case LE_OPTION: + t = (a.s64 <= b.s64); + break; + case GT_OPTION: + t = (a.s64 > b.s64); + break; + case GE_OPTION: + t = (a.s64 >= b.s64); + break; + default: + assert(0); } break; - case U8_TYPE: - case U16_TYPE: + case U8_TYPE: + case U16_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u16 == b.u16); break; - case NE_OPTION: t = (a.u16 != b.u16); break; - case LT_OPTION: t = (a.u16 < b.u16); break; - case LE_OPTION: t = (a.u16 <= b.u16); break; - case GT_OPTION: t = (a.u16 > b.u16); break; - case GE_OPTION: t = (a.u16 >= b.u16); break; - case LO_OPTION: t = (a.u16 < b.u16); break; - case LS_OPTION: t = (a.u16 <= b.u16); break; - case HI_OPTION: t = (a.u16 > b.u16); break; - case HS_OPTION: t = (a.u16 >= b.u16); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u16 == b.u16); + break; + case NE_OPTION: + t = (a.u16 != b.u16); + break; + case LT_OPTION: + t = (a.u16 < b.u16); + break; + case LE_OPTION: + t = (a.u16 <= b.u16); + break; + case GT_OPTION: + t = (a.u16 > b.u16); + break; + case GE_OPTION: + t = (a.u16 >= b.u16); + break; + case LO_OPTION: + t = (a.u16 < b.u16); + break; + case LS_OPTION: + t = (a.u16 <= b.u16); + break; + case HI_OPTION: + t = (a.u16 > b.u16); + break; + case HS_OPTION: + t = (a.u16 >= b.u16); + break; + default: + assert(0); } break; - case U32_TYPE: + case U32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u32 == b.u32); break; - case NE_OPTION: t = (a.u32 != b.u32); break; - case LT_OPTION: t = (a.u32 < b.u32); break; - case LE_OPTION: t = (a.u32 <= b.u32); break; - case GT_OPTION: t = (a.u32 > b.u32); break; - case GE_OPTION: t = (a.u32 >= b.u32); break; - case LO_OPTION: t = (a.u32 < b.u32); break; - case LS_OPTION: t = (a.u32 <= b.u32); break; - case HI_OPTION: t = (a.u32 > b.u32); break; - case HS_OPTION: t = (a.u32 >= b.u32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u32 == b.u32); + break; + case NE_OPTION: + t = (a.u32 != b.u32); + break; + case LT_OPTION: + t = (a.u32 < b.u32); + break; + case LE_OPTION: + t = (a.u32 <= b.u32); + break; + case GT_OPTION: + t = (a.u32 > b.u32); + break; + case GE_OPTION: + t = (a.u32 >= b.u32); + break; + case LO_OPTION: + t = (a.u32 < b.u32); + break; + case LS_OPTION: + t = (a.u32 <= b.u32); + break; + case HI_OPTION: + t = (a.u32 > b.u32); + break; + case HS_OPTION: + t = (a.u32 >= b.u32); + break; + default: + assert(0); } break; - case U64_TYPE: + case U64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.u64 == b.u64); break; - case NE_OPTION: t = (a.u64 != b.u64); break; - case LT_OPTION: t = (a.u64 < b.u64); break; - case LE_OPTION: t = (a.u64 <= b.u64); break; - case GT_OPTION: t = (a.u64 > b.u64); break; - case GE_OPTION: t = (a.u64 >= b.u64); break; - case LO_OPTION: t = (a.u64 < b.u64); break; - case LS_OPTION: t = (a.u64 <= b.u64); break; - case HI_OPTION: t = (a.u64 > b.u64); break; - case HS_OPTION: t = (a.u64 >= b.u64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.u64 == b.u64); + break; + case NE_OPTION: + t = (a.u64 != b.u64); + break; + case LT_OPTION: + t = (a.u64 < b.u64); + break; + case LE_OPTION: + t = (a.u64 <= b.u64); + break; + case GT_OPTION: + t = (a.u64 > b.u64); + break; + case GE_OPTION: + t = (a.u64 >= b.u64); + break; + case LO_OPTION: + t = (a.u64 < b.u64); + break; + case LS_OPTION: + t = (a.u64 <= b.u64); + break; + case HI_OPTION: + t = (a.u64 > b.u64); + break; + case HS_OPTION: + t = (a.u64 >= b.u64); + break; + default: + assert(0); } break; - case F16_TYPE: assert(0); break; - case F32_TYPE: + case F16_TYPE: + assert(0); + break; + case F32_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case NE_OPTION: t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case LT_OPTION: t = (a.f32 < b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; - case LE_OPTION: t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case GT_OPTION: t = (a.f32 > b.f32 ) && !isNaN(a.f32) && !isNaN(b.f32); break; - case GE_OPTION: t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); break; - case EQU_OPTION: t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case NEU_OPTION: t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case LTU_OPTION: t = (a.f32 < b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; - case LEU_OPTION: t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case GTU_OPTION: t = (a.f32 > b.f32 ) || isNaN(a.f32) || isNaN(b.f32); break; - case GEU_OPTION: t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); break; - case NUM_OPTION: t = !isNaN(a.f32) && !isNaN(b.f32); break; - case NAN_OPTION: t = isNaN(a.f32) || isNaN(b.f32); break; - default: - assert(0); + case EQ_OPTION: + t = (a.f32 == b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case NE_OPTION: + t = (a.f32 != b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case LT_OPTION: + t = (a.f32 < b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case LE_OPTION: + t = (a.f32 <= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case GT_OPTION: + t = (a.f32 > b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case GE_OPTION: + t = (a.f32 >= b.f32) && !isNaN(a.f32) && !isNaN(b.f32); + break; + case EQU_OPTION: + t = (a.f32 == b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case NEU_OPTION: + t = (a.f32 != b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case LTU_OPTION: + t = (a.f32 < b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case LEU_OPTION: + t = (a.f32 <= b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case GTU_OPTION: + t = (a.f32 > b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case GEU_OPTION: + t = (a.f32 >= b.f32) || isNaN(a.f32) || isNaN(b.f32); + break; + case NUM_OPTION: + t = !isNaN(a.f32) && !isNaN(b.f32); + break; + case NAN_OPTION: + t = isNaN(a.f32) || isNaN(b.f32); + break; + default: + assert(0); } break; - case F64_TYPE: - case FF64_TYPE: + case F64_TYPE: + case FF64_TYPE: switch (cmpop) { - case EQ_OPTION: t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case NE_OPTION: t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case LT_OPTION: t = (a.f64 < b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; - case LE_OPTION: t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case GT_OPTION: t = (a.f64 > b.f64 ) && !isNaN(a.f64) && !isNaN(b.f64); break; - case GE_OPTION: t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); break; - case EQU_OPTION: t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case NEU_OPTION: t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case LTU_OPTION: t = (a.f64 < b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; - case LEU_OPTION: t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case GTU_OPTION: t = (a.f64 > b.f64 ) || isNaN(a.f64) || isNaN(b.f64); break; - case GEU_OPTION: t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); break; - case NUM_OPTION: t = !isNaN(a.f64) && !isNaN(b.f64); break; - case NAN_OPTION: t = isNaN(a.f64) || isNaN(b.f64); break; - default: - assert(0); + case EQ_OPTION: + t = (a.f64 == b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case NE_OPTION: + t = (a.f64 != b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case LT_OPTION: + t = (a.f64 < b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case LE_OPTION: + t = (a.f64 <= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case GT_OPTION: + t = (a.f64 > b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case GE_OPTION: + t = (a.f64 >= b.f64) && !isNaN(a.f64) && !isNaN(b.f64); + break; + case EQU_OPTION: + t = (a.f64 == b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case NEU_OPTION: + t = (a.f64 != b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case LTU_OPTION: + t = (a.f64 < b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case LEU_OPTION: + t = (a.f64 <= b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case GTU_OPTION: + t = (a.f64 > b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case GEU_OPTION: + t = (a.f64 >= b.f64) || isNaN(a.f64) || isNaN(b.f64); + break; + case NUM_OPTION: + t = !isNaN(a.f64) && !isNaN(b.f64); + break; + case NAN_OPTION: + t = isNaN(a.f64) || isNaN(b.f64); + break; + default: + assert(0); } break; - default: assert(0); break; - } + default: + assert(0); + break; + } - return t; + return t; } -void setp_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b; +void setp_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b; - int t=0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + int t = 0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp + assert(pI->get_num_operands() < + 4); // or need to deal with "c" operand / boolOp - unsigned type = pI->get_type(); - unsigned cmpop = pI->get_cmpop(); - a = thread->get_operand_value(src1, dst, type, thread, 1); - b = thread->get_operand_value(src2, dst, type, thread, 1); + unsigned type = pI->get_type(); + unsigned cmpop = pI->get_cmpop(); + a = thread->get_operand_value(src1, dst, type, thread, 1); + b = thread->get_operand_value(src2, dst, type, thread, 1); - t = CmpOp(type,a,b,cmpop); + t = CmpOp(type, a, b, cmpop); - ptx_reg_t data; + ptx_reg_t data; - //the way ptxplus handles the zero flag, 1 = false and 0 = true - data.pred = (t==0); //inverting predicate since ptxplus uses "1" for a set zero flag + // the way ptxplus handles the zero flag, 1 = false and 0 = true + data.pred = + (t == + 0); // inverting predicate since ptxplus uses "1" for a set zero flag - thread->set_operand_value(dst,data, PRED_TYPE, thread, pI); + thread->set_operand_value(dst, data, PRED_TYPE, thread, pI); } -void set_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b; +void set_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b; - int t=0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + int t = 0; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - assert( pI->get_num_operands() < 4 ); // or need to deal with "c" operand / boolOp + assert(pI->get_num_operands() < + 4); // or need to deal with "c" operand / boolOp - unsigned src_type = pI->get_type2(); - unsigned cmpop = pI->get_cmpop(); + unsigned src_type = pI->get_type2(); + unsigned cmpop = pI->get_cmpop(); - a = thread->get_operand_value(src1, dst, src_type, thread, 1); - b = thread->get_operand_value(src2, dst, src_type, thread, 1); + a = thread->get_operand_value(src1, dst, src_type, thread, 1); + b = thread->get_operand_value(src2, dst, src_type, thread, 1); - // Take abs of first operand if needed - if(pI->is_abs()) { - switch ( src_type ) { - case S16_TYPE: a.s16 = my_abs(a.s16); break; - case S32_TYPE: a.s32 = my_abs(a.s32); break; - case S64_TYPE: a.s64 = my_abs(a.s64); break; - case U16_TYPE: a.u16 = a.u16; break; - case U32_TYPE: a.u32 = my_abs(a.u32); break; - case U64_TYPE: a.u64 = my_abs(a.u64); break; - case F32_TYPE: a.f32 = my_abs(a.f32); break; - case F64_TYPE: case FF64_TYPE: a.f64 = my_abs(a.f64); break; + // Take abs of first operand if needed + if (pI->is_abs()) { + switch (src_type) { + case S16_TYPE: + a.s16 = my_abs(a.s16); + break; + case S32_TYPE: + a.s32 = my_abs(a.s32); + break; + case S64_TYPE: + a.s64 = my_abs(a.s64); + break; + case U16_TYPE: + a.u16 = a.u16; + break; + case U32_TYPE: + a.u32 = my_abs(a.u32); + break; + case U64_TYPE: + a.u64 = my_abs(a.u64); + break; + case F32_TYPE: + a.f32 = my_abs(a.f32); + break; + case F64_TYPE: + case FF64_TYPE: + a.f64 = my_abs(a.f64); + break; default: - printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - } - - t = CmpOp(src_type,a,b,cmpop); - - ptx_reg_t data; - if ( isFloat(pI->get_type()) ) { - data.f32 = (t!=0)?1.0f:0.0f; - } else { - data.u32 = (t!=0)?0xFFFFFFFF:0; - } - - thread->set_operand_value(dst, data, pI->get_type(), thread, pI); - -} - -void shfl_impl( const ptx_instruction *pI, core_t *core, warp_inst_t inst ) -{ - unsigned i_type = pI->get_type(); - int tid; - - if(core->get_gpu()->is_functional_sim()) - tid = inst.warp_id_func() * core->get_warp_size(); - else - tid = inst.warp_id() * core->get_warp_size(); - - ptx_thread_info *thread = core->get_thread_info()[tid]; - ptx_warp_info *warp_info = thread->m_warp_info; - int lane = warp_info->get_done_threads(); - thread = core->get_thread_info()[tid + lane]; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; - int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; - int mask = cval >> 8; - bval &= 0x1F; - cval &= 0x1F; - - int maxLane = (lane & mask) | (cval & ~mask); - int minLane = lane & mask; - - int src_idx; - unsigned p; - switch(pI->shfl_op()) { - case UP_OPTION: - src_idx = lane - bval; - p = (src_idx >= maxLane); - break; - case DOWN_OPTION: - src_idx = lane + bval; - p = (src_idx <= maxLane); - break; - case BFLY_OPTION: - src_idx = lane ^ bval; - p = (src_idx <= maxLane); - break; - case IDX_OPTION: - src_idx = minLane | (bval & ~mask); - p = (src_idx <= maxLane); - break; - default: - printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); - assert(0); - break; - } - // copy from own lane - if (!p) src_idx = lane; - - // copy input from lane src_idx - ptx_reg_t data; - if (inst.active(src_idx)) { - ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; - data = source->get_operand_value(src1, dst, i_type, source, 1); - } else { - printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive threads in a warp\n"); - data.u32 = 0; - } - thread->set_operand_value(dst, data, i_type, thread, pI); - - /* - TODO: deal with predicates appropriately using the following pseudocode: - if (!isGuardPredicateTrue(src_idx)) { - printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for predicated-off threads in a warp\n"); - } - if (dest predicate selected) data.pred = p; - */ - - // keep track of the number of threads that have executed in the warp - warp_info->inc_done_threads(); - if (warp_info->get_done_threads() == inst.active_count()) { - warp_info->reset_done_threads(); - } -} - -void shl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - - switch ( i_type ) { - case B16_TYPE: - case U16_TYPE: - if ( b.u16 >= 16 ) - d.u16 = 0; + printf("Execution error: type mismatch with instruction\n"); + assert(0); + break; + } + } + + t = CmpOp(src_type, a, b, cmpop); + + ptx_reg_t data; + if (isFloat(pI->get_type())) { + data.f32 = (t != 0) ? 1.0f : 0.0f; + } else { + data.u32 = (t != 0) ? 0xFFFFFFFF : 0; + } + + thread->set_operand_value(dst, data, pI->get_type(), thread, pI); +} + +void shfl_impl(const ptx_instruction *pI, core_t *core, warp_inst_t inst) { + unsigned i_type = pI->get_type(); + int tid; + + if (core->get_gpu()->is_functional_sim()) + tid = inst.warp_id_func() * core->get_warp_size(); + else + tid = inst.warp_id() * core->get_warp_size(); + + ptx_thread_info *thread = core->get_thread_info()[tid]; + ptx_warp_info *warp_info = thread->m_warp_info; + int lane = warp_info->get_done_threads(); + thread = core->get_thread_info()[tid + lane]; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + int bval = (thread->get_operand_value(src2, dst, i_type, thread, 1)).u32; + int cval = (thread->get_operand_value(src3, dst, i_type, thread, 1)).u32; + int mask = cval >> 8; + bval &= 0x1F; + cval &= 0x1F; + + int maxLane = (lane & mask) | (cval & ~mask); + int minLane = lane & mask; + + int src_idx; + unsigned p; + switch (pI->shfl_op()) { + case UP_OPTION: + src_idx = lane - bval; + p = (src_idx >= maxLane); + break; + case DOWN_OPTION: + src_idx = lane + bval; + p = (src_idx <= maxLane); + break; + case BFLY_OPTION: + src_idx = lane ^ bval; + p = (src_idx <= maxLane); + break; + case IDX_OPTION: + src_idx = minLane | (bval & ~mask); + p = (src_idx <= maxLane); + break; + default: + printf("GPGPU-Sim PTX: ERROR: Invalid shfl option\n"); + assert(0); + break; + } + // copy from own lane + if (!p) src_idx = lane; + + // copy input from lane src_idx + ptx_reg_t data; + if (inst.active(src_idx)) { + ptx_thread_info *source = core->get_thread_info()[tid + src_idx]; + data = source->get_operand_value(src1, dst, i_type, source, 1); + } else { + printf( + "GPGPU-Sim PTX: WARNING: shfl input value unpredictable for inactive " + "threads in a warp\n"); + data.u32 = 0; + } + thread->set_operand_value(dst, data, i_type, thread, pI); + + /* + TODO: deal with predicates appropriately using the following pseudocode: + if (!isGuardPredicateTrue(src_idx)) { + printf("GPGPU-Sim PTX: WARNING: shfl input value unpredictable for + predicated-off threads in a warp\n"); + } + if (dest predicate selected) data.pred = p; + */ + + // keep track of the number of threads that have executed in the warp + warp_info->inc_done_threads(); + if (warp_info->get_done_threads() == inst.active_count()) { + warp_info->reset_done_threads(); + } +} + +void shl_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + switch (i_type) { + case B16_TYPE: + case U16_TYPE: + if (b.u16 >= 16) + d.u16 = 0; else - d.u16 = (unsigned short) ((a.u16 << b.u16) & 0xFFFF); + d.u16 = (unsigned short)((a.u16 << b.u16) & 0xFFFF); break; - case B32_TYPE: - case U32_TYPE: - if ( b.u32 >= 32 ) - d.u32 = 0; + case B32_TYPE: + case U32_TYPE: + if (b.u32 >= 32) + d.u32 = 0; else - d.u32 = (unsigned) ((a.u32 << b.u32) & 0xFFFFFFFF); + d.u32 = (unsigned)((a.u32 << b.u32) & 0xFFFFFFFF); break; - case B64_TYPE: - case U64_TYPE: - if ( b.u32 >= 64 ) - d.u64 = 0; + case B64_TYPE: + case U64_TYPE: + if (b.u32 >= 64) + d.u64 = 0; else - d.u64 = (a.u64 << b.u64); + d.u64 = (a.u64 << b.u64); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst, d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void shr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, b, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - - - switch ( i_type ) { - case U16_TYPE: - case B16_TYPE: - if ( b.u16 < 16 ) - d.u16 = (unsigned short) ((a.u16 >> b.u16) & 0xFFFF); +void shr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, b, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + + switch (i_type) { + case U16_TYPE: + case B16_TYPE: + if (b.u16 < 16) + d.u16 = (unsigned short)((a.u16 >> b.u16) & 0xFFFF); else - d.u16 = 0; + d.u16 = 0; break; - case U32_TYPE: - case B32_TYPE: - if ( b.u32 < 32 ) - d.u32 = (unsigned) ((a.u32 >> b.u32) & 0xFFFFFFFF); + case U32_TYPE: + case B32_TYPE: + if (b.u32 < 32) + d.u32 = (unsigned)((a.u32 >> b.u32) & 0xFFFFFFFF); else - d.u32 = 0; + d.u32 = 0; break; - case U64_TYPE: - case B64_TYPE: - if ( b.u32 < 64 ) - d.u64 = (a.u64 >> b.u64); + case U64_TYPE: + case B64_TYPE: + if (b.u32 < 64) + d.u64 = (a.u64 >> b.u64); else - d.u64 = 0; + d.u64 = 0; break; - case S16_TYPE: - if ( b.u16 < 16 ) - d.s64 = (a.s16 >> b.s16); + case S16_TYPE: + if (b.u16 < 16) + d.s64 = (a.s16 >> b.s16); else { - if ( a.s16 < 0 ) { - d.s64 = -1; - } else { - d.s64 = 0; - } + if (a.s16 < 0) { + d.s64 = -1; + } else { + d.s64 = 0; + } } break; - case S32_TYPE: - if ( b.u32 < 32 ) - d.s64 = (a.s32 >> b.s32); + case S32_TYPE: + if (b.u32 < 32) + d.s64 = (a.s32 >> b.s32); else { - if ( a.s32 < 0 ) { - d.s64 = -1; - } else { - d.s64 = 0; - } + if (a.s32 < 0) { + d.s64 = -1; + } else { + d.s64 = 0; + } } break; - case S64_TYPE: - if ( b.u64 < 64 ) - d.s64 = (a.s64 >> b.u64); + case S64_TYPE: + if (b.u64 < 64) + d.s64 = (a.s64 >> b.u64); else { - if ( a.s64 < 0 ) { - if ( b.s32 < 0 ) { - d.u64 = -1; - d.s32 = 0; - } else { - d.s64 = -1; - } - } else { - d.s64 = 0; - } + if (a.s64 < 0) { + if (b.s32 < 0) { + d.u64 = -1; + d.s32 = 0; + } else { + d.s64 = -1; + } + } else { + d.s64 = 0; + } } break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); + assert(0); break; - } + } - thread->set_operand_value(dst,d, i_type, thread, pI); + thread->set_operand_value(dst, d, i_type, thread, pI); } -void sin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); +void sin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); - switch ( i_type ) { - case F32_TYPE: + switch (i_type) { + case F32_TYPE: d.f32 = sin(a.f32); break; - default: + default: printf("Execution error: type mismatch with instruction\n"); - assert(0); - break; - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -void slct_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - - ptx_reg_t a, b, c, d; - - unsigned i_type = pI->get_type(); - unsigned c_type = pI->get_type2(); - bool t = false; - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - b = thread->get_operand_value(src2, dst, i_type, thread, 1); - c = thread->get_operand_value(src3, dst, c_type, thread, 1); - - switch ( c_type ) { - case S32_TYPE: t = c.s32 >= 0; break; - case F32_TYPE: t = c.f32 >= 0; break; - default: assert(0); - } - - switch ( i_type ) { - case B16_TYPE: - case S16_TYPE: - case U16_TYPE: d.u16 = t?a.u16:b.u16; break; - case F32_TYPE: - case B32_TYPE: - case S32_TYPE: - case U32_TYPE: d.u32 = t?a.u32:b.u32; break; - case F64_TYPE: - case FF64_TYPE: - case B64_TYPE: - case S64_TYPE: - case U64_TYPE: d.u64 = t?a.u64:b.u64; break; - default: assert(0); - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -void sqrt_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t a, d; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - - unsigned i_type = pI->get_type(); - a = thread->get_operand_value(src1, dst, i_type, thread, 1); - - - switch ( i_type ) { - case F32_TYPE: - if ( a.f32 < 0 ) - d.f32 = nanf(""); + assert(0); + break; + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void slct_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + + ptx_reg_t a, b, c, d; + + unsigned i_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + bool t = false; + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + b = thread->get_operand_value(src2, dst, i_type, thread, 1); + c = thread->get_operand_value(src3, dst, c_type, thread, 1); + + switch (c_type) { + case S32_TYPE: + t = c.s32 >= 0; + break; + case F32_TYPE: + t = c.f32 >= 0; + break; + default: + assert(0); + } + + switch (i_type) { + case B16_TYPE: + case S16_TYPE: + case U16_TYPE: + d.u16 = t ? a.u16 : b.u16; + break; + case F32_TYPE: + case B32_TYPE: + case S32_TYPE: + case U32_TYPE: + d.u32 = t ? a.u32 : b.u32; + break; + case F64_TYPE: + case FF64_TYPE: + case B64_TYPE: + case S64_TYPE: + case U64_TYPE: + d.u64 = t ? a.u64 : b.u64; + break; + default: + assert(0); + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void sqrt_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t a, d; + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + + unsigned i_type = pI->get_type(); + a = thread->get_operand_value(src1, dst, i_type, thread, 1); + + switch (i_type) { + case F32_TYPE: + if (a.f32 < 0) + d.f32 = nanf(""); else - d.f32 = sqrt(a.f32); break; - case F64_TYPE: - case FF64_TYPE: - if ( a.f64 < 0 ) - d.f64 = nan(""); + d.f32 = sqrt(a.f32); + break; + case F64_TYPE: + case FF64_TYPE: + if (a.f64 < 0) + d.f64 = nan(""); else - d.f64 = sqrt(a.f64); break; - default: + d.f64 = sqrt(a.f64); + break; + default: printf("Execution error: type mismatch with instruction\n"); assert(0); break; - } - - thread->set_operand_value(dst,d, i_type, thread, pI); -} - -void sst_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_instruction * cpI = const_cast(pI); // constant - const operand_info &dst = cpI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - const operand_info &src3 = pI->src3(); - unsigned type = pI->get_type(); - ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); - ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); - ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); - memory_space_t space = pI->get_space(); - memory_space *mem = NULL; - addr_t addr = src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 - ptx_cta_info *cta_info = thread->m_cta_info; - - decode_space(space,thread,src1,mem,addr); - - size_t size; - int t; - type_info_key::type_decode(type,size,t); - - // store data in sstarr memory - mem->write(addr,size/8,&src3_data.s64,thread,pI); - - // sync threads - cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 - - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; - thread->m_last_dram_callback.function = bar_callback; - thread->m_last_dram_callback.instruction = cpI; - - // the last thread that executes loads all of the data back from sstarr memory - int NUM_THREADS = cta_info->num_threads(); - cta_info->inc_bar_threads(); - if (NUM_THREADS == cta_info->get_bar_threads()) { - unsigned offset = 0; - addr = 0; - ptx_reg_t data; - float sstarr_fdata[NUM_THREADS]; - signed long long sstarr_ldata[NUM_THREADS]; - // loop through all of the threads - for (int tid = 0; tid < NUM_THREADS; tid++) { - data.u64=0; - mem->read(addr+(tid*4),size/8,&data.s64); - sstarr_fdata[tid] = data.f32; - sstarr_ldata[tid] = data.s64; - } - - // squeeze the zeros out of the array and store data back into original array - mem = NULL; - addr = src1_data.u32; - space.set_type(global_space); - decode_space(space,thread,src1,mem,addr); - // store nonzero entries and indices - for (int tid = 0; tid < NUM_THREADS; tid++) { - if (sstarr_fdata[tid] != 0) { - float ftid = (float)tid; - mem->write(addr+(offset*4),size/8,&sstarr_ldata[tid],thread,pI); - mem->write(addr+((NUM_THREADS+offset)*4),size/8,&ftid,thread,pI); - offset++; - } - } - // store the number of nonzero elements in the array - data = thread->get_operand_value(src1, dst, type, thread, 1); - data.s64 += 4*(offset-1); - thread->set_operand_value(dst, data, type, thread, pI); - - // fill the rest of the array with zeros (dst should always have a 0 in it) - while (offset < NUM_THREADS) { - mem->write(addr+(offset*4),size/8,&dst_data.s64,thread,pI); - offset++; - } - - cta_info->reset_bar_threads(); - thread->m_last_effective_address = addr+(NUM_THREADS-1)*4; - thread->m_last_memory_space = space; - } -} - -void ssy_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - //printf("Execution Warning: unimplemented ssy instruction is treated as a nop\n"); - // TODO: add implementation -} - -void st_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); //may be scalar or vector of regs - unsigned type = pI->get_type(); - ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); - ptx_reg_t data; - memory_space_t space = pI->get_space(); - unsigned vector_spec = pI->get_vector(); - - memory_space *mem = NULL; - addr_t addr = addr_reg.u32; - - decode_space(space,thread,dst,mem,addr); - - size_t size; - int t; - type_info_key::type_decode(type,size,t); - - if (!vector_spec) { - data = thread->get_operand_value(src1, dst, type, thread, 1); - mem->write(addr,size/8,&data.s64,thread,pI); - } else { - if (vector_spec == V2_TYPE) { - ptx_reg_t* ptx_regs = new ptx_reg_t[2]; - thread->get_vector_operand_values(src1, ptx_regs, 2); - mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); - mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); - delete [] ptx_regs; - } - if (vector_spec == V3_TYPE) { - ptx_reg_t* ptx_regs = new ptx_reg_t[3]; - thread->get_vector_operand_values(src1, ptx_regs, 3); - mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); - mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); - mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); - delete [] ptx_regs; - } - if (vector_spec == V4_TYPE) { - ptx_reg_t* ptx_regs = new ptx_reg_t[4]; - thread->get_vector_operand_values(src1, ptx_regs, 4); - mem->write(addr,size/8,&ptx_regs[0].s64,thread,pI); - mem->write(addr+size/8,size/8,&ptx_regs[1].s64,thread,pI); - mem->write(addr+2*size/8,size/8,&ptx_regs[2].s64,thread,pI); - mem->write(addr+3*size/8,size/8,&ptx_regs[3].s64,thread,pI); - delete [] ptx_regs; + } + + thread->set_operand_value(dst, d, i_type, thread, pI); +} + +void sst_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_instruction *cpI = const_cast(pI); // constant + const operand_info &dst = cpI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + const operand_info &src3 = pI->src3(); + unsigned type = pI->get_type(); + ptx_reg_t dst_data = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t src1_data = thread->get_operand_value(src1, src1, type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, src1, type, thread, 1); + ptx_reg_t src3_data = thread->get_operand_value(src3, src1, type, thread, 1); + memory_space_t space = pI->get_space(); + memory_space *mem = NULL; + addr_t addr = + src2_data.u32 * 4; // this assumes sstarr memory starts at address 0 + ptx_cta_info *cta_info = thread->m_cta_info; + + decode_space(space, thread, src1, mem, addr); + + size_t size; + int t; + type_info_key::type_decode(type, size, t); + + // store data in sstarr memory + mem->write(addr, size / 8, &src3_data.s64, thread, pI); + + // sync threads + cpI->set_bar_id(16); // use 16 for sst because bar uses an int from 0-15 + + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; + thread->m_last_dram_callback.function = bar_callback; + thread->m_last_dram_callback.instruction = cpI; + + // the last thread that executes loads all of the data back from sstarr memory + int NUM_THREADS = cta_info->num_threads(); + cta_info->inc_bar_threads(); + if (NUM_THREADS == cta_info->get_bar_threads()) { + unsigned offset = 0; + addr = 0; + ptx_reg_t data; + float sstarr_fdata[NUM_THREADS]; + signed long long sstarr_ldata[NUM_THREADS]; + // loop through all of the threads + for (int tid = 0; tid < NUM_THREADS; tid++) { + data.u64 = 0; + mem->read(addr + (tid * 4), size / 8, &data.s64); + sstarr_fdata[tid] = data.f32; + sstarr_ldata[tid] = data.s64; + } + + // squeeze the zeros out of the array and store data back into original + // array + mem = NULL; + addr = src1_data.u32; + space.set_type(global_space); + decode_space(space, thread, src1, mem, addr); + // store nonzero entries and indices + for (int tid = 0; tid < NUM_THREADS; tid++) { + if (sstarr_fdata[tid] != 0) { + float ftid = (float)tid; + mem->write(addr + (offset * 4), size / 8, &sstarr_ldata[tid], thread, + pI); + mem->write(addr + ((NUM_THREADS + offset) * 4), size / 8, &ftid, thread, + pI); + offset++; } - } - thread->m_last_effective_address = addr; - thread->m_last_memory_space = space; + } + // store the number of nonzero elements in the array + data = thread->get_operand_value(src1, dst, type, thread, 1); + data.s64 += 4 * (offset - 1); + thread->set_operand_value(dst, data, type, thread, pI); + + // fill the rest of the array with zeros (dst should always have a 0 in it) + while (offset < NUM_THREADS) { + mem->write(addr + (offset * 4), size / 8, &dst_data.s64, thread, pI); + offset++; + } + + cta_info->reset_bar_threads(); + thread->m_last_effective_address = addr + (NUM_THREADS - 1) * 4; + thread->m_last_memory_space = space; + } +} + +void ssy_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // printf("Execution Warning: unimplemented ssy instruction is treated as a + // nop\n"); + // TODO: add implementation } -void sub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t data; - int overflow = 0; - int carry = 0; +void st_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); // may be scalar or vector of regs + unsigned type = pI->get_type(); + ptx_reg_t addr_reg = thread->get_operand_value(dst, dst, type, thread, 1); + ptx_reg_t data; + memory_space_t space = pI->get_space(); + unsigned vector_spec = pI->get_vector(); + + memory_space *mem = NULL; + addr_t addr = addr_reg.u32; + + decode_space(space, thread, dst, mem, addr); + + size_t size; + int t; + type_info_key::type_decode(type, size, t); + + if (!vector_spec) { + data = thread->get_operand_value(src1, dst, type, thread, 1); + mem->write(addr, size / 8, &data.s64, thread, pI); + } else { + if (vector_spec == V2_TYPE) { + ptx_reg_t *ptx_regs = new ptx_reg_t[2]; + thread->get_vector_operand_values(src1, ptx_regs, 2); + mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); + mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); + delete[] ptx_regs; + } + if (vector_spec == V3_TYPE) { + ptx_reg_t *ptx_regs = new ptx_reg_t[3]; + thread->get_vector_operand_values(src1, ptx_regs, 3); + mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); + mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); + mem->write(addr + 2 * size / 8, size / 8, &ptx_regs[2].s64, thread, pI); + delete[] ptx_regs; + } + if (vector_spec == V4_TYPE) { + ptx_reg_t *ptx_regs = new ptx_reg_t[4]; + thread->get_vector_operand_values(src1, ptx_regs, 4); + mem->write(addr, size / 8, &ptx_regs[0].s64, thread, pI); + mem->write(addr + size / 8, size / 8, &ptx_regs[1].s64, thread, pI); + mem->write(addr + 2 * size / 8, size / 8, &ptx_regs[2].s64, thread, pI); + mem->write(addr + 3 * size / 8, size / 8, &ptx_regs[3].s64, thread, pI); + delete[] ptx_regs; + } + } + thread->m_last_effective_address = addr; + thread->m_last_memory_space = space; +} + +void sub_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t data; + int overflow = 0; + int carry = 0; - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); - unsigned i_type = pI->get_type(); - ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + unsigned i_type = pI->get_type(); + ptx_reg_t src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + ptx_reg_t src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - //performs addition. Sets carry and overflow if needed. - //the constant is added in during subtraction so the carry bit is set properly. - switch ( i_type ) { - case S8_TYPE: + // performs addition. Sets carry and overflow if needed. + // the constant is added in during subtraction so the carry bit is set + // properly. + switch (i_type) { + case S8_TYPE: data.s64 = (src1_data.s64 & 0xFF) - (src2_data.s64 & 0xFF) + 0x100; - if(((src1_data.s64 & 0x80)-(src2_data.s64 & 0x80)) != 0) {overflow=((src1_data.s64 & 0x80)-(data.s64 & 0x80))==0?0:1; } - carry = (data.s32 & 0x100)>>8; + if (((src1_data.s64 & 0x80) - (src2_data.s64 & 0x80)) != 0) { + overflow = ((src1_data.s64 & 0x80) - (data.s64 & 0x80)) == 0 ? 0 : 1; + } + carry = (data.s32 & 0x100) >> 8; break; - case S16_TYPE: + case S16_TYPE: data.s64 = (src1_data.s64 & 0xFFFF) - (src2_data.s64 & 0xFFFF) + 0x10000; - if(((src1_data.s64 & 0x8000)-(src2_data.s64 & 0x8000)) != 0) {overflow=((src1_data.s64 & 0x8000)-(data.s64 & 0x8000))==0?0:1; } - carry = (data.s32 & 0x10000)>>16; - break; - case S32_TYPE: - data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + 0x100000000; - if(((src1_data.s64 & 0x80000000)-(src2_data.s64 & 0x80000000)) != 0) {overflow=((src1_data.s64 & 0x80000000)-(data.s64 & 0x80000000))==0?0:1; } - carry = ((data.u64)>>32) & 0x0001; - break; - case S64_TYPE: - data.s64 = src1_data.s64 - src2_data.s64; break; - case B8_TYPE: - case U8_TYPE: + if (((src1_data.s64 & 0x8000) - (src2_data.s64 & 0x8000)) != 0) { + overflow = + ((src1_data.s64 & 0x8000) - (data.s64 & 0x8000)) == 0 ? 0 : 1; + } + carry = (data.s32 & 0x10000) >> 16; + break; + case S32_TYPE: + data.s64 = (src1_data.s64 & 0xFFFFFFFF) - (src2_data.s64 & 0xFFFFFFFF) + + 0x100000000; + if (((src1_data.s64 & 0x80000000) - (src2_data.s64 & 0x80000000)) != 0) { + overflow = ((src1_data.s64 & 0x80000000) - (data.s64 & 0x80000000)) == 0 + ? 0 + : 1; + } + carry = ((data.u64) >> 32) & 0x0001; + break; + case S64_TYPE: + data.s64 = src1_data.s64 - src2_data.s64; + break; + case B8_TYPE: + case U8_TYPE: data.u64 = (src1_data.u64 & 0xFF) - (src2_data.u64 & 0xFF) + 0x100; - carry = (data.u64 & 0x100)>>8; + carry = (data.u64 & 0x100) >> 8; break; - case B16_TYPE: - case U16_TYPE: + case B16_TYPE: + case U16_TYPE: data.u64 = (src1_data.u64 & 0xFFFF) - (src2_data.u64 & 0xFFFF) + 0x10000; - carry = (data.u64 & 0x10000)>>16; + carry = (data.u64 & 0x10000) >> 16; + break; + case B32_TYPE: + case U32_TYPE: + data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + + 0x100000000; + carry = (data.u64 & 0x100000000) >> 32; + break; + case B64_TYPE: + case U64_TYPE: + data.u64 = src1_data.u64 - src2_data.u64; + break; + case F16_TYPE: + data.f16 = src1_data.f16 - src2_data.f16; + break; // assert(0); break; + case F32_TYPE: + data.f32 = src1_data.f32 - src2_data.f32; + break; + case F64_TYPE: + case FF64_TYPE: + data.f64 = src1_data.f64 - src2_data.f64; break; - case B32_TYPE: - case U32_TYPE: - data.u64 = (src1_data.u64 & 0xFFFFFFFF) - (src2_data.u64 & 0xFFFFFFFF) + 0x100000000; - carry = (data.u64 & 0x100000000)>>32; + default: + assert(0); break; - case B64_TYPE: - case U64_TYPE: - data.u64 = src1_data.u64 - src2_data.u64; break; - case F16_TYPE: data.f16 = src1_data.f16 - src2_data.f16; break;//assert(0); break; - case F32_TYPE: data.f32 = src1_data.f32 - src2_data.f32; break; - case F64_TYPE: case FF64_TYPE: data.f64 = src1_data.f64 - src2_data.f64; break; - default: assert(0); break; - } + } - thread->set_operand_value(dst,data, i_type, thread, pI, overflow, carry); + thread->set_operand_value(dst, data, i_type, thread, pI, overflow, carry); } -void nop_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - // Do nothing +void nop_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + // Do nothing } -void subc_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void suld_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void sured_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void sust_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void suq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - +void subc_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void suld_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void sured_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void sust_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void suq_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} union intfloat { - int a; - float b; + int a; + float b; }; -float reduce_precision( float x, unsigned bits ) -{ - intfloat tmp; - tmp.b = x; - int v = tmp.a; - int man = v & ((1<<23)-1); - int mask = ((1<= mx) nx -= elem_size; - unsigned ny = (y >= my)? my - 1 : y; - return nx + mx*ny; -} - -typedef unsigned (*texAddr_t) (unsigned x, unsigned y, unsigned mx, unsigned my, size_t elem_size); -float tex_linf_sampling(memory_space* mem, unsigned tex_array_base, - int x, int y, unsigned int width, unsigned int height, size_t elem_size, - float alpha, float beta, texAddr_t b_lim) -{ - float Tij; - float Ti1j; - float Tij1; - float Ti1j1; - - mem->read(tex_array_base + b_lim(x,y,width,height,elem_size), 4, &Tij); - mem->read(tex_array_base + b_lim(x+elem_size,y,width,height,elem_size), 4, &Ti1j); - mem->read(tex_array_base + b_lim(x,y+1,width,height,elem_size), 4, &Tij1); - mem->read(tex_array_base + b_lim(x+elem_size,y+1,width,height,elem_size), 4, &Ti1j1); - - float sample = (1-alpha)*(1-beta)*Tij + - alpha*(1-beta)*Ti1j + - (1-alpha)*beta*Tij1 + - alpha*beta*Ti1j1; - - return sample; -} - -float textureNormalizeElementSigned(int element, int bits) -{ - if (bits) { - int maxN = (1 << bits) - 1; - // removing upper bits - element &= maxN; - // normalizing the number to [-1.0,1.0] - maxN >>= 1; - float output = (float) element / maxN; - if (output < -1.0f) output = -1.0f; - return output; - } else { - return 0.0f; - } -} - -float textureNormalizeElementUnsigned(unsigned int element, int bits) -{ - if (bits) { - unsigned int maxN = (1 << bits) - 1; - // removing upper bits and normalizing the number to [0.0,1.0] - return (float)(element & maxN) / maxN; - } else { - return 0.0f; - } -} - -void textureNormalizeOutput( const struct cudaChannelFormatDesc& desc, ptx_reg_t& datax, ptx_reg_t& datay, ptx_reg_t& dataz, ptx_reg_t& dataw ) -{ - if (desc.f == cudaChannelFormatKindSigned) { - datax.f32 = textureNormalizeElementSigned( datax.s32, desc.x ); - datay.f32 = textureNormalizeElementSigned( datay.s32, desc.y ); - dataz.f32 = textureNormalizeElementSigned( dataz.s32, desc.z ); - dataw.f32 = textureNormalizeElementSigned( dataw.s32, desc.w ); - } else if (desc.f == cudaChannelFormatKindUnsigned) { - datax.f32 = textureNormalizeElementUnsigned( datax.u32, desc.x ); - datay.f32 = textureNormalizeElementUnsigned( datay.u32, desc.y ); - dataz.f32 = textureNormalizeElementUnsigned( dataz.u32, desc.z ); - dataw.f32 = textureNormalizeElementUnsigned( dataw.u32, desc.w ); - } else { - assert(0 && "Undefined texture read mode: cudaReadModeNormalizedFloat expect integer elements"); - } -} - -void tex_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - unsigned dimension = pI->dimension(); - const operand_info &dst = pI->dst(); //the registers to which fetched texel will be placed - const operand_info &src1 = pI->src1(); //the name of the texture - const operand_info &src2 = pI->src2(); //the vector registers containing coordinates of the texel to be fetched - - std::string texname = src1.name(); - unsigned to_type = pI->get_type(); - unsigned c_type = pI->get_type2(); - fflush(stdout); - ptx_reg_t data1, data2, data3, data4; - if (!thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs) - thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs = new ptx_reg_t[4]; - unsigned nelem = src2.get_vect_nelem(); - thread->get_vector_operand_values(src2, thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs, nelem); //ptx_reg should be 4 entry vector type...coordinates into texture - /* - For programs with many streams, textures can be bound and unbound - asynchronously. This means we need to use the kernel's "snapshot" of - the state of the texture mappings when it was launched (so that we - don't try to access the incorrect texture mapping if it's been updated, - or that we don't access a mapping that has been unbound). - */ - gpgpu_t *gpu = thread->get_gpu(); - kernel_info_t &k = thread->get_kernel(); - const struct textureReference* texref = gpu->get_texref(texname); - const struct cudaArray* cuArray = k.get_texarray(texname); - const struct textureInfo* texInfo = k.get_texinfo(texname); - const struct textureReferenceAttr* texAttr = gpu->get_texattr(texname); - - //assume always 2D f32 input - //access array with src2 coordinates - memory_space *mem = thread->get_global_memory(); - float x_f32, y_f32; - size_t size; - int t; - unsigned tex_array_base; - unsigned int width = 0, height = 0; - int x = 0; - int y = 0; - unsigned tex_array_index; - float alpha=0, beta=0; - - type_info_key::type_decode(to_type,size,t); - tex_array_base = cuArray->devPtr32; - - switch (dimension) { - case GEOM_MODIFIER_1D: +float reduce_precision(float x, unsigned bits) { + intfloat tmp; + tmp.b = x; + int v = tmp.a; + int man = v & ((1 << 23) - 1); + int mask = ((1 << bits) - 1) << (23 - bits); + int nv = (v & ((-1) - ((1 << 23) - 1))) | (mask & man); + tmp.a = nv; + float result = tmp.b; + return result; +} + +unsigned wrap(unsigned x, unsigned y, unsigned mx, unsigned my, + size_t elem_size) { + unsigned nx = (mx + x) % mx; + unsigned ny = (my + y) % my; + return nx + mx * ny; +} + +unsigned clamp(unsigned x, unsigned y, unsigned mx, unsigned my, + size_t elem_size) { + unsigned nx = x; + while (nx >= mx) nx -= elem_size; + unsigned ny = (y >= my) ? my - 1 : y; + return nx + mx * ny; +} + +typedef unsigned (*texAddr_t)(unsigned x, unsigned y, unsigned mx, unsigned my, + size_t elem_size); +float tex_linf_sampling(memory_space *mem, unsigned tex_array_base, int x, + int y, unsigned int width, unsigned int height, + size_t elem_size, float alpha, float beta, + texAddr_t b_lim) { + float Tij; + float Ti1j; + float Tij1; + float Ti1j1; + + mem->read(tex_array_base + b_lim(x, y, width, height, elem_size), 4, &Tij); + mem->read(tex_array_base + b_lim(x + elem_size, y, width, height, elem_size), + 4, &Ti1j); + mem->read(tex_array_base + b_lim(x, y + 1, width, height, elem_size), 4, + &Tij1); + mem->read( + tex_array_base + b_lim(x + elem_size, y + 1, width, height, elem_size), 4, + &Ti1j1); + + float sample = (1 - alpha) * (1 - beta) * Tij + alpha * (1 - beta) * Ti1j + + (1 - alpha) * beta * Tij1 + alpha * beta * Ti1j1; + + return sample; +} + +float textureNormalizeElementSigned(int element, int bits) { + if (bits) { + int maxN = (1 << bits) - 1; + // removing upper bits + element &= maxN; + // normalizing the number to [-1.0,1.0] + maxN >>= 1; + float output = (float)element / maxN; + if (output < -1.0f) output = -1.0f; + return output; + } else { + return 0.0f; + } +} + +float textureNormalizeElementUnsigned(unsigned int element, int bits) { + if (bits) { + unsigned int maxN = (1 << bits) - 1; + // removing upper bits and normalizing the number to [0.0,1.0] + return (float)(element & maxN) / maxN; + } else { + return 0.0f; + } +} + +void textureNormalizeOutput(const struct cudaChannelFormatDesc &desc, + ptx_reg_t &datax, ptx_reg_t &datay, + ptx_reg_t &dataz, ptx_reg_t &dataw) { + if (desc.f == cudaChannelFormatKindSigned) { + datax.f32 = textureNormalizeElementSigned(datax.s32, desc.x); + datay.f32 = textureNormalizeElementSigned(datay.s32, desc.y); + dataz.f32 = textureNormalizeElementSigned(dataz.s32, desc.z); + dataw.f32 = textureNormalizeElementSigned(dataw.s32, desc.w); + } else if (desc.f == cudaChannelFormatKindUnsigned) { + datax.f32 = textureNormalizeElementUnsigned(datax.u32, desc.x); + datay.f32 = textureNormalizeElementUnsigned(datay.u32, desc.y); + dataz.f32 = textureNormalizeElementUnsigned(dataz.u32, desc.z); + dataw.f32 = textureNormalizeElementUnsigned(dataw.u32, desc.w); + } else { + assert(0 && + "Undefined texture read mode: cudaReadModeNormalizedFloat expect " + "integer elements"); + } +} + +void tex_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + unsigned dimension = pI->dimension(); + const operand_info &dst = + pI->dst(); // the registers to which fetched texel will be placed + const operand_info &src1 = pI->src1(); // the name of the texture + const operand_info &src2 = + pI->src2(); // the vector registers containing coordinates of the texel + // to be fetched + + std::string texname = src1.name(); + unsigned to_type = pI->get_type(); + unsigned c_type = pI->get_type2(); + fflush(stdout); + ptx_reg_t data1, data2, data3, data4; + if (!thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs) + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs = new ptx_reg_t[4]; + unsigned nelem = src2.get_vect_nelem(); + thread->get_vector_operand_values( + src2, thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs, + nelem); // ptx_reg should be 4 entry vector type...coordinates into + // texture + /* + For programs with many streams, textures can be bound and unbound + asynchronously. This means we need to use the kernel's "snapshot" of + the state of the texture mappings when it was launched (so that we + don't try to access the incorrect texture mapping if it's been updated, + or that we don't access a mapping that has been unbound). + */ + gpgpu_t *gpu = thread->get_gpu(); + kernel_info_t &k = thread->get_kernel(); + const struct textureReference *texref = gpu->get_texref(texname); + const struct cudaArray *cuArray = k.get_texarray(texname); + const struct textureInfo *texInfo = k.get_texinfo(texname); + const struct textureReferenceAttr *texAttr = gpu->get_texattr(texname); + + // assume always 2D f32 input + // access array with src2 coordinates + memory_space *mem = thread->get_global_memory(); + float x_f32, y_f32; + size_t size; + int t; + unsigned tex_array_base; + unsigned int width = 0, height = 0; + int x = 0; + int y = 0; + unsigned tex_array_index; + float alpha = 0, beta = 0; + + type_info_key::type_decode(to_type, size, t); + tex_array_base = cuArray->devPtr32; + + switch (dimension) { + case GEOM_MODIFIER_1D: width = cuArray->width; height = cuArray->height; if (texref->normalized) { - assert(c_type == F32_TYPE); - x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - if (texref->addressMode[0] == cudaAddressModeClamp) { - x_f32 = (x_f32 > 1.0)? 1.0 : x_f32; - x_f32 = (x_f32 < 0.0)? 0.0 : x_f32; - } else if (texref->addressMode[0] == cudaAddressModeWrap) { - x_f32 = x_f32 - floor(x_f32); - } - - if( texref->filterMode == cudaFilterModeLinear ) { - float xb = x_f32 * width - 0.5; - alpha = xb - floor(xb); - alpha = reduce_precision(alpha,9); - beta = 0.0; - - x = (int)floor(xb); - y = 0; - } else { - x = (int) floor(x_f32 * width); - y = 0; - } + assert(c_type == F32_TYPE); + x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; + if (texref->addressMode[0] == cudaAddressModeClamp) { + x_f32 = (x_f32 > 1.0) ? 1.0 : x_f32; + x_f32 = (x_f32 < 0.0) ? 0.0 : x_f32; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x_f32 = x_f32 - floor(x_f32); + } + + if (texref->filterMode == cudaFilterModeLinear) { + float xb = x_f32 * width - 0.5; + alpha = xb - floor(xb); + alpha = reduce_precision(alpha, 9); + beta = 0.0; + + x = (int)floor(xb); + y = 0; + } else { + x = (int)floor(x_f32 * width); + y = 0; + } } else { - switch ( c_type ) { - case S32_TYPE: + switch (c_type) { + case S32_TYPE: x = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].s32; - assert(texref->filterMode == cudaFilterModePoint); - break; - case F32_TYPE: + assert(texref->filterMode == cudaFilterModePoint); + break; + case F32_TYPE: x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - alpha = x_f32 - floor(x_f32); // offset into subtexel (for linear sampling) - x = (int) x_f32; - break; - default: assert(0 && "Unsupported texture coordinate type."); - } - // handle texture fetch that exceeded boundaries - if (texref->addressMode[0] == cudaAddressModeClamp) { - x = (x > width - 1)? (width - 1) : x; - x = (x < 0)? 0 : x; - } else if (texref->addressMode[0] == cudaAddressModeWrap) { - x = x % width; - } + alpha = x_f32 - + floor(x_f32); // offset into subtexel (for linear sampling) + x = (int)x_f32; + break; + default: + assert(0 && "Unsupported texture coordinate type."); + } + // handle texture fetch that exceeded boundaries + if (texref->addressMode[0] == cudaAddressModeClamp) { + x = (x > width - 1) ? (width - 1) : x; + x = (x < 0) ? 0 : x; + } else if (texref->addressMode[0] == cudaAddressModeWrap) { + x = x % width; + } } - width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; - x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; + width *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; + x *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; tex_array_index = tex_array_base + x; break; - case GEOM_MODIFIER_2D: + case GEOM_MODIFIER_2D: width = cuArray->width; height = cuArray->height; if (texref->normalized) { - x_f32 = reduce_precision(thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32,16); - y_f32 = reduce_precision(thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32,15); - - if (texref->addressMode[0]) {//clamp - if (x_f32<0) x_f32 = 0; - if (x_f32>=1) x_f32 = 1 - 1/x_f32; - } else {//wrap - x_f32 = x_f32 - floor(x_f32); - } - if (texref->addressMode[1]) {//clamp - if (y_f32<0) y_f32 = 0; - if (y_f32>=1) y_f32 = 1 - 1/y_f32; - } else {//wrap - y_f32 = y_f32 - floor(y_f32); - } - - if( texref->filterMode == cudaFilterModeLinear ) { - float xb = x_f32 * width - 0.5; - float yb = y_f32 * height - 0.5; - alpha = xb - floor(xb); - beta = yb - floor(yb); - alpha = reduce_precision(alpha,9); - beta = reduce_precision(beta,9); - - x = (int)floor(xb); - y = (int)floor(yb); - } else { - x = (int) floor(x_f32 * width); - y = (int) floor(y_f32 * height); - } + x_f32 = reduce_precision( + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32, 16); + y_f32 = reduce_precision( + thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32, 15); + + if (texref->addressMode[0]) { // clamp + if (x_f32 < 0) x_f32 = 0; + if (x_f32 >= 1) x_f32 = 1 - 1 / x_f32; + } else { // wrap + x_f32 = x_f32 - floor(x_f32); + } + if (texref->addressMode[1]) { // clamp + if (y_f32 < 0) y_f32 = 0; + if (y_f32 >= 1) y_f32 = 1 - 1 / y_f32; + } else { // wrap + y_f32 = y_f32 - floor(y_f32); + } + + if (texref->filterMode == cudaFilterModeLinear) { + float xb = x_f32 * width - 0.5; + float yb = y_f32 * height - 0.5; + alpha = xb - floor(xb); + beta = yb - floor(yb); + alpha = reduce_precision(alpha, 9); + beta = reduce_precision(beta, 9); + + x = (int)floor(xb); + y = (int)floor(yb); + } else { + x = (int)floor(x_f32 * width); + y = (int)floor(y_f32 * height); + } } else { - x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; - y_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32; - - alpha = x_f32 - floor(x_f32); - beta = y_f32 - floor(y_f32); - - x = (int) x_f32; - y = (int) y_f32; - if (texref->addressMode[0]) {//clamp - if (x<0) x = 0; - if (x>= (int)width) x = width-1; - } else {//wrap - x = x % width; - if (x < 0) x*= -1; - } - if (texref->addressMode[1]) {//clamp - if (y<0) y = 0; - if (y>= (int)height) y = height -1; - } else {//wrap - y = y % height; - if (y < 0) y *= -1; - } + x_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[0].f32; + y_f32 = thread->get_gpu()->gpgpu_ctx->func_sim->ptx_tex_regs[1].f32; + + alpha = x_f32 - floor(x_f32); + beta = y_f32 - floor(y_f32); + + x = (int)x_f32; + y = (int)y_f32; + if (texref->addressMode[0]) { // clamp + if (x < 0) x = 0; + if (x >= (int)width) x = width - 1; + } else { // wrap + x = x % width; + if (x < 0) x *= -1; + } + if (texref->addressMode[1]) { // clamp + if (y < 0) y = 0; + if (y >= (int)height) y = height - 1; + } else { // wrap + y = y % height; + if (y < 0) y *= -1; + } } - width *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; - x *= (cuArray->desc.w+cuArray->desc.x+cuArray->desc.y+cuArray->desc.z)/8; - tex_array_index = tex_array_base + (x + width*y); - break; - default: - assert(0); break; - } - switch ( to_type ) { - case U8_TYPE: - case U16_TYPE: - case U32_TYPE: - case B8_TYPE: - case B16_TYPE: - case B32_TYPE: - case S8_TYPE: - case S16_TYPE: - case S32_TYPE: { - unsigned long long elementOffset = 0; // offset into the next element - mem->read( tex_array_index, cuArray->desc.x/8, &data1.u32); - elementOffset += cuArray->desc.x/8; + width *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; + x *= (cuArray->desc.w + cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z) / + 8; + tex_array_index = tex_array_base + (x + width * y); + break; + default: + assert(0); + break; + } + switch (to_type) { + case U8_TYPE: + case U16_TYPE: + case U32_TYPE: + case B8_TYPE: + case B16_TYPE: + case B32_TYPE: + case S8_TYPE: + case S16_TYPE: + case S32_TYPE: { + unsigned long long elementOffset = 0; // offset into the next element + mem->read(tex_array_index, cuArray->desc.x / 8, &data1.u32); + elementOffset += cuArray->desc.x / 8; if (cuArray->desc.y) { - mem->read( tex_array_index + elementOffset, cuArray->desc.y/8, &data2.u32); - elementOffset += cuArray->desc.y/8; - if (cuArray->desc.z) { - mem->read( tex_array_index + elementOffset, cuArray->desc.z/8, &data3.u32); - elementOffset += cuArray->desc.z/8; - if (cuArray->desc.w) - mem->read( tex_array_index + elementOffset, cuArray->desc.w/8, &data4.u32); - } + mem->read(tex_array_index + elementOffset, cuArray->desc.y / 8, + &data2.u32); + elementOffset += cuArray->desc.y / 8; + if (cuArray->desc.z) { + mem->read(tex_array_index + elementOffset, cuArray->desc.z / 8, + &data3.u32); + elementOffset += cuArray->desc.z / 8; + if (cuArray->desc.w) + mem->read(tex_array_index + elementOffset, cuArray->desc.w / 8, + &data4.u32); + } } break; - } - case B64_TYPE: - case U64_TYPE: - case S64_TYPE: - mem->read( tex_array_index, 8, &data1.u64); + } + case B64_TYPE: + case U64_TYPE: + case S64_TYPE: + mem->read(tex_array_index, 8, &data1.u64); if (cuArray->desc.y) { - mem->read( tex_array_index+8, 8, &data2.u64); - if (cuArray->desc.z) { - mem->read( tex_array_index+16, 8, &data3.u64); - if (cuArray->desc.w) - mem->read( tex_array_index+24, 8, &data4.u64); - } + mem->read(tex_array_index + 8, 8, &data2.u64); + if (cuArray->desc.z) { + mem->read(tex_array_index + 16, 8, &data3.u64); + if (cuArray->desc.w) mem->read(tex_array_index + 24, 8, &data4.u64); + } } break; - case F16_TYPE: assert(0); break; - case F32_TYPE: { - if( texref->filterMode == cudaFilterModeLinear ) { - texAddr_t b_lim = wrap; - if ( texref->addressMode[0] == cudaAddressModeClamp ) { - b_lim = clamp; - } - size_t elem_size = (cuArray->desc.x + cuArray->desc.y + cuArray->desc.z + cuArray->desc.w) / 8; - size_t elem_ofst = 0; - - data1.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.x / 8; - if (cuArray->desc.y) { - data2.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.y / 8; - if (cuArray->desc.z) { - data3.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - elem_ofst += cuArray->desc.z / 8; - if (cuArray->desc.w) - data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, height, elem_size, alpha, beta, b_lim); - } - } + case F16_TYPE: + assert(0); + break; + case F32_TYPE: { + if (texref->filterMode == cudaFilterModeLinear) { + texAddr_t b_lim = wrap; + if (texref->addressMode[0] == cudaAddressModeClamp) { + b_lim = clamp; + } + size_t elem_size = (cuArray->desc.x + cuArray->desc.y + + cuArray->desc.z + cuArray->desc.w) / + 8; + size_t elem_ofst = 0; + + data1.f32 = + tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, + height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.x / 8; + if (cuArray->desc.y) { + data2.f32 = + tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, + height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.y / 8; + if (cuArray->desc.z) { + data3.f32 = + tex_linf_sampling(mem, tex_array_base, x + elem_ofst, y, width, + height, elem_size, alpha, beta, b_lim); + elem_ofst += cuArray->desc.z / 8; + if (cuArray->desc.w) + data4.f32 = tex_linf_sampling(mem, tex_array_base, x + elem_ofst, + y, width, height, elem_size, alpha, + beta, b_lim); + } + } } else { - mem->read( tex_array_index, cuArray->desc.x/8, &data1.f32); - if (cuArray->desc.y) { - mem->read( tex_array_index+4, cuArray->desc.y/8, &data2.f32); - if (cuArray->desc.z) { - mem->read( tex_array_index+8, cuArray->desc.z/8, &data3.f32); - if (cuArray->desc.w) - mem->read( tex_array_index+12, cuArray->desc.w/8, &data4.f32); - } - } + mem->read(tex_array_index, cuArray->desc.x / 8, &data1.f32); + if (cuArray->desc.y) { + mem->read(tex_array_index + 4, cuArray->desc.y / 8, &data2.f32); + if (cuArray->desc.z) { + mem->read(tex_array_index + 8, cuArray->desc.z / 8, &data3.f32); + if (cuArray->desc.w) + mem->read(tex_array_index + 12, cuArray->desc.w / 8, &data4.f32); + } + } } - } break; - case F64_TYPE: - case FF64_TYPE: - mem->read( tex_array_index, 8, &data1.f64); + } break; + case F64_TYPE: + case FF64_TYPE: + mem->read(tex_array_index, 8, &data1.f64); if (cuArray->desc.y) { - mem->read( tex_array_index+8, 8, &data2.f64); - if (cuArray->desc.z) { - mem->read( tex_array_index+16, 8, &data3.f64); - if (cuArray->desc.w) - mem->read( tex_array_index+24, 8, &data4.f64); - } + mem->read(tex_array_index + 8, 8, &data2.f64); + if (cuArray->desc.z) { + mem->read(tex_array_index + 16, 8, &data3.f64); + if (cuArray->desc.w) mem->read(tex_array_index + 24, 8, &data4.f64); + } } break; - default: assert(0); break; - } - int x_block_coord, y_block_coord, memreqindex, blockoffset; + default: + assert(0); + break; + } + int x_block_coord, y_block_coord, memreqindex, blockoffset; - switch (dimension) { - case GEOM_MODIFIER_1D: + switch (dimension) { + case GEOM_MODIFIER_1D: thread->m_last_effective_address = tex_array_index; break; - case GEOM_MODIFIER_2D: + case GEOM_MODIFIER_2D: x_block_coord = x >> (texInfo->Tx_numbits + texInfo->texel_size_numbits); y_block_coord = y >> texInfo->Ty_numbits; - memreqindex = ((y_block_coord*cuArray->width/texInfo->Tx)+x_block_coord)<<6; + memreqindex = + ((y_block_coord * cuArray->width / texInfo->Tx) + x_block_coord) << 6; - blockoffset = (x%(texInfo->Tx*texInfo->texel_size) + (y%(texInfo->Ty)<<(texInfo->Tx_numbits + texInfo->texel_size_numbits))); + blockoffset = (x % (texInfo->Tx * texInfo->texel_size) + + (y % (texInfo->Ty) + << (texInfo->Tx_numbits + texInfo->texel_size_numbits))); memreqindex += blockoffset; - thread->m_last_effective_address = tex_array_base + memreqindex;//tex_array_index; + thread->m_last_effective_address = + tex_array_base + memreqindex; // tex_array_index; break; - default: + default: assert(0); - } - thread->m_last_memory_space = tex_space; - - // normalize output into floating point numbers according to the texture read mode - if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { - textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); - } else { - assert(texAttr->m_readmode == cudaReadModeElementType); - } - - thread->set_vector_operand_values(dst,data1,data2,data3,data4); -} - -void txq_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void trap_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vabsdiff_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vadd_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vmad_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vmax_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vmin_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vset_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vshl_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vshr_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } -void vsub_impl( const ptx_instruction *pI, ptx_thread_info *thread ) { inst_not_implemented(pI); } - -void vote_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - static bool first_in_warp = true; - static bool and_all; - static bool or_all; - static unsigned int ballot_result; - static std::list threads_in_warp; - static unsigned last_tid; - - if( first_in_warp ) { - first_in_warp = false; - threads_in_warp.clear(); - and_all = true; - or_all = false; - ballot_result = 0; - int offset=31; - while( (offset>=0) && !pI->active(offset) ) - offset--; - assert( offset >= 0 ); - last_tid = (thread->get_hw_tid() - (thread->get_hw_tid()%pI->warp_size())) + offset; - } - - ptx_reg_t src1_data; - const operand_info &src1 = pI->src1(); - src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); - - //predicate value was changed so the lowest bit being set means the zero flag is set. - //As a result, the value of src1_data.pred must be inverted to get proper behavior - bool pred_value = !(src1_data.pred & 0x0001); - bool invert = src1.is_neg_pred(); - - threads_in_warp.push_back(thread); - and_all &= (invert ^ pred_value); - or_all |= (invert ^ pred_value); - - // vote.ballot - if (invert ^ pred_value) { - int lane_id = thread->get_hw_tid() % pI->warp_size(); - ballot_result |= (1 << lane_id); - } - - if( thread->get_hw_tid() == last_tid ) { - if (pI->vote_mode() == ptx_instruction::vote_ballot) { - ptx_reg_t data = ballot_result; - for( std::list::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { - const operand_info &dst = pI->dst(); - (*t)->set_operand_value(dst,data, pI->get_type(), (*t), pI); - } - } else { - bool pred_value = false; + } + thread->m_last_memory_space = tex_space; + + // normalize output into floating point numbers according to the texture read + // mode + if (texAttr->m_readmode == cudaReadModeNormalizedFloat) { + textureNormalizeOutput(cuArray->desc, data1, data2, data3, data4); + } else { + assert(texAttr->m_readmode == cudaReadModeElementType); + } - switch( pI->vote_mode() ) { - case ptx_instruction::vote_any: pred_value = or_all; break; - case ptx_instruction::vote_all: pred_value = and_all; break; - case ptx_instruction::vote_uni: pred_value = (or_all ^ and_all); break; - default: - abort(); - } - ptx_reg_t data; - data.pred = pred_value?0:1; //the way ptxplus handles the zero flag, 1 = false and 0 = true - - for( std::list::iterator t=threads_in_warp.begin(); t!=threads_in_warp.end(); ++t ) { - const operand_info &dst = pI->dst(); - (*t)->set_operand_value(dst,data, PRED_TYPE, (*t), pI); - } + thread->set_vector_operand_values(dst, data1, data2, data3, data4); +} + +void txq_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void trap_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vabsdiff_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vadd_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vmad_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vmax_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vmin_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vset_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vshl_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vshr_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} +void vsub_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + inst_not_implemented(pI); +} + +void vote_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + static bool first_in_warp = true; + static bool and_all; + static bool or_all; + static unsigned int ballot_result; + static std::list threads_in_warp; + static unsigned last_tid; + + if (first_in_warp) { + first_in_warp = false; + threads_in_warp.clear(); + and_all = true; + or_all = false; + ballot_result = 0; + int offset = 31; + while ((offset >= 0) && !pI->active(offset)) offset--; + assert(offset >= 0); + last_tid = + (thread->get_hw_tid() - (thread->get_hw_tid() % pI->warp_size())) + + offset; + } + + ptx_reg_t src1_data; + const operand_info &src1 = pI->src1(); + src1_data = thread->get_operand_value(src1, pI->dst(), PRED_TYPE, thread, 1); + + // predicate value was changed so the lowest bit being set means the zero flag + // is set. As a result, the value of src1_data.pred must be inverted to get + // proper behavior + bool pred_value = !(src1_data.pred & 0x0001); + bool invert = src1.is_neg_pred(); + + threads_in_warp.push_back(thread); + and_all &= (invert ^ pred_value); + or_all |= (invert ^ pred_value); + + // vote.ballot + if (invert ^ pred_value) { + int lane_id = thread->get_hw_tid() % pI->warp_size(); + ballot_result |= (1 << lane_id); + } + + if (thread->get_hw_tid() == last_tid) { + if (pI->vote_mode() == ptx_instruction::vote_ballot) { + ptx_reg_t data = ballot_result; + for (std::list::iterator t = threads_in_warp.begin(); + t != threads_in_warp.end(); ++t) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst, data, pI->get_type(), (*t), pI); } - first_in_warp = true; - } -} - -void xor_impl( const ptx_instruction *pI, ptx_thread_info *thread ) -{ - ptx_reg_t src1_data, src2_data, data; - - const operand_info &dst = pI->dst(); - const operand_info &src1 = pI->src1(); - const operand_info &src2 = pI->src2(); - - unsigned i_type = pI->get_type(); - src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); - src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); - - //the way ptxplus handles predicates: 1 = false and 0 = true - if(i_type == PRED_TYPE) - data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); - else - data.u64 = src1_data.u64 ^ src2_data.u64; - - thread->set_operand_value(dst,data, i_type, thread, pI); -} - -void inst_not_implemented( const ptx_instruction * pI ) -{ - printf("GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", - pI->source_file(), - pI->source_line(), - pI->get_opcode_cstr() ); - abort(); -} - -ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, operand_info dstInfo, unsigned type, ptx_thread_info *thread) -{ - ptx_reg_t result; - memory_space *mem = NULL; - size_t size; - int t; - result.u64=0; - - //complete other cases for reading from memory, such as reading from other const memory - if(opInfo.get_addr_space() == global_space) - { - mem = thread->get_global_memory(); - type_info_key::type_decode(type,size,t); - mem->read(opData.u32,size/8,&result.u64); - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(result,size,dstInfo); - } - else if(opInfo.get_addr_space() == shared_space) - { - mem = thread->m_shared_mem; - type_info_key::type_decode(type,size,t); - mem->read(opData.u32,size/8,&result.u64); - - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(result,size,dstInfo); - - } - else if(opInfo.get_addr_space() == const_space) - { - mem = thread->get_global_memory(); - type_info_key::type_decode(type,size,t); - - mem->read((opData.u32 + opInfo.get_const_mem_offset()),size/8,&result.u64); - - if( type == S16_TYPE || type == S32_TYPE ) - sign_extend(result,size,dstInfo); - } - else - { - result = opData; - } - - if(opInfo.get_operand_lohi() == 1) - { - result.u64 = result.u64 & 0xFFFF; - } - else if(opInfo.get_operand_lohi() == 2) - { - result.u64 = (result.u64>>16) & 0xFFFF; - } + } else { + bool pred_value = false; + + switch (pI->vote_mode()) { + case ptx_instruction::vote_any: + pred_value = or_all; + break; + case ptx_instruction::vote_all: + pred_value = and_all; + break; + case ptx_instruction::vote_uni: + pred_value = (or_all ^ and_all); + break; + default: + abort(); + } + ptx_reg_t data; + data.pred = pred_value ? 0 : 1; // the way ptxplus handles the zero flag, + // 1 = false and 0 = true + + for (std::list::iterator t = threads_in_warp.begin(); + t != threads_in_warp.end(); ++t) { + const operand_info &dst = pI->dst(); + (*t)->set_operand_value(dst, data, PRED_TYPE, (*t), pI); + } + } + first_in_warp = true; + } +} + +void xor_impl(const ptx_instruction *pI, ptx_thread_info *thread) { + ptx_reg_t src1_data, src2_data, data; + + const operand_info &dst = pI->dst(); + const operand_info &src1 = pI->src1(); + const operand_info &src2 = pI->src2(); + + unsigned i_type = pI->get_type(); + src1_data = thread->get_operand_value(src1, dst, i_type, thread, 1); + src2_data = thread->get_operand_value(src2, dst, i_type, thread, 1); + + // the way ptxplus handles predicates: 1 = false and 0 = true + if (i_type == PRED_TYPE) + data.pred = ~(~(src1_data.pred) ^ ~(src2_data.pred)); + else + data.u64 = src1_data.u64 ^ src2_data.u64; - if(opInfo.get_operand_neg() == true) { - result.f32 = -result.f32; - } + thread->set_operand_value(dst, data, i_type, thread, pI); +} - return result; +void inst_not_implemented(const ptx_instruction *pI) { + printf( + "GPGPU-Sim PTX: ERROR (%s:%u) instruction \"%s\" not (yet) implemented\n", + pI->source_file(), pI->source_line(), pI->get_opcode_cstr()); + abort(); } +ptx_reg_t srcOperandModifiers(ptx_reg_t opData, operand_info opInfo, + operand_info dstInfo, unsigned type, + ptx_thread_info *thread) { + ptx_reg_t result; + memory_space *mem = NULL; + size_t size; + int t; + result.u64 = 0; + + // complete other cases for reading from memory, such as reading from other + // const memory + if (opInfo.get_addr_space() == global_space) { + mem = thread->get_global_memory(); + type_info_key::type_decode(type, size, t); + mem->read(opData.u32, size / 8, &result.u64); + if (type == S16_TYPE || type == S32_TYPE) + sign_extend(result, size, dstInfo); + } else if (opInfo.get_addr_space() == shared_space) { + mem = thread->m_shared_mem; + type_info_key::type_decode(type, size, t); + mem->read(opData.u32, size / 8, &result.u64); + + if (type == S16_TYPE || type == S32_TYPE) + sign_extend(result, size, dstInfo); + + } else if (opInfo.get_addr_space() == const_space) { + mem = thread->get_global_memory(); + type_info_key::type_decode(type, size, t); + + mem->read((opData.u32 + opInfo.get_const_mem_offset()), size / 8, + &result.u64); + + if (type == S16_TYPE || type == S32_TYPE) + sign_extend(result, size, dstInfo); + } else { + result = opData; + } + + if (opInfo.get_operand_lohi() == 1) { + result.u64 = result.u64 & 0xFFFF; + } else if (opInfo.get_operand_lohi() == 2) { + result.u64 = (result.u64 >> 16) & 0xFFFF; + } + + if (opInfo.get_operand_neg() == true) { + result.f32 = -result.f32; + } + + return result; +} diff --git a/src/cuda-sim/memory.cc b/src/cuda-sim/memory.cc index 4b2acdf..1323837 100644 --- a/src/cuda-sim/memory.cc +++ b/src/cuda-sim/memory.cc @@ -7,210 +7,227 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "memory.h" #include -#include "../debug.h" #include "../../libcuda/gpgpu_context.h" +#include "../debug.h" -template memory_space_impl::memory_space_impl( std::string name, unsigned hash_size ) -{ - m_name = name; - MEM_MAP_RESIZE(hash_size); - - m_log2_block_size = -1; - for( unsigned n=0, mask=1; mask != 0; mask <<= 1, n++ ) { - if( BSIZE & mask ) { - assert( m_log2_block_size == (unsigned)-1 ); - m_log2_block_size = n; - } - } - assert( m_log2_block_size != (unsigned)-1 ); +template +memory_space_impl::memory_space_impl(std::string name, + unsigned hash_size) { + m_name = name; + MEM_MAP_RESIZE(hash_size); + + m_log2_block_size = -1; + for (unsigned n = 0, mask = 1; mask != 0; mask <<= 1, n++) { + if (BSIZE & mask) { + assert(m_log2_block_size == (unsigned)-1); + m_log2_block_size = n; + } + } + assert(m_log2_block_size != (unsigned)-1); } -template void memory_space_impl::write_only( mem_addr_t offset, mem_addr_t index, size_t length, const void *data) -{ - m_data[index].write(offset,length,(const unsigned char*)data); +template +void memory_space_impl::write_only(mem_addr_t offset, mem_addr_t index, + size_t length, const void *data) { + m_data[index].write(offset, length, (const unsigned char *)data); } -template void memory_space_impl::write( mem_addr_t addr, size_t length, const void *data, class ptx_thread_info *thd, const ptx_instruction *pI) -{ - - mem_addr_t index = addr >> m_log2_block_size; - - if ( (addr+length) <= (index+1)*BSIZE ) { - // fast route for intra-block access - unsigned offset = addr & (BSIZE-1); - unsigned nbytes = length; - m_data[index].write(offset,nbytes,(const unsigned char*)data); - } else { - // slow route for inter-block access - unsigned nbytes_remain = length; - unsigned src_offset = 0; - mem_addr_t current_addr = addr; - - while (nbytes_remain > 0) { - unsigned offset = current_addr & (BSIZE-1); - mem_addr_t page = current_addr >> m_log2_block_size; - mem_addr_t access_limit = offset + nbytes_remain; - if (access_limit > BSIZE) { - access_limit = BSIZE; - } - - size_t tx_bytes = access_limit - offset; - m_data[page].write(offset, tx_bytes, &((const unsigned char*)data)[src_offset]); - - // advance pointers - src_offset += tx_bytes; - current_addr += tx_bytes; - nbytes_remain -= tx_bytes; - } - assert(nbytes_remain == 0); - } - if( !m_watchpoints.empty() ) { - std::map::iterator i; - for( i=m_watchpoints.begin(); i!=m_watchpoints.end(); i++ ) { - mem_addr_t wa = i->second; - if( ((addr<=wa) && ((addr+length)>wa)) || ((addr>wa) && (addr < (wa+4))) ) - thd->get_gpu()->gpgpu_ctx->the_gpgpusim->g_the_gpu->hit_watchpoint(i->first,thd,pI); +template +void memory_space_impl::write(mem_addr_t addr, size_t length, + const void *data, + class ptx_thread_info *thd, + const ptx_instruction *pI) { + mem_addr_t index = addr >> m_log2_block_size; + + if ((addr + length) <= (index + 1) * BSIZE) { + // fast route for intra-block access + unsigned offset = addr & (BSIZE - 1); + unsigned nbytes = length; + m_data[index].write(offset, nbytes, (const unsigned char *)data); + } else { + // slow route for inter-block access + unsigned nbytes_remain = length; + unsigned src_offset = 0; + mem_addr_t current_addr = addr; + + while (nbytes_remain > 0) { + unsigned offset = current_addr & (BSIZE - 1); + mem_addr_t page = current_addr >> m_log2_block_size; + mem_addr_t access_limit = offset + nbytes_remain; + if (access_limit > BSIZE) { + access_limit = BSIZE; } - } + + size_t tx_bytes = access_limit - offset; + m_data[page].write(offset, tx_bytes, + &((const unsigned char *)data)[src_offset]); + + // advance pointers + src_offset += tx_bytes; + current_addr += tx_bytes; + nbytes_remain -= tx_bytes; + } + assert(nbytes_remain == 0); + } + if (!m_watchpoints.empty()) { + std::map::iterator i; + for (i = m_watchpoints.begin(); i != m_watchpoints.end(); i++) { + mem_addr_t wa = i->second; + if (((addr <= wa) && ((addr + length) > wa)) || + ((addr > wa) && (addr < (wa + 4)))) + thd->get_gpu()->gpgpu_ctx->the_gpgpusim->g_the_gpu->hit_watchpoint( + i->first, thd, pI); + } + } } -template void memory_space_impl::read_single_block( mem_addr_t blk_idx, mem_addr_t addr, size_t length, void *data) const -{ - if ((addr + length) > (blk_idx + 1) * BSIZE) { - printf("GPGPU-Sim PTX: ERROR * access to memory \'%s\' is unaligned : addr=0x%x, length=%zu\n", - m_name.c_str(), addr, length); - printf("GPGPU-Sim PTX: (addr+length)=0x%lx > 0x%x=(index+1)*BSIZE, index=0x%x, BSIZE=0x%x\n", - (addr+length),(blk_idx+1)*BSIZE, blk_idx, BSIZE); - throw 1; - } - typename map_t::const_iterator i = m_data.find(blk_idx); - if( i == m_data.end() ) { - for( size_t n=0; n < length; n++ ) - ((unsigned char*)data)[n] = (unsigned char) 0; - //printf("GPGPU-Sim PTX: WARNING reading %zu bytes from unititialized memory at address 0x%x in space %s\n", length, addr, m_name.c_str() ); - } else { - unsigned offset = addr & (BSIZE-1); - unsigned nbytes = length; - i->second.read(offset,nbytes,(unsigned char*)data); - } +template +void memory_space_impl::read_single_block(mem_addr_t blk_idx, + mem_addr_t addr, size_t length, + void *data) const { + if ((addr + length) > (blk_idx + 1) * BSIZE) { + printf( + "GPGPU-Sim PTX: ERROR * access to memory \'%s\' is unaligned : " + "addr=0x%x, length=%zu\n", + m_name.c_str(), addr, length); + printf( + "GPGPU-Sim PTX: (addr+length)=0x%lx > 0x%x=(index+1)*BSIZE, " + "index=0x%x, BSIZE=0x%x\n", + (addr + length), (blk_idx + 1) * BSIZE, blk_idx, BSIZE); + throw 1; + } + typename map_t::const_iterator i = m_data.find(blk_idx); + if (i == m_data.end()) { + for (size_t n = 0; n < length; n++) + ((unsigned char *)data)[n] = (unsigned char)0; + // printf("GPGPU-Sim PTX: WARNING reading %zu bytes from unititialized + // memory at address 0x%x in space %s\n", length, addr, m_name.c_str() ); + } else { + unsigned offset = addr & (BSIZE - 1); + unsigned nbytes = length; + i->second.read(offset, nbytes, (unsigned char *)data); + } } -template void memory_space_impl::read( mem_addr_t addr, size_t length, void *data ) const -{ - mem_addr_t index = addr >> m_log2_block_size; - if ((addr+length) <= (index+1)*BSIZE ) { - // fast route for intra-block access - read_single_block(index, addr, length, data); - } else { - // slow route for inter-block access - unsigned nbytes_remain = length; - unsigned dst_offset = 0; - mem_addr_t current_addr = addr; - - while (nbytes_remain > 0) { - unsigned offset = current_addr & (BSIZE-1); - mem_addr_t page = current_addr >> m_log2_block_size; - mem_addr_t access_limit = offset + nbytes_remain; - if (access_limit > BSIZE) { - access_limit = BSIZE; - } - - size_t tx_bytes = access_limit - offset; - read_single_block(page, current_addr, tx_bytes, &((unsigned char*)data)[dst_offset]); - - // advance pointers - dst_offset += tx_bytes; - current_addr += tx_bytes; - nbytes_remain -= tx_bytes; +template +void memory_space_impl::read(mem_addr_t addr, size_t length, + void *data) const { + mem_addr_t index = addr >> m_log2_block_size; + if ((addr + length) <= (index + 1) * BSIZE) { + // fast route for intra-block access + read_single_block(index, addr, length, data); + } else { + // slow route for inter-block access + unsigned nbytes_remain = length; + unsigned dst_offset = 0; + mem_addr_t current_addr = addr; + + while (nbytes_remain > 0) { + unsigned offset = current_addr & (BSIZE - 1); + mem_addr_t page = current_addr >> m_log2_block_size; + mem_addr_t access_limit = offset + nbytes_remain; + if (access_limit > BSIZE) { + access_limit = BSIZE; } - assert(nbytes_remain == 0); - } + + size_t tx_bytes = access_limit - offset; + read_single_block(page, current_addr, tx_bytes, + &((unsigned char *)data)[dst_offset]); + + // advance pointers + dst_offset += tx_bytes; + current_addr += tx_bytes; + nbytes_remain -= tx_bytes; + } + assert(nbytes_remain == 0); + } } -template void memory_space_impl::print( const char *format, FILE *fout ) const -{ - typename map_t::const_iterator i_page; +template +void memory_space_impl::print(const char *format, FILE *fout) const { + typename map_t::const_iterator i_page; - for ( i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { - fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); - i_page->second.print(format, fout); - } + for (i_page = m_data.begin(); i_page != m_data.end(); ++i_page) { + fprintf(fout, "%s %08x:", m_name.c_str(), i_page->first); + i_page->second.print(format, fout); + } } -template void memory_space_impl::set_watch( addr_t addr, unsigned watchpoint ) -{ - m_watchpoints[watchpoint]=addr; +template +void memory_space_impl::set_watch(addr_t addr, unsigned watchpoint) { + m_watchpoints[watchpoint] = addr; } template class memory_space_impl<32>; template class memory_space_impl<64>; template class memory_space_impl<8192>; -template class memory_space_impl<16*1024>; +template class memory_space_impl<16 * 1024>; -void g_print_memory_space(memory_space *mem, const char *format = "%08x", FILE *fout = stdout) -{ - mem->print(format,fout); +void g_print_memory_space(memory_space *mem, const char *format = "%08x", + FILE *fout = stdout) { + mem->print(format, fout); } #ifdef UNIT_TEST -int main(int argc, char *argv[] ) -{ - int errors_found=0; - memory_space *mem = new memory_space_impl<32>("test",4); - // write address to [address] - for( mem_addr_t addr=0; addr < 16*1024; addr+=4) - mem->write(addr,4,&addr,NULL,NULL); - - for( mem_addr_t addr=0; addr < 16*1024; addr+=4) { - unsigned tmp=0; - mem->read(addr,4,&tmp); - if( tmp != addr ) { - errors_found=1; - printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, addr ); - } - } - - for( mem_addr_t addr=0; addr < 16*1024; addr+=1) { - unsigned char val = (addr + 128) % 256; - mem->write(addr,1,&val,NULL,NULL); - } - - for( mem_addr_t addr=0; addr < 16*1024; addr+=1) { - unsigned tmp=0; - mem->read(addr,1,&tmp); - unsigned char val = (addr + 128) % 256; - if( tmp != val ) { - errors_found=1; - printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, (unsigned)val ); - } - } - - if( errors_found ) { - printf("SUMMARY: ERRORS FOUND\n"); - } else { - printf("SUMMARY: UNIT TEST PASSED\n"); - } +int main(int argc, char *argv[]) { + int errors_found = 0; + memory_space *mem = new memory_space_impl<32>("test", 4); + // write address to [address] + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 4) + mem->write(addr, 4, &addr, NULL, NULL); + + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 4) { + unsigned tmp = 0; + mem->read(addr, 4, &tmp); + if (tmp != addr) { + errors_found = 1; + printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, addr); + } + } + + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 1) { + unsigned char val = (addr + 128) % 256; + mem->write(addr, 1, &val, NULL, NULL); + } + + for (mem_addr_t addr = 0; addr < 16 * 1024; addr += 1) { + unsigned tmp = 0; + mem->read(addr, 1, &tmp); + unsigned char val = (addr + 128) % 256; + if (tmp != val) { + errors_found = 1; + printf("ERROR ** mem[0x%x] = 0x%x, expected 0x%x\n", addr, tmp, + (unsigned)val); + } + } + + if (errors_found) { + printf("SUMMARY: ERRORS FOUND\n"); + } else { + printf("SUMMARY: UNIT TEST PASSED\n"); + } } #endif diff --git a/src/cuda-sim/memory.h b/src/cuda-sim/memory.h index ab588bc..5850aa1 100644 --- a/src/cuda-sim/memory.h +++ b/src/cuda-sim/memory.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef memory_h_INCLUDED #define memory_h_INCLUDED @@ -33,101 +34,97 @@ #include "../tr1_hash_map.h" #define mem_map tr1_hash_map #if tr1_hash_map_ismap == 1 - #define MEM_MAP_RESIZE(hash_size) +#define MEM_MAP_RESIZE(hash_size) #else - #define MEM_MAP_RESIZE(hash_size) (m_data.rehash(hash_size)) +#define MEM_MAP_RESIZE(hash_size) (m_data.rehash(hash_size)) #endif #include -#include #include -#include -#include #include +#include +#include +#include typedef address_type mem_addr_t; -#define MEM_BLOCK_SIZE (4*1024) - -template class mem_storage { -public: - mem_storage( const mem_storage &another ) - { - m_data = (unsigned char*)calloc(1,BSIZE); - memcpy(m_data,another.m_data,BSIZE); - } - mem_storage() - { - m_data = (unsigned char*)calloc(1,BSIZE); - } - ~mem_storage() - { - free(m_data); - } - - void write( unsigned offset, size_t length, const unsigned char *data ) - { - assert( offset + length <= BSIZE ); - memcpy(m_data+offset,data,length); - } - - void read( unsigned offset, size_t length, unsigned char *data ) const - { - assert( offset + length <= BSIZE ); - memcpy(data,m_data+offset,length); - } - - void print( const char *format, FILE *fout ) const - { - unsigned int *i_data = (unsigned int*)m_data; - for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { - if (d % 1 == 0) { - fprintf(fout, "\n"); - } - fprintf(fout, format, i_data[d]); - fprintf(fout, " "); +#define MEM_BLOCK_SIZE (4 * 1024) + +template +class mem_storage { + public: + mem_storage(const mem_storage &another) { + m_data = (unsigned char *)calloc(1, BSIZE); + memcpy(m_data, another.m_data, BSIZE); + } + mem_storage() { m_data = (unsigned char *)calloc(1, BSIZE); } + ~mem_storage() { free(m_data); } + + void write(unsigned offset, size_t length, const unsigned char *data) { + assert(offset + length <= BSIZE); + memcpy(m_data + offset, data, length); + } + + void read(unsigned offset, size_t length, unsigned char *data) const { + assert(offset + length <= BSIZE); + memcpy(data, m_data + offset, length); + } + + void print(const char *format, FILE *fout) const { + unsigned int *i_data = (unsigned int *)m_data; + for (int d = 0; d < (BSIZE / sizeof(unsigned int)); d++) { + if (d % 1 == 0) { + fprintf(fout, "\n"); } - fprintf(fout, "\n"); - fflush(fout); - } - -private: - unsigned m_nbytes; - unsigned char *m_data; + fprintf(fout, format, i_data[d]); + fprintf(fout, " "); + } + fprintf(fout, "\n"); + fflush(fout); + } + + private: + unsigned m_nbytes; + unsigned char *m_data; }; class ptx_thread_info; class ptx_instruction; -class memory_space -{ -public: - virtual ~memory_space() {} - virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ) = 0; - virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data ) = 0; - virtual void read( mem_addr_t addr, size_t length, void *data ) const = 0; - virtual void print( const char *format, FILE *fout ) const = 0; - virtual void set_watch( addr_t addr, unsigned watchpoint ) = 0; +class memory_space { + public: + virtual ~memory_space() {} + virtual void write(mem_addr_t addr, size_t length, const void *data, + ptx_thread_info *thd, const ptx_instruction *pI) = 0; + virtual void write_only(mem_addr_t index, mem_addr_t offset, size_t length, + const void *data) = 0; + virtual void read(mem_addr_t addr, size_t length, void *data) const = 0; + virtual void print(const char *format, FILE *fout) const = 0; + virtual void set_watch(addr_t addr, unsigned watchpoint) = 0; }; -template class memory_space_impl : public memory_space { -public: - memory_space_impl( std::string name, unsigned hash_size ); - - virtual void write( mem_addr_t addr, size_t length, const void *data, ptx_thread_info *thd, const ptx_instruction *pI ); - virtual void write_only( mem_addr_t index, mem_addr_t offset, size_t length, const void *data); - virtual void read( mem_addr_t addr, size_t length, void *data ) const; - virtual void print( const char *format, FILE *fout ) const; - - virtual void set_watch( addr_t addr, unsigned watchpoint ); - -private: - void read_single_block( mem_addr_t blk_idx, mem_addr_t addr, size_t length, void *data) const; - std::string m_name; - unsigned m_log2_block_size; - typedef mem_map > map_t; - map_t m_data; - std::map m_watchpoints; +template +class memory_space_impl : public memory_space { + public: + memory_space_impl(std::string name, unsigned hash_size); + + virtual void write(mem_addr_t addr, size_t length, const void *data, + ptx_thread_info *thd, const ptx_instruction *pI); + virtual void write_only(mem_addr_t index, mem_addr_t offset, size_t length, + const void *data); + virtual void read(mem_addr_t addr, size_t length, void *data) const; + virtual void print(const char *format, FILE *fout) const; + + virtual void set_watch(addr_t addr, unsigned watchpoint); + + private: + void read_single_block(mem_addr_t blk_idx, mem_addr_t addr, size_t length, + void *data) const; + std::string m_name; + unsigned m_log2_block_size; + typedef mem_map > map_t; + map_t m_data; + std::map m_watchpoints; }; #endif diff --git a/src/cuda-sim/opcodes.h b/src/cuda-sim/opcodes.h index 86d3b99..dc1e8c9 100644 --- a/src/cuda-sim/opcodes.h +++ b/src/cuda-sim/opcodes.h @@ -7,71 +7,71 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef opcodes_h_included #define opcodes_h_included enum opcode_t { -#define OP_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, -#define OP_W_DEF(OP,FUNC,STR,DST,CLASSIFICATION) OP, +#define OP_DEF(OP, FUNC, STR, DST, CLASSIFICATION) OP, +#define OP_W_DEF(OP, FUNC, STR, DST, CLASSIFICATION) OP, #include "opcodes.def" - NUM_OPCODES + NUM_OPCODES #undef OP_DEF #undef OP_W_DEF }; enum special_regs { - CLOCK_REG, - HALFCLOCK_ID, - CLOCK64_REG, - CTAID_REG, - ENVREG_REG, - GRIDID_REG, - LANEID_REG, - LANEMASK_EQ_REG, - LANEMASK_LE_REG, - LANEMASK_LT_REG, - LANEMASK_GE_REG, - LANEMASK_GT_REG, - NCTAID_REG, - NTID_REG, - NSMID_REG, - NWARPID_REG, - PM_REG, - SMID_REG, - TID_REG, - WARPID_REG, - WARPSZ_REG + CLOCK_REG, + HALFCLOCK_ID, + CLOCK64_REG, + CTAID_REG, + ENVREG_REG, + GRIDID_REG, + LANEID_REG, + LANEMASK_EQ_REG, + LANEMASK_LE_REG, + LANEMASK_LT_REG, + LANEMASK_GE_REG, + LANEMASK_GT_REG, + NCTAID_REG, + NTID_REG, + NSMID_REG, + NWARPID_REG, + PM_REG, + SMID_REG, + TID_REG, + WARPID_REG, + WARPSZ_REG }; -enum wmma_type{ - LOAD_A, - LOAD_B, - LOAD_C, - STORE_D, - MMA, - ROW, - COL, - M16N16K16, - M32N8K16, - M8N32K16 - +enum wmma_type { + LOAD_A, + LOAD_B, + LOAD_C, + STORE_D, + MMA, + ROW, + COL, + M16N16K16, + M32N8K16, + M8N32K16 }; #endif diff --git a/src/cuda-sim/ptx-stats.cc b/src/cuda-sim/ptx-stats.cc index 22517df..9f7e760 100644 --- a/src/cuda-sim/ptx-stats.cc +++ b/src/cuda-sim/ptx-stats.cc @@ -7,266 +7,285 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#include "ptx_ir.h" -#include "ptx_sim.h" #include "ptx-stats.h" -#include "../option_parser.h" #include #include -#include "../tr1_hash_map.h" #include "../../libcuda/gpgpu_context.h" +#include "../option_parser.h" +#include "../tr1_hash_map.h" +#include "ptx_ir.h" +#include "ptx_sim.h" -void ptx_stats::ptx_file_line_stats_options(option_parser_t opp) -{ - option_parser_register(opp, "-enable_ptx_file_line_stats", OPT_BOOL, - &enable_ptx_file_line_stats, - "Turn on PTX source line statistic profiling. (1 = On)", "1"); - option_parser_register(opp, "-ptx_line_stats_filename", OPT_CSTR, - &ptx_line_stats_filename, - "Output file for PTX source line statistics.", "gpgpu_inst_stats.txt"); +void ptx_stats::ptx_file_line_stats_options(option_parser_t opp) { + option_parser_register( + opp, "-enable_ptx_file_line_stats", OPT_BOOL, &enable_ptx_file_line_stats, + "Turn on PTX source line statistic profiling. (1 = On)", "1"); + option_parser_register( + opp, "-ptx_line_stats_filename", OPT_CSTR, &ptx_line_stats_filename, + "Output file for PTX source line statistics.", "gpgpu_inst_stats.txt"); } // implementations // defining a PTX source line = filename + line number -class ptx_file_line -{ -public: - ptx_file_line(const char* s, int l) { - if( s == NULL ) - st = "NULL_NAME"; - else - st = s; - line = l; +class ptx_file_line { + public: + ptx_file_line(const char *s, int l) { + if (s == NULL) + st = "NULL_NAME"; + else + st = s; + line = l; + } + + bool operator<(const ptx_file_line &other) const { + if (st == other.st) { + if (line < other.line) + return true; + else + return false; + } else { + return st < other.st; } + } - bool operator<(const ptx_file_line &other) const { - if( st == other.st ) { - if( line < other.line ) - return true; - else - return false; - } else { - return st < other.st; - } - } + bool operator==(const ptx_file_line &other) const { + return (line == other.line) && (st == other.st); + } - bool operator==(const ptx_file_line &other) const { - return (line==other.line) && (st==other.st); - } - - std::string st; - unsigned line; + std::string st; + unsigned line; }; // holds all statistics collected for a singe PTX source line -class ptx_file_line_stats -{ -public: - ptx_file_line_stats() - : exec_count(0), latency(0), dram_traffic(0), - smem_n_way_bank_conflict_total(0), smem_warp_count(0), - gmem_n_access_total(0), gmem_warp_count(0), exposed_latency(0), - warp_divergence(0) - { } - - unsigned long exec_count; - unsigned long long latency; - unsigned long long dram_traffic; - unsigned long long smem_n_way_bank_conflict_total; // total number of banks accessed by this instruction - unsigned long smem_warp_count; // number of warps accessing shared memory - unsigned long long gmem_n_access_total; // number of uncoalesced access in total from this instruction - unsigned long gmem_warp_count; // number of warps causing these uncoalesced access - unsigned long long exposed_latency; // latency exposed as pipeline bubbles (attributed to this instruction) - unsigned long long warp_divergence; // number of warp divergence occured at this instruction +class ptx_file_line_stats { + public: + ptx_file_line_stats() + : exec_count(0), + latency(0), + dram_traffic(0), + smem_n_way_bank_conflict_total(0), + smem_warp_count(0), + gmem_n_access_total(0), + gmem_warp_count(0), + exposed_latency(0), + warp_divergence(0) {} + + unsigned long exec_count; + unsigned long long latency; + unsigned long long dram_traffic; + unsigned long long + smem_n_way_bank_conflict_total; // total number of banks accessed by this + // instruction + unsigned long smem_warp_count; // number of warps accessing shared memory + unsigned long long gmem_n_access_total; // number of uncoalesced access in + // total from this instruction + unsigned long + gmem_warp_count; // number of warps causing these uncoalesced access + unsigned long long exposed_latency; // latency exposed as pipeline bubbles + // (attributed to this instruction) + unsigned long long + warp_divergence; // number of warp divergence occured at this instruction }; #if (tr1_hash_map_ismap == 1) -typedef tr1_hash_map ptx_file_line_stats_map_t; +typedef tr1_hash_map + ptx_file_line_stats_map_t; #else -struct hash_ptx_file_line -{ - std::size_t operator()(const ptx_file_line & pfline) const { - std::hash hash_line; - return hash_line(pfline.line); - } +struct hash_ptx_file_line { + std::size_t operator()(const ptx_file_line &pfline) const { + std::hash hash_line; + return hash_line(pfline.line); + } }; -typedef tr1_hash_map ptx_file_line_stats_map_t; +typedef tr1_hash_map + ptx_file_line_stats_map_t; #endif static ptx_file_line_stats_map_t ptx_file_line_stats_tracker; // output statistics to a file -void ptx_stats::ptx_file_line_stats_write_file() -{ - // check if stat collection is turned on - if (enable_ptx_file_line_stats == 0) return; - - ptx_file_line_stats_map_t::iterator it; - FILE * pfile; - - pfile = fopen(ptx_line_stats_filename, "w"); - fprintf(pfile,"kernel line : count latency dram_traffic smem_bk_conflicts smem_warp gmem_access_generated gmem_warp exposed_latency warp_divergence\n"); - for( it=ptx_file_line_stats_tracker.begin(); it != ptx_file_line_stats_tracker.end(); it++ ) { - fprintf(pfile, "%s %i : ", it->first.st.c_str(), it->first.line); - fprintf(pfile, "%lu ", it->second.exec_count); - fprintf(pfile, "%llu ", it->second.latency); - fprintf(pfile, "%llu ", it->second.dram_traffic); - fprintf(pfile, "%llu ", it->second.smem_n_way_bank_conflict_total); - fprintf(pfile, "%lu ", it->second.smem_warp_count); - fprintf(pfile, "%llu ", it->second.gmem_n_access_total); - fprintf(pfile, "%lu ", it->second.gmem_warp_count); - fprintf(pfile, "%llu ", it->second.exposed_latency); - fprintf(pfile, "%llu ", it->second.warp_divergence); - fprintf(pfile, "\n"); - } - fflush(pfile); - fclose(pfile); +void ptx_stats::ptx_file_line_stats_write_file() { + // check if stat collection is turned on + if (enable_ptx_file_line_stats == 0) return; + + ptx_file_line_stats_map_t::iterator it; + FILE *pfile; + + pfile = fopen(ptx_line_stats_filename, "w"); + fprintf( + pfile, + "kernel line : count latency dram_traffic smem_bk_conflicts smem_warp " + "gmem_access_generated gmem_warp exposed_latency warp_divergence\n"); + for (it = ptx_file_line_stats_tracker.begin(); + it != ptx_file_line_stats_tracker.end(); it++) { + fprintf(pfile, "%s %i : ", it->first.st.c_str(), it->first.line); + fprintf(pfile, "%lu ", it->second.exec_count); + fprintf(pfile, "%llu ", it->second.latency); + fprintf(pfile, "%llu ", it->second.dram_traffic); + fprintf(pfile, "%llu ", it->second.smem_n_way_bank_conflict_total); + fprintf(pfile, "%lu ", it->second.smem_warp_count); + fprintf(pfile, "%llu ", it->second.gmem_n_access_total); + fprintf(pfile, "%lu ", it->second.gmem_warp_count); + fprintf(pfile, "%llu ", it->second.exposed_latency); + fprintf(pfile, "%llu ", it->second.warp_divergence); + fprintf(pfile, "\n"); + } + fflush(pfile); + fclose(pfile); } // attribute one more execution count to this ptx instruction // counting the number of threads (not warps) executing this instruction -void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) -{ - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].exec_count += 1; +void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn) { + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())] + .exec_count += 1; } // attribute pipeline latency to this ptx instruction (specified by the pc) -// pipeline latency is the number of cycles a warp with this instruction spent in the pipeline -void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].latency += latency; +// pipeline latency is the number of cycles a warp with this instruction spent +// in the pipeline +void ptx_stats::ptx_file_line_stats_add_latency(unsigned pc, unsigned latency) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())] + .latency += latency; } // attribute dram traffic to this ptx instruction (specified by the pc) -// dram traffic is counted in number of requests -void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())].dram_traffic += dram_traffic; +// dram traffic is counted in number of requests +void ptx_stats::ptx_file_line_stats_add_dram_traffic(unsigned pc, + unsigned dram_traffic) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())] + .dram_traffic += dram_traffic; } // attribute the number of shared memory access cycles to a ptx instruction -// counts both the number of warps doing shared memory access and the number of cycles involved -void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; - line_stats.smem_warp_count += 1; +// counts both the number of warps doing shared memory access and the number of +// cycles involved +void ptx_stats::ptx_file_line_stats_add_smem_bank_conflict( + unsigned pc, unsigned n_way_bkconflict) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( + pInsn->source_file(), pInsn->source_line())]; + line_stats.smem_n_way_bank_conflict_total += n_way_bkconflict; + line_stats.smem_warp_count += 1; } -// attribute a non-coalesced mem access to a ptx instruction -// counts both the number of warps causing this and the number of memory requests generated -void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.gmem_n_access_total += n_access; - line_stats.gmem_warp_count += 1; +// attribute a non-coalesced mem access to a ptx instruction +// counts both the number of warps causing this and the number of memory +// requests generated +void ptx_stats::ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, + unsigned n_access) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + + ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( + pInsn->source_file(), pInsn->source_line())]; + line_stats.gmem_n_access_total += n_access; + line_stats.gmem_warp_count += 1; } -// a class that tracks the inflight memory instructions of a shader core -// and attributes exposed latency to those instructions when signaled to do so -class ptx_inflight_memory_insn_tracker -{ -public: - typedef std::map insn_count_map; +// a class that tracks the inflight memory instructions of a shader core +// and attributes exposed latency to those instructions when signaled to do so +class ptx_inflight_memory_insn_tracker { + public: + typedef std::map insn_count_map; - void add_count(const ptx_instruction * pInsn, int count = 1) - { - ptx_inflight_memory_insns[pInsn] += count; - } + void add_count(const ptx_instruction *pInsn, int count = 1) { + ptx_inflight_memory_insns[pInsn] += count; + } - void sub_count(const ptx_instruction * pInsn, int count = 1) - { - insn_count_map::iterator i_insncount; - i_insncount = ptx_inflight_memory_insns.find(pInsn); + void sub_count(const ptx_instruction *pInsn, int count = 1) { + insn_count_map::iterator i_insncount; + i_insncount = ptx_inflight_memory_insns.find(pInsn); - assert(i_insncount != ptx_inflight_memory_insns.end()); + assert(i_insncount != ptx_inflight_memory_insns.end()); - i_insncount->second -= count; + i_insncount->second -= count; - if (i_insncount->second <= 0) { - ptx_inflight_memory_insns.erase(i_insncount); - } + if (i_insncount->second <= 0) { + ptx_inflight_memory_insns.erase(i_insncount); } - - void attribute_exposed_latency(int count = 1) - { - insn_count_map &exlat_insnmap = ptx_inflight_memory_insns; - insn_count_map::const_iterator i_exlatinsn; - - i_exlatinsn = exlat_insnmap.begin(); - for (; i_exlatinsn != exlat_insnmap.end(); ++i_exlatinsn) { - const ptx_instruction *pInsn = i_exlatinsn->first; - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.exposed_latency += count; - } + } + + void attribute_exposed_latency(int count = 1) { + insn_count_map &exlat_insnmap = ptx_inflight_memory_insns; + insn_count_map::const_iterator i_exlatinsn; + + i_exlatinsn = exlat_insnmap.begin(); + for (; i_exlatinsn != exlat_insnmap.end(); ++i_exlatinsn) { + const ptx_instruction *pInsn = i_exlatinsn->first; + ptx_file_line_stats &line_stats = + ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), + pInsn->source_line())]; + line_stats.exposed_latency += count; } + } - insn_count_map ptx_inflight_memory_insns; + insn_count_map ptx_inflight_memory_insns; }; static ptx_inflight_memory_insn_tracker *inflight_mem_tracker = NULL; -void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) -{ - inflight_mem_tracker = new ptx_inflight_memory_insn_tracker[n_shader_cores]; +void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores) { + inflight_mem_tracker = new ptx_inflight_memory_insn_tracker[n_shader_cores]; } // add an inflight memory instruction -void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); +void ptx_stats::ptx_file_line_stats_add_inflight_memory_insn(int sc_id, + unsigned pc) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - inflight_mem_tracker[sc_id].add_count(pInsn); + inflight_mem_tracker[sc_id].add_count(pInsn); } // remove an inflight memory instruction -void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); +void ptx_stats::ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, + unsigned pc) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - inflight_mem_tracker[sc_id].sub_count(pInsn); + inflight_mem_tracker[sc_id].sub_count(pInsn); } -// attribute an empty cycle in the pipeline (exposed latency) to the ptx memory instructions in flight -void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency) -{ - assert(exposed_latency > 0); - inflight_mem_tracker[sc_id].attribute_exposed_latency(exposed_latency); +// attribute an empty cycle in the pipeline (exposed latency) to the ptx memory +// instructions in flight +void ptx_file_line_stats_commit_exposed_latency(int sc_id, + int exposed_latency) { + assert(exposed_latency > 0); + inflight_mem_tracker[sc_id].attribute_exposed_latency(exposed_latency); } // attribute the number of warp divergence to a ptx instruction -void ptx_stats::ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence) -{ - const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); - - ptx_file_line_stats& line_stats = ptx_file_line_stats_tracker[ptx_file_line(pInsn->source_file(), pInsn->source_line())]; - line_stats.warp_divergence += n_way_divergence; -} +void ptx_stats::ptx_file_line_stats_add_warp_divergence( + unsigned pc, unsigned n_way_divergence) { + const ptx_instruction *pInsn = gpgpu_ctx->pc_to_instruction(pc); + ptx_file_line_stats &line_stats = ptx_file_line_stats_tracker[ptx_file_line( + pInsn->source_file(), pInsn->source_line())]; + line_stats.warp_divergence += n_way_divergence; +} diff --git a/src/cuda-sim/ptx-stats.h b/src/cuda-sim/ptx-stats.h index 246b4ce..be77c39 100644 --- a/src/cuda-sim/ptx-stats.h +++ b/src/cuda-sim/ptx-stats.h @@ -7,33 +7,33 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#pragma once +#pragma once #include "../option_parser.h" - #ifdef __cplusplus // stat collection interface to cuda-sim class ptx_instruction; -void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn); +void ptx_file_line_stats_add_exec_count(const ptx_instruction* pInsn); #endif // stat collection interface to gpgpu-sim @@ -41,28 +41,29 @@ void ptx_file_line_stats_add_exec_count(const ptx_instruction *pInsn); void ptx_file_line_stats_create_exposed_latency_tracker(int n_shader_cores); void ptx_file_line_stats_commit_exposed_latency(int sc_id, int exposed_latency); - class gpgpu_context; class ptx_stats { - public: - ptx_stats(gpgpu_context* ctx) { - ptx_line_stats_filename = NULL; - gpgpu_ctx = ctx; - } - char * ptx_line_stats_filename; - bool enable_ptx_file_line_stats; - gpgpu_context* gpgpu_ctx; - // set options - void ptx_file_line_stats_options(option_parser_t opp); + public: + ptx_stats(gpgpu_context* ctx) { + ptx_line_stats_filename = NULL; + gpgpu_ctx = ctx; + } + char* ptx_line_stats_filename; + bool enable_ptx_file_line_stats; + gpgpu_context* gpgpu_ctx; + // set options + void ptx_file_line_stats_options(option_parser_t opp); - // output stats to a file - void ptx_file_line_stats_write_file(); - // stat collection interface to gpgpu-sim - void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); - void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); - void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, unsigned n_way_bkconflict); - void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); - void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); - void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); - void ptx_file_line_stats_add_warp_divergence(unsigned pc, unsigned n_way_divergence); + // output stats to a file + void ptx_file_line_stats_write_file(); + // stat collection interface to gpgpu-sim + void ptx_file_line_stats_add_latency(unsigned pc, unsigned latency); + void ptx_file_line_stats_add_dram_traffic(unsigned pc, unsigned dram_traffic); + void ptx_file_line_stats_add_smem_bank_conflict(unsigned pc, + unsigned n_way_bkconflict); + void ptx_file_line_stats_add_uncoalesced_gmem(unsigned pc, unsigned n_access); + void ptx_file_line_stats_add_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_sub_inflight_memory_insn(int sc_id, unsigned pc); + void ptx_file_line_stats_add_warp_divergence(unsigned pc, + unsigned n_way_divergence); }; diff --git a/src/cuda-sim/ptx_ir.cc b/src/cuda-sim/ptx_ir.cc index d8943d2..aa1c25a 100644 --- a/src/cuda-sim/ptx_ir.cc +++ b/src/cuda-sim/ptx_ir.cc @@ -1,5 +1,5 @@ // Copyright (c) 2009-2011, Tor M. Aamodt, Ali Bakhoda, Wilson W.L. Fung, -// George L. Yuan +// George L. Yuan // The University of British Columbia // All rights reserved. // @@ -8,1170 +8,1263 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. -#include "ptx_parser.h" #include "ptx_ir.h" -typedef void * yyscan_t; -#include "ptx.tab.h" -#include "opcodes.h" +#include "ptx_parser.h" +typedef void *yyscan_t; +#include #include #include -#include -#include #include +#include #include "assert.h" +#include "opcodes.h" +#include "ptx.tab.h" -#include "cuda-sim.h" #include "../../libcuda/gpgpu_context.h" +#include "cuda-sim.h" #define STR_SIZE 1024 -const ptx_instruction* gpgpu_context::pc_to_instruction(unsigned pc) -{ - if( pc < s_g_pc_to_insn.size() ) - return s_g_pc_to_insn[pc]; - else - return NULL; -} - -unsigned symbol::get_uid() -{ - unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; - return result; +const ptx_instruction *gpgpu_context::pc_to_instruction(unsigned pc) { + if (pc < s_g_pc_to_insn.size()) + return s_g_pc_to_insn[pc]; + else + return NULL; } -void symbol::add_initializer( const std::list &init ) -{ - m_initializer = init; -} - -void symbol::print_info(FILE *fp) const -{ - fprintf(fp,"uid:%u, decl:%s, type:%p, ", m_uid, m_decl_location.c_str(), m_type ); - if( m_address_valid ) - fprintf(fp,"
, "); - if( m_is_label ) - fprintf(fp," is_label "); - if( m_is_shared ) - fprintf(fp," is_shared "); - if( m_is_const ) - fprintf(fp," is_const "); - if( m_is_global ) - fprintf(fp," is_global "); - if( m_is_local ) - fprintf(fp," is_local "); - if( m_is_tex ) - fprintf(fp," is_tex "); - if( m_is_func_addr ) - fprintf(fp," is_func_addr "); - if( m_function ) - fprintf(fp," %p ", m_function ); +unsigned symbol::get_uid() { + unsigned result = (gpgpu_ctx->symbol_sm_next_uid)++; + return result; } -symbol_table::symbol_table() -{ - assert(0); +void symbol::add_initializer(const std::list &init) { + m_initializer = init; } -symbol_table::symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx ) -{ - gpgpu_ctx = ctx; - m_scope_name = std::string(scope_name); - m_reg_allocator=0; - m_shared_next = 0; - m_const_next = 0; - m_global_next = 0x100; - m_local_next = 0; - m_tex_next = 0; - - //Jin: handle instruction group for cdp - m_inst_group_id = 0; - - m_parent = parent; - if ( m_parent ) { - m_shared_next = m_parent->m_shared_next; - m_global_next = m_parent->m_global_next; - } +void symbol::print_info(FILE *fp) const { + fprintf(fp, "uid:%u, decl:%s, type:%p, ", m_uid, m_decl_location.c_str(), + m_type); + if (m_address_valid) fprintf(fp, "
, "); + if (m_is_label) fprintf(fp, " is_label "); + if (m_is_shared) fprintf(fp, " is_shared "); + if (m_is_const) fprintf(fp, " is_const "); + if (m_is_global) fprintf(fp, " is_global "); + if (m_is_local) fprintf(fp, " is_local "); + if (m_is_tex) fprintf(fp, " is_tex "); + if (m_is_func_addr) fprintf(fp, " is_func_addr "); + if (m_function) fprintf(fp, " %p ", m_function); } -void symbol_table::set_name( const char *name ) -{ - m_scope_name = std::string(name); -} +symbol_table::symbol_table() { assert(0); } -const ptx_version &symbol_table::get_ptx_version() const -{ - if( m_parent == NULL ) return m_ptx_version; - else return m_parent->get_ptx_version(); -} +symbol_table::symbol_table(const char *scope_name, unsigned entry_point, + symbol_table *parent, gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_scope_name = std::string(scope_name); + m_reg_allocator = 0; + m_shared_next = 0; + m_const_next = 0; + m_global_next = 0x100; + m_local_next = 0; + m_tex_next = 0; + + // Jin: handle instruction group for cdp + m_inst_group_id = 0; -unsigned symbol_table::get_sm_target() const -{ - if( m_parent == NULL ) - return m_ptx_version.target(); - else return m_parent->get_sm_target(); + m_parent = parent; + if (m_parent) { + m_shared_next = m_parent->m_shared_next; + m_global_next = m_parent->m_global_next; + } } -void symbol_table::set_ptx_version( float ver, unsigned ext ) -{ - m_ptx_version = ptx_version(ver,ext); +void symbol_table::set_name(const char *name) { + m_scope_name = std::string(name); } -void symbol_table::set_sm_target( const char *target, const char *ext, const char *ext2 ) -{ - m_ptx_version.set_target(target,ext,ext2); +const ptx_version &symbol_table::get_ptx_version() const { + if (m_parent == NULL) + return m_ptx_version; + else + return m_parent->get_ptx_version(); } -symbol *symbol_table::lookup( const char *identifier ) -{ - std::string key(identifier); - std::map::iterator i = m_symbols.find(key); - if ( i != m_symbols.end() ) { - return i->second; - } - if ( m_parent ) { - return m_parent->lookup(identifier); - } - return NULL; +unsigned symbol_table::get_sm_target() const { + if (m_parent == NULL) + return m_ptx_version.target(); + else + return m_parent->get_sm_target(); } -symbol *symbol_table::add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ) -{ - char buf[1024]; - std::string key(identifier); - assert( m_symbols.find(key) == m_symbols.end() ); - snprintf(buf,1024,"%s:%u",filename,line); - symbol *s = new symbol(identifier,type,buf,size,gpgpu_ctx); - m_symbols[ key ] = s; - - if ( type != NULL && type->get_key().is_global() ) { - m_globals.push_back(s); - } - if ( type != NULL && type->get_key().is_const() ) { - m_consts.push_back(s); - } - - return s; +void symbol_table::set_ptx_version(float ver, unsigned ext) { + m_ptx_version = ptx_version(ver, ext); } - -void symbol_table::add_function( function_info *func, const char *filename, unsigned linenumber ) -{ - std::map::iterator i = m_symbols.find( func->get_name() ); - if( i != m_symbols.end() ) - return; - char buf[1024]; - snprintf(buf,1024,"%s:%u",filename,linenumber); - type_info *type = add_type( func ); - symbol *s = new symbol(func->get_name().c_str(),type,buf,0,gpgpu_ctx); - s->set_function(func); - m_symbols[ func->get_name() ] = s; -} - -//Jin: handle instruction group for cdp -symbol_table* symbol_table::start_inst_group() { - char inst_group_name[4096]; - snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), m_inst_group_id); - - //previous added - assert(m_inst_group_symtab.find(std::string(inst_group_name)) == m_inst_group_symtab.end()); - symbol_table *sym_table = new symbol_table(inst_group_name, 3/*inst group*/, this, gpgpu_ctx ); - - sym_table->m_global_next = m_global_next; - sym_table->m_shared_next = m_shared_next; - sym_table->m_local_next = m_local_next; - sym_table->m_reg_allocator = m_reg_allocator; - sym_table->m_tex_next = m_tex_next; - sym_table->m_const_next = m_const_next; - - m_inst_group_symtab[std::string(inst_group_name)] = sym_table; - - return sym_table; -} - -symbol_table * symbol_table::end_inst_group() { - symbol_table * sym_table = m_parent; - - sym_table->m_global_next = m_global_next; - sym_table->m_shared_next = m_shared_next; - sym_table->m_local_next = m_local_next; - sym_table->m_reg_allocator = m_reg_allocator; - sym_table->m_tex_next = m_tex_next; - sym_table->m_const_next = m_const_next; - sym_table->m_inst_group_id++; - - return sym_table; -} - -void register_ptx_function( const char *name, function_info *impl ); // either libcuda or libopencl - -bool symbol_table::add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **sym_table ) -{ - std::string key = std::string(name); - bool prior_decl = false; - if( m_function_info_lookup.find(key) != m_function_info_lookup.end() ) { - *func_info = m_function_info_lookup[key]; - prior_decl = true; - } else { - *func_info = new function_info(entry_point, gpgpu_ctx); - (*func_info)->set_name(name); - (*func_info)->set_maxnt_id(0); - m_function_info_lookup[key] = *func_info; - } - - if( m_function_symtab_lookup.find(key) != m_function_symtab_lookup.end() ) { - assert( prior_decl ); - *sym_table = m_function_symtab_lookup[key]; - } else { - assert( !prior_decl ); - *sym_table = new symbol_table( "", entry_point, this, gpgpu_ctx ); - - // Initial setup code to support a register represented as "_". - // This register is used when an instruction operand is - // not read or written. However, the parser must recognize it - // as a legitimate register but we do not want to pass - // it to the micro-architectural register to the performance simulator. - // For this purpose we add a symbol to the symbol table but - // mark it as a non_arch_reg so it does not effect the performance sim. - type_info_key null_key( reg_space, 0, 0, 0, 0, 0 ); - null_key.set_is_non_arch_reg(); - // First param is null - which is bad. - // However, the first parameter is actually unread in the constructor... - // TODO - remove the symbol_table* from type_info - type_info* null_type_info = new type_info( NULL, null_key ); - symbol *null_reg = (*sym_table)->add_variable( "_", null_type_info, 0, "", 0 ); - null_reg->set_regno(0, 0); - - (*sym_table)->set_name(name); - (*func_info)->set_symtab(*sym_table); - m_function_symtab_lookup[key] = *sym_table; - assert( (*func_info)->get_symtab() == *sym_table ); - register_ptx_function(name,*func_info); - } - return prior_decl; -} - -function_info *symbol_table::lookup_function( std::string name ) -{ - std::string key = std::string(name); - std::map::iterator it = m_function_info_lookup.find(key); - assert ( it != m_function_info_lookup.end() ); - return it->second; -} - -type_info *symbol_table::add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ) -{ - if( space_spec == param_space_unclassified ) - space_spec = param_space_local; - type_info_key t(space_spec,scalar_type_spec,vector_spec,alignment_spec,extern_spec,0); - type_info *pt; - pt = new type_info(this,t); - return pt; + +void symbol_table::set_sm_target(const char *target, const char *ext, + const char *ext2) { + m_ptx_version.set_target(target, ext, ext2); +} + +symbol *symbol_table::lookup(const char *identifier) { + std::string key(identifier); + std::map::iterator i = m_symbols.find(key); + if (i != m_symbols.end()) { + return i->second; + } + if (m_parent) { + return m_parent->lookup(identifier); + } + return NULL; +} + +symbol *symbol_table::add_variable(const char *identifier, + const type_info *type, unsigned size, + const char *filename, unsigned line) { + char buf[1024]; + std::string key(identifier); + assert(m_symbols.find(key) == m_symbols.end()); + snprintf(buf, 1024, "%s:%u", filename, line); + symbol *s = new symbol(identifier, type, buf, size, gpgpu_ctx); + m_symbols[key] = s; + + if (type != NULL && type->get_key().is_global()) { + m_globals.push_back(s); + } + if (type != NULL && type->get_key().is_const()) { + m_consts.push_back(s); + } + + return s; +} + +void symbol_table::add_function(function_info *func, const char *filename, + unsigned linenumber) { + std::map::iterator i = + m_symbols.find(func->get_name()); + if (i != m_symbols.end()) return; + char buf[1024]; + snprintf(buf, 1024, "%s:%u", filename, linenumber); + type_info *type = add_type(func); + symbol *s = new symbol(func->get_name().c_str(), type, buf, 0, gpgpu_ctx); + s->set_function(func); + m_symbols[func->get_name()] = s; +} + +// Jin: handle instruction group for cdp +symbol_table *symbol_table::start_inst_group() { + char inst_group_name[4096]; + snprintf(inst_group_name, 4096, "%s_inst_group_%u", m_scope_name.c_str(), + m_inst_group_id); + + // previous added + assert(m_inst_group_symtab.find(std::string(inst_group_name)) == + m_inst_group_symtab.end()); + symbol_table *sym_table = + new symbol_table(inst_group_name, 3 /*inst group*/, this, gpgpu_ctx); + + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; + + m_inst_group_symtab[std::string(inst_group_name)] = sym_table; + + return sym_table; +} + +symbol_table *symbol_table::end_inst_group() { + symbol_table *sym_table = m_parent; + + sym_table->m_global_next = m_global_next; + sym_table->m_shared_next = m_shared_next; + sym_table->m_local_next = m_local_next; + sym_table->m_reg_allocator = m_reg_allocator; + sym_table->m_tex_next = m_tex_next; + sym_table->m_const_next = m_const_next; + sym_table->m_inst_group_id++; + + return sym_table; +} + +void register_ptx_function(const char *name, + function_info *impl); // either libcuda or libopencl + +bool symbol_table::add_function_decl(const char *name, int entry_point, + function_info **func_info, + symbol_table **sym_table) { + std::string key = std::string(name); + bool prior_decl = false; + if (m_function_info_lookup.find(key) != m_function_info_lookup.end()) { + *func_info = m_function_info_lookup[key]; + prior_decl = true; + } else { + *func_info = new function_info(entry_point, gpgpu_ctx); + (*func_info)->set_name(name); + (*func_info)->set_maxnt_id(0); + m_function_info_lookup[key] = *func_info; + } + + if (m_function_symtab_lookup.find(key) != m_function_symtab_lookup.end()) { + assert(prior_decl); + *sym_table = m_function_symtab_lookup[key]; + } else { + assert(!prior_decl); + *sym_table = new symbol_table("", entry_point, this, gpgpu_ctx); + + // Initial setup code to support a register represented as "_". + // This register is used when an instruction operand is + // not read or written. However, the parser must recognize it + // as a legitimate register but we do not want to pass + // it to the micro-architectural register to the performance simulator. + // For this purpose we add a symbol to the symbol table but + // mark it as a non_arch_reg so it does not effect the performance sim. + type_info_key null_key(reg_space, 0, 0, 0, 0, 0); + null_key.set_is_non_arch_reg(); + // First param is null - which is bad. + // However, the first parameter is actually unread in the constructor... + // TODO - remove the symbol_table* from type_info + type_info *null_type_info = new type_info(NULL, null_key); + symbol *null_reg = + (*sym_table)->add_variable("_", null_type_info, 0, "", 0); + null_reg->set_regno(0, 0); + + (*sym_table)->set_name(name); + (*func_info)->set_symtab(*sym_table); + m_function_symtab_lookup[key] = *sym_table; + assert((*func_info)->get_symtab() == *sym_table); + register_ptx_function(name, *func_info); + } + return prior_decl; +} + +function_info *symbol_table::lookup_function(std::string name) { + std::string key = std::string(name); + std::map::iterator it = + m_function_info_lookup.find(key); + assert(it != m_function_info_lookup.end()); + return it->second; +} + +type_info *symbol_table::add_type(memory_space_t space_spec, + int scalar_type_spec, int vector_spec, + int alignment_spec, int extern_spec) { + if (space_spec == param_space_unclassified) space_spec = param_space_local; + type_info_key t(space_spec, scalar_type_spec, vector_spec, alignment_spec, + extern_spec, 0); + type_info *pt; + pt = new type_info(this, t); + return pt; +} + +type_info *symbol_table::add_type(function_info *func) { + type_info_key t; + type_info *pt; + t.set_is_func(); + pt = new type_info(this, t); + return pt; +} + +type_info *symbol_table::get_array_type(type_info *base_type, + unsigned array_dim) { + type_info_key t = base_type->get_key(); + t.set_array_dim(array_dim); + type_info *pt = new type_info(this, t); + // Where else is m_types being used? As of now, I dont find any use of it and + // causing seg fault. So disabling m_types. + // TODO: find where m_types can be used in future and solve the seg fault. + // pt = m_types[t] = new type_info(this,t); + return pt; +} + +void symbol_table::set_label_address(const symbol *label, unsigned addr) { + std::map::iterator i = m_symbols.find(label->name()); + assert(i != m_symbols.end()); + symbol *s = i->second; + s->set_label_address(addr); +} + +void symbol_table::dump() { + printf("\n\n"); + printf("Symbol table for \"%s\":\n", m_scope_name.c_str()); + std::map::iterator i; + for (i = m_symbols.begin(); i != m_symbols.end(); i++) { + printf("%30s : ", i->first.c_str()); + if (i->second) + i->second->print_info(stdout); + else + printf(" "); + printf("\n"); + } + printf("\n"); } -type_info *symbol_table::add_type( function_info *func ) -{ - type_info_key t; - type_info *pt; - t.set_is_func(); - pt = new type_info(this,t); - return pt; +unsigned operand_info::get_uid() { + unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; + return result; } -type_info *symbol_table::get_array_type( type_info *base_type, unsigned array_dim ) -{ - type_info_key t = base_type->get_key(); - t.set_array_dim(array_dim); - type_info *pt = new type_info(this,t); - //Where else is m_types being used? As of now, I dont find any use of it and causing seg fault. So disabling m_types. - //TODO: find where m_types can be used in future and solve the seg fault. - //pt = m_types[t] = new type_info(this,t); - return pt; +std::list::iterator +function_info::find_next_real_instruction( + std::list::iterator i) { + while ((i != m_instructions.end()) && (*i)->is_label()) i++; + return i; } -void symbol_table::set_label_address( const symbol *label, unsigned addr ) -{ - std::map::iterator i=m_symbols.find(label->name()); - assert( i != m_symbols.end() ); - symbol *s = i->second; - s->set_label_address(addr); -} +void function_info::create_basic_blocks() { + std::list leaders; + std::list::iterator i, l; -void symbol_table::dump() -{ - printf("\n\n"); - printf("Symbol table for \"%s\":\n", m_scope_name.c_str() ); - std::map::iterator i; - for( i=m_symbols.begin(); i!=m_symbols.end(); i++ ) { - printf("%30s : ", i->first.c_str() ); - if( i->second ) - i->second->print_info(stdout); - else - printf(" "); - printf("\n"); - } - printf("\n"); -} - -unsigned operand_info::get_uid() -{ - unsigned result = (gpgpu_ctx->operand_info_sm_next_uid)++; - return result; -} - -std::list::iterator function_info::find_next_real_instruction( std::list::iterator i) -{ - while( (i != m_instructions.end()) && (*i)->is_label() ) - i++; - return i; -} - -void function_info::create_basic_blocks() -{ - std::list leaders; - std::list::iterator i, l; - - // first instruction is a leader - i=m_instructions.begin(); - leaders.push_back(*i); - i++; - while( i!=m_instructions.end() ) { - ptx_instruction *pI = *i; - if( pI->is_label() ) { - leaders.push_back(pI); - i = find_next_real_instruction(++i); - } else { - switch( pI->get_opcode() ) { - case BRA_OP: case RET_OP: case EXIT_OP: case RETP_OP: case BREAK_OP: + // first instruction is a leader + i = m_instructions.begin(); + leaders.push_back(*i); + i++; + while (i != m_instructions.end()) { + ptx_instruction *pI = *i; + if (pI->is_label()) { + leaders.push_back(pI); + i = find_next_real_instruction(++i); + } else { + switch (pI->get_opcode()) { + case BRA_OP: + case RET_OP: + case EXIT_OP: + case RETP_OP: + case BREAK_OP: + i++; + if (i != m_instructions.end()) leaders.push_back(*i); + i = find_next_real_instruction(i); + break; + case CALL_OP: + case CALLP_OP: + if (pI->has_pred()) { + printf("GPGPU-Sim PTX: Warning found predicated call\n"); i++; - if( i != m_instructions.end() ) - leaders.push_back(*i); + if (i != m_instructions.end()) leaders.push_back(*i); i = find_next_real_instruction(i); - break; - case CALL_OP: case CALLP_OP: - if( pI->has_pred() ) { - printf("GPGPU-Sim PTX: Warning found predicated call\n"); - i++; - if( i != m_instructions.end() ) - leaders.push_back(*i); - i = find_next_real_instruction(i); - } else i++; - break; - default: + } else i++; - } - } - } - - if( leaders.empty() ) { - printf("GPGPU-Sim PTX: Function \'%s\' has no basic blocks\n", m_name.c_str()); - return; - } - - unsigned bb_id = 0; - l=leaders.begin(); - i=m_instructions.begin(); - m_basic_blocks.push_back( new basic_block_t(bb_id++,*find_next_real_instruction(i),NULL,1,0) ); - ptx_instruction *last_real_inst=*(l++); - - for( ; i!=m_instructions.end(); i++ ) { - ptx_instruction *pI = *i; - if( l != leaders.end() && *i == *l ) { - // found start of next basic block - m_basic_blocks.back()->ptx_end = last_real_inst; - if( find_next_real_instruction(i) != m_instructions.end() ) { // if not bogus trailing label - m_basic_blocks.push_back( new basic_block_t(bb_id++,*find_next_real_instruction(i),NULL,0,0) ); - last_real_inst = *find_next_real_instruction(i); - } - // start search for next leader - l++; + break; + default: + i++; } - pI->assign_bb( m_basic_blocks.back() ); - if( !pI->is_label() ) last_real_inst = pI; - } - m_basic_blocks.back()->ptx_end = last_real_inst; - m_basic_blocks.push_back( /*exit basic block*/ new basic_block_t(bb_id,NULL,NULL,0,1) ); -} - -void function_info::print_basic_blocks() -{ - printf("Printing basic blocks for function \'%s\':\n", m_name.c_str() ); - std::list::iterator ptx_itr; - unsigned last_bb=0; - for (ptx_itr = m_instructions.begin();ptx_itr != m_instructions.end(); ptx_itr++) { - if( (*ptx_itr)->get_bb() ) { - if( (*ptx_itr)->get_bb()->bb_id != last_bb ) { - printf("\n"); - last_bb = (*ptx_itr)->get_bb()->bb_id; - } - printf("bb_%02u\t: ", (*ptx_itr)->get_bb()->bb_id); - (*ptx_itr)->print_insn(); - printf("\n"); + } + } + + if (leaders.empty()) { + printf("GPGPU-Sim PTX: Function \'%s\' has no basic blocks\n", + m_name.c_str()); + return; + } + + unsigned bb_id = 0; + l = leaders.begin(); + i = m_instructions.begin(); + m_basic_blocks.push_back( + new basic_block_t(bb_id++, *find_next_real_instruction(i), NULL, 1, 0)); + ptx_instruction *last_real_inst = *(l++); + + for (; i != m_instructions.end(); i++) { + ptx_instruction *pI = *i; + if (l != leaders.end() && *i == *l) { + // found start of next basic block + m_basic_blocks.back()->ptx_end = last_real_inst; + if (find_next_real_instruction(i) != + m_instructions.end()) { // if not bogus trailing label + m_basic_blocks.push_back(new basic_block_t( + bb_id++, *find_next_real_instruction(i), NULL, 0, 0)); + last_real_inst = *find_next_real_instruction(i); } - } - printf("\nSummary of basic blocks for \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - printf("bb_%02u\t:", (*bb_itr)->bb_id); - if ((*bb_itr)->ptx_begin) - printf(" first: %s\t", ((*bb_itr)->ptx_begin)->get_opcode_cstr()); - else printf(" first: NULL\t"); - if ((*bb_itr)->ptx_end) { - printf(" last: %s\t", ((*bb_itr)->ptx_end)->get_opcode_cstr()); - } else printf(" last: NULL\t"); + // start search for next leader + l++; + } + pI->assign_bb(m_basic_blocks.back()); + if (!pI->is_label()) last_real_inst = pI; + } + m_basic_blocks.back()->ptx_end = last_real_inst; + m_basic_blocks.push_back( + /*exit basic block*/ new basic_block_t(bb_id, NULL, NULL, 0, 1)); +} + +void function_info::print_basic_blocks() { + printf("Printing basic blocks for function \'%s\':\n", m_name.c_str()); + std::list::iterator ptx_itr; + unsigned last_bb = 0; + for (ptx_itr = m_instructions.begin(); ptx_itr != m_instructions.end(); + ptx_itr++) { + if ((*ptx_itr)->get_bb()) { + if ((*ptx_itr)->get_bb()->bb_id != last_bb) { + printf("\n"); + last_bb = (*ptx_itr)->get_bb()->bb_id; + } + printf("bb_%02u\t: ", (*ptx_itr)->get_bb()->bb_id); + (*ptx_itr)->print_insn(); printf("\n"); - } - printf("\n"); -} - -void function_info::print_basic_block_links() -{ - printf("Printing basic blocks links for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - printf("ID: %d\t:", (*bb_itr)->bb_id); - if ( !(*bb_itr)->predecessor_ids.empty() ) { - printf("Predecessors:"); - std::set::iterator p; - for (p= (*bb_itr)->predecessor_ids.begin();p != (*bb_itr)->predecessor_ids.end();p++) { - printf(" %d", *p); - } - printf("\t"); + } + } + printf("\nSummary of basic blocks for \'%s\':\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + printf("bb_%02u\t:", (*bb_itr)->bb_id); + if ((*bb_itr)->ptx_begin) + printf(" first: %s\t", ((*bb_itr)->ptx_begin)->get_opcode_cstr()); + else + printf(" first: NULL\t"); + if ((*bb_itr)->ptx_end) { + printf(" last: %s\t", ((*bb_itr)->ptx_end)->get_opcode_cstr()); + } else + printf(" last: NULL\t"); + printf("\n"); + } + printf("\n"); +} + +void function_info::print_basic_block_links() { + printf("Printing basic blocks links for function \'%s\':\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + printf("ID: %d\t:", (*bb_itr)->bb_id); + if (!(*bb_itr)->predecessor_ids.empty()) { + printf("Predecessors:"); + std::set::iterator p; + for (p = (*bb_itr)->predecessor_ids.begin(); + p != (*bb_itr)->predecessor_ids.end(); p++) { + printf(" %d", *p); } - if ( !(*bb_itr)->successor_ids.empty() ) { - printf("Successors:"); - std::set::iterator s; - for (s= (*bb_itr)->successor_ids.begin();s != (*bb_itr)->successor_ids.end();s++) { - printf(" %d", *s); - } + printf("\t"); + } + if (!(*bb_itr)->successor_ids.empty()) { + printf("Successors:"); + std::set::iterator s; + for (s = (*bb_itr)->successor_ids.begin(); + s != (*bb_itr)->successor_ids.end(); s++) { + printf(" %d", *s); } - printf("\n"); - } + } + printf("\n"); + } } -operand_info* function_info::find_break_target( ptx_instruction * p_break_insn ) //find the target of a break instruction +operand_info *function_info::find_break_target( + ptx_instruction *p_break_insn) // find the target of a break instruction { - const basic_block_t *break_bb = p_break_insn->get_bb(); - // go through the dominator tree - for(const basic_block_t *p_bb = break_bb; - p_bb->immediatedominator_id != -1; - p_bb = m_basic_blocks[p_bb->immediatedominator_id]) - { - // reverse search through instructions in basic block for breakaddr instruction - unsigned insn_addr = p_bb->ptx_end->get_m_instr_mem_index(); - while (insn_addr >= p_bb->ptx_begin->get_m_instr_mem_index()) { - ptx_instruction *pI = m_instr_mem[insn_addr]; - insn_addr -= 1; - if (pI == NULL) continue; // temporary solution for variable size instructions - if (pI->get_opcode() == BREAKADDR_OP) { - return &(pI->dst()); - } + const basic_block_t *break_bb = p_break_insn->get_bb(); + // go through the dominator tree + for (const basic_block_t *p_bb = break_bb; p_bb->immediatedominator_id != -1; + p_bb = m_basic_blocks[p_bb->immediatedominator_id]) { + // reverse search through instructions in basic block for breakaddr + // instruction + unsigned insn_addr = p_bb->ptx_end->get_m_instr_mem_index(); + while (insn_addr >= p_bb->ptx_begin->get_m_instr_mem_index()) { + ptx_instruction *pI = m_instr_mem[insn_addr]; + insn_addr -= 1; + if (pI == NULL) + continue; // temporary solution for variable size instructions + if (pI->get_opcode() == BREAKADDR_OP) { + return &(pI->dst()); } - } + } + } - assert(0); + assert(0); - // lazy fallback: just traverse backwards? - for (int insn_addr = p_break_insn->get_m_instr_mem_index(); - insn_addr >= 0; insn_addr--) - { - ptx_instruction *pI = m_instr_mem[insn_addr]; - if (pI->get_opcode() == BREAKADDR_OP) { - return &(pI->dst()); - } - } + // lazy fallback: just traverse backwards? + for (int insn_addr = p_break_insn->get_m_instr_mem_index(); insn_addr >= 0; + insn_addr--) { + ptx_instruction *pI = m_instr_mem[insn_addr]; + if (pI->get_opcode() == BREAKADDR_OP) { + return &(pI->dst()); + } + } - return NULL; + return NULL; } -void function_info::connect_basic_blocks( ) //iterate across m_basic_blocks of function, connecting basic blocks together +void function_info::connect_basic_blocks() // iterate across m_basic_blocks of + // function, connecting basic blocks + // together { - std::vector::iterator bb_itr; - std::vector::iterator bb_target_itr; - basic_block_t* exit_bb = m_basic_blocks.back(); - - //start from first basic block, which we know is the entry point - bb_itr = m_basic_blocks.begin(); - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - ptx_instruction *pI = (*bb_itr)->ptx_end; - if ((*bb_itr)->is_exit) //reached last basic block, no successors to link - continue; - if (pI->get_opcode() == RETP_OP || pI->get_opcode() == RET_OP || pI->get_opcode() == EXIT_OP ) { - (*bb_itr)->successor_ids.insert(exit_bb->bb_id); - exit_bb->predecessor_ids.insert((*bb_itr)->bb_id); - if( pI->has_pred() ) { - printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n"); - // if predicated, add link to next block - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - if( next_addr < m_instr_mem_size && m_instr_mem[next_addr] ) { - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - (*bb_itr)->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } - } - continue; - } else if (pI->get_opcode() == BRA_OP) { - //find successor and link that basic_block to this one - operand_info &target = pI->dst(); //get operand, e.g. target name - unsigned addr = labels[ target.name() ]; - ptx_instruction *target_pI = m_instr_mem[addr]; - basic_block_t *target_bb = target_pI->get_bb(); - (*bb_itr)->successor_ids.insert(target_bb->bb_id); - target_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } - - if ( !(pI->get_opcode()==BRA_OP && (!pI->has_pred())) ) { - // if basic block does not end in an unpredicated branch, - // then next basic block is also successor - // (this is better than testing for .uni) - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - (*bb_itr)->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert((*bb_itr)->bb_id); - } else - assert(pI->get_opcode() == BRA_OP); - } -} -bool function_info::connect_break_targets() //connecting break instructions with proper targets -{ - std::vector::iterator bb_itr; - std::vector::iterator bb_target_itr; - bool modified = false; - - //start from first basic block, which we know is the entry point - bb_itr = m_basic_blocks.begin(); - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - basic_block_t *p_bb = *bb_itr; - ptx_instruction *pI = p_bb->ptx_end; - if (p_bb->is_exit) //reached last basic block, no successors to link - continue; - if (pI->get_opcode() == BREAK_OP) { - // backup existing successor_ids for stability check - std::set orig_successor_ids = p_bb->successor_ids; - - // erase the previous linkage with old successors - for(std::set::iterator succ_ids = p_bb->successor_ids.begin(); succ_ids != p_bb->successor_ids.end(); ++succ_ids) { - basic_block_t *successor_bb = m_basic_blocks[*succ_ids]; - successor_bb->predecessor_ids.erase(p_bb->bb_id); - } - p_bb->successor_ids.clear(); - - //find successor and link that basic_block to this one - //successor of a break is set by an preceeding breakaddr instruction - operand_info *target = find_break_target(pI); - unsigned addr = labels[ target->name() ]; - ptx_instruction *target_pI = m_instr_mem[addr]; - basic_block_t *target_bb = target_pI->get_bb(); - p_bb->successor_ids.insert(target_bb->bb_id); - target_bb->predecessor_ids.insert(p_bb->bb_id); - - if (pI->has_pred()) { - // predicated break - add link to next basic block - unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); - basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); - p_bb->successor_ids.insert(next_bb->bb_id); - next_bb->predecessor_ids.insert(p_bb->bb_id); - } - - modified = modified || (orig_successor_ids != p_bb->successor_ids); + std::vector::iterator bb_itr; + std::vector::iterator bb_target_itr; + basic_block_t *exit_bb = m_basic_blocks.back(); + + // start from first basic block, which we know is the entry point + bb_itr = m_basic_blocks.begin(); + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + ptx_instruction *pI = (*bb_itr)->ptx_end; + if ((*bb_itr)->is_exit) // reached last basic block, no successors to link + continue; + if (pI->get_opcode() == RETP_OP || pI->get_opcode() == RET_OP || + pI->get_opcode() == EXIT_OP) { + (*bb_itr)->successor_ids.insert(exit_bb->bb_id); + exit_bb->predecessor_ids.insert((*bb_itr)->bb_id); + if (pI->has_pred()) { + printf("GPGPU-Sim PTX: Warning detected predicated return/exit.\n"); + // if predicated, add link to next block + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + if (next_addr < m_instr_mem_size && m_instr_mem[next_addr]) { + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + (*bb_itr)->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } } - } - - return modified; -} -void function_info::do_pdom() -{ - create_basic_blocks(); - connect_basic_blocks(); - bool modified = false; - do { - find_dominators(); - find_idominators(); - modified = connect_break_targets(); - } while (modified == true); - - if ( g_debug_execution>=50 ) { - print_basic_blocks(); - print_basic_block_links(); - print_basic_block_dot(); - } - if ( g_debug_execution>=2 ) { - print_dominators(); - } - find_postdominators(); - find_ipostdominators(); - if ( g_debug_execution>=50 ) { - print_postdominators(); - print_ipostdominators(); - } - printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", m_name.c_str() ); - for ( unsigned ii=0; ii < m_n; ii += m_instr_mem[ii]->inst_size() ) { // handle branch instructions - ptx_instruction *pI = m_instr_mem[ii]; - pI->pre_decode(); - } - printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", m_name.c_str() ); - fflush(stdout); - m_assembled = true; -} -void intersect( std::set &A, const std::set &B ) -{ - // return intersection of A and B in A - for( std::set::iterator a=A.begin(); a!=A.end(); ) { - std::set::iterator a_next = a; - a_next++; - if( B.find(*a) == B.end() ) { - A.erase(*a); - a = a_next; - } else - a++; - } -} - -bool is_equal( const std::set &A, const std::set &B ) -{ - if( A.size() != B.size() ) - return false; - for( std::set::iterator b=B.begin(); b!=B.end(); b++ ) - if( A.find(*b) == A.end() ) - return false; - return true; -} + continue; + } else if (pI->get_opcode() == BRA_OP) { + // find successor and link that basic_block to this one + operand_info &target = pI->dst(); // get operand, e.g. target name + unsigned addr = labels[target.name()]; + ptx_instruction *target_pI = m_instr_mem[addr]; + basic_block_t *target_bb = target_pI->get_bb(); + (*bb_itr)->successor_ids.insert(target_bb->bb_id); + target_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } -void print_set(const std::set &A) + if (!(pI->get_opcode() == BRA_OP && (!pI->has_pred()))) { + // if basic block does not end in an unpredicated branch, + // then next basic block is also successor + // (this is better than testing for .uni) + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + (*bb_itr)->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert((*bb_itr)->bb_id); + } else + assert(pI->get_opcode() == BRA_OP); + } +} +bool function_info::connect_break_targets() // connecting break instructions + // with proper targets { - std::set::iterator a; - for (a= A.begin(); a != A.end(); a++) { - printf("%d ", (*a)); - } - printf("\n"); -} - -void function_info::find_dominators( ) -{ - // find dominators using algorithm of Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - printf("GPGPU-Sim PTX: Finding dominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished entry block - std::vector::iterator bb_itr = m_basic_blocks.begin(); - (*bb_itr)->dominator_ids.insert((*bb_itr)->bb_id); // the only dominator of the entry block is the entry - //copy all basic blocks to all dominator lists EXCEPT for the entry block - for (++bb_itr;bb_itr != m_basic_blocks.end(); bb_itr++) { - for (unsigned i = 0; i < m_basic_blocks.size(); i++) - (*bb_itr)->dominator_ids.insert(i); - } - bool change = true; - while (change) { - change = false; - for ( int h = 1/*skip entry*/; h < m_basic_blocks.size(); ++h ) { - assert( m_basic_blocks[h]->bb_id == (unsigned)h ); - std::set T; - for (unsigned i=0;i< m_basic_blocks.size();i++) - T.insert(i); - for ( std::set::iterator s = m_basic_blocks[h]->predecessor_ids.begin();s != m_basic_blocks[h]->predecessor_ids.end();s++) - intersect(T, m_basic_blocks[*s]->dominator_ids); - T.insert(h); - if (!is_equal(T, m_basic_blocks[h]->dominator_ids)) { - change = true; - m_basic_blocks[h]->dominator_ids = T; - } + std::vector::iterator bb_itr; + std::vector::iterator bb_target_itr; + bool modified = false; + + // start from first basic block, which we know is the entry point + bb_itr = m_basic_blocks.begin(); + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + basic_block_t *p_bb = *bb_itr; + ptx_instruction *pI = p_bb->ptx_end; + if (p_bb->is_exit) // reached last basic block, no successors to link + continue; + if (pI->get_opcode() == BREAK_OP) { + // backup existing successor_ids for stability check + std::set orig_successor_ids = p_bb->successor_ids; + + // erase the previous linkage with old successors + for (std::set::iterator succ_ids = p_bb->successor_ids.begin(); + succ_ids != p_bb->successor_ids.end(); ++succ_ids) { + basic_block_t *successor_bb = m_basic_blocks[*succ_ids]; + successor_bb->predecessor_ids.erase(p_bb->bb_id); } - } - //clean the basic block of dominators of it has no predecessors -- except for entry block - bb_itr = m_basic_blocks.begin(); - for (++bb_itr;bb_itr != m_basic_blocks.end(); bb_itr++) { - if ((*bb_itr)->predecessor_ids.empty()) - (*bb_itr)->dominator_ids.clear(); - } -} - -void function_info::find_postdominators( ) -{ - // find postdominators using algorithm of Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - printf("GPGPU-Sim PTX: Finding postdominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished exit block - std::vector::reverse_iterator bb_itr = m_basic_blocks.rbegin(); - (*bb_itr)->postdominator_ids.insert((*bb_itr)->bb_id); // the only postdominator of the exit block is the exit - for (++bb_itr;bb_itr != m_basic_blocks.rend();bb_itr++) { //copy all basic blocks to all postdominator lists EXCEPT for the exit block - for (unsigned i=0; ipostdominator_ids.insert(i); - } - bool change = true; - while (change) { - change = false; - for ( int h = m_basic_blocks.size()-2/*skip exit*/; h >= 0 ; --h ) { - assert( m_basic_blocks[h]->bb_id == (unsigned)h ); - std::set T; - for (unsigned i=0;i< m_basic_blocks.size();i++) - T.insert(i); - for ( std::set::iterator s = m_basic_blocks[h]->successor_ids.begin();s != m_basic_blocks[h]->successor_ids.end();s++) - intersect(T, m_basic_blocks[*s]->postdominator_ids); - T.insert(h); - if (!is_equal(T,m_basic_blocks[h]->postdominator_ids)) { - change = true; - m_basic_blocks[h]->postdominator_ids = T; - } + p_bb->successor_ids.clear(); + + // find successor and link that basic_block to this one + // successor of a break is set by an preceeding breakaddr instruction + operand_info *target = find_break_target(pI); + unsigned addr = labels[target->name()]; + ptx_instruction *target_pI = m_instr_mem[addr]; + basic_block_t *target_bb = target_pI->get_bb(); + p_bb->successor_ids.insert(target_bb->bb_id); + target_bb->predecessor_ids.insert(p_bb->bb_id); + + if (pI->has_pred()) { + // predicated break - add link to next basic block + unsigned next_addr = pI->get_m_instr_mem_index() + pI->inst_size(); + basic_block_t *next_bb = m_instr_mem[next_addr]->get_bb(); + p_bb->successor_ids.insert(next_bb->bb_id); + next_bb->predecessor_ids.insert(p_bb->bb_id); } - } -} -void function_info::find_ipostdominators( ) -{ - // find immediate postdominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - printf("GPGPU-Sim PTX: Finding immediate postdominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished exit block - for (unsigned i=0; iTmp_ids = m_basic_blocks[i]->postdominator_ids; - assert( m_basic_blocks[i]->bb_id == i ); - m_basic_blocks[i]->Tmp_ids.erase(i); - } - for ( int n = m_basic_blocks.size()-2; n >=0;--n) { - // point iterator to basic block before the exit - for( std::set::iterator s=m_basic_blocks[n]->Tmp_ids.begin(); s != m_basic_blocks[n]->Tmp_ids.end(); s++ ) { - int bb_s = *s; - for( std::set::iterator t=m_basic_blocks[n]->Tmp_ids.begin(); t != m_basic_blocks[n]->Tmp_ids.end(); ) { - std::set::iterator t_next = t; t_next++; // might erase thing pointed to be t, invalidating iterator t - if( *s == *t ) { - t = t_next; - continue; - } - int bb_t = *t; - if( m_basic_blocks[bb_s]->postdominator_ids.find(bb_t) != m_basic_blocks[bb_s]->postdominator_ids.end() ) - m_basic_blocks[n]->Tmp_ids.erase(bb_t); - t = t_next; - } + modified = modified || (orig_successor_ids != p_bb->successor_ids); + } + } + + return modified; +} +void function_info::do_pdom() { + create_basic_blocks(); + connect_basic_blocks(); + bool modified = false; + do { + find_dominators(); + find_idominators(); + modified = connect_break_targets(); + } while (modified == true); + + if (g_debug_execution >= 50) { + print_basic_blocks(); + print_basic_block_links(); + print_basic_block_dot(); + } + if (g_debug_execution >= 2) { + print_dominators(); + } + find_postdominators(); + find_ipostdominators(); + if (g_debug_execution >= 50) { + print_postdominators(); + print_ipostdominators(); + } + printf("GPGPU-Sim PTX: pre-decoding instructions for \'%s\'...\n", + m_name.c_str()); + for (unsigned ii = 0; ii < m_n; + ii += m_instr_mem[ii]->inst_size()) { // handle branch instructions + ptx_instruction *pI = m_instr_mem[ii]; + pI->pre_decode(); + } + printf("GPGPU-Sim PTX: ... done pre-decoding instructions for \'%s\'.\n", + m_name.c_str()); + fflush(stdout); + m_assembled = true; +} +void intersect(std::set &A, const std::set &B) { + // return intersection of A and B in A + for (std::set::iterator a = A.begin(); a != A.end();) { + std::set::iterator a_next = a; + a_next++; + if (B.find(*a) == B.end()) { + A.erase(*a); + a = a_next; + } else + a++; + } +} + +bool is_equal(const std::set &A, const std::set &B) { + if (A.size() != B.size()) return false; + for (std::set::iterator b = B.begin(); b != B.end(); b++) + if (A.find(*b) == A.end()) return false; + return true; +} + +void print_set(const std::set &A) { + std::set::iterator a; + for (a = A.begin(); a != A.end(); a++) { + printf("%d ", (*a)); + } + printf("\n"); +} + +void function_info::find_dominators() { + // find dominators using algorithm of Muchnick's Adv. Compiler Design & + // Implemmntation Fig 7.14 + printf("GPGPU-Sim PTX: Finding dominators for \'%s\'...\n", m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished entry block + std::vector::iterator bb_itr = m_basic_blocks.begin(); + (*bb_itr)->dominator_ids.insert( + (*bb_itr)->bb_id); // the only dominator of the entry block is the entry + // copy all basic blocks to all dominator lists EXCEPT for the entry block + for (++bb_itr; bb_itr != m_basic_blocks.end(); bb_itr++) { + for (unsigned i = 0; i < m_basic_blocks.size(); i++) + (*bb_itr)->dominator_ids.insert(i); + } + bool change = true; + while (change) { + change = false; + for (int h = 1 /*skip entry*/; h < m_basic_blocks.size(); ++h) { + assert(m_basic_blocks[h]->bb_id == (unsigned)h); + std::set T; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) T.insert(i); + for (std::set::iterator s = + m_basic_blocks[h]->predecessor_ids.begin(); + s != m_basic_blocks[h]->predecessor_ids.end(); s++) + intersect(T, m_basic_blocks[*s]->dominator_ids); + T.insert(h); + if (!is_equal(T, m_basic_blocks[h]->dominator_ids)) { + change = true; + m_basic_blocks[h]->dominator_ids = T; } - } - unsigned num_ipdoms=0; - for ( int n = m_basic_blocks.size()-1; n >=0;--n) { - assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); - // if the above assert fails we have an error in either postdominator - // computation, the flow graph does not have a unique exit, or some other error - if( !m_basic_blocks[n]->Tmp_ids.empty() ) { - m_basic_blocks[n]->immediatepostdominator_id = *m_basic_blocks[n]->Tmp_ids.begin(); - num_ipdoms++; + } + } + // clean the basic block of dominators of it has no predecessors -- except for + // entry block + bb_itr = m_basic_blocks.begin(); + for (++bb_itr; bb_itr != m_basic_blocks.end(); bb_itr++) { + if ((*bb_itr)->predecessor_ids.empty()) (*bb_itr)->dominator_ids.clear(); + } +} + +void function_info::find_postdominators() { + // find postdominators using algorithm of Muchnick's Adv. Compiler Design & + // Implemmntation Fig 7.14 + printf("GPGPU-Sim PTX: Finding postdominators for \'%s\'...\n", + m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished exit block + std::vector::reverse_iterator bb_itr = + m_basic_blocks.rbegin(); + (*bb_itr)->postdominator_ids.insert( + (*bb_itr) + ->bb_id); // the only postdominator of the exit block is the exit + for (++bb_itr; bb_itr != m_basic_blocks.rend(); + bb_itr++) { // copy all basic blocks to all postdominator lists EXCEPT + // for the exit block + for (unsigned i = 0; i < m_basic_blocks.size(); i++) + (*bb_itr)->postdominator_ids.insert(i); + } + bool change = true; + while (change) { + change = false; + for (int h = m_basic_blocks.size() - 2 /*skip exit*/; h >= 0; --h) { + assert(m_basic_blocks[h]->bb_id == (unsigned)h); + std::set T; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) T.insert(i); + for (std::set::iterator s = m_basic_blocks[h]->successor_ids.begin(); + s != m_basic_blocks[h]->successor_ids.end(); s++) + intersect(T, m_basic_blocks[*s]->postdominator_ids); + T.insert(h); + if (!is_equal(T, m_basic_blocks[h]->postdominator_ids)) { + change = true; + m_basic_blocks[h]->postdominator_ids = T; } - } - assert( num_ipdoms == m_basic_blocks.size()-1 ); - // the exit node does not have an immediate post dominator, but everyone else should -} - -void function_info::find_idominators( ) -{ - // find immediate dominator blocks, using algorithm of - // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - printf("GPGPU-Sim PTX: Finding immediate dominators for \'%s\'...\n", m_name.c_str() ); - fflush(stdout); - assert( m_basic_blocks.size() >= 2 ); // must have a distinquished entry block - for (unsigned i=0; iTmp_ids = m_basic_blocks[i]->dominator_ids; - assert( m_basic_blocks[i]->bb_id == i ); - m_basic_blocks[i]->Tmp_ids.erase(i); - } - for ( int n = 0; n < m_basic_blocks.size(); ++n) { - // point iterator to basic block before the exit - for( std::set::iterator s=m_basic_blocks[n]->Tmp_ids.begin(); s != m_basic_blocks[n]->Tmp_ids.end(); s++ ) { - int bb_s = *s; - for( std::set::iterator t=m_basic_blocks[n]->Tmp_ids.begin(); t != m_basic_blocks[n]->Tmp_ids.end(); ) { - std::set::iterator t_next = t; t_next++; // might erase thing pointed to be t, invalidating iterator t - if( *s == *t ) { - t = t_next; - continue; - } - int bb_t = *t; - if( m_basic_blocks[bb_s]->dominator_ids.find(bb_t) != m_basic_blocks[bb_s]->dominator_ids.end() ) - m_basic_blocks[n]->Tmp_ids.erase(bb_t); - t = t_next; - } + } + } +} + +void function_info::find_ipostdominators() { + // find immediate postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + printf("GPGPU-Sim PTX: Finding immediate postdominators for \'%s\'...\n", + m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished exit block + for (unsigned i = 0; i < m_basic_blocks.size(); + i++) { // initialize Tmp(n) to all pdoms of n except for n + m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->postdominator_ids; + assert(m_basic_blocks[i]->bb_id == i); + m_basic_blocks[i]->Tmp_ids.erase(i); + } + for (int n = m_basic_blocks.size() - 2; n >= 0; --n) { + // point iterator to basic block before the exit + for (std::set::iterator s = m_basic_blocks[n]->Tmp_ids.begin(); + s != m_basic_blocks[n]->Tmp_ids.end(); s++) { + int bb_s = *s; + for (std::set::iterator t = m_basic_blocks[n]->Tmp_ids.begin(); + t != m_basic_blocks[n]->Tmp_ids.end();) { + std::set::iterator t_next = t; + t_next++; // might erase thing pointed to be t, invalidating iterator t + if (*s == *t) { + t = t_next; + continue; + } + int bb_t = *t; + if (m_basic_blocks[bb_s]->postdominator_ids.find(bb_t) != + m_basic_blocks[bb_s]->postdominator_ids.end()) + m_basic_blocks[n]->Tmp_ids.erase(bb_t); + t = t_next; } - } - unsigned num_idoms=0; - unsigned num_nopred = 0; - for ( int n = 0; n < m_basic_blocks.size(); ++n) { - //assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); - // if the above assert fails we have an error in either dominator - // computation, the flow graph does not have a unique entry, or some other error - if( !m_basic_blocks[n]->Tmp_ids.empty() ) { - m_basic_blocks[n]->immediatedominator_id = *m_basic_blocks[n]->Tmp_ids.begin(); - num_idoms++; - } else if (m_basic_blocks[n]->predecessor_ids.empty()) { - num_nopred += 1; + } + } + unsigned num_ipdoms = 0; + for (int n = m_basic_blocks.size() - 1; n >= 0; --n) { + assert(m_basic_blocks[n]->Tmp_ids.size() <= 1); + // if the above assert fails we have an error in either postdominator + // computation, the flow graph does not have a unique exit, or some other + // error + if (!m_basic_blocks[n]->Tmp_ids.empty()) { + m_basic_blocks[n]->immediatepostdominator_id = + *m_basic_blocks[n]->Tmp_ids.begin(); + num_ipdoms++; + } + } + assert(num_ipdoms == m_basic_blocks.size() - 1); + // the exit node does not have an immediate post dominator, but everyone else + // should +} + +void function_info::find_idominators() { + // find immediate dominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + printf("GPGPU-Sim PTX: Finding immediate dominators for \'%s\'...\n", + m_name.c_str()); + fflush(stdout); + assert(m_basic_blocks.size() >= 2); // must have a distinquished entry block + for (unsigned i = 0; i < m_basic_blocks.size(); + i++) { // initialize Tmp(n) to all doms of n except for n + m_basic_blocks[i]->Tmp_ids = m_basic_blocks[i]->dominator_ids; + assert(m_basic_blocks[i]->bb_id == i); + m_basic_blocks[i]->Tmp_ids.erase(i); + } + for (int n = 0; n < m_basic_blocks.size(); ++n) { + // point iterator to basic block before the exit + for (std::set::iterator s = m_basic_blocks[n]->Tmp_ids.begin(); + s != m_basic_blocks[n]->Tmp_ids.end(); s++) { + int bb_s = *s; + for (std::set::iterator t = m_basic_blocks[n]->Tmp_ids.begin(); + t != m_basic_blocks[n]->Tmp_ids.end();) { + std::set::iterator t_next = t; + t_next++; // might erase thing pointed to be t, invalidating iterator t + if (*s == *t) { + t = t_next; + continue; + } + int bb_t = *t; + if (m_basic_blocks[bb_s]->dominator_ids.find(bb_t) != + m_basic_blocks[bb_s]->dominator_ids.end()) + m_basic_blocks[n]->Tmp_ids.erase(bb_t); + t = t_next; } - } - assert( num_idoms == m_basic_blocks.size()-num_nopred ); - // the entry node does not have an immediate dominator, but everyone else should -} - -void function_info::print_dominators() -{ - printf("Printing dominators for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - for( std::set::iterator j=m_basic_blocks[i]->dominator_ids.begin(); j!=m_basic_blocks[i]->dominator_ids.end(); j++) - printf(" %d", *j ); - printf("\n"); - } -} - -void function_info::print_postdominators() -{ - printf("Printing postdominators for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - for( std::set::iterator j=m_basic_blocks[i]->postdominator_ids.begin(); j!=m_basic_blocks[i]->postdominator_ids.end(); j++) - printf(" %d", *j ); - printf("\n"); - } -} - -void function_info::print_ipostdominators() -{ - printf("Printing immediate postdominators for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - printf("%d\n", m_basic_blocks[i]->immediatepostdominator_id); - } -} - -void function_info::print_idominators() -{ - printf("Printing immediate dominators for function \'%s\':\n", m_name.c_str() ); - std::vector::iterator bb_itr; - for (unsigned i = 0; i < m_basic_blocks.size(); i++) { - printf("ID: %d\t:", i); - printf("%d\n", m_basic_blocks[i]->immediatedominator_id); - } -} - -unsigned function_info::get_num_reconvergence_pairs() -{ - if (!num_reconvergence_pairs) { - if( m_basic_blocks.size() == 0 ) - return 0; - for (unsigned i=0; i< (m_basic_blocks.size()-1); i++) { //last basic block containing exit obviously won't have a pair - if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { - num_reconvergence_pairs++; - } + } + } + unsigned num_idoms = 0; + unsigned num_nopred = 0; + for (int n = 0; n < m_basic_blocks.size(); ++n) { + // assert( m_basic_blocks[n]->Tmp_ids.size() <= 1 ); + // if the above assert fails we have an error in either dominator + // computation, the flow graph does not have a unique entry, or some other + // error + if (!m_basic_blocks[n]->Tmp_ids.empty()) { + m_basic_blocks[n]->immediatedominator_id = + *m_basic_blocks[n]->Tmp_ids.begin(); + num_idoms++; + } else if (m_basic_blocks[n]->predecessor_ids.empty()) { + num_nopred += 1; + } + } + assert(num_idoms == m_basic_blocks.size() - num_nopred); + // the entry node does not have an immediate dominator, but everyone else + // should +} + +void function_info::print_dominators() { + printf("Printing dominators for function \'%s\':\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + for (std::set::iterator j = m_basic_blocks[i]->dominator_ids.begin(); + j != m_basic_blocks[i]->dominator_ids.end(); j++) + printf(" %d", *j); + printf("\n"); + } +} + +void function_info::print_postdominators() { + printf("Printing postdominators for function \'%s\':\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + for (std::set::iterator j = + m_basic_blocks[i]->postdominator_ids.begin(); + j != m_basic_blocks[i]->postdominator_ids.end(); j++) + printf(" %d", *j); + printf("\n"); + } +} + +void function_info::print_ipostdominators() { + printf("Printing immediate postdominators for function \'%s\':\n", + m_name.c_str()); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + printf("%d\n", m_basic_blocks[i]->immediatepostdominator_id); + } +} + +void function_info::print_idominators() { + printf("Printing immediate dominators for function \'%s\':\n", + m_name.c_str()); + std::vector::iterator bb_itr; + for (unsigned i = 0; i < m_basic_blocks.size(); i++) { + printf("ID: %d\t:", i); + printf("%d\n", m_basic_blocks[i]->immediatedominator_id); + } +} + +unsigned function_info::get_num_reconvergence_pairs() { + if (!num_reconvergence_pairs) { + if (m_basic_blocks.size() == 0) return 0; + for (unsigned i = 0; i < (m_basic_blocks.size() - 1); + i++) { // last basic block containing exit obviously won't have a pair + if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { + num_reconvergence_pairs++; } - } - return num_reconvergence_pairs; + } + } + return num_reconvergence_pairs; } -void function_info::get_reconvergence_pairs(gpgpu_recon_t* recon_points) -{ - unsigned idx=0; //array index - if( m_basic_blocks.size() == 0 ) - return; - for (unsigned i=0; i< (m_basic_blocks.size()-1); i++) { //last basic block containing exit obviously won't have a pair +void function_info::get_reconvergence_pairs(gpgpu_recon_t *recon_points) { + unsigned idx = 0; // array index + if (m_basic_blocks.size() == 0) return; + for (unsigned i = 0; i < (m_basic_blocks.size() - 1); + i++) { // last basic block containing exit obviously won't have a pair #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("i=%d\n", i); fflush(stdout); + printf("i=%d\n", i); + fflush(stdout); #endif - if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { + if (m_basic_blocks[i]->ptx_end->get_opcode() == BRA_OP) { #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("\tbranch!\n"); - printf("\tbb_id=%d; ipdom=%d\n", m_basic_blocks[i]->bb_id, m_basic_blocks[i]->immediatepostdominator_id); - printf("\tm_instr_mem index=%d\n", m_basic_blocks[i]->ptx_end->get_m_instr_mem_index()); - fflush(stdout); + printf("\tbranch!\n"); + printf("\tbb_id=%d; ipdom=%d\n", m_basic_blocks[i]->bb_id, + m_basic_blocks[i]->immediatepostdominator_id); + printf("\tm_instr_mem index=%d\n", + m_basic_blocks[i]->ptx_end->get_m_instr_mem_index()); + fflush(stdout); #endif - recon_points[idx].source_pc = m_basic_blocks[i]->ptx_end->get_PC(); - recon_points[idx].source_inst = m_basic_blocks[i]->ptx_end; + recon_points[idx].source_pc = m_basic_blocks[i]->ptx_end->get_PC(); + recon_points[idx].source_inst = m_basic_blocks[i]->ptx_end; #ifdef DEBUG_GET_RECONVERG_PAIRS - printf("\trecon_points[idx].source_pc=%d\n", recon_points[idx].source_pc); + printf("\trecon_points[idx].source_pc=%d\n", recon_points[idx].source_pc); #endif - if( m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin ) { - recon_points[idx].target_pc = m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin->get_PC(); - recon_points[idx].target_inst = m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin; - } else { - // reconverge after function return - recon_points[idx].target_pc = -2; - recon_points[idx].target_inst = NULL; - } + if (m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin) { + recon_points[idx].target_pc = + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin->get_PC(); + recon_points[idx].target_inst = + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin; + } else { + // reconverge after function return + recon_points[idx].target_pc = -2; + recon_points[idx].target_inst = NULL; + } #ifdef DEBUG_GET_RECONVERG_PAIRS - m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id]->ptx_begin->print_insn(); - printf("\trecon_points[idx].target_pc=%d\n", recon_points[idx].target_pc); fflush(stdout); + m_basic_blocks[m_basic_blocks[i]->immediatepostdominator_id] + ->ptx_begin->print_insn(); + printf("\trecon_points[idx].target_pc=%d\n", recon_points[idx].target_pc); + fflush(stdout); #endif - idx++; - } - } + idx++; + } + } } // interface with graphviz (print the graph in DOT language) for plotting -void function_info::print_basic_block_dot() -{ - printf("Basic Block in DOT\n"); - printf("digraph %s {\n", m_name.c_str()); - std::vector::iterator bb_itr; - for (bb_itr = m_basic_blocks.begin();bb_itr != m_basic_blocks.end(); bb_itr++) { - printf("\t"); - std::set::iterator s; - for (s = (*bb_itr)->successor_ids.begin();s != (*bb_itr)->successor_ids.end();s++) { - unsigned succ_bb = *s; - printf("%d -> %d; ", (*bb_itr)->bb_id, succ_bb ); - } - printf("\n"); - } - printf("}\n"); -} - -unsigned ptx_kernel_shmem_size( void *kernel_impl ) -{ - function_info *f = (function_info*)kernel_impl; - const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); - return kernel_info->smem; -} - -unsigned ptx_kernel_nregs( void *kernel_impl ) -{ - function_info *f = (function_info*)kernel_impl; - const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); - return kernel_info->regs; -} - -unsigned type_info_key::type_decode( size_t &size, int &basic_type ) const -{ - int type = scalar_type(); - return type_decode(type,size,basic_type); -} - -unsigned type_info_key::type_decode( int type, size_t &size, int &basic_type ) -{ - switch ( type ) { - case S8_TYPE: size=8; basic_type=1; return 0; - case S16_TYPE: size=16; basic_type=1; return 1; - case S32_TYPE: size=32; basic_type=1; return 2; - case S64_TYPE: size=64; basic_type=1; return 3; - case U8_TYPE: size=8; basic_type=0; return 4; - case U16_TYPE: size=16; basic_type=0; return 5; - case U32_TYPE: size=32; basic_type=0; return 6; - case U64_TYPE: size=64; basic_type=0; return 7; - case F16_TYPE: size=16; basic_type=-1; return 8; - case F32_TYPE: size=32; basic_type=-1; return 9; - case F64_TYPE: size=64; basic_type=-1; return 10; - case FF64_TYPE: size=64; basic_type=-1; return 10; - case PRED_TYPE: size=1; basic_type=2; return 11; - case B8_TYPE: size=8; basic_type=0; return 12; - case B16_TYPE: size=16; basic_type=0; return 13; - case B32_TYPE: size=32; basic_type=0; return 14; - case B64_TYPE: size=64; basic_type=0; return 15; - case BB64_TYPE: size=64; basic_type=0; return 15; - case BB128_TYPE: size=128; basic_type=0; return 16; - case TEXREF_TYPE: case SAMPLERREF_TYPE: case SURFREF_TYPE: - size=32; basic_type=3; return 16; - default: - printf("ERROR ** type_decode() does not know about \"%s\"\n", decode_token(type) ); - assert(0); +void function_info::print_basic_block_dot() { + printf("Basic Block in DOT\n"); + printf("digraph %s {\n", m_name.c_str()); + std::vector::iterator bb_itr; + for (bb_itr = m_basic_blocks.begin(); bb_itr != m_basic_blocks.end(); + bb_itr++) { + printf("\t"); + std::set::iterator s; + for (s = (*bb_itr)->successor_ids.begin(); + s != (*bb_itr)->successor_ids.end(); s++) { + unsigned succ_bb = *s; + printf("%d -> %d; ", (*bb_itr)->bb_id, succ_bb); + } + printf("\n"); + } + printf("}\n"); +} + +unsigned ptx_kernel_shmem_size(void *kernel_impl) { + function_info *f = (function_info *)kernel_impl; + const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); + return kernel_info->smem; +} + +unsigned ptx_kernel_nregs(void *kernel_impl) { + function_info *f = (function_info *)kernel_impl; + const struct gpgpu_ptx_sim_info *kernel_info = f->get_kernel_info(); + return kernel_info->regs; +} + +unsigned type_info_key::type_decode(size_t &size, int &basic_type) const { + int type = scalar_type(); + return type_decode(type, size, basic_type); +} + +unsigned type_info_key::type_decode(int type, size_t &size, int &basic_type) { + switch (type) { + case S8_TYPE: + size = 8; + basic_type = 1; + return 0; + case S16_TYPE: + size = 16; + basic_type = 1; + return 1; + case S32_TYPE: + size = 32; + basic_type = 1; + return 2; + case S64_TYPE: + size = 64; + basic_type = 1; + return 3; + case U8_TYPE: + size = 8; + basic_type = 0; + return 4; + case U16_TYPE: + size = 16; + basic_type = 0; + return 5; + case U32_TYPE: + size = 32; + basic_type = 0; + return 6; + case U64_TYPE: + size = 64; + basic_type = 0; + return 7; + case F16_TYPE: + size = 16; + basic_type = -1; + return 8; + case F32_TYPE: + size = 32; + basic_type = -1; + return 9; + case F64_TYPE: + size = 64; + basic_type = -1; + return 10; + case FF64_TYPE: + size = 64; + basic_type = -1; + return 10; + case PRED_TYPE: + size = 1; + basic_type = 2; + return 11; + case B8_TYPE: + size = 8; + basic_type = 0; + return 12; + case B16_TYPE: + size = 16; + basic_type = 0; + return 13; + case B32_TYPE: + size = 32; + basic_type = 0; + return 14; + case B64_TYPE: + size = 64; + basic_type = 0; + return 15; + case BB64_TYPE: + size = 64; + basic_type = 0; + return 15; + case BB128_TYPE: + size = 128; + basic_type = 0; + return 16; + case TEXREF_TYPE: + case SAMPLERREF_TYPE: + case SURFREF_TYPE: + size = 32; + basic_type = 3; + return 16; + default: + printf("ERROR ** type_decode() does not know about \"%s\"\n", + decode_token(type)); + assert(0); return 0xDEADBEEF; - } -} - -arg_buffer_t copy_arg_to_buffer(ptx_thread_info * thread, operand_info actual_param_op, const symbol * formal_param) -{ - if( actual_param_op.is_reg() ) { - ptx_reg_t value = thread->get_reg(actual_param_op.get_symbol()); - return arg_buffer_t(formal_param,actual_param_op,value); - } else if ( actual_param_op.is_param_local() ) { - unsigned size=formal_param->get_size_in_bytes(); - addr_t frame_offset = actual_param_op.get_symbol()->get_address(); - addr_t from_addr = thread->get_local_mem_stack_pointer() + frame_offset; - char buffer[1024]; - assert(size<1024); - thread->m_local_mem->read(from_addr,size,buffer); - return arg_buffer_t(formal_param,actual_param_op,buffer,size); - } else { - printf("GPGPU-Sim PTX: ERROR ** need to add support for this operand type in call/return\n"); - abort(); - } -} - -void copy_args_into_buffer_list( const ptx_instruction * pI, - ptx_thread_info * thread, - const function_info * target_func, - arg_buffer_list_t &arg_values ) -{ - unsigned n_return = target_func->has_return(); - unsigned n_args = target_func->num_args(); - for( unsigned arg=0; arg < n_args; arg ++ ) { - const operand_info &actual_param_op = pI->operand_lookup(n_return+1+arg); - const symbol *formal_param = target_func->get_arg(arg); - arg_values.push_back( copy_arg_to_buffer(thread, actual_param_op, formal_param) ); - } -} - -void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a) -{ - if( a.is_reg() ) { - ptx_reg_t value = a.get_reg(); - operand_info dst_reg = operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); - thread->set_reg(dst_reg.get_symbol(),value); - } else { - const void *buffer = a.get_param_buffer(); - size_t size = a.get_param_buffer_size(); - const symbol *dst = a.get_dst(); - addr_t frame_offset = dst->get_address(); - addr_t to_addr = thread->get_local_mem_stack_pointer() + frame_offset; - thread->m_local_mem->write(to_addr,size,buffer,NULL,NULL); - } -} - -void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &arg_values) -{ - arg_buffer_list_t::iterator a; - for( a=arg_values.begin(); a != arg_values.end(); a++ ) { - copy_buffer_to_frame(thread, *a); - } -} - - - -static std::list check_operands( int opcode, - const std::list &scalar_type, - const std::list &operands, - gpgpu_context* ctx) -{ - static int g_warn_literal_operands_two_type_inst; - if( (opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || (opcode == TEX_OP) || (opcode==MMA_OP) || (opcode == DP4A_OP)) { - // just make sure these do not have have const operands... - if( !g_warn_literal_operands_two_type_inst ) { - std::list::const_iterator o; - for( o = operands.begin(); o != operands.end(); o++ ) { - const operand_info &op = *o; - if( op.is_literal() ) { - printf("GPGPU-Sim PTX: PTX uses two scalar type intruction with literal operand.\n"); - g_warn_literal_operands_two_type_inst = 1; - } - } + } +} + +arg_buffer_t copy_arg_to_buffer(ptx_thread_info *thread, + operand_info actual_param_op, + const symbol *formal_param) { + if (actual_param_op.is_reg()) { + ptx_reg_t value = thread->get_reg(actual_param_op.get_symbol()); + return arg_buffer_t(formal_param, actual_param_op, value); + } else if (actual_param_op.is_param_local()) { + unsigned size = formal_param->get_size_in_bytes(); + addr_t frame_offset = actual_param_op.get_symbol()->get_address(); + addr_t from_addr = thread->get_local_mem_stack_pointer() + frame_offset; + char buffer[1024]; + assert(size < 1024); + thread->m_local_mem->read(from_addr, size, buffer); + return arg_buffer_t(formal_param, actual_param_op, buffer, size); + } else { + printf( + "GPGPU-Sim PTX: ERROR ** need to add support for this operand type in " + "call/return\n"); + abort(); + } +} + +void copy_args_into_buffer_list(const ptx_instruction *pI, + ptx_thread_info *thread, + const function_info *target_func, + arg_buffer_list_t &arg_values) { + unsigned n_return = target_func->has_return(); + unsigned n_args = target_func->num_args(); + for (unsigned arg = 0; arg < n_args; arg++) { + const operand_info &actual_param_op = + pI->operand_lookup(n_return + 1 + arg); + const symbol *formal_param = target_func->get_arg(arg); + arg_values.push_back( + copy_arg_to_buffer(thread, actual_param_op, formal_param)); + } +} + +void copy_buffer_to_frame(ptx_thread_info *thread, const arg_buffer_t &a) { + if (a.is_reg()) { + ptx_reg_t value = a.get_reg(); + operand_info dst_reg = + operand_info(a.get_dst(), thread->get_gpu()->gpgpu_ctx); + thread->set_reg(dst_reg.get_symbol(), value); + } else { + const void *buffer = a.get_param_buffer(); + size_t size = a.get_param_buffer_size(); + const symbol *dst = a.get_dst(); + addr_t frame_offset = dst->get_address(); + addr_t to_addr = thread->get_local_mem_stack_pointer() + frame_offset; + thread->m_local_mem->write(to_addr, size, buffer, NULL, NULL); + } +} + +void copy_buffer_list_into_frame(ptx_thread_info *thread, + arg_buffer_list_t &arg_values) { + arg_buffer_list_t::iterator a; + for (a = arg_values.begin(); a != arg_values.end(); a++) { + copy_buffer_to_frame(thread, *a); + } +} + +static std::list check_operands( + int opcode, const std::list &scalar_type, + const std::list &operands, gpgpu_context *ctx) { + static int g_warn_literal_operands_two_type_inst; + if ((opcode == CVT_OP) || (opcode == SET_OP) || (opcode == SLCT_OP) || + (opcode == TEX_OP) || (opcode == MMA_OP) || (opcode == DP4A_OP)) { + // just make sure these do not have have const operands... + if (!g_warn_literal_operands_two_type_inst) { + std::list::const_iterator o; + for (o = operands.begin(); o != operands.end(); o++) { + const operand_info &op = *o; + if (op.is_literal()) { + printf( + "GPGPU-Sim PTX: PTX uses two scalar type intruction with literal " + "operand.\n"); + g_warn_literal_operands_two_type_inst = 1; } - } else { - assert( scalar_type.size() < 2 ); - if( scalar_type.size() == 1 ) { - std::list result; - int inst_type = scalar_type.front(); - std::list::const_iterator o; - for( o = operands.begin(); o != operands.end(); o++ ) { - const operand_info &op = *o; - if( op.is_literal() ) { - if( (op.get_type() == double_op_t) && (inst_type == F32_TYPE) ) { - ptx_reg_t v = op.get_literal_value(); - float u = (float)v.f64; - operand_info n(u, ctx); - result.push_back(n); - } else { - result.push_back(op); - } - } else { - result.push_back(op); - } - } - return result; - } + } } - return operands; -} - - -ptx_instruction::ptx_instruction( int opcode, - const symbol *pred, - int neg_pred, - int pred_mod, - symbol *label, - const std::list &operands, - const operand_info &return_var, - const std::list &options, - const std::list &wmma_options, - const std::list &scalar_type, - memory_space_t space_spec, - const char *file, - unsigned line, - const char *source, - const core_config *config, - gpgpu_context* ctx ) : warp_inst_t(config), m_return_var(ctx) -{ - gpgpu_ctx = ctx; - m_uid = ++(ctx->g_num_ptx_inst_uid); - m_PC = 0; - m_opcode = opcode; - m_pred = pred; - m_neg_pred = neg_pred; - m_pred_mod = pred_mod; - m_label = label; - const std::list checked_operands = check_operands(opcode,scalar_type,operands, ctx); - m_operands.insert(m_operands.begin(), checked_operands.begin(), checked_operands.end() ); - m_return_var = return_var; - m_options = options; - m_wmma_options = wmma_options; - m_wide = false; - m_hi = false; - m_lo = false; - m_uni = false; - m_exit = false; - m_abs = false; - m_neg = false; - m_to_option = false; - m_cache_option = 0; - m_rounding_mode = RN_OPTION; - m_compare_op = -1; - m_saturation_mode = 0; - m_geom_spec = 0; - m_vector_spec = 0; - m_atomic_spec = 0; - m_membar_level = 0; - m_inst_size = 8; // bytes - int rr=0; - std::list::const_iterator i; - unsigned n=1; - for ( i=wmma_options.begin(); i!= wmma_options.end(); i++, n++ ) { - int last_ptx_inst_option = *i; - switch ( last_ptx_inst_option ) { - case SYNC_OPTION: - case LOAD_A: - case LOAD_B: - case LOAD_C: - case STORE_D: - case MMA: - m_wmma_type=last_ptx_inst_option; - break; - case ROW: - case COL: - m_wmma_layout[rr++]=last_ptx_inst_option; - break; - case M16N16K16: - case M32N8K16: - case M8N32K16: - break; - default: - assert(0); - break; - } - } - rr=0; - n=1; - for ( i=options.begin(); i!= options.end(); i++, n++ ) { - int last_ptx_inst_option = *i; - switch ( last_ptx_inst_option ) { + } else { + assert(scalar_type.size() < 2); + if (scalar_type.size() == 1) { + std::list result; + int inst_type = scalar_type.front(); + std::list::const_iterator o; + for (o = operands.begin(); o != operands.end(); o++) { + const operand_info &op = *o; + if (op.is_literal()) { + if ((op.get_type() == double_op_t) && (inst_type == F32_TYPE)) { + ptx_reg_t v = op.get_literal_value(); + float u = (float)v.f64; + operand_info n(u, ctx); + result.push_back(n); + } else { + result.push_back(op); + } + } else { + result.push_back(op); + } + } + return result; + } + } + return operands; +} + +ptx_instruction::ptx_instruction( + int opcode, const symbol *pred, int neg_pred, int pred_mod, symbol *label, + const std::list &operands, const operand_info &return_var, + const std::list &options, const std::list &wmma_options, + const std::list &scalar_type, memory_space_t space_spec, + const char *file, unsigned line, const char *source, + const core_config *config, gpgpu_context *ctx) + : warp_inst_t(config), m_return_var(ctx) { + gpgpu_ctx = ctx; + m_uid = ++(ctx->g_num_ptx_inst_uid); + m_PC = 0; + m_opcode = opcode; + m_pred = pred; + m_neg_pred = neg_pred; + m_pred_mod = pred_mod; + m_label = label; + const std::list checked_operands = + check_operands(opcode, scalar_type, operands, ctx); + m_operands.insert(m_operands.begin(), checked_operands.begin(), + checked_operands.end()); + m_return_var = return_var; + m_options = options; + m_wmma_options = wmma_options; + m_wide = false; + m_hi = false; + m_lo = false; + m_uni = false; + m_exit = false; + m_abs = false; + m_neg = false; + m_to_option = false; + m_cache_option = 0; + m_rounding_mode = RN_OPTION; + m_compare_op = -1; + m_saturation_mode = 0; + m_geom_spec = 0; + m_vector_spec = 0; + m_atomic_spec = 0; + m_membar_level = 0; + m_inst_size = 8; // bytes + int rr = 0; + std::list::const_iterator i; + unsigned n = 1; + for (i = wmma_options.begin(); i != wmma_options.end(); i++, n++) { + int last_ptx_inst_option = *i; + switch (last_ptx_inst_option) { + case SYNC_OPTION: + case LOAD_A: + case LOAD_B: + case LOAD_C: + case STORE_D: + case MMA: + m_wmma_type = last_ptx_inst_option; + break; + case ROW: + case COL: + m_wmma_layout[rr++] = last_ptx_inst_option; + break; + case M16N16K16: + case M32N8K16: + case M8N32K16: + break; + default: + assert(0); + break; + } + } + rr = 0; + n = 1; + for (i = options.begin(); i != options.end(); i++, n++) { + int last_ptx_inst_option = *i; + switch (last_ptx_inst_option) { case SYNC_OPTION: case ARRIVE_OPTION: case RED_OPTION: - m_barrier_op = last_ptx_inst_option; - break; + m_barrier_op = last_ptx_inst_option; + break; case EQU_OPTION: case NEU_OPTION: case LTU_OPTION: @@ -1186,16 +1279,16 @@ ptx_instruction::ptx_instruction( int opcode, case GE_OPTION: case LS_OPTION: case HS_OPTION: - m_compare_op = last_ptx_inst_option; - break; + m_compare_op = last_ptx_inst_option; + break; case NUM_OPTION: case NAN_OPTION: - m_compare_op = last_ptx_inst_option; + m_compare_op = last_ptx_inst_option; // assert(0); // finish this - break; + break; case SAT_OPTION: - m_saturation_mode = 1; - break; + m_saturation_mode = 1; + break; case RNI_OPTION: case RZI_OPTION: case RMI_OPTION: @@ -1204,38 +1297,39 @@ ptx_instruction::ptx_instruction( int opcode, case RZ_OPTION: case RM_OPTION: case RP_OPTION: - m_rounding_mode = last_ptx_inst_option; - break; + m_rounding_mode = last_ptx_inst_option; + break; case HI_OPTION: - m_compare_op = last_ptx_inst_option; - m_hi = true; - assert( !m_lo ); - assert( !m_wide ); - break; + m_compare_op = last_ptx_inst_option; + m_hi = true; + assert(!m_lo); + assert(!m_wide); + break; case LO_OPTION: - m_compare_op = last_ptx_inst_option; - m_lo = true; - assert( !m_hi ); - assert( !m_wide ); - break; + m_compare_op = last_ptx_inst_option; + m_lo = true; + assert(!m_hi); + assert(!m_wide); + break; case WIDE_OPTION: - m_wide = true; - assert( !m_lo ); - assert( !m_hi ); - break; + m_wide = true; + assert(!m_lo); + assert(!m_hi); + break; case UNI_OPTION: - m_uni = true; // don't care... < now we DO care when constructing flowgraph> - break; + m_uni = true; // don't care... < now we DO care when constructing + // flowgraph> + break; case GEOM_MODIFIER_1D: case GEOM_MODIFIER_2D: case GEOM_MODIFIER_3D: - m_geom_spec = last_ptx_inst_option; - break; + m_geom_spec = last_ptx_inst_option; + break; case V2_TYPE: case V3_TYPE: case V4_TYPE: - m_vector_spec = last_ptx_inst_option; - break; + m_vector_spec = last_ptx_inst_option; + break; case ATOMIC_AND: case ATOMIC_OR: case ATOMIC_XOR: @@ -1246,223 +1340,225 @@ ptx_instruction::ptx_instruction( int opcode, case ATOMIC_DEC: case ATOMIC_MIN: case ATOMIC_MAX: - m_atomic_spec = last_ptx_inst_option; - break; + m_atomic_spec = last_ptx_inst_option; + break; case APPROX_OPTION: - break; + break; case FULL_OPTION: - break; + break; case ANY_OPTION: - m_vote_mode = vote_any; - break; + m_vote_mode = vote_any; + break; case ALL_OPTION: - m_vote_mode = vote_all; - break; + m_vote_mode = vote_all; + break; case BALLOT_OPTION: - m_vote_mode = vote_ballot; - break; + m_vote_mode = vote_ballot; + break; case GLOBAL_OPTION: - m_membar_level = GLOBAL_OPTION; - break; + m_membar_level = GLOBAL_OPTION; + break; case CTA_OPTION: - m_membar_level = CTA_OPTION; - break; + m_membar_level = CTA_OPTION; + break; case SYS_OPTION: - m_membar_level = SYS_OPTION; - break; + m_membar_level = SYS_OPTION; + break; case FTZ_OPTION: - break; + break; case EXIT_OPTION: - m_exit = true; - break; + m_exit = true; + break; case ABS_OPTION: - m_abs = true; - break; + m_abs = true; + break; case NEG_OPTION: - m_neg = true; - break; + m_neg = true; + break; case TO_OPTION: - m_to_option = true; - break; - case CA_OPTION: case CG_OPTION: case CS_OPTION: case LU_OPTION: case CV_OPTION: - m_cache_option = last_ptx_inst_option; - break; + m_to_option = true; + break; + case CA_OPTION: + case CG_OPTION: + case CS_OPTION: + case LU_OPTION: + case CV_OPTION: + m_cache_option = last_ptx_inst_option; + break; case HALF_OPTION: - m_inst_size = 4; // bytes - break; + m_inst_size = 4; // bytes + break; case EXTP_OPTION: - break; + break; case NC_OPTION: - m_cache_option = last_ptx_inst_option; - break; + m_cache_option = last_ptx_inst_option; + break; case UP_OPTION: case DOWN_OPTION: case BFLY_OPTION: case IDX_OPTION: - m_shfl_op = last_ptx_inst_option; - break; + m_shfl_op = last_ptx_inst_option; + break; case PRMT_F4E_MODE: case PRMT_B4E_MODE: case PRMT_RC8_MODE: case PRMT_ECL_MODE: case PRMT_ECR_MODE: case PRMT_RC16_MODE: - m_prmt_op = last_ptx_inst_option; - break; + m_prmt_op = last_ptx_inst_option; + break; default: - assert(0); - break; - } - } - m_scalar_type = scalar_type; - m_space_spec = space_spec; - if( ( opcode == ST_OP || opcode == LD_OP || opcode == LDU_OP ) && (space_spec == undefined_space) ) { - m_space_spec = generic_space; - } - for( std::vector::const_iterator i=m_operands.begin(); i!=m_operands.end(); ++i) { - const operand_info &op = *i; - if( op.get_addr_space() != undefined_space ) - m_space_spec = op.get_addr_space(); // TODO: can have more than one memory space for ptxplus (g8x) inst - } - if( opcode == TEX_OP ) - m_space_spec = tex_space; - - m_source_file = file?file:""; - m_source_line = line; - m_source = source; - // Trim tabs - m_source.erase( std::remove( m_source.begin(), m_source.end(), '\t' ), m_source.end() ); - - if (opcode == CALL_OP) { - const operand_info &target = func_addr(); - assert( target.is_function_address() ); - const symbol *func_addr = target.get_symbol(); - const function_info *target_func = func_addr->get_pc(); - std::string fname = target_func->get_name(); - - if (fname =="vprintf"){ - m_is_printf = true; - } - if(fname == "cudaStreamCreateWithFlags") - m_is_cdp = 1; - if(fname == "cudaGetParameterBufferV2") - m_is_cdp = 2; - if(fname == "cudaLaunchDeviceV2") - m_is_cdp = 4; - - } -} - -void ptx_instruction::print_insn() const -{ - print_insn(stdout); - fflush(stdout); -} - -void ptx_instruction::print_insn( FILE *fp ) const -{ - fprintf( fp, "%s", to_string().c_str() ); -} - -std::string ptx_instruction::to_string() const -{ - char buf[ STR_SIZE ]; - unsigned used_bytes = 0; - if( !is_label() ) { - used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, " PC=0x%03x ", m_PC ); - } else { - used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, " " ); - } - used_bytes += snprintf( buf + used_bytes, STR_SIZE - used_bytes, - "(%s:%d) %s", - m_source_file.c_str(), m_source_line, - m_source.c_str() ); - return std::string( buf ); -} -operand_info ptx_instruction::get_pred() const -{ - return operand_info( m_pred, gpgpu_ctx); -} - - -function_info::function_info(int entry_point, gpgpu_context* ctx ) -{ - gpgpu_ctx = ctx; - m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; - m_entry_point = (entry_point==1)?true:false; - m_extern = (entry_point==2)?true:false; - num_reconvergence_pairs = 0; - m_symtab = NULL; - m_assembled = false; - m_return_var_sym = NULL; - m_kernel_info.cmem = 0; - m_kernel_info.lmem = 0; - m_kernel_info.regs = 0; - m_kernel_info.smem = 0; - m_local_mem_framesize = 0; - m_args_aligned_size = -1; - pdom_done = false; //initialize it to false -} - -unsigned function_info::print_insn( unsigned pc, FILE * fp ) const -{ - unsigned inst_size=1; // return offset to next instruction or 1 if unknown - unsigned index = pc - m_start_PC; - char command[1024]; - char buffer[1024]; - memset(command, 0, 1024); - memset(buffer, 0, 1024); - snprintf(command,1024,"c++filt -p %s",m_name.c_str()); - FILE *p = popen(command,"r"); - buffer[0]=0; - assert(fgets(buffer, 1023, p) != NULL); - // Remove trailing "\n" in buffer - char *c; - if ((c=strchr(buffer, '\n')) != NULL) *c = '\0'; - fprintf(fp,"%s",buffer); - if ( index >= m_instr_mem_size ) { - fprintf(fp, "", m_start_PC + m_instr_mem_size - 1 ); - } else { - if ( m_instr_mem[index] != NULL ) { - m_instr_mem[index]->print_insn(fp); - inst_size = m_instr_mem[index]->isize; - } else - fprintf(fp, "", pc ); - } - pclose(p); - return inst_size; -} - -std::string function_info::get_insn_str( unsigned pc ) const -{ - unsigned index = pc - m_start_PC; - if ( index >= m_instr_mem_size ) { + assert(0); + break; + } + } + m_scalar_type = scalar_type; + m_space_spec = space_spec; + if ((opcode == ST_OP || opcode == LD_OP || opcode == LDU_OP) && + (space_spec == undefined_space)) { + m_space_spec = generic_space; + } + for (std::vector::const_iterator i = m_operands.begin(); + i != m_operands.end(); ++i) { + const operand_info &op = *i; + if (op.get_addr_space() != undefined_space) + m_space_spec = + op.get_addr_space(); // TODO: can have more than one memory space for + // ptxplus (g8x) inst + } + if (opcode == TEX_OP) m_space_spec = tex_space; + + m_source_file = file ? file : ""; + m_source_line = line; + m_source = source; + // Trim tabs + m_source.erase(std::remove(m_source.begin(), m_source.end(), '\t'), + m_source.end()); + + if (opcode == CALL_OP) { + const operand_info &target = func_addr(); + assert(target.is_function_address()); + const symbol *func_addr = target.get_symbol(); + const function_info *target_func = func_addr->get_pc(); + std::string fname = target_func->get_name(); + + if (fname == "vprintf") { + m_is_printf = true; + } + if (fname == "cudaStreamCreateWithFlags") m_is_cdp = 1; + if (fname == "cudaGetParameterBufferV2") m_is_cdp = 2; + if (fname == "cudaLaunchDeviceV2") m_is_cdp = 4; + } +} + +void ptx_instruction::print_insn() const { + print_insn(stdout); + fflush(stdout); +} + +void ptx_instruction::print_insn(FILE *fp) const { + fprintf(fp, "%s", to_string().c_str()); +} + +std::string ptx_instruction::to_string() const { + char buf[STR_SIZE]; + unsigned used_bytes = 0; + if (!is_label()) { + used_bytes += + snprintf(buf + used_bytes, STR_SIZE - used_bytes, " PC=0x%03x ", m_PC); + } else { + used_bytes += + snprintf(buf + used_bytes, STR_SIZE - used_bytes, " "); + } + used_bytes += + snprintf(buf + used_bytes, STR_SIZE - used_bytes, "(%s:%d) %s", + m_source_file.c_str(), m_source_line, m_source.c_str()); + return std::string(buf); +} +operand_info ptx_instruction::get_pred() const { + return operand_info(m_pred, gpgpu_ctx); +} + +function_info::function_info(int entry_point, gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = (gpgpu_ctx->function_info_sm_next_uid)++; + m_entry_point = (entry_point == 1) ? true : false; + m_extern = (entry_point == 2) ? true : false; + num_reconvergence_pairs = 0; + m_symtab = NULL; + m_assembled = false; + m_return_var_sym = NULL; + m_kernel_info.cmem = 0; + m_kernel_info.lmem = 0; + m_kernel_info.regs = 0; + m_kernel_info.smem = 0; + m_local_mem_framesize = 0; + m_args_aligned_size = -1; + pdom_done = false; // initialize it to false +} + +unsigned function_info::print_insn(unsigned pc, FILE *fp) const { + unsigned inst_size = 1; // return offset to next instruction or 1 if unknown + unsigned index = pc - m_start_PC; + char command[1024]; + char buffer[1024]; + memset(command, 0, 1024); + memset(buffer, 0, 1024); + snprintf(command, 1024, "c++filt -p %s", m_name.c_str()); + FILE *p = popen(command, "r"); + buffer[0] = 0; + assert(fgets(buffer, 1023, p) != NULL); + // Remove trailing "\n" in buffer + char *c; + if ((c = strchr(buffer, '\n')) != NULL) *c = '\0'; + fprintf(fp, "%s", buffer); + if (index >= m_instr_mem_size) { + fprintf(fp, "", + m_start_PC + m_instr_mem_size - 1); + } else { + if (m_instr_mem[index] != NULL) { + m_instr_mem[index]->print_insn(fp); + inst_size = m_instr_mem[index]->isize; + } else + fprintf(fp, "", pc); + } + pclose(p); + return inst_size; +} + +std::string function_info::get_insn_str(unsigned pc) const { + unsigned index = pc - m_start_PC; + if (index >= m_instr_mem_size) { + char buff[STR_SIZE]; + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE, "", + m_start_PC + m_instr_mem_size - 1); + return std::string(buff); + } else { + if (m_instr_mem[index] != NULL) { + return m_instr_mem[index]->to_string(); + } else { char buff[STR_SIZE]; - buff[STR_SIZE-1] = '\0'; - snprintf(buff, STR_SIZE, "", m_start_PC + m_instr_mem_size - 1 ); + buff[STR_SIZE - 1] = '\0'; + snprintf(buff, STR_SIZE, "", pc); return std::string(buff); - } else { - if ( m_instr_mem[index] != NULL ) { - return m_instr_mem[index]->to_string(); - } else { - char buff[STR_SIZE]; - buff[STR_SIZE-1] = '\0'; - snprintf(buff, STR_SIZE, "", pc ); - return std::string(buff); - } - } -} - -void gpgpu_ptx_assemble( std::string kname, void *kinfo ) -{ - function_info *func_info = (function_info *)kinfo; - if((function_info *)kinfo == NULL) { - printf("GPGPU-Sim PTX: Warning - missing function definition \'%s\'\n", kname.c_str()); - return; - } - if( func_info->is_extern() ) { - printf("GPGPU-Sim PTX: skipping assembly for extern declared function \'%s\'\n", func_info->get_name().c_str() ); - return; } - func_info->ptx_assemble(); + } +} + +void gpgpu_ptx_assemble(std::string kname, void *kinfo) { + function_info *func_info = (function_info *)kinfo; + if ((function_info *)kinfo == NULL) { + printf("GPGPU-Sim PTX: Warning - missing function definition \'%s\'\n", + kname.c_str()); + return; + } + if (func_info->is_extern()) { + printf( + "GPGPU-Sim PTX: skipping assembly for extern declared function " + "\'%s\'\n", + func_info->get_name().c_str()); + return; + } + func_info->ptx_assemble(); } diff --git a/src/cuda-sim/ptx_ir.h b/src/cuda-sim/ptx_ir.h index f4c5c37..6627847 100644 --- a/src/cuda-sim/ptx_ir.h +++ b/src/cuda-sim/ptx_ir.h @@ -7,36 +7,37 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef ptx_ir_INCLUDED #define ptx_ir_INCLUDED #include "../abstract_hardware_model.h" +#include #include #include -#include #include #include +#include #include -#include //#include "ptx.tab.h" #include "ptx_sim.h" @@ -46,1537 +47,1517 @@ class gpgpu_context; class type_info_key { -public: - type_info_key() - { - m_is_non_arch_reg = false; - m_init = false; - } - type_info_key( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec, int array_dim ) - { - m_is_non_arch_reg = false; - m_init = true; - m_space_spec = space_spec; - m_scalar_type_spec = scalar_type_spec; - m_vector_spec = vector_spec; - m_alignment_spec = alignment_spec; - m_extern_spec = extern_spec; - m_array_dim = array_dim; - m_is_function = 0; - } - void set_is_func() - { - assert(!m_init); - m_init = true; - m_space_spec = undefined_space; - m_scalar_type_spec = 0; - m_vector_spec = 0; - m_alignment_spec = 0; - m_extern_spec = 0; - m_array_dim = 0; - m_is_function = 1; - } - - void set_array_dim( int array_dim ) { m_array_dim = array_dim; } - int get_array_dim() const { assert(m_init); return m_array_dim; } - void set_is_non_arch_reg() { m_is_non_arch_reg = true; } - - bool is_non_arch_reg() const { return m_is_non_arch_reg; } - bool is_reg() const { return m_space_spec == reg_space;} - bool is_param_kernel() const { return m_space_spec == param_space_kernel;} - bool is_param_local() const { return m_space_spec == param_space_local; } - bool is_param_unclassified() const { return m_space_spec == param_space_unclassified; } - bool is_global() const { return m_space_spec == global_space;} - bool is_local() const { return m_space_spec == local_space;} - bool is_shared() const { return m_space_spec == shared_space;} - bool is_const() const { return m_space_spec.get_type() == const_space;} - bool is_tex() const { return m_space_spec == tex_space;} - bool is_func_addr() const { return m_is_function?true:false; } - int scalar_type() const { return m_scalar_type_spec;} - int get_alignment_spec() const { return m_alignment_spec;} - unsigned type_decode( size_t &size, int &t ) const; - static unsigned type_decode( int type, size_t &size, int &t ); - memory_space_t get_memory_space() const { return m_space_spec; } -private: - bool m_init; - memory_space_t m_space_spec; - int m_scalar_type_spec; - int m_vector_spec; - int m_alignment_spec; - int m_extern_spec; - int m_array_dim; - int m_is_function; - bool m_is_non_arch_reg; - - friend struct type_info_key_compare; + public: + type_info_key() { + m_is_non_arch_reg = false; + m_init = false; + } + type_info_key(memory_space_t space_spec, int scalar_type_spec, + int vector_spec, int alignment_spec, int extern_spec, + int array_dim) { + m_is_non_arch_reg = false; + m_init = true; + m_space_spec = space_spec; + m_scalar_type_spec = scalar_type_spec; + m_vector_spec = vector_spec; + m_alignment_spec = alignment_spec; + m_extern_spec = extern_spec; + m_array_dim = array_dim; + m_is_function = 0; + } + void set_is_func() { + assert(!m_init); + m_init = true; + m_space_spec = undefined_space; + m_scalar_type_spec = 0; + m_vector_spec = 0; + m_alignment_spec = 0; + m_extern_spec = 0; + m_array_dim = 0; + m_is_function = 1; + } + + void set_array_dim(int array_dim) { m_array_dim = array_dim; } + int get_array_dim() const { + assert(m_init); + return m_array_dim; + } + void set_is_non_arch_reg() { m_is_non_arch_reg = true; } + + bool is_non_arch_reg() const { return m_is_non_arch_reg; } + bool is_reg() const { return m_space_spec == reg_space; } + bool is_param_kernel() const { return m_space_spec == param_space_kernel; } + bool is_param_local() const { return m_space_spec == param_space_local; } + bool is_param_unclassified() const { + return m_space_spec == param_space_unclassified; + } + bool is_global() const { return m_space_spec == global_space; } + bool is_local() const { return m_space_spec == local_space; } + bool is_shared() const { return m_space_spec == shared_space; } + bool is_const() const { return m_space_spec.get_type() == const_space; } + bool is_tex() const { return m_space_spec == tex_space; } + bool is_func_addr() const { return m_is_function ? true : false; } + int scalar_type() const { return m_scalar_type_spec; } + int get_alignment_spec() const { return m_alignment_spec; } + unsigned type_decode(size_t &size, int &t) const; + static unsigned type_decode(int type, size_t &size, int &t); + memory_space_t get_memory_space() const { return m_space_spec; } + + private: + bool m_init; + memory_space_t m_space_spec; + int m_scalar_type_spec; + int m_vector_spec; + int m_alignment_spec; + int m_extern_spec; + int m_array_dim; + int m_is_function; + bool m_is_non_arch_reg; + + friend struct type_info_key_compare; }; class symbol_table; struct type_info_key_compare { - bool operator()( const type_info_key &a, const type_info_key &b ) const - { - assert( a.m_init && b.m_init ); - if ( a.m_space_spec < b.m_space_spec ) return true; - if ( a.m_scalar_type_spec < b.m_scalar_type_spec ) return true; - if ( a.m_vector_spec < b.m_vector_spec ) return true; - if ( a.m_alignment_spec < b.m_alignment_spec ) return true; - if ( a.m_extern_spec < b.m_extern_spec ) return true; - if ( a.m_array_dim < b.m_array_dim ) return true; - if ( a.m_is_function < b.m_is_function ) return true; - - return false; - } + bool operator()(const type_info_key &a, const type_info_key &b) const { + assert(a.m_init && b.m_init); + if (a.m_space_spec < b.m_space_spec) return true; + if (a.m_scalar_type_spec < b.m_scalar_type_spec) return true; + if (a.m_vector_spec < b.m_vector_spec) return true; + if (a.m_alignment_spec < b.m_alignment_spec) return true; + if (a.m_extern_spec < b.m_extern_spec) return true; + if (a.m_array_dim < b.m_array_dim) return true; + if (a.m_is_function < b.m_is_function) return true; + + return false; + } }; class type_info { -public: - type_info( symbol_table *scope, type_info_key t ) - { - m_type_info = t; - } - const type_info_key &get_key() const { return m_type_info;} - -private: - symbol_table *m_scope; - type_info_key m_type_info; + public: + type_info(symbol_table *scope, type_info_key t) { m_type_info = t; } + const type_info_key &get_key() const { return m_type_info; } + + private: + symbol_table *m_scope; + type_info_key m_type_info; }; enum operand_type { - reg_t, vector_t, builtin_t, address_t, memory_t, float_op_t, double_op_t, int_t, - unsigned_t, symbolic_t, label_t, v_reg_t, v_float_op_t, v_double_op_t, - v_int_t, v_unsigned_t, undef_t + reg_t, + vector_t, + builtin_t, + address_t, + memory_t, + float_op_t, + double_op_t, + int_t, + unsigned_t, + symbolic_t, + label_t, + v_reg_t, + v_float_op_t, + v_double_op_t, + v_int_t, + v_unsigned_t, + undef_t }; class operand_info; class symbol { -public: - symbol( const char *name, const type_info *type, const char *location, unsigned size, gpgpu_context* ctx ) - { - gpgpu_ctx = ctx; - m_uid = get_uid(); - m_name = name; - m_decl_location = location; - m_type = type; - m_size = size; - m_address_valid = false; - m_is_label = false; - m_is_shared = false; - m_is_const = false; - m_is_global = false; - m_is_local = false; - m_is_param_local = false; - m_is_param_kernel = false; - m_is_tex = false; - m_is_func_addr = false; - m_reg_num_valid = false; - m_function = NULL; - m_reg_num=(unsigned)-1; - m_arch_reg_num=(unsigned)-1; - m_address=(unsigned)-1; - m_initializer.clear(); - if ( type ) m_is_shared = type->get_key().is_shared(); - if ( type ) m_is_const = type->get_key().is_const(); - if ( type ) m_is_global = type->get_key().is_global(); - if ( type ) m_is_local = type->get_key().is_local(); - if ( type ) m_is_param_local = type->get_key().is_param_local(); - if ( type ) m_is_param_kernel = type->get_key().is_param_kernel(); - if ( type ) m_is_tex = type->get_key().is_tex(); - if ( type ) m_is_func_addr = type->get_key().is_func_addr(); - } - unsigned get_size_in_bytes() const - { - return m_size; - } - const std::string &name() const { return m_name;} - const std::string &decl_location() const { return m_decl_location;} - const type_info *type() const { return m_type;} - addr_t get_address() const - { - assert( m_is_label || !m_type->get_key().is_reg() ); // todo : other assertions - assert( m_address_valid ); - return m_address; - } - function_info *get_pc() const - { - return m_function; - } - void set_regno( unsigned regno, unsigned arch_regno ) - { - m_reg_num_valid = true; - m_reg_num = regno; - m_arch_reg_num = arch_regno; - } - - void set_address( addr_t addr ) - { - m_address_valid = true; - m_address = addr; - } - void set_label_address( addr_t addr) - { - m_address_valid = true; - m_address = addr; - m_is_label = true; - } - void set_function( function_info *func ) - { - m_function = func; - m_is_func_addr = true; - } - - bool is_label() const { return m_is_label;} - bool is_shared() const { return m_is_shared;} - bool is_sstarr() const { return m_is_sstarr;} - bool is_const() const { return m_is_const;} - bool is_global() const { return m_is_global;} - bool is_local() const { return m_is_local;} - bool is_param_local() const { return m_is_param_local; } - bool is_param_kernel() const { return m_is_param_kernel; } - bool is_tex() const { return m_is_tex;} - bool is_func_addr() const { return m_is_func_addr; } - bool is_reg() const - { - if ( m_type == NULL ) { - return false; - } - return m_type->get_key().is_reg(); - } - bool is_non_arch_reg() const - { - if ( m_type == NULL ) { - return false; - } - return m_type->get_key().is_non_arch_reg(); - } - - void add_initializer( const std::list &init ); - bool has_initializer() const - { - return m_initializer.size() > 0; - } - std::list get_initializer() const - { - return m_initializer; - } - unsigned reg_num() const - { - assert( m_reg_num_valid ); - return m_reg_num; - } - unsigned arch_reg_num() const - { - assert( m_reg_num_valid ); - return m_arch_reg_num; - } - void print_info(FILE *fp) const; - unsigned uid() const { return m_uid; } - -private: - gpgpu_context* gpgpu_ctx; - unsigned get_uid(); - unsigned m_uid; - const type_info *m_type; - unsigned m_size; // in bytes - std::string m_name; - std::string m_decl_location; - - unsigned m_address; - function_info *m_function; // used for function symbols - - bool m_address_valid; - bool m_is_label; - bool m_is_shared; - bool m_is_sstarr; - bool m_is_const; - bool m_is_global; - bool m_is_local; - bool m_is_param_local; - bool m_is_param_kernel; - bool m_is_tex; - bool m_is_func_addr; - unsigned m_reg_num; - unsigned m_arch_reg_num; - bool m_reg_num_valid; - - std::list m_initializer; + public: + symbol(const char *name, const type_info *type, const char *location, + unsigned size, gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = get_uid(); + m_name = name; + m_decl_location = location; + m_type = type; + m_size = size; + m_address_valid = false; + m_is_label = false; + m_is_shared = false; + m_is_const = false; + m_is_global = false; + m_is_local = false; + m_is_param_local = false; + m_is_param_kernel = false; + m_is_tex = false; + m_is_func_addr = false; + m_reg_num_valid = false; + m_function = NULL; + m_reg_num = (unsigned)-1; + m_arch_reg_num = (unsigned)-1; + m_address = (unsigned)-1; + m_initializer.clear(); + if (type) m_is_shared = type->get_key().is_shared(); + if (type) m_is_const = type->get_key().is_const(); + if (type) m_is_global = type->get_key().is_global(); + if (type) m_is_local = type->get_key().is_local(); + if (type) m_is_param_local = type->get_key().is_param_local(); + if (type) m_is_param_kernel = type->get_key().is_param_kernel(); + if (type) m_is_tex = type->get_key().is_tex(); + if (type) m_is_func_addr = type->get_key().is_func_addr(); + } + unsigned get_size_in_bytes() const { return m_size; } + const std::string &name() const { return m_name; } + const std::string &decl_location() const { return m_decl_location; } + const type_info *type() const { return m_type; } + addr_t get_address() const { + assert(m_is_label || + !m_type->get_key().is_reg()); // todo : other assertions + assert(m_address_valid); + return m_address; + } + function_info *get_pc() const { return m_function; } + void set_regno(unsigned regno, unsigned arch_regno) { + m_reg_num_valid = true; + m_reg_num = regno; + m_arch_reg_num = arch_regno; + } + + void set_address(addr_t addr) { + m_address_valid = true; + m_address = addr; + } + void set_label_address(addr_t addr) { + m_address_valid = true; + m_address = addr; + m_is_label = true; + } + void set_function(function_info *func) { + m_function = func; + m_is_func_addr = true; + } + + bool is_label() const { return m_is_label; } + bool is_shared() const { return m_is_shared; } + bool is_sstarr() const { return m_is_sstarr; } + bool is_const() const { return m_is_const; } + bool is_global() const { return m_is_global; } + bool is_local() const { return m_is_local; } + bool is_param_local() const { return m_is_param_local; } + bool is_param_kernel() const { return m_is_param_kernel; } + bool is_tex() const { return m_is_tex; } + bool is_func_addr() const { return m_is_func_addr; } + bool is_reg() const { + if (m_type == NULL) { + return false; + } + return m_type->get_key().is_reg(); + } + bool is_non_arch_reg() const { + if (m_type == NULL) { + return false; + } + return m_type->get_key().is_non_arch_reg(); + } + + void add_initializer(const std::list &init); + bool has_initializer() const { return m_initializer.size() > 0; } + std::list get_initializer() const { return m_initializer; } + unsigned reg_num() const { + assert(m_reg_num_valid); + return m_reg_num; + } + unsigned arch_reg_num() const { + assert(m_reg_num_valid); + return m_arch_reg_num; + } + void print_info(FILE *fp) const; + unsigned uid() const { return m_uid; } + + private: + gpgpu_context *gpgpu_ctx; + unsigned get_uid(); + unsigned m_uid; + const type_info *m_type; + unsigned m_size; // in bytes + std::string m_name; + std::string m_decl_location; + + unsigned m_address; + function_info *m_function; // used for function symbols + + bool m_address_valid; + bool m_is_label; + bool m_is_shared; + bool m_is_sstarr; + bool m_is_const; + bool m_is_global; + bool m_is_local; + bool m_is_param_local; + bool m_is_param_kernel; + bool m_is_tex; + bool m_is_func_addr; + unsigned m_reg_num; + unsigned m_arch_reg_num; + bool m_reg_num_valid; + + std::list m_initializer; }; class symbol_table { -public: - symbol_table(); - symbol_table( const char *scope_name, unsigned entry_point, symbol_table *parent, gpgpu_context* ctx); - void set_name( const char *name ); - const ptx_version &get_ptx_version() const; - unsigned get_sm_target() const; - void set_ptx_version( float ver, unsigned ext ); - void set_sm_target( const char *target, const char *ext, const char *ext2 ); - symbol* lookup( const char *identifier ); - std::string get_scope_name() const { return m_scope_name; } - symbol *add_variable( const char *identifier, const type_info *type, unsigned size, const char *filename, unsigned line ); - void add_function( function_info *func, const char *filename, unsigned linenumber ); - bool add_function_decl( const char *name, int entry_point, function_info **func_info, symbol_table **symbol_table ); - function_info *lookup_function(std::string name); - type_info *add_type( memory_space_t space_spec, int scalar_type_spec, int vector_spec, int alignment_spec, int extern_spec ); - type_info *add_type( function_info *func ); - type_info *get_array_type( type_info *base_type, unsigned array_dim ); - void set_label_address( const symbol *label, unsigned addr ); - unsigned next_reg_num() { return ++m_reg_allocator;} - addr_t get_shared_next() { return m_shared_next;} - addr_t get_sstarr_next() { return m_sstarr_next;} - addr_t get_global_next() { return m_global_next;} - addr_t get_local_next() { return m_local_next;} - addr_t get_tex_next() { return m_tex_next;} - void alloc_shared( unsigned num_bytes ) { m_shared_next += num_bytes;} - void alloc_sstarr( unsigned num_bytes ) { m_sstarr_next += num_bytes;} - void alloc_global( unsigned num_bytes ) { m_global_next += num_bytes;} - void alloc_local( unsigned num_bytes ) { m_local_next += num_bytes;} - void alloc_tex( unsigned num_bytes ) { m_tex_next += num_bytes;} - - typedef std::list::iterator iterator; - - iterator global_iterator_begin() { return m_globals.begin();} - iterator global_iterator_end() { return m_globals.end();} - - iterator const_iterator_begin() { return m_consts.begin();} - iterator const_iterator_end() { return m_consts.end();} - - void dump(); - - //Jin: handle instruction group for cdp - symbol_table* start_inst_group(); - symbol_table* end_inst_group(); - - // backward pointer - class gpgpu_context* gpgpu_ctx; - -private: - unsigned m_reg_allocator; - unsigned m_shared_next; - unsigned m_sstarr_next; - unsigned m_const_next; - unsigned m_global_next; - unsigned m_local_next; - unsigned m_tex_next; - - symbol_table *m_parent; - ptx_version m_ptx_version; - std::string m_scope_name; - std::map m_symbols; //map from name of register to pointers to the registers - std::map m_types; - std::list m_globals; - std::list m_consts; - std::map m_function_info_lookup; - std::map m_function_symtab_lookup; - - //Jin: handle instruction group for cdp - unsigned m_inst_group_id; - std::map m_inst_group_symtab; + public: + symbol_table(); + symbol_table(const char *scope_name, unsigned entry_point, + symbol_table *parent, gpgpu_context *ctx); + void set_name(const char *name); + const ptx_version &get_ptx_version() const; + unsigned get_sm_target() const; + void set_ptx_version(float ver, unsigned ext); + void set_sm_target(const char *target, const char *ext, const char *ext2); + symbol *lookup(const char *identifier); + std::string get_scope_name() const { return m_scope_name; } + symbol *add_variable(const char *identifier, const type_info *type, + unsigned size, const char *filename, unsigned line); + void add_function(function_info *func, const char *filename, + unsigned linenumber); + bool add_function_decl(const char *name, int entry_point, + function_info **func_info, + symbol_table **symbol_table); + function_info *lookup_function(std::string name); + type_info *add_type(memory_space_t space_spec, int scalar_type_spec, + int vector_spec, int alignment_spec, int extern_spec); + type_info *add_type(function_info *func); + type_info *get_array_type(type_info *base_type, unsigned array_dim); + void set_label_address(const symbol *label, unsigned addr); + unsigned next_reg_num() { return ++m_reg_allocator; } + addr_t get_shared_next() { return m_shared_next; } + addr_t get_sstarr_next() { return m_sstarr_next; } + addr_t get_global_next() { return m_global_next; } + addr_t get_local_next() { return m_local_next; } + addr_t get_tex_next() { return m_tex_next; } + void alloc_shared(unsigned num_bytes) { m_shared_next += num_bytes; } + void alloc_sstarr(unsigned num_bytes) { m_sstarr_next += num_bytes; } + void alloc_global(unsigned num_bytes) { m_global_next += num_bytes; } + void alloc_local(unsigned num_bytes) { m_local_next += num_bytes; } + void alloc_tex(unsigned num_bytes) { m_tex_next += num_bytes; } + + typedef std::list::iterator iterator; + + iterator global_iterator_begin() { return m_globals.begin(); } + iterator global_iterator_end() { return m_globals.end(); } + + iterator const_iterator_begin() { return m_consts.begin(); } + iterator const_iterator_end() { return m_consts.end(); } + + void dump(); + + // Jin: handle instruction group for cdp + symbol_table *start_inst_group(); + symbol_table *end_inst_group(); + + // backward pointer + class gpgpu_context *gpgpu_ctx; + + private: + unsigned m_reg_allocator; + unsigned m_shared_next; + unsigned m_sstarr_next; + unsigned m_const_next; + unsigned m_global_next; + unsigned m_local_next; + unsigned m_tex_next; + + symbol_table *m_parent; + ptx_version m_ptx_version; + std::string m_scope_name; + std::map + m_symbols; // map from name of register to pointers to the registers + std::map m_types; + std::list m_globals; + std::list m_consts; + std::map m_function_info_lookup; + std::map m_function_symtab_lookup; + + // Jin: handle instruction group for cdp + unsigned m_inst_group_id; + std::map m_inst_group_symtab; }; class operand_info { -public: - operand_info(gpgpu_context* ctx) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = false; - m_immediate_address=false; - m_addr_offset = 0; - m_value.m_symbolic=NULL; - } - operand_info( const symbol *addr, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - if ( addr->is_label() ) { - m_type = label_t; - } else if ( addr->is_shared() ) { - m_type = symbolic_t; - } else if ( addr->is_const() ) { - m_type = symbolic_t; - } else if ( addr->is_global() ) { - m_type = symbolic_t; - } else if ( addr->is_local() ) { - m_type = symbolic_t; - } else if ( addr->is_param_local() ) { - m_type = symbolic_t; - } else if ( addr->is_param_kernel() ) { - m_type = symbolic_t; - } else if ( addr->is_tex() ) { - m_type = symbolic_t; - } else if ( addr->is_func_addr() ) { - m_type = symbolic_t; - } else if ( !addr->is_reg() ) { - m_type = symbolic_t; - } else { - m_type = reg_t; - } - - m_is_non_arch_reg = addr->is_non_arch_reg(); - m_value.m_symbolic = addr; - m_addr_offset = 0; - m_vector = false; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *addr1, const symbol *addr2, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_type = memory_t; - m_value.m_vector_symbolic = new const symbol*[8]; - m_value.m_vector_symbolic[0] = addr1; - m_value.m_vector_symbolic[1] = addr2; - m_value.m_vector_symbolic[2] = NULL; - m_value.m_vector_symbolic[3] = NULL; - m_value.m_vector_symbolic[4] = NULL; - m_value.m_vector_symbolic[5] = NULL; - m_value.m_vector_symbolic[6] = NULL; - m_value.m_vector_symbolic[7] = NULL; - m_addr_offset = 0; - m_vector = false; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( int builtin_id, int dim_mod, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = builtin_t; - m_value.m_int = builtin_id; - m_addr_offset = dim_mod; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *addr, int offset, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = address_t; - m_value.m_symbolic = addr; - m_addr_offset = offset; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( unsigned x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = unsigned_t; - m_value.m_unsigned = x; - m_addr_offset = x; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=true; - } - operand_info( int x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = int_t; - m_value.m_int = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( float x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = float_op_t; - m_value.m_float = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( double x, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = false; - m_type = double_op_t; - m_value.m_double = x; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4, gpgpu_context* ctx ) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = true; - m_type = vector_t; - m_value.m_vector_symbolic = new const symbol*[8]; - m_value.m_vector_symbolic[0] = s1; - m_value.m_vector_symbolic[1] = s2; - m_value.m_vector_symbolic[2] = s3; - m_value.m_vector_symbolic[3] = s4; - m_value.m_vector_symbolic[4] = NULL; - m_value.m_vector_symbolic[5] = NULL; - m_value.m_vector_symbolic[6] = NULL; - m_value.m_vector_symbolic[7] = NULL; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - operand_info( const symbol *s1, const symbol *s2, const symbol *s3, const symbol *s4 ,const symbol *s5,const symbol *s6,const symbol *s7, const symbol *s8, gpgpu_context* ctx) - { - init(ctx); - m_is_non_arch_reg = false; - m_addr_space = undefined_space; - m_operand_lohi = 0; - m_double_operand_type = 0; - m_operand_neg = false; - m_const_mem_offset = 0; - m_uid = get_uid(); - m_valid = true; - m_vector = true; - m_type = vector_t; - m_value.m_vector_symbolic = new const symbol*[8]; - m_value.m_vector_symbolic[0] = s1; - m_value.m_vector_symbolic[1] = s2; - m_value.m_vector_symbolic[2] = s3; - m_value.m_vector_symbolic[3] = s4; - m_value.m_vector_symbolic[4] = s5; - m_value.m_vector_symbolic[5] = s6; - m_value.m_vector_symbolic[6] = s7; - m_value.m_vector_symbolic[7] = s8; - m_addr_offset = 0; - m_neg_pred = false; - m_is_return_var = false; - m_immediate_address=false; - } - - void init(gpgpu_context* ctx) - { - gpgpu_ctx = ctx; - m_uid=(unsigned)-1; - m_valid=false; - m_vector=false; - m_type=undef_t; - m_immediate_address=false; - m_addr_space=undefined_space; - m_operand_lohi=0; - m_double_operand_type=0; - m_operand_neg=false; - m_const_mem_offset=(unsigned)-1; - m_value.m_int=0; - m_value.m_unsigned=(unsigned)-1; - m_value.m_float=0; - m_value.m_double=0; - for(unsigned i=0; i<4; i++){ - m_value.m_vint[i]=0; - m_value.m_vunsigned[i]=0; - m_value.m_vfloat[i]=0; - m_value.m_vdouble[i]=0; - } - m_value.m_symbolic=NULL; - m_value.m_vector_symbolic=NULL; - m_addr_offset=0; - m_neg_pred=0; - m_is_return_var=0; - m_is_non_arch_reg=0; - - } - void make_memory_operand() { m_type = memory_t;} - void set_return() { m_is_return_var = true; } - void set_immediate_addr() {m_immediate_address=true;} - const std::string &name() const - { - assert( m_type == symbolic_t || m_type == reg_t || m_type == address_t || m_type == memory_t || m_type == label_t); - return m_value.m_symbolic->name(); - } - - unsigned get_vect_nelem() const - { - assert( is_vector() ); - if( !m_value.m_vector_symbolic[0] ) return 0; - if( !m_value.m_vector_symbolic[1] ) return 1; - if( !m_value.m_vector_symbolic[2] ) return 2; - if( !m_value.m_vector_symbolic[3] ) return 3; - if( !m_value.m_vector_symbolic[4] ) return 4; - if( !m_value.m_vector_symbolic[5] ) return 5; - if( !m_value.m_vector_symbolic[6] ) return 6; - if( !m_value.m_vector_symbolic[7] ) return 7; - return 8; - } - - const symbol* vec_symbol(int idx) const - { - assert(idx < 8); - const symbol *result = m_value.m_vector_symbolic[idx]; - assert( result != NULL ); - return result; - } - - const std::string &vec_name1() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[0]->name(); - } - - const std::string &vec_name2() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[1]->name(); - } - - const std::string &vec_name3() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[2]->name(); - } - - const std::string &vec_name4() const - { - assert( m_type == vector_t); - return m_value.m_vector_symbolic[3]->name(); - } - - bool is_reg() const - { - if ( m_type == reg_t ) { - return true; - } - if ( m_type != symbolic_t ) { - return false; - } - return m_value.m_symbolic->type()->get_key().is_reg(); - } - bool is_param_local() const - { - if ( m_type != symbolic_t ) - return false; - return m_value.m_symbolic->type()->get_key().is_param_local(); - } - - bool is_param_kernel() const - { - if ( m_type != symbolic_t ) - return false; - return m_value.m_symbolic->type()->get_key().is_param_kernel(); - } - - bool is_vector() const - { - if ( m_vector) return true; + public: + operand_info(gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = false; + m_immediate_address = false; + m_addr_offset = 0; + m_value.m_symbolic = NULL; + } + operand_info(const symbol *addr, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + if (addr->is_label()) { + m_type = label_t; + } else if (addr->is_shared()) { + m_type = symbolic_t; + } else if (addr->is_const()) { + m_type = symbolic_t; + } else if (addr->is_global()) { + m_type = symbolic_t; + } else if (addr->is_local()) { + m_type = symbolic_t; + } else if (addr->is_param_local()) { + m_type = symbolic_t; + } else if (addr->is_param_kernel()) { + m_type = symbolic_t; + } else if (addr->is_tex()) { + m_type = symbolic_t; + } else if (addr->is_func_addr()) { + m_type = symbolic_t; + } else if (!addr->is_reg()) { + m_type = symbolic_t; + } else { + m_type = reg_t; + } + + m_is_non_arch_reg = addr->is_non_arch_reg(); + m_value.m_symbolic = addr; + m_addr_offset = 0; + m_vector = false; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *addr1, const symbol *addr2, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_type = memory_t; + m_value.m_vector_symbolic = new const symbol *[8]; + m_value.m_vector_symbolic[0] = addr1; + m_value.m_vector_symbolic[1] = addr2; + m_value.m_vector_symbolic[2] = NULL; + m_value.m_vector_symbolic[3] = NULL; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; + m_addr_offset = 0; + m_vector = false; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(int builtin_id, int dim_mod, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = builtin_t; + m_value.m_int = builtin_id; + m_addr_offset = dim_mod; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *addr, int offset, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = address_t; + m_value.m_symbolic = addr; + m_addr_offset = offset; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(unsigned x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = unsigned_t; + m_value.m_unsigned = x; + m_addr_offset = x; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = true; + } + operand_info(int x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = int_t; + m_value.m_int = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(float x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = float_op_t; + m_value.m_float = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(double x, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = false; + m_type = double_op_t; + m_value.m_double = x; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *s1, const symbol *s2, const symbol *s3, + const symbol *s4, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol *[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = NULL; + m_value.m_vector_symbolic[5] = NULL; + m_value.m_vector_symbolic[6] = NULL; + m_value.m_vector_symbolic[7] = NULL; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + operand_info(const symbol *s1, const symbol *s2, const symbol *s3, + const symbol *s4, const symbol *s5, const symbol *s6, + const symbol *s7, const symbol *s8, gpgpu_context *ctx) { + init(ctx); + m_is_non_arch_reg = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = 0; + m_uid = get_uid(); + m_valid = true; + m_vector = true; + m_type = vector_t; + m_value.m_vector_symbolic = new const symbol *[8]; + m_value.m_vector_symbolic[0] = s1; + m_value.m_vector_symbolic[1] = s2; + m_value.m_vector_symbolic[2] = s3; + m_value.m_vector_symbolic[3] = s4; + m_value.m_vector_symbolic[4] = s5; + m_value.m_vector_symbolic[5] = s6; + m_value.m_vector_symbolic[6] = s7; + m_value.m_vector_symbolic[7] = s8; + m_addr_offset = 0; + m_neg_pred = false; + m_is_return_var = false; + m_immediate_address = false; + } + + void init(gpgpu_context *ctx) { + gpgpu_ctx = ctx; + m_uid = (unsigned)-1; + m_valid = false; + m_vector = false; + m_type = undef_t; + m_immediate_address = false; + m_addr_space = undefined_space; + m_operand_lohi = 0; + m_double_operand_type = 0; + m_operand_neg = false; + m_const_mem_offset = (unsigned)-1; + m_value.m_int = 0; + m_value.m_unsigned = (unsigned)-1; + m_value.m_float = 0; + m_value.m_double = 0; + for (unsigned i = 0; i < 4; i++) { + m_value.m_vint[i] = 0; + m_value.m_vunsigned[i] = 0; + m_value.m_vfloat[i] = 0; + m_value.m_vdouble[i] = 0; + } + m_value.m_symbolic = NULL; + m_value.m_vector_symbolic = NULL; + m_addr_offset = 0; + m_neg_pred = 0; + m_is_return_var = 0; + m_is_non_arch_reg = 0; + } + void make_memory_operand() { m_type = memory_t; } + void set_return() { m_is_return_var = true; } + void set_immediate_addr() { m_immediate_address = true; } + const std::string &name() const { + assert(m_type == symbolic_t || m_type == reg_t || m_type == address_t || + m_type == memory_t || m_type == label_t); + return m_value.m_symbolic->name(); + } + + unsigned get_vect_nelem() const { + assert(is_vector()); + if (!m_value.m_vector_symbolic[0]) return 0; + if (!m_value.m_vector_symbolic[1]) return 1; + if (!m_value.m_vector_symbolic[2]) return 2; + if (!m_value.m_vector_symbolic[3]) return 3; + if (!m_value.m_vector_symbolic[4]) return 4; + if (!m_value.m_vector_symbolic[5]) return 5; + if (!m_value.m_vector_symbolic[6]) return 6; + if (!m_value.m_vector_symbolic[7]) return 7; + return 8; + } + + const symbol *vec_symbol(int idx) const { + assert(idx < 8); + const symbol *result = m_value.m_vector_symbolic[idx]; + assert(result != NULL); + return result; + } + + const std::string &vec_name1() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[0]->name(); + } + + const std::string &vec_name2() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[1]->name(); + } + + const std::string &vec_name3() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[2]->name(); + } + + const std::string &vec_name4() const { + assert(m_type == vector_t); + return m_value.m_vector_symbolic[3]->name(); + } + + bool is_reg() const { + if (m_type == reg_t) { + return true; + } + if (m_type != symbolic_t) { return false; - } - int reg_num() const { return m_value.m_symbolic->reg_num();} - int reg1_num() const { return m_value.m_vector_symbolic[0]->reg_num();} - int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num();} - int reg3_num() const { return m_value.m_vector_symbolic[2]?m_value.m_vector_symbolic[2]->reg_num():0; } - int reg4_num() const { return m_value.m_vector_symbolic[3]?m_value.m_vector_symbolic[3]->reg_num():0; } - int reg5_num() const { return m_value.m_vector_symbolic[4]?m_value.m_vector_symbolic[4]->reg_num():0; } - int reg6_num() const { return m_value.m_vector_symbolic[5]?m_value.m_vector_symbolic[5]->reg_num():0; } - int reg7_num() const { return m_value.m_vector_symbolic[6]?m_value.m_vector_symbolic[6]->reg_num():0; } - int reg8_num() const { return m_value.m_vector_symbolic[7]?m_value.m_vector_symbolic[7]->reg_num():0; } - int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } - int arch_reg_num(unsigned n) const { return (m_value.m_vector_symbolic[n])? m_value.m_vector_symbolic[n]->arch_reg_num() : -1; } - bool is_label() const { return m_type == label_t;} - bool is_builtin() const { return m_type == builtin_t;} - - // Memory operand used in ld / st instructions (ex. [__var1]) - bool is_memory_operand() const { return m_type == memory_t;} - - // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004]) - // This is used by the PTXPlus extension. The operand is assigned an address space during parsing. - bool is_memory_operand2() const { - return (m_addr_space!=undefined_space); - } - - bool is_immediate_address() const { - return m_immediate_address; - } - - bool is_literal() const { return m_type == int_t || - m_type == float_op_t || - m_type == double_op_t || - m_type == unsigned_t;} - bool is_shared() const { - if ( !(m_type == symbolic_t || m_type == address_t || m_type == memory_t) ) { - return false; - } - return m_value.m_symbolic->is_shared(); - } - bool is_sstarr() const { return m_value.m_symbolic->is_sstarr();} - bool is_const() const { return m_value.m_symbolic->is_const();} - bool is_global() const { return m_value.m_symbolic->is_global();} - bool is_local() const { return m_value.m_symbolic->is_local();} - bool is_tex() const { return m_value.m_symbolic->is_tex();} - bool is_return_var() const { return m_is_return_var; } - - bool is_function_address() const - { - if( m_type != symbolic_t ) { - return false; - } - return m_value.m_symbolic->is_func_addr(); - } - - ptx_reg_t get_literal_value() const - { - ptx_reg_t result; - switch ( m_type ) { - case int_t: result.s64 = m_value.m_int; break; - case float_op_t: result.f32 = m_value.m_float; break; - case double_op_t: result.f64 = m_value.m_double; break; - case unsigned_t: result.u32 = m_value.m_unsigned; break; + } + return m_value.m_symbolic->type()->get_key().is_reg(); + } + bool is_param_local() const { + if (m_type != symbolic_t) return false; + return m_value.m_symbolic->type()->get_key().is_param_local(); + } + + bool is_param_kernel() const { + if (m_type != symbolic_t) return false; + return m_value.m_symbolic->type()->get_key().is_param_kernel(); + } + + bool is_vector() const { + if (m_vector) return true; + return false; + } + int reg_num() const { return m_value.m_symbolic->reg_num(); } + int reg1_num() const { return m_value.m_vector_symbolic[0]->reg_num(); } + int reg2_num() const { return m_value.m_vector_symbolic[1]->reg_num(); } + int reg3_num() const { + return m_value.m_vector_symbolic[2] + ? m_value.m_vector_symbolic[2]->reg_num() + : 0; + } + int reg4_num() const { + return m_value.m_vector_symbolic[3] + ? m_value.m_vector_symbolic[3]->reg_num() + : 0; + } + int reg5_num() const { + return m_value.m_vector_symbolic[4] + ? m_value.m_vector_symbolic[4]->reg_num() + : 0; + } + int reg6_num() const { + return m_value.m_vector_symbolic[5] + ? m_value.m_vector_symbolic[5]->reg_num() + : 0; + } + int reg7_num() const { + return m_value.m_vector_symbolic[6] + ? m_value.m_vector_symbolic[6]->reg_num() + : 0; + } + int reg8_num() const { + return m_value.m_vector_symbolic[7] + ? m_value.m_vector_symbolic[7]->reg_num() + : 0; + } + int arch_reg_num() const { return m_value.m_symbolic->arch_reg_num(); } + int arch_reg_num(unsigned n) const { + return (m_value.m_vector_symbolic[n]) + ? m_value.m_vector_symbolic[n]->arch_reg_num() + : -1; + } + bool is_label() const { return m_type == label_t; } + bool is_builtin() const { return m_type == builtin_t; } + + // Memory operand used in ld / st instructions (ex. [__var1]) + bool is_memory_operand() const { return m_type == memory_t; } + + // Memory operand with immediate access (ex. s[0x0004] or g[$r1+=0x0004]) + // This is used by the PTXPlus extension. The operand is assigned an address + // space during parsing. + bool is_memory_operand2() const { return (m_addr_space != undefined_space); } + + bool is_immediate_address() const { return m_immediate_address; } + + bool is_literal() const { + return m_type == int_t || m_type == float_op_t || m_type == double_op_t || + m_type == unsigned_t; + } + bool is_shared() const { + if (!(m_type == symbolic_t || m_type == address_t || m_type == memory_t)) { + return false; + } + return m_value.m_symbolic->is_shared(); + } + bool is_sstarr() const { return m_value.m_symbolic->is_sstarr(); } + bool is_const() const { return m_value.m_symbolic->is_const(); } + bool is_global() const { return m_value.m_symbolic->is_global(); } + bool is_local() const { return m_value.m_symbolic->is_local(); } + bool is_tex() const { return m_value.m_symbolic->is_tex(); } + bool is_return_var() const { return m_is_return_var; } + + bool is_function_address() const { + if (m_type != symbolic_t) { + return false; + } + return m_value.m_symbolic->is_func_addr(); + } + + ptx_reg_t get_literal_value() const { + ptx_reg_t result; + switch (m_type) { + case int_t: + result.s64 = m_value.m_int; + break; + case float_op_t: + result.f32 = m_value.m_float; + break; + case double_op_t: + result.f64 = m_value.m_double; + break; + case unsigned_t: + result.u32 = m_value.m_unsigned; + break; default: - assert(0); - break; - } - return result; - } - int get_int() const { return m_value.m_int;} - int get_addr_offset() const { return m_addr_offset;} - const symbol *get_symbol() const { return m_value.m_symbolic;} - void set_type( enum operand_type type ) - { - m_type = type; - } - enum operand_type get_type() const { - return m_type; - } - void set_neg_pred() - { - assert( m_valid ); - m_neg_pred = true; - } - bool is_neg_pred() const { return m_neg_pred; } - bool is_valid() const { return m_valid; } - - void set_addr_space(enum _memory_space_t set_value) { m_addr_space = set_value; } - enum _memory_space_t get_addr_space() const { return m_addr_space; } - void set_operand_lohi(int set_value) { m_operand_lohi = set_value; } - int get_operand_lohi() const { return m_operand_lohi; } - void set_double_operand_type(int set_value) { m_double_operand_type = set_value; } - int get_double_operand_type() const { return m_double_operand_type; } - void set_operand_neg() { m_operand_neg = true; } - bool get_operand_neg() const { return m_operand_neg; } - void set_const_mem_offset(addr_t set_value) { m_const_mem_offset = set_value; } - addr_t get_const_mem_offset() const { return m_const_mem_offset; } - bool is_non_arch_reg() const { return m_is_non_arch_reg; } - -private: - gpgpu_context* gpgpu_ctx; - unsigned m_uid; - bool m_valid; - bool m_vector; - enum operand_type m_type; - bool m_immediate_address; - enum _memory_space_t m_addr_space; - int m_operand_lohi; - int m_double_operand_type; - bool m_operand_neg; - addr_t m_const_mem_offset; - union { - int m_int; - unsigned int m_unsigned; - float m_float; - double m_double; - int m_vint[4]; - unsigned int m_vunsigned[4]; - float m_vfloat[4]; - double m_vdouble[4]; - const symbol* m_symbolic; - const symbol** m_vector_symbolic; - } m_value; - - int m_addr_offset; - - bool m_neg_pred; - bool m_is_return_var; - bool m_is_non_arch_reg; - - unsigned get_uid(); + assert(0); + break; + } + return result; + } + int get_int() const { return m_value.m_int; } + int get_addr_offset() const { return m_addr_offset; } + const symbol *get_symbol() const { return m_value.m_symbolic; } + void set_type(enum operand_type type) { m_type = type; } + enum operand_type get_type() const { return m_type; } + void set_neg_pred() { + assert(m_valid); + m_neg_pred = true; + } + bool is_neg_pred() const { return m_neg_pred; } + bool is_valid() const { return m_valid; } + + void set_addr_space(enum _memory_space_t set_value) { + m_addr_space = set_value; + } + enum _memory_space_t get_addr_space() const { return m_addr_space; } + void set_operand_lohi(int set_value) { m_operand_lohi = set_value; } + int get_operand_lohi() const { return m_operand_lohi; } + void set_double_operand_type(int set_value) { + m_double_operand_type = set_value; + } + int get_double_operand_type() const { return m_double_operand_type; } + void set_operand_neg() { m_operand_neg = true; } + bool get_operand_neg() const { return m_operand_neg; } + void set_const_mem_offset(addr_t set_value) { + m_const_mem_offset = set_value; + } + addr_t get_const_mem_offset() const { return m_const_mem_offset; } + bool is_non_arch_reg() const { return m_is_non_arch_reg; } + + private: + gpgpu_context *gpgpu_ctx; + unsigned m_uid; + bool m_valid; + bool m_vector; + enum operand_type m_type; + bool m_immediate_address; + enum _memory_space_t m_addr_space; + int m_operand_lohi; + int m_double_operand_type; + bool m_operand_neg; + addr_t m_const_mem_offset; + union { + int m_int; + unsigned int m_unsigned; + float m_float; + double m_double; + int m_vint[4]; + unsigned int m_vunsigned[4]; + float m_vfloat[4]; + double m_vdouble[4]; + const symbol *m_symbolic; + const symbol **m_vector_symbolic; + } m_value; + + int m_addr_offset; + + bool m_neg_pred; + bool m_is_return_var; + bool m_is_non_arch_reg; + + unsigned get_uid(); }; extern const char *g_opcode_string[]; struct basic_block_t { - basic_block_t( unsigned ID, ptx_instruction *begin, ptx_instruction *end, bool entry, bool ex) - { - bb_id = ID; - ptx_begin = begin; - ptx_end = end; - is_entry=entry; - is_exit=ex; - immediatepostdominator_id = -1; - immediatedominator_id = -1; - } - - ptx_instruction* ptx_begin; - ptx_instruction* ptx_end; - std::set predecessor_ids; //indices of other basic blocks in m_basic_blocks array - std::set successor_ids; - std::set postdominator_ids; - std::set dominator_ids; - std::set Tmp_ids; - int immediatepostdominator_id; - int immediatedominator_id; - bool is_entry; - bool is_exit; - unsigned bb_id; - - // if this basic block dom B - bool dom(const basic_block_t *B) { - return (B->dominator_ids.find(this->bb_id) != B->dominator_ids.end()); - } - - // if this basic block pdom B - bool pdom(const basic_block_t *B) { - return (B->postdominator_ids.find(this->bb_id) != B->postdominator_ids.end()); - } + basic_block_t(unsigned ID, ptx_instruction *begin, ptx_instruction *end, + bool entry, bool ex) { + bb_id = ID; + ptx_begin = begin; + ptx_end = end; + is_entry = entry; + is_exit = ex; + immediatepostdominator_id = -1; + immediatedominator_id = -1; + } + + ptx_instruction *ptx_begin; + ptx_instruction *ptx_end; + std::set + predecessor_ids; // indices of other basic blocks in m_basic_blocks array + std::set successor_ids; + std::set postdominator_ids; + std::set dominator_ids; + std::set Tmp_ids; + int immediatepostdominator_id; + int immediatedominator_id; + bool is_entry; + bool is_exit; + unsigned bb_id; + + // if this basic block dom B + bool dom(const basic_block_t *B) { + return (B->dominator_ids.find(this->bb_id) != B->dominator_ids.end()); + } + + // if this basic block pdom B + bool pdom(const basic_block_t *B) { + return (B->postdominator_ids.find(this->bb_id) != + B->postdominator_ids.end()); + } }; struct gpgpu_recon_t { - address_type source_pc; - address_type target_pc; - class ptx_instruction* source_inst; - class ptx_instruction* target_inst; + address_type source_pc; + address_type target_pc; + class ptx_instruction *source_inst; + class ptx_instruction *target_inst; }; class ptx_instruction : public warp_inst_t { -public: - ptx_instruction( int opcode, - const symbol *pred, - int neg_pred, - int pred_mod, - symbol *label, - const std::list &operands, - const operand_info &return_var, - const std::list &options, - const std::list &wmma_options, - const std::list &scalar_type, - memory_space_t space_spec, - const char *file, - unsigned line, - const char *source, - const core_config *config, - gpgpu_context* ctx); - - void print_insn() const; - virtual void print_insn( FILE *fp ) const; - std::string to_string() const; - unsigned inst_size() const { return m_inst_size; } - unsigned uid() const { return m_uid;} - int get_opcode() const { return m_opcode;} - const char *get_opcode_cstr() const - { - if ( m_opcode != -1 ) { - return g_opcode_string[m_opcode]; - } else { - return "label"; - } - } - const char *source_file() const { return m_source_file.c_str();} - unsigned source_line() const { return m_source_line;} - unsigned get_num_operands() const { return m_operands.size();} - bool has_pred() const { return m_pred != NULL;} - operand_info get_pred() const; - bool get_pred_neg() const { return m_neg_pred;} - int get_pred_mod() const { return m_pred_mod;} - const char *get_source() const { return m_source.c_str();} - - typedef std::vector::const_iterator const_iterator; - - const_iterator op_iter_begin() const - { - return m_operands.begin(); - } - - const_iterator op_iter_end() const - { - return m_operands.end(); - } - - const operand_info &dst() const - { - assert( !m_operands.empty() ); - return m_operands[0]; - } - - const operand_info &func_addr() const - { - assert( !m_operands.empty() ); - if( !m_operands[0].is_return_var() ) { - return m_operands[0]; - } else { - assert( m_operands.size() >= 2 ); - return m_operands[1]; - } - } - - operand_info &dst() - { - assert( !m_operands.empty() ); + public: + ptx_instruction(int opcode, const symbol *pred, int neg_pred, int pred_mod, + symbol *label, const std::list &operands, + const operand_info &return_var, const std::list &options, + const std::list &wmma_options, + const std::list &scalar_type, memory_space_t space_spec, + const char *file, unsigned line, const char *source, + const core_config *config, gpgpu_context *ctx); + + void print_insn() const; + virtual void print_insn(FILE *fp) const; + std::string to_string() const; + unsigned inst_size() const { return m_inst_size; } + unsigned uid() const { return m_uid; } + int get_opcode() const { return m_opcode; } + const char *get_opcode_cstr() const { + if (m_opcode != -1) { + return g_opcode_string[m_opcode]; + } else { + return "label"; + } + } + const char *source_file() const { return m_source_file.c_str(); } + unsigned source_line() const { return m_source_line; } + unsigned get_num_operands() const { return m_operands.size(); } + bool has_pred() const { return m_pred != NULL; } + operand_info get_pred() const; + bool get_pred_neg() const { return m_neg_pred; } + int get_pred_mod() const { return m_pred_mod; } + const char *get_source() const { return m_source.c_str(); } + + typedef std::vector::const_iterator const_iterator; + + const_iterator op_iter_begin() const { return m_operands.begin(); } + + const_iterator op_iter_end() const { return m_operands.end(); } + + const operand_info &dst() const { + assert(!m_operands.empty()); + return m_operands[0]; + } + + const operand_info &func_addr() const { + assert(!m_operands.empty()); + if (!m_operands[0].is_return_var()) { return m_operands[0]; - } - - const operand_info &src1() const - { - assert( m_operands.size() > 1 ); + } else { + assert(m_operands.size() >= 2); return m_operands[1]; - } - - const operand_info &src2() const - { - assert( m_operands.size() > 2 ); - return m_operands[2]; - } - - const operand_info &src3() const - { - assert( m_operands.size() > 3 ); - return m_operands[3]; - } - const operand_info &src4() const - { - assert( m_operands.size() > 4 ); - return m_operands[4]; - } - const operand_info &src5() const - { - assert( m_operands.size() > 5 ); - return m_operands[5]; - } - const operand_info &src6() const - { - assert( m_operands.size() > 6 ); - return m_operands[6]; - } - const operand_info &src7() const - { - assert( m_operands.size() > 7 ); - return m_operands[7]; - } - const operand_info &src8() const - { - assert( m_operands.size() > 8 ); - return m_operands[8]; - } - - const operand_info &operand_lookup( unsigned n ) const - { - assert( n < m_operands.size() ); - return m_operands[n]; - } - bool has_return() const - { - return m_return_var.is_valid(); - } - - memory_space_t get_space() const { return m_space_spec;} - unsigned get_vector() const { return m_vector_spec;} - unsigned get_atomic() const { return m_atomic_spec;} - - int get_wmma_type() const { - return m_wmma_type; - } - int get_wmma_layout(int index) const { - return m_wmma_layout[index];//0->Matrix D,1->Matrix C - } - int get_type() const - { - assert( !m_scalar_type.empty() ); - return m_scalar_type.front(); - } - - int get_type2() const - { - assert( m_scalar_type.size()==2 ); - return m_scalar_type.back(); - } - - void assign_bb(basic_block_t* basic_block) //assign instruction to a basic block - { - m_basic_block = basic_block; - } - basic_block_t* get_bb() { return m_basic_block;} - void set_m_instr_mem_index(unsigned index) { - m_instr_mem_index = index; - } - void set_PC( addr_t PC ) - { - m_PC = PC; - } - addr_t get_PC() const - { - return m_PC; - } - - unsigned get_m_instr_mem_index() { return m_instr_mem_index;} - unsigned get_cmpop() const { return m_compare_op;} - const symbol *get_label() const { return m_label;} - bool is_label() const { if(m_label){ assert(m_opcode==-1);return true;} return false;} - bool is_hi() const { return m_hi;} - bool is_lo() const { return m_lo;} - bool is_wide() const { return m_wide;} - bool is_uni() const { return m_uni;} - bool is_exit() const { return m_exit;} - bool is_abs() const { return m_abs;} - bool is_neg() const { return m_neg;} - bool is_to() const { return m_to_option; } - unsigned cache_option() const { return m_cache_option; } - unsigned rounding_mode() const { return m_rounding_mode;} - unsigned saturation_mode() const { return m_saturation_mode;} - unsigned dimension() const { return m_geom_spec;} - unsigned barrier_op() const {return m_barrier_op;} - unsigned shfl_op() const {return m_shfl_op;} - unsigned prmt_op() const {return m_prmt_op;} - enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; - enum vote_mode_t vote_mode() const { return m_vote_mode; } - - int membar_level() const { return m_membar_level; } - - bool has_memory_read() const { - if( m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP|| m_opcode==MMA_LD_OP ) - return true; - // Check PTXPlus operand type below - // Source operands are memory operands - ptx_instruction::const_iterator op=op_iter_begin(); - for ( int n=0; op != op_iter_end(); op++, n++ ) { //process operands - if( n > 0 && op->is_memory_operand2()) // source operands only - return true; - } - return false; - } - bool has_memory_write() const { - if( m_opcode == ST_OP || m_opcode==MMA_ST_OP ) return true; - // Check PTXPlus operand type below - // Destination operand is a memory operand - ptx_instruction::const_iterator op=op_iter_begin(); - for ( int n=0; (op!=op_iter_end() && n<1); op++, n++ ) { //process operands - if( n==0 && op->is_memory_operand2()) // source operands only - return true; - } - return false; - } - - -private: - void set_opcode_and_latency(); - void set_bar_type(); - void set_fp_or_int_archop(); - void set_mul_div_or_other_archop(); - - basic_block_t *m_basic_block; - unsigned m_uid; - addr_t m_PC; - std::string m_source_file; - unsigned m_source_line; - std::string m_source; - - const symbol *m_pred; - bool m_neg_pred; - int m_pred_mod; - int m_opcode; - const symbol *m_label; - std::vector m_operands; - operand_info m_return_var; - - std::list m_options; - std::list m_wmma_options; - bool m_wide; - bool m_hi; - bool m_lo; - bool m_exit; - bool m_abs; - bool m_neg; - bool m_uni; //if branch instruction, this evaluates to true for uniform branches (ie jumps) - bool m_to_option; - unsigned m_cache_option; - int m_wmma_type; - int m_wmma_layout[2]; - int m_wmma_configuration; - unsigned m_rounding_mode; - unsigned m_compare_op; - unsigned m_saturation_mode; - unsigned m_barrier_op; - unsigned m_shfl_op; - unsigned m_prmt_op; - - std::list m_scalar_type; - memory_space_t m_space_spec; - int m_geom_spec; - int m_vector_spec; - int m_atomic_spec; - enum vote_mode_t m_vote_mode; - int m_membar_level; - int m_instr_mem_index; //index into m_instr_mem array - unsigned m_inst_size; // bytes - - virtual void pre_decode(); - friend class function_info; - // backward pointer - class gpgpu_context* gpgpu_ctx; + } + } + + operand_info &dst() { + assert(!m_operands.empty()); + return m_operands[0]; + } + + const operand_info &src1() const { + assert(m_operands.size() > 1); + return m_operands[1]; + } + + const operand_info &src2() const { + assert(m_operands.size() > 2); + return m_operands[2]; + } + + const operand_info &src3() const { + assert(m_operands.size() > 3); + return m_operands[3]; + } + const operand_info &src4() const { + assert(m_operands.size() > 4); + return m_operands[4]; + } + const operand_info &src5() const { + assert(m_operands.size() > 5); + return m_operands[5]; + } + const operand_info &src6() const { + assert(m_operands.size() > 6); + return m_operands[6]; + } + const operand_info &src7() const { + assert(m_operands.size() > 7); + return m_operands[7]; + } + const operand_info &src8() const { + assert(m_operands.size() > 8); + return m_operands[8]; + } + + const operand_info &operand_lookup(unsigned n) const { + assert(n < m_operands.size()); + return m_operands[n]; + } + bool has_return() const { return m_return_var.is_valid(); } + + memory_space_t get_space() const { return m_space_spec; } + unsigned get_vector() const { return m_vector_spec; } + unsigned get_atomic() const { return m_atomic_spec; } + + int get_wmma_type() const { return m_wmma_type; } + int get_wmma_layout(int index) const { + return m_wmma_layout[index]; // 0->Matrix D,1->Matrix C + } + int get_type() const { + assert(!m_scalar_type.empty()); + return m_scalar_type.front(); + } + + int get_type2() const { + assert(m_scalar_type.size() == 2); + return m_scalar_type.back(); + } + + void assign_bb( + basic_block_t *basic_block) // assign instruction to a basic block + { + m_basic_block = basic_block; + } + basic_block_t *get_bb() { return m_basic_block; } + void set_m_instr_mem_index(unsigned index) { m_instr_mem_index = index; } + void set_PC(addr_t PC) { m_PC = PC; } + addr_t get_PC() const { return m_PC; } + + unsigned get_m_instr_mem_index() { return m_instr_mem_index; } + unsigned get_cmpop() const { return m_compare_op; } + const symbol *get_label() const { return m_label; } + bool is_label() const { + if (m_label) { + assert(m_opcode == -1); + return true; + } + return false; + } + bool is_hi() const { return m_hi; } + bool is_lo() const { return m_lo; } + bool is_wide() const { return m_wide; } + bool is_uni() const { return m_uni; } + bool is_exit() const { return m_exit; } + bool is_abs() const { return m_abs; } + bool is_neg() const { return m_neg; } + bool is_to() const { return m_to_option; } + unsigned cache_option() const { return m_cache_option; } + unsigned rounding_mode() const { return m_rounding_mode; } + unsigned saturation_mode() const { return m_saturation_mode; } + unsigned dimension() const { return m_geom_spec; } + unsigned barrier_op() const { return m_barrier_op; } + unsigned shfl_op() const { return m_shfl_op; } + unsigned prmt_op() const { return m_prmt_op; } + enum vote_mode_t { vote_any, vote_all, vote_uni, vote_ballot }; + enum vote_mode_t vote_mode() const { return m_vote_mode; } + + int membar_level() const { return m_membar_level; } + + bool has_memory_read() const { + if (m_opcode == LD_OP || m_opcode == LDU_OP || m_opcode == TEX_OP || + m_opcode == MMA_LD_OP) + return true; + // Check PTXPlus operand type below + // Source operands are memory operands + ptx_instruction::const_iterator op = op_iter_begin(); + for (int n = 0; op != op_iter_end(); op++, n++) { // process operands + if (n > 0 && op->is_memory_operand2()) // source operands only + return true; + } + return false; + } + bool has_memory_write() const { + if (m_opcode == ST_OP || m_opcode == MMA_ST_OP) return true; + // Check PTXPlus operand type below + // Destination operand is a memory operand + ptx_instruction::const_iterator op = op_iter_begin(); + for (int n = 0; (op != op_iter_end() && n < 1); + op++, n++) { // process operands + if (n == 0 && op->is_memory_operand2()) // source operands only + return true; + } + return false; + } + + private: + void set_opcode_and_latency(); + void set_bar_type(); + void set_fp_or_int_archop(); + void set_mul_div_or_other_archop(); + + basic_block_t *m_basic_block; + unsigned m_uid; + addr_t m_PC; + std::string m_source_file; + unsigned m_source_line; + std::string m_source; + + const symbol *m_pred; + bool m_neg_pred; + int m_pred_mod; + int m_opcode; + const symbol *m_label; + std::vector m_operands; + operand_info m_return_var; + + std::list m_options; + std::list m_wmma_options; + bool m_wide; + bool m_hi; + bool m_lo; + bool m_exit; + bool m_abs; + bool m_neg; + bool m_uni; // if branch instruction, this evaluates to true for uniform + // branches (ie jumps) + bool m_to_option; + unsigned m_cache_option; + int m_wmma_type; + int m_wmma_layout[2]; + int m_wmma_configuration; + unsigned m_rounding_mode; + unsigned m_compare_op; + unsigned m_saturation_mode; + unsigned m_barrier_op; + unsigned m_shfl_op; + unsigned m_prmt_op; + + std::list m_scalar_type; + memory_space_t m_space_spec; + int m_geom_spec; + int m_vector_spec; + int m_atomic_spec; + enum vote_mode_t m_vote_mode; + int m_membar_level; + int m_instr_mem_index; // index into m_instr_mem array + unsigned m_inst_size; // bytes + + virtual void pre_decode(); + friend class function_info; + // backward pointer + class gpgpu_context *gpgpu_ctx; }; class param_info { -public: - param_info() { m_valid = false; m_value_set=false; m_size = 0; m_is_ptr = false; } - param_info( std::string name, int type, size_t size, bool is_ptr, memory_space_t ptr_space ) - { - m_valid = true; - m_value_set = false; - m_name = name; - m_type = type; - m_size = size; - m_is_ptr = is_ptr; - m_ptr_space = ptr_space; - } - void add_data( param_t v ) { - assert( (!m_value_set) || (m_value.size == v.size) ); // if this fails concurrent kernel launches might execute incorrectly - m_value_set = true; - m_value = v; - } - void add_offset( unsigned offset ) { m_offset = offset; } - unsigned get_offset() { assert(m_valid); return m_offset; } - std::string get_name() const { assert(m_valid); return m_name; } - int get_type() const { assert(m_valid); return m_type; } - param_t get_value() const { assert(m_value_set); return m_value; } - size_t get_size() const { assert(m_valid); return m_size; } - bool is_ptr_shared() const { assert(m_valid); return (m_is_ptr and m_ptr_space == shared_space); } -private: - bool m_valid; - std::string m_name; - int m_type; - size_t m_size; - bool m_value_set; - param_t m_value; - unsigned m_offset; - bool m_is_ptr; - memory_space_t m_ptr_space; + public: + param_info() { + m_valid = false; + m_value_set = false; + m_size = 0; + m_is_ptr = false; + } + param_info(std::string name, int type, size_t size, bool is_ptr, + memory_space_t ptr_space) { + m_valid = true; + m_value_set = false; + m_name = name; + m_type = type; + m_size = size; + m_is_ptr = is_ptr; + m_ptr_space = ptr_space; + } + void add_data(param_t v) { + assert((!m_value_set) || + (m_value.size == v.size)); // if this fails concurrent kernel + // launches might execute incorrectly + m_value_set = true; + m_value = v; + } + void add_offset(unsigned offset) { m_offset = offset; } + unsigned get_offset() { + assert(m_valid); + return m_offset; + } + std::string get_name() const { + assert(m_valid); + return m_name; + } + int get_type() const { + assert(m_valid); + return m_type; + } + param_t get_value() const { + assert(m_value_set); + return m_value; + } + size_t get_size() const { + assert(m_valid); + return m_size; + } + bool is_ptr_shared() const { + assert(m_valid); + return (m_is_ptr and m_ptr_space == shared_space); + } + + private: + bool m_valid; + std::string m_name; + int m_type; + size_t m_size; + bool m_value_set; + param_t m_value; + unsigned m_offset; + bool m_is_ptr; + memory_space_t m_ptr_space; }; class function_info { -public: - function_info(int entry_point, gpgpu_context* ctx ); - const ptx_version &get_ptx_version() const { return m_symtab->get_ptx_version(); } - unsigned get_sm_target() const { return m_symtab->get_sm_target(); } - bool is_extern() const { return m_extern; } - void set_name(const char *name) - { - m_name = name; - } - void set_symtab(symbol_table *symtab ) - { - m_symtab = symtab; - } - std::string get_name() const - { - return m_name; - } - unsigned print_insn( unsigned pc, FILE * fp ) const; - std::string get_insn_str( unsigned pc ) const; - void add_inst( const std::list &instructions ) - { - m_instructions = instructions; - } - std::list::iterator find_next_real_instruction( std::list::iterator i ); - void create_basic_blocks( ); - - void print_basic_blocks(); - - void print_basic_block_links(); - void print_basic_block_dot(); - - operand_info* find_break_target( ptx_instruction * p_break_insn ); //find the target of a break instruction - void connect_basic_blocks( ); //iterate across m_basic_blocks of function, connecting basic blocks together - bool connect_break_targets(); //connecting break instructions with proper targets - - //iterate across m_basic_blocks of function, - //finding dominator blocks, using algorithm of - //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - void find_dominators( ); - void print_dominators(); - void find_idominators(); - void print_idominators(); - - //iterate across m_basic_blocks of function, - //finding postdominator blocks, using algorithm of - //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 - void find_postdominators( ); - void print_postdominators(); - - //iterate across m_basic_blocks of function, - //finding immediate postdominator blocks, using algorithm of - //Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 - void find_ipostdominators( ); - void print_ipostdominators(); - void do_pdom(); //function to call pdom analysis - - unsigned get_num_reconvergence_pairs(); - - void get_reconvergence_pairs(gpgpu_recon_t* recon_points); - - unsigned get_function_size() { return m_instructions.size();} - - void ptx_assemble(); - - unsigned ptx_get_inst_op( ptx_thread_info *thread ); - void add_param( const char *name, struct param_t value ) - { - m_kernel_params[ name ] = value; - } - void add_param_name_type_size( unsigned index, std::string name, int type, size_t size, bool ptr, memory_space_t space ); - void add_param_data( unsigned argn, struct gpgpu_ptx_sim_arg *args ); - void add_return_var( const symbol *rv ) - { - m_return_var_sym = rv; - } - void add_arg( const symbol *arg ) - { - assert( arg != NULL ); - m_args.push_back(arg); - } - void remove_args() - { - m_args.clear(); - } - unsigned num_args() const - { - return m_args.size(); - } - unsigned get_args_aligned_size(); - - const symbol* get_arg( unsigned n ) const - { - assert( n < m_args.size() ); - return m_args[n]; - } - bool has_return() const - { - return m_return_var_sym != NULL; - } - const symbol *get_return_var() const - { - return m_return_var_sym; - } - const ptx_instruction *get_instruction( unsigned PC ) const - { - unsigned index = PC - m_start_PC; - if( index < m_instr_mem_size ) - return m_instr_mem[index]; - return NULL; - } - addr_t get_start_PC() const - { - return m_start_PC; - } - - void finalize( memory_space *param_mem ); - void param_to_shared( memory_space *shared_mem, symbol_table *symtab ); - void list_param( FILE *fout ) const; - void ptx_jit_config(std::map mallocPtr_Size, memory_space *param_mem, gpgpu_t* gpu, dim3 gridDim, dim3 blockDim) ; - - const struct gpgpu_ptx_sim_info* get_kernel_info () const - { - assert (m_kernel_info.maxthreads == maxnt_id); - return &m_kernel_info; - } - - const void set_kernel_info (const struct gpgpu_ptx_sim_info &info) { - m_kernel_info = info; - m_kernel_info.ptx_version = 10*get_ptx_version().ver(); - m_kernel_info.sm_target = get_ptx_version().target(); - // THIS DEPENDS ON ptxas being called after the PTX is parsed. - m_kernel_info.maxthreads = maxnt_id; - } - symbol_table *get_symtab() - { - return m_symtab; - } - - unsigned local_mem_framesize() const - { - return m_local_mem_framesize; - } - void set_framesize( unsigned sz ) - { - m_local_mem_framesize = sz; - } - bool is_entry_point() const { return m_entry_point; } - bool is_pdom_set() const { return pdom_done; } //return pdom flag - void set_pdom() { pdom_done = true; } //set pdom flag - - void add_config_param( size_t size, unsigned alignment ){ - unsigned offset = 0; - if (m_param_configs.size()>0){ - unsigned offset_nom = m_param_configs.back().first + m_param_configs.back().second; - //ensure offset matches alignment requirements - offset = offset_nom%alignment ? (offset_nom/alignment + 1) * alignment : offset_nom; - } - m_param_configs.push_back(std::pair(size, offset)); - } - - std::pair get_param_config(unsigned param_num) const { return m_param_configs[param_num]; } - - void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads;} - unsigned get_maxnt_id() { return maxnt_id;} - // backward pointer - class gpgpu_context* gpgpu_ctx; - -private: - unsigned maxnt_id; - unsigned m_uid; - unsigned m_local_mem_framesize; - bool m_entry_point; - bool m_extern; - bool m_assembled; - bool pdom_done; //flag to check whether pdom is completed or not - std::string m_name; - ptx_instruction **m_instr_mem; - unsigned m_start_PC; - unsigned m_instr_mem_size; - std::map m_kernel_params; - std::map m_ptx_kernel_param_info; - std::vector< std::pair > m_param_configs; - const symbol *m_return_var_sym; - std::vector m_args; - std::list m_instructions; - std::vector m_basic_blocks; - std::list > m_back_edges; - std::map labels; - unsigned num_reconvergence_pairs; - - //Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along with ___.ptx - struct gpgpu_ptx_sim_info m_kernel_info; - - symbol_table *m_symtab; - - //parameter size for device kernels - int m_args_aligned_size; - - addr_t m_n; // offset in m_instr_mem (used in do_pdom) + public: + function_info(int entry_point, gpgpu_context *ctx); + const ptx_version &get_ptx_version() const { + return m_symtab->get_ptx_version(); + } + unsigned get_sm_target() const { return m_symtab->get_sm_target(); } + bool is_extern() const { return m_extern; } + void set_name(const char *name) { m_name = name; } + void set_symtab(symbol_table *symtab) { m_symtab = symtab; } + std::string get_name() const { return m_name; } + unsigned print_insn(unsigned pc, FILE *fp) const; + std::string get_insn_str(unsigned pc) const; + void add_inst(const std::list &instructions) { + m_instructions = instructions; + } + std::list::iterator find_next_real_instruction( + std::list::iterator i); + void create_basic_blocks(); + + void print_basic_blocks(); + + void print_basic_block_links(); + void print_basic_block_dot(); + + operand_info *find_break_target( + ptx_instruction *p_break_insn); // find the target of a break instruction + void connect_basic_blocks(); // iterate across m_basic_blocks of function, + // connecting basic blocks together + bool + connect_break_targets(); // connecting break instructions with proper targets + + // iterate across m_basic_blocks of function, + // finding dominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + void find_dominators(); + void print_dominators(); + void find_idominators(); + void print_idominators(); + + // iterate across m_basic_blocks of function, + // finding postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.14 + void find_postdominators(); + void print_postdominators(); + + // iterate across m_basic_blocks of function, + // finding immediate postdominator blocks, using algorithm of + // Muchnick's Adv. Compiler Design & Implemmntation Fig 7.15 + void find_ipostdominators(); + void print_ipostdominators(); + void do_pdom(); // function to call pdom analysis + + unsigned get_num_reconvergence_pairs(); + + void get_reconvergence_pairs(gpgpu_recon_t *recon_points); + + unsigned get_function_size() { return m_instructions.size(); } + + void ptx_assemble(); + + unsigned ptx_get_inst_op(ptx_thread_info *thread); + void add_param(const char *name, struct param_t value) { + m_kernel_params[name] = value; + } + void add_param_name_type_size(unsigned index, std::string name, int type, + size_t size, bool ptr, memory_space_t space); + void add_param_data(unsigned argn, struct gpgpu_ptx_sim_arg *args); + void add_return_var(const symbol *rv) { m_return_var_sym = rv; } + void add_arg(const symbol *arg) { + assert(arg != NULL); + m_args.push_back(arg); + } + void remove_args() { m_args.clear(); } + unsigned num_args() const { return m_args.size(); } + unsigned get_args_aligned_size(); + + const symbol *get_arg(unsigned n) const { + assert(n < m_args.size()); + return m_args[n]; + } + bool has_return() const { return m_return_var_sym != NULL; } + const symbol *get_return_var() const { return m_return_var_sym; } + const ptx_instruction *get_instruction(unsigned PC) const { + unsigned index = PC - m_start_PC; + if (index < m_instr_mem_size) return m_instr_mem[index]; + return NULL; + } + addr_t get_start_PC() const { return m_start_PC; } + + void finalize(memory_space *param_mem); + void param_to_shared(memory_space *shared_mem, symbol_table *symtab); + void list_param(FILE *fout) const; + void ptx_jit_config(std::map mallocPtr_Size, + memory_space *param_mem, gpgpu_t *gpu, dim3 gridDim, + dim3 blockDim); + + const struct gpgpu_ptx_sim_info *get_kernel_info() const { + assert(m_kernel_info.maxthreads == maxnt_id); + return &m_kernel_info; + } + + const void set_kernel_info(const struct gpgpu_ptx_sim_info &info) { + m_kernel_info = info; + m_kernel_info.ptx_version = 10 * get_ptx_version().ver(); + m_kernel_info.sm_target = get_ptx_version().target(); + // THIS DEPENDS ON ptxas being called after the PTX is parsed. + m_kernel_info.maxthreads = maxnt_id; + } + symbol_table *get_symtab() { return m_symtab; } + + unsigned local_mem_framesize() const { return m_local_mem_framesize; } + void set_framesize(unsigned sz) { m_local_mem_framesize = sz; } + bool is_entry_point() const { return m_entry_point; } + bool is_pdom_set() const { return pdom_done; } // return pdom flag + void set_pdom() { pdom_done = true; } // set pdom flag + + void add_config_param(size_t size, unsigned alignment) { + unsigned offset = 0; + if (m_param_configs.size() > 0) { + unsigned offset_nom = + m_param_configs.back().first + m_param_configs.back().second; + // ensure offset matches alignment requirements + offset = offset_nom % alignment ? (offset_nom / alignment + 1) * alignment + : offset_nom; + } + m_param_configs.push_back(std::pair(size, offset)); + } + + std::pair get_param_config(unsigned param_num) const { + return m_param_configs[param_num]; + } + + void set_maxnt_id(unsigned maxthreads) { maxnt_id = maxthreads; } + unsigned get_maxnt_id() { return maxnt_id; } + // backward pointer + class gpgpu_context *gpgpu_ctx; + + private: + unsigned maxnt_id; + unsigned m_uid; + unsigned m_local_mem_framesize; + bool m_entry_point; + bool m_extern; + bool m_assembled; + bool pdom_done; // flag to check whether pdom is completed or not + std::string m_name; + ptx_instruction **m_instr_mem; + unsigned m_start_PC; + unsigned m_instr_mem_size; + std::map m_kernel_params; + std::map m_ptx_kernel_param_info; + std::vector > m_param_configs; + const symbol *m_return_var_sym; + std::vector m_args; + std::list m_instructions; + std::vector m_basic_blocks; + std::list > m_back_edges; + std::map labels; + unsigned num_reconvergence_pairs; + + // Registers/shmem/etc. used (from ptxas -v), loaded from ___.ptxinfo along + // with ___.ptx + struct gpgpu_ptx_sim_info m_kernel_info; + + symbol_table *m_symtab; + + // parameter size for device kernels + int m_args_aligned_size; + + addr_t m_n; // offset in m_instr_mem (used in do_pdom) }; class arg_buffer_t { -public: - arg_buffer_t(gpgpu_context* ctx) : m_src_op(ctx) - { - m_is_reg=false; - m_is_param=false; - m_param_value=NULL; - m_reg_value=ptx_reg_t(); - } - arg_buffer_t( const arg_buffer_t &another, gpgpu_context* ctx ) : m_src_op(ctx) - { - make_copy(another); - } - void make_copy( const arg_buffer_t &another ) - { - m_dst = another.m_dst; - m_src_op = another.m_src_op; - m_is_reg = another.m_is_reg; - m_is_param = another.m_is_param; - m_reg_value = another.m_reg_value; - m_param_bytes = another.m_param_bytes; - if( m_is_param ) { - m_param_value = malloc(m_param_bytes); - memcpy(m_param_value,another.m_param_value,m_param_bytes); - } - } - void operator=( const arg_buffer_t &another ) - { - make_copy(another); - } - ~arg_buffer_t() - { - if( m_is_param ) - free(m_param_value); - } - arg_buffer_t( const symbol *dst_sym, const operand_info &src_op, ptx_reg_t source_value ) : m_src_op(src_op) - { - m_dst = dst_sym; - m_reg_value=ptx_reg_t(); - if( dst_sym->is_reg() ) { - m_is_reg = true; - m_is_param = false; - assert( src_op.is_reg() ); - m_reg_value = source_value; - } else { - m_is_param = true; - m_is_reg = false; - m_param_value = calloc(sizeof(ptx_reg_t),1); - //new (m_param_value) ptx_reg_t(source_value); - memcpy(m_param_value,&source_value,sizeof(ptx_reg_t)); - m_param_bytes = sizeof(ptx_reg_t); + public: + arg_buffer_t(gpgpu_context *ctx) : m_src_op(ctx) { + m_is_reg = false; + m_is_param = false; + m_param_value = NULL; + m_reg_value = ptx_reg_t(); + } + arg_buffer_t(const arg_buffer_t &another, gpgpu_context *ctx) + : m_src_op(ctx) { + make_copy(another); + } + void make_copy(const arg_buffer_t &another) { + m_dst = another.m_dst; + m_src_op = another.m_src_op; + m_is_reg = another.m_is_reg; + m_is_param = another.m_is_param; + m_reg_value = another.m_reg_value; + m_param_bytes = another.m_param_bytes; + if (m_is_param) { + m_param_value = malloc(m_param_bytes); + memcpy(m_param_value, another.m_param_value, m_param_bytes); + } + } + void operator=(const arg_buffer_t &another) { make_copy(another); } + ~arg_buffer_t() { + if (m_is_param) free(m_param_value); + } + arg_buffer_t(const symbol *dst_sym, const operand_info &src_op, + ptx_reg_t source_value) + : m_src_op(src_op) { + m_dst = dst_sym; + m_reg_value = ptx_reg_t(); + if (dst_sym->is_reg()) { + m_is_reg = true; + m_is_param = false; + assert(src_op.is_reg()); + m_reg_value = source_value; + } else { + m_is_param = true; + m_is_reg = false; + m_param_value = calloc(sizeof(ptx_reg_t), 1); + // new (m_param_value) ptx_reg_t(source_value); + memcpy(m_param_value, &source_value, sizeof(ptx_reg_t)); + m_param_bytes = sizeof(ptx_reg_t); + } + } + arg_buffer_t(const symbol *dst_sym, const operand_info &src_op, + void *source_param_value_array, unsigned array_size) + : m_src_op(src_op) { + m_dst = dst_sym; + if (dst_sym->is_reg()) { + m_is_reg = true; + m_is_param = false; + assert(src_op.is_param_local()); + assert(dst_sym->get_size_in_bytes() == array_size); + switch (array_size) { + case 1: + m_reg_value.u8 = *(unsigned char *)source_param_value_array; + break; + case 2: + m_reg_value.u16 = *(unsigned short *)source_param_value_array; + break; + case 4: + m_reg_value.u32 = *(unsigned int *)source_param_value_array; + break; + case 8: + m_reg_value.u64 = *(unsigned long long *)source_param_value_array; + break; + default: + printf( + "GPGPU-Sim PTX: ERROR ** source param size does not match known " + "register sizes\n"); + break; } - } - arg_buffer_t( const symbol *dst_sym, const operand_info &src_op, void *source_param_value_array, unsigned array_size ) : m_src_op(src_op) - { - m_dst = dst_sym; - if( dst_sym->is_reg() ) { - m_is_reg = true; - m_is_param = false; - assert( src_op.is_param_local() ); - assert( dst_sym->get_size_in_bytes() == array_size ); - switch( array_size ) { - case 1: m_reg_value.u8 = *(unsigned char*)source_param_value_array; break; - case 2: m_reg_value.u16 = *(unsigned short*)source_param_value_array; break; - case 4: m_reg_value.u32 = *(unsigned int*)source_param_value_array; break; - case 8: m_reg_value.u64 = *(unsigned long long*)source_param_value_array; break; - default: - printf("GPGPU-Sim PTX: ERROR ** source param size does not match known register sizes\n"); - break; - } - } else { - // param - m_is_param = true; - m_is_reg = false; - m_param_value = calloc(array_size,1); - m_param_bytes = array_size; - memcpy(m_param_value,source_param_value_array,array_size); - } - } - - bool is_reg() const { return m_is_reg; } - ptx_reg_t get_reg() const - { - assert(m_is_reg); - return m_reg_value; - } - - const void *get_param_buffer() const - { - assert(m_is_param); - return m_param_value; - } - size_t get_param_buffer_size() const - { - assert(m_is_param); - return m_param_bytes; - } - - const symbol *get_dst() const { return m_dst; } - -private: - // destination of copy - const symbol *m_dst; - - // source operand - operand_info m_src_op; - - // source information - bool m_is_reg; - bool m_is_param; - - // source is register - ptx_reg_t m_reg_value; - - // source is param - void *m_param_value; - unsigned m_param_bytes; + } else { + // param + m_is_param = true; + m_is_reg = false; + m_param_value = calloc(array_size, 1); + m_param_bytes = array_size; + memcpy(m_param_value, source_param_value_array, array_size); + } + } + + bool is_reg() const { return m_is_reg; } + ptx_reg_t get_reg() const { + assert(m_is_reg); + return m_reg_value; + } + + const void *get_param_buffer() const { + assert(m_is_param); + return m_param_value; + } + size_t get_param_buffer_size() const { + assert(m_is_param); + return m_param_bytes; + } + + const symbol *get_dst() const { return m_dst; } + + private: + // destination of copy + const symbol *m_dst; + + // source operand + operand_info m_src_op; + + // source information + bool m_is_reg; + bool m_is_param; + + // source is register + ptx_reg_t m_reg_value; + + // source is param + void *m_param_value; + unsigned m_param_bytes; }; -typedef std::list< arg_buffer_t > arg_buffer_list_t; -arg_buffer_t copy_arg_to_buffer(ptx_thread_info * thread, operand_info actual_param_op, const symbol * formal_param); -void copy_args_into_buffer_list( const ptx_instruction * pI, - ptx_thread_info * thread, - const function_info * target_func, - arg_buffer_list_t &arg_values ); -void copy_buffer_list_into_frame(ptx_thread_info * thread, arg_buffer_list_t &arg_values); -void copy_buffer_to_frame(ptx_thread_info * thread, const arg_buffer_t &a); - +typedef std::list arg_buffer_list_t; +arg_buffer_t copy_arg_to_buffer(ptx_thread_info *thread, + operand_info actual_param_op, + const symbol *formal_param); +void copy_args_into_buffer_list(const ptx_instruction *pI, + ptx_thread_info *thread, + const function_info *target_func, + arg_buffer_list_t &arg_values); +void copy_buffer_list_into_frame(ptx_thread_info *thread, + arg_buffer_list_t &arg_values); +void copy_buffer_to_frame(ptx_thread_info *thread, const arg_buffer_t &a); struct textureInfo { - unsigned int texel_size; //size in bytes, e.g. (channelDesc.x+y+z+w)/8 - unsigned int Tx,Ty; //tiling factor dimensions of layout of texels per 64B cache block - unsigned int Tx_numbits,Ty_numbits; //log2(T) - unsigned int texel_size_numbits; //log2(texel_size) + unsigned int texel_size; // size in bytes, e.g. (channelDesc.x+y+z+w)/8 + unsigned int Tx, + Ty; // tiling factor dimensions of layout of texels per 64B cache block + unsigned int Tx_numbits, Ty_numbits; // log2(T) + unsigned int texel_size_numbits; // log2(texel_size) }; -extern std::map g_sym_name_to_symbol_table; - +extern std::map g_sym_name_to_symbol_table; -void gpgpu_ptx_assemble( std::string kname, void *kinfo ); +void gpgpu_ptx_assemble(std::string kname, void *kinfo); #include "../option_parser.h" -unsigned ptx_kernel_shmem_size( void *kernel_impl ); -unsigned ptx_kernel_nregs( void *kernel_impl ); +unsigned ptx_kernel_shmem_size(void *kernel_impl); +unsigned ptx_kernel_nregs(void *kernel_impl); #endif diff --git a/src/cuda-sim/ptx_loader.cc b/src/cuda-sim/ptx_loader.cc index 33bcf45..372bda4 100644 --- a/src/cuda-sim/ptx_loader.cc +++ b/src/cuda-sim/ptx_loader.cc @@ -7,522 +7,576 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "ptx_loader.h" -#include "ptx_ir.h" -#include "cuda-sim.h" -#include "ptx_parser.h" -#include #include +#include #include #include #include "../../libcuda/gpgpu_context.h" +#include "cuda-sim.h" +#include "ptx_ir.h" +#include "ptx_parser.h" /// extern prototypes -extern int ptx_error( yyscan_t yyscanner, const char *s ); -extern int ptx_lex_init(yyscan_t* scanner); -extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); -extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); +extern int ptx_error(yyscan_t yyscanner, const char *s); +extern int ptx_lex_init(yyscan_t *scanner); +extern void ptx_set_in(FILE *_in_str, yyscan_t yyscanner); +extern int ptx_parse(yyscan_t scanner, ptx_recognizer *recognizer); extern int ptx_lex_destroy(yyscan_t scanner); -extern int ptx__scan_string(const char*, yyscan_t scanner); +extern int ptx__scan_string(const char *, yyscan_t scanner); -extern std::map get_duplicate(); +extern std::map get_duplicate(); -typedef void * yyscan_t; -extern int ptxinfo_lex_init(yyscan_t* scanner); -extern void ptxinfo_set_in (FILE * _in_str ,yyscan_t yyscanner ); -extern int ptxinfo_parse(yyscan_t scanner, ptxinfo_data* ptxinfo); +typedef void *yyscan_t; +extern int ptxinfo_lex_init(yyscan_t *scanner); +extern void ptxinfo_set_in(FILE *_in_str, yyscan_t yyscanner); +extern int ptxinfo_parse(yyscan_t scanner, ptxinfo_data *ptxinfo); extern int ptxinfo_lex_destroy(yyscan_t scanner); static bool g_save_embedded_ptx; static int g_occupancy_sm_number; -bool ptxinfo_data::keep_intermediate_files() {return g_keep_intermediate_files;} - -void gpgpu_context::ptx_reg_options(option_parser_t opp) -{ - option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, &g_save_embedded_ptx, - "saves ptx files embedded in binary as .ptx", - "0"); - option_parser_register(opp, "-keep", OPT_BOOL, &(ptxinfo->g_keep_intermediate_files), - "keep intermediate files created by GPGPU-Sim when interfacing with external programs", - "0"); - option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, - &(ptxinfo->m_ptx_save_converted_ptxplus), - "Saved converted ptxplus to a file", - "0"); - option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, &g_occupancy_sm_number, - "The SM number to pass to ptxas when getting register usage for computing GPU occupancy. " - "This parameter is required in the config.", - "0"); +bool ptxinfo_data::keep_intermediate_files() { + return g_keep_intermediate_files; } -void gpgpu_context::print_ptx_file( const char *p, unsigned source_num, const char *filename ) -{ - printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num ); - char *s = strdup(p); - char *t = s; - unsigned n=1; - while ( *t != '\0' ) { - char *u = t; - while ( (*u != '\n') && (*u != '\0') ) u++; - unsigned last = (*u == '\0'); - *u = '\0'; - const ptx_instruction *pI = ptx_parser->ptx_instruction_lookup(filename,n); - char pc[64]; - if( pI && pI->get_PC() ) - snprintf(pc,64,"%4u", pI->get_PC() ); - else - snprintf(pc,64," "); - printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t ); - if ( last ) break; - t = u+1; - n++; - } - free(s); - fflush(stdout); +void gpgpu_context::ptx_reg_options(option_parser_t opp) { + option_parser_register(opp, "-save_embedded_ptx", OPT_BOOL, + &g_save_embedded_ptx, + "saves ptx files embedded in binary as .ptx", "0"); + option_parser_register(opp, "-keep", OPT_BOOL, + &(ptxinfo->g_keep_intermediate_files), + "keep intermediate files created by GPGPU-Sim when " + "interfacing with external programs", + "0"); + option_parser_register(opp, "-gpgpu_ptx_save_converted_ptxplus", OPT_BOOL, + &(ptxinfo->m_ptx_save_converted_ptxplus), + "Saved converted ptxplus to a file", "0"); + option_parser_register(opp, "-gpgpu_occupancy_sm_number", OPT_INT32, + &g_occupancy_sm_number, + "The SM number to pass to ptxas when getting register " + "usage for computing GPU occupancy. " + "This parameter is required in the config.", + "0"); } -char* ptxinfo_data::gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptxfilename, const std::string elffilename, const std::string sassfilename) -{ - - printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); - - char fname_ptxplus[1024]; - snprintf(fname_ptxplus,1024,"_ptxplus_XXXXXX"); - int fd4=mkstemp(fname_ptxplus); - close(fd4); - - // Run cuobjdump_to_ptxplus - char commandline[1024]; - int result; - snprintf(commandline, 1024, "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/cuobjdump_to_ptxplus %s %s %s %s", - ptxfilename.c_str(), - sassfilename.c_str(), - elffilename.c_str(), - fname_ptxplus); - fflush(stdout); - printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", commandline); - result = system(commandline); - if(result){fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", commandline); exit(1);} - - - // Get ptxplus from file - std::ifstream fileStream(fname_ptxplus, std::ios::in); - std::string text, line; - while(getline(fileStream,line)) { - text += (line + "\n"); - } - fileStream.close(); - - char* ptxplus_str = new char [strlen(text.c_str())+1]; - strcpy(ptxplus_str, text.c_str()); - - if (!m_ptx_save_converted_ptxplus){ - char rm_commandline[1024]; - - snprintf(rm_commandline,1024,"rm -f %s", fname_ptxplus); - - printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", rm_commandline); - int rm_result = system(rm_commandline); - if( rm_result != 0 ) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", rm_result); - exit(1); - } - } - printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); - - return ptxplus_str; +void gpgpu_context::print_ptx_file(const char *p, unsigned source_num, + const char *filename) { + printf("\nGPGPU-Sim PTX: file _%u.ptx contents:\n\n", source_num); + char *s = strdup(p); + char *t = s; + unsigned n = 1; + while (*t != '\0') { + char *u = t; + while ((*u != '\n') && (*u != '\0')) u++; + unsigned last = (*u == '\0'); + *u = '\0'; + const ptx_instruction *pI = ptx_parser->ptx_instruction_lookup(filename, n); + char pc[64]; + if (pI && pI->get_PC()) + snprintf(pc, 64, "%4u", pI->get_PC()); + else + snprintf(pc, 64, " "); + printf(" _%u.ptx %4u (pc=%s): %s\n", source_num, n, pc, t); + if (last) break; + t = u + 1; + n++; + } + free(s); + fflush(stdout); } - -symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_string( const char *p, unsigned source_num ) -{ - char buf[1024]; - snprintf(buf,1024,"_%u.ptx", source_num ); - if( g_save_embedded_ptx ) { - FILE *fp = fopen(buf,"w"); - fprintf(fp,"%s",p); - fclose(fp); +char *ptxinfo_data::gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( + const std::string ptxfilename, const std::string elffilename, + const std::string sassfilename) { + printf("GPGPU-Sim PTX: converting EMBEDDED .ptx file to ptxplus \n"); + + char fname_ptxplus[1024]; + snprintf(fname_ptxplus, 1024, "_ptxplus_XXXXXX"); + int fd4 = mkstemp(fname_ptxplus); + close(fd4); + + // Run cuobjdump_to_ptxplus + char commandline[1024]; + int result; + snprintf(commandline, 1024, + "$GPGPUSIM_ROOT/build/$GPGPUSIM_CONFIG/cuobjdump_to_ptxplus/" + "cuobjdump_to_ptxplus %s %s %s %s", + ptxfilename.c_str(), sassfilename.c_str(), elffilename.c_str(), + fname_ptxplus); + fflush(stdout); + printf("GPGPU-Sim PTX: calling cuobjdump_to_ptxplus\ncommandline: %s\n", + commandline); + result = system(commandline); + if (result) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** could not execute %s\n", + commandline); + exit(1); + } + + // Get ptxplus from file + std::ifstream fileStream(fname_ptxplus, std::ios::in); + std::string text, line; + while (getline(fileStream, line)) { + text += (line + "\n"); + } + fileStream.close(); + + char *ptxplus_str = new char[strlen(text.c_str()) + 1]; + strcpy(ptxplus_str, text.c_str()); + + if (!m_ptx_save_converted_ptxplus) { + char rm_commandline[1024]; + + snprintf(rm_commandline, 1024, "rm -f %s", fname_ptxplus); + + printf("GPGPU-Sim PTX: removing temporary files using \"%s\"\n", + rm_commandline); + int rm_result = system(rm_commandline); + if (rm_result != 0) { + fprintf(stderr, + "GPGPU-Sim PTX: ERROR ** while removing temporary files %d\n", + rm_result); + exit(1); } - symbol_table *symtab=init_parser(buf); - ptx_lex_init(&(ptx_parser->scanner)); - ptx__scan_string(p, ptx_parser->scanner); - int errors = ptx_parse (ptx_parser->scanner, ptx_parser); - if ( errors ) { - char fname[1024]; - snprintf(fname,1024,"_ptx_errors_XXXXXX"); - int fd=mkstemp(fname); - close(fd); - printf("GPGPU-Sim PTX: parser error detected, exiting... but first extracting .ptx to \"%s\"\n", fname); - FILE *ptxfile = fopen(fname,"w"); - fprintf(ptxfile,"%s", p ); - fclose(ptxfile); - abort(); - exit(40); - } - ptx_lex_destroy(ptx_parser->scanner); + } + printf("GPGPU-Sim PTX: DONE converting EMBEDDED .ptx file to ptxplus \n"); - if ( g_debug_execution >= 100 ) - print_ptx_file(p,source_num,buf); + return ptxplus_str; +} - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",buf); - return symtab; +symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_string( + const char *p, unsigned source_num) { + char buf[1024]; + snprintf(buf, 1024, "_%u.ptx", source_num); + if (g_save_embedded_ptx) { + FILE *fp = fopen(buf, "w"); + fprintf(fp, "%s", p); + fclose(fp); + } + symbol_table *symtab = init_parser(buf); + ptx_lex_init(&(ptx_parser->scanner)); + ptx__scan_string(p, ptx_parser->scanner); + int errors = ptx_parse(ptx_parser->scanner, ptx_parser); + if (errors) { + char fname[1024]; + snprintf(fname, 1024, "_ptx_errors_XXXXXX"); + int fd = mkstemp(fname); + close(fd); + printf( + "GPGPU-Sim PTX: parser error detected, exiting... but first extracting " + ".ptx to \"%s\"\n", + fname); + FILE *ptxfile = fopen(fname, "w"); + fprintf(ptxfile, "%s", p); + fclose(ptxfile); + abort(); + exit(40); + } + ptx_lex_destroy(ptx_parser->scanner); + + if (g_debug_execution >= 100) print_ptx_file(p, source_num, buf); + + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n", buf); + return symtab; } -symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_filename( const char *filename ) -{ - symbol_table *symtab=init_parser(filename); - printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n",filename); - return symtab; +symbol_table *gpgpu_context::gpgpu_ptx_sim_load_ptx_from_filename( + const char *filename) { + symbol_table *symtab = init_parser(filename); + printf("GPGPU-Sim PTX: finished parsing EMBEDDED .ptx file %s\n", filename); + return symtab; } void fix_duplicate_errors(char fname2[1024]) { - char tempfile[1024] = "_temp_ptx"; - char commandline[1024]; - - // change the name of the ptx file to _temp_ptx - snprintf(commandline,1024,"mv %s %s",fname2,tempfile); - printf("Running: %s\n", commandline); - int result = system(commandline); - if (result != 0) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", fname2, tempfile); - exit(1); - } - - // store all of the ptx into a char array - FILE *ptxsource = fopen(tempfile,"r"); - fseek(ptxsource, 0, SEEK_END); - long filesize = ftell(ptxsource); - rewind(ptxsource); - char *ptxdata = (char*)malloc((filesize+1)*sizeof(char)); - // Fail if we do not read the file - assert(fread(ptxdata, filesize, 1, ptxsource) == 1); - fclose(ptxsource); - - FILE *ptxdest = fopen(fname2,"w"); - std::map duplicate = get_duplicate(); - unsigned offset; - unsigned oldlinenum = 1; - unsigned linenum; - char *startptr = ptxdata; - char *funcptr; - char *tempptr = ptxdata - 1; - char *lineptr = ptxdata - 1; - - // recreate the ptx file without duplications - for ( std::map::iterator iter = duplicate.begin(); - iter != duplicate.end(); - iter++){ - // find the line of the next error - linenum = iter->first; - for (int i = oldlinenum; i < linenum; i++) { - lineptr = strchr(lineptr + 1, '\n'); - } - - // find the end of the current section to be copied over - // then find the start of the next section that will be copied - if (strcmp("function", iter->second) == 0) { - // get location of most recent .func - while (tempptr < lineptr && tempptr != NULL) { - funcptr = tempptr; - tempptr = strstr(funcptr + 1, ".func"); - } - - // get the start of the previous line - offset = 0; - while (*(funcptr - offset) != '\n') offset++; - - fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); - - //find next location of startptr - if (*(lineptr + 3) == ';') { - // for function definitions - startptr = lineptr + 5; - } else if (*(lineptr + 3) == '{') { - // for functions enclosed with curly brackets - offset = 5; - unsigned bracket = 1; - while (bracket != 0) { - if (*(lineptr + offset) == '{') bracket++; - else if (*(lineptr + offset) == '}') bracket--; - offset++; - } - startptr = lineptr + offset + 1; - } else { - printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); - abort(); - } - } else if (strcmp("variable", iter->second) == 0) { - fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); - - //find next location of startptr - offset = 1; - while (*(lineptr + offset) != '\n') offset++; - startptr = lineptr + offset + 1; - } else { - printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", iter->second); - } - - oldlinenum = linenum; - } - // copy over the rest of the file - fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); - - // cleanup - free(ptxdata); - fclose(ptxdest); - snprintf(commandline,1024,"rm -f %s",tempfile); - printf("Running: %s\n", commandline); - result = system(commandline); - if (result != 0) { - fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); - exit(1); - } -} + char tempfile[1024] = "_temp_ptx"; + char commandline[1024]; + + // change the name of the ptx file to _temp_ptx + snprintf(commandline, 1024, "mv %s %s", fname2, tempfile); + printf("Running: %s\n", commandline); + int result = system(commandline); + if (result != 0) { + fprintf(stderr, + "GPGPU-Sim PTX: ERROR ** while changing filename from %s to %s", + fname2, tempfile); + exit(1); + } + + // store all of the ptx into a char array + FILE *ptxsource = fopen(tempfile, "r"); + fseek(ptxsource, 0, SEEK_END); + long filesize = ftell(ptxsource); + rewind(ptxsource); + char *ptxdata = (char *)malloc((filesize + 1) * sizeof(char)); + // Fail if we do not read the file + assert(fread(ptxdata, filesize, 1, ptxsource) == 1); + fclose(ptxsource); + + FILE *ptxdest = fopen(fname2, "w"); + std::map duplicate = get_duplicate(); + unsigned offset; + unsigned oldlinenum = 1; + unsigned linenum; + char *startptr = ptxdata; + char *funcptr; + char *tempptr = ptxdata - 1; + char *lineptr = ptxdata - 1; + + // recreate the ptx file without duplications + for (std::map::iterator iter = duplicate.begin(); + iter != duplicate.end(); iter++) { + // find the line of the next error + linenum = iter->first; + for (int i = oldlinenum; i < linenum; i++) { + lineptr = strchr(lineptr + 1, '\n'); + } + + // find the end of the current section to be copied over + // then find the start of the next section that will be copied + if (strcmp("function", iter->second) == 0) { + // get location of most recent .func + while (tempptr < lineptr && tempptr != NULL) { + funcptr = tempptr; + tempptr = strstr(funcptr + 1, ".func"); + } + + // get the start of the previous line + offset = 0; + while (*(funcptr - offset) != '\n') offset++; + + fwrite(startptr, sizeof(char), funcptr - offset + 1 - startptr, ptxdest); + + // find next location of startptr + if (*(lineptr + 3) == ';') { + // for function definitions + startptr = lineptr + 5; + } else if (*(lineptr + 3) == '{') { + // for functions enclosed with curly brackets + offset = 5; + unsigned bracket = 1; + while (bracket != 0) { + if (*(lineptr + offset) == '{') + bracket++; + else if (*(lineptr + offset) == '}') + bracket--; + offset++; + } + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unrecognized function format\n"); + abort(); + } + } else if (strcmp("variable", iter->second) == 0) { + fwrite(startptr, sizeof(char), (int)(lineptr + 1 - startptr), ptxdest); + + // find next location of startptr + offset = 1; + while (*(lineptr + offset) != '\n') offset++; + startptr = lineptr + offset + 1; + } else { + printf("GPGPU-Sim PTX: ERROR ** Unsupported duplicate type: %s\n", + iter->second); + } + oldlinenum = linenum; + } + // copy over the rest of the file + fwrite(startptr, sizeof(char), ptxdata + filesize - startptr, ptxdest); + + // cleanup + free(ptxdata); + fclose(ptxdest); + snprintf(commandline, 1024, "rm -f %s", tempfile); + printf("Running: %s\n", commandline); + result = system(commandline); + if (result != 0) { + fprintf(stderr, "GPGPU-Sim PTX: ERROR ** while deleting %s", tempfile); + exit(1); + } +} -//we need the application name here too. -char* get_app_binary_name(){ - char exe_path[1025]; - char *self_exe_path; +// we need the application name here too. +char *get_app_binary_name() { + char exe_path[1025]; + char *self_exe_path; #ifdef __APPLE__ - //AMRUTH: get apple device and check the result. - printf("WARNING: not tested for Apple-mac devices \n"); - abort(); + // AMRUTH: get apple device and check the result. + printf("WARNING: not tested for Apple-mac devices \n"); + abort(); #else - std::stringstream exec_link; - exec_link << "/proc/self/exe"; - ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); - assert(path_length != -1); - exe_path[path_length] = '\0'; - - char *token = strtok(exe_path, "/"); - while(token !=NULL){ - self_exe_path = token; - token = strtok(NULL,"/"); - } + std::stringstream exec_link; + exec_link << "/proc/self/exe"; + ssize_t path_length = readlink(exec_link.str().c_str(), exe_path, 1024); + assert(path_length != -1); + exe_path[path_length] = '\0'; + + char *token = strtok(exe_path, "/"); + while (token != NULL) { + self_exe_path = token; + token = strtok(NULL, "/"); + } #endif - self_exe_path = strtok(self_exe_path, "."); - printf("self exe links to: %s\n", self_exe_path); - return self_exe_path; + self_exe_path = strtok(self_exe_path, "."); + printf("self exe links to: %s\n", self_exe_path); + return self_exe_path; } -void gpgpu_context::gpgpu_ptx_info_load_from_filename( const char *filename, unsigned sm_version) -{ - std::string ptxas_filename(std::string(filename) + "as"); - char buff[1024], extra_flags[1024]; - extra_flags[0]=0; - if(!device_runtime->g_cdp_enabled) - snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); - else - snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); - snprintf(buff,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, filename, ptxas_filename.c_str()); - int result = system(buff); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } - - FILE *ptxinfo_in; - ptxinfo->g_ptxinfo_filename = strdup(ptxas_filename.c_str()); - ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename,"r"); - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); +void gpgpu_context::gpgpu_ptx_info_load_from_filename(const char *filename, + unsigned sm_version) { + std::string ptxas_filename(std::string(filename) + "as"); + char buff[1024], extra_flags[1024]; + extra_flags[0] = 0; + if (!device_runtime->g_cdp_enabled) + snprintf(extra_flags, 1024, "--gpu-name=sm_%u", sm_version); + else + snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", sm_version); + snprintf( + buff, 1024, + "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, filename, ptxas_filename.c_str()); + int result = system(buff); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } + + FILE *ptxinfo_in; + ptxinfo->g_ptxinfo_filename = strdup(ptxas_filename.c_str()); + ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename, "r"); + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); } -void gpgpu_context::gpgpu_ptxinfo_load_from_string( const char *p_for_info, unsigned source_num, unsigned sm_version, int no_of_ptx ) -{ - //do ptxas for individual files instead of one big embedded ptx. This prevents the duplicate defs and declarations. - char ptx_file[1000]; - char *name=get_app_binary_name(); - char commandline[4096], fname[1024], fname2[1024], final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; - for (int index=1; index <= no_of_ptx; index++){ - snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); - snprintf(fname,1024,"_ptx_XXXXXX"); - int fd=mkstemp(fname); - close(fd); - - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); - snprintf(commandline,4096,"cat %s > %s",ptx_file, fname); - if (system(commandline) !=0) { - printf("ERROR: %s command failed\n", commandline); - exit(0); - } - - snprintf(fname2,1024,"_ptx2_XXXXXX"); - fd=mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf(" Ensure you have write access to simulation directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); - exit(1); - } - - snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); - char extra_flags[1024]; - extra_flags[0]=0; +void gpgpu_context::gpgpu_ptxinfo_load_from_string(const char *p_for_info, + unsigned source_num, + unsigned sm_version, + int no_of_ptx) { + // do ptxas for individual files instead of one big embedded ptx. This + // prevents the duplicate defs and declarations. + char ptx_file[1000]; + char *name = get_app_binary_name(); + char commandline[4096], fname[1024], fname2[1024], + final_tempfile_ptxinfo[1024], tempfile_ptxinfo[1024]; + for (int index = 1; index <= no_of_ptx; index++) { + snprintf(ptx_file, 1000, "%s.%d.sm_%u.ptx", name, index, sm_version); + snprintf(fname, 1024, "_ptx_XXXXXX"); + int fd = mkstemp(fname); + close(fd); + + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", + fname); + snprintf(commandline, 4096, "cat %s > %s", ptx_file, fname); + if (system(commandline) != 0) { + printf("ERROR: %s command failed\n", commandline); + exit(0); + } + + snprintf(fname2, 1024, "_ptx2_XXXXXX"); + fd = mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2, 4096, + "cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, " + "texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 " + "\\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed " + "'s/const\\[.\\]/const\\[0\\]/g' > %s", + fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf( + " Ensure you have write access to simulation " + "directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); + } + + snprintf(tempfile_ptxinfo, 1024, "%sinfo", fname); + char extra_flags[1024]; + extra_flags[0] = 0; #if CUDART_VERSION >= 3000 - if ( g_occupancy_sm_number == 0 ) { - fprintf( stderr, "gpgpusim.config must specify the sm version for the GPU that you use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" - "The register file size is specifically tied to the sm version used to querry ptxas for register usage.\n" - "A register size/SM mismatch may result in occupancy differences." ); - exit(1); + if (g_occupancy_sm_number == 0) { + fprintf( + stderr, + "gpgpusim.config must specify the sm version for the GPU that you " + "use to compute occupancy \"-gpgpu_occupancy_sm_number XX\".\n" + "The register file size is specifically tied to the sm version used " + "to querry ptxas for register usage.\n" + "A register size/SM mismatch may result in occupancy differences."); + exit(1); } - if(!device_runtime->g_cdp_enabled) - snprintf(extra_flags,1024,"--gpu-name=sm_%u", g_occupancy_sm_number); + if (!device_runtime->g_cdp_enabled) + snprintf(extra_flags, 1024, "--gpu-name=sm_%u", g_occupancy_sm_number); else - snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",g_occupancy_sm_number); + snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", + g_occupancy_sm_number); #endif - snprintf(commandline,1024,"$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + snprintf(commandline, 1024, + "$PTXAS_CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file " + "/dev/null 2> %s", extra_flags, fname2, tempfile_ptxinfo); printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); result = system(commandline); - if( result != 0 ) { - // 65280 = duplicate errors - if (result == 65280) { - FILE *ptxinfo_in; - ptxinfo_in = fopen(tempfile_ptxinfo,"r"); - ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); - - fix_duplicate_errors(fname2); - snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", commandline); - result = system(commandline); - } - if (result != 0) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } - } + if (result != 0) { + // 65280 = duplicate errors + if (result == 65280) { + FILE *ptxinfo_in; + ptxinfo_in = fopen(tempfile_ptxinfo, "r"); + ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); + + fix_duplicate_errors(fname2); + snprintf(commandline, 1024, + "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file " + "/dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: regenerating ptxinfo using \"%s\"\n", + commandline); + result = system(commandline); + } + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); + } } - - //TODO: duplicate code! move it into a function so that it can be reused! - if(no_of_ptx==0) { - //For CDP, we dump everything. So no_of_ptx will be 0. - snprintf(fname,1024,"_ptx_XXXXXX"); - int fd=mkstemp(fname); - close(fd); - - printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", fname); - FILE *ptxfile = fopen(fname,"w"); - fprintf(ptxfile,"%s", p_for_info); - fclose(ptxfile); - - snprintf(fname2,1024,"_ptx2_XXXXXX"); - fd=mkstemp(fname2); - close(fd); - char commandline2[4096]; - snprintf(commandline2,4096,"cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 \\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed 's/const\\[.\\]/const\\[0\\]/g' > %s", fname, fname2); - printf("Running: %s\n", commandline2); - int result = system(commandline2); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); - printf(" Ensure you have write access to simulation directory\n"); - printf(" and have \'cat\' and \'sed\' in your path.\n"); - exit(1); - } - //char tempfile_ptxinfo[1024]; - snprintf(tempfile_ptxinfo,1024,"%sinfo",fname); - char extra_flags[1024]; - extra_flags[0]=0; - - #if CUDART_VERSION >= 3000 - if (sm_version == 0) sm_version = 20; - if(!device_runtime->g_cdp_enabled) - snprintf(extra_flags,1024,"--gpu-name=sm_%u",sm_version); - else - snprintf(extra_flags,1024,"--compile-only --gpu-name=sm_%u",sm_version); - #endif - - snprintf(commandline,1024,"$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", - extra_flags, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); - fflush(stdout); - result = system(commandline); - if( result != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); - printf(" Ensure ptxas is in your path.\n"); - exit(1); - } + } + + // TODO: duplicate code! move it into a function so that it can be reused! + if (no_of_ptx == 0) { + // For CDP, we dump everything. So no_of_ptx will be 0. + snprintf(fname, 1024, "_ptx_XXXXXX"); + int fd = mkstemp(fname); + close(fd); + + printf("GPGPU-Sim PTX: extracting embedded .ptx to temporary file \"%s\"\n", + fname); + FILE *ptxfile = fopen(fname, "w"); + fprintf(ptxfile, "%s", p_for_info); + fclose(ptxfile); + + snprintf(fname2, 1024, "_ptx2_XXXXXX"); + fd = mkstemp(fname2); + close(fd); + char commandline2[4096]; + snprintf(commandline2, 4096, + "cat %s | sed 's/.version 1.5/.version 1.4/' | sed 's/, " + "texmode_independent//' | sed 's/\\(\\.extern \\.const\\[1\\] .b8 " + "\\w\\+\\)\\[\\]/\\1\\[1\\]/' | sed " + "'s/const\\[.\\]/const\\[0\\]/g' > %s", + fname, fname2); + printf("Running: %s\n", commandline2); + int result = system(commandline2); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (a) %d\n", result); + printf( + " Ensure you have write access to simulation " + "directory\n"); + printf(" and have \'cat\' and \'sed\' in your path.\n"); + exit(1); } + // char tempfile_ptxinfo[1024]; + snprintf(tempfile_ptxinfo, 1024, "%sinfo", fname); + char extra_flags[1024]; + extra_flags[0] = 0; - //Now that we got resource usage per kernel in a ptx file, we dump all into one file and pass it to rest of the code as usual. - if(no_of_ptx>0){ - char commandline3[4096]; - snprintf(final_tempfile_ptxinfo,1024,"f_tempfile_ptx"); - snprintf(commandline3,4096, "cat *info > %s", final_tempfile_ptxinfo); - if (system(commandline3)!=0) { - printf("ERROR: Either we dont have info files or cat is not working \n"); - printf("ERROR: %s command failed\n",commandline3); - exit(1); - } - } - - if(no_of_ptx>0) - ptxinfo->g_ptxinfo_filename = final_tempfile_ptxinfo; +#if CUDART_VERSION >= 3000 + if (sm_version == 0) sm_version = 20; + if (!device_runtime->g_cdp_enabled) + snprintf(extra_flags, 1024, "--gpu-name=sm_%u", sm_version); else - ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; - FILE *ptxinfo_in; - ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename,"r"); - - ptxinfo_lex_init(&(ptxinfo->scanner)); - ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); - ptxinfo_parse(ptxinfo->scanner, ptxinfo); - ptxinfo_lex_destroy(ptxinfo->scanner); - fclose(ptxinfo_in); - - snprintf(commandline,1024,"rm -f *info"); - if( system(commandline) != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); - exit(1); + snprintf(extra_flags, 1024, "--compile-only --gpu-name=sm_%u", + sm_version); +#endif + + snprintf( + commandline, 1024, + "$CUDA_INSTALL_PATH/bin/ptxas %s -v %s --output-file /dev/null 2> %s", + extra_flags, fname2, tempfile_ptxinfo); + printf("GPGPU-Sim PTX: generating ptxinfo using \"%s\"\n", commandline); + fflush(stdout); + result = system(commandline); + if (result != 0) { + printf("GPGPU-Sim PTX: ERROR ** while loading PTX (b) %d\n", result); + printf(" Ensure ptxas is in your path.\n"); + exit(1); } - if( ! g_save_embedded_ptx ) { - if(no_of_ptx>0) - snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, final_tempfile_ptxinfo); - else - snprintf(commandline,1024,"rm -f %s %s %s", fname, fname2, tempfile_ptxinfo); - printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); - if( system(commandline) != 0 ) { - printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); - exit(1); - } + } + + // Now that we got resource usage per kernel in a ptx file, we dump all into + // one file and pass it to rest of the code as usual. + if (no_of_ptx > 0) { + char commandline3[4096]; + snprintf(final_tempfile_ptxinfo, 1024, "f_tempfile_ptx"); + snprintf(commandline3, 4096, "cat *info > %s", final_tempfile_ptxinfo); + if (system(commandline3) != 0) { + printf("ERROR: Either we dont have info files or cat is not working \n"); + printf("ERROR: %s command failed\n", commandline3); + exit(1); + } + } + + if (no_of_ptx > 0) + ptxinfo->g_ptxinfo_filename = final_tempfile_ptxinfo; + else + ptxinfo->g_ptxinfo_filename = tempfile_ptxinfo; + FILE *ptxinfo_in; + ptxinfo_in = fopen(ptxinfo->g_ptxinfo_filename, "r"); + + ptxinfo_lex_init(&(ptxinfo->scanner)); + ptxinfo_set_in(ptxinfo_in, ptxinfo->scanner); + ptxinfo_parse(ptxinfo->scanner, ptxinfo); + ptxinfo_lex_destroy(ptxinfo->scanner); + fclose(ptxinfo_in); + + snprintf(commandline, 1024, "rm -f *info"); + if (system(commandline) != 0) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary info files\n"); + exit(1); + } + if (!g_save_embedded_ptx) { + if (no_of_ptx > 0) + snprintf(commandline, 1024, "rm -f %s %s %s", fname, fname2, + final_tempfile_ptxinfo); + else + snprintf(commandline, 1024, "rm -f %s %s %s", fname, fname2, + tempfile_ptxinfo); + printf("GPGPU-Sim PTX: removing ptxinfo using \"%s\"\n", commandline); + if (system(commandline) != 0) { + printf("GPGPU-Sim PTX: ERROR ** while removing temporary files\n"); + exit(1); } + } } diff --git a/src/cuda-sim/ptx_loader.h b/src/cuda-sim/ptx_loader.h index d8f1cbc..ff5d7ed 100644 --- a/src/cuda-sim/ptx_loader.h +++ b/src/cuda-sim/ptx_loader.h @@ -7,23 +7,24 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #ifndef PTX_LOADER_H_INCLUDED #define PTX_LOADER_H_INCLUDED @@ -31,22 +32,22 @@ #define PTXINFO_LINEBUF_SIZE 1024 class gpgpu_context; -typedef void * yyscan_t; -class ptxinfo_data{ - public: - ptxinfo_data(gpgpu_context* ctx) { - gpgpu_ctx = ctx; - } - yyscan_t scanner; - char linebuf[PTXINFO_LINEBUF_SIZE]; - unsigned col; - const char *g_ptxinfo_filename; - class gpgpu_context* gpgpu_ctx; - bool g_keep_intermediate_files; - bool m_ptx_save_converted_ptxplus; - void ptxinfo_addinfo(); - bool keep_intermediate_files(); - char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus(const std::string ptx_str, const std::string sass_str, const std::string elf_str); +typedef void* yyscan_t; +class ptxinfo_data { + public: + ptxinfo_data(gpgpu_context* ctx) { gpgpu_ctx = ctx; } + yyscan_t scanner; + char linebuf[PTXINFO_LINEBUF_SIZE]; + unsigned col; + const char* g_ptxinfo_filename; + class gpgpu_context* gpgpu_ctx; + bool g_keep_intermediate_files; + bool m_ptx_save_converted_ptxplus; + void ptxinfo_addinfo(); + bool keep_intermediate_files(); + char* gpgpu_ptx_sim_convert_ptx_and_sass_to_ptxplus( + const std::string ptx_str, const std::string sass_str, + const std::string elf_str); }; #endif diff --git a/src/cuda-sim/ptx_parser.cc b/src/cuda-sim/ptx_parser.cc index a4f4a0c..3ae8de3 100644 --- a/src/cuda-sim/ptx_parser.cc +++ b/src/cuda-sim/ptx_parser.cc @@ -7,1017 +7,991 @@ // // Redistributions of source code must retain the above copyright notice, this // list of conditions and the following disclaimer. -// Redistributions in binary form must reproduce the above copyright notice, this -// list of conditions and the following disclaimer in the documentation and/or -// other materials provided with the distribution. -// Neither the name of The University of British Columbia nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. +// Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. Neither the name of +// The University of British Columbia nor the names of its contributors may be +// used to endorse or promote products derived from this software without +// specific prior written permission. // -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. #include "ptx_parser.h" -#include "ptx_ir.h" #include "../../libcuda/gpgpu_context.h" +#include "ptx_ir.h" -typedef void * yyscan_t; -#include "ptx.tab.h" +typedef void *yyscan_t; #include +#include "ptx.tab.h" -extern int ptx_get_lineno (yyscan_t yyscanner ); -extern YYSTYPE* ptx_get_lval (yyscan_t yyscanner ); -extern int ptx_error( yyscan_t yyscanner, const char *s ); -extern int ptx_lex_init(yyscan_t* scanner); -extern void ptx_set_in(FILE * _in_str ,yyscan_t yyscanner ); -extern FILE *ptx_get_in (yyscan_t yyscanner ); -extern int ptx_parse(yyscan_t scanner, ptx_recognizer* recognizer); +extern int ptx_get_lineno(yyscan_t yyscanner); +extern YYSTYPE *ptx_get_lval(yyscan_t yyscanner); +extern int ptx_error(yyscan_t yyscanner, const char *s); +extern int ptx_lex_init(yyscan_t *scanner); +extern void ptx_set_in(FILE *_in_str, yyscan_t yyscanner); +extern FILE *ptx_get_in(yyscan_t yyscanner); +extern int ptx_parse(yyscan_t scanner, ptx_recognizer *recognizer); extern int ptx_lex_destroy(yyscan_t scanner); -void ptx_recognizer::set_ptx_warp_size(const struct core_config * warp_size) -{ - g_shader_core_config=warp_size; -} - - -#define PTX_PARSE_DPRINTF(...) \ - if( g_debug_ir_generation ) { \ - printf(" %s:%u => ",gpgpu_ctx->g_filename,ptx_get_lineno(scanner)); \ - printf(" (%s:%u) ", __FILE__, __LINE__); \ - printf(__VA_ARGS__); \ - printf("\n"); \ - fflush(stdout); \ - } - -static std::map g_ptx_token_decode; - -const char *decode_token( int type ) -{ - return g_ptx_token_decode[type].c_str(); -} - -void ptx_recognizer::read_parser_environment_variables() -{ - gpgpu_ctx->g_filename = getenv("PTX_SIM_KERNELFILE"); - char *dbg_level = getenv("PTX_SIM_DEBUG"); - if ( dbg_level && strlen(dbg_level) ) { - int debug_execution=0; - sscanf(dbg_level,"%d", &debug_execution); - if ( debug_execution >= 30 ) - g_debug_ir_generation=true; - } -} - -void ptx_recognizer::init_directive_state() -{ - PTX_PARSE_DPRINTF("init_directive_state"); - g_space_spec=undefined_space; - g_ptr_spec=undefined_space; - g_scalar_type_spec=-1; - g_vector_spec=-1; - g_opcode=-1; - g_alignment_spec = -1; - g_size = -1; - g_extern_spec = 0; - g_scalar_type.clear(); - g_operands.clear(); - g_last_symbol = NULL; -} - -void ptx_recognizer::init_instruction_state() -{ - PTX_PARSE_DPRINTF("init_instruction_state"); - g_pred = NULL; - g_neg_pred = 0; - g_pred_mod = -1; - g_label = NULL; - g_opcode = -1; - g_options.clear(); - g_wmma_options.clear(); - g_return_var = operand_info(gpgpu_ctx); - init_directive_state(); -} - -symbol_table * gpgpu_context::init_parser( const char *ptx_filename ) -{ - g_filename = strdup(ptx_filename); - if (g_global_allfiles_symbol_table == NULL) { - g_global_allfiles_symbol_table = new symbol_table("global_allfiles", 0, NULL, this); - ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = g_global_allfiles_symbol_table; - } - /*else { - g_global_symbol_table = g_current_symbol_table = new symbol_table("global",0,g_global_allfiles_symbol_table); - }*/ - -#define DEF(X,Y) g_ptx_token_decode[X] = Y; +void ptx_recognizer::set_ptx_warp_size(const struct core_config *warp_size) { + g_shader_core_config = warp_size; +} + +#define PTX_PARSE_DPRINTF(...) \ + if (g_debug_ir_generation) { \ + printf(" %s:%u => ", gpgpu_ctx->g_filename, ptx_get_lineno(scanner)); \ + printf(" (%s:%u) ", __FILE__, __LINE__); \ + printf(__VA_ARGS__); \ + printf("\n"); \ + fflush(stdout); \ + } + +static std::map g_ptx_token_decode; + +const char *decode_token(int type) { return g_ptx_token_decode[type].c_str(); } + +void ptx_recognizer::read_parser_environment_variables() { + gpgpu_ctx->g_filename = getenv("PTX_SIM_KERNELFILE"); + char *dbg_level = getenv("PTX_SIM_DEBUG"); + if (dbg_level && strlen(dbg_level)) { + int debug_execution = 0; + sscanf(dbg_level, "%d", &debug_execution); + if (debug_execution >= 30) g_debug_ir_generation = true; + } +} + +void ptx_recognizer::init_directive_state() { + PTX_PARSE_DPRINTF("init_directive_state"); + g_space_spec = undefined_space; + g_ptr_spec = undefined_space; + g_scalar_type_spec = -1; + g_vector_spec = -1; + g_opcode = -1; + g_alignment_spec = -1; + g_size = -1; + g_extern_spec = 0; + g_scalar_type.clear(); + g_operands.clear(); + g_last_symbol = NULL; +} + +void ptx_recognizer::init_instruction_state() { + PTX_PARSE_DPRINTF("init_instruction_state"); + g_pred = NULL; + g_neg_pred = 0; + g_pred_mod = -1; + g_label = NULL; + g_opcode = -1; + g_options.clear(); + g_wmma_options.clear(); + g_return_var = operand_info(gpgpu_ctx); + init_directive_state(); +} + +symbol_table *gpgpu_context::init_parser(const char *ptx_filename) { + g_filename = strdup(ptx_filename); + if (g_global_allfiles_symbol_table == NULL) { + g_global_allfiles_symbol_table = + new symbol_table("global_allfiles", 0, NULL, this); + ptx_parser->g_global_symbol_table = ptx_parser->g_current_symbol_table = + g_global_allfiles_symbol_table; + } + /*else { + g_global_symbol_table = g_current_symbol_table = new + symbol_table("global",0,g_global_allfiles_symbol_table); + }*/ + +#define DEF(X, Y) g_ptx_token_decode[X] = Y; #include "ptx_parser_decode.def" #undef DEF - g_ptx_token_decode[undefined_space] = "undefined_space"; - g_ptx_token_decode[undefined_space] = "undefined_space=0"; - g_ptx_token_decode[reg_space] = "reg_space"; - g_ptx_token_decode[local_space] = "local_space"; - g_ptx_token_decode[shared_space] = "shared_space"; - g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; - g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; - g_ptx_token_decode[param_space_local] = "param_space_local"; - g_ptx_token_decode[const_space] = "const_space"; - g_ptx_token_decode[tex_space] = "tex_space"; - g_ptx_token_decode[surf_space] = "surf_space"; - g_ptx_token_decode[global_space] = "global_space"; - g_ptx_token_decode[generic_space] = "generic_space"; - g_ptx_token_decode[instruction_space] = "instruction_space"; - - ptx_lex_init(&(ptx_parser->scanner)); - ptx_parser->init_directive_state(); - ptx_parser->init_instruction_state(); - - FILE *ptx_in; - ptx_in = fopen(ptx_filename, "r"); - ptx_set_in(ptx_in, ptx_parser->scanner); - ptx_parse(ptx_parser->scanner, ptx_parser); - ptx_in = ptx_get_in(ptx_parser->scanner); - ptx_lex_destroy(ptx_parser->scanner); - fclose(ptx_in); - return ptx_parser->g_global_symbol_table; -} - - -void ptx_recognizer::start_function( int entry_point ) -{ - PTX_PARSE_DPRINTF("start_function"); - init_directive_state(); - init_instruction_state(); - g_entry_point = entry_point; - g_func_info = NULL; - g_entry_func_param_index=0; -} - -void ptx_recognizer::add_function_name( const char *name ) -{ - PTX_PARSE_DPRINTF("add_function_name %s %s", name, ((g_entry_point==1)?"(entrypoint)":((g_entry_point==2)?"(extern)":""))); - bool prior_decl = g_global_symbol_table->add_function_decl( name, g_entry_point, &g_func_info, &g_current_symbol_table ); - if( g_add_identifier_cached__identifier ) { - add_identifier( g_add_identifier_cached__identifier, - g_add_identifier_cached__array_dim, - g_add_identifier_cached__array_ident ); - free( g_add_identifier_cached__identifier ); - g_add_identifier_cached__identifier = NULL; - g_func_info->add_return_var( g_last_symbol ); - init_directive_state(); - } - if( prior_decl ) { - g_func_info->remove_args(); - } - g_global_symbol_table->add_function( g_func_info, gpgpu_ctx->g_filename, ptx_get_lineno(scanner) ); -} - -//Jin: handle instruction group for cdp + g_ptx_token_decode[undefined_space] = "undefined_space"; + g_ptx_token_decode[undefined_space] = "undefined_space=0"; + g_ptx_token_decode[reg_space] = "reg_space"; + g_ptx_token_decode[local_space] = "local_space"; + g_ptx_token_decode[shared_space] = "shared_space"; + g_ptx_token_decode[param_space_unclassified] = "param_space_unclassified"; + g_ptx_token_decode[param_space_kernel] = "param_space_kernel"; + g_ptx_token_decode[param_space_local] = "param_space_local"; + g_ptx_token_decode[const_space] = "const_space"; + g_ptx_token_decode[tex_space] = "tex_space"; + g_ptx_token_decode[surf_space] = "surf_space"; + g_ptx_token_decode[global_space] = "global_space"; + g_ptx_token_decode[generic_space] = "generic_space"; + g_ptx_token_decode[instruction_space] = "instruction_space"; + + ptx_lex_init(&(ptx_parser->scanner)); + ptx_parser->init_directive_state(); + ptx_parser->init_instruction_state(); + + FILE *ptx_in; + ptx_in = fopen(ptx_filename, "r"); + ptx_set_in(ptx_in, ptx_parser->scanner); + ptx_parse(ptx_parser->scanner, ptx_parser); + ptx_in = ptx_get_in(ptx_parser->scanner); + ptx_lex_destroy(ptx_parser->scanner); + fclose(ptx_in); + return ptx_parser->g_global_symbol_table; +} + +void ptx_recognizer::start_function(int entry_point) { + PTX_PARSE_DPRINTF("start_function"); + init_directive_state(); + init_instruction_state(); + g_entry_point = entry_point; + g_func_info = NULL; + g_entry_func_param_index = 0; +} + +void ptx_recognizer::add_function_name(const char *name) { + PTX_PARSE_DPRINTF( + "add_function_name %s %s", name, + ((g_entry_point == 1) ? "(entrypoint)" + : ((g_entry_point == 2) ? "(extern)" : ""))); + bool prior_decl = g_global_symbol_table->add_function_decl( + name, g_entry_point, &g_func_info, &g_current_symbol_table); + if (g_add_identifier_cached__identifier) { + add_identifier(g_add_identifier_cached__identifier, + g_add_identifier_cached__array_dim, + g_add_identifier_cached__array_ident); + free(g_add_identifier_cached__identifier); + g_add_identifier_cached__identifier = NULL; + g_func_info->add_return_var(g_last_symbol); + init_directive_state(); + } + if (prior_decl) { + g_func_info->remove_args(); + } + g_global_symbol_table->add_function(g_func_info, gpgpu_ctx->g_filename, + ptx_get_lineno(scanner)); +} + +// Jin: handle instruction group for cdp void ptx_recognizer::start_inst_group() { - PTX_PARSE_DPRINTF("start_instruction_group"); - g_current_symbol_table = g_current_symbol_table->start_inst_group(); + PTX_PARSE_DPRINTF("start_instruction_group"); + g_current_symbol_table = g_current_symbol_table->start_inst_group(); } void ptx_recognizer::end_inst_group() { - PTX_PARSE_DPRINTF("end_instruction_group"); - g_current_symbol_table = g_current_symbol_table->end_inst_group(); -} - -void ptx_recognizer::add_directive() -{ - PTX_PARSE_DPRINTF("add_directive"); - init_directive_state(); -} - -#define mymax(a,b) ((a)>(b)?(a):(b)) - -void ptx_recognizer::end_function() -{ - PTX_PARSE_DPRINTF("end_function"); - - init_directive_state(); - init_instruction_state(); - g_max_regs_per_thread = mymax( g_max_regs_per_thread, (g_current_symbol_table->next_reg_num()-1)); - g_func_info->add_inst( g_instructions ); - g_instructions.clear(); - gpgpu_ptx_assemble( g_func_info->get_name(), g_func_info ); - g_current_symbol_table = g_global_symbol_table; - - PTX_PARSE_DPRINTF("function %s, PC = %d\n", g_func_info->get_name().c_str(), g_func_info->get_start_PC()); -} - -#define parse_error(msg, ...) parse_error_impl(__FILE__,__LINE__, msg, ##__VA_ARGS__) -#define parse_assert(cond,msg, ...) parse_assert_impl((cond),__FILE__,__LINE__, msg, ##__VA_ARGS__) - -void ptx_recognizer::parse_error_impl( const char *file, unsigned line, const char *msg, ... ) -{ - va_list ap; - char buf[1024]; - va_start(ap,msg); - vsnprintf(buf,1024,msg,ap); - va_end(ap); - - g_error_detected = 1; - printf("%s:%u: Parse error: %s (%s:%u)\n\n", gpgpu_ctx->g_filename, ptx_get_lineno(scanner), buf, file, line); - ptx_error(scanner, NULL); - abort(); - exit(1); -} - -void ptx_recognizer::parse_assert_impl( int test_value, const char *file, unsigned line, const char *msg, ... ) -{ - va_list ap; - char buf[1024]; - va_start(ap,msg); - vsnprintf(buf,1024,msg,ap); - va_end(ap); - - if ( test_value == 0 ) - parse_error_impl(file,line, msg); -} - - - -void ptx_recognizer::set_return() -{ - parse_assert( (g_opcode == CALL_OP || g_opcode == CALLP_OP), "only call can have return value"); - g_operands.front().set_return(); - g_return_var = g_operands.front(); -} - - -const ptx_instruction *ptx_recognizer::ptx_instruction_lookup( const char *filename, unsigned linenumber ) -{ - std::map >::iterator f=g_inst_lookup.find(filename); - if( f == g_inst_lookup.end() ) - return NULL; - std::map::iterator l=f->second.find(linenumber); - if( l == f->second.end() ) - return NULL; - return l->second; -} - -void ptx_recognizer::add_instruction() -{ - PTX_PARSE_DPRINTF("add_instruction: %s", ((g_opcode>0)?g_opcode_string[g_opcode]:"